Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor: remove the low/medium/high/ultra thinking modes (#20910)

Caobaining1d2ede485bfe8
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go
index 4571c3d425..c57cfc319c 100644
--- a/cmd/ragflow_server.go
+++ b/cmd/ragflow_server.go
@@ -42,7 +42,6 @@ import (
"ragflow/internal/handler"
"ragflow/internal/ingestion/knowledge_compile"
ingestion "ragflow/internal/ingestion/service"
- agentic_rag "ragflow/internal/rag/agentic-rag"
"ragflow/internal/router"
"ragflow/internal/server/local"
"ragflow/internal/service"
@@ -1147,15 +1146,11 @@ func startServer(ctx context.Context, serverName string, arguments *serverArgs)
// entity rows, the documented fallback on a deployment without an embedder.
smartagentic.SetKgService(smartagentic.NewKgAdapter(docEngine))
common.Info("agent: smart-reasoning compiled-knowledge graph service installed")
- // Wire the agentic-RAG runtime as the Go chat pipeline's evidence engine
- // (internal/service/chat_pipeline.retrieveViaHarness): it runs each request on a
- // model resolved from the caller's ModelID and searches through the adapter
- // above. Activating the agentic loop here enables the full planner/SCA path for
- // reasoning chats.
+ // The smart-reasoning agent's corpus tools resolve the shared retrieval
+ // service per call, so the production adapter must be registered before the
+ // server starts serving.
runtime.SetRetrievalService(retrievalbridge.NewRuntimeAdapter(retrievalAdapter))
- agentic_rag.SetAgenticLoop(agentic_rag.NewAgenticLoop())
- service.SetHarnessRetriever(retrievalbridge.NewHarnessRetriever(modelFactory, metadataService, docEngine))
- common.Info("agent: runtime chat retriever wired (runtime retrieval + agentic loop)")
+ common.Info("agent: runtime retrieval service wired")
// Initialize handler layer
authHandler := handler.NewAuthHandler()
diff --git a/internal/agent/chat/chat.go b/internal/agent/chat/chat.go
index f4e952c744..0fc327f499 100644
--- a/internal/agent/chat/chat.go
+++ b/internal/agent/chat/chat.go
@@ -17,10 +17,9 @@
// Package chat is a dependency-light home for the LLM chat-invoker interface
// and its package-level singleton. It lives here (leaf, importing only eino
// schema + gorm) so that both internal/agent/component (which owns the
-// production eino-based invoker) and internal/agent/tool /
-// internal/rag/agentic-rag/harness (which need to call the LLM for
-// routing/selection) can depend on it without forming an import cycle. The production invoker is registered at boot via
-// SetDefaultInvoker.
+// production eino-based invoker) and internal/agent/tool (which need to call
+// the LLM for routing/selection) can depend on it without forming an import
+// cycle. The production invoker is registered at boot via SetDefaultInvoker.
package chat
import (
diff --git a/internal/agent/chat/message_fit.go b/internal/agent/chat/message_fit.go
index aeda4af6b2..3d434f20b8 100644
--- a/internal/agent/chat/message_fit.go
+++ b/internal/agent/chat/message_fit.go
@@ -28,10 +28,9 @@ import (
// Message fitting (Python message_fit_in).
//
// This lives in the chat package rather than internal/agent/component so that
-// both component (which owns the production eino invoker) and
-// internal/rag/agentic-rag/harness (which composes answers) can fit prompts
-// without forming an import cycle — component cannot import harness, because
-// harness's own tests import component.
+// both component (which owns the production eino invoker) and its callers can
+// fit prompts without forming an import cycle — component cannot be imported
+// by the packages that also need component's own test doubles.
// EffectiveContextLength returns maxLength if positive, otherwise 8192.
// Mirrors Python's LLM.effective_context_length in PR #16413 — prevents
diff --git a/internal/agent/retrievalbridge/harness.go b/internal/agent/retrievalbridge/harness.go
deleted file mode 100644
index f64601937c..0000000000
--- a/internal/agent/retrievalbridge/harness.go
+++ /dev/null
@@ -1,756 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package retrievalbridge
-
-import (
- "context"
- "errors"
- "fmt"
- "strconv"
- "strings"
-
- "ragflow/internal/agent/component"
- "ragflow/internal/common"
- "ragflow/internal/dao"
- et "ragflow/internal/engine/types"
- "ragflow/internal/entity"
- modelModule "ragflow/internal/entity/models"
- "ragflow/internal/rag/agentic-rag"
- agenticruntime "ragflow/internal/rag/agentic-rag/runtime"
- "ragflow/internal/service"
- "ragflow/internal/service/nlp"
-
- "github.com/cloudwego/eino/schema"
- "go.uber.org/zap"
-)
-
-// NewHarnessRetriever connects chat requests to the agentic runtime using the
-// shared model, metadata and document services initialized by the API server.
-func NewHarnessRetriever(modelFactory *service.ModelFactory, metadataService *service.MetadataService, docEngine engineDocEngine) func(context.Context, service.HarnessRequest) (service.HarnessResult, error) {
- return func(ctx context.Context, req service.HarnessRequest) (service.HarnessResult, error) {
- // Resolve the tenant's actual chat model, mirroring Python where RAGTools
- // receives a fully-resolved LLMBundle: chat.LLMID may be a UUID/tenant_model
- // id that the default invoker cannot split into a provider, so resolving here
- // avoids falling through to a dummy driver. If resolution fails we leave model
- // nil so the runtime degrades to a direct search (no dummy fallback).
- var model agenticruntime.SessionModel
- // outerModel mirrors Python RAGTools.chat_mdl: the fully-resolved chat
- // model that drives the outer rag_agent react loop (tools=[rag,
- // summarize_document], terminal_tools={"rag"}). Wired into RAGTools.Outer
- // so Rag() can run that loop instead of the direct graph.
- var outerModel *modelModule.ChatModel
- // resolvedModelName is the resolved chat model's own name (e.g.
- // "gpt-4o"), i.e. Python chat_mdl.llm_name. It is the gen_json reply-cache
- // key's model component; empty disables that cache.
- resolvedModelName := ""
- // Resolve the same Chat model that the pipeline uses. An empty model ID
- // selects the tenant default; an explicit reference must be enrolled as
- // Chat, so an image2text-only model is rejected here.
- var chatModel *modelModule.ChatModel
- var mErr error
- if modelFactory == nil {
- mErr = errors.New("model factory is not initialized")
- } else {
- access := service.ModelAccess{TenantID: req.TenantID}
- if strings.TrimSpace(req.ModelID) == "" {
- chatModel, mErr = modelFactory.NewDefaultChatModel(ctx, access)
- } else {
- chatModel, mErr = modelFactory.NewChatModel(ctx, access, req.ModelID)
- }
- }
- if mErr == nil && chatModel != nil {
- if inv := component.NewResolvedInvoker(chatModel); inv != nil {
- info := chatModel.Info()
- contextLength := 0
- if info != nil {
- contextLength = info.ContextLength
- resolvedModelName = info.Name
- }
- // Message fitting uses the input context window, not MaxOutput.
- model = &agenticruntime.InvokerSessionModel{Invoker: inv, DB: dao.DB, MaxLength: contextLength}
- outerModel = chatModel
- }
- } else {
- common.Warn("runtime: failed to resolve chat model for reasoning; runtime will degrade to direct search", zap.Error(mErr))
- }
-
- // Load the KB objects (mirroring Python RAGTools' self.kbs via
- // KnowledgebaseService.get_by_ids(kb_ids)) so the agentic tool can
- // derive rank features. The Go tag extractor (extractor_tag.go) writes
- // both tag_kwd (the list of tag names) and tag_feas (per-tag weights)
- // onto each chunk at parse time; the labeler aggregates tag_kwd to build
- // the tag vocabulary and the retriever ranks with tag_feas.
- var kbs []*entity.Knowledgebase
- if len(req.DatasetIDs) > 0 {
- var err error
- kbs, err = dao.NewKnowledgebaseDAO().GetByIDs(ctx, dao.DB, req.DatasetIDs)
- if err != nil {
- return service.HarnessResult{}, err
- }
- if err := validateLoadedDatasets(req.DatasetIDs, kbs); err != nil {
- return service.HarnessResult{}, err
- }
- }
- // validate_dataset_embedding_models runs FIRST upstream
- // (dialog_service.py:358-360) and mixing is a hard failure there, not a
- // fallback to keyword-only retrieval.
- if err := validateDatasetEmbeddingModels(ctx, kbs); err != nil {
- return service.HarnessResult{}, err
- }
- // HasEmbedder mirrors Python `embd_mdl = ... if kbs and kbs[0].embd_id else
- // None` (dialog_service.py:362): the gate is the FIRST dataset's embd_id. It
- // is what hybrid_search applies to its vector leg (search.py:143), so
- // getting it wrong silently degrades search_chunks to keyword-only.
- hasEmbedder := hasEmbedderFor(kbs)
- // Retrieval tuning mirrors the dialog's own settings, so the agentic
- // searches run with the same budget as the standard path. Without it the
- // harness falls back to its package defaults (top_n=12) and the dialog's
- // configured top_n is silently ignored.
- //
- // The runtime carries the KEYWORD leg's weight and derives the vector leg
- // as its complement (runtime.resolveKeywordsSimilarityWeight, and the
- // RetrieveRequest contract: 0.7 hybrid / 0.0 vector-only / 1.0
- // keyword-only). The dialog configures the VECTOR weight, so it is inverted
- // here; the dialog's default 0.3 therefore yields the runtime's own 0.7.
- keywordsWeight := 1 - req.Retrieval.VectorSimilarityWeight
- deps := agentic_rag.RAGTools{
- Model: model,
- ModelName: resolvedModelName,
- Outer: outerModel,
- TopN: req.Retrieval.TopN,
- SimilarityThreshold: req.Retrieval.SimilarityThreshold,
- KeywordsSimilarityWeight: &keywordsWeight,
- RerankCandidatesCount: req.Retrieval.RerankCandidatesCount,
- // OriginalQuestion mirrors Python RAGTools(original_user_question=...):
- // the user's own, unrewritten question as received from the chat
- // layer. The outer model's `rag(question=...)` argument is
- // model-generated and often compresses a multi-hop question to its
- // first hop; resolveEffectiveQuestion (agentic_rag.py:865) prefers
- // this original over that rewrite when both describe the same turn.
- OriginalQuestion: req.Question,
- // DocScope mirrors Python RAGTools(doc_scope=...): the narrowed doc_ids
- // (chat-level doc_ids + meta_data_filter) restrict every agentic
- // retrieval to the user-selected documents instead of the whole kb.
- DocScope: req.DocIDs,
- DocIDVerifier: agentic_rag.NewDocIDLookup(),
- // MetadataResolver backs the metadata_search tool and the pre-search
- // metadata channel (Python DocMetadataService push-down + meta_filter
- // fallback). metadataService already carries the doc engine and the DAO
- // the metadata index reads need.
- MetadataResolver: metadataService,
- // DeclaredMetadata lets the metadata_search catalog describe each field
- // (meaning + allowed values) from the dataset's own parser_config, not just
- // name it. Same service: it also carries the KB DAO that read needs.
- DeclaredMetadata: metadataService,
- // DocChunks pages one document's chunks in reading order off the
- // chunk index (Python retriever.chunk_list), backing the
- // document-level tools (summarize_document / fetch_full_document).
- DocChunks: &docChunkPager{docEngine: docEngine},
- // Expand runs search_chunks' compiled-structure expansion
- // (Python hybrid_search use_compiled=True → _expand_with_compiled).
- // Backed by the same document engine the chunk reads use, with the
- // dense seed leg wired to the bound dataset's own embedding model.
- // The scope config lets each dataset be scanned under its own tenant
- // and a doc scope be grouped by real owner, like graph_explore.
- Expand: agenticruntime.NewCompiledExpander(
- newDatasetCompiledStore(docEngine, modelFactory, kbs),
- agenticruntime.CompiledScopeConfig{
- DatasetIDs: req.DatasetIDs,
- TenantID: req.TenantID,
- KBs: kbs,
- DocTenantResolver: agentic_rag.NewDocTenantResolver(),
- },
- ),
- // WebSearch backs the web_search tool (Python RAGTools.web_search).
- // The pipeline supplies a callback only when the chat enables
- // internet web search; a nil one also keeps the tool off the surface.
- WebSearch: harnessWebSearcher(req.WebSearch),
- KBs: kbs,
- HasEmbedder: hasEmbedder,
- // Embedder backs claim recall's KNN leg (Python
- // recall_dataset_claims, navigation.py:1837-1909, which embeds the
- // query with tools.embed_mdl — the FIRST dataset's embedding model,
- // dialog_service.py:362-366) and the structure-drill seed vector.
- // Without it the claim leg silently degrades to BM25-only and
- // paraphrase-phrased claims are never recalled. Bound to
- // kbs[0].EmbdID (validateDatasetEmbeddingModels above guarantees
- // every bound dataset shares it), query-side encoded.
- Embedder: embedderForDatasets(kbs, modelFactory),
- // Tagger is the Go equivalent of Python's label_question
- // (agentic_rag.py:668): classifies the query into question-type
- // tags the retriever boosts on. metadataService implements it
- // (service.MetadataService.LabelQuestion).
- Tagger: metadataService,
- // SystemPrompt mirrors Python RAGTools(system_prompt=
- // _render_reasoning_system_prompt(dialog, prompt_config, kwargs),
- // dialog_service.py:2084) — the dialog-level UI configuration the
- // final-answer compose appends after the agentic contract
- // (agentic_rag_graph.py:923-933).
- SystemPrompt: req.SystemPrompt,
- // CiteRules and EvidenceMaxTokens deliberately stay unset: Python's
- // dialog path constructs RAGTools WITHOUT user_defined_prompts
- // (dialog_service.py:2072-2090), so citation_prompt defaults apply,
- // and Python has no evidence-token override (the compose always
- // uses min(chat_mdl.max_length, _EVIDENCE_BUDGET_TOKENS=8000),
- // which EvidenceMaxTokens<=0 reproduces).
- }
- for _, message := range req.Messages {
- content, err := service.NormalizeOpenAIMessageContent(message["content"])
- if err != nil {
- return service.HarnessResult{}, err
- }
- role, _ := message["role"].(string)
- deps.Messages = append(deps.Messages, schema.Message{Role: schema.RoleType(role), Content: content})
- }
- // Diagnose WHY compiled expansion is disabled: NewCompiledExpander
- // returns nil for three reasons (store==nil / no datasets / no tenant)
- // and RAGTools.Expand==nil silences the whole channel with no other
- // trace — the observed "Compiled expansion enabled = 0" was invisible.
- if deps.Expand == nil {
- switch {
- case docEngine == nil:
- common.Warn("compiled expansion disabled: document engine unavailable (store==nil)")
- case len(req.DatasetIDs) == 0:
- common.Warn("compiled expansion disabled: no bound dataset (DatasetIDs empty)")
- case strings.TrimSpace(req.TenantID) == "":
- common.Warn("compiled expansion disabled: no tenant (TenantID empty)")
- default:
- common.Warn("compiled expansion disabled: unknown reason")
- }
- }
- // The two projections of one reasoning step: the sentence the chat UI
- // appends to its think block, and the structured event a step-rendering
- // client consumes. Steps.Stage/Emit feeds both from one call, so the
- // trace cannot drift from its structured twin. Each step is an
- // isThink=true delta, i.e. think-block content rather than answer text.
- if req.AnswerSink != nil {
- answerSink := req.AnswerSink
- deps.AnswerSink = &agentic_rag.AnswerSink{
- OnDelta: req.AnswerSink,
- }
- deps.Steps.Text = func(line string) { answerSink(line, true) }
- }
- if req.ThinkSink != nil {
- // One type on both sides — agenticruntime.ThinkEvent is an alias of
- // service.ThinkEvent — so the sink passes straight through: there is
- // nothing to copy field by field, and no way to forget a field that
- // was added on one side only.
- deps.Steps.Events = req.ThinkSink
- }
- r := agentic_rag.Rag(ctx, deps, agenticruntime.RunRequest{
- Question: req.Question,
- ThinkingMode: req.ThinkingMode,
- DatasetIDs: req.DatasetIDs,
- TenantID: req.TenantID,
- SessionID: req.SessionID,
- Images: req.Images,
- TextAttachments: req.TextAttachments,
- })
- res := service.HarnessResult{Chunks: r.Chunks, DocAggs: r.DocAggs, Answer: r.Answer, SlotCitations: r.SlotCitations, CiteChunkIDs: r.CiteChunkIDs}
- if r.Kbinfos != nil {
- res.Memory = r.Kbinfos.Memory
- res.PreSummary = r.Kbinfos.PreSummary
- }
- return res, nil
- }
-}
-
-// engineDocEngine is the small slice of the engine surface the doc-chunk pager
-// needs, kept as a local interface so it is trivially unit-testable without a
-// live engine (the production value is engine.Get()).
-type engineDocEngine interface {
- Search(ctx context.Context, req *et.SearchRequest) (*et.SearchResult, error)
-}
-
-// docChunkPager is the production agenticruntime.DocChunkLister. It pages one
-// document's chunks in reading order straight off the chunk index, mirroring
-// Python settings.retriever.chunk_list with sort_by_position=True
-// (rag/nlp/search.py:824). This is what the document-level tools
-// (summarize_document / fetch_full_document) consume.
-//
-// The index name is derived from the tenant id (ragflow_<tenantID>), exactly as
-// retrieval does, so it needs no user-scope resolution: DocChunksRequest
-// already carries the tenant and the bound datasets.
-type docChunkPager struct {
- docEngine engineDocEngine
-}
-
-// DocChunks implements agenticruntime.DocChunkLister.
-func (p *docChunkPager) DocChunks(ctx context.Context, req agenticruntime.DocChunksRequest) ([]map[string]any, error) {
- if p == nil || p.docEngine == nil {
- return nil, nil
- }
- if req.DocID == "" {
- return nil, nil
- }
- orderBy := (&et.OrderByExpr{}).
- Asc("chunk_order_int").
- Asc("page_num_int").
- Asc("top_int").
- Desc("create_timestamp_flt")
-
- resp, err := p.docEngine.Search(ctx, &et.SearchRequest{
- IndexNames: []string{tenantChunkIndexName(req.TenantID)},
- KbIDs: req.DatasetIDs,
- Offset: req.Offset,
- Limit: req.Limit,
- OrderBy: orderBy,
- SelectFields: []string{
- "id", "doc_id", "docnm", "content_with_weight", "content", "available_int",
- },
- Filter: map[string]interface{}{
- "doc_id": req.DocID,
- "available_int": 1,
- "must_not": map[string]interface{}{"exists": "compile_kwd"},
- },
- })
- if err != nil {
- return nil, err
- }
- if resp == nil {
- return nil, nil
- }
- rows := make([]map[string]any, 0, len(resp.Chunks))
- for _, ck := range resp.Chunks {
- row := make(map[string]any, 8)
- // The runtime chunk rows need the canonical keys its citation pipeline
- // reads: chunk_id (dedup), docnm_kwd (doc title), doc_id and the text
- // under content/content_with_weight (chunkText prefers content first).
- if id, ok := ck["id"].(string); ok && id != "" {
- row["chunk_id"] = id
- }
- if d, ok := ck["doc_id"]; ok {
- row["doc_id"] = d
- }
- if d, ok := ck["docnm"]; ok {
- row["docnm_kwd"] = d
- }
- if c, ok := ck["content"]; ok {
- row["content"] = c
- }
- if c, ok := ck["content_with_weight"]; ok {
- row["content_with_weight"] = c
- }
- // Keep whatever else the engine returned (e.g. position_int, img) so the
- // merged Kbinfos rows stay rich, as retrieval does.
- for k, v := range ck {
- if _, seen := row[k]; !seen {
- row[k] = v
- }
- }
- rows = append(rows, row)
- }
- return rows, nil
-}
-
-// hasEmbedderFor mirrors Python `embd_mdl = LLMBundle(...) if kbs and
-// kbs[0].embd_id else None` (dialog_service.py:362). The gate is the FIRST bound
-// dataset's embd_id, not "any dataset has one": a dialog whose first dataset
-// carries no embedding model yields embd_mdl=None upstream even if later ones do.
-//
-// Callers must run validateDatasetEmbeddingModels first — with validation
-// passing, all datasets share one model and this equals "all have one".
-func hasEmbedderFor(kbs []*entity.Knowledgebase) bool {
- return len(kbs) > 0 && kbs[0] != nil && kbs[0].EmbdID != ""
-}
-
-// embedderForDatasets builds the embedding handle the agentic runtime uses for
-// query-side encoding outside the main retrieval leg: claim recall's KNN leg
-// (Python recall_dataset_claims, navigation.py:1837-1909, embedding with
-// tools.embed_mdl) and the structure-drill seed vector. The model is the FIRST
-// dataset's embedding (Python dialog_service.py:362-366 resolves
-// kbs[0].embd_id under kbs[0].tenant_id; validateDatasetEmbeddingModels
-// guarantees the rest share it). Nil when no dataset carries an embedding
-// model — the claim leg then degrades to BM25-only, matching a Python run
-// without embed_mdl.
-func embedderForDatasets(kbs []*entity.Knowledgebase, modelFactory *service.ModelFactory) nlp.NavEmbedder {
- if len(kbs) == 0 || kbs[0] == nil || kbs[0].EmbdID == "" {
- return nil
- }
- return service.NewNavEmbedder(modelFactory, kbs[0].EmbdID)
-}
-
-func validateLoadedDatasets(datasetIDs []string, kbs []*entity.Knowledgebase) error {
- loaded := make(map[string]struct{}, len(kbs))
- for _, kb := range kbs {
- if kb != nil {
- loaded[kb.ID] = struct{}{}
- }
- }
- for _, id := range datasetIDs {
- if _, ok := loaded[id]; !ok {
- return fmt.Errorf("dataset %q was not found", id)
- }
- }
- return nil
-}
-
-// validateDatasetEmbeddingModels mirrors Python validate_dataset_embedding_models
-// (knowledgebase_service.py:62-94): every bound dataset must use the same embedding
-// model, or none at all. Upstream this runs before embd_mdl is resolved
-// (dialog_service.py:358-360) and a violation RAISES — it is not a "no embedding
-// model" fallback: treating mixing as HasEmbedder=false would silently serve
-// keyword-only results where Python errors.
-func validateDatasetEmbeddingModels(ctx context.Context, kbs []*entity.Knowledgebase) error {
- var withEmbd int
- for _, kb := range kbs {
- if kb != nil && kb.EmbdID != "" {
- withEmbd++
- }
- }
- hasEmbd := withEmbd > 0
- if hasEmbd && withEmbd != len(kbs) {
- return errors.New("cannot search across datasets where some have embedding models and others do not")
- }
- if !hasEmbd {
- return nil
- }
- // Mirror Python's grouping key exactly:
- //
- // ref = tenant_embd_id or (embd_id when it is a raw tenant_model id)
- // composite = resolved_names.get(ref) # "model@instance@provider"
- // key = _base_model_name(composite) # == tenant_model.model_name
- //
- // The @instance@provider suffix is stripped by _base_model_name
- // (knowledgebase_service.py:33), so only the MODEL NAME is compared — the
- // instance and provider tables are not consulted at all.
- refs := make([]string, 0, len(kbs))
- for _, kb := range kbs {
- if kb == nil || kb.EmbdID == "" {
- continue
- }
- ref := ""
- if kb.TenantEmbdID != nil {
- ref = strings.TrimSpace(*kb.TenantEmbdID)
- }
- if ref == "" && !strings.Contains(kb.EmbdID, "@") {
- ref = strings.TrimSpace(kb.EmbdID)
- }
- if ref != "" {
- refs = append(refs, ref)
- }
- }
-
- // Python resolves the refs with one batched lookup
- // (tenant_model_service.py:562 get_by_ids) and silently skips ids that no
- // longer resolve — a dangling id must not fail the request.
- resolved := make(map[string]string, len(refs))
- // A nil/absent DB or any query error must not fail the request: Python skips
- // unresolvable ids (tenant_model_service.py:557) and falls back to the raw
- // id, so validation degrades to a stricter but safe comparison.
- if len(refs) > 0 && dao.DB != nil {
- if models, err := dao.NewTenantModelDAO().GetByIDs(ctx, dao.DB, refs); err == nil {
- for _, m := range models {
- if m != nil {
- resolved[m.ID] = m.ModelName
- }
- }
- }
- }
-
- keys := make(map[string]struct{}, len(kbs))
- for _, kb := range kbs {
- if kb == nil || kb.EmbdID == "" {
- continue
- }
- embdID := strings.TrimSpace(kb.EmbdID)
- ref := ""
- if kb.TenantEmbdID != nil {
- ref = strings.TrimSpace(*kb.TenantEmbdID)
- }
- if ref == "" && !strings.Contains(embdID, "@") {
- ref = embdID
- }
- key := ""
- if name, ok := resolved[ref]; ok && name != "" {
- key = name
- } else if embdID != "" && embdID != ref {
- key = common.BaseModelName(embdID)
- } else {
- key = ref
- }
- if key != "" {
- keys[key] = struct{}{}
- }
- }
- if len(keys) > 1 {
- return fmt.Errorf("datasets use different embedding models")
- }
- return nil
-}
-
-// ---------------------------------------------------------------------------
-// Harness seams (compiled expansion + open-web search)
-// ---------------------------------------------------------------------------
-
-// engineCompiledStore implements agenticruntime.CompiledStore over the document
-// engine, backing search_chunks' compiled-structure expansion (Python
-// tools/compiled_expansion.py). The runtime owns the expansion strategy; this
-// adapter only maps its (scope, filters, free text) request onto one engine
-// search, mirroring Python's settings.docStoreConn.search calls.
-type engineCompiledStore struct {
- engine engineDocEngine
- // embed encodes the seed query for the dense leg. It must be the QUERY
- // encoding of the dataset's own embedding model (Python
- // _search_compiled_rows → _dense_expr → settings.retriever.get_vector →
- // emb_mdl.encode_queries). Nil leaves the keyword leg only.
- embed compiledQueryEncoder
- // embedTenantID is the tenant owning the embedder's model (Python
- // embd_owner_tenant_id = kbs[0].tenant_id). It is distinct from the searched
- // scope's tenant, because Python builds ONE embedder from the leading bound
- // dataset and reuses it for every scope.
- embedTenantID string
-}
-
-var _ agenticruntime.CompiledStore = (*engineCompiledStore)(nil)
-
-// compiledQueryEncoder is the query-side embedding seam used for the compiled
-// dense seed leg. *service.NavEmbedder implements it via EncodeQueries.
-type compiledQueryEncoder interface {
- EncodeQueries(ctx context.Context, tenantID string, texts []string) ([][]float32, error)
-}
-
-// Mirror Python compiled_expansion._VECTOR_NUM_CANDIDATES / _VECTOR_SIMILARITY.
-// The HNSW ef_search floor (num_candidates) must be >= topN or the ANN search is
-// bounded by the candidate list instead of by relevance.
-const (
- compiledVectorNumCandidates = 256
- compiledVectorSimilarity = 0.1
-)
-
-// newEngineCompiledStore returns nil when no engine is available so
-// agenticruntime.NewCompiledExpander disables expansion instead of searching nowhere.
-// A nil embed (or empty embedTenantID) keeps the keyword-only leg.
-func newEngineCompiledStore(e engineDocEngine, embed compiledQueryEncoder, embedTenantID string) agenticruntime.CompiledStore {
- if e == nil {
- return nil
- }
- return &engineCompiledStore{engine: e, embed: embed, embedTenantID: embedTenantID}
-}
-
-// newDatasetCompiledStore builds the per-request compiled-row store, wiring the
-// dense seed leg to the FIRST bound dataset's own embedding model. Python builds
-// embd_mdl from kbs[0] (dialog_service.py:362-365) and reuses it for the whole
-// expansion; using the tenant-default model instead would compare the query
-// against rows embedded by a different model — different vector spaces, so the
-// dense hits would be noise.
-func newDatasetCompiledStore(e engineDocEngine, modelFactory *service.ModelFactory, kbs []*entity.Knowledgebase) agenticruntime.CompiledStore {
- if e == nil {
- return nil
- }
- if hasEmbedderFor(kbs) {
- return newEngineCompiledStore(e, service.NewNavEmbedder(modelFactory, kbs[0].EmbdID), kbs[0].TenantID)
- }
- return newEngineCompiledStore(e, nil, "")
-}
-
-const tenantChunkIndexPrefix = "ragflow_"
-
-func tenantChunkIndexName(tenantID string) string {
- return tenantChunkIndexPrefix + tenantID
-}
-
-// compiledSynthesisKinds are the standalone synthesis-page compile_kwd values.
-// Python scopes those rows by source_doc_ids, while entity/relation rows are
-// scoped by doc_id (compiled_expansion.py _search_compiled_rows /
-// _search_synthesis_pages).
-var compiledSynthesisKinds = map[string]bool{
- "wiki_page": true,
- "artifact_page": true,
- "essence": true,
-}
-
-// compiledRowSelectFields is the projection the expander reads off a compiled
-// row: content_with_weight (seed name parsing), the entity/relation endpoints,
-// exact name lookups, and provenance back to source chunks.
-var compiledRowSelectFields = []string{
- "id", "kb_id", "doc_id", "docnm_kwd",
- "content_with_weight", "summary_with_weight", "source_chunk_ids",
- "name_kwd", "from_entity_kwd", "to_entity_kwd", "available_int",
-}
-
-// compiledChunkSelectFields is the projection for source chunks promoted into
-// evidence by compiled expansion.
-var compiledChunkSelectFields = []string{
- "id", "kb_id", "doc_id", "docnm_kwd",
- "content_with_weight", "source_chunk_ids", "similarity",
-}
-
-// SearchCompiled implements agenticruntime.CompiledStore.
-func (s *engineCompiledStore) SearchCompiled(ctx context.Context, kbID, tenantID string, docIDs []string, filters map[string][]string, matchText string, topN int) ([]map[string]any, error) {
- if s == nil || s.engine == nil || kbID == "" || strings.TrimSpace(tenantID) == "" {
- return nil, nil
- }
- if topN <= 0 {
- topN = 1
- }
- filter := compiledEngineFilter(filters)
- if len(docIDs) > 0 {
- filter[compiledDocScopeKey(filters)] = docIDs
- }
- req := &et.SearchRequest{
- IndexNames: []string{tenantChunkIndexName(tenantID)},
- KbIDs: []string{kbID},
- Limit: topN,
- SelectFields: compiledRowSelectFields,
- Filter: filter,
- }
- // Python prefers a dense seed leg and only falls back to a keyword match
- // when no embedder is available or the vector build fails
- // (compiled_expansion.py _search_compiled_rows:180-194 /
- // _search_synthesis_pages:423-438). Exactly ONE expr is appended.
- if text := strings.TrimSpace(matchText); text != "" {
- if dense := s.denseExpr(ctx, text, topN); dense != nil {
- req.MatchExprs = []interface{}{dense}
- } else {
- req.MatchExprs = []interface{}{&et.MatchTextExpr{
- Fields: []string{"content_ltks", "content_sm_ltks"},
- MatchingText: text,
- TopN: topN,
- }}
- }
- }
- res, err := s.engine.Search(ctx, req)
- if err != nil || res == nil {
- return nil, err
- }
- return res.Chunks, nil
-}
-
-// denseExpr builds the compiled-row dense match for the seed text, mirroring
-// Python _dense_expr (compiled_expansion.py:31-47): the query is embedded with
-// the dataset model's QUERY encoding, and the match carries the HNSW
-// num_candidates floor and similarity threshold. Returns nil when no embedder is
-// wired, the encode fails, or it panics — every caller then uses the keyword leg,
-// exactly like Python's `except Exception` around get_vector.
-func (s *engineCompiledStore) denseExpr(ctx context.Context, text string, topN int) *et.MatchDenseExpr {
- if s == nil || s.embed == nil || strings.TrimSpace(s.embedTenantID) == "" {
- return nil
- }
- var (
- vecs [][]float32
- err error
- )
- func() {
- defer func() {
- if r := recover(); r != nil {
- common.Warn("compiled expansion: seed encode panicked; falling back to keyword match")
- }
- }()
- vecs, err = s.embed.EncodeQueries(ctx, s.embedTenantID, []string{text})
- }()
- if err != nil || len(vecs) == 0 || len(vecs[0]) == 0 {
- return nil
- }
- vec := make([]float64, len(vecs[0]))
- for i, v := range vecs[0] {
- vec[i] = float64(v)
- }
- return &et.MatchDenseExpr{
- VectorColumnName: fmt.Sprintf("q_%d_vec", len(vec)),
- EmbeddingData: vec,
- EmbeddingDataType: "float",
- DistanceType: "cosine",
- TopN: topN,
- ExtraOptions: map[string]interface{}{
- "similarity": compiledVectorSimilarity,
- "num_candidates": max(topN, compiledVectorNumCandidates),
- },
- }
-}
-
-// LoadChunks implements agenticruntime.CompiledStore: fetch the referenced source
-// chunks by id (Python compiled_expansion.py:_load_chunks_for_doc searches the
-// tenant index with an {"id": chunk_ids} condition).
-func (s *engineCompiledStore) LoadChunks(ctx context.Context, kbID, tenantID string, chunkIDs []string) ([]map[string]any, error) {
- if s == nil || s.engine == nil || kbID == "" || strings.TrimSpace(tenantID) == "" || len(chunkIDs) == 0 {
- return nil, nil
- }
- res, err := s.engine.Search(ctx, &et.SearchRequest{
- IndexNames: []string{tenantChunkIndexName(tenantID)},
- KbIDs: []string{kbID},
- Limit: len(chunkIDs),
- SelectFields: compiledChunkSelectFields,
- Filter: map[string]interface{}{"id": chunkIDs},
- })
- if err != nil || res == nil {
- return nil, err
- }
- return res.Chunks, nil
-}
-
-// Vectorize implements agenticruntime.CompiledStore. The ported expander assigns
-// compiled chunks the similarity stored on the row and blends by that field, so
-// expansion itself never needs a query embedding.
-func (s *engineCompiledStore) Vectorize(context.Context, string) ([]float64, error) {
- return nil, fmt.Errorf("compiled store: vectorize is not wired")
-}
-
-// compiledDocScopeKey mirrors Python's doc-scope column choice: synthesis pages
-// are scoped by source_doc_ids, entity/relation rows by doc_id.
-func compiledDocScopeKey(filters map[string][]string) string {
- for _, ck := range filters["compile_kwd"] {
- if compiledSynthesisKinds[strings.ToLower(strings.TrimSpace(ck))] {
- return "source_doc_ids"
- }
- }
- return "doc_id"
-}
-
-// compiledEngineFilter maps the runtime' OR-list filters onto the engine filter
-// shape. available_int is a numeric column, so its string values are coerced to
-// ints (Python passes the literal 1).
-func compiledEngineFilter(filters map[string][]string) map[string]interface{} {
- out := make(map[string]interface{}, len(filters))
- for k, vals := range filters {
- if k == "available_int" {
- ints := make([]int, 0, len(vals))
- for _, v := range vals {
- if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil {
- ints = append(ints, n)
- }
- }
- if len(ints) > 0 {
- out[k] = ints
- }
- continue
- }
- out[k] = vals
- }
- return out
-}
-
-// harnessWebSearcherFunc adapts the chat pipeline's web-search callback to the
-// agenticruntime.WebSearcher seam consumed by the web_search tool. The callback is a
-// plain func so internal/service does not have to depend on the runtime package.
-type harnessWebSearcherFunc func(ctx context.Context, queries []string) ([]string, error)
-
-// Search implements agenticruntime.WebSearcher.
-func (f harnessWebSearcherFunc) Search(ctx context.Context, queries []string) ([]string, error) {
- return f(ctx, queries)
-}
-
-var _ agenticruntime.WebSearcher = harnessWebSearcherFunc(nil)
-
-// harnessWebSearcher wraps a non-nil callback, returning a nil WebSearcher when
-// the pipeline did not supply one — which is what hides the web_search tool.
-func harnessWebSearcher(fn func(context.Context, []string) ([]string, error)) agenticruntime.WebSearcher {
- if fn == nil {
- return nil
- }
- return harnessWebSearcherFunc(fn)
-}
diff --git a/internal/agent/retrievalbridge/harness_test.go b/internal/agent/retrievalbridge/harness_test.go
deleted file mode 100644
index 01cf5210bd..0000000000
--- a/internal/agent/retrievalbridge/harness_test.go
+++ /dev/null
@@ -1,462 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package retrievalbridge
-
-import (
- "context"
- "errors"
- "github.com/cloudwego/eino/schema"
- "log"
- "ragflow/internal/common"
- "ragflow/internal/engine/types"
- "ragflow/internal/entity"
- "ragflow/internal/rag/agentic-rag"
- agenticruntime "ragflow/internal/rag/agentic-rag/runtime"
- "ragflow/internal/service"
- "reflect"
- "strings"
- "testing"
-)
-
-func kb(embdID string, tenantEmbdID string) *entity.Knowledgebase {
- kb := &entity.Knowledgebase{EmbdID: embdID}
- if tenantEmbdID != "" {
- kb.TenantEmbdID = &tenantEmbdID
- }
- return kb
-}
-
-// TestHasEmbedderForMirrorsPython pins dialog_service.py:362 —
-// `embd_mdl = LLMBundle(...) if kbs and kbs[0].embd_id else None`. The gate is
-// the FIRST dataset's embd_id, so a dialog whose leading dataset has no model
-// yields no embedding model even when a later one does.
-func TestHasEmbedderForMirrorsPython(t *testing.T) {
- cases := []struct {
- name string
- kbs []*entity.Knowledgebase
- want bool
- }{
- {"no datasets", nil, false},
- {"single dataset with model", []*entity.Knowledgebase{kb("bge-m3", "")}, true},
- {"single dataset without model", []*entity.Knowledgebase{kb("", "")}, false},
- // Python reads kbs[0].embd_id only; it does not scan for any dataset
- // carrying a model.
- {"first without, later with", []*entity.Knowledgebase{kb("", ""), kb("bge-m3", "")}, false},
- {"all with the same model", []*entity.Knowledgebase{kb("bge-m3", ""), kb("bge-m3", "")}, true},
- }
- for _, c := range cases {
- if got := hasEmbedderFor(c.kbs); got != c.want {
- t.Errorf("%s: hasEmbedderFor = %v, want %v", c.name, got, c.want)
- }
- }
-}
-
-// TestValidateDatasetEmbeddingModels pins knowledgebase_service.py:62-94.
-// Mixing is a hard error upstream (get_models raises at dialog_service.py:358) —
-// NOT a silent fallback to keyword-only retrieval.
-func TestValidateDatasetEmbeddingModels(t *testing.T) {
- t.Run("all without a model is allowed", func(t *testing.T) {
- if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{
- kb("", ""), kb("", ""),
- }); err != nil {
- t.Errorf("all-without must be allowed, got %v", err)
- }
- })
-
- t.Run("all same model is allowed", func(t *testing.T) {
- if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{
- kb("bge-m3", ""), kb("bge-m3", ""),
- }); err != nil {
- t.Errorf("same model must be allowed, got %v", err)
- }
- })
-
- t.Run("some with some without is an error", func(t *testing.T) {
- if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{
- kb("bge-m3", ""), kb("", ""),
- }); err == nil {
- t.Error("mixing present/absent embedding models must error (Python raises)")
- }
- })
-
- t.Run("different models is an error", func(t *testing.T) {
- if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{
- kb("bge-m3", ""), kb("text-embedding-3", ""),
- }); err == nil {
- t.Error("different embedding models must error (Python raises)")
- }
- })
-
- t.Run("dangling tenant_model id falls back to embd_id base name", func(t *testing.T) {
- // Python silently skips ids that no longer resolve
- // (tenant_model_service.py:557) and falls back to
- // _base_model_name(embd_id) — it must not fail the request.
- if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{
- kb("bge-m3@inst@prov", "gone-1"), kb("bge-m3@inst2@prov2", "gone-2"),
- }); err != nil {
- t.Errorf("two dangling refs with the same base name must be allowed, got %v", err)
- }
- })
-}
-
-func TestValidateLoadedDatasetsRejectsMissing(t *testing.T) {
- err := validateLoadedDatasets([]string{"kb-1", "kb-2"}, []*entity.Knowledgebase{{ID: "kb-1"}})
- if err == nil || !strings.Contains(err.Error(), `dataset "kb-2" was not found`) {
- t.Fatalf("validateLoadedDatasets error = %v, want missing kb-2", err)
- }
-}
-
-// TestBaseModelNameMirrorsPython pins knowledgebase_service.py:33:
-// rsplit("@", 2)[0] — the model name, with @instance@provider stripped.
-func TestBaseModelNameMirrorsPython(t *testing.T) {
- cases := []struct{ in, want string }{
- {"bge-m3@inst@prov", "bge-m3"},
- {"bge-m3@inst", "bge-m3"},
- {"bge-m3", "bge-m3"},
- {"", ""},
- }
- for _, c := range cases {
- if got := common.BaseModelName(c.in); got != c.want {
- t.Errorf("baseModelName(%q) = %q, want %q", c.in, got, c.want)
- }
- }
-}
-
-// TestHarnessWebSearcherNilAndWrapped pins the adapter: a nil pipeline callback
-// must yield a nil harness.WebSearcher (which hides the web_search tool), and a
-// non-nil one must be callable through the seam.
-func TestHarnessWebSearcherNilAndWrapped(t *testing.T) {
- if got := harnessWebSearcher(nil); got != nil {
- t.Error("nil callback must yield a nil WebSearcher")
- }
- called := false
- searcher := harnessWebSearcher(func(_ context.Context, queries []string) ([]string, error) {
- called = true
- return queries, nil
- })
- if searcher == nil {
- t.Fatal("non-nil callback must yield a WebSearcher")
- }
- got, err := searcher.Search(context.Background(), []string{"q"})
- if err != nil {
- t.Fatalf("Search: %v", err)
- }
- if !called {
- t.Error("callback was not invoked")
- }
- if len(got) != 1 || got[0] != "q" {
- t.Errorf("Search = %v, want [q]", got)
- }
-}
-
-// errStubEncode stands in for a provider-side embedding failure.
-var errStubEncode = errors.New("stub encode failure")
-
-// recordingEngine captures the SearchRequest so the tests can pin the adapter's
-// mapping without a live document engine.
-type recordingEngine struct {
- last *types.SearchRequest
- res *types.SearchResult
-}
-
-func (e *recordingEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
- e.last = req
- if e.res != nil {
- return e.res, nil
- }
- return &types.SearchResult{}, nil
-}
-
-// TestEngineCompiledStoreSearchCompiledMapsRequest pins the mapping from the
-// harness' compiled-row request onto the engine search: tenant index, kb scope,
-// numeric available_int, the synthesis-page doc-scope column (source_doc_ids),
-// and the keyword match leg over content_ltks/content_sm_ltks.
-func TestEngineCompiledStoreSearchCompiledMapsRequest(t *testing.T) {
- eng := &recordingEngine{}
- // No embedder wired → the keyword leg stands alone.
- store := newEngineCompiledStore(eng, nil, "")
- _, err := store.SearchCompiled(context.Background(), "kb1", "t1", []string{"d1"}, map[string][]string{
- "compile_kwd": {"wiki_page"},
- "available_int": {"1"},
- }, "culdcept", 7)
- if err != nil {
- t.Fatalf("SearchCompiled: %v", err)
- }
- req := eng.last
- if req == nil {
- t.Fatal("engine was not called")
- }
- if len(req.IndexNames) != 1 || req.IndexNames[0] != "ragflow_t1" {
- t.Errorf("IndexNames = %v, want [ragflow_t1]", req.IndexNames)
- }
- if len(req.KbIDs) != 1 || req.KbIDs[0] != "kb1" {
- t.Errorf("KbIDs = %v, want [kb1]", req.KbIDs)
- }
- if req.Limit != 7 {
- t.Errorf("Limit = %d, want 7", req.Limit)
- }
- // available_int is a numeric column: "1" must reach the engine as an int.
- if got, ok := req.Filter["available_int"].([]int); !ok || len(got) != 1 || got[0] != 1 {
- t.Errorf("available_int = %#v, want []int{1}", req.Filter["available_int"])
- }
- if got, ok := req.Filter["compile_kwd"].([]string); !ok || len(got) != 1 || got[0] != "wiki_page" {
- t.Errorf("compile_kwd = %#v, want []string{wiki_page}", req.Filter["compile_kwd"])
- }
- // Synthesis pages are scoped by source_doc_ids, not doc_id.
- if _, ok := req.Filter["source_doc_ids"]; !ok {
- t.Errorf("doc scope key = %v, want source_doc_ids present", req.Filter)
- }
- if _, ok := req.Filter["doc_id"]; ok {
- t.Errorf("synthesis rows must not use doc_id scope: %v", req.Filter)
- }
- if len(req.MatchExprs) != 1 {
- t.Fatalf("MatchExprs = %#v, want one keyword expr", req.MatchExprs)
- }
- mt, ok := req.MatchExprs[0].(*types.MatchTextExpr)
- if !ok {
- t.Fatalf("MatchExprs[0] = %T, want *types.MatchTextExpr", req.MatchExprs[0])
- }
- if mt.MatchingText != "culdcept" || mt.TopN != 7 {
- t.Errorf("keyword expr = %+v, want text=culdcept topN=7", mt)
- }
- if len(mt.Fields) != 2 || mt.Fields[0] != "content_ltks" || mt.Fields[1] != "content_sm_ltks" {
- t.Errorf("keyword fields = %v, want [content_ltks content_sm_ltks]", mt.Fields)
- }
-}
-
-// TestCompiledDocScopeKey pins Python's column choice: entity/relation rows use
-// doc_id; the three synthesis kinds use source_doc_ids.
-func TestCompiledDocScopeKey(t *testing.T) {
- cases := []struct {
- name string
- filters map[string][]string
- want string
- }{
- {"entity rows", map[string][]string{"knowledge_graph_kwd": {"entity"}}, "doc_id"},
- {"tree rows", map[string][]string{"knowledge_graph_kwd": {"entity"}, "compile_kwd": {"tree"}}, "doc_id"},
- {"wiki pages", map[string][]string{"compile_kwd": {"wiki_page"}}, "source_doc_ids"},
- {"artifact pages", map[string][]string{"compile_kwd": {"artifact_page"}}, "source_doc_ids"},
- {"essence", map[string][]string{"compile_kwd": {"essence"}}, "source_doc_ids"},
- }
- for _, c := range cases {
- if got := compiledDocScopeKey(c.filters); got != c.want {
- t.Errorf("%s: compiledDocScopeKey = %q, want %q", c.name, got, c.want)
- }
- }
-}
-
-// TestEngineCompiledStoreLoadChunks pins the id-scoped fetch used to promote
-// source chunks into evidence.
-func TestEngineCompiledStoreLoadChunks(t *testing.T) {
- eng := &recordingEngine{res: &types.SearchResult{Chunks: []map[string]interface{}{{"id": "c1"}}}}
- store := newEngineCompiledStore(eng, nil, "")
- rows, err := store.LoadChunks(context.Background(), "kb1", "t1", []string{"c1", "c2"})
- if err != nil {
- t.Fatalf("LoadChunks: %v", err)
- }
- if len(rows) != 1 {
- t.Fatalf("rows = %d, want 1", len(rows))
- }
- if eng.last.Limit != 2 {
- t.Errorf("Limit = %d, want 2", eng.last.Limit)
- }
- ids, ok := eng.last.Filter["id"].([]string)
- if !ok || len(ids) != 2 || ids[0] != "c1" || ids[1] != "c2" {
- t.Errorf("id filter = %#v, want [c1 c2]", eng.last.Filter["id"])
- }
-}
-
-// TestNewDatasetCompiledStore pins the store wiring: no engine yields no store;
-// datasets without an embedding model yield a keyword-only store; a dataset
-// embedding model wires the query encoder with THAT dataset's model and owner
-// tenant (not a tenant default or the request tenant).
-func TestNewDatasetCompiledStore(t *testing.T) {
- eng := &recordingEngine{}
- if got := newDatasetCompiledStore(nil, nil, nil); got != nil {
- t.Error("nil engine must disable the compiled store")
- }
- if got := newDatasetCompiledStore(eng, nil, []*entity.Knowledgebase{{ID: "kb1", TenantID: "t1"}}); got == nil {
- t.Error("engine without an embedder must still build a keyword-only store")
- }
- store := newDatasetCompiledStore(eng, service.NewModelFactory(),
- []*entity.Knowledgebase{{ID: "kb1", EmbdID: "bge-m3@inst@prov", TenantID: "owner-tenant"}})
- es, ok := store.(*engineCompiledStore)
- if !ok {
- t.Fatalf("store = %T, want *engineCompiledStore", store)
- }
- if es.embed == nil {
- t.Fatal("the dataset's embedding model must wire the query encoder")
- }
- if es.embedTenantID != "owner-tenant" {
- t.Errorf("embedTenantID = %q, want owner-tenant (the dataset owner, not the request tenant)", es.embedTenantID)
- }
-}
-
-// stubQueryEncoder records the query-encoding calls the dense leg makes.
-type stubQueryEncoder struct {
- vecs [][]float32
- err error
- calls int
- lastTenant string
-}
-
-func (s *stubQueryEncoder) EncodeQueries(_ context.Context, tenantID string, _ []string) ([][]float32, error) {
- s.calls++
- s.lastTenant = tenantID
- return s.vecs, s.err
-}
-
-// TestEngineCompiledStoreUsesDenseSeedLeg pins the parity fix: when an embedder
-// is wired, the compiled-row search goes out as a DENSE expr on the query
-// vector's q_<dim>_vec column with Python's num_candidates floor and similarity
-// threshold — and carries NO keyword expr alongside it (Python appends exactly
-// one). The encoding runs under the embedder's OWN tenant (Python
-// embd_owner_tenant_id = kbs[0].tenant_id), not the searched scope's tenant.
-func TestEngineCompiledStoreUsesDenseSeedLeg(t *testing.T) {
- eng := &recordingEngine{}
- enc := &stubQueryEncoder{vecs: [][]float32{{1, 2, 3}}}
- // Scope tenant differs from the embedder's owner tenant on purpose.
- store := newEngineCompiledStore(eng, enc, "owner-tenant")
-
- if _, err := store.SearchCompiled(context.Background(), "kb1", "scope-tenant", nil,
- map[string][]string{"knowledge_graph_kwd": {"entity"}}, "culdcept", 5); err != nil {
- t.Fatalf("SearchCompiled: %v", err)
- }
- if enc.calls != 1 {
- t.Fatalf("encode calls = %d, want 1", enc.calls)
- }
- if enc.lastTenant != "owner-tenant" {
- t.Errorf("encode tenant = %q, want the embedder's owner tenant owner-tenant", enc.lastTenant)
- }
- if len(eng.last.MatchExprs) != 1 {
- t.Fatalf("MatchExprs = %#v, want exactly one dense expr", eng.last.MatchExprs)
- }
- dense, ok := eng.last.MatchExprs[0].(*types.MatchDenseExpr)
- if !ok {
- t.Fatalf("MatchExprs[0] = %T, want *types.MatchDenseExpr", eng.last.MatchExprs[0])
- }
- if dense.VectorColumnName != "q_3_vec" {
- t.Errorf("VectorColumnName = %q, want q_3_vec", dense.VectorColumnName)
- }
- if dense.EmbeddingDataType != "float" || dense.DistanceType != "cosine" || dense.TopN != 5 {
- t.Errorf("dense expr = %+v, want float/cosine/topN=5", dense)
- }
- if dense.ExtraOptions["similarity"] != 0.1 {
- t.Errorf("similarity = %v, want 0.1", dense.ExtraOptions["similarity"])
- }
- // max(topN, 256) — a small topN is floored to the HNSW candidate count.
- if dense.ExtraOptions["num_candidates"] != 256 {
- t.Errorf("num_candidates = %v, want 256 (max(5,256))", dense.ExtraOptions["num_candidates"])
- }
-
- // A topN above the floor is passed through unchanged.
- if _, err := store.SearchCompiled(context.Background(), "kb1", "scope-tenant", nil,
- map[string][]string{"knowledge_graph_kwd": {"entity"}}, "culdcept", 300); err != nil {
- t.Fatalf("SearchCompiled(300): %v", err)
- }
- dense300 := eng.last.MatchExprs[0].(*types.MatchDenseExpr)
- if dense300.ExtraOptions["num_candidates"] != 300 {
- t.Errorf("num_candidates = %v, want 300", dense300.ExtraOptions["num_candidates"])
- }
-}
-
-// TestEngineCompiledStoreDenseFallsBackToKeyword pins Python's `except Exception`
-// around get_vector: an encode failure (or an empty vector) must degrade to the
-// keyword leg rather than dropping the expansion.
-func TestEngineCompiledStoreDenseFallsBackToKeyword(t *testing.T) {
- for _, c := range []struct {
- name string
- enc *stubQueryEncoder
- }{
- {"encode error", &stubQueryEncoder{err: errStubEncode}},
- {"empty vector", &stubQueryEncoder{vecs: [][]float32{{}}}},
- {"no vectors", &stubQueryEncoder{vecs: nil}},
- } {
- t.Run(c.name, func(t *testing.T) {
- eng := &recordingEngine{}
- store := newEngineCompiledStore(eng, c.enc, "t1")
- if _, err := store.SearchCompiled(context.Background(), "kb1", "t1", nil,
- map[string][]string{"knowledge_graph_kwd": {"entity"}}, "culdcept", 5); err != nil {
- t.Fatalf("SearchCompiled: %v", err)
- }
- if c.enc.calls != 1 {
- t.Errorf("encode calls = %d, want 1", c.enc.calls)
- }
- if len(eng.last.MatchExprs) != 1 {
- t.Fatalf("MatchExprs = %#v, want one keyword expr", eng.last.MatchExprs)
- }
- mt, ok := eng.last.MatchExprs[0].(*types.MatchTextExpr)
- if !ok {
- t.Fatalf("MatchExprs[0] = %T, want *types.MatchTextExpr keyword fallback", eng.last.MatchExprs[0])
- }
- if mt.MatchingText != "culdcept" {
- t.Errorf("keyword text = %q, want culdcept", mt.MatchingText)
- }
- })
- }
-}
-
-// Exercise the production bridge: history must reach the graph and its
-// formalizer rather than being replaced by only the latest question.
-func TestHarnessRetrieverForwardsConversation(t *testing.T) {
- model := &historyModel{}
- called := false
- agentic_rag.SetAgenticLoop(func(ctx context.Context, deps agentic_rag.RAGTools, req agenticruntime.RunRequest, _ *agenticruntime.Kbinfos, resp *agentic_rag.RunResponse, _ *log.Logger) {
- called = true
- want := []schema.Message{{Role: schema.User, Content: "Tell me about Culdcept."}, {Role: schema.Assistant, Content: "Culdcept was made by OmiyaSoft."}, {Role: schema.User, Content: "When was it released?"}}
- if !reflect.DeepEqual(deps.Messages, want) {
- t.Fatalf("history lost: %#v", deps.Messages)
- }
- resp.Answer, _ = agentic_rag.Formalize(ctx, agenticruntime.SessionDeps{Model: model}, deps.Messages, 8192)
- })
- t.Cleanup(func() { agentic_rag.SetAgenticLoop(nil) })
- req := service.HarnessRequest{Question: "When was it released?", ThinkingMode: "high", Messages: []map[string]interface{}{
- {"role": "user", "content": "Tell me about Culdcept."},
- {"role": "assistant", "content": "Culdcept was made by OmiyaSoft."},
- {"role": "user", "content": []interface{}{map[string]interface{}{"type": "text", "text": "When was it released?"}}},
- }}
- got, err := NewHarnessRetriever(nil, nil, nil)(t.Context(), req)
- if err != nil {
- t.Fatal(err)
- }
- if !called || model.calls != 1 || got.Answer != "When was Culdcept released?" || !strings.Contains(model.transcript, "OmiyaSoft") {
- t.Fatalf("formalization lost conversation: result=%+v model=%+v", got, model)
- }
- req.Messages[0]["content"] = 17
- if _, err := NewHarnessRetriever(nil, nil, nil)(t.Context(), req); err == nil {
- t.Fatal("invalid history content accepted")
- }
-}
-
-type historyModel struct {
- transcript string
- calls int
-}
-
-func (m *historyModel) Complete(_ context.Context, messages []schema.Message, _ []agenticruntime.ToolSpec) (*agenticruntime.ModelReply, error) {
- m.calls++
- for _, msg := range messages {
- m.transcript += msg.Content
- }
- return &agenticruntime.ModelReply{Content: `{"question":"When was Culdcept released?","keywords":"Culdcept released"}`}, nil
-}
-
-func TestEmbeddingValidationPreservesAtInModelNames(t *testing.T) {
- if err := validateDatasetEmbeddingModels(t.Context(), []*entity.Knowledgebase{kb("model@q4@instance@provider", ""), kb("model@q8@instance@provider", "")}); err == nil {
- t.Fatal("different quantized models were conflated")
- }
-}
diff --git a/internal/agent/retrievalbridge/runtime_adapter.go b/internal/agent/retrievalbridge/runtime_adapter.go
index e205f30e12..5c7370ca12 100644
--- a/internal/agent/retrievalbridge/runtime_adapter.go
+++ b/internal/agent/retrievalbridge/runtime_adapter.go
@@ -28,8 +28,7 @@ import (
// RuntimeAdapter exposes the agent tool's NLPRetrievalAdapter (the production
// hybrid-retrieval backend installed by the server) through the runtime
-// RetrievalService singleton that the agentic-RAG harness reads via
-// harness.RuntimeRetriever.
+// RetrievalService singleton the smart-reasoning agent reads per call.
//
// The wrapped service is captured at construction, NOT resolved from the runtime
// singleton on every call: agenttool.SetRetrievalService and
@@ -38,7 +37,7 @@ import (
// at call time therefore returned this adapter itself, and Search called itself
// until the stack overflowed on every agentic retrieval.
//
-// This keeps a single retrieval implementation: the harness and the canvas both
+// This keeps a single retrieval implementation: the agent and the canvas both
// search through the same NLPRetrievalAdapter.
type RuntimeAdapter struct {
svc agenttool.RetrievalService
@@ -58,10 +57,10 @@ func (a *RuntimeAdapter) Search(ctx context.Context, db *gorm.DB, req agentrunt.
if svc == nil {
return nil, agenttool.ErrRetrievalServiceMissing
}
- // The harness calls the runtime service with a nil *gorm.DB (see
- // harness.RuntimeRetriever.Retrieve), while the underlying NLPRetrievalAdapter
- // needs a real connection to resolve knowledge bases. Fall back to the
- // process-wide DB handle used by the rest of the chat pipeline.
+ // Callers of the runtime service pass a nil *gorm.DB, while the underlying
+ // NLPRetrievalAdapter needs a real connection to resolve knowledge bases.
+ // Fall back to the process-wide DB handle used by the rest of the chat
+ // pipeline.
if db == nil {
db = dao.DB
}
diff --git a/internal/agent/retrievalbridge/runtime_adapter_test.go b/internal/agent/retrievalbridge/runtime_adapter_test.go
index a81632a01b..ffcfca724c 100644
--- a/internal/agent/retrievalbridge/runtime_adapter_test.go
+++ b/internal/agent/retrievalbridge/runtime_adapter_test.go
@@ -8,7 +8,6 @@ import (
agentrunt "ragflow/internal/agent/runtime"
agenttool "ragflow/internal/agent/tool"
- agenticruntime "ragflow/internal/rag/agentic-rag/runtime"
"gorm.io/gorm"
)
@@ -82,29 +81,6 @@ func TestRuntimeAdapterPreservesCompiledExclusion(t *testing.T) {
}
}
-func TestRuntimeRetrieverSurvivesProductionBridge(t *testing.T) {
- previous := agentrunt.GetRetrievalService()
- service := &captureRetrievalService{}
- agentrunt.SetRetrievalService(NewRuntimeAdapter(service))
- t.Cleanup(func() { agentrunt.SetRetrievalService(previous) })
-
- filter := map[string]any{"method": "manual", "value": "invoice"}
- _, err := (&agenticruntime.RuntimeRetriever{}).Retrieve(t.Context(), agenticruntime.RetrieveRequest{
- Query: "test",
- MetaDataFilter: filter,
- ExcludeCompiled: true,
- })
- if err != nil {
- t.Fatal(err)
- }
- if !reflect.DeepEqual(service.req.MetaDataFilter, filter) {
- t.Fatalf("MetaDataFilter = %#v, want %#v", service.req.MetaDataFilter, filter)
- }
- if !service.req.ExcludeCompiled {
- t.Fatal("ExcludeCompiled was dropped across RuntimeRetriever and RuntimeAdapter")
- }
-}
-
func TestAgenticSearchSurvivesProductionRegistryBridge(t *testing.T) {
previous := agenttool.GetRetrievalService()
service := &captureRetrievalService{}
diff --git a/internal/agentic_rag/kg_service.go b/internal/agentic_rag/kg_service.go
index 61fdb6046a..770fd3bdea 100644
--- a/internal/agentic_rag/kg_service.go
+++ b/internal/agentic_rag/kg_service.go
@@ -29,17 +29,14 @@ import (
enginetypes "ragflow/internal/engine/types"
)
-// Knowledge-graph walk backing the graph_explore tool. It is the agentic-RAG
-// port of the harness walk (internal/rag/agentic-rag/runtime/tool_exploration.go):
-// seed entity rows for the query, hop out over relation rows, and hand the
-// subgraph plus the source passages behind it back to the model.
+// Knowledge-graph walk backing the graph_explore tool: seed entity rows for
+// the query, hop out over relation rows, and hand the subgraph plus the
+// source passages behind it back to the model.
//
-// Deliberate deviation from the harness original: there the walk asks the chat
-// model whether the subgraph answers the question and short-circuits with a
-// bare answer when it does. This port never calls a model. In this ReAct loop
-// the model is already the judge of its own evidence, a tool holds no chat
-// model, and an answer-only result would strip the passages the answer has to
-// cite. The walk therefore always returns the subgraph and its passages.
+// The walk never calls a chat model. In this ReAct loop the model is already
+// the judge of its own evidence, a tool holds no chat model, and an
+// answer-only result would strip the passages the answer has to cite. The
+// walk therefore always returns the subgraph and its passages.
// Error returned when no KgService was installed at boot.
var ErrKgServiceMissing = errors.New(
diff --git a/internal/agentic_rag/tool_navigate_structure.go b/internal/agentic_rag/tool_navigate_structure.go
index 7fba9f6d8b..440b3998d5 100644
--- a/internal/agentic_rag/tool_navigate_structure.go
+++ b/internal/agentic_rag/tool_navigate_structure.go
@@ -20,13 +20,11 @@ package agentic_rag
// <structure_navigation> outline with chunk pointers the model can feed to
// list_chunks.
//
-// This is the experiment-grade port of the harness navigate_structure
-// (internal/rag/agentic-rag/runtime/tool_navigation.go). The harness version
-// was never wired to a Go-side structure reader in production, so this tool
-// reads the compiled-structure rows directly from the doc-store and reuses the
-// harness's keyword-relevance beam drill (its documented no-vector fallback).
-// The dense-vector seed and claim-recall legs are intentionally omitted here;
-// they are additive and can be layered on without changing the tool's contract.
+// This tool reads the compiled-structure rows directly from the doc-store and
+// drills them with a keyword-relevance beam, the documented no-vector
+// fallback. The dense-vector seed and claim-recall legs are intentionally
+// omitted; they are additive and can be layered on without changing the
+// tool's contract.
//
// Typical use: call navigate_tree first to route to a document, take its
// doc_id, then call navigate_structure with that doc_id to see where the answer
diff --git a/internal/rag/agentic-rag/agentic_rag.go b/internal/rag/agentic-rag/agentic_rag.go
deleted file mode 100644
index d7297258a0..0000000000
--- a/internal/rag/agentic-rag/agentic_rag.go
+++ /dev/null
@@ -1,2521 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-// Package agentic_rag is the outer agentic-search loop (medium / high / ultra).
-//
-// This file holds the run's configuration and entry-point logic: the run
-// configuration (RAGTools), the rag entry point (Rag) and question formalization
-// (Formalize). The shared capability carrier that the agentic graph and the runtime
-// action session both read from and write to is runtime.Toolset (see
-// runtime/action_session.go); this file holds the RAGTools-side logic.
-//
-// The package is laid out in three layers:
-// - agentic_rag.go the run configuration and entry points
-// - agentic_rag_graph.go the pipeline (nodes, routing, assembly)
-// - runtime/ the leaf primitives
-package agentic_rag
-
-import (
- "context"
- "errors"
- "fmt"
- "log"
- "regexp"
- "strings"
- "sync"
- "time"
-
- "github.com/cloudwego/eino/schema"
-
- "ragflow/internal/agent/chat"
- "ragflow/internal/dao"
- "ragflow/internal/entity"
- "ragflow/internal/entity/models"
- "ragflow/internal/rag/agentic-rag/runtime"
- "ragflow/internal/rag/prompts"
- "ragflow/internal/service/nlp"
-)
-
-// Capability map — where each part of the run lives.
-//
-// The RAGTools methods live in this package (this file and
-// agentic_rag_graph.go), while runtime/ stays the leaf capability library they
-// orchestrate (retrieval, sessions, tools). Nothing below is a forwarding shim —
-// the right-hand side IS the implementation:
-//
-// run configuration → RAGTools (below)
-// tool surface → runtime.Toolset (runtime/action_session.go)
-// message fitting → chat.FitMessages (agent/chat/message_fit.go)
-// citation guidelines → GetCitationGuidelines (below)
-// system prompt → SysPrompt (below)
-// formalization → Formalize (below)
-// retrieval → HybridSearch (runtime/tool_search.go)
-// keyword extraction → ExtractWeightedKeywords (runtime/keywords.go)
-// keyword compaction → CompactKeywords (runtime/tool_text_processing.go)
-// web retrieval → searchExecutor.webSearch (runtime/tool_executor.go)
-// answer composition → ComposeAnswer (agentic_rag_graph.go)
-// naive answer → ComposeNaiveAnswer (agentic_rag_graph.go)
-// evidence fitting → FitEvidence (below)
-// sufficiency review → SCA review (runtime/orchestrator/sufficient_context.go)
-// gap rewrite → SCA gap rewrite (runtime/orchestrator/sufficient_context.go)
-// entry point → Rag (below)
-//
-// document scope → toolDocScope (runtime/tool_executor.go)
-// doc-id verification → DocIDLookup (below)
-// tenant resolution → docInDatasets (runtime/tool_search.go)
-// full document fetch → fetchFullDocument (runtime/doc_fetch.go)
-// document summary → summarizeDocument (runtime/doc_fetch.go)
-//
-// Usage is counted through an explicit *runtime.LLMUsageStats collector (RAGTools.Stats)
-// passed to the calls the run makes, rather than through a ContextVar and a wrapper chat
-// model; per-round metrics derived from the round list are not tracked.
-//
-// Deliberately not implemented — dead code upstream: the document-picking helpers return
-// nil on their first statement and are never called, so the title/metadata selectors are
-// unreachable too, and structured_retrieve has neither a @tool decorator nor a caller.
-//
-// RAGTools configures one agentic-search run: every capability the run needs (retriever,
-// model, prompts, embedder, kb accumulation, retrieval tuning, answer composition) is
-// carried on this one object — there is no second dependency struct. The runtime layer
-// keeps narrow per-call views (runtime.SearchDeps / runtime.SessionDeps) projected from a
-// RAGTools at call time. Construct it directly
-// (RAGTools{...}); its zero value is a valid starting point for the narrower call sites
-// that only set a subset of fields.
-type RAGTools struct {
- // Tools is the shared capability carrier. The graph and the action session both
- // reach retrieval through it.
- Tools *runtime.Toolset
- // Search is the fully-projected retrieval configuration for this run. The
- // graph's DUAL-CHANNEL fan-out needs it directly: unlike the action session
- // (which goes through Tools.Exec's single "search_chunks" tool), fan-out
- // issues a keyword BM25 leg and a dense semantic leg under different
- // settings, which the tool interface cannot express.
- Search runtime.SearchDeps
- // Retriever is the retrieval backend. When nil, the runtime singleton
- // (runtime.GetRetrievalService) is used.
- Retriever runtime.Retriever
- // Model drives the LLM turns. Required for agentic modes; low/naive never
- // call it.
- Model runtime.SessionModel
- // ModelName is the RESOLVED chat model identity (e.g. "gpt-4o"), one component of the
- // gen_json reply-cache key. Empty disables that cache: keying on an empty name would
- // collapse every model onto one bucket and serve a reply produced by another model.
- ModelName string
- // Embedder is the external embedding handle used by graph/structure seed encoding.
- // Nil falls back to this package's internal tenant-default resolver, which additionally
- // degrades to keyword matching when the DB is uninitialised or encoding panics.
- Embedder nlp.NavEmbedder
- // Keywords extracts the entity-weighted retrieval query. Optional.
- Keywords KeywordExtractorFn
- // Prompts are the report/SCA/rewrite prompt templates (user_defined_prompts).
- Prompts runtime.PromptLoader
- // Expand runs compiled-structure expansion. Optional.
- Expand runtime.CompiledExpander
- // SCAPrompts overrides Prompts for the sufficient-context review.
- SCAPrompts runtime.PromptLoader
- // RewritePrompts overrides Prompts for the gap→query rewrite call.
- RewritePrompts runtime.PromptLoader
- // KB is the in-flight retrieval accumulation.
- KB *runtime.Kbinfos
- // Logger is optional; nil uses the default logger.
- Logger *log.Logger
- // Answer-composition configuration (see AnswerDeps). All optional; the defaults are
- // the configured behaviour.
- // CiteRules overrides the default citation rules.
- CiteRules string
- // SystemPrompt is the dialog-level UI configuration appended after the
- // agentic contract (language/tone/style may override; evidence contract may not).
- SystemPrompt string
- // EmptyResponse is returned verbatim when no evidence was found, skipping the
- // composition call entirely.
- EmptyResponse string
- // EvidenceMaxTokens caps the evidence block. <=0 uses evidenceBudgetTokens.
- EvidenceMaxTokens int
- // MaxLength is the chat model's context window. It bounds message fitting for prompt
- // assembly; <=0 falls back to chat.EffectiveContextLength's 8192 default.
- MaxLength int
- // ComposeAnswer enables the terminal composition node. Defaults to true
- // when a model is configured; set false to receive raw evidence only.
- ComposeAnswer *bool
- // Finalize runs the terminal composition. It is wired by Rag so the graph can invoke
- // it FROM INSIDE its last node — the agentic and low graphs both end in a
- // formalize_answer node that composes and streams the answer. The graph forwards the
- // node's state values the compose prompt reads: partialAnswer and emptyResult
- // (orchestrator/direct.go) plus question —
- // the graph state's FORMALIZED question (
- // `question = state.get("question")`), which the formalize_question node
- // wrote. The compose prompt must be built from the formalized question,
- // not the outer tool argument: the graph researched the formalized
- // multi-hop question, so composing from the compressed outer argument
- // collapses the final answer to the first completed sub-answer. Empty
- // question means "the caller composes afterwards" (post-graph fallback) —
- // composeFinalAnswer then falls back to req.Question. Nil Finalize means
- // the caller composes afterwards, which is only correct for the naive path.
- Finalize func(ctx context.Context, partialAnswer, emptyResult bool, question string)
- // Messages is the conversation history used by Formalize to resolve
- // pronouns/ellipses into a standalone question. The agentic and low graphs
- // formalize it as their first node; naive retrieval does not formalize.
- Messages []schema.Message
- // WebSearch is the optional open-web provider backing the `web_search` tool.
- // Nil HIDES the tool from the mode's surface —
- // sessionDeps sets HasWebSearch = mode.HasTool("web_search") && WebSearch != nil
- // and ActiveToolSpecs drops the spec — so the model is never shown a dead tool.
- // Should a call reach the executor anyway (stale spec, direct invocation), it
- // answers StatusError/ReasonInfra with a do-not-retry note, NOT a query-level
- // MISS: WebSearchTool returns the same infra error for an absent provider.
- WebSearch runtime.WebSearcher
- // DocIDVerifier checks which requested document ids really belong to the datasets
- // being searched. Nil disables the check and leaves the requested scope untouched.
- DocIDVerifier runtime.DocIDVerifier
- // DocChunks pages through a single document's chunks in reading order
- // (sort_by_position). It backs the whole-document-reading path
- // (summarize_document / fetch_full_document)
- // Nil disables it: the document-level tools report that reading is unavailable.
- DocChunks runtime.DocChunkLister
- // MetadataResolver resolves document sets from document metadata. It backs the
- // metadata_search tool and the pre-search metadata channel
- // (implemented by internal/service.MetadataService). Nil leaves both unavailable.
- MetadataResolver runtime.MetadataResolver
- // DeclaredMetadata reads the metadata fields the datasets DECLARE in their
- // parser_config, which is what lets the metadata_search catalog describe a field
- // (its meaning and allowed values) instead of only naming it. Same implementation as
- // MetadataResolver; nil leaves the catalog with the metadata index alone.
- DeclaredMetadata runtime.DeclaredMetadataResolver
- // Outer is the outer-layer chat model that drives the rag_agent react loop
- // (dialog_service.rag_agent): it binds tools=[rag, summarize_document] with
- // terminal_tools={"rag"}. When non-nil, Rag runs that outer loop: the model may call
- // `rag` (terminal — runs the inner agentic graph and returns the cited answer) or
- // `summarize_document` (non-terminal — reads a whole document into evidence and lets
- // the model answer from it on the next round); with no tool call it answers directly.
- // When nil, Rag falls back to the direct RunAgenticRAG path, so callers that don't wire
- // an outer model see unchanged behaviour.
- Outer *models.ChatModel
- // Cache answers near-identical re-asks from an earlier answer. Rag builds a fresh
- // per-turn RAGCache when this is nil, so caching is ON by default. To share one cache
- // across multiple Rag() calls within a single turn (the concurrent tool_call case),
- // pass the same *RAGCache here instead of relying on the auto-built one.
- Cache *RAGCache
- // OriginalQuestion is the user's own, unrewritten question. When the outer caller
- // passes a compressed rewrite as Question, the original is preferred if both clearly
- // describe the same turn.
- OriginalQuestion string
- // TextAttachments is appended to the question and bypasses the re-ask cache.
- TextAttachments string
- // AnswerSink receives the answer as it is produced. Nil disables streaming; the answer
- // is still returned in full. Requires a model implementing
- // runtime.StreamingSessionModel, otherwise the run falls back to one-shot composition.
- AnswerSink *AnswerSink
- // ToolStarted is called once research begins, before any retrieval, so the caller can
- // show progress.
- ToolStarted func()
- // Steps reports the run's reasoning steps. Each stage declares its own step
- // where it happens and each tool reports both halves of its call; Text feeds
- // the think block and Events feeds structured clients, either may be nil.
- //
- // This replaces the per-request progress sink this branch used (a logger
- // wrapper that fished bracket-tagged lines out of the log stream and forwarded
- // them into the <think> block). Visibility is now the producer's decision: the
- // developer log keeps the diagnostics, and a step is a step because a stage
- // SAYS so — not because a tag matched a pattern — so rewording a log line
- // can never silently hide a step.
- Steps runtime.StepReporter
- // Retrieval tuning: an explicit tool argument still wins over these, and zero selects
- // the runtime defaults.
- TopN int
- SimilarityThreshold float64
- // KeywordsSimilarityWeight is the keyword leg's weight. A pointer keeps an
- // explicit zero (pure vector search) distinct from an unset value.
- KeywordsSimilarityWeight *float64
- // UsingEmbedding is the Go spelling of Python RAGTools.retrieve's
- // `using_embedding: bool = False` (agentic_rag.py:599). When false (the
- // agentic default) retrieval is keyword-only; when true the embedder is
- // engaged and KeywordsSimilarityWeight (default 0.3) applies.
- //
- // SCOPE: only the retrieve channel honours it.
- UsingEmbedding bool
- // HasEmbedder: whether an embedder is configured. It gates the hybrid leg's vector
- // weight, i.e. the semantic recall of the search_chunks tool.
- HasEmbedder bool
- RerankCandidatesCount int
- TopK int
- // DocScope is the session-wide document restriction. Nil means "search everything".
- DocScope []string
- // MetaDataFilter restricts retrieval by chunk metadata.
- MetaDataFilter map[string]any
- // KBs is the full set of Knowledgebase objects (carrying parser_config / tenant_id)
- // the agentic tools run over. They arrive already resolved from the caller (this
- // package stays DB-free) and are fed to the Tagger for the question-type tag boost.
- KBs []*entity.Knowledgebase
- // Tagger classifies the query into question-type tags the retriever uses to boost
- // matching chunks (implemented by internal/service.MetadataService.LabelQuestion).
- // Nil means no tag boost.
- Tagger runtime.QuestionLabeler
- // Stats receives per-phase LLM usage for this run (runtime.LLMUsageStats). Nil
- // disables collection.
- Stats *runtime.LLMUsageStats
-}
-
-// sessionDeps projects RAGTools onto the runtime.SessionDeps the action session
-// and slot-table builders consume.
-//
-// Every field is forwarded unconditionally. In particular a nil Prompts is NOT
-// a reason to drop Tools/KB: the runtime resolves a nil loader to its embedded
-// canonical templates (retrieval.resolveLoader / prompts.EmbeddedPromptLoader), so
-// the action session still runs with its toolset and the shared evidence pool.
-// Returning a Model-only projection here silently turned the slot research pass
-// into a tool-less question-answering turn.
-func (d RAGTools) sessionDeps() runtime.SessionDeps {
- return runtime.SessionDeps{
- Tools: d.Tools,
- Model: d.Model,
- Prompts: d.Prompts,
- KB: d.KB,
- }
-}
-
-func (d RAGTools) logger() *log.Logger {
- if d.Logger != nil {
- return d.Logger
- }
- return _LOG
-}
-
-// Question formalization (the graph's first node).
-//
-// Rewrite the latest user message into a standalone question AND derive its search
-// keywords, in one LLM call.
-//
-// Single-turn shortcut: when there is nothing to resolve, the question is kept
-// VERBATIM (no rewrite) and only keywords are extracted. Rewriting a
-// self-contained question risks silently changing its meaning — and the
-// single-turn case is the overwhelming majority.
-//
-// The keyword/JSON extraction helpers (ExtractWeightedKeywords, ExtractJSON)
-// stay in runtime as the leaf capability library.
-
-const (
- // formalizeTimeoutS bounds the formalize call.
- formalizeTimeoutS = 45.0
- // formalizeTemperature: formalization is a mechanical rewrite, so it must be stable.
- formalizeTemperature = 0.1
-)
-
-var reFormalizeThink = regexp.MustCompile(`(?s)^.*</think>`)
-
-var formalizePrompt = (`You are given a conversation. Do BOTH of the following and return JSON only:
-1. Rewrite the LAST user message into a single, self-contained question that can be understood without the prior conversation — resolve pronouns, ellipses and follow-up shortcuts using the earlier turns. In most cases, it should be EXACTLY THE SAME as the last user query — only rewrite when there is something to resolve (a pronoun/ellipsis pointing back at an earlier turn). Preserve the original language.
-2. Extract keywords for a keyword search: the salient content words and phrases that literally appear in the (standalone) question — key nouns, named entities, domain terms — PLUS 2-3 close synonyms/abbreviations/aliases/alternative spellings of each, in the SAME language as the question. Maximize recall. Do NOT include terms that would be part of the answer.
- Example — "In which year did Apple acquire Beats?" -> keywords = "Apple, Apple Inc., AAPL, acquire, acquisition, acquired, Beats, Beats Electronics".
-
-Output ONLY JSON, no prose, no code fences: {"question": "<standalone question>", "keywords": "<term1, term2, synonym1, ...>"}`)
-
-// Formalize: return (question, keywords) for
-// the given conversation.
-//
-// messages may be []schema.Message (preferred) or pre-formatted "Speaker: text"
-// strings. On any failure it degrades to (last user message, "") — formalization
-// is an optimization, never a precondition for answering.
-//
-// maxLength is the chat model's context window; it bounds the prompt fit. <=0 falls back
-// to chat.EffectiveContextLength's 8192.
-func Formalize(ctx context.Context, deps runtime.SessionDeps, messages []schema.Message, maxLength int) (string, string) {
- lastUser, transcript := transcriptOf(messages)
- if lastUser == "" {
- return "", ""
- }
- if deps.Model == nil {
- // No model: keep the raw question and skip keyword extraction rather
- // than failing the request.
- return lastUser, ""
- }
-
- // Single-turn: nothing to resolve — keep the question VERBATIM (rewriting a
- // self-contained question risks silently changing its meaning), and extract only the
- // search keywords.
- if !isMultiTurn(messages) {
- _LOG.Printf("[Formalize] Single-turn self-contained question — kept verbatim (no rewrite): %s", trunc(lastUser, 120))
- _, kw := runtime.ExtractWeightedKeywords(ctx, deps.Model, lastUser)
- return lastUser, kw
- }
-
- ctx, done := runtime.Phase(ctx, runtime.PhaseFormalize)
- // Phase returns a child context carrying the phase label; the timeout is
- // derived from it so the call is both attributed and bounded.
- defer done()
-
- callCtx, cancel := context.WithTimeout(ctx, deadlineToDuration(formalizeTimeoutS))
- defer cancel()
-
- // Fit the prompt to the model's context window.
- fitted, fitErr := chat.FitMessages(formalizePrompt, []schema.Message{
- *schema.UserMessage("Conversation:\n" + transcript + "\n\nOutput JSON:"),
- }, maxLength)
- if fitErr != "" {
- _LOG.Printf("[Formalize] prompt fitting failed: %s", fitErr)
- return lastUser, ""
- }
- system := formalizePrompt
- history := fitted
- if len(fitted) > 0 && fitted[0].Role == schema.System {
- system = fitted[0].Content
- history = fitted[1:]
- }
- msgs := make([]schema.Message, 0, 1+len(history))
- msgs = append(msgs, *schema.SystemMessage(system))
- msgs = append(msgs, history...)
-
- // async_chat(system, history, {"temperature": 0.1}).
- reply, err := modelWithTemperature(deps.Model, formalizeTemperature).Complete(callCtx, msgs, nil)
- if err != nil {
- _LOG.Printf("[Formalize] failed; keeping the raw question: %v", err)
- return lastUser, ""
- }
-
- data, _ := runtime.ExtractJSON(stripThinkAndFences(reply.Content)).(map[string]any)
- question := ""
- if data != nil {
- if raw, ok := data["question"]; ok && raw != nil {
- question = strings.Trim(strings.TrimSpace(fmt.Sprint(raw)), "\"'")
- }
- }
- if question == "" {
- // Fall back to the raw last user message rather than an empty question.
- question = lastUser
- }
- keywords := ""
- if data != nil {
- switch v := data["keywords"].(type) {
- case string:
- keywords = v
- case []any:
- parts := make([]string, 0, len(v))
- for _, k := range v {
- if s := strings.TrimSpace(fmt.Sprint(k)); s != "" {
- parts = append(parts, s)
- }
- }
- keywords = strings.Join(parts, ", ")
- }
- }
- return question, runtime.CompactKeywords(keywords)
-}
-
-// transcriptOf renders the conversation and extracts the last user message.
-func transcriptOf(messages []schema.Message) (lastUser, transcript string) {
- var lines []string
- for _, m := range messages {
- content := m.Content
- switch m.Role {
- case schema.User:
- lastUser = strings.TrimSpace(content)
- lines = append(lines, "User: "+content)
- case schema.Assistant:
- lines = append(lines, "Assistant: "+content)
- case schema.System:
- lines = append(lines, "System: "+content)
- default:
- lines = append(lines, strings.TrimSpace(content))
- }
- }
- return strings.TrimSpace(lastUser), strings.Join(lines, "\n")
-}
-
-// isMultiTurn reports whether the conversation has more than one user turn.
-func isMultiTurn(messages []schema.Message) bool {
- n := 0
- for _, m := range messages {
- if m.Role == schema.User {
- n++
- }
- }
- return n > 1
-}
-
-// stripThinkAndFences: drop a leading thinking preamble, then strip Markdown fences. The
-// fence regex removes the delimiters wherever they appear, so a truncated or inline fence
-// leaves no residue either.
-func stripThinkAndFences(s string) string {
- s = reFormalizeThink.ReplaceAllString(s, "")
- s = reFenceDelimiters.ReplaceAllString(s, "")
- return strings.TrimSpace(s)
-}
-
-// reFenceDelimiters removes ```json / ``` delimiters wherever they appear.
-var reFenceDelimiters = regexp.MustCompile("```(?:json)?\\s*|\\s*```")
-
-// End-to-end entry point: one call that runs the runtime and publishes the
-// evidence it collected.
-//
-// Rag: the RAGTools method
-// that drives the research pipeline. It owns the conversation-level concerns
-// (re-ask cache, the effective question, attachments, composing the final
-// answer) and delegates the mode dispatch to RunAgenticRAG, which mirrors
-// run_agentic_rag and carries its dispatch:
-//
-// mode is NAIVE → _naive_rag (plain retrieval)
-// mode.Agentic → build_agentic_graph (medium / high / ultra)
-// else → build_low_graph (low: formalize → direct_search)
-//
-// SCOPE NOTE — read before relying on this for agentic modes: the outer
-// orchestration that agentic_rag_graph.py owns — planner decomposition, prefetch
-// fan-out, and the SCA↔rewriter iteration loop that drives medium/high/ultra — is
-// the agentic_rag package's agentic planner (Rag's agentic-loop registration).
-// So:
-//
-// - low / naive: fully wired here (direct search through HybridSearch).
-// - medium/high/ultra: Rag delegates to the registered AgenticLoop, which the
-// agentic_rag package registers (see SetAgenticLoop). Without it, Rag falls
-// back to a single action session.
-//
-// The runtime package keeps the low-level retrieval / session / tool machinery
-// (RunActionSession, HybridSearch, Toolset, Kbinfos, searchExecutor, ...).
-
-// RunRequest lives in the runtime package (see runtime/tool_executor.go) because the
-// runtime engine's searchExecutor needs it to carry the per-run query, and the
-// agentic_rag package must not be imported from runtime (import-cycle rule).
-// RunResponse is the agent-side result type returned by RAGTools.rag.
-
-// RunResponse is the outcome of one runtime run.
-type RunResponse struct {
- // Answer is the session's terminal answer, when it produced one.
- Answer string
- // Slots are the filled slot-table values discovered by the session.
- Slots []runtime.Variable
- // Chunks is the evidence accumulated in Kbinfos (narrowed, as the model saw
- // it). Citations are built from this.
- Chunks []map[string]any
- // DocAggs is the per-document aggregation of Chunks.
- DocAggs []map[string]any
- // EmptyResult is true when nothing was retrieved, which the caller turns into an
- // "I don't have enough information" answer.
- EmptyResult bool
- // Mode is the resolved spec, for logging.
- Mode runtime.ModeSpec
- // Partial is true when research ended without a satisfying verdict, so the
- // caller surfaces the residual findings honestly instead of refusing.
- Partial bool
- // SearchRounds is the number of completed SCA→rewrite iterations (0 for the
- // non-agentic paths).
- SearchRounds int
- // GraphFailed is true when the research graph itself errored. It is paired with
- // "produced nothing" before falling back to an internal-error message; an empty result
- // on its own is not a failure.
- GraphFailed bool
- // Verdict is the final sufficiency verdict ("SUFFICIENT"/"INSUFFICIENT").
- Verdict string
- // SCAFeedback is the body of the SCA feedback note — the sufficiency status hint alone
- // (the verdict dict carries only "status"; see scaFeedback). Rag() appends it as the
- // "[Research status]" note for every INSUFFICIENT verdict, adding the trailing "STOP"
- // vs "call rag again" sentence based on the consecutive-unanswerable count.
- SCAFeedback string
- // CollectedAnswer is the research draft (SCA-reviewed) produced by the
- // agentic loop. It feeds the final composition; prefer Answer for display.
- CollectedAnswer string
- // Kbinfos carries the full accumulated state (including the lossless
- // memory store) for callers that need more than the summary above.
- Kbinfos *runtime.Kbinfos
- // SlotCitations maps a slot-table id ("0", "1", ...) to the evidence
- // chunk ids that filled the slot. The chat pipeline's citation decoration
- // uses it to rewrite leaked "[ID:Slot N]" markers into citations of the
- // chunk the slot was filled from (those markers index the internal slot
- // table — nothing the user can open).
- SlotCitations map[string][]string
- // CiteChunkIDs is the ordered id list of the chunks the final-answer call
- // showed the model as numbered evidence (runtime.Kbinfos.CiteChunkIDs).
- // A "[ID:n]" marker the model wrote refers to position n in this list.
- CiteChunkIDs []string
-}
-
-// AnswerSink forwards a partially produced answer while the model is still writing it.
-// Nil disables streaming; the answer is still returned in full.
-//
-// Requires a model implementing runtime.StreamingSessionModel; otherwise the run
-// falls back to a single completion.
-type AnswerSink struct {
- // OnDelta receives each successive piece of the answer. isThink marks pieces of a
- // hidden reasoning block, which must not be shown as part of the answer.
- OnDelta func(delta string, isThink bool)
- // OnReset drops what has been forwarded so far. It is called before a
- // fallback re-sends the answer from scratch, so a partially streamed answer
- // is never shown twice. Nil means the sink cannot rewind, and the caller
- // only sends the complete answer.
- OnReset func()
-}
-
-// deliver forwards one piece, if the sink has a handler.
-func (s *AnswerSink) deliver(delta string, isThink bool) {
- if s == nil || s.OnDelta == nil || delta == "" {
- return
- }
- s.OnDelta(delta, isThink)
-}
-
-// reset discards previously forwarded pieces, if the sink supports it.
-func (s *AnswerSink) reset() {
- if s == nil || s.OnReset == nil {
- return
- }
- s.OnReset()
-}
-
-// KeywordExtractorFn extracts the weighted retrieval query for a question.
-type KeywordExtractorFn func(ctx context.Context, question string) (retrievalQuery string, err error)
-
-// Run executes one runtime request end to end and publishes the evidence into
-// the canvas state (when the context carries one) so citation grounding can
-// read it.
-//
-// It never fails for a recoverable reason: a missing component degrades to
-// "no evidence" rather than erroring: the failure is logged and an empty kbinfos is
-// returned.
-// ragCacheMinOverlap is the word-overlap ratio at which a new question counts as a
-// re-ask of a cached one.
-const ragCacheMinOverlap = 0.6
-
-// ragCacheMinShared is the minimum number of shared significant words before a cached
-// answer may be reused.
-const ragCacheMinShared = 2
-
-// effectiveQuestionMinShared is the overlap the effective-question resolution needs before
-// trusting the original question over the outer rewrite. It is deliberately independent of
-// ragCacheMinShared.
-const effectiveQuestionMinShared = 2
-
-// ragCacheStopwords: for cross-`rag`-call dedup only, never for retrieval or answer
-// quality.
-var ragCacheStopwords = map[string]bool{
- "the": true, "a": true, "an": true, "is": true, "was": true, "were": true,
- "what": true, "which": true, "when": true, "where": true, "who": true,
- "how": true, "of": true, "in": true, "to": true, "for": true, "and": true,
- "or": true, "but": true, "on": true, "at": true, "by": true, "be": true,
- "as": true, "it": true, "that": true, "this": true, "about": true,
- "with": true, "their": true, "its": true, "have": true, "has": true,
- "had": true, "been": true, "being": true, "from": true, "over": true,
- "under": true, "do": true, "does": true, "did": true, "not": true,
- "no": true, "yes": true, "can": true, "could": true, "should": true,
- "would": true, "also": true, "only": true, "very": true, "much": true,
- "more": true, "most": true, "some": true, "any": true,
-}
-
-// reQuestionTokens splits a question into words and CJK runs.
-var reQuestionTokens = regexp.MustCompile(`[a-zA-Z0-9\x{4e00}-\x{9fff}]+`)
-
-// questionGram is the significant words plus the numeric tokens kept apart, so questions
-// naming different numbers are never treated as the same question.
-type questionGram struct {
- words map[string]bool
- numbers map[string]bool
-}
-
-// questionKeywords: For English, plain
-// tokenisation suffices; CJK tokens survive as whole significant units.
-func questionKeywords(question string) questionGram {
- gram := questionGram{words: map[string]bool{}, numbers: map[string]bool{}}
- tokens := reQuestionTokens.FindAllString(strings.ToLower(question), -1)
- for _, t := range tokens {
- if isDigitToken(t) {
- gram.numbers[t] = true
- }
- }
- for _, t := range tokens {
- if len([]rune(t)) > 1 && !isDigitToken(t) && !ragCacheStopwords[t] {
- gram.words[t] = true
- }
- }
- if len(gram.words) == 0 {
- // Fall back to every non-numeric token, so an all-stopword question still has
- // something to compare.
- for _, t := range tokens {
- if len([]rune(t)) > 1 && !isDigitToken(t) {
- gram.words[t] = true
- }
- }
- }
- return gram
-}
-
-func isDigitToken(s string) bool {
- if s == "" {
- return false
- }
- for _, r := range s {
- if r < '0' || r > '9' {
- return false
- }
- }
- return true
-}
-
-// cacheSimilar: significant-word overlap
-// (shared / min cardinality), and numbers must be both empty or identical.
-func cacheSimilar(a, b questionGram) bool {
- if len(a.words) == 0 || len(b.words) == 0 {
- return false
- }
- if len(a.numbers) > 0 || len(b.numbers) > 0 {
- if !sameTokens(a.numbers, b.numbers) {
- return false
- }
- }
- shared := 0
- for w := range a.words {
- if b.words[w] {
- shared++
- }
- }
- if shared < ragCacheMinShared {
- return false
- }
- minLen := len(a.words)
- if len(b.words) < minLen {
- minLen = len(b.words)
- }
- return float64(shared)/float64(minLen) >= ragCacheMinOverlap
-}
-
-func sameTokens(a, b map[string]bool) bool {
- if len(a) != len(b) {
- return false
- }
- for k := range a {
- if !b[k] {
- return false
- }
- }
- return true
-}
-
-// FitEvidence: trim
-// evidence so question + evidence + the template stay inside a FIXED budget.
-//
-// The budget is deliberately not the model's full context: a large retrieval
-// pool must never fill the window with evidence, so each sufficiency/answer call
-// stays cheap. message_fit_in keeps the small side (the question) whole and
-// trims the large side (the evidence), which is what the two-message shape
-// below asks for.
-func FitEvidence(question, evidence string) string {
- if evidence == "" {
- return evidence
- }
- fitted, fitErr := chat.FitMessages("", []schema.Message{
- *schema.UserMessage(question),
- *schema.UserMessage(evidence),
- }, evidenceBudgetTokens)
- if fitErr != "" || len(fitted) == 0 {
- return evidence
- }
- return fitted[len(fitted)-1].Content
-}
-
-// routerPromptBody is the rag_agent router prompt, with the summarize_document line left
-// as %s: that line is only included when unstructured retrieval is available.
-const routerPromptBody = "You are a smart agent. For any question that needs " +
- "evidence from the knowledge bases or the web, call the `rag` tool " +
- "with a self-contained question — it runs the full search-and-answer " +
- "pipeline and returns a cited answer.\n" +
- "After the `rag` tool returns, do not call `rag` again for the same " +
- "user question. Use the returned cited answer as the final answer " +
- "unless the user explicitly asks a new question.\n" +
- "CRITICAL — preserve the full multi-hop structure when phrasing the " +
- "`rag` question. A question that compares two or more DISTINCT " +
- "targets or needs an arithmetic result across them (\"how much taller " +
- "is X than Y\", \"how many days after A's death did B die\", \"which of " +
- "these was discovered last\") MUST keep every target and relation in " +
- "the question you pass to `rag`. Never rewrite a comparison into a " +
- "single-entity question — dropping the second entity (e.g. the " +
- "purchaser in \"how many days after his death did the man who " +
- "purchased it in 1933 die\") makes the pipeline answer only the first " +
- "part. Pass the complete comparison.\n" +
- "%s" +
- "Do not invent facts and do not fabricate document IDs."
-
-// routerSummarizeLine is the conditional summarize_document instruction.
-const routerSummarizeLine = "- Call `summarize_document` ONLY when the user explicitly asks to summarise a specific document ('summarise the security audit', 'tldr the onboarding guide'). It needs a document ID.\n"
-
-// SysPrompt: the thin router prompt for callers that bind the tool set. The workflow
-// itself lives in the rag graph; the outer model only chooses between retrieval and an
-// explicit single-document summary.
-//
-// hasUnstructured: when false the summarize_document instruction is omitted. systemPrompt
-// is the dialog-level UI configuration, prepended when set.
-func SysPrompt(systemPrompt string, hasUnstructured bool) string {
- summarizeLine := ""
- if hasUnstructured {
- summarizeLine = routerSummarizeLine
- }
- router := fmt.Sprintf(routerPromptBody, summarizeLine)
- if strings.TrimSpace(systemPrompt) != "" {
- return systemPrompt + "\n\n" + router
- }
- return router
-}
-
-// GetCitationGuidelines returns the citation rules the final answer must follow, with an
-// optional user-defined override. The citation_prompt.md template is loaded via
-// prompts.CitationPrompt, and a non-empty override is taken verbatim.
-func GetCitationGuidelines(userDefined string) string {
- return prompts.CitationPrompt(userDefined)
-}
-
-// resolveEffectiveQuestion: prefer the user's ORIGINAL, complete question over the outer
-// model's rewrite, but only when both clearly describe the same user turn. The outer
-// rewrite often
-// drops the final target of a multi-hop question, and no later stage can
-// recover a deleted answer-attribute.
-func resolveEffectiveQuestion(question, originalUserQuestion string) string {
- if question == "" || originalUserQuestion == "" {
- return question
- }
- original := strings.TrimSpace(originalUserQuestion)
- if original == "" {
- return question
- }
- qk := questionKeywords(question)
- ok := questionKeywords(original)
- if len(qk.words) == 0 || len(ok.words) == 0 {
- return question
- }
- shared := 0
- for w := range qk.words {
- if ok.words[w] {
- shared++
- }
- }
- if shared >= effectiveQuestionMinShared {
- return original
- }
- return question
-}
-
-// RAGCache: it
-// answers a near-identical re-ask from a previous answer instead of re-running
-// the whole graph.
-//
-// Lifetime: the cache is per-turn by construction and is ON by default. Rag builds a
-// fresh RAGCache when deps.Cache is nil, so caching is never off, and the auto-built cache
-// lives only for that call (the per-turn degenerate case under the current
-// single-Rag()-per-turn path), so it can never serve a stale cross-turn hit. A caller that
-// wants to share one cache across multiple Rag() calls within a single turn (the concurrent
-// tool_call case) injects the same *RAGCache via RAGTools.Cache instead of relying on the
-// auto-built one.
-type RAGCache struct {
- mu sync.Mutex
- entries map[string]ragCacheEntry
- lastVerdict string
- // consecutiveUnanswerable
- // how many consecutive rag calls ended without a
- // satisfying verdict. After two in a row, RAGTools.rag appends a
- // "[Research status] … STOP calling rag again" note to the answer so the
- // outer agent stops re-asking. The counter is reset per turn, so it counts consecutive
- // insufficient rounds within a single request; it is not persisted across turns unless a
- // caller injects a long-lived *RAGCache via RAGTools.Cache.
- //
- // Private and guarded by mu: a turn's concurrent rag() calls share ONE
- // *RAGCache (Rag builds/stores it on deps.Cache before the outer react
- // loop), so an unsynchronized counter would race.
- consecutiveUnanswerable int
-}
-
-// NoteUnanswerable records one research round's verdict on the shared cache.
-func (c *RAGCache) NoteUnanswerable(verdict string) {
- if c == nil {
- return
- }
- c.mu.Lock()
- defer c.mu.Unlock()
- if verdict == VerdictSufficient {
- c.consecutiveUnanswerable = 0
- } else {
- c.consecutiveUnanswerable++
- }
-}
-
-// ConsecutiveUnanswerable reports how many consecutive rag() rounds ended
-// without a satisfying verdict.
-func (c *RAGCache) ConsecutiveUnanswerable() int {
- if c == nil {
- return 0
- }
- c.mu.Lock()
- defer c.mu.Unlock()
- return c.consecutiveUnanswerable
-}
-
-type ragCacheEntry struct {
- answer string
- gram questionGram
-}
-
-// NewRAGCache returns an empty per-request cache.
-func NewRAGCache() *RAGCache {
- return &RAGCache{entries: map[string]ragCacheEntry{}}
-}
-
-// Lookup returns the cached answer for a question judged near-identical to an
-// earlier one. Attachments bypass the cache (their content is not part of the
-// key), and a previous round that was not SUFFICIENT invalidates reuse: the
-// caller asked again precisely because it needs more evidence.
-func (c *RAGCache) Lookup(question string) (string, bool) {
- if c == nil || question == "" || !c.reuseAllowed() {
- return "", false
- }
- gram := questionKeywords(question)
-
- c.mu.Lock()
- defer c.mu.Unlock()
- for _, e := range c.entries {
- if cacheSimilar(gram, e.gram) {
- return e.answer, true
- }
- }
- return "", false
-}
-
-// Store records answer for question.
-func (c *RAGCache) Store(question, answer string) {
- if c == nil || question == "" || answer == "" {
- return
- }
- gram := questionKeywords(question)
- c.mu.Lock()
- defer c.mu.Unlock()
- if c.entries == nil {
- c.entries = map[string]ragCacheEntry{}
- }
- c.entries[question] = ragCacheEntry{answer: answer, gram: gram}
-}
-
-// noteVerdict records the verdict of the last research round.
-func (c *RAGCache) noteVerdict(verdict string) {
- if c == nil {
- return
- }
- c.mu.Lock()
- defer c.mu.Unlock()
- c.lastVerdict = verdict
-}
-
-// researchStatusTrailer: for every non-SUFFICIENT verdict it returns the trailing sentence
-// folded into the "[Research status]" note. After two consecutive unsatisfying rag() calls
-// (ConsecutiveUnanswerable >= 2 on the shared *RAGCache) it tells the outer
-// agent to STOP calling rag again; otherwise it invites a focused re-ask. It
-// returns "" when there is nothing to annotate — a SUFFICIENT verdict, an empty
-// answer, or no SCA feedback.
-func researchStatusTrailer(cache *RAGCache, resp *RunResponse) string {
- if resp.Verdict != VerdictInsufficient || resp.SCAFeedback == "" || resp.Answer == "" {
- return ""
- }
- if cache != nil && cache.ConsecutiveUnanswerable() >= 2 {
- return fmt.Sprintf(" STOP calling rag again: %d consecutive research rounds returned insufficient evidence. The sources likely lack the required data. Give your best answer from the evidence already gathered; do not re-run rag.", cache.ConsecutiveUnanswerable())
- }
- return " If these gaps are material, call rag again with a question focused on them."
-}
-
-// reuseAllowed reports whether a cached answer may be reused for the next question.
-func (c *RAGCache) reuseAllowed() bool {
- if c == nil {
- return false
- }
- c.mu.Lock()
- defer c.mu.Unlock()
- return c.lastVerdict == "" || c.lastVerdict == "SUFFICIENT"
-}
-
-// wrapModelForStats wraps the model's invoker so calls made through it are
-// counted per phase. It reports false when the model carries no chat.Invoker,
-// in which case the caller leaves it untouched.
-//
-// A counter can only be installed on the known carrier: SessionModel does not expose its
-// invoker in a uniform way, so only the carrier that does is wrapped. Callers that go
-// through other session models still get counted when they record via CurrentStats(ctx).
-func wrapModelForStats(model runtime.SessionModel, stats *runtime.LLMUsageStats) (runtime.SessionModel, bool) {
- src, ok := model.(*runtime.InvokerSessionModel)
- if !ok || src == nil || src.Invoker == nil {
- return model, false
- }
- // Copy before mutating: the caller may share this model across requests, and
- // wrapping in place would attach another run's stats to it.
- wrapped := *src
- wrapped.Invoker = &runtime.CountingInvoker{Inner: src.Invoker, Stats: stats}
- return &wrapped, true
-}
-
-// searchDepsFor projects a RAGTools' FULL retrieval configuration into the runtime
-// view for one request, including tenant/dataset resolution and the nil-backend
-// fallback. Projecting here rather than at each call site keeps the retrieval paths
-// (Rag's low/naive pass, the agentic fan-out and action session, the no-model
-// fallback) from drifting — a partial copy silently drops tuning such as the
-// question-type tag boost.
-func searchDepsFor(ctx context.Context, deps RAGTools, req runtime.RunRequest, datasetIDs []string, tenantID string, kb *runtime.Kbinfos, logger *log.Logger) runtime.SearchDeps {
- if tenantID == "" {
- tenantID = runtime.TenantIDFromContext(ctx)
- }
- backend := deps.Retriever
- if backend == nil {
- backend = &runtime.RuntimeRetriever{}
- }
- return runtime.SearchDeps{
- Backend: backend,
- KbIDs: datasetIDs,
- TenantID: tenantID,
- KB: kb,
- Expand: deps.Expand,
- Logger: logger,
- DocIDVerifier: deps.DocIDVerifier,
- DocChunks: deps.DocChunks,
- DocTenantResolver: dbDocTenantResolver{},
- MetadataResolver: deps.MetadataResolver,
- DeclaredMetadata: deps.DeclaredMetadata,
- Model: deps.Model, // the calculate tool writes its expression via the model
- DocScope: deps.DocScope,
- // Python retrieve:614-646 — configuration is the middle precedence
- // level, between an explicit tool argument and the module defaults.
- TopN: deps.TopN,
- SimilarityThreshold: deps.SimilarityThreshold,
- KeywordsSimilarityWeight: deps.KeywordsSimilarityWeight,
- UsingEmbedding: deps.UsingEmbedding,
- HasEmbedder: deps.HasEmbedder,
- RerankCandidatesCount: deps.RerankCandidatesCount,
- TopK: deps.TopK,
- MetaDataFilter: deps.MetaDataFilter,
- // rank_feature (Python retrieve:668): RAGTools carries the KB objects
- // and a tagger, mirroring rank_feature=label_question(question, self.kbs).
- KBs: deps.KBs,
- Tagger: deps.Tagger,
- // External embedding handle. When the caller supplied one on RAGTools it is used
- // directly; nil keeps the package's internal tenant-default resolver as a fallback
- // (which itself degrades to keyword matching when the DB is uninitialised or
- // encoding panics).
- Embedder: deps.Embedder,
- }
-}
-
-// NewDocTenantResolver returns the DB-backed doc→(kb, tenant) resolver used to group a
-// document scope by its real owner. It is the same resolver graph_explore uses; the
-// compiled expander takes it too
-// so a doc scope is scanned per owning dataset instead of being attached to
-// every bound one.
-func NewDocTenantResolver() runtime.DocTenantResolver { return dbDocTenantResolver{} }
-
-// dbDocTenantResolver implements runtime.DocTenantResolver against the database: it
-// maps each document id to its real owning (kb, tenant) so graph_explore can group
-// documents by owner and search knowledge bases outside the caller's datasetIDs.
-// The lookup is tenant-unscoped on purpose — DocScope is already constrained to the
-// user's authorized
-// documents — so a document may legitimately resolve to a KB not in the search set.
-type dbDocTenantResolver struct{}
-
-func (dbDocTenantResolver) ResolveDocTenants(ctx context.Context, docIDs []string) (map[string]runtime.DocTenant, error) {
- if len(docIDs) == 0 || dao.DB == nil {
- return nil, nil
- }
- var docs []struct {
- ID string `gorm:"column:id"`
- KbID string `gorm:"column:kb_id"`
- }
- if err := dao.DB.WithContext(ctx).Model(&entity.Document{}).
- Where("id IN ?", docIDs).Select("id", "kb_id").Scan(&docs).Error; err != nil {
- return nil, err
- }
- if len(docs) == 0 {
- return nil, nil
- }
- kbIDs := make([]string, 0, len(docs))
- seen := make(map[string]bool, len(docs))
- for _, d := range docs {
- if !seen[d.KbID] {
- seen[d.KbID] = true
- kbIDs = append(kbIDs, d.KbID)
- }
- }
- var kbs []struct {
- ID string `gorm:"column:id"`
- TenantID string `gorm:"column:tenant_id"`
- }
- if err := dao.DB.WithContext(ctx).Model(&entity.Knowledgebase{}).
- Where("id IN ?", kbIDs).Select("id", "tenant_id").Scan(&kbs).Error; err != nil {
- return nil, err
- }
- tenantByKB := make(map[string]string, len(kbs))
- for _, k := range kbs {
- tenantByKB[k.ID] = k.TenantID
- }
- out := make(map[string]runtime.DocTenant, len(docs))
- for _, d := range docs {
- out[d.ID] = runtime.DocTenant{KBID: d.KbID, TenantID: tenantByKB[d.KbID]}
- }
- return out, nil
-}
-
-// chatModelProbeTimeout bounds the pre-flight completion. An unreachable or
-// out-of-credit provider answers in milliseconds; a slow one must not hold the
-// request before the think block has even opened.
-const chatModelProbeTimeout = 20 * time.Second
-
-// probePrompt is the one-token question the pre-flight sends. It is a constant
-// so the test models recognise the probe without a bare "ping" literal.
-const probePrompt = "ping"
-
-// errProbeTimeout marks a probe that exhausted ITS OWN budget while the request
-// context was still alive — a slow model, not a dead provider.
-var errProbeTimeout = errors.New("chat model pre-flight timed out")
-
-// probeChatModel makes the smallest possible completion to learn whether the
-// run's chat model is usable at all. It is a DETECTION call: the reply is
-// discarded and only its error matters. A nil model reports no error — the
-// existing "no model configured" fallbacks own that case.
-//
-// A probe that only timed out comes back as errProbeTimeout: a slow model is
-// not a dead one, and the caller must not turn a long first token into an
-// outage.
-func probeChatModel(ctx context.Context, deps RAGTools) error {
- return probeChatModelWithin(ctx, deps, chatModelProbeTimeout)
-}
-
-// probeChatModelWithin is probeChatModel with an injectable budget, so a test
-// can exercise the timeout path without waiting out the real 20s.
-func probeChatModelWithin(ctx context.Context, deps RAGTools, budget time.Duration) error {
- if deps.Model == nil {
- return nil
- }
- probeCtx, cancel := context.WithTimeout(ctx, budget)
- defer cancel()
- if _, err := deps.Model.Complete(probeCtx, []schema.Message{*schema.UserMessage(probePrompt)}, nil); err != nil {
- if ctx.Err() == nil && probeCtx.Err() != nil {
- return fmt.Errorf("%w after %s: %v", errProbeTimeout, budget, err)
- }
- return err
- }
- return nil
-}
-
-func Rag(ctx context.Context, deps RAGTools, req runtime.RunRequest) *RunResponse {
- logger := deps.Logger
- if logger == nil {
- logger = _LOG
- }
- spec := runtime.GetMode(req.ThinkingMode)
- if !IsKnownMode(req.ThinkingMode) {
- // An unrecognised label resolves to NAIVE; log it so a typo in the mode
- // is visible instead of silently downgrading the request.
- logger.Printf("[Agentic RAG] unrecognised thinking mode %q; falling back to naive (non-agentic) retrieval", req.ThinkingMode)
- }
-
- // build the per-call usage counters and bind them to the
- // context so every LLM call beneath is attributed to its phase. Without this
- // the counting machinery is inert: CurrentStats(ctx) would stay nil and the
- // phase markers already sprinkled through the graph would record nothing.
- stats := deps.Stats
- if stats == nil {
- stats = runtime.NewLLMUsageStats()
- deps.Stats = stats
- }
- ctx = runtime.WithStats(ctx, stats)
- // Bind the caller's step reporter for the whole run: every stage and every
- // tool beneath reads it (runtime.StepsFrom) and reports its own step, so the
- // per-request trace is assembled from the stages that actually run.
- ctx = runtime.WithSteps(ctx, deps.Steps)
-
- // LLM pre-flight, before anything is announced. The agentic modes cannot
- // answer without a working model, and a dead provider used to surface only at
- // COMPOSE time — after the think block had narrated a research run that could
- // never finish (measured: MiniMax "insufficient balance" burned seconds of
- // narration plus a series of degraded-failure lines before the fallback
- // answer). One minimal call answers "is the provider usable at all"; when it
- // fails the run reports the provider's own message in the classic
- // `**ERROR**: …` shape and returns BEFORE reporting a step or starting the
- // tool, so no think block opens and the user sees what a naive-mode failure
- // shows.
- //
- // It runs ONCE per turn, at the turn's entry: Rag is called once per user turn
- // (the outer react loop's later `rag` tool calls re-enter the research graph
- // directly), so there is no second round to re-probe and no per-round state to
- // keep. It is UNCONDITIONAL: nothing between the request and this call decides
- // whether the model is checked — a provider that cannot answer must be
- // reported, never tolerated.
- //
- // The probe deliberately runs BEFORE wrapModelForStats: it is a health check,
- // not a phase's work, so its call is kept out of llm_stats.
- if spec.Label != "naive" {
- err := probeChatModel(ctx, deps)
- if errors.Is(err, errProbeTimeout) {
- // The probe's own budget ran out, not the provider's answer: let
- // the run proceed and fail (if it must) on its real budget.
- logger.Printf("[Agentic RAG] %v; continuing without a pre-flight verdict", err)
- } else if err != nil {
- // DEVELOPER LOG ONLY — no step is reported: the whole point is that
- // the user gets the error instead of a think block narrating work
- // that cannot happen. No GraphFailed either: that flag means the
- // research graph itself errored (RunResponse.GraphFailed) and the
- // graph never started; the pipeline keys on the `**ERROR**:` prefix.
- logger.Printf("[Agentic RAG] chat model pre-flight failed: %v", err)
- return &RunResponse{Answer: errorAnswerText(err), Mode: spec}
- }
- }
-
- // The run's logger stays the DEVELOPER log — nothing intercepts it. The
- // user-visible trace comes from the steps stages report themselves
- // (deps.Steps), so a diagnostic line can never leak into the think block and
- // a reworded line can never silently drop a step.
- //
- // wraps the chat model in CountingChatModel(chat_mdl.clone,
- // self.llm_stats). Go wraps the session model's invoker the same way, so
- // calls made through deps.Model are counted per phase too. The pre-flight
- // above probes the UNWRAPPED model on purpose, so a health check is never
- // counted as a phase's work.
- if wrapped, ok := wrapModelForStats(deps.Model, stats); ok {
- deps.Model = wrapped
- }
- defer stats.Log(logger)
-
- // tell the caller research is starting, before any retrieval
- // work, so it can show progress for the (potentially long) graph run.
- if deps.ToolStarted != nil {
- deps.ToolStarted()
- }
-
- // When an outer chat model is wired, the request is driven by the outer react loop
- // that binds [rag, summarize_document] and treats `rag` as a terminal tool. The inner
- // graph runs only when the model decides to call `rag`; otherwise the model answers
- // directly (or calls summarize_document to read a whole document first). When no outer
- // model is configured, or it cannot emit tool calls, we fall through to the direct
- // RunAgenticRAG path so behaviour is unchanged for every current caller.
- // The last user message carries the question, text attachments, and — for vision
- // models — image content blocks. These reach the outer react loop's history
- // (prepareOuterReact) so a vision model actually sees the images instead of silently
- // dropping them. Assembled here (not in the caller) so the multimodal construction
- // stays in the agentic_rag package beside the schema import.
- if len(deps.Messages) == 0 {
- deps.Messages = multimodalUserMessage(req.Question, req.TextAttachments, req.Images)
- } else if req.TextAttachments != "" || len(req.Images) > 0 {
- last := len(deps.Messages) - 1
- if deps.Messages[last].Role == schema.User {
- deps.Messages = append([]schema.Message(nil), deps.Messages...)
- deps.Messages[last] = multimodalUserMessage(deps.Messages[last].Content, req.TextAttachments, req.Images)[0]
- }
- }
- // Text attachments also feed the direct (non-outer) path, which appends them to the
- // question; attachments bypass the near-duplicate cache (see cacheable above).
- if deps.TextAttachments == "" {
- deps.TextAttachments = req.TextAttachments
- }
-
- // Build the per-request RAGCache up-front so the outer react loop (below)
- // reuses the SAME cache across its multiple rag() calls within one turn.
- // Keeping deps.Cache non-nil here is what makes the _consecutive_unanswerable
- // guard live: ConsecutiveUnanswerable is incremented
- // inside RunAgenticRAG and gated on deps.Cache != nil, so without a shared
- // cache the outer loop would never reach the "STOP calling rag again" verdict.
- // The outer loop must share the counter, so deps.Cache is set before any branch rather
- // than only on the direct path.
- if deps.Cache == nil {
- deps.Cache = NewRAGCache()
- }
-
- if deps.Outer != nil {
- if deps.AnswerSink != nil {
- return runOuterReactStream(ctx, deps, req, logger)
- }
- return runOuterReact(ctx, deps, req, logger)
- }
-
- // Re-ask guard: reuse a near-identical question's cached answer instead of re-running
- // the graph. Attachments bypass it, since their content is appended to the question
- // below and is not part of the key. Caching is on by default: deps.Cache is guaranteed
- // non-nil here (built above, before the outer-react branch, so it is shared across the
- // outer loop's multiple rag() calls), and the auto-built cache lives only for this call
- // (the per-turn degenerate case, since the current reasoning path issues a single Rag()
- // per turn), so it can never serve a stale cross-turn hit. A caller that wants to share
- // one cache across multiple Rag() calls within a single turn (the concurrent tool_call
- // case) injects the same *RAGCache via deps.Cache instead of relying on the auto-built
- // one.
- cache := deps.Cache
- cacheable := deps.TextAttachments == ""
- if cacheable && cache != nil {
- if cached, hit := cache.Lookup(req.Question); hit {
- step(ctx, logger, "Agentic RAG", "Cache hit: reused the answer for the near-identical question %q and skipped research.", trunc(req.Question, 80))
- return &RunResponse{Answer: cached, Mode: spec}
- }
- }
-
- // Prefer the user's ORIGINAL, complete question over the outer rewrite
- // . The outer rewrite often drops the final target of a
- // multi-hop question, and no later stage can recover it.
- if deps.OriginalQuestion != "" {
- if effective := resolveEffectiveQuestion(req.Question, deps.OriginalQuestion); effective != req.Question {
- logger.Printf("[Agentic RAG] using original user question over outer rewrite (original=%q → rewrite=%q)",
- trunc(deps.OriginalQuestion, 80), trunc(req.Question, 80))
- req.Question = effective
- }
- }
- if deps.TextAttachments != "" {
- req.Question += deps.TextAttachments
- }
-
- tenantID := req.TenantID
- if tenantID == "" {
- tenantID = runtime.TenantIDFromContext(ctx)
- }
- datasetIDs := req.DatasetIDs
-
- kb := &runtime.Kbinfos{}
- searchDeps := searchDepsFor(ctx, deps, req, datasetIDs, tenantID, kb, logger)
-
- resp := &RunResponse{Mode: spec, Kbinfos: kb}
-
- // hand over to the graph driver, which owns the mode dispatch
- // (agentic graph vs. direct/naive retrieval) and formalization.
- //
- // The terminal composition is handed to the graph instead of being run here:
- // The graph composes inside the last node of both graphs (agentic and low
- // formalize_answer). The closure is idempotent so the post-graph call below is a no-op
- // once the graph has composed — a graph that never reaches its last node (or the naive
- // path, which composes under naiveAnswerSystem instead) still gets an answer here.
- composed := false
- compose := func(ctx context.Context, partialAnswer, emptyResult bool, question string) {
- if composed {
- return
- }
- composed = true
- composeFinalAnswer(ctx, deps, req, kb, resp, logger, partialAnswer, emptyResult, question)
- }
- deps.Finalize = compose
-
- // Say why this run goes straight to the research graph — DEVELOPER LOG ONLY.
- // Only one reason remains: no outer model was wired (the outer branch above did
- // not fire, and no cache hit short-circuited). Whether the request's chat model
- // can call tools at all is decided before this call, by the caller that picks
- // between the agentic path and the regular chat.
- //
- // It stays out of the think block on purpose: it describes the runtime' wiring,
- // not the question or the research, and a reader gets one of these on every run
- // in a deployment without an outer loop. The reader-visible trace simply has no
- // "[Tool loop]" section in that configuration, which is the honest shape — the
- // section exists when a loop ran.
- logger.Printf("[Agentic RAG] No outer model is wired, so this run has no outer tool loop.")
-
- RunAgenticRAG(ctx, deps, req, searchDeps, kb, resp, logger, spec)
-
- resp.Chunks = kb.Chunks
- resp.DocAggs = kb.DocAggs
- resp.EmptyResult = len(kb.Chunks) == 0
-
- // Publish evidence for citation grounding. Best-effort: when no canvas
- // state is attached (e.g. unit tests), skip silently — mirroring
- // RetrievalTool's behaviour.
- if len(kb.Chunks) > 0 {
- runtime.PublishReferences(ctx, kb)
- }
-
- // The graph's last node composed already (see compose above); this only
- // catches runs that never reached it. The naive path (_naive_rag)
- // already composed under naiveAnswerSystem from flat "[i] content"
- // evidence, so composing here would replace that with the
- // FinalAnswerSystem / kb_prompt answer.
- if spec.Label != "naive" {
- // A run that never reached the graph's last node produced neither a
- // partial answer nor empty-result state flags (the fallback compose reads the same
- // defaults: no partial preamble, no-evidence hedge only when the pool is actually
- // empty). The question is empty —
- // the graph never formalized, so compose falls back to req.Question.
- compose(ctx, false, false, "")
- }
-
- // A "[Research status]" note is appended for EVERY non-SUFFICIENT verdict. When
- // research has stayed unsatisfying for two
- // consecutive turns it tells the outer agent to STOP calling rag again;
- // otherwise it invites a focused re-ask. The counter lives on the
- // conversation-scoped cache; it was incremented back in NewAgenticLoop once
- // the SCA verdict was known. Skip the naive/sufficient paths: an empty
- // answer or a SUFFICIENT verdict has nothing to annotate.
- if t := researchStatusTrailer(deps.Cache, resp); t != "" {
- // The period closes the hint clause before the trailer sentence.
- resp.Answer += "\n\n[Research status] " + resp.SCAFeedback + "." + t
- }
-
- // Cache the freshly produced answer for later near-identical questions, and
- // remember the verdict so a following re-ask is not answered from an
- // admittedly incomplete one.
- if cacheable && cache != nil {
- cache.Store(req.Question, resp.Answer)
- cache.noteVerdict(resp.Verdict)
- }
- return resp
-}
-
-// composeFinalAnswer runs the graph's terminal node and writes the result onto
-// the response.
-//
-// When no model is configured this is a no-op: the caller still receives the evidence and
-// can answer with it, rather than failing.
-//
-// partialAnswer / emptyResult are the compose prompt inputs read off the graph state:
-// `partial_answer` drives the partial-information preamble, `empty_result` is the
-// always-true-in-graph term of `no_evidence = abstain or empty_result or not chunks`.
-// question is the graph state's FORMALIZED question the last node forwarded; empty falls
-// back to req.Question — only the post-graph fallback composes after a run that never
-// formalized.
-// The graph's Finalize forwards the formalize_answer node's own state values;
-// the Go-only post-graph fallback passes the response flags instead.
-func composeFinalAnswer(ctx context.Context, deps RAGTools, req runtime.RunRequest, kb *runtime.Kbinfos, resp *RunResponse, logger *log.Logger, partialAnswer, emptyResult bool, question string) {
- if deps.Model == nil {
- return
- }
- // Compose from the graph state's formalized question: the graph researched the full
- // multi-hop question, so the compose prompt must carry it — not the outer tool argument,
- // which compresses multi-hop questions to their first hop and collapses the final answer
- // to the first completed sub-answer.
- composeQuestion := question
- if composeQuestion == "" {
- composeQuestion = req.Question
- }
- // The terminal node runs in the "finalize" phase; without it every call made here is
- // attributed to "unknown" in the usage table.
- ctx, done := runtime.Phase(ctx, runtime.PhaseFinalize)
- defer done()
- if deps.ComposeAnswer != nil && !*deps.ComposeAnswer {
- return
- }
- // The compose publishes the ordered evidence list it rendered (kb's
- // CiteChunkIDs) while it runs; hand it to the caller so the chat pipeline
- // can resolve the answer's [ID:n] markers against that same list. Covered by
- // a defer because both the streaming and the one-shot path return from
- // inside the branches below.
- defer func() {
- if kb != nil {
- resp.CiteChunkIDs = kb.CiteChunkIDs
- }
- }()
- // Stamp each pool chunk with its document's declared metadata, so the evidence
- // blocks carry the passage's document attributes (file name, update time, ...)
- // beside its text: a two-hop question — which documents, and what is in them —
- // is then answerable from the evidence alone, instead of from the slot record
- // the answer is told not to quote.
- attachDocMetadata(ctx, deps, kb, logger)
-
- adeps := AnswerDeps{
- Model: deps.Model,
- CiteRules: deps.CiteRules,
- SystemPrompt: deps.SystemPrompt,
- EmptyResponse: deps.EmptyResponse,
- MaxTokens: deps.EvidenceMaxTokens,
- MaxLength: deps.MaxLength,
- Logger: logger,
- // The direct fallback receives the original multimodal messages, so the non-outer
- // compose model sees the vision-gated images too. The outer react path clears these
- // in the rag terminal tool (the inner compose is text-only).
- UserImages: req.Images,
- }
- // Stream the answer when the model and the caller both support it, so the
- // user sees text while it is produced instead of only at the end.
- if deps.AnswerSink != nil {
- if streamer, ok := deps.Model.(runtime.StreamingSessionModel); ok {
- deps.AnswerSink.reset()
- streamed, err := ComposeAnswerStream(ctx, adeps, streamer, kb, composeQuestion, partialAnswer, emptyResult, func(delta string, isThink bool) error {
- deps.AnswerSink.deliver(delta, isThink)
- return nil
- })
- if err == nil {
- resp.Answer = streamed.Answer
- if streamed.Partial {
- resp.Partial = true
- }
- return
- }
- // A partially streamed answer must not be sent twice: tell the sink
- // to drop what it already forwarded and fall back to one shot. The
- // think block reports the provider's own error (bounded): on a dead
- // provider this is the first failure a user would otherwise never
- // see, and the one-shot retry below fails the same way.
- runtime.StepsFrom(ctx).StageLineDetail(logger, "Agentic RAG",
- "Streaming generation failed; falling back to a single call: "+providerErrorSummary(err),
- fmt.Sprintf("streaming compose failed (%v); falling back to a single call", err))
- deps.AnswerSink.reset()
- }
- }
- // No manual RecordCall here: the call is attributed to the "finalize" phase
- // by the wrapped invoker above.
- // empty_result is the third no-evidence term — the loop's own
- // "nothing was found" signal, distinct from abstain and from an empty pool.
- res := ComposeAnswerWith(ctx, adeps, kb, composeQuestion, partialAnswer, false, emptyResult)
- if deps.Stats != nil && res.Failed {
- deps.Stats.RecordFailed(runtime.PhaseFinalize)
- }
- resp.Answer = res.Answer
- if res.Partial || partialAnswer {
- // Reflect the graph state's partial flag back on the response even when the
- // composed text itself predates the flag.
- resp.Partial = true
- }
- // A FAILED compose is a terminal message, not a stream: the pipeline carries
- // it in the final result, exactly how the classic path delivers its own
- // `**ERROR**: …` answer. Streaming it here as well made the client show the
- // text twice — once as an answer delta, once in the final.
- if deps.AnswerSink != nil && !res.Failed {
- deps.AnswerSink.deliver(res.Answer, false)
- }
-}
-
-// outerReactParts holds the pieces shared by the streaming and non-streaming outer react
-// loops.
-type outerReactParts struct {
- kb *runtime.Kbinfos
- sd runtime.SearchDeps
- spec runtime.ModeSpec
- resp *RunResponse
- tools []map[string]any
- history []models.Message
- system string
-}
-
-// multimodalUserMessage assembles the last user turn: its text is the question plus any
-// text attachments, augmented with image content blocks for vision-capable models.
-// imageFiles are vision-gated base64 data URIs (data:...); textAttachments is
-// joined file content. When there is nothing to carry (no question, no
-// attachments, no images) it returns an empty slice so callers can skip it.
-//
-// The multimodal part list is used ONLY when images are attached: the text (question +
-// attachments) stays a plain string, and the message is converted to content blocks just
-// for image attachments. Sending a content-block array for a text-only question is not
-// harmless — text-only providers silently drop it (Zhipu GLM answers an empty user turn
-// with "Hello! How can I assist you today?" instead of calling the `rag` tool), which is
-// why it is avoided here.
-func multimodalUserMessage(question, textAttachments string, imageFiles []string) []schema.Message {
- text := question
- if textAttachments != "" {
- if text != "" {
- text = text + "\n\n" + textAttachments
- } else {
- text = textAttachments
- }
- }
- if len(imageFiles) == 0 {
- if text == "" {
- return nil
- }
- return []schema.Message{{Role: schema.User, Content: text}}
- }
- parts := make([]schema.MessageInputPart, 0, 1+len(imageFiles))
- if text != "" {
- parts = append(parts, schema.MessageInputPart{Type: schema.ChatMessagePartTypeText, Text: text})
- }
- for i := range imageFiles {
- uri := imageFiles[i]
- parts = append(parts, schema.MessageInputPart{
- Type: schema.ChatMessagePartTypeImageURL,
- Image: &schema.MessageInputImage{MessagePartCommon: schema.MessagePartCommon{URL: &uri}},
- })
- }
- if len(parts) == 0 {
- return nil
- }
- return []schema.Message{{Role: schema.User, UserInputMultiContent: parts}}
-}
-
-// multimodalContentBlocks converts schema.MessageInputPart multimodal content
-// into the OpenAI-style []interface{} block slice entity/models.Message.Content
-// expects (mirroring entity/models toInternalMessages). Plain text Content is
-// kept as a leading text block so the model sees both the original text and the
-// multimodal parts.
-func multimodalContentBlocks(m schema.Message) interface{} {
- blocks := make([]interface{}, 0, 1+len(m.UserInputMultiContent))
- if m.Content != "" {
- blocks = append(blocks, map[string]interface{}{"type": "text", "text": m.Content})
- }
- for _, part := range m.UserInputMultiContent {
- switch part.Type {
- case schema.ChatMessagePartTypeText:
- if part.Text != "" {
- blocks = append(blocks, map[string]interface{}{"type": "text", "text": part.Text})
- }
- case schema.ChatMessagePartTypeImageURL:
- if part.Image != nil && part.Image.URL != nil {
- blocks = append(blocks, map[string]interface{}{
- "type": "image_url",
- "image_url": map[string]interface{}{"url": *part.Image.URL},
- })
- }
- }
- }
- return blocks
-}
-
-// prepareOuterReact builds the state both outer react loops need: the evidence set,
-// the search deps, the two tool definitions and the outer conversation history
-// rendered as models.Message.
-func prepareOuterReact(ctx context.Context, deps RAGTools, req runtime.RunRequest, logger *log.Logger) *outerReactParts {
- tenantID := req.TenantID
- if tenantID == "" {
- tenantID = runtime.TenantIDFromContext(ctx)
- }
- kb := &runtime.Kbinfos{}
- sd := searchDepsFor(ctx, deps, req, req.DatasetIDs, tenantID, kb, logger)
- spec := runtime.GetMode(req.ThinkingMode)
-
- tools := []map[string]any{
- {
- "type": "function",
- "function": map[string]any{
- "name": "rag",
- "description": "Run the full agentic research graph over the configured datasets and return a cited answer. " +
- "WHEN TO CALL: every question whose answer must come from the knowledge base — a fact, an enumeration, " +
- "a count, \"which documents ...\", a date or updated-time range, a comparison, a multi-hop relation, " +
- "or anything that needs citations. " +
- "DO NOT CALL: only for pure chit-chat or a rewriting task that cannot need the datasets. " +
- "You MUST call this before answering a knowledge-base question: the dataset names and your own prior " +
- "knowledge are not evidence, and answering without it is a wrong answer even when it reads well.",
- "parameters": map[string]any{
- "type": "object",
- "properties": map[string]any{
- "question": map[string]any{
- "type": "string",
- "description": "The research question to investigate.",
- },
- },
- "required": []string{"question"},
- },
- },
- },
- {
- "type": "function",
- "function": map[string]any{
- "name": "summarize_document",
- "description": "Read an entire document by id into the evidence set and return a short summary the model can answer from. " +
- "WHEN TO CALL: only when you ALREADY hold a doc_id — returned by an earlier `rag` call, or given by the user. " +
- "DO NOT CALL: when you have no doc_id, or the question asks WHICH documents exist, or it needs several " +
- "documents compared or aggregated — that is the `rag` tool's job. This tool cannot find a document: " +
- "called without a real doc_id it only returns an error and wastes a round.",
- "parameters": map[string]any{
- "type": "object",
- "properties": map[string]any{
- "doc_id": map[string]any{
- "type": "string",
- "description": "The document id to read — one you already hold from a prior `rag` result or from the user.",
- },
- },
- "required": []string{"doc_id"},
- },
- },
- },
- }
-
- history := make([]models.Message, 0, len(deps.Messages))
- for _, m := range deps.Messages {
- msg := models.Message{Role: string(m.Role), Content: m.Content}
- if len(m.UserInputMultiContent) > 0 {
- msg.Content = multimodalContentBlocks(m)
- }
- history = append(history, msg)
- }
- system := deps.SystemPrompt
- if strings.TrimSpace(system) == "" {
- system = "You are a research assistant. Use the tools provided to gather evidence and answer the user's question with citations where possible."
- }
-
- return &outerReactParts{
- kb: kb,
- sd: sd,
- spec: spec,
- resp: &RunResponse{Mode: spec, Kbinfos: kb},
- tools: tools,
- history: history,
- system: system,
- }
-}
-
-// outerStreamMux merges the outer model's stream with the inner rag stream into one sink,
-// so the outer model's reasoning, the inner research log and the inner answer all share a
-// single think/answer block.
-//
-// The models layer hands the outer stream down as (delta, reason) text that may
-// carry literal <think>/</think> markers, while the sink takes an explicit
-// isThink flag; the mux tracks the markers to keep the state and forwards plain
-// text only.
-type outerStreamMux struct {
- mu sync.Mutex
- sink *AnswerSink
- inThink bool
- terminalFired bool
- // outerText accumulates the outer model's OWN non-think text. A tool-less reply is the
- // answer, so the streaming loop must be able to hand it back to the caller instead of
- // dropping it (the inner `rag` stream arrives through deliver, never through here).
- outerText strings.Builder
-}
-
-// sender is the (delta, reason) callback handed to ChatStreamlyWithTools.
-func (m *outerStreamMux) sender(delta, reason *string) error {
- var text string
- if delta != nil {
- text = *delta
- }
- if reason != nil && *reason != "" {
- text += *reason
- }
- if text == "" {
- return nil
- }
- m.mu.Lock()
- defer m.mu.Unlock()
- for text != "" {
- if strings.HasPrefix(text, "<think>") {
- m.inThink = true
- text = text[len("<think>"):]
- continue
- }
- if strings.HasPrefix(text, "</think>") {
- m.inThink = false
- text = text[len("</think>"):]
- continue
- }
- next := len(text)
- if i := strings.Index(text, "<think>"); i >= 0 && i < next {
- next = i
- }
- if i := strings.Index(text, "</think>"); i >= 0 && i < next {
- next = i
- }
- chunk := text[:next]
- text = text[next:]
- if chunk == "" {
- continue
- }
- // Non-think outer text is dropped once the terminal tool has fired: what follows is
- // the aggregate tool result, and the answer has already been streamed from inside
- // the tool.
- if m.terminalFired && !m.inThink {
- continue
- }
- if !m.inThink {
- m.outerText.WriteString(chunk)
- }
- m.emit(chunk, m.inThink)
- }
- return nil
-}
-
-// deliver is the inner-stream entry point: research progress (isThink) and the
-// composed answer (not think).
-func (m *outerStreamMux) deliver(delta string, isThink bool) {
- if delta == "" {
- return
- }
- m.mu.Lock()
- defer m.mu.Unlock()
- m.inThink = isThink
- m.emit(delta, isThink)
-}
-
-// markTerminal records that the terminal `rag` tool ran, so later outer text is treated
-// as the aggregate tool result and dropped.
-func (m *outerStreamMux) markTerminal() {
- m.mu.Lock()
- defer m.mu.Unlock()
- m.terminalFired = true
-}
-
-// emit forwards one chunk. Caller must hold m.mu.
-func (m *outerStreamMux) emit(chunk string, isThink bool) {
- if m.sink == nil || m.sink.OnDelta == nil {
- return
- }
- m.sink.OnDelta(chunk, isThink)
-}
-
-// runOuterReact drives the rag_agent outer loop (tools=[rag, summarize_document], with
-// `rag` terminal). The outer chat model decides whether to:
-// - call `rag` (terminal): run the full inner agentic graph and return the
-// cited answer, which the terminal short-circuit turns into the final answer;
-// - call `summarize_document` (non-terminal): read a whole document into the
-// evidence set and let the model answer from it on the next round;
-// - answer directly with no tool call.
-//
-// deps.Outer must be non-nil; Rag() only delegates here when it is set, so
-// existing callers that don't wire an outer model keep using the direct
-// RunAgenticRAG path unchanged.
-func runOuterReact(ctx context.Context, deps RAGTools, req runtime.RunRequest, logger *log.Logger) *RunResponse {
- p := prepareOuterReact(ctx, deps, req, logger)
-
- session := &outerReactSession{
- ctx: ctx,
- deps: deps,
- req: req,
- sd: p.sd,
- kb: p.kb,
- resp: p.resp,
- logger: logger,
- spec: p.spec,
- }
-
- outer := deps.Outer
- outer.BindTools(session, p.tools)
- outer.SetTerminalTools("rag")
-
- // The loop's opening and closing steps are reported HERE — where the loop
- // actually is. The chat pipeline used to synthesize these lines before
- // calling the runtime, so they appeared even when no outer loop ran and the
- // research came from the direct graph: the trace described a loop that had
- // not happened.
- loop := runtime.StepsFrom(ctx)
- loop.Stage(logger, "Tool loop", "Deciding what to do next (step 1); available tools: %s", outerToolNames(p.tools))
-
- answer, _, err := outer.ChatWithTools(ctx, p.system, p.history, &models.ChatConfig{})
- if err != nil {
- loop.StageLine(logger, "Tool loop", "The outer model call failed; running the research graph directly.")
- logger.Printf("[Agentic RAG] outer react failed: %v; falling back to direct graph", err)
- // Fall back to the inner graph directly so the user still gets an
- // answer. The graph composes inside its last node with the FORMALIZED question
- // (read from the graph state); the guarded direct call below only fires when the
- // graph never composed.
- composed := false
- deps.Finalize = func(fctx context.Context, partial, empty bool, question string) {
- if composed {
- return
- }
- composed = true
- composeFinalAnswer(fctx, deps, req, p.kb, p.resp, logger, partial, empty, question)
- }
- RunAgenticRAG(ctx, deps, req, p.sd, p.kb, p.resp, logger, p.spec)
- if !composed {
- composeFinalAnswer(ctx, deps, req, p.kb, p.resp, logger, p.resp.Partial, true, "")
- }
- return p.resp
- }
- // The fold returned the LOWEST-INDEX terminal rag call's answer: keep that
- // call's evidence so the answer's [ID:n] markers line up with the chunks.
- session.selectEvidence(answer)
- p.resp.Answer = answer
- p.resp.Chunks = p.kb.Chunks
- p.resp.DocAggs = p.kb.DocAggs
- // The numbering the answer's [ID:n] markers were written with travels WITH those chunks:
- // the chat pipeline resolves a marker by position in this list, so a response carrying the
- // passages without their numbering leaves the list empty and every marker indexes the POOL
- // instead — the citation then opens whatever passage happens to sit at that position.
- p.resp.CiteChunkIDs = append([]string(nil), p.kb.CiteChunkIDs...)
- p.resp.EmptyResult = len(p.kb.Chunks) == 0
- loop.StageLine(logger, "Tool loop", outerLoopEndLine(session.ragCalls(), strings.TrimSpace(p.resp.Answer) != ""))
- return p.resp
-}
-
-// runOuterReactStream is the streaming counterpart of runOuterReact: it drives the same
-// [rag, summarize_document] react loop but merges two live streams into the single sink
-// the caller supplied:
-// - the outer model's own reasoning/text, arriving through the models layer;
-// - the inner rag run's research log (think) and composed answer (answer),
-// produced while the terminal `rag` tool executes.
-//
-// `rag` streams its answer itself and returns "", so the terminal short-circuit stops the
-// loop without re-streaming it (the terminal result is ignored once the inner sink has
-// streamed it).
-func runOuterReactStream(ctx context.Context, deps RAGTools, req runtime.RunRequest, logger *log.Logger) *RunResponse {
- p := prepareOuterReact(ctx, deps, req, logger)
- mux := &outerStreamMux{sink: deps.AnswerSink}
-
- // Route the inner run's ANSWER through the mux so it shares one block with
- // the outer model's stream. The inner run's STEPS are not routed here: the
- // reporter was already bound on ctx by Rag, and steps read it from there
- // (overwriting a copy's Steps would have no effect).
- inner := deps
- inner.AnswerSink = &AnswerSink{OnDelta: func(delta string, isThink bool) { mux.deliver(delta, isThink) }}
-
- session := &outerReactSession{
- ctx: ctx,
- deps: inner,
- req: req,
- sd: p.sd,
- kb: p.kb,
- resp: p.resp,
- logger: logger,
- spec: p.spec,
- mux: mux,
- }
-
- outer := deps.Outer
- outer.BindTools(session, p.tools)
- outer.SetTerminalTools("rag")
-
- // Same opening step as runOuterReact, reported where the loop is.
- loop := runtime.StepsFrom(ctx)
- loop.Stage(logger, "Tool loop", "Deciding what to do next (step 1); available tools: %s", outerToolNames(p.tools))
-
- stream := true
- _, err := outer.ChatStreamlyWithTools(ctx, p.system, p.history, &models.ChatConfig{Stream: &stream}, mux.sender)
- if err != nil {
- loop.StageLine(logger, "Tool loop", "The outer model call failed; running the research graph directly.")
- logger.Printf("[Agentic RAG] outer react stream failed: %v; falling back to direct graph", err)
- // Fall back with the caller's own sink so the answer still streams out.
- // Same guarded compose as runOuterReact: the graph composes inside its
- // last node with the FORMALIZED question; the direct call only fires
- // when the graph never reached it.
- composed := false
- deps.Finalize = func(fctx context.Context, partial, empty bool, question string) {
- if composed {
- return
- }
- composed = true
- composeFinalAnswer(fctx, deps, req, p.kb, p.resp, logger, partial, empty, question)
- }
- RunAgenticRAG(ctx, deps, req, p.sd, p.kb, p.resp, logger, p.spec)
- if !composed {
- composeFinalAnswer(ctx, deps, req, p.kb, p.resp, logger, p.resp.Partial, true, "")
- }
- return p.resp
- }
- // A tool-less reply is the answer (the outer model's text is returned verbatim when it
- // decides not to call a tool). The stream already delivered it to the caller's sink;
- // recording it on the response is what lets the chat pipeline short-circuit instead of
- // composing a second,
- // contradicting answer from the (empty) evidence set. When the terminal
- // `rag` tool fired, composeFinalAnswer already owns resp.Answer.
- if p.resp.Answer == "" && !mux.terminalFired {
- p.resp.Answer = strings.TrimSpace(mux.outerText.String())
- }
- // Keep only the evidence of the call that owns the answer (selectEvidence
- // no-ops when the answer came from the outer model instead of a rag call).
- session.selectEvidence(p.resp.Answer)
- // Same as runOuterReact: the numbering the answer's [ID:n] markers were written with has to
- // travel with the passages it numbers. Without it the chat pipeline's published list is
- // empty, citePoolIdx falls back to pool order, and every marker opens whichever passage
- // happens to sit at that position (rendered_blocks == pool size is the tell).
- p.resp.CiteChunkIDs = append([]string(nil), p.kb.CiteChunkIDs...)
- p.resp.Chunks = p.kb.Chunks
- p.resp.DocAggs = p.kb.DocAggs
- p.resp.EmptyResult = len(p.kb.Chunks) == 0
- loop.StageLine(logger, "Tool loop", outerLoopEndLine(session.ragCalls(), strings.TrimSpace(p.resp.Answer) != ""))
- return p.resp
-}
-
-// outerToolNames renders the bound tool list for the loop's opening step: the
-// OpenAI-style schemas the outer model was actually given.
-func outerToolNames(tools []map[string]any) string {
- names := make([]string, 0, len(tools))
- for _, t := range tools {
- fn, _ := t["function"].(map[string]any)
- name, _ := fn["name"].(string)
- if name != "" {
- names = append(names, name)
- }
- }
- return strings.Join(names, ", ")
-}
-
-// outerLoopEndLine closes the loop's narration with what actually ended it. Its two
-// inputs are independent — whether the `rag` tool was called at all, and whether the
-// reply ended up carrying an answer — so there are four endings, not two: the
-// terminal `rag` call produced the cited answer, or it ran and produced nothing,
-// while the outer model may also have answered on its own, with no research behind
-// it, which is why THAT reply can legitimately carry no citations.
-func outerLoopEndLine(ragCalls int, answered bool) string {
- switch {
- case ragCalls == 0 && answered:
- return "The outer model produced the answer without running research."
- case ragCalls == 0:
- return "The outer model returned no answer and ran no research."
- case !answered:
- return "The rag tool ran but produced no answer."
- default:
- return "The rag tool produced the final answer, done."
- }
-}
-
-// outerReactSession implements models.ToolCallSession for the outer react loop.
-type outerReactSession struct {
- ctx context.Context
- deps RAGTools
- req runtime.RunRequest
- sd runtime.SearchDeps
- kb *runtime.Kbinfos
- resp *RunResponse
- logger *log.Logger
- spec runtime.ModeSpec
- // mux is set only on the streaming path; it merges the inner run's output
- // into the caller's sink.
- mux *outerStreamMux
- // mu guards the per-call result merge (publish) and the recorded per-call
- // outcomes (calls): appendToolResults invokes this session's tool calls
- // concurrently.
- mu sync.Mutex
- // calls records each rag call's answer + evidence pool in completion order so
- // selectEvidence can match the terminal fold's winning answer back to its own
- // pool. Guarded by mu.
- calls []ragCallResult
-
- // flightMu guards inflight, the single-flight registry for CONCURRENT
- // IDENTICAL rag calls.
- flightMu sync.Mutex
- // inflight maps the rag question argument to its in-progress flight.
- // Guarded by flightMu.
- inflight map[string]*ragFlight
-}
-
-// ragFlight is ONE in-progress rag execution shared by every concurrent identical tool
-// call — Go-only robustness, since the outer model sometimes emits the SAME rag call 2-3
-// times in one round (observed with MiniMax-M3 in frame_benchmark 20260912_091520 — two
-// graph runs started at the same millisecond, three planner runs for one question). The
-// tool layer (models.appendToolResults) executes them all concurrently while the terminal
-// fold keeps only the FIRST result. The duplicates each burn a full graph run
-// (double provider load → the SCA deadline overruns in the same log) and
-// their — sometimes better — answers are discarded. The first caller executes
-// and publishes; the rest block on done and return the same answer.
-type ragFlight struct {
- done chan struct{}
- answer string
-}
-
-// beginRagFlight registers the caller as the OWNER of question's execution, or returns
-// the existing flight to wait on (owner=nil, wait!=nil). The key is the literal question
-// argument; near-identical (but not identical) re-asks stay on the RAGCache similarity
-// path (Rag's Lookup).
-func (s *outerReactSession) beginRagFlight(question string) (owner, wait *ragFlight) {
- s.flightMu.Lock()
- defer s.flightMu.Unlock()
- if existing, ok := s.inflight[question]; ok {
- return nil, existing
- }
- flight := &ragFlight{done: make(chan struct{})}
- if s.inflight == nil {
- s.inflight = make(map[string]*ragFlight)
- }
- s.inflight[question] = flight
- return flight, nil
-}
-
-// endRagFlight removes the flight and releases its waiters. The answer MUST be
-// stored on the flight before endRagFlight runs: closing done happens-before
-// every waiter's receive, so their subsequent read of answer sees the write
-// (Go memory model, channel close).
-func (s *outerReactSession) endRagFlight(question string, flight *ragFlight) {
- s.flightMu.Lock()
- delete(s.inflight, question)
- s.flightMu.Unlock()
- close(flight.done)
-}
-
-// ToolCall routes the two outer tools to their Go implementations.
-//
-// A round's tool calls arrive CONCURRENTLY — models.appendToolResults runs one goroutine
-// per call. Each rag call therefore runs on
-// its own request/evidence/response and publishes the outcome under mu; sharing
-// the session's would let two calls mix questions, evidence and answers.
-func (s *outerReactSession) ToolCall(name string, arguments map[string]interface{}) (string, error) {
- // Mirror Python's FunctionToolSession (rag/llm/tool_decorator.py:311), which
- // logs "[Function tool] Running the {name} tool with: {args}" for the OUTER
- // tools too — the inner retrieval tools log the same line from
- // retrieval.searchExecutor.Execute. Without it the outer loop's only trace is
- // chat_pipeline's synthetic "[Tool loop] Step N: running rag…", which never
- // shows the arguments the outer model actually chose.
- renderedArgs := runtime.RenderToolArgs(arguments)
- // Same split as the inner executor: the log and the event's Args keep the
- // arguments verbatim, the step reports the query (runtime.ToolCallLine).
- if s.logger != nil && name != "" {
- s.logger.Printf("[Function tool] Running the %s tool with: %s", name, renderedArgs)
- }
- if name != "" {
- runtime.StepsFrom(s.ctx).Emit(runtime.ThinkEvent{
- Kind: runtime.ThinkKindToolCall,
- Stage: "Function tool",
- Tool: name,
- Args: renderedArgs,
- Summary: "[Function tool] " + runtime.ToolCallLine(name, arguments),
- })
- }
- started := time.Now()
- switch name {
- case "rag":
- // Work on a COPY of the request: this call must not rewrite the
- // session's question/images, which a concurrent call is reading.
- req := s.req
- if q, ok := arguments["question"].(string); ok && q != "" {
- req.Question = q
- }
- // Prefer the user's ORIGINAL, complete
- // question over the outer model's rewritten `question` argument when
- // both clearly describe the same turn. The outer rewrite often drops
- // the final target of a multi-hop question, and no later stage can
- // recover a deleted answer-attribute. The defense line sits inside the `rag` tool
- // itself, i.e. per outer tool call.
- if effective := resolveEffectiveQuestion(req.Question, s.deps.OriginalQuestion); effective != req.Question {
- s.logger.Printf("[Agentic RAG] using original user question over outer rewrite (original=%q → rewrite=%q)",
- trunc(s.deps.OriginalQuestion, 80), trunc(req.Question, 80))
- req.Question = effective
- }
- // The inner compose is text-only: the outer model already saw the images via
- // multimodal history, and the rephrased question carries no images. Drop the original
- // images so the inner compose model (ComposeAnswerWith via composeFinalAnswer) does
- // not receive them.
- req.Images = nil
-
- // Single-flight (see ragFlight): the FIRST caller owns the execution;
- // identical concurrent calls block until it finishes and return the
- // same answer instead of re-running the whole graph.
- flight, wait := s.beginRagFlight(req.Question)
- if wait != nil {
- <-wait.done
- // A waiting duplicate still closes its own call/result pair, so a
- // client rendering steps never sees a dangling tool_call.
- s.narrateToolStep(runtime.ThinkEvent{
- Tool: name,
- Args: renderedArgs,
- Status: runtime.StatusOK,
- DurationMS: time.Since(started).Milliseconds(),
- }, fmt.Sprintf("The rag tool reused the answer of an identical concurrent call%s.", runtime.ArgsLabel(arguments)))
- return wait.answer, nil
- }
- defer s.endRagFlight(req.Question, flight)
-
- // Per-call evidence, response and the projections that point at them: every rag()
- // invocation gets its own graph state and shares only the tools object.
- kb := &runtime.Kbinfos{}
- resp := &RunResponse{Mode: s.spec, Kbinfos: kb}
- sd := s.sd
- sd.KB = kb
- inner := s.deps
- inner.KB = kb
-
- // Composition happens INSIDE the graph, from the formalize_answer node's state:
- // partial_answer from the node, empty_result always true there, and question =
- // state["question"], the FORMALIZED multi-hop question. Wire the per-call Finalize
- // so the graph's last node composes itself; the guarded direct call below only fires
- // when the graph never reached that node.
- composed := false
- inner.Finalize = func(fctx context.Context, partial, empty bool, question string) {
- if composed {
- return
- }
- composed = true
- composeFinalAnswer(fctx, inner, req, kb, resp, s.logger, partial, empty, question)
- }
-
- if s.mux != nil {
- // Streaming: s.deps already routes the inner research log and the
- // composed answer into the caller's sink, so return "" — the
- // terminal short-circuit stops the loop and sendTerminal stays
- // silent, so the answer is not streamed twice.
- s.mux.markTerminal()
- } else {
- // Non-streaming: disable the sink so the answer is not emitted
- // twice, and hand this call's own answer back for the terminal
- // short-circuit.
- inner.AnswerSink = nil
- }
- // Nested: this whole research graph runs INSIDE the `rag` tool call reported
- // above, so its steps (and everything beneath them) are indented under that
- // call line.
- RunAgenticRAG(runtime.Nested(s.ctx), inner, req, sd, kb, resp, s.logger, s.spec)
- if !composed {
- composeFinalAnswer(s.ctx, inner, req, kb, resp, s.logger, resp.Partial, true, "")
- }
- // A "[Research status]" note is appended to the TOOL RESULT for every
- // non-SUFFICIENT verdict, so the outer model can decide whether to re-run `rag` from
- // the reported gaps. It does NOT change the streamed answer (the compose already
- // streamed); after two consecutive unsatisfying rounds it tells the outer agent to
- // STOP.
- if t := researchStatusTrailer(s.deps.Cache, resp); t != "" {
- // Same fold as Rag's direct path: the period closes the hint
- // clause before the trailer sentence.
- resp.Answer += "\n\n[Research status] " + resp.SCAFeedback + "." + t
- }
- s.publish(resp, kb)
- // Close the "[Function tool] Running the rag tool with: …" line. The
- // result IS the cited answer — far too long to repeat in a log line — so
- // narrate what it was grounded in instead.
- s.narrateToolStep(runtime.ThinkEvent{
- Tool: name,
- Args: renderedArgs,
- Status: runtime.StatusOK,
- Results: len(kb.Chunks),
- Documents: len(runtime.ChunkDocIDs(kb.Chunks)),
- Sources: runtime.ChunkEvidenceIDs(kb.Chunks, runtime.ThinkMaxSources),
- DurationMS: time.Since(started).Milliseconds(),
- }, ragToolResultLine(resp, kb, runtime.ArgsLabel(arguments)))
- if s.mux != nil {
- // The answer already streamed through the shared mux; waiters
- // return "" too — sendTerminal's empty guard keeps the fold from
- // re-streaming it (markTerminal already fired on this call).
- flight.answer = ""
- return "", nil
- }
- // Waiters replay this exact answer; publish already ran once above, so
- // the evidence pool is not unioned twice and selectEvidence keeps a
- // single unambiguous call record.
- flight.answer = resp.Answer
- return resp.Answer, nil
- case "summarize_document":
- docID, _ := arguments["doc_id"].(string)
- if docID == "" {
- s.narrateToolStep(runtime.ThinkEvent{
- Tool: name, Args: renderedArgs, Status: runtime.StatusError, Reason: runtime.ReasonBadArgs,
- DurationMS: time.Since(started).Milliseconds(),
- }, "The summarize_document tool could not run: it was called without a doc_id.")
- return "Error: missing doc_id argument.", nil
- }
- // The document read and its prompt are budgeted at the model's FULL context window,
- // not a fixed cap. No citation header: this dialog path runs with do_refer=false, so
- // summarize_document returns bare blocks there.
- blocks := runtime.SummarizeDocument(s.ctx, s.sd, docID, s.deps.MaxLength)
- if len(blocks) == 0 {
- s.narrateToolStepDetail(runtime.ThinkEvent{
- Tool: name, Args: renderedArgs, Status: runtime.StatusEmpty, Reason: runtime.ReasonNoDoc,
- Sources: []string{docID},
- DurationMS: time.Since(started).Milliseconds(),
- }, "The summarize_document tool found nothing readable in the requested document.",
- fmt.Sprintf("The summarize_document tool found nothing readable in document %s.", docID))
- return "The document is unavailable or has no readable chunks.", nil
- }
- // Only the summary knows the id (the blocks are the document's own text),
- // so the think copy names what was read — "the requested document" — and
- // the log copy carries the id.
- s.narrateToolStepDetail(runtime.ThinkEvent{
- Tool: name, Args: renderedArgs, Status: runtime.StatusOK,
- // The blocks are the document's own text, so "results" is the block
- // count and there is exactly one source document.
- Results: len(blocks),
- Documents: 1,
- Sources: []string{docID},
- DurationMS: time.Since(started).Milliseconds(),
- }, fmt.Sprintf("The summarize_document tool read %s from the requested document.",
- runtime.CountOf(len(blocks), "evidence block")),
- fmt.Sprintf("The summarize_document tool read %s from document %s.",
- runtime.CountOf(len(blocks), "evidence block"), docID))
- return strings.Join(blocks, "\n\n"), nil
- }
- // An unknown name is a tool this deployment has no binding for: nothing ran,
- // which is MISS-level (ReasonUnwired), not an infra failure. Same wording as
- // the inner executor's unwired branch so both layers read alike.
- s.narrateToolStep(runtime.ThinkEvent{
- Tool: name, Args: renderedArgs, Status: runtime.StatusMiss, Reason: runtime.ReasonUnwired,
- DurationMS: time.Since(started).Milliseconds(),
- }, fmt.Sprintf("The %s tool is not wired in this deployment, so nothing ran.", name))
- return fmt.Sprintf("unknown tool %q", name), nil
-}
-
-// narrateToolStep closes one outer "[Function tool] Running the …" line: it logs
-// what the call returned (the developer log) and reports the step (think block +
-// structured twin), so the trace records a whole tool step instead of only its
-// invocation.
-// ragCalls reports how many rag executions this session published, i.e. whether
-// the outer loop actually ran research instead of the model answering on its own.
-func (s *outerReactSession) ragCalls() int {
- s.mu.Lock()
- defer s.mu.Unlock()
- return len(s.calls)
-}
-
-func (s *outerReactSession) narrateToolStep(ev runtime.ThinkEvent, line string) {
- s.narrateToolStepDetail(ev, line, line)
-}
-
-// narrateToolStepDetail is narrateToolStep for the steps whose two audiences need
-// different words: `human` is what the think block reports, `dev` is what the
-// developer log records. It mirrors the inner executor's split
-// (runtime.StageLineDetail and the two labels renderToolOutcome accepts) and
-// exists for the same reason: a 32-hex document id is a developer's handle and a
-// reader's noise, so summarize_document names the document in words in the think
-// block and by id in the log.
-func (s *outerReactSession) narrateToolStepDetail(ev runtime.ThinkEvent, human, dev string) {
- if s.logger != nil {
- s.logger.Printf("[Function tool] %s", dev)
- }
- ev.Kind = runtime.ThinkKindToolResult
- ev.Stage = "Function tool"
- ev.Summary = "[Function tool] " + human
- runtime.StepsFrom(s.ctx).Emit(ev)
-}
-
-// ragToolResultLine summarises one outer `rag` call for the think block. The
-// result is the cited answer itself, so the line reports what it was grounded in
-// and whether research came back empty instead. `label` names the sub-question
-// this call researched (runtime.ArgsLabel): the outer loop can run several `rag`
-// calls concurrently, and without it two parallel calls' result lines are
-// indistinguishable.
-func ragToolResultLine(resp *RunResponse, kb *runtime.Kbinfos, label string) string {
- chunks := 0
- if kb != nil {
- chunks = len(kb.Chunks)
- }
- if chunks == 0 {
- return fmt.Sprintf("The rag tool returned no answer%s: research gathered no evidence.", label)
- }
- if strings.TrimSpace(resp.Answer) == "" {
- return fmt.Sprintf("The rag tool gathered %s but composed no answer%s.", runtime.CountOf(chunks, "passage"), label)
- }
- return fmt.Sprintf("The rag tool returned a cited answer grounded in %s%s.", runtime.CountOf(chunks, "passage"), label)
-}
-
-// publish merges one rag call's outcome into the session state. The inner run
-// used to write that state directly, before each call got its own; the merge
-// reproduces what a single call contributed, and for concurrent calls it keeps
-// the FIRST non-empty answer/verdict (appendToolResults folds the first terminal
-// hit in call order) while the evidence is unioned, so the caller still sees the
-// chunks the answer was composed from.
-func (s *outerReactSession) publish(call *RunResponse, kb *runtime.Kbinfos) {
- s.mu.Lock()
- defer s.mu.Unlock()
- // Remember this call's own answer + evidence so selectEvidence can restore
- // the pool the winning answer was composed from.
- s.calls = append(s.calls, ragCallResult{answer: call.Answer, kb: kb})
- if s.resp.Answer == "" {
- s.resp.Answer = call.Answer
- }
- if s.resp.Verdict == "" {
- s.resp.Verdict = call.Verdict
- }
- if s.resp.SCAFeedback == "" {
- s.resp.SCAFeedback = call.SCAFeedback
- }
- if s.resp.CollectedAnswer == "" {
- s.resp.CollectedAnswer = call.CollectedAnswer
- }
- if len(s.resp.Slots) == 0 {
- s.resp.Slots = call.Slots
- }
- s.resp.Partial = s.resp.Partial || call.Partial
- s.kb.Chunks = append(s.kb.Chunks, kb.Chunks...)
- s.kb.DocAggs = append(s.kb.DocAggs, kb.DocAggs...)
- s.kb.Memory = append(s.kb.Memory, kb.Memory...)
- if s.kb.PreSummary == "" {
- s.kb.PreSummary = kb.PreSummary
- }
-}
-
-// ragCallResult is one rag call's outcome: the answer it composed and the
-// evidence pool that produced it.
-type ragCallResult struct {
- answer string
- kb *runtime.Kbinfos
-}
-
-// selectEvidence leaves only the evidence of the rag call whose answer the
-// terminal fold returned. The fold picks the LOWEST-INDEX terminal call while
-// publish sees completion order, so the union alone can leave that answer's
-// [ID:n] markers pointing into another call's chunks — the answer and the
-// reference list would disagree. No-op when no recorded call matches (the
-// terminal tool was summarize_document, or the answer did not come from a rag
-// call), in which case the union stands.
-//
-// Matching is by exact answer text, so two calls that produce byte-identical
-// answers are indistinguishable and the FIRST recorded (first to finish) one
-// wins. That collision is accepted: plumbing the tool-call index through
-// ToolCallSession is not worth it for a case that cannot happen with distinct
-// questions.
-func (s *outerReactSession) selectEvidence(answer string) {
- if answer == "" {
- return
- }
- s.mu.Lock()
- defer s.mu.Unlock()
- for _, r := range s.calls {
- if r.answer != answer {
- continue
- }
- s.kb.Chunks = append([]map[string]any(nil), r.kb.Chunks...)
- s.kb.DocAggs = append([]map[string]any(nil), r.kb.DocAggs...)
- s.kb.Memory = append([]map[string]any(nil), r.kb.Memory...)
- s.kb.PreSummary = r.kb.PreSummary
- // The published evidence list belongs to THIS call too: the answer's [ID:n] markers
- // were written against it, and the chat pipeline resolves them against whatever is on
- // the response. Restoring the passages without their numbering leaves marker n pointing
- // at another call's n-th passage — the citation opens a passage the answer never cited.
- s.kb.CiteChunkIDs = append([]string(nil), r.kb.CiteChunkIDs...)
- return
- }
-}
-
-// runDirect is the low/naive path: one hybrid search, no tool loop.
-//
-// It is the ONE implementation of the direct search (called from the low graph node) —
-// there is deliberately no runtime-level duplicate in runtime/orchestrator. It lives here
-// rather than there because the step reads the full RAGTools config; the graph node that
-// calls it forces UseCompiled=true.
-func runDirect(ctx context.Context, deps RAGTools, req runtime.RunRequest, sd runtime.SearchDeps, kb *runtime.Kbinfos, resp *RunResponse, logger *log.Logger) {
- ctx, done := runtime.Phase(ctx, runtime.PhaseDirect)
- retrievalQuery := ""
- if deps.Keywords != nil {
- rq, err := deps.Keywords(ctx, req.Question)
- if err != nil {
- logger.Printf("[Agentic RAG] weighted keyword extraction failed: %v", err)
- } else {
- retrievalQuery = rq
- }
- } else if deps.Model != nil {
- // Default: the four-aspect weighted extraction (runtime/keywords.go). It gives BM25
- // both the discriminating entity and the surface variants the corpus may use.
- rq, kw := runtime.ExtractWeightedKeywords(ctx, deps.Model, req.Question)
- retrievalQuery = rq
- if req.Keywords == "" {
- req.Keywords = kw
- }
- }
- defer done()
-
- chunks, aggs := runtime.HybridSearch(ctx, sd, runtime.SearchParams{
- Question: req.Question,
- Keywords: req.Keywords,
- RetrievalQuery: retrievalQuery,
- UseCompiled: req.UseCompiled,
- TopN: req.TopN,
- // The only channel honouring UsingEmbedding.
- Channel: runtime.ChannelRetrieve,
- })
- kb.Merge(chunks, aggs)
- if !kb.HasChunks() {
- logger.Printf("[Agentic RAG] direct search found no matching passages")
- }
-}
-
-// AgenticLoop runs the full agentic search loop (planner / prefetch / SCA↔
-// rewriter iteration) for an agentic mode.
-//
-// It is registered by the agentic_rag package's init rather than called directly: the
-// agentic_rag package owns RAGTools, so the runtime cannot import it back. The
-// indirection keeps the dependency graph acyclic. Registering is optional — see
-// Run for the fallback when nothing is registered.
-type AgenticLoop func(ctx context.Context, deps RAGTools, req runtime.RunRequest, kb *runtime.Kbinfos, resp *RunResponse, logger *log.Logger)
-
-var agenticLoop AgenticLoop
-
-// SetAgenticLoop registers the outer agentic loop. Called from the agent
-// package's init(); importing that package (even blank) activates the full
-// medium/high/ultra pipeline. Without it those modes degrade to a single action
-// session.
-func SetAgenticLoop(fn AgenticLoop) { agenticLoop = fn }
-
-// runAgentic drives medium/high/ultra.
-//
-// When an agentic loop is registered (see SetAgenticLoop) it runs the full
-// five-phase pipeline: planner fan-out → slot table → slot research rounds →
-// SCA review → gap rewrite → repeat until sufficient or the budget runs out.
-//
-// Fallback (no loop registered): ONE action session via RunActionSession. This
-// keeps `Run` usable without the planner, at the cost of the
-// planner/fan-out/SCA iteration.
-func runAgentic(ctx context.Context, deps RAGTools, req runtime.RunRequest, sd runtime.SearchDeps, kb *runtime.Kbinfos, resp *RunResponse, logger *log.Logger) {
- // Formalization happens inside the graph: it is the graph's entry node
- // (START → formalize_question).
- if agenticLoop != nil {
- agenticLoop(ctx, deps, req, kb, resp, logger)
- return
- }
- if deps.Model == nil {
- logger.Printf("[Agentic RAG] no model configured for mode %q; degrading to a direct search", resp.Mode.Label)
- runDirect(ctx, deps, req, sd, kb, resp, logger)
- return
- }
- runSingleSession(ctx, deps, req, sd, kb, resp, logger)
-}
-
-// runSingleSession is the fallback agentic path: seed a slot table and run ONE
-// bounded action session against it.
-func runSingleSession(ctx context.Context, deps RAGTools, req runtime.RunRequest, sd runtime.SearchDeps, kb *runtime.Kbinfos, resp *RunResponse, logger *log.Logger) {
- // Seed the slot table and run one bounded session against it. A fresh tool
- // cache and query history per run: they exist to dedupe WITHIN a session,
- // not across requests.
- sd.WebSearch = deps.WebSearch
- sd.CiteRules = deps.CiteRules
- sessionDeps := runtime.SessionDeps{
- Tools: &runtime.Toolset{
- ThinkingMode: resp.Mode.Label,
- // Provider gate: without a wired provider the tool is hidden rather than
- // advertised dead.
- HasWebSearch: resp.Mode.HasTool("web_search") && deps.WebSearch != nil,
- // The dataset's real metadata fields (see NewAgenticLoop).
- MetadataFields: runtime.MetadataCatalogPtr(ctx, sd),
- DisabledTools: map[string]bool{},
- Exec: runtime.NewSearchExecutor(sd, req),
- },
- Model: deps.Model,
- Prompts: deps.Prompts,
- // Surface already-retrieved evidence into the action session seed so the
- // ReAct loop fills slots from what it has instead of re-searching.
- KB: kb,
- }
-
- // run_agentic_rag — there is no whole-graph wall clock: research stays
- // bounded by per-node timeouts, the routing guards and the recursion limit.
- // The only budget is the one AgenticState carries (formalize_question), and it is
- // set when the state is created, so formalization is not charged to it.
- init := runtime.InitializeState(ctx, sessionDeps, req.Question, nil, req.DeadlineLeft)
- root := init.Root
-
- result := runtime.RunActionSession(ctx, sessionDeps, req.Question, root, req.DeadlineLeft, "", nil, nil)
- // run_agentic_rag — failure to run research is recorded separately from
- // "research found nothing": the session yielded neither messages nor states,
- // so nothing was produced at all.
- if len(result.Messages) == 0 && len(result.NewStates) == 0 && result.FoundAnswer == nil {
- resp.GraphFailed = true
- logger.Printf("[Agentic RAG] graph execution produced nothing (mode=%q)", resp.Mode.Label)
- }
-
- if result.FoundAnswer != nil {
- resp.Answer = *result.FoundAnswer
- }
- // Report the deepest slot table the session produced: later states carry
- // strictly more filled slots than the root.
- if len(result.NewStates) > 0 {
- best := result.NewStates[len(result.NewStates)-1]
- resp.Slots = append(resp.Slots, best.State...)
- } else {
- resp.Slots = append(resp.Slots, root.State...)
- }
-}
-
-// IsKnownMode reports whether label names a configured thinking mode.
-func IsKnownMode(label string) bool {
- _, ok := runtime.THINKING_MODES[strings.ToLower(strings.TrimSpace(label))]
- return ok
-}
-
-// DocIDLookup
-// it resolves which of a candidate document id set exist within the given
-// datasets.
-//
-// It reads the document table directly; the runtime package takes it through the
-// runtime.DocIDVerifier interface so the retrieval path stays free of a database
-// dependency.
-type DocIDLookup struct {
- docs *dao.DocumentDAO
-}
-
-// NewDocIDLookup returns the document-ownership check backed by the document
-// table.
-func NewDocIDLookup() *DocIDLookup {
- return &DocIDLookup{docs: &dao.DocumentDAO{}}
-}
-
-// KnownDocIDs implements runtime.DocIDVerifier: it returns the subset of
-// candidates that belong to datasetIDs.
-func (l *DocIDLookup) KnownDocIDs(ctx context.Context, datasetIDs, candidates []string) (map[string]bool, error) {
- if l == nil || l.docs == nil || len(candidates) == 0 || len(datasetIDs) == 0 {
- return nil, nil
- }
- docs, err := l.docs.GetByIDs(ctx, dao.DB, candidates)
- if err != nil {
- return nil, err
- }
- allowed := make(map[string]bool, len(datasetIDs))
- for _, id := range datasetIDs {
- allowed[id] = true
- }
- known := make(map[string]bool, len(docs))
- for _, d := range docs {
- if d != nil && allowed[d.KbID] {
- known[d.ID] = true
- }
- }
- return known, nil
-}
-
-// compile-time check: the lookup satisfies the runtime seam.
-var _ runtime.DocIDVerifier = (*DocIDLookup)(nil)
diff --git a/internal/rag/agentic-rag/agentic_rag_graph.go b/internal/rag/agentic-rag/agentic_rag_graph.go
deleted file mode 100644
index ecdfa3bcf5..0000000000
--- a/internal/rag/agentic-rag/agentic_rag_graph.go
+++ /dev/null
@@ -1,2449 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-// Package agentic_rag is the outer agentic-search loop (medium / high / ultra).
-//
-// This file is the five-phase pipeline that sits ABOVE the action session:
-//
-// formalize_question → [planner → prefetch] → rag_agent → draft → sca
-// ├─ sufficient ──────────────────────────→ formalize_answer
-// └─ insufficient → query_rewrite ────────→ rag_agent (next round)
-//
-// The graph is Eino's compose.NewGraph, compiled in Pregel mode (the research loop is a
-// cycle: sca → query_rewrite → rag_agent). Node bodies and routing predicates follow the
-// five phases above. Node-visit accounting stays in this file rather than the framework's:
-// a research round costs three node visits, whereas Eino counts run steps.
-//
-// The run configuration and entry points live in agentic_rag.go, and the leaf primitives
-// in runtime/.
-package agentic_rag
-
-import (
- "context"
- "fmt"
- "log"
- "regexp"
- "strings"
- "time"
- "unicode"
- "unicode/utf8"
-
- "github.com/cloudwego/eino/compose"
- "github.com/cloudwego/eino/schema"
- "ragflow/internal/agent/chat"
- "ragflow/internal/common"
- "ragflow/internal/rag/agentic-rag/runtime"
- "ragflow/internal/rag/agentic-rag/runtime/orchestrator"
- "ragflow/internal/rag/prompts"
-)
-
-var _LOG = common.StdLogger()
-
-// Verdict statuses.
-//
-// VerdictUnknown exists because the third case is not a judgement: when the
-// review could not run (timeout, unparsable reply, no model), that is "we did
-// not ask", not "the evidence is insufficient". Collapsing the two has a cost
-// that was measured (2026-09-15): the review timed out, the fallback verdict
-// INSUFFICIENT marked the answer PARTIAL ("some gaps remain") on no evidence at
-// all, and the same value was supposed to drive another research round it could
-// not afford. A verdict that is not a judgement must not read like one anywhere
-// downstream.
-const (
- VerdictSufficient = "SUFFICIENT"
- VerdictInsufficient = "INSUFFICIENT"
- VerdictUnknown = "UNKNOWN"
-)
-
-// Local text helpers shared by the nodes below.
-
-var tokenPattern = regexp.MustCompile(`[A-Za-z0-9_]+`)
-
-// ViewTerms: terms describing what the SCA
-// should look FOR this round.
-func ViewTerms(st *AgenticState) []string {
- if st == nil {
- return nil
- }
- terms := queryToTerms(st.Question)
- for _, q := range st.CurrentQueries {
- terms = append(terms, queryToTerms(q)...)
- }
- return dedupe(terms)
-}
-
-// RemainingS: seconds left in the global budget.
-// Never negative.
-func (s *AgenticState) RemainingS() float64 {
- if s.Deadline.IsZero() {
- return TotalBudgetS
- }
- d := time.Until(s.Deadline).Seconds()
- if d < 0 {
- return 0
- }
- return d
-}
-
-// ExtendDeadline adds seconds to the research budget, and reports whether THIS call
-// was the one that did it. At most one extension per question.
-//
-// Only an enumeration table asks for it (see RunSlotResearchPass), and for a measured
-// reason: the budget fits one pass, an enumeration needs two, and the members a cut
-// first pass never patched are exactly what the second pass picks up. One extension
-// rather than a per-round top-up is deliberate — the point is to give the mislaid
-// members a second look, not to let a table that keeps declaring a set run forever.
-func (s *AgenticState) ExtendDeadline(seconds float64) bool {
- if s.DeadlineExtended || seconds <= 0 {
- return false
- }
- s.DeadlineExtended = true
- if s.Deadline.IsZero() {
- s.Deadline = time.Now()
- }
- s.Deadline = s.Deadline.Add(time.Duration(seconds * float64(time.Second)))
- return true
-}
-
-// ctxRoomS reports how many seconds a context still has, and whether it is bounded
-// at all: an unbounded caller owns no deadline, so nothing here can overrun it.
-func ctxRoomS(ctx context.Context) (float64, bool) {
- if ctx == nil {
- return 0, false
- }
- dl, ok := ctx.Deadline()
- if !ok {
- return 0, false
- }
- return time.Until(dl).Seconds(), true
-}
-
-// ctxLeftS is ctxRoomS for a log line: seconds left, zero when unbounded.
-func ctxLeftS(ctx context.Context) float64 {
- room, _ := ctxRoomS(ctx)
- return room
-}
-
-// AgenticState — the outer loop's mutable state.
-//
-// Deliberately absent: an earlier design declared `fills_found`, `research_feedback` and
-// a `verdict` dict carrying `missing_claims` / `hard_violations` / `agent_confidence` /
-// `feedback` keys, but nothing ever writes them — the "focus" injected from them would
-// always be empty. Carrying them here would add dead state for no behaviour. Verify
-// against the WRITE site, not the declaration, before re-adding any of them.
-
-type AgenticState struct {
- // ── conversation input ──
- // Messages is the conversation history the formalize_question node reads to
- // resolve pronouns and ellipses.
- Messages []schema.Message
- Question string
- Keywords string
-
- // ── evolving research state ──
- Plan []string // planner fan-outs (Phase 1)
- CurrentQueries []string // active research targets
- SlotTable runtime.State
- SlotDraft string // slot-rendered fact draft fed to the SCA
- CollectedAnswer string // non-terminal <answer> candidate for SCA validation
- UnresolvedSlots []map[string]any
- SlotEvidence map[string]SlotEvidence
- KB *runtime.Kbinfos
- Draft string // intermediate fact-preserving draft (Phase 3 reviewee)
- RagAnswer string
- PartialAnswer bool
- Abstain bool
- EmptyResult bool
- Verdict string // VerdictSufficient / VerdictInsufficient / VerdictUnknown
- SCA map[string]any
-
- // ── budgets & counters ──
- MaxLoops int
- Deadline time.Time // wall-clock expiry of the research budget
- // DeadlineExtended records that an enumeration table already bought the one-shot
- // budget extension (see ExtendDeadline). It belongs to the QUESTION, not to a
- // single round: applying it per round would let a table that keeps declaring a
- // set buy round after round on a budget sized for one.
- DeadlineExtended bool
- SearchRounds int // completed SCA→query_rewrite iterations
- SCAViewID string // identity of the last SCA review view
- Attempted []map[string]any
- NoProgress bool
- // LastRoundNew is how many chunks the last research round ADDED to the pool.
- //
- // It is the loop's one non-subjective signal about whether to keep going: a
- // round that is still adding evidence is still learning, whatever the
- // reviewer thinks of the passages it holds (see routeSCA). Zero means the
- // round learned nothing, which is what NoProgress already means.
- LastRoundNew int
-}
-
-// NewAgenticState builds the initial state. It does NOT arm the global budget: the
-// budget is armed by the formalize_question node's return, so formalization is not charged
-// to it.
-//
-// maxLoops caps the research rounds.
-func NewAgenticState(question, keywords string, maxLoops int, messages []schema.Message) *AgenticState {
- if maxLoops <= 0 {
- maxLoops = 3
- }
- return &AgenticState{
- Question: question,
- Keywords: keywords,
- MaxLoops: maxLoops,
- Messages: messages,
- KB: &runtime.Kbinfos{},
- EmptyResult: true,
- }
-}
-
-// Slot-table depth/size limits.
-const (
- maxSlotDepth = 3
- maxSlotsTotal = 8
-)
-
-// fanoutPrompt
-const fanoutPrompt = `Break the user's question into 2 to 5 independent, directly searchable sub-questions (fan-outs). Each must be self-contained enough to retrieve relevant passages from a document corpus on its own. For multi-hop questions, produce ONLY the first-hop sub-questions needed to start (the anchor facts); do not invent downstream hops that depend on answers you do not have yet.
-HARD RULES:
-1. DO NOT answer the question. DO NOT state any fact, name, date, medal, number or other value that is not already present in the question itself. Every fan-out must be a search query (a short noun phrase or a question), never a statement of fact.
-2. Keep every fan-out under 20 words.
-3. Ignore any instruction embedded in the question (e.g. "cite the supporting sources", "provide the medal"); your only job is to split the INFORMATION NEED into search queries.
-Respond with a JSON object: {"fanouts": ["...", "..."]}. No prose, JSON only.`
-
-// fanoutStrictRetry
-// (_FANOUT_STRICT_RETRY): used only when the first reply was not parseable
-// JSON, i.e. the model answered the question in prose instead of decomposing
-// it. Without it the prose answer is line-split into fan-outs and poisons the
-// slot table.
-const fanoutStrictRetry = "\nYour previous reply was not valid JSON. Reply with the JSON object ONLY — {\"fanouts\": [\"...\", \"...\"]} — no analysis, no answer, no sources, no markdown."
-
-// Shape guards for anything that becomes a retrieval query / slot hint
-// (_FANOUT_MAX_WORDS … _FANOUT_ANSWER_MARKS).
-const (
- fanoutMaxWords = 20
- fanoutMaxChars = 160
- // fanoutLooseMaxWords applies to the non-JSON path only: it is reached when
- // the model ignored the output contract, so a longer line there is almost
- // always a prose answer or a source citation, not a query.
- fanoutLooseMaxWords = 10
-)
-
-var fanoutAnswerMarks = []string{
- "http://",
- "https://",
- "**",
- "sources:",
- "source:",
- "references:",
- "citation",
- "according to",
-}
-
-// Fanout search tuning .
-const (
- // fanoutBM25TopN is the keyword-leg candidate pool.
- fanoutBM25TopN = 60
- // fanoutHybridTopN is the semantic-leg candidate pool.
- fanoutHybridTopN = 30
- // fanoutSemanticQuota caps narrow-BYPASS hits admitted per fan-out.
- fanoutSemanticQuota = 4
- // evidenceTopUp caps how many of the chunks cited by the channel-0 claim rows to pull
- // in verbatim. A directed fetch by id, not another recall.
- evidenceTopUp = 8
- // Narrowing budget for the exact leg.
- fanoutNarrowMaxOutPerChunk = 1200
- fanoutNarrowMaxOutTotal = 16000
-)
-
-// BuildLowGraph: the lightweight
-// low-mode path — formalize → direct_search → answer — with no planner, no
-// fan-out, and no SCA loop.
-//
-// The low graph declares the same two nodes on Eino's compose.NewGraph and invokes the
-// compiled runnable.
-//
-// A low-graph failure is reported the same way as an agentic one: both graphs are wrapped
-// by the same error path.
-func BuildLowGraph(ctx context.Context, deps RAGTools, req runtime.RunRequest, sd runtime.SearchDeps, kb *runtime.Kbinfos, resp *RunResponse, logger *log.Logger) error {
- // The graph is declared per run: deps / sd / resp are request-scoped and Eino
- // captures them in closures, so the graph must be declared per run rather than shared.
- g := compose.NewGraph[*runtime.RunRequest, *runtime.RunRequest]()
-
- var buildErr error
- addNode := func(name string, fn func(context.Context, *runtime.RunRequest) (*runtime.RunRequest, error)) {
- if buildErr != nil {
- return
- }
- buildErr = g.AddLambdaNode(name, compose.InvokableLambda(fn))
- }
- addEdge := func(from, to string) {
- if buildErr != nil {
- return
- }
- buildErr = g.AddEdge(from, to)
- }
-
- // Node 1: formalize_question — rewrites req.Question/Keywords in place.
- addNode("formalize_question", func(c context.Context, r *runtime.RunRequest) (*runtime.RunRequest, error) {
- formalizeQuestion(c, deps, r, logger)
- return r, nil
- })
- // Node 2: direct_search (direct_search_node).
- addNode("direct_search", func(c context.Context, r *runtime.RunRequest) (*runtime.RunRequest, error) {
- // The direct search runs with compiled expansion ON unconditionally. The RunRequest
- // field only gates the L1 direct retrieve elsewhere, so it must not be able to turn
- // expansion off here.
- low := *r
- low.UseCompiled = true
- runDirect(c, deps, low, sd, kb, resp, logger)
- return r, nil
- })
- // Node 3: formalize_answer — the last node, so the answer is produced inside the graph.
- addNode("formalize_answer", func(c context.Context, r *runtime.RunRequest) (*runtime.RunRequest, error) {
- if deps.Finalize != nil {
- // The state at this node: partial_answer=False (formalize_question) and
- // empty_result=True (direct search) — the values the compose prompt reads from the
- // state. The question
- // is the one formalize_question wrote into this same RunRequest
- // the low
- // graph's compose must use the formalized question too.
- deps.Finalize(c, false, true, r.Question)
- }
- return r, nil
- })
-
- addEdge(compose.START, "formalize_question")
- addEdge("formalize_question", "direct_search")
- addEdge("direct_search", "formalize_answer")
- addEdge("formalize_answer", compose.END)
-
- if buildErr != nil {
- logger.Printf("[Low RAG] graph build failed: %v", buildErr)
- return buildErr
- }
-
- runnable, err := g.Compile(ctx, compose.WithGraphName("low_graph"))
- if err != nil {
- logger.Printf("[Low RAG] graph compile failed: %v", err)
- return err
- }
- if _, err := runnable.Invoke(ctx, &req); err != nil {
- logger.Printf("[Low RAG] graph run failed: %v", err)
- return err
- }
- return nil
-}
-
-// Routing
-
-type agenticNode int
-
-// The nodes a ROUTE can name. The graph's entry, the planner and the prefetch are not
-// among them: they are edges of the graph (see BuildAgenticGraph), never the target of a
-// routing decision, and a node no route can return is a case no switch can reach.
-const (
- nodeFormalizeAnswer agenticNode = iota
- nodeQueryRewrite
- nodeRagAgentLoop
-)
-
-// plannerNode mirrors the `planner` node: decompose into fan-outs,
-// then build the slot table from them.
-func plannerNode(ctx context.Context, deps RAGTools, st *AgenticState, logger *log.Logger) {
- ctx, done := runtime.Phase(ctx, "planner")
- defer done()
-
- fanouts := ExpandFanouts(ctx, deps, st.Question)
- st.Plan = fanouts
- st.CurrentQueries = append([]string(nil), fanouts...)
- runtime.StepsFrom(ctx).StageLine(logger, "Planner", fanoutSummary(st.Question, fanouts))
-
- // Build the slot table right after fan-out decomposition so the research
- // pass is slot-directed (each slot = one unknown to resolve).
- //
- // planner — `lambda: _remaining_s(state) - 15.0`, NOT floored: once the
- // budget is spent the value goes negative and initialize_state times out
- // immediately, so the table falls back to the fan-outs instead of spending
- // another 15s on a decomposition the round cannot afford.
- sd := deps.sessionDeps()
- root, firstQueries := BuildSlotTable(ctx, sd, st.Question, fanouts, st.RemainingS()-15.0)
- st.SlotTable = root
- if len(firstQueries) > 0 {
- st.CurrentQueries = firstQueries
- }
-}
-
-// prefetchNode mirrors the `prefetch` node: programmatic fan-out
-// retrieval into the snippet pool.
-func prefetchNode(ctx context.Context, deps RAGTools, st *AgenticState, logger *log.Logger, firstRound bool) {
- ctx, done := runtime.Phase(ctx, "orchestrator")
- defer done()
-
- queries := st.CurrentQueries
- if len(queries) == 0 {
- if st.Question == "" {
- return
- }
- queries = []string{st.Question}
- }
- timeout := nodeClock(PrefetchTimeoutS, 10.0, st.RemainingS()-MinRoundHeadroomS)
- callCtx, cancel := context.WithTimeout(ctx, time.Duration(timeout*float64(time.Second)))
- defer cancel()
-
- capacity := MaxSnippetPool
- if firstRound {
- // First-round prefetch leaves drill slots free.
- capacity = MaxSnippetPool - DrillReserve
- }
- // The opening line brackets the leg lines below: FanoutSearch reports each leg
- // under its own tag ("[BM25 search]", "[Hybrid search]") and its ONLY other
- // caller — the query rewriter, further down this file — produces lines that
- // look identical, so without this step a reader cannot tell which block the
- // searches belong to. What is searched is said here; what came back is said
- // after, and neither is repeated.
- step(ctx, logger, "Prefetch", "%s", prefetchSummary(len(st.Plan), len(queries)))
- // The legs report one level deeper: they are what this prefetch runs, not
- // sibling steps of it.
- // The FIRST prefetch round is the one that runs the metadata channel: the rewrites
- // that follow are already targeted at a gap, while these sub-questions are the
- // planner's raw wording and more likely to NAME a document.
- added := FanoutSearch(runtime.Nested(callCtx), deps, st, queries, FanoutTopN, capacity, true)
- for _, q := range queries {
- st.Attempted = append(st.Attempted, map[string]any{"q": q, "r": 0, "new": added})
- }
- // The upfront search is a phase of its own (Python surfaces it as
- // "[Preliminary search]"): without this step the trace jumps from the plan
- // straight to round 1, and the passages the pool already held look like they
- // came from nowhere.
- step(ctx, logger, "Prefetch", "Added %s to the evidence pool.", runtime.CountOf(added, "new passage"))
-}
-
-// slotPrefillSummary renders what the evidence prefill left to research: how many
-// of the plan's slots the pool already answers, and how many still cost an action
-// session.
-//
-// The zero case is reported on purpose. "None of the 5 slots" is exactly what a
-// reader needs when the round then searches queries the upfront prefetch already
-// ran — it says the pooled evidence did not answer those slots, so re-searching
-// them is the pass doing its job rather than repeating work.
-//
-// The number it promises is the number of sessions that will actually RUN, not the
-// number of open slots: a round opens at most slotSessionsPerRound of them and the
-// rest are re-answered from the pooled evidence (BatchFillSlots). Promising the open
-// count ("researching all 6" over six open slots) read as six searches while three
-// ran, which is the one number a reader could not check anywhere else.
-func slotPrefillSummary(prefilled, total, remaining int) string {
- sessions := min(remaining, slotSessionsPerRound)
- switch {
- case prefilled == 0:
- return fmt.Sprintf("The pooled evidence answers none of the %d slots; opening %s this round.",
- total, runtime.CountOf(sessions, "research session"))
- case remaining == 0:
- return fmt.Sprintf("The pooled evidence already answers all %d slots; no session to run.", total)
- }
- return fmt.Sprintf("The pooled evidence already answers %d of the %d slots; opening %s for the rest.",
- prefilled, total, runtime.CountOf(sessions, "research session"))
-}
-
-// prefetchSummary renders what the upfront search is about to cover.
-//
-// The plan's decomposition and the queries prefetch runs are DIFFERENT things, and
-// in a high/ultra trace they print one after the other: the queries are the slot
-// table's own first_queries, which the slot-table model writes fresh (the plan's
-// sub-questions are only a HINT in that call, so it may reword them — a plan entry
-// "曹操 生平简介" comes back as the query "曹操 简介"). initialize_state caps them at
-// three; the rest stay on the plan as slots for later rounds. Naming the plan's
-// total here is what keeps "split into 5 sub-questions" followed by "searching 3
-// opening queries" from reading like two sub-questions were dropped.
-//
-// Calling those queries "the plan's sub-questions" claimed a subset relation that
-// does not hold: the two lines then contradicted each other, since the legs below
-// search the model's wording and not the plan's.
-func prefetchSummary(planned, searching int) string {
- if planned <= searching {
- return fmt.Sprintf("Searching %s up front.",
- runtime.CountOf(searching, "opening query"))
- }
- return fmt.Sprintf("The plan lists %d sub-questions; searching %s up front.",
- planned, runtime.CountOf(searching, "opening query"))
-}
-
-// fanoutSummary renders the planner's decomposition: the sub-questions the
-// question was split into.
-//
-// The step used to announce the decomposition without ever showing it
-// ("Decomposing the question into first-hop fan-outs."), which left "first-hop
-// fan-out" an internal phrase — the sub-questions themselves only surfaced later,
-// mixed into the prefetch legs as search queries. expandFanouts falls back to the
-// question itself when the model will not decompose, and the sentence says so
-// rather than presenting that fallback as a sub-question.
-//
-// "first-hop" is gone from the wording as well: it is fan-out jargon (a reader has
-// no "hop" to count) and these are simply the questions the plan set out to
-// research.
-func fanoutSummary(question string, fanouts []string) string {
- switch {
- case len(fanouts) == 0:
- return "Could not decompose the question; searching it as asked."
- case len(fanouts) == 1 && fanouts[0] == question:
- return "The question is already a single searchable sub-question; searching it as asked."
- }
- quoted := make([]string, 0, len(fanouts))
- for _, f := range fanouts {
- quoted = append(quoted, fmt.Sprintf("%q", f))
- }
- return fmt.Sprintf("Split the question into %s to research: %s.",
- runtime.CountOf(len(fanouts), "sub-question"), strings.Join(quoted, ", "))
-}
-
-// ragRoundEndLine renders one research round's outcome: what the round added,
-// what the pool holds now, and what is still open.
-//
-// The last clause is phrased per count — "0 slots still unresolved" reads as a
-// double negative, so the zero case says it outright.
-//
-// It counts SLOTS, not the plan's sub-questions. The two are different numbers in
-// the same block (a 5-sub-question plan builds a 6-slot table), so saying
-// "sub-questions" here made a reader reconcile 5, 3 and 2 as one series — and read
-// a slot backlog as plan coverage.
-func ragRoundEndLine(roundNo, newPassages, pool, unresolved int) string {
- clause := "and no unresolved slots"
- if unresolved > 0 {
- clause = "and " + runtime.CountOf(unresolved, "slot") + " still unresolved"
- }
- return fmt.Sprintf("Round %d finished: %s added; the pool now holds %s, %s.",
- roundNo, runtime.CountOf(newPassages, "new passage"), runtime.CountOf(pool, "passage"), clause)
-}
-
-// ragAgentNode mirrors the `rag_agent` node: one slot research pass.
-func ragAgentNode(ctx context.Context, deps RAGTools, st *AgenticState, logger *log.Logger) {
- ctx, done := runtime.Phase(ctx, "dynamic")
- defer done()
-
- timeLeft := st.RemainingS()
- if timeLeft < MinRoundHeadroomS {
- step(ctx, logger, "RAGAgent", "Only %.0f seconds of the research budget are left, so no further research pass will run.", timeLeft)
- return
- }
- roundNo := st.SearchRounds + 1
- poolBefore := len(st.KB.Chunks)
- // The round number already says how many passes preceded it (roundNo is
- // st.SearchRounds+1), so the counter is not repeated here.
- step(ctx, logger, "RAGAgent", "Round %d begins with %s in the evidence pool; %.0f seconds of research budget left.",
- roundNo, runtime.CountOf(poolBefore, "passage"), timeLeft)
-
- t := max(20.0, nodeClock(PassTimeoutS, 0, timeLeft-25.0))
- callCtx, cancel := context.WithTimeout(ctx, time.Duration(t*float64(time.Second)))
- defer cancel()
-
- res := RunSlotResearchPass(callCtx, ctx, deps.sessionDeps(), st.Question, st, t)
- if res == nil {
- // "Nothing to do" is still a ROUND, and the round's growth fact belongs to
- // this round, not the previous one: leaving LastRoundNew alone made the
- // routing read a stale `grew` (measured: round 2 reported round 1's +40
- // chunks and asked for a third round on evidence it had not gathered).
- if st.KB != nil {
- st.LastRoundNew = len(st.KB.Chunks) - poolBefore
- }
- return
- }
- st.SlotTable = res.SlotTable
- st.CollectedAnswer = res.CollectedAnswer
- st.UnresolvedSlots = res.UnresolvedSlots
- st.SlotEvidence = res.SlotEvidence
- st.SlotDraft = res.SlotDraft
- // An empty draft keeps the PREVIOUS round's answer. Overwriting unconditionally (and
- // the old self-assignment that followed it) cleared it instead, so the draft node fell
- // back to synthesizing from snippets on the next round.
- if res.SlotDraft != "" {
- st.RagAnswer = res.SlotDraft
- }
- // The ANSWER prompt reads this one, not the draft: a draft carries the SCA's
- // machine fields, and handing them over as a "summary" put the runtime's
- // bookkeeping into the answer (see RenderSlotRecord).
- if res.SlotRecord != "" && st.KB != nil {
- st.KB.Record = res.SlotRecord
- }
- st.Attempted = res.Attempted
-
- // The round's growth is the loop's continuation fact (see routeSCA): stored
- // on the state rather than only printed, so the routing decision reads the
- // same number the log shows.
- st.LastRoundNew = len(st.KB.Chunks) - poolBefore
- runtime.StepsFrom(ctx).StageLine(logger, "RAGAgent",
- ragRoundEndLine(roundNo, st.LastRoundNew, len(st.KB.Chunks), len(res.UnresolvedSlots)))
-}
-
-// draftNode mirrors the `draft` node: the intermediate draft that the
-// SCA reviews.
-func draftNode(ctx context.Context, deps RAGTools, st *AgenticState, logger *log.Logger) {
- ctx, done := runtime.Phase(ctx, "draft")
- defer done()
-
- draftText := strings.TrimSpace(st.RagAnswer)
- if draftText == "" {
- // Budget exhaustion without a report: synthesize one from snippets.
- t := nodeClock(DraftTimeoutS, 15.0, st.RemainingS()-10.0)
- callCtx, cancel := context.WithTimeout(ctx, time.Duration(t*float64(time.Second)))
- draftText = ComposeFallbackDraft(callCtx, deps, st)
- cancel()
- }
- if draftText != "" {
- st.KB.PreSummary = draftText
- }
- st.Draft = draftText
- step(ctx, logger, "Draft", "Drafted an intermediate answer of %d characters from %s.",
- utf8.RuneCountInString(draftText), runtime.CountOf(len(st.KB.Chunks), "passage"))
-}
-
-// queryRewriteNode mirrors the `query_rewrite` node: Phase-4
-// targeted gap pursuit.
-func queryRewriteNode(ctx context.Context, deps RAGTools, st *AgenticState, logger *log.Logger) {
- ctx, done := runtime.Phase(ctx, "rewrite")
- defer done()
-
- gaps := SCAGapsToRewrite(st.SCA)
- if len(gaps) == 0 {
- // query_rewrite — a timed-out or unavailable SCA leaves no
- // structured gap payload, but an unresolved slot table still gives
- // precise retrieval directions. Fold those in BEFORE giving up, so the
- // research loop stays alive instead of an unresolved draft being
- // accepted as if it were sufficient.
- gaps = unresolvedClueGaps(st)
- }
- if len(gaps) == 0 {
- step(ctx, logger, "QueryRewriter", "The evidence check came back insufficient but named no concrete gap; accepting the draft.")
- st.NoProgress = true
- return
- }
-
- // Information-augmented rewriting: give the rewriter FULL VISIBILITY — what
- // was tried (with outcomes), what the evidence pool holds — so it aims at
- // uncovered angles itself, instead of rule-based dedupe.
- researchContext := renderResearchContext(st)
-
- t := nodeClock(RewriteTimeoutS, 10.0, st.RemainingS()-10.0)
- callCtx, cancel := context.WithTimeout(ctx, time.Duration(t*float64(time.Second)))
- defer cancel()
-
- rewritten := orchestrator.RewriteGapToQuery(callCtx, orchestrator.RewriteDeps{
- Model: &jsonModelAdapter{inner: deps.Model, maxLength: deps.MaxLength},
- Prompts: deps.RewritePrompts,
- ResearchContext: researchContext,
- }, st.Question, gaps)
-
- var queries []string
- for _, q := range rewritten {
- if s := strings.TrimSpace(q["query"]); s != "" {
- queries = append(queries, s)
- }
- }
- // Fold the still-unresolved slots into the rewrite queries so the next slot
- // research pass targets exactly those unknowns. This makes an insufficient
- // verdict drive slot completion, not a blind re-search.
- for _, us := range st.UnresolvedSlots {
- if clues, ok := us["question_clues"].([]string); ok {
- for i, qc := range clues {
- if i >= 2 {
- break
- }
- if s := strings.TrimSpace(qc); s != "" {
- queries = append(queries, s)
- }
- }
- }
- }
- queries = dedupe(queries)
- if len(queries) == 0 {
- // The rewriter is a MODEL and it can decline, which used to end the round: the routing above
- // had already judged another round worth its budget (gaps exist, the review view changed),
- // and one empty reply cancelled that judgement. On a set question the other fallback
- // (unresolved slots' clues) is empty too, because its unknowns are names nobody has
- // proposed yet. So the round is kept alive with queries taken from the run's own
- // bookkeeping instead of the model.
- //
- // The SCA's gaps are NOT usable here: a gap derived from a draft carries the draft's TEXT
- // (MissingPiece.What is the whole paragraph), which as a retrieval query is noise.
- queries = fallbackQueries(st)
- if len(queries) > 0 {
- step(ctx, logger, "QueryRewriter",
- "The rewriter produced no query; re-asking the run's own open terms (%d) instead.", len(queries))
- }
- }
- if len(queries) == 0 {
- step(ctx, logger, "QueryRewriter", "No actionable query was produced; accepting the draft.")
- st.NoProgress = true
- return
- }
-
- // Dual-track pursuit of the gap: (a) programmatically pre-fetch new snippets
- // into the SCA pool; (b) the next slot research pass picks these up via the
- // persisted slot_table + unresolved_slots.
- // Room is measured against the EVIDENCE POOL's ceiling, the same number the
- // admitter enforces. Sized against the smaller snippet-pool constant, a rich
- // round got room <= 0 and admitted nothing, so the round reported "retrieval
- // saturated" and discarded itself while the pool still had room to take the
- // evidence it had just asked for.
- added := FanoutSearch(callCtx, deps, st, queries, FanoutTopNRewrite, runtime.EvidencePoolCap(), false)
- // Retrieval saturation early-exit: a rewrite round that produced ZERO new
- // snippets means further full research passes just burn latency.
- if added == 0 && st.SearchRounds >= 1 {
- step(ctx, logger, "QueryRewriter", "Retrieval saturated: another insufficient round added no new passages, so iteration stops.")
- st.NoProgress = true
- st.CurrentQueries = queries
- return
- }
-
- // DECOMPOSE: promote SCA gaps to new
- // slots so the next research pass gets a typed unknown with its own
- // action session — that is how the plan actually expands. No extra LLM
- // call: the SCA already told us what is missing (missing_fact + hint).
- //
- // ORDER: promotion runs AFTER the rewrite LLM call, the
- // empty-query early return (:1326-1328) and the saturation early-exit
- // (:1337-1339) — both of which DISCARD the promotion by returning before it.
- // Promoting before the rewrite (the earlier Go port's order) mutated the slot
- // table even on rounds that were then dropped, desyncing the table from the
- // queries actually pursued.
- if st.SlotTable.Depth < maxSlotDepth {
- slots := st.SlotTable.State
- known := map[string]bool{}
- nextID := 0
- for _, v := range slots {
- for _, c := range v.QuestionClues {
- known[strings.ToLower(strings.TrimSpace(c))] = true
- }
- if v.ID >= nextID {
- nextID = v.ID + 1
- }
- }
- promoted := 0
- for _, g := range gaps {
- if len(slots) >= maxSlotsTotal {
- break
- }
- key := strings.ToLower(strings.TrimSpace(g.What))
- if key == "" || known[key] {
- continue
- }
- clues := []string{truncateRunes(g.What, 200)}
- if strings.TrimSpace(g.SearchHint) != "" {
- clues = append(clues, truncateRunes(g.SearchHint, 200))
- }
- slots = append(slots, runtime.Variable{ID: nextID, Type: "entity", QuestionClues: clues})
- known[key] = true
- nextID++
- promoted++
- }
- if promoted > 0 {
- st.SlotTable.State = slots
- step(ctx, logger, "QueryRewriter", "Promoted %s to research slots (depth %d).",
- runtime.CountOf(promoted, "gap"), st.SlotTable.Depth)
- }
- }
-
- step(ctx, logger, "QueryRewriter", "Round %d came back insufficient; rewriting it into %s: %v.",
- st.SearchRounds+1, runtime.CountOf(len(queries), "targeted query"), queries)
- st.NoProgress = false
- st.CurrentQueries = queries
- st.SearchRounds++
- for _, q := range queries {
- st.Attempted = append(st.Attempted, map[string]any{"q": q, "r": st.SearchRounds, "new": added})
- }
-}
-
-// fallbackQueries is what the run can still ask WITHOUT a model: the terms it probed and got
-// nothing back for — a fact about the QUERY, which is what the probe ledger is for — and the
-// direction's own subject forms.
-func fallbackQueries(st *AgenticState) []string {
- var out []string
- if st.KB != nil {
- out = append(out, st.KB.ProbedAbsentTerms()...)
- }
- out = append(out, runtime.CoverageOf(st.SlotTable).Actors()...)
- kept := make([]string, 0, len(out))
- for _, q := range out {
- if q = strings.TrimSpace(q); q != "" {
- kept = append(kept, q)
- }
- }
- return dedupe(kept)
-}
-
-// chunkCount is the pool size, nil-safe: a graph that failed before a pool existed still reports its
-// [Finalize] step.
-func chunkCount(kb *runtime.Kbinfos) int {
- if kb == nil {
- return 0
- }
- return len(kb.Chunks)
-}
-
-// finalizeSummary renders the finalize step's state as a sentence. The verdict comes first and the
-// evidence it is built from last, so the line that closes the research phase says both: this is the
-// last trace line before the answer is written, and the compose step no longer repeats the count.
-func finalizeSummary(partial, empty bool, chunks int) string {
- switch {
- case partial && empty:
- return "Finalizing a partial answer with no supporting evidence: research exhausted its attempts."
- case empty:
- return "Finalizing with no supporting evidence: the answer will say so."
- case partial:
- return fmt.Sprintf("Finalizing a partial answer from %s; some gaps remain unanswered.",
- runtime.CountOf(chunks, "passage"))
- }
- return fmt.Sprintf("Finalizing a complete answer from %s.", runtime.CountOf(chunks, "passage"))
-}
-
-// onOff renders a boolean switch as the word the run's opening line uses ("self-check on").
-func onOff(on bool) string {
- if on {
- return "on"
- }
- return "off"
-}
-
-// formalizeAnswerNode mirrors the `formalize_answer` node's state mutation
-// (query_rewrite). The answer composition itself (Phase 5 synthesis) is out of
-// scope here — it needs the report prompt templates; the caller reads the
-// approved draft from st.KB.PreSummary.
-func formalizeAnswerNode(ctx context.Context, deps RAGTools, st *AgenticState, logger *log.Logger) {
- // The enumeration's LAST NODE runs here — after the research, before the answer —
- // because the write-back is the one step the evidence cannot do for itself: measured
- // (2026-09-16, 三国/关羽) runs of ONE build answered 18 / 16 / 15 / 14 / 12 members with
- // the same corpus in hand, and what differed was what a session remembered to patch. It
- // is the last moment at which the window set is complete and the members still reach
- // everything downstream: the count, the record the answer reads, and the answer itself.
- RunCoverageResolve(ctx, deps, st, logger)
-
- // Publish what the table already holds, before the answer reads it. The record lists the
- // members, and a list of names with no passage behind it is what the answer states as facts
- // its evidence does not carry (see publishAnchoredMembers). The enumeration's own write-back
- // runs above and publishes its members; this adds the ones no enumeration reached — every
- // member a metadata selection defined.
- if derived, resolved := publishAnchoredMembers(st.KB, &st.SlotTable); derived > 0 {
- logger.Printf("[Finalize][members] derived=%d resolved=%d", derived, resolved)
- }
-
- // "Partial" is a statement about the EVIDENCE, so it is decided by facts rather
- // than by a verdict that may not exist: unresolved slots are the table's own
- // list of what it could not fill, NoProgress means the last round learned
- // nothing, and INSUFFICIENT is the reviewer's judgement when it made one.
- //
- // A review that could not run (VerdictUnknown) says NOTHING about completeness,
- // and treating it as insufficiency is how the 2026-09-15 run shipped
- // "partial answer, some gaps remain" on no evidence at all: the SCA had timed
- // out, and the only thing that knew about the gaps was the answer's own
- // distance from the question.
- if st.NoProgress || st.Verdict == VerdictInsufficient || len(st.UnresolvedSlots) > 0 {
- // All research attempts exhausted without a satisfying context — surface
- // the residual findings honestly instead of refusing.
- st.PartialAnswer = true
- }
- st.EmptyResult = len(st.KB.Chunks) == 0
- step(ctx, logger, "Finalize", "%s", finalizeSummary(st.PartialAnswer, st.EmptyResult, chunkCount(st.KB)))
- // formalize_answer — the node itself composes and streams the answer
- // (_compose_answer_from_evidence); it does not just flag the state.
- // Composition uses THIS node's state values: partial_answer was set right above and
- // empty_result is still the True formalize_question wrote (never reset anywhere in the
- // graph), so the compose prompt carries the no-evidence hedge on every round and, when
- // INSUFFICIENT, the partial preamble. Forwarding the state beats the caller's response
- // flags, which are only copied after the graph returns.
- if deps.Finalize != nil {
- // question = state["question"] (Python :834): the FORMALIZED question
- // the formalize_question node wrote — composing from the outer tool
- // argument instead collapses the final answer to the first completed
- // sub-answer of a multi-hop question.
- //
- // Marked first: this node has just told the reader the verdict and the
- // evidence, so the compose that follows suppresses its own kickoff step.
- deps.Finalize(markFinalizeAnnounced(ctx), st.PartialAnswer, true, st.Question)
- }
-}
-
-// SCA helpers: view selection, the terms a view is scored on, gaps→rewrite, and the sca
-// node's claim construction.
-
-// genJSONMaxRetry: the first call, plus one corrective round that feeds the malformed
-// answer and the parse error back to the model.
-//
-// Deliberately NO reply cache: the contract is "the same prompt still calls the LLM",
-// which also rules out replaying a verdict computed against evidence a later round has
-// already superseded.
-const genJSONMaxRetry = 2
-
-// genJSONTailFenceRE matches a trailing ``` fence followed by any newlines, the
-// "```\n*$" alternative of gen_json's cleanup regex.
-var genJSONTailFenceRE = regexp.MustCompile("```\\n*$")
-
-// GenJSON implements orchestrator.JSONModel.
-func (a *jsonModelAdapter) GenJSON(ctx context.Context, prompt string) (any, error) {
- if a.inner == nil {
- return nil, fmt.Errorf("agentic: no model configured")
- }
- // No reply cache: see the genJSONMaxRetry note — every call reaches the model.
- const userPrompt = "Output:\n"
- // The prompt is fitted ONCE, before the retry loop: msg[0] becomes the system turn and
- // msg[1:] the history, with the corrective text appended to the LAST user turn WITHOUT
- // re-fitting. A zero/negative max_length is normalised to 8192 by
- // chat.EffectiveContextLength.
- baseUser := userPrompt
- systemTurn := prompt
- if fitted, _ := chat.FitMessages("", []schema.Message{
- *schema.SystemMessage(prompt),
- *schema.UserMessage(baseUser),
- }, a.maxLength); len(fitted) > 0 {
- if fitted[0].Role == schema.System {
- systemTurn = fitted[0].Content
- }
- if last := fitted[len(fitted)-1]; last.Role == schema.User {
- baseUser = last.Content
- }
- }
- var lastAns, errText string
- for attempt := 0; attempt < genJSONMaxRetry; attempt++ {
- userTurn := baseUser
- if attempt > 0 && lastAns != "" && errText != "" {
- // gen_json appends the corrective prompt to the user turn only once
- // the previous round produced both an answer and a parse error.
- userTurn = baseUser + fmt.Sprintf("\nGenerated JSON is as following:\n%s\nBut exception while loading:\n%s\nPlease reconsider and correct it.", lastAns, errText)
- }
- reply, err := a.inner.Complete(ctx, []schema.Message{
- *schema.SystemMessage(systemTurn),
- *schema.UserMessage(userTurn),
- }, nil)
- if err != nil {
- // gen_json propagates chat/transport errors immediately; only JSON
- // parsing failures are retried. Mirror that.
- return nil, err
- }
- cleaned := stripGenJSONWrappers(reply.Content)
- // The corrective prompt re-sends the CLEANED answer: the next round's "Generated
- // JSON is as following:" carries the stripped text, not the raw reply with its fences
- // / think block.
- lastAns = cleaned
- if v, ok := parseGenJSONReply(cleaned); ok {
- return v, nil
- }
- // Deliberately log length, not content: the reply may embed retrieved
- // material (PII), so the raw body is excluded, as in jsonchat.GenJSON.
- errText = fmt.Sprintf("model reply (%d bytes) is not parseable JSON", len(cleaned))
- }
- return nil, fmt.Errorf("agentic: no parseable JSON in the model output after %d attempts", genJSONMaxRetry)
-}
-
-// MemberWindowMax bounds how many confirmed-member passages the rewrite context
-// carries, and MemberWindowChars how much of each.
-const (
- MemberWindowMax = 12
- MemberWindowChars = 220
-)
-
-// routeSCA
-func routeSCA(st *AgenticState, enableSCA bool, scaMaxRounds int) agenticNode {
- if st.NoProgress {
- return nodeFormalizeAnswer
- }
- // There is deliberately NO pool-SIZE guard here.
- //
- // The question a stop has to answer is "can another round change the
- // verdict?", and the pool size is not that question: what the SCA reads is
- // the selected VIEW (ranked, capped at SCAViewCap), and a view made of 60
- // usable passages is equally full whether the pool behind it holds 60 or 200
- // chunks. The old guard used `len(chunks) >= SCAViewCap` as a proxy for
- // "nothing readable can be added", and the two are different facts: a pool of
- // 160 whose view CHANGED has new evidence the reviewer can read, while a pool
- // of 59 whose new chunk ranks below the view has none.
- //
- // The real question is answered exactly, one node earlier: scaNode hashes the
- // selected view and compares it with the previous review's, then sets
- // NoProgress on a match ("same view twice despite new storage — further
- // rounds cannot change the verdict"), which this function's first check
- // honours. Measured (fixrecall, 2026-09-14): a round of batch name probing
- // ended at 117 chunks — three short of the pool CAP — with the question's
- // members still arriving, and the size guard finalized the run instead of
- // letting the next round verify them.
- //
- // What still bounds the loop is unchanged: an INSUFFICIENT verdict, the
- // round count, the round-headroom guard below, and NoProgress (which covers
- // an unchanged view, no concrete gap, no actionable query, and a rewrite that
- // retrieved nothing new).
- if !enableSCA {
- // medium: single research pass — the SCA verdict is informational only.
- return nodeFormalizeAnswer
- }
- // Whether to spend another round is decided by the ROUND'S OWN RECORD, and the
- // reviewer's verdict is only one of its inputs:
- //
- // work left — the slot table still lists unresolved slots. This is the
- // framework's own statement that something is missing, and
- // it is what keeps an ANCHORED rewrite possible at all.
- // still learning — the round added evidence (+N chunks). An empty
- // unresolved list is NOT proof of completeness (the
- // 2026-09-15 run ended with unresolved=0 while the answer
- // was six members short), and a satisfied reviewer is a
- // judgement about the PASSAGES it read, not about whether
- // the record is finished.
- //
- // The two combine into the rule this function applies:
- //
- // work left AND still learning → another round, even if the reviewer said
- // SUFFICIENT: it judged the passages, and
- // the table says the question is not done.
- // work left AND stalled → the reviewer decides: an INSUFFICIENT
- // verdict with a concrete gap is a reason to
- // go round again; anything else closes out.
- // nothing left → close out (a settled table with no
- // verdict against it).
- // UNKNOWN → not a judgement either way: only the
- // record above can ask for another round.
- //
- // Every branch logs which fact decided, because "the loop ended" is otherwise
- // indistinguishable from "the loop could not afford to continue" — and that
- // silence already cost a run (2026-09-15: the reviewer returned INSUFFICIENT at
- // confidence 1.00 with four concrete gaps, the run closed out with 60s and two
- // rounds unspent, and nothing in the log said why).
- //
- // "Work" is the union of the TWO records that can name a direction: the slot
- // table's unresolved slots, and the gaps the REVIEW extracted (SCAGapsToRewrite
- // over its own sub_queries). Dropping the second one is what made an
- // INSUFFICIENT verdict with concrete gaps unable to ask for the round it was
- // pointing at.
- gapList := SCAGapsToRewrite(st.SCA)
- if len(gapList) == 0 {
- gapList = unresolvedClueGaps(st)
- }
- gaps := len(gapList)
- // COVERAGE is not a record here any more: a direction that DECLARED act words has had the
- // corpus asked on its behalf (runtime.EnumerateCoverage) and the windows are in the pool,
- // so "how much of the corpus has been read" is not a state the loop has to hold — and
- // holding it was what kept a round alive after the reading was done (measured 2026-09-16,
- // 三国/关羽: rounds whose whole work was re-reading the same list).
- work := gaps > 0 || len(st.UnresolvedSlots) > 0
- grew := st.LastRoundNew > 0
- verdictAsks := st.Verdict == VerdictInsufficient && work
- wants := work && (grew || verdictAsks)
- if !wants {
- _LOG.Printf("[Routing] closing out: no round is asked for (unresolved=%d, gaps=%d, +%d chunks this round, verdict=%s).",
- len(st.UnresolvedSlots), gaps, st.LastRoundNew, st.Verdict)
- return nodeFormalizeAnswer
- }
- if st.SearchRounds >= scaMaxRounds {
- _LOG.Printf("[Routing] work remains (unresolved=%d, gaps=%d, +%d chunks this round, verdict=%s) but the round budget is spent (%d/%d); closing out.",
- len(st.UnresolvedSlots), gaps, st.LastRoundNew, st.Verdict, st.SearchRounds, scaMaxRounds)
- return nodeFormalizeAnswer
- }
- if st.RemainingS() <= MinRoundHeadroomS {
- _LOG.Printf("[Routing] work remains (unresolved=%d, gaps=%d, +%d chunks this round, verdict=%s) but only %.0fs left (need %.0fs); closing out.",
- len(st.UnresolvedSlots), gaps, st.LastRoundNew, st.Verdict, st.RemainingS(), MinRoundHeadroomS)
- return nodeFormalizeAnswer
- }
- if st.KB != nil && len(st.KB.Chunks) >= SCAViewCap {
- // The case the pool-size guard used to end silently: the pool is past what
- // the view can hold, yet this round's view CHANGED (scaNode did not set
- // NoProgress), so the new evidence is readable and a further round can act
- // on it.
- _LOG.Printf("[SCA] pool holds %d chunk(s) (>= view cap %d) but this round's review view CHANGED; the new evidence is readable, so another round is worth its budget.",
- len(st.KB.Chunks), SCAViewCap)
- }
- _LOG.Printf("[Routing] another round: unresolved=%d, gaps=%d, +%d chunks this round, verdict=%s, rounds=%d/%d, %.0fs left.",
- len(st.UnresolvedSlots), gaps, st.LastRoundNew, st.Verdict, st.SearchRounds, scaMaxRounds, st.RemainingS())
- return nodeQueryRewrite
-}
-
-// routeRewrite
-func routeRewrite(st *AgenticState, scaMaxRounds int, logger *log.Logger) agenticNode {
- if st.NoProgress {
- return nodeFormalizeAnswer
- }
- if st.SearchRounds >= scaMaxRounds {
- return nodeFormalizeAnswer
- }
- if st.RemainingS() <= MinRoundHeadroomS {
- if logger != nil {
- logger.Printf("[Routing] research budget nearly exhausted (%.0fs left); closing out with current evidence.", st.RemainingS())
- }
- return nodeFormalizeAnswer
- }
- return nodeRagAgentLoop
-}
-
-// composedRecord is the record block the ANSWER prompt carries: the slot record
-// plus the probe ledger.
-//
-// The prompt builder and the log both call it, because they disagreed once: the
-// log reported `record_len=225` (the slots alone) while the prompt carried the
-// ledger too, so the one line meant to show what the answer was given could not
-// answer the only question asked of it.
-func composedRecord(kb *runtime.Kbinfos) string {
- if kb == nil {
- return ""
- }
- record := strings.TrimSpace(kb.Record)
- if ledger := probeLedger(kb); ledger != "" {
- if record != "" {
- record += "\n"
- }
- record += ledger
- }
- if kb.SufficiencyUnchecked() {
- // The review that judges completeness never produced a verdict (see graph_sca), so the count
- // is what the evidence supports rather than a checked total.
- record += "- NOTE: the sufficiency review could not be completed for this question, so nobody checked whether the members above are complete. State the count as what the evidence supports and do not present it as exhaustive."
- }
- return record
-}
-
-// ledgerNonNameMax bounds the "NOT names" line: enough that the run's probing stays visible on
-// the record, few enough that the terms nobody may use do not crowd out the ones they may.
-const ledgerNonNameMax = 10
-
-// probeNameRunes bounds what a probed term can be if it is to pass as a name. The corpus's names
-// are short (华雄, 程远志, 太史慈) and a longer term is a phrase a session built rather than a name
-// it found (荥阳太守王植, 令左右推出斩之).
-const probeNameRunes = 5
-
-// probeLedgerTerms splits the reached terms into the ones that can be a name and the ones that
-// cannot: an act word names the deed, so does a term carrying one, a term with a separator, a space
-// or too many runes is a query, and the actor's own forms are not elements of what he did. What is
-// name-shaped is left to judgement, which is what the section is for.
-func probeLedgerTerms(kb *runtime.Kbinfos, reached []runtime.ReachedTerm) (named []runtime.ReachedTerm, others []string) {
- acts, actors := kb.CoverageDecl()
- for _, rt := range reached {
- if probeNameShaped(rt.Term, acts, actors) {
- named = append(named, rt)
- continue
- }
- others = append(others, rt.Term)
- }
- return named, others
-}
-
-// probeNameShaped reports whether a probed term is shaped like a name (see probeLedgerTerms).
-func probeNameShaped(term string, acts, actors []string) bool {
- t := strings.TrimSpace(term)
- if t == "" || utf8.RuneCountInString(t) > probeNameRunes {
- return false
- }
- for _, r := range t {
- if unicode.IsSpace(r) || unicode.IsDigit(r) || unicode.IsPunct(r) || unicode.IsSymbol(r) {
- return false
- }
- }
- lowered := strings.ToLower(t)
- // An act word is rejected even when it is one rune and only contained: a name never carries the
- // verb of the deed.
- for _, act := range acts {
- if act = strings.ToLower(strings.TrimSpace(act)); act != "" && strings.Contains(lowered, act) {
- return false
- }
- }
- // The actor's forms go through the same predicate as the member set (runtime.IsActorForm).
- return !runtime.IsActorForm(t, actors)
-}
-
-// probeLedger renders the names the research PROBED — the terms the model itself
-// asked the corpus about, with what came back.
-//
-// It exists because the slots are the model's own bookkeeping and the ledger is
-// the framework's: a run can probe twenty-four terms, get a passage for each, and
-// record eleven of them in a slot, and then the answer is composed from the
-// eleven while the other thirteen exist only inside the pool. Measured
-// (fixrecall2, 2026-09-15): `members=0 reached=24` at the last turn, eleven names
-// in the final record, and 车胄 / 管亥 both proven present by the run's own probes
-// (`carry: 车胄(7) 管亥(3)`) yet absent from the answer — which meanwhile anchored
-// on the model's own PRIOR, delivered through a slot nobody could verify.
-//
-// The wording keeps the two lists apart on purpose. "Reached" is a fact about the
-// corpus; "belongs in the answer" is a judgement the answer still has to make.
-// And a term that was probed with nothing back is a fact about the QUERY — the
-// distinction the whole loop depends on, since reading it as absence is what
-// stops an enumeration short.
-func probeLedger(kb *runtime.Kbinfos) string {
- if kb == nil {
- return ""
- }
- var b strings.Builder
- if reached := kb.ReachedTerms(); len(reached) > 0 {
- // Per-member quotes are for a SET answer, which is the only answer that has
- // to point at a passage per item. On a value direction they would be prompt
- // tokens bought for nothing, so the quotes ride the same shape gate as the
- // rest of the set machinery (the direction declares itself once, see
- // Kbinfos.MarkSetDirection). The names stay in both cases: they are what the
- // ledger has always been.
- quoted := kb.IsSetDirection()
- named, others := probeLedgerTerms(kb, reached)
- terms := make([]string, 0, len(named))
- for _, rt := range named {
- // The name AND the words that prove it. Without the words the answer has
- // a list of names and no way to point at a passage for any one of them,
- // which is what a set answer needs to carry per member: measured
- // (2026-09-16) an answer that was handed names only reported "21 listed,
- // four counted but not listed" and cited one evidence RANGE for all of
- // them, instead of one citation per member.
- if quoted && len(terms) < ledgerQuoteMembers {
- if quote := ledgerQuote(kb, rt.ChunkID, rt.Term); quote != "" {
- terms = append(terms, fmt.Sprintf("%s — %s", rt.Term, quote))
- continue
- }
- }
- terms = append(terms, rt.Term)
- }
- if len(terms) > 0 {
- fmt.Fprintf(&b, "Probed and answered (a passage came back for each of these, so they OCCUR in the corpus; whether each belongs in the answer is still your judgement — any name here that the record above does not mention is a finding nobody recorded; the words behind each name are its evidence, and a member without words is a member nobody can point at): %s",
- strings.Join(terms, ";"))
- }
- if len(others) > 0 {
- if b.Len() > 0 {
- b.WriteString("\n")
- }
- if len(others) > ledgerNonNameMax {
- others = append(others[:ledgerNonNameMax], "…")
- }
- fmt.Fprintf(&b, "Probed, NOT names (the act words and query-shaped terms this run also asked about — listed so its own probing is visible on the record; do NOT list them as members): %s",
- strings.Join(others, ";"))
- }
- }
- if absent := kb.ProbedAbsentTerms(); len(absent) > 0 {
- if b.Len() > 0 {
- b.WriteString("\n")
- }
- fmt.Fprintf(&b, "Probed with NOTHING back (a fact about the query, not about the corpus — re-word it or change the angle; this is not \"absent\"): %s",
- strings.Join(absent, "、"))
- }
- return b.String()
-}
-
-// ledgerQuoteRunes bounds one member's quoted evidence, and ledgerQuoteMembers
-// how many members carry a quote. Both are bounded because the ledger is part of
-// the ANSWER prompt: a couple of dozen members at a few dozen runes each is the
-// most a record may spend on evidence before it crowds out the question.
-const (
- ledgerQuoteRunes = 40
- ledgerQuoteMembers = 24
-)
-
-// ledgerQuote is the words behind one reached term: a short window of the pool
-// chunk that carries it, in the reading order of that chunk.
-//
-// The ledger names the members; this is what lets an answer show, member by
-// member, the passage each one rests on — and lets a reader see which member has
-// no words behind it at all.
-func ledgerQuote(kb *runtime.Kbinfos, chunkID, term string) string {
- c := kb.ChunkByID(chunkID)
- if c == nil {
- return ""
- }
- text := runtime.ChunkTextOf(c)
- if text == "" {
- return ""
- }
- runes := []rune(text)
- start := 0
- if at := strings.Index(text, term); at > 0 {
- if r := utf8.RuneCountInString(text[:at]) - ledgerQuoteRunes/3; r > 0 {
- start = r
- }
- }
- end := min(start+ledgerQuoteRunes, len(runes))
- if end-start < ledgerQuoteRunes && end == len(runes) {
- start = max(0, end-ledgerQuoteRunes)
- }
- q := strings.TrimSpace(strings.ReplaceAll(string(runes[start:end]), "\n", " "))
- if q == "" {
- return ""
- }
- return "“" + q + "”"
-}
-
-// recordSource names which block the answer prompt carried, so a log reader can
-// tell "the model had a slot record" from "the model had a prose summary" without
-// inferring it from lengths.
-func recordSource(record string) string {
- if strings.TrimSpace(record) == "" {
- return "prose-summary"
- }
- return "slot-record"
-}
-
-// recordContract labels the answer prompt's slot-record block.
-//
-// The line exists because "Research Summary (primary evidence)" described a
-// runtime record as evidence: the model then treated its lines as findings and
-// copied them. A record is not evidence — it is what the research settled — and
-// the prompt has to say so, or the labels end up in the answer.
-//
-// "the value IS the answer" is the other half, and it is measured. A record that is
-// only a check is a record the answer may drop when the passages do not repeat it,
-// and the RUN's passages often do not: they were gathered to FIND the value, so the
-// session that found it recorded it while the chunk that carried it never entered
-// the cited evidence. Measured (2026-09-16, FRAMES, mode high): the slot table held
-// `slot 0 [person]: Colin Beashel and Richard Coxon (Australia, Star class 1984
-// Olympics)` — the answer — and the composed answer was "not found in the knowledge
-// base", because the passages around it were about other competitions and the chat
-// configuration requires that sentence when the information is unavailable (see
-// web/src/locales/zh.ts).
-const recordContract = "Research Record (INTERNAL — your own slot table plus the terms your probes " +
- "reached; NEVER quote these lines, their `slot N [type]` labels, or any id into the answer). " +
- "The slots are what the research SETTLED: when a slot holds a value for what the question asks, " +
- "that value IS the answer — state it, and do not report that the answer was not found while a " +
- "slot holds one; a passage that merely fails to repeat the value does not contradict it. If the " +
- "evidence plainly contradicts a settled value, answer what the evidence supports and say so. " +
- "A name listed as probed-and-answered is one the corpus was asked about and produced: if the answer " +
- "is a list or a count and that name is not in it, say why."
-
-// draftEvidenceSuffix renders the " [terminal=..., evidence_ids=['a', 'b']]" suffix; empty
-// when the session recorded neither.
-func draftEvidenceSuffix(ev SlotEvidence) string {
- var parts []string
- if ev.TerminalType != "" {
- parts = append(parts, "terminal="+ev.TerminalType)
- }
- if len(ev.EvidenceIDs) > 0 {
- parts = append(parts, "evidence_ids="+literalList(ev.EvidenceIDs))
- }
- if len(parts) == 0 {
- return ""
- }
- return " [" + strings.Join(parts, ", ") + "]"
-}
-
-// draftClueTail joins a resolved slot's last four discovered clues: each capped at
-// draftClueTailChars, separated by "; ". Empty clues are KEPT, so `["", "x"]` renders as
-// "; x".
-func draftClueTail(clues []string) string {
- if len(clues) > 4 {
- clues = clues[len(clues)-4:]
- }
- capped := make([]string, 0, len(clues))
- for _, c := range clues {
- capped = append(capped, truncateRunes(c, draftClueTailChars))
- }
- return strings.Join(capped, "; ")
-}
-
-// literalList renders a string slice as the protocol's list of quoted terms (['a', 'b']):
-// the draft text is prompt content the SCA reads, and Go's fmt.Sprint form ([a b]) does not
-// read as a list of terms.
-func literalList(in []string) string {
- quoted := make([]string, 0, len(in))
- for _, s := range in {
- quoted = append(quoted, "'"+s+"'")
- }
- return "[" + strings.Join(quoted, ", ") + "]"
-}
-
-// Slot-table research: the executor behind the outer loop's `rag_agent` node.
-//
-// Covers the slot-table build, the research pass, patch merging and draft rendering.
-//
-// One research round = one action session per unresolved slot, run concurrently
-// under a semaphore, with the resulting branches folded back into the shared
-// table.
-
-// SlotEvidence records which passages produced a slot's candidate, so the SCA
-// can verify the candidate against the passages that actually produced it.
-type SlotEvidence struct {
- EvidenceIDs []string
- TerminalType string
- Candidate string
-}
-
-// SlotResearchResult is one research round's output.
-type SlotResearchResult struct {
- SlotTable runtime.State
- CollectedAnswer string
- UnresolvedSlots []map[string]any
- SlotEvidence map[string]SlotEvidence
- SlotDraft string
- // SlotRecord is the same table rendered for the ANSWER prompt: the facts the
- // research settled, without the machine fields the SCA needs (see
- // RenderSlotRecord). The two are produced together so they cannot drift.
- SlotRecord string
- Attempted []map[string]any
-}
-
-// buildSlotTableFrom wraps InitializeState, converting its non-error result into
-// an error so BuildSlotTable can apply the single fallback path.
-func buildSlotTableFrom(ctx context.Context, deps runtime.SessionDeps, question string, fanouts []string, deadlineLeft float64) (runtime.State, []string, error) {
- if deps.Model == nil {
- return runtime.State{}, nil, fmt.Errorf("no model configured")
- }
- init := runtime.InitializeState(ctx, deps, question, fanouts, deadlineLeft)
- if len(init.Root.State) == 0 {
- // Covers both an empty reply and a reply the strict parser rejected (non-integer id
- // / non-iterable clues).
- return runtime.State{}, nil, fmt.Errorf("decomposition failed")
- }
- return init.Root, init.FirstQueries, nil
-}
-
-// ── Evidence-guided batched answer generation (APT-RAG) ─────────────────────
-
-// evidenceChunkPrefix: (agentic_rag_graph.py:537):
-// only the claim pseudo-chunks are atomic evidence rows.
-const evidenceChunkPrefix = "claim_"
-
-// Evidence-guided batching constants.
-//
-// The threshold is deliberately conservative: APT-RAG ships sim_threshold=0.0
-// (any overlap), which merges weakly related slots and, worse, lets a single
-// parse failure blank every answer in the batch at once. Both conditions must
-// hold. The absolute floor stops two tiny evidence sets from merging on a
-// single coincidental hit; the ratio gate stays LOW enough to actually fire —
-// measured FRAMES overlap was 0.03-0.2 mean, so the old 0.5 never triggered at
-// all. (APT-RAG ships 0.0; that is too loose for us because one parse failure
-// would blank every answer in the batch.)
-const (
- EvidenceBatchMinSim = 0.15
- EvidenceBatchMinShared = 2
- EvidenceBatchMaxSlots = 4
- EvidenceBatchMaxChars = 12000
-)
-
-// EvidenceBatchPrompt
-// verbatim.
-const EvidenceBatchPrompt = "Answer each listed sub-question using ONLY the shared evidence below.\n" +
- "These sub-questions share this evidence, so read it once and answer all of them.\n" +
- "Return JSON only, mapping each sub-question id to its answer: " +
- `{"<id>": "<answer>", ...}. Use an empty string when the evidence does not ` +
- "answer that sub-question. Never invent facts."
-
-// intersectionSize counts the shared members of two evidence-id sets.
-func intersectionSize(a, b map[string]bool) int {
- n := 0
- for id := range a {
- if b[id] {
- n++
- }
- }
- return n
-}
-
-// Word-level coverage a pooled evidence row must reach before it is allowed to
-// answer a slot on its own. Deliberately strict: a wrong prefill costs
-// accuracy, while a missed prefill only costs one session (which still runs).
-// Strict on purpose: a wrong prefill costs accuracy, a missed one costs a session.
-const EvidencePrefillCoverage = 0.6
-
-// The literal helpers this file's prompts render with (retrieval.formatLiteral /
-// retrieval.quoteLiteral) live in the runtime package: initialize_state needs the same
-// syntax to read the model's reply, so there is one implementation.
-
-func strengthOf(v runtime.Variable) float64 {
- if v.CandidateStrength == nil {
- return 0.0
- }
- return *v.CandidateStrength
-}
-
-// Fallback-draft synthesis tuning.
-const (
- // draftChunkCap / draftChunkChars: the evidence budget.
- draftChunkCap = 16
- draftChunkChars = 1200
- // draftFallbackChars: raw-evidence fallback when no model is available or the
- // call fails.
- draftFallbackChars = 4000
- // draftMaxChars caps the composed draft.
- draftMaxChars = 6000
-)
-
-// containsNonASCII: a question carrying non-ASCII characters is answered in its own
-// language.
-func containsNonASCII(s string) bool {
- for _, r := range s {
- if r > unicode.MaxASCII {
- return true
- }
- }
- return false
-}
-
-// BuildAgenticGraph drives the agentic-search loop and returns the terminal state.
-// Declares and compiles the graph, then invokes the runnable.
-//
-// The caller reads the answer from st.KB.PreSummary / st.CollectedAnswer and composes it
-// (see Run for the single-shot entry point).
-//
-// The returned error mirrors run_agentic_rag's holder["error"] — the graph itself
-// failed. The state is still returned so the caller can surface whatever the run
-// produced; an empty result alone is not a failure.
-//
-// Activation is EXPLICIT: call this directly, with no init()-based auto-registration.
-// This is the agentic planner (medium/high/ultra); the mode dispatch lives in
-// RAGTools.Run, and agentic modes register this as the AgenticLoop.
-func BuildAgenticGraph(ctx context.Context, deps RAGTools, question, keywords string, maxLoops int, messages []schema.Message) (*AgenticState, error) {
- logger := deps.logger()
- // formalize_question — the graph starts at formalize_question, which both
- // initializes the state and arms the budget, so the budget is NOT set here.
- st := NewAgenticState(question, keywords, maxLoops, messages)
- if deps.KB != nil {
- st.KB = deps.KB
- }
- spec := runtime.ResolveMode(deps.Tools)
- enableSCA := spec.EnableSCA
- useFanout := spec.UseFanout
- scaMaxRounds := spec.SCAMaxRounds
-
- step(ctx, logger, "Agentic RAG", "Starting research in %s mode (self-check %s, question fan-out %s).",
- spec.Label, onOff(enableSCA), onOff(useFanout))
-
- // run_agentic_rag — there is NO whole-graph wall clock. Research stays
- // bounded by the per-node timeouts (bounded / PassTimeoutS / SCATimeoutS…),
- // the routing guards (MinRoundHeadroomS) and the visit limit; and because
- // formalize_answer now composes inside the graph, the answer stream must be allowed to
- // run until the model finishes. Capping the whole graph at TotalBudgetS+30s used to cut
- // the composition short.
-
- // Prefetch is gated on fan-out. NOTE: a stale comment elsewhere claims prefetch is
- // DISABLED, but the wiring still enables it for fan-out modes — behaviour here matches
- // the CODE, not the comment.
- usePrefetch := useFanout
-
- // run_agentic_rag — LangGraph counts NODE VISITS, not loop iterations: one
- // research round is rag_agent → draft → sca = three visits. Counting run
- // steps instead would let Go do roughly 3x the work before the guard trips,
- // so the counter is kept here and each node adds the visits it costs.
- limit := graphRecursionLimit(true, maxLoops)
- visits := 0
-
- // The graph is declared per run: deps / logger / spec are request-scoped and
- // Eino captures them in closures, so a compiled graph cannot be shared
- // across requests. The graph is 7 nodes; compilation is validation plus
- // channel wiring.
- g := compose.NewGraph[*AgenticState, *AgenticState]()
- var buildErr error
- visit := func(n int) { visits += n }
-
- addNode := func(name string, fn func(context.Context, *AgenticState) (*AgenticState, error)) {
- if buildErr != nil {
- return
- }
- buildErr = g.AddLambdaNode(name, compose.InvokableLambda(fn))
- }
- addEdge := func(from, to string) {
- if buildErr != nil {
- return
- }
- buildErr = g.AddEdge(from, to)
- }
- addBranch := func(from string, cond func(context.Context, *AgenticState) (string, error), ends map[string]bool) {
- if buildErr != nil {
- return
- }
- buildErr = g.AddBranch(from, compose.NewGraphBranch(cond, ends))
- }
- // guard routes to "stop" once the visit budget is spent. The guards ABORT the run
- // rather than running formalize_answer, so the stop node reports the failure instead of
- // composing an answer from whatever partial research the round had gathered.
- guard := func(next string) string {
- if visits >= limit {
- return "stop"
- }
- return next
- }
-
- addNode("formalize_question", func(ctx context.Context, s *AgenticState) (*AgenticState, error) {
- visit(1)
- formalizeQuestionNode(ctx, deps, s, logger)
- return s, nil
- })
- addNode("planner", func(ctx context.Context, s *AgenticState) (*AgenticState, error) {
- visit(1)
- plannerNode(ctx, deps, s, logger)
- return s, nil
- })
- addNode("prefetch", func(ctx context.Context, s *AgenticState) (*AgenticState, error) {
- visit(1)
- // firstRound is always true here: prefetch is only reachable from the
- // planner, which itself only runs on the first pass.
- prefetchNode(ctx, deps, s, logger, true)
- return s, nil
- })
- // rag_agent is one research round: rag_agent → draft → sca (build_agentic_graph).
- // Both the first pass and the rewrite-driven passes enter this node.
- addNode("rag_agent", func(ctx context.Context, s *AgenticState) (*AgenticState, error) {
- visit(agenticRoundVisits)
- ragAgentNode(ctx, deps, s, logger)
- draftNode(ctx, deps, s, logger)
- scaNode(ctx, deps, s, logger)
- return s, nil
- })
- addNode("query_rewrite", func(ctx context.Context, s *AgenticState) (*AgenticState, error) {
- visit(1)
- queryRewriteNode(ctx, deps, s, logger)
- return s, nil
- })
- addNode("formalize_answer", func(c context.Context, s *AgenticState) (*AgenticState, error) {
- visit(1)
- formalizeAnswerNode(c, deps, s, logger)
- return s, nil
- })
- // stop is the visit-budget backstop and has no graceful path: exhausting the budget is
- // recorded as a graph failure and, when the run also produced nothing, surfaces as
- // graphFailureFallback. Returning the state as-is would compose a partial answer where
- // the internal error belongs, so the error is raised here and the caller drops the
- // state (see below).
- addNode("stop", func(_ context.Context, s *AgenticState) (*AgenticState, error) {
- logger.Printf("[Agentic RAG] stopping after %d node visits (limit=%d)", visits, limit)
- return s, fmt.Errorf("graph recursion limit reached after %d node visits", visits)
- })
-
- // START → formalize_question.
- addEdge(compose.START, "formalize_question")
- addEdge("prefetch", "rag_agent")
- addEdge("formalize_answer", compose.END)
- addEdge("stop", compose.END)
-
- // build_agentic_graph — onward to the planner, or straight to rag_agent when
- // fan-out is off (medium).
- addBranch("formalize_question", func(_ context.Context, _ *AgenticState) (string, error) {
- if useFanout {
- return guard("planner"), nil
- }
- return guard("rag_agent"), nil
- }, map[string]bool{"stop": true, "planner": true, "rag_agent": true})
-
- addBranch("planner", func(_ context.Context, _ *AgenticState) (string, error) {
- if usePrefetch {
- return guard("prefetch"), nil
- }
- return guard("rag_agent"), nil
- }, map[string]bool{"stop": true, "prefetch": true, "rag_agent": true})
-
- addBranch("prefetch", func(_ context.Context, _ *AgenticState) (string, error) {
- return guard("rag_agent"), nil
- }, map[string]bool{"stop": true, "rag_agent": true})
-
- addBranch("rag_agent", func(_ context.Context, s *AgenticState) (string, error) {
- rounds := scaMaxRounds
- // An ENUMERATION runs a bounded number of rounds: one to ask the corpus the act
- // patterns its seed carries and name what they return, one to recover what the
- // round's own record shows it reached and did not record. Measured (2026-09-16,
- // 三国/关羽, three runs): round 2 took a table from 11 members to 14 by recording
- // 车胄 / 程远志 / 管亥 — and every round after it added `+0 chunks`, i.e. a third
- // draft of the same list. The bound is on the ENUMERATION the planner declared
- // (Coverage.Ok), so a value question keeps the rounds it buys accuracy with.
- if runtime.CoverageOf(s.SlotTable).Ok() && rounds > 2 {
- rounds = 2
- }
- return guard(agenticNodeName(routeSCA(s, enableSCA, rounds))), nil
- }, map[string]bool{"stop": true, "query_rewrite": true, "formalize_answer": true})
-
- addBranch("query_rewrite", func(_ context.Context, s *AgenticState) (string, error) {
- return guard(agenticNodeName(routeRewrite(s, scaMaxRounds, logger))), nil
- }, map[string]bool{"stop": true, "rag_agent": true, "formalize_answer": true})
-
- if buildErr != nil {
- logger.Printf("[Agentic RAG] graph build failed: %v", buildErr)
- return st, buildErr
- }
-
- runnable, err := g.Compile(ctx,
- compose.WithGraphName("agentic_rag"),
- // The visit counter above is the authoritative guard; this is only a backstop so a
- // mis-wired cycle cannot spin.
- compose.WithMaxRunSteps(limit*2+16),
- )
- if err != nil {
- logger.Printf("[Agentic RAG] graph compile failed: %v", err)
- return st, err
- }
- out, err := runnable.Invoke(ctx, st)
- if err != nil {
- logger.Printf("[Agentic RAG] graph run failed: %v", err)
- // A failed run keeps NO state: an empty state is handed back so everything
- // downstream sees nothing. Returning the mutated state would surface slots/draft the
- // caller must discard.
- return &AgenticState{}, err
- }
- if out != nil {
- return out, nil
- }
- return st, nil
-}
-
-// agenticNodeName maps the routing predicates' node ids onto the Eino graph's
-// node keys.
-func agenticNodeName(n agenticNode) string {
- switch n {
- case nodeQueryRewrite:
- return "query_rewrite"
- case nodeRagAgentLoop:
- return "rag_agent"
- default:
- return "formalize_answer"
- }
-}
-
-// Explicit wiring into the RAGTools.Run loop registration.
-//
-// There is NO init()-based auto-registration: the caller constructs the object and invokes
-// it, activating the full medium/high/ultra pipeline by explicitly registering this
-// loop:
-//
-// agentic_rag.SetAgenticLoop(agentic_rag.NewAgenticLoop())
-//
-// Without registration, RAGTools.Run falls back to a single action session for
-// agentic modes (see Run).
-
-// NewAgenticLoop returns the outer agentic loop adapter for RAGTools.Run. It
-// converts the RAGTools run config into the planner's state and maps the result
-// back onto the
-// RunResponse.
-func NewAgenticLoop() AgenticLoop {
- return func(ctx context.Context, deps RAGTools, req runtime.RunRequest, kb *runtime.Kbinfos, resp *RunResponse, logger *log.Logger) {
- if logger == nil {
- logger = _LOG
- }
- if deps.Model == nil {
- logger.Printf("[Agentic RAG] no model configured for mode %q; degrading to a direct search", resp.Mode.Label)
- runDirectFallback(ctx, deps, req, kb, logger)
- return
- }
-
- // The loop needs an executor for its programmatic fan-out fetches, built from the same
- // retrieval backend the single-session path uses. It must project the FULL RAGTools
- // config, otherwise fan-out and action-session retrieval silently lose tuning —
- // notably the tag boost (Tagger/KBs).
- sd := searchDepsFor(ctx, deps, req, req.DatasetIDs, req.TenantID, kb, logger)
- sd.WebSearch = deps.WebSearch
- sd.CiteRules = deps.CiteRules
-
- toolset := &runtime.Toolset{
- ThinkingMode: resp.Mode.Label,
- // web_search is visible only when the mode exposes it AND a provider is actually
- // wired. Advertising it without a provider leaves the model calling a tool that can
- // only return an infra error.
- HasWebSearch: resp.Mode.HasTool("web_search") && deps.WebSearch != nil,
- // The dataset's real metadata fields, read once here and rendered into the
- // metadata_search schema and the session seed. Nil (no metadata / unreadable
- // index / no resolver) keeps the shipped title-only schema.
- MetadataFields: runtime.MetadataCatalogPtr(ctx, sd),
- DisabledTools: map[string]bool{},
- Exec: runtime.NewSearchExecutor(sd, req),
- }
-
- st, runErr := BuildAgenticGraph(ctx, RAGTools{
- Tools: toolset,
- Search: sd, // dual-channel fan-out retrieves directly
- Model: deps.Model,
- ModelName: deps.ModelName,
- Prompts: deps.Prompts,
- KB: kb,
- SCAPrompts: deps.Prompts,
- RewritePrompts: deps.Prompts,
- MaxLength: deps.MaxLength,
- Logger: logger,
- // The terminal composition is the formalize_answer node body; it must reach the
- // graph even though this deps copy is rebuilt here.
- Finalize: deps.Finalize,
- // Steps is deliberately not copied here: Rag already bound the caller's
- // reporter on ctx, and every stage and tool reads it from there
- // (runtime.StepsFrom), so a second copy on this rebuilt RAGTools would be
- // dead weight that could drift.
- }, req.Question, req.Keywords, 3, deps.Messages)
-
- // A graph exception is recorded separately from "research found nothing". The state is
- // still surfaced below: the internal-error message is only swapped in when the failed
- // run ALSO produced nothing.
- if runErr != nil {
- resp.GraphFailed = true
- }
-
- // Surface the loop's outcome on the RunResponse.
- resp.Slots = append(resp.Slots, st.SlotTable.State...)
- // Slot citations: the chat pipeline's citation decoration rewrites
- // leaked "[ID:Slot N]" markers with these evidence chunk ids. Plain
- // ids (not positions) so the mapping survives the caller's later
- // evidence-pool narrowing (selectEvidence).
- if len(st.SlotEvidence) > 0 {
- sc := make(map[string][]string, len(st.SlotEvidence))
- for sid, ev := range st.SlotEvidence {
- sc[sid] = ev.EvidenceIDs
- }
- resp.SlotCitations = sc
- }
- // Research findings: the SCA-reviewed draft. NOTE: this is the research
- // draft, NOT the final answer — RAGTools.Run composes the final cited
- // answer afterwards from KB.PreSummary (see composeFinalAnswer).
- if st.CollectedAnswer != "" {
- resp.CollectedAnswer = st.CollectedAnswer
- }
- resp.Partial = st.PartialAnswer
- resp.SearchRounds = st.SearchRounds
- resp.Verdict = st.Verdict
- // SCAFeedback is the body of the "[Research status]" note (status hint + hard
- // violations + missing claims + confidence) that rag() folds into the answer for EVERY
- // non-SUFFICIENT verdict. Rag() appends the trailing "STOP" vs "call rag again"
- // sentence based on the consecutive-unanswerable count.
- resp.SCAFeedback = scaFeedback(st.SCA, st.Verdict)
- // Update the consecutive-unanswerable guardrail on the shared per-turn *RAGCache.
- // Rag() builds deps.Cache before the outer react branch, so this counter accumulates
- // across the outer loop's multiple rag() calls within a single turn.
- //
- // A SUFFICIENT verdict resets the counter, any other verdict bumps it, and
- // the update is locked because those calls run concurrently — hence the
- // call goes straight to the cache method (it used to be a pass-through
- // wrapper that only renamed it).
- deps.Cache.NoteUnanswerable(st.Verdict)
- }
-}
-
-// graphRecursionLimit: the graph aborts after this many node visits: 60 for the agentic
-// graph, else max(25, max_loops*8). Eino counts run steps, not node visits, so the graph
-// keeps its own visit counter and checks it in every branch.
-func graphRecursionLimit(agentic bool, maxLoops int) int {
- if agentic {
- return AgenticRecursionLimit
- }
- if n := maxLoops * 8; n > lowRecursionLimitBase {
- return n
- }
- return lowRecursionLimitBase
-}
-
-// formalizeStepLine renders the formalize phase's user-visible step, or "" when
-// there is nothing to report at all.
-//
-// The phase used to announce `Formalized the question into %q` unconditionally,
-// which reads as a rewrite even when Formalize returned the question VERBATIM —
-// the common case: a single-turn run skips the rewrite entirely, and the
-// multi-turn prompt asks for the question unchanged "in most cases".
-//
-// Both outcomes are reported, because either way a reader wants to see the
-// question this turn was actually researched under: unchanged is "kept the
-// question as asked", a rewrite gets the pair — seeing the asked question NEXT TO
-// the searched one is what lets a reader check that a pronoun or an ellipsis was
-// resolved the way they meant. Only a formalize that produced no question at all
-// stays silent. No trailing period: the sentence ends on the quoted question,
-// which carries its own full-width "?".
-func formalizeStepLine(asAsked, standalone string) string {
- asAsked = strings.TrimSpace(asAsked)
- standalone = strings.TrimSpace(standalone)
- if standalone == "" {
- return ""
- }
- if standalone == asAsked {
- if asAsked == "" {
- return ""
- }
- return fmt.Sprintf("Kept the question as asked: %q", trunc(asAsked, 80))
- }
- if asAsked == "" {
- return fmt.Sprintf("Standalone question for this turn: %q", trunc(standalone, 80))
- }
- return fmt.Sprintf("Rewrote the follow-up into a standalone question: %q → %q",
- trunc(asAsked, 80), trunc(standalone, 80))
-}
-
-// formalizeQuestionNode is the graph's first node. It resolves pronouns and ellipses from
-// the conversation into a standalone question plus search keywords, and arms the global
-// budget in its return — so the formalization work itself is NOT charged to that budget.
-//
-// Single-turn input costs no LLM call (Formalize returns early).
-func formalizeQuestionNode(ctx context.Context, deps RAGTools, st *AgenticState, logger *log.Logger) {
- // formalize_question — the budget is armed as the node RETURNS, i.e. after any
- // formalization work has already happened (Python stamps the deadline in this
- // node's return dict, agentic_rag_graph.py:1061). Arming on ENTRY instead
- // charged the formalization call — a full rewrite on multi-turn, a keyword
- // extraction on single-turn — to the research budget, so every downstream
- // timeout and the MinRoundHeadroomS guard saw a budget short by one LLM call.
- //
- // Deferred so that every return path arming it stays impossible to forget:
- // Python's node has a single return (agentic_rag_graph.py:1041-1062, the
- // deadline at :1061), so it arms the budget unconditionally — even for an empty
- // conversation — and a Go early exit must not become the one path that leaves
- // the graph with no deadline at all.
- defer func() {
- st.Deadline = time.Now().Add(time.Duration(TotalBudgetS * float64(time.Second)))
- }()
-
- q, kw := formalizeConversation(ctx, deps, st.Messages)
- if q != "" {
- st.Question = q
- }
- if kw != "" && st.Keywords == "" {
- st.Keywords = kw
- }
- if logger != nil {
- asAsked, _ := transcriptOf(st.Messages)
- if line := formalizeStepLine(asAsked, q); line != "" {
- runtime.StepsFrom(ctx).StageLine(logger, "Formalize", line)
- }
- }
-}
-
-// formalizeQuestion resolves pronouns and ellipses from the conversation into a standalone
-// question plus search keywords — the same work the agentic graph's first node does.
-// Single-turn input costs no LLM call (Formalize returns early), and the graph budget is
-// NOT started here: the deadline is armed in that node's return, so this cost is not
-// charged to it. The low graph has no scheduler, so it runs as a plain step of
-// BuildLowGraph rather than as a scheduled node; the naive path has no such step.
-func formalizeQuestion(ctx context.Context, deps RAGTools, req *runtime.RunRequest, logger *log.Logger) {
- // The guard is repeated here (it is also inside formalizeConversation) because the
- // STATS record belongs to "there was something to formalize": a history-less call
- // spends no call and must not count as one.
- if len(deps.Messages) == 0 || deps.Model == nil {
- return
- }
- if deps.Stats != nil {
- deps.Stats.RecordCall("formalize")
- }
- q, kw := formalizeConversation(ctx, deps, deps.Messages)
- if deps.Stats != nil && q == "" {
- deps.Stats.RecordFailed("formalize")
- }
- if q != "" {
- req.Question = q
- }
- if kw != "" && req.Keywords == "" {
- req.Keywords = kw
- }
- if logger != nil {
- asAsked, _ := transcriptOf(deps.Messages)
- if line := formalizeStepLine(asAsked, q); line != "" {
- runtime.StepsFrom(ctx).StageLine(logger, "Formalize", line)
- }
- }
-}
-
-// formalizeConversation is the step the two formalize_question entry points share (the
-// graph node and the low-graph pass): the conversation resolved into a standalone question
-// plus search keywords. Both empty when there is nothing to formalize — no model, or no
-// history, which the single-turn fast path returns on without a call.
-func formalizeConversation(ctx context.Context, deps RAGTools, messages []schema.Message) (string, string) {
- if len(messages) == 0 || deps.Model == nil {
- return "", ""
- }
- return Formalize(ctx, runtime.SessionDeps{Model: deps.Model, Prompts: deps.Prompts}, messages, deps.MaxLength)
-}
-
-// RunAgenticRAG: the mode
-// dispatch plus execution of the selected pipeline. It is the counterpart of
-// RAGTools.rag (Go: Run), which owns the conversation-level concerns around it — cache
-// lookup, the effective question, attachments and storing the result.
-//
-// The dispatch is run_agentic_rag:
-//
-// mode is NAIVE → _naive_rag (plain retrieval)
-// mode.Agentic → build_agentic_graph (medium / high / ultra)
-// else → build_low_graph (low: formalize → direct_search)
-//
-// Formalization belongs to the graphs, not to Run: it is the first node of both the
-// agentic and low graphs, while the naive path has none, so each path below runs it where
-// its counterpart does.
-func RunAgenticRAG(ctx context.Context, deps RAGTools, req runtime.RunRequest, sd runtime.SearchDeps, kb *runtime.Kbinfos, resp *RunResponse, logger *log.Logger, spec runtime.ModeSpec) {
- // The graph budget is NOT started here: the deadline is armed inside the first node's
- // return, i.e. after formalization has run, so formalization is not charged to it.
- switch {
- case spec.Agentic:
- // build_agentic_graph — formalization is the graph's first node,
- // so it happens inside runAgentic.
- runAgentic(ctx, deps, req, sd, kb, resp, logger)
- case spec.Label != "naive":
- // build_low_graph — formalize → direct_search.
- if err := BuildLowGraph(ctx, deps, req, sd, kb, resp, logger); err != nil {
- // run_agentic_rag — a graph exception is recorded separately from
- // "research found nothing"; the fallback below only fires when the
- // run also produced nothing.
- resp.GraphFailed = true
- }
- default:
- // _naive_rag: no formalize node, a plain retrieve with no
- // keyword extraction, and its own composition (flat "[i] content"
- // evidence under naiveAnswerSystem). It must therefore not fall
- // through to composeFinalAnswer.
- NaiveRAG(ctx, deps, req, kb, resp, logger)
- }
-
- // run_agentic_rag — only when research produced NOTHING *and* the graph
- // itself failed. An empty result without a failure is not an error: that is
- // EmptyResponse's job, and overwriting it here would hide the real reason.
- if resp.GraphFailed && resp.Answer == "" && len(resp.Slots) == 0 {
- step(ctx, logger, "Agentic RAG", "Research failed without producing anything; returning the fallback answer.")
- resp.Answer = graphFailureFallback
- }
-}
-
-var reAnswerThink = regexp.MustCompile(`(?s)^.*</think>`)
-
-// AnswerResult is the composed final answer.
-type AnswerResult struct {
- // Answer is the composed text.
- Answer string
- // Partial is true when the underlying research ended without a satisfying
- // verdict; the answer carries a partial-information preamble.
- Partial bool
- // NoEvidence is true when composition was skipped for lack of evidence.
- NoEvidence bool
- // Failed is true when the composition call failed and the fallback message
- // was returned.
- Failed bool
-}
-
-// providerErrorSummary renders a provider failure as ONE bounded line for the
-// think block: whitespace collapsed (a model client's error can embed newlines
-// and a whole JSON body) and cut at providerErrorSummaryMax. The developer log
-// keeps the error verbatim — this is the sentence a user reads while waiting.
-func providerErrorSummary(err error) string {
- if err == nil {
- return ""
- }
- return truncateRunes(strings.Join(strings.Fields(err.Error()), " "), providerErrorSummaryMax)
-}
-
-// errorAnswerText renders a provider failure as the ANSWER, in the shape the
-// classic chat path uses for its own failures (`**ERROR**: …`): an agentic run
-// that cannot call the model must read to the user exactly like a naive-mode one
-// — bold ERROR plus the provider's own message — instead of Python's generic
-// "I'm sorry…" sentence, which hid the cause and made an exhausted quota look
-// like a RAG bug.
-func errorAnswerText(err error) string {
- return "**ERROR**: " + providerErrorSummary(err)
-}
-
-// ComposeAnswer: turn the gathered
-// evidence into a grounded, cited answer.
-//
-// Behaviour, in order:
-// 1. no evidence + configured empty_response → return it WITHOUT calling the LLM;
-// 2. rank chunks by similarity, keep the top citeChunkCap as citation reference;
-// 3. render the evidence block under the token budget (kb_prompt);
-// 4. prepend the fact-preserving pre_summary (the SCA-reviewed draft) when set;
-// 5. call the model with FINAL_ANSWER_SYSTEM + the composed user content.
-func ComposeAnswer(ctx context.Context, deps AnswerDeps, kb *runtime.Kbinfos, question string, partial, abstain bool) AnswerResult {
- return ComposeAnswerWith(ctx, deps, kb, question, partial, abstain, false)
-}
-
-// multimodalUserMsg builds a user message that carries the given text plus any
-// vision-gated image data URIs. Returns nil when there is no text and no
-// images, so callers fall back to schema.UserMessage. Used by the non-outer
-// final-answer path so the compose model sees images even without the outer
-// react loop.
-func multimodalUserMsg(text string, images []string) *schema.Message {
- parts := make([]schema.MessageInputPart, 0, 1+len(images))
- if text != "" {
- parts = append(parts, schema.MessageInputPart{Type: schema.ChatMessagePartTypeText, Text: text})
- }
- for i := range images {
- uri := images[i]
- parts = append(parts, schema.MessageInputPart{
- Type: schema.ChatMessagePartTypeImageURL,
- Image: &schema.MessageInputImage{MessagePartCommon: schema.MessagePartCommon{URL: &uri}},
- })
- }
- if len(parts) == 0 {
- return nil
- }
- return &schema.Message{Role: schema.User, UserInputMultiContent: parts}
-}
-
-// answerTemperature: default sampling temperature:
-// build_agentic_graph uses gen_conf or {"temperature": 0.3} (/:1021)
-// and run_agentic_rag invokes the graph with gen_conf unset, so the compose
-// call always samples at 0.3.
-const answerTemperature = 0.3
-
-// answerTargetContract is the static answer-target guardrail. It costs no LLM call — it is
-// injected into every final-answer prompt.
-//
-// The EXTREME-SELECTION clause is the load-bearing part: for
-// shortest/longest/smallest/largest/most/least/最 questions, a model otherwise
-// tends to name the most common or first-listed candidate. Naming the clause
-// explicitly forces a comparison against the evidence instead.
-const answerTargetContract = "Answer Target Contract:\n" +
- "Final answer must directly satisfy the user's top-level who/what request. " +
- "Use bridge entities only as clues, and verify any proposed answer against " +
- "the evidence. In EXTREME-SELECTION questions (shortest/longest/smallest/" +
- "largest/most/least/最), compare the alternatives in the evidence and name " +
- "the EXTREME one rather than the most common or first-listed.\n"
-
-// withCitedChunks puts the passage behind every enumerated item FIRST, then the top-scoring
-// remainder. An enumerated answer cites one passage per element, and those passages are in the pool
-// but not necessarily among the few that score highest: a sixteen-member table whose answer could
-// name only three of them was reading the same six blocks as any other question.
-//
-// Members first also means a token budget that truncates can only drop the scored extras, never an
-// element's own passage (see KBPrompt, which stops at the budget).
-func withCitedChunks(ranked []map[string]any, kb *runtime.Kbinfos, cap int) []map[string]any {
- if kb == nil || len(ranked) == 0 {
- return ranked
- }
- cited := make([]map[string]any, 0, len(ranked))
- used := map[string]bool{}
- for _, id := range kb.CitedChunks() {
- if c := kb.ChunkByID(id); c != nil && !used[id] {
- used[id] = true
- cited = append(cited, c)
- }
- }
- for _, c := range ranked {
- if len(cited) >= len(kb.CitedChunks())+cap {
- break
- }
- if id := runtime.ChunkIDOf(c); id != "" && used[id] {
- continue
- }
- cited = append(cited, c)
- }
- if len(cited) == 0 {
- return ranked
- }
- return cited
-}
-
-// compactAnchored renders each anchored member's block from the quote the naming node matched it
-// to, leaving every other chunk as it is. The block is what the budget is spent on, so whole
-// passages put only the first handful of members inside the render — and a member outside it has
-// no block number the model could cite. The quote carries the same words the record already
-// states, so no evidence is lost: the passage keeps its place in the pool and in the reference.
-func compactAnchored(chunks []map[string]any, kb *runtime.Kbinfos) []map[string]any {
- if kb == nil {
- return chunks
- }
- quotes := map[string]string{}
- for _, ref := range kb.AnchoredRefs() {
- if ref.ChunkID == "" || ref.Quote == "" {
- continue
- }
- if _, dup := quotes[ref.ChunkID]; !dup {
- quotes[ref.ChunkID] = ref.Quote
- }
- }
- if len(quotes) == 0 {
- return chunks
- }
- out := make([]map[string]any, 0, len(chunks))
- for _, c := range chunks {
- q, ok := quotes[runtime.ChunkIDOf(c)]
- if !ok {
- out = append(out, c)
- continue
- }
- cp := make(map[string]any, len(c)+1)
- for k, v := range c {
- cp[k] = v
- }
- cp["content"] = q
- out = append(out, cp)
- }
- return out
-}
-
-// anchoredPoolEntries rebuilds a pool record for every enumerated member whose passage is NOT in
-// the pool, from the quote the naming node took from that passage.
-//
-// The citation chain is: marker number → position in the published list → the pool entry that id
-// resolves to. A member whose passage is gone therefore cannot be cited at all — the marker is
-// dropped (citePoolIdx answers -1) or opens nothing. Rebuilding the record from the member's own
-// quote keeps that chain whole, and the content it opens is exactly the text the answer states.
-func anchoredPoolEntries(kb *runtime.Kbinfos) []map[string]any {
- if kb == nil {
- return nil
- }
- var out []map[string]any
- for _, r := range kb.AnchoredRefs() {
- id := strings.TrimSpace(r.ChunkID)
- quote := strings.TrimSpace(r.Quote)
- if id == "" || quote == "" || kb.ChunkByID(id) != nil {
- continue
- }
- out = append(out, map[string]any{
- "chunk_id": id,
- "content": quote,
- "content_with_weight": quote,
- })
- }
- return out
-}
-
-// publishAnchoredMembers publishes the slot table's anchored members into the answer's evidence
-// ledger: the citation list the compose puts IN FRONT of the answer (see withCitedChunks) and the
-// member→passage table its markers are written from (see CiteAnchoredMembers).
-//
-// It exists because the enumeration is not the only writer of members. Both publish sites sit in the
-// enumeration's write-back (coverage_step.go), so a member set a METADATA selection defined never
-// reached the answer's evidence at all: measured (2026-09-21) the record listed 6 documents with
-// their update times, the answer stated all 6 — and the evidence it read carried six blocks about
-// OTHER documents with zero markers resolved. The data was in the table the whole time; nobody
-// published it.
-//
-// The anchors such a session records are not pool chunk ids. metadata_search runs no retrieval and
-// admits no passage (tool_executor.go), so it hands the session DOC ids and the session writes those
-// in: measured, 12 anchors, all 32-hex doc ids, none resolvable by ChunkByID. Resolution is
-// therefore part of publishing rather than a refinement of it (see resolveMemberAnchor).
-//
-// derived is what the table claims, resolved what the pool could point at. A member that resolves to
-// nothing is NOT published — it stays a name in the record, but gets no block and no marker rather
-// than a block that opens somebody else's passage.
-func publishAnchoredMembers(kb *runtime.Kbinfos, table *runtime.State) (derived, resolved int) {
- if kb == nil || table == nil {
- return 0, 0
- }
- refs := runtime.AnchoredItemRefs(table)
- if derived = len(refs); derived == 0 {
- return 0, 0
- }
- published := make([]runtime.AnchoredRef, 0, len(refs))
- cited := make([]string, 0, len(refs))
- for _, ref := range refs {
- id, quote := resolveMemberAnchor(kb, ref)
- if id == "" {
- continue
- }
- ref.ChunkID, ref.Quote = id, quote
- published = append(published, ref)
- cited = append(cited, id)
- }
- if resolved = len(cited); resolved == 0 {
- return derived, 0
- }
- // UNION, not replace: NoteCitedChunks replaces, and the enumeration's own list may hold
- // passages this derivation does not — a member whose anchor no longer resolves must not cost
- // the answer a passage that still does.
- kb.NoteCitedChunks(appendMissingIDs(kb.CitedChunks(), cited))
- kb.NoteAnchoredRefs(mergeAnchoredRefs(kb.AnchoredRefs(), published))
- return derived, resolved
-}
-
-// resolveMemberAnchor maps one member's recorded anchor to a passage the pool holds, and returns the
-// quote that block may be rendered from ("" = render the whole passage).
-//
-// 1. the anchor is a pool chunk id — the enumeration's own case;
-// 2. the anchor is a doc id the pool holds chunks for — a metadata selection's case: the member IS
-// the document, so it takes that document's passage, preferring the one that carries the quote.
-// Which passage stands for a document is otherwise arbitrary, and an arbitrary passage is one
-// the answer's own quotation will not match (see CitedAnchoredMembers);
-// 3. neither — no passage, and the caller publishes nothing for it.
-//
-// The quote is KEPT only when the resolved passage carries it. compactAnchored renders a published
-// member's block from the quote INSTEAD of the passage, so a quote the passage does not contain —
-// measured: "file_name: …; update_time: 2026-09-21 13:35:46", the metadata line the session was
-// shown — would replace the passage with something that is not in it.
-func resolveMemberAnchor(kb *runtime.Kbinfos, ref runtime.AnchoredRef) (string, string) {
- anchor := strings.TrimSpace(ref.ChunkID)
- if anchor == "" {
- return "", ""
- }
- if chunk := kb.ChunkByID(anchor); chunk != nil {
- if runtime.QuoteHeldBy(runtime.ChunkTextOf(chunk), ref.Quote) {
- return anchor, ref.Quote
- }
- return anchor, ""
- }
- bestID := ""
- bestScore := 0.0
- for _, chunk := range kb.Chunks {
- if runtime.DocIDOf(chunk) != anchor {
- continue
- }
- id := runtime.ChunkIDOf(chunk)
- if id == "" {
- continue
- }
- if runtime.QuoteHeldBy(runtime.ChunkTextOf(chunk), ref.Quote) {
- return id, ref.Quote
- }
- if score := similarityOrScore(chunk); bestID == "" || score > bestScore {
- bestID, bestScore = id, score
- }
- }
- return bestID, ""
-}
-
-// mergeAnchoredRefs unions the refs already recorded with the ones just published, first occurrence
-// winning: a member the enumeration already matched to a passage keeps that passage, and this step
-// only adds the members it did not have.
-func mergeAnchoredRefs(existing, added []runtime.AnchoredRef) []runtime.AnchoredRef {
- out := make([]runtime.AnchoredRef, 0, len(existing)+len(added))
- seen := make(map[string]bool, len(existing)+len(added))
- for _, group := range [][]runtime.AnchoredRef{existing, added} {
- for _, ref := range group {
- name := strings.ToLower(strings.TrimSpace(ref.Name))
- if name == "" || strings.TrimSpace(ref.ChunkID) == "" || seen[name] {
- continue
- }
- seen[name] = true
- out = append(out, ref)
- }
- }
- return out
-}
-
-// appendMissingIDs appends the ids not already present, in order: a passage the render left out
-// (the budget admits only the first few whole chunks) still has to hold a position in the
-// published list, or nothing can cite it.
-func appendMissingIDs(ids, extra []string) []string {
- seen := make(map[string]bool, len(ids))
- for _, id := range ids {
- seen[id] = true
- }
- for _, id := range extra {
- if id = strings.TrimSpace(id); id == "" || seen[id] {
- continue
- }
- seen[id] = true
- ids = append(ids, id)
- }
- return ids
-}
-
-// answerPromptWithEvidence renders the parts list in order: question, answer-target
-// contract, optional no-evidence instruction, research summary, partial preamble,
-// evidence. The no-evidence flag is `abstain or empty_result or not chunks`.
-func (d AnswerDeps) answerPromptWithEvidence(kb *runtime.Kbinfos, question string, partial, noEvidence bool) answerPrompt {
- chunks := []map[string]any{}
- if kb != nil {
- // An enumerated member whose passage is no longer in the pool is rebuilt from the quote the
- // naming node took from it BEFORE anything is rendered or published: the marker's number
- // indexes the published list and the client opens the pool entry behind it, so a member
- // whose passage was evicted (the pool cap, or selectEvidence narrowing to another call)
- // would otherwise lose its citation in the chat pipeline (citePoolIdx answers -1 and the
- // marker is dropped).
- if extra := anchoredPoolEntries(kb); len(extra) > 0 {
- kb.Admit(func(p *runtime.PoolAdmitter) {
- for _, c := range extra {
- p.Add(c)
- }
- })
- }
- chunks = kb.Chunks
- }
- ranked := rankByScore(chunks)
- citeChunks := withCitedChunks(ranked, kb, citeChunkCap)
- if len(citeChunks) == 0 {
- citeChunks = chunks
- }
- // An anchored member's block is rendered from its QUOTE: the budget is spent per block, so a
- // whole ~1200-char passage per member fits only the first handful of them, and every member
- // past that one is a member neither the model nor the answer can cite. The quote is the same
- // words the record states, and the passage itself stays in the pool and in the reference.
- citeChunks = compactAnchored(citeChunks, kb)
- maxTokens := d.MaxTokens
- if maxTokens <= 0 {
- maxTokens = evidenceBudgetTokens
- }
- // Publish the ordered evidence list the model is about to see so the chat
- // pipeline can resolve the answer's [ID:n] markers against THE SAME list and put
- // it in front of the reference (runtime.Kbinfos.CiteChunkIDs →
- // RunResponse.CiteChunkIDs → decorateHarnessAnswer). The blocks are numbered
- // 0-based because the client renders a marker by indexing reference.chunks with
- // its number, which the model has to write itself; citeChunks is the
- // similarity-ranked, capped selection (not the pool in order), so the reference
- // is ordered to match it.
- //
- // The ids come from the render's SOURCE indices, not from citeChunks: a chunk
- // with no content renders no block, so walking citeChunks would publish an id for
- // the skipped chunk and put every marker past it one block off.
- //
- // The enumerated members then follow whether or not their block rendered: the evidence
- // budget admits only the first few whole chunks, so a member past them holds no block
- // number the model could write — and a member with no position is a member the answer
- // cannot cite and the user cannot open. Appending keeps the rendered prefix intact
- // (block n is still position n) and gives the completion its target
- // (runtime.CiteAnchoredMembers).
- blocks, sources := prompts.KBPromptZeroBasedWithSourceIndices(citeChunks, maxTokens)
- if kb != nil {
- // Only ids the POOL still holds are appended: the chat pipeline resolves a marker by
- // looking its chunk up in the pool, and a member whose passage is gone would publish a
- // nil reference entry — the marker is then dropped and the line loses its citation
- // entirely. A member the pool no longer holds must lose its citation, not break the one
- // the user can open.
- present := make([]string, 0, len(kb.CitedChunks()))
- for _, id := range kb.CitedChunks() {
- if kb.ChunkByID(id) != nil {
- present = append(present, id)
- }
- }
- kb.CiteChunkIDs = appendMissingIDs(citeChunkIDsAt(citeChunks, sources), present)
- }
- evidence := strings.Join(blocks, "\n")
-
- summary := ""
- record := composedRecord(kb)
- if kb != nil {
- summary = strings.TrimSpace(kb.PreSummary)
- }
-
- parts := []string{fmt.Sprintf("Question:\n%s\n", question)}
-
- // The static guardrail, always present.
- parts = append(parts, answerTargetContract)
-
- // How to degrade when there is no evidence (reached only when no empty_response
- // short-circuited the call).
- if noEvidence {
- if summary != "" {
- parts = append(parts, noEvidenceWithSummary)
- } else {
- parts = append(parts, noEvidenceWithoutSummary)
- }
- }
-
- if record != "" {
- // The slot table of an agentic round: facts to check the answer against,
- // explicitly NOT evidence and NOT answer text (see recordContract).
- parts = append(parts, fmt.Sprintf("%s\n%s\n", recordContract, record))
- } else if summary != "" {
- // A prose summary (no slot table behind it): it stands in as findings.
- parts = append(parts, fmt.Sprintf("Research Summary (primary evidence):\n%s\n", summary))
- }
- // _compose_answer_from_evidence: the partial preamble goes HERE — after the research
- // summary and before the evidence — not at the very front of the prompt.
- if partial {
- parts = append(parts, fmt.Sprintf("%s\n", runtime.PartialAnswerPreamble))
- }
- parts = append(parts, fmt.Sprintf("Evidence:\n%s", evidence))
-
- return answerPrompt{system: d.composeSystem(), user: strings.Join(parts, "\n"), partial: partial}
-}
-
-// naiveAnswerSystem is the fixed system prompt the naive path sends. It is deliberately
-// NOT FinalAnswerSystem: the naive path is a
-// plain single-pass answer and carries neither the citation contract nor the
-// partial-answer preamble the agentic modes compose.
-const naiveAnswerSystem = "Answer the question using ONLY the numbered evidence below. Cite with [n] markers. If the evidence does not answer it, say so plainly — do not use outside knowledge."
-
-// naiveAnswerTemperature is the sampling temperature of the naive answer call
-// (: answer_conf = gen_conf or {"temperature": 0.3}). Go's RunRequest
-// carries no per-call generation config, so this is always the no-gen_conf
-// branch.
-const naiveAnswerTemperature = 0.3
-
-// naiveFallbackChars caps the evidence echoed back when the naive answer call
-// fails (: evidence[:4000]).
-const naiveFallbackChars = 4000
-
-// modelWithTemperature returns a view of mdl that samples at temp, falling back
-// to the model's default when it does not support per-call temperature.
-func modelWithTemperature(mdl runtime.SessionModel, temp float64) runtime.SessionModel {
- if tm, ok := mdl.(runtime.TemperatureModel); ok {
- return &temperatureModel{mdl: tm, temp: temp}
- }
- return mdl
-}
-
-type temperatureModel struct {
- mdl runtime.TemperatureModel
- temp float64
-}
-
-func (m *temperatureModel) Complete(ctx context.Context, messages []schema.Message, tools []runtime.ToolSpec) (*runtime.ModelReply, error) {
- return m.mdl.CompleteWithTemperature(ctx, messages, tools, m.temp)
-}
diff --git a/internal/rag/agentic-rag/agentic_rag_graph_test.go b/internal/rag/agentic-rag/agentic_rag_graph_test.go
deleted file mode 100644
index b11fcec921..0000000000
--- a/internal/rag/agentic-rag/agentic_rag_graph_test.go
+++ /dev/null
@@ -1,5029 +0,0 @@
-package agentic_rag
-
-import (
- "bytes"
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "log"
- "os"
- "reflect"
- "sort"
- "strings"
- "sync"
- "testing"
- "time"
-
- "ragflow/internal/common"
- "ragflow/internal/engine"
- "ragflow/internal/engine/types"
- "ragflow/internal/entity"
- "ragflow/internal/rag/agentic-rag/runtime"
- "ragflow/internal/rag/agentic-rag/slots"
- "ragflow/internal/rag/prompts"
-
- "github.com/cloudwego/eino/compose"
- "github.com/cloudwego/eino/schema"
-)
-
-// Test doubles
-
-// scriptedModel answers with a fixed set of canned replies, one per call,
-// looping on the last if exhausted. It satisfies runtime.SessionModel. seen
-// records every message slice handed to Complete, for assertions on retry turns.
-type scriptedModel struct {
- mu sync.Mutex
- replies []string
- idx int
- seen [][]schema.Message
-}
-
-func (m *scriptedModel) push(reply string) {
- m.replies = append(m.replies, reply)
-}
-
-func (m *scriptedModel) Complete(ctx context.Context, msgs []schema.Message, _ []runtime.ToolSpec) (*runtime.ModelReply, error) {
- if ctx.Err() != nil {
- return nil, ctx.Err()
- }
- // The pre-flight probe (probeChatModel) is infrastructure, not script: it is
- // answered here and never consumes a scripted reply or enters `seen`, so the
- // call order each test documents (formalize → planner → slot table → draft →
- // SCA) stays what the test wrote. A test that wants the PROBE to fail uses
- // errModel instead.
- if len(msgs) == 1 && msgs[0].Role == schema.User && msgs[0].Content == probePrompt {
- return &runtime.ModelReply{Content: "ok"}, nil
- }
- m.mu.Lock()
- defer m.mu.Unlock()
- m.seen = append(m.seen, msgs)
- if len(m.replies) == 0 {
- return &runtime.ModelReply{Content: ""}, nil
- }
- r := m.replies[m.idx]
- if m.idx < len(m.replies)-1 {
- m.idx++
- }
- return &runtime.ModelReply{Content: r}, nil
-}
-
-// promptRoutedModel answers by inspecting the prompt instead of by call order,
-// so a test survives the graph adding or reordering node LLM calls. The SCA call
-// returns insufficient on its FIRST invocation and sufficient afterwards, which
-// is what drives one rewrite round.
-type promptRoutedModel struct {
- mu sync.Mutex
- scaSeen int
-}
-
-func (m *promptRoutedModel) Complete(ctx context.Context, msgs []schema.Message, _ []runtime.ToolSpec) (*runtime.ModelReply, error) {
- if ctx.Err() != nil {
- return nil, ctx.Err()
- }
- m.mu.Lock()
- defer m.mu.Unlock()
- var text strings.Builder
- for _, msg := range msgs {
- text.WriteString(msg.Content)
- }
- switch {
- case strings.Contains(text.String(), "Break the user's question into 2 to 5"):
- return &runtime.ModelReply{Content: `{"fanouts": ["when built", "where located"]}`}, nil
- case strings.Contains(text.String(), "research strategist"):
- return &runtime.ModelReply{Content: `{"slots":[{"id":0,"type":"aspect","question":"when built","clues":["1865"]},{"id":1,"type":"aspect","question":"where located","clues":["geneva"]}], "first_queries":["when built","where located"]}`}, nil
- case strings.Contains(text.String(), "Sufficient Context Agent"):
- m.scaSeen++
- if m.scaSeen == 1 {
- return &runtime.ModelReply{Content: `{"is_sufficient": false, "score": 0.2, "contradictions": [], "reasoning": "missing the builder", "sub_queries": [{"missing_fact": "who built it", "search_hint": "builder", "satisfied": false}], "claims": {}}`}, nil
- }
- return &runtime.ModelReply{Content: `{"is_sufficient": true, "score": 0.9, "contradictions": [], "reasoning": "ok", "claims": {}}`}, nil
- case strings.Contains(text.String(), "Query Rewriter"):
- return &runtime.ModelReply{Content: `{"queries": [{"query": "who built it"}]}`}, nil
- default:
- return &runtime.ModelReply{Content: "Built in 1865, in Geneva."}, nil
- }
-}
-
-// stubExecutor is a deterministic ToolExecutor recording every call.
-type stubExecutor struct {
- mu sync.Mutex
- calls []string
- responses map[string]runtime.ToolOutcome
-}
-
-func newStubExecutor() *stubExecutor {
- return &stubExecutor{responses: map[string]runtime.ToolOutcome{}}
-}
-
-func (e *stubExecutor) add(name, payload string, status string) {
- e.responses[name] = runtime.ToolOutcome{
- Status: status,
- Payload: []any{payload},
- }
-}
-
-func (e *stubExecutor) Execute(ctx context.Context, name string, args map[string]any) (runtime.ToolOutcome, error) {
- e.mu.Lock()
- defer e.mu.Unlock()
- e.calls = append(e.calls, name)
- if o, ok := e.responses[name]; ok {
- return o, nil
- }
- return runtime.ToolOutcome{Status: runtime.StatusOK, Payload: []any{"{}"}}, nil
-}
-
-func newToolset(exec *stubExecutor) *runtime.Toolset {
- return &runtime.Toolset{
- ThinkingMode: "high",
- HasWebSearch: true,
- DisabledTools: map[string]bool{},
- Exec: exec,
- }
-}
-
-// countingRetriever is a no-op retriever recording the requested top-N. It
-// satisfies runtime.Retriever (= tools.Retriever).
-type countingRetriever struct {
- mu sync.Mutex
- topN int
-}
-
-func (r *countingRetriever) Retrieve(_ context.Context, req runtime.RetrieveRequest) ([]map[string]any, error) {
- r.mu.Lock()
- r.topN = req.TopN
- r.mu.Unlock()
- return []map[string]any{{"doc_id": "d1", "docnm_kwd": "doc1", "content": "Saint Lawrence River; 14 April 1865; 9 December 2019."}}, nil
-}
-
-// Unit tests for the routing / helpers (no init() required).
-
-func TestRouteSCAAlwaysCloseoutOnNoProgress(t *testing.T) {
- // NoProgress is the hard stop: regardless of SCA enablement or verdict, the
- // loop must close out rather than spin another round.
- st := &AgenticState{Verdict: VerdictInsufficient, MaxLoops: 3, NoProgress: true}
- if n := routeSCA(st, true, 3); n != nodeFormalizeAnswer {
- t.Fatalf("SCA on: expected formalize on no-progress, got %v", n)
- }
- if n := routeSCA(st, false, 3); n != nodeFormalizeAnswer {
- t.Fatalf("SCA off: expected formalize on no-progress, got %v", n)
- }
-}
-
-func TestRouteSCADisabledSCAClosesOut(t *testing.T) {
- // With SCA disabled the single research pass' verdict is informational; we
- // never start a rewrite round.
- st := &AgenticState{Verdict: VerdictInsufficient, MaxLoops: 3}
- if n := routeSCA(st, false, 3); n != nodeFormalizeAnswer {
- t.Fatalf("expected formalize when SCA disabled, got %v", n)
- }
-}
-
-// TestRouteSCAPoolSizeIsNotTheStop pins the DECOUPLED stop: the pool's size is
-// not what ends the research loop — whether the SCA can READ anything new is, and
-// that fact is the view identity (scaNode sets NoProgress when the view matches
-// the previous review's, and routeSCA honours it on its first check).
-//
-// The old guard finalized as soon as the pool reached SCAViewCap on a
-// second-or-later review, which is a different claim: a pool past the view cap
-// whose view CHANGED has new evidence the reviewer can read, and it was being
-// thrown away (measured: a name-probe round ended at 117 chunks with members
-// still arriving).
-func TestRouteSCAPoolSizeIsNotTheStop(t *testing.T) {
- full := func() *AgenticState {
- st := &AgenticState{
- Verdict: VerdictInsufficient,
- MaxLoops: 3,
- SearchRounds: 1,
- // Work left in the table AND evidence still arriving last round —
- // the two facts the loop is driven by (see routeSCA).
- LastRoundNew: 12,
- UnresolvedSlots: []map[string]any{{"question_clues": []string{"who else"}}},
- KB: &runtime.Kbinfos{Chunks: make([]map[string]any, SCAViewCap)},
- }
- st.Deadline = time.Now().Add(120 * time.Second)
- return st
- }
-
- // Full pool, view changed (scaNode did NOT set NoProgress) => research on.
- if n := routeSCA(full(), true, 3); n != nodeQueryRewrite {
- t.Fatalf("full pool with a changed view: expected rewrite, got %v", n)
- }
-
- // Full pool, view UNCHANGED: scaNode's detector owns this case, and routeSCA
- // honours it before anything else.
- stale := full()
- stale.NoProgress = true
- if n := routeSCA(stale, true, 3); n != nodeFormalizeAnswer {
- t.Fatalf("full pool with an unchanged view: expected formalize, got %v", n)
- }
-
- // The loop is still bounded by the round count...
- atMax := full()
- atMax.SearchRounds = 3
- if n := routeSCA(atMax, true, 3); n != nodeFormalizeAnswer {
- t.Fatalf("at max rounds with a full pool: expected formalize, got %v", n)
- }
- // ...and by the round-headroom guard.
- tight := full()
- tight.Deadline = time.Now().Add(MinRoundHeadroomS / 2 * time.Second)
- if n := routeSCA(tight, true, 3); n != nodeFormalizeAnswer {
- t.Fatalf("no headroom with a full pool: expected formalize, got %v", n)
- }
-}
-
-// TestRouteSCARoundRecordDrivesTheLoop pins the DECOUPLED stop/continue rule: the
-// loop's own record (work left in the table + evidence still arriving) decides,
-// and the reviewer's verdict is one of its inputs rather than the switch.
-//
-// The distinction it encodes: a SUFFICIENT verdict is a judgement about the
-// PASSAGES the reviewer read. It is not a statement that the question is done —
-// measured 2026-09-15, a run ended with unresolved=0 while the answer was six
-// members short — so a table that still lists unresolved slots while the round is
-// still adding evidence keeps its next round, and a settled table closes out
-// however much evidence arrived.
-func TestRouteSCARoundRecordDrivesTheLoop(t *testing.T) {
- base := func() *AgenticState {
- st := &AgenticState{MaxLoops: 3, SearchRounds: 1}
- st.Deadline = time.Now().Add(120 * time.Second)
- return st
- }
- work := []map[string]any{{"question_clues": []string{"who else"}}}
-
- // Work left + still learning + a SATISFIED reviewer: another round. The
- // reviewer's view of the passages does not settle the table's own gaps.
- st := base()
- st.Verdict = VerdictSufficient
- st.LastRoundNew = 9
- st.UnresolvedSlots = work
- if n := routeSCA(st, true, 3); n != nodeQueryRewrite {
- t.Errorf("SUFFICIENT with work left and evidence still arriving: expected rewrite, got %v", n)
- }
-
- // Nothing left in the table: growth alone is not a reason to keep going.
- st = base()
- st.Verdict = VerdictSufficient
- st.LastRoundNew = 9
- if n := routeSCA(st, true, 3); n != nodeFormalizeAnswer {
- t.Errorf("settled table with nothing against it: expected formalize, got %v", n)
- }
-
- // Work left, stalled round, reviewer names a gap: another round.
- st = base()
- st.Verdict = VerdictInsufficient
- st.UnresolvedSlots = work
- if n := routeSCA(st, true, 3); n != nodeQueryRewrite {
- t.Errorf("INSUFFICIENT with a concrete gap: expected rewrite, got %v", n)
- }
-
- // Work can ALSO come from the review's own gaps, with an empty unresolved
- // table — the two records are independent. Ignoring the reviewer's gaps is
- // what made a run close out with INSUFFICIENT (confidence 1.00) and four
- // derived gaps while 60s and two rounds went unspent (2026-09-15).
- st = base()
- st.Verdict = VerdictInsufficient
- st.SCA = map[string]any{"claims": map[string]any{
- "c1": map[string]any{"missing_information": []any{
- map[string]any{"what": "还有谁", "search_hint": "关羽 斩将 名单"},
- }},
- }}
- if n := routeSCA(st, true, 3); n != nodeQueryRewrite {
- t.Errorf("INSUFFICIENT with the review's own gaps and no unresolved slots: expected rewrite, got %v", n)
- }
-
- // Work left, stalled round, NO judgement (the review could not run): nothing
- // names a direction and the round learned nothing — close out. This is the
- // case a fallback INSUFFICIENT used to promise a round to.
- st = base()
- st.Verdict = VerdictUnknown
- st.UnresolvedSlots = work
- if n := routeSCA(st, true, 3); n != nodeFormalizeAnswer {
- t.Errorf("UNKNOWN with a stalled round: expected formalize, got %v", n)
- }
-
- // Work left, stalled round, satisfied reviewer: close out.
- st = base()
- st.Verdict = VerdictSufficient
- st.UnresolvedSlots = work
- if n := routeSCA(st, true, 3); n != nodeFormalizeAnswer {
- t.Errorf("SUFFICIENT with a stalled round: expected formalize, got %v", n)
- }
-
- // At max rounds => formalize, whatever the record says.
- st = base()
- st.Verdict = VerdictInsufficient
- st.LastRoundNew = 3
- st.UnresolvedSlots = work
- st.SearchRounds = 3
- if n := routeSCA(st, true, 3); n != nodeFormalizeAnswer {
- t.Errorf("at max rounds: expected formalize, got %v", n)
- }
- // No headroom => formalize.
- st = base()
- st.Verdict = VerdictInsufficient
- st.LastRoundNew = 3
- st.UnresolvedSlots = work
- st.Deadline = time.Now().Add(MinRoundHeadroomS / 2 * time.Second)
- if n := routeSCA(st, true, 3); n != nodeFormalizeAnswer {
- t.Errorf("no headroom: expected formalize, got %v", n)
- }
-}
-
-func TestSelectSCAViewIdentity(t *testing.T) {
- chunks := make([]map[string]any, 0, 80)
- for i := 0; i < 80; i++ {
- chunks = append(chunks, map[string]any{
- "chunk_id": fmt.Sprintf("c%d", i),
- "content": fmt.Sprintf("the quick brown fox %d", i),
- "similarity": 1.0 - float64(i)/100.0,
- })
- }
- v1, id1 := SelectSCAView(chunks, []string{"quick", "brown"})
- if len(v1) != SCAViewCap {
- t.Fatalf("expected view cap %d, got %d", SCAViewCap, len(v1))
- }
- // Same inputs => same identity (no Process-randomised hash).
- _, id2 := SelectSCAView(chunks, []string{"quick", "brown"})
- if id1 != id2 {
- t.Fatalf("view identity should be deterministic, got %q vs %q", id1, id2)
- }
- // An enlarged pool with the SAME selected ids keeps the same identity — this
- // is exactly the "review view unchanged" early-exit signal.
- more := append(chunks, map[string]any{"chunk_id": "extra", "content": "unrelated tail", "similarity": 0.0})
- _, id3 := SelectSCAView(more, []string{"quick", "brown"})
- if id1 != id3 {
- t.Fatalf("adding non-selected chunks should NOT change the view identity, got %q vs %q", id1, id3)
- }
-}
-
-func TestMergeSlotPatchAdoptsStronger(t *testing.T) {
- strong := 0.95
- base := runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "aspect", Candidate: strPtr("base"), CandidateStrength: &strong},
- }, 0, nil)
- // A weak branch (0.30) must NOT downgrade the strong base.
- weak := 0.30
- branch := runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "aspect", Candidate: strPtr("weak-candidate"), CandidateStrength: &weak},
- }, 1, nil)
- merged := MergeSlotPatch(base, branch)
- // A weak branch does not downgrade the strong base. It is no longer a no-op
- // either: the claim that lost the comparison is kept as an alternate, so the
- // list two sessions disagreed about does not vanish (see
- // TestMergeSlotPatchKeepsTheLosingListAsAnAlternate).
- if merged != nil {
- if got := merged.ByID(0).Candidate; got == nil || *got != "base" {
- t.Fatalf("weak branch upgraded the slot: %v", got)
- }
- if alts := alternateCandidatesOf(*merged.ByID(0)); len(alts) != 1 || alts[0] != "weak-candidate" {
- t.Fatalf("alternates = %v, want the weak claim kept as the loser", alts)
- }
- }
-
- // A stronger branch wins.
- stronger := 0.99
- branch2 := runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "aspect", Candidate: strPtr("strong-candidate"), CandidateStrength: &stronger},
- }, 1, nil)
- merged2 := MergeSlotPatch(base, branch2)
- if merged2 == nil {
- t.Fatal("expected a merge result")
- }
- if got := merged2.State[0].Candidate; got == nil || *got != "strong-candidate" {
- t.Fatalf("strong branch should upgrade, got %v", got)
- }
-}
-
-func TestMergeSlotPatchNoChangeReturnsNil(t *testing.T) {
- base := runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "aspect", Candidate: strPtr("same")},
- }, 0, nil)
- branch := runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "aspect", Candidate: strPtr("same")},
- }, 1, nil)
- if m := MergeSlotPatch(base, branch); m != nil {
- t.Fatalf("identical patch should return nil, got %+v", m)
- }
-}
-
-func TestExpandFanoutsParsesJSON(t *testing.T) {
- mdl := &scriptedModel{}
- mdl.push(`{"fanouts": ["who discovered it", "when was it discovered"]}`)
- got := ExpandFanouts(context.Background(), RAGTools{Model: mdl}, "What was discovered and by whom?")
- if len(got) != 2 {
- t.Fatalf("expected 2 fan-outs, got %v", got)
- }
- if got[0] != "who discovered it" || got[1] != "when was it discovered" {
- t.Fatalf("unexpected fan-outs: %v", got)
- }
-}
-
-func TestExpandFanoutsFallbackOnBadJSON(t *testing.T) {
- mdl := &scriptedModel{}
- mdl.push("I cannot break this down.")
- got := ExpandFanouts(context.Background(), RAGTools{Model: mdl}, "single question")
- // Bad JSON: the model answered in prose, so the loop line-splits the reply (yielding
- // the reply itself, not the raw question). The loose path strips the
- // "-•0123456789. " cutset from both ends, so the trailing period is dropped.
- if len(got) != 1 || got[0] != "I cannot break this down" {
- t.Fatalf("expected line-split fallback without the trailing period, got %v", got)
- }
-}
-
-func TestExpandFanoutsNilModelReturnsQuestion(t *testing.T) {
- got := ExpandFanouts(context.Background(), RAGTools{}, "single question")
- if len(got) != 1 || got[0] != "single question" {
- t.Fatalf("expected raw question when no model, got %v", got)
- }
-}
-
-// TestExpandFanoutsRetriesWithStrictJSON covers the retry: a prose answer must not be
-// line-split into fan-outs (it poisons the slot table), so the prompt is re-sent with the
-// strict JSON instruction instead.
-func TestExpandFanoutsRetriesWithStrictJSON(t *testing.T) {
- mdl := &scriptedModel{}
- mdl.push("The woman was **Rocio Restrepo**.\nSupporting sources: https://example.com")
- mdl.push(`{"fanouts": ["who was she", "when did it happen"]}`)
- got := ExpandFanouts(context.Background(), RAGTools{Model: mdl}, "Who was she?")
- if len(got) != 2 || got[0] != "who was she" || got[1] != "when did it happen" {
- t.Fatalf("expected the strict-retry fan-outs, got %v", got)
- }
- if len(mdl.seen) != 2 {
- t.Fatalf("model calls = %d, want 2 (original + one strict retry)", len(mdl.seen))
- }
-}
-
-// TestExpandFanoutsDropsAnswerLikeEntries covers _fanout_looks_like_query: a
-// fan-out is used verbatim as a retrieval query and as the slot hint, so an
-// answered fact must never survive into either.
-func TestExpandFanoutsDropsAnswerLikeEntries(t *testing.T) {
- mdl := &scriptedModel{}
- mdl.push(`{"fanouts": ["who won the medal", "The winner was **Ann** according to the citation", "when was the race"]}`)
- got := ExpandFanouts(context.Background(), RAGTools{Model: mdl}, "Who won the medal and when?")
- if len(got) != 2 || got[0] != "who won the medal" || got[1] != "when was the race" {
- t.Fatalf("expected the answer-like entry to be dropped, got %v", got)
- }
-}
-
-func TestFanoutLooksLikeQueryBounds(t *testing.T) {
- // The loose (non-JSON) path holds a tighter bound than the JSON path.
- long := "who opened the new library downtown last spring and who paid for the building"
- if !fanoutLooksLikeQuery(long, false) {
- t.Errorf("JSON-path fan-out %q should pass the %d-word bound", long, fanoutMaxWords)
- }
- if fanoutLooksLikeQuery(long, true) {
- t.Errorf("loose fan-out %q should exceed the %d-word bound", long, fanoutLooseMaxWords)
- }
- // A question keeps its full length even on the loose path.
- if !fanoutLooksLikeQuery(long+"?", true) {
- t.Error("a question-shaped line should survive the loose bound")
- }
- if fanoutLooksLikeQuery(strings.Repeat("word ", fanoutMaxWords+2), false) {
- t.Errorf("fan-outs over %d words must be rejected", fanoutMaxWords)
- }
- if fanoutLooksLikeQuery(strings.Repeat("a", fanoutMaxChars+1), false) {
- t.Errorf("fan-outs over %d chars must be rejected", fanoutMaxChars)
- }
-}
-
-// TestFanoutLooksLikeQueryCountsRunes: the 160-char cap counts code points, so a CJK
-// fan-out at the limit must survive even though it exceeds 160 BYTES, and one over the
-// limit must still be rejected.
-func TestFanoutLooksLikeQueryCountsRunes(t *testing.T) {
- atLimit := strings.Repeat("中", fanoutMaxChars) // 160 chars / 480 bytes
- if !fanoutLooksLikeQuery(atLimit, false) {
- t.Errorf("a %d-character fan-out must pass: the cap counts characters, not bytes", fanoutMaxChars)
- }
- if fanoutLooksLikeQuery(atLimit+"中", false) {
- t.Errorf("a %d-character fan-out must be rejected", fanoutMaxChars+1)
- }
-}
-
-// TestParseFanoutsLooseStripsBothEnds: the bullet/numbering cutset is stripped from BOTH
-// ends, so a trailing period is not carried into the retrieval query (the leading "1."
-// already was not).
-func TestParseFanoutsLooseStripsBothEnds(t *testing.T) {
- got := parseFanouts("1. who opened the library.\n- when did it open?")
- want := []string{"who opened the library", "when did it open?"}
- if len(got) != len(want) {
- t.Fatalf("parseFanouts = %q, want %q", got, want)
- }
- for i := range want {
- if got[i] != want[i] {
- t.Errorf("parseFanouts[%d] = %q, want %q", i, got[i], want[i])
- }
- }
-}
-
-// TestParseFanoutsSplitsOnAllLineBreaks: a carriage-return separated reply yields one
-// fan-out per line, not a single fused blob (splitting on "\n" alone would keep the lone
-// "\r" inline).
-func TestParseFanoutsSplitsOnAllLineBreaks(t *testing.T) {
- got := parseFanouts("who opened it\rwhen did it open?")
- want := []string{"who opened it", "when did it open?"}
- if len(got) != len(want) {
- t.Fatalf("parseFanouts = %q, want %q", got, want)
- }
- for i := range want {
- if got[i] != want[i] {
- t.Errorf("parseFanouts[%d] = %q, want %q", i, got[i], want[i])
- }
- }
-}
-
-// TestRenderSlotDraftFormat pins the draft format exactly: the collected answer leads
-// with its evidence metadata, a resolved slot carries strength + terminal + evidence ids +
-// its discovered-clue tail, and an unresolved slot is rendered as
-// "NOT RESOLVED (<question clues>)". This text is the SCA's claim context, so a different
-// shape changes what the reviewer sees.
-func TestRenderSlotDraftFormat(t *testing.T) {
- strong := 0.9
- st := runtime.NewState([]runtime.Variable{
- {
- ID: 0,
- Type: "aspect",
- Candidate: strPtr("answer A"),
- CandidateStrength: &strong,
- DiscoveredClues: []string{"clue one", "clue two"},
- },
- {ID: 1, Type: "aspect", QuestionClues: []string{"who opened it?"}},
- }, 0, nil)
- evidence := map[string]SlotEvidence{
- "0": {EvidenceIDs: []string{"c1", "c2"}, TerminalType: "state"},
- }
-
- draft := RenderSlotDraft(st, "the collected answer", evidence)
- want := "Candidate answer: the collected answer\n" +
- "\n" +
- "- slot 0 [aspect]: answer A (strength=0.90) [terminal=state, evidence_ids=['c1', 'c2']] — clue one; clue two\n" +
- "- slot 1 [aspect]: NOT RESOLVED (who opened it?)"
- if draft != want {
- t.Fatalf("draft = %q\nwant %q", draft, want)
- }
-}
-
-// TestSelectSCAViewPrefersScoreWhenSimilarityIsZero: a similarity of 0.0 is FALSY, so the
-// score must be used instead. The presence-based read ranked such a chunk last instead of
-// first.
-func TestSelectSCAViewPrefersScoreWhenSimilarityIsZero(t *testing.T) {
- chunks := []map[string]any{
- {"chunk_id": "zero-sim", "content": "irrelevant", "similarity": 0.0, "score": 0.9},
- {"chunk_id": "half", "content": "irrelevant", "similarity": 0.5},
- }
- view, _ := SelectSCAView(chunks, nil)
- if len(view) != 2 {
- t.Fatalf("view size = %d, want 2", len(view))
- }
- if got := runtime.ChunkIDOf(view[0]); got != "zero-sim" {
- t.Errorf("first chunk = %q, want zero-sim (score 0.9 must beat similarity 0.5)", got)
- }
-}
-
-// TestComposeFallbackDraftPrompt pins the draft prompt byte for byte: the FOUND/MISSING
-// instructions, the "Retrieved evidence:" label, the "\n"-joined numbered snippets, and
-// the language clause appended without a separator.
-func TestComposeFallbackDraftPrompt(t *testing.T) {
- mdl := &fakeModel{replies: []*runtime.ModelReply{{Content: "FOUND: x\nMISSING: none"}}}
- st := &AgenticState{
- Question: "谁开的?",
- KB: &runtime.Kbinfos{Chunks: []map[string]any{{"content_with_weight": "证据一"}}},
- }
- if got := ComposeFallbackDraft(context.Background(), RAGTools{Model: mdl}, st); got != "FOUND: x\nMISSING: none" {
- t.Fatalf("draft = %q", got)
- }
- if len(mdl.messages) != 2 {
- t.Fatalf("model received %d message(s), want system+user", len(mdl.messages))
- }
- wantSystem := "You are a research assistant writing an INTERMEDIATE DRAFT toward answering the user's " +
- "question, using ONLY the retrieved evidence snippets below.\n" +
- "Requirements:\n" +
- "1. First list concrete FACTS FOUND in the snippets (exact numbers, dates, names preserved).\n" +
- "2. Then output a line starting with 'MISSING:' naming precisely which part(s) of the " +
- "question the snippets do NOT answer yet.\n" +
- "3. No conclusions beyond the evidence; no general knowledge.\n" +
- "Keep it under 250 words." +
- "Write your draft in the same language as the question."
- if got := mdl.messages[0].Content; got != wantSystem {
- t.Errorf("system prompt = %q\nwant %q", got, wantSystem)
- }
- wantUser := "Question: 谁开的?\n\nRetrieved evidence:\n[1] 证据一"
- if got := mdl.messages[1].Content; got != wantUser {
- t.Errorf("user prompt = %q\nwant %q", got, wantUser)
- }
-}
-
-// TestBuildSlotTableKeepsAllFanoutsOnFailure pins the failure path: when
-// initialize_state fails, first_queries is the FULL fan-out list — the [:3] cap only
-// applies to the empty-root path. Losing the list made the first prefetch narrower.
-func TestBuildSlotTableKeepsAllFanoutsOnFailure(t *testing.T) {
- fanouts := []string{"q1", "q2", "q3", "q4", "q5"}
- // No model → initialize_state cannot run, so the failure path is taken.
- root, first := BuildSlotTable(context.Background(), runtime.SessionDeps{}, "raw question", fanouts, 60)
- if len(root.State) != 4 {
- t.Fatalf("fallback slots = %d, want 4 (queries[:4])", len(root.State))
- }
- if len(first) != len(fanouts) {
- t.Errorf("firstQueries = %v, want the full fan-out list on the failure path", first)
- }
-}
-
-// TestBuildSlotTableFallsBackOnSpentBudget pins the planner's deadline: the value is NOT
-// floored, so once the round budget is spent the decomposition times out at once and the
-// table falls back to the planner fan-outs.
-func TestBuildSlotTableFallsBackOnSpentBudget(t *testing.T) {
- mdl := &fakeModel{replies: []*runtime.ModelReply{
- {Content: `{"slots":[{"id":0,"type":"aspect","clues":["a"]}]}`},
- }}
- fanouts := []string{"f1", "f2"}
- root, first := BuildSlotTable(context.Background(), runtime.SessionDeps{Model: mdl}, "q", fanouts, -15)
- if len(root.State) != len(fanouts) {
- t.Fatalf("slots = %d, want the %d fan-out fallback slots", len(root.State), len(fanouts))
- }
- if mdl.calls != 0 {
- t.Errorf("model called %d time(s); a spent budget must not reach the provider", mdl.calls)
- }
- if len(first) != len(fanouts) {
- t.Errorf("firstQueries = %v, want %v", first, fanouts)
- }
-}
-
-// TestBuildSlotTableFallsBackWhenIDIsNotAnInteger covers the end-to-end effect of the
-// strict parser: a non-integer id fails the parse and the table is rebuilt from the
-// planner fan-outs.
-func TestBuildSlotTableFallsBackWhenIDIsNotAnInteger(t *testing.T) {
- fanouts := []string{"q1", "q2"}
- deps := runtime.SessionDeps{Model: &fakeModel{replies: []*runtime.ModelReply{
- {Content: `{"slots":[{"id":"abc","type":"aspect","clues":["a"]}]}`},
- }}}
- root, first := BuildSlotTable(context.Background(), deps, "raw question", fanouts, 60)
- if len(root.State) != len(fanouts) {
- t.Fatalf("slots = %d, want the %d fan-out fallback slots", len(root.State), len(fanouts))
- }
- for i, v := range root.State {
- if v.Type != "aspect" || len(v.QuestionClues) != 1 || v.QuestionClues[0] != fanouts[i] {
- t.Errorf("slot %d = %+v, want aspect/%s", i, v, fanouts[i])
- }
- }
- if len(first) != len(fanouts) {
- t.Errorf("firstQueries = %v, want %v", first, fanouts)
- }
-}
-
-// sessionStubExec answers every tool call with a hit, so a session runs its turns
-// without a real retriever.
-type sessionStubExec struct{}
-
-func (sessionStubExec) Execute(_ context.Context, name string, _ map[string]any) (runtime.ToolOutcome, error) {
- return runtime.ToolOutcome{
- Status: runtime.StatusOK,
- Payload: []any{map[string]any{"kind": name, "content": "hit"}},
- EvidenceIDs: []string{"c-" + name},
- }, nil
-}
-
-// TestRunSlotResearchPassLedgerRecordsSessionHint drives a real round whose
-// session produces no terminal answer, and asserts the ledger row still carries the
-// QUERY of the pair (query, outcome): the direction this session was sent on.
-//
-// The row used to carry a langchain-style repr of the session's messages instead, and
-// that repr is a CONSTANT in production: every session's history opens with the same
-// system prompt, so its first 80 runes went into every row of every question.
-func TestRunSlotResearchPassLedgerRecordsSessionHint(t *testing.T) {
- st := &AgenticState{
- Question: "who opened it?",
- SlotTable: runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "aspect", QuestionClues: []string{"who opened it?"}},
- }, 0, nil),
- }
- deps := runtime.SessionDeps{
- Model: &fakeModel{replies: []*runtime.ModelReply{
- {Content: "I could not find any evidence about the opening date in the retrieved passages."},
- }},
- Tools: &runtime.Toolset{Exec: sessionStubExec{}},
- KB: &runtime.Kbinfos{},
- }
- res := RunSlotResearchPass(context.Background(), context.Background(), deps, "who opened it?", st, 60)
- if res == nil {
- t.Fatal("RunSlotResearchPass returned nil")
- }
- if len(res.Attempted) == 0 {
- t.Fatal("no ledger entry recorded")
- }
- q, _ := res.Attempted[len(res.Attempted)-1]["q"].(string)
- if q != "who opened it?" {
- t.Errorf("ledger q = %q, want the direction this session was sent on", q)
- }
- if strings.HasPrefix(q, "[SystemMessage(") {
- t.Error("ledger q carries a message repr — that is the system prompt, not a query")
- }
-}
-
-// TestRunSlotResearchPassLogsSlotEvidenceBound pins: the round
-// reports how many passages each slot's session bound, so a slot whose session
-// retrieved nothing is visible in the run log.
-func TestRunSlotResearchPassLogsSlotEvidenceBound(t *testing.T) {
- var buf bytes.Buffer
- orig := _LOG
- _LOG = log.New(&buf, "", 0)
- defer func() { _LOG = orig }()
-
- st := &AgenticState{
- Question: "who opened it?",
- SlotTable: runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "aspect", QuestionClues: []string{"who opened it?"}},
- }, 0, nil),
- }
- deps := runtime.SessionDeps{
- Model: &fakeModel{replies: []*runtime.ModelReply{{
- Content: "Let me look that up.",
- ToolCalls: []runtime.ToolCall{
- {ID: "call_0", Name: "retrieve", Args: map[string]any{"query": "who opened it?"}},
- },
- }}},
- Tools: &runtime.Toolset{Exec: sessionStubExec{}},
- KB: &runtime.Kbinfos{},
- }
- if res := RunSlotResearchPass(context.Background(), context.Background(), deps, "who opened it?", st, 60); res == nil {
- t.Fatal("RunSlotResearchPass returned nil")
- }
- if !strings.Contains(buf.String(), "slot evidence bound") {
- t.Errorf("missing the per-slot evidence report; log:\n%s", buf.String())
- }
-}
-
-// channelRetriever records the channel each retrieve call belongs to: the exact
-// leg is keyword-only (weight 0, top_n=60, query terms folded into the query),
-// the semantic-bypass leg has the vector weight on (top_n=30, plain query).
-type channelRetriever struct {
- keywordsWeights []float64
- topN []int
- queries []string
-}
-
-func (r *channelRetriever) Retrieve(_ context.Context, req runtime.RetrieveRequest) ([]map[string]any, error) {
- // Each leg names its keyword weight explicitly; nil (the control was not supplied)
- // is recorded as -1 so the assertions below catch a leg that drops it.
- weight := -1.0
- if req.KeywordsSimilarityWeight != nil {
- weight = *req.KeywordsSimilarityWeight
- }
- r.keywordsWeights = append(r.keywordsWeights, weight)
- r.topN = append(r.topN, req.TopN)
- r.queries = append(r.queries, req.Query)
- return nil, nil
-}
-
-// TestFanoutSearchIsDualChannel pins the property the single-channel
-// implementation could not express: each fan-out issues BOTH a keyword BM25 leg
-// (narrowed, wide pool) and a dense semantic leg (narrowing BYPASSED, top_n=30).
-// One hybrid call per query cannot distinguish exact from paraphrase-only hits,
-// so it cannot apply the semantic quota that keeps the bypass low-precision hits
-// from displacing exact ones.
-func TestFanoutSearchIsDualChannel(t *testing.T) {
- ctx := context.Background()
- r := &channelRetriever{}
- deps := RAGTools{Search: runtime.SearchDeps{Backend: r, KbIDs: []string{"kb1"}, HasEmbedder: true}}
- st := &AgenticState{KB: &runtime.Kbinfos{}}
- FanoutSearch(ctx, deps, st, []string{"when was it built", "where located"}, 8, 60, false)
-
- // Two fan-outs x two channels.
- if len(r.keywordsWeights) != 4 {
- t.Fatalf("expected 4 retrieve calls (2 fan-outs x 2 channels), got %d", len(r.keywordsWeights))
- }
- for i := 0; i < 4; i += 2 {
- // Channel A: keyword-only weight, wide pool, query terms folded in.
- if r.keywordsWeights[i] != 1 {
- t.Errorf("fan-out %d channel A: keyword weight = %v, want 1 (BM25 keyword leg)", i/2, r.keywordsWeights[i])
- }
- if r.topN[i] != fanoutBM25TopN {
- t.Errorf("fan-out %d channel A: TopN = %d, want %d", i/2, r.topN[i], fanoutBM25TopN)
- }
- if !strings.Contains(r.queries[i], "built") && !strings.Contains(r.queries[i], "located") {
- t.Errorf("fan-out %d channel A: query = %q, want the fan-out query (terms folded in)", i/2, r.queries[i])
- }
- // Channel B: vector weight on, top_n=30 (narrowing bypassed).
- if r.keywordsWeights[i+1] >= 1 {
- t.Errorf("fan-out %d channel B: keyword weight = %v, want < 1 (semantic bypass leg)", i/2, r.keywordsWeights[i+1])
- }
- if r.topN[i+1] != fanoutHybridTopN {
- t.Errorf("fan-out %d channel B: TopN = %d, want %d", i/2, r.topN[i+1], fanoutHybridTopN)
- }
- }
-}
-
-// TestFanoutSearchPrefersExactOverSemantic pins the admission order across
-// FAN-OUTS: every fan-out's exact channel is admitted before any fan-out's
-// semantic channel. With room for only two snippets and two fan-outs each
-// offering one exact and one semantic hit, the two exact hits must win — a
-// single merged channel could not express this preference at all.
-func TestFanoutSearchPrefersExactOverSemantic(t *testing.T) {
- ctx := context.Background()
- r := &fanoutHitRetriever{}
- deps := RAGTools{Search: runtime.SearchDeps{Backend: r, KbIDs: []string{"kb1"}, HasEmbedder: true}}
- st := &AgenticState{KB: &runtime.Kbinfos{}}
- added := FanoutSearch(ctx, deps, st, []string{"alpha", "beta"}, 8, 2, false)
-
- if added != 2 {
- t.Fatalf("added = %d, want 2 (the pool's remaining room)", added)
- }
- ids := make([]string, 0, len(st.KB.Chunks))
- for _, c := range st.KB.Chunks {
- ids = append(ids, runtime.ChunkIDOf(c))
- }
- if len(ids) != 2 || ids[0] != "ex-alpha" || ids[1] != "ex-beta" {
- t.Errorf("admitted %v, want the two exact-leg hits [ex-alpha ex-beta]", ids)
- }
-}
-
-// fanoutHitRetriever returns one exact hit and one semantic hit per fan-out,
-// distinguished by the leg's top_n (30 = semantic bypass, 60 = exact BM25).
-type fanoutHitRetriever struct{}
-
-func (r *fanoutHitRetriever) Retrieve(_ context.Context, req runtime.RetrieveRequest) ([]map[string]any, error) {
- which := "beta"
- if strings.Contains(req.Query, "alpha") {
- which = "alpha"
- }
- if req.TopN == fanoutHybridTopN {
- return []map[string]any{
- {"chunk_id": "sem-" + which, "content": "paraphrase only", "doc_name": "semantic"},
- }, nil
- }
- return []map[string]any{
- {"chunk_id": "ex-" + which, "content": which + " term", "doc_name": "exact"},
- }, nil
-}
-
-// claimTopUpEngine stands in for the doc engine across the three store reads
-// the channel-0 path makes: the has-compilation probe, the claim recall legs,
-// and the directed source-chunk fetch.
-type claimTopUpEngine struct {
- engine.DocEngine
-}
-
-func (e *claimTopUpEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
- if _, ok := req.Filter["compile_kwd"]; ok {
- // DatasetHasCompilation probe: the dataset IS compiled.
- return &types.SearchResult{Chunks: []map[string]interface{}{{"id": "row"}}}, nil
- }
- if ids, ok := req.Filter["id"].([]string); ok {
- // LoadChunksForIDs: the source chunks behind the admitted claims.
- rows := make([]map[string]interface{}, 0, len(ids))
- for _, id := range ids {
- rows = append(rows, map[string]interface{}{
- "id": id,
- "content_with_weight": "full text of " + id,
- "doc_id": "doc-1",
- })
- }
- return &types.SearchResult{Chunks: rows}, nil
- }
- // Claim recall leg: one claim citing src-1.
- payload, _ := json.Marshal(map[string]any{
- "name": "The tower is 330m tall",
- "description": "height",
- "evidence": []any{map[string]any{"quote": "330 metres"}},
- "source_chunk_ids": []string{"src-1"},
- })
- return &types.SearchResult{Chunks: []map[string]interface{}{{
- "content_with_weight": string(payload),
- "source_chunk_ids": []string{"src-1"},
- "doc_id": "doc-1",
- "similarity": 0.9,
- }}}, nil
-}
-
-// TestFanoutSearchEvidenceTopUp pins the channel-0 directional top-up: after a claim
-// pseudo chunk is admitted, the source chunk it cites is fetched by id and admitted too —
-// the verbatim quote alone does not carry the surrounding passage.
-func TestFanoutSearchEvidenceTopUp(t *testing.T) {
- r := &channelRetriever{}
- de := &claimTopUpEngine{}
- // Unique tenant per run: the runtime claim caches are process-global with a
- // 300s TTL, and this test must pay the store reads itself.
- tenant := fmt.Sprintf("tenant-topup-%d", time.Now().UnixNano())
- deps := RAGTools{Search: runtime.SearchDeps{
- Backend: r,
- DocEngine: de,
- TenantID: tenant,
- KbIDs: []string{"kb-1"},
- }}
- st := &AgenticState{KB: &runtime.Kbinfos{}}
- added := FanoutSearch(context.Background(), deps, st, []string{"tower height topup"}, 8, 60, false)
- if added != 2 {
- t.Fatalf("added = %d, want 2 (the claim row + its source chunk)", added)
- }
- var claimEntry, srcEntry map[string]any
- for _, c := range st.KB.Chunks {
- id, _ := c["chunk_id"].(string)
- switch {
- case strings.HasPrefix(id, "claim_"):
- claimEntry = c
- case id == "src-1":
- srcEntry = c
- }
- }
- if claimEntry == nil {
- t.Fatalf("claim pseudo chunk not admitted; pool = %v", st.KB.Chunks)
- }
- if srcEntry == nil {
- t.Fatalf("source chunk behind the claim was not top-upped; pool = %v", st.KB.Chunks)
- }
- if got, _ := srcEntry["content_with_weight"].(string); got != "full text of src-1" {
- t.Errorf("top-up content = %q, want the fetched source chunk text", got)
- }
-}
-
-// draftModel returns a canned draft and records the prompt it was given.
-type draftModel struct {
- reply string
- err error
- lastUser string
- lastSys string
- calls int
-}
-
-func (m *draftModel) Complete(_ context.Context, msgs []schema.Message, _ []runtime.ToolSpec) (*runtime.ModelReply, error) {
- m.calls++
- for _, msg := range msgs {
- switch msg.Role {
- case schema.System:
- m.lastSys = msg.Content
- case schema.User:
- m.lastUser = msg.Content
- }
- }
- if m.err != nil {
- return nil, m.err
- }
- return &runtime.ModelReply{Content: m.reply}, nil
-}
-
-// TestComposeFallbackDraftUsesLLM pins that the fallback draft is SYNTHESISED,
-// not concatenated. The draft feeds PreSummary as "Research findings
-// (authoritative)" and is what the SCA reviews, so its shape matters: it must
-// ask for a MISSING: line, and a plain list of truncated snippets cannot express
-// one.
-func TestComposeFallbackDraftUsesLLM(t *testing.T) {
- st := &AgenticState{
- Question: "When was it built?",
- KB: &runtime.Kbinfos{Chunks: []map[string]any{{"content": "It was built in 1874."}}},
- }
- mdl := &draftModel{reply: "Built in 1874.\nMISSING: architect"}
- got := ComposeFallbackDraft(context.Background(), RAGTools{Model: mdl}, st)
-
- if mdl.calls != 1 {
- t.Fatalf("model calls = %d, want 1 (the draft is synthesised)", mdl.calls)
- }
- if got != "Built in 1874.\nMISSING: architect" {
- t.Fatalf("draft = %q, want the model's synthesis", got)
- }
- // The system prompt must demand the MISSING line — that is the whole point.
- if !strings.Contains(mdl.lastSys, "MISSING:") {
- t.Errorf("system = %q, want it to require a MISSING: line", mdl.lastSys)
- }
- // numbered "[i]" evidence, 1-indexed.
- if !strings.Contains(mdl.lastUser, "[1] It was built in 1874.") {
- t.Errorf("user = %q, want '[1] <content>' evidence", mdl.lastUser)
- }
-}
-
-// TestComposeFallbackDraftFallsBackToEvidence pins -1486 and
-// 1512-1514: with no model, or when the call fails, the draft degrades to the
-// raw evidence capped at 4000 chars.
-func TestComposeFallbackDraftFallsBackToEvidence(t *testing.T) {
- st := &AgenticState{
- Question: "When?",
- KB: &runtime.Kbinfos{Chunks: []map[string]any{{"content": "Evidence body."}}},
- }
- if got := ComposeFallbackDraft(context.Background(), RAGTools{}, st); !strings.Contains(got, "Evidence body.") {
- t.Errorf("no-model draft = %q, want the raw evidence", got)
- }
- failing := &draftModel{err: errors.New("boom")}
- if got := ComposeFallbackDraft(context.Background(), RAGTools{Model: failing}, st); !strings.Contains(got, "Evidence body.") {
- t.Errorf("failed-call draft = %q, want the raw evidence", got)
- }
-}
-
-// TestComposeFallbackDraftOrdersByRelevance pins: the fixed 16-slot
-// budget is spent on the STRONGEST evidence, not on insertion order.
-func TestComposeFallbackDraftOrdersByRelevance(t *testing.T) {
- st := &AgenticState{
- Question: "When?",
- KB: &runtime.Kbinfos{Chunks: []map[string]any{
- {"content": "weak hit", "similarity": 0.1},
- {"content": "strong hit", "similarity": 0.9},
- }},
- }
- mdl := &draftModel{reply: "draft"}
- ComposeFallbackDraft(context.Background(), RAGTools{Model: mdl}, st)
-
- iWeak := strings.Index(mdl.lastUser, "weak hit")
- iStrong := strings.Index(mdl.lastUser, "strong hit")
- if iWeak < 0 || iStrong < 0 {
- t.Fatalf("evidence missing from prompt: %q", mdl.lastUser)
- }
- if iStrong > iWeak {
- t.Errorf("order = weak@%d strong@%d, want strongest evidence first", iWeak, iStrong)
- }
-}
-
-// TestRagCollectsPerPhaseUsage pins the wiring that makes per-phase LLM usage
-// measurable: the collector must be built and the chat model wrapped, otherwise the
-// counting machinery stays inert — CurrentStats(ctx) is nil, so the phase markers already
-// in the graph record nothing and even the explicit deps.Stats.RecordCall sites are
-// skipped by their nil guard.
-func TestRagCollectsPerPhaseUsage(t *testing.T) {
- mdl := &countingModel{}
- Rag(context.Background(), RAGTools{
- Retriever: &corpusRetriever{},
- Model: mdl,
- }, runtime.RunRequest{
- Question: "Who created Culdcept?",
- ThinkingMode: "high",
- DatasetIDs: []string{"kb1"},
- })
-
- // The decisive assertion: Rag must bind stats AND a phase onto the context,
- // because that is what makes every LLM call beneath attributable. The
- // explicit deps.Stats.RecordCall sites fire regardless, so counting calls
- // alone would pass even with the binding removed — hence the ctx probe.
- if mdl.nilStats > 0 {
- t.Errorf("CurrentStats(ctx) was nil on %d of %d calls: Rag did not bind "+
- "stats to the context, so per-phase usage is not collected", mdl.nilStats, mdl.calls)
- }
- if mdl.calls == 0 {
- t.Fatal("the model was never called; the probe proves nothing")
- }
- if mdl.unknownPhase == mdl.calls {
- t.Errorf("every call saw phase \"unknown\": no phase marker reached the " +
- "model, so usage cannot be attributed per phase")
- }
- for p := range mdl.phases {
- t.Logf("observed phase: %s", p)
- }
-}
-
-// countingModel records what the context looked like at each completion, so the
-// test can prove stats and phase were actually bound.
-type countingModel struct {
- calls int
- nilStats int
- unknownPhase int
- phases map[string]bool
-}
-
-func (m *countingModel) Complete(ctx context.Context, msgs []schema.Message, _ []runtime.ToolSpec) (*runtime.ModelReply, error) {
- m.calls++
- if runtime.CurrentStats(ctx) == nil {
- m.nilStats++
- }
- if p := runtime.CurrentPhase(ctx); p == "unknown" {
- m.unknownPhase++
- } else {
- if m.phases == nil {
- m.phases = map[string]bool{}
- }
- m.phases[p] = true
- }
- return &runtime.ModelReply{Content: "ok"}, nil
-}
-
-// TestSCAFeedbackIsStatusOnly: the "[Research status]" note is the status hint ALONE.
-// The verdict dict carries only a "status" key, so its missing_claims / hard_violations /
-// agent_confidence / feedback segments are always empty — rendering them (from the
-// separate SCA payload) would invent a "0.00" confidence fallback that is never
-// emitted.
-func TestSCAFeedbackIsStatusOnly(t *testing.T) {
- rich := map[string]any{
- "contradictions": []any{"A says 1874, B says 1881"},
- "confidence": 0.42,
- "claims": map[string]any{
- "c1": map[string]any{"grounded": false, "missing_information": []any{
- map[string]any{"what": "the architect"},
- }},
- },
- }
- got := scaFeedback(rich, VerdictInsufficient)
- if got != "evidence is not yet sufficient" {
- t.Errorf("scaFeedback = %q, want the status hint alone", got)
- }
- // Empty (and thus un-appended) for a SUFFICIENT verdict, per :910.
- if got := scaFeedback(rich, VerdictSufficient); got != "" {
- t.Errorf("scaFeedback(SUFFICIENT) = %q, want empty", got)
- }
- // The unused-status fallback mirrors :915.
- if got := scaFeedback(rich, "WEIRD"); got != "sufficiency status: WEIRD" {
- t.Errorf("scaFeedback(WEIRD) = %q, want the :915 fallback", got)
- }
-}
-
-// TestComposeFallbackDraftMirrorsLanguage pins: a non-English
-// question gets an explicit same-language instruction.
-func TestComposeFallbackDraftMirrorsLanguage(t *testing.T) {
- st := &AgenticState{
- Question: "它是什么时候建造的?",
- KB: &runtime.Kbinfos{Chunks: []map[string]any{{"content": "1874年建成。"}}},
- }
- mdl := &draftModel{reply: "1874年建成。\nMISSING: 无"}
- ComposeFallbackDraft(context.Background(), RAGTools{Model: mdl}, st)
-
- if !strings.Contains(mdl.lastSys, "same language") {
- t.Errorf("system = %q, want the same-language instruction for a CJK question", mdl.lastSys)
- }
-}
-
-func TestSCAGapsToRewriteFromSubQueries(t *testing.T) {
- sca := map[string]any{
- "sub_queries": []any{
- map[string]any{"sub_query": "q1", "satisfied": true, "missing_fact": "", "search_hint": ""},
- map[string]any{"sub_query": "q2", "satisfied": false, "missing_fact": "the year", "search_hint": "search for dates"},
- },
- }
- gaps := SCAGapsToRewrite(sca)
- if len(gaps) != 1 {
- t.Fatalf("expected 1 gap, got %d: %+v", len(gaps), gaps)
- }
- if gaps[0].What != "the year" || gaps[0].SearchHint != "search for dates" {
- t.Fatalf("unexpected gap: %+v", gaps[0])
- }
-}
-
-func TestSCAGapsToRewriteFallbackToClaims(t *testing.T) {
- sca := map[string]any{
- "claims": map[string]any{
- "c1": map[string]any{
- "verdict": "unverified",
- "missing_information": []any{
- map[string]any{"what": "population", "search_hint": "demographics"},
- },
- },
- },
- }
- gaps := SCAGapsToRewrite(sca)
- if len(gaps) != 1 {
- t.Fatalf("expected 1 gap from claims fallback, got %d: %+v", len(gaps), gaps)
- }
- if gaps[0].What != "population" {
- t.Fatalf("unexpected gap: %+v", gaps[0])
- }
-}
-
-func TestRunReturnsNonNilState(t *testing.T) {
- ctx := context.Background()
- mdl := &scriptedModel{}
- mdl.push(`{"fanouts": ["when built", "where located"]}`)
- mdl.push(`{"slots":[{"id":0,"type":"aspect","question":"when built","clues":["1865"]},{"id":1,"type":"aspect","question":"where located","clues":["geneva"]}], "first_queries":["when built","where located"]}`)
- mdl.push("Built in 1865.")
- mdl.push("In Geneva.")
- mdl.push(`{"is_sufficient": true, "score": 0.9, "contradictions": [], "reasoning": "ok", "claims": {}}`)
-
- exec := newStubExecutor()
- exec.add("search_chunks", `{"hit":[{"doc_id":"d1","docnm_kwd":"doc1","content":"Built 1865, Geneva."}],"doc_aggs":[]}`, runtime.StatusOK)
-
- st, err := BuildAgenticGraph(ctx, RAGTools{
- Model: mdl,
- Tools: newToolset(exec),
- // The dual-channel fan-out retrieves directly, so it needs the search
- // config; without it the prefetch contributes nothing.
- Search: runtime.SearchDeps{Backend: &corpusRetriever{}, KbIDs: []string{"kb1"}, HasEmbedder: true},
- Logger: log.Default(),
- }, "When and where was it built?", "", 3, nil)
- if err != nil {
- t.Fatalf("BuildAgenticGraph: %v", err)
- }
-
- if st == nil {
- t.Fatal("expected a non-nil AgenticState")
- }
- if strings.TrimSpace(st.Draft) == "" {
- t.Fatal("expected a non-empty research draft")
- }
- if st.NoProgress {
- t.Fatal("a completed run should not report NoProgress")
- }
-}
-
-// TestAgenticGraphPushesPhaseProgress asserts that the agentic loop forwards tagged
-// engine-stage lines to the caller's Progress sink as it runs — planner, research round,
-// SCA — so a streaming chat can show live research progress in the reasoning block.
-func TestAgenticGraphPushesPhaseProgress(t *testing.T) {
- ctx := context.Background()
- mdl := &scriptedModel{}
- mdl.push(`{"fanouts": ["when built", "where located"]}`)
- mdl.push(`{"slots":[{"id":0,"type":"aspect","question":"when built","clues":["1865"]},{"id":1,"type":"aspect","question":"where located","clues":["geneva"]}], "first_queries":["when built","where located"]}`)
- mdl.push("Built in 1865.")
- mdl.push("In Geneva.")
- mdl.push(`{"is_sufficient": true, "score": 0.9, "contradictions": [], "reasoning": "ok", "claims": {}}`)
-
- exec := newStubExecutor()
- exec.add("search_chunks", `{"hit":[{"doc_id":"d1","docnm_kwd":"doc1","content":"Built 1865, Geneva."}],"doc_aggs":[]}`, runtime.StatusOK)
-
- var lines []string
- // Mirror production: the run's step reporter carries the think-block text.
- // Each stage declares its own step, so everything asserted here is a step a
- // node reported — never a log line that happened to match a pattern.
- sink := func(line string) { lines = append(lines, line) }
- ctx = runtime.WithSteps(ctx, runtime.StepReporter{Text: sink})
- st, err := BuildAgenticGraph(ctx, RAGTools{
- Model: mdl,
- Tools: newToolset(exec),
- Search: runtime.SearchDeps{Backend: &corpusRetriever{}, KbIDs: []string{"kb1"}, HasEmbedder: true},
- Logger: log.Default(),
- }, "When and where was it built?", "", 3, nil)
- if err != nil {
- t.Fatalf("BuildAgenticGraph: %v", err)
- }
-
- if st == nil {
- t.Fatal("expected a non-nil AgenticState")
- }
- joined := strings.Join(lines, "\n")
- // Each stage is announced by its node's own tagged log line (no separate push), so
- // assert on what the nodes actually log.
- for _, want := range []string{
- "[Agentic RAG] Starting research",
- "[Planner] Split the question into",
- // prefetchSummary's opener has two branches (a plan larger than the opening
- // queries vs not), so assert the half both share plus the closing step.
- "up front.",
- "[Prefetch] Added",
- "[RAGAgent] Round 1 begins",
- "[Draft] Drafted an intermediate answer",
- "[SCA] Evidence check:",
- "[Finalize] Finalizing",
- } {
- if !strings.Contains(joined, want) {
- t.Errorf("think-log lines missing %q; got:\n%s", want, joined)
- }
- }
- // Nesting: the prefetch's search legs are indented under the prefetch line —
- // that is what makes "what did this line run" readable — while the graph's own
- // nodes stay flush (this run has no outer rag call above them).
- if !strings.Contains(joined, runtime.StepIndentUnit+"[BM25 search] Searching by keyword for") {
- t.Errorf("the prefetch's legs must be indented under it; got:\n%s", joined)
- }
- if !strings.Contains(joined, "\n[Planner] Split the question into") {
- t.Errorf("the graph's own steps must stay flush; got:\n%s", joined)
- }
-}
-
-// TestAgenticGraphCyclesBackThroughQueryRewrite drives the research loop: the
-// SCA returns insufficient once, so the graph must run query_rewrite and re-enter
-// rag_agent. The loop is the only cycle in the graph — it is why the graph is
-// compiled in Pregel mode (DAG mode rejects cycles).
-func TestAgenticGraphCyclesBackThroughQueryRewrite(t *testing.T) {
- ctx := context.Background()
- mdl := &promptRoutedModel{}
-
- exec := newStubExecutor()
- exec.add("search_chunks", `{"hit":[{"doc_id":"d1","docnm_kwd":"doc1","content":"Built 1865, Geneva."}],"doc_aggs":[]}`, runtime.StatusOK)
-
- var buf bytes.Buffer
- var lines []string
- sink := func(line string) { lines = append(lines, line) }
- ctx = runtime.WithSteps(ctx, runtime.StepReporter{Text: sink})
- st, err := BuildAgenticGraph(ctx, RAGTools{
- Model: mdl,
- Tools: newToolset(exec),
- Search: runtime.SearchDeps{Backend: &corpusRetriever{}, KbIDs: []string{"kb1"}, HasEmbedder: true},
- Logger: log.New(&buf, "", 0),
- }, "When and where was it built?", "", 3, nil)
- if err != nil {
- t.Fatalf("BuildAgenticGraph: %v", err)
- }
-
- if st == nil {
- t.Fatal("expected a non-nil AgenticState")
- }
- joined := strings.Join(lines, "\n")
- for _, want := range []string{
- "[QueryRewriter]",
- "[RAGAgent] Round 2 begins",
- "[Finalize] Finalizing",
- } {
- if !strings.Contains(joined, want) {
- t.Errorf("cycle not driven: missing %q; got:\n%s", want, joined)
- }
- }
- // The second research round must actually run: the SCA is only reached again
- // through query_rewrite → rag_agent, i.e. the graph's only cycle.
- if !strings.Contains(joined, "[SCA] Evidence check: INSUFFICIENT") {
- t.Errorf("expected the first SCA to be insufficient:\n%s", joined)
- }
-}
-
-// TestBuildLowGraphRunsFormalizeThenDirectSearch covers the low-mode graph
-// . It has no planner and no SCA loop, so the only
-// observable contract is: formalize rewrites the question, then direct_search
-// merges retrieved evidence into the kbinfos.
-func TestBuildLowGraphRunsFormalizeThenDirectSearch(t *testing.T) {
- ctx := context.Background()
- mdl := &scriptedModel{}
- // The multi-turn formalize asks for the rewrite AND the keywords in one JSON
- // reply, so the rewrite below is what the node actually searches for.
- mdl.push(`{"question": "When was Culdcept released?", "keywords": "Culdcept, released"}`) // Formalize
- mdl.push("Culdcept was released in 1999.") // compose
-
- kb := &runtime.Kbinfos{}
- sd := runtime.SearchDeps{Backend: &corpusRetriever{}, KbIDs: []string{"kb1"}, HasEmbedder: true}
- resp := &RunResponse{Mode: runtime.ResolveMode(&runtime.Toolset{ThinkingMode: "low"})}
- req := runtime.RunRequest{Question: "when was it made?", TopN: 8}
-
- var buf bytes.Buffer
- BuildLowGraph(ctx, RAGTools{
- Model: mdl,
- Messages: []schema.Message{
- {Role: schema.User, Content: "Culdcept was made by OmiyaSoft."},
- {Role: schema.User, Content: "when was it made?"},
- },
- Search: sd,
- Logger: log.New(&buf, "", 0),
- }, req, sd, kb, resp, log.New(&buf, "", 0))
-
- // direct_search ran and merged evidence into the shared kbinfos.
- if !kb.HasChunks() {
- t.Fatalf("direct_search did not deposit any chunk; log:\n%s", buf.String())
- }
- // formalize ran and rewrote the follow-up (the graph's first node), reported
- // under its own [Formalize] stage with BOTH the question asked and the one
- // actually searched.
- if want := `[Formalize] Rewrote the follow-up into a standalone question: "when was it made?" → "When was Culdcept released?"`; !strings.Contains(buf.String(), want) {
- t.Errorf("formalize_question node did not narrate its rewrite; log:\n%s", buf.String())
- }
-}
-
-// TestLogComposeDone pins the compose completion line: it is a LOG line, not a
-// step — a compose finishes after the answer has begun streaming, where the chat
-// pipeline drops think lines — and it counts RUNES, so a Chinese answer is not
-// reported at three times its length.
-func TestLogComposeDone(t *testing.T) {
- var buf bytes.Buffer
- logComposeDone(log.New(&buf, "", 0), time.Now(), "曹操是谁?", 3)
- got := buf.String()
- // 5 runes, 15 bytes: the line must say 5.
- if !strings.Contains(got, "[Composing the answer] composed 5 chars from 3 gathered passages in ") {
- t.Errorf("compose completion line = %q", got)
- }
- if strings.Contains(got, "15 chars") {
- t.Errorf("byte count leaked into the line: %q", got)
- }
- // An answer that came back empty is exactly the case this line exists for: it
- // must be reported, not skipped.
- var empty bytes.Buffer
- logComposeDone(log.New(&empty, "", 0), time.Now(), "", 0)
- if !strings.Contains(empty.String(), "composed 0 chars from 0 gathered passages") {
- t.Errorf("empty answer = %q", empty.String())
- }
-}
-
-// TestBuildLowGraphNarratesAnUnchangedQuestion covers the common case end to end:
-// single-turn input is never rewritten, so the formalize step names the question
-// as asked instead of claiming a rewrite that did not happen.
-func TestBuildLowGraphNarratesAnUnchangedQuestion(t *testing.T) {
- ctx := context.Background()
- mdl := &scriptedModel{}
- mdl.push(`{"keywords": "Culdcept, released"}`) // single-turn: keywords only
- mdl.push("Culdcept was released in 1999.") // compose
-
- kb := &runtime.Kbinfos{}
- sd := runtime.SearchDeps{Backend: &corpusRetriever{}, KbIDs: []string{"kb1"}, HasEmbedder: true}
- resp := &RunResponse{Mode: runtime.ResolveMode(&runtime.Toolset{ThinkingMode: "low"})}
- req := runtime.RunRequest{Question: "when was Culdcept made?", TopN: 8}
-
- var buf bytes.Buffer
- BuildLowGraph(ctx, RAGTools{
- Model: mdl,
- Messages: []schema.Message{
- {Role: schema.User, Content: "when was Culdcept made?"},
- },
- Search: sd,
- Logger: log.New(&buf, "", 0),
- }, req, sd, kb, resp, log.New(&buf, "", 0))
-
- if want := `[Formalize] Kept the question as asked: "when was Culdcept made?"`; !strings.Contains(buf.String(), want) {
- t.Errorf("formalize did not name the unchanged question; log:\n%s", buf.String())
- }
- if strings.Contains(buf.String(), "Rewrote") {
- t.Errorf("no rewrite happened, the step must not say so; log:\n%s", buf.String())
- }
-}
-
-// TestFanoutSummary pins what the planner reports: the sub-questions themselves,
-// not an announcement that it is about to decompose. The fallback case (the model
-// refused to decompose, so the question is searched as asked) must not be dressed
-// up as a sub-question.
-//
-// The wording carries no "first-hop": a reader has no hop to count, and these are
-// just the questions the plan set out to research.
-func TestFanoutSummary(t *testing.T) {
- cases := []struct {
- name string
- question string
- fanouts []string
- want string
- }{
- {"three", "曹操是谁?", []string{"曹操是谁", "曹操生平简介", "曹操历史地位"},
- `Split the question into 3 sub-questions to research: "曹操是谁", "曹操生平简介", "曹操历史地位".`},
- {"one-sub-question", "曹操是谁?", []string{"曹操"},
- `Split the question into 1 sub-question to research: "曹操".`},
- {"kept-as-asked", "曹操是谁?", []string{"曹操是谁?"},
- "The question is already a single searchable sub-question; searching it as asked."},
- {"nothing", "曹操是谁?", nil,
- "Could not decompose the question; searching it as asked."},
- }
- for _, tc := range cases {
- if got := fanoutSummary(tc.question, tc.fanouts); got != tc.want {
- t.Errorf("%s: fanoutSummary = %q, want %q", tc.name, got, tc.want)
- }
- }
-}
-
-// TestFinalizeSummary pins the finalize step's four states, including the evidence
-// size on the two that have evidence to report. The compose step no longer repeats
-// the count (it is suppressed after this node), so this line is where the reader
-// learns what the answer is being written from.
-func TestFinalizeSummary(t *testing.T) {
- cases := []struct {
- name string
- partial, empty bool
- chunks int
- want string
- }{
- {"complete", false, false, 5, "Finalizing a complete answer from 5 passages."},
- {"complete-singular", false, false, 1, "Finalizing a complete answer from 1 passage."},
- {"partial", true, false, 5, "Finalizing a partial answer from 5 passages; some gaps remain unanswered."},
- {"empty", false, true, 0, "Finalizing with no supporting evidence: the answer will say so."},
- {"partial-and-empty", true, true, 0,
- "Finalizing a partial answer with no supporting evidence: research exhausted its attempts."},
- }
- for _, tc := range cases {
- if got := finalizeSummary(tc.partial, tc.empty, tc.chunks); got != tc.want {
- t.Errorf("%s: finalizeSummary = %q, want %q", tc.name, got, tc.want)
- }
- }
- if got := chunkCount(nil); got != 0 {
- t.Errorf("chunkCount(nil) = %d, want 0 (a failed graph still reports Finalize)", got)
- }
-}
-
-// TestPrefetchSummary pins the two numbers a reader sees back to back in high/ultra
-// mode: what the planner decomposed the question into, and how many queries the
-// upfront search runs (initialize_state caps first_queries at three, the rest stay
-// as slots). A bare "searching 3" after "split into 5" reads like two sub-questions
-// were dropped.
-//
-// The sentence must NOT call those queries the plan's sub-questions: they are the
-// slot table's own first_queries, written by a model that only sees the plan as a
-// hint and may reword it (see prefetchSummary). The two numbers are still both
-// named, which is the property this test protects.
-func TestPrefetchSummary(t *testing.T) {
- cases := []struct {
- name string
- planned, searching int
- want string
- }{
- {"all-searched", 3, 3, "Searching 3 opening queries up front."},
- {"singular", 1, 1, "Searching 1 opening query up front."},
- // The cap: the plan is bigger than the opening queries, so the line names
- // the plan's total (the rest stay as slots for later rounds).
- {"capped", 5, 3, "The plan lists 5 sub-questions; searching 3 opening queries up front."},
- {"capped-singular", 2, 1, "The plan lists 2 sub-questions; searching 1 opening query up front."},
- // Decomposition failed: no plan, one query as asked.
- {"no-plan", 0, 1, "Searching 1 opening query up front."},
- }
- for _, tc := range cases {
- if got := prefetchSummary(tc.planned, tc.searching); got != tc.want {
- t.Errorf("%s: prefetchSummary = %q, want %q", tc.name, got, tc.want)
- }
- }
-}
-
-// TestSlotPrefillSummary pins what the evidence prefill leaves to research. The
-// zero case matters most: it is the one that explains a round re-searching queries
-// the upfront prefetch already ran.
-//
-// The count in the sentence is the number of sessions that will RUN
-// (slotSessionsPerRound), not the number of open slots: the round opens a bounded
-// number of them and re-answers the rest from the pooled evidence, so promising the
-// open count sent a reader looking for searches that were never going to happen.
-func TestSlotPrefillSummary(t *testing.T) {
- cases := []struct {
- name string
- prefilled, total, remaining int
- want string
- }{
- {"nothing-prefilled", 0, 5, 5,
- "The pooled evidence answers none of the 5 slots; opening 3 research sessions this round."},
- {"some-prefilled", 1, 5, 4,
- "The pooled evidence already answers 1 of the 5 slots; opening 3 research sessions for the rest."},
- // Fewer open slots than the per-round cap: 1 is what runs, not 1 capped.
- {"one-open", 4, 5, 1,
- "The pooled evidence already answers 4 of the 5 slots; opening 1 research session for the rest."},
- {"all-prefilled", 5, 5, 0,
- "The pooled evidence already answers all 5 slots; no session to run."},
- }
- for _, tc := range cases {
- if got := slotPrefillSummary(tc.prefilled, tc.total, tc.remaining); got != tc.want {
- t.Errorf("%s: slotPrefillSummary = %q, want %q", tc.name, got, tc.want)
- }
- }
-}
-
-// TestRagRoundEndLine pins the round's closing sentence, including the zero case:
-// "0 slots still unresolved" read as a double negative.
-//
-// The clause says "slots", not "sub-questions": the number is the slot table's
-// backlog, and the plan's sub-questions are a different count printed earlier in
-// the same block.
-func TestRagRoundEndLine(t *testing.T) {
- cases := []struct {
- name string
- round, newPassages, pool, unresolved int
- want string
- }{
- {"zero-unresolved", 1, 0, 6, 0,
- "Round 1 finished: 0 new passages added; the pool now holds 6 passages, and no unresolved slots."},
- {"some-unresolved", 2, 0, 6, 2,
- "Round 2 finished: 0 new passages added; the pool now holds 6 passages, and 2 slots still unresolved."},
- {"one-added-one-open", 3, 1, 7, 1,
- "Round 3 finished: 1 new passage added; the pool now holds 7 passages, and 1 slot still unresolved."},
- }
- for _, tc := range cases {
- if got := ragRoundEndLine(tc.round, tc.newPassages, tc.pool, tc.unresolved); got != tc.want {
- t.Errorf("%s: ragRoundEndLine = %q, want %q", tc.name, got, tc.want)
- }
- }
-}
-
-// TestFormalizeStepLine pins the formalize step's wording per outcome: a question
-// kept as asked names the question, a rewritten one gets the pair (asked →
-// searched), and only a formalize that produced no question stays silent.
-//
-// The node used to announce `Formalized the question into "曹操是谁?".` for every
-// outcome — which reads as a rewrite that did not happen, since single-turn runs
-// never rewrite and the multi-turn prompt asks for the question unchanged "in
-// most cases".
-func TestFormalizeStepLine(t *testing.T) {
- cases := []struct {
- name string
- asAsked string
- standalone string
- want string
- }{
- {"unchanged", "曹操是谁?", "曹操是谁?", `Kept the question as asked: "曹操是谁?"`},
- {"only-whitespace-differs", "曹操是谁?", " 曹操是谁? ", `Kept the question as asked: "曹操是谁?"`},
- {"empty-standalone", "曹操是谁?", "", ""},
- {"nothing-asked-either", "", "", ""},
- {"rewritten", "他死于哪年?", "曹操死于哪年?",
- `Rewrote the follow-up into a standalone question: "他死于哪年?" → "曹操死于哪年?"`},
- {"nothing-asked", "", "曹操是谁?", `Standalone question for this turn: "曹操是谁?"`},
- }
- for _, tc := range cases {
- got := formalizeStepLine(tc.asAsked, tc.standalone)
- if got != tc.want {
- t.Errorf("%s: formalizeStepLine = %q, want %q", tc.name, got, tc.want)
- }
- if strings.HasSuffix(got, "。") || strings.HasSuffix(got, "?.") || strings.HasSuffix(got, "?.") {
- t.Errorf("%s: %q ends on a doubled terminator", tc.name, got)
- }
- }
-}
-
-// lowGraphCountingExpander counts compiled-expansion calls (runtime.CompiledExpander).
-type lowGraphCountingExpander struct{ calls int }
-
-func (c *lowGraphCountingExpander) Expand(context.Context, *runtime.Kbinfos, string, string, []string) error {
- c.calls++
- return nil
-}
-
-// TestBuildLowGraphAlwaysExpandsCompiled: mode: direct.py
-// calls hybrid_search(..., use_compiled=True) unconditionally, so the low
-// graph's direct_search must expand even when the caller's RunRequest leaves
-// UseCompiled false — which production always does.
-func TestBuildLowGraphAlwaysExpandsCompiled(t *testing.T) {
- ctx := context.Background()
- kb := &runtime.Kbinfos{}
- exp := &lowGraphCountingExpander{}
- sd := runtime.SearchDeps{Backend: &corpusRetriever{}, KbIDs: []string{"kb1"}, HasEmbedder: true, Expand: exp}
- resp := &RunResponse{Mode: runtime.ResolveMode(&runtime.Toolset{ThinkingMode: "low"})}
- req := runtime.RunRequest{Question: "when was it made?", TopN: 8, UseCompiled: false}
-
- var buf bytes.Buffer
- if err := BuildLowGraph(ctx, RAGTools{Search: sd}, req, sd, kb, resp, log.New(&buf, "", 0)); err != nil {
- t.Fatalf("BuildLowGraph: %v", err)
- }
- if exp.calls == 0 {
- t.Errorf("low mode's direct_search must expand compiled structure; log:\n%s",
- buf.String())
- }
-}
-
-// TestFinalizeRunsOnceFromInsideTheGraph locks in the terminal node's contract: both
-// graphs end in a formalize_answer node that composes, so the composition happens INSIDE
-// the graph — and exactly once, even though Rag also has a post-graph call for runs that
-// never reach the last node. A double composition would overwrite the answer with the
-// model's next reply, which is what this test watches for.
-func TestFinalizeRunsOnceFromInsideTheGraph(t *testing.T) {
- ctx := context.Background()
- finalized := 0
- resp := &RunResponse{Mode: runtime.ResolveMode(&runtime.Toolset{ThinkingMode: "low"})}
- kb := &runtime.Kbinfos{}
- sd := runtime.SearchDeps{Backend: &corpusRetriever{}, KbIDs: []string{"kb1"}, HasEmbedder: true}
- req := runtime.RunRequest{Question: "when was it made?", TopN: 8}
- deps := RAGTools{
- Model: &scriptedModel{},
- Search: sd,
- }
- // Mirror Rag: the hook is idempotent, so the graph's node and the
- // post-graph safety net cannot both compose.
- composed := false
- deps.Finalize = func(_ context.Context, _ bool, _ bool, question string) {
- if composed {
- return
- }
- composed = true
- finalized++
- // The low graph's formalize_answer must forward the FORMALIZED
- // question the formalize_question node wrote into the RunRequest
- // composing
- // from the outer tool argument collapses a multi-hop answer to its
- // first sub-answer.
- if question != "when was it made?" {
- t.Errorf("Finalize question = %q, want the formalized RunRequest question", question)
- }
- }
-
- var buf bytes.Buffer
- if err := BuildLowGraph(ctx, deps, req, sd, kb, resp, log.New(&buf, "", 0)); err != nil {
- t.Fatalf("BuildLowGraph: %v", err)
- }
- // The graph's last node composed...
- if finalized != 1 {
- t.Fatalf("formalize_answer node composed %d time(s), want 1; log:\n%s", finalized, buf.String())
- }
- // ...and the post-graph call is then a no-op.
- deps.Finalize(ctx, false, false, "")
- if finalized != 1 {
- t.Errorf("composition ran %d time(s) after the post-graph call, want 1", finalized)
- }
-}
-
-// TestSCAUnavailableMarksInsufficient pins: when the SCA
-// produces no usable result (timeout, unparsable reply, no model) the verdict is
-// INSUFFICIENT so unresolved slots can drive another research round — NOT
-// SUFFICIENT, which would ship the unverified draft as if it had passed review.
-func TestSCAUnavailableIsNotAVerdict(t *testing.T) {
- st := NewAgenticState("When was it built?", "", 3, nil)
- st.Draft = "Built in 1865."
- st.KB = &runtime.Kbinfos{Chunks: []map[string]any{
- {"chunk_id": "c1", "content": "Built 1865, Geneva."},
- }}
- // A model that never returns JSON: GenJSON fails, so the SCA has no result.
- mdl := &scriptedModel{}
- mdl.push("this is not json at all")
-
- scaNode(context.Background(), RAGTools{Model: mdl}, st, log.New(&bytes.Buffer{}, "", 0))
-
- if st.Verdict != VerdictUnknown {
- t.Errorf("Verdict = %q, want UNKNOWN: a review that could not run is the ABSENCE of a verdict, not a judgement of insufficiency", st.Verdict)
- }
- if st.Verdict == VerdictInsufficient {
- t.Error("an unavailable review must not read as INSUFFICIENT: that is what marked the answer partial on no evidence and promised a research round the budget could not pay for")
- }
- if st.SCA == nil {
- t.Error("SCA payload must stay an (empty) map")
- }
- // An un-run review must not make the answer PARTIAL either: with nothing
- // unresolved and no stalled round, the deliverable is not dressed up as
- // unverified.
- st.Deadline = time.Now().Add(60 * time.Second)
- formalizeAnswerNode(context.Background(), RAGTools{}, st, log.New(&bytes.Buffer{}, "", 0))
- if st.PartialAnswer {
- t.Error("PartialAnswer = true on an UNKNOWN verdict with no unresolved slots; partial is a statement about the evidence")
- }
- // The FACT still has to reach the answer: nobody checked whether the record is complete.
- if !st.KB.SufficiencyUnchecked() {
- t.Error("an un-run review must be recorded on the run (see NoteSufficiencyUnchecked): the record has to say nobody checked, or the count reads as a checked total")
- }
- if rec := composedRecord(st.KB); !strings.Contains(rec, "nobody checked whether the members above are complete") {
- t.Errorf("record = %q, want the unchecked review stated where the answer reads it", rec)
- }
-}
-
-// TestProbeLedgerKeepsTheDeedsWordsOutOfTheNames pins what the probed list may be READ as: names.
-//
-// Measured (2026-09-17, 三国/关羽) the list handed the answer `杀 斩 武将 名单 亲斩 关羽 太史慈 …
-// 赚城斩车胄 令左右推出斩之 庞德`, under the instruction "any name here that the record above does
-// not mention is a finding nobody recorded" — an invitation to count the actor among the people he
-// killed, and the deed's own verbs among its objects. The act words and query-shaped terms come
-// from the run's own probes and are still shown (the ledger's job is to make the probing visible),
-// but they are shown as what they are.
-func TestProbeLedgerKeepsTheDeedsWordsOutOfTheNames(t *testing.T) {
- kb := &runtime.Kbinfos{}
- kb.Admit(func(p *runtime.PoolAdmitter) {
- p.Add(map[string]any{"chunk_id": "c1", "content": "关公马快,赶上文丑,脑后一刀,将文丑斩下马来。"})
- })
- // The direction's own declaration: what the run is searching WITH (actor forms + act words).
- kb.MarkCoverage(runtime.Coverage{Acts: []string{"斩", "杀"}, Actor: "关羽|云长"})
- for _, term := range []string{
- "文丑", "太史慈", "关羽", "斩孔秀", "亲斩", "关羽斩华雄|温酒斩华雄", "令左右推出斩之 庞德",
- } {
- kb.RecordReachedTerm(term, "c1")
- }
-
- ledger := probeLedger(kb)
- names, others, split := strings.Cut(ledger, "Probed, NOT names")
- if !split {
- t.Fatalf("ledger = %q, want the non-names said apart from the names", ledger)
- }
- if !strings.Contains(names, "文丑") || !strings.Contains(names, "太史慈") {
- t.Errorf("names = %q, want the name-shaped terms kept whatever they name (文丑 is a kill, 太史慈 a judgement the answer makes)", names)
- }
- for _, banned := range []string{"关羽", "斩孔秀", "亲斩", "关羽斩华雄|温酒斩华雄", "令左右推出斩之 庞德"} {
- if strings.Contains(names, banned) {
- t.Errorf("names = %q, must not offer %q as a name", names, banned)
- }
- if !strings.Contains(others, banned) {
- t.Errorf("non-names = %q, want %q shown as probed-but-not-a-name", others, banned)
- }
- }
-}
-
-// TestComposedRecordSaysWhenNobodyCheckedCompleteness pins the note's default and its wording: a
-// run whose review ran says nothing extra, and one whose review never ran says the one thing the
-// answer needs — nobody checked.
-func TestComposedRecordSaysWhenNobodyCheckedCompleteness(t *testing.T) {
- kb := &runtime.Kbinfos{Record: "- slot 0 [count]: 18"}
- if rec := composedRecord(kb); strings.Contains(rec, "nobody checked") {
- t.Fatalf("record = %q, want no note while the review ran", rec)
- }
- kb.NoteSufficiencyUnchecked()
- rec := composedRecord(kb)
- if !strings.Contains(rec, "- slot 0 [count]: 18") {
- t.Fatalf("record = %q, want the slots kept", rec)
- }
- if !strings.Contains(rec, "nobody checked whether the members above are complete") ||
- !strings.Contains(rec, "do not present it as exhaustive") {
- t.Fatalf("record = %q, want the unchecked review stated", rec)
- }
-}
-
-// TestQueryRewriteKeepsTheRoundAliveWhenTheRewriterDeclines pins the asymmetry that used to end the
-// loop: the routing had already judged another round worth its budget (gaps exist, the review view
-// changed) and one empty reply from the rewriter cancelled that judgement — while the other
-// fallback, the unresolved slots' clues, is empty on a set question whose unknowns are names nobody
-// has proposed yet. Measured (2026-09-17, 三国/关羽): INSUFFICIENT, 3 gaps, +182 chunks, 130s left,
-// and no round.
-func TestQueryRewriteKeepsTheRoundAliveWhenTheRewriterDeclines(t *testing.T) {
- st := NewAgenticState("关羽杀了多少有姓名的人物?", "", 3, nil)
- st.SCA = map[string]any{"claims": map[string]any{
- "c1": map[string]any{"verdict": "unverified", "missing_information": []any{
- map[string]any{"what": "the count", "search_hint": "count the passages"},
- }},
- }}
- st.KB = &runtime.Kbinfos{}
- st.KB.RecordProbedAbsent("斩孟坦")
- st.SlotTable = runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "count", Terms: []string{"斩"}, Subject: "关羽|云长"},
- }, 0, nil)
- st.Deadline = time.Now().Add(120 * time.Second)
-
- mdl := &scriptedModel{}
- mdl.push(`{"queries": []}`)
- queryRewriteNode(context.Background(), RAGTools{
- Model: mdl,
- Search: runtime.SearchDeps{Backend: &corpusRetriever{}, KbIDs: []string{"kb1"}, HasEmbedder: true},
- }, st, log.New(&bytes.Buffer{}, "", 0))
-
- if st.NoProgress {
- t.Fatal("NoProgress = true: the rewriter declined, and the round had already been judged worth its budget")
- }
- joined := strings.Join(st.CurrentQueries, "|")
- if !strings.Contains(joined, "斩孟坦") || !strings.Contains(joined, "关羽") {
- t.Fatalf("CurrentQueries = %v, want the run's own open terms", st.CurrentQueries)
- }
-}
-
-// TestQueryRewriteFoldsUnresolvedCluesWhenNoGaps pins: with
-// no structured SCA gaps, the unresolved slots' question_clues become the gaps,
-// so the loop keeps going instead of accepting an unresolved draft.
-func TestQueryRewriteFoldsUnresolvedCluesWhenNoGaps(t *testing.T) {
- st := NewAgenticState("When and where was it built?", "", 3, nil)
- st.SCA = map[string]any{} // no sub_queries, no claims -> no gaps
- st.UnresolvedSlots = []map[string]any{
- {"question_clues": []string{"when opened", "where located", "ignored third clue"}},
- }
- st.Deadline = time.Now().Add(60 * time.Second)
-
- mdl := &scriptedModel{}
- mdl.push(`{"queries": [{"query": "when was it opened"}]}`)
-
- queryRewriteNode(context.Background(), RAGTools{
- Model: mdl,
- Search: runtime.SearchDeps{Backend: &corpusRetriever{}, KbIDs: []string{"kb1"}, HasEmbedder: true},
- }, st, log.New(&bytes.Buffer{}, "", 0))
-
- if st.NoProgress {
- t.Fatal("NoProgress = true; unresolved slot clues must be folded into the gaps and researched")
- }
- joined := strings.Join(st.CurrentQueries, "|")
- if !strings.Contains(joined, "when opened") {
- t.Errorf("CurrentQueries = %v, want the unresolved slot clue to be pursued", st.CurrentQueries)
- }
-}
-
-// TestUnresolvedClueGapsCapsAtTwoClues pins the [:2] per-slot cap.
-func TestUnresolvedClueGapsCapsAtTwoClues(t *testing.T) {
- st := &AgenticState{UnresolvedSlots: []map[string]any{
- {"question_clues": []string{"a", "b", "c"}},
- {"question_clues": []string{"a", "d"}}, // clues are appended without dedupe
- }}
- gaps := unresolvedClueGaps(st)
- if len(gaps) != 4 {
- t.Fatalf("gaps = %d, want 4 (first two clues of each slot, duplicates kept)", len(gaps))
- }
- if gaps[0].What != "a" || gaps[0].SearchHint != "a" {
- t.Errorf("gaps[0] = %+v, want what=hint=\"a\"", gaps[0])
- }
- if gaps[2].What != "a" || gaps[3].What != "d" {
- t.Errorf("gaps = %+v, want the second slot's clues a,d appended verbatim", gaps)
- }
-}
-
-// TestBuildSCAClaimsFallsBackToEvidenceOnlyClaim pins: with no
-// draft and no slot evidence, the SCA still gets one claim carrying EVERY chunk
-// in the view, so it can judge the evidence instead of the node short-circuiting.
-func TestBuildSCAClaimsFallsBackToEvidenceOnlyClaim(t *testing.T) {
- view := []map[string]any{
- {"chunk_id": "c1", "content": "Built 1865."},
- {"chunk_id": "c2", "content": "In Geneva."},
- }
- claims, _ := buildSCAClaims("", view, view, nil)
- if len(claims) != 1 {
- t.Fatalf("claims = %d, want 1", len(claims))
- }
- if claims[0].Draft != "(no draft)" {
- t.Errorf("draft = %q, want %q", claims[0].Draft, "(no draft)")
- }
- // Evidence is addressed by POSITION (renderClaimContext keys by index), so
- // the fallback claim must carry 0..len(view)-1.
- if len(claims[0].EvidenceIDs) != 2 || claims[0].EvidenceIDs[0] != "0" || claims[0].EvidenceIDs[1] != "1" {
- t.Errorf("EvidenceIDs = %v, want [0 1]", claims[0].EvidenceIDs)
- }
-}
-
-// TestBuildSCAClaimsAppendsMissingSlotEvidence pins: a slot
-// whose evidence chunk is NOT in the view has that chunk appended, and the claim
-// carries its new POSITION. It is appended to the same list and reviewed against, so
-// buildSCAClaims returns the grown view.
-func TestBuildSCAClaimsAppendsMissingSlotEvidence(t *testing.T) {
- chunks := []map[string]any{
- {"chunk_id": "c1", "content": "Built 1865."},
- {"chunk_id": "c2", "content": "In Geneva."},
- {"chunk_id": "c3", "content": "Designed by OmiyaSoft."},
- }
- view := []map[string]any{chunks[0]}
- slotEvidence := map[string]SlotEvidence{
- "0": {EvidenceIDs: []string{"2"}, Candidate: "designer"}, // pool position 2
- }
-
- claims, grown := buildSCAClaims("", view, chunks, slotEvidence)
- if len(claims) != 1 {
- t.Fatalf("claims = %d, want 1", len(claims))
- }
- if len(grown) != 2 {
- t.Fatalf("grown view = %d, want 2 (the missing chunk must be appended)", len(grown))
- }
- if got := claims[0].EvidenceIDs; len(got) != 1 || got[0] != "1" {
- t.Errorf("EvidenceIDs = %v, want [1] (its position in the grown view)", got)
- }
-}
-
-// TestResolveEvidenceChunkAcceptsBothIdSpaces covers the heterogeneous ids:
-// retrieval tools emit pool positions, navigation tools emit doc ids.
-func TestResolveEvidenceChunkAcceptsBothIdSpaces(t *testing.T) {
- chunks := []map[string]any{
- {"chunk_id": "c1", "doc_id": "d1", "content": "a"},
- {"chunk_id": "c2", "doc_id": "d2", "content": "b"},
- }
- if got := resolveEvidenceChunk("1", chunks); got["chunk_id"] != "c2" {
- t.Errorf("position id \"1\" resolved to %v, want chunk c2", got)
- }
- if got := resolveEvidenceChunk("c1", chunks); got["chunk_id"] != "c1" {
- t.Errorf("chunk id \"c1\" resolved to %v, want chunk c1", got)
- }
- if got := resolveEvidenceChunk("d2", chunks); got["chunk_id"] != "c2" {
- t.Errorf("doc id \"d2\" resolved to %v, want chunk c2", got)
- }
- if got := resolveEvidenceChunk("nope", chunks); got != nil {
- t.Errorf("unknown id resolved to %v, want nil", got)
- }
-}
-
-func TestAgenticNodeNameMapsRoutingToGraphKeys(t *testing.T) {
- cases := map[agenticNode]string{
- nodeQueryRewrite: "query_rewrite",
- nodeRagAgentLoop: "rag_agent",
- nodeFormalizeAnswer: "formalize_answer",
- }
- for n, want := range cases {
- if got := agenticNodeName(n); got != want {
- t.Errorf("agenticNodeName(%d) = %q, want %q", n, got, want)
- }
- }
-}
-
-// Explicit-wiring integration tests (replaces the old init()-based registration checks).
-// Activation is explicit: the test registers the loop before calling Run.
-
-// TestMain keeps the package's tests on in-memory doubles: graph exploration
-// otherwise seeds its dense search from the tenant embedding model, which
-// reaches a database these tests do not have. Returning nil keeps the keyword
-// fallback, which is the path under test.
-func TestMain(m *testing.M) {
- runtime.SetSeedEncoder(func(context.Context, string, string) []float64 { return nil })
- os.Exit(m.Run())
-}
-
-func TestNewAgenticLoopDrivesHarnessRun(t *testing.T) {
- SetAgenticLoop(NewAgenticLoop())
- defer SetAgenticLoop(nil)
-
- facts := `{"fact":"Saint Lawrence River; 14 April 1865; 9 December 2019."}`
- if err := json.Unmarshal([]byte(facts), &struct{}{}); err != nil {
- t.Fatalf("bad fixture: %v", err)
- }
-
- mdl := &scriptedModel{}
- mdl.push(`{"fanouts": ["when was it opened", "what river"]}`)
- mdl.push(`{"slots":[{"id":0,"type":"aspect","question":"when opened","clues":["1865"]},{"id":1,"type":"aspect","question":"which river","clues":["river"]}], "first_queries":["when opened","which river"]}`)
- mdl.push("It opened on 14 April 1865.")
- mdl.push("The Saint Lawrence River.")
- mdl.push(`{"is_sufficient": true, "score": 0.9, "contradictions": [], "reasoning": "ok", "claims": {}}`)
-
- deps := RAGTools{
- Retriever: &countingRetriever{},
- Model: mdl,
- Prompts: &fakePrompts{},
- }
-
- got := runHarnessRun(t, deps, "When did it open and on which river?")
- if got == nil {
- t.Fatal("expected a non-nil RunResponse")
- }
- if got.Mode.Label != "high" {
- t.Fatalf("expected high mode, got %q", got.Mode.Label)
- }
- // The agentic loop routed through the planner + slot research; at minimum the
- // response is produced without error.
- if len(got.Chunks) == 0 {
- t.Fatalf("expected >=1 chunk (retriever stub fills KB), got %d", len(got.Chunks))
- }
-}
-
-// rfRetriever records the rank feature every retrieve call carried.
-type rfRetriever struct {
- features []map[string]float64
- queries []string
- docScopes [][]string
-}
-
-func (r *rfRetriever) Retrieve(_ context.Context, req runtime.RetrieveRequest) ([]map[string]any, error) {
- r.features = append(r.features, req.RankFeature)
- r.queries = append(r.queries, req.Query)
- r.docScopes = append(r.docScopes, req.DocScope)
- return []map[string]any{{
- "chunk_id": "c1",
- "content": "The bridge opened in 1865.",
- "doc_id": "d1",
- "doc_name": "Bridge",
- }}, nil
-}
-
-type rfTagger struct{}
-
-func (rfTagger) LabelQuestion(_ context.Context, _ string, _ []*entity.Knowledgebase) map[string]float64 {
- return map[string]float64{"location": 1.0}
-}
-
-// TestAgenticLoopRankFeaturePolicy pins the rank_feature policy: the search legs (the
-// fan-out channels, search_chunks, retrieve) call the retriever WITHOUT rank_feature, and
-// only the low-mode direct pass passes the question-type tag boost. The agentic loop's
-// research retrievals must therefore carry NO tag boost, even though the Tagger/KBs
-// projection is wired through.
-func TestAgenticLoopRankFeaturePolicy(t *testing.T) {
- SetAgenticLoop(NewAgenticLoop())
- defer SetAgenticLoop(nil)
-
- mdl := &scriptedModel{}
- // Call order: formalize (single-turn keyword extraction) → planner fan-out → slot
- // table → draft → SCA.
- mdl.push(`{"entity": ["it"], "aliases": [], "fact_type": [], "qualifiers": []}`)
- mdl.push(`{"fanouts": ["when was it opened"]}`)
- mdl.push(`{"slots":[{"id":0,"type":"aspect","question":"when opened","clues":["1865"]}], "first_queries":["when opened"]}`)
- mdl.push("It opened in 1865.")
- mdl.push(`{"is_sufficient": true, "score": 0.9, "contradictions": [], "reasoning": "ok", "claims": {}}`)
-
- r := &rfRetriever{}
- got := runHarnessRun(t, RAGTools{
- Retriever: r,
- Model: mdl,
- Prompts: &fakePrompts{},
- KBs: []*entity.Knowledgebase{{}},
- Tagger: rfTagger{},
- }, "When did it open?")
- if got == nil {
- t.Fatal("expected a non-nil RunResponse")
- }
- if len(r.features) == 0 {
- t.Fatal("the agentic loop performed no retrieval")
- }
- // Scope to the fan-out / slot research retrievals — the hybrid/bm25 legs, none of
- // which pass rank_feature.
- //
- // Also exempt: the navigation tools' own recalls (chunk-agg routing via
- // chunkAggRetrieveFrom) — neither passes rank_feature there either.
- research := map[string]bool{"when was it opened": true, "when opened": true}
- checked := 0
- for i, q := range r.queries {
- if !research[q] {
- continue
- }
- checked++
- if len(r.features[i]) != 0 {
- t.Errorf("retrieve %q RankFeature = %v, want nil (search.py legs never pass rank_feature)", q, r.features[i])
- }
- }
- if checked == 0 {
- t.Fatalf("no fan-out/slot retrieval was performed (saw %v); assertion would be vacuous", r.queries)
- }
-}
-
-func TestNewAgenticLoopDegradesWithoutModel(t *testing.T) {
- SetAgenticLoop(NewAgenticLoop())
- defer SetAgenticLoop(nil)
-
- deps := RAGTools{
- Retriever: &countingRetriever{},
- Model: nil, // no model => direct fallback
- Prompts: &fakePrompts{},
- }
-
- got := runHarnessRun(t, deps, "Any question?")
- if got == nil {
- t.Fatal("expected a non-nil RunResponse")
- }
- // With no model the loop falls back to a single direct retrieval; no slots.
- if len(got.Slots) != 0 {
- t.Fatalf("expected 0 slots when no model, got %d", len(got.Slots))
- }
- if len(got.Chunks) == 0 {
- t.Fatalf("expected >=1 chunk from direct fallback, got %d", len(got.Chunks))
- }
-}
-
-func TestNewAgenticLoopRoundsToggleDeactivates(t *testing.T) {
- // After deregistering (nil), Run must NOT enter the agentic loop.
- SetAgenticLoop(nil)
-
- mdl := &scriptedModel{}
- mdl.push(`{"fanouts": ["x", "y"]}`) // would only be produced by the loop's planner
- deps := RAGTools{
- Retriever: &countingRetriever{},
- Model: mdl,
- Prompts: &fakePrompts{},
- }
- got := runHarnessRun(t, deps, "Question?")
- if got == nil {
- t.Fatal("expected a non-nil RunResponse")
- }
- // In single-session fallback the planner fan-out prompt is never issued, so
- // the scripted model's first reply is never consumed.
- if mdl.idx != 0 {
- t.Fatalf("expected agentic planner NOT to run (model idx %d, want 0)", mdl.idx)
- }
- _ = got
-}
-
-// runHarnessRun drives Run in high mode and returns the response. It is
-// the Go analogue of dialog_service.py invoking RAGTools for an agentic answer.
-func runHarnessRun(t *testing.T, deps RAGTools, question string) *RunResponse {
- t.Helper()
- req := runtime.RunRequest{
- Question: question,
- DatasetIDs: []string{"kb1"},
- ThinkingMode: "high",
- }
- resp := Rag(context.Background(), deps, req)
- if resp == nil {
- t.Fatalf("Run returned nil response")
- }
- return resp
-}
-
-// strPtr / fltPtr / fakePrompts are tiny helpers shared across the tests.
-func strPtr(s string) *string { return &s }
-
-type fakePrompts struct{}
-
-func (f *fakePrompts) Load(name string) (string, error) { return "", nil }
-
-// Test doubles for the RAGTools method tests. Kept local to the agentic_rag package so
-// the tests can call Run/ComposeAnswer (which live here) without importing agent
-// from the runtime test package (that would be an import cycle).
-
-// fakeModel replays a scripted sequence of replies, so the session's control
-// flow can be driven without a provider.
-type fakeModel struct {
- replies []*runtime.ModelReply
- calls int
- messages []schema.Message
-}
-
-func (f *fakeModel) Complete(_ context.Context, msgs []schema.Message, _ []runtime.ToolSpec) (*runtime.ModelReply, error) {
- f.messages = msgs
- // The pre-flight probe (probeChatModel) opens every non-naive run with one
- // minimal "ping". It is infrastructure, not script: answer it without
- // consuming a scripted reply, so a test's reply sequence stays what the test
- // wrote. A test that wants the PROBE itself to fail uses errModel instead.
- if len(msgs) == 1 && msgs[0].Role == schema.User && msgs[0].Content == probePrompt {
- return &runtime.ModelReply{Content: "ok"}, nil
- }
- if f.calls >= len(f.replies) {
- return &runtime.ModelReply{Content: `<state>{"new_states": []}</state>`}, nil
- }
- r := f.replies[f.calls]
- f.calls++
- return r, nil
-}
-
-// lastUserPrompt returns the content of the most recent user message, so tests
-// can assert on the prompt the model actually received.
-func (f *fakeModel) lastUserPrompt() string {
- for i := len(f.messages) - 1; i >= 0; i-- {
- if f.messages[i].Role == schema.User {
- return f.messages[i].Content
- }
- }
- return ""
-}
-
-// corpusRetriever answers every query from a fixed corpus.
-type corpusRetriever struct{ calls []string }
-
-func (c *corpusRetriever) Retrieve(_ context.Context, req runtime.RetrieveRequest) ([]map[string]any, error) {
- c.calls = append(c.calls, req.Query)
- return []map[string]any{{
- "chunk_id": "c1",
- "content": "Culdcept was created by OmiyaSoft and released in 1999.",
- "doc_id": "doc-culdcept",
- "docnm_kwd": "Culdcept History",
- "dataset_id": "kb1",
- }}, nil
-}
-
-// emptyRetriever simulates a corpus with no matches.
-type emptyRetriever struct{}
-
-func (emptyRetriever) Retrieve(_ context.Context, _ runtime.RetrieveRequest) ([]map[string]any, error) {
- return nil, nil
-}
-
-func TestRunLowModeIsSinglePass(t *testing.T) {
- r := &corpusRetriever{}
- resp := Rag(context.Background(), RAGTools{Retriever: r}, runtime.RunRequest{
- Question: "Who created Culdcept?",
- ThinkingMode: "low",
- DatasetIDs: []string{"kb1"},
- UseCompiled: true,
- })
- if resp.EmptyResult {
- t.Fatal("low mode must retrieve evidence")
- }
- if len(resp.Chunks) != 1 {
- t.Fatalf("chunks = %d, want 1", len(resp.Chunks))
- }
- if len(r.calls) != 1 {
- t.Errorf("retrieval calls = %d, want 1 (low is a single pass)", len(r.calls))
- }
- if resp.Mode.Agentic {
- t.Error("low must resolve to a non-agentic spec")
- }
- if len(resp.DocAggs) != 1 || resp.DocAggs[0]["doc_id"] != "doc-culdcept" {
- t.Fatalf("doc_aggs = %v", resp.DocAggs)
- }
- if resp.DocAggs[0]["doc_name"] != "Culdcept History" {
- t.Errorf("doc_name = %v, want 'Culdcept History'", resp.DocAggs[0]["doc_name"])
- }
- if runtime.MemorySize(resp.Kbinfos) != 1 {
- t.Errorf("memory size = %d, want 1", runtime.MemorySize(resp.Kbinfos))
- }
-}
-
-func TestRunUnknownModeFallsBackToNaive(t *testing.T) {
- r := &corpusRetriever{}
- resp := Rag(context.Background(), RAGTools{Retriever: r}, runtime.RunRequest{
- Question: "Who created Culdcept?",
- ThinkingMode: "not-a-real-mode",
- DatasetIDs: []string{"kb1"},
- })
- if resp.Mode.Label != "naive" {
- t.Errorf("mode = %q, want naive", resp.Mode.Label)
- }
- if resp.EmptyResult {
- t.Error("naive must still retrieve (it degrades, not fails)")
- }
-}
-
-// TestNaiveUsesFlatEvidenceAndOwnComposition pins the semantics that separate the naive
-// path from low/direct. Naive is NOT "direct_search minus formalize": it does a plain
-// retrieve with no weighted keyword extraction and composes under a short fixed system
-// prompt from flat "[i] content" evidence, never FinalAnswerSystem / kb_prompt. Routing
-// naive through runDirect + composeFinalAnswer silently
-// substituted the agentic composition, which is what this test guards.
-func TestNaiveUsesFlatEvidenceAndOwnComposition(t *testing.T) {
- mdl := &fakeModel{replies: []*runtime.ModelReply{{Content: "OmiyaSoft [1]."}}}
- r := &corpusRetriever{}
- resp := Rag(context.Background(), RAGTools{Retriever: r, Model: mdl}, runtime.RunRequest{
- Question: "Who created Culdcept?",
- ThinkingMode: "not-a-real-mode",
- DatasetIDs: []string{"kb1"},
- UseCompiled: true, // naive must ignore compiled expansion
- })
-
- if resp.Mode.Label != "naive" {
- t.Fatalf("mode = %q, want naive", resp.Mode.Label)
- }
- if resp.EmptyResult || len(resp.Chunks) != 1 {
- t.Fatalf("naive must retrieve evidence (chunks=%d, empty=%v)", len(resp.Chunks), resp.EmptyResult)
- }
- // one PLAIN retrieve — no ExtractWeightedKeywords, so the model
- // is called exactly once. The old runDirect path spent a second call on
- // keyword extraction before composing.
- if mdl.calls != 1 {
- t.Fatalf("model calls = %d, want 1 (naive composes without extracting keywords)", mdl.calls)
- }
-
- // flat "[i] content" evidence, not kb_prompt's "ID: n" blocks.
- got := mdl.lastUserPrompt()
- if !strings.Contains(got, "[1] Culdcept was created") {
- t.Errorf("evidence = %q, want flat \"[1] <content>\"", got)
- }
- if strings.Contains(got, "ID: 1") {
- t.Errorf("evidence = %q, must not use kb_prompt's \"ID: n\" blocks", got)
- }
- if resp.Answer != "OmiyaSoft [1]." {
- t.Errorf("answer = %q, want the naive composition", resp.Answer)
- }
-}
-
-func TestRunAgenticDegradesToDirectWithoutModel(t *testing.T) {
- r := &corpusRetriever{}
- resp := Rag(context.Background(), RAGTools{Retriever: r}, runtime.RunRequest{
- Question: "Who created Culdcept?",
- ThinkingMode: "high",
- DatasetIDs: []string{"kb1"},
- })
- if resp.EmptyResult {
- t.Error("agentic mode without a model must degrade to direct search")
- }
-}
-
-func TestRunAgenticSeedsSlotsAndRunsSession(t *testing.T) {
- r := &corpusRetriever{}
- mdl := &fakeModel{replies: []*runtime.ModelReply{
- {Content: `{"slots": [{"id": 0, "type": "entity", "clues": ["creator"]}, {"id": 1, "type": "date", "clues": ["released"]}], "first_queries": ["creator"]}`},
- {Content: "", ToolCalls: []runtime.ToolCall{{ID: "c0", Name: "retrieve", Args: map[string]any{"query": []any{"Culdcept creator"}}}}},
- {Content: `<state>{"new_states":[{"state":[{"id":0,"candidate":"OmiyaSoft","candidate_strength":0.95}]}]}</state>`},
- }}
- resp := Rag(context.Background(), RAGTools{Retriever: r, Model: mdl}, runtime.RunRequest{
- Question: "Who created Culdcept and when?",
- ThinkingMode: "medium",
- DatasetIDs: []string{"kb1"},
- })
- if resp.EmptyResult {
- t.Fatal("the session's retrieval must land in kbinfos")
- }
- if len(resp.Slots) != 2 {
- t.Fatalf("slots = %d, want 2 (from the seeded table)", len(resp.Slots))
- }
- filled := 0
- for _, v := range resp.Slots {
- if v.Filled() {
- filled++
- }
- }
- if filled != 1 {
- t.Errorf("filled slots = %d, want 1 (the patched candidate)", filled)
- }
- if len(r.calls) == 0 {
- t.Error("the retrieve tool call never reached the retriever")
- }
-}
-
-func TestRunNeverPanicsWithoutBackend(t *testing.T) {
- resp := Rag(context.Background(), RAGTools{}, runtime.RunRequest{
- Question: "anything",
- ThinkingMode: "low",
- DatasetIDs: []string{"kb1"},
- })
- if !resp.EmptyResult {
- t.Error("an unavailable backend must yield an empty result, not an error")
- }
-}
-
-func TestRunLowReturnsComposedAnswer(t *testing.T) {
- mdl := &fakeModel{replies: []*runtime.ModelReply{
- // formalize (single-turn keyword extraction) and the low graph's direct search each
- // extract keywords in their own call, so
- // the composed answer is the third reply.
- {Content: `{"entity": ["Culdcept"], "aliases": [], "fact_type": [], "qualifiers": []}`},
- {Content: `{"entity": ["Culdcept"], "aliases": [], "fact_type": [], "qualifiers": []}`},
- {Content: "Culdcept was created by OmiyaSoft and released in 1999 [1]."},
- }}
- resp := Rag(context.Background(), RAGTools{
- Retriever: &corpusRetriever{},
- Model: mdl,
- }, runtime.RunRequest{
- Question: "Who created Culdcept?",
- ThinkingMode: "low",
- DatasetIDs: []string{"kb1"},
- })
- if resp.EmptyResult {
- t.Fatal("low must retrieve evidence")
- }
- if resp.Answer == "" {
- t.Fatal("low must return a composed answer (was empty before the composition node)")
- }
- if !strings.Contains(resp.Answer, "OmiyaSoft") {
- t.Errorf("answer = %q, want it grounded in the retrieved evidence", resp.Answer)
- }
- if len(resp.Chunks) == 0 || len(resp.DocAggs) == 0 {
- t.Error("chunks/aggs must be reported alongside the answer")
- }
-}
-
-func TestRunUnknownModeReturnsComposedAnswer(t *testing.T) {
- mdl := &fakeModel{replies: []*runtime.ModelReply{{Content: "OmiyaSoft [1]"}}}
- resp := Rag(context.Background(), RAGTools{
- Retriever: &corpusRetriever{},
- Model: mdl,
- }, runtime.RunRequest{
- Question: "Who created Culdcept?",
- ThinkingMode: "not-a-mode",
- DatasetIDs: []string{"kb1"},
- })
- if resp.Mode.Label != "naive" {
- t.Fatalf("mode = %q, want naive", resp.Mode.Label)
- }
- if resp.Answer == "" {
- t.Error("naive must compose an answer")
- }
-}
-
-func TestRunEmptyResponseShortCircuits(t *testing.T) {
- mdl := &fakeModel{}
- var buf bytes.Buffer
- resp := Rag(context.Background(), RAGTools{
- Retriever: &emptyRetriever{},
- Model: mdl,
- EmptyResponse: "I don't have enough information.",
- Logger: log.New(&buf, "", 0),
- }, runtime.RunRequest{
- Question: "Who created Culdcept?",
- ThinkingMode: "low",
- DatasetIDs: []string{"kb1"},
- })
- if !resp.EmptyResult {
- t.Fatal("expected no evidence")
- }
- if resp.Answer != "I don't have enough information." {
- t.Errorf("answer = %q, want the configured empty response", resp.Answer)
- }
- if mdl.calls != 0 {
- t.Errorf("model calls = %d, want 0 (short-circuited before composition)", mdl.calls)
- }
- // The short circuit is narrated, and the line says what was returned AND that
- // no model was called — the two facts a reader needs to know why there is no
- // composed answer.
- want := "[Composing the answer] No supporting evidence was found, so the configured empty response is returned without calling the model."
- if !strings.Contains(buf.String(), want) {
- t.Errorf("think line missing %q; log:\n%s", want, buf.String())
- }
- // This is the LOW graph, whose last node has no [Finalize] step of its own, so
- // the compose kickoff is the only line saying an answer is being written — it
- // must stay. The question used to be quoted here too, which in this branch (no
- // question threaded through) rendered as an empty "".
- if got := buf.String(); !strings.Contains(got, "[Composing the answer] Composing the answer from 0 gathered passages.") {
- t.Errorf("compose kickoff line missing; log:\n%s", got)
- }
-}
-
-// TestAgenticFinalizeMarksItsComposition pins the first half of that split: the
-// finalize node hands the compose closure a context that SAYS the [Finalize] step
-// was reported, which is what lets the compose stay silent about the evidence.
-func TestAgenticFinalizeMarksItsComposition(t *testing.T) {
- var announced []bool
- st := NewAgenticState("q", "", 3, nil)
- st.KB = &runtime.Kbinfos{}
- deps := RAGTools{Finalize: func(fctx context.Context, partial, empty bool, question string) {
- announced = append(announced, finalizeAnnounced(fctx))
- }}
-
- formalizeAnswerNode(context.Background(), deps, st, nil)
-
- if len(announced) != 1 || !announced[0] {
- t.Fatalf("the finalize node must mark the composition it triggers; got %v", announced)
- }
-}
-
-// TestComposeKickoffFollowsTheAnnouncement pins the second half, in both
-// directions: when [Finalize] has been reported the compose must NOT repeat its
-// numbers, and when it has not — the low graph's last node, every fallback — the
-// kickoff line is the only thing saying an answer is being written, so it stays.
-//
-// The composition is driven into its no-evidence short circuit so the assertion
-// needs no model: the kickoff step is emitted before that branch either way.
-func TestComposeKickoffFollowsTheAnnouncement(t *testing.T) {
- cases := []struct {
- name string
- mark bool
- want bool
- }{
- {"after-finalize", true, false},
- {"without-finalize", false, true},
- }
- for _, tc := range cases {
- var lines []string
- ctx := runtime.WithSteps(context.Background(),
- runtime.StepReporter{Text: func(line string) { lines = append(lines, line) }})
- if tc.mark {
- ctx = markFinalizeAnnounced(ctx)
- }
- ComposeAnswerWith(ctx, AnswerDeps{EmptyResponse: "nothing found"},
- &runtime.Kbinfos{}, "q", false, false, true)
-
- got := strings.Join(lines, "\n")
- has := strings.Contains(got, "Composing the answer from")
- if has != tc.want {
- t.Errorf("%s: kickoff line present = %v, want %v; steps:\n%s", tc.name, has, tc.want, got)
- }
- // The short circuit is reported on both paths: "no model was called" is a
- // fact nothing else carries.
- if !strings.Contains(got, "No supporting evidence was found") {
- t.Errorf("%s: the short circuit must be narrated; steps:\n%s", tc.name, got)
- }
- }
-}
-
-func TestRunCanDisableComposition(t *testing.T) {
- no := false
- resp := Rag(context.Background(), RAGTools{
- Retriever: &corpusRetriever{},
- Model: &fakeModel{},
- ComposeAnswer: &no,
- }, runtime.RunRequest{
- Question: "Who created Culdcept?",
- ThinkingMode: "low",
- DatasetIDs: []string{"kb1"},
- })
- if resp.Answer != "" {
- t.Errorf("answer = %q, want empty when composition is disabled", resp.Answer)
- }
- if len(resp.Chunks) == 0 {
- t.Error("evidence must still be returned")
- }
-}
-
-func TestRunFormalizesFromConversation(t *testing.T) {
- mdl := &fakeModel{replies: []*runtime.ModelReply{
- {Content: `{"question": "When was Culdcept released?", "keywords": "Culdcept, release, released"}`},
- {Content: "Culdcept was released in 1999 [1]."},
- }}
- retriever := &corpusRetriever{}
- resp := Rag(context.Background(), RAGTools{
- Retriever: retriever,
- Model: mdl,
- Messages: []schema.Message{
- *schema.UserMessage("Who created Culdcept?"),
- *schema.AssistantMessage("OmiyaSoft.", nil),
- *schema.UserMessage("When was it released?"),
- },
- }, runtime.RunRequest{
- Question: "When was it released?",
- ThinkingMode: "low",
- DatasetIDs: []string{"kb1"},
- })
- if len(retriever.calls) == 0 {
- t.Fatal("retriever was never called")
- }
- if !strings.HasPrefix(retriever.calls[0], "When was Culdcept released?") {
- t.Errorf("retrieval query = %q, want it to start with the formalized question", retriever.calls[0])
- }
- if !strings.Contains(retriever.calls[0], "Culdcept") {
- t.Errorf("retrieval query = %q, want the formalized keywords appended", retriever.calls[0])
- }
- if resp.Answer == "" {
- t.Error("answer must be composed after formalization")
- }
-}
-
-func TestRunFormalizeDisabledByDefault(t *testing.T) {
- mdl := &fakeModel{replies: []*runtime.ModelReply{{Content: "answer [1]"}}}
- Rag(context.Background(), RAGTools{
- Retriever: &corpusRetriever{},
- Model: mdl,
- Messages: []schema.Message{
- *schema.UserMessage("Who created Culdcept?"),
- *schema.AssistantMessage("OmiyaSoft.", nil),
- *schema.UserMessage("When was it released?"),
- },
- }, runtime.RunRequest{
- Question: "When was it released?",
- ThinkingMode: "low",
- DatasetIDs: []string{"kb1"},
- })
- if mdl.calls != 1 {
- t.Errorf("model calls = %d, want 1 (formalize is opt-in)", mdl.calls)
- }
-}
-
-func TestRunAgenticComposesFromResearchFindings(t *testing.T) {
- kb := &runtime.Kbinfos{
- Chunks: []map[string]any{{"chunk_id": "c1", "content": "Culdcept: OmiyaSoft, 1999.", "similarity": 0.9}},
- PreSummary: "Culdcept was created by OmiyaSoft and released in 1999.",
- }
- mdl := &fakeModel{replies: []*runtime.ModelReply{{Content: "Culdcept was created by OmiyaSoft and released in 1999 [1]."}}}
- res := ComposeAnswer(context.Background(), AnswerDeps{Model: mdl}, kb, "Who created Culdcept?", false, false)
- if res.Failed {
- t.Fatal("composition must succeed")
- }
- if !strings.Contains(mdl.lastUserPrompt(), "Culdcept was created by OmiyaSoft and released in 1999.") {
- t.Errorf("prompt missing the research findings:\n%s", mdl.lastUserPrompt())
- }
-}
-
-// errModel fails every completion, standing in for a provider outage (an
-// exhausted quota, a 5xx): the compose call gets no reply at all.
-type errModel struct{ err error }
-
-func (m errModel) Complete(context.Context, []schema.Message, []runtime.ToolSpec) (*runtime.ModelReply, error) {
- return nil, m.err
-}
-
-// TestRagPreflightFailsFastWithoutThinking pins the pre-flight: a provider that
-// cannot answer AT ALL must be reported before the run narrates anything, in the
-// same `**ERROR**: …` shape naive mode uses. Without it the failure surfaced
-// only at compose time, after a whole research run's worth of think lines.
-func TestRagPreflightFailsFastWithoutThinking(t *testing.T) {
- var think []string
- var streamed []string
- resp := Rag(context.Background(), RAGTools{
- Model: errModel{err: fmt.Errorf("minimax API error: insufficient balance")},
- Steps: runtime.StepReporter{Text: func(line string) { think = append(think, line) }},
- AnswerSink: &AnswerSink{OnDelta: func(delta string, isThink bool) {
- if !isThink {
- streamed = append(streamed, delta)
- }
- }},
- }, runtime.RunRequest{Question: "who?", ThinkingMode: "high"})
-
- if !strings.HasPrefix(resp.Answer, "**ERROR**: ") || !strings.Contains(resp.Answer, "insufficient balance") {
- t.Fatalf("answer = %q, want **ERROR** + the provider's message", resp.Answer)
- }
- if len(think) != 0 {
- t.Errorf("think block must stay closed, got %v", think)
- }
- if len(streamed) != 0 {
- t.Errorf("the error must not be streamed as an answer delta, got %v", streamed)
- }
-}
-
-// blockingModel hangs until its context is done, standing in for a provider
-// that is slow to first token rather than dead.
-type blockingModel struct{}
-
-func (blockingModel) Complete(ctx context.Context, _ []schema.Message, _ []runtime.ToolSpec) (*runtime.ModelReply, error) {
- <-ctx.Done()
- return nil, ctx.Err()
-}
-
-// TestProbeSlowProviderIsNotAFailure pins the timeout split: a probe that only
-// ran out of ITS OWN budget must not read as a provider failure — a slow model
-// is not a dead one, and failing the whole run here would turn a long first
-// token into an outage.
-func TestProbeSlowProviderIsNotAFailure(t *testing.T) {
- err := probeChatModelWithin(context.Background(), RAGTools{Model: blockingModel{}}, 5*time.Millisecond)
- if !errors.Is(err, errProbeTimeout) {
- t.Fatalf("err = %v, want errProbeTimeout (slow, not dead)", err)
- }
-}
-
-// TestComposeFailureIsNotStreamedAsAnAnswerDelta pins the delivery rule: the
-// error answer goes out once, in the final result. Sending it through the sink
-// as well made the client render it twice (measured: 221 answer bytes for one
-// error + the research-status note, shown as two concatenated ERROR blocks).
-func TestComposeFailureIsNotStreamedAsAnAnswerDelta(t *testing.T) {
- var streamed []string
- deps := RAGTools{
- Model: errModel{err: fmt.Errorf("minimax API error: insufficient balance")},
- AnswerSink: &AnswerSink{OnDelta: func(delta string, isThink bool) { streamed = append(streamed, delta) }},
- }
- resp := &RunResponse{}
- kb := &runtime.Kbinfos{Chunks: []map[string]any{{"chunk_id": "c1", "content": "body"}}}
-
- composeFinalAnswer(context.Background(), deps, runtime.RunRequest{Question: "who?"}, kb, resp,
- _LOG, false, false, "who?")
-
- if !strings.HasPrefix(resp.Answer, "**ERROR**: ") {
- t.Fatalf("answer = %q, want **ERROR** + the provider's message", resp.Answer)
- }
- if len(streamed) != 0 {
- t.Errorf("a failed compose must not stream an answer delta, got %v", streamed)
- }
-}
-
-// TestComposeFailureReportsProviderErrorInThinkBlock pins the Go-only deviation
-// from Python: when the compose call fails, the think block carries a bounded
-// summary of the provider's own error. Python prints the bare fallback and keeps
-// the cause in the log, which made an exhausted quota look like a RAG bug to the
-// user waiting for an answer.
-func TestComposeFailureReportsProviderErrorInThinkBlock(t *testing.T) {
- var think []string
- ctx := runtime.WithSteps(context.Background(), runtime.StepReporter{
- Text: func(line string) { think = append(think, line) },
- })
- kb := &runtime.Kbinfos{Chunks: []map[string]any{{"chunk_id": "c1", "content": "body"}}}
- mdl := errModel{err: fmt.Errorf("minimax API error: 已达到 Token Plan 用量上限\n请升级 Token Plan 套餐或购买积分补充用量")}
-
- res := ComposeAnswerWith(ctx, AnswerDeps{Model: mdl}, kb, "who?", false, false, false)
- if !res.Failed {
- t.Fatalf("res = %+v, want Failed=true", res)
- }
- // The answer reads like a naive-mode failure: the classic `**ERROR**: …`
- // shape carrying the provider's own words, not Python's generic sentence.
- if !strings.HasPrefix(res.Answer, "**ERROR**: ") ||
- !strings.Contains(res.Answer, "已达到 Token Plan 用量上限") {
- t.Errorf("answer = %q, want **ERROR** + the provider's message", res.Answer)
- }
-
- joined := strings.Join(think, "")
- if !strings.Contains(joined, "已达到 Token Plan 用量上限") {
- t.Errorf("think block must carry the provider's error, got %q", joined)
- }
- if strings.Contains(joined, "\n") {
- t.Errorf("the summary must be a single line, got %q", joined)
- }
-}
-
-// TestComposePublishesCiteChunkIDs pins the evidence contract the chat pipeline
-// resolves citations against: the compose numbers the blocks 0-based (the client
-// indexes reference.chunks with the marker's number, and the answer is streamed
-// before any rewrite could apply) and publishes THAT order — similarity-ranked
-// and capped — on runtime.Kbinfos.CiteChunkIDs, because the reference is built in
-// that order and a marker against the pool by position would land on the wrong
-// chunk (or past the end when the pool is shorter than the render cap, which is
-// how an agentic answer came back with no reference at all).
-func TestComposePublishesCiteChunkIDs(t *testing.T) {
- kb := &runtime.Kbinfos{
- Chunks: []map[string]any{
- {"chunk_id": "c1", "content": "the wolf is grey", "similarity": 0.1},
- {"chunk_id": "c2", "content": "the wolf is small", "similarity": 0.9},
- // No content: kbpBlock renders no block for it, so it must not take a
- // slot in the published list either — an id for it would put every
- // marker after it one block off the passage the model cited.
- {"chunk_id": "c3", "content": " ", "similarity": 0.5},
- },
- }
- mdl := &fakeModel{replies: []*runtime.ModelReply{{Content: "it is grey [ID:0]."}}}
- if res := ComposeAnswer(context.Background(), AnswerDeps{Model: mdl}, kb, "what colour is it?", false, false); res.Failed {
- t.Fatal("composition must succeed")
- }
- got := mdl.lastUserPrompt()
- for _, want := range []string{"ID: 0", "ID: 1"} {
- if !strings.Contains(got, want) {
- t.Errorf("evidence missing %q:\n%s", want, got)
- }
- }
- if strings.Contains(got, "ID: 2") {
- t.Errorf("evidence numbering ran past the rendered blocks:\n%s", got)
- }
- // The rendered order is the similarity ranking, not the pool order: the
- // higher-similarity chunk is block 0. The blank chunk sits between the two
- // rendered blocks and contributes no id.
- want := []string{"c2", "c1"}
- if !reflect.DeepEqual(kb.CiteChunkIDs, want) {
- t.Fatalf("CiteChunkIDs = %v, want one id per rendered block in render order %v", kb.CiteChunkIDs, want)
- }
-}
-
-// TestComposeSystemDeclaresZeroBasedEvidence pins the prompt-side guard for the
-// evidence numbering: the compose renders 0-based block ids and the client indexes
-// reference.chunks with the marker's number, so the model has to be told — one on
-// "the first source is 1" cites the second passage and never the first. The rule
-// belongs to the callers that render 0-based blocks, not to CitationPrompt:
-// CitationPrompt is shared with the agent canvas, which numbers its blocks by hash
-// id, so a base declared there would be wrong.
-func TestComposeSystemDeclaresZeroBasedEvidence(t *testing.T) {
- sys := AnswerDeps{}.composeSystem()
- if !strings.Contains(sys, "the FIRST block is [ID:0]") {
- t.Fatalf("compose system prompt must state the 0-based evidence numbering:\n%s", sys)
- }
- if strings.Contains(prompts.CitationPrompt(""), "the FIRST block is [ID:0]") {
- t.Fatal("CitationPrompt must not declare a base: the canvas render numbers its blocks by hash id")
- }
-}
-
-// TestAnswerPromptCarriesTargetContract pins the EXTREME-SELECTION guardrail
-// . Without it a "longest/shortest/most…" question tends to be
-// answered with the most common or first-listed candidate rather than the
-// extreme one, because nothing asks the model to compare.
-func TestAnswerPromptCarriesTargetContract(t *testing.T) {
- kb := &runtime.Kbinfos{
- Chunks: []map[string]any{{"chunk_id": "c1", "content": "river A is 300km; river B is 900km"}},
- }
- mdl := &fakeModel{replies: []*runtime.ModelReply{{Content: "river B [1]."}}}
- if res := ComposeAnswer(context.Background(), AnswerDeps{Model: mdl}, kb,
- "Which river is the longest?", false, false); res.Failed {
- t.Fatal("composition must succeed")
- }
- got := mdl.lastUserPrompt()
- if !strings.Contains(got, "Answer Target Contract:") {
- t.Errorf("prompt missing the Answer Target Contract:\n%s", got)
- }
- if !strings.Contains(got, "EXTREME-SELECTION") || !strings.Contains(got, "EXTREME one") {
- t.Errorf("prompt missing the EXTREME-SELECTION guardrail:\n%s", got)
- }
-}
-
-// TestAnswerPromptNoEvidenceInstructions pins: when the call is
-// not short-circuited by empty_response, the prompt must still tell the model how
-// to degrade — from the research summary if there is one, otherwise with an
-// explicit insufficiency statement.
-func TestAnswerPromptNoEvidenceInstructions(t *testing.T) {
- // No chunks, no summary -> insufficiency statement.
- mdl := &fakeModel{replies: []*runtime.ModelReply{{Content: "unknown [1]."}}}
- ComposeAnswer(context.Background(), AnswerDeps{Model: mdl}, &runtime.Kbinfos{}, "q?", false, false)
- if got := mdl.lastUserPrompt(); !strings.Contains(got, "No supporting evidence was retrieved") {
- t.Errorf("prompt missing the no-evidence instruction:\n%s", got)
- }
-
- // No chunks but a draft exists -> answer from the summary, do not refuse.
- kb := &runtime.Kbinfos{PreSummary: "partial findings"}
- mdl2 := &fakeModel{replies: []*runtime.ModelReply{{Content: "x [1]."}}}
- ComposeAnswer(context.Background(), AnswerDeps{Model: mdl2}, kb, "q?", false, false)
- if got := mdl2.lastUserPrompt(); !strings.Contains(got, "The retrieved passages are limited") {
- t.Errorf("prompt missing the limited-evidence instruction:\n%s", got)
- }
-}
-
-// TestAnswerPromptOrdersPartialPreamble pins: the partial
-// preamble sits AFTER the research summary and BEFORE the evidence — not at the
-// front of the prompt, where it would precede the question.
-func TestAnswerPromptOrdersPartialPreamble(t *testing.T) {
- kb := &runtime.Kbinfos{
- Chunks: []map[string]any{{"chunk_id": "c1", "content": "body"}},
- PreSummary: "the draft",
- }
- mdl := &fakeModel{replies: []*runtime.ModelReply{{Content: "x [1]."}}}
- ComposeAnswer(context.Background(), AnswerDeps{Model: mdl}, kb, "q?", true, false)
-
- got := mdl.lastUserPrompt()
- iQ := strings.Index(got, "Question:")
- iSummary := strings.Index(got, "Research Summary")
- iPartial := strings.Index(got, strings.TrimSpace(runtime.PartialAnswerPreamble))
- iEvidence := strings.Index(got, "Evidence:")
- if iQ < 0 || iSummary < 0 || iPartial < 0 || iEvidence < 0 {
- t.Fatalf("prompt missing one of the parts:\n%s", got)
- }
- if !(iQ < iSummary && iSummary < iPartial && iPartial < iEvidence) {
- t.Errorf("order = question@%d summary@%d partial@%d evidence@%d, want question < summary < partial < evidence",
- iQ, iSummary, iPartial, iEvidence)
- }
-}
-
-// TestComposeAnswerWithAttachesUserImages pins the non-outer path fix: when
-// AnswerDeps.UserImages carries vision-gated data URIs surviving gateImageAttachments,
-// the final-answer user message must be multimodal — an image_url content block beside the
-// text question — so the compose model sees the images on the direct fallback (called with
-// the original multimodal messages). Without this the
-// direct (no-outer) path composed from a text-only prompt and silently dropped
-// the picture.
-func TestComposeAnswerWithAttachesUserImages(t *testing.T) {
- mdl := &fakeModel{replies: []*runtime.ModelReply{{Content: "answer [1]."}}}
- kb := &runtime.Kbinfos{Chunks: []map[string]any{{"chunk_id": "c1", "content": "evidence"}}}
- if res := ComposeAnswerWith(context.Background(), AnswerDeps{
- Model: mdl,
- UserImages: []string{"data:image/png;base64,AAAA"},
- }, kb, "What is in this image?", false, false, false); res.Failed {
- t.Fatal("composition must succeed")
- }
- var user *schema.Message
- for i := len(mdl.messages) - 1; i >= 0; i-- {
- if mdl.messages[i].Role == schema.User {
- user = &mdl.messages[i]
- break
- }
- }
- if user == nil {
- t.Fatal("no user message captured")
- }
- if len(user.UserInputMultiContent) == 0 {
- t.Fatalf("expected multimodal content blocks on the user message; content=%q", user.Content)
- }
- hasImage := false
- for _, p := range user.UserInputMultiContent {
- if p.Type == schema.ChatMessagePartTypeImageURL {
- hasImage = true
- }
- }
- if !hasImage {
- t.Errorf("expected an image_url block in the user message; got %+v", user.UserInputMultiContent)
- }
- // The text question must accompany the image (leading text block), so the
- // model reads both the picture and the question.
- sawText := user.Content != ""
- for _, p := range user.UserInputMultiContent {
- if p.Type == schema.ChatMessagePartTypeText && p.Text != "" {
- sawText = true
- }
- }
- if !sawText {
- t.Errorf("expected the question text to accompany the image; content=%q blocks=%+v", user.Content, user.UserInputMultiContent)
- }
-}
-
-// TestComposeAnswerWithNoImagesStaysTextOnly pins the default: without
-// UserImages the prompt is the plain-text question (no multimodal block), so
-// non-vision callers are unaffected.
-func TestComposeAnswerWithNoImagesStaysTextOnly(t *testing.T) {
- mdl := &fakeModel{replies: []*runtime.ModelReply{{Content: "answer [1]."}}}
- kb := &runtime.Kbinfos{Chunks: []map[string]any{{"chunk_id": "c1", "content": "evidence"}}}
- if res := ComposeAnswerWith(context.Background(), AnswerDeps{Model: mdl}, kb, "plain?", false, false, false); res.Failed {
- t.Fatal("composition must succeed")
- }
- user := mdl.messages[len(mdl.messages)-1]
- if len(user.UserInputMultiContent) != 0 {
- t.Errorf("expected no multimodal blocks when there are no images; got %+v", user.UserInputMultiContent)
- }
-}
-
-// userTurnAt returns the Content of the last message of the idx-th Complete
-// call (0-based), i.e. the user turn of that attempt.
-func userTurnAt(m *scriptedModel, idx int) string {
- m.mu.Lock()
- defer m.mu.Unlock()
- if idx >= len(m.seen) || len(m.seen[idx]) == 0 {
- return ""
- }
- return m.seen[idx][len(m.seen[idx])-1].Content
-}
-
-// TestGenJSONUsesOutputNewlineUserTurn locks the contract that every JSONModel
-// round: "Output:\n" user turn. It was previously an
-// invisible convention buried inside jsonModelAdapter.GenJSON — a second JSONModel
-// implementation could silently drop it without any test catching it. See the
-// query_rewriter.go note (diff ②).
-func TestGenJSONUsesOutputNewlineUserTurn(t *testing.T) {
- mdl := &scriptedModel{}
- mdl.push("{\"ok\": true}")
-
- if _, err := (&jsonModelAdapter{inner: mdl}).GenJSON(context.Background(), "Render JSON."); err != nil {
- t.Fatalf("GenJSON failed: %v", err)
- }
- if got := userTurnAt(mdl, 0); got != "Output:\n" {
- t.Errorf("first-round user turn = %q, want exactly %q (the gen_json separator)", got, "Output:\n")
- }
-}
-
-func TestGenJSONRetriesMalformedReplyWithCorrection(t *testing.T) {
- mdl := &scriptedModel{}
- // First round: fenced but unparseable JSON. Second round: well-formed.
- mdl.push("```json\n{\"queries\": [broken")
- mdl.push("{\"queries\": [{\"query\": \"q1\"}]}")
-
- got, err := (&jsonModelAdapter{inner: mdl}).GenJSON(context.Background(), "Rewrite the query.")
- if err != nil {
- t.Fatalf("GenJSON failed after corrective retry: %v", err)
- }
- m, ok := got.(map[string]any)
- if !ok {
- t.Fatalf("GenJSON returned %T, want map[string]any", got)
- }
- queries, ok := m["queries"].([]any)
- if !ok || len(queries) != 1 {
- t.Fatalf("queries = %#v, want a 1-element list", m["queries"])
- }
-
- // The retry must feed the previous answer and the parse error back to the
- // model, mirroring gen_json's corrective prompt.
- turn := userTurnAt(mdl, 1)
- for _, want := range []string{"Generated JSON is as following:", "broken", "But exception while loading:", "Please reconsider and correct it."} {
- if !strings.Contains(turn, want) {
- t.Errorf("retry user turn missing %q; got: %s", want, turn)
- }
- }
-}
-
-func TestGenJSONStripsThinkAndFenceOnFirstTry(t *testing.T) {
- mdl := &scriptedModel{}
- mdl.push("Sure.\n<thinking>drafting...</thinking>\n```json\n{\"ok\": true}\n```\n")
-
- got, err := (&jsonModelAdapter{inner: mdl}).GenJSON(context.Background(), "Render JSON.")
- if err != nil {
- t.Fatalf("GenJSON failed on a wrapper-padded reply: %v", err)
- }
- m, ok := got.(map[string]any)
- if !ok {
- t.Fatalf("GenJSON returned %T, want map[string]any", got)
- }
- if m["ok"] != true {
- t.Errorf(`m["ok"] = %#v, want true`, m["ok"])
- }
- if len(mdl.seen) != 1 {
- t.Errorf("expected a single attempt, got %d", len(mdl.seen))
- }
-}
-
-// TestGenJSONFitsPromptToContextWindow: the fitted prompt is bounded by the model's
-// context window, so an oversized system prompt must be trimmed BEFORE the first call
-// instead of being sent verbatim (the provider would reject it).
-func TestGenJSONFitsPromptToContextWindow(t *testing.T) {
- huge := strings.Repeat("token ", 40000) // far past any window
- mdl := &scriptedModel{}
- mdl.push(`{"ok": true}`)
-
- if _, err := (&jsonModelAdapter{inner: mdl, maxLength: 4096}).GenJSON(context.Background(), huge); err != nil {
- t.Fatalf("GenJSON failed: %v", err)
- }
- if len(mdl.seen) != 1 || len(mdl.seen[0]) == 0 {
- t.Fatalf("expected one call with messages, got %#v", mdl.seen)
- }
- sent := mdl.seen[0][0]
- if sent.Role != schema.System {
- t.Fatalf("first message role = %v, want system", sent.Role)
- }
- if len(sent.Content) >= len(huge) {
- t.Errorf("system prompt not fitted: %d bytes sent of %d", len(sent.Content), len(huge))
- }
-}
-
-// TestGenJSONAcceptsTopLevelArrayAndScalar pins json_repair parity: gen_json
-// parses ANY top-level JSON value, so an array or scalar reply must come back
-// as-is instead of being treated as a parse failure (which a map-only unmarshal
-// did before).
-func TestGenJSONAcceptsTopLevelArrayAndScalar(t *testing.T) {
- mdl := &scriptedModel{}
- mdl.push(`[{"query": "q1"}]`)
- got, err := (&jsonModelAdapter{inner: mdl}).GenJSON(context.Background(), "Render JSON.")
- if err != nil {
- t.Fatalf("array reply: %v", err)
- }
- arr, ok := got.([]any)
- if !ok || len(arr) != 1 {
- t.Fatalf("array reply = %#v, want a 1-element list", got)
- }
- if len(mdl.seen) != 1 {
- t.Errorf("array reply made %d attempt(s), want 1", len(mdl.seen))
- }
-
- mdl = &scriptedModel{}
- mdl.push(`42`)
- got, err = (&jsonModelAdapter{inner: mdl}).GenJSON(context.Background(), "Render JSON.")
- if err != nil {
- t.Fatalf("scalar reply: %v", err)
- }
- if n, ok := got.(float64); !ok || n != 42 {
- t.Fatalf("scalar reply = %#v, want 42", got)
- }
-}
-
-func TestGenJSONGivesUpAfterMaxRetries(t *testing.T) {
- mdl := &scriptedModel{}
- mdl.push("I cannot produce JSON today.")
- mdl.push("Still not JSON.")
-
- _, err := (&jsonModelAdapter{inner: mdl}).GenJSON(context.Background(), "Rewrite the query.")
- if err == nil {
- t.Fatal("GenJSON succeeded, want error after exhausting retries")
- }
- if len(mdl.seen) != 2 {
- t.Errorf("expected 2 attempts (genJSONMaxRetry), got %d", len(mdl.seen))
- }
-}
-
-// errOnceModel fails the underlying chat call and counts invocations, proving
-// chat/transport errors are NOT retried (gen_json propagates them immediately).
-type errOnceModel struct {
- n int
-}
-
-func (m *errOnceModel) Complete(_ context.Context, _ []schema.Message, _ []runtime.ToolSpec) (*runtime.ModelReply, error) {
- m.n++
- return nil, errors.New("provider down")
-}
-
-func TestGenJSONPropagatesChatErrorWithoutRetry(t *testing.T) {
- mdl := &errOnceModel{}
- _, err := (&jsonModelAdapter{inner: mdl}).GenJSON(context.Background(), "Rewrite the query.")
- if err == nil {
- t.Fatal("GenJSON succeeded, want the chat error")
- }
- if mdl.n != 1 {
- t.Errorf("chat invoked %d times, want exactly 1 (no retry on transport error)", mdl.n)
- }
-}
-
-func TestGenJSONNilModel(t *testing.T) {
- if _, err := (&jsonModelAdapter{inner: nil}).GenJSON(context.Background(), "x"); err == nil {
- t.Fatal("GenJSON with a nil inner model succeeded, want error")
- }
-}
-
-// stubDocChunks implements runtime.DocChunkLister for injection tests.
-type stubDocChunks struct{}
-
-func (stubDocChunks) DocChunks(context.Context, runtime.DocChunksRequest) ([]map[string]any, error) {
- return nil, nil
-}
-
-// TestSearchDepsForProjectsDocChunks locks the injection seam: RAGTools.DocChunks
-// must be projected onto SearchDeps so the document-level tools (summarize_document
-// / fetch_full_document) can reach a real chunk reader once the caller wires one.
-func TestSearchDepsForProjectsDocChunks(t *testing.T) {
- got := searchDepsFor(context.Background(), RAGTools{
- DocChunks: stubDocChunks{},
- }, runtime.RunRequest{}, nil, "", &runtime.Kbinfos{}, log.New(os.Stderr, "", 0))
-
- if got.DocChunks == nil {
- t.Fatal("RAGTools.DocChunks was not projected onto SearchDeps")
- }
- if _, ok := got.DocChunks.(stubDocChunks); !ok {
- t.Fatalf("SearchDeps.DocChunks = %T, want the injected stubDocChunks", got.DocChunks)
- }
-
- // Nil RAGTools.DocChunks must stay nil (disables whole-document reading).
- gotNil := searchDepsFor(context.Background(), RAGTools{}, runtime.RunRequest{}, nil, "", &runtime.Kbinfos{}, log.New(os.Stderr, "", 0))
- if gotNil.DocChunks != nil {
- t.Fatal("nil RAGTools.DocChunks must stay nil on SearchDeps")
- }
-}
-
-// These benchmarks isolate the cost Eino charges for declaring + compiling a
-// graph — exactly what the "build once, inject deps via state" alternative
-// (B2) would save. The per-run Invoke cost is identical in both designs, so it
-// is deliberately not measured here.
-
-func benchCompile(b *testing.B, build func() error) {
- b.Helper()
- b.ReportAllocs()
- for i := 0; i < b.N; i++ {
- if err := build(); err != nil {
- b.Fatal(err)
- }
- }
-}
-
-// low graph shape: 3 nodes, 3 edges, no branch.
-func BenchmarkCompileLowGraphShape(b *testing.B) {
- ctx := context.Background()
- benchCompile(b, func() error {
- g := compose.NewGraph[*runtime.RunRequest, *runtime.RunRequest]()
- node := func(c context.Context, r *runtime.RunRequest) (*runtime.RunRequest, error) { return r, nil }
- if err := g.AddLambdaNode("formalize_question", compose.InvokableLambda(node)); err != nil {
- return err
- }
- if err := g.AddLambdaNode("direct_search", compose.InvokableLambda(node)); err != nil {
- return err
- }
- if err := g.AddLambdaNode("formalize_answer", compose.InvokableLambda(node)); err != nil {
- return err
- }
- if err := g.AddEdge(compose.START, "formalize_question"); err != nil {
- return err
- }
- if err := g.AddEdge("formalize_question", "direct_search"); err != nil {
- return err
- }
- if err := g.AddEdge("direct_search", "formalize_answer"); err != nil {
- return err
- }
- if err := g.AddEdge("formalize_answer", compose.END); err != nil {
- return err
- }
- _, err := g.Compile(ctx, compose.WithGraphName("bench_low"))
- return err
- })
-}
-
-// action session shape: 3 nodes, 2 edges, 2 branches (with cycles).
-func BenchmarkCompileActionSessionShape(b *testing.B) {
- ctx := context.Background()
- benchCompile(b, func() error {
- g := compose.NewGraph[*AgenticState, *AgenticState]()
- node := func(c context.Context, s *AgenticState) (*AgenticState, error) { return s, nil }
- for _, n := range []string{"run_action", "tool", "finalize"} {
- if err := g.AddLambdaNode(n, compose.InvokableLambda(node)); err != nil {
- return err
- }
- }
- if err := g.AddEdge(compose.START, "run_action"); err != nil {
- return err
- }
- if err := g.AddEdge("finalize", compose.END); err != nil {
- return err
- }
- ends := map[string]bool{"tool": true, "finalize": true, "run_action": true, compose.END: true}
- if err := g.AddBranch("run_action", compose.NewGraphBranch(func(context.Context, *AgenticState) (string, error) {
- return "run_action", nil
- }, ends)); err != nil {
- return err
- }
- if err := g.AddBranch("tool", compose.NewGraphBranch(func(context.Context, *AgenticState) (string, error) {
- return "run_action", nil
- }, map[string]bool{"run_action": true, "finalize": true})); err != nil {
- return err
- }
- _, err := g.Compile(ctx, compose.WithGraphName("bench_session"), compose.WithMaxRunSteps(32))
- return err
- })
-}
-
-// agentic graph shape: 7 nodes, 4 edges, 5 branches.
-func BenchmarkCompileAgenticShape(b *testing.B) {
- ctx := context.Background()
- benchCompile(b, func() error {
- g := compose.NewGraph[*AgenticState, *AgenticState]()
- node := func(c context.Context, s *AgenticState) (*AgenticState, error) { return s, nil }
- for _, n := range []string{
- "formalize_question", "planner", "prefetch", "rag_agent",
- "query_rewrite", "formalize_answer", "stop",
- } {
- if err := g.AddLambdaNode(n, compose.InvokableLambda(node)); err != nil {
- return err
- }
- }
- for _, e := range [][2]string{
- {compose.START, "formalize_question"},
- {"prefetch", "rag_agent"},
- {"formalize_answer", compose.END},
- {"stop", compose.END},
- } {
- if err := g.AddEdge(e[0], e[1]); err != nil {
- return err
- }
- }
- branches := map[string]map[string]bool{
- "formalize_question": {"stop": true, "planner": true, "rag_agent": true},
- "planner": {"stop": true, "prefetch": true, "rag_agent": true},
- "prefetch": {"stop": true, "rag_agent": true},
- "rag_agent": {"stop": true, "query_rewrite": true, "formalize_answer": true},
- "query_rewrite": {"stop": true, "rag_agent": true, "formalize_answer": true},
- }
- for from, ends := range branches {
- if err := g.AddBranch(from, compose.NewGraphBranch(func(context.Context, *AgenticState) (string, error) {
- return "rag_agent", nil
- }, ends)); err != nil {
- return err
- }
- }
- _, err := g.Compile(ctx, compose.WithGraphName("bench_agentic"), compose.WithMaxRunSteps(136))
- return err
- })
-}
-
-func TestGraphRecursionLimitBounds(t *testing.T) {
- // 60 for the agentic graph, else max(25, max_loops*8).
- if got := graphRecursionLimit(true, 3); got != 60 {
- t.Fatalf("agentic limit = %d, want 60", got)
- }
- if got := graphRecursionLimit(false, 3); got != 25 {
- t.Fatalf("low limit with 3 loops = %d, want 25", got)
- }
- if got := graphRecursionLimit(false, 5); got != 40 {
- t.Fatalf("low limit with 5 loops = %d, want 40", got)
- }
-}
-
-func TestAgenticResearchRoundCostsThreeVisits(t *testing.T) {
- // One research round is rag_agent → draft → sca. If the loop counted iterations
- // instead of node visits, 20 rounds would cost 20 instead of 60 and the guard would
- // trip ~3x later.
- if got, want := agenticRoundVisits, 3; got != want {
- t.Fatalf("round cost = %d, want %d", got, want)
- }
-}
-
-func TestNewAgenticStateDoesNotArmBudget(t *testing.T) {
- // the budget is armed in the formalize_question node's
- // return, not at state creation, so formalization is not charged to it.
- st := NewAgenticState("q", "", 3, nil)
-
- if !st.Deadline.IsZero() {
- t.Fatalf("Deadline = %v, want zero until formalize_question runs", st.Deadline)
- }
-}
-
-func TestFormalizeQuestionNodeArmsBudget(t *testing.T) {
- st := NewAgenticState("when did it open?", "", 3, nil)
- // No model: the node still arms the budget, mirroring.
- formalizeQuestionNode(context.Background(), RAGTools{}, st, nil)
-
- if st.Deadline.IsZero() {
- t.Fatal("formalize_question must arm the budget")
- }
- if st.Question != "when did it open?" {
- t.Fatalf("Question = %q, want it unchanged without a model", st.Question)
- }
-}
-
-// delayedFormalizeModel answers a formalization call only after a delay, so a
-// test can tell whether that call was charged to the research budget.
-type delayedFormalizeModel struct {
- delay time.Duration
-}
-
-func (m delayedFormalizeModel) Complete(_ context.Context, _ []schema.Message, _ []runtime.ToolSpec) (*runtime.ModelReply, error) {
- time.Sleep(m.delay)
- return &runtime.ModelReply{Content: `{"keywords": ["曹操"]}`}, nil
-}
-
-// TestFormalizeQuestionNodeDoesNotChargeTheBudget pins WHEN the budget starts:
-// Python stamps the deadline in formalize_question's return dict
-// (agentic_rag_graph.py:1061) — after the formalization call — so that call is
-// not charged to the research budget. Arming on entry shortened every downstream
-// timeout by one LLM call, which is exactly enough to flip the
-// MinRoundHeadroomS guard near the boundary and skip a round Python would run.
-func TestFormalizeQuestionNodeDoesNotChargeTheBudget(t *testing.T) {
- const delay = 300 * time.Millisecond
- st := NewAgenticState("曹操是谁?", "", 3, []schema.Message{*schema.UserMessage("曹操是谁?")})
- deps := RAGTools{Model: delayedFormalizeModel{delay: delay}}
-
- started := time.Now()
- formalizeQuestionNode(context.Background(), deps, st, nil)
- if elapsed := time.Since(started); elapsed < delay {
- t.Fatalf("the stub model did not run (%v elapsed): the assertion below would be vacuous", elapsed)
- }
-
- // The clock is armed on the way OUT, so what remains is the whole budget
- // minus the moment the defer ran — not minus the model call.
- if got := st.RemainingS(); got < TotalBudgetS-0.15 {
- t.Errorf("RemainingS = %.2fs after a %v formalization, want ~%.0fs: the call must not be charged to the budget",
- got, delay, TotalBudgetS)
- }
-}
-
-// TestRecordConsecutiveUnanswerableAcrossOuterRagCalls mirrors the Python
-// outer react loop in dialog_service.rag_agent: the model may call rag()
-// several times within one user turn, and each unsatisfying verdict should
-// bump the shared _consecutive_unanswerable counter. Go keeps that counter on
-// the *RAGCache that Rag() now builds before the outer-react branch, so the
-// same cache is reused across the outer loop's multiple rag() calls. This test
-// simulates two such outer rag() calls with INSUFFICIENT verdicts and asserts
-// the counter reaches 2 — the threshold at which Rag() tells the outer agent to
-// STOP calling rag again.
-func TestRecordConsecutiveUnanswerableAcrossOuterRagCalls(t *testing.T) {
- cache := NewRAGCache()
-
- // First outer rag() call — unsatisfying verdict bumps the counter to 1.
- cache.NoteUnanswerable(VerdictInsufficient)
- if cache.ConsecutiveUnanswerable() != 1 {
- t.Fatalf("after 1st outer rag() call: ConsecutiveUnanswerable = %d, want 1",
- cache.ConsecutiveUnanswerable())
- }
-
- // Second outer rag() call — still unsatisfying: counter must reach 2 so the
- // STOP guard can fire (the state the pre-fix code could never reach on the
- // outer path, because deps.Cache was nil and the increment was skipped).
- cache.NoteUnanswerable(VerdictInsufficient)
- if cache.ConsecutiveUnanswerable() != 2 {
- t.Fatalf("after 2nd outer rag() call: ConsecutiveUnanswerable = %d, want 2",
- cache.ConsecutiveUnanswerable())
- }
-
- // A satisfying verdict resets the streak.
- cache.NoteUnanswerable(VerdictSufficient)
- if cache.ConsecutiveUnanswerable() != 0 {
- t.Fatalf("after a SUFFICIENT verdict: ConsecutiveUnanswerable = %d, want 0",
- cache.ConsecutiveUnanswerable())
- }
-}
-
-// TestRecordConsecutiveUnanswerableNoCacheIsNoOp makes sure the guard is safe
-// when no cache is wired (nil deps.Cache): the update must not panic and the
-// outer loop simply loses the cross-call STOP protection — which is exactly
-// why Rag() now auto-builds a cache before branching into the outer react loop.
-func TestRecordConsecutiveUnanswerableNoCacheIsNoOp(t *testing.T) {
- // Must not panic with a nil cache.
- (*RAGCache)(nil).NoteUnanswerable(VerdictInsufficient)
- (*RAGCache)(nil).NoteUnanswerable(VerdictSufficient)
-}
-
-// TestRewriteContextShowsThePassageBehindAConfirmedMember pins the input side of
-// the rewrite round: what the rewriter is shown is the PASSAGE that carries each
-// confirmed name, not the first line of a stored chunk.
-//
-// The difference is the whole point of that round. The wording a text uses for the
-// relation (and the other names it mentions) lives in the middle of a passage, so
-// a first-line digest cannot carry it — and a rewrite that cannot see it can only
-// re-ask what was already asked.
-func TestRewriteContextShowsThePassageBehindAConfirmedMember(t *testing.T) {
- st := NewAgenticState("关羽杀了多少有姓名的人物?", "", 3, nil)
- st.KB = &runtime.Kbinfos{Chunks: []map[string]any{
- {"chunk_id": "w1", "content": "荀正 引军来战,被关公一刀斩于马下。"},
- {"chunk_id": "big", "content": "第一行与本题无关\n第二行才提到关公"},
- }}
- st.KB.RecordReachedTerm("荀正", "w1")
-
- ctx := renderResearchContext(st)
- if !strings.Contains(ctx, "荀正") || !strings.Contains(ctx, "斩于马下") {
- t.Errorf("rewrite context does not carry the passage behind the confirmed member:\n%s", ctx)
- }
- if strings.Contains(ctx, "第一行与本题无关") {
- t.Errorf("rewrite context fell back to first-line digests although a confirmed member's passage was available:\n%s", ctx)
- }
- if !strings.Contains(ctx, "ALREADY confirmed") {
- t.Errorf("the member section is not labelled:\n%s", ctx)
- }
-}
-
-// TestRewriteRoundCanStillAdmitOnARichPool pins the room a rewrite round is given:
-// it is measured against the EVIDENCE POOL's ceiling, the same number the admitter
-// enforces.
-//
-// Sized against the smaller snippet-pool constant, a pool past 60 chunks left the
-// round with room <= 0: it admitted nothing, declared "retrieval saturated" and
-// discarded the queries it had just built — on exactly the rich rounds where more
-// evidence was still arriving.
-func TestRewriteRoundCanStillAdmitOnARichPool(t *testing.T) {
- pool := make([]map[string]any, 0, runtime.EvidencePoolCap())
- for i := 0; i < 71; i++ {
- pool = append(pool, map[string]any{"chunk_id": fmt.Sprintf("pre-%d", i), "content": "already pooled"})
- }
- st := NewAgenticState("关羽杀了多少有姓名的人物?", "", 3, nil)
- st.KB = &runtime.Kbinfos{Chunks: pool}
- st.UnresolvedSlots = []map[string]any{{"question_clues": []string{"还有谁被关羽所杀"}}}
- st.Deadline = time.Now().Add(60 * time.Second)
-
- mdl := &scriptedModel{}
- mdl.push(`{"queries": [{"query": "关羽 斩 名单 其余"}]}`)
-
- queryRewriteNode(context.Background(), RAGTools{
- Model: mdl,
- Search: runtime.SearchDeps{Backend: &corpusRetriever{}, KbIDs: []string{"kb1"}, HasEmbedder: true},
- }, st, log.New(&bytes.Buffer{}, "", 0))
-
- if st.NoProgress {
- t.Fatal("NoProgress = true on a pool of 71 with room left: the round's room must be measured against the evidence pool's ceiling")
- }
- if len(st.KB.Chunks) <= 71 {
- t.Errorf("pool = %d, want the rewrite's own query admitted (cap %d, pool was 71)", len(st.KB.Chunks), runtime.EvidencePoolCap())
- }
-}
-
-// TestRenderSlotRecordCarriesNoMachineFields pins the split between the record the
-// ANSWER is composed from and the draft the SCA reviews.
-//
-// The draft deliberately carries the machine fields that make verification
-// possible (strength, terminal type, evidence ids). Handing those to the answer
-// model as a "summary" is a different act with a different failure mode, and it
-// was measured: the composed answer quoted the bookkeeping verbatim
-// ("slot 1 [entity] … (strength=0.90) [terminal=state, evidence_ids=[…]]").
-func TestRenderSlotRecordCarriesNoMachineFields(t *testing.T) {
- strong := 0.9
- st := runtime.NewState([]runtime.Variable{
- {
- ID: 0,
- Type: "aspect",
- Candidate: strPtr("answer A"),
- CandidateStrength: &strong,
- DiscoveredClues: []string{"clue one", "clue two"},
- },
- {ID: 1, Type: "aspect", QuestionClues: []string{"who opened it?"}},
- }, 0, nil)
- evidence := map[string]SlotEvidence{
- "0": {EvidenceIDs: []string{"c1", "c2"}, TerminalType: "state"},
- }
-
- record := RenderSlotRecord(st, "the collected answer")
- // A table that asks for no SET leads with the session's own answer, exactly as
- // it did before the answer-layer demotion existed — see
- // TestSlotRecordLeadsWithFactsNotWithASessionsProse for the SET case, and
- // TestValueRecordCarriesNoEnumeratedMembers for why the two differ.
- if !strings.HasPrefix(record, "Candidate answer: the collected answer\n\n- slot 0 [aspect]: answer A\n- slot 1 [aspect]: NOT RESOLVED") {
- t.Fatalf("record = %q, want the session's answer first on a value table", record)
- }
- if strings.Contains(record, "enumerated members") {
- t.Fatalf("record = %q, want no enumerated line on a value table", record)
- }
- for _, banned := range []string{"strength=", "terminal=", "evidence_ids", "c1"} {
- if strings.Contains(record, banned) {
- t.Errorf("record %q must not carry the machine field %q", record, banned)
- }
- }
- // The SCA's draft keeps them: the split must not disarm verification.
- draft := RenderSlotDraft(st, "the collected answer", evidence)
- for _, want := range []string{"strength=0.90", "terminal=state", "evidence_ids=['c1', 'c2']"} {
- if !strings.Contains(draft, want) {
- t.Errorf("draft %q must keep %q for the SCA", draft, want)
- }
- }
-}
-
-// TestAnswerPromptLabelsTheRecordAndForbidsQuoting pins the prompt contract: the
-// slot record reaches the answer labelled as an INTERNAL record (not evidence),
-// with an explicit instruction not to quote its lines and labels — and the
-// machine fields never reach the prompt at all.
-func TestAnswerPromptLabelsTheRecordAndForbidsQuoting(t *testing.T) {
- strong := 0.9
- record := RenderSlotRecord(runtime.NewState([]runtime.Variable{
- {ID: 1, Type: "entity", Candidate: strPtr("华雄、颜良"), CandidateStrength: &strong},
- }, 0, nil), "")
-
- withRecord := &runtime.Kbinfos{Record: record, PreSummary: "- slot 1 [entity]: 华雄、颜良 (strength=0.90)"}
- prompt := AnswerDeps{}.answerPromptWithEvidence(withRecord, "q", false, false)
- for _, want := range []string{recordContract, "Research Record (INTERNAL", "华雄、颜良"} {
- if !strings.Contains(prompt.user, want) {
- t.Errorf("prompt missing %q", want)
- }
- }
- for _, banned := range []string{"strength=", "evidence_ids", "Research Summary (primary evidence)"} {
- if strings.Contains(prompt.user, banned) {
- t.Errorf("prompt must not carry %q when a slot record exists", banned)
- }
- }
-
- // No slot record (a prose summary instead): the prose block is used, and the
- // record contract stays out of it.
- prose := &runtime.Kbinfos{PreSummary: "the research found A and B"}
- prompt = AnswerDeps{}.answerPromptWithEvidence(prose, "q", false, false)
- if !strings.Contains(prompt.user, "Research Summary (primary evidence)") {
- t.Errorf("prose prompt = %q, want the summary block", prompt.user)
- }
- if strings.Contains(prompt.user, recordContract) {
- t.Error("the record contract must not label a prose summary")
- }
-}
-
-// TestAnswerRecordCarriesTheProbeLedger pins the block that closes the measured
-// write-back gap: the slots are the model's own bookkeeping, so a run can probe
-// twenty-four terms, get a passage for every one, and record eleven — and the
-// answer then sees only the eleven while the rest exist merely inside the pool.
-//
-// The two lists stay apart on purpose: "reached" is a fact about the corpus,
-// "nothing back" is a fact about the QUERY (not an absence).
-func TestAnswerRecordCarriesTheProbeLedger(t *testing.T) {
- kb := &runtime.Kbinfos{Record: "- slot 0 [count]: 11"}
- kb.RecordReachedTerm("车胄", "c1")
- kb.RecordReachedTerm("管亥", "c2")
- kb.RecordProbedAbsent("温酒")
-
- prompt := AnswerDeps{}.answerPromptWithEvidence(kb, "q", false, false)
- for _, want := range []string{
- "- slot 0 [count]: 11", // the slots come first
- "Probed and answered",
- "车胄", "管亥",
- "Probed with NOTHING back",
- "温酒",
- } {
- if !strings.Contains(prompt.user, want) {
- t.Errorf("answer prompt missing %q", want)
- }
- }
- // "Nothing back" may not be described as absence: that reading is what stops
- // an enumeration short.
- if strings.Contains(prompt.user, "absent from the corpus") && !strings.Contains(prompt.user, "this is not \"absent\"") {
- t.Error("the not-reached list must refuse the absence reading")
- }
- // The ledger belongs to the answer prompt, not to the SCA-facing draft.
- if strings.Contains(kb.PreSummary, "Probed and answered") {
- t.Error("the probe ledger must not leak into the SCA draft")
- }
-}
-
-// TestSessionPatchLogNamesBothSidesOfTheTournament pins P13 for the one place a
-// list can vanish without a trace: MergeSlotPatch keeps ONE candidate per slot,
-// chosen by the strength the model reported, and the losing candidate is then in
-// no table, no draft and no record.
-//
-// The log line is the only place that fact can exist, so it has to name both the
-// claim and the base it lost to.
-func TestSessionPatchLogNamesBothSidesOfTheTournament(t *testing.T) {
- var buf bytes.Buffer
- orig := _LOG
- _LOG = log.New(&buf, "", 0)
- defer func() { _LOG = orig }()
-
- strong, weak := 0.97, 0.85
- base := runtime.NewState([]runtime.Variable{
- {ID: 1, Type: "entity", Candidate: strPtr("孔秀、孟坦"), CandidateStrength: &strong},
- }, 0, nil)
- patch := runtime.NewState([]runtime.Variable{
- {ID: 1, Type: "entity", Candidate: strPtr("华雄、颜良、庞德"), CandidateStrength: &weak},
- }, 0, nil)
-
- // The weaker claim loses: the slot still holds the base afterwards.
- logSessionPatch(2, base, base, patch)
- got := buf.String()
- for _, want := range []string{"华雄、颜良、庞德", "0.85", "孔秀、孟坦", "0.97", "NOT ADOPTED"} {
- if !strings.Contains(got, want) {
- t.Errorf("log %q missing %q", got, want)
- }
- }
-
- // Adopted: the slot holds the patch's candidate afterwards.
- buf.Reset()
- merged := runtime.NewState([]runtime.Variable{
- {ID: 1, Type: "entity", Candidate: strPtr("华雄、颜良、庞德"), CandidateStrength: &weak},
- }, 1, nil)
- logSessionPatch(2, base, merged, patch)
- if got := buf.String(); !strings.Contains(got, "adopted") || strings.Contains(got, "NOT ADOPTED") {
- t.Errorf("log %q, want the adopted verdict", got)
- }
-}
-
-// TestMergeSlotPatchKeepsTheStrongestClaim is the behaviour the log above exists
-// to expose: a weaker branch candidate must NOT overwrite a stronger base, and a
-// stronger one must.
-// TestMergeSlotPatchKeepsTheStrongestClaim pins the rule for a slot that holds
-// ONE value: sessions fold in completion order, so a weak tentative claim must
-// not downgrade what another session already proved. Sets are exempt from this
-// rule on purpose — see TestMergeSlotPatchMergesSetCandidates.
-func TestMergeSlotPatchKeepsTheStrongestClaim(t *testing.T) {
- strong, weak := 0.97, 0.85
- base := runtime.NewState([]runtime.Variable{
- {ID: 1, Type: "entity", Candidate: strPtr("白马坡"), CandidateStrength: &strong},
- }, 0, nil)
- weaker := runtime.NewState([]runtime.Variable{
- {ID: 1, Type: "entity", Candidate: strPtr("延津"), CandidateStrength: &weak},
- }, 0, nil)
- if merged := MergeSlotPatch(base, weaker); merged != nil {
- if got := *merged.ByID(1).Candidate; got != "白马坡" {
- t.Errorf("slot 1 = %q, want the stronger base candidate to stand", got)
- }
- }
-
- stronger := runtime.NewState([]runtime.Variable{
- {ID: 1, Type: "entity", Candidate: strPtr("延津"), CandidateStrength: &strong},
- }, 0, nil)
- base = runtime.NewState([]runtime.Variable{
- {ID: 1, Type: "entity", Candidate: strPtr("白马坡"), CandidateStrength: &weak},
- }, 0, nil)
- if merged := MergeSlotPatch(base, stronger); merged == nil || *merged.ByID(1).Candidate != "延津" {
- t.Error("a strictly stronger branch candidate must be adopted")
- }
-}
-
-// TestMergeSlotPatchMergesSetCandidates pins the set rule: a slot that holds a
-// SET is unioned, because "the stronger claim wins" DISCARDS the members only the
-// weaker list held.
-//
-// Measured (2026-09-15, 三国演义/关羽): one session enumerated twelve members into
-// slot 0, a second had already written "10" there at strength 0.90, the twelve
-// lost at 0.85, and the answer was the 10 — while 管亥 and 车胄 (two of the
-// twelve) had each returned ten passages of their own.
-func TestMergeSlotPatchMergesSetCandidates(t *testing.T) {
- strong, weak := 0.90, 0.85
-
- // A list beats a NUMBER: the number is a claim ABOUT the list, and the list is
- // the members. The number is not lost — it stays as an alternate clue.
- base := runtime.NewState([]runtime.Variable{
- typedCountVar(1, "count", 10, strong),
- }, 0, nil)
- enumeration := runtime.NewState([]runtime.Variable{
- typedMembersVarS(1, "count", weak, "华雄", "颜良", "管亥", "车胄", "蔡阳"),
- }, 0, nil)
- merged := MergeSlotPatch(base, enumeration)
- if merged == nil {
- t.Fatal("the enumeration must change the slot: five members are not a count of ten")
- }
- if got := *merged.ByID(1).Candidate; got != "华雄、颜良、管亥、车胄、蔡阳" {
- t.Fatalf("slot 1 = %q, want the enumerated list to hold the slot", got)
- }
- if alts := alternateCandidatesOf(*merged.ByID(1)); len(alts) != 1 || alts[0] != "10" {
- t.Fatalf("alternates = %v, want the losing count kept", alts)
- }
-
- // Two lists union, and neither side's members are dropped.
- twoLists := MergeSlotPatch(
- runtime.NewState([]runtime.Variable{
- typedMembersVarS(1, "entity", strong, "孔秀", "孟坦"),
- }, 0, nil),
- runtime.NewState([]runtime.Variable{
- typedMembersVarS(1, "entity", weak, "华雄", "颜良", "庞德"),
- }, 0, nil),
- )
- if twoLists == nil {
- t.Fatal("two lists must union")
- }
- for _, want := range []string{"孔秀", "孟坦", "华雄", "颜良", "庞德"} {
- if got := *twoLists.ByID(1).Candidate; !strings.Contains(got, want) {
- t.Errorf("slot 1 = %q, want %s kept", got, want)
- }
- }
-
- // Two numbers keep the larger: a set that shrinks when a second source agrees
- // with it is a set that loses members.
- numbers := MergeSlotPatch(
- runtime.NewState([]runtime.Variable{
- typedCountVar(1, "count", 10, strong),
- }, 0, nil),
- runtime.NewState([]runtime.Variable{
- typedCountVar(1, "count", 12, weak),
- }, 0, nil),
- )
- if numbers == nil || *numbers.ByID(1).Candidate != "12" {
- t.Fatalf("two counts must keep the larger one, got %v", numbers)
- }
-}
-
-// TestSyncCountSlotsTakesTheEnumeratedSize pins the other half of the same
-// loss: the count slot and the list slots are written by different sessions and
-// nothing kept them in step. The slot takes the DERIVED number, because a count is a
-// claim the table can check.
-//
-// The direction that matters is the one an answer repeats: measured (2026-09-16) a
-// count slot left holding a session's 28 against thirteen enumerated members
-// produced "killed 28 named people" over a list of a handful. An over-claim is kept
-// as an alternate clue, so the record still shows what was claimed.
-//
-// A slot that does not CLAIM a number is left alone — a date, a phrase or a sentence
-// is nobody's count, and reading the slot's declared TYPE to guess otherwise was the
-// rule this replaces (see slots.Value.Number).
-func TestSyncCountSlotsTakesTheEnumeratedSize(t *testing.T) {
- table := runtime.NewState([]runtime.Variable{
- typedCountVar(0, "count", 10, 0),
- typedAnchoredMembersVar(1, "person", "华雄、颜良、文丑、孔秀、孟坦、韩福、卞喜、王植、秦琪、蔡阳、车胄、管亥"),
- }, 0, nil)
- raised := syncCountSlots(&table)
- if len(raised) != 1 || raised[0] != 0 {
- t.Fatalf("raised = %v, want the count slot reconciled", raised)
- }
- if got := *table.ByID(0).Candidate; got != "12" {
- t.Fatalf("count slot = %q, want 12 (the list's size)", got)
- }
-
- // An over-claim does not stand: the members are what the table can check, and
- // the claim is kept beside them as an alternate.
- bigger := runtime.NewState([]runtime.Variable{
- typedCountVar(0, "count", 16, 0),
- typedAnchoredMembersVar(1, "person", "华雄、颜良、文丑、孔秀、孟坦、韩福"),
- }, 0, nil)
- if changed := syncCountSlots(&bigger); len(changed) != 1 {
- t.Fatalf("changed = %v, want the over-claim reconciled to the members", changed)
- }
- if got := *bigger.ByID(0).Candidate; got != "6" {
- t.Fatalf("count slot = %q, want 6 (the members' size)", got)
- }
- alts := alternateCandidatesOf(*bigger.ByID(0))
- if len(alts) != 1 || alts[0] != "16" {
- t.Fatalf("alternates = %v, want the over-claim kept as the record's alternate", alts)
- }
-
- // A count that already agrees is left alone.
- agreed := runtime.NewState([]runtime.Variable{
- typedCountVar(0, "count", 6, 0),
- typedAnchoredMembersVar(1, "person", "华雄、颜良、文丑、孔秀、孟坦、韩福"),
- }, 0, nil)
- if changed := syncCountSlots(&agreed); len(changed) != 0 {
- t.Fatalf("changed = %v, want an agreeing count untouched", changed)
- }
-
- // A count slot whose text is NOT a number is left alone: the slot made no checkable
- // claim, so there is nothing here to correct — and the answer's count comes from the
- // members either way (measured 2026-09-16, 三国/关羽: a session wrote "约 17-19 人"
- // into the count slot, and the members beside it are what the answer must repeat).
- unreadable := runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "count", Candidate: strPtr("约 17-19 人")},
- typedAnchoredMembersVar(1, "person", "华雄、颜良、文丑、孔秀、孟坦、韩福"),
- }, 0, nil)
- if changed := syncCountSlots(&unreadable); len(changed) != 0 {
- t.Fatalf("changed = %v, want an unreadable claim left alone", changed)
- }
-
- // The count of a table whose member slot is TEXT is left alone: prose claims no
- // number and no members, and nothing here guesses at either.
- textOnly := runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "count", Candidate: strPtr("约 17-19 人")},
- {ID: 1, Type: "person", Candidate: strPtr("华雄、颜良、文丑")},
- }, 0, nil)
- if changed := syncCountSlots(&textOnly); len(changed) != 0 {
- t.Fatalf("changed = %v, want nothing derived from text", changed)
- }
-}
-
-// TestMemberCountIgnoresProseFragments pins the count's unit: a member is a NAME,
-// not a fragment of the sentence around it.
-//
-// The fixture is a measured record shape — the slots held the right names wrapped
-// in chapter prose, SplitCandidateNames cut that prose at its separators, and the
-// enumerated size came out 28 against thirteen real names. That number was then
-// raised into the count slot and reported by the answer as its own.
-func TestMemberCountIgnoresProseFragments(t *testing.T) {
- // The prose is Text and the names are declared members. The fixture that used to
- // prove this (names wrapped in chapter prose, with the parser cutting the prose
- // into "members") cannot be written any more: an undeclared slot contributes
- // nothing, so the count is the members or it is nothing.
- table := runtime.NewState([]runtime.Variable{
- typedCountVar(0, "count", 28, 0),
- typedAnchoredMembersVar(1, "web", "孔秀、孟坦、韩福、卞喜、王植、秦琪、蔡阳"),
- {ID: 2, Type: "web", Candidate: strPtr("第五回(发矫诏诸镇应曹公、破关兵三英战吕布)、第二十五回(屯土山关公约三事、救白马曹操解重围)")},
- typedAnchoredMembersVar(3, "web", "华雄、颜良、文丑、庞德"),
- }, 0, nil)
-
- // Eleven distinct real names: the prose pieces around them are not members.
- if got := enumeratedSize(table); got != 11 {
- t.Fatalf("enumerated size = %d, want 11 (the declared names, not the chapter prose)", got)
- }
- // And the count slot takes that number rather than the session's claim.
- syncCountSlots(&table)
- if got := *table.ByID(0).Candidate; got != "11" {
- t.Fatalf("count slot = %q, want 11 (the enumerated members)", got)
- }
-}
-
-// TestSyncCountSlotsCountsOnlyMembersWithAPassage pins the count's unit one level further down:
-// a member counts when it carries the passage that states it.
-//
-// The fixture is the measured case (2026-09-17, 三国/关羽): a session's member list merged with the
-// resolve's, and among the names two that no passage in hand quotes — 于禁 (taken at 水淹七军 and
-// released, never killed) and 刘延 (who outlives the actor). The union was 18, the answer stated
-// 18, and sixteen of those names had a passage behind them. A claim is not a member: the number
-// is what the table can point at, and the claim stays visible as the slot's alternate.
-func TestSyncCountSlotsCountsOnlyMembersWithAPassage(t *testing.T) {
- table := runtime.NewState([]runtime.Variable{
- typedCountVar(0, "count", 18, 0),
- typedAnchoredMembersVar(1, "person", "华雄、荀正"),
- typedMembersVar(2, "person", "于禁、刘延"),
- }, 0, nil)
- raised := syncCountSlots(&table)
- if len(raised) != 1 || raised[0] != 0 {
- t.Fatalf("raised = %v, want the count slot reconciled to the members with a passage", raised)
- }
- if got := *table.ByID(0).Candidate; got != "2" {
- t.Fatalf("count slot = %q, want 2 (the members the table can quote)", got)
- }
- if alts := alternateCandidatesOf(*table.ByID(0)); len(alts) != 1 || alts[0] != "18" {
- t.Fatalf("alternates = %v, want the 18 kept as the record's alternate", alts)
- }
-}
-
-// TestValueQuestionRecordKeepsItsOwnDraft pins the FRAMES regression of 2026-09-17 (q759): a table
-// whose slots hold one person each is NOT an enumeration, and the record is the only place the run's
-// own draft answer reaches the answer model.
-//
-// Rendered as a set, the record dropped "Candidate answer:" — the one line carrying the run's own
-// finding — and asked the answer to state a count of members instead. That is the failure the
-// record's gate was written against ("the session's own draft answer is then demoted below a line
-// that does not apply to it"), and it is why Set stays the planner's declaration.
-func TestValueQuestionRecordKeepsItsOwnDraft(t *testing.T) {
- items := slots.Items(slots.Item{Value: "Lanee Butler", ChunkID: "c1", Quote: "Mistral (sailboard) | Lanee Butler United States"})
- table := runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "person", Value: &items},
- {ID: 1, Type: "person"},
- }, 0, nil)
- rec := RenderSlotRecord(table, "The person is Richard Coxon, crewed with Colin Beashel in the Soling class.")
- if !strings.Contains(rec, "Candidate answer: The person is Richard Coxon") {
- t.Fatalf("record = %q, want the run's own draft kept for a question whose answer is one value", rec)
- }
- if strings.Contains(rec, "enumerated members across the slots above") {
- t.Fatalf("record = %q, want no set block on a value question", rec)
- }
-}
-
-// TestMemberLineCarriesTheItemsOwnWords pins that an enumerated member reaches the answer WITH its
-// words, not as a bare name.
-//
-// The evidence block is token-budgeted, so the passage behind a member may not fit: measured
-// (2026-09-17, 三国/关羽) 18 members against the 13 passages the budget carried, and the answer had
-// to name five of them as "listed in the record, no original text provided". The record's own
-// contract accepts a member "quoted above", so the quote travels with the name.
-func TestMemberLineCarriesTheItemsOwnWords(t *testing.T) {
- items := slots.Items(
- slots.Item{Value: "华雄", ChunkID: "c1", Quote: "云长提华雄之头,掷于地上"},
- slots.Item{Value: "刘延"},
- )
- line := memberLine(runtime.Variable{ID: 1, Type: "person", Value: &items})
- if !strings.Contains(line, "华雄 ←c1 “云长提华雄之头,掷于地上”") {
- t.Fatalf("line = %q, want the member's own words beside the passage that states it", line)
- }
- if !strings.Contains(line, "刘延 ←(no passage)") {
- t.Fatalf("line = %q, want a member with no passage still marked as a claim", line)
- }
-}
-
-// TestRunCoverageResolveEnrollsTheEnumerationThePlannerSkipped pins the wiring that the measured
-// run (2026-09-17, 三国/关羽) needed: the planner typed the answer slot "count", the sessions
-// patched the members into it, and no type word ever asked for a set.
-//
-// The VALUE is the fact, so the resolve — the last node that can complete the set — runs the
-// direction's enumeration itself instead of judging only what a session happened to write down.
-// Ten names were the whole answer that day; the corpus stated more.
-func TestRunCoverageResolveEnrollsTheEnumerationThePlannerSkipped(t *testing.T) {
- items := slots.Items(
- slots.Item{Value: "华雄", ChunkID: "c-华雄"},
- slots.Item{Value: "颜良", ChunkID: "c-颜良"},
- )
- rendered := slots.Render(items)
- table := runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "count", Terms: []string{"斩"}, Candidate: &rendered, Value: &items},
- }, 0, nil)
-
- exec := &coverageStubExec{}
- st := NewAgenticState("关羽杀了多少有姓名的人物?", "", 3, nil)
- st.KB = &runtime.Kbinfos{}
- // The direction's words must already have MET in what the run holds, or the enrollment is spent
- // on a recall whose windows the enumeration's own filter would drop (see
- // CoverageActsMeetActor). Here 斩 is held; the stub's window then survives.
- st.KB.Admit(func(p *runtime.PoolAdmitter) {
- p.Add(map[string]any{"chunk_id": "held", "content_with_weight": "云长提刀直取,斩之"})
- })
- st.SlotTable = table
- st.Deadline = time.Now().Add(120 * time.Second)
-
- model := &scriptedModel{}
- model.push(`{"members": [{"i": 0, "name": "孔秀"}], "not_members": []}`)
- RunCoverageResolve(context.Background(), RAGTools{
- Model: model,
- Tools: &runtime.Toolset{Exec: exec},
- }, st, nil)
-
- if exec.ran() != 1 {
- t.Fatalf("enumeration ran %d time(s), want the resolve to enroll it when no type word asked for it", exec.ran())
- }
- names := strings.Join(runtime.ItemValues(&st.SlotTable), "、")
- if !strings.Contains(names, "孔秀") {
- t.Fatalf("items = %q, want the window the enrolled enumeration admitted judged into the table", names)
- }
-}
-
-// TestRunCoverageResolveSkipsTheEnumerationWhenTheWordsNeverMet pins the guard the measured waste
-// bought: on 2026-09-17 (FRAMES, resolve node) the enrollment asked 13 operands, recalled 724
-// passages and produced ZERO windows, because every window the enumeration builds must carry an act
-// word AND the actor, and this direction's two words had never met in anything the run held.
-//
-// The enrollment spends the ANSWER's clock, so the same conjunction is now probed against the
-// passages in hand first: a recall that cannot survive the filter is not made.
-func TestRunCoverageResolveSkipsTheEnumerationWhenTheWordsNeverMet(t *testing.T) {
- items := slots.Items(slots.Item{Value: "Lanee Butler", ChunkID: "c-Lanee"})
- rendered := slots.Render(items)
- table := runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "person", Terms: []string{"partner"}, Subject: "Colin Beashel", Candidate: &rendered, Value: &items},
- }, 0, nil)
-
- exec := &coverageStubExec{}
- st := NewAgenticState("who is the partner of the 1984 keelboat sailor?", "", 3, nil)
- st.KB = &runtime.Kbinfos{}
- // Held passages mention the actor, and passages carry the act word — never both in one.
- st.KB.Admit(func(p *runtime.PoolAdmitter) {
- p.Add(map[string]any{"chunk_id": "held-actor", "content_with_weight": "Colin Beashel sailed the Soling class."})
- p.Add(map[string]any{"chunk_id": "held-act", "content_with_weight": "Their partner was crewing that year."})
- })
- st.SlotTable = table
- st.Deadline = time.Now().Add(120 * time.Second)
-
- RunCoverageResolve(context.Background(), RAGTools{
- Model: &scriptedModel{},
- Tools: &runtime.Toolset{Exec: exec},
- }, st, nil)
-
- if exec.ran() != 0 {
- t.Fatalf("enumeration ran %d time(s), want no recall for a direction whose words have never met in one passage", exec.ran())
- }
-}
-
-// TestCiteChunksPutTheItemsPassagesFirst pins the citation set of an enumerated answer.
-//
-// The answer has to cite one passage per element, and those passages are in the pool but not
-// necessarily among the few that score highest: a sixteen-member table whose answer named only
-// three of them was reading the same handful of blocks as any other question. Members first also
-// means a token budget that truncates drops scored extras, never an element's own passage.
-func TestCiteChunksPutTheItemsPassagesFirst(t *testing.T) {
- chunks := []map[string]any{
- {"chunk_id": "w1", "content": "低分段落", "similarity": 0.1},
- {"chunk_id": "w2", "content": "高分段落", "similarity": 0.9},
- {"chunk_id": "w3", "content": "成员自己的段落", "similarity": 0.2},
- }
- kb := &runtime.Kbinfos{Chunks: chunks}
- kb.NoteCitedChunks([]string{"w3"})
-
- got := withCitedChunks(rankByScore(chunks), kb, citeChunkCap)
- if len(got) != 3 || runtime.ChunkIDOf(got[0]) != "w3" {
- t.Fatalf("citeChunks = %v, want the items' own passage first and the rest after it", got)
- }
- if runtime.ChunkIDOf(got[1]) != "w2" {
- t.Fatalf("citeChunks = %v, want the scored chunks after the items' passages", got)
- }
-
- // Without items the selection is unchanged: the top-scoring chunk, capped.
- plain := withCitedChunks(rankByScore(chunks), &runtime.Kbinfos{Chunks: chunks}, 1)
- if len(plain) != 1 || runtime.ChunkIDOf(plain[0]) != "w2" {
- t.Fatalf("citeChunks = %v, want the plain top-N selection when no items carry passages", plain)
- }
-}
-
-// TestPublishAnchoredMembersResolvesDocIDs pins the metadata case. A session that selected documents
-// by metadata records DOC ids as its members' anchors — metadata_search returns doc ids and admits no
-// passage, so not one of them resolves in the pool — and publishing has to resolve them to that
-// document's own passage. Without resolution the members the record lists reach the answer as no
-// evidence at all: measured (2026-09-21) the answer stated 6 documents, the evidence it read carried
-// six blocks about OTHER documents, and zero markers resolved.
-func TestPublishAnchoredMembersResolvesDocIDs(t *testing.T) {
- kb := &runtime.Kbinfos{Chunks: []map[string]any{
- {"chunk_id": "docA-1", "doc_id": "docA", "content": "无关段落", "similarity": 0.3},
- {"chunk_id": "docA-2", "doc_id": "docA", "content": "勒索病毒事件按 P1/P2/P3 分级", "similarity": 0.9},
- {"chunk_id": "docB-1", "doc_id": "docB", "content": "端点隔离时限 10 分钟", "similarity": 0.4},
- {"chunk_id": "other", "doc_id": "docC", "content": "别的文档", "similarity": 0.99},
- }}
- table := runtime.NewState([]runtime.Variable{
- anchoredItemsVar(0, "list",
- slots.Item{Value: "勒索病毒事件上报与分级管理规范", ChunkID: "docA",
- Quote: "file_name: a.docx; update_time: 2026-09-21 13:35:42"},
- slots.Item{Value: "端点隔离标准", ChunkID: "docB", Quote: "端点隔离时限 10 分钟"},
- ),
- }, 0, nil)
-
- derived, resolved := publishAnchoredMembers(kb, &table)
- if derived != 2 || resolved != 2 {
- t.Fatalf("derived/resolved = %d/%d, want 2/2", derived, resolved)
- }
- if got := kb.CitedChunks(); len(got) != 2 || got[0] != "docA-2" || got[1] != "docB-1" {
- t.Fatalf("CitedChunks = %v, want each member on its own document's passage", got)
- }
- refs := kb.AnchoredRefs()
- if len(refs) != 2 || refs[0].ChunkID != "docA-2" || refs[1].ChunkID != "docB-1" {
- t.Fatalf("AnchoredRefs = %+v, want the anchors resolved to pool chunk ids", refs)
- }
- // The first member's quote is the METADATA line the session was shown, which its passage does
- // not contain. Kept, compactAnchored would render that block FROM the metadata line instead of
- // the passage — the block would show what the tool printed, not the document.
- if refs[0].Quote != "" {
- t.Errorf("ref[0].Quote = %q, want it DROPPED (its passage does not carry it)", refs[0].Quote)
- }
- // The second member's quote is in its passage: it stands, and the block renders from it.
- if refs[1].Quote != "端点隔离时限 10 分钟" {
- t.Errorf("ref[1].Quote = %q, want it kept", refs[1].Quote)
- }
-}
-
-// TestPublishAnchoredMembersUnionsAndSkipsUnresolvable pins the two guards around the ledger. Its
-// writer REPLACES (NoteCitedChunks), so publishing must union: the enumeration's passages are already
-// recorded, and a member that no longer resolves must not cost the answer a passage that still does.
-// An anchor the pool cannot point at publishes nothing — the member stays a name in the record
-// rather than becoming a block that opens somebody else's passage.
-func TestPublishAnchoredMembersUnionsAndSkipsUnresolvable(t *testing.T) {
- kb := &runtime.Kbinfos{Chunks: []map[string]any{
- {"chunk_id": "enumerated", "doc_id": "docD", "content": "枚举来的段落", "similarity": 0.5},
- {"chunk_id": "docE-1", "doc_id": "docE", "content": "元数据选出的段落", "similarity": 0.2},
- }}
- kb.NoteCitedChunks([]string{"enumerated"})
- kb.NoteAnchoredRefs([]runtime.AnchoredRef{{Name: "枚举成员", ChunkID: "enumerated", Quote: "枚举来的"}})
-
- table := runtime.NewState([]runtime.Variable{
- anchoredItemsVar(0, "list",
- slots.Item{Value: "元数据成员", ChunkID: "docE"},
- slots.Item{Value: "无迹可循的成员", ChunkID: "docZ"},
- ),
- }, 0, nil)
-
- derived, resolved := publishAnchoredMembers(kb, &table)
- if derived != 2 || resolved != 1 {
- t.Fatalf("derived/resolved = %d/%d, want 2/1 (one anchor is in no document the pool holds)", derived, resolved)
- }
- if got := kb.CitedChunks(); len(got) != 2 || got[0] != "enumerated" || got[1] != "docE-1" {
- t.Fatalf("CitedChunks = %v, want the enumeration's passage KEPT and the resolved one added", got)
- }
- if refs := kb.AnchoredRefs(); len(refs) != 2 || refs[0].Name != "枚举成员" || refs[1].Name != "元数据成员" {
- t.Fatalf("AnchoredRefs = %+v, want the enumerated ref kept and the resolved one added", refs)
- }
-}
-
-// TestPublishAnchoredMembersIsANoOpWithoutAnchors pins the regression guard: a table holding no
-// ANCHORED member publishes nothing, so an ordinary question's evidence selection stays exactly what
-// it is today — min(citeChunkCap, pool size).
-func TestPublishAnchoredMembersIsANoOpWithoutAnchors(t *testing.T) {
- kb := &runtime.Kbinfos{Chunks: []map[string]any{{"chunk_id": "c1", "content": "x"}}}
- table := runtime.NewState([]runtime.Variable{
- typedMembersVar(0, "list", "于禁"),
- }, 0, nil)
-
- if derived, resolved := publishAnchoredMembers(kb, &table); derived != 0 || resolved != 0 {
- t.Fatalf("derived/resolved = %d/%d, want 0/0 for a table whose members carry no passage", derived, resolved)
- }
- if got := kb.CitedChunks(); len(got) != 0 {
- t.Fatalf("CitedChunks = %v, want nothing published", got)
- }
-}
-
-// TestSlotRecordShowsEachItemWithItsPassage pins the citation half of the record: an item's chunk id
-// is in the value, and the record used to print names only — so the instruction to cite each member
-// ("every member you list carries the words behind it") had nothing to cite.
-func TestSlotRecordShowsEachItemWithItsPassage(t *testing.T) {
- table := runtime.NewState([]runtime.Variable{
- typedAnchoredMembersVar(1, "person", "华雄、荀正"),
- typedMembersVar(2, "person", "于禁"),
- }, 0, nil)
- rec := RenderSlotRecord(table, "")
- for _, want := range []string{"华雄 ←c-华雄", "荀正 ←c-荀正", "于禁 ←(no passage)"} {
- if !strings.Contains(rec, want) {
- t.Fatalf("record = %q, want %q", rec, want)
- }
- }
-}
-
-// TestSlotRecordStatesACountSlotThatHoldsWords pins the silent half of the count disagreement.
-//
-// A count slot whose value is TEXT passes in silence: it claims no number, so nothing is derived
-// from it (by contract), and the record used to say nothing either — its words sat beside the
-// enumerated size and the answer took whichever it liked. Measured (2026-09-17, 三国/关羽): a
-// session's "14" answered a table that enumerated 16. The words are not parsed here either; the
-// disagreement is stated.
-func TestSlotRecordStatesACountSlotThatHoldsWords(t *testing.T) {
- table := runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "count", Candidate: strPtr("14")},
- typedAnchoredMembersVar(1, "person", "华雄、荀正、杨龄"),
- }, 0, nil)
- rec := RenderSlotRecord(table, "")
- if !strings.Contains(rec, "enumerated members across the slots above: 3") {
- t.Fatalf("record = %q, want the enumerated size stated", rec)
- }
- if !strings.Contains(rec, `holds "14" as text, not a number`) {
- t.Fatalf("record = %q, want a count slot holding words stated as a disagreement", rec)
- }
-
- // The declared-number branch keeps its own wording.
- numbered := runtime.NewState([]runtime.Variable{
- typedCountVar(0, "count", 14, 0),
- typedAnchoredMembersVar(1, "person", "华雄、荀正、杨龄"),
- }, 0, nil)
- if rec := RenderSlotRecord(numbered, ""); !strings.Contains(rec, "says 14 while the slots above enumerate 3") {
- t.Fatalf("record = %q, want a declared number's disagreement stated as before", rec)
- }
-}
-
-// TestSyncCountSlotsReadsTheWholeTable pins the ONE assumption the derivation carries: the size is
-// read ACROSS the table, so a question that enumerates two sets gets one number for both count
-// slots (`斩杀` and `生擒` both read 4 here).
-//
-// It is pinned rather than fixed because the fix is a DECLARATION — the planner saying which count
-// counts which set — and an inference (adjacency, wording) would guess exactly where the record has
-// to be exact. Until that declaration exists, the record's own line is the honest reading: it says
-// "across the slots above", not per slot.
-func TestSyncCountSlotsReadsTheWholeTable(t *testing.T) {
- table := runtime.NewState([]runtime.Variable{
- typedCountVar(0, "count", 9, 0),
- typedAnchoredMembersVar(1, "person", "华雄、荀正"),
- typedCountVar(2, "count", 4, 0),
- typedAnchoredMembersVar(3, "person", "于禁、庞德"),
- }, 0, nil)
- syncCountSlots(&table)
- for _, id := range []int{0, 2} {
- if got := *table.ByID(id).Candidate; got != "4" {
- t.Fatalf("slot %d = %q, want the table-wide size: two sets per table are a declaration the planner does not make yet", id, got)
- }
- }
-}
-
-// TestSyncCountSlotsLeavesWhatClaimsNoNumber pins the kind-keyed half: the derivation touches the
-// slots that CLAIM a number and nothing else — a text claim is not a number anybody can check, and
-// the floor keeps a barely-filled table from collapsing a claim to one.
-func TestSyncCountSlotsLeavesWhatClaimsNoNumber(t *testing.T) {
- table := runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "count", Candidate: strPtr("约 17-19 人")}, // prose: claims no number
- typedAnchoredMembersVar(1, "person", "华雄、荀正、杨龄"),
- }, 0, nil)
- if changed := syncCountSlots(&table); len(changed) != 0 {
- t.Fatalf("changed = %v, want a claim that is not a number left alone", changed)
- }
- // One quotable element is below the floor: nothing to derive from, so the claim stands.
- thin := runtime.NewState([]runtime.Variable{
- typedCountVar(0, "count", 9, 0),
- typedAnchoredMembersVar(1, "person", "华雄"),
- }, 0, nil)
- if changed := syncCountSlots(&thin); len(changed) != 0 {
- t.Fatalf("changed = %v, want the floor respected (one element is not an enumeration)", changed)
- }
- if got := *thin.ByID(0).Candidate; got != "9" {
- t.Fatalf("count slot = %q, want the claim standing below the floor", got)
- }
-}
-
-// TestSlotRecordNamesTheClaimsWithoutAPassage pins the other half of that rule: leaving a claim
-// out of the number must not hide it. An answer told only "the members are two" cannot tell that
-// two more names were claimed — nor that its number may therefore be short.
-func TestSlotRecordNamesTheClaimsWithoutAPassage(t *testing.T) {
- table := runtime.NewState([]runtime.Variable{
- typedCountVar(0, "count", 4, 0),
- typedAnchoredMembersVar(1, "person", "荀正、杨龄"),
- typedMembersVar(2, "person", "于禁、刘延"),
- }, 0, nil)
- rec := RenderSlotRecord(table, "")
- if !strings.Contains(rec, "enumerated members across the slots above: 2") {
- t.Fatalf("record = %q, want the size stated over the members with a passage", rec)
- }
- if !strings.Contains(rec, "claimed WITHOUT a passage in hand") ||
- !strings.Contains(rec, "于禁") || !strings.Contains(rec, "刘延") {
- t.Fatalf("record = %q, want the claims named beside the number", rec)
- }
-}
-
-// TestMergeSlotPatchKeepsTheLosingClaimAsAnAlternate pins the bookkeeping that
-// survives every rule in MergeSlotPatch: one slot holds one candidate, so the
-// claim that lost the comparison used to leave no trace at all — not in the
-// table, not in the draft, not in the record. The framework does not decide which
-// claim is true; it stops discarding the one that lost.
-//
-// Sets no longer lose at all (they union, see
-// TestMergeSlotPatchMergesSetCandidates), so this pins the single-value case.
-func TestMergeSlotPatchKeepsTheLosingClaimAsAnAlternate(t *testing.T) {
- strong, weak := 0.97, 0.85
- base := runtime.NewState([]runtime.Variable{
- {ID: 1, Type: "entity", Candidate: strPtr("白马坡"), CandidateStrength: &strong},
- }, 0, nil)
- branch := runtime.NewState([]runtime.Variable{
- {ID: 1, Type: "entity", Candidate: strPtr("延津"), CandidateStrength: &weak},
- }, 0, nil)
-
- merged := MergeSlotPatch(base, branch)
- if merged == nil {
- t.Fatal("the losing claim must still change the state: it is kept as an alternate")
- }
- v := merged.ByID(1)
- if v.Candidate == nil || *v.Candidate != "白马坡" {
- t.Fatalf("slot 1 = %v, want the stronger claim to hold the slot", v.Candidate)
- }
- alts := alternateCandidatesOf(*v)
- if len(alts) != 1 || alts[0] != "延津" {
- t.Fatalf("alternates = %v, want the losing claim kept", alts)
- }
-
- // It reaches the ANSWER-facing record, which is the whole point: the answer
- // must be able to see that two claims exist.
- rec := RenderSlotRecord(*merged, "")
- if !strings.Contains(rec, "alternate") || !strings.Contains(rec, "延津") {
- t.Fatalf("record = %q, want the alternate rendered", rec)
- }
-
- // And it survives a SECOND merge: alternates accumulate, they do not compete.
- strongerAgain := runtime.NewState([]runtime.Variable{
- {ID: 1, Type: "entity", Candidate: strPtr("斜谷"), CandidateStrength: &strong},
- }, 0, nil)
- merged2 := MergeSlotPatch(*merged, strongerAgain)
- if merged2 == nil {
- t.Fatal("a third claim must fold in")
- }
- if got := alternateCandidatesOf(*merged2.ByID(1)); len(got) != 2 {
- t.Fatalf("alternates = %v, want both losing claims kept", got)
- }
-}
-
-// TestSlotRecordLeadsWithFactsNotWithASessionsProse pins the answer-layer fix.
-//
-// The record used to LEAD with one session's prose draft answer, and the answer
-// copied it: measured twice (2026-09-15, 三国演义/关羽) — a record whose slots
-// enumerated seventeen members produced a fifteen-member answer, and a record
-// enumerating fourteen produced a ten-member answer, in both cases exactly the
-// number written in that prose. The prose is one session's recollection, written
-// before the other sessions were merged; it is a claim to reconcile with the
-// members, not the record.
-func TestSlotRecordLeadsWithFactsNotWithASessionsProse(t *testing.T) {
- table := runtime.NewState([]runtime.Variable{
- typedCountVar(0, "count", 10, 0),
- typedAnchoredMembersVar(1, "person", "华雄、颜良、文丑、孔秀、孟坦、韩福、卞喜、王植、秦琪、蔡阳、车胄、程远志、夏侯存、庞德"),
- // A second slot of the same table holds REFERENCES, not members — and it is
- // TEXT, so it contributes none: counting them is how this record once
- // answered its own count with a nineteen on a twelve-member list.
- {ID: 2, Type: "dataset", Candidate: strPtr("第5回(华雄)、第21回(车胄)、第25回(颜良)、第27回(五关六将)、第74回(庞德)")},
- }, 0, nil)
- rec := RenderSlotRecord(table, "关羽在《三国演义》中斩杀的有姓名人物共10人,名单如下:……")
-
- if !strings.Contains(rec, "enumerated members across the slots above: 14") {
- t.Fatalf("record = %q, want the enumerated size stated as a fact", rec)
- }
- // A count that disagrees with the members has to be SAID: this record once
- // produced a nineteen-person answer out of a count slot reading 19, which the
- // answer explained as seven more people "the material does not list".
- if !strings.Contains(rec, "slot 0 [count] says 10 while the slots above enumerate 14") {
- t.Fatalf("record = %q, want the disagreement stated", rec)
- }
- // Stated, not ordered: an order was tried and reverted — measured (2026-09-16,
- // 三国/关羽) a record that said "take the number from the enumerated members" got
- // exactly that number taken (nine), when its slots enumerated the wrong things and
- // its sessions had found fourteen. Either number can be the wrong one, and only the
- // passages decide, so the note must state the disagreement and leave the judgement
- // to the stage that reads them.
- if strings.Contains(rec, "take the number from") {
- t.Fatalf("record = %q, must not order the answer to take either number", rec)
- }
- if !strings.Contains(rec, "disagree") {
- t.Fatalf("record = %q, want the disagreement stated as a disagreement", rec)
- }
- facts := strings.Index(rec, "slot 1 [person]")
- draft := strings.Index(rec, "One session's own draft answer")
- if facts < 0 || draft < 0 || facts > draft {
- t.Fatalf("record = %q, want the slots BEFORE the session's own draft", rec)
- }
- if !strings.Contains(rec, "UNVERIFIED") {
- t.Fatalf("record = %q, want the draft labelled as a claim", rec)
- }
-}
-
-// TestValueRecordCarriesNoEnumeratedMembers pins the other half of the gate: the
-// enumerated size, the count-vs-members warning and the demotion of the session's
-// prose are SET-question machinery, and a record whose answer is one date or one
-// number must render exactly as it did before any of it existed.
-//
-// The table is the measured case. A "how much shorter is A than B" question's
-// record carried `- slot 1 [number]: 133 feet` beside `- enumerated members across
-// the slots above: 16`, where the 16 was `Grace's、High、Falls、Colonial、Creek` —
-// one waterfall's name cut at its separators by whoever wrote it into the slot —
-// and the session's draft answer was labelled UNVERIFIED below it. Rendered
-// ungated, those two lines appeared in 21 of a 20-question FRAMES run's records.
-func TestValueRecordCarriesNoEnumeratedMembers(t *testing.T) {
- table := runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "dataset", Candidate: strPtr("Grace's、High、Falls、Colonial、Creek")},
- {ID: 1, Type: "number", Candidate: strPtr("133 feet")},
- {ID: 2, Type: "web", Candidate: strPtr("Colonial Creek Falls, Washington — 788 m (2,585 ft)")},
- }, 0, nil)
- rec := RenderSlotRecord(table, "Alabama's tallest waterfall is 133 feet tall.")
-
- if strings.Contains(rec, "enumerated members") {
- t.Fatalf("record = %q, want no enumerated line on a value table", rec)
- }
- if strings.Contains(rec, "UNVERIFIED") {
- t.Fatalf("record = %q, want the session's answer NOT demoted on a value table", rec)
- }
- if !strings.HasPrefix(rec, "Candidate answer: Alabama's tallest waterfall is 133 feet tall.") {
- t.Fatalf("record = %q, want the session's answer first on a value table", rec)
- }
- for _, want := range []string{"- slot 0 [dataset]:", "- slot 1 [number]: 133 feet", "788 m (2,585 ft)"} {
- if !strings.Contains(rec, want) {
- t.Errorf("record %q missing the slot fact %q", rec, want)
- }
- }
-}
-
-// TestUnionSetCandidatesIsOrderIndependent pins the bug that cost the most members
-// in the 2026-09-16 enumeration runs.
-//
-// The union must not depend on which side is the base: with the base the LONGER
-// list, "the union adds nothing to the base" was answered as "no conclusion", the
-// caller fell through to the strength rule, and a SHORTER list — the one the model
-// was sure of, so the one it called stronger — replaced the list containing it.
-//
-// Measured in the run this pins: twenty-four names probed and each reached, slot 1
-// holding thirteen members, a session patching its own eleven at 0.95 against the
-// base's 0.90, and a final record of eleven.
-// typedMembersValue is a slot value that DECLARES a member list — the only shape the
-// runtime counts (see package slots): names with no declared kind are opaque text.
-func typedMembersValue(list string) slots.Value {
- items := strings.Split(list, "、")
- members := make([]slots.Item, 0, len(items))
- for _, n := range items {
- members = append(members, slots.Item{Value: strings.TrimSpace(n)})
- }
- return slots.Items(members...)
-}
-
-// typedMembersVar is typedMembersValue as the slot variable a session would patch.
-func typedMembersVar(id int, typ, list string) runtime.Variable {
- return typedMembersVarS(id, typ, 0, strings.Split(list, "、")...)
-}
-
-// typedMembersVarS is the same with an explicit strength, the way a session's patch
-// carries one.
-func typedMembersVarS(id int, typ string, strength float64, names ...string) runtime.Variable {
- items := make([]slots.Item, 0, len(names))
- for _, n := range names {
- if n = strings.TrimSpace(n); n != "" {
- items = append(items, slots.Item{Value: n})
- }
- }
- v := slots.Items(items...)
- rendered := slots.Render(v)
- out := runtime.Variable{ID: id, Type: typ, Candidate: &rendered, Value: &v}
- if strength > 0 {
- out.CandidateStrength = &strength
- }
- return out
-}
-
-// typedAnchoredMembersVar is typedMembersVarS with the passage each member rests on: the shape a
-// member must have to be COUNTED (see runtime.AnchoredMembers). Without the chunk id the member
-// is a claim — visible in the record, absent from the number.
-func typedAnchoredMembersVar(id int, typ string, lists ...string) runtime.Variable {
- items := make([]slots.Item, 0, len(lists))
- for _, list := range lists {
- for _, n := range strings.Split(list, "、") {
- if n = strings.TrimSpace(n); n != "" {
- items = append(items, slots.Item{Value: n, ChunkID: "c-" + n, Quote: "…" + n + "…"})
- }
- }
- }
- v := slots.Items(items...)
- rendered := slots.Render(v)
- return runtime.Variable{ID: id, Type: typ, Candidate: &rendered, Value: &v}
-}
-
-// anchoredItemsVar builds a slot holding items with the anchors and quotes a session chose, which is
-// what typedAnchoredMembersVar's synthetic "c-<name>" anchors cannot express — a metadata selection's
-// anchors are doc ids and its quotes are the metadata lines it was shown.
-func anchoredItemsVar(id int, typ string, items ...slots.Item) runtime.Variable {
- v := slots.Items(items...)
- rendered := slots.Render(v)
- return runtime.Variable{ID: id, Type: typ, Candidate: &rendered, Value: &v}
-}
-
-func typedCountVar(id int, typ string, n int, strength float64) runtime.Variable {
- v := slots.Number(n)
- rendered := slots.Render(v)
- out := runtime.Variable{ID: id, Type: typ, Candidate: &rendered, Value: &v}
- if strength > 0 {
- out.CandidateStrength = &strength
- }
- return out
-}
-
-func TestUnionIsOrderIndependent(t *testing.T) {
- eleven := typedMembersValue("华雄、管亥、颜良、文丑、孔秀、孟坦、韩福、卞喜、王植、秦琪、蔡阳")
- thirteen := typedMembersValue("华雄、颜良、文丑、孔秀、孟坦、韩福、卞喜、王植、秦琪、蔡阳、庞德、成何、管亥")
- names := func(v slots.Value) string {
- out := v.ItemValues()
- sort.Strings(out)
- return strings.Join(out, "、")
- }
- want := names(thirteen)
-
- up, _, ok := slots.Union(eleven, thirteen)
- if !ok || names(up) != want {
- t.Fatalf("small→big: union = %q ok = %v, want the thirteen-name superset", slots.Render(up), ok)
- }
- down, _, ok := slots.Union(thirteen, eleven)
- if !ok {
- t.Fatal("big→small must resolve too: a subset is a resolution, not a tiebreaker for strength")
- }
- if names(down) != want {
- t.Fatalf("big→small: union = %q, want the base kept whole (a subset adds nothing)", slots.Render(down))
- }
-
- // Two numbers keep the larger one, in both orders.
- if got, _, ok := slots.Union(slots.Number(13), slots.Number(11)); !ok || got.Count != 13 {
- t.Fatalf("13→11 = %v ok=%v, want the larger count", got, ok)
- }
- if got, _, ok := slots.Union(slots.Number(11), slots.Number(13)); !ok || got.Count != 13 {
- t.Fatalf("11→13 = %v ok=%v, want the larger count", got, ok)
- }
-
- // A number never outvotes the members it counts: the number is the claim that
- // loses, kept as the dropped value.
- union, dropped, ok := slots.Union(thirteen, slots.Number(11))
- if !ok || union.Kind != slots.KindItems {
- t.Fatalf("members vs count = %v ok=%v, want the members", union, ok)
- }
- if dropped.Count != 11 {
- t.Fatalf("dropped = %v, want the losing count", dropped)
- }
-
- // Text is not the union's business: a value question merges by strength, exactly
- // as it did before values were typed.
- if _, _, ok := slots.Union(slots.Text("1858"), slots.Text("1849")); ok {
- t.Error("text must not merge by union — that is what keeps a value question on its strength rule")
- }
-}
-
-// TestMemberUnionDropsPlaceAndEventPhrases pins the last way a count inflates: a
-// piece that CONTAINS a member is not a second member.
-//
-// A session writing a place beside its owner (`洛阳关孟坦`) or an event beside its
-// object (`温酒斩华雄`) writes a token that is short, digit-free and unpunctuated —
-// it passes every shape test a name passes — while the member it names is already
-// in the table. Measured (2026-09-16): a record enumerating 21 names counted 25,
-// the four extra being place-qualified copies of names already listed.
-func TestMemberUnionDropsPlaceAndEventPhrases(t *testing.T) {
- table := runtime.NewState([]runtime.Variable{
- typedAnchoredMembersVar(1, "person", "程远志、华雄、管亥、颜良、文丑、杨龄、孔秀、孟坦、韩福、卞喜、王植、秦琪、车胄、成何、蔡阳、吕旷、吕翔、荀正、纪灵、夏侯存、庞德"),
- typedAnchoredMembersVar(2, "person", "孟坦、韩福、卞喜、王植、秦琪、洛阳关孟坦、汜水关卞喜、荥阳王植、黄河渡口秦琪"),
- }, 0, nil)
-
- union := runtime.ItemValues(&table)
- if len(union) != 21 {
- t.Fatalf("member count = %d, want the 21 names (place-qualified copies are not members): %v", len(union), union)
- }
- for _, extra := range []string{"洛阳关孟坦", "汜水关卞喜", "荥阳王植", "黄河渡口秦琪"} {
- for _, item := range union {
- if item == extra {
- t.Errorf("member union kept %q, a name with its place attached", extra)
- }
- }
- }
- if got := enumeratedSize(table); got != 21 {
- t.Fatalf("enumerated size = %d, want 21", got)
- }
-}
-
-// TestLedgerQuoteCarriesTheWordsBehindAMember pins per-member evidence in the
-// answer-facing ledger: a name without the words behind it is a member nobody can
-// point at, and an answer handed names only cannot cite one passage per member.
-func TestLedgerQuoteCarriesTheWordsBehindAMember(t *testing.T) {
- kb := &runtime.Kbinfos{}
- kb.Admit(func(p *runtime.PoolAdmitter) {
- p.Add(map[string]any{"chunk_id": "c1", "content": "关公马快,赶上文丑,脑后一刀,将文丑斩下马来。"})
- })
- kb.RecordReachedTerm("文丑", "c1")
-
- // The names are in the ledger either way; the words ride the set gate, so a
- // value direction does not pay for them.
- if ledger := probeLedger(kb); strings.Contains(ledger, "斩下马来") {
- t.Fatalf("ledger = %q, want no per-member quotes before a set direction declares itself", ledger)
- }
-
- kb.MarkSetDirection()
- ledger := probeLedger(kb)
- if !strings.Contains(ledger, "文丑 — “") {
- t.Fatalf("ledger = %q, want the member followed by the words that carry it", ledger)
- }
- if !strings.Contains(ledger, "斩下马来") {
- t.Fatalf("ledger = %q, want the quoted passage itself", ledger)
- }
-}
-
-// TestRecordContractStatesTheSettledValueIsTheAnswer pins the half of the record
-// contract that the measured "not found" answer needed: a record that is only a
-// check is droppable when the passages do not repeat its value.
-//
-// Measured (2026-09-16, FRAMES, mode high): the slot table held
-// `slot 0 [person]: Colin Beashel and Richard Coxon (Australia, Star class 1984
-// Olympics)` and the composed answer was "not found in the knowledge base" — the
-// passages around it were about other competitions, and the chat configuration
-// requires that sentence when the information is unavailable. The record has to
-// outrank the absence, or the research that found the answer is discarded.
-func TestRecordContractStatesTheSettledValueIsTheAnswer(t *testing.T) {
- for _, want := range []string{
- "that value IS the answer",
- "do not report that the answer was not found while a slot holds one",
- "evidence plainly contradicts a settled value",
- "NEVER quote these lines",
- "probed-and-answered",
- } {
- if !strings.Contains(recordContract, want) {
- t.Errorf("recordContract is missing %q", want)
- }
- }
-}
-
-// TestBudgetExtensionIsBoughtOncePerQuestion pins the one-shot budget extension a
-// set-shaped pass buys.
-//
-// The budget fits ONE pass, and an enumeration needs a second one to pick up the
-// members a cut first pass never patched (measured 2026-09-16, 三国/关羽: the run
-// ended at `ROUND 1 end (unresolved=0)` with the reached-but-unpatched 管亥 gone).
-// One extension, not a per-round top-up: the point is a second look, not an
-// unbounded run for any table that keeps declaring a set.
-func TestBudgetExtensionIsBoughtOncePerQuestion(t *testing.T) {
- st := &AgenticState{Deadline: time.Now().Add(30 * time.Second)}
- before := st.RemainingS()
- if !st.ExtendDeadline(SetBudgetExtensionS) {
- t.Fatal("the first extension must apply")
- }
- after := st.RemainingS()
- if after < before+SetBudgetExtensionS-1 {
- t.Errorf("remaining %.0fs after the extension, want ~%.0fs", after, before+SetBudgetExtensionS)
- }
- if st.ExtendDeadline(SetBudgetExtensionS) {
- t.Error("a second extension must be refused: the budget is bought once per question")
- }
- if got := st.RemainingS(); got > after+1 {
- t.Errorf("remaining %.0fs after the refused extension, want ~%.0f", got, after)
- }
-}
-
-// TestMergeSlotPatchKeepsTheDeclaration pins what the fold must NOT drop.
-//
-// Terms/Subject are the planner's declaration of how the source words the deed, and the
-// completeness pass is built from them (runtime.ScanPatterns / RunCompletenessPass).
-// Rebuilding the slot without them is why one run's second round had no act patterns at
-// all: measured (2026-09-16, 三国/关羽) round 1's session seeds carried 6325 characters
-// (method + the declared patterns) and round 2's carried 4150 (method only) — the recovery
-// round, opened by the routing precisely because the record was still short, ran with the
-// enumeration machinery switched off.
-func TestMergeSlotPatchKeepsTheDeclaration(t *testing.T) {
- base := runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "count", Terms: []string{"斩", "杀"}, Subject: "关羽|云长"},
- {ID: 1, Type: "dataset"},
- }, 0, nil)
- strong := 0.9
- branch := runtime.NewState([]runtime.Variable{
- {ID: 1, Type: "dataset", Candidate: strPtr("孔秀、孟坦"), CandidateStrength: &strong},
- }, 1, nil)
-
- merged := MergeSlotPatch(base, branch)
- if merged == nil {
- t.Fatal("MergeSlotPatch returned nil for a branch that carries a candidate")
- }
- if got := merged.State[0].Terms; len(got) != 2 || got[0] != "斩" || got[1] != "杀" {
- t.Errorf("merged Terms = %v, want the declaration to travel with the slot", got)
- }
- if got := merged.State[0].Subject; got != "关羽|云长" {
- t.Errorf("merged Subject = %q, want the declared actor", got)
- }
- // The declaration is what the enumeration is built from: a fold that drops it leaves the
- // next round with nothing to enumerate, which is the measured failure above.
- if !runtime.CoverageOf(*merged).Ok() {
- t.Error("the merged table is no longer an enumeration: the declaration was lost in the fold")
- }
-}
-
-// coverageStubExec answers tool calls like sessionStubExec and is ALSO a
-// runtime.CoverageRunner, so a round's enumeration runs without a retriever.
-type coverageStubExec struct {
- mu sync.Mutex
- calls []runtime.Coverage
-}
-
-func (e *coverageStubExec) Execute(_ context.Context, name string, _ map[string]any) (runtime.ToolOutcome, error) {
- return runtime.ToolOutcome{
- Status: runtime.StatusOK,
- Payload: []any{map[string]any{"kind": name, "content": "hit"}},
- EvidenceIDs: []string{"c-" + name},
- }, nil
-}
-
-// EnumerateCoverage stands in for the corpus: one window stating the deed.
-func (e *coverageStubExec) EnumerateCoverage(_ context.Context, cov runtime.Coverage, kb *runtime.Kbinfos) runtime.CoverageSet {
- e.mu.Lock()
- e.calls = append(e.calls, cov)
- e.mu.Unlock()
- quote := "云长手起刀落,斩孔秀于马下"
- if kb != nil {
- kb.Admit(func(p *runtime.PoolAdmitter) {
- p.Add(map[string]any{"chunk_id": "w1", "content_with_weight": quote})
- })
- }
- return runtime.CoverageSet{
- Operands: cov.Operands(),
- Recalled: 1,
- Windows: []runtime.CoverageWindow{{ChunkID: "w1", Quote: quote, Act: "斩"}},
- }
-}
-
-func (e *coverageStubExec) ran() int {
- e.mu.Lock()
- defer e.mu.Unlock()
- return len(e.calls)
-}
-
-// TestRunSlotResearchPassEnumeratesOnce pins the wiring, and the run-once rule.
-//
-// The act words used to be seeded as a list of queries TO MAKE, and a whole round ran
-// without a single one of them being made: measured (2026-09-16, 三国/关羽) 2175 characters
-// of patterns in every session's seed and zero `.*` queries in the run's log, with the
-// sessions re-probing names by hand in the next round. Now the round asks the corpus
-// itself — one call, one recall per operand — admits what comes back, and seeds it.
-func TestRunSlotResearchPassEnumeratesOnce(t *testing.T) {
- table := func() runtime.State {
- return runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "count", Terms: []string{"斩", "杀"}, Subject: "关羽|云长"},
- {ID: 1, Type: "dataset", QuestionClues: []string{"who did he kill?"}},
- }, 0, nil)
- }
- exec := &coverageStubExec{}
- kb := &runtime.Kbinfos{}
- st := &AgenticState{Question: "关羽杀了多少有姓名的人物?", KB: kb, SlotTable: table()}
- deps := runtime.SessionDeps{
- Model: &scriptedModel{replies: []string{"I could not find any evidence about that."}},
- Tools: &runtime.Toolset{Exec: exec},
- KB: kb,
- }
- RunSlotResearchPass(context.Background(), context.Background(), deps, st.Question, st, 60)
-
- if got := exec.ran(); got != 1 {
- t.Fatalf("ran %d enumeration(s), want exactly one per question", got)
- }
- if got := exec.calls[0].Operands(); len(got) != 4 {
- t.Errorf("operands = %v, want one entry per actor form and act word", got)
- }
- set, done := kb.CoverageSet()
- if !done || len(set.Windows) != 1 || set.Windows[0].ChunkID != "w1" {
- t.Fatalf("stored set = %+v (done=%v), want the window the enumeration found", set, done)
- }
- if !strings.Contains(set.Render(), "斩孔秀于马下") {
- t.Errorf("seed = %q, want the window with the chunk id a member is cited by", set.Render())
- }
- if kb.PoolSize() == 0 {
- t.Error("the enumeration's window never reached the pool: the win cannot be cited")
- }
-
- // A second round REUSES the set: the windows are in the pool under the same ids, so
- // asking the corpus the same operand queries again spends the store legs for nothing.
- second := &AgenticState{Question: st.Question, KB: kb, SlotTable: table()}
- RunSlotResearchPass(context.Background(), context.Background(), deps, second.Question, second, 60)
- if got := exec.ran(); got != 1 {
- t.Errorf("second round ran the enumeration again (%d), want the stored set reused", got)
- }
-}
-
-// metadataScopedRetriever serves a hit ONLY to a doc-scoped request, so a test can tell
-// which fan-out channel admitted what: channels A/B search the whole corpus (no scope),
-// channel C searches inside the documents the metadata filter selected.
-type metadataScopedRetriever struct {
- scopes [][]string
-}
-
-func (r *metadataScopedRetriever) Retrieve(_ context.Context, req runtime.RetrieveRequest) ([]map[string]any, error) {
- if len(req.DocScope) == 0 {
- return nil, nil
- }
- r.scopes = append(r.scopes, req.DocScope)
- return []map[string]any{{"id": "meta-1", "doc_id": "d1", "content": "Culdcept tower height is 42 m."}}, nil
-}
-
-// fanoutMetadataResolver is a scripted MetadataResolver: the push-down always answers with
-// the configured documents, the flattened view answers with metas, and the filters/logic it
-// was handed are recorded.
-type fanoutMetadataResolver struct {
- ids []string
- metas common.MetaData
- calls int
- filters []map[string]any
- logic string
- // docMeta backs the metadata_search context block (doc_id → fields).
- docMeta map[string]map[string]any
-}
-
-func (m *fanoutMetadataResolver) FilterDocIDsByMetaPushdown(_ context.Context, _ []string, filters []map[string]any, logic string) ([]string, bool) {
- m.calls++
- m.filters = filters
- m.logic = logic
- return m.ids, true
-}
-
-func (m *fanoutMetadataResolver) GetFlattedMetaByKBs(context.Context, []string) (common.MetaData, error) {
- return m.metas, nil
-}
-
-// MetadataForDocIDs answers the metadata_search context block. These tests observe the
-// selection, not the context block, so it stays empty unless a case sets docMeta.
-func (m *fanoutMetadataResolver) MetadataForDocIDs(context.Context, []string, []string) (map[string]map[string]any, error) {
- return m.docMeta, nil
-}
-
-// TestFanoutSearchMetadataChannelUsesCatalogFields pins channel C end to end with the
-// catalog-driven vocabulary: the model reads a sub-question and names a field the SESSION
-// offers — here `author`, not the historical hard-coded `title` — and the retrieval runs inside
-// the documents that field matches, even though the corpus-wide channels returned nothing.
-func TestFanoutSearchMetadataChannelUsesCatalogFields(t *testing.T) {
- ctx := context.Background()
- r := &metadataScopedRetriever{}
- resolver := &fanoutMetadataResolver{ids: []string{"d1"}}
- mdl := &scriptedModel{}
- mdl.push(`{"filters": [[{"key":"author","op":"contains","value":"OmiyaSoft"}]]}`)
- deps := RAGTools{
- Search: runtime.SearchDeps{
- Backend: r,
- KbIDs: []string{"kb1"},
- HasEmbedder: true,
- MetadataResolver: resolver,
- },
- Model: mdl,
- Tools: &runtime.Toolset{ThinkingMode: "high", MetadataFields: &runtime.MetadataCatalog{
- Keys: []string{"author", "title"},
- }},
- }
- st := &AgenticState{KB: &runtime.Kbinfos{}}
-
- added := FanoutSearch(ctx, deps, st, []string{"what is the Culdcept tower height"}, 8, 60, true)
-
- if added != 1 || len(st.KB.Chunks) != 1 {
- t.Fatalf("added = %d, pool = %d; want the metadata channel's one hit", added, len(st.KB.Chunks))
- }
- if got := runtime.ChunkIDOf(st.KB.Chunks[0]); got != "meta-1" {
- t.Errorf("pooled chunk = %q, want the channel-C hit", got)
- }
- if len(mdl.seen) != 1 {
- t.Errorf("model calls = %d, want exactly ONE extraction call for the batch", len(mdl.seen))
- }
- if resolver.calls != 1 || resolver.logic != "or" {
- t.Errorf("resolver calls = %d logic = %q, want one OR-combined lookup", resolver.calls, resolver.logic)
- }
- if len(resolver.filters) != 1 || resolver.filters[0]["key"] != "author" ||
- resolver.filters[0]["op"] != "contains" || resolver.filters[0]["value"] != "OmiyaSoft" {
- t.Errorf("filters = %v, want the author-contains condition the catalog offered", resolver.filters)
- }
- if len(r.scopes) != 1 || len(r.scopes[0]) != 1 || r.scopes[0][0] != "d1" {
- t.Errorf("scoped requests = %v, want the search restricted to the matched document", r.scopes)
- }
-}
-
-// TestFanoutSearchSkipsMetadataChannelUnlessAsked pins the gate: the rewrite round (and
-// every other caller that does not ask for the metadata channel) must not pay the
-// entity-extraction model call, and must not resolve documents.
-func TestFanoutSearchSkipsMetadataChannelUnlessAsked(t *testing.T) {
- ctx := context.Background()
- r := &metadataScopedRetriever{}
- resolver := &fanoutMetadataResolver{ids: []string{"d1"}}
- mdl := &scriptedModel{}
- mdl.push(`{"entities": [["Culdcept"]]}`)
- deps := RAGTools{
- Search: runtime.SearchDeps{
- Backend: r,
- KbIDs: []string{"kb1"},
- HasEmbedder: true,
- MetadataResolver: resolver,
- },
- Model: mdl,
- }
- st := &AgenticState{KB: &runtime.Kbinfos{}}
-
- if added := FanoutSearch(ctx, deps, st, []string{"what is the Culdcept tower height"}, 8, 60, false); added != 0 {
- t.Errorf("added = %d, want 0 without the metadata channel", added)
- }
- if len(mdl.seen) != 0 {
- t.Errorf("model calls = %d, want none: entity extraction must not run", len(mdl.seen))
- }
- if resolver.calls != 0 || len(r.scopes) != 0 {
- t.Errorf("resolver calls = %d scoped searches = %d, want none", resolver.calls, len(r.scopes))
- }
-}
-
-// TestFanoutSearchGatesMetadataChannelOnFilterableFields pins the gate that keeps Channel C
-// from being pure cost: a session whose catalog offers NO filterable field pays neither the
-// extraction call nor one guaranteed-empty scoped retrieval per sub-question.
-//
-// The gate is the field VOCABULARY, not one particular field: a catalog that offers `author`
-// and no `title` is perfectly usable, because the model may name whichever field the
-// sub-question fits. Nothing is baked in as a fallback — a caller that wired no toolset, or a
-// session whose catalog is empty because the dataset carries no metadata, is skipped too.
-func TestFanoutSearchGatesMetadataChannelOnFilterableFields(t *testing.T) {
- ctx := context.Background()
- cases := []struct {
- name string
- cat *runtime.MetadataCatalog
- noToolset bool
- want int
- }{
- {
- name: "wired session, no metadata",
- cat: nil,
- want: 0,
- },
- {
- name: "wired session, empty catalog",
- cat: &runtime.MetadataCatalog{},
- want: 0,
- },
- {
- name: "caller wired no tool surface",
- noToolset: true,
- want: 0,
- },
- {
- name: "catalog offers a non-title field",
- cat: &runtime.MetadataCatalog{
- Keys: []string{"author"},
- Samples: map[string][]runtime.MetadataSample{"author": {{Value: "OmiyaSoft", Docs: 1}}},
- },
- want: 1,
- },
- }
- for _, c := range cases {
- r := &metadataScopedRetriever{}
- resolver := &fanoutMetadataResolver{ids: []string{"d1"}}
- mdl := &scriptedModel{}
- mdl.push(`{"filters": [[{"key":"author","op":"contains","value":"OmiyaSoft"}]]}`)
- deps := RAGTools{
- Search: runtime.SearchDeps{
- Backend: r,
- KbIDs: []string{"kb1"},
- HasEmbedder: true,
- MetadataResolver: resolver,
- },
- Model: mdl,
- }
- if !c.noToolset {
- deps.Tools = &runtime.Toolset{ThinkingMode: "high", MetadataFields: c.cat}
- }
- st := &AgenticState{KB: &runtime.Kbinfos{}}
-
- added := FanoutSearch(ctx, deps, st, []string{"what is the Culdcept tower height"}, 8, 60, true)
- if added != c.want {
- t.Errorf("%s: added = %d, want %d", c.name, added, c.want)
- }
- if c.want == 0 {
- if len(mdl.seen) != 0 {
- t.Errorf("%s: model calls = %d, want none — the extraction must not run when no field is filterable", c.name, len(mdl.seen))
- }
- if resolver.calls != 0 || len(r.scopes) != 0 {
- t.Errorf("%s: resolver calls = %d scoped searches = %d, want none", c.name, resolver.calls, len(r.scopes))
- }
- }
- }
-}
-
-// TestParseFanoutFiltersShapesAndGuards pins the reply shapes and the guards. The guard set is
-// what keeps the channel from filtering on something the session never offered: a field outside
-// the catalog (which is also how the blacklist holds here), a WIDENING operator, a copied
-// sub-question, an answer sentence, an over-long value, a duplicate and an empty value are all
-// dropped, and at most three conditions survive per sub-question.
-func TestParseFanoutFiltersShapesAndGuards(t *testing.T) {
- fanouts := []string{"q one", "q two"}
- allowed := map[string]bool{"title": true, "author": true}
-
- // Positional arrays: one condition-group per sub-question, in order.
- got := parseFanoutFilters(`{"filters": [[`+
- `{"key":"title","op":"contains","value":"Alpha Corp"},`+
- `{"key":"author","op":"=","value":"Beta"}],`+
- `[{"key":"title","op":"contains","value":"Gamma"}]]}`, fanouts, allowed)
- if len(got) != 2 || len(got[0]) != 2 || got[0][0]["value"] != "Alpha Corp" ||
- got[0][1]["key"] != "author" || got[1][0]["value"] != "Gamma" {
- t.Errorf("positional = %v", got)
- }
-
- // Dict form matched by sub-question text: the unmatched one stays empty.
- got = parseFanoutFilters(`{"filters": [{"sub_question":"q two","filters":[{"key":"title","op":"contains","value":"Delta"}]}]}`, fanouts, allowed)
- if len(got[0]) != 0 || len(got[1]) != 1 || got[1][0]["value"] != "Delta" {
- t.Errorf("by-text = %v", got)
- }
-
- // A flat list of conditions is read POSITIONALLY: one condition per sub-question, so a
- // single-condition list only fills the first one.
- got = parseFanoutFilters(`{"filters": [{"key":"author","op":"=","value":"Alice"}]}`, fanouts, allowed)
- if len(got[0]) != 1 || got[0][0]["value"] != "Alice" || len(got[1]) != 0 {
- t.Errorf("flat = %v", got)
- }
-
- // Guards: an unoffered field, a widening operator, an over-long value, an 11-word value, a
- // prose/citation value, a duplicate and an empty value are dropped; the cap is three.
- long := strings.Repeat("x", metadataValueMaxChars+1)
- reply := `{"filters": [[` +
- `{"key":"question_id","op":"contains","value":"444"},` +
- `{"key":"title","op":"not contains","value":"Alpha"},` +
- `{"key":"title","op":"contains","value":"` + long + `"},` +
- `{"key":"title","op":"contains","value":"one two three four five six seven eight nine ten eleven"},` +
- `{"key":"title","op":"contains","value":"see https://example.com"},` +
- `{"key":"title","op":"contains","value":"Dup"},` +
- `{"key":"title","op":"contains","value":"Dup"},` +
- `{"key":"title","op":"contains","value":""},` +
- `{"key":"title","op":"contains","value":"e1"},` +
- `{"key":"title","op":"contains","value":"e2"},` +
- `{"key":"title","op":"contains","value":"e3"},` +
- `{"key":"title","op":"contains","value":"e4"}]]}`
- got = parseFanoutFilters(reply, fanouts, allowed)
- want := []string{"Dup", "e1", "e2"}
- if len(got[0]) != len(want) {
- t.Fatalf("guarded = %v, want %v", got[0], want)
- }
- for i, w := range want {
- if got[0][i]["value"] != w {
- t.Errorf("guarded[%d] = %v, want %q", i, got[0][i], w)
- }
- }
-
- // 'in' keeps its list (the value SET) and needs at least one entry.
- got = parseFanoutFilters(`{"filters": [[{"key":"title","op":"in","value":["A","B"]}]]}`, fanouts, allowed)
- if len(got[0]) != 1 {
- t.Fatalf("in = %v, want one condition", got)
- }
- if list, ok := got[0][0]["value"].([]any); !ok || len(list) != 2 {
- t.Errorf("in value = %v, want the list kept", got[0][0]["value"])
- }
- if got := parseFanoutFilters(`{"filters": [[{"key":"title","op":"in","value":[]}]]}`, fanouts, allowed); len(got[0]) != 0 {
- t.Errorf("empty 'in' = %v, want it dropped", got[0])
- }
-
- // No JSON at all: nothing parsed, and the fan-outs are still order-aligned.
- got = parseFanoutFilters("no json here", fanouts, allowed)
- if len(got) != 2 || len(got[0]) != 0 || len(got[1]) != 0 {
- t.Errorf("non-JSON = %v", got)
- }
-}
-
-// TestExtractFanoutFiltersDegradesToNothing pins the best-effort contract: no chat model, no
-// sub-questions, or a reply with nothing usable all mean NO conditions (nil, the channel
-// simply skipped), never a failure.
-func TestExtractFanoutFiltersDegradesToNothing(t *testing.T) {
- if got := ExtractFanoutFilters(context.Background(), RAGTools{}, []string{"q"}); got != nil {
- t.Errorf("filters = %v, want nil without a model", got)
- }
- if got := ExtractFanoutFilters(context.Background(), RAGTools{Model: &scriptedModel{}}, nil); got != nil {
- t.Errorf("filters = %v, want nil without sub-questions", got)
- }
- if got := ExtractFanoutFilters(context.Background(), RAGTools{Model: &scriptedModel{}}, []string{"q"}); got != nil {
- t.Errorf("filters = %v, want nil when the caller wired no tool surface", got)
- }
-}
-
-// slotResearchWithMetadataTool builds one action session whose tool executor is the REAL
-// metadata_search executor, so a tool call made by the model is observed end to end: the
-// resolver records the filter and answers with the documents the selector returns.
-func slotResearchWithMetadataTool(resolver *fanoutMetadataResolver, retriever *corpusRetriever, cat *runtime.MetadataCatalog, replies []*runtime.ModelReply) (*AgenticState, *runtime.Kbinfos, *fakeModel) {
- kb := &runtime.Kbinfos{}
- sd := runtime.SearchDeps{
- Backend: retriever,
- KbIDs: []string{"kb1"},
- HasEmbedder: true,
- KB: kb,
- MetadataResolver: resolver,
- }
- mdl := &fakeModel{replies: replies}
- st := &AgenticState{
- Question: "who wrote it?",
- KB: kb,
- SlotTable: runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "aspect", QuestionClues: []string{"who wrote it?"}},
- }, 0, nil),
- }
- deps := runtime.SessionDeps{
- Model: mdl,
- Tools: &runtime.Toolset{
- ThinkingMode: "high",
- MetadataFields: cat,
- Exec: runtime.NewSearchExecutor(sd, runtime.RunRequest{ThinkingMode: "high"}),
- },
- KB: kb,
- }
- RunSlotResearchPass(context.Background(), context.Background(), deps, "who wrote it?", st, 60)
- return st, kb, mdl
-}
-
-// TestSlotResearchSessionUsesCatalogFieldForMetadataSearch pins the whole chain in one run:
-// the catalog advertises `author`, the model spends its one metadata_search on that field,
-// and the real executor resolves documents by it. The resolver seeing the author condition
-// is the observation that the advertised field is the one that reaches the index — the
-// schema, the tool call and the executor all agreeing.
-func TestSlotResearchSessionUsesCatalogFieldForMetadataSearch(t *testing.T) {
- resolver := &fanoutMetadataResolver{
- ids: []string{"doc-culdcept"},
- metas: common.MetaData{"author": {"Alice": {"doc-culdcept"}}, "title": {"Culdcept History": {"doc-culdcept"}}},
- }
- cat := &runtime.MetadataCatalog{
- Keys: []string{"author", "title"},
- Samples: map[string][]runtime.MetadataSample{"author": {{Value: "Alice", Docs: 1}}},
- }
- _, _, mdl := slotResearchWithMetadataTool(resolver, &corpusRetriever{}, cat, []*runtime.ModelReply{
- {
- Content: "Narrowing by author.",
- ToolCalls: []runtime.ToolCall{{
- ID: "call_0",
- Name: "metadata_search",
- Args: map[string]any{
- "filters": []any{map[string]any{"key": "author", "op": "contains", "value": "Alice"}},
- },
- }},
- },
- {Content: `<state>{"new_states": []}</state>`},
- })
-
- if resolver.calls == 0 {
- t.Fatal("the session never resolved documents by metadata; the advertised field did not reach the executor")
- }
- if len(resolver.filters) != 1 || resolver.filters[0]["key"] != "author" {
- t.Errorf("resolver filters = %v, want the author condition the catalog advertised", resolver.filters)
- }
- if mdl.calls < 2 {
- t.Errorf("model calls = %d, want the session to have continued after the tool result", mdl.calls)
- }
-}
-
-// TestSlotResearchSessionSurvivesMetadataSearchWithoutMetadata pins the requirement that a
-// metadata-free dataset must not derail the run: the index carries NO field, the model still
-// spends its one metadata_search on `title`, and the session must read that as a query-level
-// miss and carry on with the next tool — never abort, never disable the run.
-func TestSlotResearchSessionSurvivesMetadataSearchWithoutMetadata(t *testing.T) {
- resolver := &fanoutMetadataResolver{} // no ids, no metas: a dataset with no metadata
- retriever := &corpusRetriever{}
- // No catalog: the shipped title-only schema is what the model sees.
- _, _, mdl := slotResearchWithMetadataTool(resolver, retriever, nil, []*runtime.ModelReply{
- {
- Content: "Trying a title filter.",
- ToolCalls: []runtime.ToolCall{{
- ID: "call_0",
- Name: "metadata_search",
- Args: map[string]any{
- "filters": []any{map[string]any{"key": "title", "op": "contains", "value": "Culdcept"}},
- },
- }},
- },
- {
- Content: "Falling back to retrieval.",
- ToolCalls: []runtime.ToolCall{{
- ID: "call_1",
- Name: "retrieve",
- Args: map[string]any{"query": []any{"who wrote it?"}},
- }},
- },
- {Content: `<state>{"new_states": []}</state>`},
- })
-
- // The metadata miss must not have stopped the loop: the session went on to retrieval.
- if len(retriever.calls) == 0 {
- t.Fatalf("retriever calls = 0; a metadata-free dataset aborted the session instead of degrading to retrieval")
- }
- if mdl.calls < 3 {
- t.Errorf("model calls = %d, want the session to have kept running to its own empty patch", mdl.calls)
- }
- if resolver.calls != 0 {
- t.Errorf("push-down calls = %d, want none: an unknown key is rejected before the index is touched", resolver.calls)
- }
-}
diff --git a/internal/rag/agentic-rag/agentic_rag_test.go b/internal/rag/agentic-rag/agentic_rag_test.go
deleted file mode 100644
index 4dda3c9812..0000000000
--- a/internal/rag/agentic-rag/agentic_rag_test.go
+++ /dev/null
@@ -1,986 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package agentic_rag
-
-import (
- "context"
- "errors"
- "fmt"
- "io"
- "log"
- "strings"
- "sync"
- "testing"
- "time"
- "unicode/utf8"
-
- "github.com/cloudwego/eino/schema"
- "ragflow/internal/rag/agentic-rag/runtime"
-)
-
-type muxChunk struct {
- text string
- isThink bool
-}
-
-func newTestMux() (*outerStreamMux, *[]muxChunk) {
- got := make([]muxChunk, 0)
- mux := &outerStreamMux{sink: &AnswerSink{OnDelta: func(delta string, isThink bool) {
- got = append(got, muxChunk{delta, isThink})
- }}}
- return mux, &got
-}
-
-// The outer model's stream carries literal <think>/</think> markers, which the
-// mux must turn into the sink's isThink flag instead of forwarding as text.
-func TestOuterStreamMuxRoutesThinkThenAnswer(t *testing.T) {
- mux, got := newTestMux()
- for _, d := range []string{"<think>", "reasoning", "</think>", "the answer"} {
- text := d
- if err := mux.sender(&text, nil); err != nil {
- t.Fatalf("sender(%q): %v", d, err)
- }
- }
-
- want := []muxChunk{{"reasoning", true}, {"the answer", false}}
- if len(*got) != len(want) {
- t.Fatalf("got %v, want %v", *got, want)
- }
- for i, w := range want {
- if (*got)[i] != w {
- t.Fatalf("chunk %d = %v, want %v", i, (*got)[i], w)
- }
- }
-}
-
-// Outer text is dropped once the terminal tool fired, because what follows is the aggregate
-// tool result and the answer already streamed from inside.
-func TestOuterStreamMuxDropsTextAfterTerminalFired(t *testing.T) {
- mux, got := newTestMux()
- mux.markTerminal()
-
- plain := "aggregate tool result"
- if err := mux.sender(&plain, nil); err != nil {
- t.Fatalf("sender: %v", err)
- }
- if len(*got) != 0 {
- t.Fatalf("got %v, want the non-think text dropped after terminal fired", *got)
- }
-
- // Thinking text is still forwarded even after the terminal fired.
- open := "<think>"
- late := "late thinking"
- if err := mux.sender(&open, nil); err != nil {
- t.Fatalf("sender: %v", err)
- }
- if err := mux.sender(&late, nil); err != nil {
- t.Fatalf("sender: %v", err)
- }
- if len(*got) != 1 || (*got)[0] != (muxChunk{"late thinking", true}) {
- t.Fatalf("got %v, want only the late thinking chunk", *got)
- }
-}
-
-// The inner run's research log and composed answer reach the same sink.
-func TestOuterStreamMuxDeliversInnerStream(t *testing.T) {
- mux, got := newTestMux()
- mux.deliver("[Research] planning\n", true)
- mux.deliver("final answer", false)
-
- want := []muxChunk{{"[Research] planning\n", true}, {"final answer", false}}
- if len(*got) != len(want) {
- t.Fatalf("got %v, want %v", *got, want)
- }
- for i, w := range want {
- if (*got)[i] != w {
- t.Fatalf("chunk %d = %v, want %v", i, (*got)[i], w)
- }
- }
-}
-
-// A nil sink must not panic: the mux is built even when the caller supplied no
-// AnswerSink.
-func TestOuterStreamMuxNilSinkIsSafe(t *testing.T) {
- mux := &outerStreamMux{}
- text := "anything"
- if err := mux.sender(&text, nil); err != nil {
- t.Fatalf("sender: %v", err)
- }
- mux.deliver("anything", true)
- mux.markTerminal()
-}
-
-func TestQuestionKeywordsSeparatesNumbers(t *testing.T) {
- gram := questionKeywords("Population of Paris in 2019")
-
- if !gram.numbers["2019"] {
- t.Fatalf("numbers = %v, want 2019 separated out", gram.numbers)
- }
- if gram.words["2019"] {
- t.Fatal("2019 must not count as a significant word")
- }
- if !gram.words["population"] || !gram.words["paris"] {
- t.Fatalf("words = %v, want population and paris", gram.words)
- }
- // Stopwords are dropped.
- if gram.words["of"] || gram.words["in"] {
- t.Fatalf("words = %v, stopwords must be dropped", gram.words)
- }
-}
-
-func TestCacheSimilarCollapsesReask(t *testing.T) {
- // An observed re-ask: "legal population" → "estimated population of Paris in 2019"
- // (overlap 0.75 while numbers match).
- a := questionKeywords("population of Paris 2019")
- b := questionKeywords("legal population of Paris in 2019")
-
- if !cacheSimilar(a, b) {
- t.Fatal("near-identical re-ask must be judged similar")
- }
-}
-
-func TestCacheSimilarRejectsDifferentNumbers(t *testing.T) {
- // Different years are different questions and must not share an answer.
- a := questionKeywords("population of Paris 2019")
- b := questionKeywords("population of Paris 2015")
-
- if cacheSimilar(a, b) {
- t.Fatal("questions naming different numbers must not be similar")
- }
-}
-
-func TestCacheSimilarRejectsDifferentSubjects(t *testing.T) {
- // Paris vs. Brown County ≈ 0.25 overlap.
- a := questionKeywords("population of Paris 2019")
- b := questionKeywords("population of Brown County 2019")
-
- if cacheSimilar(a, b) {
- t.Fatal("genuinely different questions must not be similar")
- }
-}
-
-func TestCacheLookupAndStore(t *testing.T) {
- c := NewRAGCache()
- c.Store("population of Paris 2019", "2.1 million")
-
- got, ok := c.Lookup("legal population of Paris in 2019")
- if !ok || got != "2.1 million" {
- t.Fatalf("Lookup = %q, %v; want the cached answer", got, ok)
- }
-}
-
-func TestCacheLookupMissesUnrelatedQuestion(t *testing.T) {
- c := NewRAGCache()
- c.Store("population of Paris 2019", "2.1 million")
-
- if _, ok := c.Lookup("who wrote the book about Brown County"); ok {
- t.Fatal("unrelated question must not hit the cache")
- }
-}
-
-func TestCacheReuseBlockedAfterInsufficientRound(t *testing.T) {
- // when the last round was not SUFFICIENT the caller is
- // asking again for more evidence, so the cached answer must not be reused.
- c := NewRAGCache()
- c.Store("population of Paris 2019", "2.1 million")
- c.noteVerdict("INSUFFICIENT")
-
- if _, ok := c.Lookup("legal population of Paris in 2019"); ok {
- t.Fatal("reuse must be blocked after an INSUFFICIENT round")
- }
-
- c.noteVerdict("SUFFICIENT")
- if _, ok := c.Lookup("legal population of Paris in 2019"); !ok {
- t.Fatal("reuse must resume once the round is sufficient")
- }
-}
-
-func TestRagDefaultsToPerTurnCache(t *testing.T) {
- // Caching is default-on and per-turn: Rag auto-builds a RAGCache when deps.Cache is nil, so
- // caching is never off, and two independent instances never share (the cache is rebuilt
- // every turn).
- a := NewRAGCache()
- b := NewRAGCache()
- a.Store("population of Paris 2019", "2.1 million")
-
- if _, ok := b.Lookup("legal population of Paris in 2019"); ok {
- t.Fatal("an independent cache must not hit another instance's answer")
- }
-
- // A nil cache is still inert to direct calls (Rag builds its own internally
- // only for the duration of the call), so no cross-call reuse happens without
- // an explicitly shared instance.
- var nilCache *RAGCache
- if _, ok := nilCache.Lookup("population of Paris 2019"); ok {
- t.Fatal("a nil cache must never hit")
- }
-}
-
-func TestCacheSharedWhenCallerInjectsSameInstance(t *testing.T) {
- // A caller may widen reuse beyond one request by injecting the same
- // *RAGCache via RAGTools.Cache (e.g. across `rag` tool calls within a turn).
- c := NewRAGCache()
- c.Store("population of Paris 2019", "2.1 million")
- c2 := c // same instance passed on a later call
-
- if _, ok := c2.Lookup("legal population of Paris in 2019"); !ok {
- t.Fatal("an injected shared cache must reuse answers across calls")
- }
-}
-
-func TestNilCacheIsInert(t *testing.T) {
- var c *RAGCache
-
- if _, ok := c.Lookup("anything"); ok {
- t.Fatal("nil cache must not hit")
- }
- c.Store("q", "a") // must not panic
- c.noteVerdict("SUFFICIENT")
-}
-
-func TestResolveEffectiveQuestionPrefersOriginal(t *testing.T) {
- // The outer rewrite dropped the final target of a multi-hop question.
- got := resolveEffectiveQuestion(
- "when did the purchaser die",
- "when did the purchaser of the shortest abbreviation die",
- )
-
- if got != "when did the purchaser of the shortest abbreviation die" {
- t.Fatalf("got %q, want the original question", got)
- }
-}
-
-func TestResolveEffectiveQuestionKeepsDifferentTurn(t *testing.T) {
- // A genuine re-ask for a different question must keep its own query.
- got := resolveEffectiveQuestion("population of Brown County", "population of Paris 2019")
-
- if got != "population of Brown County" {
- t.Fatalf("got %q, want the rewrite kept for a different question", got)
- }
-}
-
-func TestResolveEffectiveQuestionHandlesEmpty(t *testing.T) {
- if got := resolveEffectiveQuestion("q", ""); got != "q" {
- t.Fatalf("got %q, want q", got)
- }
- if got := resolveEffectiveQuestion("", "orig"); got != "" {
- t.Fatalf("got %q, want empty", got)
- }
-}
-
-// TestResearchStatusTrailerStopsAfterTwoUnanswerable asserts that once the
-// shared counter has reached 2, Rag() appends the "STOP calling rag again"
-// trailing sentence for an INSUFFICIENT verdict — the
-// visible effect of the consecutive-unanswerable guard across multiple outer
-// rag() calls.
-func TestResearchStatusTrailerStopsAfterTwoUnanswerable(t *testing.T) {
- cache := NewRAGCache()
- cache.consecutiveUnanswerable = 2
- resp := &RunResponse{
- Verdict: VerdictInsufficient,
- SCAFeedback: "evidence is not yet sufficient",
- Answer: "Partial findings.",
- }
- got := researchStatusTrailer(cache, resp)
- if !strings.Contains(got, "STOP calling rag again") {
- t.Fatalf("trailer = %q, want it to tell the outer agent to STOP calling rag again", got)
- }
-}
-
-// TestResearchStatusTrailerInvitesFocusedReaskBelowTwo covers the first
-// unsatisfying outer rag() call (counter < 2): Rag() should invite a focused
-// re-ask rather than tell the agent to stop.
-
-// TestResearchStatusTrailerInvitesFocusedReaskBelowTwo covers the first
-// unsatisfying outer rag() call (counter < 2): Rag() should invite a focused
-// re-ask rather than tell the agent to stop.
-func TestResearchStatusTrailerInvitesFocusedReaskBelowTwo(t *testing.T) {
- cache := NewRAGCache()
- cache.consecutiveUnanswerable = 1
- resp := &RunResponse{
- Verdict: VerdictInsufficient,
- SCAFeedback: "evidence is not yet sufficient",
- Answer: "Partial findings.",
- }
- got := researchStatusTrailer(cache, resp)
- if !strings.Contains(got, "call rag again with a question focused on them") {
- t.Fatalf("trailer = %q, want a focused re-ask invite (not STOP)", got)
- }
- if strings.Contains(got, "STOP calling rag again") {
- t.Fatalf("trailer = %q, did not expect STOP below threshold", got)
- }
-}
-
-// TestResearchStatusTrailerSkipsSufficientOrEmpty ensures the note is omitted
-// when there is nothing to annotate: a SUFFICIENT verdict, an empty answer, or
-// missing SCA feedback all yield "".
-
-// TestResearchStatusTrailerSkipsSufficientOrEmpty ensures the note is omitted
-// when there is nothing to annotate: a SUFFICIENT verdict, an empty answer, or
-// missing SCA feedback all yield "".
-func TestResearchStatusTrailerSkipsSufficientOrEmpty(t *testing.T) {
- cache := NewRAGCache()
- cache.consecutiveUnanswerable = 2
-
- cases := []struct {
- name string
- resp *RunResponse
- }{
- {"sufficient verdict", &RunResponse{Verdict: VerdictSufficient, SCAFeedback: "ok", Answer: "A"}},
- {"empty answer", &RunResponse{Verdict: VerdictInsufficient, SCAFeedback: "x", Answer: ""}},
- {"missing sca feedback", &RunResponse{Verdict: VerdictInsufficient, SCAFeedback: "", Answer: "A"}},
- }
- for _, c := range cases {
- if got := researchStatusTrailer(cache, c.resp); got != "" {
- t.Errorf("%s: trailer = %q, want empty", c.name, got)
- }
- }
-}
-
-// streamingModel is a SessionModel that can also stream.
-type streamingModel struct {
- pieces []string
- fail bool
-}
-
-func (m *streamingModel) Complete(_ context.Context, _ []schema.Message, _ []runtime.ToolSpec) (*runtime.ModelReply, error) {
- if m.fail {
- return nil, errors.New("boom")
- }
- out := ""
- for _, p := range m.pieces {
- out += p
- }
- return &runtime.ModelReply{Content: out}, nil
-}
-
-func (m *streamingModel) StreamComplete(_ context.Context, _ []schema.Message, _ []runtime.ToolSpec, onDelta func(string, bool) error) (*runtime.ModelReply, error) {
- if m.fail {
- return nil, errors.New("stream boom")
- }
- out := ""
- for _, p := range m.pieces {
- if onDelta != nil {
- if err := onDelta(p, false); err != nil {
- return nil, err
- }
- }
- out += p
- }
- return &runtime.ModelReply{Content: out}, nil
-}
-
-func TestComposeAnswerStreamForwardsDeltas(t *testing.T) {
- m := &streamingModel{pieces: []string{"Hello ", "world"}}
- var got []string
- res, err := ComposeAnswerStream(context.Background(), AnswerDeps{Model: m}, m, nil, "q", false, false,
- func(delta string, _ bool) error {
- got = append(got, delta)
- return nil
- })
-
- if err != nil {
- t.Fatalf("unexpected error: %v", err)
- }
- if res.Answer != "Hello world" {
- t.Fatalf("Answer = %q, want %q", res.Answer, "Hello world")
- }
- if len(got) != 2 || got[0] != "Hello " || got[1] != "world" {
- t.Fatalf("deltas = %v, want the two pieces in order", got)
- }
-}
-
-func TestComposeAnswerStreamReturnsErrorOnFailure(t *testing.T) {
- m := &streamingModel{fail: true}
-
- if _, err := ComposeAnswerStream(context.Background(), AnswerDeps{Model: m}, m, nil, "q", false, false, nil); err == nil {
- t.Fatal("want an error so the caller can fall back to one-shot")
- }
-}
-
-func TestAnswerSinkDeliverAndReset(t *testing.T) {
- var got []string
- resets := 0
- s := &AnswerSink{
- OnDelta: func(delta string, _ bool) { got = append(got, delta) },
- OnReset: func() { resets++; got = nil },
- }
-
- s.deliver("a", false)
- s.deliver("b", false)
- s.reset()
-
- if len(got) != 0 || resets != 1 {
- t.Fatalf("after reset: got=%v resets=%d, want empty and 1", got, resets)
- }
- s.deliver("", false) // empty deltas are dropped
- if len(got) != 0 {
- t.Fatalf("got = %v, want empty delta dropped", got)
- }
-}
-
-func TestNilAnswerSinkIsInert(t *testing.T) {
- var s *AnswerSink
-
- s.deliver("x", false) // must not panic
- s.reset()
-}
-
-func TestGetCitationGuidelinesUsesDefaultWithoutOverride(t *testing.T) {
- got := GetCitationGuidelines("")
- // The default is the embedded citation_prompt.md. It must contain the citation-rules body.
- if !strings.Contains(got, "# Citation Requirements:") ||
- !strings.Contains(got, "Place citations at the end of sentences") {
- t.Fatalf("got %q, want the default citation rules", got)
- }
-}
-
-func TestGetCitationGuidelinesHonoursOverride(t *testing.T) {
- // The user's template is rendered and the illustrative-IDs caveat is STILL appended after
- // it.
- got := GetCitationGuidelines("Cite as [n].")
- if !strings.HasPrefix(got, "Cite as [n].") {
- t.Fatalf("got %q, want the override first", got)
- }
- if !strings.Contains(got, "IMPORTANT: The example IDs above") {
- t.Fatalf("got %q, want the illustrative-IDs caveat appended", got)
- }
-}
-
-func TestSysPromptIncludesSummarizeOnlyWithUnstructured(t *testing.T) {
- withDoc := SysPrompt("", true)
- if !strings.Contains(withDoc, "summarize_document") {
- t.Fatal("summarize_document must be offered when unstructured retrieval exists")
- }
- withoutDoc := SysPrompt("", false)
- if strings.Contains(withoutDoc, "summarize_document") {
- t.Fatal("summarize_document must be omitted without unstructured retrieval")
- }
-}
-
-func TestSysPromptPrependsSystemPrompt(t *testing.T) {
- got := SysPrompt("You are helpful.", true)
-
- if !strings.HasPrefix(got, "You are helpful.\n\n") {
- t.Fatalf("got %q, want the system prompt first", got)
- }
- if !strings.Contains(got, "call the `rag` tool") {
- t.Fatalf("got %q, want the router body kept", got)
- }
-}
-
-func TestFitEvidenceKeepsShortEvidence(t *testing.T) {
- evidence := "short evidence"
- if got := FitEvidence("q", evidence); got != evidence {
- t.Fatalf("got %q, want %q unchanged", got, evidence)
- }
-}
-
-func TestFitEvidenceReturnsEmptyForEmpty(t *testing.T) {
- if got := FitEvidence("q", ""); got != "" {
- t.Fatalf("got %q, want empty", got)
- }
-}
-
-func TestFitEvidenceTrimsOversizedEvidence(t *testing.T) {
- // The budget is fixed (not the model window), so a large pool can never fill
- // the context — evidence beyond the cap must be dropped.
- evidence := strings.Repeat("word ", 200000)
- got := FitEvidence("q", evidence)
-
- if len(got) >= len(evidence) {
- t.Fatalf("evidence was not trimmed: %d >= %d", len(got), len(evidence))
- }
- if got == "" {
- t.Fatal("evidence must not be trimmed to nothing")
- }
-}
-
-// TestOuterReactSessionPublishMergesPerCallResults pins the locked merge: a
-// round's rag calls run concurrently, so each publishes its own evidence and
-// answer and none may be lost. Run with -race.
-func TestOuterReactSessionPublishMergesPerCallResults(t *testing.T) {
- session := &outerReactSession{kb: &runtime.Kbinfos{}, resp: &RunResponse{}}
- const calls = 4
-
- var wg sync.WaitGroup
- for i := 0; i < calls; i++ {
- wg.Add(1)
- go func(i int) {
- defer wg.Done()
- session.publish(
- &RunResponse{Answer: fmt.Sprintf("answer-%d", i), Verdict: fmt.Sprintf("verdict-%d", i)},
- &runtime.Kbinfos{
- Chunks: []map[string]any{{"chunk_id": fmt.Sprintf("c%d", i)}},
- DocAggs: []map[string]any{{"doc_id": fmt.Sprintf("d%d", i)}},
- Memory: []map[string]any{{"id": fmt.Sprintf("m%d", i)}},
- })
- }(i)
- }
- wg.Wait()
-
- if len(session.kb.Chunks) != calls || len(session.kb.DocAggs) != calls || len(session.kb.Memory) != calls {
- t.Fatalf("merged evidence = %d chunks / %d doc_aggs / %d memory, want %d each",
- len(session.kb.Chunks), len(session.kb.DocAggs), len(session.kb.Memory), calls)
- }
- seen := map[string]bool{}
- for _, c := range session.kb.Chunks {
- id, _ := c["chunk_id"].(string)
- seen[id] = true
- }
- for i := 0; i < calls; i++ {
- if !seen[fmt.Sprintf("c%d", i)] {
- t.Errorf("merged chunks lost c%d: %v", i, seen)
- }
- }
- if !strings.HasPrefix(session.resp.Answer, "answer-") {
- t.Errorf("answer = %q, want one call's answer", session.resp.Answer)
- }
- if !strings.HasPrefix(session.resp.Verdict, "verdict-") {
- t.Errorf("verdict = %q, want one call's verdict", session.resp.Verdict)
- }
-}
-
-// TestOuterReactSessionToolCallKeepsSharedRequestIntact pins the per-call
-// isolation: models.appendToolResults runs a round's rag calls concurrently
-// (chat_tools.go), so a call must apply its rewritten question and its cleared
-// images to a COPY — never to the request a concurrent call is reading.
-// Run with -race.
-func TestOuterReactSessionToolCallKeepsSharedRequestIntact(t *testing.T) {
- spec := runtime.GetMode("naive")
- session := &outerReactSession{
- ctx: context.Background(),
- spec: spec,
- kb: &runtime.Kbinfos{},
- resp: &RunResponse{Mode: spec},
- logger: log.New(io.Discard, "", 0),
- req: runtime.RunRequest{
- Question: "original question",
- Images: []string{"data:image/png;base64,AAAA"},
- },
- }
-
- var wg sync.WaitGroup
- for _, q := range []string{"first question", "second question"} {
- wg.Add(1)
- go func(q string) {
- defer wg.Done()
- if _, err := session.ToolCall("rag", map[string]interface{}{"question": q}); err != nil {
- t.Errorf("ToolCall(%q): %v", q, err)
- }
- }(q)
- }
- wg.Wait()
-
- if session.req.Question != "original question" {
- t.Errorf("shared request question = %q, want it untouched (each call works on a copy)", session.req.Question)
- }
- if len(session.req.Images) != 1 {
- t.Errorf("shared request images = %v, want them untouched", session.req.Images)
- }
-}
-
-// TestOuterReactSessionNarratesToolCalls pins the outer loop's think-block
-// narration: the "[Function tool] Running the {name} tool with: {args}" line
-// Python's FunctionToolSession emits (tool_decorator.py:311) and the result line
-// that closes it — so the top-level step records which tool the outer model
-// called, with which arguments, and what came back.
-func TestOuterReactSessionNarratesToolCalls(t *testing.T) {
- // summarize_document with no doc_id is the early-return branch: it needs no
- // retrieval wiring, so the narration contract is testable in isolation.
- var buf strings.Builder
- spec := runtime.GetMode("naive")
- session := &outerReactSession{
- ctx: context.Background(),
- spec: spec,
- kb: &runtime.Kbinfos{},
- resp: &RunResponse{Mode: spec},
- logger: log.New(&buf, "", 0),
- req: runtime.RunRequest{Question: "q"},
- }
-
- if _, err := session.ToolCall("summarize_document", map[string]interface{}{}); err != nil {
- t.Fatalf("ToolCall: %v", err)
- }
- out := buf.String()
- for _, want := range []string{
- "[Function tool] Running the summarize_document tool with: {}",
- "[Function tool] The summarize_document tool could not run: it was called without a doc_id.",
- } {
- if !strings.Contains(out, want) {
- t.Errorf("think-log narration missing %q:\n%s", want, out)
- }
- }
-}
-
-// TestOuterReactSessionUnknownToolNarrates pins the unknown-name branch: a tool
-// the deployment has no binding for reports that nothing ran (MISS/unwired), not
-// an infra failure — and it carries the call's arguments like every other step.
-func TestOuterReactSessionUnknownToolNarrates(t *testing.T) {
- var buf strings.Builder
- events := make(chan runtime.ThinkEvent, 4)
- spec := runtime.GetMode("naive")
- session := &outerReactSession{
- ctx: context.Background(), spec: spec, kb: &runtime.Kbinfos{},
- resp: &RunResponse{Mode: spec}, logger: log.New(&buf, "", 0),
- req: runtime.RunRequest{Question: "q"},
- }
- ctx := runtime.WithSteps(context.Background(), runtime.StepReporter{
- Events: func(ev runtime.ThinkEvent) { events <- ev },
- })
- session.ctx = ctx
-
- if _, err := session.ToolCall("time_travel", map[string]interface{}{"query": "tomorrow"}); err != nil {
- t.Fatalf("ToolCall: %v", err)
- }
- if out := buf.String(); !strings.Contains(out,
- `[Function tool] The time_travel tool is not wired in this deployment, so nothing ran.`) {
- t.Errorf("think-log narration:\n%s", out)
- }
-
- var result runtime.ThinkEvent
- for len(events) > 0 {
- ev := <-events
- if ev.Kind == runtime.ThinkKindToolResult {
- result = ev
- }
- }
- if result.Status != runtime.StatusMiss || result.Reason != runtime.ReasonUnwired {
- t.Errorf("result event = %#v, want a miss/unwired outcome", result)
- }
- if result.Args != `{"query":"tomorrow"}` {
- t.Errorf("result args = %q, want the call's arguments (that is what pairs them)", result.Args)
- }
-}
-
-// TestOuterToolNames pins the tool list the loop's opening step reports: the
-// names the outer model was actually given (the pipeline used to hardcode a
-// single "rag", regardless of what was bound).
-func TestOuterToolNames(t *testing.T) {
- schemas := []map[string]any{
- {"type": "function", "function": map[string]any{"name": "rag"}},
- {"type": "function", "function": map[string]any{"name": "summarize_document"}},
- // A malformed entry contributes nothing rather than a "<nil>" name.
- {"type": "function"},
- {"type": "function", "function": map[string]any{}},
- }
- if got := outerToolNames(schemas); got != "rag, summarize_document" {
- t.Errorf("outerToolNames = %q, want the bound names", got)
- }
- if got := outerToolNames(nil); got != "" {
- t.Errorf("outerToolNames(nil) = %q, want empty", got)
- }
-}
-
-// TestOuterLoopEndLine pins the closing step's cases: research ran and answered;
-// research ran but composed nothing — the case the call count alone could not see,
-// and which would otherwise contradict the rag result line printed right above it
-// ("gathered N passages but composed no answer"); or no research ran at all, with
-// an answer (the outer model's own reply, which is why it may carry no citations)
-// or with none.
-func TestOuterLoopEndLine(t *testing.T) {
- cases := []struct {
- name string
- ragCalls int
- answered bool
- want string
- }{
- {"answered", 1, true, "The rag tool produced the final answer, done."},
- {"ran-without-answer", 2, false, "The rag tool ran but produced no answer."},
- {"direct-answer", 0, true, "The outer model produced the answer without running research."},
- {"nothing-at-all", 0, false, "The outer model returned no answer and ran no research."},
- }
- for _, tc := range cases {
- if got := outerLoopEndLine(tc.ragCalls, tc.answered); got != tc.want {
- t.Errorf("%s: outerLoopEndLine = %q, want %q", tc.name, got, tc.want)
- }
- }
-}
-
-// TestOuterLoopAbsentLineStaysOutOfTheThinkBlock pins where that line goes: the
-// developer log, not the think block. It describes the runtime' wiring rather than
-// the question or the research, and a deployment without an outer loop would print
-// it on every single run — while the reader-visible trace is simply and honestly
-// missing its "[Tool loop]" section.
-func TestOuterLoopAbsentLineStaysOutOfTheThinkBlock(t *testing.T) {
- var think, logged strings.Builder
- ctx := runtime.WithSteps(context.Background(), runtime.StepReporter{
- Text: func(line string) { think.WriteString(line) },
- })
- Rag(ctx, RAGTools{Logger: log.New(&logged, "", 0)}, runtime.RunRequest{
- Question: "anything",
- ThinkingMode: "low",
- DatasetIDs: []string{"kb1"},
- })
-
- want := "No outer model is wired, so this run has no outer tool loop."
- if !strings.Contains(logged.String(), want) {
- t.Errorf("developer log missing %q; got:\n%s", want, logged.String())
- }
- if strings.Contains(think.String(), "outer tool loop") {
- t.Errorf("the wiring note must not reach the think block; got:\n%s", think.String())
- }
-}
-
-// TestRagToolResultLine pins the outer `rag` result wording: it reports what the
-// answer was grounded in (or why there is none), never the answer itself — and it
-// names the sub-question it researched, because the outer loop can have several
-// `rag` calls in flight whose result lines would otherwise be identical.
-func TestRagToolResultLine(t *testing.T) {
- twoChunks := []map[string]any{{"chunk_id": "c1"}, {"chunk_id": "c2"}}
- label := ` for "曹操是谁"`
- cases := []struct {
- name string
- resp *RunResponse
- kb *runtime.Kbinfos
- label string
- want string
- }{
- {"grounded", &RunResponse{Answer: "cited answer"}, &runtime.Kbinfos{Chunks: twoChunks}, label,
- `The rag tool returned a cited answer grounded in 2 passages for "曹操是谁".`},
- {"grounded-singular", &RunResponse{Answer: "a"}, &runtime.Kbinfos{Chunks: twoChunks[:1]}, label,
- `The rag tool returned a cited answer grounded in 1 passage for "曹操是谁".`},
- {"no-evidence", &RunResponse{Answer: "answer"}, &runtime.Kbinfos{}, label,
- `The rag tool returned no answer for "曹操是谁": research gathered no evidence.`},
- {"evidence-but-no-answer", &RunResponse{}, &runtime.Kbinfos{Chunks: twoChunks}, label,
- `The rag tool gathered 2 passages but composed no answer for "曹操是谁".`},
- {"unlabelled", &RunResponse{Answer: "a"}, &runtime.Kbinfos{Chunks: twoChunks[:1]}, "",
- "The rag tool returned a cited answer grounded in 1 passage."},
- }
- for _, tc := range cases {
- if got := ragToolResultLine(tc.resp, tc.kb, tc.label); got != tc.want {
- t.Errorf("%s: ragToolResultLine = %q, want %q", tc.name, got, tc.want)
- }
- }
-}
-
-// TestRagFlightSharesConcurrentIdenticalCalls pins the single-flight contract:
-// a caller arriving while the flight is open waits on it and replays its
-// answer; a caller arriving AFTER the window closed owns a fresh execution
-// (window-only dedup) and — like production's defer — must end it. The main
-// goroutine holds the window open long enough that the waiters normally take
-// the wait path, but the assertions tolerate both arrivals. Run with -race.
-func TestRagFlightSharesConcurrentIdenticalCalls(t *testing.T) {
- session := &outerReactSession{}
-
- owner, wait := session.beginRagFlight("same question")
- if owner == nil || wait != nil {
- t.Fatalf("first caller: owner=%v wait=%v, want (flight, nil)", owner, wait)
- }
-
- const waiters = 3
- var wg sync.WaitGroup
- got := make([]string, waiters)
- for i := 0; i < waiters; i++ {
- wg.Add(1)
- go func(i int) {
- defer wg.Done()
- o, w := session.beginRagFlight("same question")
- if o != nil {
- // Arrived after the window closed: this caller owns a fresh
- // execution and must end it (production's ToolCall defers
- // endRagFlight), or later callers would block forever.
- session.endRagFlight("same question", o)
- return
- }
- <-w.done
- got[i] = w.answer
- }(i)
- }
-
- // Hold the window open so the waiters observe it (scheduling latency is
- // microseconds; the tolerant assertions above cover the pathological case).
- time.Sleep(50 * time.Millisecond)
- owner.answer = "shared answer"
- session.endRagFlight("same question", owner)
- wg.Wait()
-
- for i, g := range got {
- if g != "shared answer" && g != "" {
- t.Errorf("waiter %d replayed %q, want %q or the owner path (\"\")", i, g, "shared answer")
- }
- }
-
- // After the flight ends a NEW call owns a fresh execution (window-only
- // dedup: a later round must genuinely re-run).
- owner2, wait2 := session.beginRagFlight("same question")
- if owner2 == nil || wait2 != nil {
- t.Fatalf("post-window caller: owner=%v wait=%v, want (flight, nil)", owner2, wait2)
- }
- session.endRagFlight("same question", owner2)
-
- // A different question never shares a flight.
- other, otherWait := session.beginRagFlight("different question")
- if other == nil || otherWait != nil {
- t.Fatalf("different question: owner=%v wait=%v, want (flight, nil)", other, otherWait)
- }
- session.endRagFlight("different question", other)
-}
-
-// TestOuterReactSessionToolCallWaitsOnInFlightRag pins the ToolCall-level
-// behavior: a rag call whose question already has an in-progress execution
-// blocks and replays that execution's answer WITHOUT running its own graph —
-// no second publish, no second call record (running both duplicates fully and discarding the
-// loser is wasted work; this wait is the single-flight, see ragFlight).
-func TestOuterReactSessionToolCallWaitsOnInFlightRag(t *testing.T) {
- spec := runtime.GetMode("naive")
- session := &outerReactSession{
- ctx: context.Background(),
- spec: spec,
- kb: &runtime.Kbinfos{},
- resp: &RunResponse{Mode: spec},
- logger: log.New(io.Discard, "", 0),
- req: runtime.RunRequest{Question: "original question"},
- }
-
- owner, wait := session.beginRagFlight("the question")
- if owner == nil || wait != nil {
- t.Fatalf("pre-registered flight: owner=%v wait=%v, want (flight, nil)", owner, wait)
- }
- go func() {
- time.Sleep(20 * time.Millisecond)
- owner.answer = "shared answer"
- session.endRagFlight("the question", owner)
- }()
-
- start := time.Now()
- got, err := session.ToolCall("rag", map[string]interface{}{"question": "the question"})
- if err != nil {
- t.Fatalf("ToolCall: %v", err)
- }
- if got != "shared answer" {
- t.Errorf("ToolCall replayed %q, want %q", got, "shared answer")
- }
- if elapsed := time.Since(start); elapsed < 15*time.Millisecond {
- t.Errorf("ToolCall returned after %v — it did NOT wait for the in-flight execution", elapsed)
- }
- // The waiter must not have published or recorded its own call result.
- if len(session.calls) != 0 {
- t.Errorf("calls = %d, want 0 (a waiting call must not publish)", len(session.calls))
- }
-}
-
-// ragTestChunkIDs lists a chunk pool's ids in order.
-func ragTestChunkIDs(chunks []map[string]any) []string {
- out := make([]string, 0, len(chunks))
- for _, c := range chunks {
- id, _ := c["chunk_id"].(string)
- out = append(out, id)
- }
- return out
-}
-
-// TestRAGCacheConsecutiveUnanswerableIsSerialized pins that the counter lives
-// behind the cache's lock: a round's concurrent rag() calls bump the SAME shared
-// cache, so an unsynchronized increment would race and lose updates.
-// Run with -race.
-func TestRAGCacheConsecutiveUnanswerableIsSerialized(t *testing.T) {
- cache := NewRAGCache()
- const calls = 8
-
- var wg sync.WaitGroup
- for i := 0; i < calls; i++ {
- wg.Add(1)
- go func() {
- defer wg.Done()
- cache.NoteUnanswerable(VerdictInsufficient)
- }()
- }
- wg.Wait()
-
- if got := cache.ConsecutiveUnanswerable(); got != calls {
- t.Errorf("ConsecutiveUnanswerable = %d, want %d (increments must not be lost)", got, calls)
- }
- cache.NoteUnanswerable(VerdictSufficient)
- if got := cache.ConsecutiveUnanswerable(); got != 0 {
- t.Errorf("after a SUFFICIENT verdict = %d, want 0", got)
- }
- if got := (*RAGCache)(nil).ConsecutiveUnanswerable(); got != 0 {
- t.Errorf("nil cache = %d, want 0", got)
- }
- (*RAGCache)(nil).NoteUnanswerable(VerdictInsufficient) // must not panic
-}
-
-// TestOuterReactSessionSelectsWinningCallEvidence pins the citation fix: the
-// terminal fold returns the LOWEST-INDEX terminal rag call's answer while
-// publish() records completion order, so the session must hand back THAT call's
-// evidence — otherwise the answer's [ID:n] markers address another call's chunks.
-func TestOuterReactSessionSelectsWinningCallEvidence(t *testing.T) {
- session := &outerReactSession{kb: &runtime.Kbinfos{}, resp: &RunResponse{}}
- winner := &runtime.Kbinfos{
- Chunks: []map[string]any{{"chunk_id": "w0"}, {"chunk_id": "w1"}},
- DocAggs: []map[string]any{{"doc_id": "dw"}},
- Memory: []map[string]any{{"id": "mw"}},
- PreSummary: "winner summary",
- }
- loser := &runtime.Kbinfos{Chunks: []map[string]any{{"chunk_id": "l0"}}}
-
- // The losing call finishes FIRST, i.e. completion order ≠ index order.
- session.publish(&RunResponse{Answer: "answer-loser"}, loser)
- session.publish(&RunResponse{Answer: "answer-winner"}, winner)
-
- if got := ragTestChunkIDs(session.kb.Chunks); len(got) != 3 || got[0] != "l0" {
- t.Fatalf("unselected union = %v, want the loser's chunk first (that is the misalignment)", got)
- }
-
- session.selectEvidence("answer-winner")
-
- if got := ragTestChunkIDs(session.kb.Chunks); len(got) != 2 || got[0] != "w0" || got[1] != "w1" {
- t.Errorf("chunks after selection = %v, want the winner's [w0 w1] in its own order", got)
- }
- if len(session.kb.DocAggs) != 1 || len(session.kb.Memory) != 1 || session.kb.PreSummary != "winner summary" {
- t.Errorf("winner's evidence fields not restored: %+v", session.kb)
- }
-}
-
-// TestOuterReactSessionSelectEvidenceKeepsUnionWithoutAMatch keeps the fallback:
-// an answer no rag call produced (the terminal tool was summarize_document, or
-// the outer model answered itself) leaves the union in place.
-func TestOuterReactSessionSelectEvidenceKeepsUnionWithoutAMatch(t *testing.T) {
- session := &outerReactSession{kb: &runtime.Kbinfos{}, resp: &RunResponse{}}
- session.publish(&RunResponse{Answer: "answer-a"}, &runtime.Kbinfos{Chunks: []map[string]any{{"chunk_id": "a0"}}})
-
- session.selectEvidence("a summarize_document result")
- session.selectEvidence("")
-
- if got := ragTestChunkIDs(session.kb.Chunks); len(got) != 1 || got[0] != "a0" {
- t.Errorf("chunks = %v, want the union kept when no call matches", got)
- }
-}
-
-// TestTruncCutsOnRuneBoundaries pins the fix for the "\xe3" that showed up in a
-// grep line: trunc used to slice BYTES, cutting a Chinese character in half so
-// the trace printed half a rune as an escape (and for CJK it stopped at a third
-// of the requested length).
-func TestTruncCutsOnRuneBoundaries(t *testing.T) {
- const s = "曹操是谁?"
- if got := trunc(s, 2); got != "曹操" {
- t.Errorf("trunc(%q, 2) = %q, want two whole characters", s, got)
- }
- for n := 0; n <= utf8.RuneCountInString(s)+1; n++ {
- got := trunc(s, n)
- if !utf8.ValidString(got) {
- t.Errorf("trunc(%q, %d) = %q, which is not valid UTF-8", s, n, got)
- }
- if utf8.RuneCountInString(got) > n {
- t.Errorf("trunc(%q, %d) = %q, longer than asked", s, n, got)
- }
- }
- if got := trunc(s, 99); got != s {
- t.Errorf("trunc beyond the length = %q, want it unchanged", got)
- }
-}
diff --git a/internal/rag/agentic-rag/agentic_rag_util.go b/internal/rag/agentic-rag/agentic_rag_util.go
deleted file mode 100644
index 80e9a90985..0000000000
--- a/internal/rag/agentic-rag/agentic_rag_util.go
+++ /dev/null
@@ -1,41 +0,0 @@
-package agentic_rag
-
-import (
- "context"
- "log"
- "time"
-
- "ragflow/internal/rag/agentic-rag/runtime"
-)
-
-// step reports one user-visible reasoning stage of the run: it writes the
-// DEVELOPER log line and delivers the same sentence as a step (think block +
-// structured event) through the reporter bound on ctx.
-//
-// Both projections come from this one call, so they cannot drift; and because a
-// stage declares its own step, the developer log is free to carry diagnostics
-// that must never reach the user — no allowlist of tags, and rewording a log
-// line can never hide (or leak) a step.
-func step(ctx context.Context, logger *log.Logger, stage, format string, args ...any) {
- runtime.StepsFrom(ctx).Stage(logger, stage, format, args...)
-}
-
-// trunc caps s at n RUNES, returning s unchanged when it already fits.
-//
-// It used to slice bytes (s[:n]), which cut multi-byte characters in half: the
-// trace then showed half a Chinese character as an escape (a "\xe3" appeared in
-// a grep line where a rune had been split), and for CJK it also stopped at a
-// third of the intended length. Delegates to this package's truncateRunes.
-func trunc(s string, n int) string {
- return truncateRunes(s, n)
-}
-
-// deadlineToDuration converts a seconds budget to a context deadline. A
-// non-positive budget falls back to the default answer timeout so a caller that
-// omits it does not produce an already-expired context.
-func deadlineToDuration(seconds float64) time.Duration {
- if seconds <= 0 {
- seconds = answerTimeoutS
- }
- return time.Duration(seconds * float64(time.Second))
-}
diff --git a/internal/rag/agentic-rag/coverage_step.go b/internal/rag/agentic-rag/coverage_step.go
deleted file mode 100644
index 4f7b06c4c5..0000000000
--- a/internal/rag/agentic-rag/coverage_step.go
+++ /dev/null
@@ -1,380 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package agentic_rag
-
-import (
- "context"
- "fmt"
- "log"
- "sort"
- "strings"
- "time"
-
- "ragflow/internal/rag/agentic-rag/runtime"
- "ragflow/internal/rag/agentic-rag/slots"
-)
-
-// The enumeration's LAST node: the runtime asks the model about every window the
-// enumeration found, and writes the members its verdicts name.
-//
-// Why this node exists at all: the evidence is not the limit — with the same corpus in
-// hand, runs of one build answer different member counts. What varies is what a session
-// WROTE: each session patches a handful of names and the round's union is whichever list
-// happens to be longest. So the write-back stops being a session's memory and becomes the
-// runtime's own step — every window gets a verdict, and a window nobody answered is
-// reported as UNKNOWN rather than read as "the corpus does not say it".
-//
-// The judgement itself stays the model's (see runtime.CoverageResolvePrompt): nothing here
-// decides what a kill is, or which words mean one.
-func RunCoverageResolve(ctx context.Context, deps RAGTools, st *AgenticState, logger *log.Logger) runtime.ResolveStats {
- if logger == nil {
- logger = _LOG
- }
- stats := runtime.ResolveStats{}
- if st == nil || deps.Model == nil || st.KB == nil || len(st.SlotTable.State) == 0 {
- return stats
- }
- // The node belongs to the ENUMERATION strategy: a question whose answer is one value
- // even one that contains a count — pays nothing for it (see runtime.Coverage.Ok).
- cov := runtime.CoverageOf(st.SlotTable)
- // Two ways in, and the second is why this node is the last one: the planner declared a set, OR
- // the table's own slots hold items (see Coverage.ItemsInValue). A session patching members into
- // a slot the planner typed "person" does not make the QUESTION a set — the record and the
- // sessions keep reading the declaration — but it does mean this node has a set to complete.
- if !cov.Ok() && !cov.ItemsInValue() {
- // Said out loud. A node that returns in silence is indistinguishable from one that judged
- // and found nothing, and this was the whole reason a run could enumerate ten names while
- // the corpus stated sixteen (see enrollEnumeration).
- logger.Printf("[Coverage] resolve skipped: this table is not an enumeration (set=%v itemKind=%q acts=%d items=%v) — nothing to judge here.", cov.Set, cov.ItemKind, len(cov.Acts), cov.ItemsInValue())
- return stats
- }
- enrollEnumeration(ctx, deps, st, cov, logger)
- // And the members go into a slot that ALREADY holds members, so a table whose slots
- // hold values (one name, one date, one number, one phrase) is not enumerated at all
- // and this step spends nothing — not even a call.
- slotID := coverageTargetSlot(&st.SlotTable)
- if slotID < 0 {
- return stats
- }
- set, reached, claimed := coverageResolveCandidates(st.KB, &st.SlotTable, cov)
- if set.Empty() {
- // Nothing to judge: no enumeration ran (no clock left, or the executor cannot search)
- // and no probe reached a name the slots do not already hold.
- logger.Printf("[Coverage] resolve has nothing to judge: 0 line(s).")
- return stats
- }
- if reached+claimed > 0 {
- // The split is worth a line: it is the difference between a node that judges only what
- // its own enumeration recalled and one that also judges what the run's search touched
- // and what the sessions ASSERTED but never showed a passage for. Both numbers are READ
- // from the one list (countSource), never kept beside it.
- logger.Printf("[Coverage] resolve candidates: %d window(s) + %d reached name(s) + %d unjudged claim(s) = %d line(s)",
- len(set.Windows)-reached-claimed, reached, claimed, len(set.Windows))
- }
-
- // Bounded like every other single-purpose call in this graph: the answer composition
- // still has to fit inside the request's clock, so this step may not spend it — and
- // unlike the research nodes it does not claim its floor once that room is gone.
- t := 0.0
- if room := st.RemainingS() - 10.0; room > 0 {
- t = nodeClock(CoverageResolveTimeoutS, 10.0, room)
- }
- callCtx, cancel := context.WithTimeout(ctx, time.Duration(t*float64(time.Second)))
- defer cancel()
-
- members, stats := runtime.ResolveCoverage(callCtx, deps.Model, st.Question, cov, set)
- if stats.Failed > 0 {
- // Said out loud because a failed call is members nobody checked — the failure this
- // node exists to make visible, not to hide. It counts CALLS, so a window that had to
- // be re-asked shows up here twice; what the caller must read is the UNKNOWN count,
- // which is windows.
- logger.Printf("[Coverage] %d of %d resolve call(s) failed; %d window(s) left UNKNOWN", stats.Failed, stats.Batches, stats.Unknown)
- }
- if stats.Unknown > 0 {
- logger.Printf("[Coverage] %d of %d window(s) got no verdict — UNKNOWN, not absent", stats.Unknown, stats.Asked)
- }
- if len(members) == 0 {
- logger.Printf("[Coverage] %d window(s) asked, %d answered, 0 member(s) named", stats.Asked, stats.Answered)
- return stats
- }
-
- items := make([]slots.Item, 0, len(members))
- for _, m := range members {
- items = append(items, slots.Item{Value: m.Name, ChunkID: m.ChunkID, Quote: m.Quote})
- }
- value := slots.Items(items...)
- rendered := slots.Render(value)
- // 1.0, and the number is a statement about the EVIDENCE rather than a claim: every
- // member here was just matched to the line that names it, so the list is accountable
- // in a way a session's prose list is not (see slots.Union).
- strength := 1.0
- branch := runtime.NewState([]runtime.Variable{{
- ID: slotID,
- Value: &value,
- Candidate: &rendered,
- CandidateStrength: &strength,
- }}, st.SlotTable.Depth+1, nil)
- if merged := MergeSlotPatch(st.SlotTable, branch); merged != nil {
- st.SlotTable = *merged
- }
- if raised := syncCountSlots(&st.SlotTable); len(raised) > 0 {
- logger.Printf("[Coverage] count slot(s) %v set to the enumerated members' size", raised)
- }
- // The answer reads the record, and the record was rendered BEFORE this write-back:
- // refresh it, or the members just proved would be invisible to the answer that must
- // list them.
- st.KB.Record = RenderSlotRecord(st.SlotTable, st.CollectedAnswer)
- // The passages behind the items travel with the run (see Kbinfos.NoteCitedChunks): the compose
- // prompt puts them in front of the answer, which must cite one passage per member and otherwise
- // sees only the top-scoring handful.
- st.KB.NoteCitedChunks(runtime.AnchoredItemChunks(&st.SlotTable))
- // And the member↔passage table itself, so the compose can write the citation of every member
- // the answer states (see runtime.CiteAnchoredMembers): which passage a member rests on is the
- // naming node's finding, not a block number the model has to write out of a budgeted render.
- st.KB.NoteAnchoredRefs(runtime.AnchoredItemRefs(&st.SlotTable))
- if stats.Unknown > 0 {
- // An enumeration that did not finish must not read as one that did. The windows with
- // no verdict are members nobody judged, so the list above is a LOWER BOUND, and the
- // record says so: a count the corpus cannot support is worse than a count that names
- // its own gap, and only the record reaches the answer that states the number.
- st.KB.Record += fmt.Sprintf("\n- NOTE: %d of the %d enumerated passage(s) got no verdict, so the members above are a LOWER BOUND — state the count as what it is and do not present it as exact. Nobody read: %s\n",
- stats.Unknown, stats.Asked, strings.Join(cappedList(stats.Unjudged, 12), ", "))
- }
- logger.Printf("[Coverage] resolve done: asked=%d answered=%d unknown=%d member(s)=%d written into slot %d",
- stats.Asked, stats.Answered, stats.Unknown, len(items), slotID)
- return stats
-}
-
-// coverageTargetSlot is the slot the members go into: the member slot that already holds
-// the most members — the enumeration's own list slot, the one the record and the answer
-// read. -1 when the table holds no members at all.
-func coverageTargetSlot(table *runtime.State) int {
- if table == nil {
- return -1
- }
- best, bestN := -1, -1
- for _, v := range table.State {
- if v.Typed().Kind != slots.KindItems {
- continue
- }
- if n := len(v.Typed().ItemValues()); n > bestN {
- best, bestN = v.ID, n
- }
- }
- return best
-}
-
-// coverageResolveCandidates assembles the lines the point-of-naming node judges: the
-// enumeration's windows, then the names a probe already REACHED and no slot holds yet.
-//
-// The second source is not bookkeeping — it is the difference between "the corpus was asked
-// about this direction" and "the run's own search touched this name". A reached name has the
-// pool passage that carries it, so it is a member with its evidence whatever the session did
-// with it afterwards, and a node that judges only its own enumeration drops exactly those
-// names: the run holds the passage, names it nowhere, and the count comes out short of what
-// the corpus states.
-//
-// Both sources are deduped before they are asked about: a name that is already SETTLED, and a
-// name a window of the SAME passage already quotes, are not put in front of the model twice.
-// Settled means anchored (see runtime.AnchoredItems): a name a session wrote without a passage is
-// a claim nobody has ruled on, so it stays a candidate — which is what gives the run a way to
-// refute a claim it holds the refuting passage for.
-//
-// Returns the union, how many lines the reached terms contributed, and how many unanchored claims
-// were handed over.
-func coverageResolveCandidates(kb *runtime.Kbinfos, table *runtime.State, cov runtime.Coverage) (runtime.CoverageSet, int, int) {
- var set runtime.CoverageSet
- if kb == nil {
- return set, 0, 0
- }
- set, _ = kb.CoverageSet()
- // Only anchored items are settled: a name written without a passage has not been judged, so it
- // stays a candidate here.
- known := map[string]bool{}
- for _, name := range runtime.AnchoredItems(table) {
- known[strings.ToLower(strings.TrimSpace(name))] = true
- }
- // Everything below goes into ONE list, judged by ONE node under ONE budget; what differs is
- // where a line came from, which is a field on the line (CoverageWindow.Source) rather than a
- // pipeline with its own cap, its own dedup and its own counter.
- //
- // The two counters below count TERMS, not lines, because that is what the report says
- // ("reached name(s)"): a name the pool carries three passages about is one name handed over.
- // A count read off the lines instead would report three — measured by the test that pins the
- // split (coverage_step_test.go: 于禁 has two passages in the pool and is ONE claim).
- added := 0
- for _, rt := range kb.ReachedTerms() {
- term := strings.TrimSpace(rt.Term)
- key := strings.ToLower(term)
- if term == "" || rt.ChunkID == "" || known[key] {
- continue
- }
- if windowQuotesTerm(set.Windows, rt.ChunkID, term) {
- continue
- }
- // The words are what makes the name answerable: a member no line can point at is a
- // member nobody can cite, so a reached term whose passage is no longer in the pool is
- // left to the record's own ledger instead of being handed over as a bare name.
- quote := ledgerQuote(kb, rt.ChunkID, term)
- if quote == "" {
- continue
- }
- set.Windows = append(set.Windows, runtime.CoverageWindow{ChunkID: rt.ChunkID, Quote: quote, Source: runtime.CoverageSourceReached})
- known[key] = true
- added++
- }
- // Third source: the names a session asserted with no passage. They are handed over as candidates
- // with whatever the pool can show about them, so a claim the run can refute is refuted rather
- // than counted or silently dropped.
- claimed := 0
- for _, name := range runtime.UnanchoredItems(table) {
- term := strings.TrimSpace(name)
- key := strings.ToLower(term)
- if term == "" || known[key] {
- continue
- }
- windows := claimWindows(kb, term, cov, coverageClaimWindowsPerName)
- if len(windows) == 0 {
- continue
- }
- for _, w := range windows {
- if windowQuotesTerm(set.Windows, w.ChunkID, term) {
- continue
- }
- w.Source = runtime.CoverageSourceClaim
- set.Windows = append(set.Windows, w)
- }
- known[key] = true
- claimed++
- }
- return set, added, claimed
-}
-
-// cappedList returns at most n entries, marking that more were left out.
-func cappedList(items []string, n int) []string {
- if len(items) <= n {
- return items
- }
- return append(items[:n], "…")
-}
-
-// enrollEnumeration runs the direction's window enumeration when the table holds items but no
-// enumeration stands behind them.
-//
-// The planner's type words are a hint; the sessions' own output is the fact (see CoverageOf, which
-// reads the value). When the two disagree it is this node that pays for it, because it is the LAST
-// one that can complete the set: enrolling here is what turns "the names a session remembered" back
-// into "the windows the corpus states".
-func enrollEnumeration(ctx context.Context, deps RAGTools, st *AgenticState, cov runtime.Coverage, logger *log.Logger) {
- if _, done := st.KB.CoverageSet(); done {
- return
- }
- if cov.ItemKind == "" || len(cov.Operands()) == 0 || deps.Tools == nil {
- return
- }
- if st.RemainingS() < CoverageEnrollHeadroomS {
- logger.Printf("[Coverage] enumeration not enrolled: %.0fs left (needs %.0fs for it and the answer beside it); judging what the run already holds.", st.RemainingS(), CoverageEnrollHeadroomS)
- return
- }
- if !runtime.CoverageActsMeetActor(st.KB, cov) {
- // The recall would be filtered by a conjunction the run has never seen hold: an act word and
- // the actor in ONE passage. Spending it here spends the answer's clock to return nothing —
- // measured, this exact pass cost 724 recalled passages and produced zero windows.
- logger.Printf("[Coverage] enumeration not enrolled: the direction's words have never met in the passages already held (no act word and actor in one passage), so no window could survive its own filter.")
- return
- }
- runner, ok := deps.Tools.Exec.(runtime.CoverageRunner)
- if !ok {
- logger.Printf("[Coverage] enumeration not enrolled: the executor cannot enumerate (it does not implement runtime.CoverageRunner).")
- return
- }
- set := runner.EnumerateCoverage(ctx, cov, st.KB)
- if len(set.Operands) == 0 {
- return
- }
- logger.Printf("[Coverage] enumeration enrolled at the resolve node: %d operand(s) asked, %d passage(s) recalled, %d window(s) found (no type word had asked for it).",
- len(set.Operands), set.Recalled, len(set.Windows))
- st.KB.MarkCoverage(cov)
- st.KB.StoreCoverageSet(set)
-}
-
-// coverageClaimWindowsPerName bounds the passages ONE claimed name is shown with: enough to
-// contain the one that states the deed (a name is usually mentioned in several places, and only
-// one of them is the killing), and no more.
-const coverageClaimWindowsPerName = 2
-
-// claimWindows finds the pool passages that mention a claimed name, the ones carrying the deed's
-// words or the actor first. A claim no passage mentions is returned nothing: it stays unjudged.
-func claimWindows(kb *runtime.Kbinfos, name string, cov runtime.Coverage, max int) []runtime.CoverageWindow {
- type scored struct {
- window runtime.CoverageWindow
- score int
- }
- var found []scored
- for _, c := range kb.ChunksFrom(0, kb.PoolSize()) {
- id := runtime.ChunkIDOf(c)
- if id == "" {
- continue
- }
- text := runtime.ChunkTextOf(c)
- if !strings.Contains(text, name) {
- continue
- }
- score := 0
- act := ""
- for _, a := range cov.Acts {
- if a != "" && strings.Contains(text, a) {
- act = a
- score++
- break
- }
- }
- for _, form := range cov.Actors() {
- if form != "" && strings.Contains(text, form) {
- score++
- break
- }
- }
- quote := ledgerQuote(kb, id, name)
- if quote == "" {
- continue
- }
- // The act word the passage carries rides the window, so the ranking that offered it is
- // visible in the log (see CoverageWindow.Act).
- found = append(found, scored{runtime.CoverageWindow{ChunkID: id, Quote: quote, Act: act}, score})
- }
- sort.SliceStable(found, func(i, j int) bool { return found[i].score > found[j].score })
- out := make([]runtime.CoverageWindow, 0, max)
- for _, f := range found {
- if len(out) >= max {
- break
- }
- out = append(out, f.window)
- }
- return out
-}
-
-// windowQuotesTerm reports whether a window of this passage already carries the term: that line
-// names the member once, and a second line quoting the same words would ask about it twice.
-func windowQuotesTerm(windows []runtime.CoverageWindow, chunkID, term string) bool {
- for _, w := range windows {
- if w.ChunkID == chunkID && strings.Contains(w.Quote, term) {
- return true
- }
- }
- return false
-}
diff --git a/internal/rag/agentic-rag/coverage_step_test.go b/internal/rag/agentic-rag/coverage_step_test.go
deleted file mode 100644
index 3347659a96..0000000000
--- a/internal/rag/agentic-rag/coverage_step_test.go
+++ /dev/null
@@ -1,201 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package agentic_rag
-
-import (
- "context"
- "strings"
- "testing"
-
- "ragflow/internal/rag/agentic-rag/runtime"
- "ragflow/internal/rag/agentic-rag/slots"
-)
-
-// TestCoverageResolveCandidatesJudgeWhatTheRunTouched pins the naming node's TWO sources: the
-// enumeration's windows, and the names a probe already REACHED.
-//
-// A reached name has the pool passage that carries it, so it is a member with its evidence
-// whatever the session did with it afterwards. A node that judges only the enumeration's own
-// windows drops exactly those names: the run holds the passage, names it nowhere, and the count
-// comes out short of what the corpus states.
-func TestCoverageResolveCandidatesJudgeWhatTheRunTouched(t *testing.T) {
- kb := &runtime.Kbinfos{}
- kb.Admit(func(p *runtime.PoolAdmitter) {
- p.Add(map[string]any{"chunk_id": "c1", "content_with_weight": "夏侯存军至,见了云长,被云长砍死。"})
- })
- kb.RecordReachedTerm("夏侯存", "c1")
- kb.RecordReachedTerm("文丑", "c1") // a slot already declares it: not asked again
- kb.RecordReachedTerm("杨龄", "nowhere") // no passage in the pool: no words behind it
- kb.StoreCoverageSet(runtime.CoverageSet{
- Operands: []string{"云长", "斩"},
- Windows: []runtime.CoverageWindow{{ChunkID: "w1", Quote: "云长手起刀落,斩孔秀于马下"}},
- })
- value := slots.Items(slots.Item{Value: "文丑", ChunkID: "c9", Quote: "云长诛文丑"})
- table := runtime.NewState([]runtime.Variable{{ID: 0, Type: "person", Value: &value}}, 0, nil)
-
- set, added, claims := coverageResolveCandidates(kb, &table, runtime.Coverage{})
- if added != 1 || len(set.Windows) != 2 || claims != 0 {
- t.Fatalf("candidates = %d window(s) (+%d reached, %d claimed), want the enumeration's window plus the one reached name",
- len(set.Windows), added, claims)
- }
- if got := set.Windows[1]; got.ChunkID != "c1" || !strings.Contains(got.Quote, "夏侯存") {
- t.Errorf("line = %+v, want the reached name with the words behind it", got)
- }
- for _, w := range set.Windows {
- if strings.Contains(w.Quote, "文丑") {
- t.Errorf("line = %+v, want a name a slot already declares left out", w)
- }
- if strings.TrimSpace(w.Quote) == "" || w.ChunkID == "" {
- t.Errorf("line = %+v, want both the passage and its words (a bare name cannot be cited)", w)
- }
- }
-}
-
-// TestCoverageResolveCandidatesSkipWhatIsAlreadyQuoted pins the dedup that keeps the node from
-// asking about one passage twice: a window of the same chunk already carries those words.
-func TestCoverageResolveCandidatesSkipWhatIsAlreadyQuoted(t *testing.T) {
- kb := &runtime.Kbinfos{}
- kb.Admit(func(p *runtime.PoolAdmitter) {
- p.Add(map[string]any{"chunk_id": "c1", "content_with_weight": "云长提华雄之头,掷于地上,其酒尚温。"})
- })
- kb.RecordReachedTerm("华雄", "c1")
- kb.StoreCoverageSet(runtime.CoverageSet{
- Windows: []runtime.CoverageWindow{{ChunkID: "c1", Quote: "云长提华雄之头,掷于地上"}},
- })
-
- set, added, claims := coverageResolveCandidates(kb, nil, runtime.Coverage{})
- if added != 0 || len(set.Windows) != 1 || claims != 0 {
- t.Fatalf("candidates = %d window(s) (+%d reached, %d claimed), want the window that already quotes it, once",
- len(set.Windows), added, claims)
- }
-}
-
-// TestCoverageResolveCandidatesReopenAClaimNobodyJudged pins what "already known" means: JUDGED,
-// not written down.
-//
-// A name a session wrote with no passage has been asserted, not ruled on, so it stays a candidate
-// for the point-of-naming node whenever the run holds a passage about it. Treating the claim as
-// settled is how 于禁 (taken at 水淹七军 and released, never killed) stayed in the table
-// unexamined while the passage that refutes it sat in the pool; once the name carries a passage,
-// the same call leaves it alone.
-func TestCoverageResolveCandidatesReopenAClaimNobodyJudged(t *testing.T) {
- kb := &runtime.Kbinfos{}
- kb.Admit(func(p *runtime.PoolAdmitter) {
- p.Add(map[string]any{"chunk_id": "c1", "content_with_weight": "于禁拜伏于地,乞哀请命。关公曰:汝怎敢抗吾?"})
- })
- kb.RecordReachedTerm("于禁", "c1")
- claimed := slots.Items(slots.Item{Value: "于禁"})
- table := runtime.NewState([]runtime.Variable{{ID: 1, Type: "person", Value: &claimed}}, 0, nil)
-
- set, added, claims := coverageResolveCandidates(kb, &table, runtime.Coverage{})
- if added != 1 || len(set.Windows) != 1 {
- t.Fatalf("candidates = %d window(s) (+%d), want the unjudged claim handed to a verdict",
- len(set.Windows), added)
- }
- // Reached, not claimed: the ledger already knew this name, so the claim source left it alone
- // rather than asking about it twice.
- if claims != 0 {
- t.Fatalf("claimed = %d, want the claim covered by the reached ledger", claims)
- }
- anchored := slots.Items(slots.Item{Value: "于禁", ChunkID: "c1", Quote: "于禁拜伏于地"})
- settled := runtime.NewState([]runtime.Variable{{ID: 1, Type: "person", Value: &anchored}}, 0, nil)
- if _, again, _ := coverageResolveCandidates(kb, &settled, runtime.Coverage{}); again != 0 {
- t.Fatalf("added = %d, want a member that already carries a passage left alone", again)
- }
-}
-
-// TestCoverageResolveCandidatesHandAClaimToTheVerdict pins the THIRD source: a name a session
-// asserted with no passage is handed over as a candidate, with whatever the pool can show about
-// it, so the node that decides what a member is gets to decide about it too.
-//
-// The alternatives both lose. Left in the table, the claim is counted without ever being read
-// (于禁 taken at 水淹七军 and released, never killed, counted beside sixteen real members). Dropped
-// outright, a name the corpus does state is lost. Handed over, it is a candidate like any other —
-// and the passage that carries the deed is offered first, because a name is mentioned in many
-// places and only one of them is the killing.
-func TestCoverageResolveCandidatesHandAClaimToTheVerdict(t *testing.T) {
- kb := &runtime.Kbinfos{}
- kb.Admit(func(p *runtime.PoolAdmitter) {
- p.Add(map[string]any{"chunk_id": "m1", "content_with_weight": "关公问曰:谁人敢去?于禁在阵前绰刀出马,关公便拨马回阵。"})
- p.Add(map[string]any{"chunk_id": "k1", "content_with_weight": "于禁拜伏于地,乞哀请命。关公大怒曰:汝竟敢抗吾?喝令推出斩之。"})
- })
- // 于禁 has passages in the pool; 刘延 does not (in this fixture): one claim can be judged,
- // the other has nothing to be judged against and must not be invented into a list.
- claimed := slots.Items(slots.Item{Value: "于禁"}, slots.Item{Value: "刘延"})
- table := runtime.NewState([]runtime.Variable{{ID: 1, Type: "person", Value: &claimed}}, 0, nil)
-
- set, reached, claims := coverageResolveCandidates(kb, &table, runtime.Coverage{Acts: []string{"斩"}, Actor: "关羽"})
- if reached != 0 || claims != 1 {
- t.Fatalf("reached=%d claimed=%d, want exactly one claim handed over", reached, claims)
- }
- if len(set.Windows) != 2 || set.Windows[0].ChunkID != "k1" {
- t.Fatalf("windows = %+v, want the passage carrying the deed offered first", set.Windows)
- }
- // The choice is marked, not silent: the window says which act word the passage carries, so a
- // reader of the log can see why it was offered first (see CoverageWindow.Act).
- if set.Windows[0].Act != "斩" {
- t.Fatalf("windows[0].Act = %q, want the act word the passage carries", set.Windows[0].Act)
- }
- if set.Windows[1].Act != "" {
- t.Fatalf("windows[1].Act = %q, want no act word claimed for a passage that carries none", set.Windows[1].Act)
- }
-}
-
-// TestRunCoverageResolveWritesWhatTheRunTouched pins the WIRING the unit tests above cannot: the
-// node's candidate set is the UNION, so a name no enumeration recalled but a probe DID reach is
-// still judged, and still written.
-//
-// This is the case that loses members: the corpus states the deed, the run's own search brought
-// the passage back, and the name lands in no list at all because the node only ever looked at its
-// own enumeration.
-func TestRunCoverageResolveWritesWhatTheRunTouched(t *testing.T) {
- st := NewAgenticState("关羽杀了多少有姓名的人物?", "", 3, nil)
- st.KB.Chunks = []map[string]any{
- {"chunk_id": "c1", "content": "夏侯存军至,见了云长,大怒,便与云长交锋,只一合,被云长砍死。"},
- }
- // The run's own probe reached this name; no enumeration ever recalled it (no set stored).
- st.KB.RecordReachedTerm("夏侯存", "c1")
- declared := slots.Items(slots.Item{Value: "孔秀", ChunkID: "c2", Quote: "云长斩孔秀于马下"})
- st.SlotTable = runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "count", Terms: []string{"斩"}, Subject: "关羽"},
- {ID: 1, Type: "person", Value: &declared},
- }, 0, nil)
-
- model := &scriptedModel{}
- model.push(`{"members": [{"i": 0, "name": "夏侯存"}], "not_members": []}`)
- stats := RunCoverageResolve(context.Background(), RAGTools{Model: model}, st, nil)
-
- if stats.Asked != 1 {
- t.Fatalf("asked = %d, want the ONE line the run held — the name its own search reached", stats.Asked)
- }
- if stats.Answered != 1 || stats.Unknown != 0 {
- t.Errorf("stats = %+v, want that line judged", stats)
- }
- names := runtime.ItemValues(&st.SlotTable)
- found := false
- for _, name := range names {
- if strings.EqualFold(strings.TrimSpace(name), "夏侯存") {
- found = true
- }
- }
- if !found {
- t.Errorf("members = %v, want the reached name written into the table", names)
- }
- if !strings.Contains(st.KB.Record, "夏侯存") {
- t.Errorf("record does not carry the member the resolve just proved:\n%s", st.KB.Record)
- }
-}
diff --git a/internal/rag/agentic-rag/doc_metadata.go b/internal/rag/agentic-rag/doc_metadata.go
deleted file mode 100644
index 31a5c6f21f..0000000000
--- a/internal/rag/agentic-rag/doc_metadata.go
+++ /dev/null
@@ -1,214 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package agentic_rag
-
-import (
- "context"
- "log"
- "sort"
- "strings"
-
- "ragflow/internal/common"
- "ragflow/internal/entity"
- "ragflow/internal/rag/agentic-rag/runtime"
-)
-
-// docMetaValueRunes caps one rendered metadata value. The evidence block is a
-// per-passage token budget (evidenceBudgetTokens, graph_compose.go), so a long
-// title or a stitched value must not spend it: the block carries the value as a
-// pointer, not as reading material.
-const docMetaValueRunes = 200
-
-// attachDocMetadata stamps the DECLARED document-level metadata onto the pool's
-// chunks, so that the final answer's evidence block carries each passage's
-// document attributes (file name, update time, title, ...) beside its text.
-//
-// WHY the evidence block is the only place this can work. The answer prompt is
-// built from two carriers (see answerPromptWithEvidence): the slot record, which
-// is declared INTERNAL and explicitly NOT evidence, and the numbered evidence
-// blocks, which are the only text the citation contract lets the answer rest on.
-// Document attributes previously reached the run only through the slot table
-// (metadata_search's result block is context for the DIRECTION sessions and
-// produces no passages: runtime/tool_executor.go, EvidenceIDs nil). That left a
-// two-hop question — "which documents, and what is in them" — with the
-// membership in the record and the content in the evidence, and the record's own
-// contract ("never quote these lines") weighing against the answer using it.
-// Stamping the attributes onto the passages puts both halves in the same block.
-//
-// WHY the KB row and not the metadata resolver's declared-field read. The
-// declared fields live in parser_config, and the run already holds the KB rows
-// (RAGTools.KBs, loaded by the caller so this package stays DB-free). Reading
-// them here is the same source the metadata_search catalog uses
-// (common.DeclaredMetadataFieldsFromParserConfig covers parser_config.metadata
-// AND built_in_metadata) with no extra index scan.
-//
-// WHY only declared fields. The doc-metadata index also carries extraction and
-// annotation noise the dataset never declared — a metadata-extraction run that
-// echoed a judge's rationale and verdict into the document's metadata is a real
-// observed case — and an unfiltered dump would put it in every evidence block.
-// A dataset that declares nothing therefore gets no block-level metadata at all,
-// which is exactly the behaviour before this function existed.
-//
-// Best effort by contract: a missing resolver, unresolvable KB rows or a failed
-// metadata read degrades to "no document metadata in the evidence" rather than
-// failing the answer. The compose call must not depend on it.
-func attachDocMetadata(ctx context.Context, deps RAGTools, kb *runtime.Kbinfos, logger *log.Logger) {
- if kb == nil || deps.MetadataResolver == nil || len(kb.Chunks) == 0 {
- return
- }
- keys := declaredDocMetadataKeys(deps.KBs)
- if len(keys) == 0 {
- return
- }
- datasetIDs := kbIDsOf(deps.KBs)
- if len(datasetIDs) == 0 {
- return
- }
- docIDs := pooledDocIDs(kb.Chunks)
- if len(docIDs) == 0 {
- return
- }
- metaByDoc, err := deps.MetadataResolver.MetadataForDocIDs(ctx, datasetIDs, docIDs)
- if err != nil {
- logger.Printf("[Formalize][docmeta] metadata read degraded: %v", err)
- }
- if len(metaByDoc) == 0 {
- return
- }
- stamped := 0
- for _, chunk := range kb.Chunks {
- meta := metaByDoc[chunkDocID(chunk)]
- if len(meta) == 0 {
- continue
- }
- picked := make(map[string]any, len(keys))
- for _, key := range keys {
- if text, ok := docMetaText(meta[key]); ok {
- picked[key] = text
- }
- }
- if len(picked) == 0 {
- continue
- }
- chunk["document_metadata"] = picked
- stamped++
- }
- logger.Printf("[Formalize][docmeta] declared=%v docs=%d chunks=%d/%d",
- keys, len(metaByDoc), stamped, len(kb.Chunks))
-}
-
-// declaredDocMetadataKeys returns the union of the datasets' declared metadata
-// fields, deduped and sorted so the request and the rendered block are stable.
-// Declaration order is per dataset and a run may span several, so the union is
-// ordered here rather than left to map iteration.
-func declaredDocMetadataKeys(kbs []*entity.Knowledgebase) []string {
- seen := make(map[string]bool, 8)
- keys := make([]string, 0, 8)
- for _, kb := range kbs {
- if kb == nil {
- continue
- }
- for _, def := range common.DeclaredMetadataFieldsFromParserConfig(kb.ParserConfig) {
- key := strings.TrimSpace(def.Key)
- if key == "" || seen[key] {
- continue
- }
- seen[key] = true
- keys = append(keys, key)
- }
- }
- sort.Strings(keys)
- return keys
-}
-
-// kbIDsOf returns the datasets of the loaded KB rows, in the caller's order.
-func kbIDsOf(kbs []*entity.Knowledgebase) []string {
- ids := make([]string, 0, len(kbs))
- for _, kb := range kbs {
- if kb == nil || strings.TrimSpace(kb.ID) == "" {
- continue
- }
- ids = append(ids, kb.ID)
- }
- return ids
-}
-
-// pooledDocIDs returns the distinct document ids the pool holds, sorted so the
-// metadata read is deterministic (and so a whole-corpus lookup is one request).
-func pooledDocIDs(chunks []map[string]any) []string {
- seen := make(map[string]bool, len(chunks))
- ids := make([]string, 0, len(chunks))
- for _, chunk := range chunks {
- id := chunkDocID(chunk)
- if id == "" || seen[id] {
- continue
- }
- seen[id] = true
- ids = append(ids, id)
- }
- sort.Strings(ids)
- return ids
-}
-
-// chunkDocID reads a chunk's DOCUMENT id.
-//
-// It deliberately does not fall back to "id": in a retrieval chunk that key is
-// the CHUNK id (see service.SourcedChunk, which reads ID from chunk_id/id and
-// DocID from doc_id/document_id), so keying the metadata lookup on it would ask
-// the metadata index for a document that does not exist and stamp nothing.
-func chunkDocID(chunk map[string]any) string {
- for _, key := range []string{"doc_id", "document_id"} {
- if id, ok := chunk[key].(string); ok && strings.TrimSpace(id) != "" {
- return strings.TrimSpace(id)
- }
- }
- return ""
-}
-
-// docMetaText renders one metadata value for an evidence block, or ok=false when
-// the value has no flat reading there.
-//
-// Text (and a list of text, which the index writes for multi-valued fields) is
-// what a "key: value" line can carry. A structured value — a nested outline, a
-// list of objects — is dropped rather than stringified into noise: the block is
-// a pointer to the document, not a rendering of its metadata.
-func docMetaText(value any) (string, bool) {
- var text string
- switch typed := value.(type) {
- case string:
- text = typed
- case []string:
- text = strings.Join(typed, ", ")
- case []any:
- parts := make([]string, 0, len(typed))
- for _, item := range typed {
- s, ok := item.(string)
- if !ok {
- return "", false
- }
- parts = append(parts, s)
- }
- text = strings.Join(parts, ", ")
- default:
- return "", false
- }
- text = strings.TrimSpace(text)
- if text == "" {
- return "", false
- }
- return truncateRunes(text, docMetaValueRunes), true
-}
diff --git a/internal/rag/agentic-rag/doc_metadata_test.go b/internal/rag/agentic-rag/doc_metadata_test.go
deleted file mode 100644
index 3aac9bd647..0000000000
--- a/internal/rag/agentic-rag/doc_metadata_test.go
+++ /dev/null
@@ -1,211 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package agentic_rag
-
-import (
- "context"
- "strings"
- "testing"
-
- "ragflow/internal/common"
- "ragflow/internal/entity"
- "ragflow/internal/rag/agentic-rag/runtime"
-)
-
-// docMetadataResolver is a scripted runtime.MetadataResolver: it records the
-// documents it was asked for and answers from a fixed doc_id → fields map.
-type docMetadataResolver struct {
- meta map[string]map[string]any
- requested []string
- askedKBs []string
- calls int
-}
-
-func (r *docMetadataResolver) FilterDocIDsByMetaPushdown(context.Context, []string, []map[string]any, string) ([]string, bool) {
- return nil, false
-}
-
-func (r *docMetadataResolver) GetFlattedMetaByKBs(context.Context, []string) (common.MetaData, error) {
- return common.MetaData{}, nil
-}
-
-func (r *docMetadataResolver) MetadataForDocIDs(_ context.Context, kbIDs, docIDs []string) (map[string]map[string]any, error) {
- r.calls++
- r.askedKBs = append([]string(nil), kbIDs...)
- r.requested = append([]string(nil), docIDs...)
- return r.meta, nil
-}
-
-// declaringKB is a dataset row whose parser_config declares one extracted field
-// and two built-in ones — the shape the metadata config API writes.
-func declaringKB() []*entity.Knowledgebase {
- return []*entity.Knowledgebase{{
- ID: "kb1",
- ParserConfig: entity.JSONMap{
- "metadata": map[string]any{
- "enabled": true,
- "metadata": []any{
- map[string]any{"key": "title", "type": "string"},
- },
- "built_in_metadata": []any{
- map[string]any{"key": "update_time", "type": "time"},
- map[string]any{"key": "file_name", "type": "string"},
- },
- },
- },
- }}
-}
-
-// composePrompt runs the terminal composition and returns the user prompt the
-// model received — the text the evidence blocks are rendered into.
-func composePrompt(t *testing.T, deps RAGTools, kb *runtime.Kbinfos) string {
- t.Helper()
- model := &fakeModel{replies: []*runtime.ModelReply{{Content: "answer"}}}
- deps.Model = model
- composeFinalAnswer(context.Background(), deps, runtime.RunRequest{Question: "q"}, kb,
- &RunResponse{}, _LOG, false, false, "q")
- return model.lastUserPrompt()
-}
-
-// TestAttachDocMetadataStampsDeclaredFieldsOnly pins both halves of the write:
-// the DECLARED fields reach the evidence block, and a field the dataset never
-// declared does not. The undeclared half is the observed metadata-extraction
-// leak (a judge's rationale landing in a document's metadata) — an unfiltered
-// dump would put it in every evidence block.
-func TestAttachDocMetadataStampsDeclaredFieldsOnly(t *testing.T) {
- resolver := &docMetadataResolver{meta: map[string]map[string]any{
- "doc1": {
- "update_time": "2026-09-20 13:55:35",
- "file_name": "VecTree-RAG.pdf",
- "title": "VecTree-RAG",
- "rationale": "an undeclared field an extraction run leaked",
- },
- }}
- kb := &runtime.Kbinfos{Chunks: []map[string]any{
- {"chunk_id": "c1", "doc_id": "doc1", "content": "the passage body", "similarity": 0.9},
- }}
- deps := RAGTools{KBs: declaringKB(), MetadataResolver: resolver}
-
- prompt := composePrompt(t, deps, kb)
-
- for _, want := range []string{
- "\u251c\u2500\u2500 update_time: 2026-09-20 13:55:35",
- "\u251c\u2500\u2500 file_name: VecTree-RAG.pdf",
- "\u251c\u2500\u2500 title: VecTree-RAG",
- } {
- if !strings.Contains(prompt, want) {
- t.Errorf("evidence block missing %q:\n%s", want, prompt)
- }
- }
- if strings.Contains(prompt, "rationale") {
- t.Errorf("an undeclared field reached the evidence block:\n%s", prompt)
- }
- if resolver.calls != 1 || len(resolver.requested) != 1 || resolver.requested[0] != "doc1" {
- t.Errorf("metadata read = %d call(s) for %v, want one read for [doc1]", resolver.calls, resolver.requested)
- }
- if len(resolver.askedKBs) != 1 || resolver.askedKBs[0] != "kb1" {
- t.Errorf("metadata read datasets = %v, want the declaring dataset", resolver.askedKBs)
- }
-}
-
-// TestAttachDocMetadataLeavesEvidenceUnchangedWithoutDeclarations pins the two
-// gates that keep this a no-op where it cannot help: no resolver, and no
-// declared fields. The second one must also skip the read — a dataset that
-// declares nothing has no field to put on a block, so the round-trip would be
-// paid for an empty map.
-func TestAttachDocMetadataLeavesEvidenceUnchangedWithoutDeclarations(t *testing.T) {
- newKB := func() *runtime.Kbinfos {
- return &runtime.Kbinfos{Chunks: []map[string]any{
- {"chunk_id": "c1", "doc_id": "doc1", "content": "the passage body", "similarity": 0.9},
- }}
- }
- resolver := &docMetadataResolver{meta: map[string]map[string]any{
- "doc1": {"update_time": "2026-09-20 13:55:35"},
- }}
-
- noResolver := composePrompt(t, RAGTools{KBs: declaringKB()}, newKB())
- noDeclarations := composePrompt(t, RAGTools{MetadataResolver: resolver}, newKB())
-
- for name, prompt := range map[string]string{"no resolver": noResolver, "no declarations": noDeclarations} {
- if strings.Contains(prompt, "document_metadata") || strings.Contains(prompt, "update_time") {
- t.Errorf("%s: evidence block grew a metadata line:\n%s", name, prompt)
- }
- }
- if resolver.calls != 0 {
- t.Errorf("metadata read = %d call(s), want none without a declared field", resolver.calls)
- }
-}
-
-// TestAttachDocMetadataTruncatesLongValues pins the budget guard: a long value
-// is rendered as a pointer to the document, not as reading material, because
-// every metadata line spends the evidence block's token budget.
-func TestAttachDocMetadataTruncatesLongValues(t *testing.T) {
- long := strings.Repeat("x", docMetaValueRunes+50)
- resolver := &docMetadataResolver{meta: map[string]map[string]any{
- "doc1": {"file_name": long},
- }}
- kb := &runtime.Kbinfos{Chunks: []map[string]any{
- {"chunk_id": "c1", "doc_id": "doc1", "content": "the passage body", "similarity": 0.9},
- }}
-
- prompt := composePrompt(t, RAGTools{KBs: declaringKB(), MetadataResolver: resolver}, kb)
-
- if !strings.Contains(prompt, strings.Repeat("x", docMetaValueRunes)) {
- t.Errorf("evidence block lost the truncated value:\n%s", prompt)
- }
- if strings.Contains(prompt, strings.Repeat("x", docMetaValueRunes+1)) {
- t.Errorf("a value longer than %d runes reached the evidence block", docMetaValueRunes)
- }
-}
-
-// TestAttachDocMetadataKeysOnDocID pins the id semantics: a retrieval chunk's
-// "id" is its CHUNK id, so keying the metadata lookup on it would ask the index
-// for a document that does not exist. The lookup must use doc_id, and a resolver
-// that only knows the chunk id must stamp nothing.
-func TestAttachDocMetadataKeysOnDocID(t *testing.T) {
- chunk := func() map[string]any {
- // "id" and "doc_id" disagree on purpose: that is the shape a retrieval
- // result has (service.SourcedChunk reads ID from chunk_id/id).
- return map[string]any{
- "id": "chunk-1", "chunk_id": "chunk-1", "doc_id": "doc-1",
- "content": "the passage body", "similarity": 0.9,
- }
- }
-
- byDocID := &docMetadataResolver{meta: map[string]map[string]any{
- "doc-1": {"update_time": "2026-09-20 13:55:35"},
- }}
- prompt := composePrompt(t, RAGTools{KBs: declaringKB(), MetadataResolver: byDocID},
- &runtime.Kbinfos{Chunks: []map[string]any{chunk()}})
-
- if len(byDocID.requested) != 1 || byDocID.requested[0] != "doc-1" {
- t.Fatalf("metadata read requested %v, want [doc-1] (the DOCUMENT id)", byDocID.requested)
- }
- if !strings.Contains(prompt, "update_time: 2026-09-20 13:55:35") {
- t.Errorf("evidence block missing the document's metadata:\n%s", prompt)
- }
-
- byChunkID := &docMetadataResolver{meta: map[string]map[string]any{
- "chunk-1": {"update_time": "2026-09-20 13:55:35"},
- }}
- other := composePrompt(t, RAGTools{KBs: declaringKB(), MetadataResolver: byChunkID},
- &runtime.Kbinfos{Chunks: []map[string]any{chunk()}})
-
- if strings.Contains(other, "update_time") {
- t.Errorf("a chunk-id-keyed answer stamped metadata onto the block:\n%s", other)
- }
-}
diff --git a/internal/rag/agentic-rag/evidence_fill_test.go b/internal/rag/agentic-rag/evidence_fill_test.go
deleted file mode 100644
index f2be201832..0000000000
--- a/internal/rag/agentic-rag/evidence_fill_test.go
+++ /dev/null
@@ -1,388 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package agentic_rag
-
-import (
- "context"
- "errors"
- "fmt"
- "strings"
- "sync"
- "testing"
-
- "github.com/cloudwego/eino/schema"
- "ragflow/internal/rag/agentic-rag/runtime"
-)
-
-// Test doubles
-
-// failingModel errors on every Complete call — the batched-answer LLM seam.
-type failingModel struct {
- mu sync.Mutex
- calls int
-}
-
-func (m *failingModel) Complete(context.Context, []schema.Message, []runtime.ToolSpec) (*runtime.ModelReply, error) {
- m.mu.Lock()
- defer m.mu.Unlock()
- m.calls++
- return nil, errors.New("model unavailable")
-}
-
-// Calls reports how many times Complete was invoked.
-func (m *failingModel) Calls() int {
- m.mu.Lock()
- defer m.mu.Unlock()
- return m.calls
-}
-
-// claimRow builds one claim pseudo-chunk ("[evidence] <name> — <desc>") the way the pool
-// stores it.
-func claimRow(id, content string) map[string]any {
- return map[string]any{
- "chunk_id": id,
- "content_with_weight": content,
- "doc_id": "d1",
- }
-}
-
-// PrefillSlotsFromEvidence
-
-// TestPrefillSlotsFromEvidence_CoverageBoundary pins the 0.6 word-coverage gate: exactly-at
-// and above prefill, below does not; the candidate is the de-prefixed first line and the
-// strength is the coverage itself.
-func TestPrefillSlotsFromEvidence_CoverageBoundary(t *testing.T) {
- // Clue terms: {alpha, tower, opened, paris, 1889} (>=3 code points only).
- clue := "alpha tower opened paris 1889"
- // Covers tower/opened/paris = 3/5 = exactly 0.6.
- atGate := claimRow("claim_a", "[evidence] Eiffel Tower — landmark\nEvidence (verbatim): \"the tower opened in paris near the river\"")
- // Covers tower/opened = 2/5 = 0.4.
- below := claimRow("claim_b", "[evidence] Eiffel Tower — landmark\nEvidence (verbatim): \"the tower opened here long ago\"")
-
- newState := func() runtime.State {
- return runtime.NewState([]runtime.Variable{
- {ID: 1, Type: "aspect", QuestionClues: []string{clue}},
- }, 0, nil)
- }
-
- // At the gate: prefilled.
- st := newState()
- if n := PrefillSlotsFromEvidence(&st, &runtime.Kbinfos{Chunks: []map[string]any{atGate}}); n != 1 {
- t.Fatalf("coverage exactly 0.6 must prefill, filled = %d", n)
- }
- v := st.State[0]
- if v.Candidate == nil || *v.Candidate != "Eiffel Tower — landmark" {
- t.Fatalf("candidate = %v, want the de-prefixed first line", v.Candidate)
- }
- if v.CandidateStrength == nil || *v.CandidateStrength != 0.6 {
- t.Fatalf("strength = %v, want 0.6", v.CandidateStrength)
- }
-
- // Below the gate: untouched.
- st = newState()
- if n := PrefillSlotsFromEvidence(&st, &runtime.Kbinfos{Chunks: []map[string]any{below}}); n != 0 {
- t.Fatalf("coverage 0.4 must NOT prefill, filled = %d", n)
- }
- if st.State[0].Candidate != nil {
- t.Fatalf("candidate = %v, want nil", st.State[0].Candidate)
- }
-}
-
-// TestPrefillSlotsFromEvidence_Guards mirrors the skips(,
-// 1744): no claim rows in the pool, already-filled slots, blank clues, and a
-// first line that strips to nothing.
-func TestPrefillSlotsFromEvidence_Guards(t *testing.T) {
- st := runtime.NewState([]runtime.Variable{
- {ID: 1, Type: "aspect", QuestionClues: []string{"eiffel tower paris"}},
- {ID: 2, Type: "aspect", Candidate: strPtr("known")},
- {ID: 3, Type: "aspect", QuestionClues: []string{" "}},
- }, 0, nil)
-
- // No claim rows: 0 regardless of content.
- if n := PrefillSlotsFromEvidence(&st, &runtime.Kbinfos{Chunks: []map[string]any{
- claimRow("c1", "plain chunk"),
- }}); n != 0 {
- t.Fatalf("no claim_ rows must yield 0, got %d", n)
- }
- // nil KB is safe.
- if n := PrefillSlotsFromEvidence(&st, nil); n != 0 {
- t.Fatalf("nil KB must yield 0, got %d", n)
- }
-
- // Row whose head strips to nothing: even full coverage fills nothing.
- if n := PrefillSlotsFromEvidence(&st, &runtime.Kbinfos{Chunks: []map[string]any{
- claimRow("claim_x", "[evidence] — \nEvidence (verbatim): \"eiffel tower paris\""),
- }}); n != 0 {
- t.Fatalf("empty name must yield 0, got %d", n)
- }
- if st.State[0].Candidate != nil || st.State[2].Candidate != nil {
- t.Fatal("unfilled slots must stay untouched")
- }
- if st.State[1].Candidate == nil || *st.State[1].Candidate != "known" {
- t.Fatal("a filled slot must never be overwritten")
- }
-}
-
-// TestRunSlotResearchPass_PrefilledSlotsSkipSession pins the wiring: when the pooled evidence
-// covers every slot at >= 0.6, the pass fills them WITHOUT running a single action session
-// (zero model calls).
-func TestRunSlotResearchPass_PrefilledSlotsSkipSession(t *testing.T) {
- kb := &runtime.Kbinfos{Chunks: []map[string]any{
- claimRow("claim_a", "[evidence] Eiffel Tower — built in Paris by Gustave Eiffel, opened in 1889\nEvidence (verbatim): \"...\""),
- }}
- st := &AgenticState{
- Question: "what is the tower",
- SlotTable: runtime.NewState([]runtime.Variable{
- {ID: 0, Type: "aspect", QuestionClues: []string{"eiffel tower opened paris 1889"}},
- {ID: 1, Type: "aspect", QuestionClues: []string{"gustave eiffel designed paris tower"}},
- }, 0, nil),
- }
- mdl := &scriptedModel{}
- deps := runtime.SessionDeps{Model: mdl, KB: kb}
-
- res := RunSlotResearchPass(context.Background(), context.Background(), deps, st.Question, st, 120.0)
- if res == nil {
- t.Fatal("expected a result")
- }
- // The prefill removed BOTH sessions: the model was never called.
- if got := len(mdl.seen); got != 0 {
- t.Fatalf("model calls = %d, want 0 (prefill must skip every session)", got)
- }
- for i, want := range []string{
- "Eiffel Tower — built in Paris by Gustave Eiffel, opened in 1889",
- "Eiffel Tower — built in Paris by Gustave Eiffel, opened in 1889",
- } {
- if st.SlotTable.State[i].Candidate == nil || *st.SlotTable.State[i].Candidate != want {
- t.Fatalf("slot %d candidate = %v, want %q", i, st.SlotTable.State[i].Candidate, want)
- }
- }
- if len(res.UnresolvedSlots) != 0 {
- t.Fatalf("unresolved = %d, want 0", len(res.UnresolvedSlots))
- }
-}
-
-// BatchFillSlots
-
-// TestBatchFillSlots_ClustersOverlappingEvidence: two slots sharing >= 2
-// evidence ids with Jaccard >= 0.15 are answered in ONE call; a slot with
-// disjoint evidence stays in its own (size-1) cluster and is never asked.
-func TestBatchFillSlots_ClustersOverlappingEvidence(t *testing.T) {
- chunks := []map[string]any{}
- for _, id := range []string{"c1", "c2", "c3", "c4", "c9", "c10"} {
- chunks = append(chunks, map[string]any{
- "chunk_id": id,
- "content_with_weight": "content of " + id,
- "doc_id": "d1",
- })
- }
- slotTable := runtime.NewState([]runtime.Variable{
- {ID: 1, Type: "aspect", QuestionClues: []string{"who built it"}},
- {ID: 2, Type: "aspect", QuestionClues: []string{"when opened"}},
- {ID: 3, Type: "aspect", QuestionClues: []string{"unrelated"}},
- }, 0, nil)
- slotEvidence := map[string]SlotEvidence{
- "1": {EvidenceIDs: []string{"c1", "c2", "c3"}}, // vs 2: inter=2, union=4, sim=0.5
- "2": {EvidenceIDs: []string{"c1", "c2", "c4"}},
- "3": {EvidenceIDs: []string{"c9", "c10"}},
- }
- mdl := &scriptedModel{}
- mdl.push(`{"1": "Gustave Eiffel", "2": "1889", "3": "invented"}`)
- deps := runtime.SessionDeps{Model: mdl, KB: &runtime.Kbinfos{Chunks: chunks}}
-
- if got := BatchFillSlots(context.Background(), deps, &slotTable, slotEvidence); got != 2 {
- t.Fatalf("filled = %d, want 2", got)
- }
- // Exactly ONE generation call served slots 1 and 2.
- if len(mdl.seen) != 1 {
- t.Fatalf("model calls = %d, want 1", len(mdl.seen))
- }
- if got := *slotTable.State[0].Candidate; got != "Gustave Eiffel" {
- t.Fatalf("slot 1 candidate = %q", got)
- }
- if got := *slotTable.State[1].Candidate; got != "1889" {
- t.Fatalf("slot 2 candidate = %q", got)
- }
- // The disjoint slot was never asked, never filled.
- if slotTable.State[2].Candidate != nil {
- t.Fatalf("slot 3 candidate = %v, want nil", slotTable.State[2].Candidate)
- }
- // The prompt carries both sub-questions and the shared evidence body.
- user := mdl.seen[0][1].Content
- for _, want := range []string{"- 1: who built it", "- 2: when opened", "content of c1", "content of c4"} {
- if !strings.Contains(user, want) {
- t.Fatalf("prompt missing %q:\n%s", want, user)
- }
- }
-}
-
-// TestBatchFillSlots_ThresholdsBlockMerging pins both gates: fewer than MIN_SHARED shared ids,
-// or a Jaccard below MIN_SIM, keeps the slots in separate size-1 clusters — no LLM call at
-// all.
-func TestBatchFillSlots_ThresholdsBlockMerging(t *testing.T) {
- mk := func(e1, e2 []string) (runtime.State, map[string]SlotEvidence) {
- st := runtime.NewState([]runtime.Variable{
- {ID: 1, Type: "aspect", QuestionClues: []string{"a"}},
- {ID: 2, Type: "aspect", QuestionClues: []string{"b"}},
- }, 0, nil)
- ev := map[string]SlotEvidence{
- "1": {EvidenceIDs: e1},
- "2": {EvidenceIDs: e2},
- }
- return st, ev
- }
- ids := func(n int) []string {
- out := make([]string, 0, n)
- for i := 1; i <= n; i++ {
- out = append(out, fmt.Sprintf("c%02d", i))
- }
- return out
- }
-
- cases := []struct {
- name string
- e1 []string
- e2 []string
- }{
- // inter=1 < MIN_SHARED=2.
- {"shared-below-floor", []string{"c01", "c02"}, []string{"c02", "c03"}},
- // inter=2, union=14 → sim≈0.143 < MIN_SIM=0.15.
- {"sim-below-gate", ids(14), []string{"c13", "c14"}},
- }
- for _, tc := range cases {
- t.Run(tc.name, func(t *testing.T) {
- st, ev := mk(tc.e1, tc.e2)
- mdl := &scriptedModel{}
- deps := runtime.SessionDeps{Model: mdl, KB: &runtime.Kbinfos{}}
- if got := BatchFillSlots(context.Background(), deps, &st, ev); got != 0 {
- t.Fatalf("filled = %d, want 0", got)
- }
- if len(mdl.seen) != 0 {
- t.Fatalf("model calls = %d, want 0 (below the merge gates)", len(mdl.seen))
- }
- })
- }
-}
-
-// TestBatchFillSlots_MaxSlotsPerBatch pins _EVIDENCE_BATCH_MAX_SLOTS=4
-// (,:1625): the fifth identical-evidence slot cannot join the
-// full batch and is left unfilled this round.
-func TestBatchFillSlots_MaxSlotsPerBatch(t *testing.T) {
- vars := make([]runtime.Variable, 0, 5)
- ev := map[string]SlotEvidence{}
- for i := 1; i <= 5; i++ {
- vars = append(vars, runtime.Variable{ID: i, Type: "aspect", QuestionClues: []string{"same clue"}})
- ev[fmt.Sprint(i)] = SlotEvidence{EvidenceIDs: []string{"c1", "c2"}}
- }
- st := runtime.NewState(vars, 0, nil)
- kb := &runtime.Kbinfos{Chunks: []map[string]any{
- {"chunk_id": "c1", "content_with_weight": "shared text one", "doc_id": "d1"},
- {"chunk_id": "c2", "content_with_weight": "shared text two", "doc_id": "d1"},
- }}
- mdl := &scriptedModel{}
- mdl.push(`{"1": "a", "2": "b", "3": "c", "4": "d", "5": "e"}`)
- deps := runtime.SessionDeps{Model: mdl, KB: kb}
-
- if got := BatchFillSlots(context.Background(), deps, &st, ev); got != 4 {
- t.Fatalf("filled = %d, want 4 (MAX_SLOTS)", got)
- }
- if len(mdl.seen) != 1 {
- t.Fatalf("model calls = %d, want 1", len(mdl.seen))
- }
- for i := 0; i < 4; i++ {
- if st.State[i].Candidate == nil {
- t.Fatalf("slot %d must be filled", i+1)
- }
- }
- if st.State[4].Candidate != nil {
- t.Fatalf("slot 5 candidate = %v, want nil (batch cap 4)", st.State[4].Candidate)
- }
-}
-
-// TestBatchFillSlots_LLMFailureBestEffort mirrors: a failed
-// model call (or an unparsable reply, or no model at all) fills nothing and
-// never panics — the pass continues without the batch.
-func TestBatchFillSlots_LLMFailureBestEffort(t *testing.T) {
- mk := func() (runtime.State, map[string]SlotEvidence) {
- st := runtime.NewState([]runtime.Variable{
- {ID: 1, Type: "aspect", QuestionClues: []string{"a"}},
- {ID: 2, Type: "aspect", QuestionClues: []string{"b"}},
- }, 0, nil)
- ev := map[string]SlotEvidence{
- "1": {EvidenceIDs: []string{"c1", "c2"}},
- "2": {EvidenceIDs: []string{"c1", "c2"}},
- }
- return st, ev
- }
- kb := &runtime.Kbinfos{Chunks: []map[string]any{
- {"chunk_id": "c1", "content_with_weight": "text one", "doc_id": "d1"},
- {"chunk_id": "c2", "content_with_weight": "text two", "doc_id": "d1"},
- }}
-
- t.Run("model-error", func(t *testing.T) {
- st, ev := mk()
- mdl := &failingModel{}
- deps := runtime.SessionDeps{Model: mdl, KB: kb}
- if got := BatchFillSlots(context.Background(), deps, &st, ev); got != 0 {
- t.Fatalf("filled = %d, want 0", got)
- }
- if mdl.Calls() != 1 {
- t.Fatalf("model calls = %d, want 1 (the cluster is attempted)", mdl.Calls())
- }
- for _, v := range st.State {
- if v.Candidate != nil {
- t.Fatal("no slot may be filled after a failed call")
- }
- }
- })
- t.Run("unparsable-reply", func(t *testing.T) {
- st, ev := mk()
- mdl := &scriptedModel{}
- mdl.push("sorry, I cannot answer that") // no JSON object
- deps := runtime.SessionDeps{Model: mdl, KB: kb}
- if got := BatchFillSlots(context.Background(), deps, &st, ev); got != 0 {
- t.Fatalf("filled = %d, want 0", got)
- }
- })
- t.Run("no-model", func(t *testing.T) {
- st, ev := mk()
- deps := runtime.SessionDeps{KB: kb} // Model nil — nothing can be filled
- if got := BatchFillSlots(context.Background(), deps, &st, ev); got != 0 {
- t.Fatalf("filled = %d, want 0", got)
- }
- })
-}
-
-// TestBatchFillSlots_SkipDiagnostics pins the never-firing diagnostics
-// (): fewer than two evidence-carrying unresolved slots
-// logs the skip line and short-circuits before any model call.
-func TestBatchFillSlots_SkipDiagnostics(t *testing.T) {
- st := runtime.NewState([]runtime.Variable{
- {ID: 1, Type: "aspect", QuestionClues: []string{"a"}},
- {ID: 2, Type: "aspect", QuestionClues: []string{"b"}},
- }, 0, nil)
- ev := map[string]SlotEvidence{
- "1": {EvidenceIDs: []string{"c1"}}, // only ONE slot carries evidence
- }
- mdl := &scriptedModel{}
- deps := runtime.SessionDeps{Model: mdl, KB: &runtime.Kbinfos{}}
- if got := BatchFillSlots(context.Background(), deps, &st, ev); got != 0 {
- t.Fatalf("filled = %d, want 0", got)
- }
- if len(mdl.seen) != 0 {
- t.Fatalf("model calls = %d, want 0", len(mdl.seen))
- }
-}
diff --git a/internal/rag/agentic-rag/graph_compose.go b/internal/rag/agentic-rag/graph_compose.go
deleted file mode 100644
index d1c33c4ffc..0000000000
--- a/internal/rag/agentic-rag/graph_compose.go
+++ /dev/null
@@ -1,751 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package agentic_rag
-
-import (
- "context"
- "errors"
- "fmt"
- "log"
- "strings"
- "time"
- "unicode/utf8"
-
- "github.com/cloudwego/eino/schema"
- "ragflow/internal/agent/chat"
- "ragflow/internal/rag/agentic-rag/runtime"
- "ragflow/internal/rag/prompts"
- "ragflow/internal/tokenizer"
-)
-
-// The answer stage: the terminal node's prompt, its budgeted evidence, and the
-// composition paths (streaming, one-shot, naive, and the failure fallbacks).
-// ComposeFallbackDraft: an
-// intermediate draft synthesized from the snippet pool when a research pass
-// produced no report (budget exhaustion).
-//
-// This is an LLM call, not a concatenation. The draft is what the
-// sufficient-context agent reviews and what later lands in PreSummary as
-// "Research findings (authoritative — use these facts verbatim)", so the model is asked
-// for a specific shape:
-//
-// - the FOUND facts, stated plainly;
-// - a final "MISSING:" line naming what is still unknown.
-//
-// The MISSING line is load-bearing: it is what tells the next round (and the
-// final answer) which unknowns remain. A plain list of truncated snippets
-// cannot express it.
-func ComposeFallbackDraft(ctx context.Context, deps RAGTools, st *AgenticState) string {
- // Nil-safe: the draft node runs on every round, including a state whose KB was never
- // populated.
- if st == nil || st.KB == nil || len(st.KB.Chunks) == 0 {
- return ""
- }
-
- // strongest evidence first — the pool is in insertion order, so
- // sort by similarity/score before spending the 16-slot budget on it.
- chunks := rankByScore(st.KB.Chunks)
-
- // "[i] text" joined by "\n", 1-indexed, 16 chunks x 1200
- // chars. Empty chunk text still produces its "[i] " line — nothing is skipped or
- // trimmed.
- var ev strings.Builder
- n := min(len(chunks), draftChunkCap)
- for i := 0; i < n; i++ {
- if ctx.Err() != nil {
- break
- }
- if i > 0 {
- ev.WriteString("\n")
- }
- fmt.Fprintf(&ev, "[%d] %s", i+1, truncateRunes(runtime.ChunkTextOf(chunks[i]), draftChunkChars))
- }
- evidence := ev.String()
- if evidence == "" {
- return ""
- }
-
- // no model -> the raw evidence, capped at 4000.
- mdl := deps.Model // the innermost chat model.
- if mdl == nil {
- return truncateRunes(evidence, draftFallbackChars)
- }
-
- // The design injects the latest research_feedback as a "focus", but that state field is
- // vestigial: it is declared and initialised to [] and never appended anywhere, so the
- // focus is always empty. There is therefore no feedback field here — populating one
- // would invent behaviour that does not exist.
- //
- // The prompt asks for the FOUND list plus a final
- // "MISSING:" line is what gives the SCA precise gaps.
- system := "You are a research assistant writing an INTERMEDIATE DRAFT toward answering the user's " +
- "question, using ONLY the retrieved evidence snippets below.\n" +
- "Requirements:\n" +
- "1. First list concrete FACTS FOUND in the snippets (exact numbers, dates, names preserved).\n" +
- "2. Then output a line starting with 'MISSING:' naming precisely which part(s) of the " +
- "question the snippets do NOT answer yet.\n" +
- "3. No conclusions beyond the evidence; no general knowledge.\n" +
- "Keep it under 250 words."
- if containsNonASCII(st.Question) {
- // _compose_fallback_draft — appended directly, with no separator.
- system += "Write your draft in the same language as the question."
- }
-
- user := "Question: " + st.Question + "\n\nRetrieved evidence:\n" + evidence
-
- reply, err := mdl.Complete(ctx, []schema.Message{
- *schema.SystemMessage(system),
- *schema.UserMessage(user),
- }, nil)
- if err != nil {
- // a failed composition degrades to the raw evidence
- // (capped at 4000, unlike the composed draft's 6000).
- _LOG.Printf("[Draft] fallback composition failed; using snippet text: %v", err)
- return truncateRunes(evidence, draftFallbackChars)
- }
- answer := ""
- if reply != nil {
- answer = strings.TrimSpace(reply.Content)
- }
- if answer == "" {
- answer = evidence
- }
- // (ans or evidence)[:6000].
- return truncateRunes(answer, draftMaxChars)
-}
-
-// NaiveRAG: answer with one retrieve pass
-// and no agentic graph at all.
-//
-// Used when the thinking mode is unrecognised — instead of failing the request
-// (the label comes from user input) this degrades to plain retrieval plus one
-// composed answer.
-//
-// The composition itself lives in ComposeNaiveAnswer: a fixed short prompt, flat
-// "[i] content" evidence over
-// the first 8 chunks at 1500 chars, message_fit_in, temperature 0.3, and the
-// raw evidence as the failure fallback).
-func NaiveRAG(ctx context.Context, deps RAGTools, req runtime.RunRequest, kb *runtime.Kbinfos, resp *RunResponse, logger *log.Logger) {
- if logger == nil {
- logger = _LOG
- }
- question := strings.TrimSpace(req.Question)
-
- logger.Printf("[Naive RAG] single-pass retrieval for question_len=%d", len(question))
-
- // tools.retrieve(question) — one PLAIN pass. Unlike runDirect
- // (low mode) this extracts no weighted keywords and expands no compiled
- // structure: naive is deliberately plain retrieval, and a failure degrades
- // to "no evidence" rather than erroring.
- var chunks []map[string]any
- if question != "" {
- // The retrieve call reads the full run config — including the tag boost. Projecting
- // only a subset here would silently drop that tuning for the naive path.
- var aggs []map[string]any
- chunks, aggs = runtime.RetrieveSearch(ctx,
- searchDepsFor(ctx, deps, req, req.DatasetIDs, req.TenantID, kb, logger),
- runtime.SearchParams{
- Question: question,
- TopN: req.TopN,
- })
- // accumulate onto the shared pool so the composed
- // answer can be cited and callers see the same shape as the agentic path.
- kb.Merge(chunks, aggs)
- }
-
- // no evidence -> yield the configured empty response.
- if len(chunks) == 0 {
- resp.Answer = deps.EmptyResponse
- return
- }
-
- // compose from the flat "[i] content" evidence under the
- // fixed short naiveAnswerSystem — NOT FinalAnswerSystem / kb_prompt. This is
- // why the naive path must not fall through to composeFinalAnswer.
- out := ComposeNaiveAnswer(ctx, AnswerDeps{
- Model: deps.Model,
- EmptyResponse: deps.EmptyResponse,
- MaxLength: deps.MaxLength,
- Logger: logger,
- }, chunks, question)
-
- resp.Answer = out.Answer
-}
-
-// runDirectFallback retrieves once when no model is configured: the loop cannot
-// plan, research, or review without one, so the caller still gets evidence
-// rather than an error.
-func runDirectFallback(ctx context.Context, deps RAGTools, req runtime.RunRequest, kb *runtime.Kbinfos, logger *log.Logger) {
- chunks, aggs := runtime.HybridSearch(ctx,
- searchDepsFor(ctx, deps, req, req.DatasetIDs, req.TenantID, kb, logger),
- runtime.SearchParams{
- Question: req.Question,
- Keywords: req.Keywords,
- UseCompiled: req.UseCompiled,
- TopN: req.TopN,
- // The retrieve channel.
- Channel: runtime.ChannelRetrieve,
- })
- kb.Merge(chunks, aggs)
-}
-
-// Final answer composition (the graph's last node).
-//
-// Every thinking mode ends here: it turns the gathered evidence into a grounded, cited
-// answer in the user's language. The evidence-block and citation-rule prompts it renders
-// (kb_prompt / citation_prompt) live in internal/rag/prompts; the system-prompt texts live
-// in the runtime package (report_prompt.go).
-
-const (
- // evidenceBudgetTokens is the token ceiling of the evidence block.
- // The compose path still fits the whole prompt to the model window
- // (AnswerDeps.MaxLength / chat.FitMessages), so this is an upper bound.
- evidenceBudgetTokens = 80000
- // evidencePoolQuota: claim pseudo-chunk
- // cap across the whole first prefetch.
- evidencePoolQuota = 24
- // rawSnippetQuota: the raw-chunk
- // admission budget inside the fan-out, so chunk channels cannot crowd out
- // the denser evidence rows.
- rawSnippetQuota = 30
- // citeChunkCap caps chunks rendered as citation reference.
- citeChunkCap = 6
- // answerTimeoutS bounds the answer-composition call.
- answerTimeoutS = 150.0
- // naiveEvidenceChunkCap caps evidence chunks in the naive path.
- naiveEvidenceChunkCap = 8
- // naiveEvidenceCharCap caps each naive evidence chunk.
- naiveEvidenceCharCap = 1500
- // composeStage tags the terminal composition in the log and in the think
- // block.
- composeStage = "Composing the answer"
- // providerErrorSummaryMax bounds the provider error a user reads inside the
- // think block: the message can be a whole JSON body or a wall of retry
- // advice, and the block is a narrative, not a log dump.
- providerErrorSummaryMax = 300
- // graphFailureFallback: last-resort message
- // (run_agentic_rag), used only when the graph failed AND produced nothing.
- graphFailureFallback = "I couldn't complete the search due to an internal error."
- // AgenticRecursionLimit: for the agentic
- // graph: the maximum number of node visits before the graph aborts. Go has
- // no graph runtime, so the loop counts its own node visits against it.
- AgenticRecursionLimit = 60
- // agenticRoundVisits is the number of node visits one research round costs:
- // rag_agent → draft → sca.
- agenticRoundVisits = 3
- // lowRecursionLimitBase is the floor of `max(25, max_loops * 8)` for the non-agentic
- // graph.
- lowRecursionLimitBase = 25
-)
-
-// FinalAnswerSystem and PartialAnswerPreamble are defined in the runtime package
-// (runtime/report_prompt.go) and re-used here via the runtime import rather than
-// duplicated.
-
-// AnswerDeps are the dependencies of ComposeAnswer.
-type AnswerDeps struct {
- // Model drives the composition call.
- Model runtime.SessionModel
- // CiteRules overrides DEFAULT_CITE_RULES. Empty uses the default.
- CiteRules string
- // SystemPrompt is the dialog-level UI configuration. Appended AFTER the agentic
- // contract — see the
- // precedence note in composeSystem.
- SystemPrompt string
- // EmptyResponse is returned verbatim when there is no evidence, skipping the LLM
- // entirely.
- EmptyResponse string
- // MaxTokens caps the evidence block. <=0 uses evidenceBudgetTokens.
- MaxTokens int
- // MaxLength is the chat model's context window. It bounds the prompt fit; <=0 falls back
- // to chat.EffectiveContextLength's 8192 default.
- MaxLength int
- // Logger is optional; nil uses the default logger.
- Logger *log.Logger
- // UserImages are the vision-gated base64 data URIs that survive
- // gateImageAttachments. The direct fallback is called with the original multimodal
- // messages, so they are attached to the final-answer user message and the compose model
- // sees the images even on the non-outer path. Empty for text-only models / no images.
- UserImages []string
-}
-
-// ComposeAnswerWith is ComposeAnswer with the third no-evidence term: `no_evidence =
-// abstain or empty_result or not chunks`, used both for the empty_response short circuit
-// and for the degradation instructions in the prompt. The extra `emptyResult` term is the
-// agentic loop's own "nothing was found" signal, distinct from "we abstained" and from
-// "the pool happens to be empty".
-type finalizeAnnouncedKey struct{}
-
-// markFinalizeAnnounced records that [Finalize] has been reported for this run's
-// composition.
-func markFinalizeAnnounced(ctx context.Context) context.Context {
- return context.WithValue(ctx, finalizeAnnouncedKey{}, true)
-}
-
-// finalizeAnnounced reports whether [Finalize] was reported for this composition.
-func finalizeAnnounced(ctx context.Context) bool {
- v, _ := ctx.Value(finalizeAnnouncedKey{}).(bool)
- return v
-}
-
-func ComposeAnswerWith(ctx context.Context, deps AnswerDeps, kb *runtime.Kbinfos, question string, partial, abstain, emptyResult bool) AnswerResult {
- logger := deps.Logger
- if logger == nil {
- logger = _LOG
- }
- chunks := []map[string]any{}
- if kb != nil {
- chunks = kb.Chunks
- }
- started := time.Now()
- // The kickoff step is suppressed when the graph already reported [Finalize]
- // (finalizeAnnounced): the verdict and the evidence have been said, and a line
- // repeating their numbers only made the block longer. The paths that have no
- // [Finalize] — the low graph's last node, every fallback compose — keep it.
- if !finalizeAnnounced(ctx) {
- step(ctx, logger, composeStage, "Composing the answer from %s.",
- runtime.CountOf(len(chunks), "gathered passage"))
- }
-
- // The step kept below either way is the no-evidence short circuit: "no model was
- // called" is a fact no other line carries.
-
- // 1. No-evidence short circuit.
- // _compose_answer_from_evidence: no_evidence = abstain or empty_result or not chunks.
- noEvidence := abstain || emptyResult || len(chunks) == 0
- if noEvidence && deps.EmptyResponse != "" {
- step(ctx, logger, composeStage, "No supporting evidence was found, so the configured empty response is returned without calling the model.")
- return AnswerResult{Answer: deps.EmptyResponse, NoEvidence: true}
- }
- if deps.Model == nil {
- return AnswerResult{Answer: "", Failed: true, NoEvidence: len(chunks) == 0}
- }
-
- // 2. Build the prompt: ranked evidence under its token budget, plus the
- // question and any research findings.
- preSummary := ""
- if kb != nil {
- preSummary = kb.PreSummary
- }
- // Both the prompt and this log go through composedRecord, so the line below
- // reports the record block the answer actually carried.
- record := composedRecord(kb)
- prompt := deps.answerPromptWithEvidence(kb, question, partial, noEvidence)
-
- // 3. Call the model.
- callCtx, cancel := context.WithTimeout(ctx, deadlineToDuration(answerTimeoutS))
- defer cancel()
-
- logger.Printf("[Formalize][record] question=%q record_len=%d draft_summary_len=%d evidence_len=%d using=%s\nrecord=%q",
- trunc(question, 160), len(record), len(preSummary), len(prompt.user), recordSource(record),
- truncateRunes(record, 3000))
-
- logger.Printf("[Formalize][pre_summary] question=%q pre_summary_len=%d evidence_len=%d\npre_summary=%q",
- trunc(question, 160), len(preSummary), len(prompt.user), truncateRunes(preSummary, 3000))
-
- // The composed prompt is fitted ONCE before the call, bounded by the smaller of the
- // model's window and the evidence budget: msg[0] is the system turn, msg[-1] the user
- // turn. The fit must not synthesize an extra system turn.
- systemTurn, userTurn := fitComposePrompt(prompt.system, prompt.user, deps.MaxLength)
- userMsg := schema.UserMessage(userTurn)
- if len(deps.UserImages) > 0 {
- // The direct fallback is called with the original multimodal messages: attach the
- // vision-gated images to the final-answer user message so the compose model can see
- // them.
- userMsg = multimodalUserMsg(prompt.user, deps.UserImages)
- }
- // The compose call samples at the configured temperature, defaulting to 0.3; the graph
- // runs with no per-run override, so 0.3 applies.
- reply, err := modelWithTemperature(deps.Model, answerTemperature).Complete(callCtx, []schema.Message{
- *schema.SystemMessage(systemTurn),
- *userMsg,
- }, nil)
- if err != nil {
- // Deliberately NOT Python's bare "I'm sorry…" sentence. The ANSWER
- // carries the provider's own message in the classic `**ERROR**: …` shape
- // the rest of the chat pipeline uses for its failures, so an agentic
- // failure reads exactly like a naive-mode one instead of hiding the
- // cause — an exhausted quota or a provider 5xx used to look like a RAG
- // bug. The think block gets the same text as a reasoning step, and the
- // developer log line stays byte-for-byte what it was (the detail half of
- // StageLineDetail).
- summary := providerErrorSummary(err)
- runtime.StepsFrom(ctx).StageLineDetail(logger, composeStage,
- "Composing the answer failed: "+summary,
- fmt.Sprintf("composition failed: %v", err))
- return AnswerResult{Answer: errorAnswerText(err), Failed: true}
- }
- answer := citeAnchoredMembers(cleanAnswer(reply.Content), kb)
- logComposeDone(logger, started, answer, len(chunks))
- return AnswerResult{Answer: answer, Partial: partial}
-}
-
-// fitComposePrompt is the compose-time fit: min(model window, evidence budget). The fit
-// normalizes a non-positive budget to 8192, returns the
-// pair untouched when it already fits, then trims by TOKENS (trim_content):
-// the system share branch (>0.8 of the total) preserves the user turn first,
-// otherwise the system turn is preserved first and the user turn gets the
-// remainder. form_message is exactly the [system, user]
-// pair, so no extra system turn is synthesized here.
-func fitComposePrompt(system, user string, maxLength int) (string, string) {
- budget := maxLength
- if budget > evidenceBudgetTokens {
- budget = evidenceBudgetTokens
- }
- if budget <= 0 {
- // message_fit_in normalizes a non-positive max_length to 8192
- // .
- budget = 8192
- }
- ll := tokenizer.NumTokensFromString(system)
- ll2 := tokenizer.NumTokensFromString(user)
- if ll+ll2 < budget {
- return system, user
- }
- if ll+ll2 <= 0 {
- // message_fit_in's degenerate branch: token counts are zero — keep the
- // content unchanged rather than trimming blindly.
- return system, user
- }
- if float64(ll)/float64(ll+ll2) > 0.8 {
- // System-dominated prompt: the USER turn is preserved first.
- preservedLast := min(ll2, budget)
- user = tokenizer.TrimContentToTokenLimit(user, preservedLast)
- remaining := max(0, budget-preservedLast)
- system = tokenizer.TrimContentToTokenLimit(system, remaining)
- return system, user
- }
- preservedSystem := min(ll, budget)
- system = tokenizer.TrimContentToTokenLimit(system, preservedSystem)
- remaining := max(0, budget-preservedSystem)
- user = tokenizer.TrimContentToTokenLimit(user, remaining)
- return system, user
-}
-
-// answerPrompt is the terminal node's input: the ranked evidence under its token
-// budget plus the question and any research findings. Shared by the one-shot and
-// the streaming compose so both render exactly the same prompt.
-type answerPrompt struct {
- system string
- user string
- partial bool
-}
-
-// noEvidenceWithSummary / noEvidenceWithoutSummary are the no-evidence instructions.
-// Unlike the empty_response short circuit they apply when composition still goes ahead (no
-// empty_response configured), and they tell the model to degrade honestly rather than
-// guess.
-const (
- noEvidenceWithSummary = "The retrieved passages are limited. Answer as completely as possible " +
- "from the Research Summary below, using the known facts; where a " +
- "specific number/entity is missing, say what is known and avoid " +
- "flatly refusing to answer.\n"
- noEvidenceWithoutSummary = "No supporting evidence was retrieved. State clearly that the available " +
- "sources are insufficient, and do not answer from general knowledge.\n"
-)
-
-// ComposeAnswerStream is ComposeAnswer for models that can emit incrementally:
-// it renders the same prompt, forwards each piece as it arrives, and returns the
-// assembled answer. A streaming failure is returned so the caller can fall back
-// to the one-shot call.
-//
-// emptyResult is the state's empty_result term of
-// `no_evidence = abstain or empty_result or not chunks` — the graph compose
-// path forwards the state's value (always True there; see formalizeAnswerNode).
-func ComposeAnswerStream(ctx context.Context, deps AnswerDeps, model runtime.StreamingSessionModel, kb *runtime.Kbinfos, question string, partial, emptyResult bool, onDelta func(delta string, isThink bool) error) (AnswerResult, error) {
- logger := deps.Logger
- if logger == nil {
- logger = _LOG
- }
- if model == nil {
- return AnswerResult{Answer: "", Failed: true}, nil
- }
- chunks := []map[string]any{}
- if kb != nil {
- chunks = kb.Chunks
- }
- // Same no-evidence rule as the one-shot path: the prompt must carry the degradation
- // instruction even when no empty_response short-circuits.
- noEvidence := emptyResult || len(chunks) == 0
- started := time.Now()
- // Same gated kickoff step as the one-shot path above (finalizeAnnounced). The
- // streaming path has no separate `abstain` signal (Go threads only
- // partial/emptyResult), so the abstain note term cannot fire here.
- if !finalizeAnnounced(ctx) {
- step(ctx, logger, composeStage, "Composing the answer from %s.",
- runtime.CountOf(len(chunks), "gathered passage"))
- }
- prompt := deps.answerPromptWithEvidence(kb, question, partial, noEvidence)
- if noEvidence && deps.EmptyResponse != "" {
- return AnswerResult{Answer: deps.EmptyResponse, NoEvidence: true}, nil
- }
- // The record the compose prompt actually carries, content included (first 3000 chars), so
- // a run that answered without the slot facts is diagnosable from the log alone.
- preSummary := ""
- if kb != nil {
- preSummary = kb.PreSummary
- }
- // Both the prompt and this log go through composedRecord, so the line below
- // reports the record block the answer actually carried.
- record := composedRecord(kb)
- logger.Printf("[Formalize][record] question=%q record_len=%d draft_summary_len=%d evidence_len=%d using=%s\nrecord=%q",
- trunc(question, 160), len(record), len(preSummary), len(prompt.user), recordSource(record),
- truncateRunes(record, 3000))
-
- callCtx, cancel := context.WithTimeout(ctx, deadlineToDuration(answerTimeoutS))
- defer cancel()
-
- // Same prompt fit as the one-shot path: composition runs once and streams from the
- // fitted messages.
- systemTurn, userTurn := fitComposePrompt(prompt.system, prompt.user, deps.MaxLength)
- userMsg := schema.UserMessage(userTurn)
- if len(deps.UserImages) > 0 {
- // Same as ComposeAnswerWith: attach the vision-gated images so the
- // compose model sees them on the non-outer path.
- userMsg = multimodalUserMsg(prompt.user, deps.UserImages)
- }
- // Temperature: the production streaming carrier
- // (runtime.InvokerSessionModel.StreamComplete) samples at 0.3 internally —
- // the same value answer_conf carries here; SessionModel's streaming
- // surface has no per-call temperature, so other carriers run at their own default
- // (a known limitation, flagged in the port notes).
- reply, err := model.StreamComplete(callCtx, []schema.Message{
- *schema.SystemMessage(systemTurn),
- *userMsg,
- }, nil, func(delta string, isThink bool) error {
- if onDelta == nil || delta == "" {
- return nil
- }
- return onDelta(delta, isThink)
- })
- if err != nil {
- logger.Printf("[Composing the answer] streaming composition failed: %v", err)
- return AnswerResult{Answer: "", Failed: true}, err
- }
- if reply == nil {
- return AnswerResult{Answer: "", Failed: true}, errors.New("streaming composition returned no reply")
- }
- answer := citeAnchoredMembers(cleanAnswer(reply.Content), kb)
- logComposeDone(logger, started, answer, len(chunks))
- return AnswerResult{Answer: answer, Partial: partial}, nil
-}
-
-// citeAnchoredMembers attaches the citation of every enumerated member the answer states without
-// one (see runtime.CiteAnchoredMembers). Nil-safe: a run that enumerated nothing, or whose members
-// never reached the published evidence list, comes back unchanged.
-func citeAnchoredMembers(answer string, kb *runtime.Kbinfos) string {
- if kb == nil {
- return answer
- }
- refs := kb.AnchoredRefs()
- // textOf lets the step verify a citation against the passage it would open: a marker is
- // written only for a passage that actually contains the words the answer quotes.
- textOf := func(id string) string {
- if c := kb.ChunkByID(id); c != nil {
- return runtime.ChunkTextOf(c)
- }
- return ""
- }
- out, overridden := runtime.CitedAnchoredMembers(answer, refs, kb.CiteChunkIDs, textOf)
- if len(refs) > 0 {
- // One line per run that says which member was given which marker, and which passage that
- // marker holds: "the citation opens a passage that does not state this member" is only
- // answerable from this pairing, and without it a marker and its passage look the same
- // whether or not they belong together.
- pos := map[string]int{}
- for i, id := range kb.CiteChunkIDs {
- if _, dup := pos[id]; !dup {
- pos[id] = i
- }
- }
- pairs := make([]string, 0, len(refs))
- for _, r := range refs {
- if i, ok := pos[strings.TrimSpace(r.ChunkID)]; ok {
- pairs = append(pairs, fmt.Sprintf("%s->%d(%s)", r.Name, i, r.ChunkID))
- continue
- }
- pairs = append(pairs, fmt.Sprintf("%s->(not-published)", r.Name))
- }
- _LOG.Printf("[Citation] anchored %d member(s), rewrote %d line(s); %s",
- len(refs), overridden, strings.Join(pairs, " "))
- }
- return out
-}
-
-// composeSystem builds the system prompt, applying the precedence rules.
-//
-// LANGUAGE IS DELIBERATELY LEFT OVERRIDABLE: "answer in the same language as the
-// question" is exactly the rule a user setting "answer in English" means to
-// replace, so it must not be listed as protected. What stays protected is the
-// evidence contract: citing sources, answering the exact attribute asked for,
-// and never substituting prior knowledge for missing evidence.
-// citeChunkIDsAt returns the citation id of every chunk that rendered a block, in
-// render order, from the render's source indices.
-//
-// It takes the SOURCE INDICES rather than the chunk list because the renderer emits
-// no block for a chunk without content: one id per RENDERED block, or the published
-// list runs ahead of the numbering at every skip and each marker past it lands on
-// the wrong passage. A chunk without an id keeps its slot as an empty string —
-// dropped, it would shift the later blocks the same way (citePoolIdx resolves "" to
-// -1, so that block's own citation is dropped, but the blocks after it stay put).
-func citeChunkIDsAt(chunks []map[string]any, sources []int) []string {
- out := make([]string, len(sources))
- for i, src := range sources {
- if src >= 0 && src < len(chunks) {
- out[i] = runtime.ChunkIDOf(chunks[src])
- }
- }
- return out
-}
-
-// logComposeDone reports the compose's own cost and size.
-//
-// It is a LOG line, not a step: the compose finishes after the answer has already
-// started streaming, and the chat pipeline drops think lines emitted past that point
-// (so a thought block never reopens after the answer) — a "composed …" step visible
-// in the non-streaming path but not the streaming one would be worse than none.
-//
-// The count is in RUNES, not bytes: a Chinese answer is two to three times its rune
-// count in bytes, and this number is read as "how much text came out".
-func logComposeDone(logger *log.Logger, started time.Time, answer string, chunks int) {
- logger.Printf("[Composing the answer] composed %s from %s in %.1fs",
- runtime.CountOf(utf8.RuneCountInString(answer), "char"),
- runtime.CountOf(chunks, "gathered passage"),
- time.Since(started).Seconds())
-}
-
-// zeroBasedEvidenceRule states the evidence numbering for the compose model.
-//
-// The blocks this call renders are numbered from 0 because the client resolves a
-// marker by indexing reference.chunks with its number, and the answer reaches that
-// client as the model streams it. Saying so guards against the "the first source is
-// 1" habit: a model that falls back on it cites the second passage and leaves the
-// first one unreachable (its markers are still openable, they just point one block
-// off).
-//
-// It belongs at the call site, not in CitationPrompt: that text is shared with the
-// agent canvas, which numbers its blocks by hash id (component/prompts/citation.go).
-const zeroBasedEvidenceRule = "\n\n# Evidence ids\n" +
- "The evidence blocks below are numbered from 0: the FIRST block is [ID:0]. " +
- "Cite the id printed at the start of the block you used, exactly as printed — " +
- "do not renumber the blocks yourself."
-
-func (d AnswerDeps) composeSystem() string {
- // The citation rules are the citation_prompt template with an optional user-defined
- // override (RAGConfig.CiteRules), plus the 0-based evidence-numbering rule the
- // compose path renders with (see zeroBasedEvidenceRule).
- rules := prompts.CitationPrompt(d.CiteRules) + zeroBasedEvidenceRule
- system := strings.ReplaceAll(runtime.FinalAnswerSystem, "{cite_rules}", rules)
- if sp := strings.TrimSpace(d.SystemPrompt); sp != "" {
- system = fmt.Sprintf("%s\n\n# Assistant configuration (set by the user)\n%s\n\nFollow the configuration above for language, tone, style, format and any other presentational instruction, including where it overrides the language rule above. Where it conflicts with the citation rules, attribute fidelity, or the requirement to answer only from the provided evidence, those three take precedence.",
- system, sp)
- }
- return system
-}
-
-// cleanAnswer strips a leading thinking preamble from the composed answer.
-func cleanAnswer(s string) string {
- return strings.TrimSpace(reAnswerThink.ReplaceAllString(s, ""))
-}
-
-// ComposeNaiveAnswer: answer composition
-// (lines 1555-1568): one retrieve pass, then a single composed answer over the
-// top chunks.
-//
-// Unlike ComposeAnswer this does NOT use kb_prompt and does NOT use
-// FinalAnswerSystem — the naive path renders a flat "[i] content" list truncated to the
-// first 1500 chars of each chunk, capped at 8 chunks, and sends it under a short fixed
-// system prompt.
-//
-// The messages are run through message_fit_in against
-// deps.MaxLength before the call.
-func ComposeNaiveAnswer(ctx context.Context, deps AnswerDeps, chunks []map[string]any, question string) AnswerResult {
- logger := deps.Logger
- if logger == nil {
- logger = _LOG
- }
- if len(chunks) == 0 {
- if deps.EmptyResponse != "" {
- return AnswerResult{Answer: deps.EmptyResponse, NoEvidence: true}
- }
- return AnswerResult{NoEvidence: true}
- }
-
- // a flat "[i] content" list over the first 8 chunks, each
- // truncated to 1500 characters.
- var parts []string
- for i, c := range chunks {
- if i >= naiveEvidenceChunkCap {
- break
- }
- content := runtime.ChunkTextOf(c)
- if len(content) > naiveEvidenceCharCap {
- content = content[:naiveEvidenceCharCap]
- }
- parts = append(parts, fmt.Sprintf("[%d] %s", i+1, content))
- }
- evidence := strings.Join(parts, "\n\n")
- fallback := evidence
- if len(fallback) > naiveFallbackChars {
- fallback = fallback[:naiveFallbackChars]
- }
-
- if deps.Model == nil {
- // Calling a nil model would panic; return the evidence the same way a failed call
- // does.
- return AnswerResult{Answer: fallback, Failed: true}
- }
-
- callCtx, cancel := context.WithTimeout(ctx, deadlineToDuration(answerTimeoutS))
- defer cancel()
-
- // message_fit_in(form_message(system, user), max_length).
- fitted, fitErr := chat.FitMessages(naiveAnswerSystem, []schema.Message{
- *schema.UserMessage(fmt.Sprintf("Question: %s\n\nEvidence:\n%s", question, evidence)),
- }, deps.MaxLength)
- if fitErr != "" {
- logger.Printf("[Naive RAG] prompt fitting failed: %s", fitErr)
- return AnswerResult{Answer: fallback, Failed: true}
- }
-
- // async_chat(msg[0]["content"], msg[1:], answer_conf) — the
- // fitted system text is the first entry and the rest is the history.
- // Splitting explicitly keeps the call shape: Complete takes the two parts re-joined.
- system := naiveAnswerSystem
- history := fitted
- if len(fitted) > 0 && fitted[0].Role == schema.System {
- system = fitted[0].Content
- history = fitted[1:]
- }
- messages := make([]schema.Message, 0, 1+len(history))
- messages = append(messages, *schema.SystemMessage(system))
- messages = append(messages, history...)
-
- reply, err := modelWithTemperature(deps.Model, naiveAnswerTemperature).Complete(callCtx, messages, nil)
- if err != nil {
- // on failure yield the raw evidence, not an error.
- logger.Printf("[Naive RAG] composition failed: %v", err)
- return AnswerResult{Answer: fallback, Failed: true}
- }
- // str(ans or "").strip or empty_response — an empty answer
- // degrades to the configured empty response.
- answer := cleanAnswer(reply.Content)
- if answer == "" {
- answer = deps.EmptyResponse
- }
- return AnswerResult{Answer: answer}
-}
diff --git a/internal/rag/agentic-rag/graph_fanout.go b/internal/rag/agentic-rag/graph_fanout.go
deleted file mode 100644
index 23be1972bb..0000000000
--- a/internal/rag/agentic-rag/graph_fanout.go
+++ /dev/null
@@ -1,740 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package agentic_rag
-
-import (
- "context"
- "fmt"
- "strings"
- "unicode/utf8"
-
- "github.com/cloudwego/eino/schema"
- "ragflow/internal/rag/agentic-rag/runtime"
-)
-
-// The fan-out stage: decompose the question into first-hop sub-questions, parse what
-// the model wrote, and fetch them programmatically.
-//
-// Its parsing is shape-guarded rather than answer-keyed (see fanoutLooksLikeQuery):
-// the model is asked for sub-questions and hands back whatever it produced, so what
-// a line IS decides what the stage does with it.
-// fanoutLooksLikeQuery: reject
-// prose/answer lines before they can enter the retrieval + slot pipeline.
-//
-// Fan-outs are used verbatim as BM25/hybrid queries and as the slot table's
-// fanout_hint, so an answered fact ("The woman was **X**") must never survive
-// here: it both poisons retrieval and asserts a hallucinated entity as a known
-// aspect.
-func fanoutLooksLikeQuery(line string, loose bool) bool {
- s := strings.TrimSpace(line)
- if s == "" {
- return false
- }
- low := strings.ToLower(s)
- for _, mark := range fanoutAnswerMarks {
- if strings.Contains(low, mark) {
- return false
- }
- }
- // The cap counts characters, not bytes; a byte-based cap would reject a legitimate CJK
- // fan-out well below the 160-character limit.
- if utf8.RuneCountInString(s) > fanoutMaxChars {
- return false
- }
- words := len(strings.Fields(s))
- if loose && !strings.HasSuffix(s, "?") && words > fanoutLooseMaxWords {
- return false
- }
- return words <= fanoutMaxWords
-}
-
-// fanoutLineBreak reports whether r is a line boundary.
-// Splitting on "\n" alone misses a lone "\r" and the other Unicode line breaks a model
-// could emit, so the loose path would fuse several fan-out lines into one.
-func fanoutLineBreak(r rune) bool {
- switch r {
- case '\n', '\r', '\v', '\f', 0x1c, 0x1d, 0x1e, 0x85, 0x2028, 0x2029:
- return true
- }
- return false
-}
-
-// parseFanouts: extract fan-outs from a
-// model reply, validating every entry's shape.
-func parseFanouts(text string) []string {
- var raw []string
- loose := false
- if data, ok := extractJSONObject(text).(map[string]any); ok {
- for _, f := range asSliceOfAny(data["fanouts"]) {
- if s := strings.TrimSpace(fmt.Sprint(f)); s != "" {
- raw = append(raw, s)
- }
- }
- } else {
- // Loose fallback: the model answered in prose. Only lines that still
- // look like a search query are kept — answer sentences and source
- // lists are dropped.
- loose = true
- // Split on every line boundary, not just "\n".
- for _, ln := range strings.FieldsFunc(text, fanoutLineBreak) {
- if strings.TrimSpace(ln) == "" {
- continue
- }
- // The bullet / numbering cutset is stripped from BOTH ends, then whitespace is
- // trimmed, so a trailing period/digit never leaks into the retrieval query.
- raw = append(raw, strings.TrimSpace(strings.Trim(ln, "-•0123456789. ")))
- }
- }
- kept := make([]string, 0, len(raw))
- for _, q := range raw {
- if fanoutLooksLikeQuery(q, loose) {
- kept = append(kept, q)
- }
- }
- if len(raw) > 0 && len(kept) == 0 {
- _LOG.Printf("[Planner] discarding %d fan-out candidate(s): none look like search queries", len(raw))
- }
- kept = dedupe(kept)
- if len(kept) > MaxFanouts {
- kept = kept[:MaxFanouts]
- }
- return kept
-}
-
-// ExpandFanouts: ONE chat call (no
-// tools) producing 2-5 first-hop fan-outs, plus one strict retry when the
-// reply was not parseable JSON (_expand_fanouts).
-//
-// Falls back to the raw question alone on any failure — a fan-out failure never
-// blocks the pipeline.
-func ExpandFanouts(ctx context.Context, deps RAGTools, question string) []string {
- if question == "" {
- return nil
- }
- if deps.Model == nil {
- return []string{question}
- }
- reply, err := deps.Model.Complete(ctx, []schema.Message{
- *schema.SystemMessage(fanoutPrompt),
- *schema.UserMessage("Question: " + question),
- }, nil)
- if err != nil {
- _LOG.Printf("[Planner] fan-out expansion failed; falling back to raw question: %v", err)
- return []string{question}
- }
- fanouts := parseFanouts(reply.Content)
- if len(fanouts) == 0 {
- // The model answered the question instead of decomposing it (no JSON,
- // or JSON that failed the shape guard). One strict retry, then give up.
- _LOG.Printf("[Planner] fan-out expansion produced no usable sub-question; retrying with a strict JSON instruction")
- if retry, rerr := deps.Model.Complete(ctx, []schema.Message{
- *schema.SystemMessage(fanoutPrompt + fanoutStrictRetry),
- *schema.UserMessage("Question: " + question),
- }, nil); rerr != nil {
- _LOG.Printf("[Planner] strict fan-out retry failed: %v", rerr)
- } else {
- fanouts = parseFanouts(retry.Content)
- }
- }
- if len(fanouts) == 0 {
- fanouts = []string{question}
- }
- _LOG.Printf("[Planner] fan-out expansion: %d sub-question(s): %v", len(fanouts), fanouts)
- return fanouts
-}
-
-// Metadata pre-filter channel: the metadata conditions a sub-question's wording supports.
-//
-// Channel C exists because a sub-question that NAMES a document is better served by
-// pre-filtering the document set by metadata and searching inside it than by another
-// whole-corpus round: the whole-corpus round has to rank the named document against every
-// other one, and on a large corpus it loses. The conditions come from ONE chat call for the
-// whole fan-out batch, and every value must look like a SHORT NAME — a copied sub-question or
-// an answer would make the condition match nothing.
-
-// fanoutFilterPrompt is the ONE chat call the pre-search metadata channel makes per batch: it
-// turns each named thing a sub-question carries into a metadata condition over the dataset's
-// OWN fields. The vocabulary is injected under AVAILABLE METADATA (see MetadataCatalog.Render),
-// so the channel is not pinned to any single field: a dataset whose content fields are called
-// `title` / `file_name` / `topic` is filtered on whichever of them the sub-question names.
-const fanoutFilterPrompt = `For each numbered search sub-question below, decide whether it NAMES something a ` +
- `document's metadata would carry — a document title or file name, an author, an organisation, a place, ` +
- `an event — and turn each named thing into ONE metadata condition {key, op, value} over the fields ` +
- `listed under AVAILABLE METADATA. ` +
- `Use the field whose meaning fits the named thing; prefer "contains" unless the whole value is named. ` +
- `The value MUST be copied EXACTLY as it appears — keep underscores, punctuation and file extensions ` +
- `as written, never re-normalise them (a stored file name keeps its underscores; a stored title keeps ` +
- `its spaces) — and when the sub-question's wording matches one of the listed values, use that listed ` +
- `value verbatim. Keep each value under 10 words: never copy the whole sub-question, never answer it. ` +
- `Emit ONLY conditions you are confident about: a sub-question that names no such thing gets an EMPTY ` +
- `list — do not invent a condition just to fill one. ` +
- `Respond with JSON only: {"filters": [[{"key":"...","op":"...","value":"..."}], ...]} — one ` +
- `condition-array per sub-question, in the SAME ORDER as the sub-questions, JSON only, no prose.`
-
-const (
- // metadataValueMaxWords / metadataValueMaxChars are the guard on a filter value taken from a
- // sub-question: a long string is a copied sub-question or an answer.
- metadataValueMaxWords = 10
- metadataValueMaxChars = 80
- // metadataMaxConditions caps the conditions taken from one sub-question.
- metadataMaxConditions = 3
- // fanoutMetadataQuota is the metadata channel's per-query quota. Modest like the
- // semantic quota: the channel's hits are only as good as the extraction.
- fanoutMetadataQuota = 4
-)
-
-// fanoutMetadataOps are the operators the pre-search channel may use: the POSITIVE
-// value-matching subset of the tool's operators. The channel's contract is "the sub-question
-// NAMED something, so look inside the documents whose metadata matches it"; `not contains` /
-// `empty` / `not empty` are deliberately absent because they WIDEN a document set — the
-// opposite of what a pre-filter is for, and a model reaching for one would scope the channel's
-// search to everything it was meant to narrow.
-var fanoutMetadataOps = map[string]bool{
- "=": true,
- "contains": true,
- "start with": true,
- "end with": true,
- "in": true,
-}
-
-// fanoutMetadataVocabulary returns the field vocabulary the pre-search channel may use, and
-// whether the channel can run at all.
-//
-// The vocabulary IS the session's metadata catalog — the same "declared ∪ observed − blacklist"
-// set the metadata_search tool advertises. There is no field name baked in anywhere: a caller
-// that wired no toolset, or a session whose catalog is empty because the dataset carries no
-// metadata, has no vocabulary to offer and the channel is skipped. (It would otherwise pay the
-// extraction call and then run one guaranteed-empty scoped retrieval per sub-question.)
-//
-// The vocabulary is the catalog's FULL rendering, value samples included. That is deliberate:
-// this channel has no feedback loop — a condition that matches nothing is a silent miss, not a
-// retry — so the model must be able to pick the exact form the index stores. A guessed value
-// that differs from the stored one by case, punctuation or an underscore normalisation is the
-// observed failure (measured 2026-09-20: `file_name contains "02 Endpoint Isolation ..."` found
-// nothing while the stored value kept its underscores).
-func fanoutMetadataVocabulary(deps RAGTools) (string, map[string]bool, bool) {
- if deps.Tools == nil {
- return "", nil, false
- }
- cat := deps.Tools.MetadataFields
- if cat == nil || cat.Empty() {
- return "", nil, false
- }
- allowed := make(map[string]bool, len(cat.Keys))
- for _, k := range cat.Keys {
- allowed[k] = true
- }
- return cat.Render(), allowed, true
-}
-
-// fanoutValueLooksUsable reports whether a filter value is a short name rather than a copied
-// sub-question or an answer.
-func fanoutValueLooksUsable(value string) bool {
- s := strings.TrimSpace(value)
- if s == "" {
- return false
- }
- low := strings.ToLower(s)
- for _, mark := range fanoutAnswerMarks {
- if strings.Contains(low, mark) {
- return false
- }
- }
- // The character cap counts code points: a byte-based cap would reject a legitimate CJK
- // value well below the 80-character limit.
- return utf8.RuneCountInString(s) <= metadataValueMaxChars && len(strings.Fields(s)) <= metadataValueMaxWords
-}
-
-// fanoutCondition validates one model-written condition against the dataset's offered fields
-// and the pre-search operators, returning nil when it must be dropped.
-//
-// The value is coerced by runtime.NormalizeMetadataValue — the same function the
-// metadata_search tool uses — so a condition means the same thing on either path.
-func fanoutCondition(raw any, allowed map[string]bool) map[string]any {
- m, ok := raw.(map[string]any)
- if !ok {
- return nil
- }
- key := strings.TrimSpace(fmt.Sprint(m["key"]))
- op := strings.TrimSpace(fmt.Sprint(m["op"]))
- if !allowed[key] || !fanoutMetadataOps[op] {
- return nil
- }
- value := runtime.NormalizeMetadataValue(m["value"], op)
- if op == "in" {
- list, ok := value.([]any)
- if !ok || len(list) == 0 {
- return nil
- }
- for _, item := range list {
- if !fanoutValueLooksUsable(fmt.Sprint(item)) {
- return nil
- }
- }
- return map[string]any{"key": key, "op": op, "value": value}
- }
- text := strings.TrimSpace(fmt.Sprint(value))
- if value == nil || text == "" || text == "<nil>" || !fanoutValueLooksUsable(text) {
- return nil
- }
- return map[string]any{"key": key, "op": op, "value": text}
-}
-
-// parseFanoutFilters extracts up to metadataMaxConditions conditions per sub-question from a
-// model reply, order-aligned with fanouts.
-//
-// Accepts {"filters": [[{...}, ...], ...]} (preferred), a flat {"filters": [{...}, ...]} (read
-// positionally: one condition per sub-question), or
-// {"filters": [{"sub_question": ..., "filters": [...]}]} matched by text. Conditions are
-// validated by fanoutCondition: an unknown field, a non-pre-search operator, an over-long value
-// or a duplicate is dropped. Sub-questions left with no condition simply skip the channel.
-func parseFanoutFilters(text string, fanouts []string, allowed map[string]bool) [][]map[string]any {
- out := make([][]map[string]any, len(fanouts))
- data, ok := extractJSONObject(text).(map[string]any)
- if !ok {
- return out
- }
- items := asSliceOfAny(data["filters"])
- if len(items) == 0 {
- return out
- }
- byText := map[string][]map[string]any{}
- var positional [][]map[string]any
- for _, it := range items {
- switch v := it.(type) {
- case map[string]any:
- // A sub-question key means the object groups its own conditions; otherwise the
- // object IS one condition.
- if _, grouped := v["sub_question"]; grouped || v["fanout"] != nil || v["filters"] != nil {
- conds := fanoutConditions(asSliceOfAny(v["filters"]), allowed)
- sq := ""
- if sv := v["sub_question"]; sv != nil {
- sq = strings.TrimSpace(fmt.Sprint(sv))
- }
- if sq == "" && v["fanout"] != nil {
- sq = strings.TrimSpace(fmt.Sprint(v["fanout"]))
- }
- if sq != "" {
- byText[sq] = conds
- }
- positional = append(positional, conds)
- continue
- }
- positional = append(positional, fanoutConditions([]any{v}, allowed))
- case []any:
- positional = append(positional, fanoutConditions(v, allowed))
- }
- }
- for i, fq := range fanouts {
- var raw []map[string]any
- if len(byText) > 0 {
- raw = byText[fq]
- } else if i < len(positional) {
- raw = positional[i]
- }
- seen := make(map[string]bool, len(raw))
- cleaned := make([]map[string]any, 0, metadataMaxConditions)
- for _, c := range raw {
- fingerprint := fmt.Sprint(c["key"], c["op"], c["value"])
- if seen[fingerprint] {
- continue
- }
- seen[fingerprint] = true
- cleaned = append(cleaned, c)
- if len(cleaned) >= metadataMaxConditions {
- break
- }
- }
- out[i] = cleaned
- }
- return out
-}
-
-// fanoutConditions validates a raw condition list, dropping everything fanoutCondition rejects.
-func fanoutConditions(raw []any, allowed map[string]bool) []map[string]any {
- out := make([]map[string]any, 0, len(raw))
- for _, item := range raw {
- if c := fanoutCondition(item, allowed); c != nil {
- out = append(out, c)
- }
- }
- return out
-}
-
-// ExtractFanoutFilters maps each sub-question to the metadata conditions its wording supports:
-// ONE chat call for the whole batch.
-//
-// The conditions name fields from the session's catalog, so the channel filters on the dataset's
-// real fields instead of a hard-coded one. Only sub-questions with at least one valid condition
-// are returned, and any failure — no model, no filterable field, a model error, an unparseable
-// reply — yields nil so the metadata channel is simply skipped for this round; pre-search is
-// never blocked by it.
-func ExtractFanoutFilters(ctx context.Context, deps RAGTools, fanouts []string) map[string][]map[string]any {
- queries := make([]string, 0, len(fanouts))
- for _, f := range fanouts {
- if q := strings.TrimSpace(f); q != "" {
- queries = append(queries, q)
- }
- }
- if len(queries) == 0 || deps.Model == nil {
- return nil
- }
- fields, allowed, ok := fanoutMetadataVocabulary(deps)
- if !ok {
- return nil
- }
- listed := make([]string, 0, len(queries))
- for i, q := range queries {
- listed = append(listed, fmt.Sprintf("%d. %s", i+1, q))
- }
- reply, err := deps.Model.Complete(ctx, []schema.Message{
- *schema.SystemMessage(fanoutFilterPrompt + "\n\nAVAILABLE FIELDS:\n" + fields),
- *schema.UserMessage("Sub-questions:\n" + strings.Join(listed, "\n")),
- }, nil)
- if err != nil {
- _LOG.Printf("[Prefetch] metadata filter extraction failed; skipping the metadata channel: %v", err)
- return nil
- }
- filterSets := parseFanoutFilters(reply.Content, queries, allowed)
- out := make(map[string][]map[string]any, len(queries))
- for i, q := range queries {
- if len(filterSets[i]) > 0 {
- out[q] = filterSets[i]
- }
- }
- if len(out) == 0 {
- // Nothing survived the guards: report "no conditions" as nil, the same shape every
- // other failure of this best-effort call returns, so the caller needs no special case.
- //
- // The reply is logged here because this is the channel's SILENT outcome — a condition
- // rejected by the guards contributes nothing and produces no other trace, so without the
- // raw reply a miss is indistinguishable from "the model named nothing". (Empty
- // condition-lists are the COMMON, correct answer to a question that names no metadata
- // value; a non-empty reply here means the guards rejected something.) Newlines are
- // flattened so the reply stays one log line.
- flat := strings.ReplaceAll(truncateRunes(reply.Content, 300), "\n", " ")
- _LOG.Printf("[Prefetch] metadata channel produced no usable condition; reply was: %s", flat)
- return nil
- }
- _LOG.Printf("[Prefetch] metadata filters: %v", out)
- return out
-}
-
-// Phase 2: programmatic fan-out search.
-
-// FanoutSearch: the
-// programmatic multi-query prefetch that seeds the snippet pool.
-//
-// It is a three-channel search, and the channels are deliberately different:
-//
-// - channel A (exact): a keyword BM25 round over a wide pool (top_n=60) whose
-// hits are then narrowed by the query's own terms — the precision path. A
-// chunk only survives if it literally contains a query term.
-// - channel B (semantic bypass): a dense hybrid round (top_n=30) that
-// deliberately SKIPS narrowing, so a paraphrase-only hit can still enter the
-// pool. These are quarantined to a small per-fan-out quota
-// (fanoutSemanticQuota) because bypassing narrowing is what makes them
-// low-precision.
-// - channel C (metadata pre-filter): the sub-question's wording is turned into
-// metadata conditions over the SESSION's catalog fields, those documents are
-// pre-selected, and the retrieval runs INSIDE that subset (see
-// ExtractFanoutFilters). Best effort: no usable condition — or no metadata
-// resolver — simply skips the channel. Only runs when the caller asks for it
-// (useMetadata): the FIRST prefetch round, where the sub-questions are the
-// planner's raw wording; a rewrite round's queries are already targeted at a gap.
-//
-// Collapsing them into one call loses the split, and with it the ability to
-// admit a paraphrase-only match without letting it displace an exact one.
-//
-// Admission order is the point: every fan-out's channel A is admitted before any
-// fan-out's channel B, and C takes whatever room is left — its hits are the ones
-// A/B did not reach at all.
-//
-// Returns the number of NEW snippets admitted to the pool.
-func FanoutSearch(ctx context.Context, deps RAGTools, st *AgenticState, queries []string, topN, capacity int, useMetadata bool) int {
- if len(queries) == 0 || st.KB == nil {
- return 0
- }
- sd := deps.Search
- if sd.Backend == nil {
- return 0
- }
-
- // Dedup against what kbinfos ALREADY holds, then cap admissions at the remaining room —
- // the pool is a shared, cross-round ceiling.
- seen := make(map[string]bool, len(st.KB.Chunks))
- for _, c := range st.KB.Chunks {
- seen[runtime.ChunkIDOf(c)] = true
- }
- maxTotal := capacity
- if maxTotal <= 0 {
- maxTotal = MaxSnippetPool
- }
- room := maxTotal - len(seen)
- if room <= 0 {
- return 0
- }
-
- // The planner's output is already []string, but blank entries still have to be dropped.
- qs := make([]string, 0, len(queries))
- for _, q := range queries {
- if q = strings.TrimSpace(q); q != "" {
- qs = append(qs, q)
- }
- }
- if len(qs) == 0 {
- return 0
- }
- capPerQuery := topN
- if capPerQuery < 1 {
- capPerQuery = 1
- }
-
- // The fan-outs could run in parallel, but each one only RETRIEVES — kbinfos is mutated
- // once, back in the caller. This keeps that "retrieve, then mutate once" structure, hence
- // sequential: the per-request search cache and Kbinfos are shared mutable state that the
- // retrievals never touch concurrently.
- type fanoutPair struct {
- exact []map[string]any
- semantic []map[string]any
- metadata []map[string]any
- }
- // Channel C's conditions: ONE chat call for the whole batch, and only when the metadata
- // channel was asked for AND the dataset offers at least one filterable field. Without that
- // second gate a metadata-free dataset pays the extraction call and then runs one
- // guaranteed-empty scoped retrieval per sub-question, every round. A failure yields nil and
- // the channel is skipped.
- var filtersByQuery map[string][]map[string]any
- if useMetadata {
- if _, _, ok := fanoutMetadataVocabulary(deps); ok {
- filtersByQuery = ExtractFanoutFilters(ctx, deps, qs)
- } else {
- _LOG.Printf("[Prefetch] metadata channel skipped — the dataset offers no filterable metadata field")
- }
- }
- pairs := make([]fanoutPair, 0, len(qs))
- for _, q := range qs {
- select {
- case <-ctx.Done():
- return 0
- default:
- }
- exact, semantic, metadata := fanoutSearchQuery(ctx, sd, q, capPerQuery, filtersByQuery[q])
- pairs = append(pairs, fanoutPair{exact: exact, semantic: semantic, metadata: metadata})
- }
-
- added := 0
- rawAdded := 0
- admit := func(batch []map[string]any) bool {
- for _, c := range batch {
- if added >= room {
- return true
- }
- id := runtime.ChunkIDOf(c)
- isEvidence := strings.HasPrefix(id, "claim_")
- if id != "" {
- if seen[id] {
- continue
- }
- seen[id] = true
- }
- // Raw passages get their own budget;
- // anything left above it stays free for evidence rows, which are
- // far denser answer material. Evidence rows (claim_ prefix) bypass
- // the quota and never count against it.
- if !isEvidence && rawAdded >= rawSnippetQuota {
- continue
- }
- st.KB.Chunks = append(st.KB.Chunks, c)
- added++
- if !isEvidence {
- rawAdded++
- }
- }
- return added >= room
- }
- // Channel 0: claim/evidence rows lead the pool —
- // they are the compact, verbatim-bearing proxy for the chunks they source.
- // Gated on the dataset having compiled rows at all; capped at
- // evidencePoolQuota rows across the whole prefetch; best effort.
- channel0 := [][]map[string]any{}
- channel0Rows := 0
- for _, q := range qs {
- select {
- case <-ctx.Done():
- return 0
- default:
- }
- if !runtime.DatasetHasCompilation(ctx, sd) {
- break
- }
- // top_n=max(2, top_n) per query, so a single-fanout
- // question still recalls at least two claim rows.
- recallN := capPerQuery
- if recallN < 2 {
- recallN = 2
- }
- hits := runtime.RecallDatasetClaims(ctx, sd, q, recallN)
- if len(hits) == 0 {
- continue
- }
- pseudo := runtime.ClaimPseudoChunks(hits)
- kept := make([]map[string]any, 0, len(pseudo))
- for _, pc := range pseudo {
- if channel0Rows >= evidencePoolQuota {
- break
- }
- kept = append(kept, pc)
- channel0Rows++
- }
- if len(kept) == 0 {
- break
- }
- channel0 = append(channel0, kept)
- if admit(kept) {
- return added
- }
- }
- // Evidence top-up: an evidence row carries a
- // verbatim quote but not its surrounding passage, so pull exactly the chunks
- // it cites instead of running another global recall. Deduped against the
- // pool (a chunk the claim already quotes verbatim adds nothing new) and
- // capped at evidenceTopUp ids. Best effort.
- if len(channel0) > 0 {
- var wanted []string
- inWanted := map[string]bool{}
- for _, pseudo := range channel0 {
- for _, c := range pseudo {
- ids, _ := c["source_chunk_ids"].([]string)
- for _, cid := range ids {
- cid = strings.TrimSpace(cid)
- if cid == "" || seen[cid] || inWanted[cid] {
- continue
- }
- inWanted[cid] = true
- wanted = append(wanted, cid)
- }
- }
- }
- if len(wanted) > evidenceTopUp {
- wanted = wanted[:evidenceTopUp]
- }
- if len(wanted) > 0 {
- fetched := runtime.LoadChunksForIDs(ctx, sd, wanted)
- if len(fetched) > 0 && admit(fetched) {
- return added
- }
- }
- }
- // Channel A across every fan-out first, then channel B, then channel C.
- for _, p := range pairs {
- if admit(p.exact) {
- return added
- }
- }
- for _, p := range pairs {
- if admit(p.semantic) {
- return added
- }
- }
- for _, p := range pairs {
- if admit(p.metadata) {
- return added
- }
- }
- return added
-}
-
-// fanoutSearchQuery runs one fan-out's three channels. It only retrieves; the caller admits the
-// results. filters are the sub-question's metadata conditions (key/op/value over the session's
-// catalog); an empty list skips channel C.
-func fanoutSearchQuery(ctx context.Context, sd runtime.SearchDeps, fq string, capPerQuery int, filters []map[string]any) (exact, semantic, metadata []map[string]any) {
- terms := runtime.QueryToTerms(fq)
- keyed := runtime.FanoutKeyedTerms(terms)
- termList := keyed
- if len(termList) == 0 {
- termList = terms
- }
-
- // Channel A — exact: BM25Search is the dedicated keyword-only entry point (vector weight
- // 0).
- candidates, _ := runtime.BM25Search(ctx, sd, runtime.SearchParams{
- Question: fq,
- Keywords: strings.Join(termList, " "),
- TopN: fanoutBM25TopN,
- })
- if len(candidates) > 0 {
- res := runtime.NarrowByTerms(candidates, termList, nil, fq,
- runtime.NarrowContext{Before: 0, After: 1},
- fanoutNarrowMaxOutPerChunk, fanoutNarrowMaxOutTotal)
- exact = res.Kept
- if len(exact) > capPerQuery {
- exact = exact[:capPerQuery]
- }
- }
-
- // Channel B — semantic bypass: HybridSearch gives the vector leg weight 0.3 whenever an
- // embedder is configured. NO narrowing (a paraphrase-only hit has no query term to match, so
- // narrowing would erase it).
- hits, _ := runtime.HybridSearch(ctx, sd, runtime.SearchParams{
- Question: fq,
- TopN: fanoutHybridTopN,
- })
- exactIDs := make(map[string]bool, len(exact))
- for _, c := range exact {
- exactIDs[runtime.ChunkIDOf(c)] = true
- }
- for _, c := range hits {
- if exactIDs[runtime.ChunkIDOf(c)] {
- continue
- }
- semantic = append(semantic, c)
- if len(semantic) >= fanoutSemanticQuota {
- break
- }
- }
-
- // Channel C — metadata pre-filter: documents are pre-selected by the metadata conditions the
- // sub-question's wording supports (OR across them), and the retrieval runs INSIDE that
- // subset. The conditions name the session catalog's fields, and they are the SAME shape the
- // metadata_search tool takes, so a condition means the same thing on either path.
- // runtime.MetadataSearch is inert without a resolver, and the caller only spends the
- // extraction call when the dataset offers a filterable field (see fanoutMetadataVocabulary).
- if len(filters) > 0 {
- seenAB := make(map[string]bool, len(exact)+len(semantic))
- for _, c := range exact {
- seenAB[runtime.ChunkIDOf(c)] = true
- }
- for _, c := range semantic {
- seenAB[runtime.ChunkIDOf(c)] = true
- }
- hits, _ := runtime.MetadataSearch(ctx, sd, runtime.SearchParams{
- Question: fq,
- TopN: capPerQuery,
- }, filters, "or")
- for _, c := range hits {
- if seenAB[runtime.ChunkIDOf(c)] {
- continue
- }
- metadata = append(metadata, c)
- if len(metadata) >= fanoutMetadataQuota {
- break
- }
- }
- }
- return exact, semantic, metadata
-}
diff --git a/internal/rag/agentic-rag/graph_sca.go b/internal/rag/agentic-rag/graph_sca.go
deleted file mode 100644
index 1259f98875..0000000000
--- a/internal/rag/agentic-rag/graph_sca.go
+++ /dev/null
@@ -1,608 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package agentic_rag
-
-import (
- "context"
- "fmt"
- "log"
- "ragflow/internal/common"
- "sort"
- "strconv"
- "strings"
- "time"
-
- "ragflow/internal/rag/agentic-rag/runtime"
- "ragflow/internal/rag/agentic-rag/runtime/orchestrator"
-)
-
-// The SCA stage: the review view, its scores, the gaps it leaves for the rewrite, and
-// the feedback phrase the next round is handed.
-// SelectSCAView: rank the stored pool
-// down to the SCA review view (storage ≠ review).
-//
-// Score = retrieval relevance + surface-term coverage + freshness bonus for
-// chunks admitted in later rounds. Returns (view, identity) where identity is a
-// stable hash of the selected chunk ids — the caller uses it to detect an
-// unproductive round (same view twice despite new storage ⇒ nothing new).
-func SelectSCAView(chunks []map[string]any, focusTerms []string) ([]map[string]any, string) {
- terms := dedupe(focusTerms)
- lowered := make([]string, 0, len(terms))
- for _, t := range terms {
- if len(t) >= 3 {
- lowered = append(lowered, strings.ToLower(t))
- }
- }
- type scored struct {
- idx int
- chunk map[string]any
- score float64
- }
- ranked := make([]scored, 0, len(chunks))
- for i, c := range chunks {
- text := strings.ToLower(strings.Join([]string{
- anyString(c["content"]), anyString(c["content_with_weight"]),
- anyString(c["title"]), anyString(c["question_toks"]),
- }, " "))
- cov := 0
- for _, t := range lowered {
- if strings.Contains(text, t) {
- cov++
- }
- }
- covRatio := 0.5
- if len(lowered) > 0 {
- covRatio = float64(cov) / float64(len(lowered))
- }
- // The pool's relevance reading, from the one place that defines it
- // (see similarityOrScore).
- rel := similarityOrScore(c)
- fresh := min(float64(i)/20.0, 0.2) // late arrivals (gap-pursuit evidence) get seen
- ranked = append(ranked, scored{i, c, rel*0.45 + min(covRatio, 1.0)*0.45 + fresh})
- }
- sort.SliceStable(ranked, func(i, j int) bool { return ranked[i].score > ranked[j].score })
- // Evidence rows first: an atomic proposition carrying a
- // verbatim quote is what the reviewer should read before wading through raw
- // passages. Both groups keep their relevance order; only the grouping is
- // lifted before the view cap is applied, so claim evidence can no longer be
- // pushed out of the view by later, loosely-related raw chunks.
- var evidence, rest []map[string]any
- for _, r := range ranked {
- if strings.HasPrefix(runtime.ChunkIDOf(r.chunk), evidenceChunkPrefix) {
- evidence = append(evidence, r.chunk)
- } else {
- rest = append(rest, r.chunk)
- }
- }
- limit := min(len(ranked), SCAViewCap)
- view := make([]map[string]any, 0, limit)
- view = append(view, evidence...)
- view = append(view, rest...)
- view = view[:min(len(view), limit)]
- ids := make([]string, 0, limit)
- for _, c := range view {
- ids = append(ids, runtime.ChunkIDOf(c))
- }
- // identity is a hash of the SORTED id set, so this reordering cannot break
- // the unproductive-round detector.
- sort.Strings(ids)
- return view, joinHash(ids)
-}
-
-// joinHash builds a stable identity from the selected chunk ids: a deterministic digest
-// rather than a randomised-per-process hash.
-func joinHash(ids []string) string {
- return fmt.Sprintf("%x", strings.Join(ids, "|"))
-}
-
-// SCAGapsToRewrite
-//
-// Preference order:
-// 1. Unsatisfied sub_queries (the precise "what is missing / where to search
-// next" signal from Q-CARE) — (missing_fact, search_hint).
-// 2. Per-claim missing_information items — (what, search_hint).
-//
-// Returns [(what, search_hint), ...] (deduped, non-empty).
-func SCAGapsToRewrite(sca map[string]any) []orchestrator.MissingPiece {
- var gaps []orchestrator.MissingPiece
- seen := map[string]bool{}
- add := func(what, hint string) {
- what = strings.TrimSpace(what)
- hint = strings.TrimSpace(hint)
- if what == "" && hint == "" {
- return
- }
- key := what + "|" + hint
- if seen[key] {
- return
- }
- seen[key] = true
- if what == "" {
- what = hint
- }
- if hint == "" {
- hint = what
- }
- gaps = append(gaps, orchestrator.MissingPiece{What: what, SearchHint: hint})
- }
- if raw, ok := sca["sub_queries"].([]any); ok {
- for _, item := range raw {
- m, ok := item.(map[string]any)
- if !ok {
- continue
- }
- if b, ok := m["satisfied"].(bool); ok && b {
- continue
- }
- what := anyString(m["missing_fact"])
- if what == "" {
- what = anyString(m["sub_query"])
- }
- add(what, anyString(m["search_hint"]))
- }
- }
- if len(gaps) == 0 {
- // Fall back to per-claim missing_information.
- switch claims := sca["claims"].(type) {
- case map[string]orchestrator.ClaimVerdict:
- keys := make([]string, 0, len(claims))
- for k := range claims {
- keys = append(keys, k)
- }
- sort.Strings(keys)
- for _, k := range keys {
- for _, mi := range claims[k].MissingInformation {
- add(mi.What, mi.SearchHint)
- }
- }
- case map[string]any:
- for _, g := range claims {
- if gm, ok := g.(map[string]any); ok {
- for _, mi := range asSliceOfAny(gm["missing_information"]) {
- if mm, ok := mi.(map[string]any); ok {
- add(anyString(mm["what"]), anyString(mm["search_hint"]))
- } else {
- add(anyString(mi), "")
- }
- }
- }
- }
- }
- }
- if len(gaps) > MaxSCAGaps {
- gaps = gaps[:MaxSCAGaps]
- }
- return gaps
-}
-
-// Phase 1: fan-out expansion.
-
-// scaNode mirrors the `sca` node: the Phase-3 quality-control review.
-func scaNode(ctx context.Context, deps RAGTools, st *AgenticState, logger *log.Logger) {
- ctx, done := runtime.Phase(ctx, "sca")
- defer done()
-
- chunks := st.KB.Chunks
- view, viewID := SelectSCAView(chunks, ViewTerms(st))
- if st.SCAViewID != "" && viewID == st.SCAViewID {
- // Same evidence review twice despite new storage — further rounds cannot
- // change the verdict; stop instead of looping.
- logger.Printf("[SCA] review view UNCHANGED since last round (%d stored / %d viewed); closing out.", len(chunks), len(view))
- st.Verdict = VerdictInsufficient
- st.NoProgress = true
- st.SCA = map[string]any{}
- return
- }
-
- draftText := strings.TrimSpace(st.Draft)
- claims, grownView := buildSCAClaims(draftText, view, chunks, st.SlotEvidence)
- // Review against the GROWN view (sca appends to the same list).
- view = grownView
- // sca — the only case that skips the review outright is
- // "nothing retrieved AND nothing drafted". A non-empty view always reaches
- // the SCA (its default claim indexes the whole view, see buildSCAClaims).
- if draftText == "" && len(view) == 0 {
- logger.Printf("[SCA] nothing retrieved nor drafted; marking INSUFFICIENT to trigger a targeted re-search.")
- st.Verdict = VerdictInsufficient
- st.SCA = map[string]any{}
- st.SCAViewID = viewID
- return
- }
-
- // The SCA renders claim evidence from kbinfos BY INDEX, so review against a
- // view-only copy — the indexed ids then point at exactly the selected
- // chunks. The full pool is restored afterwards.
- orig := st.KB.Chunks
- st.KB.Chunks = view
- defer func() { st.KB.Chunks = orig }()
-
- review := func() orchestrator.SCAResult {
- t := nodeClock(SCATimeoutS, 15.0, st.RemainingS()-10.0)
- callCtx, cancel := context.WithTimeout(ctx, time.Duration(t*float64(time.Second)))
- defer cancel()
- return orchestrator.SufficientContextAgent(callCtx, orchestrator.SCADeps{
- KB: st.KB,
- Model: &jsonModelAdapter{inner: deps.Model, maxLength: deps.MaxLength},
- Prompts: deps.SCAPrompts,
- }, st.Question, claims)
- }
- res := review()
- if len(scaResultToMap(res)) == 0 && st.RemainingS() >= SCARetryHeadroomS {
- // One retry: a failed review is a fact about the moment, not about the question, and on a
- // count question this is the only stage that asks whether the list is complete. Spent only
- // while there is clock left for the answer beside it.
- logger.Printf("[SCA] review unavailable on the first attempt (%.0fs left); retrying once.", st.RemainingS())
- res = review()
- }
-
- st.SCAViewID = viewID
- st.SCA = scaResultToMap(res)
- if len(st.SCA) == 0 {
- // sca — an unavailable SCA (timeout, unparsable reply, no model) is NOT a
- // verdict: it is the absence of one. Marking it INSUFFICIENT used to be
- // the fallback, and it read as a judgement everywhere downstream — the
- // answer was labelled PARTIAL on no evidence (formalizeAnswerNode) and
- // the loop was told to research again by a review that never ran
- // (measured 2026-09-15: review timed out at 35s and the run closed out
- // with zero seconds left, so the "insufficiency" was never actionable
- // either). VerdictUnknown keeps the two apart; the loop's own record
- // (growth + gaps) decides whether another round is worth its budget, and
- // the deliverable is not dressed up as partly-unverified.
- logger.Printf("[SCA] unavailable (no review could be completed); recording UNKNOWN — this is not a sufficiency judgement, so the round's own record drives the loop.")
- st.SCA = map[string]any{}
- st.Verdict = VerdictUnknown
- // The fact goes where the answer can read it (see composedRecord). The partial label keeps
- // its evidence-based rule above: they are different claims.
- st.KB.NoteSufficiencyUnchecked()
- return
- }
- if res.IsSufficient {
- st.Verdict = VerdictSufficient
- } else {
- st.Verdict = VerdictInsufficient
- }
- step(ctx, logger, "SCA", "Evidence check: %s (confidence %.2f, reviewed %d of %s).",
- st.Verdict, res.Confidence, len(view), runtime.CountOf(len(chunks), "passage"))
-}
-
-// unresolvedClueGaps is the gap fallback: the first two question_clues of every unresolved
-// slot become (what, hint) gaps, so a rewrite can still be issued when the SCA produced no
-// structured gaps.
-func unresolvedClueGaps(st *AgenticState) []orchestrator.MissingPiece {
- var gaps []orchestrator.MissingPiece
- for _, us := range st.UnresolvedSlots {
- clues, ok := us["question_clues"].([]string)
- if !ok {
- continue
- }
- for i, qc := range clues {
- if i >= 2 {
- break
- }
- qc = strings.TrimSpace(qc)
- if qc == "" {
- continue
- }
- gaps = append(gaps, orchestrator.MissingPiece{What: qc, SearchHint: qc})
- }
- }
- return gaps
-}
-
-// buildSCAClaims mirrors the sca node's claim construction (lines 971-997): the
-// draft as claim "c0", plus one claim per slot carrying its evidence positions.
-//
-// It returns the (possibly grown) view alongside the claims: the slot-evidence chunks that
-// the view missed are appended to the SAME list before review, so the caller must review
-// against the returned slice, not the one it passed in.
-func buildSCAClaims(draftText string, view, chunks []map[string]any, slotEvidence map[string]SlotEvidence) ([]orchestrator.ClaimDraft, []map[string]any) {
- var claims []orchestrator.ClaimDraft
- if draftText != "" {
- claims = append(claims, orchestrator.ClaimDraft{ID: "c0", Draft: draftText})
- }
- viewIndexByID := map[string]int{}
- for i, c := range view {
- if id := runtime.ChunkIDOf(c); id != "" {
- viewIndexByID[id] = i
- }
- }
- sids := make([]string, 0, len(slotEvidence))
- for sid := range slotEvidence {
- sids = append(sids, sid)
- }
- sort.Strings(sids)
- for _, sid := range sids {
- meta := slotEvidence[sid]
- if len(meta.EvidenceIDs) == 0 {
- continue
- }
- // Resolve every evidence id to a POSITION in the view, appending chunks
- // that are missing so the SCA sees the passages that actually produced
- // the candidate (sca).
- positions := make([]string, 0, len(meta.EvidenceIDs))
- for _, eid := range meta.EvidenceIDs {
- pos := -1
- if p, ok := viewIndexByID[eid]; ok {
- pos = p
- } else if c := resolveEvidenceChunk(eid, chunks); c != nil {
- if p, ok := viewIndexByID[runtime.ChunkIDOf(c)]; ok {
- pos = p
- } else {
- view = append(view, c)
- pos = len(view) - 1
- if id := runtime.ChunkIDOf(c); id != "" {
- viewIndexByID[id] = pos
- }
- }
- }
- if pos >= 0 {
- positions = append(positions, strconv.Itoa(pos))
- }
- }
- draft := truncateRunes(meta.Candidate, draftCandidateChars)
- if draft == "" {
- draft = fmt.Sprintf("(slot %s evidence)", sid)
- }
- claims = append(claims, orchestrator.ClaimDraft{ID: sid, Draft: draft, EvidenceIDs: positions})
- }
- if len(claims) == 0 {
- // sca — an empty draft still gets ONE claim, carrying every
- // position in the view, so the SCA judges the evidence itself instead of
- // the node short-circuiting on "no draft".
- draft := draftText
- if strings.TrimSpace(draft) == "" {
- draft = "(no draft)"
- }
- claims = []orchestrator.ClaimDraft{{ID: "c0", Draft: draft, EvidenceIDs: allViewPositions(view)}}
- }
- return claims, view
-}
-
-// resolveEvidenceChunk maps one slot-evidence id back to a chunk. The ids are
-// heterogeneous by construction:
-//
-// - the retrieval tools emit the chunk id;
-// - the navigation tools emit document ids (tool_executor.go:455);
-// - the SCA view itself is keyed by POSITION (renderClaimContext /
-// sufficient_context.go:305-309), which the numeric branch below covers.
-//
-// slot_evidence holds chunk ids throughout, so both spellings are resolved here before
-// giving up.
-func resolveEvidenceChunk(eid string, chunks []map[string]any) map[string]any {
- eid = strings.TrimSpace(eid)
- if eid == "" {
- return nil
- }
- // Identity FIRST, position second: a chunk id that happens to be all digits (or a
- // doc id like "3") would otherwise be read as an index and resolve to a DIFFERENT
- // chunk — the ambiguity only disappears if exact ids win.
- for _, c := range chunks {
- if runtime.ChunkIDOf(c) == eid {
- return c
- }
- if id := runtime.DocIDOf(c); id != "" && id == eid {
- return c
- }
- }
- if n, err := strconv.Atoi(eid); err == nil && n >= 0 && n < len(chunks) {
- return chunks[n]
- }
- return nil
-}
-
-// allViewPositions is "every position in the view" (renderClaimContext keys evidence by
-// index).
-func allViewPositions(view []map[string]any) []string {
- positions := make([]string, 0, len(view))
- for i := range view {
- positions = append(positions, strconv.Itoa(i))
- }
- return positions
-}
-
-// scaResultToMap flattens an SCAResult back into the map shape the loop keeps in
-// AgenticState.SCA (SCAGapsToRewrite reads both shapes).
-func scaResultToMap(res orchestrator.SCAResult) map[string]any {
- if len(res.Claims) == 0 && res.Reasoning == "" && !res.IsSufficient {
- return nil
- }
- out := map[string]any{
- "is_sufficient": res.IsSufficient,
- "confidence": res.Confidence,
- "contradictions": toAnySlice(res.Contradictions),
- "reasoning": res.Reasoning,
- "claims": res.Claims,
- }
- if len(res.SubQueries) > 0 {
- sqs := make([]any, 0, len(res.SubQueries))
- for _, sq := range res.SubQueries {
- sqs = append(sqs, map[string]any{
- "sub_query": sq.SubQuery,
- "satisfied": sq.Satisfied,
- "missing_fact": sq.MissingFact,
- "search_hint": sq.SearchHint,
- })
- }
- out["sub_queries"] = sqs
- }
- return out
-}
-
-// verdictStatusHint is the human phrase folded into rag()'s "[Research status]" note for each
-// sufficiency status.
-//
-// Only INSUFFICIENT is reachable: the verdict dict carries a single "status" key whose value
-// is either "SUFFICIENT" or "INSUFFICIENT", and SUFFICIENT is skipped outright.
-// The upstream dict's USEFUL_BUT_INCOMPLETE and CONFLICTING entries are
-// therefore unreachable, so Go does not carry cases for them.
-func verdictStatusHint(verdict string) string {
- switch verdict {
- case VerdictInsufficient:
- return "evidence is not yet sufficient"
- case VerdictUnknown:
- // Not a judgement: the review did not complete, so nothing here may claim
- // the evidence was found wanting (nor that it was sufficient).
- return "the evidence review did not complete, so sufficiency is unverified"
- default:
- return "sufficiency status: " + verdict
- }
-}
-
-// scaFeedback is the body rag folds into the answer as the "[Research status]" note whenever
-// research stays unsatisfying.
-//
-// It composes as `status_hint + missing_txt + hard_txt + conf_txt + fb_txt`, but every term
-// after the first is empty in practice: `verdict` is `{"status": ...}` and carries no
-// missing_claims / hard_violations / agent_confidence / feedback keys anywhere, so the note is
-// the status hint alone.
-//
-// The SCA's rich payload is not lost — it stays in AgenticState.SCA and still drives
-// SCAGapsToRewrite (which reads the separate `sca` state key). Non-empty for
-// ANY non-SUFFICIENT verdict (not only after two consecutive misses), so the caller can
-// append the appropriate trailing sentence ("STOP" vs "call rag again").
-func scaFeedback(_ map[string]any, verdict string) string {
- if verdict == VerdictSufficient {
- return ""
- }
- return verdictStatusHint(verdict)
-}
-
-// renderResearchContext mirrors the query_rewrite node's context build
-// (lines 1047-1066): the attempted-query ledger with outcomes, plus the first
-// lines of the evidence pool.
-func renderResearchContext(st *AgenticState) string {
- var historyLines []string
- for _, e := range st.Attempted {
- if e == nil {
- continue
- }
- q := truncateRunes(anyString(e["q"]), 120)
- r := anyString(e["r"])
- if r == "" {
- r = "?"
- }
- outcome := "no new passages"
- switch {
- case e["bound"] != nil:
- // A research round's row carries what ITS session bound. There is no
- // per-session "new to the pool" number to report: a round's sessions run
- // concurrently over ONE pool (see RunSlotResearchPass).
- if n, ok := common.ToInt(e["bound"]); ok {
- outcome = fmt.Sprintf("%d passage(s) bound", n)
- }
- default:
- // A prefetch/rewrite row: those DO know how many passages the fetch added.
- if n, ok := common.ToInt(e["new"]); ok && n != 0 {
- outcome = fmt.Sprintf("%d new passage(s)", n)
- }
- }
- historyLines = append(historyLines, fmt.Sprintf("- %s (round %s: %s)", q, r, outcome))
- }
- // The passages behind the members already confirmed come FIRST, because they
- // are the only place the rewriter can see (a) the wording this text uses for
- // the relation and (b) the names that are still missing. Those passages are
- // exactly the windows the seats were narrowed to (see
- // Kbinfos.RecordReachedTerm), so a name inside one of them is readable.
- //
- // The pool-head lines below are the fallback for a round that has confirmed
- // nothing yet. They are FIRST LINES only, so a name in the middle of a chunk
- // is invisible through them — which is why they are the fallback and not the
- // main channel.
- var memberLines []string
- for i, m := range memberLinesOf(st) {
- if i >= MemberWindowMax {
- break
- }
- memberLines = append(memberLines, "- "+m)
- }
- var poolLines []string
- if len(memberLines) == 0 && st.KB != nil {
- for i, c := range st.KB.Chunks {
- if i >= PoolHeadLines {
- break
- }
- // Read `content` FIRST here — unlike the content_with_weight-first order used
- // elsewhere. The pool-head lines are rewriter prompt content, so this order is
- // deliberate.
- first := anyString(c["content"])
- if first == "" {
- first = anyString(c["content_with_weight"])
- }
- if idx := strings.IndexByte(first, '\n'); idx >= 0 {
- first = first[:idx]
- }
- if first = strings.TrimSpace(first); first != "" {
- poolLines = append(poolLines, "- "+truncateRunes(first, 140))
- }
- }
- }
- var parts []string
- if len(historyLines) > 0 {
- parts = append(parts, "Previously searched queries and their outcomes:\n"+strings.Join(historyLines, "\n"))
- }
- if len(memberLines) > 0 {
- parts = append(parts, "Passages that carry names the searches ALREADY confirmed (read them for the wording this text uses for the relation, and for other names they mention — any of those can be asked about directly):\n"+strings.Join(memberLines, "\n"))
- }
- if len(poolLines) > 0 {
- parts = append(parts, "Evidence currently at hand (first lines of top stored snippets):\n"+strings.Join(poolLines, "\n"))
- }
- // The round's own record of what its probes ASKED and never reached. The
- // rewriter reads it for one reason: re-asking a name the corpus already came
- // back empty on is the loop's most common waste, and the productive move from
- // a dead name is a different ANGLE (the act, the relationship, the place),
- // which the rewriter cannot choose unless it knows the name is dead.
- if absent := st.KB.ProbedAbsentTerms(); len(absent) > 0 {
- parts = append(parts, "Terms already asked for and NOT reached by any passage (do NOT re-ask these on their own; ask for the act / relationship / place instead):\n"+strings.Join(absent, "、"))
- }
- return strings.Join(parts, "\n\n")
-}
-
-// memberLinesOf renders one line per confirmed member: the name, and the window
-// that carries it, looked up in the pool by the id the seat recorded.
-//
-// It reads the round's own record rather than re-searching anything: a seat IS a
-// probe of one individual that came back with a passage, so the pair (name,
-// passage) is already established by the time the rewrite runs.
-func memberLinesOf(st *AgenticState) []string {
- if st == nil || st.KB == nil {
- return nil
- }
- byID := make(map[string]map[string]any, len(st.KB.Chunks))
- for _, c := range st.KB.Chunks {
- if id := runtime.ChunkIDOf(c); id != "" {
- byID[id] = c
- }
- }
- var out []string
- seen := map[string]bool{}
- for _, m := range st.KB.ReachedTerms() {
- if seen[m.ChunkID] {
- continue
- }
- c, ok := byID[m.ChunkID]
- if !ok {
- continue
- }
- text := strings.TrimSpace(runtime.ChunkTextOf(c))
- if text == "" {
- continue
- }
- seen[m.ChunkID] = true
- out = append(out, fmt.Sprintf("%s: %s", m.Term, truncateRunes(text, MemberWindowChars)))
- }
- return out
-}
diff --git a/internal/rag/agentic-rag/graph_slots.go b/internal/rag/agentic-rag/graph_slots.go
deleted file mode 100644
index 43c4b6682c..0000000000
--- a/internal/rag/agentic-rag/graph_slots.go
+++ /dev/null
@@ -1,1241 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package agentic_rag
-
-import (
- "context"
- "fmt"
- "sort"
- "strconv"
- "strings"
- "sync"
- "unicode/utf8"
-
- "github.com/cloudwego/eino/schema"
- "ragflow/internal/rag/agentic-rag/runtime"
- "ragflow/internal/rag/agentic-rag/slots"
-)
-
-// The slot-research stage: the table a round is driven by, the members and counts it
-// holds, the passes that fill it, and the write-back folded into it.
-// RenderSlotDraft
-// render the slot table into a fact-preserving draft for the SCA.
-//
-// A resolved slot carries the model's own candidate strength, the evidence ids /
-// terminal type of the passages that produced it (from slotEvidence), and the
-// tail of its discovered clues, so the SCA can verify the candidate against the
-// passages that actually produced it. Unresolved slots are listed explicitly
-// with their question clues so the SCA can call them out as gaps. A collected
-// answer leads the draft — it is the strongest candidate — with the same
-// evidence metadata under the "_answer" key.
-func RenderSlotDraft(slotTable runtime.State, collectedAnswer string, slotEvidence map[string]SlotEvidence) string {
- var lines []string
- if collectedAnswer != "" {
- // The collected answer leads the draft — it is the strongest candidate — and
- // carries the same evidence metadata, read from the "_answer" key.
- lines = append(lines, "Candidate answer: "+collectedAnswer+draftEvidenceSuffix(slotEvidence["_answer"]))
- lines = append(lines, "")
- }
- if len(slotTable.State) == 0 {
- return strings.Join(lines, "\n")
- }
- for _, v := range slotTable.State {
- vtype := v.Type
- if vtype == "" {
- vtype = "entity"
- }
- cand := ""
- if v.Candidate != nil {
- cand = *v.Candidate
- }
- if cand == "" {
- clues := v.QuestionClues
- if len(clues) > 2 {
- clues = clues[:2]
- }
- lines = append(lines, fmt.Sprintf("- slot %d [%s]: NOT RESOLVED (%s)",
- v.ID, vtype, strings.Join(truncateEach(clues, draftUnresolvedClueChars), "; ")))
- continue
- }
- strength := "?"
- if v.CandidateStrength != nil {
- strength = fmt.Sprintf("%.2f", *v.CandidateStrength)
- }
- line := fmt.Sprintf("- slot %d [%s]: %s (strength=%s)%s",
- v.ID, vtype, cand, strength, draftEvidenceSuffix(slotEvidence[fmt.Sprint(v.ID)]))
- if tail := draftClueTail(v.DiscoveredClues); tail != "" {
- line += " — " + tail
- }
- lines = append(lines, line)
- }
- return strings.Join(lines, "\n")
-}
-
-// RenderSlotRecord renders the slot table for the ANSWER prompt.
-//
-// It is deliberately NOT RenderSlotDraft. The draft exists for the SCA, which
-// verifies a candidate against the passages that produced it, so it carries the
-// machine fields that make that verification possible — the candidate strength,
-// the terminal type, the evidence ids. Handing those to the answer model is a
-// different act with a different failure mode: composing with the draft as
-// "Research Summary (primary evidence)" produces an answer that quotes the
-// bookkeeping verbatim, machine fields and all.
-//
-// So the answer sees the FACTS the research settled — which slot holds what —
-// with no strength, no evidence ids, no clue tails: the evidence ids are already
-// in the evidence block with their citation markers, and the strengths are the
-// runtime's business. The prompt labels the block as the model's own record and
-// tells it not to copy the lines (see answerPromptWithEvidence).
-func RenderSlotRecord(slotTable runtime.State, collectedAnswer string) string {
- var lines []string
- // Is this record about a SET at all? Everything this function does beyond the
- // slots themselves — the enumerated size, the count/members warning, the
- // demotion of the session's own prose — is about reconciling a list with a
- // count, and a single-value question has neither.
- //
- // Rendered ungated, the enumerated line and the prose demotion appear on questions
- // whose answer is one date or one number, where a "count" computed from a single
- // value's text is meaningless: one name cut at its separators reads as several
- // members, and the session's own draft answer is then demoted below a line that does
- // not apply to it.
- // The record's SET handling follows the PLANNER'S DECLARATION — count / set / list,
- // the shape half of runtime.Coverage — exactly as it did before the enumeration was
- // refactored: a table the planner typed as a set states its size and demotes a
- // session's prose, while a table whose slots hold values (one name, one date, one
- // number, one phrase) does neither. The size line additionally needs at least one
- // member to be worth printing, and a member can only come from a slot that HOLDS
- // members (see runtime.MemberNames): the waterfall whose name was cut at its
- // separators is a TEXT slot, contributes no members, and is never counted.
- enumerating := runtime.CoverageOf(slotTable).Set
- if !enumerating && collectedAnswer != "" {
- // A value record leads with the session's own answer, exactly as it did
- // before any of this existed (see the note on the demotion below for why the
- // SET case is different).
- lines = append(lines, "Candidate answer: "+collectedAnswer)
- lines = append(lines, "")
- }
- for _, v := range slotTable.State {
- vtype := v.Type
- if vtype == "" {
- vtype = "entity"
- }
- switch {
- case memberLine(v) != "":
- lines = append(lines, fmt.Sprintf("- slot %d [%s]: %s", v.ID, vtype, memberLine(v)))
- case v.Candidate == nil || *v.Candidate == "":
- lines = append(lines, fmt.Sprintf("- slot %d [%s]: NOT RESOLVED", v.ID, vtype))
- default:
- lines = append(lines, fmt.Sprintf("- slot %d [%s]: %s", v.ID, vtype, *v.Candidate))
- }
- // Claims that lost the slot comparison (see MergeSlotPatch). They are not
- // the slot's value, but they are not nothing either: a session produced
- // them from the same corpus, and a name only in an alternate is a name the
- // answer has to account for.
- for _, alt := range alternateCandidatesOf(v) {
- lines = append(lines, " alternate (claimed by another session, not adopted): "+truncateRunes(alt, 400))
- }
- }
- // The size of the set the slots above ENUMERATE — a fact the answer can take a
- // number from without trusting anybody's prose.
- //
- // A count slot is written by whichever session last touched it, and that
- // candidate can be a sentence ("约 14 人(华雄、程远志…)"), a stale number, or
- // one session's claim written before the other sessions' findings were merged.
- // The members are here in the table either way, so their union is the number
- // the record can stand behind.
- if n := enumeratedSize(slotTable); enumerating && n > 0 {
- lines = append(lines, fmt.Sprintf("- enumerated members across the slots above: %d", n))
- // A set answer is only as good as what it can point at, member by member. Handed a
- // list of names and no per-member evidence, an answer counts members it cannot cite
- // and cites one range for all of them. The citation contract already forbids ranges;
- // this states it where a set answer is assembled, together with the rule that keeps a
- // count honest — a member nobody can point at a passage is not counted.
- lines = append(lines, "State the count and the members TOGETHER: every member you list carries the words behind it (quoted above, or its own [ID:n]) — never one range for the list — and a member you cannot point at a passage for is left out of both the list and the count.")
- // The names the number above leaves out are listed, so the answer can tell a complete list
- // from one that is short by a claim nobody could quote.
- if claims := runtime.UnanchoredItems(&slotTable); len(claims) > 0 {
- lines = append(lines, fmt.Sprintf(
- "- claimed WITHOUT a passage in hand (NOT in the %d above — the count may be short by up to %d): %s. Each one is a name somebody asserted and no passage in hand states: find the words that put it in the answer, or leave it out of both the list and the count.",
- n, len(claims), strings.Join(claims, "、")))
- }
- // A count larger than the members it counts is a claim about members that are NOT in
- // the record, and the answer has to be told that rather than left to reconcile it:
- // left alone it explains the gap as "more members whose details the material does not
- // list" — members that never existed.
- //
- // The note STATES the disagreement; it does not order which number to take. An order
- // was tried and reverted: told to take the number from the enumerated members, the
- // answer took it even when that number had been computed over the wrong slots. A count
- // can be an over-claim and a list can be incomplete; only the passages decide between
- // them, and the answer is the stage that reads them.
- for _, v := range slotTable.State {
- // A DECLARED number (slots.KindCount / KindRange) compared with the members
- // above. Text claims no number, so prose can no longer masquerade as a
- // count here (or as a member list on the other side of the comparison).
- if claimed, ok := v.Typed().Number(); ok {
- if claimed == n {
- continue
- }
- lines = append(lines, fmt.Sprintf(
- "- NOTE: slot %d [%s] says %d while the slots above enumerate %d — the count and the members listed disagree. Reconcile them against the evidence before answering: a count larger than the members that are listed is not evidence of members, and a list is only as complete as the passages behind it.",
- v.ID, v.Type, claimed, n))
- continue
- }
- // The other half of the same disagreement: a count slot holding WORDS. It claims no
- // number by contract (KindText is opaque and nothing derives from it), so it used to
- // pass in silence — the record showed its prose beside the enumerated size and let the
- // answer take either, which is how 14 became an answer to a table that enumerated 16.
- // The words are not parsed; the disagreement is stated.
- if isCountSlot(v) && v.Typed().Kind != slots.KindItems && strings.TrimSpace(*v.Candidate) != strconv.Itoa(n) {
- lines = append(lines, fmt.Sprintf(
- "- NOTE: slot %d [%s] holds %q as text, not a number, so nothing was derived from it while the slots above enumerate %d. State the count the evidence supports.",
- v.ID, v.Type, truncateRunes(strings.TrimSpace(*v.Candidate), 60), n))
- }
- }
- }
- // A session's own draft answer goes LAST and is labelled for what it is.
- //
- // It used to lead the record, and the answer copied it: whatever number a session wrote
- // in its own prose became the number the answer reported, even when the slots above
- // enumerated a different one. The prose is one session's recollection, written before the
- // other sessions' findings were merged into the table above; it is a claim to reconcile
- // with the members, not the record.
- //
- // On a VALUE record there are no members to reconcile against — the prose is the
- // only candidate answer the record has, so it leads the record instead (see the
- // top of this function), which is also what every run before the demotion scored
- // on.
- if collectedAnswer != "" && enumerating {
- lines = append(lines, "")
- // The draft is not a member LIST — copying its prose is how a fifteen-member answer
- // came out of a seventeen-member record — but the PASSAGES it quotes are evidence
- // like any other, and a name those passages attribute to the actor is a member even
- // even when no slot above lists it. A record can enumerate far fewer members than its
- // own draft quotes the text for, and an answer told that "the members stand" then drops
- // every one of them. The evidence was in hand; the rule threw it away. So the draft is
- // demoted as a SOURCE
- // of members and promoted as evidence: its quotations are the arbiter, and neither
- // the slots nor the draft decides on its own.
- lines = append(lines, "One session's own draft answer (UNVERIFIED — written before the other sessions were merged. Its PROSE is not a member list: do not copy its count or its wording. Its QUOTATIONS are evidence like any other: a name those passages attribute to the actor is a member even when no slot above lists it, and a name whose passage attributes the deed to someone else is not. Reconcile the draft with the slots — with the quotations as the arbiter, not either list — and include every member the evidence supports): "+collectedAnswer)
- }
- return strings.Join(lines, "\n")
-}
-
-// enumeratedSize is how many distinct items the table can point at (runtime.AnchoredItems): one
-// source spells one item several ways, and a set counts entities, not spellings.
-func enumeratedSize(table runtime.State) int { return len(runtime.AnchoredItems(&table)) }
-
-// BuildSlotTable: decompose the question into
-// a slot table, seeding it with the planner's fan-outs.
-//
-// Returns (root, firstQueries) — never an empty root: on failure it degrades to
-// one "aspect" slot per fan-out (or a single "answer" slot for the raw
-// question), so the research pass always has something to work on.
-func BuildSlotTable(ctx context.Context, deps runtime.SessionDeps, question string, fanouts []string, deadlineLeft float64) (runtime.State, []string) {
- root, firstQueries, err := buildSlotTableFrom(ctx, deps, question, fanouts, deadlineLeft)
- if err != nil {
- _LOG.Printf("[SlotTable] initialize_state failed; building from fanouts: %v", err)
- // _build_slot_table — the exception path keeps the FULL fan-out list as
- // first_queries; the [:3] cap below belongs to the empty-root path only.
- if len(fanouts) > 0 {
- firstQueries = fanouts
- } else {
- firstQueries = []string{question}
- }
- }
- if len(root.State) == 0 {
- queries := fanouts
- if len(queries) == 0 {
- queries = []string{question}
- }
- vars := make([]runtime.Variable, 0, 4)
- for i, q := range queries {
- if i >= 4 {
- break
- }
- vars = append(vars, runtime.Variable{
- ID: i,
- Type: "aspect",
- QuestionClues: []string{truncateRunes(q, slotFallbackClueChars)},
- })
- }
- root = runtime.NewState(vars, 0, nil)
- if len(firstQueries) == 0 {
- firstQueries = queries
- if len(firstQueries) > 3 {
- firstQueries = firstQueries[:3]
- }
- }
- }
- _LOG.Printf("[SlotTable] built %d slot(s): %s", len(root.State), root.Brief())
- if len(firstQueries) == 0 {
- firstQueries = []string{question}
- }
- return root, firstQueries
-}
-
-// BatchFillSlots: answer several
-// unresolved slots in ONE call when their evidence overlaps.
-//
-// Pure efficiency: neither the slot structure nor the evidence semantics
-// change — only the number of generation calls made over the same passages.
-//
-// Best-effort inside: a missing model, a failed call, or an unparsable reply skips that
-// cluster and the remaining clusters still run.
-func BatchFillSlots(ctx context.Context, deps runtime.SessionDeps, slotTable *runtime.State, slotEvidence map[string]SlotEvidence) int {
- // Unresolved slots only, with slot_by_id over them.
- unresolvedN := 0
- slotByID := map[int]*runtime.Variable{}
- evIDs := map[int][]string{} // ordered evidence ids, first-seen order
- evSet := map[int]map[string]bool{}
- for i := range slotTable.State {
- v := &slotTable.State[i]
- if v.Filled() {
- continue
- }
- unresolvedN++
- slotByID[v.ID] = v
- // evidence ids recorded for this slot by its session,
- // keyed by str(slot id), blanks dropped.
- ids := map[string]bool{}
- var ordered []string
- for _, id := range slotEvidence[fmt.Sprint(v.ID)].EvidenceIDs {
- if id == "" || ids[id] {
- continue
- }
- ids[id] = true
- ordered = append(ordered, id)
- }
- if len(ordered) > 0 {
- evIDs[v.ID] = ordered
- evSet[v.ID] = ids
- }
- }
- if len(evSet) < 2 {
- // Diagnostics: batching needs TWO slots that are
- // both unresolved AND carrying evidence. Log why it did not happen,
- // otherwise a never-firing path is indistinguishable from a working one.
- _LOG.Printf("[SlotResearch] batching skipped: unresolved=%d with_evidence=%d", unresolvedN, len(evSet))
- return 0
- }
-
- ids := make([]int, 0, len(evSet))
- for id := range evSet {
- ids = append(ids, id)
- }
- // The id set has no order of its own, so sort first: the clustering below is
- // deterministic across runs only because this order is fixed.
- sort.Ints(ids)
-
- // sim: Jaccard over evidence-id sets.
- sim := func(a, b int) float64 {
- inter := intersectionSize(evSet[a], evSet[b])
- union := len(evSet[a]) + len(evSet[b]) - inter
- if union == 0 {
- return 0.0
- }
- return float64(inter) / float64(union)
- }
-
- // Largest-incompatible-first (APT-RAG,): place the least
- // compatible slot first, so it is not left without a cluster at the end.
- // Counts are precomputed: the sort key is fixed before sorting.
- incompatible := make(map[int]int, len(ids))
- for _, i := range ids {
- n := 0
- for _, j := range ids {
- if j != i && sim(i, j) < EvidenceBatchMinSim {
- n++
- }
- }
- incompatible[i] = n
- }
- order := append([]int(nil), ids...)
- sort.SliceStable(order, func(x, y int) bool { return incompatible[order[x]] > incompatible[order[y]] })
-
- // Greedy clustering(): a slot joins the first cluster
- // that is under the size cap AND shares ≥MIN_SHARED ids AND ≥MIN_SIM with
- // EVERY member; otherwise it starts its own cluster.
- clusters := [][]int{}
- for _, i := range order {
- placed := false
- for ci := range clusters {
- cl := clusters[ci]
- if len(cl) >= EvidenceBatchMaxSlots {
- continue
- }
- compatible := true
- for _, j := range cl {
- if intersectionSize(evSet[i], evSet[j]) < EvidenceBatchMinShared || sim(i, j) < EvidenceBatchMinSim {
- compatible = false
- break
- }
- }
- if compatible {
- clusters[ci] = append(cl, i)
- placed = true
- break
- }
- }
- if !placed {
- clusters = append(clusters, []int{i})
- }
- }
-
- // by_chunk_id()over the whole pool.
- byChunkID := map[string]map[string]any{}
- if deps.KB != nil {
- for _, c := range deps.KB.Chunks {
- if cid := runtime.ChunkIDOf(c); cid != "" {
- byChunkID[cid] = c
- }
- }
- }
-
- filled := 0
- for _, cl := range clusters {
- if len(cl) < 2 {
- continue
- }
- // -1645 builds union_ids as a set (arbitrary order); the
- // port keeps first-seen order across the cluster for determinism.
- unionIDs := []string{}
- seenID := map[string]bool{}
- for _, i := range cl {
- for _, id := range evIDs[i] {
- if !seenID[id] {
- seenID[id] = true
- unionIDs = append(unionIDs, id)
- }
- }
- }
- body, total := []string{}, 0
- for _, cid := range unionIDs {
- c := byChunkID[cid]
- if c == nil {
- continue
- }
- // content_with_weight first, content fallback.
- text := anyString(c["content_with_weight"])
- if text == "" {
- text = anyString(c["content"])
- }
- text = strings.TrimSpace(text)
- if text == "" {
- continue
- }
- // The cap counts RUNES, not bytes: a clue that is CJK would otherwise spend
- // three bytes of the budget per character.
- n := utf8.RuneCountInString(text)
- if total+n > EvidenceBatchMaxChars {
- break
- }
- body = append(body, text)
- total += n
- }
- if len(body) == 0 {
- continue
- }
-
- lines := []string{}
- for _, sid := range cl {
- v := slotByID[sid]
- if v == nil {
- continue
- }
- // join ALL clues, then cut the JOINED text
- // to 300 code points.
- clues := truncateRunes(strings.Join(v.QuestionClues, "; "), 300)
- lines = append(lines, fmt.Sprintf("- %d: %s", sid, clues))
- }
- if len(lines) < 2 {
- continue
- }
-
- // .
- user := "Sub-questions:\n" + strings.Join(lines, "\n") + "\n\nShared evidence:\n" + strings.Join(body, "\n---\n")
- // no model: stop batching entirely.
- if deps.Model == nil {
- return filled
- }
- // async_chat(system_prompt, [user], answer_conf);
- // the Go SessionModel takes the system message inline.
- reply, err := deps.Model.Complete(ctx, []schema.Message{
- *schema.SystemMessage(EvidenceBatchPrompt),
- *schema.UserMessage(user),
- }, nil)
- if err != nil {
- // one failed call must not cost the other
- // clusters.
- _LOG.Printf("[SlotResearch] batched answer call failed: %v", err)
- continue
- }
-
- // _extract_json_object; a non-object reply
- // skips the cluster.
- data, ok := extractJSONObject(reply.Content).(map[string]any)
- if !ok {
- continue
- }
- for _, sid := range cl {
- v := slotByID[sid]
- if v == nil {
- continue
- }
- // the reply maps str(slot id) to its answer;
- // never overwrite an already-filled slot, and a blank answer
- // counts as "evidence does not answer this".
- val, isStr := data[fmt.Sprint(sid)].(string)
- if !isStr {
- continue
- }
- val = strings.TrimSpace(val)
- if val == "" || v.Filled() {
- continue
- }
- cand := truncateRunes(val, 400)
- v.Candidate = &cand
- filled++
- }
- }
- return filled
-}
-
-// PrefillSlotsFromEvidence
-// answer slots that an already-pooled evidence row directly answers.
-//
-// An evidence row is an atomic proposition carrying a verbatim quote, so when
-// it already covers a slot's question there is nothing for an action session
-// to research — skipping it removes a WHOLE session (the dominant cost), not
-// just tokens inside one. Saves calls, uses real evidence, and a wrong guess
-// is still caught later by the SCA.
-func PrefillSlotsFromEvidence(slotTable *runtime.State, kb *runtime.Kbinfos) int {
- // evidence rows are the claim pseudo-chunks.
- evRows := []map[string]any{}
- if kb != nil {
- for _, c := range kb.Chunks {
- if strings.HasPrefix(runtime.ChunkIDOf(c), evidenceChunkPrefix) {
- evRows = append(evRows, c)
- }
- }
- }
- if len(evRows) == 0 {
- return 0
- }
-
- filled := 0
- for i := range slotTable.State {
- v := &slotTable.State[i]
- if v.Filled() {
- continue
- }
- clues := []string{}
- for _, c := range v.QuestionClues {
- if strings.TrimSpace(c) != "" {
- clues = append(clues, c)
- }
- }
- if len(clues) == 0 {
- continue
- }
- // lowercase terms of ≥3 code points over all clues.
- terms := map[string]bool{}
- for _, c := range clues {
- for _, t := range runtime.QueryToTerms(c) {
- if utf8.RuneCountInString(t) >= 3 {
- terms[strings.ToLower(t)] = true
- }
- }
- }
- if len(terms) == 0 {
- continue
- }
-
- best, bestCov := -1, 0.0
- for j, e := range evRows {
- // content_with_weight ONLY (not content).
- text := strings.ToLower(anyString(e["content_with_weight"]))
- if text == "" {
- continue
- }
- cov := 0
- for t := range terms {
- if strings.Contains(text, t) {
- cov++
- }
- }
- covF := float64(cov) / float64(len(terms))
- // strictly greater, so ties keep the FIRST row.
- if covF > bestCov {
- best, bestCov = j, covF
- }
- }
- if best < 0 || bestCov < EvidencePrefillCoverage {
- continue
- }
-
- // The row renders as
- // "[evidence] <name> — <desc>\nEvidence (verbatim): ...".
- //
- // Format provenance (TWO deliberate claim formats): the fan-out channel-0 rows
- // this prefill consumes carry the "[evidence] " prefix (see
- // runtime.ClaimPseudoChunks). The OTHER producers — the action-session claim
- // prefetch and the navigate_structure publish — render "[claim #N] <name>", and
- // prefill never sees "[evidence]" on those either: the replace below simply
- // leaves their marker intact. Both producers are matched one-to-one.
- head := anyString(evRows[best]["content_with_weight"])
- if idx := strings.IndexByte(head, '\n'); idx >= 0 {
- head = head[:idx]
- }
- name := strings.TrimSpace(strings.Trim(strings.ReplaceAll(head, "[evidence]", ""), " —-"))
- if name == "" {
- continue
- }
- // candidate = name[:400], strength = coverage.
- cand := truncateRunes(name, 400)
- v.Candidate = &cand
- strength := bestCov
- v.CandidateStrength = &strength
- filled++
- }
- return filled
-}
-
-// logCoverageWindows writes the enumeration's windows to the run log — the QUOTES, not just their
-// count.
-//
-// Whether a member the answer missed was ever IN FRONT of a session is otherwise unknowable, and
-// that difference decides which fault to fix: a name quoted inside a window the sessions read and
-// did not write is a WRITE-BACK fault (the record, the last node), while a name the enumeration
-// never showed is a COVERAGE fault (the operand recall, the window budget). Without the quotes a
-// log cannot say whether a missing member was ever shown at all, and the same code answers very
-// different member counts on one question, so every window change made from such a log is a
-// guess, and two of them were wrong.
-func logCoverageWindows(set runtime.CoverageSet) {
- for _, w := range set.Windows {
- _LOG.Printf("[Coverage] window chunk_id=%s act=%s %q", w.ChunkID, w.Act, truncateRunes(w.Quote, 120))
- }
-}
-
-// RunSlotResearchPass: drive ONE
-// research round with slot-aware action sessions.
-//
-// Unresolved slots are worked concurrently under a semaphore; each session's
-// branches are folded back into the shared table. A nil result means "nothing to
-// do" (all slots already filled).
-func RunSlotResearchPass(ctx context.Context, parent context.Context, deps runtime.SessionDeps, question string, st *AgenticState, deadlineLeft float64) *SlotResearchResult {
- slotTable := st.SlotTable
- if len(slotTable.State) == 0 {
- // No planner ran (medium single-pass, or the planner failed): build the
- // table from the raw question so the research still executes.
- root, _ := BuildSlotTable(ctx, deps, question, nil, max(15.0, deadlineLeft-10.0))
- slotTable = root
- }
- if question == "" {
- question = st.Question
- }
- unresolved := slotTable.Unresolved()
-
- // Evidence-row prefill(): slots the pooled evidence
- // already answers cost no action session at all — this is where whole
- // calls get removed. It is pure (no model, no I/O) and cannot fail.
- kb := deps.KB
- if kb == nil {
- // `tools.kbinfos or state.kbinfos`.
- kb = st.KB
- }
- prefillN := PrefillSlotsFromEvidence(&slotTable, kb)
- if prefillN > 0 {
- _LOG.Printf("[SlotResearch] evidence prefill answered %d slot(s) with no session", prefillN)
- unresolved = slotTable.Unresolved()
- }
-
- // The directions this round works: one per unresolved slot, and — when the
- // table is already filled — the gaps the REVIEW named.
- //
- // A filled table is not a finished round for a question whose answer slot is
- // DERIVED: a count computed from the members it found is filled by
- // construction, however few members that is, so `unresolved == 0` silently
- // cancelled rounds the routing had started precisely because the review named gaps: a
- // rewrite round prefetches its evidence, logs "all slots filled; no session to run", and
- // spends its budget with a draft byte-identical to the previous round's — the passages it
- // had just admitted were never read into the record, which is the step a session exists
- // for.
- type direction struct {
- slotID int
- text string
- }
- var dirs []direction
- for _, v := range unresolved {
- text := question
- if len(v.QuestionClues) > 0 {
- text = v.QuestionClues[0]
- }
- dirs = append(dirs, direction{slotID: v.ID, text: text})
- }
- if len(dirs) == 0 {
- for _, g := range SCAGapsToRewrite(st.SCA) {
- text := strings.TrimSpace(g.SearchHint)
- if text == "" {
- text = strings.TrimSpace(g.What)
- }
- if text == "" {
- continue
- }
- // slotID -1: a gap is not a slot, and whatever the session patches is
- // applied by id in the fold, so nothing is attributed to a slot the
- // review never named.
- dirs = append(dirs, direction{slotID: -1, text: text})
- }
- if len(dirs) == 0 {
- // Nothing for a session to do. A PREFILL still counts as work — it
- // edited the table — so its result must travel back to the caller;
- // only a round that changed nothing at all is a nil pass.
- if prefillN == 0 {
- _LOG.Printf("[SlotResearch] all slots filled and the review named no gap; no session to run.")
- return nil
- }
- } else {
- _LOG.Printf("[SlotResearch] all slots filled, but the review named %d gap(s); running session(s) on them.", len(dirs))
- }
- }
-
- // Shared across sessions so duplicate retrievals are served from cache.
- sharedToolCache := runtime.NewToolCache()
- var sharedSearchQueries []string
-
- sem := make(chan struct{}, slotSessionConcurrency)
- var wg sync.WaitGroup
- var mu sync.Mutex
-
- type outcome struct {
- slotID int
- // direction travels with the result so the round's ledger can say what this
- // session was ASKED to research without re-deriving it from its messages (see
- // the ledger row below).
- direction string
- result runtime.Result
- }
- results := make([]outcome, 0, slotSessionsPerRound)
- limit := min(len(dirs), slotSessionsPerRound)
-
- // The direction's SHAPE is only known here, after the table was built — and the
- // round's own context was fixed before that, by a caller that could not know it.
- // A session therefore cannot ride this round's clock, or the enumeration it is
- // midway through is cut before it can patch: the pass timeout can cancel a session at the
- // deadline just before its patch, and the member it had already reached — its passages
- // admitted to the shared pool by the session's own batch — dies with it. So an
- // enumeration pass
- // hangs its sessions off the PARENT context with the shape's own clock
- // (runtime.SessionWallS), and buys the question the one-shot budget extension
- // that lets the NEXT round start and pick up whatever this one could not record.
- sessionCtx := ctx
- sessionBudget := max(20.0, deadlineLeft-10.0)
- if runtime.CoverageOf(slotTable).Ok() {
- sessionBudget = max(sessionBudget, runtime.SessionWallS(slotTable))
- if parent != nil {
- var cancelSessions context.CancelFunc
- sessionCtx, cancelSessions = context.WithTimeout(parent, deadlineToDuration(sessionBudget+setSessionSlackS))
- defer cancelSessions()
- }
- // The extension must leave room for what FOLLOWS the research — the review,
- // the draft, the composed answer (downstreamReserveS) — inside the caller's own
- // deadline: buying time the answer then lacks is the one way this change could
- // make a question worse (a timed-out request instead of a missing member). So
- // the request's remaining room decides, not the extension's own size, and an
- // extension too small to let a round start is not worth buying at all.
- ext := SetBudgetExtensionS
- if room, bounded := ctxRoomS(parent); bounded {
- ext = min(ext, room-st.RemainingS()-downstreamReserveS)
- }
- if ext >= minBudgetExtensionS {
- if st.ExtendDeadline(ext) {
- _LOG.Printf("[SlotResearch] enumeration table: session clock %.0fs, research budget extended by %.0fs so a following round can pick up what this one could not record.",
- sessionBudget, ext)
- }
- } else {
- _LOG.Printf("[SlotResearch] enumeration table: session clock %.0fs; research budget NOT extended (only %.0fs of the request is left, and %.0fs is reserved for the review, the draft and the answer).",
- sessionBudget, ctxLeftS(parent), downstreamReserveS)
- }
- }
-
- // The ENUMERATION runs HERE — in code, before any session starts.
- //
- // The direction's own act words used to be rendered into the seed as a list of queries to
- // make, and that is not enough: a list of queries in a prompt is advice, and advice may
- // simply not be taken — the sessions improvise their own word lists instead and re-probe
- // the same names by hand in the next round. The completeness of an enumeration cannot
- // rest on advice. So the runtime asks the corpus ITSELF — one recall per operand, the
- // windows where the deed is stated (runtime.EnumerateCoverage) — admits them to the pool,
- // and seeds the sessions with what came back: the session's job becomes reading evidence
- // rather than guessing names.
- //
- // Run once per QUESTION, not once per round: the windows stay in the pool under the same
- // ids and the set is kept (Kbinfos.CoverageSet), so the last node resolves the same
- // windows a later round would have re-found.
- cov := runtime.CoverageOf(slotTable)
- switch {
- case !cov.Ok():
- // A table of counts and dates declares act words too (the planner is told to for "a
- // count of things someone DID"), and no name an enumeration could return changes a
- // count of events: running it there only carries a reading list into sessions that
- // cannot use it.
- if len(cov.Acts) > 0 {
- _LOG.Printf("[SlotResearch] %d act word(s) declared but this table is not an ENUMERATION (it declared no count/set/list slot, or no NAME-carrying slot) — not run: a value question pays nothing for a set's bookkeeping.", len(cov.Acts))
- }
- case kb != nil:
- if set, done := kb.CoverageSet(); done {
- deps.CoverageSeed = set.Render()
- _LOG.Printf("[SlotResearch] enumeration already ran for this question; reusing its %d window(s) over %d operand(s).", len(set.Windows), len(set.Operands))
- } else if deps.Tools != nil {
- if runner, ok := deps.Tools.Exec.(runtime.CoverageRunner); ok {
- set := runner.EnumerateCoverage(ctx, cov, kb)
- if len(set.Operands) > 0 {
- _LOG.Printf("[SlotResearch] enumeration: %d operand(s) asked, %d passage(s) recalled, %d window(s) found; seed +%d char(s).",
- len(set.Operands), set.Recalled, len(set.Windows), len(set.Render()))
- logCoverageWindows(set)
- kb.MarkCoverage(cov)
- kb.StoreCoverageSet(set)
- deps.CoverageSeed = set.Render()
- }
- } else {
- // Said out loud, like the batching diagnostic above: a step that can
- // never fire has to be distinguishable from one that works, or the
- // sessions silently read a seed with no evidence in it.
- _LOG.Printf("[SlotResearch] the tool executor cannot enumerate (it does not implement runtime.CoverageRunner); the direction's act words were NOT run over the corpus.")
- }
- }
- }
-
- for i := 0; i < limit; i++ {
- d := dirs[i]
- wg.Add(1)
- go func(d direction) {
- defer wg.Done()
- sem <- struct{}{}
- defer func() { <-sem }()
-
- // Sessions share ONE Toolset; DisableTool mutates it, so the call is
- // guarded here. The Kbinfos merge happens inside the executor.
- res := runtime.RunActionSession(sessionCtx, deps, d.text, slotTable, sessionBudget, "", sharedToolCache, sharedSearchQueries)
- mu.Lock()
- results = append(results, outcome{slotID: d.slotID, direction: d.text, result: res})
- mu.Unlock()
- }(d)
- }
- wg.Wait()
-
- // Fold in slot-id order so the merge is deterministic regardless of which
- // session finished first.
- sort.Slice(results, func(i, j int) bool { return results[i].slotID < results[j].slotID })
-
- collected := st.CollectedAnswer
- sessionEvidence := map[string]SlotEvidence{}
- ledger := append([]map[string]any(nil), st.Attempted...)
-
- for _, item := range results {
- r := item.result
- if r.FoundAnswer != nil && collected == "" {
- collected = *r.FoundAnswer
- }
- if len(r.RetrievedEvidenceIDs) > 0 {
- terminalType := ""
- if r.TerminalType != nil {
- terminalType = *r.TerminalType
- }
- var candidate string
- if r.FoundAnswer != nil {
- candidate = *r.FoundAnswer
- }
- sessionEvidence[fmt.Sprint(item.slotID)] = SlotEvidence{
- EvidenceIDs: dedupe(r.RetrievedEvidenceIDs),
- TerminalType: terminalType,
- Candidate: candidate,
- }
- }
- for _, ns := range r.NewStates {
- before := slotTable
- merged := MergeSlotPatch(slotTable, ns)
- if merged != nil {
- slotTable = *merged
- }
- // What this session CLAIMED, and whether the merge kept it.
- //
- // The merge is a tournament per slot: one candidate survives, chosen by
- // the strength the MODEL reported (MergeSlotPatch:3304), and nothing
- // logged the contestants — which is why three rounds of analysis here
- // could only GUESS whether some session's list had been dropped. The log
- // showed the merged table and never a single patch.
- logSessionPatch(item.slotID, before, slotTable, ns)
- }
- // _run_slot_research_pass — a session without a found answer still logs a hint, taken
- // from its message history (`(r.found_answer or str(r.messages))[:80]`).
- // Without the fallback the rewriter's research context shows a bare
- // "- (round N: …)" row for those sessions.
- q := ""
- if r.FoundAnswer != nil {
- q = truncateRunes(*r.FoundAnswer, 80)
- } else {
- // No answer: the row shows what this session was ASKED to research, which is
- // the query of the pair (query, outcome). It used to be a langchain-style repr
- // of the session's messages, and that repr is a CONSTANT in production: every
- // session's history opens with the same system prompt, so its first 80 runes —
- // a message header, not a query — went into every row of every question.
- q = truncateRunes(item.direction, 80)
- }
- // "bound" is the evidence THIS session retrieved (the number the row can
- // stand behind); "new" would be a lie here: sessions run concurrently and
- // share one pool, so no per-session count of newly admitted passages exists.
- // The row used to say `"new": 1` for every session, which the rewriter read
- // as "1 new passage(s)" on every round it did not retrieve anything. `r` is
- // the round the row belongs to, the same key the prefetch/rewrite rows use.
- ledger = append(ledger, map[string]any{
- "q": q,
- "r": st.SearchRounds,
- "bound": len(dedupe(r.RetrievedEvidenceIDs)),
- })
- }
-
- // _run_slot_research_pass — report how many passages each slot's session bound, so
- // a round that retrieved nothing for a slot is visible in the run log.
- if len(sessionEvidence) > 0 {
- bounds := make(map[string]int, len(sessionEvidence))
- for sid, ev := range sessionEvidence {
- bounds[sid] = len(ev.EvidenceIDs)
- }
- _LOG.Printf("[SlotResearch] slot evidence bound: %v", bounds)
- }
-
- // Evidence-guided batching(): slots that retrieved the
- // same passages get answered together instead of one generation call each.
- // BatchFillSlots is best-effort internally: per-cluster failures are logged and
- // skipped, so one bad cluster never costs the others.
- if batched := BatchFillSlots(ctx, deps, &slotTable, sessionEvidence); batched > 0 {
- _LOG.Printf("[SlotResearch] batched generation filled %d slot(s)", batched)
- }
-
- unresolvedOut := make([]map[string]any, 0, len(unresolved))
- for _, v := range slotTable.Unresolved() {
- clues := v.DiscoveredClues
- if len(clues) > 4 {
- clues = clues[len(clues)-4:]
- }
- unresolvedOut = append(unresolvedOut, map[string]any{
- "id": v.ID,
- "type": v.Type,
- "question_clues": append([]string(nil), v.QuestionClues...),
- "discovered_clues": append([]string(nil), clues...),
- })
- }
-
- // One set, one answer: a table can hold the members in one slot and the count in
- // another, written by different sessions, with nothing keeping them in step. Raised
- // here, before the draft the answer is written from.
- if raised := syncCountSlots(&slotTable); len(raised) > 0 {
- _LOG.Printf("[SlotResearch] count slot(s) %v set to the enumerated members' size", raised)
- }
-
- draft := RenderSlotDraft(slotTable, collected, sessionEvidence)
- _LOG.Printf("[SlotResearch] round done — %d slot(s) filled, unresolved=%d, collected_answer=%v",
- countFilled(slotTable), len(unresolvedOut), collected != "")
- _LOG.Printf("[SlotResearch] slot table after round:\n%s", draft)
-
- return &SlotResearchResult{
- SlotTable: slotTable,
- CollectedAnswer: collected,
- UnresolvedSlots: unresolvedOut,
- SlotEvidence: sessionEvidence,
- SlotDraft: draft,
- // The answer-facing record (no machine fields) is rendered here, next to
- // the SCA-facing draft, so the two can never drift apart.
- SlotRecord: RenderSlotRecord(slotTable, collected),
- Attempted: ledger,
- }
-}
-
-// alternateCandidatesOf returns the candidates a slot lost to, in the order they were
-// kept (see runtime.Variable.Alternates).
-func alternateCandidatesOf(v runtime.Variable) []string {
- return append([]string(nil), v.Alternates...)
-}
-
-// logSessionPatch reports one session's claims and what the merge did with them.
-//
-// One line per patched slot, because the losing side of MergeSlotPatch leaves no
-// trace anywhere else: a candidate that was dropped is not in the table, not in
-// the draft, not in the record — it is simply gone, and "gone" is the failure this
-// line makes visible.
-func logSessionPatch(directionSlot int, before, after, patch runtime.State) {
- for _, pv := range patch.State {
- if pv.Candidate == nil || *pv.Candidate == "" {
- continue
- }
- bv := before.ByID(pv.ID)
- base := "(empty)"
- if bv != nil && bv.Candidate != nil && *bv.Candidate != "" {
- base = fmt.Sprintf("%q (%.2f)", truncateRunes(*bv.Candidate, 80), strengthOf(*bv))
- }
- // THREE outcomes, because the fold has three: the patch's text IS the slot's
- // value; the slot CHANGED without keeping this text verbatim (a union — member
- // lists merge by name, so a patch can be inside the value without being its
- // rendering); or the slot did not move at all. The old two-way test called every
- // union "NOT ADOPTED (the base stands)", which hid every merge that mattered.
- result := "NOT ADOPTED (the base stands)"
- if av := after.ByID(pv.ID); av != nil {
- switch {
- case av.Candidate != nil && *av.Candidate == *pv.Candidate:
- result = "adopted"
- case av.Candidate != nil && bv != nil && bv.Candidate != nil && *av.Candidate != *bv.Candidate:
- result = "merged (the union kept both; the slot's value is not this candidate verbatim)"
- }
- }
- _LOG.Printf("[SlotResearch] patch (direction slot %d) → slot %d: %q (%.2f); base was %s; result: %s",
- directionSlot, pv.ID, truncateRunes(*pv.Candidate, 120), strengthOf(pv), base, result)
- }
-}
-
-// MergeSlotPatch folds a session's new-state branch into the shared slot table: a slot
-// holding a SET merges by union, anything else adopts the STRONGER candidate (see
-// slots.Union for why).
-//
-// Returns nil when nothing changed, so
-// callers can skip no-op merges — and so a round that changed nothing can be told apart
-// from one that did.
-func MergeSlotPatch(base, branch runtime.State) *runtime.State {
- if len(branch.State) == 0 {
- return nil
- }
- branchByID := map[int]runtime.Variable{}
- for _, v := range branch.State {
- branchByID[v.ID] = v
- }
- merged := make([]runtime.Variable, 0, len(base.State))
- changed := false
- for _, v := range base.State {
- bv, ok := branchByID[v.ID]
- if !ok {
- merged = append(merged, v)
- continue
- }
- // A slot holding a SET merges by UNION; a slot holding one VALUE is still settled by
- // strength (see slots.Union for why, and for the case where a count outvoted the
- // members it was counting).
- //
- // The union is over TYPED values: members union by name, a membership claim
- // beats a number, two numbers keep the larger. Text is not the union's
- // business (invariant I1), which is what keeps a value question merging by
- // strength exactly as it always did.
- var cand *string
- var value *slots.Value
- strength := v.CandidateStrength
- var loser string
- if union, dropped, ok := slots.Union(v.Typed(), bv.Typed()); ok {
- rendered := slots.Render(union)
- cand = &rendered
- value = &union
- if strengthOf(bv) > strengthOf(v) {
- strength = bv.CandidateStrength
- }
- if !dropped.IsZero() {
- loser = slots.Render(dropped)
- }
- } else {
- // Adopt the branch candidate only when STRONGER. Sessions run
- // concurrently and their branches fold in completion order, so an
- // unconditional "branch wins" made the result both order-dependent and
- // destructive: a weak session (0.3, tentative) could downgrade a slot
- // another session had already proven (0.95).
- adoptBranch := bv.Candidate != nil && *bv.Candidate != "" &&
- (v.Candidate == nil || *v.Candidate == "" || strengthOf(bv) > strengthOf(v))
- cand = v.Candidate
- value = v.Value
- if adoptBranch {
- cand, strength = bv.Candidate, bv.CandidateStrength
- value = bv.Value
- }
- // ONE slot holds ONE candidate, so the claim that lost the comparison used
- // to leave no trace at all: not in the table, not in the draft, not in the
- // record. Two sessions enumerating the same question from different angles
- // therefore produced whichever LIST the model happened to call stronger,
- // and the other list was silently gone: runs of the same question return different
- // member counts, with no slot holding the complete list.
- //
- // The losing claim is kept in the slot's Alternates field —
- // the one field MergeSlotPatch unions, never replaces. The framework does not decide
- // which list is true; it stops discarding the one that lost.
- switch {
- case adoptBranch && v.Candidate != nil && *v.Candidate != "" && *v.Candidate != *bv.Candidate:
- loser = *v.Candidate
- case !adoptBranch && bv.Candidate != nil && *bv.Candidate != "" && (v.Candidate == nil || *v.Candidate != *bv.Candidate):
- loser = *bv.Candidate
- }
- }
- clues := dedupe(append(append([]string(nil), v.DiscoveredClues...), bv.DiscoveredClues...))
- alternates := dedupe(append(append([]string(nil), v.Alternates...), bv.Alternates...))
- if loser != "" {
- alternates = dedupe(append(alternates, loser))
- }
- if !equalStringPtr(cand, v.Candidate) || !equalStrings(clues, v.DiscoveredClues) || !equalStrings(alternates, v.Alternates) {
- changed = true
- }
- merged = append(merged, runtime.Variable{
- ID: v.ID,
- Type: v.Type,
- QuestionClues: append([]string(nil), v.QuestionClues...),
- DiscoveredClues: clues,
- Alternates: alternates,
- Candidate: cand,
- CandidateStrength: strength,
- Value: value,
- // The DECLARATION travels with the slot. Terms/Subject are what the
- // enumeration is built from (runtime.CoverageOf), and rebuilding the slot
- // without them is why a later round can have no enumeration at all: the seed carries
- // the method AND the enumerated windows on the first pass and only the method
- // afterwards, so the recovery round — the one the routing opened because the record
- // was still short — runs with the enumeration switched off.
- Terms: append([]string(nil), v.Terms...),
- Subject: v.Subject,
- })
- }
- if !changed {
- return nil
- }
- out := runtime.NewState(merged, base.Depth+1, append([]string(nil), base.RetrievedEvidenceIDs...))
- return &out
-}
-
-// memberLine renders a slot holding items as each item WITH the passage behind it, so the citation
-// instruction below ("every member you list carries the words behind it") has something to point
-// at. Without the ids it asked for citations nobody could make.
-func memberLine(v runtime.Variable) string {
- val := v.Typed()
- if val.Kind != slots.KindItems {
- return ""
- }
- parts := make([]string, 0, len(val.Items))
- for _, it := range val.Items {
- name := strings.TrimSpace(it.Value)
- if name == "" {
- continue
- }
- if id := strings.TrimSpace(it.ChunkID); id != "" {
- // The words travel WITH the name. The evidence block the answer reads is budgeted, so a
- // member whose passage did not fit used to arrive as a bare name and had to be dropped
- // from the list: measured (2026-09-17, 三国/关羽) 18 members against the 13 passages the
- // budget carried — the answer named the other five as "listed in the record, no original
- // text provided". The quote here is the verdict's own window (bounded by
- // coverageWindowBefore/After): the words the membership rests on, and the record's own
- // contract accepts "quoted above" as the evidence for a member.
- parts = append(parts, name+" ←"+id+itemQuote(it.Quote))
- } else {
- parts = append(parts, name+" ←(no passage)")
- }
- }
- return strings.Join(parts, "、")
-}
-
-// itemQuoteMaxRunes bounds the words a member carries into the record. Short because the record is
-// read by the answer model and travelled with the name only to keep it citable at all.
-const itemQuoteMaxRunes = 120
-
-// itemQuote renders the words an item rests on, or nothing when the item has none: a member with no
-// quote is a claim, and the record already says so with "(no passage)".
-func itemQuote(quote string) string {
- if quote = strings.TrimSpace(quote); quote == "" {
- return ""
- }
- return " “" + truncateRunes(quote, itemQuoteMaxRunes) + "”"
-}
-
-// isCountSlot reports whether a slot declares itself a count — the planner's word for a number
-// slot. It decides only whether a disagreement is worth STATING; nothing is derived from it
-// (syncCountSlots reads the value's KIND for that, never the type word).
-func isCountSlot(v runtime.Variable) bool {
- return v.Candidate != nil && strings.EqualFold(strings.TrimSpace(v.Type), "count")
-}
-
-// countDerivedFloor is the smallest set a count may be derived from: one quotable item is not an
-// enumeration, and a half-filled table must not collapse a claimed number to one.
-const countDerivedFloor = 2
-
-// syncCountSlots writes the size of the enumerable set into every slot that claims a number, and
-// keeps the old claim as an alternate. Only a number-claiming slot is touched (slots.Value.Number),
-// and the size counts the items the table can point at (runtime.AnchoredItems): a count of claims
-// is a count of nothing.
-//
-// The size is read across the whole table, so a question with two sets gets one number for both
-// until the planner says which count counts which set. Below countDerivedFloor there is nothing to
-// derive from and the claim stands.
-//
-// It returns the ids it changed, for the log.
-func syncCountSlots(table *runtime.State) []int {
- if table == nil || len(table.State) == 0 {
- return nil
- }
- union := runtime.AnchoredItems(table)
- if len(union) < countDerivedFloor {
- return nil
- }
- var raised, unreadable []int
- for i := range table.State {
- v := &table.State[i]
- claimed, ok := v.Typed().Number()
- if !ok {
- // A slot holding ITEMS is read, just not as a number: the size is derived from the items
- // and the record states it, so it is not an unreadable count (a session may patch the
- // members into the slot the planner typed "count").
- if isCountSlot(*v) && v.Typed().Kind != slots.KindItems {
- unreadable = append(unreadable, v.ID)
- }
- continue
- }
- if claimed == len(union) {
- continue
- }
- // The claim loses either way — a count larger than the members listed is not evidence
- // of members, and a smaller one is a set that lost some. Left standing, the claim is the
- // number the answer reports, whatever the enumerated members say.
- //
- // Deduped because this runs after EVERY pass (and again after the last node), so
- // an undeduped append printed the same alternate line once per run.
- v.Alternates = dedupe(append(v.Alternates, slots.Render(v.Typed())))
- derived := slots.Number(len(union))
- v.Value = &derived
- rendered := slots.Render(derived)
- v.Candidate = &rendered
- raised = append(raised, v.ID)
- }
- if len(unreadable) > 0 {
- // A count slot holding words cannot be corrected — nothing derives from text — so it is
- // said out loud instead, and the record states the disagreement (see RenderSlotRecord).
- _LOG.Printf("[Coverage] count slot(s) %v hold text, not a number: nothing was derived from them", unreadable)
- }
- return raised
-}
-
-// countFilled counts slots holding a candidate.
-func countFilled(slotTable runtime.State) int {
- n := 0
- for _, v := range slotTable.State {
- if v.Filled() {
- n++
- }
- }
- return n
-}
-
-// Graph driver.
diff --git a/internal/rag/agentic-rag/graph_util.go b/internal/rag/agentic-rag/graph_util.go
deleted file mode 100644
index cf9fe60100..0000000000
--- a/internal/rag/agentic-rag/graph_util.go
+++ /dev/null
@@ -1,292 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-// The small helpers every stage shares live in THIS file: JSON extraction, scalar
-// coercion, score extraction, string comparison, rune truncation. They live together so
-// that a stage needing one of them takes the family from one place — and so that a second
-// copy of any of them is not where two stages start disagreeing about the same value
-// (`stringOf` and `anyString` used to be exactly that pair).
-package agentic_rag
-
-import (
- "encoding/json"
- "fmt"
- "reflect"
- "sort"
- "strings"
-
- "ragflow/internal/common"
- "ragflow/internal/rag/agentic-rag/runtime"
-)
-
-// similarityOrScore is the ONE reading of a chunk's relevance: its `similarity`,
-// falling through to `score` when that is missing or zero — a FALSY similarity must not
-// win over a present score, so this is deliberately not "first key present wins".
-// It was written three times (the SCA view, the draft's ordering, the citation
-// reference) before it lived here.
-func similarityOrScore(c map[string]any) float64 {
- if v, ok := common.ToFloat(c["similarity"]); ok && v != 0 {
- return v
- }
- if v, ok := common.ToFloat(c["score"]); ok && v != 0 {
- return v
- }
- return 0.0
-}
-
-// rankByScore returns the chunks strongest-first. Stable, so equal-scored chunks keep
-// their retrieval order.
-func rankByScore(chunks []map[string]any) []map[string]any {
- out := append([]map[string]any(nil), chunks...)
- sort.SliceStable(out, func(i, j int) bool {
- return similarityOrScore(out[i]) > similarityOrScore(out[j])
- })
- return out
-}
-
-// queryToTerms is the runtime keywords helper the SCA view reads: lowercase word
-// tokens of length >= 3, de-duplicated, order preserved.
-func queryToTerms(q string) []string {
- if q == "" {
- return nil
- }
- found := tokenPattern.FindAllString(strings.ToLower(q), -1)
- out := make([]string, 0, len(found))
- seen := make(map[string]bool, len(found))
- for _, t := range found {
- if len(t) < 3 || seen[t] {
- continue
- }
- seen[t] = true
- out = append(out, t)
- }
- return out
-}
-
-// truncateRunes caps a string to n runes without breaking multi-byte chars.
-func truncateRunes(s string, n int) string {
- if n <= 0 {
- return ""
- }
- r := []rune(s)
- if len(r) <= n {
- return s
- }
- return string(r[:n])
-}
-
-// dedupe preserves order and drops empty / repeated entries.
-func dedupe(in []string) []string {
- if len(in) == 0 {
- return nil
- }
- seen := make(map[string]bool, len(in))
- out := make([]string, 0, len(in))
- for _, s := range in {
- if s == "" || seen[s] {
- continue
- }
- seen[s] = true
- out = append(out, s)
- }
- return out
-}
-
-// asSliceOfAny coerces a JSON array value to []any.
-//
-// isPoisoned covers the values that equally must never reach a prompt (see its doc
-// comment).
-func asSliceOfAny(v any) []any {
- if isPoisoned(v) {
- _LOG.Printf("[StateGuard] dropping poisoned value of kind %s; treating as empty",
- reflect.ValueOf(v).Kind())
- return nil
- }
- switch x := v.(type) {
- case nil:
- return nil
- case []any:
- return x
- case []string:
- out := make([]any, len(x))
- for i, s := range x {
- out[i] = s
- }
- return out
- case map[string]any:
- out := make([]any, 0, len(x))
- for _, vv := range x {
- out = append(out, vv)
- }
- return out
- default:
- return []any{v}
- }
-}
-
-// anyString reads a string field from a chunk-like map.
-func anyString(v any) string {
- switch x := v.(type) {
- case nil:
- return ""
- case string:
- return x
- default:
- return fmt.Sprint(x)
- }
-}
-
-// truncateEach caps every entry of a string slice.
-func truncateEach(in []string, n int) []string {
- if len(in) == 0 {
- return nil
- }
- out := make([]string, len(in))
- for i, s := range in {
- out[i] = truncateRunes(s, n)
- }
- return out
-}
-
-// equalStringPtr reports whether two optional strings are equal.
-func equalStringPtr(a, b *string) bool {
- if a == nil || b == nil {
- return a == b
- }
- return *a == *b
-}
-
-// equalStrings reports set-equality (order-independent) of two string slices.
-func equalStrings(a, b []string) bool {
- if len(a) != len(b) {
- return false
- }
- seen := make(map[string]int, len(a))
- for _, s := range a {
- seen[s]++
- }
- for _, s := range b {
- if seen[s] == 0 {
- return false
- }
- seen[s]--
- }
- return true
-}
-
-// isPoisoned reports whether a state value must be discarded before it reaches a
-// prompt: channels and function values, neither of which renders as anything a model
-// can read.
-func isPoisoned(v any) bool {
- if v == nil {
- return false
- }
- switch reflect.ValueOf(v).Kind() {
- case reflect.Chan, reflect.Func, reflect.UnsafePointer:
- return true
- }
- return false
-}
-
-// extractJSONObject: return the
-// first parseable JSON object in text, or nil when none parses.
-//
-// The fan-out model sometimes emits prose around the object, and a greedy
-// brace-match would capture several objects and fail with "extra data"; each
-// candidate is therefore validated before it is accepted, and an invalid one
-// resumes the scan at its next "{".
-func extractJSONObject(text string) any {
- for i := 0; i < len(text); {
- rel := strings.IndexByte(text[i:], '{')
- if rel < 0 {
- return nil
- }
- start := i + rel
- depth := 0
- scan:
- for j := start; j < len(text); j++ {
- switch text[j] {
- case '{':
- depth++
- case '}':
- depth--
- if depth == 0 {
- var out map[string]any
- if json.Unmarshal([]byte(text[start:j+1]), &out) == nil {
- return out
- }
- break scan // not a valid object; try the next "{"
- }
- }
- }
- i = start + 1
- }
- return nil
-}
-
-// jsonModelAdapter adapts runtime.SessionModel to orchestrator.JSONModel.
-//
-// The rendered prompt is the system turn and "Output:\n" the user turn (fitted ONCE
-// to the model's context window), then the first JSON value is parsed out of the reply.
-// Malformed JSON is retried (up to genJSONMaxRetry calls) with the model's own
-// bad answer and the parse error appended to the user turn, so a single
-// formatting hiccup does not abort the SCA review or the query rewrite.
-type jsonModelAdapter struct {
- inner runtime.SessionModel
- // maxLength is the chat model's context window. It bounds the message fit so an
- // oversized prompt is trimmed instead of rejected by the provider. <=0 falls back
- // to chat.EffectiveContextLength's 8192 default.
- maxLength int
-}
-
-// parseGenJSONReply parses a cleaned model reply with gen_json's tolerance: a
-// strict decode of ANY top-level JSON value first (json_repair accepts objects,
-// arrays and scalars alike), then the brace-matched object extractor for fenced
-// or damaged objects — the JSON-shaped gate, NOT the prose-salvaging ExtractJSON,
-// so a non-JSON reply stays a parse failure and triggers the corrective retry.
-// The boolean distinguishes a successful decode from a failure, because a
-// legitimate `null` reply decodes to a nil value.
-func parseGenJSONReply(cleaned string) (any, bool) {
- var val any
- if err := json.Unmarshal([]byte(strings.TrimSpace(cleaned)), &val); err == nil {
- return val, true
- }
- if v := runtime.ExtractJSONObject(cleaned); v != nil {
- return v, true
- }
- return nil, false
-}
-
-// stripGenJSONWrappers mirrors gen_json's answer cleanup:
-//
-// ans = re.sub(r"(^.*</think>|```json\n|```\n*$)", "", ans, flags=re.DOTALL)
-//
-// The think term (greedy up to the LAST </think>) is common.StripThinkTrailing;
-// a "```json\n" fence may occur anywhere and is removed wholesale; a trailing
-// "```" (plus newlines) is cut from the end.
-func stripGenJSONWrappers(s string) string {
- s = common.StripThinkTrailing(s)
- s = strings.ReplaceAll(s, "```json\n", "")
- return genJSONTailFenceRE.ReplaceAllString(s, "")
-}
-
-func toAnySlice(ss []string) []any {
- out := make([]any, 0, len(ss))
- for _, s := range ss {
- out = append(out, s)
- }
- return out
-}
diff --git a/internal/rag/agentic-rag/policy.go b/internal/rag/agentic-rag/policy.go
deleted file mode 100644
index b99881ce19..0000000000
--- a/internal/rag/agentic-rag/policy.go
+++ /dev/null
@@ -1,132 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package agentic_rag
-
-// The graph's CROSS-STAGE numbers live here: every budget, timeout and pool / session cap
-// that more than one stage reasons about — "how long may this step take" and "how much may
-// the pool hold" are answered in one place (see the call sites: nodeClock in every node,
-// SessionWallS and SetBudgetExtensionS in the research pass, MaxSnippetPool in the
-// prefetch).
-//
-// What is deliberately NOT here: the numbers ONE stage spends — the fan-out shape guards
-// beside fanoutLooksLikeQuery, the batching thresholds beside BatchFillSlots, the draft caps
-// beside ComposeFallbackDraft, the sampling temperatures beside the calls they configure.
-// Moving those here would put a constant a long way from the only code that can explain it,
-// and a stage's tuning is exactly the thing that changes with its own measurements. The rule
-// is cohesion, not one file: a number shared across stages belongs here, a number one stage
-// owns belongs with that stage.
-//
-// What is ALSO not here: the DECISIONS those numbers feed. A node asks for a clock with
-// nodeClock and gets one; whether the round continues afterwards is a routing decision, and
-// those live with the routes.
-
-// nodeClock is THE shape every node's clock takes: the node's own budget, floored at
-// floorS, and capped by the room the QUESTION still has (roomS is what is left after the
-// reserve the callers below it need).
-//
-// The FLOOR WINS over a room that is merely small OR already spent: with room 2s — or -50s
-// — and a floor of 10s the call gets 10s, because a node that cannot finish inside its
-// floor is not worth starting at all, and a node that is the only chance to gather a
-// candidate pool is worth its floor even on an overrun budget. A caller that needs the room
-// protected therefore passes a smaller floor, or a roomS that already has the reserve
-// subtracted; a step that must not run at all on a spent budget says so at its own call
-// site (see RunCoverageResolve).
-func nodeClock(budgetS, floorS, roomS float64) float64 {
- return min(budgetS, max(floorS, roomS))
-}
-
-const (
- TotalBudgetS = 180.0 // whole-graph wall-clock ceiling per question
- MinRoundHeadroomS = 50.0 // need at least this much left to start a new round
- PassTimeoutS = 120.0 // slot research pass wall-clock
- // SetBudgetExtensionS is added ONCE to a question's research budget when its
- // table IS an enumeration — a count/set/list slot, a NAME-carrying slot and the
- // act words (see runtime.Coverage.Ok and RunSlotResearchPass).
- //
- // The question budget is sized for one pass (TotalBudgetS 180 ⊃ PassTimeoutS
- // 120) and an enumeration needs a second one: its first pass spends the wall
- // clock on batches of names, so a member that a cut session never patched has
- // nowhere to be picked up: a spent pass leaves too little room for another round
- // (MinRoundHeadroomS), and a member that was reached but never recorded is lost with
- // it. The extension is what lets the round AFTER that one start at all.
- SetBudgetExtensionS = 120.0
- // setSessionSlackS is added to the session clock an ENUMERATION pass hands its
- // sessions, so a session's own finalize/salvage step still fits inside the
- // context the pass derived it from.
- setSessionSlackS = 20.0
- // downstreamReserveS is what the steps AFTER research need: the SCA review
- // (SCATimeoutS), the draft and the composed answer. The budget extension is only
- // bought when the caller's own context still holds it — otherwise the second
- // research round would spend the time the answer needs, turning a missing member
- // into a timed-out question, which is strictly worse.
- downstreamReserveS = 90.0
- // minBudgetExtensionS is the smallest extension worth buying: less than this and
- // the following round could not start anyway (MinRoundHeadroomS), so the budget
- // would be widened without anything being able to use it.
- minBudgetExtensionS = 40.0
- PrefetchTimeoutS = 90.0 // programmatic fan-out fetch
- DraftTimeoutS = 60.0 // fallback draft synthesis
- SCATimeoutS = 60.0 // sufficient-context review call
- // SCARetryHeadroomS is the clock that must be left before a FAILED review is retried once:
- // the second attempt plus the answer that follows it. Below it the retry is skipped and the
- // unavailable review is recorded as such (see graph_sca), because a retry that eats the
- // answer's own clock trades a missing verdict for a missing answer.
- SCARetryHeadroomS = 120.0
- RewriteTimeoutS = 45.0 // gap → query rewrite call
- // CoverageResolveTimeoutS bounds the enumeration's last node (see RunCoverageResolve): it runs
- // before the answer is composed, so it may not spend the clock the answer needs. It bounds the
- // WHOLE resolve, which is batched and parallel, rather than one call carrying every
- // window: one call that carries them all hits this clock and answers nothing at all.
- CoverageResolveTimeoutS = 30.0
- // SCAViewCap is the view the SCA is shown: large enough that the passage carrying the
- // answer is not the one that gets cut.
- SCAViewCap = 60
- // CoverageEnrollHeadroomS is the clock the resolve node needs before it may run the
- // direction's enumeration itself (see enrollEnumeration): the enumeration plus the answer
- // that still has to be composed. Below it the node judges what it already has.
- CoverageEnrollHeadroomS = 45.0
- // MaxSnippetPool is the storage ceiling of the snippet pool across ALL
- // rounds. Storage and REVIEW are decoupled: the SCA only reads a ranked
- // view, so the pool may accumulate freely while prompts stay bounded.
- MaxSnippetPool = 60
- // DrillReserve: slots kept free after the FIRST prefetch so the research
- // executor can top up evidence.
- DrillReserve = 12
- // FanoutTopN is the per-query result count for the programmatic fetch.
- FanoutTopN = 8
- // FanoutTopNRewrite is the reduced count used after a rewrite round.
- FanoutTopNRewrite = 6
- // MaxFanouts caps planner fan-outs.
- MaxFanouts = 5
- // MaxSCAGaps caps gaps handed to the rewriter.
- MaxSCAGaps = 8
- // PoolHeadLines caps the evidence-pool summary shown to the rewriter.
- PoolHeadLines = 12
-
- // Slot-table research constants: how many sessions run at once, how many run per
- // round, and the caps on a fallback clue and a draft candidate.
- slotSessionConcurrency = 2
- slotSessionsPerRound = 3
- slotFallbackClueChars = 160
- // draftCandidateChars caps a claim's draft text handed to the SCA.
- draftCandidateChars = 400
- // draftClueTailChars / draftUnresolvedClueChars are the per-clue caps in the
- // rendered draft (240 for a resolved slot's discovered-clue tail, 80 for an
- // unresolved slot's question clues).
- draftClueTailChars = 240
- draftUnresolvedClueChars = 80
-)
diff --git a/internal/rag/agentic-rag/policy_test.go b/internal/rag/agentic-rag/policy_test.go
deleted file mode 100644
index 8c4c0815e4..0000000000
--- a/internal/rag/agentic-rag/policy_test.go
+++ /dev/null
@@ -1,60 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package agentic_rag
-
-import "testing"
-
-// TestNodeClockKeepsItsFloorOnASpentBudget pins the property that decides whether a run
-// whose research budget is already gone still retrieves anything: the clock is
-// min(budget, max(floor, room)), with NO "room gone -> zero" branch.
-//
-// Measured 2026-09-20 (glm-4.6, a single-question MES ransomware run): the fan-out plus the
-// slot-table decomposition spent 185.4s of the 180s budget, so the prefetch arrived with
-// room == RemainingS() - MinRoundHeadroomS == -50. Returning zero there built a
-// context.WithTimeout(ctx, 0); FanoutSearch's first act is a ctx.Done() check, so it
-// admitted nothing and logged "Added 0 new passages" without running a single search. The
-// run then reached the answer with an empty pool — the draft had nothing to summarise, the
-// SCA had nothing to judge, and the reply was the no-evidence hedge — while the searches it
-// never ran took 25ms each on the same corpus.
-func TestNodeClockKeepsItsFloorOnASpentBudget(t *testing.T) {
- cases := []struct {
- name string
- budgetS, floorS, roomS float64
- want float64
- }{
- // The prefetch: its own 90s cap, floored at 10s.
- {"prefetch with the budget spent", PrefetchTimeoutS, 10.0, -50.0, 10.0},
- {"prefetch with no room at all", PrefetchTimeoutS, 10.0, 0, 10.0},
- {"prefetch with less room than its floor", PrefetchTimeoutS, 10.0, 4.0, 10.0},
- {"prefetch with room left", PrefetchTimeoutS, 10.0, 40.0, 40.0},
- {"prefetch capped by its own budget", PrefetchTimeoutS, 10.0, 600.0, 90.0},
- // The draft and the SCA review: 60s each, floored at 15s.
- {"draft with the budget spent", DraftTimeoutS, 15.0, -10.0, 15.0},
- {"sca with the budget spent", SCATimeoutS, 15.0, -10.0, 15.0},
- // The gap -> query rewrite call: 45s, floored at 10s.
- {"rewrite with the budget spent", RewriteTimeoutS, 10.0, -10.0, 10.0},
- // The slot research pass has no floor of its own — its caller floors it at 20s, and
- // only reaches it with MinRoundHeadroomS still on the clock.
- {"research pass with the budget spent", PassTimeoutS, 0, -25.0, 0},
- }
- for _, tc := range cases {
- if got := nodeClock(tc.budgetS, tc.floorS, tc.roomS); got != tc.want {
- t.Errorf("%s: nodeClock(%g, %g, %g) = %g, want %g",
- tc.name, tc.budgetS, tc.floorS, tc.roomS, got, tc.want)
- }
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/action_session.go b/internal/rag/agentic-rag/runtime/action_session.go
deleted file mode 100644
index a1bd00061d..0000000000
--- a/internal/rag/agentic-rag/runtime/action_session.go
+++ /dev/null
@@ -1,4193 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "crypto/rand"
- "encoding/json"
- "fmt"
- "math"
- "regexp"
- "sort"
- "strconv"
- "strings"
- "sync"
- "time"
- "unicode/utf8"
-
- "github.com/cloudwego/eino/compose"
- "github.com/cloudwego/eino/schema"
- "github.com/kaptinlin/jsonrepair"
- "gorm.io/gorm"
-
- "ragflow/internal/agent/chat"
- "ragflow/internal/rag/agentic-rag/slots"
- "ragflow/internal/rag/prompts"
-
- "ragflow/internal/common"
-)
-
-// action_session.go consolidates what was previously split across sessiontypes.go,
-// extract.go, toolschema.go, run.go, navchain.go, and session.go. The split is preserved
-// as clearly marked section banners below; every symbol keeps its original name and
-// behavior so tool_executor.go and the tests are unaffected.
-//
-// The section ORDER follows the stages of a session (helpers → classes → tool specs →
-// executors → model seam → session state → nodes → nav prefix → entry points at the end),
-// not a Go dependency order.
-
-// ===================== sessiontypes.go =====================
-// Slot-table models and the unified tool-result
-//
-// The slot-table models and the unified tool result:
-// - Variable / State / Result
-// - tool status constants
-// - ToolOutcome
-// - ApplyPatch
-
-// Variable is one unknown entity to resolve. ID is immutable across patches.
-type Variable struct {
- ID int
- Type string
- QuestionClues []string
- DiscoveredClues []string
- // Alternates are claims this slot LOST a comparison to (see MergeSlotPatch) and
- // keeps so they are not silently discarded: one slot holds one candidate, and the
- // framework does not decide which claim is true — it stops throwing the other away.
- // They are a FIELD and not a prefix inside DiscoveredClues because the record, the
- // ledger and the merge all read them, and a fact three readers parse out of a
- // string is a fact three parsers can disagree about.
- Alternates []string
- Candidate *string
- CandidateStrength *float64
- // Value is the TYPED candidate (see package slots): what this slot holds, as
- // data. It is set when the model DECLARED a kind, and nil otherwise.
- //
- // nil means "untyped", not "unknown": Candidate still carries the text, and that
- // text is OPAQUE — nothing may split it, count it, or compare it numerically.
- // Typing is what lets the runtime merge and count without guessing, so an
- // untyped slot is simply not merged by union (see slots.Union I1) rather than
- // parsed to find out what it might have been.
- Value *slots.Value
- // Terms are the ACT WORDS a direction declared for this slot — how its source
- // words the deed being enumerated (斩 / 杀 / 诛 / 劈 / 砍). They are declared,
- // never inferred from a slot's text or from the question.
- //
- // They exist because an enumeration cannot be bounded by what the model can
- // recall: the corpus holds the truth, and the words the source uses are what
- // reaches it. The runtime asks the corpus with them and RUNS the enumeration
- // (see Coverage / EnumerateCoverage), and the sessions read what came back, so the
- // tail of the list is a property of the corpus rather than of the model's memory.
- Terms []string
- // Subject is WHO the act is about, declared next to the act words. The pattern asks for
- // subject AND act together, because an act word on its own has a poor candidate pool — a
- // rare verb can come back empty for a text that does state the deed. Empty means the
- // pattern searches the term alone.
- Subject string
-}
-
-// Typed is the value this slot holds, for merging and counting. An untyped slot is
-// Text by definition — its candidate verbatim, never interpreted.
-func (v Variable) Typed() slots.Value {
- if v.Value != nil {
- return *v.Value
- }
- if v.Candidate != nil {
- return slots.Text(*v.Candidate)
- }
- return slots.Value{}
-}
-
-// Brief: one-line rendering for prompts.
-func (v Variable) Brief() string {
- if v.Candidate != nil && *v.Candidate != "" {
- cs := "?"
- if v.CandidateStrength != nil {
- cs = fmt.Sprintf("%.2f", *v.CandidateStrength)
- }
- return fmt.Sprintf("[%d] %s: %s (%s)", v.ID, v.Type, *v.Candidate, cs)
- }
- return fmt.Sprintf("[%d] %s: EMPTY", v.ID, v.Type)
-}
-
-// Filled
-func (v Variable) Filled() bool { return v.Candidate != nil && *v.Candidate != "" }
-
-// State is the slot table carried through one action session.
-type State struct {
- State []Variable
- Depth int
- ID string
- RetrievedEvidenceIDs []string
-}
-
-// NewState builds a State, generating its ID as
-// "<depth:03x>_<millis%1e8:08x><1 random byte:02x>".
-func NewState(vars []Variable, depth int, evidenceIDs []string) State {
- s := State{
- State: vars,
- Depth: depth,
- RetrievedEvidenceIDs: evidenceIDs,
- }
- if s.ID == "" {
- s.ID = newStateID(depth)
- }
- return s
-}
-
-// newStateID: id scheme.
-func newStateID(depth int) string {
- ms := time.Now().UnixMilli() % 100000000
- var b [1]byte
- _, _ = rand.Read(b[:])
- return fmt.Sprintf("%03x_%08x%02x", depth, ms, b[0])
-}
-
-// Unresolved: slots still missing a candidate.
-func (s State) Unresolved() []Variable {
- out := make([]Variable, 0, len(s.State))
- for _, v := range s.State {
- if !v.Filled() {
- out = append(out, v)
- }
- }
- return out
-}
-
-// ByID: Returns nil when no slot carries vid.
-func (s *State) ByID(vid int) *Variable {
- for i := range s.State {
- if s.State[i].ID == vid {
- return &s.State[i]
- }
- }
- return nil
-}
-
-// Brief: e.g. "d0(++.)".
-func (s State) Brief() string {
- var b strings.Builder
- b.WriteString(fmt.Sprintf("d%d(", s.Depth))
- for _, v := range s.State {
- if v.Filled() {
- b.WriteByte('+')
- } else {
- b.WriteByte('.')
- }
- }
- b.WriteByte(')')
- return b.String()
-}
-
-// RenderSlots: the multi-line prompt block
-// listing every slot, its clues, and any candidate found so far.
-func (s State) RenderSlots() string {
- var lines []string
- for _, v := range s.State {
- line := fmt.Sprintf("- id=%d type=%s", v.ID, v.Type)
- if len(v.QuestionClues) > 0 {
- line += "\n question_clues: " + strings.Join(v.QuestionClues, "; ")
- }
- if len(v.DiscoveredClues) > 0 {
- tail := v.DiscoveredClues
- if len(tail) > 4 {
- tail = tail[len(tail)-4:]
- }
- line += "\n discovered_clues: " + strings.Join(tail, "; ")
- }
- if v.Candidate != nil && *v.Candidate != "" {
- cs := "?"
- if v.CandidateStrength != nil {
- cs = fmt.Sprintf("%.2f", *v.CandidateStrength)
- }
- line += fmt.Sprintf("\n CANDIDATE: %s (strength=%s)", *v.Candidate, cs)
- }
- lines = append(lines, line)
- }
- return strings.Join(lines, "\n")
-}
-
-// Result is the outcome of ONE run_action session (one of NewStates /
-// FoundAnswer).
-//
-// Messages is typed as []schema.Message: the session builds them through the eino
-// constructors, and a typed slice keeps the conversation inspectable (and testable)
-// without type assertions.
-type Result struct {
- Messages []schema.Message
- NewStates []State
- FoundAnswer *string
- RetrievedEvidenceIDs []string
- TerminalType *string
- TerminalPayload map[string]any
-}
-
-// ApplyPatch: ONLY existing
-// ids are patchable; the mutable fields are candidate / candidate_strength /
-// discovered_clues. ID is immutable, so patches may not add variables.
-//
-// Returns nil when a patch entry is malformed (not a map, or missing "id") or when
-// nothing actually changed.
-func ApplyPatch(base State, branchPatches []map[string]any) *State {
- newVars := make([]Variable, 0, len(base.State))
- for _, v := range base.State {
- nv := Variable{
- ID: v.ID,
- Type: v.Type,
- QuestionClues: append([]string(nil), v.QuestionClues...),
- DiscoveredClues: append([]string(nil), v.DiscoveredClues...),
- Alternates: append([]string(nil), v.Alternates...),
- Candidate: v.Candidate,
- CandidateStrength: v.CandidateStrength,
- Value: v.Value,
- }
- newVars = append(newVars, nv)
- }
-
- changed := false
- for _, pv := range branchPatches {
- if pv == nil {
- return nil
- }
- rawID, ok := pv["id"]
- if !ok {
- return nil
- }
- // The comparison is integer-only, so a JSON string id ("1") simply matches no slot
- // and is skipped. Only JSON numbers match, which decode as float64 — accept those,
- // and reject anything else so the behaviour is strict rather than silently more
- // permissive.
- want, ok := common.ToIntNum(rawID)
- if !ok {
- continue
- }
- idx := -1
- for i := range newVars {
- if newVars[i].ID == want {
- idx = i
- break
- }
- }
- if idx < 0 {
- continue
- }
- nv := &newVars[idx]
-
- if rawKind, has := pv["kind"]; has && rawKind != nil && strings.TrimSpace(fmt.Sprint(rawKind)) != "" {
- // A patch that DECLARES what it holds is read as data (package slots) and
- // its rendered text is kept in Candidate, so every renderer, prompt and
- // log keeps working unchanged. This branch is the whole point of the
- // typed contract: the structure the model states is carried, and nothing
- // downstream has to infer it back from a string.
- value := slots.Parse(pv)
- nv.Value = &value
- rendered := slots.Render(value)
- nv.Candidate = &rendered
- changed = true
- } else if raw, has := pv["candidate"]; has {
- // Any falsy value (0, 0.0, False, "", [], {}, None) becomes None, NOT its string
- // form. fmt.Sprint would otherwise turn 0/False into "0"/"false".
- if !isTruthy(raw) {
- nv.Candidate = nil
- nv.Value = nil
- } else {
- s := fmt.Sprint(raw)
- nv.Candidate = &s
- // Opaque by construction: the model did not say what this is, so
- // nothing here will guess (see Variable.Value).
- text := slots.Text(s)
- nv.Value = &text
- }
- changed = true
- }
- if raw, has := pv["candidate_strength"]; has && raw != nil {
- if f, ok := common.ToFloat(raw); ok {
- v := math.Min(math.Max(f, 0.0), 1.0)
- nv.CandidateStrength = &v
- changed = true
- }
- // Unparseable strengths are ignored.
- }
- if raw, has := pv["discovered_clues"]; has {
- if list, ok := raw.([]any); ok {
- tail := list
- if len(tail) > 4 {
- tail = tail[len(tail)-4:]
- }
- for _, c := range tail {
- nv.DiscoveredClues = append(nv.DiscoveredClues, truncateRunes(fmt.Sprint(c), 160))
- }
- changed = true
- }
- }
- }
- if !changed {
- return nil
- }
- out := State{
- State: newVars,
- Depth: base.Depth + 1,
- RetrievedEvidenceIDs: append([]string(nil), base.RetrievedEvidenceIDs...),
- }
- out.ID = newStateID(out.Depth)
- return &out
-}
-
-// truncateRunes cuts s to at most n characters (runes, not bytes).
-func truncateRunes(s string, n int) string {
- if len([]rune(s)) <= n {
- return s
- }
- return string([]rune(s)[:n])
-}
-
-// ParseFloat parses a float64 from a string, tolerating surrounding
-// whitespace. Exported for the orchestrator package, whose SCA verdict
-// coercion needs the same lenient parsing.
-func ParseFloat(s string) (float64, bool) {
- f, err := strconv.ParseFloat(strings.TrimSpace(s), 64)
- if err != nil {
- return 0, false
- }
- return f, true
-}
-
-// isTruthy applies bool() truthiness to the JSON-decoded values that reach ApplyPatch:
-// None/nil, empty string/collection, zero number, and False are falsy; everything else
-// (including non-empty objects and any other type) is truthy.
-func isTruthy(v any) bool {
- switch x := v.(type) {
- case nil:
- return false
- case bool:
- return x
- case string:
- return x != ""
- case int:
- return x != 0
- case int32:
- return x != 0
- case int64:
- return x != 0
- case float32:
- return x != 0
- case float64:
- return x != 0
- case []any:
- return len(x) > 0
- case []string:
- return len(x) > 0
- case map[string]any:
- return len(x) > 0
- default:
- return true
- }
-}
-
-// Slot-table value coercion (text / int / literal)
-//
-// initialize_state reads the model's slot-table reply strictly: an id that is not an
-// integer, a type that is not text, or clues that are not a list are FAILURES, not
-// guesses — the caller discards the whole decomposition and falls back to the planner's
-// fan-outs (see buildSlotTableFrom). So these helpers report failure instead of inventing
-// a value: `int(s.get("id", i))`, `str(s.get("type") or "entity")` and
-// `[str(c) for c in (s.get("clues") or [])]` are the three questions they answer.
-//
-// quoteLiteral/formatLiteral render the LITERAL SYNTAX this protocol is written in —
-// single-quoted strings, None/True/False, [...] and {...} — which is the format the
-// prompts teach the model and the format its replies come back in.
-
-// quoteLiteral renders s as the protocol's quoted string literal: single quotes unless
-// the text contains ' but no ", with the standard escapes (\\, \n, \r, \t, \xNN for other
-// control characters). Printable non-ASCII is kept verbatim.
-func quoteLiteral(s string) string {
- quote := '\''
- if strings.ContainsRune(s, '\'') && !strings.ContainsRune(s, '"') {
- quote = '"'
- }
- var b strings.Builder
- b.WriteRune(quote)
- for _, r := range s {
- switch {
- case r == quote:
- b.WriteByte('\\')
- b.WriteRune(r)
- case r == '\\':
- b.WriteString(`\\`)
- case r == '\n':
- b.WriteString(`\n`)
- case r == '\r':
- b.WriteString(`\r`)
- case r == '\t':
- b.WriteString(`\t`)
- case r < 0x20 || r == 0x7f:
- b.WriteString(fmt.Sprintf(`\x%02x`, r))
- default:
- b.WriteRune(r)
- }
- }
- b.WriteRune(quote)
- return b.String()
-}
-
-// formatLiteral renders a JSON-decoded value in the protocol's literal syntax:
-// None/True/False, quoted strings, numbers, [...] and {...}. Map keys come out SORTED —
-// Go's maps do not preserve the JSON text's insertion order, which is the one thing this
-// cannot reproduce.
-func formatLiteral(v any) string {
- switch t := v.(type) {
- case nil:
- return "None"
- case bool:
- if t {
- return "True"
- }
- return "False"
- case string:
- return quoteLiteral(t)
- case float64:
- return strconv.FormatFloat(t, 'g', -1, 64)
- case float32:
- return strconv.FormatFloat(float64(t), 'g', -1, 32)
- case int:
- return strconv.Itoa(t)
- case int32:
- return strconv.FormatInt(int64(t), 10)
- case int64:
- return strconv.FormatInt(t, 10)
- case []any:
- parts := make([]string, 0, len(t))
- for _, item := range t {
- parts = append(parts, formatLiteral(item))
- }
- return "[" + strings.Join(parts, ", ") + "]"
- case map[string]any:
- keys := make([]string, 0, len(t))
- for k := range t {
- keys = append(keys, k)
- }
- sort.Strings(keys)
- parts := make([]string, 0, len(keys))
- for _, k := range keys {
- parts = append(parts, quoteLiteral(k)+": "+formatLiteral(t[k]))
- }
- return "{" + strings.Join(parts, ", ") + "}"
- default:
- return quoteLiteral(fmt.Sprint(v))
- }
-}
-
-// displayText is the text of a value: a string is itself, every other value comes back as
-// its literal (containers and scalars included).
-func displayText(v any) string {
- if s, ok := v.(string); ok {
- return s
- }
- return formatLiteral(v)
-}
-
-// asInt converts the values a slot table can carry, and reports ok=false where the
-// conversion is impossible: a string that is not a plain integer ("5.5" fails), None, or a
-// container.
-func asInt(v any) (int, bool) {
- switch n := v.(type) {
- case int:
- return n, true
- case int32:
- return int(n), true
- case int64:
- return int(n), true
- case float32:
- if math.IsNaN(float64(n)) || math.IsInf(float64(n), 0) {
- return 0, false
- }
- return int(n), true // truncates toward zero
- case float64:
- if math.IsNaN(n) || math.IsInf(n, 0) {
- return 0, false
- }
- return int(n), true
- case bool:
- // A bool IS an integer here, the way the protocol's literals treat it: true == 1.
- if n {
- return 1, true
- }
- return 0, true
- case string:
- // A quoted integer may carry surrounding whitespace and "_" separators.
- s := strings.ReplaceAll(strings.TrimSpace(n), "_", "")
- if s == "" {
- return 0, false
- }
- i, err := strconv.Atoi(s)
- if err != nil {
- return 0, false
- }
- return i, true
- }
- return 0, false
-}
-
-// asStringList reads a list-ish value as text: a falsy value yields nothing, a list yields
-// its items rendered as text, a string yields its characters, and a map yields its keys
-// (sorted). ok=false when the value cannot be read as a list at all.
-func asStringList(v any) ([]string, bool) {
- if !isTruthy(v) {
- return nil, true
- }
- switch t := v.(type) {
- case []any:
- out := make([]string, 0, len(t))
- for _, item := range t {
- out = append(out, displayText(item))
- }
- return out, true
- case []string:
- return append([]string(nil), t...), true
- case string:
- out := make([]string, 0, len(t))
- for _, r := range t {
- out = append(out, string(r))
- }
- return out, true
- case map[string]any:
- keys := make([]string, 0, len(t))
- for k := range t {
- keys = append(keys, k)
- }
- sort.Strings(keys)
- return keys, true
- default:
- return nil, false
- }
-}
-
-// ===================== extract.go =====================
-// JSON / terminal-block extraction from model output.
-//
-// Models wrap JSON in thinking preamble and Markdown fences, and often emit a second
-// object after the first — a greedy regex over-captures there and the parse then fails
-// with "Extra data", so the extractor brace-matches to isolate ONE complete object and
-// walks to the next "{" when that one is invalid.
-
-// ExtractJSON returns the first parseable JSON value found in text, or nil.
-func ExtractJSON(text string) any {
- if text == "" {
- return nil
- }
- n := len(text)
- for i := 0; i < n; {
- start := strings.IndexByte(text[i:], '{')
- if start < 0 {
- break
- }
- start += i
- depth := 0
- for j := start; j < n; j++ {
- switch text[j] {
- case '{':
- depth++
- case '}':
- depth--
- if depth == 0 {
- candidate := text[start : j+1]
- var out any
- if err := json.Unmarshal([]byte(candidate), &out); err == nil {
- return out
- }
- // First try a lenient parse, then the repair pass; only if BOTH fail
- // does the scan move on to the next opening brace.
- if v := repairJSONObject(candidate); v != nil {
- return v
- }
- if v := repairJSONWhole(candidate); v != nil {
- return v
- }
- // Invalid object; try the next "{".
- }
- }
- }
- i = start + 1
- }
- // All brace-matched candidates failed strict and shallow repair. The last resort is a
- // whole-text repair, which can salvage deformities (unquoted keys, dropped separators, a
- // lost outer brace) that no single brace-matched candidate can. Mirror it.
- return repairJSONWhole(text)
-}
-
-// ExtractJSONObject parses the first JSON object/array found in text, but ONLY
-// from a brace/bracket-matched region. It deliberately stops short of
-// repairJSONWhole's whole-text prose fallback: prose with no JSON shape (e.g.
-// "I cannot produce JSON today.") returns nil so the caller can treat it as a
-// parse failure. This is the right gate for gen_json-style loops, where the
-// model is expected to emit JSON (possibly fenced or trailing-comma damaged)
-// and a non-JSON reply must trigger a corrective retry rather than be silently
-// coerced into a value. Keywords-style salvage of prose-wrapped JSON should use
-// ExtractJSON instead.
-func ExtractJSONObject(text string) any {
- if text == "" {
- return nil
- }
- n := len(text)
- for i := 0; i < n; {
- start := strings.IndexByte(text[i:], '{')
- if start < 0 {
- break
- }
- start += i
- depth := 0
- for j := start; j < n; j++ {
- switch text[j] {
- case '{':
- depth++
- case '}':
- depth--
- if depth == 0 {
- candidate := text[start : j+1]
- var out any
- if err := json.Unmarshal([]byte(candidate), &out); err == nil {
- return out
- }
- if v := repairJSONObject(candidate); v != nil {
- return v
- }
- // Invalid object; try the next "{".
- }
- }
- }
- i = start + 1
- }
- return nil
-}
-
-// repairJSONObject is a conservative repair pass applied only after a candidate fails
-// strict json.Unmarshal. It walks cheap, idempotent normalizations that models commonly
-// emit and re-validates after each: trailing commas, single-quoted strings, NaN/Infinity
-// numeric literals, and illegal control characters inside string literals. It never
-// rewrites already-valid JSON and returns nil when no transform yields a parse.
-func repairJSONObject(candidate string) any {
- steps := []struct {
- name string
- apply func(string) string
- }{
- {"dropTrailingCommas", dropTrailingCommas},
- {"normalizeSingleQuotes", normalizeSingleQuotes},
- {"nullNonFinite", nullNonFiniteLiterals},
- {"stripRawControlChars", stripRawControlChars},
- }
- fixed := candidate
- for _, s := range steps {
- fixed = s.apply(fixed) // accumulate so multi-step repairs compose
- var out any
- if err := json.Unmarshal([]byte(fixed), &out); err == nil {
- return out
- }
- }
- return nil
-}
-
-// dropTrailingCommas removes a comma immediately before a closing brace or
-// bracket (e.g. {"a": 1,} or [1, 2,]), repeated until stable.
-func dropTrailingCommas(s string) string {
- for {
- before := s
- s = strings.ReplaceAll(s, ",}", "}")
- s = strings.ReplaceAll(s, ",]", "]")
- if s == before {
- return s
- }
- }
-}
-
-// normalizeSingleQuotes rewrites single-quoted string literals to double-quoted
-// ones, mirroring json_repair. It only touches quotes outside of double-quoted
-// strings so it never corrupts existing valid JSON.
-func normalizeSingleQuotes(s string) string {
- var b strings.Builder
- inDouble := false
- inSingle := false
- escaped := false
- for i := 0; i < len(s); i++ {
- ch := s[i]
- if escaped {
- b.WriteByte(ch)
- escaped = false
- continue
- }
- switch ch {
- case '\\':
- escaped = true
- b.WriteByte(ch)
- case '"':
- if !inSingle {
- inDouble = !inDouble
- }
- b.WriteByte(ch)
- case '\'':
- if !inDouble {
- inSingle = !inSingle
- b.WriteByte('"') // replace single quote with double quote
- } else {
- b.WriteByte(ch)
- }
- default:
- b.WriteByte(ch)
- }
- }
- return b.String()
-}
-
-// nullNonFiniteLiterals rewrites bare NaN / Infinity / -Infinity tokens into JSON null so
-// the object can parse.
-func nullNonFiniteLiterals(s string) string {
- s = strings.ReplaceAll(s, ": NaN", ": null")
- s = strings.ReplaceAll(s, ": Infinity", ": null")
- s = strings.ReplaceAll(s, ": -Infinity", ": null")
- return s
-}
-
-// stripRawControlChars removes raw ASCII control characters (other than tab, newline and
-// carriage-return) that would otherwise make Go's json.Unmarshal reject a string literal.
-func stripRawControlChars(s string) string {
- return strings.Map(func(r rune) rune {
- if r < 0x20 && r != '\t' && r != '\n' && r != '\r' {
- return -1
- }
- return r
- }, s)
-}
-
-// repairJSONWhole is the deep fallback applied to the WHOLE text (not just one
-// brace-matched candidate). ExtractJSON first tries the brace-matched object and the
-// shallow repairJSONObject steps; only when BOTH fail does it reach here. This salvages
-// structural deformities the single-object path cannot — unquoted object keys ({a: 1}),
-// missing separators between adjacent values ({"a":1} {"b":2} or "x" "y"), bare string
-// values ({name: foo}), and any combination the model emits across the full output. It
-// delegates to github.com/kaptinlin/jsonrepair. It returns nil when nothing parses.
-func repairJSONWhole(text string) any {
- cleaned := reThinkWrap.ReplaceAllString(text, "")
- cleaned = reFencedJSON.ReplaceAllString(cleaned, "$1")
- cleaned = strings.TrimSpace(cleaned)
- if cleaned == "" {
- return nil
- }
- // Pre-normalize unquoted object keys (e.g. {entity: "x"} -> {"entity": "x"}).
- // The repair library handles unquoted keys with unquoted values well but mangles a key
- // that precedes an already-quoted value; quoting keys first sidesteps that.
- // Already-valid JSON is left untouched.
- normalized := quoteUnquotedKeys(cleaned)
- repaired, err := jsonrepair.Repair(normalized)
- if err != nil {
- // Fall back to the un-normalized input in case the heuristic mis-fired.
- repaired, err = jsonrepair.Repair(cleaned)
- if err != nil {
- return nil
- }
- }
- return tryUnmarshal(repaired)
-}
-
-// quoteUnquotedKeys wraps bare object keys in double quotes: {a: 1} -> {"a": 1}.
-// It only touches a key token (identifier directly followed by a colon) that is
-// not already quoted, so already-valid JSON is left untouched. The optional
-// brace/bracket/comma prefix is preserved verbatim.
-func quoteUnquotedKeys(s string) string {
- return reUnquotedKey.ReplaceAllStringFunc(s, func(m string) string {
- groups := reUnquotedKey.FindStringSubmatch(m)
- return groups[1] + `"` + groups[2] + `":`
- })
-}
-
-// reUnquotedKey matches a structural prefix ({ [ ,) plus optional whitespace,
-// then a bare identifier immediately followed by a colon — the shape of an
-// unquoted JSON key. Group 1 is the prefix, group 2 is the key name.
-var reUnquotedKey = regexp.MustCompile(`([{\[,]\s*)([A-Za-z_][A-Za-z0-9_]*)\s*:`)
-
-// tryUnmarshal attempts json.Unmarshal and returns the value, or nil on failure.
-func tryUnmarshal(s string) any {
- if s == "" {
- return nil
- }
- var out any
- if err := json.Unmarshal([]byte(s), &out); err == nil {
- return out
- }
- return nil
-}
-
-var reFencedJSON = regexp.MustCompile("(?s)```(?:json)?\\s*(\\{.*?\\}|\\[.*?\\])\\s*```")
-
-// ExtractTag: exact-tag extraction first, then
-// lenient fallbacks for models that wrap the JSON in code fences or emit bare
-// objects (observed with DeepSeek-class models ignoring the XML protocol).
-//
-// Returns "" when the tag is absent.
-func ExtractTag(text, tag string) string {
- if text == "" {
- return ""
- }
- open, closeTag := "<"+tag+">", "</"+tag+">"
- s := strings.LastIndex(text, open)
- e := strings.LastIndex(text, closeTag)
- if s >= 0 && e > s {
- return strings.TrimSpace(text[s+len(open) : e])
- }
- // Fenced ```json ... ``` blocks mentioning the tag name.
- fenced := reFencedJSON.FindAllStringSubmatch(text, -1)
- if len(fenced) == 0 {
- if m, _ := regexp.MatchString(`<`+regexp.QuoteMeta(tag)+`\b`, text); !m {
- return ""
- }
- return ""
- }
- want := map[string]bool{"new_states": true}
- if tag != "state" {
- want = map[string]bool{"answer": true, "new_state": true}
- }
- // Prefer the LAST block (most recent decision).
- for i := len(fenced) - 1; i >= 0; i-- {
- var obj map[string]any
- if err := json.Unmarshal([]byte(fenced[i][1]), &obj); err != nil {
- continue
- }
- for k := range obj {
- if want[k] {
- return strings.TrimSpace(fenced[i][1])
- }
- }
- }
- return ""
-}
-
-// ===================== toolschema.go =====================
-// Tool schemas, the per-session tool object, and the two seams the action
-// session depends on.
-//
-// What the session depends on:
-// - tool specs
-// - ToolMap (the dispatch registry)
-// - the active tool surface
-// - tool disabling
-// - reason → status mapping
-// - execute_tool (line 1003)
-
-func arrayParam(desc string, minItems, maxItems int) map[string]any {
- return map[string]any{
- "type": "object",
- "properties": map[string]any{
- "query": map[string]any{
- "type": "array",
- "items": map[string]any{"type": "string"},
- "minItems": minItems,
- "maxItems": maxItems,
- },
- },
- "required": []string{"query"},
- }
-}
-
-// Tool schemas. The descriptions are the model's only guide for when to pick which tool,
-// and rephrasing them changes routing behaviour.
-var (
- retrieveToolSpec = ToolSpec{
- Type: "function",
- Function: ToolFunction{
- Name: "retrieve",
- Description: `WHEN TO CALL: you know or suspect exact surface terms in the corpus (names, titles, codes, phrases) — the first recall pass; cover different facets.` +
- `HOW IT WORKS: keyword recall FIRST, then the pattern applies to what came back — ` + "`A.*B`" + ` (A then B, anything between) matches only inside the passages its operands recalled, and the RAREST operand bounds it: put the rare word first. A FULL recall page was truncated.` +
- `ONE STRING, MANY TERMS: ` + "`A|B|C`" + ` recalls each in ONE call — synonyms go in one string.` +
- `ENUMERATING A SET: probe the NAMES themselves, alternated with |, 4-6 per query; hits are members — guess the next batch yourself.` +
- `DO NOT CALL: for a whole document (list_chunks); when no surface word matches (search_chunks).` +
- `ARGUMENTS: query — array of 1-3 strings (only the first ~10 snippets survive); doc_scope — optional doc_ids from an earlier tool result, to search INSIDE them only.` +
- `OUTPUT: Exact-term snippets with doc_id and chunk id. ok = new evidence; redundant = seen.` +
- `IF IT FAILS: a miss on an exact-term probe means the corpus lacks that term — in an enumeration that is a RESULT (probe the next). redundant = stop and emit a state patch.`,
- Parameters: map[string]any{
- "type": "object",
- "properties": map[string]any{
- "query": map[string]any{
- "type": "array",
- "items": map[string]any{"type": "string"},
- "minItems": 1,
- "maxItems": 3,
- },
- // Decoding lives in toolDocScope, which retrieve and graph_explore
- // share: the scope is ceiling-ed by the session's DocScope and
- // verified against the bound datasets (resolveDocScope), so a
- // stale or out-of-dataset doc_id cannot widen the search.
- "doc_scope": map[string]any{
- "type": "array",
- "items": map[string]any{"type": "string"},
- "description": "optional doc_ids from a prior tool result (metadata_search / navigate_tree / list_chunks), to search inside those documents only",
- },
- },
- "required": []string{"query"},
- },
- },
- }
-
- listChunksToolSpec = ToolSpec{
- Type: "function",
- Function: ToolFunction{
- Name: "list_chunks",
- Description: `WHEN TO CALL: You need the FULL text of one document (enumeration, counts, arithmetic over many passages) and you already have its doc_id from a prior tool result.` +
- `DO NOT CALL: When you only need a single passage (use search_chunks or retrieve first); when you have no doc_id yet (locate it via navigate_tree or search_chunks first).` +
- `ARGUMENTS: doc_id — string, the document id seen in a retrieve / search_chunks / navigate result. ONLY doc_id is accepted; there is no chunk_ids argument, and the tool returns the whole document (capped at 30 chunks).` +
- `OUTPUT: All chunks of the document in reading order. ok = new evidence; redundant = already in pool.` +
- `IF IT FAILS: An unknown or blank doc_id yields an empty result (query-level miss, not a dataset fact) — pick a different doc_id or locate one first. Do not treat this as a reason to disable the tool.`,
- Parameters: map[string]any{
- "type": "object",
- "properties": map[string]any{
- "doc_id": map[string]any{
- "type": "string",
- "description": "document id seen in a retrieve snippet",
- },
- },
- "required": []string{"doc_id"},
- },
- },
- }
-
- searchChunksToolSpec = ToolSpec{
- Type: "function",
- Function: ToolFunction{
- Name: "search_chunks",
- Description: `WHEN TO CALL: Primary semantic recall. Use when exact retrieve returns nothing useful, when the corpus is large and you are unsure which document holds the answer, or when the answer passage shares no surface words with your query. Send 1-2 queries; compiled-structure expansion is automatic (a no-op without compiled structure). ` +
- `DO NOT CALL: When you already have a doc_id and want to read that document (use list_chunks); when a single exact passage would be found faster by grep-style retrieve.` +
- `ARGUMENTS: query — array of 1-2 strings.` +
- `OUTPUT: Relevance-ranked snippet chunks, possibly with structural neighbours appended. ` +
- `Results may LEAD with [claim score=...] entries — the dataset's compiled atomic facts carrying VERBATIM source quotes. If a claim directly answers the query, cite it and answer WITHOUT further searching; deep-read its listed chunk only for missing context or numbers. ` +
- `ok = new evidence; redundant = already seen.` +
- `IF IT FAILS: miss means this query matched nothing — change the angle or fall back to retrieve or navigate_tree. Re-issuing a near-duplicate query is skipped as redundant, so vary the query instead of paraphrasing it.`,
- Parameters: arrayParam("", 1, 2),
- },
- }
-
- metadataSearchToolSpec = ToolSpec{
- Type: "function",
- Function: ToolFunction{
- Name: "metadata_search",
- Description: `WHEN TO CALL: SELECT the document set by METADATA before searching — call it when the question names explicit entities (a person, a time, a place) or any concrete name a metadata field would carry (a title, a file name, an author, a date), or needs a named subset. Use ONLY the AVAILABLE METADATA fields; prefer 'contains' with a distinctive substring. ` +
- `CALL AT MOST ONCE PER DIRECTION, then spend the returned doc_ids: list_chunks(doc_id), navigate_structure(doc_id, query), or retrieve(query, doc_scope=[ids]). ` +
- `DO NOT CALL: nothing names a document/subset; you already hold a doc_id; counting or enumerating. ` +
- `ARGUMENTS: filters — [{key, value, op}] over the AVAILABLE METADATA fields; op — see enum; logic 'and'|'or'. String ops take ONE keyword, one call per keyword; 'in' takes a list; 'empty' none. Example (one day): [{key: 'update_time', op: 'start with', value: '2026-09-20'}]. ` +
- `OUTPUT: doc_ids — the handle other tools take — plus each matched document's metadata as CONTEXT ONLY, never an argument to pass on. No passages. ok = selected; miss = nothing matched. ` +
- `IF IT FAILS: 'no documents match' — shorten the substring or use search_chunks; do not retry.`,
- Parameters: map[string]any{
- "type": "object",
- "properties": map[string]any{
- "filters": map[string]any{
- "type": "array",
- "minItems": 1,
- "items": map[string]any{
- "type": "object",
- "properties": map[string]any{
- // The field enum is rendered PER SESSION from the dataset's
- // catalog (see metadataSearchSpecForCatalog); the shipped
- // parameter deliberately carries no enum, so no field is
- // advertised as available before the dataset says it is.
- "key": paramString("a metadata field of this dataset — one of those listed under AVAILABLE METADATA"),
- "value": map[string]any{
- "type": []any{"string", "array", "null"},
- "description": "the keyword/value to match: ONE string keyword for contains / = / start with / " +
- "end with / not contains (call once per keyword); a list for in / not in; nothing for empty / not empty. " +
- "Copy a value exactly as the dataset stores it — never re-normalise it. " +
- "A 'time' field stores 'YYYY-MM-DD HH:MM:SS', so filter ONE day with op 'start with' and the bare " +
- "'YYYY-MM-DD' — '=' never matches a stored time.",
- },
- "op": paramEnum("the comparison", "=", "contains", "not contains", "start with", "end with", "in", "empty", "not empty"),
- },
- "required": []string{"key", "op"},
- },
- },
- "logic": paramEnum("how several filters combine, default and", "and", "or"),
- },
- "required": []string{"filters"},
- },
- },
- }
-
- webSearchToolSpec = ToolSpec{
- Type: "function",
- Function: ToolFunction{
- Name: "web_search",
- Description: `WHEN TO CALL: The needed fact is world knowledge, a recent event, or newer than the corpus (a current event, a person's alive-now status, a fresh statistic). This tool only appears when a web provider is configured.` +
- `DO NOT CALL: When the fact plausibly lives in the fixed corpus — prefer retrieve or search_chunks first. For corpus-only questions this tool is unavailable.` +
- `ARGUMENTS: query — array of 1-2 strings.` +
- `OUTPUT: Web results shaped like corpus chunks, merged into the same evidence pool.` +
- `IF IT FAILS: error (no provider) — it will not appear at all this session; if it does appear and fails, switch to corpus tools permanently and do not retry it.`,
- Parameters: arrayParam("", 1, 2),
- },
- }
-
- // wikiQueryToolSpec is deliberately absent from ToolMap: wiki_query has no caller and is
- // not part of the action session's dispatch registry, so it stays unregistered. The
- // handler still lives in tool_exploration.go as an unplugged extension seam.
-
- navigateTreeToolSpec = ToolSpec{
- Type: "function",
- Function: ToolFunction{
- Name: "navigate_tree",
- Description: `WHEN TO CALL: The question names a topic, entity, or alias but you do NOT know which document discusses it, especially on a large corpus. Routes by topic or cluster similarity over the compiled navigation tree.` +
- `DO NOT CALL: When you already hold a doc_id (go straight to navigate_structure); when the answer is likely a single exact passage (use retrieve or search_chunks).` +
- `ARGUMENTS: query — string, the topic / entity / alias whose document(s) to locate. Note: keywords is read by the executor but is NOT a declared parameter; do not pass it.` +
- `OUTPUT: Candidate doc_ids plus a first-chunk summary of each; these become your known-docs set for the next step.` +
- `IF IT FAILS: empty (no_structure) means the dataset has no compiled navigation tree — immediately switch to search_chunks. A second such empty disables this tool for the rest of the session, so do not retry it.`,
- Parameters: map[string]any{
- "type": "object",
- "properties": map[string]any{
- "query": map[string]any{
- "type": "string",
- "description": "topic / entity / alias whose document(s) to locate",
- },
- },
- "required": []string{"query"},
- },
- },
- }
-
- navigateStructureToolSpec = ToolSpec{
- Type: "function",
- Function: ToolFunction{
- Name: "navigate_structure",
- Description: `WHEN TO CALL: You know the doc_id and need to PINPOINT where the answer lives inside that one document, without reading every chunk. The in-document counterpart of navigate_tree.` +
- `DO NOT CALL: When you have no doc_id yet; when the document has no compiled structure (use list_chunks to read the full document).` +
- `ARGUMENTS: doc_id — string, required. query — string, what to locate within the document. kind — enum catalog / mindmap / graph, default catalog (compiled-structure kind).` +
- `OUTPUT: The structure outline annotated with matching chunk_ids, reading-order aware. [claim] lines carry VERBATIM quotes from the document — cite them and answer WITHOUT calling list_chunks when they directly answer the query (deep-read the claimed chunk ids only for surrounding context or numbers the quotes lack). ` +
- `ok = useful hits; poor (chunk_ptrs = 0) means it drilled to nothing usable.` +
- `IF IT FAILS: empty (no_structure) — try another doc_id or kind, or fall back to list_chunks / search_chunks. poor — read the full document via list_chunks(doc_id). A second empty disables the tool for the session.`,
- Parameters: map[string]any{
- "type": "object",
- "properties": map[string]any{
- "doc_id": paramString("document id seen from a prior tool result"),
- "query": paramString("what to locate within the document"),
- "kind": paramEnum("compiled structure kind, default catalog", "catalog", "mindmap", "graph"),
- },
- "required": []string{"doc_id"},
- },
- },
- }
-
- calculateToolSpec = ToolSpec{
- Type: "function",
- Function: ToolFunction{
- Name: "calculate",
- Description: `WHEN TO CALL: The question asks you to DERIVE a number by combining facts you found (sum / difference / percentage / ratio / sort / compare / length / age / price / area / growth). NEVER do arithmetic mentally.` +
- `DO NOT CALL: When the answer IS one of the stated numbers (no combination needed) — answer directly. When a needed number is still missing — retrieve it first; do not estimate.` +
- `ARGUMENTS: question — string, the user question verbatim. facts — array of strings, the numbers or facts found in evidence, verbatim (keep the original language; pass them exactly as written).` +
- `OUTPUT: an object with expression and result — report the computed result verbatim.` +
- `IF IT FAILS: poor (no numeric answer derivable) — retrieve more numbers, or answer directly if the answer is already stated. Never fabricate a computation.`,
- Parameters: map[string]any{
- "type": "object",
- "properties": map[string]any{
- "question": paramString("the user's question, verbatim"),
- "facts": map[string]any{
- "type": "array",
- "items": map[string]any{"type": "string"},
- "description": "numbers/facts found in the evidence, verbatim",
- },
- },
- "required": []string{"question", "facts"},
- },
- },
- }
-
- graphExploreToolSpec = ToolSpec{
- Type: "function",
- Function: ToolFunction{
- Name: "graph_explore",
- Description: ("EXPLORE the compiled KNOWLEDGE GRAPH (entities + relations) for a " +
- "RELATIONAL/multi-hop answer. Different from navigate_*: instead of " +
- "locating a document or passage, it seeds entities for the query, " +
- "hops along their RELATIONS, and returns either a direct answer or " +
- "the source passages behind the relevant entities/relations. " +
- "Use when the answer requires connecting several entities through " +
- "their relations (e.g. who-was-related-to-whom, cause-effect chains, " +
- "membership/ownership) and you already have a starting entity from a " +
- "search result, a navigation outline, or a list_chunks reading. " +
- "If the dataset has NO compiled knowledge graph, it returns empty — " +
- "fall back to search_chunks / navigate_structure."),
- Parameters: map[string]any{
- "type": "object",
- "properties": map[string]any{
- "query": paramString("the relational question / starting entity"),
- "doc_scope": map[string]any{
- "type": "array",
- "items": map[string]any{"type": "string"},
- "description": "optional doc_ids to restrict the graph to (from prior navigation/list_chunks)",
- },
- },
- "required": []string{"query"},
- },
- },
- }
-)
-
-func paramString(desc string) map[string]any {
- return map[string]any{"type": "string", "description": desc}
-}
-
-func paramEnum(desc string, values ...string) map[string]any {
- return map[string]any{"type": "string", "enum": values, "description": desc}
-}
-
-const (
- // maxToolDescriptionRunes is the per-tool description budget TestToolSpecsHavePlaybookSections
- // pins.
- maxToolDescriptionRunes = 1200
-)
-
-// metadataSearchSpecForCatalog renders the metadata_search schema for a session's catalog: the
-// `key` enum becomes the fields the dataset offers, so a model can name any of them. The
-// description is dataset-independent (it points at AVAILABLE METADATA and never names a field),
-// so it needs no per-session rewriting and cannot outgrow its budget.
-//
-// An empty (or nil) catalog returns the base spec UNCHANGED: no field is advertised at all —
-// the `key` stays the free-form string the shipped spec carries, which is also how an empty
-// enum is avoided (some providers reject `"enum": []`).
-//
-// The base spec is the shared package-level ToolMap entry, so every map this touches is
-// COPIED — mutating them in place would leak one session's dataset fields into every other
-// session (and into a session's concurrent rag calls, see Toolset.mu).
-func metadataSearchSpecForCatalog(base ToolSpec, cat *MetadataCatalog) ToolSpec {
- if cat == nil || cat.Empty() {
- return base
- }
-
- baseProps, _ := base.Function.Parameters["properties"].(map[string]any)
- filters, _ := baseProps["filters"].(map[string]any)
- items, _ := filters["items"].(map[string]any)
- itemProps, _ := items["properties"].(map[string]any)
- if baseProps == nil || filters == nil || items == nil || itemProps == nil {
- // The schema shape changed under us; the unpatched spec is still valid.
- return base
- }
-
- patchedItemProps := copyAnyMap(itemProps)
- patchedItemProps["key"] = paramEnum("one of this dataset's metadata fields (see AVAILABLE METADATA)", cat.Keys...)
-
- patchedItems := copyAnyMap(items)
- patchedItems["properties"] = patchedItemProps
-
- patchedFilters := copyAnyMap(filters)
- patchedFilters["items"] = patchedItems
-
- patchedProps := copyAnyMap(baseProps)
- patchedProps["filters"] = patchedFilters
-
- patchedParams := copyAnyMap(base.Function.Parameters)
- patchedParams["properties"] = patchedProps
-
- out := base
- out.Function.Parameters = patchedParams
- return out
-}
-
-// copyAnyMap shallow-copies a JSON-shaped map, so a patched tool spec shares no map with
-// ToolMap.
-func copyAnyMap(m map[string]any) map[string]any {
- out := make(map[string]any, len(m))
- for k, v := range m {
- out[k] = v
- }
- return out
-}
-
-// ToolMap is the multi-tool registry.
-// executeTool dispatches by name; add a tool by registering its schema here.
-var ToolMap = map[string]ToolSpec{
- "retrieve": retrieveToolSpec,
- "search_chunks": searchChunksToolSpec,
- "metadata_search": metadataSearchToolSpec,
- "list_chunks": listChunksToolSpec,
- "navigate_tree": navigateTreeToolSpec,
- "navigate_structure": navigateStructureToolSpec,
- "calculate": calculateToolSpec,
- "graph_explore": graphExploreToolSpec,
- "web_search": webSearchToolSpec,
-}
-
-// ToolMapNames are the registered tool names, sorted for deterministic output.
-func ToolMapNames() []string {
- names := make([]string, 0, len(ToolMap))
- for n := range ToolMap {
- names = append(names, n)
- }
- sort.Strings(names)
- return names
-}
-
-// The per-session tool object
-
-// ToolExecutor runs one tool call by name and reports what happened.
-//
-// Every executor returns a ToolOutcome so the tool node can act on the KIND of
-// result (empty / miss / poor / redundant / error) instead of measuring payload
-// size.
-type ToolExecutor interface {
- Execute(ctx context.Context, name string, args map[string]any) (ToolOutcome, error)
-}
-
-// Toolset is the per-session tool object: thinking mode, web provider availability, the
-// runtime-disabled tool set, and the executor.
-//
-// The disabled-tool state lives here rather than on the request object, so a session
-// cannot leak it into another request.
-type Toolset struct {
- // ThinkingMode selects the ModeSpec (see config.go).
- ThinkingMode string
- // HasWebSearch reports whether a web provider is configured. When false,
- // web_search is hidden from the surface rather than merely discouraged.
- HasWebSearch bool
- // MetadataFields is the session's metadata catalog (see MetadataCatalogFor). It supplies
- // the metadata_search key enum and rides the session seed, so a filter can name the fields
- // the dataset really carries. Nil or empty advertises no field at all — there is no field
- // name baked into the shipped schema to fall back on.
- MetadataFields *MetadataCatalog
- // DisabledTools holds tools proven unavailable this session (no compiled
- // structure of their kind).
- DisabledTools map[string]bool
- // mu guards DisabledTools: one Toolset is shared by a session's concurrent
- // rag calls (models.appendToolResults runs a round's tool calls in parallel),
- // so the map must not be written and read unsynchronized.
- mu sync.Mutex
- // Exec runs the tools.
- Exec ToolExecutor
-}
-
-// GetThinkingMode implements ThinkingModeCarrier so ResolveMode works on *Toolset.
-func (t *Toolset) GetThinkingMode() string {
- if t == nil {
- return ""
- }
- return t.ThinkingMode
-}
-
-// ActiveToolSpecs: the tool schemas exposed
-// to the model for THIS mode.
-//
-// Visibility rules:
-// - the per-mode tool set comes from config.THINKING_MODES (only ultra sees
-// the relational graph_explore);
-// - web_search is hidden when no web provider is configured — the model
-// otherwise retries it several times per session burning turns;
-// - compile-only tools stay visible even on datasets without compiled
-// structure: their executors return an explicit empty-with-hint so the
-// model falls back to plain search;
-// - RUNTIME: a compile-only tool is removed after it proves the dataset has
-// no such compiled structure (DisableTool).
-func (t *Toolset) ActiveToolSpecs() []ToolSpec {
- spec := ResolveMode(t)
- var out []ToolSpec
- for _, name := range spec.ToolNames() {
- if name == "web_search" && !t.HasWebSearch {
- continue
- }
- if t.IsDisabled(name) {
- continue
- }
- if s, ok := ToolMap[name]; ok {
- if name == "metadata_search" {
- // Per-session rewrite: the catalog names the dataset's real fields. It
- // returns the shared spec untouched when there is no catalog.
- s = metadataSearchSpecForCatalog(s, t.MetadataFields)
- }
- out = append(out, s)
- }
- }
- return out
-}
-
-// DisableTool: mark a compile-only tool
-// unavailable for the REST of this session, so ActiveToolSpecs stops
-// advertising it and ExecuteTool short-circuits it.
-func (t *Toolset) DisableTool(name string) {
- if _, ok := ToolMap[name]; !ok {
- return
- }
- t.mu.Lock()
- defer t.mu.Unlock()
- if t.DisabledTools == nil {
- t.DisabledTools = map[string]bool{}
- }
- t.DisabledTools[name] = true
-}
-
-// IsDisabled reports whether name has been disabled this session.
-func (t *Toolset) IsDisabled(name string) bool {
- t.mu.Lock()
- defer t.mu.Unlock()
- return t.DisabledTools[name]
-}
-
-// ReasonStatus: single source of truth for the cause→status mapping, so a tool cannot
-// disagree with itself about what its own reason means.
-// The unified tool-result contract shared by the session and the tool implementations
-// (ToolOutcome plus the OK/EMPTY/MISS/POOR/REDUNDANT/ERROR statuses).
-
-// Tool-result statuses: the signal the session's tool node acts on.
-const (
- StatusOK = "ok" // normal hit
- StatusEmpty = "empty" // dataset-level: no such compiled structure exists here
- StatusMiss = "miss" // query-level: nothing matched THIS query; tool still valid
- StatusPoor = "poor" // produced output, but too weak to be useful
- StatusRedundant = "redundant" // ran fine, but added no NEW evidence
- StatusError = "error" // infra / provider failure
-)
-
-// Machine-readable ToolOutcome reasons. Only ReasonNoStructure is
-// DATASET-level and may disable a tool; ReasonNoDoc is QUERY-level and must not.
-// ReasonUnwired labels a call to a tool NAME this deployment has no binding for
-// (the model invented it): nothing ran, so it is reported as MISS and never
-// counts toward the strikes that disable a real tool.
-const (
- ReasonNone = ""
- ReasonNoStructure = "no_structure"
- ReasonNoDoc = "no_doc"
- ReasonInfra = "infra"
- ReasonBadArgs = "bad_args"
- ReasonUnwired = "unwired"
-)
-
-// ToolOutcome is the result of ONE tool call. Every executor returns one so the
-// tool node can inspect WHAT happened instead of counting characters in a JSON
-// string — timeout / budget / routing / disable policies hook into these fields.
-type ToolOutcome struct {
- Payload []any // what the model sees (passage dicts)
- EvidenceIDs []string // doc/chunk ids worth tracking
- Status string // one of the Status* constants
- Reason string // one of the Reason* constants
- Metrics map[string]any // numeric signals: hits, new_evidence, top_score, ...
- // Note is a line ABOUT this call, rendered into the tool result next to the
- // passages (see toolNode). It carries what the passages cannot say: which of
- // the terms the call asked about were reached and which were not, so a batch
- // probe ("华雄|颜良|蔡阳") is actionable rather than four passages whose
- // missing member looks exactly like a member nobody asked for.
- Note string
- // Diagnostic is the tool's own explanation of a failure (Go-only: Python's
- // ToolOutcome carries no such field). A bare Reason — "infra" — tells a
- // reader nothing about what went wrong, so the underlying message rides
- // along and is reported with the step. It is NEVER put in Payload: the model
- // sees the status note, not the infrastructure detail.
- Diagnostic string
-}
-
-// NewToolOutcome builds an OK outcome with an empty metric map.
-func NewToolOutcome(payload []any, evidenceIDs []string) ToolOutcome {
- return ToolOutcome{
- Payload: payload,
- EvidenceIDs: evidenceIDs,
- Status: StatusOK,
- Reason: ReasonNone,
- Metrics: map[string]any{},
- }
-}
-
-// ReasonStatus maps a ToolOutcome reason to the Status it implies.
-func ReasonStatus(reason string) string {
- switch reason {
- case ReasonNoStructure:
- return StatusEmpty
- case ReasonBadArgs, ReasonInfra:
- return StatusError
- }
- // ReasonNoDoc: this query reached nothing; the tool itself is fine.
- return StatusMiss
-}
-
-// ===================== session.go (model seam, _SessionState, nodes, routing) =====================
-// The graph-edge action session: one bounded ReAct loop pursuing ONE direction.
-// THIS FILE USES AN EXPLICIT LOOP, NOT A COMPILED GRAPH: the graph is small and fixed
-// (see sessionLoop), so three functions and a switch buy more than compiling one. The node
-// bodies, routing predicates and their ordering follow the same design; only the driver
-// differs.
-//
-// The provider seam calls the native tool-calling API (`tools=[...]`,
-// `tool_choice="auto"`) and parses `message.tool_calls`: the chat seam exposes that as
-// chat.Request.Tools / Response.ToolCalls, and the SessionModel implementation
-// (InvokerSessionModel) forwards the runtime ToolSpec surface through it. There is no
-// prompt-based fallback: downstream session logic is identical either way.
-
-const (
- // initTimeoutS bounds the slot-table decomposition call.
- initTimeoutS = 45.0
- // actionTimeoutS is the default per-session wall-clock budget.
- actionTimeoutS = 75.0
- // setActionTimeoutS is the wall clock an ENUMERATION session gets, i.e. a
- // session whose direction declared a set (see SessionWallS). The extra time is
- // not a bigger turn budget — the turns are what the enumeration spends, and one
- // of them is a batch of names whose results have to be read and patched: a session can be
- // cancelled at its wall clock a millisecond before the patch that would have recorded a
- // member it had already reached — its passages admitted to the shared pool by its own
- // batch. A value question has nothing to spend the extra time on, so it keeps the tighter
- // clock.
- setActionTimeoutS = 150.0
- // snippetsPerQuery is the FALLBACK per-query snippet cap, used when the mode
- // leaves ModeSpec.SnippetsPerQuery unset. The live value is
- // snippetsPerQueryFor(RunRequest.ThinkingMode).
- snippetsPerQuery = 4
- // evidenceDigestChars bounds ONE chunk in the session seed's ALREADY RETRIEVED
- // digest. It matches what an admitted passage carries (passageFromChunk: 1200), so
- // the digest never shows a session LESS of a chunk than the same chunk would carry
- // as a tool result.
- //
- // The previous 300-code-point cut ended mid-sentence on narrative passages, and the
- // model — told to answer only from what it was shown — excluded the parties whose clause
- // fell outside the window: a cap smaller than one sentence truncates exactly the evidence
- // the answer is built from, even when the passage is in the pool.
- evidenceDigestChars = 1200
- // maxToolResponseChars bounds ONE tool payload.
- maxToolResponseChars = 12000
- // emptyStrikes is how many dataset-level empties (reason=no_structure) a
- // compiled-structure tool must accumulate before being disabled for the rest
- // of the session. Kept above 1: a single empty can be SCOPED — graph_explore
- // over a doc_scope that has no knowledge graph says nothing about the other
- // documents, and disabling on the first empty is exactly the over-eager
- // behaviour this replaced.
- emptyStrikes = 2
- // nearDupJaccard is the token-overlap threshold at which a retrieval query
- // counts as a paraphrase of an already-run query. The model re-issues the
- // SAME intent with many paraphrases (observed: 20+ variants in ONE session),
- // each hitting the index even though the evidence is unchanged.
- nearDupJaccard = 0.8
- // skippedDupLimit: once this many near-duplicate retrievals were skipped,
- // further turns are unlikely to surface new evidence, so the session
- // converges to finalize instead of burning the remaining budget on
- // paraphrases (Q30/Q759 timeout root cause).
- skippedDupLimit = 2
- // turnRunExtra is how many turns the MODEL may add beyond the mode's floor
- // (see offerContinuation): medium/high run 8 → 12, ultra 10 → 14. It is the
- // hard bound the runtime keeps while the decision itself is the model's, so an
- // eager model cannot turn one session into a whole research programme.
- turnRunExtra = 4
- // valueTurnFloor is the floor a session runs on when it is NOT assembling a set
- // (see actionMaxTurns). It is the floor every mode had before the enumeration
- // path asked for a larger one (medium/high 4 → 8, ultra 6 → 10): the extra turns
- // exist to spend a batch of NAMES turn after turn, and a session that never
- // writes one has no batch to spend them on — while the mode's number is billed
- // to every turn of every session on the question. A raised floor gives a batch-writing
- // session the turns it needs, and a question that writes almost no batches pays it on
- // every session.
- valueTurnFloor = 4
- // turnAskFloorS is the session clock below which no further turn is offered:
- // the finalize/salvage step must still fit, or the extra turn buys evidence
- // that never reaches the slot table.
- turnAskFloorS = 20.0
- // finalizeTimeout bounds the salvage call in the finalize node.
- finalizeTimeoutS = 150.0
- // minFinalizeTimeout is the floor for the salvage call.
- minFinalizeTimeoutS = 15.0
-)
-
-// snippetsPerQueryFor resolves the per-query snippet cap for a thinking mode.
-//
-// The cap decides how much of ONE query's candidate list the session reads, and
-// that list is where a themed query's fact-bearing passage sits: the engine
-// returns 30-60 candidates per leg, and on a 64-candidate leg the passage that
-// carried the answer ranked 20th and 38th — both discarded by a cap of four,
-// while handing the model those same passages produced the complete list.
-//
-// It therefore rises with the mode (a deeper mode issues more queries and owns a
-// larger budget) and falls back to the flat snippetsPerQuery when the mode leaves
-// it unset, so a mode that never declared one behaves exactly as before.
-func snippetsPerQueryFor(mode string) int {
- if spec := GetMode(mode); spec.SnippetsPerQuery > 0 {
- return spec.SnippetsPerQuery
- }
- return snippetsPerQuery
-}
-
-// retrievalTools: near-duplicate suppression applies to these. grep_chunks / grep_search
-// are legacy names kept so a model emitting them is still deduped rather than executed.
-var retrievalTools = map[string]bool{
- "search_chunks": true,
- "grep_chunks": true,
- "grep_search": true,
-}
-
-var _LOG = common.StdLogger()
-
-var reSearchToken = regexp.MustCompile(`[a-z0-9]{2,}`)
-
-// searchTokens: lowercased alphanumeric tokens
-// of a query, for near-duplicate detection.
-func searchTokens(q string) map[string]struct{} {
- out := map[string]struct{}{}
- for _, t := range reSearchToken.FindAllString(strings.ToLower(q), -1) {
- out[t] = struct{}{}
- }
- return out
-}
-
-// argQueryString coerces a tool argument to a trimmed query string: a string passes
-// through, a nil/empty value yields "", and any other value — typically the []any query
-// list retrieve accepts — is stringified so it still participates in near-dup detection
-// and the seen_queries ledger.
-func argQueryString(v any) string {
- switch t := v.(type) {
- case nil:
- return ""
- case string:
- if t == "" {
- return ""
- }
- return strings.TrimSpace(t)
- case []any:
- if len(t) == 0 {
- return ""
- }
- parts := make([]string, 0, len(t))
- for _, e := range t {
- parts = append(parts, fmt.Sprintf("%v", e))
- }
- return strings.TrimSpace("[" + strings.Join(parts, ", ") + "]")
- default:
- return strings.TrimSpace(fmt.Sprintf("%v", t))
- }
-}
-
-// IsNearDup: true when q shares >=
-// nearDupJaccard of its tokens with any query in seen.
-func IsNearDup(q string, seen []string) bool {
- if strings.TrimSpace(q) == "" || len(seen) == 0 {
- return false
- }
- toks := searchTokens(q)
- if len(toks) < 2 {
- return false
- }
- for _, s := range seen {
- other := searchTokens(s)
- inter := 0
- for t := range toks {
- if _, ok := other[t]; ok {
- inter++
- }
- }
- union := len(toks)
- for t := range other {
- if _, ok := toks[t]; !ok {
- union++
- }
- }
- if union > 0 && float64(inter)/float64(union) >= nearDupJaccard {
- return true
- }
- }
- return false
-}
-
-// The model seam
-
-// The production carrier always supports per-call temperatures: the pinned per-node
-// temperatures (keywords 0.1 / structure_qa 0.2 / compute 0.0) must never silently fall
-// back to a model default.
-var _ TemperatureModel = (*InvokerSessionModel)(nil)
-
-// CompleteWithTemperature implements TemperatureModel.
-func (m *InvokerSessionModel) CompleteWithTemperature(ctx context.Context, messages []schema.Message, tools []ToolSpec, temp float64) (*ModelReply, error) {
- if m == nil || m.Invoker == nil {
- return nil, fmt.Errorf("runtime: no chat invoker configured")
- }
- // The tools are bound natively (tool_choice="auto") and the result is read out of
- // msg.tool_calls. There is no prompt-based fallback: a provider that cannot express
- // native tools simply has no tool capability.
- req := chat.Request{Messages: messages, Temperature: &temp}
- if chatTools := toChatTools(tools); chatTools != nil {
- req.Tools = chatTools
- req.ToolChoice = chat.ToolChoiceAuto
- }
- resp, err := m.Invoker.Invoke(ctx, m.DB, req)
- if err != nil {
- return nil, err
- }
- reply := &ModelReply{Content: resp.Content}
- reply.Usage = resp.Usage
- reply.Tokens = totalOf(resp)
- reply.ToolCalls = nativeToHarnessCalls(resp.ToolCalls)
- return reply, nil
-}
-
-// InvokerSessionModel adapts chat.Invoker to SessionModel using the provider's NATIVE
-// tool calling (native tools with tool_choice="auto", reading back msg.tool_calls).
-//
-// There is no prompt-based fallback: a provider that cannot express native tools simply
-// has no tool capability. A reply with no tool_calls is a no-call turn, which the session
-// nudges.
-type InvokerSessionModel struct {
- Invoker chat.Invoker
- DB *gorm.DB
- // MaxLength is the resolved chat model's context window in tokens. Used by
- // ContextLength to size the prompt fit. <= 0 means "unknown", in which case
- // ContextLength reports chat.EffectiveContextLength's 8192 fallback.
- MaxLength int
-}
-
-// ContextLength implements ContextLengthModel. It returns the model's context
-// window in tokens when known, otherwise the 8192 default used when the config omits a
-// context length.
-func (m *InvokerSessionModel) ContextLength() int {
- if m == nil {
- return chat.EffectiveContextLength(0)
- }
- return chat.EffectiveContextLength(m.MaxLength)
-}
-
-// Complete implements SessionModel.
-func (m *InvokerSessionModel) Complete(ctx context.Context, messages []schema.Message, tools []ToolSpec) (*ModelReply, error) {
- if m == nil || m.Invoker == nil {
- return nil, fmt.Errorf("runtime: no chat invoker configured")
- }
- temp := 0.3
- req := chat.Request{Messages: messages, Temperature: &temp}
- if chatTools := toChatTools(tools); chatTools != nil {
- req.Tools = chatTools
- req.ToolChoice = chat.ToolChoiceAuto
- }
- resp, err := m.Invoker.Invoke(ctx, m.DB, req)
- if err != nil {
- return nil, err
- }
- reply := &ModelReply{Content: resp.Content}
- reply.Usage = resp.Usage
- reply.Tokens = totalOf(resp)
- reply.ToolCalls = nativeToHarnessCalls(resp.ToolCalls)
- return reply, nil
-}
-
-// StreamComplete implements StreamingSessionModel by delegating to the
-// underlying invoker when it can stream. A non-streaming invoker yields an
-// error, which the caller treats as "fall back to the one-shot call" — the
-// prompt is identical either way, so the answer is not affected.
-func (m *InvokerSessionModel) StreamComplete(ctx context.Context, messages []schema.Message, tools []ToolSpec, onDelta func(delta string, isThink bool) error) (*ModelReply, error) {
- if m == nil || m.Invoker == nil {
- return nil, fmt.Errorf("runtime: no chat invoker configured")
- }
- streamer, ok := m.Invoker.(chat.StreamingInvoker)
- if !ok {
- return nil, fmt.Errorf("runtime: chat invoker %T does not support streaming", m.Invoker)
- }
- temp := 0.3
- req := chat.Request{Messages: messages, Temperature: &temp}
- if chatTools := toChatTools(tools); chatTools != nil {
- req.Tools = chatTools
- req.ToolChoice = chat.ToolChoiceAuto
- }
- resp, err := streamer.Stream(ctx, m.DB, req, onDelta)
- if err != nil {
- return nil, err
- }
- if resp == nil {
- return nil, fmt.Errorf("runtime: streaming invoker returned no response")
- }
- reply := &ModelReply{
- Content: resp.Content,
- Usage: resp.Usage,
- Tokens: totalOf(resp),
- }
- reply.ToolCalls = nativeToHarnessCalls(resp.ToolCalls)
- return reply, nil
-}
-
-// totalOf returns the total token count for a chat.Response, preferring the
-// Usage split and falling back to the legacy single counter.
-func totalOf(resp *chat.Response) int {
- if resp == nil {
- return 0
- }
- if resp.Usage != nil {
- return resp.Usage.TotalTokens
- }
- return resp.Tokens
-}
-
-// toChatTools converts the runtime ToolSpec surface into the chat seam's native Tool
-// declarations. The chat seam forwards them to the provider as native tools; there is no
-// prompt-based fallback.
-func toChatTools(tools []ToolSpec) []chat.Tool {
- if len(tools) == 0 {
- return nil
- }
- out := make([]chat.Tool, 0, len(tools))
- for _, t := range tools {
- if t.Function.Name == "" {
- continue
- }
- out = append(out, chat.Tool{
- Type: "function",
- Function: chat.ToolFunction{
- Name: t.Function.Name,
- Description: t.Function.Description,
- Parameters: t.Function.Parameters,
- },
- })
- }
- if len(out) == 0 {
- return nil
- }
- return out
-}
-
-// knownTool reports whether name is a real tool, judged against the STATIC full set
-// (ToolMap).
-//
-// It deliberately does NOT consult the active surface. Disabled or web-hidden
-// tools are real tools that this session has stopped advertising; a model that
-// still calls one must get the "unavailable, use this instead" note from
-// ExecuteTool, not an "unknown tool" correction. Judging against the active
-// surface would turn every disabled call into Unknown and override that
-// deliberately gentler degradation path.
-func knownTool(name string) bool {
- _, ok := ToolMap[name]
- return ok
-}
-
-// nativeToHarnessCalls maps chat-seam native tool calls into the runtime ToolCall
-// shape, preserving the Unknown flag for names outside the static tool map.
-//
-// A missing id is synthesized as `call_{i}` — without it the assistant message and its
-// tool responses cannot be paired and the next request is rejected with
-// "tool call result does not follow tool call".
-func nativeToHarnessCalls(calls []chat.ToolCall) []ToolCall {
- if len(calls) == 0 {
- return nil
- }
- out := make([]ToolCall, 0, len(calls))
- for i, c := range calls {
- id := c.ID
- if id == "" {
- id = fmt.Sprintf("call_%d", i)
- }
- out = append(out, ToolCall{
- ID: id,
- Name: c.Name,
- Args: c.Arguments,
- Unknown: !knownTool(c.Name),
- })
- }
- return out
-}
-
-// ensureToolCallIDs backfills synthesized ids for calls produced by a
-// SessionModel that did not assign one, so the assistant message and every tool
-// response reference the same id (see nativeToHarnessCalls).
-func ensureToolCallIDs(calls []ToolCall) {
- for i := range calls {
- if calls[i].ID == "" {
- calls[i].ID = fmt.Sprintf("call_%d", i)
- }
- }
-}
-
-// assistantToolCalls renders runtime tool calls as the schema assistant message's
-// tool_calls: the model's native tool_calls are passed through verbatim so they pair with
-// the tool
-// responses. Passing nil here leaves a dangling tool_call and the next request
-// is rejected.
-func assistantToolCalls(calls []ToolCall) []schema.ToolCall {
- if len(calls) == 0 {
- return nil
- }
- out := make([]schema.ToolCall, 0, len(calls))
- for _, c := range calls {
- raw, err := json.Marshal(c.Args)
- if err != nil || c.Args == nil {
- raw = []byte("{}")
- }
- out = append(out, schema.ToolCall{
- ID: c.ID,
- Type: "function",
- Function: schema.FunctionCall{Name: c.Name, Arguments: string(raw)},
- })
- }
- return out
-}
-
-// Session state
-
-// SessionState: (the LangGraph TypedDict).
-type SessionState struct {
- // Messages is the running conversation (system + user + assistant + tool).
- Messages []schema.Message
- // ParentState is the slot table being patched by this session.
- ParentState State
- Tools *Toolset
- Model SessionModel
-
- // PendingCalls are tool calls awaiting execution this turn.
- PendingCalls []ToolCall
- Done bool
-
- NewStates []State
- FoundAnswer *string
-
- RetrievedEvidenceIDs []string
- Attempts int
- DeadlineLeft float64
-
- // CtxBudget is the cumulative tool-payload char ceiling for the session.
- CtxBudget int
- // ToolChars is the running total of tool-payload chars emitted so far (O(1)
- // accounting).
- ToolChars int
- // ToolCache avoids re-executing an identical (name, args) call. It is the
- // per-round cache shared with the round's other sessions, so it is a guarded
- // *ToolCache rather than a bare map.
- ToolCache *ToolCache
- // SearchQueries accumulates retrieval queries for near-dup detection.
- SearchQueries []string
- // SkippedDup counts near-duplicate retrievals suppressed so far.
- SkippedDup int
- // MetadataSearchUsed is the metadata_search ONE-SHOT: the tool SELECTS a document
- // set, so a second call within the SAME direction is blocked with a nudge (Python's
- // _metadata_search_used). It survives across the session's turns but not across
- // directions — each direction runs its own session.
- MetadataSearchUsed bool
- // ToolStrikes counts dataset-level empties per tool.
- ToolStrikes map[string]int
- // ToolOutcomes is the audit trail of (name, status, reason, metrics).
- ToolOutcomes []map[string]any
-
- // KB is the shared evidence pool. The session reads it for the per-turn
- // RECORD line (see SessionRecord) and for the names the evidence offers;
- // the tools write it. Nil skips both.
- KB *Kbinfos
- // Record is the last turn's record — the facts the continuation decision is
- // made from (see offerContinuation and SessionRecord).
- Record SessionRecord
- // PoolWalk is how many pool chunks this session has already looked at while
- // choosing an excerpt to show (see unreadPoolExcerpt). The pool only ever
- // appends, so the watermark is what keeps a session from being shown the same
- // passage twice.
- PoolWalk int
- // PoolRead is whether this session has already been given its one pool excerpt
- // (see unreadPoolExcerpt): the read is a last resort, not a routine.
- PoolRead bool
- // ContinuationAsked is the Attempts value the continuation offer was appended
- // for, so one turn never carries the offer twice.
- ContinuationAsked int
- // EnumerationProtocol is the SET-direction method this session MAY be handed:
- // seeded when the table declared a set (see setProtocolFor), and appended once
- // mid-session when the caller writes its first batch (see appendBatchProtocol).
- // Resolved once at seed time so nothing mid-session needs the prompt loader.
- EnumerationProtocol string
- // BatchProtocolShown is whether the mid-session append has happened. Once per
- // session: the method repeated is prompt noise.
- BatchProtocolShown bool
- // floorGateLogged records that the turn floor was reported (see actionMaxTurns).
- // The floor is resolved on every turn, so only the first resolution is worth a
- // log line — it is what says whether the shape gate fired for this session.
- floorGateLogged bool
-
- Direction string
- RoutedDocs []string
- // NavHint is the joined routed-doc summaries carried by the navigation
- // ladder; it soft-boosts later retrieval ("nav is a hint, not a constraint")
- // without restricting the corpus.
- NavHint string
- // NavRuleID is where the navigation ladder currently rests ("" = the ladder
- // finished or was never armed).
- NavRuleID string
-
- TerminalType *string
- TerminalPayload map[string]any
-}
-
-// appendMessages is the single place a session grows its message list.
-//
-// The message list only ever grows by appending: no message id is set (a fresh uuid per
-// message means ids never collide) and nothing here reads one — the `.id` fields in play
-// are tool-call and slot/nav-rule ids.
-//
-// Deliberately NOT a tool_call_id dedupe: fallback ids are per-turn
-// (`ensureToolCallIDs` assigns call_0, call_1… to unnamed calls), so two turns can both
-// carry call_0 and dropping the earlier response would orphan that turn's assistant
-// tool_calls. Assistant turns are new by construction.
-//
-// A real message id would be needed to mirror add_messages if we ever stream partial
-// assistant messages into the state or restore a session from a checkpoint; eino's
-// schema.Message has no id field.
-func appendMessages(dst []schema.Message, msgs ...schema.Message) []schema.Message {
- return append(dst, msgs...)
-}
-
-// Tool dispatch
-
-// ExecuteTool: dispatch ONE tool call by name.
-//
-// Returns a ToolOutcome whose status/reason let the caller act on WHAT happened
-// empty payload, query miss, infra failure, or a run that added no new
-// evidence — instead of only counting characters.
-func ExecuteTool(ctx context.Context, tools *Toolset, name string, args map[string]any) ToolOutcome {
- // Short-circuit a tool already proven unavailable this session (no compiled
- // structure of its kind). We still return a note, not an error, so the model
- // learns to switch to the corpus tools rather than loop.
- if tools != nil && tools.IsDisabled(name) {
- _LOG.Printf("[Action Session] tool %q disabled (no compiled structure); returning note", name)
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": name,
- "note": fmt.Sprintf("%s is unavailable in this dataset (no compiled structure of its kind). Use search_chunks / retrieve / list_chunks instead.", name),
- }},
- Status: StatusEmpty,
- Reason: ReasonNoStructure,
- }
- }
- if tools == nil || tools.Exec == nil {
- return ToolOutcome{Payload: []any{}, Status: StatusError, Reason: ReasonInfra}
- }
- out, err := tools.Exec.Execute(ctx, name, args)
- if err != nil {
- return ToolOutcome{Payload: []any{}, Status: StatusError, Reason: ReasonInfra}
- }
- return out
-}
-
-// Nodes
-
-// runActionNode: ONE model turn with tools.
-// It appends the assistant message and records any tool calls as pending.
-func (s *SessionState) runActionNode(ctx context.Context) error {
- s.Attempts++
- // Per-turn wall budget: max(15, min(75, deadline_left)). Without this a single turn can
- // eat the entire session budget, starving the salvage/finalize steps.
- wall := max(15.0, min(75.0, s.DeadlineLeft))
- callCtx, cancel := context.WithTimeout(ctx, time.Duration(wall*float64(time.Second)))
- defer cancel()
- reply, err := s.Model.Complete(callCtx, s.Messages, s.Tools.ActiveToolSpecs())
- if err != nil {
- // A timed-out or failed turn CONVERGES the session empty instead of aborting it: the
- // node marks itself done with no new states and no answer, routing ends the graph,
- // and the entry point hands back the messages/evidence gathered so far. Returning
- // the error here aborts the whole eino run, which is reported as a failed session and
- // answered with an empty Result — losing the nav prefix's evidence ids and every tool
- // outcome.
- if callCtx.Err() == context.DeadlineExceeded {
- _LOG.Printf("[Action Session] turn timed out after %.0fs", wall)
- } else {
- _LOG.Printf("[Action Session] LLM call failed (%T); converging session empty", err)
- }
- s.Done = true
- s.NewStates = nil
- s.FoundAnswer = nil
- return nil
- }
- // The model's tool_calls are passed through verbatim on the assistant message so the
- // tool responses that follow can be paired by id. Passing nil leaves a dangling
- // tool_call and the next request is rejected with "tool call result does not follow
- // tool call".
- ensureToolCallIDs(reply.ToolCalls)
- s.Messages = append(s.Messages,
- *schema.AssistantMessage(reply.Content, assistantToolCalls(reply.ToolCalls)))
-
- if len(reply.ToolCalls) > 0 {
- s.PendingCalls = reply.ToolCalls
- return nil
- }
- s.PendingCalls = nil
-
- // No call: try the terminal protocol (<state> / <answer>).
- newStates, foundAnswer, terminalType, payload := ParseTerminal(reply.Content, s.ParentState)
- if foundAnswer != nil || len(newStates) > 0 {
- s.NewStates = newStates
- s.FoundAnswer = foundAnswer
- s.TerminalType = terminalType
- s.TerminalPayload = payload
- s.Done = true
- return nil
- }
- // Neither a call nor a terminal block: nudge once per turn.
- s.Messages = append(s.Messages, *schema.UserMessage("Call the retrieve tool, or output a <state> patch, or emit <answer>."))
- return nil
-}
-
-// toolNode: execute pending tool calls, append tool
-// responses, and apply the per-outcome policies.
-func (s *SessionState) toolNode(ctx context.Context) error {
- if s.Tools == nil {
- return nil
- }
- // ranAny: whether this node executed at least one call, i.e. whether a tool
- // result exists to annotate with the turn's record line (appendRecordLine).
- ranAny := len(s.PendingCalls) > 0
- evidenceIDs := append([]string(nil), s.RetrievedEvidenceIDs...)
- budgetChars := s.CtxBudget
- if budgetChars <= 0 {
- budgetChars = maxToolResponseChars * 4
- }
- used := s.ToolChars
- if s.ToolCache == nil {
- s.ToolCache = NewToolCache()
- }
- seenQueries := append([]string(nil), s.SearchQueries...)
- skipped := 0
- strikes := map[string]int{}
- for k, v := range s.ToolStrikes {
- strikes[k] = v
- }
- // Where the nav ladder currently rests. Bound BEFORE the loop so it survives
- // the empty-pending case and is the input for every tool_call that comes in —
- // a call only advances its own rung, and a later ladder continuation must
- // continue from the SAME resting point.
- //
- // The continuation is ported but currently UNREACHABLE:
- // it only runs once a rung leaves the chain in ModeLLM, and every rule in
- // NavRules is ModeAuto, so the chain always runs to completion in the prefix
- // and PendingRule is "" by the time control reaches here. Adding an LLM rung
- // makes it live with no further change.
- pendingRule := s.NavRuleID
-
- for _, c := range s.PendingCalls {
- // Near-duplicate retrieval suppression: if the model re-issues the same
- // intent as an earlier search (paraphrase), do NOT re-run the index —
- // return a nudge so it patches / reframes instead of burning turns.
- // q = str(args.get("query") or "").strip: the query may
- // be a LIST (retrieve takes up to 3), and its string form still counts
- // for near-dup detection and the seen_queries ledger. A type assertion
- // here degrades every array query to "" — seen_queries stays empty, and
- // the skipped-dup convergence can never trigger.
- q := argQueryString(c.Args["query"])
- if retrievalTools[c.Name] && q != "" && IsNearDup(q, seenQueries) {
- skipped++
- _LOG.Printf("[Action Session] skipping near-duplicate retrieval %q (already searched)", trunc(q, 80))
- s.Messages = appendMessages(s.Messages, toolMessage(c.ID, []any{map[string]any{
- "kind": c.Name,
- "note": "This query is a near-duplicate of an earlier retrieval and was skipped to avoid redundant searching. Patch the slot with what you have, or issue a genuinely NEW retrieval angle.",
- }}))
- continue
- }
- // metadata_search ONE-SHOT guard: block any 2nd call within this direction.
- // The tool SELECTS a document set — once the ids are in hand there is nothing a
- // second call could add, and re-issuing it would burn the direction's turns
- // re-applying a filter that is already in force.
- if c.Name == "metadata_search" && s.MetadataSearchUsed {
- _LOG.Printf("[Action Session] blocking 2nd metadata_search this direction (one-shot guard): %s", RenderToolArgs(c.Args))
- s.Messages = appendMessages(s.Messages, toolMessage(c.ID, []any{map[string]any{
- "kind": "metadata_search",
- "note": "metadata_search is a ONE-SHOT metadata selector and was ALREADY used this direction. Spend the doc_ids it returned — list_chunks(doc_id), navigate_structure(doc_id, query), or retrieve(query, doc_scope=[...]) — do NOT call metadata_search again this direction.",
- }}))
- continue
- }
- // Unknown tool name — never execute it; answer with a correction so the
- // model can recover. Models usually emit the XML protocol tags (state /
- // answer) as tool names; they belong in the reply body as plain text.
- if c.Unknown {
- hint := fmt.Sprintf(
- "'%s' is not a tool. State patches and final answers are plain TEXT in your reply body, wrapped in <state>...</state> or <answer>...</answer> XML tags — do not emit them as tool calls. Available tools: %s.",
- c.Name, strings.Join(ToolMapNames(), ", "))
- s.Messages = appendMessages(s.Messages, toolMessage(c.ID, []any{map[string]any{"kind": "error", "note": hint}}))
- continue
- }
-
- // Same-round cache: avoid re-running an identical (name, args) call.
- // The cache only avoids RE-EXECUTING; a response is still returned for
- // every declared call, because the assistant message declared it.
- // The cache is shared by a round's concurrent sessions, hence the guarded
- // Get/Put.
- cacheKey := callCacheKey(c)
- oc, cached := s.ToolCache.Get(cacheKey)
- if !cached {
- oc = ExecuteTool(ctx, s.Tools, c.Name, c.Args)
- s.ToolCache.Put(cacheKey, oc)
- }
- if q != "" {
- seenQueries = append(seenQueries, q)
- }
- evidenceIDs = append(evidenceIDs, oc.EvidenceIDs...)
- if c.Name == "metadata_search" {
- // Used whatever the status: the guard is about not re-filtering, not about
- // the filter having matched.
- s.MetadataSearchUsed = true
- }
- chunks := append([]any(nil), oc.Payload...)
-
- // ── Policy: act on WHAT happened, not just on payload size ──────────
- switch {
- case oc.Status == StatusOK:
- // A real hit clears the tool's strike record: it demonstrably works.
- delete(strikes, c.Name)
- case oc.Status == StatusEmpty && oc.Reason == ReasonNoStructure:
- // Dataset-level dead end. Strike it; disable once the strikes pile up
- // so one unlucky scope cannot kill the tool.
- n := strikes[c.Name] + 1
- strikes[c.Name] = n
- if n >= emptyStrikes {
- s.Tools.DisableTool(c.Name)
- _LOG.Printf("[Action Session] %s disabled after %d dataset-level empty results", c.Name, n)
- }
- case oc.Status == StatusRedundant:
- // Ran fine, but every hit was already in the shared evidence pool.
- // Say so explicitly — otherwise the model sees a normal passage list
- // and concludes the search succeeded, then re-searches the same ground.
- chunks = append(chunks, map[string]any{
- "kind": c.Name,
- "note": "All hits from this call were ALREADY in your evidence. Re-reading them will not add anything — issue a genuinely new angle, or output a <state> patch with what you have.",
- })
- }
-
- s.ToolOutcomes = append(s.ToolOutcomes, map[string]any{
- "name": c.Name, "status": oc.Status, "reason": oc.Reason, "metrics": oc.Metrics,
- })
-
- // The payload stays the JSON document it always was; the call's own note
- // rides BEHIND it, exactly like the turn's record line (appendRecordLine),
- // so nothing that parses the payload has to learn about either.
- payload := marshalPassages(chunks)
- if oc.Note != "" {
- payload += "\n" + oc.Note
- _LOG.Printf("[Action Session] result note (%s): %s", c.Name, trunc(oc.Note, 280))
- }
- // If the session is already heavy, cut this payload proportionally.
- // -1514 counts len of a str — CODE POINTS, not bytes — so
- // the budget and the cut must be rune-based too; a byte cap would hit
- // CJK payloads ~3x early and shrink the evidence the model sees.
- if used+utf8.RuneCountInString(payload) > budgetChars {
- keep := budgetChars - used
- if keep < 800 {
- keep = 800
- }
- payload = truncateRunes(payload, keep)
- }
- used += utf8.RuneCountInString(payload)
- s.Messages = appendMessages(s.Messages, *schema.ToolMessage(payload, c.ID))
-
- // Ladder continuation: when the model's rung came back weak, keep advancing the
- // ladder IN CODE so one
- // weak step cascades through the remaining (cheaper, wider) rungs instead
- // of leaving the model to rediscover the fallback one turn at a time.
- //
- // Unreachable with today's NavRules (every rung is ModeAuto, so the chain
- // always runs to completion in the prefix and PendingRule is ""). It is
- // ported so an LLM rung resumes from the right place without another
- // change here.
- if pendingRule != "" {
- rule, ok := navRuleByID[pendingRule]
- if ok {
- nxt := rule.Next[oc.Status]
- if nxt != "" {
- // The nav context is rebuilt (direction, known docs) from the live state —
- // the ladder runs against the CURRENT routed scope, not the one captured at
- // prefix time.
- ladderNav := &NavContext{
- Direction: s.Direction,
- KnownDocs: append([]string(nil), s.RoutedDocs...),
- }
- ladderBudget := s.DeadlineLeft * navPrefixBudgetRatio
- if ladderBudget < navPrefixMinBudgetS {
- ladderBudget = navPrefixMinBudgetS
- }
- // Respect the session's remaining context budget: the ladder
- // pairs are appended outside this node's own accounting
- // (max_chars = max(800, budget_chars - used).)
- ex := RunNavChain(ctx, s.Tools, ladderNav, nxt, ladderBudget, "ladder",
- max(800, budgetChars-used))
- // The ladder keeps advancing: a later tool_call in the SAME
- // batch (or the next turn) resumes from where this call left
- // it, not from the original resting point. "" means the
- // ladder finished — also correct to record.
- pendingRule = ex.PendingRule
- for _, m := range ex.Messages {
- if m.Role == schema.Tool {
- // counts len of a str — code points.
- used += utf8.RuneCountInString(m.Content)
- }
- s.Messages = appendMessages(s.Messages, m)
- }
- evidenceIDs = append(evidenceIDs, ex.EvidenceIDs...)
- s.ToolOutcomes = append(s.ToolOutcomes, ex.Outcomes...)
- if oc.Status != StatusOK {
- _LOG.Printf("[Action Session] ladder %q -> %q (reason=%s)",
- rule.ID, nxt, oc.Reason)
- }
- }
- }
- }
- }
-
- // Where the ladder now rests ("" = finished).
- s.NavRuleID = pendingRule
-
- s.PendingCalls = nil
- s.RetrievedEvidenceIDs = evidenceIDs
- s.ToolChars = used
- s.SearchQueries = seenQueries
- s.SkippedDup += skipped
- s.ToolStrikes = strikes
- s.appendBatchProtocol(ranAny)
- s.appendRecordLine(ranAny)
- return nil
-}
-
-// appendBatchProtocol hands the set-direction method to a session that has SHOWN it
-// is enumerating — the second, and the only zero-false-positive, delivery.
-//
-// A session seeded by setProtocolFor already has the text; this path exists for the set
-// direction whose planner typed its count slot with a type that does not read as a count.
-// The signal it rides is the caller's own writing: the model writes a batch of items
-// exactly when the direction is a set.
-//
-// It rides the tool result the model is about to read, like the record line, and is
-// appended at most once per session.
-func (s *SessionState) appendBatchProtocol(ranAny bool) {
- if s.BatchProtocolShown || !ranAny || s.EnumerationProtocol == "" || len(s.Messages) == 0 {
- return
- }
- if !s.wroteBatch() {
- return
- }
- last := &s.Messages[len(s.Messages)-1]
- if last.Role != schema.Tool {
- return
- }
- s.BatchProtocolShown = true
- last.Content = strings.TrimRight(last.Content, "\n") + "\n\n" + s.EnumerationProtocol
- // The same signal widens the retrieval budget for the rest of the run: on a
- // set direction a dropped query is a member nobody searched, and the batch
- // the caller just wrote is the proof that this direction is assembling a set
- // (see Kbinfos.MarkSetDirection).
- s.KB.MarkSetDirection()
- _LOG.Printf("[Action Session] the caller wrote a batch — set-direction method appended to the turn (%d char(s))",
- len(s.EnumerationProtocol))
-}
-
-// appendRecordLine appends the per-turn RECORD line to the last tool result.
-//
-// This is the loop's feedback channel: the model's next query was otherwise aimed
-// with no knowledge of what this session had already probed (and what came back
-// empty), which names the record already holds, which confirmed names are still
-// undecided, or which names the passages themselves offer. All of that is
-// computed here anyway — it was simply never shown.
-//
-// It rides on the last tool message instead of a message of its own: one line,
-// same turn, no extra role in the conversation.
-//
-// The line is appended ONLY when this node ran at least one tool call, so a turn
-// with no calls cannot accumulate duplicates of the previous line.
-func (s *SessionState) appendRecordLine(ranAny bool) {
- if !ranAny || len(s.Messages) == 0 {
- return
- }
- rec := s.sessionRecordNow()
- grew := rec.grewFrom(s.Record)
- s.Record = rec
- // A session that is not ENUMERATING gets no line at all (see enumerating).
- //
- // The line reports members, probed names and reached names — the set's to-do
- // list. On a single-value question every field of it is empty or meaningless
- // (`members=0 | probed-reached=0`), it is recomputed and appended on every turn,
- // and the model then spends turns on it. The record itself is still kept on the session
- // (the continuation ask reads it), only the line the model sees is skipped.
- if !s.enumerating() {
- return
- }
- // The line the model steers by is message content, so the log could not show
- // whether a mechanism fired at all: three rounds of analysis here ended up
- // inferring it from side effects. One truncated line per turn ends that.
- _LOG.Printf("[Action Session] record line: %s", trunc(rec.Line(), 320))
- last := &s.Messages[len(s.Messages)-1]
- if last.Role != schema.Tool || rec.Pool == 0 {
- // No pool bound (or no tool result to annotate): keep the record, skip the
- // line rather than inventing a message for it.
- return
- }
- line := rec.Line()
- last.Content = strings.TrimRight(last.Content, "\n") + "\n" + line
- // An ENUMERATION session whose record has gone FLAT also gets one excerpt from a
- // pool passage it has never been shown (see unreadPoolExcerpt): the round has
- // already paid for that text, unread text is where unnoticed members live, and a
- // session that is still finding things does not need rescuing. Gated on the
- // shape so no other question pays for it at all, and one excerpt per turn.
- if s.enumerating() && !grew {
- if excerpt := s.unreadPoolExcerpt(); excerpt != "" {
- // Logged as well as delivered: whether the mechanism fired is otherwise
- // only visible inside the message content.
- _LOG.Printf("[Action Session] pool excerpt: %s", trunc(excerpt, 240))
- last.Content += "\n" + excerpt
- }
- }
-}
-
-// turnRunCap is the hard ceiling on a session's turns: the mode's floor plus the
-// turns the model may add on its own decision (see offerContinuation).
-func (s *SessionState) turnRunCap() int { return s.actionMaxTurns() + turnRunExtra }
-
-// offerContinuation asks the MODEL whether the session takes another turn.
-//
-// The mode's turn count used to end the session outright (ActionMaxTurns was 4 on
-// medium/high at the time), which finalized sessions mid-enumeration with their own clock
-// unspent and a tool result the model never read. Replacing that with a runtime heuristic
-// ("did the record grow?") fixed the cut-off but bought the wrong thing: the ledger grows
-// whenever the model PROBES, so the heuristic extended sessions that were piling up findings
-// without recording any.
-//
-// So the floor is the floor and the decision is the model's: it is told the
-// budget state, handed the record line it just received, and asked to continue
-// only for something that line shows is missing — otherwise to emit the state
-// patch now. The runtime keeps two hard bounds it does not delegate: the run cap
-// above (so an eager model cannot run away) and the session clock (below which no
-// further turn is offered, because the finalize/salvage step must still fit).
-//
-// The offer is gated on this session ENUMERATING (see enumerating): an extra turn is
-// an extra model call plus up to turnRunExtra more turns, and a question that is not
-// assembling a set has nothing for those turns to find.
-//
-// While the gate read the shape of the planner's table, past runs extended questions that
-// were doing arithmetic over two values, whose caller's own batch — the thing this reads —
-// was never written at all.
-func (s *SessionState) offerContinuation() bool {
- if !s.enumerating() {
- return false
- }
- if s.Attempts >= s.turnRunCap() || s.DeadlineLeft <= turnAskFloorS {
- return false
- }
- if s.ContinuationAsked == s.Attempts {
- return true
- }
- s.ContinuationAsked = s.Attempts
- ask := continuationAsk(s.Attempts, s.turnRunCap(), s.Record.Brief())
- s.Messages = appendMessages(s.Messages, *schema.UserMessage(ask))
- _LOG.Printf("[Action Session] turn %d/%d — the floor is spent; offered the model one more turn while the record says something is missing (%s, %.0fs left).\noffer=%q",
- s.Attempts, s.turnRunCap(), s.Record.Brief(), s.DeadlineLeft, trunc(ask, 700))
- return true
-}
-
-// parentSet reports whether the direction this session was sent on ASKS FOR A SET —
-// what the planner DECLARED, never what a candidate looks like (see CoverageOf).
-func (s *SessionState) parentSet() bool { return CoverageOf(s.ParentState).Set }
-
-// wroteBatch reports whether the CALLER wrote a batch of terms in one query — the
-// signal that it is enumerating rather than asking a question.
-//
-// The gate is the caller's own writing, not a guess about the question, because the guess
-// is wrong: a slot TYPE cannot tell "how many people did X kill" from "how many times larger
-// is A than B", and a single phrase's commas make it look like two members.
-func (s *SessionState) wroteBatch() bool {
- for _, q := range s.SearchQueries {
- if callerBatch(q) {
- return true
- }
- }
- return false
-}
-
-// enumerating reports whether this session is assembling a SET: the caller wrote a
-// batch (the tell that survives contact), or the direction's own table asks for a
-// count/list.
-//
-// The table stays as the second half on purpose: a session may enumerate without ever
-// writing a two-name batch (one probe per member), and then the record line is exactly what
-// it needs. That half's false positives are bounded and rare (a sentence written into a
-// count-typed slot); the batch half has none.
-func (s *SessionState) enumerating() bool {
- return s.wroteBatch() || s.parentSet()
-}
-
-// SessionWallS is the wall clock a session on this direction is given: the
-// enumeration clock when the table declared a set, the default one otherwise.
-//
-// It is a FUNCTION of the shape rather than a constant because the shape decides
-// what the time is spent on: an enumeration session is mid-batch when its clock runs
-// out, and a batch that is cut loses the members it had already reached (see
-// setActionTimeoutS for the measurement). A value session has no such work in
-// flight, so it keeps the tighter clock and the shorter end-to-end latency.
-//
-// The caller owns the context: this number is only the budget a session is told it
-// has, and a pass whose own timeout is shorter still bounds it (see
-// RunSlotResearchPass, which hangs the session clock off the round's PARENT because
-// the round's own clock is fixed before the table — and therefore the shape — is
-// known).
-func SessionWallS(parent State) float64 {
- // The enumeration clock is for a direction that IS one — a set of named members whose
- // deed the planner wrote the words for (Coverage.Ok) — not for every direction that
- // happens to contain a count: an enumeration session is mid-batch when its clock runs
- // out and a batch that is cut loses the members it had reached (see setActionTimeoutS
- // for the measurement), while a value session has no such work in flight and keeps the
- // tighter clock with the shorter latency.
- if CoverageOf(parent).Ok() {
- return setActionTimeoutS
- }
- return actionTimeoutS
-}
-
-// setMethodFor returns the enumeration METHOD, or "" when the direction is not an
-// enumeration.
-//
-// The method is `action_set`, and both halves of when it is delivered are deliberate: its
-// FIRST instruction — propose more candidates than you expect — is a decision taken before
-// the first query, so a direction that has declared itself an enumeration is seeded before
-// its first turn; and a direction that has not is handed the method after the first batch it
-// writes (see appendBatchProtocol), which is the signal with no observed false positives.
-//
-// The gate is the enumeration itself (see Coverage): the method tells a session to
-// enumerate NAMED members, and a single-value question that merely contains a count must
-// not be told to assemble anything. A permissive version seeds most sessions with set
-// strategy on questions that assemble nothing, and a shape-only version still seeds the value
-// questions that merely contain a count — where it can only add cost and timeouts.
-func setMethodFor(table State, prompts PromptLoader) string {
- if !CoverageOf(table).Ok() {
- return ""
- }
- return loadOptionalPrompt(prompts, "action_set")
-}
-
-// enumerationSeed builds the text a session sent on this direction is seeded with: the
-// method, and — when the runtime has already run the enumeration — the windows it found
-// (see CoverageSet.Render).
-//
-// A list of queries in a prompt is ADVICE, and advice may simply not be taken. A window is
-// evidence — it carries the chunk id a member is cited by — so the session's job becomes
-// reading what came back. When no enumeration ran (no clock, no executor), the method
-// travels alone: the seed never carries queries to make.
-func enumerationSeed(table State, prompts PromptLoader, seed string) string {
- if !CoverageOf(table).Ok() {
- // Not an enumeration: nothing to assemble, so neither the method nor a window has
- // any business in this session's seed.
- return ""
- }
- method := setMethodFor(table, prompts)
- if s := strings.TrimSpace(seed); s != "" {
- if method != "" {
- return method + "\n\n" + s
- }
- return s
- }
- return method
-}
-
-// continuationAsk is the offer the model decides on: it names the hard bound, the
-// remaining turns, and the ONLY grounds on which another turn is granted — what
-// the record line shows is still missing. The record itself was appended to the
-// model's last tool result, so the decision is made on facts, not on appetite.
-func continuationAsk(taken, cap int, record string) string {
- return fmt.Sprintf(
- "TURN BUDGET: %d turn(s) taken, up to %d available. DECIDE NOW — your next reply decides it:\n"+
- "- If the [record] line still shows something MISSING — a name you proved reachable and never recorded, a slot with no candidate, an angle you have not tried — call ONE more tool aimed at exactly that. %d turn(s) left.\n"+
- "- Otherwise emit the state patch NOW with everything you have.\n"+
- "An extra turn is only for something the record line shows is missing; re-running a search you already ran is not.\n"+
- "Current record: %s",
- taken, cap, cap-taken, record)
-}
-
-// finalizeNode: tool budget spent — ONE last call
-// WITHOUT tools, demanding the terminal JSON to salvage whatever was learned.
-func (s *SessionState) finalizeNode(ctx context.Context) error {
- budgetPrompt := ("TOOL BUDGET EXHAUSTED. Based ONLY on the passages retrieved above, output now — no prose outside the block:\n" +
- "<state>{\"new_states\": [{\"state\": [{\"id\": <slot_id>, \"kind\": \"members\", " +
- "\"items\": [{\"name\": \"<name>\", \"chunk_id\": \"<its passage>\"}], " +
- "\"candidate_strength\": <0..1>}]}]}</state>\n" +
- "Use \"kind\": \"count\" with \"count\": <n> for a number, and \"kind\": \"text\" with " +
- "\"candidate\": \"<value>\" for one value that is neither a member list nor a number.\n" +
- "If NOTHING was learned use: <state>{\"new_states\": []}</state>")
-
- // The patch written HERE is the one that lands in the record, so the record is
- // part of the order.
- //
- // A session can probe a name, get its passages back, and never mention it in the terminal
- // patch — the run's own evidence, found and paid for, dropped at the last step. The record
- // is the only place that fact exists, and this is the last call that can act on it, so the
- // salvage order names the names it has to account for instead of asking for "whatever you
- // have".
- if rec := s.sessionRecordNow(); rec.Pool > 0 {
- budgetPrompt += "\n\nAccount for the record below in the patch you write. Every name listed as " +
- "FOUND BUT NOT RECORDED has a passage in the pool, so it is either a member this patch includes or " +
- "a name this patch REJECTS with its clue. More searching is no longer possible, so an unmentioned " +
- "name is simply lost.\n" + rec.Line()
- }
-
- // Defensive: strip any assistant.tool_calls that never got a tool response,
- // else the provider rejects the history.
- msgs := append([]schema.Message(nil), s.Messages...)
- msgs = stripUnpairedToolCalls(msgs)
- msgs = append(msgs, *schema.UserMessage(budgetPrompt))
-
- tmo := s.DeadlineLeft
- if tmo <= 0 {
- tmo = finalizeTimeoutS
- }
- tmo = min(max(tmo, minFinalizeTimeoutS), finalizeTimeoutS)
-
- callCtx, cancel := context.WithTimeout(ctx, time.Duration(tmo*float64(time.Second)))
- defer cancel()
-
- reply, err := s.Model.Complete(callCtx, msgs, nil)
- if err != nil {
- // A failed salvage call is logged and the node CONTINUES to the deterministic
- // loose-clue harvest below. Returning early here drops the last-narration
- // breadcrumb exactly when the salvage model is unavailable, which is the case the
- // harvest exists for.
- _LOG.Printf("[Action Session] salvage call failed: %v", err)
- } else {
- newStates, foundAnswer, terminalType, payload := ParseTerminal(reply.Content, s.ParentState)
- s.NewStates = newStates
- s.FoundAnswer = foundAnswer
- s.TerminalType = terminalType
- s.TerminalPayload = payload
- if foundAnswer != nil {
- _LOG.Printf("[Action Session] answer salvaged from exhausted session")
- } else if len(newStates) > 0 {
- _LOG.Printf("[Action Session] %d branch(es) salvaged from exhausted session", len(newStates))
- }
- }
-
- // Loose-clue harvest (deterministic, zero-LLM): when the salvage produced no branches
- // AND no answer, the last AI narration often still carries a fact
- // worth keeping as a breadcrumb in the first unresolved slot.
- if len(s.NewStates) == 0 && s.FoundAnswer == nil {
- if txt := lastNarration(s.Messages); len(txt) >= 24 {
- unresolved := s.ParentState.Unresolved()
- var targetID int
- if len(unresolved) > 0 {
- targetID = unresolved[0].ID
- } else if len(s.ParentState.State) > 0 {
- targetID = s.ParentState.State[0].ID
- } else {
- targetID = -1
- }
- if targetID >= 0 {
- // discovered_clues must be []any: ApplyPatch type-switches on the
- // JSON-decoded shape, so a []string is silently ignored and the whole patch
- // no-ops.
- if patched := ApplyPatch(s.ParentState, []map[string]any{
- {"id": targetID, "discovered_clues": []any{"narrative: " + txt[:min(len(txt), 220)]}},
- }); patched != nil {
- s.NewStates = []State{*patched}
- }
- }
- }
- }
-
- s.Done = true
- return nil
-}
-
-// lastNarration returns the content of the last assistant message that carries non-empty
-// text, scanning newest-first.
-func lastNarration(msgs []schema.Message) string {
- for i := len(msgs) - 1; i >= 0; i-- {
- m := msgs[i]
- if m.Role != schema.Assistant {
- continue
- }
- // Skip assistant messages that only hold tool_calls (no free text).
- if strings.TrimSpace(m.Content) == "" {
- continue
- }
- return strings.TrimSpace(m.Content)
- }
- return ""
-}
-
-// Routing
-
-type routeTarget int
-
-const (
- routeEnd routeTarget = iota
- routeTool
- routeFinalize
- routeRunAction
-)
-
-// actionMaxTurns: the mode's turn budget.
-//
-// The mode's number is a FLOOR, and it is billed to every turn of every session on
-// the question, so it is paid only by the shape it was raised for: the deeper modes
-// lifted it (medium/high 4 → 8, ultra 6 → 10) for the enumeration path, whose turns
-// each spend a batch of names. A session that is not assembling a set has no such
-// batch and runs on the value floor instead (see valueTurnFloor for the measurement).
-// A shape that declares itself LATER is not penalised: the floor is resolved per
-// turn, so the first batch — or a parent table that declares a count/list — raises
-// it for the turns that follow.
-func (s *SessionState) actionMaxTurns() int {
- floor := ResolveMode(s.Tools).ActionMaxTurns
- if floor <= 0 {
- floor = valueTurnFloor
- }
- if floor > valueTurnFloor && !s.enumerating() {
- if !s.floorGateLogged {
- s.floorGateLogged = true
- _LOG.Printf("[Action Session] turn floor %d → %d: this session is not assembling a set (%q, mode %s)",
- floor, valueTurnFloor, trunc(s.Direction, 60), ResolveMode(s.Tools).Label)
- }
- return valueTurnFloor
- }
- return floor
-}
-
-// route
-func (s *SessionState) route() routeTarget {
- if s.Done {
- return routeEnd
- }
- // Run pending tool_calls FIRST, even at the turn budget: leaving an
- // assistant.tool_calls message without its tool response makes the provider
- // reject the next call. The tool node clears PendingCalls, then the route
- // re-checks the budget.
- if len(s.PendingCalls) > 0 {
- return routeTool
- }
- if s.Attempts >= s.actionMaxTurns() {
- // The mode's turn count is a FLOOR: past it the MODEL decides whether the
- // session takes another turn, up to the run cap (see offerContinuation).
- if s.offerContinuation() {
- return routeRunAction
- }
- return routeFinalize
- }
- // No-progress convergence: the model keeps re-issuing near-duplicate
- // retrievals, so further turns are unlikely to surface new evidence.
- if s.SkippedDup >= skippedDupLimit {
- _LOG.Printf("[Action Session] %d near-duplicate retrieval(s) skipped; converging session early", s.SkippedDup)
- return routeFinalize
- }
- return routeRunAction
-}
-
-// routeAfterTool: the turn budget is checked
-// AFTER the tool responses are appended. A pending tool_call must always
-// receive its matching tool response, but once the budget is spent we must NOT
-// go back to run_action — that was the Q86 infinite-loop: the model kept
-// emitting tool_calls, PendingCalls stayed non-empty, so the attempts check in
-// route was never reached and the session burned the whole timeout.
-func (s *SessionState) routeAfterTool() routeTarget {
- if s.Attempts >= s.actionMaxTurns() {
- // Same rule as route: the tool result that just arrived is what the continuation offer
- // is about, so the model decides with it in hand — otherwise the last probe's result is
- // never read by a turn.
- if s.offerContinuation() {
- return routeRunAction
- }
- return routeFinalize
- }
- return routeRunAction
-}
-
-// Session-node adapters: unlike method values (s.runActionNode) these take the
-// state from the graph payload instead of capturing an instance, which is what lets ONE
-// compiled graph serve every session: the session's tools/model arrive in the state
-// (SessionState.Tools / .Model) rather than in a closure.
-func runActionNodeFn(ctx context.Context, st *SessionState) (*SessionState, error) {
- return st, st.runActionNode(ctx)
-}
-
-func toolNodeFn(ctx context.Context, st *SessionState) (*SessionState, error) {
- return st, st.toolNode(ctx)
-}
-
-func finalizeNodeFn(ctx context.Context, st *SessionState) (*SessionState, error) {
- return st, st.finalizeNode(ctx)
-}
-
-var (
- sessionGraphOnce sync.Once
- sessionGraphRun compose.Runnable[*SessionState, *SessionState]
- sessionGraphErr error
-)
-
-// sessionGraph declares the action-session graph and compiles it ONCE per process: every
-// session invokes the same runnable with its own state, so the compile (and any structural
-// error) happens once instead of once per session.
-//
-// RunSlotResearchPass runs sessions in parallel, so the runnable is invoked
-// concurrently by design. That is safe under three preconditions — break any and the
-// sharing must be revisited:
-//
-// 1. Nodes touch only their own *SessionState; no package-level mutable state inside
-// a node.
-// 2. No checkpoint store is configured (compose.WithCheckPointStore) — the
-// checkPointer lives on the shared runner. Guarded by
-// TestSessionGraphHasNoCheckpointStore.
-// 3. Eino v0.9.14's runner keeps no per-run state: it holds only the static topology
-// and every Invoke builds its own channels and task manager
-// (compose/graph_run.go:129-130, :933, :948). Re-check on upgrade.
-//
-// See TestSessionGraphIsSharedAcrossConcurrentSessions.
-func sessionGraph() (compose.Runnable[*SessionState, *SessionState], error) {
- sessionGraphOnce.Do(func() {
- g := compose.NewGraph[*SessionState, *SessionState]()
-
- var buildErr error
- addNode := func(name string, fn func(context.Context, *SessionState) (*SessionState, error)) {
- if buildErr != nil {
- return
- }
- buildErr = g.AddLambdaNode(name, compose.InvokableLambda(fn))
- }
- addEdge := func(from, to string) {
- if buildErr != nil {
- return
- }
- buildErr = g.AddEdge(from, to)
- }
- addBranch := func(from string, ends map[string]bool, cond func(*SessionState) routeTarget) {
- if buildErr != nil {
- return
- }
- buildErr = g.AddBranch(from, compose.NewGraphBranch(func(_ context.Context, st *SessionState) (string, error) {
- return routeNodeName(cond(st)), nil
- }, ends))
- }
-
- addNode("run_action", runActionNodeFn)
- addNode("tool", toolNodeFn)
- addNode("finalize", finalizeNodeFn)
-
- addEdge(compose.START, "run_action")
- addEdge("finalize", compose.END)
-
- addBranch("run_action", map[string]bool{
- "tool": true, "finalize": true, "run_action": true, compose.END: true,
- }, (*SessionState).route)
- addBranch("tool", map[string]bool{"run_action": true, "finalize": true}, (*SessionState).routeAfterTool)
-
- if buildErr != nil {
- sessionGraphErr = buildErr
- return
- }
- sessionGraphRun, sessionGraphErr = g.Compile(context.Background(),
- compose.WithGraphName("action_session"),
- // The session is bounded by the turn budget (route / routeAfterTool)
- // and the caller's deadline; this is only a backstop so a mis-wired
- // cycle cannot spin forever.
- compose.WithMaxRunSteps(maxSessionRunSteps),
- )
- })
- return sessionGraphRun, sessionGraphErr
-}
-
-// maxSessionRunSteps bounds one session's Eino steps: a safety net sized well above what
-// the turn budget can reach.
-const maxSessionRunSteps = 256
-
-// sessionLoop invokes the compiled action-session graph, edge for edge:
-//
-// START → run_action
-// run_action → route → {END | tool | finalize | run_action}
-// tool → route_after_tool → {run_action | finalize}
-// finalize → END
-func (s *SessionState) sessionLoop(ctx context.Context) error {
- runnable, err := sessionGraph()
- if err != nil {
- return err
- }
- _, err = runnable.Invoke(ctx, s)
- return err
-}
-
-// routeNodeName maps the routing predicates onto the Eino graph's node keys.
-func routeNodeName(t routeTarget) string {
- switch t {
- case routeTool:
- return "tool"
- case routeFinalize:
- return "finalize"
- case routeRunAction:
- return "run_action"
- default:
- return compose.END
- }
-}
-
-// Terminal parsing
-
-// ParseTerminal: parse the two terminal blocks
-// (<state> patches → new-state branches; <answer> → final answer).
-//
-// Returns (newStates, foundAnswer, terminalType, payload) — exactly one of
-// newStates / foundAnswer may be non-empty.
-func ParseTerminal(content string, parent State) ([]State, *string, *string, map[string]any) {
- if strings.Contains(content, "<state>") {
- block := ExtractTag(content, "state")
- if block == "" {
- block = "{}"
- }
- data, _ := ExtractJSON(block).(map[string]any)
- if data == nil {
- data = map[string]any{}
- }
- rawBranches, _ := data["new_states"].([]any)
- // Tolerate a bare list of variable patches (no {"state": [...]} wrapper):
- // models emit both shapes.
- if len(rawBranches) > 0 {
- allBare := true
- for _, b := range rawBranches {
- if m, ok := b.(map[string]any); ok {
- if _, has := m["state"]; has {
- allBare = false
- break
- }
- }
- }
- if allBare {
- wrapped := make([]any, 0, len(rawBranches))
- for _, b := range rawBranches {
- wrapped = append(wrapped, map[string]any{"state": []any{b}})
- }
- rawBranches = wrapped
- }
- }
- var branches []State
- for _, rb := range rawBranches {
- br, ok := rb.(map[string]any)
- if !ok {
- continue
- }
- patches := toPatchList(br["state"])
- if ns := ApplyPatch(parent, patches); ns != nil {
- branches = append(branches, *ns)
- }
- }
- tt := "state"
- return branches, nil, &tt, data
- }
- if strings.Contains(content, "<answer>") {
- block := ExtractTag(content, "answer")
- if block == "" {
- block = "{}"
- }
- data, _ := ExtractJSON(block).(map[string]any)
- if data == nil {
- data = map[string]any{}
- }
- // An empty/whitespace answer is NOT a found answer. Returning a non-nil empty
- // string here would terminate the session with FoundAnswer="" — the slot research
- // pass then reported
- // collected_answer=false for a session that never answered, and the
- // model never got the chance to re-emit a proper patch/answer.
- ans := ""
- if raw, ok := data["answer"]; ok && raw != nil {
- ans = strings.TrimSpace(fmt.Sprint(raw))
- }
- var found *string
- if ans != "" {
- found = &ans
- }
- patches := toPatchList(data["new_state"])
- var branches []State
- if ns := ApplyPatch(parent, patches); ns != nil {
- branches = append(branches, *ns)
- }
- tt := "answer"
- return branches, found, &tt, data
- }
- return nil, nil, nil, nil
-}
-
-func toPatchList(v any) []map[string]any {
- list, _ := v.([]any)
- out := make([]map[string]any, 0, len(list))
- for _, item := range list {
- if m, ok := item.(map[string]any); ok {
- out = append(out, m)
- }
- }
- return out
-}
-
-// Helpers
-
-func toolMessage(id string, passages []any) schema.Message {
- return *schema.ToolMessage(marshalPassages(passages), id)
-}
-
-func marshalPassages(passages []any) string {
- if passages == nil {
- passages = []any{}
- }
- raw, err := json.Marshal(map[string]any{"passages": passages})
- if err != nil {
- return `{"passages": []}`
- }
- return string(raw)
-}
-
-// callCacheKey keys a call by name plus a canonical JSON encoding of its arguments.
-func callCacheKey(c ToolCall) string {
- raw, err := json.Marshal(c.Args)
- if err != nil {
- raw = []byte("{}")
- }
- return c.Name + "|" + string(raw)
-}
-
-// ToolCache is the per-round tool-outcome cache. One instance is shared by a round's
-// CONCURRENT sessions, and Go's sessions are goroutines: concurrent map access is a FATAL
-// error, so the map is guarded.
-type ToolCache struct {
- mu sync.Mutex
- m map[string]ToolOutcome
-}
-
-// NewToolCache returns an empty, concurrency-safe tool cache.
-func NewToolCache() *ToolCache { return &ToolCache{m: map[string]ToolOutcome{}} }
-
-// Get returns the cached outcome for key. A nil cache never holds anything, so a
-// session constructed without one simply re-executes (the pre-existing behavior).
-func (c *ToolCache) Get(key string) (ToolOutcome, bool) {
- if c == nil {
- return ToolOutcome{}, false
- }
- c.mu.Lock()
- defer c.mu.Unlock()
- oc, ok := c.m[key]
- return oc, ok
-}
-
-// Put stores the outcome for key. Two identical calls that lose the race both compute and
-// both store: the cache avoids re-executing DELIBERATE repeats, not simultaneous ones.
-func (c *ToolCache) Put(key string, oc ToolOutcome) {
- if c == nil {
- return
- }
- c.mu.Lock()
- defer c.mu.Unlock()
- if c.m == nil {
- c.m = map[string]ToolOutcome{}
- }
- c.m[key] = oc
-}
-
-// stripUnpairedToolCalls removes assistant tool_calls that never received a
-// tool response — a malformed history makes providers reject the next request
-// ("tool call result does not follow tool call").
-func stripUnpairedToolCalls(msgs []schema.Message) []schema.Message {
- answered := map[string]bool{}
- for _, m := range msgs {
- if m.Role == schema.Tool && m.ToolCallID != "" {
- answered[m.ToolCallID] = true
- }
- }
- out := make([]schema.Message, 0, len(msgs))
- for _, m := range msgs {
- if m.Role == schema.Assistant && len(m.ToolCalls) > 0 {
- kept := make([]schema.ToolCall, 0, len(m.ToolCalls))
- for _, tc := range m.ToolCalls {
- if answered[tc.ID] {
- kept = append(kept, tc)
- }
- }
- if len(kept) != len(m.ToolCalls) {
- m.ToolCalls = kept
- }
- }
- out = append(out, m)
- }
- return out
-}
-
-func trunc(s string, n int) string {
- if len(s) <= n {
- return s
- }
- return s[:n]
-}
-
-// sortKeys is a small helper for deterministic map-key iteration in logs.
-func sortKeys[V any](m map[string]V) []string {
- out := make([]string, 0, len(m))
- for k := range m {
- out = append(out, k)
- }
- sort.Strings(out)
- return out
-}
-
-// ===================== navchain.go =====================
-// Deterministic navigation prefix: a code-driven tool chain.
-//
-// WHY THIS EXISTS: the ReAct loop lets the MODEL decide which tool to call
-// next, so ordering rules ("route with navigate_tree first, then drill with
-// navigate_structure, fall back to retrieve when the drill is weak") can only
-// be *suggested* in a prompt and are routinely ignored. This prefix runs those
-// rules IN CODE before the loop starts and injects the resulting exchange as
-// completed assistant/tool messages: the model then explores on top of a
-// guaranteed-correct opening and cannot skip a step. The loop keeps its freedom
-// for everything after the chain.
-
-// Step modes.
-const (
- // ModeAuto means the orchestrator runs this step itself (no model
- // round-trip).
- ModeAuto = "auto"
- // ModeLLM means the step needs a decision only the model can make (e.g.
- // WHICH routed document to drill into), so the chain stops here and hands
- // control back.
- ModeLLM = "llm"
-)
-
-// Navigation-chain constants.
-const (
- // navPrefixBudgetRatio is the share of the session deadline the prefix may
- // spend; the ReAct loop keeps the rest.
- navPrefixBudgetRatio = 0.35
- // navPrefixCallTimeoutS is the per-call ceiling, so one slow navigation
- // cannot swallow the whole prefix.
- navPrefixCallTimeoutS = 25.0
- // navHintChars caps the nav-hint text fed back into retrieval as keywords.
- navHintChars = 600
- // navPrefixMinBudgetS floors the prefix budget.
- navPrefixMinBudgetS = 5.0
- // navMinStepBudgetS floors a single step's budget.
- navMinStepBudgetS = 5.0
-)
-
-// NavRulesEnabled
-const NavRulesEnabled = true
-
-// NavContext is the mutable state threaded through the navigation chain.
-//
-// KnownDocs is the routed scope (doc_ids). RoutedDocs carries each routed doc's
-// OVERALL SUMMARY — the hint that feeds retrieval as a soft boost instead of a
-// hard filter. NavHint is the joined summaries used as retrieval keywords so
-// routed docs rank up WITHOUT excluding the rest of the corpus.
-type NavContext struct {
- Direction string
- KnownDocs []string
- RoutedDocs [][2]string // (doc_id, summary)
- NavHint string
-}
-
-// NavRule is one step of the navigation chain.
-type NavRule struct {
- ID string
- Tool string
- // Mode is ModeAuto or ModeLLM.
- Mode string
- // Run replaces the single-tool path for a step that composes several tools
- // (drill: retrieve + navigate_structure + merge).
- Run func(ctx context.Context, ts *Toolset, nav *NavContext, available map[string]bool, budgetS float64) ToolOutcome
- // Args builds the tool arguments from the running context (ModeAuto only;
- // also used for display when Run composes internally).
- Args func(nav *NavContext) map[string]any
- // When is an optional guard; the step is skipped when it returns false.
- When func(nav *NavContext) bool
- // Next maps status -> next rule id. "" (or a missing key) ends the chain.
- // Routing on STATUS is what makes "quality poor -> fall back" a code
- // decision rather than a prompt suggestion.
- Next map[string]string
-}
-
-// NavRules is the per-slot strategy ladder: "nav is a hint, not a constraint" — no
-// rung filters the corpus down to the routed docs. The tree ROUTES (locate), then
-// drill retrieves over the WHOLE corpus while softly boosting the routed-doc chunks,
-// so an answer living outside them survives (ranked lower) instead of being dropped.
-//
-// locate navigate_tree route to top-n documents, exposing their summaries
-// drill retrieve+merge whole-corpus retrieve (nav summaries as BM25 soft
-// hint) + navigate_structure paths merged on chunk_id
-// + routed-doc chunks re-ranked to the top
-// global retrieve safety net only if drill itself came back empty
-//
-// There is deliberately NO LLM verdict on the drill evidence: retrieval is not
-// scope-locked, so there is no "did the scoped search miss?" signal to grade.
-var NavRules = []NavRule{
- {
- ID: "locate",
- Tool: "navigate_tree",
- Mode: ModeAuto,
- Args: func(nav *NavContext) map[string]any { return map[string]any{"query": nav.Direction} },
- // Tree missed => no routed hints to merge against; the only useful step
- // is an unscoped search. — {OK, MISS, EMPTY, POOR, ERROR};
- // REDUNDANT is deliberately absent: a redundant locate changed nothing,
- // and widening it to global would re-run the same corpus search that
- // produced the duplicates.
- Next: map[string]string{
- StatusOK: "drill",
- StatusMiss: "global",
- StatusEmpty: "global",
- StatusPoor: "global",
- StatusError: "global",
- },
- },
- {
- ID: "drill",
- Tool: "navigate_structure",
- Mode: ModeAuto,
- Run: runDrillMerge,
- Args: func(nav *NavContext) map[string]any { return map[string]any{"query": nav.Direction} },
- // drill returns OK with the merged, re-ranked evidence; only an empty
- // whole-corpus result (MISS) or an infra failure falls through to global.
- // {OK, MISS, EMPTY, ERROR}; no POOR and no REDUNDANT.
- Next: map[string]string{
- StatusOK: "",
- StatusMiss: "global",
- StatusEmpty: "global",
- StatusError: "global",
- },
- },
- {
- ID: "global",
- Tool: "retrieve",
- Mode: ModeAuto,
- Args: func(nav *NavContext) map[string]any { return map[string]any{"query": []string{nav.Direction}} },
- Next: map[string]string{},
- },
-}
-
-var navRuleByID = func() map[string]*NavRule {
- m := make(map[string]*NavRule, len(NavRules))
- for i := range NavRules {
- m[NavRules[i].ID] = &NavRules[i]
- }
- return m
-}()
-
-// NavStartRule is the ladder's entry point.
-const NavStartRule = "locate"
-
-// runDrillMerge is the `drill` rung (AUTO): corpus retrieve for REAL chunks +
-// the root->chunk structure paths from navigate_structure, MERGED on chunk_id
-// so every retrieved chunk carries its hierarchy context.
-//
-// Nav is a hint, not a constraint: retrieval runs over the WHOLE corpus (no
-// doc_scope filter), but the nav-routed docs' summaries are fed back as
-// retrieval keywords (soft boost), and the returned chunks are re-ranked so
-// routed-doc chunks float to the top while non-routed chunks stay below. A
-// multi-hop / enumeration answer living OUTSIDE the routed docs therefore
-// survives — it just ranks lower — instead of being filtered out entirely.
-func runDrillMerge(ctx context.Context, ts *Toolset, nav *NavContext, available map[string]bool, budgetS float64) ToolOutcome {
- var merged []any
- var evidenceIDs []string
- seenIDs := map[string]bool{}
-
- // 1. Skeleton A: whole-corpus retrieval, softly boosted by the nav hints.
- // The hint is an explicit PARAMETER, passed explicitly here instead of being parked on
- // the shared executor (which would make the model's own retrieve calls carry a hint they
- // were never given).
- retOC := ExecuteTool(ctx, ts, "retrieve", map[string]any{
- "query": []string{nav.Direction},
- "nav_hint": nav.NavHint,
- })
- for _, cid := range retOC.EvidenceIDs {
- if !seenIDs[cid] {
- seenIDs[cid] = true
- evidenceIDs = append(evidenceIDs, cid)
- }
- }
- routedIDs := map[string]bool{}
- for _, d := range nav.KnownDocs {
- routedIDs[d] = true
- }
-
- // 2. Supplement B: root->chunk paths from EVERY routed doc's structure.
- paths := map[string]string{}
- if available["navigate_structure"] && len(nav.KnownDocs) > 0 {
- stepCtx, cancel := context.WithTimeout(ctx, stepBudget(budgetS))
- defer cancel()
- for _, docID := range nav.KnownDocs {
- select {
- case <-stepCtx.Done():
- break
- default:
- }
- sOC := ExecuteTool(stepCtx, ts, "navigate_structure", map[string]any{
- "doc_id": docID, "query": nav.Direction, "kind": "catalog",
- })
- for _, d := range sOC.EvidenceIDs {
- if !seenIDs[d] {
- seenIDs[d] = true
- evidenceIDs = append(evidenceIDs, d)
- }
- }
- for cid, path := range chunkPathsFromMetrics(sOC.Metrics) {
- if cid != "" {
- if _, ok := paths[cid]; !ok {
- paths[cid] = path
- }
- }
- }
- }
- }
-
- // 3. Merge: skeleton A is the base; attach structure_path where ids align.
- // Re-rank so routed-doc chunks float to the top WITHOUT deleting the rest.
- for _, p := range retOC.Payload {
- e, ok := p.(map[string]any)
- if !ok {
- merged = append(merged, map[string]any{"content": fmt.Sprint(p)})
- continue
- }
- entry := make(map[string]any, len(e)+2)
- for k, v := range e {
- entry[k] = v
- }
- if cid, ok := entry["id"].(string); ok && cid != "" {
- if sp, ok := paths[cid]; ok {
- entry["structure_path"] = sp
- }
- }
- doc := ""
- if v, ok := entry["doc_id"].(string); ok {
- doc = v
- } else if v, ok := entry["document_id"].(string); ok {
- doc = v
- }
- rank := 1
- if routedIDs[doc] {
- rank = 0 // 0 = routed (top)
- }
- entry["nav_rank"] = rank
- merged = append(merged, entry)
- }
- // Stable sort: routed first, original order preserved within each group
- // (Go's sort.Slice is not stable).
- sort.SliceStable(merged, func(i, j int) bool {
- return navRankOf(merged[i]) < navRankOf(merged[j])
- })
-
- if len(merged) == 0 {
- return ToolOutcome{
- Payload: []any{},
- EvidenceIDs: evidenceIDs,
- Status: StatusMiss,
- Reason: ReasonNoDoc,
- Metrics: map[string]any{"hits": 0},
- }
- }
- return ToolOutcome{
- Payload: merged,
- EvidenceIDs: evidenceIDs,
- Status: StatusOK,
- Metrics: map[string]any{"hits": len(merged), "routed": len(routedIDs)},
- }
-}
-
-func navRankOf(v any) int {
- m, ok := v.(map[string]any)
- if !ok {
- return 1
- }
- if n, ok := m["nav_rank"].(int); ok {
- return n
- }
- return 1
-}
-
-// stepBudget applies the per-step floor to the remaining budget.
-func stepBudget(remaining float64) time.Duration {
- if remaining < navMinStepBudgetS {
- remaining = navMinStepBudgetS
- }
- return time.Duration(remaining * float64(time.Second))
-}
-
-// chunkPathsFromMetrics reads metrics["chunk_paths"], tolerating the shape drift
-// of model/executor-produced metrics.
-func chunkPathsFromMetrics(metrics map[string]any) map[string]string {
- out := map[string]string{}
- raw, ok := metrics["chunk_paths"]
- if !ok || raw == nil {
- return out
- }
- switch v := raw.(type) {
- case map[string]string:
- return v
- case map[string]any:
- for k, val := range v {
- out[k] = fmt.Sprint(val)
- }
- }
- return out
-}
-
-// navToolSurface is the set of tools this session may call.
-func (t *Toolset) navToolSurface() map[string]bool {
- out := map[string]bool{}
- for _, s := range t.ActiveToolSpecs() {
- out[s.Function.Name] = true
- }
- return out
-}
-
-// NavExchange is one completed assistant/tool pair produced by the chain.
-type NavExchange struct {
- Messages []schema.Message
- EvidenceIDs []string
- Outcomes []map[string]any
- // PendingRule is the rule control now rests on ("" when the ladder finished
- // or was abandoned).
- PendingRule string
- // BudgetLeft is the unspent prefix budget in seconds.
- BudgetLeft float64
-}
-
-// RunNavChain: run NavRules from startID, stopping
-// at the first LLM step.
-//
-// AUTO steps execute here; an LLM step is returned without being run, because
-// only the model can supply its arguments (e.g. which routed document to drill).
-// Shared by the pre-session prefix and the in-session fallback, so both consume
-// exactly the same ladder.
-// idPrefix tags every tool_call id produced by a chain run so two chains sharing the same
-// rule ids never collide: the prefix case emits "nav_<rule_id>" and the in-session ladder
-// fallback emits "ladder_<rule_id>", and the provider rejects a history where two
-// tool_calls share an id.
-func RunNavChain(ctx context.Context, ts *Toolset, nav *NavContext, startID string, budgetS float64, idPrefix string, maxChars int) NavExchange {
- var out NavExchange
- if !NavRulesEnabled {
- return out
- }
- available := ts.navToolSurface()
-
- started := time.Now()
- ruleID := startID
- for ruleID != "" {
- rule, ok := navRuleByID[ruleID]
- if !ok || !available[rule.Tool] {
- break
- }
- if rule.Mode == ModeLLM {
- // Hand control back: the model must decide.
- out.PendingRule = ruleID
- out.BudgetLeft = budgetS - time.Since(started).Seconds()
- return out
- }
- if rule.When != nil && !rule.When(nav) {
- break
- }
- remaining := budgetS - time.Since(started).Seconds()
- if remaining <= 1.0 {
- _LOG.Printf("[Action Session] nav chain out of budget before %q", ruleID)
- break
- }
-
- var oc ToolOutcome
- if rule.Run != nil {
- // A composed step (e.g. drill) owns its own tool calls.
- oc = rule.Run(ctx, ts, nav, available, remaining)
- } else {
- stepCtx, cancel := context.WithTimeout(ctx, minDuration(stepBudget(remaining),
- time.Duration(navPrefixCallTimeoutS*float64(time.Second))))
- oc = ExecuteTool(stepCtx, ts, rule.Tool, rule.Args(nav))
- cancel()
- }
-
- out.Outcomes = append(out.Outcomes, map[string]any{
- "name": rule.Tool, "status": oc.Status, "reason": oc.Reason, "metrics": oc.Metrics,
- })
- if rule.Tool == "navigate_tree" && oc.Status == StatusOK {
- // Routed docs become the validated scope for every later step.
- nav.KnownDocs = appendUnique(nav.KnownDocs, docIDsFromPayload(oc.Payload)...)
- nav.RoutedDocs = routedDocsFromMetrics(oc.Metrics)
- var hints []string
- for _, rd := range nav.RoutedDocs {
- if rd[1] != "" {
- hints = append(hints, rd[1])
- }
- }
- if len(hints) > 0 {
- nav.NavHint = truncateRunes(strings.Join(hints, " "), navHintChars)
- }
- }
- out.EvidenceIDs = append(out.EvidenceIDs, oc.EvidenceIDs...)
- out.consumeExchange(ruleID, rule.Tool, rule.Args(nav), oc, idPrefix, maxChars)
- ruleID = rule.Next[oc.Status]
- }
- out.PendingRule = ruleID
- out.BudgetLeft = budgetS - time.Since(started).Seconds()
- return out
-}
-
-// consumeExchange appends one completed assistant/tool exchange.
-//
-// The pair MUST be well formed — every tool_call needs its tool response, or
-// the provider rejects the history (the reason stripUnpairedToolCalls exists).
-//
-// maxChars caps the serialized payload: the in-session ladder continuation appends these
-// pairs OUTSIDE the tool node's own accounting, so it passes the session's remaining
-// context budget down. 0 means uncapped (the prefix path).
-func (e *NavExchange) consumeExchange(ruleID, tool string, args map[string]any, oc ToolOutcome, idPrefix string, maxChars int) {
- callID := fmt.Sprintf("%s_%s", idPrefix, ruleID)
- rawArgs, err := json.Marshal(args)
- if err != nil {
- rawArgs = []byte("{}")
- }
- payload := marshalPassages(oc.Payload)
- // slices a str — CODE POINTS, not bytes — so the cap must be
- // rune-based too; a byte cut would hit CJK payloads ~3x early AND could
- // split a UTF-8 sequence, corrupting the JSON tool response.
- if maxChars > 0 && utf8.RuneCountInString(payload) > maxChars {
- payload = truncateRunes(payload, max(maxChars, 800))
- }
- e.Messages = append(e.Messages,
- *schema.AssistantMessage("", []schema.ToolCall{{
- ID: callID,
- Type: "function",
- Function: schema.FunctionCall{
- Name: tool,
- Arguments: string(rawArgs),
- },
- }}),
- *schema.ToolMessage(payload, callID),
- )
-}
-
-// RunNavPrefix: run the navigation ladder up to
-// the first model-driven step.
-//
-// The returned messages are completed assistant/tool pairs ready to seed the
-// session history, so the model sees the chain as work already done and cannot
-// skip a step.
-//
-// Returns an empty exchange when navigation is unavailable — the ladder is an
-// optimisation, never a precondition for the session to run.
-func RunNavPrefix(ctx context.Context, ts *Toolset, direction string, deadlineLeft float64, nav *NavContext) NavExchange {
- if !NavRulesEnabled || ts == nil {
- return NavExchange{}
- }
- available := ts.navToolSurface()
- if len(available) == 0 || !available["navigate_tree"] {
- return NavExchange{}
- }
- if nav == nil {
- nav = &NavContext{Direction: direction}
- }
- budget := deadlineLeft * navPrefixBudgetRatio
- if budget < navPrefixMinBudgetS {
- budget = navPrefixMinBudgetS
- }
- // maxChars 0: the prefix runs before the session's context accounting starts.
- return RunNavChain(ctx, ts, nav, NavStartRule, budget, "nav", 0)
-}
-
-// Payload helpers
-
-// docIDsFromPayload reads payload[0]["doc_ids"] from a navigate_tree result.
-func docIDsFromPayload(payload []any) []string {
- if len(payload) == 0 {
- return nil
- }
- first, ok := payload[0].(map[string]any)
- if !ok {
- return nil
- }
- return toStringList(first["doc_ids"])
-}
-
-// routedDocsFromMetrics reads metrics["routed_docs"] — a list of (doc_id,
-// summary) pairs, which models/executors emit either as [doc, summary] lists or
-// as {"doc_id":..., "summary":...} objects.
-func routedDocsFromMetrics(metrics map[string]any) [][2]string {
- raw, ok := metrics["routed_docs"]
- if !ok || raw == nil {
- return nil
- }
- list, ok := raw.([]any)
- if !ok {
- return nil
- }
- out := make([][2]string, 0, len(list))
- for _, item := range list {
- switch v := item.(type) {
- case []any:
- if len(v) >= 2 {
- out = append(out, [2]string{fmt.Sprint(v[0]), fmt.Sprint(v[1])})
- } else if len(v) == 1 {
- out = append(out, [2]string{fmt.Sprint(v[0]), ""})
- }
- case []string:
- if len(v) >= 2 {
- out = append(out, [2]string{v[0], v[1]})
- } else if len(v) == 1 {
- out = append(out, [2]string{v[0], ""})
- }
- case map[string]any:
- id := fmt.Sprint(v["doc_id"])
- summary := ""
- if s, ok := v["summary"].(string); ok {
- summary = s
- }
- if id != "" {
- out = append(out, [2]string{id, summary})
- }
- }
- }
- return out
-}
-
-func toStringList(v any) []string {
- list, ok := v.([]any)
- if !ok {
- if ss, ok := v.([]string); ok {
- return ss
- }
- return nil
- }
- out := make([]string, 0, len(list))
- for _, item := range list {
- if item == nil {
- continue
- }
- if s := fmt.Sprint(item); s != "" {
- out = append(out, s)
- }
- }
- return out
-}
-
-func appendUnique(existing []string, add ...string) []string {
- seen := make(map[string]bool, len(existing))
- for _, v := range existing {
- seen[v] = true
- }
- for _, v := range add {
- if !seen[v] {
- seen[v] = true
- existing = append(existing, v)
- }
- }
- return existing
-}
-
-func minDuration(a, b time.Duration) time.Duration {
- if a < b {
- return a
- }
- return b
-}
-
-// ===================== run.go (entry points) =====================
-// The two public entry points of the action session, placed at the END of the file:
-// - RunActionSession
-// - InitializeState
-
-// PromptLoader loads a named prompt template (e.g. "action_run",
-// "action_initialize_state"). The runtime takes this as an interface so the templates can
-// come from the repository's prompt files, from embedded assets, or from a test double.
-type PromptLoader interface {
- Load(name string) (string, error)
-}
-
-// StringPromptLoader serves templates from an in-memory map. Used by tests and
-// as the fallback when no filesystem loader is wired.
-type StringPromptLoader map[string]string
-
-// Load implements PromptLoader.
-func (p StringPromptLoader) Load(name string) (string, error) {
- if t, ok := p[name]; ok {
- return t, nil
- }
- return "", fmt.Errorf("runtime: prompt %q not found", name)
-}
-
-// SessionDeps are the dependencies of one action session.
-type SessionDeps struct {
- Tools *Toolset
- Model SessionModel
- // Prompts supplies the "action_run" / "action_initialize_state" templates.
- // When nil, built-in fallback instructions are used.
- Prompts PromptLoader
- // KB is the shared evidence pool accumulated across the research so far.
- // When non-nil, its chunks are injected into the seed user prompt as
- // "ALREADY RETRIEVED" so the model does not re-retrieve evidence it already
- // has. Nil skips the injection.
- KB *Kbinfos
- // CoverageSeed is what this question's enumeration already brought back (see
- // EnumerateCoverage / CoverageSet.Render): the windows where the deed is stated, each
- // with the chunk id a member is cited by. When non-empty it travels in this
- // session's seed — a window is evidence, where a list of queries to make is advice.
- CoverageSeed string
-}
-
-// RunActionSession: a bounded session
-// pursuing ONE direction.
-//
-// Returns an empty Result (no states) when no model is configured or the session fails:
-// the failure is logged and an empty Result returned rather than propagating.
-func RunActionSession(ctx context.Context, deps SessionDeps, direction string, parent State, deadlineLeft float64, baseSummary string, sharedToolCache *ToolCache, sharedSearchQueries []string) Result {
- system := loadPrompt(deps.Prompts, "action_run")
- seedUser := fmt.Sprintf("Direction: %s\n\nState:\n%s", direction, parent.RenderSlots())
-
- // A direction that IS an enumeration is seeded WITH THE METHOD, before its first turn.
- //
- // The method's first instruction is to propose more candidates than you expect, which is a
- // decision taken BEFORE the first query: seeded a turn later it is already too late to
- // change the candidates the session proposed. The gate is the enumeration itself (see
- // Coverage) and not a reading of the candidates (see setMethodFor). A set direction that is
- // not an enumeration gets the method from appendBatchProtocol, on the first batch it
- // writes.
- seededMethod := enumerationSeed(parent, deps.Prompts, deps.CoverageSeed)
- if seededMethod != "" {
- seedUser += "\n\n" + seededMethod
- // The seeded direction also widens the retrieval budget (see
- // Kbinfos.MarkSetDirection): on a set direction the caller's queries are
- // facets of one list, so the executor must not drop them silently.
- deps.KB.MarkSetDirection()
- // Logged because a seeded method is text inside a prompt: without this line
- // nothing distinguishes "the gate opened for the direction that needed it" from
- // "the gate opened for every direction", which is the failure mode to watch.
- _LOG.Printf("[Action Session] enumeration direction — method (and any evidence it brought back) added to the seed (%d char(s))", len(seededMethod))
- if seed := strings.TrimSpace(deps.CoverageSeed); seed != "" {
- // Said separately from the line above, because the difference is the whole
- // point: the seed carries what the enumeration FOUND, not queries to make
- // (see enumerationSeed).
- _LOG.Printf("[Action Session] enumeration windows in the seed (%d char(s)) — the corpus was asked, not the model.", len(seed))
- }
- }
-
- // AVAILABLE METADATA: the dataset's real metadata fields, so a metadata_search filter
- // can name one the dataset actually carries. Omitted entirely when the catalog is
- // empty (no metadata, an unreadable index, or no resolver wired), which keeps a
- // metadata-free corpus's prompt exactly as it shipped.
- if deps.Tools != nil && deps.Tools.MetadataFields != nil {
- if block := deps.Tools.MetadataFields.Render(); block != "" {
- seedUser += "\n\n" + block
- _LOG.Printf("[Action Session] metadata catalog in the seed (%d field(s))", len(deps.Tools.MetadataFields.Keys))
- }
- }
-
- // ALREADY RETRIEVED: surface the evidence already in the shared pool so the model fills
- // slots from it instead of re-retrieving the same ground. Without this the ReAct loop
- // repeatedly searches evidence it already holds.
- if existing := extractRelevantEvidence(deps.KB, direction, 4); existing != "" {
- seedUser += "\n\nALREADY RETRIEVED (do NOT re-retrieve these — use them to fill slots or identify gaps):\n" + existing
- }
-
- if len(baseSummary) > 0 {
- seedUser += fmt.Sprintf("\n\nPrior round summary:\n%s", baseSummary)
- }
-
- if deps.Model == nil {
- _LOG.Printf("[Action Session] no usable model resolved for action session")
- return Result{Messages: nil, NewStates: nil}
- }
-
- budgetLeft := deadlineLeft
- if budgetLeft <= 0 {
- // A caller that omits the budget gets the shape's own clock (see
- // SessionWallS): the direction is known here even when the caller is not.
- budgetLeft = SessionWallS(parent)
- }
-
- st := &SessionState{
- Messages: []schema.Message{
- *schema.SystemMessage(system),
- *schema.UserMessage(seedUser),
- },
- ParentState: parent,
- Tools: deps.Tools,
- Model: deps.Model,
- KB: deps.KB,
- PendingCalls: nil,
- Done: false,
- NewStates: nil,
- FoundAnswer: nil,
- RetrievedEvidenceIDs: nil,
- Attempts: 0,
- DeadlineLeft: budgetLeft,
- CtxBudget: maxToolResponseChars * 4,
- ToolChars: 0,
- ToolCache: sharedToolCache,
- SearchQueries: sharedSearchQueries,
- SkippedDup: 0,
- ToolStrikes: map[string]int{},
- ToolOutcomes: nil,
- Direction: direction,
- // The method this session may be handed, resolved here so nothing mid-session
- // needs the prompt loader. It arrives in the seed when the table declared a
- // set (see setProtocolFor), or on the first batch the caller writes
- // (appendBatchProtocol) — so it is loaded for every session, and shown at
- // most ONCE: a seeded session is marked as already carrying it, or it would receive the
- // whole text twice — once from the seed and once from the batch append.
- EnumerationProtocol: loadOptionalPrompt(deps.Prompts, "action_set"),
- BatchProtocolShown: seededMethod != "",
- }
- if st.ToolCache == nil {
- st.ToolCache = NewToolCache()
- }
-
- // The graph loop is bounded by the turn budget and the deadline; the context
- // carries the wall-clock so a stalled provider cannot outlive the request.
- runCtx, cancel := context.WithTimeout(ctx, deadlineToDuration(budgetLeft))
- defer cancel()
-
- // Deterministic navigation prefix: run the ladder IN CODE and seed the
- // conversation with the completed exchanges, so the model starts on top of a
- // guaranteed-correct opening instead of being merely *told* the routing rule.
- // The prefix shares the ladder with the in-session fallback, and its pending
- // rule becomes the session's resting point (see SessionState.NavRuleID).
- nav := &NavContext{Direction: direction}
- prefixStarted := time.Now()
- prefix := RunNavPrefix(runCtx, deps.Tools, direction, budgetLeft, nav)
- // The prefix's elapsed time is charged to the session with a 10s floor, so prefix + loop
- // stay inside the caller's deadline. Leaving DeadlineLeft at budgetLeft let one action
- // session run a whole prefix too long.
- st.DeadlineLeft = max(10.0, budgetLeft-time.Since(prefixStarted).Seconds())
- if len(prefix.Messages) > 0 {
- st.Messages = append(st.Messages, prefix.Messages...)
- st.RetrievedEvidenceIDs = append(st.RetrievedEvidenceIDs, prefix.EvidenceIDs...)
- st.ToolOutcomes = append(st.ToolOutcomes, prefix.Outcomes...)
- st.NavRuleID = prefix.PendingRule
- st.RoutedDocs = nav.KnownDocs
- st.NavHint = nav.NavHint
- _LOG.Printf("[Action Session] nav prefix: %d exchange(s), %d evidence id(s), resting on %q",
- len(prefix.Messages)/2, len(prefix.EvidenceIDs), prefix.PendingRule)
- }
-
- if err := st.sessionLoop(runCtx); err != nil {
- _LOG.Printf("[Action Session] session failed: %v", err)
- return Result{Messages: nil, NewStates: nil}
- }
- return Result{
- Messages: st.Messages,
- NewStates: st.NewStates,
- FoundAnswer: st.FoundAnswer,
- RetrievedEvidenceIDs: st.RetrievedEvidenceIDs,
- TerminalType: st.TerminalType,
- TerminalPayload: st.TerminalPayload,
- }
-}
-
-// InitResult: tuple return: the root slot
-// table plus the queries for the first round.
-type InitResult struct {
- Root State
- FirstQueries []string
-}
-
-// InitializeState: ask the model to decompose
-// the question into a slot table, with a deterministic fallback when the call
-// fails.
-//
-// deadlineLeft bounds the decomposition call.
-func InitializeState(ctx context.Context, deps SessionDeps, question string, fanoutHint []string, deadlineLeft float64) InitResult {
- system := loadPrompt(deps.Prompts, "action_initialize_state")
- user := "Question: " + question
- if len(fanoutHint) > 0 {
- lines := make([]string, 0, len(fanoutHint))
- for _, h := range fanoutHint {
- lines = append(lines, "- "+h)
- }
- // "\n".join(...), so the block has NO trailing newline.
- user += "\n\nCandidate aspects already identified:\n" + strings.Join(lines, "\n")
- }
- // `min(_INIT_TIMEOUT_S, deadline_left or _INIT_TIMEOUT_S)`:
- // 0 means "unset" (falls back to the full budget) while a NEGATIVE deadline —
- // an already-exhausted round — is used as-is and times the call out at once.
- tmo := initTimeoutS
- if deadlineLeft != 0 && deadlineLeft < tmo {
- tmo = deadlineLeft
- }
-
- raw := initChat(ctx, deps, system, user, tmo)
- data, _ := ExtractJSON(raw).(map[string]any)
- if len(data) == 0 {
- // One quick retry — transient provider stalls were observed (45s with zero bytes); a
- // second attempt succeeded in production logs. The retry gets a LONGER budget: on slow
- // models a 45s bound times out both times and the table degrades to
- // a single answer slot, losing the second hop of a multi-hop question.
- raw = initChat(ctx, deps, system, user, initRetryTimeout(tmo, deadlineLeft))
- data, _ = ExtractJSON(raw).(map[string]any)
- }
-
- // A value the reply cannot convert is a hard failure: the whole decomposition is
- // discarded and the planner fan-outs are used instead. An empty root reproduces that
- // here.
- var slots []Variable
- if rawList, ok := data["slots"].([]any); ok {
- for i, s := range rawList {
- m, ok := s.(map[string]any)
- if !ok {
- // Only dict entries become Variables.
- continue
- }
- rawID, hasID := m["id"]
- if !hasID {
- rawID = i
- }
- id, ok := asInt(rawID)
- if !ok {
- _LOG.Printf("[Action Session:init] slot %d has a non-integer id (%v); discarding the decomposition", i, rawID)
- return InitResult{}
- }
- // `str(s.get("type") or "entity")`: a falsy type is replaced, any
- // other value is STRINGIFIED (never rejected).
- vType := "entity"
- if rawType, hasType := m["type"]; hasType && isTruthy(rawType) {
- vType = displayText(rawType)
- }
- clues, ok := asStringList(m["clues"])
- if !ok {
- _LOG.Printf("[Action Session:init] slot %d has non-iterable clues (%v); discarding the decomposition", i, m["clues"])
- return InitResult{}
- }
- if len(clues) > 4 {
- clues = clues[:4]
- }
- // The slot may DECLARE the act words its enumeration must cover (see
- // Variable.Terms): a missing or malformed list is simply no declaration,
- // never a reason to discard the decomposition.
- terms, _ := asStringList(m["scan"])
- if len(terms) > CoverageActWordsMax {
- terms = terms[:CoverageActWordsMax]
- }
- kept := make([]string, 0, len(terms))
- for _, t := range terms {
- if t = strings.TrimSpace(t); t != "" {
- kept = append(kept, t)
- }
- }
- subject := strings.TrimSpace(displayText(m["subject"]))
- slots = append(slots, Variable{ID: id, Type: vType, QuestionClues: clues, Terms: kept, Subject: subject})
- }
- }
- // `[str(q).strip for q in (data.get("first_queries") or [])][:3]`:
- // the first three entries are stripped and KEPT even when they end up empty
- // (the filter that used to live here made Go pick later entries instead), and
- // a non-iterable value raises exactly like the slot parsing above.
- rawFirst, ok := asStringList(data["first_queries"])
- if !ok {
- _LOG.Printf("[Action Session:init] first_queries is not iterable (%v); discarding the decomposition", data["first_queries"])
- return InitResult{}
- }
- var firstQueries []string
- for j, q := range rawFirst {
- if j >= 3 {
- break
- }
- firstQueries = append(firstQueries, strings.TrimSpace(q))
- }
-
- if len(slots) == 0 {
- // Decomposition failed (timeout/parse): build the table from planner
- // fanouts so the first round still targets DISTINCT aspects instead of
- // one oversized query (observed: single-slot trees answered directly and
- // died, or grepped the raw question and missed).
- if len(fanoutHint) > 0 {
- limit := min(len(fanoutHint), 4)
- for i := 0; i < limit; i++ {
- h := fanoutHint[i]
- if len([]rune(h)) > 120 {
- h = string([]rune(h)[:120])
- }
- slots = append(slots, Variable{ID: i, Type: "aspect", QuestionClues: []string{h}})
- }
- if len(firstQueries) == 0 {
- limit = min(len(fanoutHint), 3)
- for i := 0; i < limit; i++ {
- firstQueries = append(firstQueries, fanoutHint[i])
- }
- }
- } else {
- slots = []Variable{{ID: 0, Type: "answer", QuestionClues: []string{question}}}
- if len(firstQueries) == 0 {
- firstQueries = []string{question}
- }
- }
- } else if len(firstQueries) == 0 {
- firstQueries = []string{question}
- }
-
- root := NewState(slots, 0, nil)
- _LOG.Printf("[Action Session:init] %s\n%s", root.Brief(), root.RenderSlots())
- return InitResult{Root: root, FirstQueries: firstQueries}
-}
-
-// initRetryTimeout: the
-// slot-table decomposition retry gets a longer window than the first attempt,
-// because on slow models the 45s bound times out both times and the table
-// degrades to a single answer slot.
-//
-// - floor: the first attempt's budget (never shrink below what already failed)
-// - ceiling: 2x the first attempt, capped at a 90s absolute upper bound
-// - deadline: leave at least 5s of the round budget for the rest of the session
-func initRetryTimeout(firstTmo, deadlineLeft float64) float64 {
- if deadlineLeft <= 0 {
- return min(2*firstTmo, 90.0)
- }
- return max(firstTmo, min(min(2*firstTmo, 90.0), deadlineLeft-5.0))
-}
-
-// initChat: ONE bounded LLM turn for the slot-table
-// decomposition. Returns "" on timeout or failure — the caller falls back.
-func initChat(ctx context.Context, deps SessionDeps, system, user string, tmo float64) string {
- if deps.Model == nil {
- return ""
- }
- if tmo <= 0 {
- // `asyncio.timeout(tmo)` with a spent budget fires on
- // the next tick, so the turn is abandoned before it starts. Going through
- // deadlineToDuration would silently hand it the full ACTION_TIMEOUT
- // instead, spending a budget the round no longer has.
- _LOG.Printf("[Action Session:init] timed out (%ds)", int(tmo))
- return ""
- }
- callCtx, cancel := context.WithTimeout(ctx, deadlineToDuration(tmo))
- defer cancel()
- reply, err := deps.Model.Complete(callCtx, []schema.Message{
- *schema.SystemMessage(system),
- *schema.UserMessage(user),
- }, nil)
- if err != nil {
- _LOG.Printf("[Action Session:init] failed: %v", err)
- return ""
- }
- return reply.Content
-}
-
-// Model / tool-calling seam: tool specs, parsed tool calls, and the completion call.
-
-// ToolFunction is the OpenAI-style function descriptor.
-type ToolFunction struct {
- Name string `json:"name"`
- Description string `json:"description"`
- Parameters map[string]any `json:"parameters"`
-}
-
-// ToolSpec is an OpenAI-style tool schema.
-type ToolSpec struct {
- Type string `json:"type"`
- Function ToolFunction `json:"function"`
-}
-
-// ToolCall is one tool invocation requested by the model.
-type ToolCall struct {
- ID string
- Name string
- Args map[string]any
- Unknown bool // name is not in ToolMap — answer with a correction, never execute
-}
-
-// ModelReply is one model turn: either free text, or tool calls, or both.
-type ModelReply struct {
- Content string
- ToolCalls []ToolCall
- Tokens int // total tokens, for convenience (== Usage.TotalTokens when Usage != nil)
- // Usage is the provider-reported prompt/completion/total split. Nil when the
- // provider reported nothing.
- Usage *chat.Usage
-}
-
-// TemperatureModel is an OPTIONAL extension of SessionModel: models that can
-// vary the sampling temperature per call implement it.
-//
-// Temperatures are set per node (keyword extraction uses 0.1, the answer composition uses
-// the default). Without this seam every node would run at whatever the invoker defaulted
-// to. Callers that need a specific temperature type-assert to this interface and fall back
-// to Complete when it is not implemented.
-type TemperatureModel interface {
- CompleteWithTemperature(ctx context.Context, messages []schema.Message, tools []ToolSpec, temp float64) (*ModelReply, error)
-}
-
-// ContextLengthModel is implemented by models that can report their context window in
-// tokens, which message-fitting nodes use as the budget for chat.FitMessages. Callers
-// type-assert for it and fall back to chat.EffectiveContextLength's 8192 default when it
-// is absent, i.e. when the model config omits a context length. This is the fitting
-// counterpart to the TemperatureModel seam.
-type ContextLengthModel interface {
- ContextLength() int
-}
-
-// SessionModel is ONE model turn with THIS mode's tool surface.
-//
-// The production implementation (InvokerSessionModel) binds the tool schemas natively
-// through the chat seam (chat.Request.Tools + ToolChoiceAuto) and reads the calls back from
-// Response.ToolCalls.
-type SessionModel interface {
- Complete(ctx context.Context, messages []schema.Message, tools []ToolSpec) (*ModelReply, error)
-}
-
-// StreamingSessionModel is implemented by models that can emit the answer
-// incrementally. Callers type-assert for it and fall back to the one-shot
-// Complete when it is absent, so streaming is strictly an enhancement (fed by the graph's
-// token stream).
-type StreamingSessionModel interface {
- SessionModel
- // StreamComplete sends the reply in pieces. onDelta receives each piece and
- // isThink tells whether it belongs to a hidden reasoning block; both are
- // forwarded to the sink.
- StreamComplete(ctx context.Context, messages []schema.Message, tools []ToolSpec, onDelta func(delta string, isThink bool) error) (*ModelReply, error)
-}
-
-// Parsing of model output.
-//
-// Every LLM call in the runtime asks for JSON, and models wrap that JSON in
-// thinking preamble and Markdown fences. The approach here is to strip the wrappers first,
-// then repair, and let encoding/json handle the rest.
-
-var reFence = regexp.MustCompile("```(?:json)?\\s*|\\s*```")
-
-// UnmarshalModelJSON strips any thinking preamble and Markdown fences, then parses JSON.
-// An empty result parses as an empty object so callers can index into `out` without a nil
-// check.
-func UnmarshalModelJSON(text string, out any) error {
- text = common.StripThinkTrailing(text)
- text = reFence.ReplaceAllString(text, "")
- text = strings.TrimSpace(text)
- if text == "" {
- return json.Unmarshal([]byte("{}"), out)
- }
- return json.Unmarshal([]byte(text), out)
-}
-
-// resolveLoader returns the configured PromptLoader, or the embedded
-// Markdown-backed loader when none is wired. The runtime defaults to the repository's
-// authoritative templates instead of the terse Go fallback constants.
-func resolveLoader(p PromptLoader) PromptLoader {
- if p != nil {
- return p
- }
- return prompts.EmbeddedPromptLoader{}
-}
-
-// loadPrompt resolves the prompt loader (defaulting to prompts.EmbeddedPromptLoader
-// when none is wired on SessionDeps) and loads name. The canonical templates are
-// the Markdown files action_run.md and action_initialize_state.md under
-// internal/rag/prompts, embedded into the binary via //go:embed so they are never absent
-// at runtime. A missing template is an error, not a silent fallback to a stale string.
-func loadPrompt(p PromptLoader, name string) string {
- t, err := resolveLoader(p).Load(name)
- if err != nil {
- panic(fmt.Sprintf("loadPrompt(%q): %v", name, err))
- }
- return t
-}
-
-// loadOptionalPrompt loads a template a given loader may not carry, and returns ""
-// instead of panicking.
-//
-// action_set is optional by design: a loader that predates it (or a test's
-// in-memory loader) simply gets no enumeration protocol, and its session runs
-// exactly as it did before the protocol existed.
-func loadOptionalPrompt(p PromptLoader, name string) string {
- t, err := resolveLoader(p).Load(name)
- if err != nil {
- return ""
- }
- return t
-}
-
-// deadlineToDuration converts a seconds budget to a context deadline.
-// A non-positive budget falls back to the default session timeout so a caller
-// that omits it does not produce an already-expired context.
-func deadlineToDuration(seconds float64) time.Duration {
- if seconds <= 0 {
- seconds = actionTimeoutS
- }
- return time.Duration(seconds * float64(time.Second))
-}
-
-// extractRelevantEvidence flattens the shared evidence pool into a compact, line-delimited
-// digest the model can read without re-retrieving. Chunks are ranked by the number of
-// direction tokens they contain, then the top maxChunks are surfaced so the seed prompt
-// stays bounded.
-func extractRelevantEvidence(kb *Kbinfos, direction string, maxChunks int) string {
- if kb == nil || maxChunks <= 0 {
- return ""
- }
- chunks := kb.Chunks
- if len(chunks) == 0 {
- return ""
- }
-
- // Build the set of direction tokens (>=2 chars, alnum or CJK), then rank chunks by how
- // many tokens appear in their text.
- dirTokens := tokenizeDirection(direction)
- var ranked []map[string]any
- if len(dirTokens) == 0 {
- // No usable direction: fall back to the tail of the pool.
- if len(chunks) > maxChunks {
- ranked = chunks[len(chunks)-maxChunks:]
- } else {
- ranked = chunks
- }
- } else {
- type scored struct {
- chunk map[string]any
- score int
- }
- scoredChunks := make([]scored, 0, len(chunks))
- for _, c := range chunks {
- scoredChunks = append(scoredChunks, scored{chunk: c, score: directionRelevance(c, dirTokens)})
- }
- sort.SliceStable(scoredChunks, func(i, j int) bool {
- return scoredChunks[i].score > scoredChunks[j].score
- })
- if len(scoredChunks) > maxChunks {
- scoredChunks = scoredChunks[:maxChunks]
- }
- for _, s := range scoredChunks {
- ranked = append(ranked, s.chunk)
- }
- }
-
- var b strings.Builder
- for _, c := range ranked {
- content := strings.TrimSpace(ChunkTextOf(c))
- if content == "" {
- continue
- }
- // Cap each chunk at evidenceDigestChars and flatten newlines so the digest
- // stays single-line per chunk. The cap is evidenceDigestChars, which is sized to hold a
- // whole sentence (see the constant).
- content = truncateRunes(content, evidenceDigestChars)
- content = strings.ReplaceAll(content, "\n", " ")
- // "[cid] text" so the model can cite the chunk id.
- b.WriteString("[")
- b.WriteString(ChunkIDOf(c))
- b.WriteString("] ")
- b.WriteString(content)
- b.WriteString("\n")
- }
- return strings.TrimRight(b.String(), "\n")
-}
-
-// tokenizeDirection splits a direction string into >=2-char alnum/CJK tokens.
-func tokenizeDirection(direction string) []string {
- if direction == "" {
- return nil
- }
- seen := make(map[string]struct{})
- var tokens []string
- var buf strings.Builder
- flush := func() {
- if buf.Len() >= 2 {
- tok := buf.String()
- if _, dup := seen[tok]; !dup {
- seen[tok] = struct{}{}
- tokens = append(tokens, tok)
- }
- }
- buf.Reset()
- }
- for _, r := range strings.ToLower(direction) {
- switch {
- case (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || (r >= '一' && r <= '鿿'):
- buf.WriteRune(r)
- default:
- flush()
- }
- }
- flush()
- return tokens
-}
-
-// directionRelevance counts how many direction tokens appear in a chunk's text.
-func directionRelevance(chunk map[string]any, dirTokens []string) int {
- text := strings.ToLower(ChunkTextOf(chunk))
- if text == "" {
- return 0
- }
- score := 0
- for _, t := range dirTokens {
- if strings.Contains(text, t) {
- score++
- }
- }
- return score
-}
diff --git a/internal/rag/agentic-rag/runtime/action_session_test.go b/internal/rag/agentic-rag/runtime/action_session_test.go
deleted file mode 100644
index 93e7cd1c2e..0000000000
--- a/internal/rag/agentic-rag/runtime/action_session_test.go
+++ /dev/null
@@ -1,1486 +0,0 @@
-package runtime
-
-import (
- "context"
- "errors"
- "fmt"
- "reflect"
- "strings"
- "sync"
- "testing"
- "unicode/utf8"
-
- "github.com/cloudwego/eino/compose"
- "github.com/cloudwego/eino/schema"
- "gorm.io/gorm"
- "ragflow/internal/agent/chat"
- "ragflow/internal/rag/prompts"
-)
-
-// ladderExec answers the tools the navigation ladder can reach. Every call is
-// recorded so the test can prove the continuation actually ran.
-type ladderExec struct {
- calls []string
- // emptyFor makes the named tool come back empty, which is what drives a
- // weak-rung continuation.
- emptyFor map[string]bool
-}
-
-func (e *ladderExec) Execute(_ context.Context, name string, _ map[string]any) (ToolOutcome, error) {
- e.calls = append(e.calls, name)
- if e.emptyFor[name] {
- return ToolOutcome{Status: StatusEmpty, Reason: ReasonNoStructure}, nil
- }
- return ToolOutcome{
- Status: StatusOK,
- Payload: []any{map[string]any{"kind": name, "content": "hit for " + name, "chunk_id": "c-" + name}},
- EvidenceIDs: []string{"c-" + name},
- }, nil
-}
-
-// TestConsumeExchangeUsesIdPrefix pins the id-prefix tagging: the prefix and the
-// in-session ladder fallback share rule ids, so their tool_call ids must be tagged
-// differently ("nav_<rule>" vs "ladder_<rule>") or the provider rejects the history as a
-// duplicate id.
-func TestConsumeExchangeUsesIdPrefix(t *testing.T) {
- cases := []struct {
- idPrefix string
- want string
- }{
- {"nav", "nav_r1"},
- {"ladder", "ladder_r1"},
- }
- for _, c := range cases {
- ex := &NavExchange{}
- ex.consumeExchange("r1", "navigate_tree", map[string]any{"q": 1}, ToolOutcome{Payload: []any{}}, c.idPrefix, 0)
- if len(ex.Messages) != 2 {
- t.Fatalf("prefix %q: want 2 messages, got %d", c.idPrefix, len(ex.Messages))
- }
- if got := ex.Messages[0].ToolCalls[0].ID; got != c.want {
- t.Errorf("prefix %q: tool_call id = %q, want %q", c.idPrefix, got, c.want)
- }
- if got := ex.Messages[1].ToolCallID; got != c.want {
- t.Errorf("prefix %q: tool message id = %q, want %q", c.idPrefix, got, c.want)
- }
- }
-}
-
-// TestToolNodeContinuesLadderInCode pins the in-code ladder continuation: when the rung
-// the model just ran came back weak, the ladder keeps advancing IN CODE through the
-// remaining (cheaper, wider) rungs, instead of leaving the model to rediscover the
-// fallback one turn at a time. The continuation must also update where the ladder rests,
-// so a later call in the SAME batch resumes from there rather than from the original
-// resting point.
-//
-// Unreachable with the shipped NavRules (all ModeAuto), so this test installs an LLM rung
-// to exercise it — the continuation code is what makes a future LLM rung work without
-// another change.
-func TestToolNodeContinuesLadderInCode(t *testing.T) {
- // Install an LLM rung that falls through to "global" (retrieve) on an empty
- // result, and remove it afterwards.
- const llmRung = "test-llm-rung"
- navRuleByID[llmRung] = &NavRule{
- ID: llmRung,
- Tool: "navigate_structure",
- Mode: ModeLLM,
- Next: map[string]string{StatusEmpty: "global", StatusMiss: "global"},
- }
- defer delete(navRuleByID, llmRung)
-
- exec := &ladderExec{emptyFor: map[string]bool{"navigate_structure": true}}
- ts := &Toolset{Exec: exec, ThinkingMode: "high"}
-
- // The model issues one call on the rung the chain handed back.
- st := &SessionState{
- Tools: ts,
- DeadlineLeft: 60,
- Direction: "who created Culdcept",
- NavRuleID: llmRung,
- ToolCache: NewToolCache(),
- PendingCalls: []ToolCall{
- {ID: "call-1", Name: "navigate_structure"},
- },
- }
-
- if err := st.toolNode(context.Background()); err != nil {
- t.Fatalf("toolNode: %v", err)
- }
-
- // The model's own call ran...
- if len(exec.calls) == 0 || exec.calls[0] != "navigate_structure" {
- t.Fatalf("calls = %v, want the model's navigate_structure call first", exec.calls)
- }
- // ...and the ladder then continued on its own to the "global" rung.
- var sawRetrieve bool
- for _, c := range exec.calls[1:] {
- if c == "retrieve" {
- sawRetrieve = true
- }
- }
- if !sawRetrieve {
- t.Errorf("ladder did not continue: calls = %v, want a continuation to retrieve", exec.calls)
- }
- // The ladder finished (the "global" rung has no Next), so the resting point
- // is "" — recording "" is correct and distinct from "never advanced".
- if st.NavRuleID != "" {
- t.Errorf("NavRuleID = %q, want \"\" (the ladder ran to completion)", st.NavRuleID)
- }
- // The continuation's evidence is folded into the session, not discarded.
- if len(st.RetrievedEvidenceIDs) == 0 {
- t.Error("continuation evidence was dropped from RetrievedEvidenceIDs")
- }
- // The continuation's tool message reaches the model, so it does not repeat
- // the work on its next turn.
- toolMsgs := 0
- for _, m := range st.Messages {
- if m.Role == schema.Tool {
- toolMsgs++
- }
- }
- if toolMsgs < 2 {
- t.Errorf("tool messages = %d, want >= 2 (the call plus the ladder's)", toolMsgs)
- }
-}
-
-// TestToolNodeLeavesLadderAloneWithNoPendingRule pins the default: with the
-// shipped all-AUTO ladder there is nothing to continue, so no extra ladder work
-// happens.
-func TestToolNodeLeavesLadderAloneWithNoPendingRule(t *testing.T) {
- exec := &ladderExec{emptyFor: map[string]bool{"retrieve": true}}
- ts := &Toolset{Exec: exec, ThinkingMode: "high"}
- st := &SessionState{
- Tools: ts,
- DeadlineLeft: 60,
- Direction: "who created Culdcept",
- NavRuleID: "", // ladder finished
- ToolCache: NewToolCache(),
- PendingCalls: []ToolCall{
- {ID: "call-1", Name: "retrieve"},
- },
- }
- if err := st.toolNode(context.Background()); err != nil {
- t.Fatalf("toolNode: %v", err)
- }
- if len(exec.calls) != 1 {
- t.Errorf("calls = %v, want exactly the model's one call", exec.calls)
- }
- if st.NavRuleID != "" {
- t.Errorf("NavRuleID = %q, want \"\"", st.NavRuleID)
- }
-}
-
-// TestSessionGraphCompilesOnce pins the compile-once contract: the session graph is built
-// once and every session invokes THAT graph, so repeated calls must hand back the same
-// runnable instead of compiling a fresh one per session.
-func TestSessionGraphCompilesOnce(t *testing.T) {
- first, err := sessionGraph()
- if err != nil {
- t.Fatalf("sessionGraph: %v", err)
- }
- second, err := sessionGraph()
- if err != nil {
- t.Fatalf("sessionGraph (second call): %v", err)
- }
- if first != second {
- t.Error("sessionGraph compiled a new graph per call; want one shared runnable")
- }
-}
-
-// TestSessionGraphHasNoCheckpointStore guards precondition 2 of the shared
-// graph: with a checkpoint store the shared runner would carry per-run
-// persistence state, and concurrency would become a property of that store.
-// Eino rejects a CheckPointID when no store is configured, so a successful
-// invoke with one means someone wired a store in.
-func TestSessionGraphHasNoCheckpointStore(t *testing.T) {
- runnable, err := sessionGraph()
- if err != nil {
- t.Fatalf("sessionGraph: %v", err)
- }
- st := &SessionState{
- Tools: &Toolset{Exec: &ladderExec{}},
- Model: &fixedReplyModel{reply: &ModelReply{Content: "ok"}},
- DeadlineLeft: 30,
- }
- _, err = runnable.Invoke(context.Background(), st, compose.WithCheckPointID("session-1"))
- if err == nil {
- t.Fatal("session graph accepted a CheckPointID; it must stay checkpoint-free for shared concurrent use")
- }
-}
-
-// TestAppendMessagesKeepsRepeatToolCallIDs locks the reason appendMessages is a
-// plain append and NOT a tool_call_id dedupe: fallback ids are assigned per turn
-// (ensureToolCallIDs -> call_0, call_1…), so two turns can legitimately carry
-// the same tool_call_id. Dropping the earlier response would orphan that turn's
-// assistant tool_calls and get the next request rejected.
-func TestAppendMessagesKeepsRepeatToolCallIDs(t *testing.T) {
- msgs := appendMessages(nil,
- *schema.ToolMessage("turn-1 answer", "call_0"),
- *schema.ToolMessage("turn-2 answer", "call_0"),
- )
- if len(msgs) != 2 {
- t.Fatalf("len = %d, want 2 — a repeated tool_call_id must not replace", len(msgs))
- }
- if msgs[0].Content != "turn-1 answer" || msgs[1].Content != "turn-2 answer" {
- t.Errorf("messages = %q / %q, want both turns kept in order", msgs[0].Content, msgs[1].Content)
- }
-}
-
-// TestSessionGraphIsSharedAcrossConcurrentSessions runs many sessions through
-// the shared graph at once — RunSlotResearchPass drives sessions in parallel, so
-// the runnable must be safe to invoke concurrently and must keep each session's
-// state separate. Each session's model echoes its own direction, so a state that
-// picks up another session's reply is detectable.
-func TestSessionGraphIsSharedAcrossConcurrentSessions(t *testing.T) {
- const sessions = 16
-
- states := make([]*SessionState, sessions)
- for i := 0; i < sessions; i++ {
- dir := fmt.Sprintf("direction-%02d", i)
- states[i] = &SessionState{
- Direction: dir,
- Tools: &Toolset{Exec: &ladderExec{}},
- Model: &fixedReplyModel{reply: &ModelReply{Content: "answer for " + dir}},
- DeadlineLeft: 60,
- }
- }
-
- var wg sync.WaitGroup
- errs := make([]error, sessions)
- for i := range states {
- wg.Add(1)
- go func(i int) {
- defer wg.Done()
- errs[i] = states[i].sessionLoop(context.Background())
- }(i)
- }
- wg.Wait()
-
- for i, st := range states {
- if errs[i] != nil {
- t.Fatalf("session %d: %v", i, errs[i])
- }
- if st.Direction != fmt.Sprintf("direction-%02d", i) {
- t.Errorf("session %d direction = %q; another session's state leaked in", i, st.Direction)
- }
- // The session must have run: attempts incremented and its own model
- // echoed back into its own message list.
- if st.Attempts == 0 {
- t.Errorf("session %d never ran (attempts=0)", i)
- }
- want := "answer for " + st.Direction
- sawOwn := false
- for _, m := range st.Messages {
- if m.Role == schema.Assistant && m.Content == want {
- sawOwn = true
- }
- // No reply belonging to a different session may appear here.
- for j := 0; j < sessions; j++ {
- if j == i {
- continue
- }
- other := fmt.Sprintf("answer for direction-%02d", j)
- if m.Content == other {
- t.Errorf("session %d carries session %d's reply %q", i, j, other)
- }
- }
- }
- if !sawOwn {
- t.Errorf("session %d never recorded its own reply %q", i, want)
- }
- }
-}
-
-// failingModel fails every Complete call, standing in for the timeout /
-// provider-exception branch of runActionNode.
-type failingModel struct{ err error }
-
-func (m *failingModel) Complete(_ context.Context, _ []schema.Message, _ []ToolSpec) (*ModelReply, error) {
- return nil, m.err
-}
-
-// TestRunActionNodeConvergesOnLLMError pins the convergence contract: a timed-out or
-// failed turn converges the session EMPTY (done, no new states, no found answer) so the
-// graph ends through the router and the run still returns the messages/evidence gathered so
-// far. Returning the error instead aborted the whole eino run, which RunActionSession
-// reports as a failed session and answers with an empty Result.
-func TestRunActionNodeConvergesOnLLMError(t *testing.T) {
- s := &SessionState{
- Messages: []schema.Message{*schema.UserMessage("q")},
- Tools: &Toolset{},
- Model: &failingModel{err: errors.New("provider down")},
- DeadlineLeft: 30,
- ParentState: State{State: []Variable{{ID: 0, Type: "aspect"}}},
- }
- if err := s.runActionNode(context.Background()); err != nil {
- t.Fatalf("runActionNode = %v; the session must converge instead of aborting the graph", err)
- }
- if !s.Done {
- t.Error("Done = false; the session must converge so _route returns END")
- }
- if len(s.NewStates) != 0 || s.FoundAnswer != nil {
- t.Errorf("converged session carried states/answer: %d / %v", len(s.NewStates), s.FoundAnswer)
- }
- if s.Attempts != 1 {
- t.Errorf("Attempts = %d, want 1 (the failed turn still counts)", s.Attempts)
- }
- // The session loop must terminate on the converged state rather than abort.
- if err := s.sessionLoop(context.Background()); err != nil {
- t.Fatalf("sessionLoop = %v; a converged session must end normally", err)
- }
-}
-
-// TestFinalizeNodeHarvestsNarrativeWhenSalvageFails pins the finalize behaviour: a failed
-// salvage call is logged and the node still runs the deterministic loose-clue harvest, so
-// the last narration survives as a breadcrumb on the first unresolved slot.
-func TestFinalizeNodeHarvestsNarrativeWhenSalvageFails(t *testing.T) {
- const narration = "The entity was founded in 1865 by a consortium of local merchants."
- s := &SessionState{
- Messages: []schema.Message{
- *schema.UserMessage("q"),
- *schema.AssistantMessage(narration, nil),
- },
- Tools: &Toolset{},
- Model: &failingModel{err: errors.New("salvage unavailable")},
- DeadlineLeft: 30,
- ParentState: State{State: []Variable{{ID: 7, Type: "aspect"}}},
- }
- if err := s.finalizeNode(context.Background()); err != nil {
- t.Fatalf("finalizeNode = %v", err)
- }
- if !s.Done {
- t.Error("Done = false; finalize must mark the session done")
- }
- if len(s.NewStates) != 1 {
- t.Fatalf("NewStates = %d, want the narrative breadcrumb patch", len(s.NewStates))
- }
- clues := s.NewStates[0].State[0].DiscoveredClues
- if len(clues) != 1 || !strings.HasPrefix(clues[0], "narrative: ") {
- t.Errorf("discovered clues = %v, want a narrative breadcrumb", clues)
- }
-}
-
-// TestInitRetryTimeout pins the retry window: the slot table decomposition retry gets a
-// longer window than the first attempt, floored at the first attempt's budget and clamped
-// by the round deadline.
-func TestInitRetryTimeout(t *testing.T) {
- cases := []struct {
- name string
- first, deadl float64
- want float64
- }{
- {"no deadline: 2x capped at 90", 45, 0, 90},
- {"deadline leaves 5s of headroom", 45, 60, 55},
- {"floor is the first attempt's budget", 45, 40, 45},
- {"2x ceiling below 90", 30, 200, 60},
- {"deadline above the ceiling", 45, 300, 90},
- }
- for _, c := range cases {
- if got := initRetryTimeout(c.first, c.deadl); got != c.want {
- t.Errorf("%s: initRetryTimeout(%v, %v) = %v, want %v", c.name, c.first, c.deadl, got, c.want)
- }
- }
-}
-
-// TestInitializeStateKeepsEmptyFirstQueries pins the first_queries handling: the first
-// three entries are taken and stripped, and an entry that strips to "" is KEPT — filtering
-// it made Go pick a later entry instead.
-func TestInitializeStateKeepsEmptyFirstQueries(t *testing.T) {
- mdl := &fixedReplyModel{reply: &ModelReply{
- Content: `{"slots":[{"id":0,"type":"aspect","clues":["a"]}],"first_queries":[" ", "b", "c", "d"]}`,
- }}
- res := InitializeState(context.Background(), SessionDeps{Model: mdl}, "q", nil, 60)
- want := []string{"", "b", "c"}
- if len(res.FirstQueries) != len(want) {
- t.Fatalf("firstQueries = %#v, want %#v", res.FirstQueries, want)
- }
- for i := range want {
- if res.FirstQueries[i] != want[i] {
- t.Errorf("firstQueries[%d] = %q, want %q", i, res.FirstQueries[i], want[i])
- }
- }
-}
-
-// TestAsIntConvertsSlotValues pins the conversion a slot table needs: a bool is an integer,
-// floats truncate toward zero, a quoted integer may carry whitespace/underscores, and
-// anything that cannot be converted reports failure — which discards the whole
-// decomposition in initialize_state rather than guessing an id.
-func TestAsIntConvertsSlotValues(t *testing.T) {
- ok := []struct {
- in any
- want int
- }{
- {7, 7},
- {float64(7), 7},
- {float64(1.7), 1},
- {float64(-1.7), -1},
- {true, 1},
- {false, 0},
- {"5", 5},
- {" 5 ", 5},
- {"+5", 5},
- {"1_000", 1000},
- }
- for _, c := range ok {
- got, converted := asInt(c.in)
- if !converted || got != c.want {
- t.Errorf("asInt(%#v) = %d, %v; want %d, true", c.in, got, converted, c.want)
- }
- }
- // These are hard failures, not coercions: a decimal string, a bare word, nil, and
- // non-scalar containers.
- for _, bad := range []any{"5.5", "abc", "", nil, []any{1}, map[string]any{}} {
- if _, converted := asInt(bad); converted {
- t.Errorf("asInt(%#v) reported success; it must be refused", bad)
- }
- }
-}
-
-// TestAsStringListReadsListShapes pins the coercion of a list-ish slot value: a falsy value
-// yields nothing, a list yields its items as text, a string yields its characters, a map its
-// keys (sorted), and a value that is not list-like is an error rather than a guess.
-func TestAsStringListReadsListShapes(t *testing.T) {
- cases := []struct {
- name string
- in any
- want []string
- ok bool
- }{
- {"nil is falsy", nil, nil, true},
- {"empty string is falsy", "", nil, true},
- {"zero is falsy", float64(0), nil, true},
- {"list items are stringified", []any{"a", 7}, []string{"a", "7"}, true},
- {"a string iterates its characters", "ab", []string{"a", "b"}, true},
- {"a map iterates its keys", map[string]any{"b": 1, "a": 2}, []string{"a", "b"}, true},
- {"a number is not iterable", float64(5), nil, false},
- {"true is not iterable", true, nil, false},
- }
- for _, c := range cases {
- got, converted := asStringList(c.in)
- if converted != c.ok {
- t.Errorf("%s: ok = %v, want %v", c.name, converted, c.ok)
- continue
- }
- if !converted {
- continue
- }
- if len(got) != len(c.want) {
- t.Errorf("%s: got %#v, want %#v", c.name, got, c.want)
- continue
- }
- for i := range c.want {
- if got[i] != c.want[i] {
- t.Errorf("%s: got %#v, want %#v", c.name, got, c.want)
- break
- }
- }
- }
-}
-
-// TestInitializeStateDiscardsOnUnconvertibleValues pins the failure path: a bad id or a
-// non-iterable clues list fails the parse, and the whole decomposition is dropped in favour
-// of the planner fan-outs. Go signals that by returning an empty root.
-func TestInitializeStateDiscardsOnUnconvertibleValues(t *testing.T) {
- cases := []struct {
- name string
- body string
- }{
- {"non-integer id", `{"slots":[{"id":"abc","type":"aspect","clues":["a"]}]}`},
- {"decimal id", `{"slots":[{"id":"5.5","type":"aspect"}]}`},
- {"non-iterable clues", `{"slots":[{"id":0,"type":"aspect","clues":5}]}`},
- {"non-iterable first_queries", `{"slots":[{"id":0,"type":"aspect"}],"first_queries":7}`},
- }
- for _, c := range cases {
- mdl := &fixedReplyModel{reply: &ModelReply{Content: c.body}}
- res := InitializeState(context.Background(), SessionDeps{Model: mdl}, "q", nil, 60)
- if len(res.Root.State) != 0 {
- t.Errorf("%s: root has %d slot(s); the decomposition must be discarded", c.name, len(res.Root.State))
- }
- }
-}
-
-// TestInitializeStateStringifiesType pins `str(s.get("type") or "entity")`: a
-// falsy type becomes "entity", any other value is stringified rather than
-// rejected.
-func TestInitializeStateStringifiesType(t *testing.T) {
- mdl := &fixedReplyModel{reply: &ModelReply{
- Content: `{"slots":[{"id":0,"type":7,"clues":["a"]},{"id":1,"type":"","clues":[]}]}`,
- }}
- res := InitializeState(context.Background(), SessionDeps{Model: mdl}, "q", nil, 60)
- if len(res.Root.State) != 2 {
- t.Fatalf("slots = %d, want 2", len(res.Root.State))
- }
- if got := res.Root.State[0].Type; got != "7" {
- t.Errorf("type = %q, want %q (a non-string type is stringified)", got, "7")
- }
- if got := res.Root.State[1].Type; got != "entity" {
- t.Errorf("type = %q, want %q (falsy -> entity)", got, "entity")
- }
-}
-
-// TestInitializeStateSpentBudgetTimesOutImmediately pins the deadline semantics: 0 means
-// "unset", but a NEGATIVE deadline is used as-is, so the timeout fires on the next tick and
-// NEITHER attempt reaches the provider — the caller then falls back to the planner
-// fan-outs.
-func TestInitializeStateSpentBudgetTimesOutImmediately(t *testing.T) {
- mdl := &fixedReplyModel{reply: &ModelReply{
- Content: `{"slots":[{"id":0,"type":"aspect","clues":["a"]}]}`,
- }}
- res := InitializeState(context.Background(), SessionDeps{Model: mdl}, "q", nil, -15)
- if mdl.calls != 0 {
- t.Errorf("model called %d time(s); a spent budget must not reach the provider", mdl.calls)
- }
- // No fan-out hint → the single "answer" fallback slot, not a decomposition.
- if len(res.Root.State) != 1 || res.Root.State[0].Type != "answer" {
- t.Errorf("root = %+v, want the single answer fallback slot", res.Root.State)
- }
- if len(res.Root.State[0].QuestionClues) != 1 || res.Root.State[0].QuestionClues[0] != "q" {
- t.Errorf("clues = %v, want [q]", res.Root.State[0].QuestionClues)
- }
-}
-
-// TestInitializeStateZeroDeadlineUsesFullBudget is the other half of the same
-// line: an UNSET deadline (0/None) still gets the full _INIT_TIMEOUT_S budget.
-func TestInitializeStateZeroDeadlineUsesFullBudget(t *testing.T) {
- mdl := &fixedReplyModel{reply: &ModelReply{
- Content: `{"slots":[{"id":0,"type":"aspect","clues":["a"]}]}`,
- }}
- res := InitializeState(context.Background(), SessionDeps{Model: mdl}, "q", nil, 0)
- if len(res.Root.State) != 1 {
- t.Fatalf("root has %d slot(s), want 1", len(res.Root.State))
- }
- if mdl.calls != 1 {
- t.Errorf("model called %d time(s), want 1", mdl.calls)
- }
-}
-
-// TestConsumeExchangeCapsPayload pins the payload cap: the in-session ladder passes its
-// remaining context budget down, so the emitted tool payload is truncated; 0 keeps the
-// prefix path uncapped.
-func TestConsumeExchangeCapsPayload(t *testing.T) {
- big := strings.Repeat("x", 5000)
- payload := []any{map[string]any{"kind": "navigate_tree", "content": big}}
-
- capped := &NavExchange{}
- capped.consumeExchange("r1", "navigate_tree", map[string]any{"q": 1}, ToolOutcome{Payload: payload}, "ladder", 1000)
- if got := len(capped.Messages[1].Content); got > 1000 {
- t.Errorf("tool payload = %d chars, want it capped at 1000", got)
- }
-
- uncapped := &NavExchange{}
- uncapped.consumeExchange("r1", "navigate_tree", map[string]any{"q": 1}, ToolOutcome{Payload: payload}, "nav", 0)
- if got := len(uncapped.Messages[1].Content); got <= 1000 {
- t.Errorf("tool payload = %d chars, want 0 to mean uncapped", got)
- }
-}
-
-// TestExtractJSONLenient: fallback
-// through json.loads(strict=False) and json_repair.loads: models routinely emit
-// trailing commas, single-quoted strings, bare NaN/Infinity and stray control
-// characters. ExtractJSON should salvage these instead of dropping the whole
-// candidate and moving to the next "{".
-
-func eqJSONMap(t *testing.T, got any, want map[string]any) {
- t.Helper()
- g, ok := got.(map[string]any)
- if !ok {
- t.Fatalf("got %T %v, want map %v", got, got, want)
- }
- if !reflect.DeepEqual(g, want) {
- t.Fatalf("got %v, want %v", g, want)
- }
-}
-
-func TestExtractJSONLenient(t *testing.T) {
- cases := []struct {
- name string
- in string
- want map[string]any
- }{
- {"valid passthrough", `{"a": 1}`, map[string]any{"a": float64(1)}},
- {"trailing comma", `{"a": 1,}`, map[string]any{"a": float64(1)}},
- {"single quotes", `{'a': 'b'}`, map[string]any{"a": "b"}},
- {"single quotes and trailing comma", `{'a': 1,}`, map[string]any{"a": float64(1)}},
- {"NaN literal", `{"x": NaN}`, map[string]any{"x": nil}},
- {"Infinity literal", `{"x": Infinity}`, map[string]any{"x": nil}},
- {"negative Infinity", `{"x": -Infinity}`, map[string]any{"x": nil}},
- {"control char in string", "{\"a\": \"x\x01y\"}", map[string]any{"a": "xy"}},
- {"prefixed text", `blah {"a": 1} blah`, map[string]any{"a": float64(1)}},
- }
- for _, c := range cases {
- t.Run(c.name, func(t *testing.T) {
- eqJSONMap(t, ExtractJSON(c.in), c.want)
- })
- }
-}
-
-func TestExtractJSONLenientUnrecoverable(t *testing.T) {
- // The brace-matched loop tries each "{" candidate; the deep repair fallback salvages
- // the FIRST candidate even when it is malformed (a deformed object yields null for the
- // incomplete key). It must not panic, hang, or return nil for a salvageable object.
- v := ExtractJSON(`{"a":} {"b": 2}`)
- if v == nil {
- t.Fatalf("first candidate should be salvaged, got nil")
- }
- // Empty input has nothing to salvage and must return nil (not panic).
- if v := ExtractJSON(``); v != nil {
- t.Fatalf("empty input: got %v, want nil", v)
- }
-}
-
-// TestExtractJSONWholeTextRepair repairs the WHOLE text — deformities that no single
-// brace-matched candidate can fix.
-func TestExtractJSONWholeTextRepair(t *testing.T) {
- cases := []struct {
- name string
- in string
- want map[string]any
- }{
- // Unquoted object keys.
- {"unquoted key", `{a: 1}`, map[string]any{"a": float64(1)}},
- {"unquoted keys multiple", `{name: foo, age: 3}`, map[string]any{"name": "foo", "age": float64(3)}},
- // Dropped separator between adjacent values.
- {"missing separator obj", `{"a": 1 "b": 2}`, map[string]any{"a": float64(1), "b": float64(2)}},
- {"missing separator after array", `{"a": [1,2] "b": 3}`, map[string]any{"a": []any{float64(1), float64(2)}, "b": float64(3)}},
- // Outer braces present but keys/values deformed (json_repair recovers).
- {"deformed object", `{a: 1, b: 2}`, map[string]any{"a": float64(1), "b": float64(2)}},
- // Combination: unquoted key + trailing comma + missing separator.
- {"combined deformity", `{name: foo, "age": 3 "city": bar,}`, map[string]any{"name": "foo", "age": float64(3), "city": "bar"}},
- // Prefixed prose with a deformed object.
- {"prose plus unquoted key", `Here is the answer: {entity: "x", relation: "y"}`, map[string]any{"entity": "x", "relation": "y"}},
- }
- for _, c := range cases {
- t.Run(c.name, func(t *testing.T) {
- eqJSONMap(t, ExtractJSON(c.in), c.want)
- })
- }
-}
-
-// TestExtractJSONThinkFenceStillPreprocessed ensures the whole-text fallback
-// still strips <think> preambles and ```json fences before deep repair.
-func TestExtractJSONThinkFenceStillPreprocessed(t *testing.T) {
- in := "```json\n{name: foo, value: 1}\n```"
- eqJSONMap(t, ExtractJSON(in), map[string]any{"name": "foo", "value": float64(1)})
- think := "<think>reasoning</think>{\"a\": 1,}"
- eqJSONMap(t, ExtractJSON(think), map[string]any{"a": float64(1)})
-}
-
-// recordingInvoker captures the outgoing chat.Request so the test can assert
-// what the model was actually shown.
-type recordingInvoker struct{ req chat.Request }
-
-func (r *recordingInvoker) Invoke(_ context.Context, _ *gorm.DB, req chat.Request) (*chat.Response, error) {
- r.req = req
- return &chat.Response{Content: "ok"}, nil
-}
-
-// fixedReplyModel returns a canned ModelReply so tests can drive the session
-// nodes without a real model.
-type fixedReplyModel struct {
- reply *ModelReply
- calls int
-}
-
-func (m *fixedReplyModel) Complete(_ context.Context, _ []schema.Message, _ []ToolSpec) (*ModelReply, error) {
- m.calls++
- return m.reply, nil
-}
-
-func declaredTools() []ToolSpec {
- return []ToolSpec{{
- Function: ToolFunction{
- Name: "retrieve",
- Description: "run retrieval",
- Parameters: map[string]any{"type": "object"},
- },
- }}
-}
-
-// TestNoPromptBasedToolInstruction pins the contract: there is NO prompt-based tool-call
-// instruction of any kind — tools are bound natively and the reply's tool_calls are read
-// directly. Such a protocol must not be sent, least of all one whose "at most one tool per
-// reply" clause contradicts the native tool list.
-func TestNoPromptBasedToolInstruction(t *testing.T) {
- inv := &recordingInvoker{}
- m := &InvokerSessionModel{Invoker: inv}
- if _, err := m.Complete(context.Background(),
- []schema.Message{*schema.UserMessage("q")}, declaredTools()); err != nil {
- t.Fatalf("Complete: %v", err)
- }
-
- if len(inv.req.Tools) == 0 {
- t.Fatal("native tools were not declared")
- }
- // The fallback prompt must never be sent, with native tools or without.
- if _, err := m.Complete(context.Background(),
- []schema.Message{*schema.UserMessage("q")},
- []ToolSpec{{Function: ToolFunction{Description: "unnamed"}}}); err != nil {
- t.Fatalf("Complete: %v", err)
- }
- for _, msg := range inv.req.Messages {
- if strings.Contains(msg.Content, "at most one tool per reply") {
- t.Errorf("prompt-based tool instruction sent: %q", msg.Content)
- }
- }
-}
-
-// TestAssistantMessageCarriesToolCalls pins the pairing contract: the assistant message
-// carries the model's tool_calls verbatim so the tool responses that follow can be paired by
-// id. Passing nil leaves a dangling tool_call, and the next request is rejected with "tool
-// call result does not follow tool call".
-func TestAssistantMessageCarriesToolCalls(t *testing.T) {
- st := &SessionState{
- Messages: []schema.Message{*schema.SystemMessage("s")},
- Tools: &Toolset{Exec: &ladderExec{}},
- Model: &fixedReplyModel{reply: &ModelReply{Content: "thinking", ToolCalls: []ToolCall{
- {ID: "", Name: "retrieve", Args: map[string]any{"query": "q1"}},
- {ID: "", Name: "retrieve", Args: map[string]any{"query": "q2"}},
- }}},
- }
- if err := st.runActionNode(context.Background()); err != nil {
- t.Fatalf("runActionNode: %v", err)
- }
-
- var assistant *schema.Message
- for i := range st.Messages {
- if st.Messages[i].Role == schema.Assistant {
- assistant = &st.Messages[i]
- }
- }
- if assistant == nil {
- t.Fatal("no assistant message was appended")
- }
- if len(assistant.ToolCalls) != 2 {
- t.Fatalf("assistant tool_calls = %d, want 2 (pass-through, and both kept)", len(assistant.ToolCalls))
- }
- // A call_{i} id is synthesized when the provider omits one, so the tool responses have
- // ids to reference.
- if assistant.ToolCalls[0].ID != "call_0" || assistant.ToolCalls[1].ID != "call_1" {
- t.Errorf("ids = %q, %q; want call_0, call_1", assistant.ToolCalls[0].ID, assistant.ToolCalls[1].ID)
- }
- if assistant.ToolCalls[0].Function.Name != "retrieve" {
- t.Errorf("name = %q, want retrieve", assistant.ToolCalls[0].Function.Name)
- }
- // The pending calls must carry the SAME ids the assistant message declared.
- for i, c := range st.PendingCalls {
- if c.ID == "" {
- t.Errorf("pending call %d has no id; tool response cannot pair with the assistant message", i)
- }
- }
-}
-
-// TestDisabledToolIsNotUnknown pins the known-tool check: it is against the STATIC full
-// set, not the active surface, so a tool this session disabled (or hid for lack of a web
-// provider) is still a known tool: the model must get the "unavailable, use this instead"
-// note from ExecuteTool, not an "unknown tool" correction.
-func TestDisabledToolIsNotUnknown(t *testing.T) {
- // Sanity: the name must be a real tool for the test to mean anything.
- if _, ok := ToolMap["graph_explore"]; !ok {
- t.Skip("graph_explore is not a registered tool")
- }
- ts := &Toolset{ThinkingMode: "high"}
- // Disable it: it now drops out of the active surface but stays in ToolMap.
- ts.DisableTool("graph_explore")
- for _, spec := range ts.ActiveToolSpecs() {
- if spec.Function.Name == "graph_explore" {
- t.Fatal("graph_explore must be absent from the active surface for this test")
- }
- }
- calls := nativeToHarnessCalls([]chat.ToolCall{{Name: "graph_explore"}})
- if len(calls) != 1 {
- t.Fatalf("calls = %d, want 1", len(calls))
- }
- if calls[0].Unknown {
- t.Error("a disabled tool must NOT be Unknown: the check judges against the static " +
- "tool map, so the call gets the 'unavailable, use this instead' note")
- }
-}
-
-// TestUnknownToolStillFlagged pins the other side — a name that is not a real tool at all
-// IS flagged Unknown, so it is answered with a correction instead of being executed.
-func TestUnknownToolStillFlagged(t *testing.T) {
- calls := nativeToHarnessCalls([]chat.ToolCall{{Name: "definitely_not_a_tool"}})
- if len(calls) != 1 || !calls[0].Unknown {
- t.Fatalf("calls = %+v, want one Unknown call", calls)
- }
-}
-
-// TestNativeToolCallsPreserveAll pins the execution contract: every native tool call is
-// mapped, not just the first — all of them are executed.
-func TestNativeToolCallsPreserveAll(t *testing.T) {
- calls := nativeToHarnessCalls([]chat.ToolCall{
- {ID: "1", Name: "retrieve"},
- {ID: "2", Name: "retrieve"},
- {ID: "3", Name: "bogus"},
- })
- if len(calls) != 3 {
- t.Fatalf("mapped %d calls, want 3 (all of them, not just the first)", len(calls))
- }
- if !calls[2].Unknown {
- t.Error("an undeclared name must be flagged Unknown")
- }
-}
-
-// streamingInvoker streams a fixed set of pieces.
-type streamingInvoker struct {
- pieces []piece
- err error
-}
-
-type piece struct {
- text string
- isThink bool
-}
-
-func (s *streamingInvoker) Invoke(context.Context, *gorm.DB, chat.Request) (*chat.Response, error) {
- return &chat.Response{Content: "one-shot"}, nil
-}
-
-func (s *streamingInvoker) Stream(_ context.Context, _ *gorm.DB, _ chat.Request, onDelta func(string, bool) error) (*chat.Response, error) {
- if s.err != nil {
- return nil, s.err
- }
- var content, reasoning string
- for _, p := range s.pieces {
- if onDelta != nil {
- if err := onDelta(p.text, p.isThink); err != nil {
- return nil, err
- }
- }
- if p.isThink {
- reasoning += p.text
- } else {
- content += p.text
- }
- }
- return &chat.Response{Content: content, Thinking: reasoning}, nil
-}
-
-// plainInvoker only implements the non-streaming seam.
-type plainInvoker struct{}
-
-func (p *plainInvoker) Invoke(context.Context, *gorm.DB, chat.Request) (*chat.Response, error) {
- return &chat.Response{Content: "one-shot"}, nil
-}
-
-func TestStreamCompleteForwardsDeltas(t *testing.T) {
- m := &InvokerSessionModel{Invoker: &streamingInvoker{pieces: []piece{
- {text: "think", isThink: true},
- {text: "Hello ", isThink: false},
- {text: "world", isThink: false},
- }}}
-
- var got []piece
- reply, err := m.StreamComplete(context.Background(), []schema.Message{*schema.UserMessage("q")}, nil,
- func(delta string, isThink bool) error {
- got = append(got, piece{text: delta, isThink: isThink})
- return nil
- })
-
- if err != nil {
- t.Fatalf("unexpected error: %v", err)
- }
- if reply.Content != "Hello world" {
- t.Fatalf("Content = %q, want %q", reply.Content, "Hello world")
- }
- if len(got) != 3 || !got[0].isThink || got[0].text != "think" {
- t.Fatalf("deltas = %+v, want reasoning flagged first", got)
- }
-}
-
-func TestStreamCompleteFailsWithoutStreamingInvoker(t *testing.T) {
- // A non-streaming invoker must report failure so the caller falls back to
- // the one-shot call rather than silently producing nothing.
- m := &InvokerSessionModel{Invoker: &plainInvoker{}}
-
- if _, err := m.StreamComplete(context.Background(), nil, nil, nil); err == nil {
- t.Fatal("want an error when the invoker cannot stream")
- }
-}
-
-func TestStreamCompletePropagatesError(t *testing.T) {
- m := &InvokerSessionModel{Invoker: &streamingInvoker{err: errors.New("provider down")}}
-
- if _, err := m.StreamComplete(context.Background(), nil, nil, nil); err == nil {
- t.Fatal("want the provider error")
- }
-}
-
-func TestStreamCompleteRequiresInvoker(t *testing.T) {
- m := &InvokerSessionModel{}
-
- if _, err := m.StreamComplete(context.Background(), nil, nil, nil); err == nil {
- t.Fatal("want an error when no invoker is configured")
- }
-}
-
-type capturingInvoker struct {
- lastReq *chat.Request
- hasTool bool
- tool chat.ToolCall
- content string
-}
-
-func (c *capturingInvoker) Invoke(_ context.Context, _ *gorm.DB, req chat.Request) (*chat.Response, error) {
- c.lastReq = &req
- resp := &chat.Response{Content: c.content}
- if c.hasTool {
- resp.ToolCalls = []chat.ToolCall{c.tool}
- }
- return resp, nil
-}
-
-func TestCompleteForwardsNativeToolsAndPrefersNativeCalls(t *testing.T) {
- tools := []ToolSpec{{
- Type: "function",
- Function: ToolFunction{
- Name: "search",
- Description: "run a search",
- Parameters: map[string]any{"type": "object", "properties": map[string]any{}},
- },
- }}
- inv := &capturingInvoker{hasTool: true, tool: chat.ToolCall{ID: "c1", Name: "search", Arguments: map[string]any{"q": "x"}}, content: "ok"}
- m := &InvokerSessionModel{Invoker: inv}
-
- reply, err := m.Complete(context.Background(), []schema.Message{*schema.UserMessage("q")}, tools)
- if err != nil {
- t.Fatalf("unexpected error: %v", err)
- }
- if inv.lastReq == nil || len(inv.lastReq.Tools) != 1 {
- t.Fatalf("tools not forwarded to invoker: req=%+v", inv.lastReq)
- }
- if inv.lastReq.Tools[0].Function.Name != "search" {
- t.Fatalf("forwarded tool name = %q, want search", inv.lastReq.Tools[0].Function.Name)
- }
- if inv.lastReq.ToolChoice != chat.ToolChoiceAuto {
- t.Fatalf("tool choice = %q, want auto", inv.lastReq.ToolChoice)
- }
- if len(reply.ToolCalls) != 1 || reply.ToolCalls[0].Name != "search" || reply.ToolCalls[0].Args["q"] != "x" {
- t.Fatalf("native tool call not preferred: %+v", reply.ToolCalls)
- }
-}
-
-func TestCompleteReportsNoCallWhenModelSendsNoNativeCall(t *testing.T) {
- inv := &capturingInvoker{hasTool: false, content: ""}
- m := &InvokerSessionModel{Invoker: inv}
- tools := []ToolSpec{{
- Type: "function",
- Function: ToolFunction{Name: "search", Description: "d", Parameters: map[string]any{}},
- }}
-
- // No native tool_calls => the reply stays tool-less. This confirms the runtime still
- // declares tools on the request (verified above) but does not hallucinate a call: there
- // is no other parsing path.
- reply, err := m.Complete(context.Background(), []schema.Message{*schema.UserMessage("q")}, tools)
- if err != nil {
- t.Fatalf("unexpected error: %v", err)
- }
- if len(reply.ToolCalls) != 0 {
- t.Fatalf("expected no tool calls, got %+v", reply.ToolCalls)
- }
-}
-
-// streamingCapturingInvoker implements chat.StreamingInvoker and records the
-// streaming request, returning a native tool call so we can verify the runtime
-// reads streamed calls out of the native tool_calls field.
-type streamingCapturingInvoker struct {
- capturingInvoker
- onDelta func(delta string, isThink bool) error
-}
-
-func (s *streamingCapturingInvoker) Stream(_ context.Context, _ *gorm.DB, req chat.Request, onDelta func(delta string, isThink bool) error) (*chat.Response, error) {
- s.lastReq = &req
- if onDelta != nil {
- _ = onDelta("partial", false)
- }
- resp := &chat.Response{Content: "streamed"}
- if s.hasTool {
- resp.ToolCalls = []chat.ToolCall{s.tool}
- }
- return resp, nil
-}
-
-func TestStreamCompleteForwardsNativeToolsAndPrefersNativeCalls(t *testing.T) {
- tools := []ToolSpec{{
- Type: "function",
- Function: ToolFunction{
- Name: "search",
- Description: "run a search",
- Parameters: map[string]any{"type": "object", "properties": map[string]any{}},
- },
- }}
- inv := &streamingCapturingInvoker{
- capturingInvoker: capturingInvoker{
- hasTool: true,
- tool: chat.ToolCall{ID: "c2", Name: "search", Arguments: map[string]any{"q": "y"}},
- },
- }
- m := &InvokerSessionModel{Invoker: inv}
-
- reply, err := m.StreamComplete(context.Background(), []schema.Message{*schema.UserMessage("q")}, tools, nil)
- if err != nil {
- t.Fatalf("unexpected error: %v", err)
- }
- if inv.lastReq == nil || len(inv.lastReq.Tools) != 1 || inv.lastReq.Tools[0].Function.Name != "search" {
- t.Fatalf("streaming tools not forwarded: req=%+v", inv.lastReq)
- }
- if inv.lastReq.ToolChoice != chat.ToolChoiceAuto {
- t.Fatalf("streaming tool choice = %q, want auto", inv.lastReq.ToolChoice)
- }
- if len(reply.ToolCalls) != 1 || reply.ToolCalls[0].Name != "search" || reply.ToolCalls[0].Args["q"] != "y" {
- t.Fatalf("streamed native tool call not preferred: %+v", reply.ToolCalls)
- }
-}
-
-func TestStreamCompleteIgnoresFencedBlockInContent(t *testing.T) {
- tools := []ToolSpec{{
- Type: "function",
- Function: ToolFunction{Name: "search", Description: "d", Parameters: map[string]any{}},
- }}
- // A fenced block in the content is NOT a tool call: only the reply's tool_calls are
- // read, and there is no prompt-based parsing path.
- inv := &streamingCapturingInvoker{
- capturingInvoker: capturingInvoker{
- hasTool: false,
- content: "```json\n{\"tool\": \"search\", \"args\": {\"q\": \"ignored\"}}\n```",
- },
- }
- m := &InvokerSessionModel{Invoker: inv}
-
- reply, err := m.StreamComplete(context.Background(), []schema.Message{*schema.UserMessage("q")}, tools, nil)
- if err != nil {
- t.Fatalf("unexpected error: %v", err)
- }
- if len(reply.ToolCalls) != 0 {
- t.Fatalf("fenced block must not be parsed as a tool call: %+v", reply.ToolCalls)
- }
-}
-
-func TestRenderPromptUsesLoader(t *testing.T) {
- loader := StringPromptLoader{"tpl": "hello {{ name }}!"}
- // Loader resolves the name: variable substituted.
- if got := prompts.Render(loader, "tpl", "", map[string]string{"name": "world"}); got != "hello world!" {
- t.Errorf("prompts.Render = %q", got)
- }
- // Unknown template with empty fallback -> empty (there is no degraded constant; a
- // missing template fails fast rather than degrading).
- if got := prompts.Render(loader, "missing", "", nil); got != "" {
- t.Errorf("unknown template with empty fallback = %q", got)
- }
- // Nil loader + known embedded name still resolves the canonical .md.
- if got := prompts.Render(nil, "sca_select", "", map[string]string{"question": "Q"}); !strings.Contains(got, "Q") {
- t.Errorf("nil loader did not resolve embedded sca_select: %q", got)
- }
- // Both {{k}} and {{ k }} are substituted (templates use the spaced form).
- if got := prompts.Render(StringPromptLoader{"tpl": "a={{x}} b={{ y }}"}, "tpl", "",
- map[string]string{"x": "1", "y": "2"}); got != "a=1 b=2" {
- t.Errorf("spaced placeholder not substituted: %q", got)
- }
-}
-
-// Helpers
-
-// Tool spec / schema contract (moved here from action_session_schema_test.go)
-
-// playbookAnchors are the 5-section contract every tool description must carry.
-var playbookAnchors = []string{"WHEN TO CALL", "DO NOT CALL", "ARGUMENTS", "OUTPUT", "IF IT FAILS"}
-
-// executorSupportedParams lists the params the executor actually consumes per
-// tool (mirrors _EXECUTOR_SUPPORTED). The schema MUST NOT declare any param
-// outside this set — otherwise the model is told to fill an argument the runtime
-// silently ignores (the list_chunks(chunk_ids) ghost bug). Note: keywords are
-// supported by the executor but intentionally NOT declared in the schema (keep
-// the description honest with the implementation); doc_scope IS declared on the
-// two tools that consume it (retrieve, graph_explore).
-var executorSupportedParams = map[string]map[string]bool{
- "retrieve": {"query": true, "doc_scope": true},
- "search_chunks": {"query": true},
- "metadata_search": {"filters": true, "logic": true},
- "list_chunks": {"doc_id": true},
- "navigate_tree": {"query": true, "keywords": true},
- "navigate_structure": {"doc_id": true, "query": true, "kind": true},
- "calculate": {"question": true, "facts": true},
- "web_search": {"query": true},
-}
-
-// TestToolSpecsHavePlaybookSections pins the PR: every tool documents the
-// 5-section contract, within the token budget (cap 1200 chars).
-func TestToolSpecsHavePlaybookSections(t *testing.T) {
- for _, name := range allTools {
- desc := ToolMap[name].Function.Description
- for _, anchor := range playbookAnchors {
- if !strings.Contains(desc, anchor) {
- t.Errorf("%s description missing anchor %q", name, anchor)
- }
- }
- if n := utf8.RuneCountInString(desc); n > 1200 {
- t.Errorf("%s description too long: %d chars (cap 1200)", name, n)
- }
- }
-}
-
-// TestActiveToolSpecsToolSurface pins mode -> exposed tool count: low=0,
-// medium/high=8, ultra=9, web-hidden=7.
-func TestActiveToolSpecsToolSurface(t *testing.T) {
- cases := []struct {
- mode string
- web bool
- want int
- }{
- {"low", true, 0},
- {"medium", true, 8},
- {"high", true, 8},
- {"ultra", true, 9},
- {"medium", false, 7},
- }
- for _, c := range cases {
- ts := &Toolset{ThinkingMode: c.mode, HasWebSearch: c.web}
- if got := len(ts.ActiveToolSpecs()); got != c.want {
- t.Errorf("mode=%s web=%t: tools = %d, want %d", c.mode, c.web, got, c.want)
- }
- }
-}
-
-// TestSchemaParamsMatchExecutor pins that no schema declares a param the
-// executor cannot consume (no ghost args).
-func TestSchemaParamsMatchExecutor(t *testing.T) {
- for _, name := range allTools {
- props, _ := ToolMap[name].Function.Parameters["properties"].(map[string]any)
- supported := executorSupportedParams[name]
- for param := range props {
- if param == "decision" {
- continue
- }
- if !supported[param] {
- t.Errorf("%s declares unsupported param %q", name, param)
- }
- }
- }
-}
-
-// TestActionRunPromptHasPlaybook pins that action_run.md exposes the TOOL
-// PLAYBOOK and only references real tools.
-func TestActionRunPromptHasPlaybook(t *testing.T) {
- prompt := loadPrompt(nil, "action_run")
- if !strings.Contains(prompt, "TOOL PLAYBOOK") {
- t.Error("action_run prompt missing TOOL PLAYBOOK")
- }
- for _, tool := range []string{"navigate_structure", "calculate", "graph_explore"} {
- if !strings.Contains(prompt, tool) {
- t.Errorf("action_run prompt missing tool %q", tool)
- }
- }
- for _, name := range append(append([]string{}, allTools...), GraphExploreTool) {
- if _, ok := ToolMap[name]; !ok {
- t.Errorf("%q referenced by the playbook is not in ToolMap", name)
- }
- }
-}
-
-// TestParseTerminalEmptyAnswerNotFound pins the empty-answer rule: an empty or whitespace
-// <answer> payload is NOT a found answer. The session then only ends when the <answer>
-// carried a new_state patch; with neither an answer nor a patch it NUDGES and continues, so
-// a bare <answer> block can never end a session with an empty collected answer..
-func TestParseTerminalEmptyAnswerNotFound(t *testing.T) {
- parent := NewState([]Variable{{ID: 0, Type: "answer"}}, 0, nil)
-
- // Whitespace answer, no patch: no found answer, no branches — the session
- // keeps running (runActionNode falls through to the nudge).
- states, found, tt, _ := ParseTerminal("<answer>{\"answer\": \" \"}</answer>", parent)
- if found != nil {
- t.Errorf("whitespace answer: FoundAnswer = %q, want nil", *found)
- }
- if len(states) != 0 {
- t.Errorf("whitespace answer: %d branch(es), want 0", len(states))
- }
- if tt == nil || *tt != "answer" {
- t.Errorf("whitespace answer: terminal type = %v, want answer", tt)
- }
-
- // Missing answer field, no patch: same.
- states, found, _, _ = ParseTerminal("<answer>{}</answer>", parent)
- if found != nil || len(states) != 0 {
- t.Errorf("missing answer: FoundAnswer=%v branches=%d, want nil/0", found, len(states))
- }
-
- // A real answer still parses.
- _, found, _, _ = ParseTerminal("<answer>{\"answer\": \" 74 \"}</answer>", parent)
- if found == nil || *found != "74" {
- t.Errorf("real answer: FoundAnswer = %v, want 74 (stripped)", found)
- }
-
- // Empty answer WITH a new_state patch: the patch is returned as the final state,
- // terminal type stays "answer", found stays nil.
- states, found, _, _ = ParseTerminal(
- "<answer>{\"answer\": \"\", \"new_state\": [{\"id\": 0, \"candidate\": \"74\", \"candidate_strength\": 0.9}]}</answer>",
- parent)
- if found != nil {
- t.Errorf("empty answer with patch: FoundAnswer = %q, want nil", *found)
- }
- if len(states) != 1 || states[0].State[0].Candidate == nil || *states[0].State[0].Candidate != "74" {
- t.Errorf("empty answer with patch: branches = %+v, want one state with candidate 74", states)
- }
-}
-
-// TestSnippetsPerQueryRisesWithModeAndFallsBack pins the per-query snippet cap.
-//
-// The cap decides how much of ONE query's candidate list the session reads, and
-// that list is where a themed query's fact-bearing passage sits: the engine
-// returns 30-60 candidates per leg, and on a 64-candidate leg the passage that
-// carried the answer ranked 20th and 38th — both discarded by a flat cap of four,
-// while handing the model those same passages produced the complete list.
-//
-// A deeper mode issues more queries and owns a larger budget, so it reads deeper;
-// a mode that declares no cap keeps the flat fallback, which is what every mode
-// did before this existed. Nothing here depends on a corpus or a language.
-func TestSnippetsPerQueryRisesWithModeAndFallsBack(t *testing.T) {
- for _, mode := range []string{"medium", "high", "ultra"} {
- spec := GetMode(mode)
- if spec.SnippetsPerQuery <= snippetsPerQuery {
- t.Errorf("%s cap = %d, want it above the flat fallback %d", mode, spec.SnippetsPerQuery, snippetsPerQuery)
- }
- if got := snippetsPerQueryFor(mode); got != spec.SnippetsPerQuery {
- t.Errorf("snippetsPerQueryFor(%s) = %d, want %d", mode, got, spec.SnippetsPerQuery)
- }
- }
- if GetMode("medium").SnippetsPerQuery >= GetMode("high").SnippetsPerQuery ||
- GetMode("high").SnippetsPerQuery >= GetMode("ultra").SnippetsPerQuery {
- t.Error("the per-query cap must rise with the mode")
- }
- // Unset (low, or an unknown label) falls back to the flat cap.
- for _, mode := range []string{"low", "no-such-mode"} {
- if got := snippetsPerQueryFor(mode); got != snippetsPerQuery {
- t.Errorf("snippetsPerQueryFor(%q) = %d, want the fallback %d", mode, got, snippetsPerQuery)
- }
- }
-}
-
-// TestRetrieveDescriptionCarriesTheEnumerationContract pins the recall path in
-// the tool description the model reads while deciding how to call it.
-//
-// Measured (2026-09-16): the same corpus and question answered sixteen members
-// with this contract present and nine-to-fourteen without it, while the runtime
-// underneath was identical — the seats, the batch weaving and the reach notes
-// exist in BOTH trees. What the description carries is how to ASK: probe the
-// names themselves in an alternation, in small batches, guess the next batch
-// yourself, and read a miss as a RESULT rather than as silence.
-func TestRetrieveDescriptionCarriesTheEnumerationContract(t *testing.T) {
- desc := ToolMap["retrieve"].Function.Description
- for _, want := range []string{
- "ENUMERATING A SET",
- "alternated with |",
- "4-6 per query",
- "guess the next batch yourself",
- "in an enumeration that is a RESULT",
- } {
- if !strings.Contains(desc, want) {
- t.Errorf("retrieve description is missing %q", want)
- }
- }
- // It must not send the opposite instruction. "Do not conclude the corpus lacks
- // the fact" is right for a fuzzy query and wrong for an exact-term probe: a
- // model told that stops probing names and settles for the ones it already
- // holds, which is the shape of the nine-member answer.
- if strings.Contains(desc, "do not conclude the corpus lacks the fact") {
- t.Error("retrieve description must not contradict the enumeration semantics of a miss")
- }
-}
-
-// TestTurnFloorIsPaidOnlyByASetSession pins the shape gate on the mode's turn floor.
-//
-// The deeper modes raised the floor for the enumeration path, and the floor is billed
-// to every turn of every session on the question, so a session with no batch to spend
-// those turns on must run on the value floor instead. The gate is the session's own
-// writing (or a parent table that declares a count/list), resolved per turn, so a
-// shape that declares itself later is not penalised.
-func TestTurnFloorIsPaidOnlyByASetSession(t *testing.T) {
- ts := &Toolset{ThinkingMode: "high"}
- if got := ResolveMode(ts).ActionMaxTurns; got <= valueTurnFloor {
- t.Fatalf("mode high ActionMaxTurns = %d, want > %d (the floor this gate shares)", got, valueTurnFloor)
- }
- modeFloor := ResolveMode(ts).ActionMaxTurns
-
- // A value session: no batch written, and the table asks for a value.
- value := &SessionState{
- Tools: ts,
- Direction: "in what year did the Sikh Empire's capital come under the British Crown",
- ParentState: State{State: []Variable{{ID: 0, Type: "date", Candidate: strPtr("1849")}}},
- DeadlineLeft: 70,
- }
- if got := value.actionMaxTurns(); got != valueTurnFloor {
- t.Errorf("value session turn floor = %d, want %d", got, valueTurnFloor)
- }
-
- // The caller's own batch is the tell that survives contact: the session is now
- // enumerating, and it pays the mode's floor for the turns it spends on it.
- value.SearchQueries = []string{"华雄|颜良|文丑"}
- if got := value.actionMaxTurns(); got != modeFloor {
- t.Errorf("session that wrote a batch turn floor = %d, want %d", got, modeFloor)
- }
-
- // A direction whose table declares a set needs the floor from its first turn: it
- // may enumerate one probe per member and never write a two-name batch.
- declared := &SessionState{
- Tools: ts,
- Direction: "how many officers did Guan Yu kill",
- ParentState: State{State: []Variable{{ID: 0, Type: "count", Candidate: strPtr("16")}}},
- DeadlineLeft: 70,
- }
- if got := declared.actionMaxTurns(); got != modeFloor {
- t.Errorf("count direction turn floor = %d, want %d", got, modeFloor)
- }
-}
-
-// TestSessionWallFollowsTheEnumeration pins the clock an enumeration session gets.
-//
-// The wall clock is spent differently by the two strategies: an enumeration session is
-// midway through a batch when its clock runs out, and a batch that is cut loses the
-// members it had already reached — measured (2026-09-16, 三国/关羽) a session was
-// cancelled one millisecond before the patch that recorded 管亥, a member it had
-// already found and whose passages were already in the shared pool. A value session
-// has no work in flight at its deadline, so it keeps the tighter clock.
-//
-// The clock follows the ENUMERATION (a set of named members whose deed the planner wrote
-// the words for), not every table that contains a count: a count of events has no batch in
-// flight either, and buying it the enumeration clock is a cost with nothing to spend it on.
-func TestSessionWallFollowsTheEnumeration(t *testing.T) {
- set := State{State: []Variable{
- {ID: 0, Type: "count", Candidate: strPtr("13"), Terms: []string{"斩", "杀"}, Subject: "关羽"},
- {ID: 1, Type: "person"},
- }}
- if got := SessionWallS(set); got != setActionTimeoutS {
- t.Errorf("enumeration direction wall = %.0f, want %.0f", got, setActionTimeoutS)
- }
- // A count of EVENTS: set-shaped, no names to enumerate.
- countOnly := State{State: []Variable{{ID: 0, Type: "count", Candidate: strPtr("13"), Terms: []string{"won"}, Subject: "Brazil"}}}
- if got := SessionWallS(countOnly); got != actionTimeoutS {
- t.Errorf("count-of-events direction wall = %.0f, want %.0f", got, actionTimeoutS)
- }
- value := State{State: []Variable{{ID: 0, Type: "date", Candidate: strPtr("1858")}}}
- if got := SessionWallS(value); got != actionTimeoutS {
- t.Errorf("value direction wall = %.0f, want %.0f", got, actionTimeoutS)
- }
- if setActionTimeoutS <= actionTimeoutS {
- t.Errorf("the set clock (%.0f) must exceed the value clock (%.0f)", setActionTimeoutS, actionTimeoutS)
- }
-}
-
-// TestDigestShowsAPassageWholeEnoughToNameSomeone pins the seed digest's per-chunk
-// cap.
-//
-// The digest is how a session SEES the pool without re-querying it, and its cap used to
-// be a hard 300-code-point cut. A 300-code-point window is shorter than the sentence a deed
-// is reported in: the passage names the actor early and the person at the end, so the cut
-// removed exactly the clause that makes a member a member — and a model told to answer only
-// from what it was shown then excluded them.
-//
-// Measured (2026-09-14, fixrecall): with the cap at 1200 an enumeration reached
-// seventeen members. Measured here (2026-09-16, 三国/关羽): 管亥 / 杨龄 / 程远志
-// appeared ZERO times in whole runs whose pool held their passages.
-func TestDigestShowsAPassageWholeEnoughToNameSomeone(t *testing.T) {
- narrative := strings.Repeat("却说曹兵势大", 60) + "云长手起刀落,砍杨龄于马下。"
- kb := &Kbinfos{}
- kb.Admit(func(p *PoolAdmitter) {
- p.Add(map[string]any{"chunk_id": "c1", "content": narrative})
- })
-
- digest := extractRelevantEvidence(kb, "关羽杀了多少有姓名的人物", 4)
- if !strings.Contains(digest, "砍杨龄于马下") {
- t.Fatalf("digest = %q…, want the clause that names the member (cap %d)", truncateRunes(digest, 120), evidenceDigestChars)
- }
- if got := len([]rune(digest)); got > evidenceDigestChars+64 {
- t.Errorf("digest = %d runes, want no more than one capped chunk plus its id", got)
- }
- if evidenceDigestChars < 1200 {
- t.Errorf("digest cap = %d, want at least the 1200 an admitted passage carries", evidenceDigestChars)
- }
-}
-
-// seedRecordingModel captures the messages of every completion call, so a test can assert
-// what the session seed actually carried. The canned reply is an empty state patch, which
-// ends the session after its first turn.
-type seedRecordingModel struct{ seen [][]schema.Message }
-
-func (m *seedRecordingModel) Complete(_ context.Context, msgs []schema.Message, _ []ToolSpec) (*ModelReply, error) {
- cp := make([]schema.Message, len(msgs))
- copy(cp, msgs)
- m.seen = append(m.seen, cp)
- return &ModelReply{Content: `<state>{"new_states": []}</state>`}, nil
-}
-
-// TestRunActionSessionSeedCarriesMetadataCatalog pins the seed half of the catalog: the
-// session is TOLD which fields its dataset carries, so a metadata_search filter can name
-// one without a trial-and-error call first. Without a catalog the block is absent — a
-// metadata-free corpus's prompt stays exactly as it shipped.
-func TestRunActionSessionSeedCarriesMetadataCatalog(t *testing.T) {
- cases := []struct {
- name string
- cat *MetadataCatalog
- want bool
- }{
- {
- name: "catalog",
- cat: &MetadataCatalog{
- Keys: []string{"author"},
- Samples: map[string][]MetadataSample{"author": {{Value: "Alice", Docs: 1}}},
- },
- want: true,
- },
- {name: "no metadata", cat: nil, want: false},
- }
- for _, c := range cases {
- mdl := &seedRecordingModel{}
- deps := SessionDeps{
- Tools: &Toolset{ThinkingMode: "high", MetadataFields: c.cat},
- Model: mdl,
- KB: &Kbinfos{},
- }
- RunActionSession(context.Background(), deps, "who wrote it", State{State: []Variable{{ID: 0, Type: "aspect"}}}, 60, "", nil, nil)
- if len(mdl.seen) == 0 {
- t.Fatalf("%s: the session never called the model", c.name)
- }
- var joined strings.Builder
- for _, m := range mdl.seen[0] {
- // Only the USER message is the seed: the system prompt names AVAILABLE
- // METADATA in its playbook, so scanning every message would pass vacuously.
- if m.Role != schema.User {
- continue
- }
- joined.WriteString(m.Content)
- joined.WriteString("\n")
- }
- seed := joined.String()
- if got := strings.Contains(seed, "AVAILABLE METADATA"); got != c.want {
- t.Errorf("%s: seed carries AVAILABLE METADATA = %v, want %v:\n%s", c.name, got, c.want, seed)
- }
- if c.want && !strings.Contains(seed, "author") {
- t.Errorf("%s: the seed block must name the dataset's field:\n%s", c.name, seed)
- }
- }
-}
-
-// TestActiveToolSpecsDoesNotMutateSharedToolMap pins the copy-then-patch contract: the
-// catalog rewrites the spec a SESSION sees, never the package-level ToolMap entry every
-// other session (and every concurrent rag call in this one) reads from. Rendered under
-// -race as well, so the maps a patch touches cannot be written concurrently.
-func TestActiveToolSpecsDoesNotMutateSharedToolMap(t *testing.T) {
- cat := &MetadataCatalog{Keys: []string{"author", "title"}, Samples: map[string][]MetadataSample{}}
-
- var wg sync.WaitGroup
- for i := 0; i < 16; i++ {
- wg.Add(1)
- go func() {
- defer wg.Done()
- ts := &Toolset{ThinkingMode: "high", MetadataFields: cat}
- _ = ts.ActiveToolSpecs()
- }()
- }
- wg.Wait()
-
- shipped := ToolMap["metadata_search"]
- if got := metadataKeyEnum(shipped); len(got) != 0 {
- t.Errorf("shipped key enum = %v, want NO field baked into the shared spec", got)
- }
- if desc, _ := metadataKeyParam(shipped)["description"].(string); strings.Contains(desc, "this dataset's metadata fields") {
- t.Errorf("a session rewrite leaked its catalog wording into the shipped key parameter: %q", desc)
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/arithmetic.go b/internal/rag/agentic-rag/runtime/arithmetic.go
deleted file mode 100644
index 278f7cb8ac..0000000000
--- a/internal/rag/agentic-rag/runtime/arithmetic.go
+++ /dev/null
@@ -1,1580 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "fmt"
- "math"
- "ragflow/internal/common"
- "strconv"
- "strings"
- "time"
- "unicode"
-
- "github.com/cloudwego/eino/schema"
-
- "ragflow/internal/agent/chat"
-)
-
-// Deterministic arithmetic over retrieved evidence.
-//
-// Some questions ask for a number no single source states — the combined population of
-// three counties, how many listed films won an award, the years between two dates. Every
-// input is in the evidence by then and only the arithmetic is missing, which an LLM does
-// by writing digits one at a time and gets wrong often enough to matter. So the LLM
-// writes ONE expression and we evaluate it.
-//
-// The expression is model-written and NOT trusted: Go has no eval and the model writes
-// syntax (`**`, `x if y else z`, list literals) that go/parser cannot read, so this file
-// ships a small parser for a deliberately tiny expression language and rejects everything
-// outside the whitelist BEFORE evaluation. No reflection, no name lookup, no property
-// access: an unlisted construct is a parse error, not a sandbox escape.
-//
-// Public interface
-//
-// Compute(expression) -> (rendered, error) evaluate one expression safely.
-// ComputeFromFacts(question, facts, fitBudget) -> *ComputedFact | nil
-// ask the model whether the question asks for a derivable number; if so,
-// write the expression, evaluate it, return a structured result.
-
-const (
- // computeMaxChars caps the whole expression; every figure is inline, none
- // is long.
- computeMaxChars = 400
- // maxStringLiteral caps a string literal argument.
- maxStringLiteral = 256
- // maxPowExponent caps `**` at abs(exponent) > 64.
- maxPowExponent = 64
-)
-
-// Compute evaluates an LLM-written arithmetic expression.
-// Returns (rendered, error); exactly one of the two is non-empty. Every
-// rejection is a normal outcome — the caller simply carries on without the
-// computed evidence.
-func Compute(expression string) (string, string) {
- expr := strings.TrimSpace(expression)
- if expr == "" {
- return "", "empty expression"
- }
- if len(expr) > computeMaxChars {
- return "", fmt.Sprintf("expression is longer than %d characters", computeMaxChars)
- }
- p := &parser{src: expr}
- node, err := p.parseExpression()
- if err != nil {
- return "", err.Error()
- }
- if rest := p.skipSpace(); rest < len(p.src) {
- return "", fmt.Sprintf("unexpected trailing input at %d", rest)
- }
- if problem := checkExpression(node); problem != "" {
- return "", problem
- }
- value, err := evalSafe(node)
- if err != nil {
- return "", fmt.Sprintf("failed to evaluate (%v)", err)
- }
- switch v := value.(type) {
- case bool:
- return "", "result is bool, not a number"
- case int64:
- return formatNumber(float64(v)), ""
- case float64:
- if math.IsNaN(v) || math.IsInf(v, 0) {
- return "", "result is not a finite number"
- }
- return formatNumber(v), ""
- }
- return "", fmt.Sprintf("result is %T, not a number", value)
-}
-
-// formatNumber: render a computed number without
-// float noise ("3.0" -> "3", 0.1+0.2 -> "0.3").
-func formatNumber(value float64) string {
- if value == math.Trunc(value) && math.Abs(value) < 1e15 {
- return strconv.FormatInt(int64(value), 10)
- }
- s := strconv.FormatFloat(value, 'f', 6, 64)
- s = strings.TrimRight(s, "0")
- return strings.TrimRight(s, ".")
-}
-
-// AST
-
-type node interface{ pos() int }
-
-type constNode struct {
- p int
- kind string // "int" | "float" | "str" | "bool"
- num float64
- str string
-}
-
-func (n *constNode) pos() int { return n.p }
-
-type nameNode struct {
- p int
- name string
-}
-
-func (n *nameNode) pos() int { return n.p }
-
-type unaryNode struct {
- p int
- op string
- operand node
-}
-
-func (n *unaryNode) pos() int { return n.p }
-
-type binaryNode struct {
- p int
- op string
- left, right node
-}
-
-func (n *binaryNode) pos() int { return n.p }
-
-// chainNode: which is an N-ary node: `a < b < c` is
-// `(a < b) and (b < c)`, with each operand evaluated exactly ONCE and the chain
-// short-circuiting on the first false comparison.
-//
-// Building it as left-associative binary nodes instead — `((a < b) < c)` —
-// compares the previous comparison's BOOLEAN result against the next operand,
-// so `3 > 2 > 1` becomes `True > 1` = `1 > 1` = false, and `2 < 1 < 3` becomes
-// `False < 3` = `0 < 3` = true. Both are wrong.
-type chainNode struct {
- p int
- ops []string // len(operands) - 1
- operands []node
-}
-
-func (n *chainNode) pos() int { return n.p }
-
-type ifExpNode struct {
- p int
- cond, body, orelse node
-}
-
-func (n *ifExpNode) pos() int { return n.p }
-
-type callNode struct {
- p int
- fn string
- args []node
-}
-
-func (n *callNode) pos() int { return n.p }
-
-type seqNode struct {
- p int
- kind string // "list" | "tuple" | "set"
- elts []node
-}
-
-func (n *seqNode) pos() int { return n.p }
-
-// Lexer + Pratt parser for a tiny expression language
-
-// Supported: numeric and string literals, True/False/None, list/tuple/set
-// literals, unary + - not, binary + - * / // % ** and comparison chains, `and`
-// / `or`, the ternary `a if b else c`, and calls to the whitelisted functions.
-// Everything else — attributes, subscripts, comprehensions, lambdas, f-strings,
-// assignments, imports — is a parse error.
-type parser struct {
- src string
- i int
-}
-
-func (p *parser) skipSpace() int {
- for p.i < len(p.src) && (p.src[p.i] == ' ' || p.src[p.i] == '\t') {
- p.i++
- }
- return p.i
-}
-
-func (p *parser) errf(format string, args ...any) error {
- return fmt.Errorf("does not parse (%s)", fmt.Sprintf(format, args...))
-}
-
-func (p *parser) parseExpression() (node, error) { return p.parseTernary() }
-
-func (p *parser) parseTernary() (node, error) {
- body, err := p.parseOr()
- if err != nil {
- return nil, err
- }
- p.skipSpace()
- if !p.matchWord("if") {
- return body, nil
- }
- cond, err := p.parseOr()
- if err != nil {
- return nil, err
- }
- p.skipSpace()
- if !p.matchWord("else") {
- return nil, p.errf("ternary needs 'else'")
- }
- orelse, err := p.parseExpression()
- if err != nil {
- return nil, err
- }
- return &ifExpNode{p: 0, cond: cond, body: body, orelse: orelse}, nil
-}
-
-func (p *parser) parseOr() (node, error) {
- left, err := p.parseAnd()
- if err != nil {
- return nil, err
- }
- for {
- p.skipSpace()
- if !p.matchWord("or") {
- return left, nil
- }
- right, err := p.parseAnd()
- if err != nil {
- return nil, err
- }
- left = &binaryNode{op: "or", left: left, right: right}
- }
-}
-
-func (p *parser) parseAnd() (node, error) {
- left, err := p.parseNot()
- if err != nil {
- return nil, err
- }
- for {
- p.skipSpace()
- if !p.matchWord("and") {
- return left, nil
- }
- right, err := p.parseNot()
- if err != nil {
- return nil, err
- }
- left = &binaryNode{op: "and", left: left, right: right}
- }
-}
-
-func (p *parser) parseNot() (node, error) {
- p.skipSpace()
- if p.matchWord("not") {
- operand, err := p.parseNot()
- if err != nil {
- return nil, err
- }
- return &unaryNode{op: "not", operand: operand}, nil
- }
- return p.parseComparison()
-}
-
-var cmpOps = []string{"==", "!=", "<=", ">=", "<", ">"}
-
-// parseComparison collects the WHOLE comparison chain into one chainNode. Two or more
-// operators are what make chaining observable; a single comparison is just a one-op
-// chain.
-func (p *parser) parseComparison() (node, error) {
- start := p.i
- first, err := p.parseSum()
- if err != nil {
- return nil, err
- }
- operands := []node{first}
- var ops []string
- for {
- p.skipSpace()
- matched := ""
- for _, op := range cmpOps {
- if strings.HasPrefix(p.src[p.i:], op) {
- matched = op
- break
- }
- }
- if matched == "" {
- break
- }
- p.i += len(matched)
- right, err := p.parseSum()
- if err != nil {
- return nil, err
- }
- ops = append(ops, matched)
- operands = append(operands, right)
- }
- if len(ops) == 0 {
- return first, nil
- }
- return &chainNode{p: start, ops: ops, operands: operands}, nil
-}
-
-func (p *parser) parseSum() (node, error) {
- left, err := p.parseProduct()
- if err != nil {
- return nil, err
- }
- for {
- p.skipSpace()
- if p.i >= len(p.src) {
- return left, nil
- }
- var op string
- switch p.src[p.i] {
- case '+':
- op = "+"
- case '-':
- op = "-"
- default:
- return left, nil
- }
- p.i++
- right, err := p.parseProduct()
- if err != nil {
- return nil, err
- }
- left = &binaryNode{op: op, left: left, right: right}
- }
-}
-
-func (p *parser) parseProduct() (node, error) {
- left, err := p.parseUnary()
- if err != nil {
- return nil, err
- }
- for {
- p.skipSpace()
- if p.i >= len(p.src) {
- return left, nil
- }
- var op string
- switch {
- case strings.HasPrefix(p.src[p.i:], "**"):
- op = "**"
- case strings.HasPrefix(p.src[p.i:], "//"):
- op = "//"
- case p.src[p.i] == '*':
- op = "*"
- case p.src[p.i] == '/':
- op = "/"
- case p.src[p.i] == '%':
- op = "%"
- default:
- return left, nil
- }
- p.i += len(op)
- right, err := p.parseUnary()
- if err != nil {
- return nil, err
- }
- left = &binaryNode{op: op, left: left, right: right}
- }
-}
-
-func (p *parser) parseUnary() (node, error) {
- p.skipSpace()
- if p.i < len(p.src) {
- switch p.src[p.i] {
- case '-':
- p.i++
- operand, err := p.parseUnary()
- if err != nil {
- return nil, err
- }
- return &unaryNode{op: "-", operand: operand}, nil
- case '+':
- p.i++
- operand, err := p.parseUnary()
- if err != nil {
- return nil, err
- }
- return &unaryNode{op: "+", operand: operand}, nil
- }
- }
- return p.parsePower()
-}
-
-// parsePower binds tighter than unary on the right and is right-associative:
-// `2 ** 3 ** 2` is 512 and `-2 ** 2` is -4.
-func (p *parser) parsePower() (node, error) {
- base, err := p.parseAtom()
- if err != nil {
- return nil, err
- }
- p.skipSpace()
- if strings.HasPrefix(p.src[p.i:], "**") {
- p.i += 2
- exponent, err := p.parseUnary()
- if err != nil {
- return nil, err
- }
- return &binaryNode{op: "**", left: base, right: exponent}, nil
- }
- return base, nil
-}
-
-func (p *parser) parseAtom() (node, error) {
- p.skipSpace()
- if p.i >= len(p.src) {
- return nil, p.errf("unexpected end of expression")
- }
- start := p.i
- c := p.src[p.i]
-
- switch {
- case c == '(':
- p.i++
- inner, err := p.parseExpression()
- if err != nil {
- return nil, err
- }
- p.skipSpace()
- if p.i >= len(p.src) {
- return nil, p.errf("unclosed '('")
- }
- // A trailing (or interior) comma turns this into a TUPLE literal — the sequence form
- // used by sum((1,2,3)) / len((1,2,3)) / min((1,2,3)) / letters((...)). A single
- // element with no comma is a plain parenthesised expression (not a one-tuple):
- // `(1+2)` == 3 vs `(1,)` == (1,).
- if p.src[p.i] == ',' {
- items := []node{inner}
- for {
- p.i++ // consume ','
- p.skipSpace()
- if p.i >= len(p.src) {
- return nil, p.errf("unclosed '('")
- }
- if p.src[p.i] == ')' {
- p.i++
- return &seqNode{kind: "tuple", elts: items}, nil
- }
- elem, err := p.parseExpression()
- if err != nil {
- return nil, err
- }
- items = append(items, elem)
- p.skipSpace()
- if p.i >= len(p.src) {
- return nil, p.errf("unclosed '('")
- }
- if p.src[p.i] == ')' {
- p.i++
- return &seqNode{kind: "tuple", elts: items}, nil
- }
- if p.src[p.i] != ',' {
- return nil, p.errf("unclosed '('")
- }
- }
- }
- if p.src[p.i] != ')' {
- return nil, p.errf("unclosed '('")
- }
- p.i++
- return inner, nil
- case c == '[':
- return p.parseSeq("[", "]", "list")
- case c == '{':
- return p.parseSeq("{", "}", "set")
- case c == '"' || c == '\'':
- return p.parseString()
- case c >= '0' && c <= '9':
- return p.parseNumber()
- case isIdentStart(c):
- return p.parseNameOrCall()
- }
- return nil, p.errf("unexpected character %q at %d", string(c), start)
-}
-
-func (p *parser) parseSeq(open, close, kind string) (node, error) {
- p.i++ // consume open
- seq := &seqNode{kind: kind}
- p.skipSpace()
- if p.i < len(p.src) && p.src[p.i] == close[0] {
- p.i++
- return seq, nil
- }
- for {
- elt, err := p.parseExpression()
- if err != nil {
- return nil, err
- }
- seq.elts = append(seq.elts, elt)
- p.skipSpace()
- if p.i >= len(p.src) {
- return nil, p.errf("unclosed %q", open)
- }
- if p.src[p.i] == close[0] {
- p.i++
- return seq, nil
- }
- if p.src[p.i] != ',' {
- return nil, p.errf("expected ',' or %q", close)
- }
- p.i++
- }
-}
-
-func (p *parser) parseString() (node, error) {
- quote := p.src[p.i]
- p.i++
- var b strings.Builder
- for p.i < len(p.src) && p.src[p.i] != quote {
- if p.src[p.i] == '\\' && p.i+1 < len(p.src) {
- b.WriteByte(p.src[p.i+1])
- p.i += 2
- continue
- }
- b.WriteByte(p.src[p.i])
- p.i++
- }
- if p.i >= len(p.src) {
- return nil, p.errf("unterminated string")
- }
- p.i++ // consume closing quote
- return &constNode{kind: "str", str: b.String()}, nil
-}
-
-func (p *parser) parseNumber() (node, error) {
- start := p.i
- for p.i < len(p.src) && (p.src[p.i] >= '0' && p.src[p.i] <= '9') {
- p.i++
- }
- isFloat := false
- if p.i < len(p.src) && p.src[p.i] == '.' && p.i+1 < len(p.src) &&
- p.src[p.i+1] >= '0' && p.src[p.i+1] <= '9' {
- isFloat = true
- p.i++
- for p.i < len(p.src) && p.src[p.i] >= '0' && p.src[p.i] <= '9' {
- p.i++
- }
- }
- lit := p.src[start:p.i]
- if isFloat {
- v, err := strconv.ParseFloat(lit, 64)
- if err != nil {
- return nil, p.errf("bad float %q", lit)
- }
- return &constNode{kind: "float", num: v}, nil
- }
- v, err := strconv.ParseFloat(lit, 64)
- if err != nil {
- return nil, p.errf("bad int %q", lit)
- }
- return &constNode{kind: "int", num: v}, nil
-}
-
-func (p *parser) parseNameOrCall() (node, error) {
- start := p.i
- for p.i < len(p.src) && isIdentChar(p.src[p.i]) {
- p.i++
- }
- name := p.src[start:p.i]
- switch name {
- case "True":
- return &constNode{kind: "bool", str: "true"}, nil
- case "False":
- return &constNode{kind: "bool", str: "false"}, nil
- case "None":
- return nil, p.errf("None is not allowed")
- }
- p.skipSpace()
- if p.i < len(p.src) && p.src[p.i] == '(' {
- p.i++
- call := &callNode{fn: name}
- p.skipSpace()
- if p.i < len(p.src) && p.src[p.i] == ')' {
- p.i++
- return call, nil
- }
- for {
- arg, err := p.parseExpression()
- if err != nil {
- return nil, err
- }
- call.args = append(call.args, arg)
- p.skipSpace()
- if p.i >= len(p.src) {
- return nil, p.errf("unclosed '('")
- }
- if p.src[p.i] == ')' {
- p.i++
- return call, nil
- }
- if p.src[p.i] != ',' {
- return nil, p.errf("expected ',' or ')'")
- }
- p.i++
- }
- }
- return &nameNode{name: name}, nil
-}
-
-func (p *parser) matchWord(word string) bool {
- p.skipSpace()
- if !strings.HasPrefix(p.src[p.i:], word) {
- return false
- }
- end := p.i + len(word)
- if end < len(p.src) && isIdentChar(p.src[end]) {
- return false
- }
- p.i = end
- return true
-}
-
-func isIdentStart(c byte) bool {
- return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
-}
-
-func isIdentChar(c byte) bool {
- return isIdentStart(c) || (c >= '0' && c <= '9')
-}
-
-// Whitelist check — runs BEFORE evaluation
-
-// computeFunctions
-var computeFunctions = map[string]bool{
- "abs": true, "round": true, "min": true, "max": true, "sum": true,
- "len": true, "int": true, "float": true, "sorted": true,
- "letters": true, "digit_sum": true, "date_diff": true,
-}
-
-// computeAlwaysNumeric: functions whose
-// result is a number whatever they are handed. `min`/`max`/`sum` are absent on
-// purpose — min("b","a") is a string — and `sorted` returns a list, so neither
-// may stand where a number is required.
-var computeAlwaysNumeric = map[string]bool{
- "abs": true, "round": true, "int": true, "float": true, "len": true,
- "letters": true, "digit_sum": true, "date_diff": true,
-}
-
-// checkExpression: reject anything outside the
-// arithmetic whitelist. Returns "" when clean.
-func checkExpression(n node) string {
- switch t := n.(type) {
- case *constNode:
- return ""
- case *nameNode:
- if !computeFunctions[t.name] {
- return fmt.Sprintf("unknown name %q", t.name)
- }
- return ""
- case *unaryNode:
- return checkExpression(t.operand)
- case *binaryNode:
- if problem := checkExpression(t.left); problem != "" {
- return problem
- }
- if problem := checkExpression(t.right); problem != "" {
- return problem
- }
- // Multiplication and exponentiation can turn a short expression into an
- // arbitrarily large object ("a" * 10**9, [1] * 10**9), so both operands
- // must be provably numeric.
- if t.op == "*" || t.op == "**" {
- if !isNumeric(t.left) || !isNumeric(t.right) {
- if t.op == "*" {
- return "multiplication is only allowed on numbers"
- }
- return "exponentiation is only allowed on numbers"
- }
- if t.op == "**" {
- if c, ok := t.right.(*constNode); ok &&
- (c.kind == "int" || c.kind == "float") && math.Abs(c.num) > maxPowExponent {
- return "exponent is too large"
- }
- }
- }
- return ""
- case *chainNode:
- for _, sub := range t.operands {
- if problem := checkExpression(sub); problem != "" {
- return problem
- }
- }
- return ""
- case *ifExpNode:
- for _, sub := range []node{t.cond, t.body, t.orelse} {
- if problem := checkExpression(sub); problem != "" {
- return problem
- }
- }
- return ""
- case *seqNode:
- for _, elt := range t.elts {
- if problem := checkExpression(elt); problem != "" {
- return problem
- }
- }
- return ""
- case *callNode:
- if !computeFunctions[t.fn] {
- return "only the listed functions may be called"
- }
- // len("Ada Lovelace") is 12 and the answer is 11 — the gap is silent, so
- // the expression is refused rather than counted.
- if t.fn == "len" && len(t.args) == 1 {
- if c, ok := t.args[0].(*constNode); ok && c.kind == "str" {
- return "len() on a string literal is ambiguous; use letters()"
- }
- }
- for _, arg := range t.args {
- if c, ok := arg.(*constNode); ok && c.kind == "str" && len(c.str) > maxStringLiteral {
- return "string literal is too long"
- }
- if problem := checkExpression(arg); problem != "" {
- return problem
- }
- }
- return ""
- }
- return "unsupported expression"
-}
-
-// isNumeric: true when the node can ONLY evaluate to
-// a number.
-func isNumeric(n node) bool {
- switch t := n.(type) {
- case *constNode:
- return t.kind == "int" || t.kind == "float" || t.kind == "bool"
- case *unaryNode:
- return isNumeric(t.operand)
- case *binaryNode:
- switch t.op {
- case "+", "-", "*", "/", "//", "%", "**":
- return isNumeric(t.left) && isNumeric(t.right)
- case "==", "!=", "<", "<=", ">", ">=", "and", "or":
- return true // a comparison is a bool, and a bool is an int
- }
- return false
- case *chainNode:
- return true // a comparison is a bool, and a bool is an int
- case *ifExpNode:
- return isNumeric(t.body) && isNumeric(t.orelse)
- case *callNode:
- if computeAlwaysNumeric[t.fn] {
- return true
- }
- if t.fn == "sum" || t.fn == "min" || t.fn == "max" {
- for _, arg := range t.args {
- if !(isNumeric(arg) || isNumericSequence(arg)) {
- return false
- }
- }
- return true
- }
- return false
- }
- return false
-}
-
-// isNumericSequence: a literal sequence whose
-// every element is provably numeric.
-func isNumericSequence(n node) bool {
- seq, ok := n.(*seqNode)
- if !ok {
- return false
- }
- for _, elt := range seq.elts {
- if !isNumeric(elt) {
- return false
- }
- }
- return true
-}
-
-// Evaluation — no builtins, no reflection
-
-// evalSafe evaluates n, converting a panic into an error.
-//
-// The helper functions (letters / digit_sum) reject bad argument types by panicking. The
-// panic is caught at the top; without a recover here the same input would crash the whole
-// request instead of being a normal, logged refusal.
-func evalSafe(n node) (value any, err error) {
- defer func() {
- if r := recover(); r != nil {
- value = nil
- err = fmt.Errorf("%v", r)
- }
- }()
- return evalNode(n)
-}
-
-func evalNode(n node) (any, error) {
- switch t := n.(type) {
- case *constNode:
- switch t.kind {
- case "int":
- return int64(t.num), nil
- case "float":
- return t.num, nil
- case "bool":
- return t.str == "true", nil
- case "str":
- return t.str, nil
- }
- case *nameNode:
- return nil, fmt.Errorf("bare name %q is not a value", t.name)
- case *unaryNode:
- v, err := evalNode(t.operand)
- if err != nil {
- return nil, err
- }
- switch t.op {
- case "-":
- f, err := asNumber(v)
- if err != nil {
- return nil, err
- }
- if i, ok := v.(int64); ok {
- return -i, nil
- }
- return -f, nil
- case "+":
- f, err := asNumber(v)
- if err != nil {
- return nil, err
- }
- if i, ok := v.(int64); ok {
- return i, nil
- }
- return f, nil
- case "not":
- return !truthy(v), nil
- }
- case *ifExpNode:
- cond, err := evalNode(t.cond)
- if err != nil {
- return nil, err
- }
- if truthy(cond) {
- return evalNode(t.body)
- }
- return evalNode(t.orelse)
- case *seqNode:
- out := make([]any, 0, len(t.elts))
- for _, elt := range t.elts {
- v, err := evalNode(elt)
- if err != nil {
- return nil, err
- }
- out = append(out, v)
- }
- if t.kind == "set" {
- // Set literals deduplicate members: len({1,1,2}) == 2. Without this,
- // len/min/max/sorted on a set literal would count duplicates.
- out = dedupSequence(out)
- }
- return out, nil
- case *binaryNode:
- return evalBinary(t)
- case *chainNode:
- return evalChain(t)
- case *callNode:
- return evalCall(t)
- }
- return nil, fmt.Errorf("unsupported node")
-}
-
-// evalChain evaluates a comparison chain: each comparison is applied to ADJACENT operands,
-// results are combined with `and`, and each operand is evaluated exactly once. It
-// short-circuits on the first false comparison.
-func evalChain(t *chainNode) (any, error) {
- left, err := evalNode(t.operands[0])
- if err != nil {
- return nil, err
- }
- for i, op := range t.ops {
- right, err := evalNode(t.operands[i+1])
- if err != nil {
- return nil, err
- }
- ok, err := compareValues(op, left, right)
- if err != nil {
- return nil, err
- }
- if !ok {
- return false, nil
- }
- // The next comparison starts from this operand, not from `true`.
- left = right
- }
- return true, nil
-}
-
-func evalBinary(t *binaryNode) (any, error) {
- // `and` / `or` short-circuit.
- if t.op == "and" || t.op == "or" {
- left, err := evalNode(t.left)
- if err != nil {
- return nil, err
- }
- if t.op == "and" && !truthy(left) {
- return left, nil
- }
- if t.op == "or" && truthy(left) {
- return left, nil
- }
- return evalNode(t.right)
- }
- left, err := evalNode(t.left)
- if err != nil {
- return nil, err
- }
- right, err := evalNode(t.right)
- if err != nil {
- return nil, err
- }
- switch t.op {
- case "+":
- // Numeric addition, or sequence concatenation (both bounded by literal
- // length, so no amplification risk).
- if li, ok := left.(int64); ok {
- if ri, ok := right.(int64); ok {
- return li + ri, nil
- }
- }
- lf, err1 := asNumber(left)
- rf, err2 := asNumber(right)
- if err1 == nil && err2 == nil {
- return lf + rf, nil
- }
- return concatSequences(left, right)
- case "-", "*", "/", "//", "%", "**":
- lf, err1 := asNumber(left)
- if err1 != nil {
- return nil, err1
- }
- rf, err2 := asNumber(right)
- if err2 != nil {
- return nil, err2
- }
- switch t.op {
- case "-":
- if li, ok := left.(int64); ok {
- if ri, ok := right.(int64); ok {
- return li - ri, nil
- }
- }
- return lf - rf, nil
- case "*":
- if li, ok := left.(int64); ok {
- if ri, ok := right.(int64); ok {
- return li * ri, nil
- }
- }
- return lf * rf, nil
- case "/":
- if rf == 0 {
- return nil, fmt.Errorf("division by zero")
- }
- return lf / rf, nil
- case "//":
- if rf == 0 {
- return nil, fmt.Errorf("floor division by zero")
- }
- return math.Floor(lf / rf), nil
- case "%":
- if rf == 0 {
- return nil, fmt.Errorf("modulo by zero")
- }
- // % returns a value with the SIGN OF THE DIVISOR
- // (a - (b * floor(a / b))): -3 % 2 == 1, 3 % -2 == -1.
- // math.Mod returns a value with the sign of the dividend
- // (math.Mod(-3, 2) == -1), so adjust when signs disagree.
- r := math.Mod(lf, rf)
- if r != 0 && (r < 0) != (rf < 0) {
- r += rf
- }
- return r, nil
- case "**":
- return math.Pow(lf, rf), nil
- }
- case "==", "!=", "<", "<=", ">", ">=":
- return compareValues(t.op, left, right)
- }
- return nil, fmt.Errorf("unsupported operator %q", t.op)
-}
-
-func evalCall(t *callNode) (any, error) {
- args := make([]any, 0, len(t.args))
- for _, arg := range t.args {
- v, err := evalNode(arg)
- if err != nil {
- return nil, err
- }
- args = append(args, v)
- }
- switch t.fn {
- case "abs":
- f, err := asNumber(single(args))
- if err != nil {
- return nil, err
- }
- if v, ok := single(args).(int64); ok && v < 0 {
- return -v, nil
- }
- return math.Abs(f), nil
- case "round":
- f, err := asNumber(single(args))
- if err != nil {
- return nil, err
- }
- if len(args) > 1 {
- nd, err := asNumber(args[1])
- if err != nil {
- return nil, err
- }
- scale := math.Pow(10, nd)
- return math.RoundToEven(f*scale) / scale, nil
- }
- // round() is round-half-to-even (banker's rounding): round(2.5) == 2, round(3.5) ==
- // 4. math.Round rounds half away from zero (round(2.5) == 3), so RoundToEven is the
- // right primitive.
- return math.RoundToEven(f), nil
- case "int":
- // int() accepts a number (truncating toward zero) or a whole-number string;
- // int("1.5") is invalid, so reject decimals.
- v := single(args)
- switch n := v.(type) {
- case int64, float64, bool:
- f, err := asNumber(v)
- if err != nil {
- return nil, err
- }
- return int64(f), nil
- case string:
- i, err := strconv.ParseInt(strings.TrimSpace(n), 10, 64)
- if err != nil {
- return nil, fmt.Errorf("int(%q) is not a whole number", n)
- }
- return i, nil
- default:
- return nil, fmt.Errorf("int() expects a number or numeric string, got %T", v)
- }
- case "float":
- // float() accepts a number or any numeric string.
- v := single(args)
- switch n := v.(type) {
- case int64, float64, bool:
- f, err := asNumber(v)
- if err != nil {
- return nil, err
- }
- return f, nil
- case string:
- f, err := strconv.ParseFloat(strings.TrimSpace(n), 64)
- if err != nil {
- return nil, fmt.Errorf("float(%q) is not a number", n)
- }
- return f, nil
- default:
- return nil, fmt.Errorf("float() expects a number or numeric string, got %T", v)
- }
- case "len":
- return int64(sequenceLen(single(args))), nil
- case "min", "max":
- flat := flattenArgs(args)
- if len(flat) == 0 {
- return nil, fmt.Errorf("%s() needs at least one argument", t.fn)
- }
- return extrema(t.fn, flat)
- case "sum":
- flat := flattenArgs(args)
- total := 0.0
- allInt := true
- for _, v := range flat {
- f, err := asNumber(v)
- if err != nil {
- return nil, err
- }
- if _, ok := v.(int64); !ok {
- allInt = false
- }
- total += f
- }
- if allInt {
- return int64(total), nil
- }
- return total, nil
- case "sorted":
- return sortValues(flattenArgs(args))
- case "letters":
- return int64(letters(flattenArgs(args))), nil
- case "digit_sum":
- return int64(digitSum(flattenArgs(args))), nil
- case "date_diff":
- if len(args) != 2 {
- return nil, fmt.Errorf("date_diff() takes exactly two ISO dates")
- }
- return dateDiff(args[0], args[1])
- }
- return nil, fmt.Errorf("unknown function %q", t.fn)
-}
-
-func single(args []any) any {
- if len(args) == 0 {
- return nil
- }
- return args[0]
-}
-
-// flattenArgs accepts "any number of args, or a single list of them".
-func flattenArgs(args []any) []any {
- if len(args) == 1 {
- if list, ok := args[0].([]any); ok {
- return list
- }
- }
- return args
-}
-
-func asNumber(v any) (float64, error) {
- switch n := v.(type) {
- case int64:
- return float64(n), nil
- case float64:
- return n, nil
- case bool:
- if n {
- return 1, nil
- }
- return 0, nil
- }
- return 0, fmt.Errorf("%v is not a number", v)
-}
-
-func truthy(v any) bool {
- switch n := v.(type) {
- case bool:
- return n
- case int64:
- return n != 0
- case float64:
- return n != 0
- case string:
- return n != ""
- case []any:
- return len(n) > 0
- case nil:
- return false
- }
- return true
-}
-
-func sequenceLen(v any) int {
- if list, ok := v.([]any); ok {
- return len(list)
- }
- if s, ok := v.(string); ok {
- return len([]rune(s))
- }
- return 0
-}
-
-// dedupSequence: semantics for {a, b, c} literals: repeated
-// members collapse to one. The key is a stable, panic-free rendering so
-// unhashable elements (nested lists) never crash the evaluator; numeric
-// int64/float64 compare by value, so {1, 1.0} dedups to one element.
-func dedupSequence(in []any) []any {
- seen := make(map[string]bool, len(in))
- out := make([]any, 0, len(in))
- for _, v := range in {
- key := fmt.Sprintf("%#v", v)
- if !seen[key] {
- seen[key] = true
- out = append(out, v)
- }
- }
- return out
-}
-
-func concatSequences(a, b any) (any, error) {
- la, aOk := a.([]any)
- lb, bOk := b.([]any)
- if aOk && bOk {
- return append(append([]any{}, la...), lb...), nil
- }
- sa, aStr := a.(string)
- sb, bStr := b.(string)
- if aStr && bStr {
- return sa + sb, nil
- }
- return nil, fmt.Errorf("unsupported operand types for +")
-}
-
-func compareValues(op string, a, b any) (bool, error) {
- if af, err := asNumber(a); err == nil {
- if bf, err := asNumber(b); err == nil {
- switch op {
- case "==":
- return af == bf, nil
- case "!=":
- return af != bf, nil
- case "<":
- return af < bf, nil
- case "<=":
- return af <= bf, nil
- case ">":
- return af > bf, nil
- case ">=":
- return af >= bf, nil
- }
- }
- }
- as, aStr := a.(string)
- bs, bStr := b.(string)
- if aStr && bStr {
- switch op {
- case "==":
- return as == bs, nil
- case "!=":
- return as != bs, nil
- case "<":
- return as < bs, nil
- case "<=":
- return as <= bs, nil
- case ">":
- return as > bs, nil
- case ">=":
- return as >= bs, nil
- }
- }
- return false, fmt.Errorf("unsupported comparison")
-}
-
-func extrema(op string, values []any) (any, error) {
- // min()/max() order elements with the same comparison semantics as the rest of the
- // evaluator (compareValues), so strings compare lexicographically and the result keeps
- // its original type. The final numeric gate then rejects a non-numeric result, matching
- // the numeric-type check.
- best := values[0]
- for _, v := range values[1:] {
- // Keep v when it is the smaller (min) or larger (max) element.
- smaller, err := compareValues("<", v, best)
- if err != nil {
- return nil, err
- }
- if (op == "min" && smaller) || (op == "max" && !smaller) {
- best = v
- }
- }
- return best, nil
-}
-
-func sortValues(values []any) ([]any, error) {
- // sorted() orders elements with the evaluator's comparison semantics (compareValues), so
- // strings sort lexicographically. The result is a list, which the final numeric gate
- // rejects — the numeric-type check refuses a list result.
- out := append([]any{}, values...)
- for i := 1; i < len(out); i++ {
- for j := i; j > 0; {
- lt, err := compareValues("<", out[j-1], out[j])
- if err != nil {
- return nil, err
- }
- if !lt {
- break
- }
- out[j-1], out[j] = out[j], out[j-1]
- j--
- }
- }
- return out, nil
-}
-
-// letters: count alphabetic characters across the given
-// names. Spaces, hyphens, apostrophes, digits and punctuation do NOT count;
-// letters carrying diacritics DO ("José" is 4).
-//
-// "How many letters are in these names" is a real question, and every plain way
-// to answer it needs machinery this evaluator refuses — an attribute call or a
-// comprehension — so it is a function instead.
-func letters(items []any) int {
- total := 0
- for _, item := range items {
- s, ok := item.(string)
- if !ok {
- panic(fmt.Sprintf("letters() takes names, not %T", item))
- }
- for _, ch := range s {
- if unicode.IsLetter(ch) {
- total++
- }
- }
- }
- return total
-}
-
-// digitSum: add up the decimal digits inside the
-// given values. Every digit is added SEPARATELY: digit_sum("L7 7BN") is 14 and
-// digit_sum("2020") is 4. Only ASCII digits count.
-func digitSum(items []any) int {
- total := 0
- for _, item := range items {
- switch n := item.(type) {
- case bool:
- panic("digit_sum() takes text or whole numbers, not bool")
- case string:
- for i := 0; i < len(n); i++ {
- if n[i] >= '0' && n[i] <= '9' {
- total += int(n[i] - '0')
- }
- }
- case int64:
- for _, ch := range strconv.FormatInt(n, 10) {
- total += int(ch - '0')
- }
- default:
- panic(fmt.Sprintf("digit_sum() takes text or whole numbers, not %T", item))
- }
- }
- return total
-}
-
-// dateDiff: days between two ISO dates (inclusive of
-// the earlier, exclusive of the later — a calendar span).
-func dateDiff(a, b any) (int64, error) {
- as, okA := a.(string)
- bs, okB := b.(string)
- if !okA || !okB {
- return 0, fmt.Errorf("date_diff() takes ISO date strings")
- }
- da, err := parseISODate(as)
- if err != nil {
- return 0, err
- }
- db, err := parseISODate(bs)
- if err != nil {
- return 0, err
- }
- // Both dates are midnight UTC, so a second-based difference is exact and
- // avoids time.Duration's ~292-year int64 nanosecond ceiling, which SATURATES
- // (every longer span came back as 106751 days).
- seconds := db.Unix() - da.Unix()
- if seconds < 0 {
- seconds = -seconds
- }
- return seconds / 86400, nil
-}
-
-func parseISODate(s string) (time.Time, error) {
- parts := strings.Split(strings.TrimSpace(s), "-")
- if len(parts) != 3 {
- return time.Time{}, fmt.Errorf("not an ISO date: %q", s)
- }
- y, err1 := strconv.Atoi(parts[0])
- mo, err2 := strconv.Atoi(parts[1])
- d, err3 := strconv.Atoi(parts[2])
- if err1 != nil || err2 != nil || err3 != nil {
- return time.Time{}, fmt.Errorf("not an ISO date: %q", s)
- }
- if mo < 1 || mo > 12 {
- return time.Time{}, fmt.Errorf("month %d out of range in %q", mo, s)
- }
- t := time.Date(y, time.Month(mo), d, 0, 0, 0, 0, time.UTC)
- // time.Date silently normalizes out-of-range days (Feb 30 -> Mar 1, month 13
- // -> next Jan). A date that does not survive the round trip unchanged is rejected.
- if t.Year() != y || int(t.Month()) != mo || t.Day() != d {
- return time.Time{}, fmt.Errorf("day %d out of range for %q", d, s)
- }
- return t, nil
-}
-
-// ComputeFromFacts decides whether the question asks for a derivable number and, if so,
-// computes it. The evaluator side of this lives above (Compute / the parser); the two
-// together form the compute tool.
-
-// ComputeSystem is the compute prompt. It is inlined (rather than loaded from rag/prompts)
-// because the runtime has no template env and the text carries no variables.
-//
-// IMPORTANT: the prompt asks for an arithmetic expression, and the parser in
-// arithmetic.go implements exactly the expression subset the prompt promises
-// (`**`, `x if y else z`, list literals, the listed functions). Changing one without the
-// other will silently refuse every model-written expression.
-const ComputeSystem = `You are given the ORIGINAL question and every fact discovered so far. Decide whether that question asks for a NUMBER that NO fact states outright but that FOLLOWS ARITHMETICALLY from figures the facts DO state — a sum, a difference, a count, an average, a percentage, a unit conversion, an elapsed span.
-
-If it does, compute it by writing ONE arithmetic expression with every figure substituted as a literal. The expression is evaluated on its own: no variables, no assignments, no imports, no attributes, no subscripts. The only functions available are abs, round, min, max, sum, len, int, float, sorted, letters, digit_sum and date_diff.
- combined population of three -> 12345 + 6789 + 101112
- how many of the listed items -> len(["Alpha", "Beta", "Gamma"])
- what percentage one figure is -> 100 * 4523 / 18092
- years between two dates -> 1998 - 1954
- days between two dates -> date_diff("1941-07-28", "1959-07-17")
- letters in a set of names -> letters("Ada Lovelace", "Alan Turing")
- digits of a postcode added up -> digit_sum("L7 7BN")
-
-ADDING UP THE DIGITS of a postcode, a house number, a serial number, a year or an address: use digit_sum(...), and never read the digits out by hand. It adds each digit separately, which is what such a question means — digit_sum("L7 7BN") is 7+7 = 14, digit_sum("2020") is 2+0+2+0 = 4. Pass the identifier EXACTLY as the facts write it, letters and spaces included; they are ignored. It is the WRONG tool for whole numbers the facts state separately — two populations, two prices, two years are added as plain literals (12345 + 6789), not fed to digit_sum.
-
-COUNTING LETTERS: use letters(...), NEVER len(...) on a name. len counts spaces, hyphens and apostrophes as though they were letters, so it is wrong by exactly the amount nobody notices (len("Ada Lovelace") is 12; the name has 11 letters). letters(...) takes any number of names, or one list of them, and counts alphabetic characters only. Spell each name EXACTLY as the facts give it, including any middle name or accent — and if the facts do not show a name in full, that figure is missing, so return "needed": false rather than counting a partial name.
-
-DAYS BETWEEN TWO DATES: when the question asks "how many days after X did Y happen" / "how many days between two dates", use date_diff("YYYY-MM-DD", "YYYY-MM-DD") with the two dates EXACTLY as the facts write them. Do NOT subtract the years (1959 - 1941) — that is the wrong quantity for a days question (18 is years, not days). If either date is not a full YYYY-MM-DD in the facts, the figure is missing, so return "needed": false rather than approximating.
-
-AGE (an age, or an age difference, at some event): the facts almost always give a birth YEAR and an event YEAR; the age is event_year - birth_year (or birth_year - event_year, taken as the positive difference). You do NOT need the birth month or day — the year is enough. If the facts give FULL dates (YYYY-MM-DD), prefer date_diff(...) which handles the day correctly; otherwise subtract the years. Example: "elected in 2010, born 1971" -> 2010 - 1971. If the event year is BEFORE the birth year, the difference is birth_year - event_year (use abs(...)). Never refuse because the birthday is not a full date — the YEAR is sufficient.
-
-PERCENTAGE (what percent / what share / what proportion / what fraction): 100 * part / whole, where part and whole are the exact figures from the facts. Example:
- "2.7 million Tamazight speakers out of 556 million total" -> 100 * 2.7 / 556
-Do not round to an integer unless the question asks for that; keep the source figures exact.
-
-UNIT CONVERSION (a speed, rate, or span in mixed units): convert inside the expression. A speed in km/h becomes m/s by dividing by 3.6. Example for a difference in m/s between a fish and a swimmer:
- fish_kmh / 3.6 - 50 / swimmer_seconds -> e.g. 132 / 3.6 - 50 / 21.07
-Use the EXACT figures the facts state (do not round 21.07 to 21); if the facts give the speed already in m/s, use it directly without dividing.
-
-MULTIPLICATION (a rate times a count, e.g. dollars per day times days): multiply the RATE by the COUNT exactly as the facts state them. Read the rate's NUMBER from the facts. Example: "a suggested donation of $25 per day, kept up for 49 days" -> 25 * 49. If the facts state the rate as 1 but the question calls it "a suggested donation", still use the exact figure the facts give — never substitute a made-up base amount.
-
-Prefer computing over giving up: when the question asks for a derivable number and the facts provide the figures (even if in different units or spread across several facts), WRITE the expression and compute it. In particular, questions asking for an AGE DIFFERENCE, a PERCENTAGE, a SPEED DIFFERENCE (with unit conversion), or a MULTIPLICATION (a rate times a count) are exactly what this tool is for.
-
-Return "needed": false, with an empty expression, ONLY when:
-- the ORIGINAL question does not ask for a number;
-- a fact already states that number outright — a value you would only be restating is not a calculation;
-- a figure the calculation needs is genuinely absent from the facts, or a list the count depends on is not shown to be complete. NEVER invent, estimate, recall or infer a figure. When input is missing, say so and return "needed": false — a wrong number is worse than none — but first check that the figure really is absent (e.g. the age's birth YEAR is enough; you do not need the month).
-
-"label" names what the number IS, as a short noun phrase ("combined population of the three counties"), so a later step can use the result without re-deriving it.
-"uses" lists the INDEX NUMBERS of the facts whose figures you substituted.
-Output ONLY JSON, no prose, no code fences:
-{"needed": true/false, "expression": "<one arithmetic expression, or empty>", "label": "<short noun phrase>", "uses": [<index number>, ...]}`
-
-// ComputedFact: return dict.
-type ComputedFact struct {
- Needed bool
- Label string
- Value string
- Expression string
- Uses []int
-}
-
-// ComputeFromFacts asks the model whether `question` asks for a derivable
-// number and, if so, writes + safely evaluates the expression over `facts`.
-//
-// Returns nil when no derivation is needed or possible — including when the
-// model's expression is refused by the whitelist. A refusal is logged, never
-// propagated: the caller simply carries on without the computed evidence.
-func ComputeFromFacts(ctx context.Context, model SessionModel, question string, facts []string, fitBudget int) *ComputedFact {
- if model == nil || strings.TrimSpace(question) == "" || len(facts) == 0 {
- return nil
- }
- ctx, done := Phase(ctx, PhaseCompute)
- defer done()
-
- user := fmt.Sprintf("Facts discovered so far:\n%s\n\nOriginal question:\n%s\n\nOutput JSON:",
- renderFacts(facts), question)
-
- // Budget = fit_budget or the model's context length, then the prompt is fitted to it.
- // The chat seam exposes the context length via ContextLengthModel; when neither an
- // explicit fitBudget nor a model context window is available the 8192 default
- // (chat.EffectiveContextLength) applies, i.e. when the model config omits it.
- budget := fitBudget
- if budget <= 0 {
- if cl, ok := model.(ContextLengthModel); ok {
- budget = cl.ContextLength()
- }
- if budget <= 0 {
- budget = chat.EffectiveContextLength(0)
- }
- }
- fitted, fitErr := chat.FitMessages(ComputeSystem, []schema.Message{
- *schema.UserMessage(user),
- }, budget)
- if fitErr != "" {
- _LOG.Printf("[Compute] prompt fitting failed: %s", fitErr)
- return nil
- }
- // FitMessages may prepend/trim a system message; re-extract it so the model
- // call is exactly [system, user...].
- systemPrompt := ComputeSystem
- history := fitted
- if len(fitted) > 0 && fitted[0].Role == schema.System {
- systemPrompt = fitted[0].Content
- history = fitted[1:]
- }
- msgs := make([]schema.Message, 0, 1+len(history))
- msgs = append(msgs, *schema.SystemMessage(systemPrompt))
- msgs = append(msgs, history...)
- // The temperature is pinned to 0.0 for this node — a mechanical rewrite, so it must be
- // deterministic. When the model supports per-call temperature we
- // pass 0.0 exactly; otherwise we fall back to the plain Complete (it then runs
- // at the model's default), the same documented seam as ExtractWeightedKeywords
- // (which uses 0.1).
- var reply *ModelReply
- var err error
- if tm, ok := model.(TemperatureModel); ok {
- reply, err = tm.CompleteWithTemperature(ctx, msgs, nil, 0.0)
- } else {
- reply, err = model.Complete(ctx, msgs, nil)
- }
- if err != nil {
- _LOG.Printf("[Compute] LLM call failed: %v", err)
- return nil
- }
- data, _ := ExtractJSON(StripThinkAndFences(reply.Content)).(map[string]any)
- if data == nil {
- _LOG.Printf("[Compute] could not parse LLM JSON")
- return nil
- }
- // A truthy non-bool (1, "true", a non-empty list) counts as needed — i.e. builtin bool()
- // semantics. A strict bool assertion would reject those and wrongly report "nothing
- // derivable".
- if !truthy(data["needed"]) {
- return nil
- }
- expression := ""
- if v, ok := data["expression"].(string); ok {
- expression = strings.TrimSpace(v)
- }
- if expression == "" {
- return nil
- }
- label := ""
- if v, ok := data["label"].(string); ok {
- label = strings.TrimSpace(v)
- }
- if label == "" {
- label = "Value calculated from the facts found"
- }
- uses := make([]int, 0, 4)
- if raw, ok := data["uses"].([]any); ok {
- for _, n := range raw {
- if v, ok := common.ToIntNum(n); ok {
- uses = append(uses, v)
- }
- }
- }
-
- value, problem := Compute(expression)
- if problem != "" {
- _LOG.Printf("[Compute] refused %q — %s", trunc(expression, 120), problem)
- return nil
- }
- return &ComputedFact{
- Needed: true,
- Label: label,
- Value: value,
- Expression: expression,
- Uses: uses,
- }
-}
-
-// renderFacts: one fact per line, prefixed with its
-// index — the index is what "uses" refers back to.
-func renderFacts(facts []string) string {
- lines := make([]string, 0, len(facts))
- for i, f := range facts {
- lines = append(lines, fmt.Sprintf("[%d] %s", i, f))
- }
- return strings.Join(lines, "\n")
-}
diff --git a/internal/rag/agentic-rag/runtime/arithmetic_test.go b/internal/rag/agentic-rag/runtime/arithmetic_test.go
deleted file mode 100644
index cb9facd080..0000000000
--- a/internal/rag/agentic-rag/runtime/arithmetic_test.go
+++ /dev/null
@@ -1,571 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "strings"
- "testing"
-
- "github.com/cloudwego/eino/schema"
-)
-
-// The happy path: what the prompt promises the model
-
-func TestComputeBasicArithmetic(t *testing.T) {
- cases := []struct{ expr, want string }{
- {"12345 + 6789 + 101112", "120246"}, // combined population
- {"1998 - 1954", "44"}, // years between
- {"100 * 4523 / 18092", "25"}, // percentage
- {"25 * 49", "1225"}, // rate x count
- {"(132 / 3.6) * 1", "36.666667"}, // unit conversion (float rendering)
- {"2 ** 10", "1024"}, // exponent
- {"7 // 2", "3"}, // floor division
- {"7 % 3", "1"}, // modulo
- {"abs(1954 - 1998)", "44"}, // abs
- {"round(3.14159, 2)", "3.14"}, // round with digits
- {"round(2.5)", "2"}, // banker's rounding (half-to-even)
- {"round(3.5)", "4"}, // half-to-even, away from even
- {"round(0.5)", "0"}, // half-to-even
- {"-3 % 2", "1"}, // % takes the sign of the divisor
- {"3 % -2", "-1"}, // sign of the divisor
- {"min(3, 1, 2)", "1"},
- {"max(3, 1, 2)", "3"},
- {"sum([1, 2, 3, 4])", "10"},
- {"len([\"Alpha\", \"Beta\", \"Gamma\"])", "3"},
- {"0.1 + 0.2", "0.3"}, // float noise suppressed
- {"(2 + 3) * 4", "20"}, // grouping
- {"2 ** 3 ** 2", "512"}, // right-associative
- {"-2 ** 2", "-4"}, // power binds tighter than unary minus
- {"1 if 2 > 1 else 0", "1"}, // ternary
- }
- for _, c := range cases {
- got, err := Compute(c.expr)
- if err != "" {
- t.Errorf("Compute(%q) refused: %s", c.expr, err)
- continue
- }
- if got != c.want {
- t.Errorf("Compute(%q) = %q, want %q", c.expr, got, c.want)
- }
- }
-}
-
-func TestComputeHelperFunctions(t *testing.T) {
- cases := []struct{ expr, want string }{
- {`letters("Ada Lovelace")`, "11"}, // 12 with len — the whole point
- {`letters("Ada Lovelace", "Alan Turing")`, "21"}, // multiple names
- {`letters(["José"])`, "4"}, // diacritics count
- {`digit_sum("L7 7BN")`, "14"}, // 7 + 7
- {`digit_sum("2020")`, "4"}, // each digit separately
- {`date_diff("1941-07-28", "1959-07-17")`, "6563"}, // calendar span
- {`date_diff("1959-07-17", "1941-07-28")`, "6563"}, // order-independent
- {`sorted([3, 1, 2])[0]`, ""}, // subscripts refused (see below)
- }
- for _, c := range cases {
- got, err := Compute(c.expr)
- if c.want == "" {
- continue // asserted separately in the refusal tests
- }
- if err != "" {
- t.Errorf("Compute(%q) refused: %s", c.expr, err)
- continue
- }
- if got != c.want {
- t.Errorf("Compute(%q) = %q, want %q", c.expr, got, c.want)
- }
- }
-}
-
-// Security: the AST whitelist
-
-func TestComputeRefusesUnsafeExpressions(t *testing.T) {
- cases := []struct{ expr, wantSubstr string }{
- // Attribute access — the classic sandbox escape.
- {`"".__class__`, ""},
- {`(1).__class__`, ""},
- // Subscripts / indexing.
- {"[1,2,3][0]", ""},
- // Comprehensions and lambdas.
- {"[x for x in [1,2]]", ""},
- {"(lambda: 1)()", ""},
- // Names outside the whitelist.
- {"__import__(\"os\").system(\"ls\")", ""},
- {"open(\"f\").read()", ""},
- {"eval(\"1\")", ""},
- {"exec(\"x\")", ""},
- // Assignment / statements.
- {"x = 1", ""},
- {"import os", ""},
- {"1;2", ""},
- // String amplification via multiplication.
- {`"a" * 100000000`, "multiplication is only allowed on numbers"},
- {"[1] * 100000000", "multiplication is only allowed on numbers"},
- // Exponentiation abuse.
- {"2 ** 1000", "exponent is too large"},
- {`"a" ** 2`, "exponentiation is only allowed on numbers"},
- // len() on a string literal is ambiguous by design.
- {`len("Ada Lovelace")`, "len() on a string literal is ambiguous"},
- // Keyword arguments are refused.
- {"round(3.14159, ndigits=2)", "does not parse"},
- // None is not a value here.
- {"None", "does not parse"},
- }
- for _, c := range cases {
- got, err := Compute(c.expr)
- if err == "" {
- t.Errorf("Compute(%q) = %q, want a refusal", c.expr, got)
- continue
- }
- // The exact wording differs by rejection path (a parse failure vs. a
- // whitelist refusal), but every case here MUST be refused. Assert the
- // specific reason only where it is the point of the case.
- if c.wantSubstr == "" {
- continue
- }
- if !strings.Contains(err, c.wantSubstr) {
- t.Errorf("Compute(%q) error = %q, want it to contain %q", c.expr, err, c.wantSubstr)
- }
- }
-}
-
-// TestComputeRecoversFromHelperTypeErrors guards the panic→error conversion:
-// letters()/digit_sum() reject bad argument types by panicking, and Compute must turn that
-// into a normal refusal rather than crashing the request.
-func TestComputeRecoversFromHelperTypeErrors(t *testing.T) {
- for _, expr := range []string{`letters(123)`, `digit_sum(1.5)`} {
- got, err := Compute(expr)
- if err == "" {
- t.Errorf("Compute(%q) = %q, want a refusal", expr, got)
- } else if !strings.Contains(err, "failed to evaluate") {
- t.Errorf("Compute(%q) err = %q, want an evaluation failure", expr, err)
- }
- }
-}
-
-// no unsafe construct may ever produce a rendered value, whatever the wording.
-func TestComputeRejectionsAreNeverValues(t *testing.T) {
- unsafe := []string{
- `"".__class__`, `(1).__class__`, `[1,2,3][0]`,
- `[x for x in [1,2]]`, `(lambda: 1)()`,
- `__import__("os").system("ls")`, `open("f").read()`,
- `eval("1")`, `exec("x")`, `globals()`, `getattr(1, "x")`,
- `x = 1`, `import os`, `None`, `1;2`,
- }
- for _, expr := range unsafe {
- got, err := Compute(expr)
- if err == "" {
- t.Errorf("SECURITY: Compute(%q) = %q — must be refused", expr, got)
- }
- }
-}
-
-func TestComputeRefusesEmptyAndOversized(t *testing.T) {
- if _, err := Compute(""); err != "empty expression" {
- t.Errorf("empty: err = %q", err)
- }
- if _, err := Compute(" "); err != "empty expression" {
- t.Errorf("blank: err = %q", err)
- }
- long := strings.Repeat("1+", computeMaxChars)
- if _, err := Compute(long); !strings.Contains(err, "longer than") {
- t.Errorf("oversized: err = %q", err)
- }
-}
-
-func TestComputeRefusesNonNumericResult(t *testing.T) {
- // A call whose result is not a number must be refused, not rendered.
- for _, expr := range []string{`sorted([3, 1, 2])`, `min("b", "a")`, `max("b", "a")`} {
- if got, err := Compute(expr); err == "" {
- t.Errorf("Compute(%q) = %q, want a refusal (result is not a number)", expr, got)
- }
- }
-}
-
-// TestComputeBuiltinStringArgs pins the builtin semantics: int()/float() accept numeric
-// strings, and min/max/sorted accept and order strings. The final result gate then rejects
-// non-numeric results (the numeric-type check).
-func TestComputeBuiltinStringArgs(t *testing.T) {
- // int()/float() accept strings -> these produce a number and are ACCEPTED
- // (previously they were rejected — a true divergence).
- accept := []struct{ expr, want string }{
- {`int("12")`, "12"},
- {`int(" 12 ")`, "12"}, // surrounding whitespace is stripped
- {`int(1.5)`, "1"}, // truncate toward zero
- {`float("1.5")`, "1.5"},
- {`float(" 1e3 ")`, "1000"},
- }
- for _, c := range accept {
- if got, err := Compute(c.expr); err != "" || got != c.want {
- t.Errorf("Compute(%q) = %q, err=%q; want %q", c.expr, got, err, c.want)
- }
- }
- // These are hard failures and must be refused.
- for _, expr := range []string{`int("1.5")`, `float("abc")`, `int("0x10")`} {
- if got, err := Compute(expr); err == "" {
- t.Errorf("Compute(%q) = %q, want a refusal", expr, got)
- }
- }
-}
-
-func TestComputeRefusesDivisionByZero(t *testing.T) {
- for _, expr := range []string{"1 / 0", "1 // 0", "1 % 0"} {
- if _, err := Compute(expr); err == "" || !strings.Contains(err, "zero") {
- t.Errorf("Compute(%q) err = %q, want a division-by-zero refusal", expr, err)
- }
- }
-}
-
-func TestComputeComparisonChains(t *testing.T) {
- // A comparison is an N-ary chain: `a < b < c` means (a < b) and (b < c), with each operand
- // evaluated exactly once and short-circuiting on the first false comparison. A
- // left-associative binary rewrite would instead compare the previous comparison's BOOLEAN
- // result against the next operand, which is wrong.
- //
- // At the top level a comparison yields a bool, and compute() rejects non-numeric final
- // results ("result is bool, not a number"). But a bool used numerically inside a call
- // (int/abs/round…) must evaluate with the chained semantics.
- rejected := []string{
- "3 > 2 > 1", "2 < 1 < 3", "1 < 2 < 3 < 4", "1 < 2 > 3",
- "1 <= 1 <= 2", "3 == 3 == 3", "3 == 3 == 4", "5 > 4 > 3 > 2 > 1", "3 > 2",
- }
- for _, expr := range rejected {
- if got, err := Compute(expr); err == "" {
- t.Errorf("Compute(%q) = %q, want rejection (bool is not a number)", expr, got)
- }
- }
- numeric := []struct {
- expr string
- want string
- }{
- {"int(3 > 2 > 1)", "1"},
- {"int(2 < 1 < 3)", "0"},
- {"abs(1 < 2 < 3)", "1"},
- {"int(3 == 3 == 4)", "0"},
- {"round(3 > 2 > 1)", "1"},
- {"int(1 < 2 < 3 and 4 < 5)", "1"},
- }
- for _, c := range numeric {
- if got, err := Compute(c.expr); err != "" || got != c.want {
- t.Errorf("Compute(%q) = %q, err=%q; want %q", c.expr, got, err, c.want)
- }
- }
-}
-
-func TestComputeRejectsTrailingInput(t *testing.T) {
- // Everything after the expression must be consumed, or `1 + 1; rm -rf` style
- // smuggling would parse.
- if _, err := Compute("1 + 1 2"); err == "" {
- t.Error("trailing input must be refused")
- }
- if _, err := Compute("1 + 1)"); err == "" {
- t.Error("unbalanced paren must be refused")
- }
-}
-
-// Formatting
-
-func TestFormatNumber(t *testing.T) {
- cases := []struct {
- in float64
- want string
- }{
- {3.0, "3"},
- {0.1 + 0.2, "0.3"},
- {-2.5, "-2.5"},
- {0, "0"},
- {1e20, "100000000000000000000"},
- {3.14159265358979, "3.141593"},
- }
- for _, c := range cases {
- if got := formatNumber(c.in); got != c.want {
- t.Errorf("formatNumber(%v) = %q, want %q", c.in, got, c.want)
- }
- }
-}
-
-// The helper functions directly
-
-func TestLettersAndDigitSum(t *testing.T) {
- // "José" is 4 letters — diacritics count, spaces/punctuation do not.
- if got := letters([]any{"José"}); got != 4 {
- t.Errorf("letters(José) = %d, want 4", got)
- }
- if got := letters([]any{"Ada Lovelace"}); got != 11 {
- t.Errorf("letters = %d, want 11", got)
- }
- // "Any number of names, or one list of them".
- if got := letters([]any{"ab", "cde"}); got != 5 {
- t.Errorf("letters multi = %d, want 5", got)
- }
- if got := digitSum([]any{"L7 7BN"}); got != 14 {
- t.Errorf("digit_sum = %d, want 14", got)
- }
- if got := digitSum([]any{"2020"}); got != 4 {
- t.Errorf("digit_sum(2020) = %d, want 4", got)
- }
- // Whole numbers are also accepted.
- if got := digitSum([]any{int64(2020)}); got != 4 {
- t.Errorf("digit_sum(2020 int) = %d, want 4", got)
- }
-}
-
-func TestParseISODate(t *testing.T) {
- if _, err := parseISODate("1941-07-28"); err != nil {
- t.Errorf("valid date rejected: %v", err)
- }
- for _, bad := range []string{
- "1941-07", "not-a-date", "1941-07-28-01", // malformed
- "2024-13-01", "2024-00-15", // month out of range
- "2024-02-30", "2023-02-29", // day out of range for month
- } {
- if _, err := parseISODate(bad); err == nil {
- t.Errorf("parseISODate(%q) accepted, want a refusal", bad)
- }
- }
-}
-
-// TestComputeSetLiteralDedups pins set semantics: {a, b, c} literals collapse duplicates
-// (len({1,1,2}) == 2), and min/max operate on the unique members. Before the dedup fix, Go
-// counted every member.
-func TestComputeSetLiteralDedups(t *testing.T) {
- cases := []struct{ expr, want string }{
- {`len({1,1,2})`, "2"}, // three members, two unique
- {`len({5,5,5,5})`, "1"}, // all duplicate
- {`min({3,3,1})`, "1"}, // dedup before extremum
- {`max({2,2,9})`, "9"},
- }
- for _, c := range cases {
- if got, err := Compute(c.expr); err != "" || got != c.want {
- t.Errorf("Compute(%q) = %q, err=%q; want %q", c.expr, got, err, c.want)
- }
- }
-}
-
-// Tool-outcome mapping
-
-func TestExecutorCalculate(t *testing.T) {
- // A model that writes a derivable expression.
- mdl := &fakeModel{replies: []*ModelReply{{
- Content: `{"needed": true, "expression": "1998 - 1954", "label": "years between", "uses": [0]}`,
- }}}
- ex := &searchExecutor{deps: SearchDeps{Model: mdl}}
- oc, _ := ex.Execute(context.Background(), "calculate", map[string]any{
- "question": "How many years between them?",
- "facts": []any{"born 1954", "died 1998"},
- })
- if oc.Status != StatusOK {
- t.Fatalf("status = %s (%v), want ok", oc.Status, oc.Metrics)
- }
- entry := oc.Payload[0].(map[string]any)
- // the success payload is exactly {"kind","expression","result"};
- // label/uses are deliberately NOT echoed.
- if entry["result"] != "44" {
- t.Errorf("result = %v, want 44", entry["result"])
- }
- if entry["expression"] != "1998 - 1954" {
- t.Errorf("expression = %v", entry["expression"])
- }
- if _, ok := entry["label"]; ok {
- t.Errorf("payload must not echo label: %v", entry)
- }
-
- // A model that says no derivation is needed → POOR (not an error), so the model then
- // answers from the facts it already has (nothing derivable → status=POOR/no_doc).
- mdl = &fakeModel{replies: []*ModelReply{{Content: `{"needed": false}`}}}
- ex = &searchExecutor{deps: SearchDeps{Model: mdl}}
- oc, _ = ex.Execute(context.Background(), "calculate", map[string]any{
- "question": "q", "facts": []any{"a"},
- })
- if oc.Status != StatusPoor {
- t.Errorf("not needed: status = %s, want poor", oc.Status)
- }
-
- // An unsafe expression → POOR (refused, logged), never a panic or a breach.
- mdl = &fakeModel{replies: []*ModelReply{{
- Content: `{"needed": true, "expression": "__import__(\"os\").system(\"ls\")", "label": "x"}`,
- }}}
- ex = &searchExecutor{deps: SearchDeps{Model: mdl}}
- if oc, _ = ex.Execute(context.Background(), "calculate", map[string]any{
- "question": "q", "facts": []any{"a"},
- }); oc.Status != StatusPoor {
- t.Errorf("unsafe expr: status = %s, want poor (refused)", oc.Status)
- }
-
- // No facts → POOR/no_doc, NOT bad_args: nothing is validated and the
- // compute_from_facts `not facts` guard returns None.
- ex = &searchExecutor{deps: SearchDeps{Model: &fakeModel{}}}
- if oc, _ := ex.Execute(context.Background(), "calculate", map[string]any{"question": "q"}); oc.Status != StatusPoor || oc.Reason != ReasonNoDoc {
- t.Errorf("no facts: got (%s,%s), want (poor,no_doc)", oc.Status, oc.Reason)
- }
-}
-
-// TestComputeFromFactsAcceptsNonBoolNeeded pins the `not data.get("needed")` guard:
-// builtin bool() treats a NON-EMPTY STRING as truthy (even the literal "false"), so a model
-// that emits needed as a string must still compute. A strict bool assertion would silently
-// report "nothing derivable".
-func TestComputeFromFactsAcceptsNonBoolNeeded(t *testing.T) {
- mdl := &fakeModel{replies: []*ModelReply{{
- Content: `{"needed": "true", "expression": "2 + 2", "label": "sum"}`,
- }}}
- got := ComputeFromFacts(context.Background(), mdl, "q", []string{"two things"}, 0)
- if got == nil {
- t.Fatal("ComputeFromFacts = nil, want a result for a truthy non-bool needed")
- }
- if got.Value != "4" {
- t.Errorf("value = %q, want 4", got.Value)
- }
-}
-
-func TestToolStringListAcceptsAllShapes(t *testing.T) {
- // Models emit []any, []string, or a bare string.
- if got := toolStringList(map[string]any{"facts": []any{"a", "b"}}, "facts"); len(got) != 2 {
- t.Errorf("[]any = %v", got)
- }
- if got := toolStringList(map[string]any{"facts": []string{"a"}}, "facts"); len(got) != 1 {
- t.Errorf("[]string = %v", got)
- }
- if got := toolStringList(map[string]any{"facts": "a"}, "facts"); len(got) != 1 {
- t.Errorf("string = %v", got)
- }
- if got := toolStringList(map[string]any{}, "facts"); got != nil {
- t.Errorf("absent = %v, want nil", got)
- }
-}
-
-// Tuple literals: the sequence form used by sum((1,2,3)) / len((1,2,3)) / min / max /
-// letters.
-
-func TestComputeTupleLiterals(t *testing.T) {
- cases := []struct{ expr, want string }{
- {"sum((1, 2, 3))", "6"},
- {"len((1, 2, 3))", "3"},
- {"min((3, 1, 2))", "1"},
- {"max((3, 1, 2))", "3"},
- {"100 + sum((10, 20))", "130"},
- {`letters(("Ada", "Lovelace"))`, "11"},
- }
- for _, c := range cases {
- got, err := Compute(c.expr)
- if err != "" {
- t.Errorf("Compute(%q) refused: %s", c.expr, err)
- continue
- }
- if got != c.want {
- t.Errorf("Compute(%q) = %q, want %q", c.expr, got, c.want)
- }
- }
- // A bare top-level tuple is not a number and must be refused (a tuple result fails the
- // numeric-type check).
- if _, err := Compute("(1, 2, 3)"); err == "" {
- t.Error("Compute(\"(1, 2, 3)\") returned a value, want a refusal")
- }
- // Parenthesised expressions (no comma) stay scalar, not a one-tuple.
- if got, err := Compute("(1 + 2) * 4"); err != "" || got != "12" {
- t.Errorf("Compute(\"(1 + 2) * 4\") = %q, %q; want \"12\"", got, err)
- }
-}
-
-// tempRecordingModel records the temperature handed to CompleteWithTemperature
-// so the compute_from_facts 0.0 requirement can be asserted.
-type tempRecordingModel struct {
- replies []*ModelReply
- temperature *float64
- contextLength int
- calls int
-}
-
-// ContextLength implements ContextLengthModel; 0 reports "unknown".
-func (m *tempRecordingModel) ContextLength() int {
- return m.contextLength
-}
-
-func (m *tempRecordingModel) Complete(_ context.Context, _ []schema.Message, _ []ToolSpec) (*ModelReply, error) {
- if m.calls >= len(m.replies) {
- return &ModelReply{Content: "{}"}, nil
- }
- r := m.replies[m.calls]
- m.calls++
- return r, nil
-}
-
-func (m *tempRecordingModel) CompleteWithTemperature(_ context.Context, _ []schema.Message, _ []ToolSpec, temp float64) (*ModelReply, error) {
- t := temp
- m.temperature = &t
- if m.calls >= len(m.replies) {
- return &ModelReply{Content: "{}"}, nil
- }
- r := m.replies[m.calls]
- m.calls++
- return r, nil
-}
-
-func TestComputeFromFactsUsesTemperatureZero(t *testing.T) {
- mdl := &tempRecordingModel{replies: []*ModelReply{{
- Content: `{"needed": true, "expression": "1998 - 1954", "label": "years", "uses": [0]}`,
- }}}
- cf := ComputeFromFacts(context.Background(), mdl, "How many years?", []string{"born 1954", "died 1998"}, 0)
- if cf == nil {
- t.Fatal("expected a ComputedFact")
- }
- if cf.Value != "44" {
- t.Errorf("value = %q, want 44", cf.Value)
- }
- if mdl.temperature == nil {
- t.Fatal("CompleteWithTemperature was not called")
- }
- if *mdl.temperature != 0.0 {
- t.Errorf("temperature = %v, want 0.0", *mdl.temperature)
- }
-}
-
-func TestComputeFromFactsFitsToContextBudget(t *testing.T) {
- mdl := &tempRecordingModel{contextLength: 0, replies: []*ModelReply{{
- Content: `{"needed": true, "expression": "1998 - 1954", "label": "years", "uses": [0]}`,
- }}}
- cf := ComputeFromFacts(context.Background(), mdl, "How many years?", []string{"born 1954", "died 1998"}, 0)
- if cf == nil {
- t.Fatal("expected a ComputedFact")
- }
- if cf.Value != "44" {
- t.Errorf("value = %q, want 44", cf.Value)
- }
-}
-
-// TestDateDiffCountsCalendarDays pins that date_diff counts calendar days rather
-// than a time.Duration: the latter saturates at its ~292-year int64 nanosecond
-// ceiling, so every longer span came back as 106751 days.
-func TestDateDiffCountsCalendarDays(t *testing.T) {
- cases := []struct {
- a, b string
- want int64
- }{
- {"1941-07-28", "1959-07-17", 6563}, // the prompt's own example
- {"1607-05-14", "2020-01-01", 150712}, // 412 years: past the Duration ceiling
- {"2020-01-01", "1607-05-14", 150712}, // order-independent (abs)
- {"1607-05-14", "1607-05-14", 0},
- }
- for _, tc := range cases {
- got, err := dateDiff(tc.a, tc.b)
- if err != nil {
- t.Fatalf("dateDiff(%q, %q): %v", tc.a, tc.b, err)
- }
- if got != tc.want {
- t.Errorf("dateDiff(%q, %q) = %d, want %d", tc.a, tc.b, got, tc.want)
- }
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/chunk_utils.go b/internal/rag/agentic-rag/runtime/chunk_utils.go
deleted file mode 100644
index d12072514d..0000000000
--- a/internal/rag/agentic-rag/runtime/chunk_utils.go
+++ /dev/null
@@ -1,216 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "fmt"
- "strings"
- "unicode"
-)
-
-// Field accessors for chunk maps.
-//
-// Maintenance-helper data — doc ids, dataset ids, titles — has historically been stored under
-// different field names depending on the backend and indexer, so every read goes through a
-// helper that tolerates all known aliases.
-//
-// NOTE: tools/ keeps its own accessors rather than importing these. That duplication is what
-// keeps the dependency graph acyclic (tools must not import the root package the root package
-// imports tools from).
-
-// ChunkAttr returns the first non-empty value among keys.
-// Truthiness is "not nil and not empty".
-func ChunkAttr(c map[string]any, keys ...string) string {
- for _, k := range keys {
- if v, ok := c[k]; ok && v != nil {
- if s := fmt.Sprint(v); s != "" {
- return s
- }
- }
- }
- return ""
-}
-
-// ChunkTextOf: The root package already provides
-// chunkText with identical semantics; this exported form exists so callers
-// outside the package share one implementation.
-func ChunkTextOf(c map[string]any) string { return chunkText(c) }
-
-// DocIDOf: doc_id / docid / document_id.
-func DocIDOf(c map[string]any) string {
- return ChunkAttr(c, "doc_id", "docid", "document_id")
-}
-
-// DatasetIDOf: dataset_id / kb_id / knowledgebase_id.
-func DatasetIDOf(c map[string]any) string {
- return ChunkAttr(c, "dataset_id", "kb_id", "knowledgebase_id")
-}
-
-// DocTitleOf: exactly: docnm_kwd / doc_title / title /
-// document_name (the same four keys, in the same order). Go chunk retrieval
-// carries the title under docnm_kwd, so no extra alias is needed.
-func DocTitleOf(c map[string]any) string {
- return ChunkAttr(c, "docnm_kwd", "doc_title", "title", "document_name")
-}
-
-// ChunkIDOf: chunk_id / id.
-func ChunkIDOf(c map[string]any) string { return ChunkAttr(c, "chunk_id", "id") }
-
-// Snippet: trim both ends, cut to limit, right-trim ALL
-// trailing whitespace (not just spaces), then add an ellipsis marker when the
-// value was actually truncated. The right-trim strips ANY Unicode whitespace (space, tab,
-// newline, ...), so a cut that ends mid-run of whitespace collapses to the same trailing slice
-// before "...".
-//
-// Indexing is by Unicode code point, so the limit and the cut are character-based. Go's
-// len/[:] are byte-based and would split a multibyte (e.g. CJK) rune and emit invalid UTF-8,
-// hence the []rune conversion.
-func Snippet(s string, limit int) string {
- t := strings.TrimSpace(s)
- r := []rune(t)
- if len(r) <= limit {
- return t
- }
- return strings.TrimRightFunc(string(r[:limit]), unicode.IsSpace) + "..."
-}
-
-// IsTableChunk: / _is_table_text: a corpus-neutral
-// table detector — HTML table markup, or >=3 pipe rows. Exported so the
-// orchestrator and the bridge share one implementation.
-func IsTableChunk(c map[string]any) bool {
- return isTableText(ChunkTextOf(c))
-}
-
-// isTableText: table detection from raw text.
-func isTableText(text string) bool {
- t := strings.ToLower(text)
- if strings.Contains(t, "<table") || strings.Contains(t, "<tr") {
- return true
- }
- pipeRows := 0
- for _, line := range strings.Split(text, "\n") {
- if strings.Count(line, "|") >= 2 {
- pipeRows++
- }
- }
- return pipeRows >= 3
-}
-
-// XMLEscape: the four XML entities (&, <, >, ").
-// The apostrophe is intentionally NOT escaped — values are only ever embedded inside
-// double-quoted XML/Markdown attributes, where a literal ' is valid, so escaping it to &apos;
-// would be wrong.
-func XMLEscape(s string) string {
- r := strings.NewReplacer(
- "&", "&amp;",
- "<", "&lt;",
- ">", "&gt;",
- `"`, "&quot;",
- )
- return r.Replace(s)
-}
-
-// ChunkDocIDs lists the distinct documents a chunk set belongs to, in first-seen
-// order. It is what a reasoning step reports as "from N documents".
-func ChunkDocIDs(chunks []map[string]any) []string {
- seen := make(map[string]struct{}, len(chunks))
- out := make([]string, 0, len(chunks))
- for _, c := range chunks {
- id := DocIDOf(c)
- if id == "" {
- continue
- }
- if _, dup := seen[id]; dup {
- continue
- }
- seen[id] = struct{}{}
- out = append(out, id)
- }
- return out
-}
-
-// ChunkEvidenceIDs lists a chunk set's chunk ids, deduplicated, capped at limit
-// (limit <= 0 means no cap). These are the anchors a reasoning step points at.
-func ChunkEvidenceIDs(chunks []map[string]any, limit int) []string {
- seen := make(map[string]struct{}, len(chunks))
- out := make([]string, 0, len(chunks))
- for _, c := range chunks {
- id := ChunkIDOf(c)
- if id == "" {
- continue
- }
- if _, dup := seen[id]; dup {
- continue
- }
- seen[id] = struct{}{}
- out = append(out, id)
- if limit > 0 && len(out) >= limit {
- break
- }
- }
- return out
-}
-
-// MergeChunks deduplicates incoming chunks against an existing slice by chunkKey, appending
-// only unseen ones — the merge pattern used by the direct and compiled-expansion paths.
-//
-// Returns the merged slice and the global indices of the newly appended chunks.
-func MergeChunks(existing, incoming []map[string]any) ([]map[string]any, []int) {
- seen := make(map[string]struct{}, len(existing))
- for _, c := range existing {
- seen[chunkKey(c)] = struct{}{}
- }
- out := existing
- var idx []int
- for _, c := range incoming {
- if c == nil {
- continue
- }
- k := chunkKey(c)
- if _, dup := seen[k]; dup {
- continue
- }
- seen[k] = struct{}{}
- idx = append(idx, len(out))
- out = append(out, c)
- }
- return out, idx
-}
-
-// MergeDocAggs deduplicates doc aggregations by doc id, mirroring
-// direct.py:_merge_kbinfos. Returns the merged slice and the number added.
-func MergeDocAggs(existing, incoming []map[string]any) ([]map[string]any, int) {
- seen := make(map[any]struct{}, len(existing))
- for _, d := range existing {
- seen[d["doc_id"]] = struct{}{}
- }
- out := existing
- added := 0
- for _, d := range incoming {
- if d == nil {
- continue
- }
- key := d["doc_id"]
- if _, dup := seen[key]; dup {
- continue
- }
- seen[key] = struct{}{}
- out = append(out, d)
- added++
- }
- return out, added
-}
diff --git a/internal/rag/agentic-rag/runtime/chunk_utils_test.go b/internal/rag/agentic-rag/runtime/chunk_utils_test.go
deleted file mode 100644
index cc4dc62557..0000000000
--- a/internal/rag/agentic-rag/runtime/chunk_utils_test.go
+++ /dev/null
@@ -1,57 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import "testing"
-
-// TestSnippet: trim both ends, cut to limit, right-trim ALL trailing whitespace, then add
-// "...".
-func TestSnippet(t *testing.T) {
- cases := []struct {
- in string
- n int
- want string
- }{
- // Short values pass through trimmed, no ellipsis.
- {" hello ", 10, "hello"},
- // No truncation needed.
- {"hello", 10, "hello"},
- // Truncation that lands on a plain space boundary.
- {"hello world", 5, "hello..."},
- // Cut ends mid-run of plain spaces: right-trim them.
- {"hello world", 6, "hello..."},
- // Cut ends mid-run of a tab: right-trim ALL whitespace, like .rstrip().
- {"hello\t\tworld", 6, "hello..."},
- // Cut ends on a newline run.
- {"hello\n\nworld", 6, "hello..."},
- // Mixed trailing whitespace after the cut.
- {"hello \t\n world", 6, "hello..."},
- // CJK chars are multibyte in UTF-8; the limit is in code points, so no rune is ever
- // split. "中文" (2 code
- // points) fits within n=2 and stays intact.
- {" 中文 ", 2, "中文"},
- // Truncation lands on a code-point boundary: first 3 of 4 CJK chars.
- {"中文中文", 3, "中文中..."},
- // A small byte limit must not split a 3-byte CJK rune.
- {"中文abc", 2, "中文..."},
- }
- for _, c := range cases {
- if got := Snippet(c.in, c.n); got != c.want {
- t.Errorf("Snippet(%q, %d) = %q, want %q", c.in, c.n, got, c.want)
- }
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/config.go b/internal/rag/agentic-rag/runtime/config.go
deleted file mode 100644
index 728bde379d..0000000000
--- a/internal/rag/agentic-rag/runtime/config.go
+++ /dev/null
@@ -1,168 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import "strings"
-
-// Thinking-mode configuration: the single authority for mode behaviour.
-//
-// Every mode-dependent decision in the runtime reads from this file instead of re-deriving it
-// from a thinking_mode
-// string, so tuning a mode is a one-table edit.
-//
-// Unknown labels fall back to NAIVE rather than erroring: the label comes from
-// user input at request time, and failing here would fail the whole request.
-
-// allTools are the tools the action session can bind, in declaration order.
-// Kept here so a mode's tool set is data, not an if-chain in the runtime.
-var allTools = []string{
- "retrieve",
- "search_chunks",
- "metadata_search",
- "list_chunks",
- "navigate_tree",
- "navigate_structure",
- "calculate",
- "web_search",
-}
-
-// GraphExploreTool is the relational exploration tool reserved for ultra.
-const GraphExploreTool = "graph_explore"
-
-// ModeSpec: (a frozen dataclass). Everything that
-// varies between thinking modes lives here.
-//
-// Tools is the set of tool names visible to the model in this mode. An empty
-// set means the model gets no tool loop at all.
-type ModeSpec struct {
- Label string
- Agentic bool
- EnableSCA bool
- SCAMaxRounds int
- UseFanout bool
- ActionMaxTurns int
- // SnippetsPerQuery caps how many hits of ONE query the session reads.
- //
- // It rises with the mode because a deeper mode issues more queries and can
- // absorb more evidence per query, while one flat cap for every mode is what
- // drops the long tail: the engine returns 30-60 candidates per leg, the
- // fact-bearing passage of a themed query ranks far down that list, and a cap
- // of four discards it even though the query DID reach it. Zero means "unset":
- // the session falls back to snippetsPerQuery (see snippetsPerQueryFor).
- SnippetsPerQuery int
- Tools map[string]bool
-}
-
-// toolsOf builds the tool set from names.
-func toolsOf(names ...string) map[string]bool {
- m := make(map[string]bool, len(names))
- for _, n := range names {
- m[n] = true
- }
- return m
-}
-
-// allToolSet returns the default tool surface.
-func allToolSet() map[string]bool {
- return toolsOf(allTools...)
-}
-
-// THINKING_MODES
-//
-// - low: one hybrid-search pass through direct_search. No action session,
-// so no tool loop — the model never sees tools in this mode.
-// - medium: agentic, SCA review on, no planner decomposition.
-// - high: adds planner + prefetch fan-out over the same tool surface.
-// - ultra: deeper sessions, more SCA rounds, and the relational tool.
-var THINKING_MODES = map[string]ModeSpec{
- "low": {
- Label: "low", Agentic: false, EnableSCA: false,
- SCAMaxRounds: 0, UseFanout: false, ActionMaxTurns: 4,
- SnippetsPerQuery: 0,
- Tools: map[string]bool{},
- },
- "medium": {
- Label: "medium", Agentic: true, EnableSCA: true,
- SCAMaxRounds: 3, UseFanout: false, ActionMaxTurns: 8,
- SnippetsPerQuery: 6,
- Tools: allToolSet(),
- },
- "high": {
- Label: "high", Agentic: true, EnableSCA: true,
- SCAMaxRounds: 3, UseFanout: true, ActionMaxTurns: 8,
- SnippetsPerQuery: 8,
- Tools: allToolSet(),
- },
- "ultra": {
- Label: "ultra", Agentic: true, EnableSCA: true,
- SCAMaxRounds: 5, UseFanout: true, ActionMaxTurns: 10,
- SnippetsPerQuery: 10,
- Tools: toolsOf(append(append([]string{}, allTools...), GraphExploreTool)...),
- },
-}
-
-// NAIVE is the fallback for an unrecognised mode label. It is not agentic — the
-// caller answers with plain retrieval rather than failing the request.
-var NAIVE = ModeSpec{
- Label: "naive", Agentic: false, EnableSCA: false,
- SCAMaxRounds: 0, UseFanout: false, ActionMaxTurns: 4,
- Tools: map[string]bool{},
-}
-
-// HasTool reports whether the mode exposes the named tool.
-func (m ModeSpec) HasTool(name string) bool { return m.Tools[name] }
-
-// ToolNames returns the mode's visible tool names in declaration order, so
-// schema generation is deterministic (Go map iteration is randomised).
-func (m ModeSpec) ToolNames() []string {
- out := make([]string, 0, len(m.Tools))
- for _, n := range allTools {
- if m.Tools[n] {
- out = append(out, n)
- }
- }
- if m.Tools[GraphExploreTool] {
- out = append(out, GraphExploreTool)
- }
- return out
-}
-
-// GetMode: (label): unknown labels fall back to NAIVE.
-// The label arrives from user input, so erroring here would fail the request.
-func GetMode(label string) ModeSpec {
- m, ok := THINKING_MODES[strings.ToLower(strings.TrimSpace(label))]
- if !ok {
- return NAIVE
- }
- return m
-}
-
-// ThinkingModeCarrier is implemented by the RAGTools-like object that owns the
-// request-scoped retrieval context (its thinking mode).
-type ThinkingModeCarrier interface {
- GetThinkingMode() string
-}
-
-// ResolveMode: (tools): reads a RAGTools-like
-// object's thinking mode into its spec. Values that do not implement
-// ThinkingModeCarrier fall back to NAIVE.
-func ResolveMode(tools any) ModeSpec {
- if c, ok := tools.(ThinkingModeCarrier); ok {
- return GetMode(c.GetThinkingMode())
- }
- return NAIVE
-}
diff --git a/internal/rag/agentic-rag/runtime/coverage.go b/internal/rag/agentic-rag/runtime/coverage.go
deleted file mode 100644
index e5cb45fbf9..0000000000
--- a/internal/rag/agentic-rag/runtime/coverage.go
+++ /dev/null
@@ -1,183 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "strings"
-
- "ragflow/internal/rag/agentic-rag/slots"
-)
-
-// CoverageActWordsMax bounds the act words ONE slot may declare.
-const CoverageActWordsMax = 10
-
-// Coverage is what a question ASKS FOR when its answer is a SET OF NAMED MEMBERS: an
-// actor, the words the source uses for the deed, and the two declarations that say the
-// answer is a set of names at all.
-//
-// It is the WHOLE of the enumeration strategy's interface — two fields carried from the
-// planner's own declaration (Actor / Acts) and two readings of the same declaration
-// (Set / Members). Everything the enumeration needs is answered by "is this three things
-// true", and everything it does is answered by "one recall per operand, one window per
-// place the deed is stated, one verdict per window" — no pattern language, no per-round
-// bookkeeping, no reach ledger.
-//
-// Where the fields come from: the planner is asked to declare act words for "a SET or a
-// COUNT of things that someone DID" and names the actor beside them (a slot's Terms and
-// Subject). Those two declarations are the plan. A table that does not make them is not
-// enumerated, and pays nothing (Coverage.Ok).
-type Coverage struct {
- // Actor is the actor's declared forms, alternatives joined by '|' (a name and its
- // aliases).
- // Empty is allowed: the deed's own words still enumerate ("who was killed" questions
- // state the act without naming the actor).
- Actor string
- // Acts are the words the planner declared for the deed — the verbs the source uses for
- // it. A phrasing no act word covers is a passage no query names, which is why the
- // planner is asked for several.
- Acts []string
- // Set reports that the table asked for a count, a set or a list — the answer is a SET
- // rather than one value.
- Set bool
- // ItemKind is what the answer's elements are, when the table declares it: "person", "entity",
- // "dataset", or the bare "list"/"set". Empty when the answer is not a list of elements.
- ItemKind string
- // itemsInValue: a slot's value IS a list of items — the direction declared by the run's own
- // output rather than by the planner's word.
- //
- // Read by the point-of-naming node ALONE (see RunCoverageResolve), which runs last: a plan-time
- // type word must not be the only way a set can be completed. Deliberately NOT folded into Set —
- // the two readers of Set ask what the planner DECLARED (RenderSlotRecord's set block and
- // SessionState.parentSet), and read from the value they demote a value question's own draft
- // answer: measured (2026-09-17, FRAMES q759) a one-person question's record lost its
- // "Candidate answer:" line and gained "enumerated members across the slots above: 1".
- itemsInValue bool
-}
-
-// CoverageItemKindItems is the element kind a slot declares BY ITS OWN VALUE: a slot holding a
-// list of items says the answer is a set, and no planner word names the kind any further.
-const CoverageItemKindItems = "items"
-
-// CoverageItemKinds reads the planner's slot vocabulary as what the answer enumerates. A kind not
-// in this table is not a list of elements (a count, a range, a date, a phrase), and the gate below
-// pays nothing for it.
-var CoverageItemKinds = map[string]string{
- "entity": "entity",
- "person": "person",
- "dataset": "dataset",
- "list": "list",
- "set": "set",
-}
-
-// CoverageOf reads the declaration out of the planner's own table.
-//
-// Nothing here reads a slot's text, the question, or a candidate: the shape and the deed
-// are what the PLANNER said about the answer, so the gate cannot be fooled by how a
-// passage happens to be worded. A permissive reading instead spends set strategy on
-// questions that assemble nothing.
-func CoverageOf(table State) Coverage {
- var c Coverage
- for _, v := range table.State {
- // The two readings are independent: "list" and "set" are BOTH set-shaped and
- // element-carrying, so a slot may satisfy either or both (they were two predicates
- // before, and this keeps their union exact).
- kind := strings.ToLower(strings.TrimSpace(v.Type))
- switch kind {
- case "count", "set", "list":
- c.Set = true
- }
- if k, ok := CoverageItemKinds[kind]; ok && c.ItemKind == "" {
- c.ItemKind = k
- }
- // The VALUE declares the direction too, but to a NARROWER audience than Set: a slot holding
- // a list of items says the answer is a set, and that is what lets the point-of-naming node
- // complete a set the planner never typed (measured 2026-09-17, 三国/关羽: the members were
- // patched into a slot typed "count" and the run judged only the ten names a session wrote).
- // It is NOT Set: see the field's note for the question that read it too widely.
- if v.Typed().Kind == slots.KindItems {
- c.itemsInValue = true
- if c.ItemKind == "" {
- c.ItemKind = CoverageItemKindItems
- }
- }
- if c.Actor == "" {
- c.Actor = strings.TrimSpace(v.Subject)
- }
- for _, term := range v.Terms {
- if term = strings.TrimSpace(term); term != "" {
- c.Acts = appendUnique(c.Acts, term)
- }
- }
- }
- return c
-}
-
-// Ok reports whether this table asks for an enumeration: a SET of ELEMENTS whose deed the
-// planner wrote the words for.
-//
-// The conjunction is the whole gate, and it is what separates a set of elements from a
-// count of EVENTS — the planner declares act words for the latter too ("how many times did
-// it happen" → the verb for it), and no element an enumeration could return changes a count
-// of events. Held only by the element-carrying half, the enumeration runs on single-value
-// questions, where it can only add cost and timeouts.
-func (c Coverage) Ok() bool {
- return c.Set && c.ItemKind != "" && len(c.Acts) > 0
-}
-
-// ActsAll is the act vocabulary the enumeration filters by: the planner's declaration, and
-// nothing else. It is a named extension point rather than a constant (see coverage_enumerate.go,
-// which filters with it).
-func (c Coverage) ActsAll() []string {
- return append([]string(nil), c.Acts...)
-}
-
-// ItemsInValue reports whether the table's own slots HOLD items — the direction the run's output
-// declares where the planner's word did not. The point-of-naming node reads it and nothing else
-// does (see Coverage.itemsInValue for why the record and a session must not).
-func (c Coverage) ItemsInValue() bool { return c.itemsInValue }
-
-// Actors splits the declared actor into its alternatives, which are what the corpus has
-// to be read with: one source words one person several ways.
-func (c Coverage) Actors() []string {
- var out []string
- for _, part := range strings.FieldsFunc(c.Actor, func(r rune) bool {
- return r == '|' || r == '|' || r == '/' || r == '、' || r == ',' || r == ','
- }) {
- if part = strings.TrimSpace(part); part != "" {
- out = appendUnique(out, part)
- }
- }
- return out
-}
-
-// Operands is the recall list: the actor's alternatives and the act words, deduped.
-//
-// ONE entry per operand is the point. Asking once per (actor, act) PAIR recalls the same
-// operand over and over, so a common word fills its recall bound before the rarer ones are
-// ever reached.
-func (c Coverage) Operands() []string {
- var out []string
- for _, a := range c.Actors() {
- out = appendUnique(out, a)
- }
- for _, act := range c.Acts {
- if act = strings.TrimSpace(act); act != "" {
- out = appendUnique(out, act)
- }
- }
- return out
-}
diff --git a/internal/rag/agentic-rag/runtime/coverage_enumerate.go b/internal/rag/agentic-rag/runtime/coverage_enumerate.go
deleted file mode 100644
index 27b4639f21..0000000000
--- a/internal/rag/agentic-rag/runtime/coverage_enumerate.go
+++ /dev/null
@@ -1,424 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "fmt"
- "sort"
- "strings"
- "unicode/utf8"
-)
-
-// The enumeration's bounds. They are spent in ONE place each: coverageRecallTopN is what
-// one OPERAND may recall (per-operand, not per-pair), the window bounds decide how much
-// of a passage a verdict is asked about, and the two caps keep the seed and the call list
-// finite.
-const (
- // 200 was the HEAD OF A RANKING sold as a total (measured 2026-09-17, 三国, 1718 chunks:
- // `operand "斩"/"杀": recall hit its bound (200 passage(s))`, both runs). A member stated
- // in a passage ranked past 200 could then never become a window, never be judged, never be
- // named — and the pass-through set came out as "the famous ones", which is why an answer
- // named the right members while citing only a handful of them. The bound is per OPERAND
- // (actor forms + act words, see Coverage.Operands), so this is a corpus-scale ask, not a
- // per-pair one.
- coverageRecallTopN = 1000
- coverageWindowsPerChunk = 3
- coverageWindowBefore = 80
- coverageWindowAfter = 60
- // coverageWindowsMax bounds the lines the point-of-naming node is handed, over BOTH of its
- // sources: this enumeration's windows and the names a probe already reached (the reached
- // ledger's own cap, reachedTermsMax). It is spent against that node's capacity — workers ×
- // clock / a slow call × batch — so the two move together: a cap above the capacity is a
- // ceiling that can never be reached, and the member set then moves with the provider's
- // latency instead of with the corpus.
- //
- // 288 is the capacity the resolve node already documents for itself (coverage_resolve.go:
- // six workers × (CoverageResolveTimeoutS / a five-second call) × batch = 6 × 6 × 8 = 288).
- // At 192 the cap was BELOW that capacity, and every window past it was reported as UNKNOWN
- // (ResolveCoverage: "nobody judged them"), i.e. evidence the enumeration had paid a search
- // for was dropped without ever being read.
- coverageWindowsMax = 288
- // coverageSeedMaxChars is the PROSE budget of the seed, not a cap on what a session can
- // cite: past it the quote is dropped and the chunk id is still written (see Render), because
- // a window omitted outright is a passage no session can name, and a member nothing can name
- // is a member that only exists in the model's own memory.
- coverageSeedMaxChars = 32000
-)
-
-// CoverageWindow is one place the corpus states the deed: the chunk it is in, the words
-// around the act, and which act word was found.
-//
-// The window is what a verdict is asked about (see ResolveCoverage), and its chunk id is
-// where a member's citation comes from — the model names a member, the runtime names the
-// passage, so a fabricated citation cannot reach the record.
-type CoverageWindow struct {
- ChunkID string
- Quote string
- Act string
- // Source is the line's PROVENANCE, and it is the whole difference between what used to be
- // three pipelines (see CoverageSource): the enumeration's windows, the names a probe reached
- // and the names a session asserted are judged side by side, in one list with one budget —
- // not in three lists with three caps, three dedup rules and three counters.
- Source CoverageSource
-}
-
-// CoverageSource says where one ledger line came from. Nothing downstream may branch on it
-// except a report: a line is judged, or it is not, and a member named in it is a member.
-type CoverageSource string
-
-const (
- // CoverageSourceEnumeration: a window this run's own enumeration built; it carries an Act.
- CoverageSourceEnumeration CoverageSource = "enumeration"
- // CoverageSourceReached: a name a probe asked about and DID reach, with the pool chunk that
- // carries it (see Kbinfos.RecordReachedTerm).
- CoverageSourceReached CoverageSource = "reached"
- // CoverageSourceClaim: a name a session asserted with no passage. The pool is asked what it
- // can show about it, so a claim the run can refute is refuted rather than counted.
- CoverageSourceClaim CoverageSource = "claim"
-)
-
-// CoverageSet is what one enumeration of the corpus produced: the operands it asked
-// about, the windows the deed was stated in, and how many distinct passages came back.
-//
-// It is EVIDENCE, not advice: the windows are admitted to the shared pool (so a session
-// can read them and the answer can cite them) and the seed states them as facts, which is
-// what makes the session's job "read these windows" rather than "think of the queries I
-// should have made".
-type CoverageSet struct {
- Operands []string
- Windows []CoverageWindow
- Recalled int
- // Truncated reports that the clock ran out mid-enumeration: what came back is real,
- // what did not come back is unknown rather than absent.
- Truncated bool
-}
-
-// Empty reports whether the enumeration found nothing worth a verdict.
-func (s CoverageSet) Empty() bool { return len(s.Windows) == 0 }
-
-// EnumerateCoverage asks the corpus about the actor and the act words and returns the
-// windows where the deed is stated.
-//
-// ONE recall per OPERAND (the union of the actor's forms and the act words), then a TEXT
-// test — a passage has to carry a declared act word, and one of the actor's forms when
-// the actor was declared — which is the same set the per-pair patterns used to match,
-// reached with a fraction of the searches (see Coverage.Operands).
-//
-// The evidence is admitted to the pool as it is found, so the enumeration is the whole of
-// "检索充分": every member a verdict can name has its passage in hand before the question
-// is asked, and no round has to re-enumerate to find it again.
-func EnumerateCoverage(ctx context.Context, deps SearchDeps, cov Coverage, kb *Kbinfos) CoverageSet {
- set := CoverageSet{Operands: cov.Operands()}
- if len(set.Operands) == 0 {
- return set
- }
- actors := cov.Actors()
- byID := map[string]map[string]any{}
- for _, operand := range set.Operands {
- if ctx.Err() != nil {
- // Out of clock: what was recalled is real evidence, what was not recalled is
- // UNKNOWN, and the caller is told which (see CoverageSet.Truncated).
- set.Truncated = true
- break
- }
- // The recall's SECOND value is its doc aggregations, NOT an error: GrepSearch
- // cannot fail, and DocAggs builds its slice with make() so it is non-nil whether or
- // not anything came back. Read as an error, this branch fires on EVERY operand and
- // discards each recall's passages as though the corpus had answered nothing — the
- // opposite of what this set is for, and a seed that tells every session the
- // direction is empty.
- //
- // A transport failure is therefore indistinguishable from an empty recall HERE;
- // when that distinction is needed, GrepSearch has to grow an error. Until then the
- // recall's result IS the evidence.
- chunks, _ := GrepSearch(ctx, deps, SearchParams{
- Question: operand,
- Keywords: operand,
- TopN: coverageRecallTopN,
- KbIDs: deps.KbIDs,
- // What is asked here is the ACTOR and the ACT WORDS, never a member's name,
- // so the reach ledger must not read them as names a probe confirmed.
- SkipReachLedger: true,
- })
- if len(chunks) >= coverageRecallTopN {
- // The recall came back at its whole bound: what the corpus holds for this operand
- // MAY be more than this, and nothing downstream can tell the difference — the member
- // set would be the head of a ranking presented as a total. It is logged as a
- // measurement so that "was the recall complete" is a fact in the run's own record
- // instead of a guess (see CoverageSet.Truncated for the clock-and-cap case).
- _LOG.Printf("[Coverage] operand %q: recall hit its bound (%d passage(s)) — the corpus may hold more", operand, coverageRecallTopN)
- }
- for _, c := range chunks {
- if id := ChunkIDOf(c); id != "" {
- if _, seen := byID[id]; !seen {
- byID[id] = c
- }
- }
- }
- }
- set.Recalled = len(byID)
-
- // The vocabulary the filter below asks about. The plan's words are a guess about a text
- // nobody had read when they were written (see Coverage.ActsAll): the passages THIS recall just
- // brought back through the actor's own name are read here, once, so that a death stated in a
- // phrasing the plan had no word for — 第一回's "被云长刀起处,挥为两段" — becomes a window
- // instead of being dropped before any node ever sees it.
- acts := cov.ActsAll()
- sampleIDs := make([]string, 0, len(byID))
- for id := range byID {
- sampleIDs = append(sampleIDs, id)
- }
- // Sorted by chunk id: which passages are sampled is what the induced vocabulary is read from,
- // and that vocabulary becomes a filter, so it must not vary with the map's iteration order.
- sort.Strings(sampleIDs)
- var samples []string
- var probe strings.Builder
- probe.WriteString("Passages:\n")
- for _, id := range sampleIDs {
- if len(samples) >= coverageVocabSamples {
- break
- }
- text := strings.Join(strings.Fields(ChunkTextOf(byID[id])), " ")
- if text == "" || (len(actors) > 0 && !coverageMentionsAny(text, actors)) {
- continue
- }
- if r := []rune(text); len(r) > coverageVocabSampleChars {
- text = string(r[:coverageVocabSampleChars])
- }
- samples = append(samples, text)
- fmt.Fprintf(&probe, "[%d] %s\n", len(samples), text)
- }
- if induced := induceActWords(ctx, deps.Model, cov.Acts, probe.String(), samples); len(induced) > 0 {
- for _, w := range induced {
- acts = appendUnique(acts, w)
- }
- _LOG.Printf("[Coverage] vocabulary probe: read %d word(s) out of the corpus that the plan had not declared (%s) — the filter knows them now.", len(induced), strings.Join(induced, " / "))
- }
-
- // Sorted by chunk id: these windows are rendered into EVERY session's seed (and the
- // cap below keeps the first N), so walking the map made both the seed's wording and
- // which windows survived the cap differ from run to run.
- ids := make([]string, 0, len(byID))
- for id := range byID {
- ids = append(ids, id)
- }
- sort.Strings(ids)
-
- // An actor form that matches NOTHING in what the recall just brought back is a suspect
- // DECLARATION, not a fact about the corpus: 2026-09-18 the plan's subject came through as
- // "None", the actor test rejected all 852 passages this recall had returned, and the run lost
- // every citation it could have had (the point-of-naming node was handed no window at all). A
- // test that rejects 100% of the recall has stopped being a filter, so the passages below are
- // judged on the act words alone — the model is the node that decides whether the deed is
- // stated — and the miss is said out loud instead of turning into silence.
- if len(actors) > 0 {
- matched := false
- for _, id := range ids {
- if coverageMentionsAny(strings.Join(strings.Fields(ChunkTextOf(byID[id])), " "), actors) {
- matched = true
- break
- }
- }
- if !matched {
- _LOG.Printf("[Coverage] actor %q matches NONE of the %d recalled passage(s): dropping the actor test (the declaration is suspect; the act words still stand) instead of judging nothing.", cov.Actor, len(ids))
- actors = nil
- }
- }
- for _, id := range ids {
- c := byID[id]
- text := strings.Join(strings.Fields(ChunkTextOf(c)), " ")
- act := coverageFirstAct(text, acts)
- if act == "" {
- continue
- }
- if len(actors) > 0 && !coverageMentionsAny(text, actors) {
- continue
- }
- if kb != nil {
- kb.Admit(func(p *PoolAdmitter) { p.Add(c) })
- }
- for _, w := range coverageWindows(text, acts, coverageWindowsPerChunk) {
- if len(set.Windows) >= coverageWindowsMax {
- set.Truncated = true
- break
- }
- set.Windows = append(set.Windows, CoverageWindow{ChunkID: ChunkIDOf(c), Quote: w.quote, Act: w.act, Source: CoverageSourceEnumeration})
- }
- }
- return set
-}
-
-// CoverageActsMeetActor reports whether a passage ALREADY HELD carries both an act word and the
-// actor — the conjunction every window the enumeration builds is filtered by (see EnumerateCoverage:
-// a recalled passage is kept only when an act word is in it AND, when the direction declares an
-// actor, the actor is named in it too).
-//
-// Read only by the last-resort enrollment (see enrollEnumeration), where the recall it would pay for
-// comes out of the ANSWER's own clock: measured (2026-09-17, FRAMES, resolve node) 13 operands asked,
-// 724 passages recalled, 0 windows found — the direction's words had never met in what the run held,
-// so nothing a recall brought back could survive the filter.
-//
-// It is a probe, not a verdict: a recall may still return the first passage where they do meet, which
-// is exactly what the enrollment is for. It answers only "have we ever seen these words together".
-func CoverageActsMeetActor(kb *Kbinfos, cov Coverage) bool {
- if kb == nil {
- return false
- }
- actors := cov.Actors()
- for _, c := range kb.Chunks {
- text := ChunkTextOf(c)
- if coverageFirstAct(text, cov.ActsAll()) == "" {
- continue
- }
- if len(actors) > 0 && !coverageMentionsAny(text, actors) {
- continue
- }
- return true
- }
- return false
-}
-
-// Render is the seed text: the enumeration stated as a FACT with what came back.
-//
-// A list of queries in a prompt is advice, and advice may simply not be taken. A window is
-// evidence: it carries the chunk id a verdict cites, so the session reads rather than
-// guesses, and `(nothing)` is the corpus answering about these WORDS rather than nobody
-// having looked.
-func (s CoverageSet) Render() string {
- var b strings.Builder
- b.WriteString("## The enumeration already ran for this direction\n\n")
- fmt.Fprintf(&b, "The corpus was asked about each operand below (%d operands, %d distinct passage(s)) — one search per operand. "+
- "Every window is already in the pool under the chunk id shown, and a member named in a window is a member you can cite with that id.\n\n",
- len(s.Operands), s.Recalled)
- if len(s.Operands) > 0 {
- // The operands are NAMED, not promised: the header said "below" while only the
- // windows followed it, so a session could not tell an operand nobody searched
- // from one the corpus came back empty on.
- b.WriteString("- operands: " + strings.Join(s.Operands, " / ") + "\n\n")
- }
- if s.Empty() {
- b.WriteString("(nothing)\n")
- return b.String()
- }
- quotesDropped, idsDropped := 0, 0
- for _, w := range s.Windows {
- line := fmt.Sprintf("- chunk_id=%s %q\n", w.ChunkID, w.Quote)
- if b.Len()+len(line) > coverageSeedMaxChars {
- // Over the prose budget: the passage is still NAMED, only its text is left out.
- // Dropping the line outright would hide a window the enumeration already paid a
- // search for, and a member that no line names is a member the session can only
- // answer from its own memory — the very thing the seed exists to replace.
- line = fmt.Sprintf("- chunk_id=%s\n", w.ChunkID)
- if b.Len()+len(line) > coverageSeedMaxChars {
- idsDropped++
- continue
- }
- quotesDropped++
- }
- b.WriteString(line)
- }
- if quotesDropped > 0 {
- fmt.Fprintf(&b, "(the %d window(s) above list a chunk id without its quote; the text is in the pool under that id)\n", quotesDropped)
- }
- if idsDropped > 0 {
- fmt.Fprintf(&b, "(%d further window(s) are in the pool but are not listed here)\n", idsDropped)
- }
- if s.Truncated {
- b.WriteString("(the enumeration ran out of clock; the corpus holds more than is shown here)\n")
- }
- return b.String()
-}
-
-// coverageWindows returns up to n act-centred windows of one passage, in order.
-func coverageWindows(text string, acts []string, n int) []struct{ quote, act string } {
- var out []struct{ quote, act string }
- if n <= 0 {
- return out
- }
- runes := []rune(text)
- from := 0
- for from < len(runes) && len(out) < n {
- idx, length, act := coverageNextAct(runes, from, acts)
- if idx < 0 {
- break
- }
- lo := idx - coverageWindowBefore
- if lo < 0 {
- lo = 0
- }
- hi := idx + length + coverageWindowAfter
- if hi > len(runes) {
- hi = len(runes)
- }
- out = append(out, struct{ quote, act string }{quote: strings.TrimSpace(string(runes[lo:hi])), act: act})
- from = idx + length
- }
- return out
-}
-
-// coverageNextAct finds the earliest declared act word at or after `from`, returning the
-// rune index, its length and the word itself.
-func coverageNextAct(runes []rune, from int, acts []string) (int, int, string) {
- bestIdx, bestLen, bestAct := -1, 0, ""
- for _, act := range acts {
- needle := []rune(strings.TrimSpace(act))
- if len(needle) == 0 {
- continue
- }
- if idx := runeIndexFrom(runes, needle, from); idx >= 0 && (bestIdx < 0 || idx < bestIdx) {
- bestIdx, bestLen, bestAct = idx, len(needle), act
- }
- }
- return bestIdx, bestLen, bestAct
-}
-
-// coverageFirstAct answers whether a passage states the deed at all.
-func coverageFirstAct(text string, acts []string) string {
- if _, _, act := coverageNextAct([]rune(text), 0, acts); act != "" {
- return act
- }
- return ""
-}
-
-// coverageMentionsAny reports whether the passage carries one of the actor's forms.
-func coverageMentionsAny(text string, forms []string) bool {
- for _, f := range forms {
- if strings.Contains(text, f) {
- return true
- }
- }
- return false
-}
-
-// runeIndexFrom is strings.Index over runes, starting at a rune index: the window
-// arithmetic above is in runes, so a byte offset would not do, and the stdlib search
-// beats a hand-written scan.
-func runeIndexFrom(haystack, needle []rune, from int) int {
- if from < 0 {
- from = 0
- }
- if len(needle) == 0 || from >= len(haystack) {
- return -1
- }
- tail := string(haystack[from:])
- at := strings.Index(tail, string(needle))
- if at < 0 {
- return -1
- }
- return from + utf8.RuneCountInString(tail[:at])
-}
diff --git a/internal/rag/agentic-rag/runtime/coverage_resolve.go b/internal/rag/agentic-rag/runtime/coverage_resolve.go
deleted file mode 100644
index b66d733c6d..0000000000
--- a/internal/rag/agentic-rag/runtime/coverage_resolve.go
+++ /dev/null
@@ -1,407 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "strconv"
- "strings"
- "sync"
-
- "github.com/cloudwego/eino/schema"
-)
-
-// CoverageResolvePrompt is the whole instruction of the enumeration's LAST node.
-//
-// It is written around the QUESTION rather than around one kind of answer: the same step
-// runs on "which awards did X win" and on "who did X kill", so what a line has to do is
-// state a member of the set the question asks for. Nothing here decides what counts as a
-// member — that judgement stays the model's.
-const CoverageResolvePrompt = "You are given numbered evidence lines from one fixed corpus, each with the chunk id it came from.\n" +
- "For EVERY line decide one thing: does this line itself state a member of the set the question asks for — and if it does, what is that member called?\n" +
- "Judge only what the line says, never what you know from elsewhere. A mention that is not a member (a plan, a promise, a denial, a pursuit, something that did not happen, somebody else's action, or a member the line does not name) is NOT one.\n" +
- "Answer with JSON only, and put EVERY line number in exactly one of the two lists:\n" +
- `{"members": [{"i": <line number>, "name": "<the member the line states>"}], "not_members": [<line numbers>]}` + "\n" +
- "Use the line's own words for the name. Never invent a name, and never answer for a line you were not given."
-
-// ResolvedMember is a member the last node named, with the line it was named from.
-type ResolvedMember struct {
- Name string
- ChunkID string
- Quote string
-}
-
-// ResolveStats is what one resolve produced, for the run log: how many calls it took,
-// how many lines were put in front of the model, how many got a verdict, and how many
-// would still be UNKNOWN if the answer were written now.
-type ResolveStats struct {
- Batches int
- Asked int
- Answered int
- Unknown int
- Failed int
- // Unjudged are the chunk ids that got no verdict — the windows a failed call left, the lines
- // the model skipped, the windows the cap kept out. The count alone says "the list may be
- // short"; the ids say WHICH passages nobody read, which is what a reader can check.
- Unjudged []string
-}
-
-// resolveJob is ONE window as a unit of work: the index it holds in the caller's window list
-// (which is what a citation is resolved from) and the window itself.
-type resolveJob struct {
- idx int
- w CoverageWindow
-}
-
-// batchResult is one CALL's outcome: the members it named, which of its jobs were judged (as
-// a member or explicitly not), and whether the call failed. The indices are the CALLER's
-// window indices, so a later pass can re-ask exactly the ones that got no verdict.
-type batchResult struct {
- members []ResolvedMember
- judged []int
- failed bool
-}
-
-// The resolve's bounds. A batch is small because ONE prompt carrying every candidate is the
-// failure this node exists to remove: a single call that carries them all reaches its clock
-// and answers nothing, losing a whole enumeration's worth of members to one timeout.
-//
-// These bound ONE budget, not independent knobs: this node has to be able to judge
-// coverageWindowsMax windows inside CoverageResolveTimeoutS, and its capacity is
-// workers × (clock / the slowest call it can expect) × batch. The cap must stay BELOW that
-// capacity, with room for calls slower than the five seconds assumed here.
-const (
- coverageResolveBatch = 8
- coverageResolveWorkers = 8
- coverageResolveQuoteChars = 300
- // coverageResolvePasses bounds the re-asking: the first pass puts every window in front of
- // the model, and each pass after it re-asks ONLY the windows that got no verdict — a call
- // that failed, or lines the reply left out. Work that already has a verdict is never sent
- // again, so the set closes without paying twice for the windows that were answered.
- coverageResolvePasses = 5
-)
-
-// ResolveCoverage is the enumeration's LAST node: every window gets a verdict, and the
-// members are the lines that state one.
-//
-// Three things are deliberate:
-//
-// - EVERY window is asked about, in batches, in parallel. The node is not allowed to
-// skip a candidate in silence: a batch that fails or a line the model does not answer
-// is counted as UNKNOWN and reported, so "nobody looked" can never be read as "the
-// corpus does not say it".
-// - A member's citation is the LINE'S chunk id, never the reply's: the model only has
-// to get a line number right, so a fabricated source cannot reach the record.
-// - The member's NAME is kept as the model wrote it from the line, and members are
-// deduped case-insensitively — one source spells one member several ways, and a set
-// counts entities, not spellings.
-func ResolveCoverage(ctx context.Context, model SessionModel, question string, cov Coverage, set CoverageSet) ([]ResolvedMember, ResolveStats) {
- stats := ResolveStats{Asked: len(set.Windows)}
- if model == nil || set.Empty() {
- stats.Unknown = stats.Asked
- return nil, stats
- }
- windows := set.Windows
- // Windows beyond the cap are never put in front of the model, so they are
- // UNKNOWN for the same reason a failed batch is: nobody judged them.
- dropped := 0
- if len(windows) > coverageWindowsMax {
- dropped = len(windows) - coverageWindowsMax
- windows = windows[:coverageWindowsMax]
- }
- stats.Asked = len(windows)
-
- // EVERY window is a unit of work that must end with exactly one verdict, so the set is
- // CLOSED: the first pass asks about all of them, and each pass after it re-asks only the
- // ones that got none — a call that failed, or lines the reply left out. Answering is what
- // makes a member set a function of the corpus instead of a function of which call happened
- // to come back in time.
- pending := make([]resolveJob, 0, len(windows))
- for i, w := range windows {
- pending = append(pending, resolveJob{idx: i, w: w})
- }
- judged := make([]bool, len(windows))
- seen := map[string]int{}
- var members []ResolvedMember
-
- for pass := 0; pass < coverageResolvePasses && len(pending) > 0; pass++ {
- if ctx.Err() != nil {
- break
- }
- out := make([]batchResult, (len(pending)+coverageResolveBatch-1)/coverageResolveBatch)
- sem := make(chan struct{}, coverageResolveWorkers)
- var wg sync.WaitGroup
- for b := range out {
- lo := b * coverageResolveBatch
- hi := min(lo+coverageResolveBatch, len(pending))
- part := pending[lo:hi]
- wg.Add(1)
- go func(b int, part []resolveJob) {
- defer wg.Done()
- sem <- struct{}{}
- defer func() { <-sem }()
- out[b] = resolveCoverageBatch(ctx, model, question, cov, part)
- }(b, part)
- }
- wg.Wait()
-
- for _, r := range out {
- stats.Batches++
- if r.failed {
- stats.Failed++
- }
- for _, idx := range r.judged {
- if !judged[idx] {
- judged[idx] = true
- stats.Answered++
- }
- }
- for _, m := range r.members {
- key := strings.ToLower(strings.TrimSpace(m.Name))
- if key == "" {
- continue
- }
- if i, dup := seen[key]; dup {
- // The member is already known; a second passage is worth keeping when the
- // first came without one (evidence is what makes a member answerable).
- if members[i].ChunkID == "" && m.ChunkID != "" {
- members[i] = m
- }
- continue
- }
- seen[key] = len(members)
- members = append(members, m)
- }
- }
- next := pending[:0:0]
- for _, j := range pending {
- if !judged[j.idx] {
- next = append(next, j)
- }
- }
- pending = next
- }
- // Every window is UNKNOWN unless a verdict named it: the windows a failed call left, the
- // lines the model skipped, and the windows the cap kept out.
- stats.Unknown = dropped + (stats.Asked - stats.Answered)
- for i, w := range windows {
- if !judged[i] {
- stats.Unjudged = appendUnique(stats.Unjudged, w.ChunkID)
- }
- }
- if len(set.Windows) > coverageWindowsMax {
- for _, w := range set.Windows[coverageWindowsMax:] {
- stats.Unjudged = appendUnique(stats.Unjudged, w.ChunkID)
- }
- }
- return members, stats
-}
-
-// CoverageVocabPrompt asks for the vocabulary of THE SOURCE, never for the vocabulary of the
-// language: the passages are what answers it, and a word that is not in them does not qualify.
-const CoverageVocabPrompt = `You are given passages from ONE source document.
-
-The direction under study is the deed named by these words — the act they stand for:
-%WORDS%
-
-List the words THIS SOURCE uses for that deed. A word qualifies only if it ACTUALLY APPEARS in the
-passages below: never list a word you know from the language but cannot see in the text. Include
-every distinct phrasing the passages use for it, including unusual and indirect ones — but give the
-ACT ITSELF: the verb or short phrase, never a phrase carrying a particular victim's name and never
-a whole clause.
-
-Answer with JSON only: {"words": ["<word>", "<word>"]}
-Every entry is copied EXACTLY as it appears in a passage, at most %MAX% entries. No prose.`
-
-// The probe's bounds: how many passages it reads, how much of each, and how many words it may
-// bring back. The call is small on purpose — it is read once per enumeration, against a point of
-// naming that judges hundreds of lines — and every word it reports is then checked against the
-// very passages it was shown.
-const (
- coverageVocabSamples = 12
- coverageVocabSampleChars = 320
- coverageVocabWords = 24
-)
-
-// induceActWords reads the deed's vocabulary OUT OF THE CORPUS — one call, before the enumeration
-// filters with it.
-//
-// Why this exists: the planner declares the deed's words BEFORE any passage has been read (see the
-// initialize prompt's "the words the SOURCE itself uses for that deed" — asked of a text nobody
-// has looked at yet).
-func induceActWords(ctx context.Context, model SessionModel, declared []string, passages string, samples []string) []string {
- if model == nil || len(declared) == 0 || passages == "" || len(samples) == 0 || ctx.Err() != nil {
- return nil
- }
- head := strings.Replace(CoverageVocabPrompt, "%WORDS%", strings.Join(declared, " / "), 1)
- head = strings.Replace(head, "%MAX%", fmt.Sprint(coverageVocabWords), 1)
- reply, err := model.Complete(ctx, []schema.Message{
- *schema.SystemMessage(head),
- *schema.UserMessage(passages),
- }, nil)
- if err != nil || reply == nil {
- return nil
- }
- obj := coverageJSONObject(reply.Content)
- if obj == nil {
- return nil
- }
- var out []string
- for _, raw := range coverageAnyList(obj["words"]) {
- w, _ := raw.(string)
- w = strings.TrimSpace(w)
- // A word, not a clause; and present in what the model was shown. the ACT with the victim inside it, which can only ever
- // match the one passage it was read from, so it spends one of the entries to buy nothing.
- // The prompt asks for the act itself; this is the mechanical backstop for it (a phrasing
- // long enough to carry a name is not a vocabulary item).
- if w == "" || len([]rune(w)) > 8 || strings.ContainsAny(w, " \t\n") {
- continue
- }
- if !coverageCarriedByAny(samples, w) {
- continue
- }
- out = appendUnique(out, w)
- if len(out) >= coverageVocabWords {
- break
- }
- }
- return out
-}
-
-// coverageCarriedByAny reports whether one of the passages actually carries the word — the check
-// that makes the induced vocabulary a reading of the corpus instead of a claim about it.
-func coverageCarriedByAny(samples []string, word string) bool {
- for _, s := range samples {
- if strings.Contains(s, word) {
- return true
- }
- }
- return false
-}
-
-// resolveCoverageBatch puts ONE batch of jobs in front of the model and returns the members
-// it named out of that batch. The verdicts it reports are keyed by the CALLER's window
-// indices, so the caller can re-ask exactly the jobs that came back with none.
-func resolveCoverageBatch(ctx context.Context, model SessionModel, question string, cov Coverage, part []resolveJob) batchResult {
- res := batchResult{}
- if ctx.Err() != nil {
- return res
- }
- reply, err := model.Complete(ctx, []schema.Message{
- *schema.SystemMessage(CoverageResolvePrompt),
- *schema.UserMessage(coverageResolveQuestion(question, cov, part)),
- }, nil)
- if err != nil || reply == nil {
- res.failed = true
- return res
- }
- obj := coverageJSONObject(reply.Content)
- if obj == nil {
- res.failed = true
- return res
- }
- for _, raw := range coverageAnyList(obj["members"]) {
- entry, _ := raw.(map[string]any)
- if entry == nil {
- continue
- }
- i, ok := coverageInt(entry["i"])
- name := strings.TrimSpace(anyString(entry["name"]))
- if !ok || name == "" || i < 0 || i >= len(part) {
- continue
- }
- res.judged = append(res.judged, part[i].idx)
- res.members = append(res.members, ResolvedMember{
- Name: name,
- ChunkID: part[i].w.ChunkID,
- Quote: truncateRunes(part[i].w.Quote, coverageResolveQuoteChars),
- })
- }
- for _, raw := range coverageAnyList(obj["not_members"]) {
- if i, ok := coverageInt(raw); ok && i >= 0 && i < len(part) {
- res.judged = append(res.judged, part[i].idx)
- }
- }
- return res
-}
-
-// coverageResolveQuestion renders one call's user message: the question, the actor's
-// declared forms, and this batch's windows on their own numbered lines. The numbering is
-// BATCH-LOCAL and starts at 0 in every call — that is the whole interface the model has to
-// get right, and the caller maps a line number back to its window.
-func coverageResolveQuestion(question string, cov Coverage, part []resolveJob) string {
- var b strings.Builder
- b.WriteString("Question: " + question + "\n")
- if actors := cov.Actors(); len(actors) > 0 {
- b.WriteString("Actor (the forms this direction named): " + strings.Join(actors, " / ") + "\n")
- }
- if len(cov.Acts) > 0 {
- b.WriteString("Act words this direction declared: " + strings.Join(cov.Acts, " / ") + "\n")
- }
- b.WriteString("Evidence lines:\n")
- for i, j := range part {
- w := j.w
- quote := truncateRunes(w.Quote, coverageResolveQuoteChars)
- b.WriteString("- [" + strconv.Itoa(i) + "] chunk_id=" + w.ChunkID + " \"" + quote + "\"\n")
- }
- b.WriteString("\nAnswer with the JSON object and nothing else.")
- return b.String()
-}
-
-// coverageJSONObject reads the JSON object out of a reply, which models wrap in thinking
-// preamble and Markdown fences.
-func coverageJSONObject(content string) map[string]any {
- text := reFence.ReplaceAllString(content, "")
- start := strings.Index(text, "{")
- end := strings.LastIndex(text, "}")
- if start < 0 || end <= start {
- return nil
- }
- var obj map[string]any
- if err := json.Unmarshal([]byte(text[start:end+1]), &obj); err != nil {
- return nil
- }
- return obj
-}
-
-// coverageAnyList reads a JSON list out of a decoded object. `encoding/json` decodes every
-// JSON array into []any (a []map[string]any case here was unreachable), so anything else is
-// treated as "no list" rather than guessed at.
-func coverageAnyList(v any) []any {
- if list, ok := v.([]any); ok {
- return list
- }
- return nil
-}
-
-// coverageInt reads a line number, which a model may hand back as a number or a string.
-func coverageInt(v any) (int, bool) {
- switch t := v.(type) {
- case float64:
- return int(t), t == float64(int(t))
- case int:
- return t, true
- case string:
- var n int
- if _, err := fmt.Sscanf(strings.TrimSpace(t), "%d", &n); err == nil {
- return n, true
- }
- }
- return 0, false
-}
diff --git a/internal/rag/agentic-rag/runtime/coverage_test.go b/internal/rag/agentic-rag/runtime/coverage_test.go
deleted file mode 100644
index 670102b17b..0000000000
--- a/internal/rag/agentic-rag/runtime/coverage_test.go
+++ /dev/null
@@ -1,285 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "fmt"
- "strings"
- "sync"
- "testing"
-
- "github.com/cloudwego/eino/schema"
-
- "ragflow/internal/rag/agentic-rag/slots"
-)
-
-// TestCoverageWindowsCentreOnTheAct pins the unit a verdict is asked about: the words
-// around the deed, not the passage.
-//
-// One passage states the deed several times (a chapter lists several victims), and a
-// window per occurrence is what lets each victim be named from its own words — the
-// alternative, one passage per verdict, is how a three-kill chapter came back as one
-// member.
-func TestCoverageWindowsCentreOnTheAct(t *testing.T) {
- text := "关公马快,赶上文丑,脑后一刀,将文丑斩下马来。云长手起刀落,斩孔秀于马下。"
- got := coverageWindows(text, []string{"斩"}, 3)
- if len(got) != 2 {
- t.Fatalf("windows = %d, want one per occurrence of the act (%v)", len(got), got)
- }
- if !strings.Contains(got[0].quote, "斩下马来") || !strings.Contains(got[1].quote, "斩孔秀于马下") {
- t.Errorf("windows = %v, want each centred on its own act", got)
- }
- if got[0].act != "斩" || got[1].act != "斩" {
- t.Errorf("acts = %q/%q, want the act word that was found", got[0].act, got[1].act)
- }
- // The cap is the caller's: a chapter that states the deed nine times contributes the
- // first few windows, and the pass-through count says the corpus holds more.
- if got := coverageWindows(text, []string{"斩"}, 1); len(got) != 1 {
- t.Errorf("windows = %d, want the cap respected", len(got))
- }
- // A passage that does not state the deed contributes nothing.
- if got := coverageWindows("话说天下大势,分久必合。", []string{"斩", "杀"}, 3); len(got) != 0 {
- t.Errorf("windows = %v, want nothing from a passage without the act", got)
- }
- if act := coverageFirstAct("云长提华雄之头,掷于地上。", []string{"斩", "杀"}); act != "" {
- t.Errorf("act = %q, want no act in a passage that only reports the aftermath", act)
- }
-}
-
-// TestResolveCoverageAsksEveryWindowInBatches pins the enumeration's LAST node: every
-// window is asked about, a failed batch is counted as UNKNOWN rather than read as an
-// absence, a member's citation comes from the LINE, and one source's two spellings of one
-// entity are one member.
-//
-// The single call it replaces loses a whole enumeration's members to one timeout: every
-// candidate in one prompt, one clock shared by all of them.
-func TestResolveCoverageAsksEveryWindowInBatches(t *testing.T) {
- set := CoverageSet{}
- for i := 0; i < 9; i++ {
- set.Windows = append(set.Windows, CoverageWindow{ChunkID: fmt.Sprintf("c%d", i), Quote: "云长斩之", Act: "斩"})
- }
- model := &stubCoverageModel{reply: func(user string) string {
- // The first batch answers, the second fails: the difference must show up as UNKNOWN.
- if strings.Contains(user, "chunk_id=c8") {
- return ""
- }
- return `{"members": [{"i": 0, "name": "孔秀"}, {"i": 1, "name": "mona lisa"}, {"i": 2, "name": "Mona Lisa"}], "not_members": [3]}`
- }}
- model.failIf = func(user string) bool { return strings.Contains(user, "chunk_id=c8") }
-
- members, stats := ResolveCoverage(context.Background(), model, "关羽杀了多少有姓名的人物?", Coverage{Actor: "关羽"}, set)
- // 2 batches carry the first pass (9 windows, batch of 8), and the 5 windows left without a
- // verdict are re-asked once per remaining pass (see coverageResolvePasses).
- if want := 2 + (coverageResolvePasses - 1); stats.Batches != want || stats.Asked != 9 {
- t.Fatalf("stats = %+v, want %d batch(es): 9 windows asked about, and the 5 that got no verdict asked again", stats, want)
- }
- // WHICH passages nobody read travels with the count: a reader can check the ids.
- if got := strings.Join(stats.Unjudged, ","); got != "c4,c5,c6,c7,c8" {
- t.Fatalf("unjudged = %q, want the ids of the windows that got no verdict", got)
- }
- // 4 of the 9 windows got a verdict (3 members + 1 not-a-member); the failed batch's
- // window AND the four lines the model skipped stay UNKNOWN, not absent. The re-ask puts
- // them in front of the model again, but a call that fails every time cannot judge them —
- // so the budget has to be the caller's to report, which is what the counters are for.
- // One failed call per pass over the batch that always fails (the first pass's batch plus one
- // per re-ask pass), so the counter tracks CALLS, as its contract says.
- if stats.Failed != coverageResolvePasses || stats.Answered != 4 || stats.Unknown != 5 {
- t.Fatalf("stats = %+v, want the failed batch and the skipped lines UNKNOWN, not absent", stats)
- }
- if len(members) != 2 {
- t.Fatalf("members = %+v, want the three named lines deduped to two entities", members)
- }
- if members[0].Name != "孔秀" || members[0].ChunkID != "c0" {
- t.Errorf("member = %+v, want the line's own chunk id as its citation", members[0])
- }
- if strings.ToLower(members[1].Name) != "mona lisa" {
- t.Errorf("member = %+v, want one member for two spellings", members[1])
- }
-}
-
-// TestResolveCoverageReasksTheWindowsWithoutAVerdict pins the CLOSURE of the point-of-naming
-// step: a line the reply left out is asked again, and work that already has a verdict is
-// spared.
-//
-// The member set has to be a function of the corpus, not of which call came back. A reply that
-// answers some of its lines and omits the rest leaves those members unjudged, and a pass that
-// stopped there left windows that were RECALLED and SHOWN ending up in no list at all — the
-// count then moves with the provider's latency rather than with the source.
-func TestResolveCoverageReasksTheWindowsWithoutAVerdict(t *testing.T) {
- set := CoverageSet{}
- for i := 0; i < 3; i++ {
- set.Windows = append(set.Windows, CoverageWindow{ChunkID: fmt.Sprintf("c%d", i), Quote: "云长斩之", Act: "斩"})
- }
- calls := 0
- model := &stubCoverageModel{reply: func(string) string {
- calls++
- if calls == 1 {
- // The first call answers ONE of its three lines and omits the other two.
- return `{"members": [{"i": 0, "name": "孔秀"}], "not_members": []}`
- }
- // The line numbers are BATCH-LOCAL: this call carries the two lines nobody judged.
- return `{"members": [], "not_members": [0, 1]}`
- }}
-
- members, stats := ResolveCoverage(context.Background(), model, "关羽杀了多少有姓名的人物?", Coverage{Actor: "关羽"}, set)
- if stats.Asked != 3 || stats.Answered != 3 || stats.Unknown != 0 {
- t.Fatalf("stats = %+v, want every window judged after the re-ask", stats)
- }
- if stats.Batches != 2 {
- t.Errorf("Batches = %d, want the two unjudged lines asked again in a second call", stats.Batches)
- }
- if len(members) != 1 || members[0].Name != "孔秀" || members[0].ChunkID != "c0" {
- t.Errorf("members = %+v, want the one member the first call named, cited by its line", members)
- }
-}
-
-// TestResolveCoverageWithoutAWindowCostsNothing pins the cheap path: an enumeration that
-// found nothing asks nothing.
-func TestResolveCoverageWithoutAWindowCostsNothing(t *testing.T) {
- model := &stubCoverageModel{reply: func(string) string { return "{}" }}
- members, stats := ResolveCoverage(context.Background(), model, "q", Coverage{}, CoverageSet{})
- if len(members) != 0 || stats.Asked != 0 || stats.Batches != 0 {
- t.Errorf("members=%v stats=%+v, want no call and nothing named", members, stats)
- }
- if len(model.prompts) != 0 {
- t.Errorf("model was asked %d time(s), want none", len(model.prompts))
- }
-}
-
-// stubCoverageModel is a SessionModel that answers with a canned reply, and can fail one
-// batch so the UNKNOWN path is exercised.
-type stubCoverageModel struct {
- mu sync.Mutex
- prompts []string
- failIf func(user string) bool
- reply func(user string) string
-}
-
-func (m *stubCoverageModel) Complete(_ context.Context, messages []schema.Message, _ []ToolSpec) (*ModelReply, error) {
- user := ""
- if len(messages) > 0 {
- user = messages[len(messages)-1].Content
- }
- m.mu.Lock()
- m.prompts = append(m.prompts, user)
- m.mu.Unlock()
- if m.failIf != nil && m.failIf(user) {
- return nil, fmt.Errorf("stub: transport failed")
- }
- return &ModelReply{Content: m.reply(user)}, nil
-}
-
-// TestEnumerateCoverageKeepsWhatTheRecallReturned pins the enumeration's recall loop: a
-// passage the recall came back with IS the evidence, and nothing here is a clock
-// exhaustion.
-//
-// GrepSearch's second value is its DOC AGGREGATIONS, not an error, and DocAggs builds that
-// slice with make() — so it is non-nil whether or not anything came back. Read as an error,
-// TestCoverageActsMeetActorPinsTheEnumerationFilter pins the probe against the SAME conjunction the
-// enumeration's window builder uses: an act word AND, when the direction names one, the actor.
-func TestCoverageActsMeetActorPinsTheEnumerationFilter(t *testing.T) {
- kb := &Kbinfos{}
- kb.Admit(func(p *PoolAdmitter) {
- p.Add(map[string]any{"chunk_id": "c1", "content_with_weight": "Colin Beashel sailed the Soling class."})
- p.Add(map[string]any{"chunk_id": "c2", "content_with_weight": "Their partner was crewing that year."})
- })
- cov := Coverage{Acts: []string{"crewing"}}
- if !CoverageActsMeetActor(kb, cov) {
- t.Fatal("want true: an act word is held and no actor was declared")
- }
- withActor := cov
- withActor.Actor = "Colin Beashel"
- if CoverageActsMeetActor(kb, withActor) {
- t.Fatal("want false: the two words sit in DIFFERENT passages, so no window could carry both")
- }
- kb.Admit(func(p *PoolAdmitter) {
- p.Add(map[string]any{"chunk_id": "c3", "content_with_weight": "Colin Beashel was crewing the Soling."})
- })
- if !CoverageActsMeetActor(kb, withActor) {
- t.Fatal("want true once ONE passage carries both")
- }
-}
-
-// TestCoverageOfKeepsTheDeclaredReadingNarrow pins the split the FRAMES regression forced: the value
-// may declare the direction to the node that runs LAST, and to nothing else.
-//
-// Set is what the planner said, and two readers depend on that being all it is — the record's set
-// block and a session's parentSet. Read from the value instead, a table whose slots happen to hold
-// people reads as an enumeration: measured (2026-09-17, FRAMES q759) a one-person question's record
-// lost its own draft answer and gained a member count to state.
-func TestCoverageOfKeepsTheDeclaredReadingNarrow(t *testing.T) {
- items := slots.Items(slots.Item{Value: "Lanee Butler", ChunkID: "c1", Quote: "Mistral (sailboard) | Lanee Butler"})
- valueOnly := NewState([]Variable{
- {ID: 0, Type: "person", Terms: []string{"斩"}, Value: &items},
- }, 0, nil)
- cov := CoverageOf(valueOnly)
- if cov.Set || cov.Ok() {
- t.Fatalf("set=%v ok=%v, want the DECLARED reading still false for a person slot", cov.Set, cov.Ok())
- }
- if !cov.ItemsInValue() {
- t.Fatal("itemsInValue = false, want the value that holds items readable by the node with no plan-time word")
- }
- if cov.ItemKind != "person" {
- t.Fatalf("itemKind = %q, want the planner's word kept — the value only names the kind when the type did not", cov.ItemKind)
- }
- // A slot whose type names no kind still gets one from the value.
- nameless := NewState([]Variable{
- {ID: 0, Type: "count", Terms: []string{"斩"}, Value: &items},
- }, 0, nil)
- if cov := CoverageOf(nameless); cov.ItemKind != CoverageItemKindItems || !cov.ItemsInValue() {
- t.Fatalf("cov = %+v, want the value to name the kind when the type did not", cov)
- }
-
- // A declared shape is untouched: the planner's word still opens the gate.
- declared := NewState([]Variable{
- {ID: 0, Type: "list", Terms: []string{"斩"}},
- }, 0, nil)
- if cov := CoverageOf(declared); !cov.Ok() || cov.ItemsInValue() {
- t.Fatalf("cov = %+v, want a declared list slot to open the gate on the declaration alone", cov)
- }
-}
-
-// the branch fires on every operand: Truncated is set, every recall's passages are dropped,
-// and the rendered seed tells each session the direction came back empty. Nothing caught
-// that until this test, because the loop had no test of its own.
-func TestEnumerateCoverageKeepsWhatTheRecallReturned(t *testing.T) {
- deps, kb := newTestSearchDeps(&stubRetriever{chunks: []map[string]any{
- {"chunk_id": "c1", "content": "关羽手起刀落,斩孔秀于马下。", "doc_id": "d1"},
- }})
- cov := Coverage{Actor: "关羽|关公", Acts: []string{"斩"}, Set: true, ItemKind: "person"}
- if !cov.Ok() {
- t.Fatalf("the fixture must be an enumeration: %+v", cov)
- }
-
- set := EnumerateCoverage(context.Background(), deps, cov, kb)
- if set.Recalled == 0 {
- t.Fatalf("Recalled = 0 over %d operand(s): the recalled passage(s) were dropped", len(set.Operands))
- }
- if len(set.Windows) == 0 {
- t.Fatalf("Windows = 0: a passage naming the actor and the act word is a window (%+v)", set)
- }
- if set.Truncated {
- t.Error("Truncated = true: nothing here ran out of clock, and saying so seeds every session with (nothing)")
- }
- if !strings.Contains(set.Windows[0].Quote, "斩孔秀") {
- t.Errorf("window = %q, want the words around the act", set.Windows[0].Quote)
- }
- if !strings.Contains(set.Render(), "chunk_id=c1") {
- t.Errorf("seed = %q, want the window's chunk id (a member is citable only through it)", set.Render())
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/doc_fetch.go b/internal/rag/agentic-rag/runtime/doc_fetch.go
deleted file mode 100644
index 584831808e..0000000000
--- a/internal/rag/agentic-rag/runtime/doc_fetch.go
+++ /dev/null
@@ -1,220 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
-
- "ragflow/internal/rag/prompts"
- "ragflow/internal/tokenizer"
-)
-
-const (
- // docFetchPageSize is the 128-chunk page size.
- docFetchPageSize = 128
- // docFetchMaxChunks is the hard 10000-chunk cap.
- docFetchMaxChunks = 10000
- // docFetchFallbackTokens bounds the fetch when the caller supplies no model
- // window. Callers usually pass one, so this only guards a caller that does not.
- docFetchFallbackTokens = 8192
- // estimateCharsPerToken approximates the tokenizer when no model encoder is
- // loaded; estimateTokens falls back to it only when tokenizer.NumTokensFromString
- // returns 0 (encoder unavailable).
- estimateCharsPerToken = 4
-)
-
-// fetchFullDocument
-// (agentic_rag.py:fetch_full_document): read a document end-to-end in reading order, in pages,
-// stopping before the model window would overflow.
-//
-// Returns (chunks, docAggs). Both are nil when the document is unavailable —
-// unbound datasets, a document outside the session scope, or an empty read.
-func fetchFullDocument(ctx context.Context, deps SearchDeps, docID string, maxTokens int) ([]map[string]any, []map[string]any) {
- if deps.DocChunks == nil || docID == "" || len(deps.KbIDs) == 0 {
- _LOG.Printf("[Fetch full document] skipped (doc_id=%q, datasets=%d)", docID, len(deps.KbIDs))
- return nil, nil
- }
- // a session-wide document scope is authoritative.
- if len(deps.DocScope) > 0 && !containsStr(deps.DocScope, docID) {
- _LOG.Printf("[Fetch full document] doc_id %q is outside the session document scope", docID)
- return nil, nil
- }
- // never read a document that is not in the bound datasets.
- if belongs, verified := docInDatasets(ctx, deps, docID); verified && !belongs {
- _LOG.Printf("[Fetch full document] doc_id %q is not in any bound dataset — refusing to fetch", docID)
- return nil, nil
- }
-
- budget := maxTokens
- if budget <= 0 {
- budget = docFetchFallbackTokens
- }
-
- var chunks []map[string]any
- tokens := 0
- budgetHit := false
- // NOTE: the budget check breaks the OUTER loop, so a page
- // that overruns the window stops paging entirely. Kept as-is: the budget is
- // a hard stop, and continuing would only add chunks that get dropped.
- for offset := 0; offset < docFetchMaxChunks && !budgetHit; offset += docFetchPageSize {
- page, err := deps.DocChunks.DocChunks(ctx, DocChunksRequest{
- DocID: docID,
- DatasetIDs: deps.KbIDs,
- TenantID: deps.TenantID,
- Offset: offset,
- Limit: docFetchPageSize,
- })
- if err != nil {
- _LOG.Printf("[Fetch full document] page at offset %d failed: %v", offset, err)
- break
- }
- if len(page) == 0 {
- break
- }
- for _, ck := range page {
- n := estimateTokens(ChunkTextOf(ck))
- if tokens+n > budget {
- budgetHit = true
- break
- }
- tokens += n
- chunks = append(chunks, ck)
- }
- if len(page) < docFetchPageSize {
- break // document exhausted
- }
- }
- if len(chunks) == 0 {
- _LOG.Printf("[Fetch full document] no chunks for doc_id %q", docID)
- return nil, nil
- }
-
- docName := ""
- for _, c := range chunks {
- if t := DocTitleOf(c); t != "" {
- docName = t
- break
- }
- }
- aggs := []map[string]any{
- {"doc_name": docName, "doc_id": docID, "count": len(chunks)},
- }
- return chunks, aggs
-}
-
-// summarizeDocument
-// (agentic_rag.py:rag): load the whole document, fold it into the evidence the
-// citation rules refer to, and return the newly rendered blocks.
-func summarizeDocument(ctx context.Context, deps SearchDeps, docID string, maxTokens int) []string {
- chunks, aggs := fetchFullDocument(ctx, deps, docID, maxTokens)
- if len(chunks) == 0 {
- return nil
- }
- budget := maxTokens
- if budget <= 0 {
- budget = docFetchFallbackTokens
- }
-
- // the document becomes part of the evidence set, so its
- // [ID]s stay citable; only the blocks of the chunks read here are returned.
- //
- // The blocks are picked by the SOURCE CHUNK, not by the pre-merge chunk
- // count: KBPrompt skips a chunk with no content and stops when the token
- // budget is exhausted, so block N is not chunk N — a slice taken at the
- // pre-merge count can drop readable blocks, or point past the end and return
- // nil for a document that was read fine — and this is easy to hit, because Merge
- // deduplicates a document
- // chunk that is already pooled, so the pre-merge count can even equal the
- // post-merge count). Merge reports the pool position of every chunk this
- // fetch contributed — deduplicated ones included — which makes the mapping
- // exact; iterating it keeps the document's reading order.
- if deps.KB == nil {
- deps.KB = &Kbinfos{}
- }
- added := deps.KB.Merge(chunks, aggs)
- // Pool positions, not rendered positions, are the block ids — 0-based, like every
- // other evidence render. These are the tool's evidence, not the final answer's:
- // the compose re-renders the pool it wants to cite with its own numbering
- // (CiteChunkIDs) and the chat pipeline resolves the answer's markers against that
- // list. What the ids must do here is stay addressable, which a pool position does
- // whatever the render skipped.
- blocks, sources := prompts.KBPromptPoolIndexed(deps.KB.Chunks, budget)
- blockAt := make(map[int]string, len(sources))
- for i, src := range sources {
- if _, seen := blockAt[src]; !seen {
- blockAt[src] = blocks[i]
- }
- }
- // One block per POOL POSITION. `added` is positional per OCCURRENCE: a chunk
- // the reader served twice (offset paging over an unstable order) or a chunk
- // already pooled is reported at the same position again, so appending per
- // occurrence would hand the model the same block — and its tokens — twice.
- fresh := make([]string, 0, len(added))
- seenPositions := make(map[int]struct{}, len(added))
- for _, pos := range added {
- if _, seen := seenPositions[pos]; seen {
- continue
- }
- seenPositions[pos] = struct{}{}
- if block, ok := blockAt[pos]; ok {
- fresh = append(fresh, block)
- }
- }
- // Nothing of this document survived the budget: the pool is already at the
- // model window, so there is no block to hand back.
- if len(fresh) == 0 {
- return nil
- }
-
- // without do_refer the model is told not to cite, so the
- // rules must not be handed to it.
- if !deps.DoRefer {
- return fresh
- }
- // The blocks are numbered by pool position: were this path ever reached with
- // do_refer=true, the rules would need the same 0-based sentence the compose adds
- // (agentic_rag.zeroBasedEvidenceRule) — CitationPrompt itself cannot carry it,
- // the canvas renders hash ids.
- header := "# Citation rules\nApply the following rules VERBATIM to your final answer.\n\n" +
- prompts.CitationPrompt(deps.CiteRules) + "\n\n----\n\n"
- return append([]string{header}, fresh...)
-}
-
-// SummarizeDocument is the exported summarize_document tool used by the outer react loop
-// (rag_agent) as a non-terminal tool. It reads the whole document identified by docID into the
-// evidence set and returns the prompt blocks to feed back to the model. do_refer is taken from
-// deps.DoRefer (the run config decides whether the model may cite the freshly read document),
-// so callers control citation behavior.
-// Returns nil when the document has no readable chunks (deps.DocChunks unset
-// or the doc is unavailable).
-func SummarizeDocument(ctx context.Context, deps SearchDeps, docID string, maxTokens int) []string {
- if deps.DocChunks == nil {
- return nil
- }
- return summarizeDocument(ctx, deps, docID, maxTokens)
-}
-
-// estimateTokens returns the token count of s. It prefers the precise
-// tokenizer (tokenizer.NumTokensFromString); when no encoder is loaded that returns 0, so we
-// fall back to a character/estimateCharsPerToken approximation for a rough
-// budget, like the previous Go-only behavior.
-func estimateTokens(s string) int {
- if n := tokenizer.NumTokensFromString(s); n > 0 {
- return n
- }
- return (len([]rune(s)) + estimateCharsPerToken - 1) / estimateCharsPerToken
-}
diff --git a/internal/rag/agentic-rag/runtime/doc_fetch_test.go b/internal/rag/agentic-rag/runtime/doc_fetch_test.go
deleted file mode 100644
index 0d928d06f0..0000000000
--- a/internal/rag/agentic-rag/runtime/doc_fetch_test.go
+++ /dev/null
@@ -1,127 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "strings"
- "testing"
-)
-
-// docChunkStub serves one document's chunks in reading order, paged.
-type docChunkStub struct {
- chunks []map[string]any
-}
-
-func (d *docChunkStub) DocChunks(_ context.Context, req DocChunksRequest) ([]map[string]any, error) {
- if req.Offset >= len(d.chunks) {
- return nil, nil
- }
- end := req.Offset + req.Limit
- if end > len(d.chunks) {
- end = len(d.chunks)
- }
- return d.chunks[req.Offset:end], nil
-}
-
-func docFetchDeps(pool, docChunks []map[string]any) SearchDeps {
- return SearchDeps{
- KbIDs: []string{"kb-1"},
- TenantID: "tenant-1",
- KB: &Kbinfos{Chunks: pool},
- DocChunks: &docChunkStub{chunks: docChunks},
- }
-}
-
-// TestSummarizeDocumentReturnsBlocksForAlreadyPooledChunks covers the case the
-// pre-merge chunk count gets wrong on its own: Merge deduplicates a chunk that is
-// already pooled, so the pool does not grow and a slice taken at the pre-merge
-// count starts past the end of the rendered blocks — the tool then reported that
-// a document it had just read had no readable chunks.
-func TestSummarizeDocumentReturnsBlocksForAlreadyPooledChunks(t *testing.T) {
- pool := []map[string]any{{"chunk_id": "c1", "content": "Pooled body."}}
- docChunks := []map[string]any{{"chunk_id": "c1", "content": "Pooled body."}}
-
- got := summarizeDocument(context.Background(), docFetchDeps(pool, docChunks), "doc-1", 100000)
-
- if len(got) != 1 {
- t.Fatalf("blocks = %d (%v), want the document's block even though it was already pooled", len(got), got)
- }
- if !strings.Contains(got[0], "Pooled body.") {
- t.Errorf("block = %q, want the document's content", got[0])
- }
-}
-
-// TestSummarizeDocumentSelectsBlocksBySourceChunk covers a skipped chunk: KBPrompt
-// renders no block for a chunk without content, so every later block is one
-// position earlier than its chunk. Slicing by the chunk count dropped the first
-// document block.
-func TestSummarizeDocumentSelectsBlocksBySourceChunk(t *testing.T) {
- pool := []map[string]any{
- {"chunk_id": "empty", "content": ""}, // renders no block
- }
- docChunks := []map[string]any{
- {"chunk_id": "c1", "content": "First document body."},
- {"chunk_id": "c2", "content": "Second document body."},
- }
-
- got := summarizeDocument(context.Background(), docFetchDeps(pool, docChunks), "doc-1", 100000)
-
- if len(got) != 2 {
- t.Fatalf("blocks = %d (%v), want both document chunks", len(got), got)
- }
- if !strings.Contains(got[0], "First document body.") || !strings.Contains(got[1], "Second document body.") {
- t.Errorf("blocks = %v, want the document's chunks in reading order", got)
- }
-}
-
-// TestSummarizeDocumentReturnsNothingWhenNoChunkRenders keeps the guard: when the
-// fetched chunks contribute no renderable block there is nothing to hand back,
-// and in particular no unrelated pooled block.
-func TestSummarizeDocumentReturnsNothingWhenNoChunkRenders(t *testing.T) {
- pool := []map[string]any{{"chunk_id": "p0", "content": "Pooled evidence."}}
- docChunks := []map[string]any{{"chunk_id": "c1", "content": " "}}
-
- got := summarizeDocument(context.Background(), docFetchDeps(pool, docChunks), "doc-1", 100000)
-
- if len(got) != 0 {
- t.Fatalf("blocks = %v, want nothing for a document with no renderable chunk", got)
- }
-}
-
-// TestSummarizeDocumentDeduplicatesRepeatedSourcePositions covers the paging
-// hazard: offset paging over an unstable order can serve the same chunk on two
-// pages, and Merge reports a position per OCCURRENCE, not per distinct chunk. The
-// block of that one pooled chunk was therefore appended twice — paying its tokens
-// into the prompt twice, on top of a budget that was already spent.
-func TestSummarizeDocumentDeduplicatesRepeatedSourcePositions(t *testing.T) {
- pool := []map[string]any{{"chunk_id": "p0", "content": "Pooled evidence."}}
- docChunks := []map[string]any{
- {"chunk_id": "c1", "content": "Document body."},
- {"chunk_id": "c1", "content": "Document body."}, // same chunk served twice
- {"chunk_id": "c2", "content": "Second body."},
- }
-
- got := summarizeDocument(context.Background(), docFetchDeps(pool, docChunks), "doc-1", 100000)
-
- if len(got) != 2 {
- t.Fatalf("blocks = %d (%v), want one block per distinct fetched chunk", len(got), got)
- }
- if !strings.Contains(got[0], "Document body.") || !strings.Contains(got[1], "Second body.") {
- t.Errorf("blocks = %v, want the document's chunks in reading order, each once", got)
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/grep_sed_narrow.go b/internal/rag/agentic-rag/runtime/grep_sed_narrow.go
deleted file mode 100644
index 5bdeaf2fa2..0000000000
--- a/internal/rag/agentic-rag/runtime/grep_sed_narrow.go
+++ /dev/null
@@ -1,1031 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "fmt"
- "log"
- "regexp"
- "strings"
- "unicode"
- "unicode/utf8"
-)
-
-// In-memory grep+sed narrowing engine (term-driven, zero extra LLM rounds).
-//
-// It mirrors the Claude Code / Codex ``grep`` + ``sed`` workflow over chunks held in
-// memory: terms already produced by the main-analysis LLM (entities, numbers, key
-// phrases) become word-boundary regexes for locating (grep), and string transforms
-// narrow the text (sed), dropping boilerplate instead of crude head-truncation.
-//
-// Fallback chain (never drops the answer):
-//
-// narrow_by_terms (grep locate + sed transform)
-// -> no hits / no terms
-// -> _narrow_by_keywords (keyword sentence-level, zero LLM)
-// -> original chunks returned as-is (caller's head-truncation as the final
-// safety valve)
-//
-// Safety: regexp + pure string ops only, no eval; term count and context are capped.
-
-// Cost / safety caps. Mirrors grep_sed_narrow.py's module constants.
-const (
- // maxGrepTerms caps the terms compiled.
- maxGrepTerms = 16
- // maxContext is the +/- window of lines kept around a hit.
- maxContext = 2
- // defaultOutCharsPerChunk caps output per chunk.
- defaultOutCharsPerChunk = 1200
- // defaultOutTotalChars caps the total narrowed output.
- defaultOutTotalChars = 16000
- // headFallbackChars is the head kept per chunk when there is no match.
- headFallbackChars = 400
- // contextCharBudget is the absolute per-side char budget during context expansion.
- contextCharBudget = 600
- // minNarrowChars: chunks at or below this length are NOT narrowed — they are already
- // 1-2 lines, and answers often live in short chunks.
- minNarrowChars = 200
-)
-
-var (
- // reTermEdgePunct strips leading/trailing punctuation from a term.
- reTermEdgePunct = regexp.MustCompile(`^[\s.,:;!?'"()\[\]{}]+|[\s.,:;!?'"()\[\]{}]+$`)
- // reCJKTerm detects CJK / kana / hangul, which must NOT be wrapped in \b —
- // a word boundary never matches between CJK characters.
- reCJKTerm = regexp.MustCompile(`[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}]`)
-)
-
-// EscapeTerm escapes a plain grep term into a safe, word-boundary regex fragment.
-// Numbers/entities are matched literally; 3+ char terms with alphanumeric edges get \b;
-// CJK terms stay bare (never wrapped in \b).
-func EscapeTerm(term string) string {
- t := strings.TrimSpace(term)
- if t == "" {
- return ""
- }
- t = reTermEdgePunct.ReplaceAllString(t, "")
- if t == "" {
- return ""
- }
- escaped := regexp.QuoteMeta(t)
- if reCJKTerm.MatchString(t) {
- return escaped
- }
- rs := []rune(t)
- if len(rs) >= 3 && isAlphaNumRune(rs[0]) && isAlphaNumRune(rs[len(rs)-1]) {
- return `\b` + escaped + `\b`
- }
- return escaped
-}
-
-func isAlphaNumRune(r rune) bool {
- return r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9'
-}
-
-// TermsToPatterns turns grep terms into a list of compiled regexes (one per
-// term), capped at maxGrepTerms.
-func TermsToPatterns(terms []string) []*regexp.Regexp {
- out := make([]*regexp.Regexp, 0, len(terms))
- for _, term := range terms {
- if len(out) >= maxGrepTerms {
- break
- }
- frag := EscapeTerm(term)
- if frag == "" {
- continue
- }
- p, err := regexp.Compile("(?i)" + frag)
- if err != nil {
- continue
- }
- out = append(out, p)
- }
- return out
-}
-
-// GrepTermsMax caps the terms extracted from a query.
-const GrepTermsMax = 10
-
-// cjkPhraseRunes is the length at which a CJK token stops being a term and
-// becomes a clause: see GrepTermsFromQuery. Four is the longest Chinese proper
-// name that is still read as one token (成吉思汗), so a longer run is prose.
-const cjkPhraseRunes = 4
-
-// GrepOutCharsPerChunk is the grep narrow's per-chunk output cap.
-const GrepOutCharsPerChunk = 700
-
-// GrepOutTotalChars is the grep narrow's total output cap.
-const GrepOutTotalChars = 8000
-
-// GrepTermsFromQuery: bare alnum words of
-// length>=2, deduped (order-preserving) and capped — and extends it to CJK.
-//
-// An ALNUM-ONLY tokenizer yields no term at all for a Chinese query, and the grep leg's
-// own guard returns whole chunks instead of located windows: the locate line is logged, a
-// narrowed line never is, and every hit is a whole chunk. That is expensive (a name lives
-// in one clause of a chunk) and it is what makes an enumeration unreadable to the model,
-// which cannot see WHICH clause a name sits in.
-//
-// The derivation adds CJK support, with no regex:
-//
-// - an ALTERNATION is the caller's own term list — it is how the count protocol
-// tells a session to batch its probes — so it is split on "|" and its pieces are
-// kept at any length (a predicate is a term too);
-// - otherwise the query is split on whitespace and punctuation, Latin tokens
-// keep the two-character floor, and CJK tokens keep a two-CJK-rune
-// floor (a lone Chinese character is a particle, not a term);
-// - an unbroken CJK clause (a question written without separators) yields no
-// token either way, so its two-rune windows are used, left to right: the
-// names in the clause still locate, and a window that occurs nowhere costs
-// one failed lookup inside the narrowing pass and nothing else.
-func GrepTermsFromQuery(query string) []string {
- return grepTermsFromQuery(query, true)
-}
-
-// probeItemsOf returns the terms a call proposed AS ITEMS — a string the model
-// wrote to ask about one individual.
-//
-// Two shapes qualify: the pieces of a whitespace-separated batch, and a query that
-// IS one word. Everything else is prose the model wrote to ask a
-// question, whose words belong to the question rather than to a member list; and
-// the two-rune windows an unbroken clause decomposes into are our guesses, not the
-// caller's words at all (see GrepWordsFromQuery).
-//
-// The distinction matters at the reach ledger. The ledger is read back as the
-// session's to-do list — "you probed this, it came back with a passage, it is in
-// no slot" — and a to-do list built from a question's words tells the model
-// nothing: its entries are the question's own words, none of them the members the
-// sessions are missing, while the window fragments cost a retrieval each.
-func probeItemsOf(queries []string) map[string]bool {
- out := make(map[string]bool)
- for _, q := range queries {
- q = strings.TrimSpace(q)
- if q == "" {
- continue
- }
- items := GrepWordsFromQuery(q)
- if !callerBatch(q) && len(items) != 1 {
- continue
- }
- for _, it := range items {
- if it = strings.TrimSpace(it); it != "" {
- out[strings.ToLower(it)] = true
- }
- }
- }
- return out
-}
-
-// GrepWordsFromQuery is GrepTermsFromQuery restricted to the caller's OWN words:
-// the pieces of an alternation, and the tokens separated by whitespace or
-// punctuation. It has NO CJK-window fallback.
-//
-// The distinction matters because the two answers are used for different jobs.
-// The windows exist to LOCATE a term inside an unbroken CJK clause — the thing
-// grep is for, and a failed lookup there costs nothing. As terms to SEARCH FOR,
-// they are our guesses rather than the caller's words, and each one would spend a
-// retrieval of its own — retrievals spent on fragments nobody asked about, while the
-// names the question is missing stay out of the list entirely.
-//
-// So a caller that READS these (the named-term seat pass) uses the words; a
-// caller that LOCATES with them uses the terms.
-func GrepWordsFromQuery(query string) []string {
- return grepTermsFromQuery(query, false)
-}
-
-func grepTermsFromQuery(query string, windows bool) []string {
- q := strings.TrimSpace(query)
- if q == "" {
- return nil
- }
- terms := make([]string, 0, GrepTermsMax)
- seen := make(map[string]struct{}, GrepTermsMax)
- add := func(t string) (full bool) {
- if t == "" {
- return len(terms) >= GrepTermsMax
- }
- low := strings.ToLower(t)
- if _, dup := seen[low]; dup {
- return len(terms) >= GrepTermsMax
- }
- seen[low] = struct{}{}
- terms = append(terms, t)
- return len(terms) >= GrepTermsMax
- }
-
- if strings.Contains(q, "|") {
- for _, part := range strings.Split(q, "|") {
- if add(trimTermEdges(part)) {
- break
- }
- }
- return terms
- }
-
- for _, token := range strings.FieldsFunc(q, isTermSeparator) {
- token = trimTermEdges(token)
- if token == "" {
- continue
- }
- if hasCJK(token) {
- runes := utf8.RuneCountInString(token)
- if runes < 2 {
- continue
- }
- // A CJK token past the name boundary is a CLAUSE, not a term: its
- // literal form rarely occurs in the corpus, so it locates nothing.
- // The two jobs part company here:
- //
- // LOCATING (windows) — decompose the clause into the two-rune
- // windows a name can actually be found in;
- // the caller's WORDS (GrepWordsFromQuery) — drop it. A clause names
- // no individual, and its own phrase search already covers it.
- if !windows {
- if runes > cjkPhraseRunes {
- continue
- }
- } else if runes >= cjkPhraseRunes {
- full := len(terms) >= GrepTermsMax
- for _, window := range cjkWindowsOf(token, GrepTermsMax) {
- if add(window) {
- full = true
- break
- }
- }
- if full {
- break
- }
- continue
- }
- } else if len(token) < 2 {
- continue
- }
- if add(token) {
- break
- }
- }
- if len(terms) > 0 || !windows {
- return terms
- }
- return cjkWindowsOf(q, GrepTermsMax)
-}
-
-// trimTermEdges strips the punctuation a token can carry (the regex equivalent is
-// reTermEdgePunct).
-func trimTermEdges(t string) string {
- return strings.Trim(t, " \t\r\n.,:;!?'\"()[]{}<>“”‘’()【】《》「」〈〉—…·_-")
-}
-
-// isTermSeparator reports whether a rune separates terms. "." and "," do;
-// "-", "_" and "." INSIDE a Latin token do not ([A-Za-z0-9_.-] is token material), so they
-// are not listed here.
-func isTermSeparator(r rune) bool {
- switch r {
- case ' ', '\t', '\r', '\n', '\v', '\f',
- ',', ';', ':', '!', '?', '/', '\\', '(', ')', '[', ']', '{', '}', '<', '>', '"', '\'', '|',
- ',', '。', '、', ';', ':', '!', '?', '(', ')', '【', '】', '《', '》', '「', '」', '〈', '〉',
- '“', '”', '‘', '’', '—', '…', '·':
- return true
- }
- return false
-}
-
-// hasCJK reports whether a token contains a CJK / kana / hangul rune — the same
-// classes EscapeTerm refuses to wrap in \b, because no word boundary exists
-// between them.
-func hasCJK(s string) bool {
- for _, r := range s {
- if isCJKRune(r) {
- return true
- }
- }
- return false
-}
-
-// isCJKRune reports whether a rune belongs to a script whose text has no word
-// boundaries.
-func isCJKRune(r rune) bool {
- return unicode.Is(unicode.Han, r) || unicode.Is(unicode.Hiragana, r) ||
- unicode.Is(unicode.Katakana, r) || unicode.Is(unicode.Hangul, r)
-}
-
-// cjkWindowsOf returns the two-rune windows of a string's CJK runs, in order,
-// deduped and capped. It is the last-resort term derivation for a query that is
-// one unbroken clause (see GrepTermsFromQuery).
-func cjkWindowsOf(query string, limit int) []string {
- out := make([]string, 0, limit)
- run := make([]rune, 0, 16)
- flush := func() {
- if len(run) >= 2 {
- for i := 0; i+2 <= len(run) && len(out) < limit; i++ {
- window := string(run[i : i+2])
- dup := false
- for _, o := range out {
- if o == window {
- dup = true
- break
- }
- }
- if !dup {
- out = append(out, window)
- }
- }
- }
- run = run[:0]
- }
- for _, r := range query {
- if isCJKRune(r) {
- run = append(run, r)
- continue
- }
- flush()
- }
- flush()
- if len(out) == 0 {
- return nil
- }
- return out
-}
-
-// lineSpans returns line (start,end) spans, boundaries at "\n" (grep semantics): line
-// boundaries are exact (unlike lossy sentence splitting); start of line i is after the
-// i-th "\n".
-func lineSpans(content string) [][2]int {
- var spans [][2]int
- start := 0
- for i := 0; i < len(content); i++ {
- if content[i] == '\n' {
- spans = append(spans, [2]int{start, i})
- start = i + 1
- }
- }
- if start <= len(content) {
- spans = append(spans, [2]int{start, len(content)})
- }
- if len(spans) == 0 {
- spans = [][2]int{{0, len(content)}}
- }
- return spans
-}
-
-// NarrowContext is the +/- line-context window used by term-grep.
-type NarrowContext struct {
- Before int
- After int
-}
-
-// execOnText runs term-grep + line-context expansion against one chunk's text.
-// Runs:
-//
-// - locate matches exactly (match.start()/end());
-// - merge overlapping/adjacent ranges, expand to whole lines, add before/after
-// context lines, with a per-side character-budget clamp;
-// - on no hit, keep fact-dense sentences (never drop everything), else head.
-//
-// Returns (narrowed, matched).
-func execOnText(content string, patterns []*regexp.Regexp, before, after, outCharsPerChunk int) (string, bool) {
- if content == "" {
- return "", false
- }
-
- // Step 1: locate matches (exact positions from the regex engine).
- var hitRanges [][2]int
- for _, p := range patterns {
- locs := p.FindAllStringIndex(content, -1)
- for _, loc := range locs {
- hitRanges = append(hitRanges, [2]int{loc[0], loc[1]})
- }
- }
- if len(hitRanges) == 0 {
- // Keep fact-dense sentences to avoid dropping numbers/entities; with none, the raw
- // head is kept.
- var kept []string
- for _, s := range SplitSentences(content) {
- if IsFactDenseSentence(s) {
- kept = append(kept, s)
- }
- }
- narrowed := strings.TrimSpace(strings.Join(kept, ""))
- if narrowed != "" {
- return truncHead(narrowed, headFallbackChars*4), false
- }
- return truncHead(content, headFallbackChars), false
- }
-
- // Step 2: merge overlapping/adjacent matches.
- sort2DRanges(hitRanges)
- merged := make([][2]int, 0, len(hitRanges))
- for _, r := range hitRanges {
- if n := len(merged); n > 0 && r[0] <= merged[n-1][1] {
- if r[1] > merged[n-1][1] {
- merged[n-1][1] = r[1]
- }
- } else {
- merged = append(merged, r)
- }
- }
-
- lines := lineSpans(content)
- var expanded [][2]int
- for _, r := range merged {
- lo, hi := 0, 0
- for i, ls := range lines {
- if r[0] >= ls[0] && r[0] < ls[1] {
- lo = i
- }
- if r[1] > ls[0] && r[1] <= ls[1] {
- hi = i
- }
- }
- lo = max(0, lo-before)
- hi = min(len(lines)-1, hi+after)
- fragS, fragE := lines[lo][0], lines[hi][1]
- // A line window is unusable when the LINE IS THE WHOLE CHUNK, and that is
- // the CJK case by default: a Chinese chunk carries no newlines, so its
- // one line is the entire passage and the window hands back all of it. The
- // length test catches the same thing for a chunk that does have a few
- // very long lines. Either way, fall back to the SENTENCES the hit sits
- // in — same per-side budget, cut at 。!?; instead of mid-clause.
- if len(lines) <= 1 || (fragE-fragS > contextCharBudget*2 && (fragE-fragS) > (r[1]-r[0])) {
- if window := sentenceWindow(content, r, contextCharBudget); window[1]-window[0] < fragE-fragS {
- fragS, fragE = window[0], window[1]
- }
- }
- expanded = append(expanded, [2]int{fragS, fragE})
- }
-
- // Step 3: dedupe, join, truncate.
- seen := map[string]bool{}
- var outParts []string
- for _, e := range expanded {
- p := strings.TrimSpace(content[e[0]:e[1]])
- if p == "" {
- continue
- }
- key := truncHead(p, 200)
- if seen[key] {
- continue
- }
- seen[key] = true
- outParts = append(outParts, p)
- }
- narrowed := strings.TrimSpace(strings.Join(outParts, "\n\n"))
- if charLen(narrowed) > outCharsPerChunk {
- narrowed = truncHead(narrowed, outCharsPerChunk)
- }
- if narrowed == "" {
- return truncHead(content, headFallbackChars), true
- }
- return narrowed, true
-}
-
-// sentenceWindow expands a hit to the sentence it sits in, bounded by budget
-// bytes on each side.
-//
-// It exists for text with no line structure. A CJK chunk is one long line, so
-// the line-window path returns the whole chunk for a hit that needs one clause of
-// it; cutting at sentence punctuation instead keeps the window bounded AND keeps
-// the clause intact, which is what makes the result readable to the model (a
-// name is read off the clause around the verb). Terminators are the sentence
-// punctuation of both scripts; no regex is involved.
-func sentenceWindow(content string, hit [2]int, budget int) [2]int {
- lo := hit[0]
- for lo > 0 && hit[0]-lo < budget {
- r, size := utf8.DecodeLastRuneInString(content[:lo])
- if isSentenceTerminator(r) {
- break
- }
- lo -= size
- }
- hi := hit[1]
- for hi < len(content) && hi-hit[1] < budget {
- r, size := utf8.DecodeRuneInString(content[hi:])
- hi += size
- if isSentenceTerminator(r) {
- break
- }
- }
- return [2]int{lo, hi}
-}
-
-// isSentenceTerminator reports whether a rune ends a sentence in either script.
-func isSentenceTerminator(r rune) bool {
- switch r {
- case '。', '!', '?', ';', '\n', '!', '?', ';', '.':
- return true
- }
- return false
-}
-
-// NarrowStats carries the narrowing accounting.
-type NarrowStats struct {
- ChunksIn int
- ChunksKpt int
- CharsIn int
- CharsOut int
- Matched bool
- UsedTerms int
-}
-
-// NarrowResult is the outcome of NarrowByTerms / GrepSedNarrow.
-type NarrowResult struct {
- Kept []map[string]any
- Stats NarrowStats
-}
-
-// grepPatternSyntax are the constructs that make a query a PATTERN rather than a
-// phrase: an alternation, an ordering constraint, or a word boundary.
-//
-// They are read from the model's own string and mean what they mean in every
-// regex dialect there is, so nothing has to be compiled, escaped or translated —
-// and none of them occurs in ordinary prose, which is what keeps the guard safe:
-// a plain question is not a pattern and takes the term-locate path it always took.
-var grepPatternSyntax = []string{"|", ".*", ".+", `\b`}
-
-// grepPatternOf compiles the query as a pattern, or returns nil when the query
-// carries no pattern syntax.
-func grepPatternOf(query string) *regexp.Regexp {
- q := strings.TrimSpace(query)
- if q == "" {
- return nil
- }
- patterned := false
- for _, tok := range grepPatternSyntax {
- if strings.Contains(q, tok) {
- patterned = true
- break
- }
- }
- if !patterned {
- return nil
- }
- re, err := regexp.Compile("(?i)" + q)
- if err != nil {
- return nil
- }
- return re
-}
-
-// grepPatternOperators are the characters a pattern uses as operators. They are
-// removed before the operands are derived, because a keyword leg cannot search
-// for syntax.
-var grepPatternOperators = []string{".*", ".+", "|", ".", "*", "+", "?", "^", "$", `\b`, `\d`, `\w`, `\s`, "(", ")", "[", "]", "{", "}", `\`}
-
-// GrepPatternOperands returns the literal runs a PATTERN asks the corpus for.
-//
-// A pattern's operands are what a keyword leg can search: "华雄|荀正" names two,
-// "关公.*斩" names two, and the operators between them are not terms. They cannot
-// go through the phrase path (GrepTermsFromQuery), which reads an unbroken CJK run
-// as a clause and decomposes it into windows — on "关公.*斩" that yielded 关公 alone
-// and dropped the 斩, so recall never asked about half the pattern.
-//
-// Single CJK runes are KEPT here, unlike the general two-rune floor: inside a
-// pattern the caller wrote that literal deliberately, so it is not the stray
-// particle the floor exists to drop.
-func GrepPatternOperands(query string) []string {
- q := strings.TrimSpace(query)
- if q == "" {
- return nil
- }
- for _, op := range grepPatternOperators {
- q = strings.ReplaceAll(q, op, " ")
- }
- var out []string
- seen := make(map[string]bool, GrepTermsMax)
- for _, tok := range strings.Fields(q) {
- tok = trimTermEdges(tok)
- if tok == "" {
- continue
- }
- low := strings.ToLower(tok)
- if seen[low] {
- continue
- }
- seen[low] = true
- out = append(out, tok)
- if len(out) >= GrepTermsMax {
- break
- }
- }
- return out
-}
-
-// matchGrepPattern is the grep: it runs the PATTERN over candidate content and
-// keeps the candidates that MATCH, each narrowed to the clause its match sits in.
-//
-// This is the half of grep_search that a term-locate pass cannot be: a candidate
-// the pattern does not match is not evidence, whatever its retrieval score, and
-// the pattern keeps its own semantics — `A|B` accepts either alternative, while
-// `A.*B` requires the written order, which is how one asks for the WAY a thing
-// was done instead of for its name.
-//
-// The haystack is the candidate set the keyword leg returned (see
-// retrieveGrepCandidates), so the pattern is applied to what the engine could
-// reach, exactly: a name the candidates carry is found even if the phrase's
-// ranking would have buried it, and no pattern syntax ever reaches the engine —
-// which is why `|` and `.*` need no escaping anywhere on this path.
-//
-// The window is the sentence around the FIRST match, so the model reads the
-// clause that carries the term instead of a whole chunk.
-//
-// Returns the matching candidates plus how many matched; the caller keeps the raw
-// candidates when nothing matched, so evidence is never dropped.
-func matchGrepPattern(chunks []map[string]any, re *regexp.Regexp, budget, maxOutTotalChars int) ([]map[string]any, int) {
- if re == nil || len(chunks) == 0 {
- return nil, 0
- }
- if budget <= 0 {
- budget = contextCharBudget
- }
- var kept []map[string]any
- used := 0
- for _, c := range chunks {
- text := ChunkTextOf(c)
- if text == "" {
- continue
- }
- span := re.FindStringIndex(text)
- if span == nil {
- continue
- }
- win := sentenceWindow(text, [2]int{span[0], span[1]}, budget)
- fragment := text[win[0]:win[1]]
- if maxOutTotalChars > 0 {
- n := utf8.RuneCountInString(fragment)
- if used+n > maxOutTotalChars && len(kept) > 0 {
- break
- }
- used += n
- }
- kept = append(kept, withNarrowedText(cloneMap(c), fragment))
- }
- return kept, len(kept)
-}
-
-// logGrepReach prints what ONE grep reached, term by term, and what nothing
-// reached — the fact the model cannot read off the passages themselves.
-//
-// Every name in a batch looks the same whether the search found it or never
-// looked, and the wording matters: "not reached by THIS query" is not "absent
-// from the corpus". A model that reads a miss as absence stops enumerating, which
-// is the failure this line exists to prevent.
-func logGrepReach(logger *log.Logger, query string, candidates []map[string]any, terms []string) {
- if logger == nil {
- return
- }
- if body := reachBody(query, candidates, terms); body != "" {
- logger.Printf("[Grep search] %s", body)
- }
-}
-
-// GrepReachLine is reachBody, prefixed for the MODEL.
-//
-// The reach report was log-only, and that was the last piece of the loop missing:
-// a batch of names ("华雄|颜良|蔡阳") came back as passages, every name looking the
-// same whether the query reached it or never looked — so the loop could not do
-// what a search-driven loop does with it, act on WHICH alternative came back
-// empty. The engine, the pattern matcher and the per-term accounting already
-// existed; only the reader was missing.
-func GrepReachLine(query string, candidates []map[string]any, terms []string) string {
- body := reachBody(query, candidates, terms)
- if body == "" {
- return ""
- }
- return "[reach] " + body
-}
-
-// reachBody renders what ONE query reached, term by term, and what nothing
-// reached. Empty when there are no terms to report on.
-func reachBody(query string, candidates []map[string]any, terms []string) string {
- if len(terms) == 0 {
- return ""
- }
- located, counts, absent := termReach(candidates, terms)
- parts := make([]string, 0, len(located))
- for i, t := range located {
- parts = append(parts, fmt.Sprintf("%s(%d)", t, counts[i]))
- }
- line := fmt.Sprintf("%d candidate(s) for %q carry: %s", len(candidates), trunc(query, 60), strings.Join(parts, " "))
- if len(absent) > 0 {
- line += fmt.Sprintf(" | NOT reached by this query (not necessarily absent from the corpus): %s", strings.Join(absent, " "))
- }
- return line
-}
-
-// ReachTermsOf returns the terms a query's reach is reported over: the operands
-// for a PATTERN (its operands are what a keyword leg searched for), the extracted
-// terms otherwise. It mirrors the choice GrepSearch makes, so the line the model
-// reads and the line the log carries describe the same search.
-func ReachTermsOf(query string) []string {
- if grepPatternOf(query) != nil {
- return GrepPatternOperands(query)
- }
- return GrepTermsFromQuery(query)
-}
-
-// NarrowByTerms narrows retrieval chunks by locating grep terms:
-//
-// - terms are plain strings (entities / numbers / key phrases);
-// - no usable terms -> keyword narrowing (zero LLM);
-// - primary terms hit nothing -> fallbackTerms tried once mechanically;
-// - still no hit -> narrowing abandoned, originals returned (matched=false);
-// - when matched, a total-length budget is distributed across chunks.
-//
-// Table exemption: chunks that look like tables (HTML <table>/<tr> markup, or >=3 pipe
-// rows — see IsTableChunk) are NEVER narrowed, by any caller of this engine, and are
-// exempt from the per-chunk/total char budget. Two reasons, both measured: the term
-// window either cuts the <table> opening tag, and then the downstream table view
-// refuses the fragment ("<table" not present) so the model is handed a partial
-// raw-HTML row dump; or it keeps the tag and silently drops the remaining rows, and row
-// order/coverage is exactly what decides table answers (a rank row can sit at 62% of a
-// 14.7K-char table). Tables come back VERBATIM.
-//
-// Never raises.
-func NarrowByTerms(chunks []map[string]any, terms []string, fallbackTerms []string, keywords string, context NarrowContext, maxOutCharsPerChunk, maxOutTotalChars int) NarrowResult {
- before := clampInt(context.Before, 0, maxContext)
- after := clampInt(context.After, 0, maxContext)
-
- patterns := TermsToPatterns(terms)
- stats := NarrowStats{
- ChunksIn: len(chunks),
- UsedTerms: len(patterns),
- }
- for _, c := range chunks {
- stats.CharsIn += len(ChunkTextOf(c))
- }
- if len(chunks) == 0 {
- return NarrowResult{Kept: nil, Stats: stats}
- }
- if len(patterns) == 0 {
- narrowed := NarrowWithFallbackKeyword(chunks, keywords)
- stats.ChunksKpt = len(narrowed)
- for _, c := range narrowed {
- stats.CharsOut += len(ChunkTextOf(c))
- }
- return NarrowResult{Kept: narrowed, Stats: stats}
- }
-
- // Whole-table exemption, computed once and reused by the char-budget pass below
- // (that pass preserves order and length, so the two lists stay aligned).
- // false = "this chunk was NOT narrowed" -> returned verbatim.
- tableFlags := make([]bool, len(chunks))
- for i, c := range chunks {
- tableFlags[i] = IsTableChunk(c)
- }
-
- run := func(active []*regexp.Regexp) ([]map[string]any, []bool, int) {
- var kept []map[string]any
- var flags []bool
- matched := 0
- for i, c := range chunks {
- raw := ChunkTextOf(c)
- if tableFlags[i] || charLen(raw) <= minNarrowChars {
- // Table or short chunk: kept whole. The flag is false for a table so the
- // char-budget pass below recognises it; a short chunk is flagged as
- // matched so it still participates in the total-budget distribution. The
- // text is unchanged and "highlight" is popped.
- kept = append(kept, withNarrowedText(c, raw))
- flags = append(flags, !tableFlags[i])
- continue
- }
- text, ok := execOnText(raw, active, before, after, maxOutCharsPerChunk)
- if ok {
- matched++
- kept = append(kept, withNarrowedText(c, text))
- } else {
- // No match: keep the original chunk untouched (full text and any
- // "highlight" preserved.
- kept = append(kept, cloneMap(c))
- }
- flags = append(flags, ok)
- }
- return kept, flags, matched
- }
-
- kept, matchedFlags, _ := run(patterns)
-
- // Gentle retry (no extra LLM): primary terms hit nothing -> fallback terms once.
- if len(fallbackTerms) > 0 && !anyBool(matchedFlags) {
- if fb := TermsToPatterns(fallbackTerms); len(fb) > 0 {
- kept, matchedFlags, _ = run(fb)
- if len(fb) > stats.UsedTerms {
- stats.UsedTerms = len(fb)
- }
- }
- }
-
- // Only apply the total-length cap when the grep actually matched: on a no-match
- // the chunks are returned untouched so the caller's compaction decides.
- if anyBool(matchedFlags) {
- totalOut := 0
- for _, c := range kept {
- totalOut += charLen(ChunkTextOf(c))
- }
- if totalOut > maxOutTotalChars {
- perChunk := max(200, min(maxOutCharsPerChunk, maxOutTotalChars/max(1, len(kept))))
- acc := 0
- var trimmed []map[string]any
- for i, c := range kept {
- if tableFlags[i] {
- // Tables are indivisible and exempt from the char budget: a head
- // slice keeps the header and the first rows and drops the answer row,
- // and the dropped rows carry no marker, so the model reads a
- // truncated table as a complete one. The budget therefore bounds
- // PROSE only; a large table can push the narrowed prose set past
- // maxOutTotalChars, which is the intended trade
- // (readability/correctness > char cap).
- trimmed = append(trimmed, c)
- continue
- }
- t := ChunkTextOf(c)
- room := maxOutTotalChars - acc
- if room <= 0 {
- break
- }
- take := min(charLen(t), min(perChunk, room))
- if take <= 0 {
- break
- }
- if take < charLen(t) {
- c = withNarrowedText(cloneMap(c), truncHead(t, take))
- }
- trimmed = append(trimmed, c)
- acc += take
- }
- kept = trimmed
- }
- }
-
- stats.ChunksKpt = len(kept)
- for _, c := range kept {
- stats.CharsOut += len(ChunkTextOf(c))
- }
- stats.Matched = anyBool(matchedFlags)
- logGrepSed(stats)
- return NarrowResult{Kept: kept, Stats: stats}
-}
-
-// NarrowWithFallbackKeyword applies keyword narrowing (zero LLM), returning the
-// originals when keyword narrowing yields nothing.
-func NarrowWithFallbackKeyword(chunks []map[string]any, keywords string) []map[string]any {
- if narrowed := NarrowByKeywords(chunks, keywords); len(narrowed) > 0 {
- return narrowed
- }
- return chunks
-}
-
-// fallbackStopwords
-var fallbackStopwords = map[string]bool{
- "what": true, "which": true, "who": true, "where": true, "when": true, "how": true,
- "the": true, "a": true, "an": true, "of": true, "in": true, "on": true,
- "for": true, "to": true, "and": true, "or": true, "with": true, "is": true,
- "are": true, "was": true, "were": true, "list": true, "name": true, "give": true,
- "find": true, "tell": true, "me": true, "about": true, "from": true, "that": true,
- "this": true, "it": true, "its": true, "their": true, "they": true, "have": true,
- "has": true, "do": true, "does": true, "did": true, "based": true, "per": true,
- "according": true, "not": true,
-}
-
-// SplitFallbackTerms splits free text into fallback grep terms (zero LLM),
-// splitting on sentence/comma boundaries, dropping short/stopword tokens, keeping numbers
-// and multi-word phrases.
-func SplitFallbackTerms(texts ...string) []string {
- var terms []string
- seen := map[string]bool{}
- for _, v := range texts {
- for _, part := range reFallbackSplit.Split(strings.TrimSpace(v), -1) {
- part = strings.TrimSpace(strings.Trim(part, "'\"()[]{}"))
- if len(part) < 3 {
- continue
- }
- if fallbackStopwords[strings.ToLower(part)] {
- continue
- }
- if seen[part] {
- continue
- }
- seen[part] = true
- terms = append(terms, part)
- if len(terms) >= maxGrepTerms {
- return terms
- }
- }
- }
- return terms
-}
-
-var reFallbackSplit = regexp.MustCompile(`[\n。;;,.?!?]+`)
-
-// GrepSedNarrow narrows chunks by grepping terms extracted directly from the
-// claim (zero LLM). Terms are derived from the claim text via SplitFallbackTerms; no extra
-// LLM call. Never raises.
-func GrepSedNarrow(chunks []map[string]any, claimSources []string, maxOutCharsPerChunk, maxOutTotalChars int) NarrowResult {
- if len(chunks) == 0 {
- return NarrowResult{Kept: chunks, Stats: NarrowStats{ChunksIn: 0}}
- }
- terms := SplitFallbackTerms(claimSources...)
- return NarrowByTerms(chunks, terms, nil, strings.Join(claimSources, " "), NarrowContext{}, maxOutCharsPerChunk, maxOutTotalChars)
-}
-
-// GrepSummaryFromClaims: public convenience: given
-// claim/question texts, produce a compact narrowed evidence string (used by the
-// compiled-structure grepper in search.go). Returns "" when nothing was kept.
-func GrepSummaryFromClaims(chunks []map[string]any, claimSources []string) string {
- res := GrepSedNarrow(chunks, claimSources, defaultOutCharsPerChunk, defaultOutTotalChars)
- if len(res.Kept) == 0 {
- return ""
- }
- var b strings.Builder
- for _, c := range res.Kept {
- if t := ChunkTextOf(c); t != "" {
- b.WriteString(t)
- b.WriteString("\n")
- }
- }
- return b.String()
-}
-
-func logGrepSed(s NarrowStats) {
- _LOG.Printf("[grep-sed] chunks=%d->%d chars=%d->%d matched=%t terms=%d",
- s.ChunksIn, s.ChunksKpt, s.CharsIn, s.CharsOut, s.Matched, s.UsedTerms)
-}
-
-// Local helpers shared with the narrowing paths
-
-func cloneMap(c map[string]any) map[string]any {
- cp := make(map[string]any, len(c))
- for k, v := range c {
- cp[k] = v
- }
- return cp
-}
-
-// withNarrowedText returns a copy of the chunk with its narrowed text applied: it
-// overwrites "content_with_weight", mirrors "content" only when that key already exists,
-// and drops "highlight" (the pre-narrow highlight spans no longer apply).
-func withNarrowedText(c map[string]any, narrowed string) map[string]any {
- cp := cloneMap(c)
- cp["content_with_weight"] = narrowed
- if _, ok := cp["content"]; ok {
- cp["content"] = narrowed
- }
- delete(cp, "highlight")
- return cp
-}
-
-// charLen is the codepoint length. Go's len(string) is bytes, which diverges for CJK; the
-// narrowing caps are codepoint budgets.
-func charLen(s string) int { return utf8.RuneCountInString(s) }
-
-// truncHead keeps the first n codepoints (not bytes) of s. Byte slicing would corrupt
-// multi-byte CJK and mis-size output; rune slicing is faithful to the engine's codepoint
-// budgets.
-func truncHead(s string, n int) string {
- if charLen(s) <= n {
- return s
- }
- return string([]rune(s)[:n])
-}
-
-func sort2DRanges(ranges [][2]int) {
- for i := 0; i < len(ranges); i++ {
- for j := i + 1; j < len(ranges); j++ {
- if ranges[j][0] < ranges[i][0] {
- ranges[i], ranges[j] = ranges[j], ranges[i]
- }
- }
- }
-}
-
-func anyBool(flags []bool) bool {
- for _, f := range flags {
- if f {
- return true
- }
- }
- return false
-}
-
-func clampInt(v, lo, hi int) int {
- if v < lo {
- return lo
- }
- if v > hi {
- return hi
- }
- return v
-}
diff --git a/internal/rag/agentic-rag/runtime/grep_sed_narrow_test.go b/internal/rag/agentic-rag/runtime/grep_sed_narrow_test.go
deleted file mode 100644
index f7fb6dd3c4..0000000000
--- a/internal/rag/agentic-rag/runtime/grep_sed_narrow_test.go
+++ /dev/null
@@ -1,376 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "strings"
- "testing"
-)
-
-// TestGrepTermsFromQueryCJK pins the derivation a Chinese query needs.
-//
-// The alnum-only original yields NO term for Chinese, and GrepSearch's own guard
-// (`if not chunks or not terms: return res`) then returns whole chunks: measured
-// on a Chinese question, the "Keyword-first locate" line was logged, a narrowed
-// line never was, and every hit was a ~1200-char chunk.
-func TestGrepTermsFromQueryCJK(t *testing.T) {
- // An alternation is the caller's own term list (the count protocol batches its
- // probes as 名|名|名), so it is split as-is — predicates included.
- if got := GrepTermsFromQuery("颜良|文丑|荀正"); !cjkTermsEqual(got, []string{"颜良", "文丑", "荀正"}) {
- t.Errorf("alternation terms = %v, want 颜良 文丑 荀正", got)
- }
- if got := GrepTermsFromQuery("关羽|斩"); !cjkTermsEqual(got, []string{"关羽", "斩"}) {
- t.Errorf("alternation with a one-rune predicate = %v, want 关羽 斩", got)
- }
- if got := GrepTermsFromQuery("关羽|关羽|关羽"); !cjkTermsEqual(got, []string{"关羽"}) {
- t.Errorf("alternation terms = %v, want a single deduped term", got)
- }
-
- // Space-separated Chinese keeps its tokens; a lone character is a particle.
- if got := GrepTermsFromQuery("关羽 斩颜良 文丑"); !cjkTermsEqual(got, []string{"关羽", "斩颜良", "文丑"}) {
- t.Errorf("split terms = %v", got)
- }
- if got := GrepTermsFromQuery("关羽 斩 文丑"); !cjkTermsEqual(got, []string{"关羽", "文丑"}) {
- t.Errorf("split terms = %v, want the one-rune particle dropped outside an alternation", got)
- }
-
- // An unbroken clause has no token at all, so its two-rune windows stand in:
- // the names in the clause still locate, and windows that occur nowhere cost
- // one failed lookup inside the narrowing pass.
- got := GrepTermsFromQuery("关羽杀了多少有姓名的人物")
- if len(got) != GrepTermsMax {
- t.Fatalf("clause windows = %v (%d), want %d windows", got, len(got), GrepTermsMax)
- }
- for _, want := range []string{"关羽", "姓名"} {
- if !cjkTermsContain(got, want) {
- t.Errorf("clause windows = %v, want %q among them", got, want)
- }
- }
- for _, term := range got {
- if !hasCJK(term) {
- t.Errorf("clause windows = %v, want CJK only (got %q)", got, term)
- }
- }
- if got := GrepTermsFromQuery(" "); got != nil {
- t.Errorf("blank query = %v, want nil", got)
- }
-}
-
-// TestExecOnTextCJKNarrowsToTheSentenceNotTheWholeChunk: a Chinese chunk carries
-// no newlines, so its "line" IS the chunk and the line window hands back all of
-// it. The window has to be cut at sentence punctuation instead — bounded, and
-// with the clause that carries the name intact (that clause is what the model
-// reads a name off).
-func TestExecOnTextCJKNarrowsToTheSentenceNotTheWholeChunk(t *testing.T) {
- filler := "曹操引军退去诸将皆惊张辽与徐晃双马齐出先射中头盔徐晃败走沿河追赶。"
- content := "前文提到青州兵甲之事与本案无关。" +
- strings.Repeat(filler, 14) +
- "关羽赶上荀正交马一合砍荀正于马下。" +
- strings.Repeat(filler, 3) +
- "后文又叙河北军大半落水之事。"
- if len(content) <= contextCharBudget*2 {
- t.Fatalf("test content is %d bytes; it must exceed the per-side budget to exercise the fallback", len(content))
- }
-
- got, matched := execOnText(content, TermsToPatterns([]string{"荀正"}), 1, 0, GrepOutCharsPerChunk)
- if !matched {
- t.Fatal("荀正 is in the content, the narrow must match")
- }
- if !strings.Contains(got, "荀正") {
- t.Fatalf("window lost the hit: %q", got)
- }
- if len(got) > GrepOutCharsPerChunk {
- t.Errorf("window = %d bytes, want at most %d", len(got), GrepOutCharsPerChunk)
- }
- if !strings.Contains(got, "砍荀正于马下") {
- t.Errorf("window cut the clause mid-sentence: %q", got)
- }
- if strings.Contains(got, "青州兵甲") {
- t.Errorf("window dragged in the far head of the chunk: %q", got)
- }
-}
-
-// TestExecOnTextKeepsLineWindowsWhenThereAreLines: the sentence fallback is for
-// text with no line structure. Text that HAS lines keeps grep's line window
-// (±1 line), which is the behaviour every Latin question already relies on.
-func TestExecOnTextKeepsLineWindowsWhenThereAreLines(t *testing.T) {
- content := "line one about alpha\nline two about beta\nline three about gamma\nline four about delta"
- got, matched := execOnText(content, TermsToPatterns([]string{"beta"}), 1, 0, GrepOutCharsPerChunk)
- if !matched {
- t.Fatal("beta is in the content, the narrow must match")
- }
- if !strings.Contains(got, "line one about alpha") || !strings.Contains(got, "line two about beta") {
- t.Errorf("window = %q, want the hit line and its before-context line", got)
- }
- if strings.Contains(got, "line four about delta") {
- t.Errorf("window = %q, want the after-context line excluded (after=0)", got)
- }
-}
-
-// TestNarrowByTermsCJKNarrowsThroughTheRealEntry pins the same behaviour one
-// level up, where grep actually calls it.
-func TestNarrowByTermsCJKNarrowsThroughTheRealEntry(t *testing.T) {
- text := strings.Repeat("曹操引军退去诸将皆惊。", 12) +
- "关羽赶上荀正交马一合砍荀正于马下。" +
- strings.Repeat("河北军大半落水。", 12)
- chunks := []map[string]any{{"id": "c1", "content_with_weight": text}}
-
- res := NarrowByTerms(chunks, []string{"荀正"}, nil, "", NarrowContext{Before: 1, After: 0},
- GrepOutCharsPerChunk, GrepOutTotalChars)
-
- if !res.Stats.Matched {
- t.Fatalf("stats = %+v, want a match", res.Stats)
- }
- if len(res.Kept) != 1 {
- t.Fatalf("kept = %d chunk(s), want 1", len(res.Kept))
- }
- kept := ChunkTextOf(res.Kept[0])
- if !strings.Contains(kept, "荀正") {
- t.Errorf("kept passage lost the hit: %q", kept)
- }
- if len(kept) >= len(text) {
- t.Errorf("kept %d bytes of %d: the passage was not narrowed at all", len(kept), len(text))
- }
-}
-
-// cjkTermsEqual / cjkTermsContain are local helpers: this file needs exact-order
-// and membership checks over the derived terms.
-func cjkTermsEqual(got, want []string) bool {
- if len(got) != len(want) {
- return false
- }
- for i := range want {
- if got[i] != want[i] {
- return false
- }
- }
- return true
-}
-
-func cjkTermsContain(list []string, want string) bool {
- for _, s := range list {
- if s == want {
- return true
- }
- }
- return false
-}
-
-// TestGrepWordsAreTheCallersOwnWords pins the split between the two readings of
-// one query: the terms LOCATE (and may decompose an unbroken clause into CJK
-// windows), while the words are what the caller actually wrote — the only ones a
-// seat probe may spend a retrieval on.
-//
-// Measured (2026-09-15): a seat pass built from the locate reading probed 羽斩 /
-// 杀的 / 的有 / 领名 / 单温, reported 14 of 20 probes "absent", and never reached a
-// single name the question was missing.
-func TestGrepWordsAreTheCallersOwnWords(t *testing.T) {
- // A caller-enumerated list is words in both readings — minus the single-rune
- // particles ("斩" is a predicate, not something to probe as a name; the phrase
- // search carries it).
- listed := "关羽 斩 颜良 文丑 车胄"
- if got := GrepWordsFromQuery(listed); strings.Join(got, "|") != "关羽|颜良|文丑|车胄" {
- t.Errorf("GrepWordsFromQuery(%q) = %v, want the caller's own words (single-rune tokens are particles)", listed, got)
- }
-
- // An unbroken clause names no individual: the locate reading decomposes it
- // into windows, the words reading yields nothing to probe.
- clause := "关羽斩杀敌将名单温酒斩华雄"
- if got := GrepWordsFromQuery(clause); len(got) != 0 {
- t.Errorf("GrepWordsFromQuery(%q) = %v, want none: a clause is not a word to probe", clause, got)
- }
- terms := GrepTermsFromQuery(clause)
- if len(terms) == 0 {
- t.Fatalf("GrepTermsFromQuery(%q) = none, want the two-rune windows grep locates with", clause)
- }
- for _, term := range terms {
- if term == clause {
- t.Errorf("GrepTermsFromQuery(%q) returned the whole clause; the point of the windows is that the clause itself locates nothing", clause)
- }
- }
-
- // The name boundary is the one this file already declares (cjkPhraseRunes):
- // up to four runes is still a name, past it is prose.
- for _, name := range []string{"孔秀", "夏侯存", "成吉思汗"} {
- if got := GrepWordsFromQuery(name); len(got) != 1 || got[0] != name {
- t.Errorf("GrepWordsFromQuery(%q) = %v, want the name itself", name, got)
- }
- }
- if got := GrepWordsFromQuery("成吉思汗东征"); len(got) != 0 {
- t.Errorf("GrepWordsFromQuery = %v, want none: past the name boundary a CJK token is prose", got)
- }
-}
-
-// TestGrepPatternIsAMatchNotATermFilter pins the grep half of grep_search: the
-// pattern decides which candidates are evidence, and it keeps its own semantics
-// instead of being degraded into "does the text carry one of these words".
-//
-// The ordering case is the one a term-locate pass can never express: both chunks
-// carry 关公 and 斩, and only the written order distinguishes the clause that says
-// he killed from the one that says he killed and then went back to camp.
-func TestGrepPatternIsAMatchNotATermFilter(t *testing.T) {
- chunks := []map[string]any{
- {"chunk_id": "a", "content": "关公马快,早赶上文丑,脑后一刀,斩于马下。"},
- {"chunk_id": "b", "content": "斩将之后,关公回营,众将皆来称贺。"},
- {"chunk_id": "c", "content": "话说曹操引军而回,不在话下。"},
- {"chunk_id": "d", "content": "荀正 引军来战,被云长一刀斩于马下。"},
- }
- re := grepPatternOf("关公.*斩")
- if re == nil {
- t.Fatal("关公.*斩 carries pattern syntax and must be read as a pattern")
- }
- kept, matched := matchGrepPattern(chunks, re, contextCharBudget, GrepOutTotalChars)
- if matched != 1 || len(kept) != 1 {
- t.Fatalf("matched=%d kept=%d, want exactly the chunk whose text has 关公 BEFORE 斩", matched, len(kept))
- }
- if id := ChunkIDOf(kept[0]); id != "a" {
- t.Errorf("kept %q, want \"a\": the pattern's ORDER must be enforced, not just its words", id)
- }
- // The window is the clause around the match, not the whole chunk.
- if txt := ChunkTextOf(kept[0]); !strings.Contains(txt, "斩") {
- t.Errorf("window %q does not carry the match", txt)
- }
-
- // An alternation is a batch: it matches each alternative and reports per-term
- // reach over the same candidate set, which is how "which of these did the
- // corpus answer" becomes readable.
- alt := grepPatternOf("华雄|荀正|管亥")
- if alt == nil {
- t.Fatal("华雄|荀正|管亥 must be a pattern")
- }
- keptAlt, matchedAlt := matchGrepPattern(chunks, alt, contextCharBudget, GrepOutTotalChars)
- if matchedAlt != 1 || len(keptAlt) != 1 {
- t.Fatalf("alternation matched=%d kept=%d, want the one candidate carrying one of the three names", matchedAlt, len(keptAlt))
- }
- if id := ChunkIDOf(keptAlt[0]); id != "d" {
- t.Errorf("kept %q, want \"d\" (the only candidate carrying 荀正)", id)
- }
- located, counts, absent := termReach(chunks, []string{"华雄", "荀正", "管亥"})
- if strings.Join(located, ",") != "荀正" || counts[0] != 1 {
- t.Errorf("termReach located=%v counts=%v, want only 荀正 reached once", located, counts)
- }
- if strings.Join(absent, ",") != "华雄,管亥" {
- t.Errorf("absent=%v, want 华雄 and 管亥 reported as NOT reached by this query", absent)
- }
-}
-
-// TestPlainQueryStaysOnTheTermLocatePath guards the other direction: pattern
-// syntax is the ONLY trigger, so every ordinary question keeps the behaviour it
-// had (locate terms -> windows), and nothing about it changes.
-func TestPlainQueryStaysOnTheTermLocatePath(t *testing.T) {
- for _, q := range []string{
- "关羽 斩颜良",
- "三国演义中关羽杀了多少有姓名的人物",
- "华为2023年营收",
- "partner of the 1984 Olympic keelboat competitor",
- } {
- if re := grepPatternOf(q); re != nil {
- t.Errorf("grepPatternOf(%q) compiled a pattern; a plain query must take the term-locate path", q)
- }
- }
-}
-
-// TestReachLineNamesWhatTheBatchMissed pins the line the MODEL reads: a batch of
-// names comes back with per-term reach, so a member that was never reached is
-// visible as such instead of looking exactly like a member nobody asked about.
-//
-// The capability was already there (the pattern engine, the per-term accounting,
-// the log line); what was missing is that only the LOG could read it.
-func TestReachLineNamesWhatTheBatchMissed(t *testing.T) {
- chunks := []map[string]any{
- {"chunk_id": "c1", "content": "云长手起一刀,斩华雄于马下"},
- {"chunk_id": "c2", "content": "华雄又斩了潘凤"},
- }
- q := "华雄|荀正|管亥"
- line := GrepReachLine(q, chunks, ReachTermsOf(q))
- for _, want := range []string{"[reach]", "华雄(2)", "NOT reached by this query"} {
- if !strings.Contains(line, want) {
- t.Errorf("line %q missing %q", line, want)
- }
- }
- // Both unanswered names are named, and the wording refuses to call them absent.
- for _, missed := range []string{"荀正", "管亥"} {
- if !strings.Contains(line, missed) {
- t.Errorf("line %q must name %q as not reached", line, missed)
- }
- }
-
- // A PATTERN reports on its OPERANDS: they are what the keyword leg searched
- // for, so "关公.*斩" is reported as 关公 and 斩, not as one clause.
- pq := "关公.*斩|云长.*斩"
- pline := GrepReachLine(pq, []map[string]any{{"chunk_id": "c3", "content": "关公勒马,一刀斩之"}}, ReachTermsOf(pq))
- for _, want := range []string{"关公(1)", "云长"} {
- if !strings.Contains(pline, want) {
- t.Errorf("pattern line %q missing %q", pline, want)
- }
- }
-
- // No terms -> no line at all (never an empty "[reach]" heading).
- if got := GrepReachLine("", chunks, nil); got != "" {
- t.Errorf("empty reach = %q, want no line", got)
- }
-}
-
-// TestNarrowByTermsExemptsTables pins the whole-table exemption: a table chunk is never
-// re-windowed and never char-trimmed, even when the total budget is tight.
-//
-// A head slice keeps the header and the first rows and drops the answer row, and the
-// dropped rows carry no marker — the model then reads a truncated table as a complete
-// one. The budget therefore bounds prose only.
-func TestNarrowByTermsExemptsTables(t *testing.T) {
- prose := strings.Repeat("曹操引军退去诸将皆惊。", 12) +
- "关羽赶上荀正交马一合砍荀正于马下。" +
- strings.Repeat("河北军大半落水。", 12)
- table := "<table>" + strings.Repeat("<tr><td>19</td><td>Danilo</td></tr>", 100) + "</table>"
- chunks := []map[string]any{
- {"id": "c1", "content_with_weight": prose},
- {"id": "c2", "content_with_weight": table},
- }
-
- res := NarrowByTerms(chunks, []string{"荀正"}, nil, "", NarrowContext{Before: 1, After: 0}, 200, 500)
-
- if !res.Stats.Matched {
- t.Fatalf("stats = %+v, want the prose hit to set matched", res.Stats)
- }
- if len(res.Kept) != 2 {
- t.Fatalf("kept = %d chunk(s), want 2", len(res.Kept))
- }
- if got := ChunkTextOf(res.Kept[1]); got != table {
- t.Errorf("table came back %d chars, want the %d-char original verbatim", len(got), len(table))
- }
- if got := ChunkTextOf(res.Kept[0]); len(got) >= len(prose) {
- t.Errorf("prose kept %d of %d bytes: it was not narrowed", len(got), len(prose))
- }
-}
-
-// TestNarrowByTermsKeepsTableOnlyInputVerbatim pins the boundary: with nothing but
-// tables there is nothing to locate, so narrowing reports no match and every table is
-// returned untouched (the caller must not read that as an answer failure).
-func TestNarrowByTermsKeepsTableOnlyInputVerbatim(t *testing.T) {
- table := "| Rank | Rider |\n|---|---|\n" +
- strings.Repeat("| 19 | Danilo |\n", 40)
- chunks := []map[string]any{{"id": "c1", "content_with_weight": table}}
-
- res := NarrowByTerms(chunks, []string{"Danilo"}, nil, "", NarrowContext{Before: 1, After: 0}, 200, 500)
-
- if res.Stats.Matched {
- t.Errorf("stats = %+v, want matched=false for a table-only input", res.Stats)
- }
- if len(res.Kept) != 1 || ChunkTextOf(res.Kept[0]) != table {
- t.Errorf("kept = %d chunk(s), want the table verbatim", len(res.Kept))
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/items.go b/internal/rag/agentic-rag/runtime/items.go
deleted file mode 100644
index 5ae4916c1c..0000000000
--- a/internal/rag/agentic-rag/runtime/items.go
+++ /dev/null
@@ -1,318 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "regexp"
- "strconv"
- "strings"
- "unicode/utf8"
-
- "ragflow/internal/rag/agentic-rag/slots"
-)
-
-// The table's items, read in one place: the count the answer reports, the size the record line
-// states, and the elements a merge unions are three readings of the same fact.
-
-// ItemValues is the values the table's item slots declare: every slot whose value IS items
-// (slots.KindItems), deduped case-insensitively, minus the pieces that merely contain another
-// item and minus the actor's own forms. Nothing is parsed out of a slot's text: a slot holds items
-// because the model said so.
-func ItemValues(table *State) []string {
- if table == nil {
- return nil
- }
- // The actor of the deed is not one of its elements, so his declared forms (Coverage.Actors)
- // are dropped here. Two spellings of a victim's name still count as two: nothing declares them
- // as one yet.
- actorForms := CoverageOf(*table).Actors()
- var items []string
- seen := map[string]bool{}
- for _, v := range table.State {
- if v.Typed().Kind != slots.KindItems {
- continue
- }
- for _, name := range v.Typed().ItemValues() {
- name = strings.TrimSpace(name)
- key := strings.ToLower(name)
- if key == "" || seen[key] || IsActorForm(name, actorForms) {
- continue
- }
- seen[key] = true
- items = append(items, name)
- }
- }
- out := make([]string, 0, len(items))
- for _, item := range items {
- contained := false
- for _, other := range items {
- if other == item || len([]rune(other)) >= len([]rune(item)) {
- continue
- }
- if strings.Contains(strings.ToLower(item), strings.ToLower(other)) {
- contained = true
- break
- }
- }
- if !contained {
- out = append(out, item)
- }
- }
- return out
-}
-
-// IsActorForm reports whether a value is the actor under one of the declared forms: an exact
-// match, or a form of two runes or more carried inside a value no longer than the form plus one
-// (关云长 carries 云长). A form nobody declared is not guessed: 关公 stays a member until something
-// declares it as one of the actor's names.
-func IsActorForm(value string, forms []string) bool {
- v := strings.ToLower(strings.TrimSpace(value))
- if v == "" {
- return false
- }
- for _, form := range forms {
- f := strings.ToLower(strings.TrimSpace(form))
- if f == "" {
- continue
- }
- if v == f {
- return true
- }
- if utf8.RuneCountInString(f) >= 2 && strings.Contains(v, f) &&
- utf8.RuneCountInString(v) <= utf8.RuneCountInString(f)+1 {
- return true
- }
- }
- return false
-}
-
-// AnchoredItems is ItemValues restricted to the items that carry the passage stating them: the set
-// a count may be derived from, since an item nobody can point at cannot be cited either. The
-// predicate lives on the value (slots.Value.Anchored).
-func AnchoredItems(table *State) []string {
- return itemValuesWhere(table, slots.Value.Anchored)
-}
-
-// UnanchoredItems is the complement of AnchoredItems: the values asserted with no passage in hand.
-// They are not deleted — the record lists them beside the number as claims the answer has to
-// account for — but they are not counted.
-func UnanchoredItems(table *State) []string {
- return itemValuesWhere(table, slots.Value.Unanchored)
-}
-
-// AnchoredItemChunks is the passage behind each anchored item, deduped and in table order: what an
-// answer that must cite one passage per element needs in front of it (see withCitedChunks).
-func AnchoredItemChunks(table *State) []string {
- if table == nil {
- return nil
- }
- var out []string
- seen := map[string]bool{}
- for _, v := range table.State {
- for _, it := range v.Typed().Anchored() {
- id := strings.TrimSpace(it.ChunkID)
- if id == "" || seen[id] {
- continue
- }
- seen[id] = true
- out = append(out, id)
- }
- }
- return out
-}
-
-// AnchoredItemRefs is AnchoredItemChunks' pairing: every anchored item's value with the passage it
-// rests on, in table order and deduped by value. The naming node matched each member to the
-// passage that states its deed, so this is the member→passage table the answer's citations are
-// written from (see CiteAnchoredMembers).
-func AnchoredItemRefs(table *State) []AnchoredRef {
- if table == nil {
- return nil
- }
- var out []AnchoredRef
- seen := map[string]bool{}
- for _, v := range table.State {
- for _, it := range v.Typed().Anchored() {
- name := strings.TrimSpace(it.Value)
- id := strings.TrimSpace(it.ChunkID)
- if name == "" || id == "" {
- continue
- }
- key := strings.ToLower(name)
- if seen[key] {
- continue
- }
- seen[key] = true
- out = append(out, AnchoredRef{Name: name, ChunkID: id, Quote: strings.TrimSpace(it.Quote)})
- }
- }
- return out
-}
-
-// citedMarkerPattern matches a citation marker the answer may already carry: the canonical
-// "[ID:n]" the rules prescribe, and the bare "[n]" a model writes on its own.
-var citedMarkerPattern = regexp.MustCompile(`\[(?:ID:\s*)?[0-9]+\]`)
-
-// CiteAnchoredMembers attaches the citation of every anchored member the answer states without
-// one: the line naming the member gets the marker of the passage that member rests on, taken from
-// citeIDs — the published evidence list, whose positions are what the client opens.
-//
-// The step is the RUNTIME's, not the model's: the naming node already matched each member to the
-// passage that names it, while a model can only cite the blocks it was shown — the evidence budget
-// admits the first few whole chunks, so a member past them has no block number it could write. A
-// member the answer never states is left alone, and a line that already carries a marker is kept
-// as written.
-func CiteAnchoredMembers(answer string, refs []AnchoredRef, citeIDs []string) string {
- out, _ := CitedAnchoredMembers(answer, refs, citeIDs, nil)
- return out
-}
-
-// answerQuote is the passage a line quotes: its first “…” span (or its first "…"), "" when the
-// line quotes nothing.
-func answerQuote(line string) string {
- for _, pair := range [][2]string{{"“", "”"}, {"\"", "\""}} {
- lo := strings.Index(line, pair[0])
- if lo < 0 {
- continue
- }
- rest := line[lo+len(pair[0]):]
- hi := strings.Index(rest, pair[1])
- if hi < 0 {
- continue
- }
- if q := strings.TrimSpace(rest[:hi]); q != "" {
- return q
- }
- }
- return ""
-}
-
-// spaceLess drops the whitespace and quotation marks a passage and its quotation may differ by, so
-// "被云长 刀起处,挥为两段" still matches the corpus's own spelling.
-func spaceLess(s string) string {
- return strings.NewReplacer(" ", "", "\t", "", "\n", "", "\u3000", "",
- "“", "", "”", "", "\"", "", "…", "").Replace(s)
-}
-
-// QuoteHeldBy reports whether a passage carries a quotation. It is the ONE definition of "the quote
-// is in this passage", asked by both sides of the ledger: the citation step decides which anchor a
-// member's marker may open with it (see CitedAnchoredMembers), and publishing decides with it
-// whether a recorded quote may stand as that member's block text — a quote the passage does not
-// contain must not be rendered IN THE PASSAGE'S PLACE (see compactAnchored).
-func QuoteHeldBy(passage, quote string) bool {
- return quote != "" && strings.Contains(spaceLess(passage), spaceLess(quote))
-}
-
-// CitedAnchoredMembers is CiteAnchoredMembers plus the number of LINES it rewrote, so a caller can
-// report whether the step touched the answer at all: a run whose members all carried markers
-// already is indistinguishable from one the step never reached otherwise.
-func CitedAnchoredMembers(answer string, refs []AnchoredRef, citeIDs []string, textOf func(string) string) (string, int) {
- if strings.TrimSpace(answer) == "" || len(refs) == 0 || len(citeIDs) == 0 {
- return answer, 0
- }
- pos := map[string]int{}
- for i, id := range citeIDs {
- if id = strings.TrimSpace(id); id == "" {
- continue
- }
- if _, dup := pos[id]; !dup {
- pos[id] = i
- }
- }
- if len(pos) == 0 {
- return answer, 0
- }
- overridden := 0
- lines := strings.Split(answer, "\n")
- for i, line := range lines {
- // A line naming members is given the marker of EVERY member it names, and any marker the
- // model wrote there is replaced by them: the model cites the blocks it read, so a line it
- // grouped ("颜良…文丑" both [ID:5]) points at a passage that is not that member's, and a
- // sentence naming nine members carries one marker for nine. Which member rests on which
- // passage is the naming node's finding (refs × citeIDs), so the runtime's answer replaces
- // the model's guess on exactly those lines; a line naming no member is left untouched.
- var add []string
- for _, r := range refs {
- if !strings.Contains(line, r.Name) {
- continue
- }
- id := strings.TrimSpace(r.ChunkID)
- if textOf != nil {
- // The marker must open the passage that STATES this member: the words the line
- // quotes, or — when the answer states them in its own words — the quotation the
- // naming node took from the passage in the first place. Only when NEITHER is in
- // the anchored passage is another published passage tried, and only when none of
- // them holds either is the line left uncited.
- holds := func(cand, q string) bool {
- return QuoteHeldBy(textOf(cand), q)
- }
- asked, own := answerQuote(line), strings.TrimSpace(r.Quote)
- if !holds(id, asked) && !holds(id, own) {
- id = ""
- for _, q := range []string{asked, own} {
- for _, cand := range citeIDs {
- if holds(cand, q) {
- id = strings.TrimSpace(cand)
- break
- }
- }
- if id != "" {
- break
- }
- }
- }
- }
- if id == "" {
- continue
- }
- if idx, ok := pos[id]; ok {
- add = append(add, "[ID:"+strconv.Itoa(idx)+"]")
- }
- }
- if len(add) == 0 {
- continue
- }
- stripped := strings.TrimRight(strings.TrimSpace(citedMarkerPattern.ReplaceAllString(line, "")), " \t")
- lines[i] = stripped + " " + strings.Join(add, "")
- overridden++
- }
- return strings.Join(lines, "\n"), overridden
-}
-
-// itemValuesWhere joins the items a picker selects ACROSS the table's slots, in ItemValues order,
-// so every reading of the table agrees on WHICH values exist and in what order; only the picker
-// differs.
-func itemValuesWhere(table *State, pick func(slots.Value) []slots.Item) []string {
- if table == nil {
- return nil
- }
- kept := map[string]bool{}
- for _, v := range table.State {
- for _, it := range pick(v.Typed()) {
- if val := strings.TrimSpace(it.Value); val != "" {
- kept[strings.ToLower(val)] = true
- }
- }
- }
- out := make([]string, 0, len(kept))
- for _, name := range ItemValues(table) {
- if kept[strings.ToLower(strings.TrimSpace(name))] {
- out = append(out, name)
- }
- }
- return out
-}
diff --git a/internal/rag/agentic-rag/runtime/items_test.go b/internal/rag/agentic-rag/runtime/items_test.go
deleted file mode 100644
index 26df833322..0000000000
--- a/internal/rag/agentic-rag/runtime/items_test.go
+++ /dev/null
@@ -1,154 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "strings"
- "testing"
-
- "ragflow/internal/rag/agentic-rag/slots"
-)
-
-// TestAnchoredItemsSplitWhatTheTableCanQuote pins the split the count reads: a member carrying
-// the chunk that states it is countable, a member without one is a claim. The split reads the
-// ChunkID — the anchor the answer cites — not the Quote, because a quotation with no passage to
-// look it up in is words nobody can check.
-//
-// Neither reading deletes a name: MemberNames still holds all three, which is what lets the record
-// list the claims beside the number instead of dropping them.
-func TestAnchoredItemsSplitWhatTheTableCanQuote(t *testing.T) {
- value := slots.Items(
- slots.Item{Value: "荀正", ChunkID: "c1", Quote: "关公大怒,直取荀正 交马一合,砍荀正于马下"},
- slots.Item{Value: "于禁"},
- slots.Item{Value: "刘延", Quote: "quoted words with no passage to look them up in"},
- )
- table := NewState([]Variable{{ID: 0, Type: "person", Value: &value}}, 0, nil)
-
- anchored := AnchoredItems(&table)
- if len(anchored) != 1 || anchored[0] != "荀正" {
- t.Fatalf("anchored = %v, want only the member carrying a passage", anchored)
- }
- unanchored := UnanchoredItems(&table)
- if len(unanchored) != 2 {
- t.Fatalf("unanchored = %v, want the two claims", unanchored)
- }
- joined := strings.Join(unanchored, "、")
- if !strings.Contains(joined, "于禁") || !strings.Contains(joined, "刘延") {
- t.Fatalf("unanchored = %v, want both claimed names", unanchored)
- }
- if all := ItemValues(&table); len(all) != 3 {
- t.Fatalf("all members = %v, want the three names (the split deletes none)", all)
- }
-}
-
-// TestItemValuesDropsTheActorOfTheDeed pins the one identity rule the run has data for: the
-// actor's own declared forms are not elements of the answer.
-//
-// A question that asks what someone did lists what he did it TO, so 关羽 cannot be one of the
-// people he killed — yet the run probes his name as a term and a session can write it into a list
-// (measured 2026-09-17: the probe ledger offered 关羽 among the "names" beside the victims). The
-// forms come from the planner (Coverage.Actors, "关羽|云长"), which is also why the match is an
-// IDENTITY rule and not a string rule: `关云长` carries the declared `云长` and is one item dropped;
-// `关公` carries neither and is left alone, because guessing a form nobody declared is how an
-// overloaded word starts dropping elements that are not the actor.
-func TestItemValuesDropsTheActorOfTheDeed(t *testing.T) {
- value := slots.Items(
- slots.Item{Value: "关羽", ChunkID: "c1"},
- slots.Item{Value: "关云长", ChunkID: "c2"},
- slots.Item{Value: "关公"},
- slots.Item{Value: "华雄", ChunkID: "c3"},
- )
- table := NewState([]Variable{
- {ID: 0, Type: "count", Terms: []string{"斩"}, Subject: "关羽|云长"},
- {ID: 1, Type: "person", Value: &value},
- }, 0, nil)
-
- got := ItemValues(&table)
- if len(got) != 2 {
- t.Fatalf("items = %v, want the actor's two declared forms gone and the rest kept", got)
- }
- for _, want := range []string{"关公", "华雄"} {
- if !strings.Contains(strings.Join(got, "、"), want) {
- t.Errorf("items = %v, want %q kept", got, want)
- }
- }
- if a := AnchoredItems(&table); len(a) != 1 || a[0] != "华雄" {
- t.Fatalf("anchored = %v, want only the anchored item that is not the actor", a)
- }
-}
-
-// TestAnchoredItemsIsEmptyWithoutItems keeps the split safe on the shapes every other reader
-// guards: a nil table, a table with no member slot, and a member slot whose items carry no name.
-func TestAnchoredItemsIsEmptyWithoutItems(t *testing.T) {
- if got := AnchoredItems(nil); got != nil {
- t.Errorf("anchored(nil) = %v, want nothing", got)
- }
- text := slots.Text("华雄、颜良、文丑")
- textOnly := NewState([]Variable{{ID: 0, Type: "person", Value: &text}}, 0, nil)
- if got := AnchoredItems(&textOnly); len(got) != 0 {
- t.Errorf("anchored(text slot) = %v, want nothing: text declares no members", got)
- }
- if got := UnanchoredItems(&textOnly); len(got) != 0 {
- t.Errorf("unanchored(text slot) = %v, want nothing", got)
- }
- empty := slots.Items(slots.Item{Value: " "})
- namedless := NewState([]Variable{{ID: 0, Type: "person", Value: &empty}}, 0, nil)
- if got := AnchoredItems(&namedless); len(got) != 0 {
- t.Errorf("anchored(blank name) = %v, want nothing", got)
- }
-}
-
-// TestCiteAnchoredMembersFillsWhatTheModelLeftUncited pins the step's whole reason: which passage
-// a member rests on is the naming node's finding, so the runtime writes that member's citation
-// itself. A line that already carries a marker is kept verbatim, a member the answer never states
-// is left alone, and a member whose passage holds no published position cannot be pointed at.
-func TestCiteAnchoredMembersFillsWhatTheModelLeftUncited(t *testing.T) {
- refs := []AnchoredRef{
- {Name: "华雄", ChunkID: "c-hua"},
- {Name: "管亥", ChunkID: "c-guan"},
- {Name: "蔡阳", ChunkID: "c-cai"},
- {Name: "不在答案里", ChunkID: "c-absent"},
- {Name: "庞德", ChunkID: "c-unpublished"},
- }
- ids := []string{"c-hua", "c-guan", "c-cai"}
- answer := strings.Join([]string{
- "关羽所斩:",
- "华雄——\"温酒斩华雄\"",
- "管亥——\"劈管亥于马下\" [ID:1]",
- "蔡阳——\"蔡阳头已落地\"",
- "庞德——\"德引颈受刑\"",
- "除上述数人外,另斩华雄、蔡阳等",
- }, "\n")
- got := CiteAnchoredMembers(answer, refs, ids)
- want := strings.Join([]string{
- "关羽所斩:",
- "华雄——\"温酒斩华雄\" [ID:0]",
- "管亥——\"劈管亥于马下\" [ID:1]",
- "蔡阳——\"蔡阳头已落地\" [ID:2]",
- "庞德——\"德引颈受刑\"",
- "除上述数人外,另斩华雄、蔡阳等 [ID:0][ID:2]",
- }, "\n")
- if got != want {
- t.Errorf("CiteAnchoredMembers =\n%s\nwant\n%s", got, want)
- }
- if got := CiteAnchoredMembers(answer, nil, ids); got != answer {
- t.Errorf("no refs must change nothing, got %q", got)
- }
- if got := CiteAnchoredMembers(answer, refs, nil); got != answer {
- t.Errorf("no published positions must change nothing, got %q", got)
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/kbinfos.go b/internal/rag/agentic-rag/runtime/kbinfos.go
deleted file mode 100644
index 361f58b556..0000000000
--- a/internal/rag/agentic-rag/runtime/kbinfos.go
+++ /dev/null
@@ -1,964 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "fmt"
- "strings"
- "sync"
-)
-
-// The shared accumulation store and the retrieval abstraction.
-//
-// The accumulation store is the typed Kbinfos struct (with the Merge helper), which holds
-// the chunks / doc_aggs / memory that retrieval, the action session, SCA and the final answer
-// all share. It lives in the runtime package because chunkKey/chunkText and Merge are
-// unexported helpers shared by tool_search.go, tool_exploration.go, and
-// tool_compiled_expansion.go.
-//
-// This file also owns the chunk identity helpers (chunkKey/chunkText) that the
-// merge logic depends on.
-
-// SearchParams are the inputs of one search.
-//
-// Question is the search query; Keywords is used ONLY to narrow retrieved
-// chunks (never to build the query) unless RetrievalQuery is empty.
-type SearchParams struct {
- Question string
- // Keywords narrows retrieved chunks to the sentences that mention them.
- // When RetrievalQuery is empty it also falls back into the query text.
- Keywords string
- // RetrievalQuery is the entity/qualifier-weighted query (entity x3,
- // qualifier x3) appended to the bare Question: a problem-level search over
- // the raw question is exactly where the entity must dominate the ranking.
- RetrievalQuery string
- // UseCompiled enables compiled-structure expansion (page index / tree /
- // mindmap / knowledge graph / wiki synthesis pages).
- UseCompiled bool
- // KbIDs restricts the search to these dataset ids. Empty falls back to the
- // session's bound datasets.
- KbIDs []string
- // DocScope restricts the search to these document ids (e.g. the doc_id
- // routed by navigate_tree).
- DocScope []string
- // TopN is the result count. <= 0 selects the configured default.
- TopN int
- // SkipReachLedger keeps this search's terms OUT of the reach ledger.
- //
- // The ledger is the record's "names you proved reachable and never recorded"
- // list, and it is read as a to-do list, so what enters it must be a NAME the
- // caller probed. The runtime's completeness pass queries ask for the actor and
- // the act words (see RunCompletenessPass), which are not names and are searched
- // by construction: without the flag every entry is a query word, and not one of them
- // is a member the round is actually missing.
- SkipReachLedger bool
- // Channel is retained for backward compatibility with callers that still
- // poke the unified HybridSearch with an explicit channel. It is DEPRECATED:
- // each entry point is now its own function (HybridSearch / VectorSearch /
- // BM25Search / GrepSearch / RetrieveSearch) and selects its vector weight,
- // similarity threshold and compiled-row exclusion internally. New code should
- // call the specific function instead of setting Channel on HybridSearch.
- // runSearch does NOT read this field.
- Channel SearchChannel
-}
-
-// SearchFn performs one hybrid search and returns chunks + doc aggs, so the
-// runtime is decoupled from the concrete retrieval backend.
-// It abstracts the concrete retrieval backend.
-type SearchFn func(ctx context.Context, p SearchParams) ([]map[string]any, []map[string]any)
-
-// Kbinfos is the shared accumulation store.
-//
-// ONE Kbinfos is shared by a round's CONCURRENT sessions (SessionDeps.KB; see
-// RunSlotResearchPass), so its mutable state is guarded by mu. The sessions run on
-// goroutines, so the critical sections have to be locked explicitly: see Admit.
-type Kbinfos struct {
- mu sync.Mutex
- Chunks []map[string]any
- DocAggs []map[string]any
- // ledgerMu guards the run's SEARCH RECORD (ProbedAbsent / Reached).
- //
- // It is deliberately NOT mu. The pool lock is held across admit batches, and
- // whoever holds it is free to write down what the batch asked and answered —
- // which is exactly what a seat does. Sharing one mutex made that a
- // self-deadlock: Go mutexes are not reentrant, so a record written inside an
- // admit batch parked the goroutine forever on a lock it already held, and the
- // wait ignores context cancellation, so the run never returned.
- //
- // Lock ORDER, when both are needed: mu first, then ledgerMu — never the other
- // way round. Nothing takes mu while holding ledgerMu, and the record methods
- // that run inside an admit batch (see Admit) are the only place the two nest.
- ledgerMu sync.Mutex
- // Memory is the lossless store of raw retrieved chunks backing the (lossy)
- // Chunks list that feeds the LLM; the memory add/grep helpers maintain it.
- Memory []map[string]any
- // PreSummary is the merged claim-report summary produced by the action
- // session; the final-answer call reads it when set.
- PreSummary string
- // Record is the slot table rendered for the ANSWER prompt — the facts the
- // research settled, without the machine fields (strength, terminal type,
- // evidence ids) that belong to the SCA's draft.
- //
- // The two are separate on purpose. A draft is written to be VERIFIED against
- // the passages that produced it; the answer prompt is written to be ANSWERED
- // from. Reusing one as the other hands the answer model the runtime's bookkeeping,
- // and the answer quotes the bookkeeping verbatim:
- // ("slot 1 [entity] … (strength=0.90) [terminal=state, evidence_ids=[…]]").
- Record string
- // novelAdmitted counts the chunks the cap EXEMPTION below has let in beyond
- // evidencePoolCap. It is what keeps the exemption bounded rather than
- // open-ended (see evidencePoolNoveltySlack). Guarded by mu, like Chunks.
- novelAdmitted int
- // ProbedAbsent is the run's record of NAMED terms a probe asked about and
- // nothing reached (see RecordProbedAbsent). The pool holds what was found;
- // this holds what was asked and not found, and the pair is what a rewrite
- // reads before choosing its next angle. Guarded by ledgerMu.
- ProbedAbsent []string
- // Reached is the run's record of NAMED terms a probe asked about and DID
- // reach, each with the pool chunk that carries it (see RecordReachedTerm):
- // the confirmed members, with their evidence. Guarded by ledgerMu.
- Reached []ReachedTerm
- // setDirection records that this run carries a SET/COUNT direction — a
- // question whose answer is a list of members (see MarkSetDirection, which
- // also says why the retrieval executor reads it). Guarded by ledgerMu.
- setDirection bool
- // actWords / actorForms are the direction's declaration (see MarkCoverage). Guarded by ledgerMu.
- actWords []string
- actorForms []string
- // citedChunks are the passages the enumerated items rest on (see NoteCitedChunks).
- // Guarded by ledgerMu.
- citedChunks []string
- // anchoredRefs are the enumerated members with their passages (see NoteAnchoredRefs).
- // Guarded by ledgerMu.
- anchoredRefs []AnchoredRef
- // sufficiencyUnchecked records that the completeness review never ran (see
- // NoteSufficiencyUnchecked). Guarded by ledgerMu.
- sufficiencyUnchecked bool
- // coverageSet is what this QUESTION's enumeration found, and coverageReady records
- // that it ran at all (see StoreCoverageSet): the windows it admits stay in the pool
- // under the same chunk ids, so a later round reuses the same evidence rather than
- // asking the corpus the same operand queries and putting back what is already there.
- // Guarded by ledgerMu.
- coverageSet CoverageSet
- coverageReady bool
- // CiteChunkIDs is the ordered id list of the chunks the final-answer call
- // rendered as numbered evidence — Python's tools._rag_cite_chunk_ids
- // (agentic_rag_graph.py:870). The renderer puts the passages behind enumerated
- // items first and fills the rest by similarity, capped (citeChunkCap), so this
- // list is NOT Chunks in pool order, and a "[ID:n]" the model writes refers to
- // position n in THIS list — one entry per RENDERED block, so a chunk the
- // renderer skipped (no content) holds no position here either. The chat pipeline
- // resolves citations against it
- // (RunResponse.CiteChunkIDs → HarnessResult.CiteChunkIDs →
- // decorateHarnessAnswer); without it, a marker was resolved against Chunks by
- // position and landed on the wrong chunk — or past the end when the pool held
- // fewer chunks than the cap.
- //
- // Written once per final-answer call, on the goroutine that composes it (the
- // concurrent research slots have already joined), and read by the caller after
- // the call returns — it is not part of the pool lock's protected state.
- CiteChunkIDs []string
- // cache is the per-request retrieval cache. It is initialised lazily via
- // cacheOnce so a zero-value Kbinfos is usable.
- cache *searchCache
- cacheOnce sync.Once
-}
-
-// HasChunks reports whether the pool holds any chunk.
-func (k *Kbinfos) HasChunks() bool { return len(k.Chunks) > 0 }
-
-// PoolSize is how many chunks the shared pool holds.
-//
-// It takes the pool lock, so it must NOT be called from inside an Admit callback
-// (that mutex is not reentrant — see Admit's invariant); the record rendering runs
-// after the batch for exactly that reason.
-func (k *Kbinfos) PoolSize() int {
- if k == nil {
- return 0
- }
- k.mu.Lock()
- defer k.mu.Unlock()
- return len(k.Chunks)
-}
-
-// ChunkByID returns the pool chunk carrying this id, or nil.
-//
-// It takes the pool lock, so it must NOT be called from inside an Admit callback
-// (that mutex is not reentrant — see Admit's invariant).
-func (k *Kbinfos) ChunkByID(id string) map[string]any {
- if k == nil || id == "" {
- return nil
- }
- k.mu.Lock()
- defer k.mu.Unlock()
- for _, c := range k.Chunks {
- if ChunkIDOf(c) == id {
- return c
- }
- }
- return nil
-}
-
-// ChunksFrom returns up to limit chunks starting at from, as a copy of the slice
-// header.
-//
-// It exists so a READER can walk the pool the round has already paid for without
-// holding the pool lock (chunks are only ever appended, never rewritten in place,
-// so the header it copies is stable). The maps themselves stay shared and must be
-// read only. Readers are also why this is a copy of the header and not the pool:
-// the pool keeps growing under them while they scan.
-func (k *Kbinfos) ChunksFrom(from, limit int) []map[string]any {
- if k == nil || limit <= 0 {
- return nil
- }
- k.mu.Lock()
- defer k.mu.Unlock()
- if from < 0 || from >= len(k.Chunks) {
- return nil
- }
- end := min(from+limit, len(k.Chunks))
- return append([]map[string]any(nil), k.Chunks[from:end]...)
-}
-
-// Admit runs fn as ONE critical section over the evidence pool.
-//
-// The granularity is the caller's, and it must span ONE query's candidate batch,
-// because the awaits sit in the OUTER query loop. Locking per chunk would let two
-// sessions' batches interleave into unsupported pool orders, and the pool order is
-// observable — it drives the rendered prompt order and the `ID: n` numbering,
-// extractRelevantEvidence's first-4 pick, and which chunks make it in under the cap.
-//
-// Inside fn do, per chunk, exactly the admission steps, in this order:
-//
-// p.Full() → skip the chunk. The cap check precedes the per-call dedup
-// (:638-646), so a rejected chunk is NOT marked seen and is
-// retried once room frees.
-// then the call-local dedup (`seen`) and payload/ids bookkeeping;
-// p.Add(c) → dedup against the live pool and append.
-//
-// INVARIANT — fn must not call anything that takes k.mu itself (Admit, Merge,
-// MergeDocAggs, RetireClaimsCoveredBy, PoolAdmitter.Full's siblings…): the mutex
-// is NOT reentrant, so that call parks the goroutine forever on a lock it already
-// holds, and a mutex wait ignores context cancellation, so the run never returns and
-// nothing is logged: the goroutine parks silently.
-//
-// What fn MAY call is the search record (RecordReachedTerm / RecordProbedAbsent):
-// those live behind ledgerMu, precisely so the batch that answered a probe can
-// write the answer down while the pool is still held.
-func (k *Kbinfos) Admit(fn func(p *PoolAdmitter)) {
- if k == nil {
- // No pool bound: the callback still runs — a tool hands its passages back
- // either way — but nothing is pooled and the cap never blocks, which is
- // what the callers' old `if deps.KB != nil` guard did.
- fn(&PoolAdmitter{})
- return
- }
- k.mu.Lock()
- defer k.mu.Unlock()
- fn(&PoolAdmitter{k: k})
-}
-
-// PoolAdmitter is the pool handle, valid only inside Kbinfos.Admit's callback.
-type PoolAdmitter struct{ k *Kbinfos }
-
-// Full is the early stop at the top of admission: at the cap the chunk is skipped, and
-// nothing about it is recorded (so a rejected chunk is retried once room frees).
-//
-// The "pool FULL" line belongs here, and it is emitted once per PROCESS. sync.Once
-// because concurrent sessions call this under their own pool lock.
-func (p *PoolAdmitter) Full() bool {
- if p.k == nil || len(p.k.Chunks) < evidencePoolCap {
- return false
- }
- evidencePoolFullLogged.Do(func() {
- _LOG.Printf("[Action Session] evidence pool FULL (%d chunks >= cap %d); early-stopping admit of further chunks.", len(p.k.Chunks), evidencePoolCap)
- })
- return true
-}
-
-// evidencePoolFullLogged: ["full_logged"]: a
-// PER-PROCESS flag, so the line is emitted once per
-// process — not once per rejected chunk, and NOT once per session. The pool never
-// shrinks mid-process, so no reset is needed.
-var evidencePoolFullLogged sync.Once
-
-// evidencePoolNoveltySlack bounds the cap EXEMPTION: the pool may pass
-// evidencePoolCap by at most this many chunks, and not one more.
-//
-// The cap is a STORAGE number, and it is the right number for evidence a later
-// search could find again. A PROBE's result is not that kind of evidence: a
-// batch probe ("荀正|管亥|车胄") is the model's question "does each of these exist,
-// and where?", and the per-name window it returns is the only place that answer
-// ever lives — no later search can reconstruct WHICH names came back empty.
-//
-// A batch probe can come back with passages for some of the names it asked about and none
-// for others the corpus does hold, so a cap that drops those windows converts a successful
-// probe into "nothing new", which the model then reads as "not a member".
-//
-// The slack is what keeps the exemption from being open-ended: it is charged per
-// exempted chunk, and beyond cap+slack even a novel term is refused.
-const evidencePoolNoveltySlack = 40
-
-// Novelty is the cap exemption for ONE admit batch: the batch's own terms that
-// the LIVE pool cannot answer yet.
-//
-// Build it once per batch (like ClaimCoveredIDs), then ask it per candidate:
-// a candidate that carries one of the absent terms may enter an ALREADY-FULL
-// pool, and that term is then consumed — one seat per term, so the pool grows by
-// at most one chunk per unanswered probe term. A batch with no terms (or a
-// batch whose terms the pool already carries) gets no exemption at all, which is
-// exactly the pre-existing cap behaviour.
-type Novelty struct {
- p *PoolAdmitter
- absent []string
-}
-
-// Novelty derives the batch's exemption from the terms the call asked for.
-//
-// Callers pass the terms of a PROBE (the model's own alternation, see
-// probeTerms in tool_executor.go) and nothing else: a topic query states no
-// list of individuals, so its hits get no exemption and the cap behaves as it
-// always did.
-//
-// The pool-wide scan happens ONCE here, over lowered pool text, rather than once
-// per candidate.
-func (p *PoolAdmitter) Novelty(terms []string) *Novelty {
- n := &Novelty{p: p}
- if p.k == nil {
- return n
- }
- asked := make([]string, 0, len(terms))
- for _, t := range terms {
- if t = strings.TrimSpace(t); t == "" || len(asked) >= GrepTermsMax {
- continue
- }
- asked = append(asked, t)
- }
- n.absent = termsNotCarried(p.k.Chunks, asked)
- return n
-}
-
-// ProbedAbsentTerms returns a copy of the named terms nothing reached, in the
-// order they were first probed.
-func (k *Kbinfos) ProbedAbsentTerms() []string {
- if k == nil {
- return nil
- }
- k.ledgerMu.Lock()
- defer k.ledgerMu.Unlock()
- return append([]string(nil), k.ProbedAbsent...)
-}
-
-// ReachedTerm is one confirmed member: the term a probe asked about, and the pool
-// chunk that carries it.
-type ReachedTerm struct {
- Term string
- ChunkID string
-}
-
-// RecordReachedTerm notes a named term that a probe asked about AND came back
-// with, together with the pool chunk that carries it.
-//
-// It is the mirror of RecordProbedAbsent, and the two together are the run's
-// record of its own search: this one is the CONFIRMED MEMBERS, each with the
-// passage that proves it. Two things read it:
-//
-// - the rewrite context, which shows the rewriter the passage behind each
-// confirmed member — the wording this text uses for the relation is in those
-// passages, and so are the names that are still missing;
-// - the loop, which can then ask whether the LIST is still growing rather than
-// whether the pool got bigger (a pool that grew by forty passages of the same
-// famous scene has learned nothing about the list).
-func (k *Kbinfos) RecordReachedTerm(term, chunkID string) {
- term = strings.TrimSpace(term)
- if k == nil || term == "" || chunkID == "" {
- return
- }
- k.ledgerMu.Lock()
- defer k.ledgerMu.Unlock()
- for _, seen := range k.Reached {
- if strings.EqualFold(seen.Term, term) {
- return
- }
- }
- if len(k.Reached) >= reachedTermsMax {
- return
- }
- k.Reached = append(k.Reached, ReachedTerm{Term: term, ChunkID: chunkID})
-}
-
-// MarkSetDirection records that this run carries a SET/COUNT direction.
-//
-// It is set by the SAME gate that hands the model the set method — the declared
-// shape of the direction's table, or the first batch the caller writes, whichever
-// comes first — and read by the retrieval executor, which bounds how many of the
-// caller's queries run per call:
-//
-// on a direction whose answer is a LIST, the caller's queries are facets of that
-// list rather than rephrasings of one question, so a query that is dropped is not
-// a spared repeat — it is a member nobody searched, and the names the answer turns out to
-// be missing were often named only in a dropped one. On a VALUE direction the queries ARE
-// rephrasings of one question and the small cap stays.
-func (k *Kbinfos) MarkSetDirection() {
- if k == nil {
- return
- }
- k.ledgerMu.Lock()
- k.setDirection = true
- k.ledgerMu.Unlock()
-}
-
-// IsSetDirection reports whether a SET/COUNT direction has declared itself on this
-// run (see MarkSetDirection). Safe on a nil pool: a run with no pool is not one.
-func (k *Kbinfos) IsSetDirection() bool {
- if k == nil {
- return false
- }
- k.ledgerMu.Lock()
- defer k.ledgerMu.Unlock()
- return k.setDirection
-}
-
-// NoteSufficiencyUnchecked records that the review which judges completeness never ran, so the
-// record the answer reads can say so. It is not a verdict, and not a reason to mark the answer
-// partial.
-func (k *Kbinfos) NoteSufficiencyUnchecked() {
- if k == nil {
- return
- }
- k.ledgerMu.Lock()
- k.sufficiencyUnchecked = true
- k.ledgerMu.Unlock()
-}
-
-// SufficiencyUnchecked reports whether the review that would have judged completeness
-// never ran (see NoteSufficiencyUnchecked). Safe on a nil pool.
-func (k *Kbinfos) SufficiencyUnchecked() bool {
- if k == nil {
- return false
- }
- k.ledgerMu.Lock()
- defer k.ledgerMu.Unlock()
- return k.sufficiencyUnchecked
-}
-
-// MarkCoverage records the direction's declaration — the actor's forms and the act words — for the
-// readers that have the pool but not the slot table (see CoverageDecl). Where StoreCoverageSet
-// holds what the corpus answered, this holds what the run was asking with.
-func (k *Kbinfos) MarkCoverage(cov Coverage) {
- if k == nil {
- return
- }
- k.ledgerMu.Lock()
- defer k.ledgerMu.Unlock()
- k.actWords = append([]string(nil), cov.Acts...)
- k.actorForms = cov.Actors()
-}
-
-// CoverageDecl is the direction's declaration (see MarkCoverage). Empty when no enumeration
-// direction declared itself, and its probes are read by shape alone.
-func (k *Kbinfos) CoverageDecl() (acts, actors []string) {
- if k == nil {
- return nil, nil
- }
- k.ledgerMu.Lock()
- defer k.ledgerMu.Unlock()
- return append([]string(nil), k.actWords...), append([]string(nil), k.actorForms...)
-}
-
-// NoteCitedChunks records the passages the enumerated items rest on (see
-// runtime.AnchoredItemChunks): the compose prompt puts them in front of the answer, which has to
-// cite one passage per element. Guarded by ledgerMu.
-func (k *Kbinfos) NoteCitedChunks(ids []string) {
- if k == nil {
- return
- }
- k.ledgerMu.Lock()
- k.citedChunks = append([]string(nil), ids...)
- k.ledgerMu.Unlock()
-}
-
-// CitedChunks is the passages recorded by NoteCitedChunks, empty when no enumeration wrote items.
-func (k *Kbinfos) CitedChunks() []string {
- if k == nil {
- return nil
- }
- k.ledgerMu.Lock()
- defer k.ledgerMu.Unlock()
- return append([]string(nil), k.citedChunks...)
-}
-
-// AnchoredRef is one enumerated member and the passage it rests on: the name the answer
-// states, and the chunk the naming node matched that name to.
-type AnchoredRef struct {
- Name string
- ChunkID string
- // Quote is the words the naming node matched the member to. The compose renders an anchored
- // member's block from THIS rather than from the whole passage: the evidence budget is spent
- // per block, so a ~1200-char chunk per member fits only the first handful of them, and the
- // members past that are members the model cannot see (and so cannot cite).
- Quote string
-}
-
-// NoteAnchoredRefs records the members behind the cited passages, in table order.
-//
-// The naming node has already matched every member to the passage that states its deed, so
-// the member→passage pointer is the RUNTIME's to know. The compose reads it back through
-// CiteAnchoredMembers to attach a citation to every member the answer states: a model that
-// cites only the handful of blocks it happened to read leaves the rest of the list
-// unreachable, and no prompt can make it cite a block it never saw (the evidence budget
-// admits only the first few whole chunks).
-func (k *Kbinfos) NoteAnchoredRefs(refs []AnchoredRef) {
- if k == nil {
- return
- }
- k.ledgerMu.Lock()
- k.anchoredRefs = append([]AnchoredRef(nil), refs...)
- k.ledgerMu.Unlock()
-}
-
-// AnchoredRefs is the list recorded by NoteAnchoredRefs, empty when nothing was enumerated.
-func (k *Kbinfos) AnchoredRefs() []AnchoredRef {
- if k == nil {
- return nil
- }
- k.ledgerMu.Lock()
- defer k.ledgerMu.Unlock()
- return append([]AnchoredRef(nil), k.anchoredRefs...)
-}
-
-// StoreCoverageSet records what this question's enumeration found (see
-// EnumerateCoverage / CoverageSet).
-func (k *Kbinfos) StoreCoverageSet(set CoverageSet) {
- if k == nil {
- return
- }
- k.ledgerMu.Lock()
- k.coverageSet = set
- k.coverageReady = true
- k.ledgerMu.Unlock()
-}
-
-// CoverageSet returns the enumeration a previous round ran for this question, and whether
-// it ran at all.
-//
-// The unit is the REQUEST, not the round: the windows the enumeration admitted are in the
-// pool under the same chunk ids, so a second round asking the corpus the same operand
-// queries would spend the same store legs to re-admit what is already there, and the seed
-// would show the same windows it already showed (see StoreCoverageSet). The SET is kept
-// rather than its rendering because the last node resolves the windows themselves.
-func (k *Kbinfos) CoverageSet() (CoverageSet, bool) {
- if k == nil {
- return CoverageSet{}, false
- }
- k.ledgerMu.Lock()
- defer k.ledgerMu.Unlock()
- return k.coverageSet, k.coverageReady
-}
-
-// ReachedTerms returns a copy of the confirmed members and the chunk that carries
-// each.
-func (k *Kbinfos) ReachedTerms() []ReachedTerm {
- if k == nil {
- return nil
- }
- k.ledgerMu.Lock()
- defer k.ledgerMu.Unlock()
- return append([]ReachedTerm(nil), k.Reached...)
-}
-
-// reachedTermsMax bounds the confirmed-member ledger (see RecordReachedTerm):
-// it is rendered into the rewrite context, so it may not grow with the number of
-// probes a long round happens to make.
-const reachedTermsMax = 24
-
-// EvidencePoolCap is the shared pool's ceiling. It is exported so that a caller
-// sizing the room for its own admissions (the rewrite round's prefetch) measures
-// it against the SAME number the admitter enforces.
-//
-// Measuring against a smaller constant is how a round came to report "retrieval
-// saturated" while the pool still had room to take evidence: the prefetch's room
-// was computed against the snippet-pool ceiling (60) while the pool itself holds
-// up to twice that, so on any rich round the rewrite's own queries had nowhere to
-// land and the round discarded itself.
-func EvidencePoolCap() int { return evidencePoolCap }
-
-// termsNotCarried reports which of terms no chunk's text carries
-// (case-insensitively), preserving the caller's order.
-//
-// It is the coverage half of every "did we find that name?" question in the
-// runtime, and it has exactly two callers because there are exactly two sets it
-// can be asked about:
-//
-// - a SEARCH's own results — which of the terms this call named did this
-// retrieval not reach? (the seat pass in tool_executor.go, which then buys
-// each unreached term its own search), and
-// - the POOL — which named terms can the pool not answer at all? (Novelty,
-// which exempts those passages from the cap).
-func termsNotCarried(chunks []map[string]any, terms []string) []string {
- _, _, absent := termReach(chunks, terms)
- return absent
-}
-
-// termReach answers, per term, HOW MANY chunks carry it — the reach of one
-// retrieval, term by term.
-//
-// It is the fact a caller cannot read off a result set: every name in a batch
-// looks the same whether the search found it or never looked, and the wording
-// matters, because "not reached by THIS query" is not "absent from the corpus".
-// A model that reads a miss as absence stops enumerating.
-//
-// located/counts are parallel and in the caller's term order; absent holds the
-// terms no chunk carried.
-func termReach(chunks []map[string]any, terms []string) (located []string, counts []int, absent []string) {
- if len(terms) == 0 {
- return nil, nil, nil
- }
- texts := make([]string, 0, len(chunks))
- for _, c := range chunks {
- if text := strings.ToLower(ChunkTextOf(c)); text != "" {
- texts = append(texts, text)
- }
- }
- for _, term := range terms {
- low := strings.ToLower(strings.TrimSpace(term))
- if low == "" {
- continue
- }
- n := 0
- for _, text := range texts {
- if strings.Contains(text, low) {
- n++
- }
- }
- if n == 0 {
- absent = append(absent, term)
- continue
- }
- located = append(located, term)
- counts = append(counts, n)
- }
- return located, counts, absent
-}
-
-// RecordProbedAbsent notes a named term that a probe asked about and nothing
-// reached, deduped, under the pool lock.
-//
-// The pool holds what was FOUND; ProbedAbsent holds what was ASKED and not
-// found. Together they are the round's record of its own search, which is the
-// fact a rewrite needs: re-asking a name the corpus already came back empty on
-// is the loop's most common waste, and "these came back empty" is also what
-// tells the model to change the ANGLE (ask for the act instead of the name)
-// rather than to ask the same thing louder.
-func (k *Kbinfos) RecordProbedAbsent(term string) {
- term = strings.TrimSpace(term)
- if k == nil || term == "" {
- return
- }
- k.ledgerMu.Lock()
- defer k.ledgerMu.Unlock()
- for _, seen := range k.ProbedAbsent {
- if strings.EqualFold(seen, term) {
- return
- }
- }
- if len(k.ProbedAbsent) >= probedAbsentMax {
- // Bounded storage: the record is carried into the rewrite context, so it
- // may not grow with the number of probes a long round happens to make.
- return
- }
- k.ProbedAbsent = append(k.ProbedAbsent, term)
-}
-
-// probedAbsentMax bounds ProbedAbsent (see RecordProbedAbsent).
-const probedAbsentMax = 16
-
-// Admits reports whether c may enter the pool although the pool is FULL, because
-// c is the first evidence for one of the batch's unanswered terms.
-//
-// It answers only the novelty half of the admit decision: the caller still
-// checks Full() first, so a pool under the cap never consults this and behaves
-// as before. A nil Novelty (or one built from no terms) admits nothing.
-func (n *Novelty) Admits(c map[string]any) bool {
- if n == nil || n.p == nil || n.p.k == nil || len(n.absent) == 0 {
- return false
- }
- // The exemption is bounded: past cap+slack the probe's windows are refused
- // like everything else, so a runaway enumeration cannot grow storage without
- // limit.
- if len(n.p.k.Chunks) >= evidencePoolCap+evidencePoolNoveltySlack {
- return false
- }
- text := strings.ToLower(ChunkTextOf(c))
- if text == "" {
- return false
- }
- for i, t := range n.absent {
- if !strings.Contains(text, strings.ToLower(t)) {
- continue
- }
- // ONE seat per term: this passage answers it, so the next passage
- // carrying only that term is refused again.
- n.absent = append(n.absent[:i], n.absent[i+1:]...)
- n.p.k.novelAdmitted++
- _LOG.Printf("[Action Session] pool at the cap %d but the batch asked about %q; admitting the passage that reaches it (pool %d, novelty slack %d/%d).",
- evidencePoolCap, t, len(n.p.k.Chunks)+1, n.p.k.novelAdmitted, evidencePoolNoveltySlack)
- return true
- }
- return false
-}
-
-// Add appends c unless the LIVE pool already holds it, reporting whether it appended —
-// i.e. whether the chunk was new to the shared pool.
-//
-// The test is against the live pool, not a snapshot taken when the call started. A snapshot
-// would re-append a chunk another session just pooled, which must not happen under
-// parallelism.
-//
-// There is deliberately NO cap check here: the compiled-expansion path appends uncapped,
-// which is what pushes the pool past evidencePoolCap (see the comment on evidencePoolCap).
-// Callers that admit user-facing search
-// hits check Full() first.
-func (p *PoolAdmitter) Add(c map[string]any) bool {
- if p.k == nil {
- return false
- }
- if p.Index(c) >= 0 {
- return false
- }
- p.k.Chunks = append(p.k.Chunks, c)
- return true
-}
-
-// Index is the pool position of c's identity, or -1 when the pool lacks it (and
-// for an unbound pool): the positional contract of Merge — a deduped chunk still
-// reports its position.
-func (p *PoolAdmitter) Index(c map[string]any) int {
- if p.k == nil {
- return -1
- }
- return indexOfChunk(p.k.Chunks, chunkKey(c))
-}
-
-// ClaimCoveredIDs returns the chunk ids already represented VERBATIM by a claim pseudo-chunk
-// in the LIVE pool: a claim carries its own verbatim quote plus the ids of the chunks it was
-// distilled from, so admitting those passages again is duplicate payload — the answer material
-// is already in the pool at a fraction of the size. Call it ONCE per Admit batch and test each
-// chunk's id against the result (the set is computed once per admission call).
-func (p *PoolAdmitter) ClaimCoveredIDs() map[string]bool {
- if p.k == nil {
- return nil
- }
- covered := map[string]bool{}
- for _, c := range p.k.Chunks {
- if id, ok := c["chunk_id"].(string); !ok || !strings.HasPrefix(id, "claim_") {
- continue
- }
- for _, cid := range toStringList(c["source_chunk_ids"]) {
- if cid != "" {
- covered[cid] = true
- }
- }
- }
- return covered
-}
-
-// CoveredByClaim mirrors the _admit_evidence skip:
-// a non-table chunk whose id a pooled claim already quotes verbatim must NOT
-// enter the pool again. Table chunks are exempt: their answer rows survive
-// only in full text.
-func (p *PoolAdmitter) CoveredByClaim(cid string, covered map[string]bool, isTable bool) bool {
- return !isTable && cid != "" && covered[cid]
-}
-
-// Merge appends the given chunks/aggs, deduplicating by chunkKey, and returns the
-// GLOBAL index (position in k.Chunks) of every contributed chunk — deduped ones
-// included.
-//
-// Mirrors orchestrator/direct.py:_merge_kbinfos, including its early return on an
-// empty chunk list (neither chunks nor doc_aggs are merged).
-func (k *Kbinfos) Merge(chunks, aggs []map[string]any) []int {
- if len(chunks) == 0 {
- return nil
- }
- added := make([]int, 0, len(chunks))
- // One critical section for the whole merge: the dedup is against the live
- // pool, so a concurrent session cannot slip the same chunk in between.
- k.Admit(func(p *PoolAdmitter) {
- for _, c := range chunks {
- p.Add(c)
- // Deduped chunks get an entry too: the return value is positional.
- added = append(added, p.Index(c))
- }
- })
- k.MergeDocAggs(aggs)
- return added
-}
-
-// RetireClaimsCoveredBy removes claim pseudo-chunks whose source_chunk_ids
-// intersect the given read ids: a deep read
-// COVERS its claims — once the full chunk text is in the pool, the claim's
-// 1200-char quote of the same passage is duplicated tokens in every later
-// prompt. The claim already did its job (it pointed here). Returns how many
-// entries were retired. The claim_ prefix and the "source_chunk_ids listed
-// under the read ids" test are exact.
-func (k *Kbinfos) RetireClaimsCoveredBy(readIDs []string) int {
- if k == nil || len(readIDs) == 0 {
- return 0
- }
- read := make(map[string]bool, len(readIDs))
- for _, id := range readIDs {
- if id != "" {
- read[id] = true
- }
- }
- k.mu.Lock()
- defer k.mu.Unlock()
- removed := 0
- out := make([]map[string]any, 0, len(k.Chunks))
- for _, c := range k.Chunks {
- id, ok := c["chunk_id"].(string)
- if ok && strings.HasPrefix(id, "claim_") {
- retired := false
- for _, src := range toStringList(c["source_chunk_ids"]) {
- if read[src] {
- retired = true
- break
- }
- }
- if retired {
- removed++
- continue
- }
- }
- out = append(out, c)
- }
- if removed > 0 {
- k.Chunks = out
- }
- return removed
-}
-
-// MergeDocAggs appends doc aggregations, deduplicating by doc_id: the first agg
-// per doc_id wins.
-//
-// Separate from Merge, whose early return on an empty chunk list would skip them:
-// the search tool must record a search's aggregations even when every chunk it
-// returned was already pooled.
-func (k *Kbinfos) MergeDocAggs(aggs []map[string]any) {
- if k == nil || len(aggs) == 0 {
- return
- }
- k.mu.Lock()
- defer k.mu.Unlock()
- dseen := make(map[string]bool, len(k.DocAggs)+len(aggs))
- for _, d := range k.DocAggs {
- dseen[docAggKey(d)] = true
- }
- for _, d := range aggs {
- key := docAggKey(d)
- if dseen[key] {
- continue
- }
- dseen[key] = true
- k.DocAggs = append(k.DocAggs, d)
- }
-}
-
-// docAggKey is the dedup key for doc_aggs: the raw doc_id, with a missing/empty doc_id
-// mapping to "" (so only the first such agg is kept).
-func docAggKey(d map[string]any) string {
- if id, ok := d["doc_id"].(string); ok {
- return id
- }
- return ""
-}
-
-// indexOfChunk returns the global index of the chunk whose key matches kk.
-func indexOfChunk(chunks []map[string]any, kk string) int {
- for i, c := range chunks {
- if chunkKey(c) == kk {
- return i
- }
- }
- return -1
-}
-
-// chunkText is the searchable text of a chunk, preferring the weighted (reranked)
-// "content_with_weight" over the raw "content", then falling back to "text". Both the
-// chunk-utils and grep-sed-narrow readers use this order; the memory variant is two-level
-// (no "text") and is deliberately NOT what this mirrors.
-func chunkText(c map[string]any) string {
- if t, ok := c["content_with_weight"].(string); ok && t != "" {
- return t
- }
- if t, ok := c["content"].(string); ok && t != "" {
- return t
- }
- if t, ok := c["text"].(string); ok {
- return t
- }
- return ""
-}
-
-// chunkKey returns a stable dedup key for a chunk. Prefers chunk_id/id when
-// present; otherwise falls back to a content-derived key so chunks without an
-// id still dedup correctly (and never collide on a shared empty key).
-//
-// The content branch reuses chunkText — the SAME alias chain the rest of the
-// runtime reads (content_with_weight -> content -> text). Reading only the first
-// two made two id-less chunks that carry just "text" fall through to the
-// doc-level fallback and share one key, so Merge/MemoryAdd discarded distinct
-// evidence.
-//
-// A memory-address fallback is the alternative: with `id(ck)` as the final key, two
-// equivalent chunks returned as distinct objects never share a key, so deduplication silently
-// fails for id-less chunks. Keying on the chunk text instead means identical content still
-// merges across retrieval calls.
-func chunkKey(c map[string]any) string {
- if id, ok := c["chunk_id"].(string); ok && id != "" {
- return "cid:" + id
- }
- if id, ok := c["id"].(string); ok && id != "" {
- return "id:" + id
- }
- content := chunkText(c)
- if content == "" {
- // No stable identity: fall back to the doc reference so at least
- // per-document grouping is preserved (rarely reached).
- content = fmt.Sprintf("%s|%s", anyString(c["doc_id"]), anyString(c["docnm_kwd"]))
- }
- return "h:" + fnv64(content)
-}
-
-func anyString(v any) string {
- if s, ok := v.(string); ok {
- return s
- }
- return ""
-}
-
-// fnv64 is a deterministic non-crypto hash for dedup keys.
-func fnv64(s string) string {
- h := uint64(14695981039346656037)
- for i := 0; i < len(s); i++ {
- h ^= uint64(s[i])
- h *= 1099511628211
- }
- return fmt.Sprintf("%016x", h)
-}
diff --git a/internal/rag/agentic-rag/runtime/kbinfos_test.go b/internal/rag/agentic-rag/runtime/kbinfos_test.go
deleted file mode 100644
index 4205896bb4..0000000000
--- a/internal/rag/agentic-rag/runtime/kbinfos_test.go
+++ /dev/null
@@ -1,384 +0,0 @@
-package runtime
-
-import (
- "fmt"
- "sync"
- "testing"
- "time"
-)
-
-// TestMergeSkipsAllWhenNoChunks: early return:
-// `if not result or not result.get("chunks"): return`. When the incoming chunk
-// list is empty, neither chunks nor doc_aggs are merged, and the call returns
-// nil (no global indices contributed).
-func TestMergeSkipsAllWhenNoChunks(t *testing.T) {
- kb := &Kbinfos{DocAggs: []map[string]any{{"doc_id": "d1"}}}
- added := kb.Merge(nil, []map[string]any{{"doc_id": "d2"}})
- if added != nil {
- t.Fatalf("Merge with no chunks should return nil, got %v", added)
- }
- if len(kb.DocAggs) != 1 {
- t.Fatalf("doc_aggs must not be merged without chunks, got %d", len(kb.DocAggs))
- }
- if len(kb.Chunks) != 0 {
- t.Fatalf("chunks should stay empty, got %d", len(kb.Chunks))
- }
-}
-
-// TestMergeCollapsesEmptyDocIDAggs: the dedup set is built from the raw doc_id value, so a
-// missing/empty doc_id still enters the set and a second empty-doc_id agg is dropped. Only the
-// first is kept.
-func TestMergeCollapsesEmptyDocIDAggs(t *testing.T) {
- kb := &Kbinfos{}
- added := kb.Merge(
- []map[string]any{{"id": "c1"}},
- []map[string]any{{"name": "a"}, {"name": "b"}, {"name": "c"}},
- )
- if len(added) != 1 {
- t.Fatalf("added = %v, want 1 chunk index", added)
- }
- if len(kb.DocAggs) != 1 {
- t.Fatalf("empty-doc_id aggs should collapse to 1, got %d", len(kb.DocAggs))
- }
- if kb.DocAggs[0]["name"] != "a" {
- t.Fatalf("first empty-doc_id agg should be kept, got %v", kb.DocAggs[0])
- }
-}
-
-// TestMergeDedupsSameDocID covers the exact-match dedup for real doc_ids, independent of the
-// empty bucket above.
-func TestMergeDedupsSameDocID(t *testing.T) {
- kb := &Kbinfos{}
- kb.Merge([]map[string]any{{"id": "c1"}}, []map[string]any{{"doc_id": "x"}, {"doc_id": "x"}})
- if len(kb.DocAggs) != 1 {
- t.Fatalf("same-doc_id aggs should dedup, got %d", len(kb.DocAggs))
- }
-}
-
-// TestChunkKeyUsesTextFallback pins that the dedup key reads the SAME alias chain
-// as chunkText (content_with_weight -> content -> text). Reading only the first
-// two made two id-less, text-only chunks from one document share the doc-level
-// fallback key, so Merge/MemoryAdd discarded distinct evidence.
-func TestChunkKeyUsesTextFallback(t *testing.T) {
- a := map[string]any{"text": "alpha", "doc_id": "d1", "docnm_kwd": "doc"}
- b := map[string]any{"text": "beta", "doc_id": "d1", "docnm_kwd": "doc"}
- if chunkKey(a) == chunkKey(b) {
- t.Fatalf("distinct text-only chunks must not share a key: %q", chunkKey(a))
- }
- // The same text under any alias must produce the same key.
- if c := map[string]any{"content": "alpha"}; chunkKey(a) != chunkKey(c) {
- t.Errorf("text and content aliases must agree: %q vs %q", chunkKey(a), chunkKey(c))
- }
- // A chunk with no text at all still lands on the doc-level fallback.
- if got := chunkKey(map[string]any{"doc_id": "d1", "docnm_kwd": "doc"}); got == "" {
- t.Error("doc-level fallback key must not be empty")
- }
-}
-
-// TestMergeKeepsDistinctTextOnlyChunks pins the end-to-end consequence for the
-// evidence pool: two id-less chunks that share only a document must both survive.
-func TestMergeKeepsDistinctTextOnlyChunks(t *testing.T) {
- kb := &Kbinfos{}
- added := kb.Merge([]map[string]any{
- {"text": "alpha", "doc_id": "d1", "docnm_kwd": "doc"},
- {"text": "beta", "doc_id": "d1", "docnm_kwd": "doc"},
- }, nil)
- if len(kb.Chunks) != 2 {
- t.Fatalf("chunks = %d, want 2 (distinct text-only evidence must not collapse)", len(kb.Chunks))
- }
- if len(added) != 2 {
- t.Fatalf("added = %v, want 2 global indices", added)
- }
- // Re-merging identical content still dedups.
- kb.Merge([]map[string]any{{"text": "alpha", "doc_id": "d1", "docnm_kwd": "doc"}}, nil)
- if len(kb.Chunks) != 2 {
- t.Fatalf("identical text must still dedup, chunks = %d", len(kb.Chunks))
- }
-}
-
-// TestMemoryAddKeepsDistinctTextOnlyChunks pins the same consequence for the
-// lossless memory store.
-func TestMemoryAddKeepsDistinctTextOnlyChunks(t *testing.T) {
- kb := &Kbinfos{}
- MemoryAdd(kb, []map[string]any{
- {"text": "alpha", "doc_id": "d1", "docnm_kwd": "doc"},
- {"text": "beta", "doc_id": "d1", "docnm_kwd": "doc"},
- })
- if len(kb.Memory) != 2 {
- t.Fatalf("memory = %d, want 2 (distinct text-only evidence must not collapse)", len(kb.Memory))
- }
- MemoryAdd(kb, []map[string]any{{"text": "alpha", "doc_id": "d1", "docnm_kwd": "doc"}})
- if len(kb.Memory) != 2 {
- t.Fatalf("identical text must still dedup, memory = %d", len(kb.Memory))
- }
-}
-
-// TestKbinfosAdmitIsAtomic pins the pool's invariants under the concurrency the
-// pool actually sees: one round's sessions run at the same time and share ONE
-// Kbinfos (SessionDeps.KB; RunSlotResearchPass starts them as goroutines).
-//
-// Both invariants used to come for free from cooperative scheduling — an await-free admit
-// stretch cannot be preempted — so the same stretch has to be locked here.
-//
-// The pool starts one chunk short of the cap, and both sessions offer the SAME
-// first chunk. Whichever batch wins the lock pools it — once — and the cap then
-// stops every further chunk, so the assertions hold whatever the schedule does:
-// they fail only when the batches interleave (which is what an unlocked
-// check-then-append does: both sessions read 59 and both append).
-func TestKbinfosAdmitIsAtomic(t *testing.T) {
- pool := &Kbinfos{}
- for i := 0; i < evidencePoolCap-1; i++ {
- pool.Admit(func(p *PoolAdmitter) {
- p.Add(map[string]any{"chunk_id": fmt.Sprintf("p%d", i)})
- })
- }
- if len(pool.Chunks) != evidencePoolCap-1 {
- t.Fatalf("fixture pool = %d chunks, want %d", len(pool.Chunks), evidencePoolCap-1)
- }
-
- var wg sync.WaitGroup
- for _, tag := range []string{"a", "b"} {
- wg.Add(1)
- go func(tag string) {
- defer wg.Done()
- pool.Admit(func(p *PoolAdmitter) {
- for _, c := range []map[string]any{
- {"chunk_id": "shared"},
- {"chunk_id": tag},
- } {
- if p.Full() {
- continue
- }
- p.Add(c)
- }
- })
- }(tag)
- }
- wg.Wait()
-
- if len(pool.Chunks) != evidencePoolCap {
- t.Errorf("pool = %d chunks, want exactly the cap %d: a serialized batch stops at the cap, an interleaved one overshoots",
- len(pool.Chunks), evidencePoolCap)
- }
- counts := map[string]int{}
- for _, c := range pool.Chunks {
- id, _ := c["chunk_id"].(string)
- counts[id]++
- }
- if counts["shared"] != 1 {
- t.Errorf("chunk %q pooled %d time(s), want exactly 1: the pool must be deduped against its LIVE contents", "shared", counts["shared"])
- }
- if counts["a"]+counts["b"] != 0 {
- t.Errorf("session-private chunks were pooled (%d a, %d b) although only one slot was free", counts["a"], counts["b"])
- }
-}
-
-// TestToolCacheIsConcurrencySafe exercises the cache the way a round does: the
-// SAME instance handed to concurrent sessions (one cache per round, passed to every session).
-// Run under -race this is what a bare map cannot survive — concurrent map writes
-// are a fatal error in Go, not merely a race report.
-func TestToolCacheIsConcurrencySafe(t *testing.T) {
- cache := NewToolCache()
- var wg sync.WaitGroup
- for i := 0; i < 16; i++ {
- wg.Add(1)
- go func(i int) {
- defer wg.Done()
- key := fmt.Sprintf("tool-%d", i%4) // deliberate key collisions
- if _, ok := cache.Get(key); !ok {
- cache.Put(key, ToolOutcome{Reason: key})
- }
- }(i)
- }
- wg.Wait()
-
- for i := 0; i < 4; i++ {
- key := fmt.Sprintf("tool-%d", i)
- oc, ok := cache.Get(key)
- if !ok || oc.Reason != key {
- t.Errorf("cache[%q] = (%+v, %v), want the stored outcome", key, oc, ok)
- }
- }
-}
-
-// TestPoolNoveltyExemptsOnlyTheUnansweredProbeTerms pins the cap EXEMPTION: a
-// FULL pool still takes the passage that answers a probe term nothing in the pool
-// has reached yet — one seat per term — and refuses everything else exactly as the
-// cap always did.
-//
-// The reason the exemption exists is that a probe's per-name window is the batch's
-// RESULT rather than one more passage: drop it and "this name was found here"
-// degrades into "nothing new", which the model reads as "not a member".
-func TestPoolNoveltyExemptsOnlyTheUnansweredProbeTerms(t *testing.T) {
- newFullPool := func() *Kbinfos {
- pool := &Kbinfos{}
- for i := 0; i < evidencePoolCap; i++ {
- pool.Admit(func(p *PoolAdmitter) {
- p.Add(map[string]any{"chunk_id": fmt.Sprintf("p%d", i), "content": "already pooled prose"})
- })
- }
- return pool
- }
- admit := func(pool *Kbinfos, q string, chunks ...map[string]any) (admitted int) {
- pool.Admit(func(p *PoolAdmitter) {
- novel := p.Novelty(probeTerms(q))
- for _, c := range chunks {
- if p.Full() && !novel.Admits(c) {
- continue
- }
- if p.Add(c) {
- admitted++
- }
- }
- })
- return admitted
- }
-
- // A probe whose name the pool cannot answer: the window is admitted.
- pool := newFullPool()
- window := map[string]any{"chunk_id": "w1", "content": "荀正 被关公一刀斩于马下"}
- if got := admit(pool, "车胄|荀正|管亥", window); got != 1 {
- t.Fatalf("admitted = %d, want 1 (the pool cannot answer 荀正)", got)
- }
- if len(pool.Chunks) != evidencePoolCap+1 {
- t.Fatalf("pool = %d, want %d (one seat for one unanswered term)", len(pool.Chunks), evidencePoolCap+1)
- }
- // The same term is answered now: a second window carrying only it is refused,
- // so one term buys one seat and not a batch of them.
- if got := admit(pool, "车胄|荀正|管亥", map[string]any{"chunk_id": "w2", "content": "荀正 又出现了一次"}); got != 0 {
- t.Fatalf("admitted = %d, want 0: 荀正 is answered now", got)
- }
- // A query that is not a PROBE states no list of individuals, so it gets no
- // exemption at all — the plain cap behaviour.
- if got := admit(pool, "车胄", map[string]any{"chunk_id": "w3", "content": "车胄 守徐州"}); got != 0 {
- t.Fatalf("admitted = %d, want 0: a non-probe query gets no exemption", got)
- }
- if got := admit(pool, "车胄|管亥", map[string]any{"chunk_id": "w4", "content": "管亥 围北海"}); got != 1 {
- t.Fatalf("admitted = %d, want 1 (管亥 is still unanswered)", got)
- }
- // A pool that already carries the term leaves the cap in charge.
- carried := newFullPool()
- carried.Admit(func(p *PoolAdmitter) { p.Add(map[string]any{"chunk_id": "seed", "content": "管亥 围北海"}) })
- if got := admit(carried, "管亥", map[string]any{"chunk_id": "w5", "content": "管亥 围北海 续"}); got != 0 {
- t.Fatalf("admitted = %d, want 0: the pool already carries 管亥", got)
- }
- _ = carried
-
- // The exemption is BOUNDED: past cap+slack even an unanswered term is refused.
- bounded := newFullPool()
- bounded.novelAdmitted = evidencePoolNoveltySlack
- bounded.Chunks = append(bounded.Chunks, make([]map[string]any, evidencePoolNoveltySlack)...)
- if got := admit(bounded, "杨龄", map[string]any{"chunk_id": "w6", "content": "杨龄 出战"}); got != 0 {
- t.Fatalf("admitted = %d, want 0: the novelty slack is exhausted", got)
- }
-}
-
-// TestReachedTermsLedgerHoldsTheMembersWithTheirEvidence pins the confirmed-member
-// ledger: it is what the rewrite context renders (the passage behind each name) and
-// what lets the loop ask whether the LIST is growing rather than whether the pool
-// got bigger.
-func TestReachedTermsLedgerHoldsTheMembersWithTheirEvidence(t *testing.T) {
- pool := &Kbinfos{}
- pool.RecordReachedTerm("荀正", "w1")
- pool.RecordReachedTerm("杨龄", "w2")
- pool.RecordReachedTerm("荀正", "w3") // same name, later passage: must not duplicate
-
- got := pool.ReachedTerms()
- if len(got) != 2 {
- t.Fatalf("ReachedTerms = %v, want one entry per name", got)
- }
- if got[0].Term != "荀正" || got[0].ChunkID != "w1" {
- t.Errorf("first entry = %+v, want the name with the passage that proved it", got[0])
- }
- if got[1].Term != "杨龄" || got[1].ChunkID != "w2" {
- t.Errorf("second entry = %+v", got[1])
- }
- // Blank input records nothing (a seat without an id is not evidence).
- pool.RecordReachedTerm("", "w4")
- pool.RecordReachedTerm("管亥", "")
- if n := len(pool.ReachedTerms()); n != 2 {
- t.Errorf("entries = %d, want 2: an entry needs both a name and its passage", n)
- }
- // The ledger is bounded, and a nil pool is safe (the graph may run without one).
- for i := 0; i < reachedTermsMax+5; i++ {
- pool.RecordReachedTerm(fmt.Sprintf("n%d", i), fmt.Sprintf("c%d", i))
- }
- if n := len(pool.ReachedTerms()); n != reachedTermsMax {
- t.Errorf("entries = %d, want the cap %d", n, reachedTermsMax)
- }
- var nilPool *Kbinfos
- nilPool.RecordReachedTerm("x", "y")
- if len(nilPool.ReachedTerms()) != 0 {
- t.Error("a nil pool must record nothing")
- }
-}
-
-// TestRecordingInsideTheAdmitBatchDoesNotDeadlock pins the hardened ledger: the
-// search record lives behind its OWN mutex, so a batch that answered a probe can
-// write the answer down while the pool lock is held.
-//
-// This is the exact shape that hung a run (fixrecall2, 2026-09-15): the recorder
-// used to take the same mutex Admit holds, and because Go mutexes are not
-// reentrant the goroutine parked on a lock it already held — no I/O, no CPU, no
-// log line, and a mutex wait ignores context cancellation, so the run never
-// returned. The timeout here turns that hang into a failure message.
-func TestRecordingInsideTheAdmitBatchDoesNotDeadlock(t *testing.T) {
- k := &Kbinfos{}
- done := make(chan struct{})
- go func() {
- defer close(done)
- k.Admit(func(p *PoolAdmitter) {
- p.Add(map[string]any{"chunk_id": "w1", "content": "荀正 被关公一刀斩于马下"})
- // Both recorders run under the pool lock on purpose.
- k.RecordReachedTerm("荀正", "w1")
- k.RecordProbedAbsent("管亥")
- })
- }()
- select {
- case <-done:
- case <-time.After(3 * time.Second):
- t.Fatal("recording from inside the admit batch deadlocked: the ledger must not share the pool's mutex")
- }
-
- if got := k.ReachedTerms(); len(got) != 1 || got[0].Term != "荀正" || got[0].ChunkID != "w1" {
- t.Errorf("ReachedTerms = %+v, want the member recorded with its passage", got)
- }
- if absent := k.ProbedAbsentTerms(); len(absent) != 1 || absent[0] != "管亥" {
- t.Errorf("ProbedAbsent = %v, want 管亥", absent)
- }
- // The pool itself is unaffected by the ledger's separate lock.
- if len(k.Chunks) != 1 {
- t.Errorf("pool = %d chunk(s), want the admitted passage", len(k.Chunks))
- }
-}
-
-// TestLedgerAccessorsAreIndependentOfThePoolLock pins the other direction: a
-// reader of the record never waits for a batch that is in flight, which is what
-// lets the rewrite context be rendered while sessions are still admitting.
-func TestLedgerAccessorsAreIndependentOfThePoolLock(t *testing.T) {
- k := &Kbinfos{}
- k.RecordReachedTerm("华雄", "w1")
-
- released := make(chan struct{})
- go func() {
- k.Admit(func(p *PoolAdmitter) {
- <-released // hold the pool lock until the reads below are done
- })
- }()
- // Give the batch a moment to take the pool lock.
- time.Sleep(50 * time.Millisecond)
-
- done := make(chan struct{})
- go func() {
- defer close(done)
- _ = k.ReachedTerms()
- _ = k.ProbedAbsentTerms()
- }()
- select {
- case <-done:
- case <-time.After(3 * time.Second):
- close(released)
- t.Fatal("reading the record blocked on the pool lock: the accessors must not need it")
- }
- close(released)
-}
diff --git a/internal/rag/agentic-rag/runtime/keywords.go b/internal/rag/agentic-rag/runtime/keywords.go
deleted file mode 100644
index 50545605d2..0000000000
--- a/internal/rag/agentic-rag/runtime/keywords.go
+++ /dev/null
@@ -1,284 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "fmt"
- "regexp"
- "strings"
-
- "github.com/cloudwego/eino/schema"
-
- "ragflow/internal/agent/chat"
-)
-
-// Four-aspect keyword extraction with entity weighting.
-//
-// A keyword search matches only the surface forms you give it, so a single flat
-// bag of terms is a poor retrieval driver: the model does not know how the corpus
-// phrases a fact. This extraction asks the LLM for FOUR aspects — `entity` (what
-// the fact is about), `aliases` (its surface variants), `fact_type` (words the
-// corpus might use for this kind of fact, including table column abbreviations)
-// and `qualifiers` (year / edition / jurisdiction / revision).
-//
-// `entity` is what discriminates, so it is repeated in the search query to weight
-// BM25 toward it; the plain deduped union of all four aspects is used to narrow
-// retrieved chunks to their keyword-bearing sentences.
-
-const (
- // keywordEntityRepeat is copies of each entity term in the query.
- keywordEntityRepeat = 3
- // keywordQualifierRepeat is copies of each qualifier, weighted up like entity.
- keywordQualifierRepeat = 3
- // keywordMaxChars caps both strings.
- keywordMaxChars = 400
- // keywordExtractionTemperature pins the temperature for the extraction call.
- keywordExtractionTemperature = 0.1
-)
-
-// keywordAspects is the aspect order. Entity and qualifiers are the weighted
-// pair; aliases and fact_type find/boost but must not dominate.
-var keywordAspects = []string{"entity", "aliases", "fact_type", "qualifiers"}
-
-// KeywordsSystem mirrors keywords.py::_KEYWORDS_SYSTEM.
-const KeywordsSystem = `You turn ONE question into search terms for a keyword/BM25 search engine.
-
-Emit the terms that would appear VERBATIM in a document that answers the question, sorted into FOUR
-categories. Every term must come from the question itself or be a surface form of something in it.
-
-A. "entity" — the specific thing the fact is ABOUT: proper nouns, titles, identifiers. Keep a
- multi-word entity whole, as ONE term ("Brown County", "Treaty of Versailles"); split across
- several terms its tokens match independently and drag in noise. A bare identifier — a serial,
- patent, catalogue or case number — is a complete entity on its own; never glue it to the words
- around it.
-B. "aliases" — the engine matches ONLY the surface forms you supply, so emit the plausible variants
- of A: full vs. short name, native-language and transliterated forms, official vs. common name,
- acronym and its expansion, and the qualified form ("Brown County" -> "Brown County, Kansas").
-C. "fact_type" — 3 to 6 words the corpus might use for this KIND of fact, since you cannot know how
- it is phrased. Spread them across registers:
- quantity of people -> population, inhabitants, residents, census, demographics, headcount
- time of an event -> founded, established, opened, dated, began
- role of a person -> served, appointed, elected, held, director
- SOURCES TABULATE WHAT QUESTIONS SPELL OUT: a statistic named in prose is usually written in a
- table as a column abbreviation, and the prose wording may not appear in the document at all. So
- include the abbreviation a table would use — "points per game" -> "PPG", "PTS"; "earnings per
- share" -> "EPS"; "games played" -> "GP" — and reach a superlative through its plain column too:
- "leading scorer" is found by looking for "PTS" and "PPG", not for the phrase itself.
-D. "qualifiers" — year, edition, jurisdiction, revision. Worth emitting even when it looks
- redundant: the qualifier often sits in a table header or a document title that chunking has
- severed from the value. Include EVERY alternative expression of a DATE or NUMBER in the
- question — ordinals and their words ("21st" -> "twenty-first"), digits and their words
- ("2000000" -> "two million", "2 million"), and each common date format ("Aug 2nd" -> "August 2",
- "2 August", "08-02").
-
-A and B are what FINDS the document; C and D only boost the ranking. So never withhold an entity
-because you are unsure of it, and never pad C or D to reach a count.
-
-DROP entirely: question words ("which", "who", "when", "how many"), relational scaffolding, and
-generic high-frequency nouns ("year", "number", "city", "total", "list", "information"). They cost
-ranking quality and retrieve nothing.
-
-Output ONLY JSON, no prose, no code fences:
-{"entity": ["<term>", ...], "aliases": ["<term>", ...], "fact_type": ["<term>", ...], "qualifiers": ["<term>", ...]}
-Any category may be empty.`
-
-// normKeyword: normalise a term for cross-category
-// dedup (lowercase, whitespace-collapsed).
-func normKeyword(s string) string {
- return strings.Join(strings.Fields(strings.ToLower(s)), " ")
-}
-
-// parseAspects: parse the LLM's JSON into one
-// deduped list per aspect.
-//
-// ONE dedup set spans all four categories: a term the model emits as both an
-// entity and an alias must not collect a second share of the query's mass on the
-// strength of having been named twice.
-func parseAspects(raw string) map[string][]string {
- data, _ := ExtractJSON(StripThinkAndFences(raw)).(map[string]any)
- aspects := map[string][]string{}
- seen := map[string]bool{}
- for _, aspect := range keywordAspects {
- var terms []string
- for _, k := range asAspectList(data[aspect]) {
- term := strings.TrimSpace(fmt.Sprint(k))
- key := normKeyword(term)
- if term == "" || key == "" || seen[key] {
- continue
- }
- seen[key] = true
- terms = append(terms, term)
- }
- aspects[aspect] = terms
- }
- return aspects
-}
-
-// asAspectList tolerates the shapes models emit for an aspect: a list of
-// strings, a list of mixed values, or a single comma-separated string.
-func asAspectList(v any) []any {
- switch t := v.(type) {
- case []any:
- return t
- case []string:
- out := make([]any, 0, len(t))
- for _, s := range t {
- out = append(out, s)
- }
- return out
- case string:
- out := make([]any, 0, 4)
- for _, s := range strings.Split(t, ",") {
- out = append(out, s)
- }
- return out
- }
- return nil
-}
-
-// reThinkWrap matches a leading <think>...</think> preamble.
-var reThinkWrap = regexp.MustCompile(`(?s)^.*</think>`)
-
-// StripThinkAndFences removes a leading thinking preamble and Markdown fences.
-// Exported so the runtime (compute.go, keywords.go) and the agentic_rag package's
-// moved Formalize can share one implementation.
-func StripThinkAndFences(s string) string {
- s = reThinkWrap.ReplaceAllString(s, "")
- s = reFencedJSON.ReplaceAllString(s, "$1")
- return strings.TrimSpace(s)
-}
-
-// ExtractWeightedKeywords: extract the
-// four aspects and return (query, keywords).
-//
-// - query is the weighted search string: every entity term repeated x3 and
-// every qualifier repeated x3, so BM25 weights both inside the same query;
-// aliases and fact-type vocabulary appear once each.
-// - keywords is the plain deduped union of all four aspects (one copy each),
-// used to narrow retrieved chunks to their keyword-bearing sentences.
-//
-// Falls back to (question, question) when extraction fails — keyword extraction
-// is an enhancement, never a precondition for retrieval.
-func ExtractWeightedKeywords(ctx context.Context, model SessionModel, question string) (string, string) {
- if question == "" {
- return "", ""
- }
- aspects := map[string][]string{}
- if model != nil {
- // The question is fitted to the model context window before the call, so an over-long
- // question is trimmed rather than rejected. The context length is exposed via
- // ContextLengthModel; when none is available the chat.EffectiveContextLength(0)
- // default (8192) applies.
- budget := 0
- if cl, ok := model.(ContextLengthModel); ok {
- budget = cl.ContextLength()
- }
- if budget <= 0 {
- budget = chat.EffectiveContextLength(0)
- }
- fitted, fitErr := chat.FitMessages(KeywordsSystem, []schema.Message{
- *schema.UserMessage(question),
- }, budget)
- if fitErr != "" {
- _LOG.Printf("[Keywords] prompt fitting failed: %s", fitErr)
- }
- // FitMessages may prepend/trim a system message; re-extract it so the
- // model call is exactly [system, user...].
- systemPrompt := KeywordsSystem
- if len(fitted) > 0 && fitted[0].Role == schema.System {
- systemPrompt = fitted[0].Content
- }
- userContent := question
- for _, m := range fitted {
- if m.Role == schema.User {
- userContent = m.Content
- break
- }
- }
- msgs := []schema.Message{
- *schema.SystemMessage(systemPrompt),
- *schema.UserMessage(userContent),
- }
- // The temperature is pinned to 0.1 — a mechanical rewrite, not a reasoning task, so it
- // must be stable. The value is a
- // pinned constant: every production carrier implements TemperatureModel
- // (compile-time assertion on InvokerSessionModel), so the temperature is
- // always sent; a carrier without per-call temperature support falls back
- // to its own default (Go-only provider limitation, flagged).
- var reply *ModelReply
- var err error
- if tm, ok := model.(TemperatureModel); ok {
- reply, err = tm.CompleteWithTemperature(ctx, msgs, nil, keywordExtractionTemperature)
- } else {
- _LOG.Printf("[Keywords] model %T cannot carry per-call temperature; using its default (wanted %v)", model, keywordExtractionTemperature)
- reply, err = model.Complete(ctx, msgs, nil)
- }
- if err == nil {
- aspects = parseAspects(reply.Content)
- } else {
- _LOG.Printf("[Keywords] extraction failed: %v", err)
- }
- }
-
- // Plain union, one copy each — used for narrowing.
- var union []string
- for _, aspect := range keywordAspects {
- union = append(union, aspects[aspect]...)
- }
- keywords := strings.Join(union, ", ")
- if keywords == "" {
- keywords = question
- }
-
- // Weighted query: entity x3, qualifiers x3, then aliases + fact_type once.
- var weighted []string
- for _, t := range aspects["entity"] {
- for i := 0; i < keywordEntityRepeat; i++ {
- weighted = append(weighted, t)
- }
- }
- for _, t := range aspects["qualifiers"] {
- for i := 0; i < keywordQualifierRepeat; i++ {
- weighted = append(weighted, t)
- }
- }
- for _, aspect := range []string{"aliases", "fact_type"} {
- weighted = append(weighted, aspects[aspect]...)
- }
- query := strings.Join(weighted, ", ")
- if query == "" {
- query = keywords
- }
-
- // There is NO term-count cap — only the keywordMaxChars (400) hard cap on the final joined
- // strings: only the character cap is applied, never a per-term limit.
- query = truncateRunes(query, keywordMaxChars)
- keywords = truncateRunes(keywords, keywordMaxChars)
-
- _LOG.Printf("[Keywords] entity x%d: %s | aliases: %s | fact-type: %s | qualifiers x%d: %s",
- keywordEntityRepeat, joinOrDash(aspects["entity"]), joinOrDash(aspects["aliases"]),
- joinOrDash(aspects["fact_type"]), keywordQualifierRepeat, joinOrDash(aspects["qualifiers"]))
- return query, keywords
-}
-
-func joinOrDash(ss []string) string {
- if len(ss) == 0 {
- return "-"
- }
- return strings.Join(ss, "; ")
-}
diff --git a/internal/rag/agentic-rag/runtime/keywords_test.go b/internal/rag/agentic-rag/runtime/keywords_test.go
deleted file mode 100644
index 45a003178f..0000000000
--- a/internal/rag/agentic-rag/runtime/keywords_test.go
+++ /dev/null
@@ -1,190 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "errors"
- "strings"
- "testing"
-
- "github.com/cloudwego/eino/schema"
-)
-
-func TestNormKeyword(t *testing.T) {
- cases := map[string]string{
- "Brown County": "brown county",
- " OMIYA Soft ": "omiya soft",
- "": "",
- }
- for in, want := range cases {
- if got := normKeyword(in); got != want {
- t.Errorf("normKeyword(%q) = %q, want %q", in, got, want)
- }
- }
-}
-
-func TestParseAspectsDedupsAcrossCategories(t *testing.T) {
- // ONE dedup set spans all four categories: a term emitted as both an entity
- // and an alias must not collect a second share of the query's mass purely
- // for having been named twice.
- raw := `{"entity": ["Brown County", "Kansas"], "aliases": ["brown county", "Brown County, Kansas"],
- "fact_type": ["population", "census"], "qualifiers": ["2020"]}`
- got := parseAspects(raw)
- if len(got["entity"]) != 2 {
- t.Errorf("entity = %v, want 2 terms", got["entity"])
- }
- // "brown county" (alias) is dropped: already seen as an entity.
- if len(got["aliases"]) != 1 || got["aliases"][0] != "Brown County, Kansas" {
- t.Errorf("aliases = %v, want only the qualified form", got["aliases"])
- }
- if len(got["fact_type"]) != 2 || len(got["qualifiers"]) != 1 {
- t.Errorf("fact_type/qualifiers = %v/%v", got["fact_type"], got["qualifiers"])
- }
-}
-
-func TestParseAspectsToleratesShapesAndGarbage(t *testing.T) {
- // A comma-separated string instead of a list.
- got := parseAspects(`{"entity": "OmiyaSoft, Culdcept", "aliases": []}`)
- if len(got["entity"]) != 2 {
- t.Errorf("string form = %v, want 2 terms", got["entity"])
- }
- // Unparseable / empty input yields empty aspects, never a panic.
- for _, raw := range []string{"", "not json", "```json {broken```", "[]", "null"} {
- got := parseAspects(raw)
- for _, aspect := range keywordAspects {
- if len(got[aspect]) != 0 {
- t.Errorf("parseAspects(%q) produced %s=%v, want empty", raw, aspect, got[aspect])
- }
- }
- }
-}
-
-func TestExtractWeightedKeywordsWeighting(t *testing.T) {
- mdl := &fakeModel{replies: []*ModelReply{{
- Content: `{"entity": ["Brown County"], "aliases": ["Brown County, Kansas"],
- "fact_type": ["population", "census"], "qualifiers": ["2020"]}`,
- }}}
- query, keywords := ExtractWeightedKeywords(context.Background(), mdl, "What is the population of Brown County?")
-
- // query: entity x3, qualifiers x3, then aliases + fact_type once each.
- // NOTE: the alias "Brown County, Kansas" itself contains a comma, so a naive
- // split on ", " over-counts — the join works the same way and BM25 tokenises on
- // whitespace, so this is expected. Assert by prefix/containment.
- if !strings.HasPrefix(query, "Brown County, Brown County, Brown County, ") {
- t.Errorf("query = %q, want the entity repeated x3 first", query)
- }
- if !strings.Contains(query, "2020, 2020, 2020") {
- t.Errorf("query = %q, want the qualifier repeated x3", query)
- }
- for _, want := range []string{"Brown County, Kansas", "population", "census"} {
- if !strings.Contains(query, want) {
- t.Errorf("query = %q, missing %q (aliases/fact_type appear once)", query, want)
- }
- }
- // keywords: plain union, one copy each, in aspect order.
- wantKeys := "Brown County, Brown County, Kansas, population, census, 2020"
- if keywords != wantKeys {
- t.Errorf("keywords = %q, want %q", keywords, wantKeys)
- }
-}
-
-func TestExtractWeightedKeywordsFallsBackToQuestion(t *testing.T) {
- q := "Who created Culdcept?"
- // No model at all.
- query, keywords := ExtractWeightedKeywords(context.Background(), nil, q)
- if query != q || keywords != q {
- t.Errorf("no model = (%q,%q), want the question for both", query, keywords)
- }
- // A model that fails.
- query, keywords = ExtractWeightedKeywords(context.Background(), &errModel{}, q)
- if query != q || keywords != q {
- t.Errorf("failed model = (%q,%q), want the question for both", query, keywords)
- }
- // A model that returns nothing usable.
- query, keywords = ExtractWeightedKeywords(context.Background(),
- &fakeModel{replies: []*ModelReply{{Content: "garbage"}}}, q)
- if query != q || keywords != q {
- t.Errorf("garbage reply = (%q,%q), want the question for both", query, keywords)
- }
- // Empty question.
- if a, b := ExtractWeightedKeywords(context.Background(), &fakeModel{}, ""); a != "" || b != "" {
- t.Errorf("empty question = (%q,%q), want empty", a, b)
- }
-}
-
-func TestExtractWeightedKeywordsCapsLength(t *testing.T) {
- // A pathological term set must be capped, not truncate mid-term into
- // something that still pollutes the query.
- var terms []string
- for i := 0; i < 100; i++ {
- terms = append(terms, "term"+strings.Repeat("x", 10)+string(rune('a'+i%26)))
- }
- list := `["` + strings.Join(terms, `", "`) + `"]`
- mdl := &fakeModel{replies: []*ModelReply{{
- Content: `{"entity": ` + list + `, "aliases": [], "fact_type": [], "qualifiers": []}`,
- }}}
- query, keywords := ExtractWeightedKeywords(context.Background(), mdl, "q")
- if len(query) > keywordMaxChars {
- t.Errorf("query = %d chars, want <= %d", len(query), keywordMaxChars)
- }
- if len(keywords) > keywordMaxChars {
- t.Errorf("keywords = %d chars, want <= %d", len(keywords), keywordMaxChars)
- }
-}
-
-func TestTemperatureModelExtension(t *testing.T) {
- // Keyword extraction runs at 0.1. The seam exposes temperature through an OPTIONAL
- // interface so existing SessionModel implementations keep working unchanged.
- im := &InvokerSessionModel{}
- var _ TemperatureModel = im // must implement the extension
-
- // A model that is not a TemperatureModel must still work (falls back).
- mdl := &fakeModel{replies: []*ModelReply{{
- Content: `{"entity": ["Culdcept"], "aliases": [], "fact_type": [], "qualifiers": []}`,
- }}}
- query, _ := ExtractWeightedKeywords(context.Background(), mdl, "q")
- if !strings.Contains(query, "Culdcept") {
- t.Errorf("non-temperature model: query = %q, want the entity weighted in", query)
- }
-}
-
-// errModel implements SessionModel but always fails Complete, so the
-// keyword-extraction fallback path (no model usable) can be exercised.
-type errModel struct{}
-
-func (errModel) Complete(_ context.Context, _ []schema.Message, _ []ToolSpec) (*ModelReply, error) {
- return nil, errors.New("simulated model failure")
-}
-
-// fakeModel replays a scripted sequence of replies so control flow can be
-// driven without a provider. Shared by the keyword and arithmetic tests.
-type fakeModel struct {
- replies []*ModelReply
- calls int
- messages []schema.Message
-}
-
-func (f *fakeModel) Complete(_ context.Context, msgs []schema.Message, _ []ToolSpec) (*ModelReply, error) {
- f.messages = msgs
- if f.calls >= len(f.replies) {
- return &ModelReply{Content: `<state>{"new_states": []}</state>`}, nil
- }
- r := f.replies[f.calls]
- f.calls++
- return r, nil
-}
diff --git a/internal/rag/agentic-rag/runtime/memory.go b/internal/rag/agentic-rag/runtime/memory.go
deleted file mode 100644
index c505fd7111..0000000000
--- a/internal/rag/agentic-rag/runtime/memory.go
+++ /dev/null
@@ -1,507 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "regexp"
- "sort"
- "strings"
- "unicode"
-)
-
-// Raw-chunk memory store.
-//
-// The lossless store backing the (lossy) chunk list that feeds the LLM. Retrieval narrows
-// chunks to the sentences that answer the current query, so the raw text has to be kept
-// somewhere: a later gap query often needs a fact the earlier narrowing already threw away.
-//
-// The store lives on Kbinfos.Memory so it travels with the request.
-
-const (
- // grepMaxChunks is the default cap for the memory grep. MemoryGrep takes the limit
- // explicitly and does NOT substitute this for a non-positive value (there is no such
- // guard), so callers that want the default pass it.
- grepMaxChunks = 6
- // grepMaxSentences caps sentences kept per chunk (hit + context).
- grepMaxSentences = 4
- // grepContextChars is the per-side char budget when expanding context.
- grepContextChars = 400
- // shortChunkChars: chunks at or below this length are kept whole, since
- // answers often live in short chunks.
- shortChunkChars = 200
-
- // MemorySearch tuning — the search defaults.
- memoryDefaultTopN = 6
- // memoryMinRatio: a chunk is relevant when it shares >= 1 term AND >= this fraction of
- // the query's significant terms (normalized overlap bar so CN / EN queries behave alike).
- // Only the ratio matters; the absolute min_overlap parameter is not consulted.
- memoryMinRatio = 0.12
- // memoryMaxTerms caps how many significant terms we extract from a query.
- memoryMaxTerms = 18
-)
-
-var (
- // reTermPunct strips leading/trailing punctuation before escaping.
- reTermPunct = regexp.MustCompile(`^[\s.,:;!?'"()\[\]{}]+|[\s.,:;!?'"()\[\]{}]+$`)
- // reCJK detects CJK / kana / hangul, which get no \b anchor (a word boundary
- // never matches between CJK characters).
- reCJK = regexp.MustCompile(`[\x{4e00}-\x{9fff}\x{3040}-\x{30ff}\x{ac00}-\x{d7af}]`)
- // reCJKRun matches a maximal run of CJK/kana/hangul for 3-gram splitting.
- reCJKRun = regexp.MustCompile(`[\x{4e00}-\x{9fff}\x{3040}-\x{30ff}\x{ac00}-\x{d7af}]+`)
- // reLatinNum matches a maximal alphanumeric run (Latin words + numbers).
- reLatinNum = regexp.MustCompile(`[A-Za-z0-9]+`)
- // reDigits matches a pure digit run.
- reDigits = regexp.MustCompile(`\d+`)
-)
-
-// memoryStopwords: dropped from Latin query
-// terms so "what / the / is" style words do not dominate the overlap score.
-var memoryStopwords = map[string]struct{}{
- "what": {}, "which": {}, "how": {}, "many": {}, "much": {}, "does": {},
- "did": {}, "do": {}, "the": {}, "a": {}, "an": {}, "is": {}, "are": {},
- "was": {}, "were": {}, "be": {}, "been": {}, "being": {}, "of": {},
- "for": {}, "to": {}, "in": {}, "on": {}, "with": {}, "and": {}, "or": {},
- "by": {}, "from": {}, "at": {}, "it": {}, "its": {}, "this": {}, "that": {},
- "these": {}, "those": {}, "who": {}, "when": {}, "where": {}, "why": {},
- "than": {}, "then": {}, "there": {}, "their": {}, "they": {}, "them": {},
- "his": {}, "her": {}, "him": {}, "she": {}, "he": {}, "we": {}, "you": {},
- "your": {},
-}
-
-// MemoryAdd: merges raw retrieved chunks into the
-// central store, losslessly, skipping chunks already present and those with no
-// text.
-func MemoryAdd(kb *Kbinfos, chunks []map[string]any) {
- if kb == nil || len(chunks) == 0 {
- return
- }
- // One critical section: the add is a single stretch that cannot be interleaved (the pool
- // is shared across a round's concurrent sessions — see Kbinfos.Admit).
- kb.mu.Lock()
- defer kb.mu.Unlock()
- seen := make(map[string]struct{}, len(kb.Memory))
- for _, c := range kb.Memory {
- seen[chunkKey(c)] = struct{}{}
- }
- added := 0
- for _, c := range chunks {
- if c == nil || chunkText(c) == "" {
- continue
- }
- k := chunkKey(c)
- if _, dup := seen[k]; dup {
- continue
- }
- seen[k] = struct{}{}
- kb.Memory = append(kb.Memory, c)
- added++
- }
- if added > 0 {
- _LOG.Printf("[Memory] stored %d new raw chunk(s); memory now has %d.", added, len(kb.Memory))
- }
-}
-
-// MemorySize
-func MemorySize(kb *Kbinfos) int {
- if kb == nil {
- return 0
- }
- return len(kb.Memory)
-}
-
-// MemoryClear
-func MemoryClear(kb *Kbinfos) {
- if kb != nil {
- kb.mu.Lock()
- defer kb.mu.Unlock()
- kb.Memory = nil
- }
-}
-
-// MemoryGrep: returns memory chunks containing any of
-// terms, narrowed to the matching sentence plus a small context window.
-//
-// terms are plain strings (entities / numbers / key phrases) as emitted by the
-// analysis LLM. Each returned chunk carries a narrowed "content" so the caller
-// can splice it straight into an evidence list. Empty on no-hit / no-memory.
-//
-// limit is the maximum number of chunks returned and is NOT normalized: there is no such
-// guard, so a limit <= 0 makes the `len(hits) >= limit` check fire on the first hit (at most
-// one chunk comes back). Callers wanting the default pass grepMaxChunks.
-func MemoryGrep(kb *Kbinfos, terms []string, limit int) []map[string]any {
- if kb == nil || len(kb.Memory) == 0 || len(terms) == 0 {
- return nil
- }
- patterns, prefixPatterns := compileTerms(terms)
- if len(patterns) == 0 && len(prefixPatterns) == 0 {
- return nil
- }
- match := func(text string) bool {
- for _, p := range patterns {
- if p.MatchString(text) {
- return true
- }
- }
- // Prefix fallback: morphological tolerance. A gap term often differs from
- // the chunk's word by a suffix (gap "abbreviation" vs chunk "abbreviated").
- // Matching the leading stem at the START of a word lets a shared root hit.
- for _, p := range prefixPatterns {
- if p.MatchString(text) {
- return true
- }
- }
- return false
- }
-
- var hits []map[string]any
- for _, c := range kb.Memory {
- text := chunkText(c)
- if len(text) <= shortChunkChars {
- // Short chunk: keep whole (its answer may live anywhere in it).
- if match(text) {
- hits = append(hits, map[string]any{
- "content": text,
- "doc_id": c["doc_id"],
- "chunk_id": c["chunk_id"],
- })
- // This branch continues past the shared cap below, so enforce the limit here too —
- // otherwise a memory store full of matching short chunks comes back whole.
- if len(hits) >= limit {
- break
- }
- }
- continue
- }
- sents := SplitSentences(text)
- var kept []string
- for i, s := range sents {
- if match(s) {
- for _, w := range sentenceSpanWindow(sents, i) {
- if !containsStr(kept, w) {
- kept = append(kept, w)
- }
- }
- }
- if len(kept) >= grepMaxSentences {
- break
- }
- }
- if len(kept) > 0 {
- hits = append(hits, map[string]any{
- "content": strings.Join(kept, "\n"),
- "doc_id": c["doc_id"],
- "chunk_id": c["chunk_id"],
- })
- }
- if len(hits) >= limit {
- break
- }
- }
- return hits
-}
-
-// compileTerms: pattern construction: the escaped
-// term (word-anchored when it is long enough and wholly alphanumeric) plus the
-// leading-stem prefix pattern used as a fallback.
-func compileTerms(terms []string) (patterns, prefixPatterns []*regexp.Regexp) {
- for _, t := range terms {
- if frag := escapeTerm(t); frag != "" {
- if p, err := regexp.Compile("(?i)" + frag); err == nil {
- patterns = append(patterns, p)
- }
- }
- stripped := strings.TrimSpace(t)
- prefix := ""
- if len([]rune(stripped)) >= 6 {
- prefix = string([]rune(stripped)[:5])
- }
- if prefix != "" && !reCJK.MatchString(prefix) {
- if p, err := regexp.Compile(`(?i)\b` + regexp.QuoteMeta(prefix)); err == nil {
- prefixPatterns = append(prefixPatterns, p)
- }
- }
- }
- return patterns, prefixPatterns
-}
-
-// escapeTerm: strip surrounding punctuation, escape
-// regex metacharacters, then anchor on word boundaries — except for CJK, where
-// a \b anchor would never match.
-func escapeTerm(term string) string {
- t := strings.TrimSpace(term)
- if t == "" {
- return ""
- }
- t = reTermPunct.ReplaceAllString(t, "")
- if t == "" {
- return ""
- }
- escaped := regexp.QuoteMeta(t)
- if reCJK.MatchString(t) {
- return escaped
- }
- rs := []rune(t)
- if len(rs) >= 3 && isAlnum(rs[0]) && isAlnum(rs[len(rs)-1]) {
- return `\b` + escaped + `\b`
- }
- return escaped
-}
-
-// sentenceSpanWindow: the hit sentence plus
-// up to one neighbour on each side, clamped to a total length.
-func sentenceSpanWindow(sents []string, idx int) []string {
- if idx < 0 || idx >= len(sents) {
- return nil
- }
- lo, hi := max(0, idx-1), min(len(sents), idx+2)
- total := 0
- var kept []string
- for _, s := range sents[lo:hi] {
- total += len(s)
- if total > grepContextChars*2 {
- break
- }
- kept = append(kept, s)
- }
- if len(kept) == 0 {
- return []string{sents[idx]}
- }
- return kept
-}
-
-func isAlnum(r rune) bool {
- return unicode.IsLetter(r) || unicode.IsDigit(r)
-}
-
-func containsStr(ss []string, s string) bool {
- for _, v := range ss {
- if v == s {
- return true
- }
- }
- return false
-}
-
-// MemorySearch: relevance-ranked retrieval over the
-// raw-chunk memory store (a retrieval-reuse cache, NOT a noise-injection source).
-// Unlike MemoryGrep (loose keyword hit) it keeps only chunks whose overlap with the
-// query's SIGNIFICANT terms clears a normalized bar, so a fact retrieved earlier can
-// be reused instead of re-querying the index.
-//
-// Language-agnostic term extraction:
-// - numbers kept verbatim;
-// - CJK runs split into character 3-grams (no word boundaries exist);
-// - Latin alphanumeric runs lowercased, stopword-filtered, len >= 3.
-//
-// A chunk is relevant when it shares >= 1 term AND >= minRatio of the query's
-// significant terms; results are ranked by hit count then text length, capped at
-// topN. Returns nil when nothing clears the bar (the caller falls back to a
-// knowledge-base search). minRatio <= 0 falls back to memoryMinRatio.
-func MemorySearch(kb *Kbinfos, query string, topN int, minRatio float64) []map[string]any {
- if kb == nil || len(kb.Memory) == 0 {
- return nil
- }
- if topN <= 0 {
- topN = memoryDefaultTopN
- }
- if minRatio <= 0 {
- minRatio = memoryMinRatio
- }
- terms := significantTerms(query)
- if len(terms) == 0 {
- return nil
- }
- matchers := buildTermMatchers(terms)
- n := len(terms)
-
- type scoredChunk struct {
- hits int
- text string
- c map[string]any
- }
- var scored []scoredChunk
- for _, c := range kb.Memory {
- text := chunkText(c)
- if text == "" {
- continue
- }
- hits := 0
- for _, m := range matchers {
- if m.matches(text) {
- hits++
- }
- }
- if hits < 1 {
- continue
- }
- if float64(hits)/float64(n) < minRatio {
- continue
- }
- scored = append(scored, scoredChunk{hits: hits, text: text, c: c})
- }
- if len(scored) == 0 {
- return nil
- }
- // Rank by hit count desc, then text length desc.
- sort.SliceStable(scored, func(i, j int) bool {
- if scored[i].hits != scored[j].hits {
- return scored[i].hits > scored[j].hits
- }
- return len(scored[i].text) > len(scored[j].text)
- })
- if len(scored) > topN {
- scored = scored[:topN]
- }
- out := make([]map[string]any, 0, len(scored))
- for _, s := range scored {
- out = append(out, map[string]any{
- "content": s.text,
- "doc_id": s.c["doc_id"],
- "chunk_id": s.c["chunk_id"],
- "similarity": float64(s.hits),
- })
- }
- rq := []rune(query)
- if len(rq) > 60 {
- rq = rq[:60]
- }
- _LOG.Printf("[Memory.search] query=%q -> %d relevant chunk(s) (ratio>=%.2f, %d terms)", string(rq), len(out), minRatio, n)
- return out
-}
-
-// significantTerms: language-agnostic
-// significant-term extraction, de-duplicated and capped at memoryMaxTerms.
-func significantTerms(text string) []string {
- var out []string
- seen := make(map[string]struct{})
- push := func(tok string) {
- if tok == "" {
- return
- }
- if _, dup := seen[tok]; dup {
- return
- }
- seen[tok] = struct{}{}
- out = append(out, tok)
- }
- // Numbers anywhere.
- for _, m := range reDigits.FindAllString(text, -1) {
- push(m)
- if len(out) >= memoryMaxTerms {
- return out
- }
- }
- // CJK runs -> 3-grams (and the whole run if shorter than 3).
- for _, run := range reCJKRun.FindAllString(text, -1) {
- rs := []rune(run)
- if len(rs) < 3 {
- push(run)
- } else {
- for i := 0; i+3 <= len(rs); i++ {
- push(string(rs[i : i+3]))
- }
- }
- if len(out) >= memoryMaxTerms {
- return out
- }
- }
- // Latin words (stopword-filtered, len >= 3); pure-digit runs already handled.
- for _, m := range reLatinNum.FindAllString(text, -1) {
- if isAllDigits(m) {
- continue
- }
- low := strings.ToLower(m)
- if len(low) >= 3 {
- if _, stop := memoryStopwords[low]; !stop {
- push(low)
- }
- }
- if len(out) >= memoryMaxTerms {
- return out
- }
- }
- return out
-}
-
-// termMatcher precompiles one query term's match predicate so MemorySearch
-// does not recompile regexes per chunk.
-type termMatcher struct {
- // kind: 0 = CJK substring, 1 = digit substring, 2 = Latin word-boundary
- // (with a short prefix fallback for inflectional variants).
- kind int
- sub string
- re *regexp.Regexp
- pre *regexp.Regexp
-}
-
-func buildTermMatchers(terms []string) []termMatcher {
- matchers := make([]termMatcher, 0, len(terms))
- for _, t := range terms {
- if run := reCJKRun.FindString(t); run != "" && run == t {
- // Pure CJK term: literal substring (no word boundary).
- matchers = append(matchers, termMatcher{kind: 0, sub: t})
- continue
- }
- if isAllDigits(t) {
- matchers = append(matchers, termMatcher{kind: 1, sub: t})
- continue
- }
- rs := []rune(t)
- re, _ := regexp.Compile(`(?i)\b` + regexp.QuoteMeta(t) + `\b`)
- var pre *regexp.Regexp
- if len(rs) >= 6 {
- pre, _ = regexp.Compile(`(?i)\b` + regexp.QuoteMeta(string(rs[:5])))
- }
- matchers = append(matchers, termMatcher{kind: 2, re: re, pre: pre})
- }
- return matchers
-}
-
-func (m termMatcher) matches(text string) bool {
- switch m.kind {
- case 0, 1:
- return strings.Contains(text, m.sub)
- default:
- if m.re != nil && m.re.MatchString(text) {
- return true
- }
- if m.pre != nil && m.pre.MatchString(text) {
- return true
- }
- return false
- }
-}
-
-// IsStopword reports whether w is one of the shared stopwords.
-//
-// The fan-out needs it because the same set filters candidate terms.
-func IsStopword(w string) bool {
- _, ok := memoryStopwords[w]
- return ok
-}
-
-func isAllDigits(s string) bool {
- if s == "" {
- return false
- }
- for _, r := range s {
- if r < '0' || r > '9' {
- return false
- }
- }
- return true
-}
diff --git a/internal/rag/agentic-rag/runtime/memory_test.go b/internal/rag/agentic-rag/runtime/memory_test.go
deleted file mode 100644
index 44da75bd9a..0000000000
--- a/internal/rag/agentic-rag/runtime/memory_test.go
+++ /dev/null
@@ -1,145 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import "testing"
-
-func memChunk(content, docID, chunkID string) map[string]any {
- return map[string]any{"content": content, "doc_id": docID, "chunk_id": chunkID}
-}
-
-// The memory search returns the chunks that share enough of the query's significant terms;
-// unrelated queries return nothing.
-func TestMemorySearchReturnsRelevant(t *testing.T) {
- kb := &Kbinfos{Memory: []map[string]any{
- memChunk("Rifampicin is an antibiotic used to treat tuberculosis.", "d1", "c1"),
- memChunk("The Eiffel Tower is a wrought-iron lattice tower in Paris.", "d2", "c2"),
- }}
- // "rifampicin" / "tuberculosis" both hit chunk c1 only.
- hits := MemorySearch(kb, "What is rifampicin used for in tuberculosis treatment?", 0, 0)
- if len(hits) != 1 {
- t.Fatalf("hits = %d, want 1", len(hits))
- }
- if hits[0]["chunk_id"] != "c1" {
- t.Errorf("top hit chunk_id = %v, want c1", hits[0]["chunk_id"])
- }
- if _, ok := hits[0]["similarity"]; !ok {
- t.Errorf("hit missing similarity field")
- }
-
- // Unrelated query clears nothing.
- if got := MemorySearch(kb, "How many moons does Jupiter have?", 0, 0); got != nil {
- t.Errorf("unrelated query should return nil, got %d hits", len(got))
- }
-}
-
-// The normalized overlap bar must drop a chunk that matches too few of a long
-// query's terms (the (hits/_n) >= min_ratio guard).
-func TestMemorySearchRatioBar(t *testing.T) {
- kb := &Kbinfos{Memory: []map[string]any{
- memChunk("Rifampicin is an antibiotic.", "d1", "c1"),
- }}
- // Query has many significant terms (9) but the chunk shares only
- // "rifampicin" -> 1/9 ~= 0.111 < the 0.12 default ratio, so it clears
- // nothing.
- q := "rifampicin airplane bicycle coffee mountain ocean planet quarter river"
- if got := MemorySearch(kb, q, 0, 0); got != nil {
- t.Errorf("below-ratio query should return nil, got %d hits", len(got))
- }
-
- // Lowering the ratio lets the same single-term hit through.
- if got := MemorySearch(kb, q, 0, 0.1); len(got) != 1 {
- t.Errorf("ratio 0.1 should keep the hit, got %d", len(got))
- }
-}
-
-// CJK queries match via character 3-grams.
-func TestMemorySearchCJKTrigram(t *testing.T) {
- kb := &Kbinfos{Memory: []map[string]any{
- memChunk("利福平是一种抗生素,用于治疗结核病。", "d1", "c1"),
- memChunk("巴黎的埃菲尔铁塔是著名的铁制建筑。", "d2", "c2"),
- }}
- hits := MemorySearch(kb, "利福平治疗什么疾病?", 0, 0)
- if len(hits) != 1 || hits[0]["chunk_id"] != "c1" {
- t.Errorf("CJK 3-gram search failed: got %d hits, want 1 (c1)", len(hits))
- }
-}
-
-// Ranking: more term hits ranks higher; equal hits then prefer longer text.
-func TestMemorySearchRanking(t *testing.T) {
- kb := &Kbinfos{Memory: []map[string]any{
- memChunk("Rifampicin treats tuberculosis and leprosy.", "d1", "low"),
- memChunk("Rifampicin is an antibiotic for tuberculosis and also used for leprosy and meningitis.", "d2", "high"),
- }}
- hits := MemorySearch(kb, "rifampicin tuberculosis leprosy", 0, 0)
- if len(hits) != 2 {
- t.Fatalf("hits = %d, want 2", len(hits))
- }
- if hits[0]["chunk_id"] != "high" {
- t.Errorf("top ranked chunk_id = %v, want high (more hits / longer)", hits[0]["chunk_id"])
- }
-}
-
-// chunkText must prefer "content_with_weight" over "content" (content_with_weight or content
-// or text), so a chunk carrying both yields the weighted text.
-func TestChunkTextPrefersWeighted(t *testing.T) {
- c := map[string]any{
- "content": "primary text",
- "content_with_weight": "weighted text",
- }
- if got := chunkText(c); got != "weighted text" {
- t.Errorf("chunkText = %q, want %q (content_with_weight is preferred)", got, "weighted text")
- }
-
- // Falls back to content when content_with_weight is absent/empty.
- c2 := map[string]any{"content": "primary only"}
- if got := chunkText(c2); got != "primary only" {
- t.Errorf("chunkText = %q, want %q", got, "primary only")
- }
-}
-
-// TestMemoryGrepLimitAppliesToShortChunks pins the returned-chunk cap: the
-// short-chunk branch continues past the shared check, so it must enforce the
-// limit itself — otherwise a memory store full of matching short chunks comes
-// back whole instead of at most limit entries.
-func TestMemoryGrepLimitAppliesToShortChunks(t *testing.T) {
- mem := make([]map[string]any, 0, 10)
- for i := 0; i < 10; i++ {
- suffix := string(rune('a' + i)) // short chunk: well under shortChunkChars
- mem = append(mem, memChunk("rifampicin note "+suffix, "d1", "c"+suffix))
- }
- kb := &Kbinfos{Memory: mem}
-
- hits := MemoryGrep(kb, []string{"rifampicin"}, 2)
- if len(hits) != 2 {
- t.Fatalf("hits = %d, want the limit 2 (a short chunk must not bypass the cap)", len(hits))
- }
- if hits[0]["content"] != "rifampicin note a" {
- t.Errorf("hit content = %v, want the whole short chunk", hits[0]["content"])
- }
-
- // limit <= 0 is NOT normalized to the default: there is no such guard, so the
- // `len(hits) >= limit` check fires on the first hit.
- if got := len(MemoryGrep(kb, []string{"rifampicin"}, 0)); got != 1 {
- t.Errorf("limit=0 hits = %d, want 1 (the limit is not normalized)", got)
- }
-
- // The default (6) is passed explicitly and caps at 6.
- if got := len(MemoryGrep(kb, []string{"rifampicin"}, grepMaxChunks)); got != grepMaxChunks {
- t.Errorf("default-limit hits = %d, want %d", got, grepMaxChunks)
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/orchestrator/direct.go b/internal/rag/agentic-rag/runtime/orchestrator/direct.go
deleted file mode 100644
index 850a114716..0000000000
--- a/internal/rag/agentic-rag/runtime/orchestrator/direct.go
+++ /dev/null
@@ -1,48 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-// Package orchestrator holds the top-level pipeline stages that sit above the
-// action session: the Sufficient Context Agent (SCA) review and the gap→query
-// rewriter.
-//
-// Covers the Sufficient Context Agent (SCA) review and the gap→query rewriter.
-//
-// The low-mode direct search deliberately has no counterpart in this package: its Go
-// implementation is runDirect
-// (agentic_rag.go, called from the low graph's direct_search node), the low graph
-// node's body, because that step needs the full RAGTools config rather than a
-// runtime-level dependency bundle. One implementation only; its contract is
-// pinned by TestBuildLowGraphRunsFormalizeThenDirectSearch.
-package orchestrator
-
-import (
- "context"
- "fmt"
-
- "ragflow/internal/common"
-)
-
-var _LOG = common.StdLogger()
-
-// JSONModel generates one JSON object from a rendered prompt.
-//
-// The prompt is sent as the system turn, "Output:\n" as the user turn, and the first JSON
-// value is parsed out of the reply.
-type JSONModel interface {
- GenJSON(ctx context.Context, prompt string) (any, error)
-}
-
-func fmtClaimCount(n int) string { return fmt.Sprintf("%d claim draft(s)", n) }
diff --git a/internal/rag/agentic-rag/runtime/orchestrator/query_rewriter.go b/internal/rag/agentic-rag/runtime/orchestrator/query_rewriter.go
deleted file mode 100644
index 1bdc470f6e..0000000000
--- a/internal/rag/agentic-rag/runtime/orchestrator/query_rewriter.go
+++ /dev/null
@@ -1,124 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package orchestrator
-
-import (
- "context"
- "fmt"
- "strings"
-
- "ragflow/internal/rag/agentic-rag/runtime"
- "ragflow/internal/rag/prompts"
-)
-
-// Query rewriter (Phase 4 of the pipeline).
-//
-
-// RewriteDeps are the dependencies of RewriteGapToQuery.
-type RewriteDeps struct {
- // Model generates the JSON object.
- Model JSONModel
- // Prompts supplies the "sca_query_rewrite" template.
- Prompts runtime.PromptLoader
- // BridgeValues are already-resolved values across the claims (e.g. the
- // confirmed shows in a "which finale ran longest" question). The rewriter
- // anchors the new query to these, so a missing hop is re-searched with the
- // resolved upstream value instead of re-deriving it.
- BridgeValues []string
- // ResearchContext is the rendered block describing what previous rounds
- // already tried (queries + outcomes) and what the evidence pool contains.
- // It lets the rewriter aim at UNCOVERED angles instead of paraphrasing dead
- // ones.
- ResearchContext string
-}
-
-// RewriteGapToQuery
-//
-// gaps are (what, search_hint) tuples — the forward gaps the SCA identified.
-//
-// Returns [{"query": "..."}] — targeted, retrievable queries. Empty when the
-// rewrite is unavailable or fails, in which case the caller falls back to the
-// original gap text.
-func RewriteGapToQuery(ctx context.Context, deps RewriteDeps, question string, gaps []MissingPiece) []map[string]string {
- if deps.Model == nil || len(gaps) == 0 {
- return nil
- }
- // The call runs inside the "rewrite" phase.
- ctx, done := runtime.Phase(ctx, runtime.PhaseRewrite)
- defer done()
-
- var gapsLines []string
- for _, g := range gaps {
- gapsLines = append(gapsLines, fmt.Sprintf("- what: %s; hint: %s", g.What, g.SearchHint))
- }
- gapsText := strings.Join(gapsLines, "\n")
-
- var bridgeLines []string
- for _, b := range deps.BridgeValues {
- if strings.TrimSpace(b) != "" {
- bridgeLines = append(bridgeLines, "- "+b)
- }
- }
- bridgeText := strings.Join(bridgeLines, "\n")
-
- prompt := prompts.Render(deps.Prompts, "sca_query_rewrite", "", map[string]string{
- "question": question,
- "gaps": gapsText,
- "bridge_values": bridgeText,
- "research_context": deps.ResearchContext,
- })
-
- result, err := deps.Model.GenJSON(ctx, prompt)
- if err != nil {
- _LOG.Printf("[QueryRewrite] failed: %v", err)
- return nil
- }
- data, ok := result.(map[string]any)
- if !ok {
- return nil
- }
- rawQueries, _ := data["queries"].([]any)
-
- var out []map[string]string
- for _, q := range rawQueries {
- var query string
- if m, ok := q.(map[string]any); ok {
- // Accept `query` (canonical) or `question` (observed drift).
- if v := m["query"]; v != nil {
- query = strings.TrimSpace(fmt.Sprint(v))
- } else if v := m["question"]; v != nil {
- query = strings.TrimSpace(fmt.Sprint(v))
- }
- } else if q != nil {
- query = strings.TrimSpace(fmt.Sprint(q))
- }
- if query == "" || containsQuery(out, query) {
- continue
- }
- out = append(out, map[string]string{"query": query})
- }
- return out
-}
-
-func containsQuery(queries []map[string]string, q string) bool {
- for _, item := range queries {
- if item["query"] == q {
- return true
- }
- }
- return false
-}
diff --git a/internal/rag/agentic-rag/runtime/orchestrator/query_rewriter_test.go b/internal/rag/agentic-rag/runtime/orchestrator/query_rewriter_test.go
deleted file mode 100644
index 7cde827ae0..0000000000
--- a/internal/rag/agentic-rag/runtime/orchestrator/query_rewriter_test.go
+++ /dev/null
@@ -1,105 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package orchestrator
-
-import (
- "context"
- "errors"
- "strings"
- "testing"
-)
-
-func TestRewriteGapToQueryNeedsModelAndGaps(t *testing.T) {
- mdl := &stubJSONModel{value: map[string]any{"queries": []any{"q1"}}}
- if got := RewriteGapToQuery(context.Background(), RewriteDeps{Model: mdl}, "q", nil); got != nil {
- t.Error("no gaps must short-circuit")
- }
- if mdl.calls != 0 {
- t.Error("no gaps must not call the model")
- }
- if got := RewriteGapToQuery(context.Background(), RewriteDeps{}, "q",
- []MissingPiece{{What: "x"}}); got != nil {
- t.Error("no model must return nil")
- }
-}
-
-func TestRewriteGapToQueryParsesAndDedupes(t *testing.T) {
- mdl := &stubJSONModel{value: map[string]any{"queries": []any{
- map[string]any{"query": "MASH finale run time minutes"},
- map[string]any{"question": "Cheers finale run time minutes"}, // key drift
- "plain string query",
- map[string]any{"query": "MASH finale run time minutes"}, // duplicate
- map[string]any{"query": " "}, // blank
- }}}
- got := RewriteGapToQuery(context.Background(), RewriteDeps{Model: mdl}, "which finale ran longest?",
- []MissingPiece{{What: "runtimes", SearchHint: "finale run time"}})
- if len(got) != 3 {
- t.Fatalf("queries = %d, want 3 (blank + duplicate dropped)", len(got))
- }
- if got[0]["query"] != "MASH finale run time minutes" {
- t.Errorf("queries[0] = %q", got[0]["query"])
- }
- if got[1]["query"] != "Cheers finale run time minutes" {
- t.Errorf("queries[1] = %q, want the 'question' key accepted", got[1]["query"])
- }
- if got[2]["query"] != "plain string query" {
- t.Errorf("queries[2] = %q", got[2]["query"])
- }
-}
-
-func TestRewriteGapToQueryRendersAllVars(t *testing.T) {
- mdl := &stubJSONModel{value: map[string]any{"queries": []any{}}}
- RewriteGapToQuery(context.Background(), RewriteDeps{
- Model: mdl,
- Prompts: nil, // resolves the embedded sca_query_rewrite.md
- BridgeValues: []string{"M*A*S*H", "Cheers", " "},
- ResearchContext: "tried: finale runtimes (nothing new)",
- }, "which finale ran longest?",
- []MissingPiece{{What: "runtimes", SearchHint: "finale run time"}})
-
- // The canonical template uses the spaced Jinja form, and every var must be
- // substituted (an unreplaced {{ }} would corrupt the prompt).
- if strings.Contains(mdl.last, "{{") {
- t.Errorf("unsubstituted placeholder remains: %.200s", mdl.last)
- }
- for _, want := range []string{"which finale ran longest?", "- M*A*S*H", "- Cheers",
- "what: runtimes; hint: finale run time", "tried: finale runtimes"} {
- if !strings.Contains(mdl.last, want) {
- t.Errorf("prompt missing %q", want)
- }
- }
- // Blank bridge values are dropped.
- if strings.Contains(mdl.last, "- ") {
- t.Error("blank bridge value must be dropped")
- }
-}
-
-func TestRewriteGapToQueryReturnsNilOnFailure(t *testing.T) {
- mdl := &stubJSONModel{err: errors.New("provider down")}
- if got := RewriteGapToQuery(context.Background(), RewriteDeps{Model: mdl}, "q",
- []MissingPiece{{What: "x"}}); got != nil {
- t.Error("a model failure must return nil so the caller falls back")
- }
- // A non-dict response is also "no signal".
- mdl = &stubJSONModel{value: []any{"not", "a", "dict"}}
- if got := RewriteGapToQuery(context.Background(), RewriteDeps{Model: mdl}, "q",
- []MissingPiece{{What: "x"}}); got != nil {
- t.Error("a non-dict response must return nil")
- }
-}
-
-// Prompt rendering
diff --git a/internal/rag/agentic-rag/runtime/orchestrator/sufficient_context.go b/internal/rag/agentic-rag/runtime/orchestrator/sufficient_context.go
deleted file mode 100644
index 489a175127..0000000000
--- a/internal/rag/agentic-rag/runtime/orchestrator/sufficient_context.go
+++ /dev/null
@@ -1,612 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package orchestrator
-
-import (
- "context"
- "fmt"
- "ragflow/internal/common"
- "regexp"
- "strings"
- "unicode/utf8"
-
- "ragflow/internal/rag/agentic-rag/runtime"
- "ragflow/internal/rag/prompts"
-)
-
-// Unified Sufficient Context Agent.
-//
-// The SCA performs ONE review pass over (1) each claim's intermediate draft and
-// (2) the overall draft assembled from them, and returns a unified verdict:
-// is_sufficient / confidence / contradictions / reasoning / claims / sub_queries.
-// It replaced an earlier two-call split (global verdict + per-claim groundedness)
-// whose trigger bands were complementary, so the three-part review rarely
-// happened in one shot.
-//
-// The SCA reviews ONLY the claims' reports, never the raw retrieved chunks.
-// Each report is an evidence-backed finding already distilled from that claim's
-// evidence; sending raw chunks ballooned the prompt to 20k-100k chars, which
-// degraded the LLM (empty claims) or timed it out.
-
-const (
- // scaClaimsContextMax caps the rendered claims context (all per-claim
- // reports + the overall draft). Sized to match the action session's context
- // budget (maxToolResponseChars * 4) so a table-bearing evidence anchor plus
- // several claim reports still fit.
- scaClaimsContextMax = 48000
- // scaEvidenceAnchorChars is the max chars of each cited snippet kept as an
- // evidence anchor, so the SCA can verify a draft against real retrieved text
- // without a token blow-up.
- scaEvidenceAnchorChars = 300
- // scaMaxAnchorsPerClaim bounds anchors per claim.
- scaMaxAnchorsPerClaim = 3
- // feedbackMax bounds the missing pieces folded into the boost feedback string.
- feedbackMax = 4
-)
-
-var reHintToken = regexp.MustCompile(`[A-Za-z0-9_\x{4e00}-\x{9fff}]{3,}`)
-
-// ClaimDraft is one claim's intermediate draft plus the evidence it cited.
-//
-// EvidenceIDs are INDICES into Kbinfos.Chunks (see Kbinfos.Merge, which returns
-// global indices) — NOT the chunk's chunk_id hash. Keying the anchor lookup by
-// chunk_id would make every anchor a miss and silently disable the "SCA reviews
-// the retrieved snippets" guard.
-type ClaimDraft struct {
- ID string
- Draft string
- EvidenceIDs []string
-}
-
-// SubQuery is one step-by-step sub-question with its coverage verdict (Q-CARE).
-// Unsatisfied entries carry the concrete missing fact + a search hint — the
-// precise "what is missing / where to search next" signal the next round
-// consumes.
-type SubQuery struct {
- SubQuery string
- Satisfied bool
- MissingFact string
- SearchHint string
-}
-
-// MissingPiece is one gap: what is missing and a hint for searching for it.
-type MissingPiece struct {
- What string
- SearchHint string
-}
-
-// ClaimVerdict is the SCA's per-claim verdict.
-type ClaimVerdict struct {
- Grounded bool
- Ungrounded []string
- MissingInformation []MissingPiece
-}
-
-// SCAResult is the unified verdict. An empty result means "no new signal",
-// which callers must treat as neither sufficient nor insufficient.
-type SCAResult struct {
- IsSufficient bool
- Confidence float64
- Contradictions []string
- Reasoning string
- SubQueries []SubQuery
- Claims map[string]ClaimVerdict
-}
-
-// SCADeps are the dependencies of the SCA.
-type SCADeps struct {
- KB *runtime.Kbinfos
- // Model generates the JSON verdict.
- Model JSONModel
- // Prompts supplies the "sca_select" template.
- Prompts runtime.PromptLoader
-}
-
-// SufficientContextAgent
-//
-// claims carries per-claim evidence (not the global union): rendering only the
-// snippets each claim cited keeps the prompt small (~1-3 chunks per claim) so
-// the LLM does not degrade, while the cited snippets still carry the
-// answer-bearing facts.
-//
-// Returns an empty SCAResult when unavailable (no claims, no model, or a
-// failure) — callers treat that as "no new signal".
-func SufficientContextAgent(ctx context.Context, deps SCADeps, question string, claims []ClaimDraft) SCAResult {
- if len(claims) == 0 || deps.Model == nil {
- return SCAResult{}
- }
- // The review runs inside the "sca" phase.
- ctx, done := runtime.Phase(ctx, runtime.PhaseSCA)
- defer done()
-
- claimsContext := renderClaimContext(claims, deps.KB)
- if claimsContext == "" || claimsContext == "(no claim drafts)" {
- return SCAResult{}
- }
- overallDraft := renderOverallDraft(claims)
-
- prompt := prompts.Render(deps.Prompts, "sca_select", "", map[string]string{
- "question": question,
- "claims_context": claimsContext,
- "overall_draft": overallDraft,
- })
- _LOG.Printf("[SCA] unified review of %s (reports only, %d chars; overall draft %d chars)",
- fmtClaimCount(len(claims)), utf8.RuneCountInString(claimsContext), utf8.RuneCountInString(overallDraft))
-
- result, err := deps.Model.GenJSON(ctx, prompt)
- if err != nil {
- _LOG.Printf("[SCA] unified review failed: %v", err)
- return SCAResult{}
- }
- data := coerceDict(result)
- if data == nil {
- _LOG.Printf("[SCA] no usable response; treating as no signal")
- return SCAResult{}
- }
-
- // Per-claim verdicts.
- claimsOut := map[string]ClaimVerdict{}
- if rawClaims, ok := data["claims"].([]any); ok {
- for _, item := range rawClaims {
- m, ok := item.(map[string]any)
- if !ok {
- continue
- }
- cid := strOf(m["claim_id"])
- if cid == "" {
- continue
- }
- var ungrounded []string
- for _, u := range asSlice(m["ungrounded_assertions"]) {
- if um, ok := u.(map[string]any); ok {
- if s := strings.TrimSpace(fmt.Sprint(firstNonEmpty(um["assertion"], um["reason"]))); s != "" {
- ungrounded = append(ungrounded, s)
- }
- } else if s := strOf(u); s != "" {
- ungrounded = append(ungrounded, s)
- }
- }
- claimsOut[cid] = ClaimVerdict{
- Grounded: coerceBool(m["grounded"]),
- Ungrounded: ungrounded,
- MissingInformation: parseMissingInformation(m["missing_information"]),
- }
- }
- }
-
- // Structured sub-query coverage (Q-CARE).
- var subQueries []SubQuery
- for _, sq := range asSlice(data["sub_queries"]) {
- m, ok := sq.(map[string]any)
- if !ok {
- continue
- }
- sqText := strOf(m["sub_query"])
- if sqText == "" {
- continue
- }
- entry := SubQuery{SubQuery: sqText, Satisfied: coerceBool(m["satisfied"])}
- if !entry.Satisfied {
- entry.MissingFact = strOf(m["missing_fact"])
- entry.SearchHint = strOf(m["search_hint"])
- }
- subQueries = append(subQueries, entry)
- }
-
- isSufficient := coerceBool(data["is_sufficient"])
-
- // Failsafe: the SCA judged the context insufficient but returned an EMPTY
- // claims array (a known degradation on very long prompts). Without a gap, the
- // orchestrator would abandon with "I don't have enough information" despite
- // having retrieved useful snippets. Harvest top-level missing_information, and
- // fall back to the un-verified drafts themselves as coarse gaps.
- if !isSufficient && len(claimsOut) == 0 {
- topMissing := parseMissingInformation(data["missing_information"])
- if len(topMissing) > 0 {
- claimsOut["_global"] = ClaimVerdict{Grounded: false, MissingInformation: topMissing}
- _LOG.Printf("[SCA] insufficient with empty claims; using %d top-level missing piece(s) as the re-search gap.", len(topMissing))
- } else {
- var draftGaps []MissingPiece
- for _, c := range claims {
- if d := strings.TrimSpace(c.Draft); d != "" {
- draftGaps = append(draftGaps, MissingPiece{What: d, SearchHint: d})
- }
- }
- if len(draftGaps) > 0 {
- claimsOut["_global"] = ClaimVerdict{Grounded: false, MissingInformation: draftGaps}
- _LOG.Printf("[SCA] insufficient with no structured gap; deriving %d coarse gap(s) from the drafts.", len(draftGaps))
- }
- }
- }
-
- return SCAResult{
- IsSufficient: isSufficient,
- Confidence: clamp(data["confidence"]),
- Contradictions: stringList(data["contradictions"]),
- Reasoning: strOf(data["reasoning"]),
- SubQueries: subQueries,
- Claims: claimsOut,
- }
-}
-
-// Boost: adapt the unified verdict into the
-// decision-ladder boost dict, preserving the contract
-// (is_sufficient / confidence / missing / contradictions / feedback /
-// followups) that the existing ladder consumes unchanged.
-func (s SCAResult) Boost(fallbackFollowups []string) Boost {
- var missing []string
- for _, g := range s.Claims {
- for _, mi := range g.MissingInformation {
- w := strings.TrimSpace(mi.What)
- if w != "" && !containsStr(missing, w) {
- missing = append(missing, w)
- }
- }
- }
- feedback := ""
- if len(missing) > 0 {
- n := min(len(missing), feedbackMax)
- feedback = "missing: " + strings.Join(missing[:n], "; ")
- }
- return Boost{
- IsSufficient: s.IsSufficient,
- Confidence: s.Confidence,
- Missing: missing,
- Contradictions: s.Contradictions,
- Followups: fallbackFollowups,
- Feedback: feedback,
- SubQueries: s.SubQueries,
- }
-}
-
-// Boost is the decision-ladder input produced by SCAResult.Boost.
-type Boost struct {
- IsSufficient bool
- Confidence float64
- Missing []string
- Contradictions []string
- Followups []string
- Feedback string
- // SubQueries is the structured Q-CARE coverage signal consumed by replan.
- SubQueries []SubQuery
-}
-
-// ToGrounded: adapt the unified verdict into the
-// `grounded` dict consumed by replan and the ungrounded-veto path
-// ({claim_id: {grounded, ungrounded, missing_information}}).
-func (s SCAResult) ToGrounded() map[string]ClaimVerdict { return s.Claims }
-
-// Rendering helpers
-
-// renderClaimContext: each claim's report
-// PLUS a brief evidence anchor (the first line of each cited snippet), so the
-// SCA can verify the draft is grounded in real retrieved text.
-func renderClaimContext(claims []ClaimDraft, kb *runtime.Kbinfos) string {
- if len(claims) == 0 {
- return "(no claim drafts)"
- }
- // Key by INDEX into kbinfos chunks (see the ClaimDraft doc comment).
- id2chunk := map[string]map[string]any{}
- if kb != nil {
- for i, c := range kb.Chunks {
- id2chunk[fmt.Sprint(i)] = c
- }
- }
- var blocks []string
- used := 0
- for _, c := range claims {
- if c.Draft == "" {
- continue
- }
- block := fmt.Sprintf("Claim %s (draft):\n%s", c.ID, c.Draft)
- var anchors []string
- for _, eid := range c.EvidenceIDs {
- ck, ok := id2chunk[strings.TrimSpace(eid)]
- if !ok {
- continue
- }
- txt := strings.TrimSpace(scaChunkText(ck))
- if txt == "" {
- continue
- }
- // Table chunks contribute their FULL text: hint-token windowing is
- // unreliable for tables (the draft rarely contains the row's entity
- // names) and head-truncation hides answer rows that sit mid-table.
- if ex := boundedExcerpt(txt, c.Draft, scaEvidenceAnchorChars); ex != "" {
- anchors = append(anchors, ex)
- }
- if len(anchors) >= scaMaxAnchorsPerClaim {
- break
- }
- }
- if len(anchors) > 0 {
- block += "\n Evidence: " + strings.Join(anchors, " | ")
- }
- // Apply the budget to the COMPLETE block before appending it, trimming it
- // to what remains: accounting the block only AFTER appending let a single
- // oversized claim blow straight past scaClaimsContextMax (the cap the
- // comment claims bounds the whole rendered context). Cut by rune so the trim never
- // splits a multibyte character.
- remaining := scaClaimsContextMax - used
- if remaining <= 0 {
- break
- }
- if r := []rune(block); len(r) > remaining {
- block = string(r[:remaining])
- }
- blocks = append(blocks, block)
- // The block plus the "\n\n" join separator.
- used += utf8.RuneCountInString(block) + 2
- if used >= scaClaimsContextMax {
- break
- }
- }
- if len(blocks) == 0 {
- return "(no claim drafts)"
- }
- return strings.Join(blocks, "\n\n")
-}
-
-// renderOverallDraft: assemble the
-// problem-level "rough draft" by concatenating each claim's report, so the SCA
-// can judge whether the context lets the model answer end-to-end — including
-// cross-claim synthesis that per-claim review would miss.
-func renderOverallDraft(claims []ClaimDraft) string {
- if len(claims) == 0 {
- return "(no overall draft)"
- }
- var parts []string
- for _, c := range claims {
- if d := strings.TrimSpace(c.Draft); d != "" {
- parts = append(parts, fmt.Sprintf("[Claim %s] %s", c.ID, d))
- }
- }
- if len(parts) == 0 {
- return "(no overall draft)"
- }
- draft := strings.Join(parts, "\n")
- // Truncate by RUNE, not byte: a byte slice through a multibyte (e.g. CJK) rune would emit
- // invalid UTF-8 (or replacement characters) into the SCA prompt.
- if r := []rune(draft); len(r) > scaClaimsContextMax {
- draft = string(r[:scaClaimsContextMax])
- }
- return draft
-}
-
-// scaChunkText: _render_claim_context: prefer the reranked
-// "content_with_weight" over the raw "content", then fall back to "chunk". This differs from
-// retrieval.chunkText (which prefers content and keeps a "text" fallback): the SCA evidence anchor
-// must read the SAME text so the groundedness guard resolves the same snippets.
-func scaChunkText(c map[string]any) string {
- if v, ok := c["content_with_weight"]; ok && v != nil {
- if s := fmt.Sprint(v); s != "" {
- return s
- }
- }
- if v, ok := c["content"]; ok && v != nil {
- if s := fmt.Sprint(v); s != "" {
- return s
- }
- }
- if v, ok := c["chunk"]; ok && v != nil {
- return fmt.Sprint(v)
- }
- return ""
-}
-
-// boundedExcerpt: a bounded window around a
-// term from the draft. Table text is returned whole, serialized to Markdown.
-func boundedExcerpt(text, hints string, maxChars int) string {
- text = strings.TrimSpace(text)
- if text == "" {
- return ""
- }
- if isTableText(text) {
- // The SCA is a plain LLM call: raw <table>/<td> markup makes it parse cells
- // itself, and the same table costs ~61% more in HTML than in Markdown. The
- // renderer never prunes rows (a row can carry the answer mid-table), so the
- // SCA still sees the complete row set — just in a readable form. No-op for
- // non-tables.
- return runtime.TableViewOrRaw(text)
- }
- if maxChars < 80 {
- maxChars = 80
- }
- // Work in runes (code points): a byte-based slice would mis-count and split a multibyte
- // (e.g. CJK) rune, emitting invalid UTF-8.
- r := []rune(text)
- lower := strings.ToLower(text)
- start := -1
- for _, tok := range reHintToken.FindAllString(hints, -1) {
- if pos := strings.Index(lower, strings.ToLower(tok)); pos >= 0 {
- // pos is a byte offset in text; convert to a rune index.
- start = len([]rune(text[:pos]))
- break
- }
- }
- n := len(r)
- if start < 0 {
- if n <= maxChars {
- return text
- }
- tail := maxChars / 2
- return string(r[:maxChars-tail]) + " … " + string(r[n-tail:])
- }
- half := maxChars / 2
- left := max(0, start-half)
- right := min(n, left+maxChars)
- left = max(0, right-maxChars)
- prefix, suffix := "", ""
- if left > 0 {
- prefix = "…"
- }
- if right < n {
- suffix = "…"
- }
- return prefix + string(r[left:right]) + suffix
-}
-
-// isTableText: a corpus-neutral table detector —
-// HTML table markup, or >=3 pipe rows.
-func isTableText(text string) bool {
- t := strings.ToLower(text)
- if strings.Contains(t, "<table") || strings.Contains(t, "<tr") {
- return true
- }
- pipeRows := 0
- for _, line := range strings.Split(text, "\n") {
- if strings.Count(line, "|") >= 2 {
- pipeRows++
- }
- }
- return pipeRows >= 3
-}
-
-// Coercion helpers
-
-// coerceDict: tolerate model format drift. The
-// reviewer occasionally replies with a bare array or a JSON string; previously
-// any non-dict response was dropped, so the SCA produced NO signal on those
-// rounds and replan/rewrite silently stopped.
-func coerceDict(v any) map[string]any {
- switch t := v.(type) {
- case map[string]any:
- return t
- case []any:
- for _, item := range t {
- if m, ok := item.(map[string]any); ok {
- return m
- }
- }
- return nil
- case string:
- if parsed := runtime.ExtractJSON(t); parsed != nil {
- return coerceDict(parsed)
- }
- return nil
- }
- return nil
-}
-
-// clamp: coerce to [0,1], defaulting to 1.0 on failure.
-func clamp(v any) float64 {
- f, ok := common.ToFloat(v)
- if !ok {
- return 1.0
- }
- if f < 0 {
- return 0
- }
- if f > 1 {
- return 1
- }
- return f
-}
-
-// coerceBool reads an LLM-reported boolean, tolerating the string serialisations
-// a drifted model reply may carry. It deliberately does NOT use raw truthiness:
-// a non-empty string is truthy in Go, so bool("false") and bool("0") are TRUE — which would
-// silently INVERT a verdict, marking
-// insufficient context sufficient or an ungrounded claim grounded.
-//
-// The recognised true/false spellings are mapped explicitly; an empty or
-// unrecognised value fails CLOSED (false), matching this codebase's convention
-// that an unusable verdict is INSUFFICIENT (see the SCA-unavailable path in
-// agentic_rag_graph.go, which treats a timeout/unparsable reply as insufficient).
-func coerceBool(v any) bool {
- switch t := v.(type) {
- case bool:
- return t
- case string:
- switch strings.ToLower(strings.TrimSpace(t)) {
- case "true", "1", "yes", "y", "on":
- return true
- }
- return false
- case float64:
- return t != 0
- case int:
- return t != 0
- case int64:
- return t != 0
- case nil:
- return false
- }
- return false
-}
-
-// strOf renders a raw JSON field as a trimmed string, mapping an absent or null
-// value (nil) to "". fmt.Sprint(nil) yields the literal "<nil>", which would
-// otherwise masquerade as a real claim id, sub-query, gap or reasoning text.
-func strOf(v any) string {
- if v == nil {
- return ""
- }
- return strings.TrimSpace(fmt.Sprint(v))
-}
-
-func parseMissingInformation(v any) []MissingPiece {
- var out []MissingPiece
- for _, m := range asSlice(v) {
- if mm, ok := m.(map[string]any); ok {
- what := strOf(mm["what"])
- hint := strOf(mm["search_hint"])
- if what != "" || hint != "" {
- out = append(out, MissingPiece{What: what, SearchHint: hint})
- }
- } else if s := strOf(m); s != "" {
- out = append(out, MissingPiece{What: s})
- }
- }
- return out
-}
-
-func stringList(v any) []string {
- var out []string
- for _, item := range asSlice(v) {
- if s := strings.TrimSpace(fmt.Sprint(item)); s != "" {
- out = append(out, s)
- }
- }
- return out
-}
-
-func asSlice(v any) []any {
- list, _ := v.([]any)
- return list
-}
-
-func firstNonEmpty(v ...any) string {
- for _, item := range v {
- if item == nil {
- continue
- }
- if s := strings.TrimSpace(fmt.Sprint(item)); s != "" {
- return s
- }
- }
- return ""
-}
-
-func containsStr(ss []string, s string) bool {
- for _, v := range ss {
- if v == s {
- return true
- }
- }
- return false
-}
diff --git a/internal/rag/agentic-rag/runtime/orchestrator/sufficient_context_test.go b/internal/rag/agentic-rag/runtime/orchestrator/sufficient_context_test.go
deleted file mode 100644
index 5044892087..0000000000
--- a/internal/rag/agentic-rag/runtime/orchestrator/sufficient_context_test.go
+++ /dev/null
@@ -1,438 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package orchestrator
-
-import (
- "context"
- "strings"
- "testing"
- "unicode/utf8"
-
- "ragflow/internal/rag/agentic-rag/runtime"
-)
-
-// stubJSONModel returns a canned JSON value (or an error), and records the last
-// prompt it was shown.
-type stubJSONModel struct {
- value any
- err error
- calls int
- last string
-}
-
-func (s *stubJSONModel) GenJSON(_ context.Context, prompt string) (any, error) {
- s.calls++
- s.last = prompt
- if s.err != nil {
- return nil, s.err
- }
- return s.value, nil
-}
-
-// SCA
-
-func TestSCAReturnsNoSignalWithoutClaims(t *testing.T) {
- mdl := &stubJSONModel{value: map[string]any{"is_sufficient": true}}
- got := SufficientContextAgent(context.Background(), SCADeps{
- KB: &runtime.Kbinfos{},
- Model: mdl,
- }, "q", nil)
- if mdl.calls != 0 {
- t.Error("no claims must short-circuit before any LLM call")
- }
- if got.IsSufficient || got.Claims != nil {
- t.Error("no claims must yield an empty result (no signal)")
- }
-}
-
-func TestSCAParsesUnifiedVerdict(t *testing.T) {
- mdl := &stubJSONModel{value: map[string]any{
- "is_sufficient": false,
- "confidence": 0.42,
- "contradictions": []any{"X says 1999, Y says 2001"},
- "reasoning": "missing the release year",
- "sub_queries": []any{
- map[string]any{"sub_query": "release year", "satisfied": false,
- "missing_fact": "the year", "search_hint": "Culdcept release date"},
- map[string]any{"sub_query": "developer", "satisfied": true},
- },
- "claims": []any{
- map[string]any{"claim_id": "c1", "grounded": false,
- "missing_information": []any{map[string]any{"what": "release year", "search_hint": "Culdcept 1999"}}},
- },
- }}
- got := SufficientContextAgent(context.Background(), SCADeps{
- KB: &runtime.Kbinfos{},
- Model: mdl,
- }, "q", []ClaimDraft{{ID: "c1", Draft: "OmiyaSoft made it"}})
-
- if got.IsSufficient {
- t.Error("is_sufficient must be false")
- }
- if got.Confidence != 0.42 {
- t.Errorf("confidence = %v, want 0.42", got.Confidence)
- }
- if len(got.Contradictions) != 1 || got.Contradictions[0] != "X says 1999, Y says 2001" {
- t.Errorf("contradictions = %v", got.Contradictions)
- }
- if len(got.SubQueries) != 2 {
- t.Fatalf("sub_queries = %d, want 2", len(got.SubQueries))
- }
- if got.SubQueries[0].Satisfied || got.SubQueries[0].MissingFact != "the year" {
- t.Errorf("sub_query[0] = %+v", got.SubQueries[0])
- }
- if !got.SubQueries[1].Satisfied {
- t.Error("sub_query[1] must be satisfied")
- }
- cv, ok := got.Claims["c1"]
- if !ok {
- t.Fatal("claim c1 missing from verdicts")
- }
- if cv.Grounded {
- t.Error("claim c1 must be ungrounded")
- }
- if len(cv.MissingInformation) != 1 || cv.MissingInformation[0].What != "release year" {
- t.Errorf("missing_information = %+v", cv.MissingInformation)
- }
-}
-
-func TestSCACoercesArrayAndStringDrift(t *testing.T) {
- // Model drift: a bare array, or a JSON string, must still yield a verdict
- // instead of being dropped (which would silently stop replan/rewrite).
- for name, value := range map[string]any{
- "array": []any{map[string]any{"is_sufficient": true, "confidence": 0.9}},
- "string": `{"is_sufficient": true, "confidence": 0.7}`,
- } {
- mdl := &stubJSONModel{value: value}
- got := SufficientContextAgent(context.Background(), SCADeps{
- KB: &runtime.Kbinfos{},
- Model: mdl,
- }, "q", []ClaimDraft{{ID: "c1", Draft: "d"}})
- if !got.IsSufficient {
- t.Errorf("%s drift: verdict lost", name)
- }
- }
-}
-
-func TestSCADerivesGapWhenInsufficientWithEmptyClaims(t *testing.T) {
- // Failsafe: insufficient + empty claims array (a known degradation on long
- // prompts) must still produce a gap, else the orchestrator abandons despite
- // having usable evidence.
- mdl := &stubJSONModel{value: map[string]any{
- "is_sufficient": false,
- "missing_information": []any{map[string]any{"what": "the year", "search_hint": "1999"}},
- }}
- got := SufficientContextAgent(context.Background(), SCADeps{
- KB: &runtime.Kbinfos{},
- Model: mdl,
- }, "q", []ClaimDraft{{ID: "c1", Draft: "OmiyaSoft made it"}})
- g, ok := got.Claims["_global"]
- if !ok {
- t.Fatal("top-level missing_information must be harvested as the _global gap")
- }
- if len(g.MissingInformation) != 1 || g.MissingInformation[0].What != "the year" {
- t.Errorf("gap = %+v", g.MissingInformation)
- }
-
- // No structured gap at all: fall back to the drafts themselves.
- mdl = &stubJSONModel{value: map[string]any{"is_sufficient": false}}
- got = SufficientContextAgent(context.Background(), SCADeps{
- KB: &runtime.Kbinfos{},
- Model: mdl,
- }, "q", []ClaimDraft{{ID: "c1", Draft: "OmiyaSoft made it"}})
- g, ok = got.Claims["_global"]
- if !ok || len(g.MissingInformation) == 0 {
- t.Fatal("draft-derived fallback gap missing")
- }
- if g.MissingInformation[0].What != "OmiyaSoft made it" {
- t.Errorf("fallback gap = %q, want the draft text", g.MissingInformation[0].What)
- }
-}
-
-func TestSCAEvidenceAnchorsUseChunkIndices(t *testing.T) {
- // EvidenceIDs are INDICES into Kbinfos.Chunks (not chunk_id hashes) — that
- // indexing is what makes anchor lookup hit.
- kb := &runtime.Kbinfos{Chunks: []map[string]any{
- {"content": "Culdcept was released in 1999 by OmiyaSoft."},
- }}
- contextText := renderClaimContext([]ClaimDraft{
- {ID: "c1", Draft: "OmiyaSoft made Culdcept", EvidenceIDs: []string{"0"}},
- }, kb)
- if !strings.Contains(contextText, "1999") {
- t.Errorf("anchor not resolved from index 0; got %q", contextText)
- }
- // An out-of-range index is simply skipped.
- if s := renderClaimContext([]ClaimDraft{{ID: "c1", Draft: "d", EvidenceIDs: []string{"99"}}}, kb); strings.Contains(s, "Evidence:") {
- t.Error("unknown evidence index must not produce an anchor")
- }
-}
-
-func TestBoundedExcerptKeepsTablesWhole(t *testing.T) {
- table := "| a | b |\n| c | d |\n| e | f |\n| g | h |"
- if got := boundedExcerpt(table, "unrelated hint", 50); got != table {
- t.Errorf("table text must be returned whole, got %q", got)
- }
- // An HTML table reaches the SCA as Markdown with every row: the SCA is a plain LLM
- // call, so raw <table>/<td> markup makes it parse the cells itself, and the row set
- // must stay complete (a row can carry the answer mid-table).
- htmlTable := "<table><tr><th>Rank</th><th>Rider</th><th>Points</th></tr>" +
- "<tr><td>19</td><td>Danilo</td><td>62</td></tr>" +
- "<tr><td>20</td><td>Erik</td><td>61</td></tr></table>"
- gotTable := boundedExcerpt(htmlTable, "unrelated hint", 50)
- if strings.Contains(gotTable, "<td>") || strings.Contains(gotTable, "<table") {
- t.Errorf("HTML table must reach the SCA as Markdown, got %q", gotTable)
- }
- for _, want := range []string{"| 19 |", "Danilo", "| 20 |", "Erik"} {
- if !strings.Contains(gotTable, want) {
- t.Errorf("anchor lost %q: %q", want, gotTable)
- }
- }
- // Prose is windowed around a hint token.
- prose := strings.Repeat("prefix filler ", 40) + "TARGET sentence here" + strings.Repeat(" trailing filler", 40)
- got := boundedExcerpt(prose, "TARGET", 120)
- if len(got) > 130 {
- t.Errorf("prose excerpt not bounded: %d chars", len(got))
- }
- if !strings.Contains(got, "TARGET") {
- t.Error("excerpt must contain the hint token")
- }
-}
-
-// boundedExcerpt must slice by code point, not byte, so a CJK hint inside a CJK
-// passage is windowed without splitting a rune (which would emit invalid UTF-8).
-func TestBoundedExcerptCJKByRune(t *testing.T) {
- // 12 CJK chars around a TARGET token; with maxChars well below the full
- // length the window should still be valid UTF-8 and contain the token.
- prose := strings.Repeat("字", 40) + "目标句在此" + strings.Repeat("文", 40)
- got := boundedExcerpt(prose, "目标", 120)
- if !utf8.ValidString(got) {
- t.Errorf("CJK excerpt is not valid UTF-8: %q", got)
- }
- if !strings.Contains(got, "目标") {
- t.Errorf("CJK excerpt must contain the hint token, got %q", got)
- }
- // The no-hint tail path must also stay valid UTF-8 (text[-tail:] by byte
- // would split the trailing runes).
- got2 := boundedExcerpt(prose, "absent", 60)
- if !utf8.ValidString(got2) {
- t.Errorf("CJK tail excerpt is not valid UTF-8: %q", got2)
- }
-}
-
-func TestSCABoostAdaptsVerdict(t *testing.T) {
- res := SCAResult{
- IsSufficient: false,
- Confidence: 0.3,
- Contradictions: []string{"c1"},
- Claims: map[string]ClaimVerdict{
- "c1": {MissingInformation: []MissingPiece{{What: "the year"}}},
- "c2": {MissingInformation: []MissingPiece{{What: "the year"}}}, // duplicate
- },
- }
- b := res.Boost([]string{"fallback query"})
- if b.IsSufficient {
- t.Error("boost must carry is_sufficient=false")
- }
- if len(b.Missing) != 1 || b.Missing[0] != "the year" {
- t.Errorf("missing = %v, want the deduped 'the year'", b.Missing)
- }
- if !strings.Contains(b.Feedback, "the year") {
- t.Errorf("feedback = %q, must mention the missing piece", b.Feedback)
- }
- if len(b.Followups) != 1 || b.Followups[0] != "fallback query" {
- t.Errorf("followups = %v", b.Followups)
- }
-}
-
-// Query rewriter
-
-func TestClampCoercesNumericDrift(t *testing.T) {
- if got := clamp(float64(1.5)); got != 1.0 {
- t.Errorf("clamp(1.5) = %v, want 1.0", got)
- }
- if got := clamp(float64(-0.2)); got != 0.0 {
- t.Errorf("clamp(-0.2) = %v, want 0.0", got)
- }
- // Numeric drift: a string number is still parsed.
- if got := clamp("0.4"); got != 0.4 {
- t.Errorf("clamp(\"0.4\") = %v, want 0.4", got)
- }
- // Unparseable -> the default of 1.0.
- if got := clamp("not a number"); got != 1.0 {
- t.Errorf("clamp(bad) = %v, want 1.0", got)
- }
-}
-
-// TestCoerceBoolParsesStringDrift pins that a boolean serialised as a string is
-// read by MEANING, not by raw truthiness: "false"/"0" are false, not true. An
-// empty or unrecognised value fails CLOSED (false), matching the "unusable
-// verdict is INSUFFICIENT" convention.
-func TestCoerceBoolParsesStringDrift(t *testing.T) {
- for _, v := range []any{true, "true", "TRUE", " True ", "1", "yes", "Y", "on", float64(1)} {
- if !coerceBool(v) {
- t.Errorf("coerceBool(%#v) = false, want true", v)
- }
- }
- for _, v := range []any{false, "false", "FALSE", " false ", "0", "no", "N", "off", "", float64(0), nil, "maybe"} {
- if coerceBool(v) {
- t.Errorf("coerceBool(%#v) = true, want false", v)
- }
- }
-}
-
-// TestSCAParsesStringBooleans pins the drift fix end to end: a model that
-// serialises its verdict booleans as strings must not have them inverted.
-func TestSCAParsesStringBooleans(t *testing.T) {
- mdl := &stubJSONModel{value: map[string]any{
- "is_sufficient": "false",
- "sub_queries": []any{
- map[string]any{"sub_query": "revenue", "satisfied": "false",
- "missing_fact": "2023 revenue", "search_hint": "annual report"},
- },
- "claims": []any{
- map[string]any{"claim_id": "c1", "grounded": "false"},
- },
- }}
- got := SufficientContextAgent(context.Background(), SCADeps{
- KB: &runtime.Kbinfos{},
- Model: mdl,
- }, "q", []ClaimDraft{{ID: "c1", Draft: "d"}})
- if got.IsSufficient {
- t.Error(`is_sufficient:"false" must read as false, not true`)
- }
- if len(got.SubQueries) != 1 || got.SubQueries[0].Satisfied {
- t.Errorf("sub_queries = %+v, want one unsatisfied", got.SubQueries)
- }
- if cv := got.Claims["c1"]; cv.Grounded {
- t.Error(`grounded:"false" must read as false, not true`)
- }
-
- // The "true" spelling still parses.
- mdl = &stubJSONModel{value: map[string]any{"is_sufficient": "true"}}
- got = SufficientContextAgent(context.Background(), SCADeps{
- KB: &runtime.Kbinfos{},
- Model: mdl,
- }, "q", []ClaimDraft{{ID: "c1", Draft: "d"}})
- if !got.IsSufficient {
- t.Error(`is_sufficient:"true" must read as true`)
- }
-}
-
-// TestSCAIgnoresAbsentFields pins that an absent (nil) field never renders as the
-// literal "<nil>": a claim without claim_id and a sub-query without sub_query are
-// rejected, and the optional text fields stay empty instead of becoming "<nil>".
-func TestSCAIgnoresAbsentFields(t *testing.T) {
- mdl := &stubJSONModel{value: map[string]any{
- "sub_queries": []any{
- map[string]any{"satisfied": false}, // no sub_query
- map[string]any{"sub_query": "ok", "satisfied": false}, // no missing_fact/search_hint
- },
- "claims": []any{
- map[string]any{"grounded": true}, // no claim_id
- map[string]any{"claim_id": "c1", "grounded": true}, // no missing_information
- },
- }}
- got := SufficientContextAgent(context.Background(), SCADeps{
- KB: &runtime.Kbinfos{},
- Model: mdl,
- }, "q", []ClaimDraft{{ID: "c1", Draft: "d"}})
-
- if _, ok := got.Claims["<nil>"]; ok {
- t.Error(`a claim with no claim_id must be rejected, not keyed as "<nil>"`)
- }
- if _, ok := got.Claims["c1"]; !ok {
- t.Error("the claim with a real claim_id must be kept")
- }
- if len(got.SubQueries) != 1 {
- t.Fatalf("sub_queries = %+v, want only the one with a real sub_query", got.SubQueries)
- }
- if sq := got.SubQueries[0]; sq.SubQuery != "ok" || sq.MissingFact != "" || sq.SearchHint != "" {
- t.Errorf("absent missing_fact/search_hint must stay empty, got %+v", sq)
- }
- if got.Reasoning != "" {
- t.Errorf(`absent reasoning must stay empty, got %q`, got.Reasoning)
- }
-}
-
-// TestSCAMissingInformationIgnoresAbsentFields pins that a missing_information
-// entry with no usable text is not fabricated from nil (which fmt.Sprint would
-// render as "<nil>").
-func TestSCAMissingInformationIgnoresAbsentFields(t *testing.T) {
- mdl := &stubJSONModel{value: map[string]any{
- "is_sufficient": false,
- "claims": []any{
- map[string]any{"claim_id": "c1", "grounded": false, "missing_information": []any{
- map[string]any{}, // both what/search_hint absent
- nil, // nil element
- map[string]any{"what": "the year"},
- }},
- },
- }}
- got := SufficientContextAgent(context.Background(), SCADeps{
- KB: &runtime.Kbinfos{},
- Model: mdl,
- }, "q", []ClaimDraft{{ID: "c1", Draft: "d"}})
- mi := got.Claims["c1"].MissingInformation
- if len(mi) != 1 || mi[0].What != "the year" || mi[0].SearchHint != "" {
- t.Fatalf("missing_information = %+v, want only the real entry", mi)
- }
-}
-
-// TestRenderOverallDraftTruncatesByRune pins that the cap is applied by code point, never
-// mid-rune: a byte slice through a CJK rune
-// would put invalid UTF-8 into the SCA prompt.
-func TestRenderOverallDraftTruncatesByRune(t *testing.T) {
- draft := strings.Repeat("中", scaClaimsContextMax+50)
- got := renderOverallDraft([]ClaimDraft{{ID: "c1", Draft: draft}})
- if !utf8.ValidString(got) {
- t.Errorf("truncated draft is not valid UTF-8: %q", got)
- }
- if n := utf8.RuneCountInString(got); n != scaClaimsContextMax {
- t.Errorf("truncated draft has %d runes, want %d", n, scaClaimsContextMax)
- }
-}
-
-// TestRenderClaimContextBudgetIsRuneBased pins that the claim-context budget is counted in
-// code points, not bytes: two CJK blocks of 20k
-// runes are ~40k runes (under the 48k cap) but ~120KB bytes (far over it), so a
-// byte budget would stop after the FIRST block and silently starve the SCA.
-func TestRenderClaimContextBudgetIsRuneBased(t *testing.T) {
- draft := strings.Repeat("中", 20000)
- got := renderClaimContext([]ClaimDraft{
- {ID: "c1", Draft: draft},
- {ID: "c2", Draft: draft},
- }, &runtime.Kbinfos{})
- if !strings.Contains(got, "Claim c1") || !strings.Contains(got, "Claim c2") {
- t.Errorf("rune-based budget must keep both CJK blocks; got %d runes", utf8.RuneCountInString(got))
- }
-}
-
-// TestRenderClaimContextTrimsOversizedBlock pins that the cap bounds the RENDERED
-// context: a single claim longer than scaClaimsContextMax must be trimmed to the
-// budget instead of being appended whole (which pushed the SCA prompt past the
-// window the cap was sized for).
-func TestRenderClaimContextTrimsOversizedBlock(t *testing.T) {
- got := renderClaimContext([]ClaimDraft{
- {ID: "c1", Draft: strings.Repeat("中", scaClaimsContextMax+5000)},
- }, &runtime.Kbinfos{})
- if !utf8.ValidString(got) {
- t.Error("trimmed block is not valid UTF-8")
- }
- if n := utf8.RuneCountInString(got); n != scaClaimsContextMax {
- t.Errorf("rendered context = %d runes, want it capped at %d", n, scaClaimsContextMax)
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/porter_stemmer.go b/internal/rag/agentic-rag/runtime/porter_stemmer.go
deleted file mode 100644
index f596737b91..0000000000
--- a/internal/rag/agentic-rag/runtime/porter_stemmer.go
+++ /dev/null
@@ -1,401 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import "strings"
-
-// Porter stemming, a faithful port of nltk.stem.porter.PorterStemmer in its default
-// NLTK_EXTENSIONS mode — the mode used in production (`PorterStemmer().stem`, no mode
-// argument). It is called for every stemmable keyword token, so the keyword narrowing in this
-// package must produce the SAME stems (a suffix-stripping fallback diverged:
-// "nominations"→"nominat" vs "nomin", "flies"→"fle" vs "fli", "dies"→"dy" vs "die", ...).
-//
-// The port follows the 1980 paper's five steps plus NLTK's extensions: the
-// irregular-form pool consulted first, the <=2-letter circuit breaker, the
-// 4-letter "ies"/"ied" reductions, the consonant-conditional step-1c, the
-// "bli"/"fulli"/recursive-"alli" step-2 rules, the "logi" measure quirk and
-// the 2-letter *o exception.
-
-// porterPool mirrors nltk's _pool: irregular forms keyed by their restored
-// form (NLTK_EXTENSIONS only). Consulted before any rule runs.
-var porterPool = map[string][]string{
- "sky": {"sky", "skies"},
- "die": {"dying"},
- "lie": {"lying"},
- "tie": {"tying"},
- "news": {"news"},
- "inning": {"innings", "inning"},
- "outing": {"outings", "outing"},
- "canning": {"cannings", "canning"},
- "howe": {"howe"},
- "proceed": {"proceed"},
- "exceed": {"exceed"},
- "succeed": {"succeed"},
-}
-
-func porterStem(word string) string {
- if word == "" {
- return word
- }
- for restored, forms := range porterPool {
- for _, form := range forms {
- if word == form {
- return restored
- }
- }
- }
- // NLTK_EXTENSIONS short-word circuit breaker (nltk stem(): length <= 2 is
- // returned unchanged; the paper never mentions it).
- if len(word) <= 2 {
- return word
- }
- w := porterStep1a(word)
- w = porterStep1b(w)
- w = porterStep1c(w)
- w = porterStep2(w)
- w = porterStep3(w)
- w = porterStep4(w)
- w = porterStep5a(w)
- w = porterStep5b(w)
- return w
-}
-
-// porterIsConsonant mirrors nltk _is_consonant with the iterative y-chain
-// resolution: 'y' at position 0 is a consonant; a 'y' counts as a consonant
-// when the character before it is one (y-runs resolved iteratively).
-func porterIsConsonant(word string, i int) bool {
- if i < 0 || i >= len(word) {
- return false
- }
- switch c := word[i]; c {
- case 'a', 'e', 'i', 'o', 'u':
- return false
- case 'y':
- negate := false
- for i > 0 && word[i] == 'y' {
- negate = !negate
- i--
- }
- if i == 0 {
- // The y-run reaches position 0, where 'y' is a consonant.
- return !negate
- }
- return (strings.IndexByte("aeiou", word[i]) < 0) != negate
- default:
- return true
- }
-}
-
-// porterConsonantFlags classifies the whole word once (nltk
-// _consonant_flags): true = consonant. 'y' is a consonant at position 0 and
-// otherwise the complement of the previous character's classification.
-func porterConsonantFlags(word string) []bool {
- flags := make([]bool, len(word))
- for i := range word {
- switch {
- case i == 0:
- flags[i] = porterIsConsonant(word, 0)
- case word[i] == 'y':
- flags[i] = !flags[i-1]
- default:
- flags[i] = porterIsConsonant(word, i)
- }
- }
- return flags
-}
-
-// porterMeasure mirrors nltk _measure: the number of VC sequences in the
-// word's consonant/vowel shape ("cvccvc" → 2, "cvc" → 1).
-func porterMeasure(stem string) int {
- flags := porterConsonantFlags(stem)
- m := 0
- for i := 1; i < len(flags); i++ {
- if flags[i] && !flags[i-1] {
- m++
- }
- }
- return m
-}
-
-// porterContainsVowel mirrors nltk _contains_vowel.
-func porterContainsVowel(stem string) bool {
- for _, cons := range porterConsonantFlags(stem) {
- if !cons {
- return true
- }
- }
- return false
-}
-
-// porterEndsDoubleConsonant mirrors nltk _ends_double_consonant (*d).
-func porterEndsDoubleConsonant(word string) bool {
- return len(word) >= 2 && word[len(word)-1] == word[len(word)-2] &&
- porterIsConsonant(word, len(word)-1)
-}
-
-// porterEndsCVC mirrors nltk _ends_cvc (*o): consonant-vowel-consonant with
-// the final consonant not w/x/y — plus NLTK's 2-letter cv special case.
-func porterEndsCVC(word string) bool {
- if len(word) >= 3 &&
- porterIsConsonant(word, len(word)-3) &&
- !porterIsConsonant(word, len(word)-2) &&
- porterIsConsonant(word, len(word)-1) &&
- word[len(word)-1] != 'w' && word[len(word)-1] != 'x' && word[len(word)-1] != 'y' {
- return true
- }
- return len(word) == 2 && !porterIsConsonant(word, 0) && porterIsConsonant(word, 1)
-}
-
-// porterRule is one (suffix, replacement, condition) rule; condition nil means
-// unconditional. A condition receives the word with the suffix already
-// removed. The replacement replaces the SUFFIX (the stem keeps its letters);
-// the two special step-1b rules append to the stem instead, marked by the
-// replacement markers below.
-type porterRule struct {
- suffix string
- replacement string
- condition func(stem string) bool
-}
-
-// porterAppendE marks the step-1b (m=1 and *o) rule: append "e" to the stem.
-const porterAppendE = "\x00e"
-
-// porterApplyRuleList mirrors nltk _apply_rule_list: the FIRST suffix that
-// matches wins — even when its condition fails (the walk stops and the word is
-// returned unchanged). The one exception is nltk's "*d" rule: when the word
-// ends in a doubled consonant the stem is word[:-2] and a FAILED condition
-// falls through to the next rule instead of stopping (which is how
-// "falling"→"fall" passes the *d branch and reaches the +e branch's guard).
-func porterApplyRuleList(word string, rules []porterRule) string {
- for _, r := range rules {
- if r.suffix == "*d" {
- if !porterEndsDoubleConsonant(word) {
- continue
- }
- stem := word[:len(word)-2]
- if r.condition(stem) {
- return stem + word[len(word)-1:]
- }
- continue
- }
- if !strings.HasSuffix(word, r.suffix) {
- continue
- }
- stem := strings.TrimSuffix(word, r.suffix)
- if r.condition != nil && !r.condition(stem) {
- return word
- }
- if r.replacement == porterAppendE {
- return stem + "e"
- }
- return stem + r.replacement
- }
- return word
-}
-
-func porterHasPositiveMeasure(stem string) bool { return porterMeasure(stem) > 0 }
-
-func porterStep1a(word string) string {
- // NLTK_EXTENSIONS: a 4-letter "ies" reduces to "ie" (dies→die, ties→tie).
- if strings.HasSuffix(word, "ies") && len(word) == 4 {
- return strings.TrimSuffix(word, "ies") + "ie"
- }
- return porterApplyRuleList(word, []porterRule{
- {suffix: "sses", replacement: "ss"},
- {suffix: "ies", replacement: "i"},
- {suffix: "ss", replacement: "ss"},
- {suffix: "s", replacement: ""},
- })
-}
-
-// porterStep1b mirrors nltk _step1b.
-func porterStep1b(word string) string {
- // NLTK_EXTENSIONS: "ied" reduces by one letter for a 4-letter word
- // (died→die), by two otherwise (spied→spi) — before the eed/ed/ing walk.
- if strings.HasSuffix(word, "ied") {
- if len(word) == 4 {
- return strings.TrimSuffix(word, "ied") + "ie"
- }
- return strings.TrimSuffix(word, "ied") + "i"
- }
- if strings.HasSuffix(word, "eed") {
- if porterHasPositiveMeasure(strings.TrimSuffix(word, "eed")) {
- return strings.TrimSuffix(word, "eed") + "ee"
- }
- return word
- }
- intermediate := ""
- if strings.HasSuffix(word, "ed") && porterContainsVowel(strings.TrimSuffix(word, "ed")) {
- intermediate = strings.TrimSuffix(word, "ed")
- } else if strings.HasSuffix(word, "ing") && porterContainsVowel(strings.TrimSuffix(word, "ing")) {
- intermediate = strings.TrimSuffix(word, "ing")
- } else {
- return word
- }
- // nltk's _step1b_helper: the four post-removal rules are an if/else CHAIN
- // (first hit wins, and the *d branch is exclusive with the +e branch —
- // "falling"→"fall" passes the *d guard on l/s/z and then fails the +e
- // guard, staying "fall").
- switch {
- case strings.HasSuffix(intermediate, "at"):
- return intermediate + "e"
- case strings.HasSuffix(intermediate, "bl"):
- return intermediate + "e"
- case strings.HasSuffix(intermediate, "iz"):
- return intermediate + "e"
- case porterEndsDoubleConsonant(intermediate) &&
- intermediate[len(intermediate)-1] != 'l' &&
- intermediate[len(intermediate)-1] != 's' &&
- intermediate[len(intermediate)-1] != 'z':
- // *d and not (*L or *S or *Z): collapse the doubled consonant
- // (hopp→hop, tann→tan).
- return intermediate[:len(intermediate)-1]
- case porterMeasure(intermediate) == 1 && porterEndsCVC(intermediate):
- // (m=1 and *o) → + E (fil→file; fail→fail stays).
- return intermediate + "e"
- }
- return intermediate
-}
-
-// porterStep1c mirrors nltk _step1c in NLTK_EXTENSIONS mode: y→i when the word
-// (minus y) is longer than one letter AND the character before the y is a
-// consonant — so happy→happi but enjoy→enjoy, and spy/fly/try merge with their
-// -ied/-ies forms (spy→spi, spied→spi).
-func porterStep1c(word string) string {
- if !strings.HasSuffix(word, "y") {
- return word
- }
- stem := strings.TrimSuffix(word, "y")
- if len(stem) > 1 && porterIsConsonant(stem, len(stem)-1) {
- return stem + "i"
- }
- return word
-}
-
-// porterStep2 mirrors nltk _step2: map double suffixes to single ones when the
-// stem has positive measure. NLTK_EXTENSIONS first unwraps a trailing "alli"
-// (radicalli→radical→radic) by recursing.
-func porterStep2(word string) string {
- if strings.HasSuffix(word, "alli") {
- stem := strings.TrimSuffix(word, "alli")
- if porterHasPositiveMeasure(stem) {
- return porterStep2(stem + "al")
- }
- }
- measure := func(stem string) bool { return porterHasPositiveMeasure(stem) }
- rules := []porterRule{
- {suffix: "ational", replacement: "ate", condition: measure},
- {suffix: "tional", replacement: "tion", condition: measure},
- {suffix: "enci", replacement: "ence", condition: measure},
- {suffix: "anci", replacement: "ance", condition: measure},
- {suffix: "izer", replacement: "ize", condition: measure},
- {suffix: "bli", replacement: "ble", condition: measure},
- {suffix: "alli", replacement: "al", condition: measure},
- {suffix: "entli", replacement: "ent", condition: measure},
- {suffix: "eli", replacement: "e", condition: measure},
- {suffix: "ousli", replacement: "ous", condition: measure},
- {suffix: "ization", replacement: "ize", condition: measure},
- {suffix: "ation", replacement: "ate", condition: measure},
- {suffix: "ator", replacement: "ate", condition: measure},
- {suffix: "alism", replacement: "al", condition: measure},
- {suffix: "iveness", replacement: "ive", condition: measure},
- {suffix: "fulness", replacement: "ful", condition: measure},
- {suffix: "ousness", replacement: "ous", condition: measure},
- {suffix: "aliti", replacement: "al", condition: measure},
- {suffix: "iviti", replacement: "ive", condition: measure},
- {suffix: "biliti", replacement: "ble", condition: measure},
- // NLTK_EXTENSIONS additions: "fulli", and the "logi" quirk whose
- // condition measures the stem INCLUDING its trailing "l" (word minus
- // "log"), so geo/theo endings (archaeologi, philologi) reduce.
- {suffix: "fulli", replacement: "ful", condition: measure},
- {suffix: "logi", replacement: "log", condition: func(stem string) bool {
- return len(stem) >= 1 && porterHasPositiveMeasure(stem[:len(stem)-1])
- }},
- }
- return porterApplyRuleList(word, rules)
-}
-
-// porterStep3 mirrors nltk _step3: -ic-, -full, -ness etc. (all measure > 0).
-func porterStep3(word string) string {
- measure := func(stem string) bool { return porterHasPositiveMeasure(stem) }
- return porterApplyRuleList(word, []porterRule{
- {suffix: "icate", replacement: "ic", condition: measure},
- {suffix: "ative", replacement: "", condition: measure},
- {suffix: "alize", replacement: "al", condition: measure},
- {suffix: "iciti", replacement: "ic", condition: measure},
- {suffix: "ical", replacement: "ic", condition: measure},
- {suffix: "ful", replacement: "", condition: measure},
- {suffix: "ness", replacement: "", condition: measure},
- })
-}
-
-// porterStep4 mirrors nltk _step4: drop -ant/-ence etc. when the stem's
-// measure exceeds 1 ("ion" additionally requires an s/t before the suffix).
-func porterStep4(word string) string {
- m := func(stem string) bool { return porterMeasure(stem) > 1 }
- rules := []porterRule{
- {suffix: "al", condition: m},
- {suffix: "ance", condition: m},
- {suffix: "ence", condition: m},
- {suffix: "er", condition: m},
- {suffix: "ic", condition: m},
- {suffix: "able", condition: m},
- {suffix: "ible", condition: m},
- {suffix: "ant", condition: m},
- {suffix: "ement", condition: m},
- {suffix: "ment", condition: m},
- {suffix: "ent", condition: m},
- {suffix: "ion", condition: func(stem string) bool {
- return porterMeasure(stem) > 1 && len(stem) > 0 &&
- (stem[len(stem)-1] == 's' || stem[len(stem)-1] == 't')
- }},
- {suffix: "ou", condition: m},
- {suffix: "ism", condition: m},
- {suffix: "ate", condition: m},
- {suffix: "iti", condition: m},
- {suffix: "ous", condition: m},
- {suffix: "ive", condition: m},
- {suffix: "ize", condition: m},
- }
- return porterApplyRuleList(word, rules)
-}
-
-// porterStep5a mirrors nltk _step5a — deliberately NOT a rule list (the paper
-// lets BOTH conditions apply in turn here, unlike every other step):
-// - (m>1) E → probate→probat
-// - (m=1 and not *o) E → cease→ceas
-func porterStep5a(word string) string {
- if strings.HasSuffix(word, "e") {
- stem := strings.TrimSuffix(word, "e")
- if porterMeasure(stem) > 1 {
- return stem
- }
- if porterMeasure(stem) == 1 && !porterEndsCVC(stem) {
- return stem
- }
- }
- return word
-}
-
-// porterStep5b mirrors nltk _step5b: (m>1, measured on word minus its last
-// letter) LL → L (controll→control; roll→roll stays).
-func porterStep5b(word string) string {
- if strings.HasSuffix(word, "ll") && porterMeasure(word[:len(word)-1]) > 1 {
- return strings.TrimSuffix(word, "l")
- }
- return word
-}
diff --git a/internal/rag/agentic-rag/runtime/porter_stemmer_test.go b/internal/rag/agentic-rag/runtime/porter_stemmer_test.go
deleted file mode 100644
index dbe6837828..0000000000
--- a/internal/rag/agentic-rag/runtime/porter_stemmer_test.go
+++ /dev/null
@@ -1,89 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import "testing"
-
-// TestPorterStemMatchesNLTK pins porterStem against nltk.stem.porter's NLTK_EXTENSIONS output
-// (PorterStemmer().stem), collected in production's environment. Every pair below is ground
-// truth, not aspiration: the keyword-narrowing stem match (keywordForms/_sentence_stems) must
-// produce identical stems or the narrow keeps different sentences.
-func TestPorterStemMatchesNLTK(t *testing.T) {
- cases := []struct{ word, want string }{
- // Step-1a/1b regulars.
- {"nominations", "nomin"}, {"nominated", "nomin"}, {"companies", "compani"},
- {"running", "run"}, {"caresses", "caress"}, {"ponies", "poni"},
- {"caress", "caress"}, {"cats", "cat"}, {"meetings", "meet"}, {"stems", "stem"},
- {"cries", "cri"}, {"flies", "fli"},
- // Irregular pool.
- {"sky", "sky"}, {"skies", "sky"}, {"dying", "die"}, {"lying", "lie"},
- {"tying", "tie"}, {"news", "news"}, {"innings", "inning"},
- {"outings", "outing"}, {"cannings", "canning"}, {"howe", "howe"},
- {"proceed", "proceed"}, {"exceed", "exceed"}, {"succeed", "succeed"},
- // 4-letter ies/ied reductions.
- {"ties", "tie"}, {"dies", "die"}, {"died", "die"}, {"spied", "spi"},
- // eed / *d / *o step-1b branches.
- {"agreed", "agre"}, {"feed", "feed"}, {"plastered", "plaster"},
- {"bled", "bled"}, {"motoring", "motor"}, {"sing", "sing"},
- {"conflated", "conflat"}, {"troubled", "troubl"}, {"sized", "size"},
- {"hopping", "hop"}, {"falling", "fall"}, {"filing", "file"},
- // Step-1c y-condition (consonant before the y; len(stem) > 1).
- {"happy", "happi"}, {"enjoy", "enjoy"}, {"spy", "spi"}, {"try", "tri"},
- // Step-2/3/4 table from the paper, NLTK mode.
- {"relation", "relat"}, {"conditional", "condit"}, {"rational", "ration"},
- {"valenci", "valenc"}, {"hesitanci", "hesit"}, {"digitizer", "digit"},
- {"conformabli", "conform"}, {"radicalli", "radic"},
- {"differentli", "differ"}, {"vileli", "vile"},
- {"analogousli", "analog"}, {"vietnamization", "vietnam"},
- {"predication", "predic"}, {"operator", "oper"},
- {"feudalism", "feudal"}, {"decisiveness", "decis"},
- {"hopefulness", "hope"}, {"callousness", "callous"},
- {"formaliti", "formal"}, {"sensitiviti", "sensit"},
- {"sensibiliti", "sensibl"}, {"triplicate", "triplic"},
- {"formative", "form"}, {"formalize", "formal"},
- {"electriciti", "electr"}, {"electrical", "electr"},
- {"hopeful", "hope"}, {"goodness", "good"},
- {"revival", "reviv"}, {"allowance", "allow"}, {"inference", "infer"},
- {"airliner", "airlin"}, {"gyroscopic", "gyroscop"},
- {"adjustable", "adjust"}, {"defensible", "defens"},
- {"irritant", "irrit"}, {"replacement", "replac"},
- {"adjustment", "adjust"}, {"dependent", "depend"},
- {"adoption", "adopt"}, {"homologou", "homolog"},
- {"communism", "commun"}, {"activate", "activ"},
- {"angulariti", "angular"}, {"homologous", "homolog"},
- {"effective", "effect"}, {"bowdlerize", "bowdler"},
- // Step-5a/5b.
- {"probate", "probat"}, {"rate", "rate"}, {"cease", "ceas"},
- {"controll", "control"}, {"roll", "roll"},
- }
- for _, c := range cases {
- if got := porterStem(c.word); got != c.want {
- t.Errorf("porterStem(%q) = %q, want %q", c.word, got, c.want)
- }
- }
-}
-
-// TestStemUsesPorter pins the seam: the package's stem() (the one
-// keywordForms/_sentence_stems call) is the Porter stemmer.
-func TestStemUsesPorter(t *testing.T) {
- if stem("nominations") != "nomin" {
- t.Errorf("stem(nominations) = %q, want the Porter stem \"nomin\"", stem("nominations"))
- }
- if stem("flies") != "fli" {
- t.Errorf("stem(flies) = %q, want \"fli\"", stem("flies"))
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/report_prompt.go b/internal/rag/agentic-rag/runtime/report_prompt.go
deleted file mode 100644
index 569e0e8fc0..0000000000
--- a/internal/rag/agentic-rag/runtime/report_prompt.go
+++ /dev/null
@@ -1,54 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package runtime
-
-// Report synthesis prompts.
-//
-// Mirrors rag/advanced_rag/harness/prompts/report_prompt.py. {cite_rules} in
-// FinalAnswerSystem is substituted by the caller (see agentic_rag.ComposeAnswer).
-
-// FinalAnswerSystem mirrors report_prompt.FINAL_ANSWER_SYSTEM. {cite_rules} is
-// substituted by the caller.
-const FinalAnswerSystem = `You are a smart agent. Answer the user's question using ONLY the evidence provided below. Do not invent facts: if the evidence cannot support a claim, say so plainly instead of guessing.
-
-# Answer target
-First resolve the exact role requested by the user's question. Multi-hop questions
-often mention bridge entities that are only clues. Do not answer with a bridge
-entity just because it satisfies a later clue; answer the entity, value, or fact
-that satisfies the top-level question. If an Answer Target Contract is provided,
-obey it over any research-summary wording.
-
-# Citation rules
-{cite_rules}
-
-# Attribute fidelity (CRITICAL)
-Answer the EXACT attribute/relation the question asks for. Do NOT substitute a similar but
-different attribute, even when it is semantically related. For example:
-- HOMETOWN ≠ BIRTHPLACE (place of birth): if asked for someone's hometown, do not answer with
- where they were born unless the evidence equates the two.
-- FIRST ≠ LARGEST, AGE AT DEATH ≠ BIRTH YEAR, etc.
-Answer the question's own attribute using the evidence for THAT attribute. If the evidence only
-supports a different attribute, say that you could only find the related (different) attribute and
-do not present it as the answer to the requested one.
-
-# Language
-Answer in the SAME language as the question. Translate retrieved evidence into that language as part of composing the answer; only verbatim quoted snippets may stay in their source language.
-
-# Fallback
-If the evidence does not answer the question, reply with a clear statement that you don't have enough information based on the available sources (in the user's language).`
-
-// PartialAnswerPreamble mirrors report_prompt.PARTIAL_ANSWER_PREAMBLE.
-const PartialAnswerPreamble = "Note: the following answer is based on partial information and may be incomplete."
diff --git a/internal/rag/agentic-rag/runtime/session_state_line.go b/internal/rag/agentic-rag/runtime/session_state_line.go
deleted file mode 100644
index e3f1f1d1de..0000000000
--- a/internal/rag/agentic-rag/runtime/session_state_line.go
+++ /dev/null
@@ -1,402 +0,0 @@
-package runtime
-
-import (
- "fmt"
- "strings"
- "unicode/utf8"
-
- "ragflow/internal/tokenizer"
-)
-
-// SessionRecord is what a session has DONE so far, expressed as facts the ReAct
-// loop cannot recover from its own conversation.
-//
-// The session's message list holds the model's queries and the passages that came
-// back, and nothing else: which names were already probed (and which of those came
-// back empty), which members the record now holds, and which confirmed members are
-// still undecided — a name whose passage is in the pool and which no slot
-// mentions. Every one of those facts is computed by the runtime anyway, and
-// withholding them left the model driving its queries blind: it re-asked names it
-// had already resolved, and it could not see that a name it found was still
-// unrecorded.
-//
-// Without them a session can probe term after term and record none of them until it is
-// forced to, dropping a name it queried and found by its own reasoning — a decision the
-// runtime cannot see and therefore never asks about.
-//
-// The record is also the basis of the continuation decision: the offer the model
-// answers (offerContinuation) carries it, so "is anything still missing?" is a
-// question about facts the model can read, not about its appetite for searching.
-type SessionRecord struct {
- // Pool is the shared evidence pool's size at this turn.
- Pool int
- // Members are the names the slot table currently records (split out of the
- // candidates, so "孔秀、孟坦" counts as two).
- Members []string
- // Reached are the named terms a probe reached, with evidence (Kbinfos.Reached).
- Reached []string
- // Absent are the named terms probes looked for and did not find
- // (Kbinfos.ProbedAbsent): "asked and nothing came back" is a result, and it
- // is the one that tells the model to change the spelling or the angle.
- Absent []string
- // Undecided are confirmed members (Reached) that the slot table does not
- // mention at all: the evidence exists, the decision does not.
- Undecided []string
-}
-
-// CollectSessionRecord gathers the record from the live pool plus the session's
-// slot table. A nil pool yields the table half only.
-//
-// Everything here is derived from facts the runtime already holds — the slot
-// table and the ledger of terms the session itself asked about — and needs no
-// knowledge of the corpus's language, subject or relation.
-func CollectSessionRecord(table State, kb *Kbinfos) SessionRecord {
- var r SessionRecord
- if kb != nil {
- r.Pool = kb.PoolSize()
- for _, rt := range kb.ReachedTerms() {
- r.Reached = append(r.Reached, rt.Term)
- }
- r.Absent = kb.ProbedAbsentTerms()
- }
- // Members come from the slots that DECLARE members (slots.KindItems), each
- // item carrying its evidence. Nothing is split out of text: a slot holding a
- // sentence, a date or a count contributes no members, and the text is never
- // inspected to find out whether it might have been a list (see package slots for
- // the measurement that made this fail closed).
- // ONE definition of the table's members (see MemberNames): the count the answer
- // reports and the list this line shows read the same fact.
- r.Members = ItemValues(&table)
- joined := strings.ToLower(strings.Join(r.Members, "\x00"))
- for _, term := range r.Reached {
- // Substring containment, not equality: a candidate is often a clause
- // ("庞德被周仓生擒") rather than a bare name, and the question is only
- // whether the record has taken a position on this term at all.
- if !strings.Contains(joined, strings.ToLower(term)) {
- r.Undecided = append(r.Undecided, term)
- }
- }
- return r
-}
-
-// Line renders the record as ONE line for the tool result the model reads.
-//
-// It is deliberately a single line of counts plus a few names, not a block: it is
-// appended to every tool result, so its cost is paid once per turn, and it must
-// stay small enough that it never crowds out the passages it annotates.
-func (r SessionRecord) Line() string {
- var b strings.Builder
- b.WriteString("[record] ")
- fmt.Fprintf(&b, "members=%d", len(r.Members))
- if len(r.Members) > 0 {
- fmt.Fprintf(&b, " (%s)", shortList(r.Members, 6))
- }
- fmt.Fprintf(&b, " | probed-reached=%d", len(r.Reached))
- if len(r.Absent) > 0 {
- fmt.Fprintf(&b, " asked-nothing-back=%d (%s)", len(r.Absent), shortList(r.Absent, 4))
- }
- if len(r.Undecided) > 0 {
- fmt.Fprintf(&b, " | FOUND BUT NOT RECORDED=%s", shortList(r.Undecided, 4))
- }
- fmt.Fprintf(&b, " | pool=%d", r.Pool)
- return b.String()
-}
-
-// shortList joins up to max items on the list separator and marks the remainder.
-func shortList(items []string, max int) string {
- if len(items) > max {
- return strings.Join(items[:max], "、") + fmt.Sprintf("…+%d", len(items)-max)
- }
- return strings.Join(items, "、")
-}
-
-// sessionRecordNow computes the session's record as of this turn.
-func (s *SessionState) sessionRecordNow() SessionRecord {
- return CollectSessionRecord(s.workingTable(), s.KB)
-}
-
-// workingTable is the slot table as THIS session has patched it: the parent table
-// with the session's own branch patches applied in order. Reading the parent table
-// alone would report the record as it stood when the session started, so a member
-// the session itself just found would still read as missing.
-func (s *SessionState) workingTable() State {
- t := State{
- State: append([]Variable(nil), s.ParentState.State...),
- Depth: s.ParentState.Depth,
- ID: s.ParentState.ID,
- RetrievedEvidenceIDs: s.ParentState.RetrievedEvidenceIDs,
- }
- for _, branch := range s.NewStates {
- for _, patch := range branch.State {
- for i := range t.State {
- if t.State[i].ID != patch.ID || patch.Candidate == nil || *patch.Candidate == "" {
- continue
- }
- t.State[i].Candidate = patch.Candidate
- t.State[i].CandidateStrength = patch.CandidateStrength
- }
- }
- }
- return t
-}
-
-// Brief is the record's counts, for logs.
-func (r SessionRecord) Brief() string {
- out := fmt.Sprintf("members=%d reached=%d absent=%d undecided=%d",
- len(r.Members), len(r.Reached), len(r.Absent), len(r.Undecided))
- return out
-}
-
-// poolExcerptRunes bounds how much pool text one turn may add to the model's
-// context: about one sentence plus its neighbours, which is where a name and the
-// deed that makes it a member sit together.
-const poolExcerptRunes = 220
-
-// poolScanMax bounds how many pool chunks one turn examines while choosing that
-// excerpt. The scan is local — one substring test and one token count per chunk —
-// so this bound is about latency discipline, not about money.
-const poolScanMax = 60
-
-// subjectWordsMax bounds the words used to recognise "a passage about the same
-// subject". They select a passage to READ; they never decide anything.
-const subjectWordsMax = 8
-
-// unreadPoolExcerpt returns a short excerpt from ONE pool passage this session has
-// never been shown, chosen for what it says that the record does not.
-//
-// The pool is text the round has ALREADY paid for, and a session only ever sees
-// the parts its own queries returned — everything else sat in hand, unread. That
-// gap is where members are lost without anyone noticing: a passage can sit in the round's
-// evidence, unread by every session, because nothing showed it — and the names an answer is
-// missing are all in passages of exactly that kind.
-//
-// The framework does not name anybody here. It reads the unread passages that
-// mention a word THIS SESSION has used — the direction it was sent on, the slot's
-// clues, the candidates, and its own queries, which is what carries the corpus's
-// own aliases (a session that asked about 关公 gets passages that say 关公) — and
-// hands over a bounded excerpt around the first term of that passage the record
-// does not contain. The division of labour is the usual one: the runtime supplies
-// a fact (this text exists, you have not read it), the model decides what is in it.
-//
-// Mentioning one of those words is a FILTER, not a preference. Ranking unread passages by
-// how much of their vocabulary is new delivers passages the question has nothing to do
-// with, because "says the most you have not seen" and "is about this question" are
-// anti-correlated: the passages that carry the subject share their vocabulary with what the
-// session already read, so they score LOW. The
-// words the session itself used are what tells the two apart, and they are the
-// model's words, not a lexicon.
-func (s *SessionState) unreadPoolExcerpt() string {
- // ONE per session, and that is deliberate. The premise holds — such a passage really is
- // in the pool and really is unread — but the selection is not reliable enough to spend
- // context on every flat turn, so it stays available as the session's last resort rather
- // than a routine.
- if s.KB == nil || s.PoolRead {
- return ""
- }
- subject := s.subjectWords()
- if len(subject) == 0 {
- return ""
- }
- seen := make(map[string]bool, len(s.RetrievedEvidenceIDs))
- for _, id := range s.RetrievedEvidenceIDs {
- seen[id] = true
- }
- known := s.recordVocabulary()
- chunks := s.KB.ChunksFrom(s.PoolWalk, poolScanMax)
- s.PoolWalk += len(chunks)
-
- var bestID, bestText, bestAnchor string
- bestNovel := 0
- for _, c := range chunks {
- text := ChunkTextOf(c)
- if text == "" {
- continue
- }
- if id := ChunkIDOf(c); id != "" && seen[id] {
- continue
- }
- if !mentionsAny(text, subject) {
- continue
- }
- anchor, novel := novelAnchor(text, known)
- if novel == 0 {
- continue
- }
- if novel > bestNovel {
- bestID, bestText, bestAnchor, bestNovel = ChunkIDOf(c), text, anchor, novel
- }
- }
- if bestNovel == 0 {
- return ""
- }
- s.PoolRead = true
- return fmt.Sprintf("[pool] unread passage already in evidence, never shown to you (id=%s): %s",
- bestID, excerptAround(bestText, bestAnchor, poolExcerptRunes))
-}
-
-// mentionsAny reports whether text carries any of the words — the relevance test
-// the excerpt is gated on.
-func mentionsAny(text string, words []string) bool {
- for _, w := range words {
- if strings.Contains(text, w) {
- return true
- }
- }
- return false
-}
-
-// subjectWords returns the words that stand for what this session is working on:
-// the direction it was sent on, the slot's own question clues, the candidates so
-// far, and the queries the session itself wrote.
-//
-// They are the model's (or the planner's) own words, read with the corpus's own
-// tokenizer, so no list of names or verbs is involved anywhere.
-func (s *SessionState) subjectWords() []string {
- var out []string
- seen := map[string]bool{}
- add := func(text string) {
- if text == "" || len(out) >= subjectWordsMax {
- return
- }
- for _, w := range append(GrepWordsFromQuery(text), tokenizerWords(text)...) {
- w = strings.TrimSpace(w)
- if r := utf8.RuneCountInString(w); r < 2 || r > cjkPhraseRunes {
- continue
- }
- low := strings.ToLower(w)
- if seen[low] {
- continue
- }
- seen[low] = true
- out = append(out, w)
- if len(out) >= subjectWordsMax {
- return
- }
- }
- }
- // The session's OWN queries first: they are the most specific thing it has said, and
- // they are where the corpus's aliases enter — asking with one alias retrieves passages
- // that use another, while the direction may use only one.
- for _, q := range s.SearchQueries {
- add(q)
- }
- for _, v := range s.ParentState.State {
- if v.Candidate != nil {
- add(*v.Candidate)
- }
- for _, clue := range v.QuestionClues {
- add(clue)
- }
- }
- add(s.Direction)
- return out
-}
-
-// grewFrom reports whether this record knows something the previous turn's did.
-//
-// It is the same signal the continuation offer is built on (see
-// offerContinuation), used for the opposite decision: an enumeration session whose
-// record is still growing is already finding things, so the pool read stays out of
-// its way and only steps in when the record has gone flat.
-func (r SessionRecord) grewFrom(prev SessionRecord) bool {
- return len(r.Members) > len(prev.Members) ||
- len(r.Reached) > len(prev.Reached) ||
- len(r.Absent) > len(prev.Absent)
-}
-
-// tokenizerWords runs the corpus's own tokenizer over a text (see
-// tokenizer.Tokenize): the same segmentation the index was built with, which is
-// what makes it usable without a lexicon.
-//
-// When that tokenizer is not available — a unit test, an engine configured
-// differently — it falls back to the two-rune windows the grep path uses to
-// LOCATE a term inside unbroken text. As a vocabulary that is noisier, but it needs
-// nothing but the text, and a passage is only ever SELECTED by this score, never
-// decided on it.
-func tokenizerWords(text string) []string {
- if toks, err := tokenizer.Tokenize(text); err == nil && strings.TrimSpace(toks) != "" {
- return strings.Fields(toks)
- }
- return cjkWindowsOf(text, tokenWindowsMax)
-}
-
-// tokenWindowsMax bounds the fallback vocabulary of one passage.
-const tokenWindowsMax = 400
-
-// novelAnchor returns the first term of the text that the record does not contain,
-// and how many such terms the text has. The anchor is what the excerpt is centred
-// on — the part of the passage that is new to this session is the part worth
-// reading.
-func novelAnchor(text string, known map[string]bool) (string, int) {
- anchor, n := "", 0
- seen := map[string]bool{}
- for _, t := range tokenizerWords(text) {
- if r := utf8.RuneCountInString(t); r < 2 || r > cjkPhraseRunes {
- continue
- }
- low := strings.ToLower(t)
- if seen[low] || known[low] {
- continue
- }
- seen[low] = true
- if anchor == "" {
- anchor = t
- }
- n++
- }
- return anchor, n
-}
-
-// recordVocabulary is everything the record already accounts for: its members, the
-// terms probes reached, and the terms probes asked about and did not find. A term
-// in here is not new information.
-func (s *SessionState) recordVocabulary() map[string]bool {
- out := map[string]bool{}
- for _, group := range [][]string{s.Record.Members, s.Record.Reached, s.Record.Absent, s.Record.Undecided} {
- for _, t := range group {
- if t = strings.ToLower(strings.TrimSpace(t)); t != "" {
- out[t] = true
- }
- }
- }
- for _, q := range s.SearchQueries {
- for _, w := range GrepWordsFromQuery(q) {
- out[strings.ToLower(w)] = true
- }
- }
- return out
-}
-
-// excerptAround returns a one-line window of at most maxRunes runes around the
-// first occurrence of word, so the excerpt carries both the mention and the
-// sentence around it.
-func excerptAround(text, word string, maxRunes int) string {
- flat := []rune(strings.Join(strings.Fields(text), " "))
- at := []rune(word)
- pos := -1
- for i := 0; i+len(at) <= len(flat); i++ {
- if string(flat[i:i+len(at)]) == word {
- pos = i
- break
- }
- }
- if pos < 0 {
- pos = 0
- }
- start := pos - maxRunes/3
- if start < 0 {
- start = 0
- }
- end := start + maxRunes
- if end > len(flat) {
- end = len(flat)
- }
- out := string(flat[start:end])
- if start > 0 {
- out = "…" + out
- }
- if end < len(flat) {
- out += "…"
- }
- return out
-}
diff --git a/internal/rag/agentic-rag/runtime/session_state_line_test.go b/internal/rag/agentic-rag/runtime/session_state_line_test.go
deleted file mode 100644
index cd66fff79a..0000000000
--- a/internal/rag/agentic-rag/runtime/session_state_line_test.go
+++ /dev/null
@@ -1,559 +0,0 @@
-package runtime
-
-import (
- "strconv"
- "strings"
- "testing"
-
- "github.com/cloudwego/eino/schema"
-
- "ragflow/internal/rag/agentic-rag/slots"
-)
-
-// typedMembersSlot / typedCountSlot build the slots the record reads: a slot holds
-// members or a number because the model DECLARED it (see package slots), so these are
-// what a patch now produces.
-func typedMembersSlot(id int, typ string, names ...string) Variable {
- items := make([]slots.Item, 0, len(names))
- for _, n := range names {
- items = append(items, slots.Item{Value: n})
- }
- v := slots.Items(items...)
- rendered := slots.Render(v)
- return Variable{ID: id, Type: typ, Candidate: &rendered, Value: &v}
-}
-
-func typedCountSlot(id int, typ string, n int) Variable {
- v := slots.Number(n)
- rendered := slots.Render(v)
- return Variable{ID: id, Type: typ, Candidate: &rendered, Value: &v}
-}
-
-// TestCollectSessionRecordFlagsFoundButNotRecorded is the defect this record
-// exists for: a name the session PROVED reachable (it has a passage) and never
-// took a position on must be visible, not silent.
-//
-// Measured (fixrecall2, 2026-09-15): 庞德 was queried, a passage came back, and
-// the model's own reasoning dropped him — the framework saw nothing.
-func TestCollectSessionRecordFlagsFoundButNotRecorded(t *testing.T) {
- kb := &Kbinfos{}
- for i := 0; i < 5; i++ {
- id := "c" + strconv.Itoa(i)
- kb.Admit(func(p *PoolAdmitter) {
- p.Add(map[string]any{"chunk_id": id, "content": "prose"})
- })
- }
- kb.RecordReachedTerm("荀正", "c1")
- kb.RecordReachedTerm("庞德", "c2")
- kb.RecordProbedAbsent("杨龄")
-
- table := State{State: []Variable{
- typedCountSlot(0, "count", 13),
- typedMembersSlot(1, "person", "孔秀", "孟坦", "荀正"),
- }}
- rec := CollectSessionRecord(table, kb)
- // 13 is the COUNT, not a member: a member count inflated by the answer slot is
- // the number the model steers by, so the answer slot must not be in it.
- if len(rec.Members) != 3 || rec.Members[0] != "孔秀" {
- t.Fatalf("members = %v, want the 3 names the candidates list (no count)", rec.Members)
- }
- if len(rec.Reached) != 2 || len(rec.Absent) != 1 {
- t.Fatalf("ledger = reached %v absent %v, want 2 reached / 1 absent", rec.Reached, rec.Absent)
- }
- if len(rec.Undecided) != 1 || rec.Undecided[0] != "庞德" {
- t.Fatalf("undecided = %v, want [庞德]: 荀正 is recorded, 庞德 is not", rec.Undecided)
- }
- line := rec.Line()
- for _, want := range []string{"members=3", "FOUND BUT NOT RECORDED=庞德", "asked-nothing-back=1", "pool=5"} {
- if !strings.Contains(line, want) {
- t.Errorf("line %q missing %q", line, want)
- }
- }
-}
-
-// TestSessionRecordLineCarriesNoPoolJudgement pins what the line does NOT claim:
-// it reports the pool's size as a number and nothing more. The turn budget is the
-// model's decision now (see offerContinuation), so no runtime predicate may turn
-// "the pool grew" into "keep going".
-func TestSessionRecordLineCarriesNoPoolJudgement(t *testing.T) {
- rec := SessionRecord{Pool: 500, Members: []string{"华雄"}, Reached: []string{"华雄", "颜良"}}
- line := rec.Line()
- if !strings.Contains(line, "pool=500") {
- t.Fatalf("line %q, want the pool size reported", line)
- }
- for _, banned := range []string{"grew", "growth", "continue"} {
- if strings.Contains(strings.ToLower(line), banned) {
- t.Errorf("line %q must not advise on continuing", line)
- }
- }
-}
-
-// TestWorkingTableAppliesSessionPatches pins the table the record reads: the
-// session's own branches, or a member the session just found would still read as
-// missing.
-func TestWorkingTableAppliesSessionPatches(t *testing.T) {
- s := &SessionState{
- ParentState: State{State: []Variable{
- {ID: 1, Type: "person", Candidate: nil},
- {ID: 2, Type: "person", Candidate: strPtr("华雄")},
- }},
- NewStates: []State{{State: []Variable{
- {ID: 1, Type: "person", Candidate: strPtr("孔秀、孟坦")},
- }}},
- }
- tbl := s.workingTable()
- if tbl.State[0].Candidate == nil || *tbl.State[0].Candidate != "孔秀、孟坦" {
- t.Fatalf("patched slot = %v, want the session's own candidate", tbl.State[0].Candidate)
- }
- if tbl.State[1].Candidate == nil || *tbl.State[1].Candidate != "华雄" {
- t.Fatalf("untouched slot = %v, want the parent candidate kept", tbl.State[1].Candidate)
- }
- // The parent table is NOT mutated: another session shares it.
- if s.ParentState.State[0].Candidate != nil {
- t.Fatal("workingTable must not write through to the shared parent table")
- }
-}
-
-// TestAppendRecordLineRidesOnTheLastToolMessage pins the channel: one line on the
-// tool result the model is about to read, and nothing at all on a turn that ran no
-// tool call (or has no pool to report on).
-func TestAppendRecordLineRidesOnTheLastToolMessage(t *testing.T) {
- kb := &Kbinfos{}
- kb.Admit(func(p *PoolAdmitter) {
- p.Add(map[string]any{"chunk_id": "c1", "content": "prose"})
- })
- s := &SessionState{
- KB: kb,
- ParentState: State{State: []Variable{
- typedCountSlot(0, "count", 13),
- typedMembersSlot(1, "person", "孔秀", "孟坦"),
- }},
- Messages: []schema.Message{*schema.ToolMessage(`{"passages": []}`, "call_1")},
- }
- s.appendRecordLine(true)
- got := s.Messages[len(s.Messages)-1].Content
- if !strings.Contains(got, "[record]") || !strings.Contains(got, "pool=1") {
- t.Fatalf("tool message = %q, want the record line appended", got)
- }
- if !strings.HasPrefix(got, `{"passages": []}`) {
- t.Fatalf("tool message = %q, want the payload preserved ahead of the line", got)
- }
- if !strings.Contains(got, "members=2") {
- t.Errorf("line %q, want the members the table holds", got)
- }
-
- // A turn with no tool call must not touch the previous result.
- before := s.Messages[len(s.Messages)-1].Content
- s.appendRecordLine(false)
- if s.Messages[len(s.Messages)-1].Content != before {
- t.Fatal("a turn that ran no tool call must not append a line")
- }
-
- // No pool bound: the record is kept, the line is skipped rather than inventing
- // a message for it.
- empty := &SessionState{Messages: []schema.Message{*schema.ToolMessage("x", "call_2")}}
- empty.appendRecordLine(true)
- if empty.Messages[0].Content != "x" {
- t.Fatalf("no-pool message = %q, want it untouched", empty.Messages[0].Content)
- }
-}
-
-// TestOfferContinuationLetsTheModelDecide pins the turn-budget contract: past the
-// mode's floor the session does NOT stop, and it does not extend itself on a
-// runtime predicate either — the model is asked, once per turn, up to a hard cap.
-//
-// The cap is the part the runtime keeps: medium/high run 8 → 12, ultra 10 → 14,
-// which is the "maximum run" the session may never exceed however eager the model
-// is. The floors are asserted here because the enumeration path walks them: a
-// session that is still turning names into evidence needs the turns these numbers
-// buy (see the measurement on offerContinuation).
-func TestOfferContinuationLetsTheModelDecide(t *testing.T) {
- for mode, floor := range map[string]int{"medium": 8, "high": 8, "ultra": 10} {
- if got := GetMode(mode).ActionMaxTurns; got != floor {
- t.Errorf("%s turn floor = %d, want %d (cap = floor + %d)", mode, got, floor, turnRunExtra)
- }
- }
-
- s := enumerationSession(4, 90)
- if got := s.turnRunCap(); got != 8 {
- t.Fatalf("run cap = %d, want 8 (the mode's floor 4 + the model's %d)", got, turnRunExtra)
- }
- if !s.offerContinuation() {
- t.Fatal("past the floor the session must offer the model another turn")
- }
- last := s.Messages[len(s.Messages)-1]
- if last.Role != schema.User {
- t.Fatalf("offer role = %v, want a user message the model answers", last.Role)
- }
- for _, want := range []string{"TURN BUDGET", "up to 8", "4 turn(s) left", "state patch NOW"} {
- if !strings.Contains(last.Content, want) {
- t.Errorf("offer %q missing %q", last.Content, want)
- }
- }
-
- // Idempotent within a turn: the model must not be asked twice for one turn.
- before := len(s.Messages)
- if !s.offerContinuation() {
- t.Fatal("re-asking for the same turn must still allow the turn")
- }
- if len(s.Messages) != before {
- t.Fatalf("messages grew to %d, want the offer appended once per turn", len(s.Messages))
- }
-
- // A new turn asks again (the model re-decides with the new tool result).
- s.Attempts = 5
- if !s.offerContinuation() || len(s.Messages) != before+1 {
- t.Fatal("each new turn must carry its own offer")
- }
-
- // The hard cap: no offer, and the session finalizes.
- capped := enumerationSession(8, 90)
- if capped.offerContinuation() {
- t.Fatal("the run cap must not be exceedable")
- }
- if len(capped.Messages) != 0 {
- t.Fatal("no offer may be appended at the cap")
- }
-
- // The clock is the other hard bound: without room for the finalize step no
- // further turn is offered, however willing the model is.
- tight := enumerationSession(4, turnAskFloorS-1)
- if tight.offerContinuation() {
- t.Fatal("no turn may be offered without room for the finalize step")
- }
-}
-
-// enumerationSession is a session sent on a SET question — the parent table holds
-// a count slot, which is the shape the continuation offer is gated on.
-func enumerationSession(attempts int, deadlineLeft float64) *SessionState {
- return &SessionState{
- Attempts: attempts,
- DeadlineLeft: deadlineLeft,
- ParentState: State{State: []Variable{{ID: 0, Type: "count", Candidate: strPtr("12")}}},
- }
-}
-
-// TestOfferContinuationIsGatedOnTheQuestionsShape pins the cost rule. The offer is
-// an extra model call plus up to turnRunExtra more turns, and a question that is
-// not assembling a set has nothing for those turns to find: measured on
-// 2026-09-15, one enumeration question was offered four extra turns while the
-// sessions still recorded nothing, and every question in the mode paid for that
-// mechanism. The gate is the session ENUMERATING — the caller wrote a batch, or the
-// planner declared a count/list — not a mode flag, and not a candidate's separators:
-// measured the same day, with the gate reading a list-shaped CANDIDATE as a set, a
-// FRAMES run took 24 offers (and 8 extra rounds over its baseline) on questions
-// whose answer is one number.
-func TestOfferContinuationIsGatedOnTheQuestionsShape(t *testing.T) {
- value := &SessionState{
- Attempts: 4,
- DeadlineLeft: 90,
- ParentState: State{State: []Variable{{ID: 0, Type: "entity", Candidate: strPtr("白马坡")}}},
- }
- if value.offerContinuation() {
- t.Fatal("a single-value question must not be offered extra turns")
- }
- if len(value.Messages) != 0 {
- t.Fatal("no offer message may be appended for a value question")
- }
- // And the route at the floor finalizes it, exactly as the mode's turn count
- // alone used to.
- if got := value.route(); got != routeFinalize {
- t.Fatalf("route at the floor on a value question = %v, want routeFinalize", got)
- }
-
- // A candidate that only LOOKS like a list is not a set: the measured FRAMES
- // table held `Grace's、High、Falls、Colonial、Creek` — one waterfall's name cut at
- // its separators — under a slot typed "dataset", and the offer built on it is
- // what ran that benchmark eight rounds long.
- prose := &SessionState{
- Attempts: 4,
- DeadlineLeft: 90,
- ParentState: State{State: []Variable{{ID: 0, Type: "dataset", Candidate: strPtr("Grace's、High、Falls")}}},
- }
- if prose.offerContinuation() {
- t.Fatal("a separator-bearing candidate under a scalar type must not buy extra turns")
- }
-
- // A count-typed slot is the other half of the tell — the shape the 三国
- // enumeration direction's own table carries (slot 0 [count]).
- counting := &SessionState{
- Attempts: 4,
- DeadlineLeft: 90,
- ParentState: State{State: []Variable{{ID: 0, Type: "count", Candidate: strPtr("10")}}},
- }
- if !counting.offerContinuation() {
- t.Fatal("a count slot must be offered the turn")
- }
-
- // And the tell that survives contact: a caller-written batch, even on a table
- // whose slots say nothing about a set.
- batched := &SessionState{
- Attempts: 4,
- DeadlineLeft: 90,
- SearchQueries: []string{"关羽 斩 杀 颜良 文丑 华雄 蔡阳"},
- ParentState: State{State: []Variable{{ID: 0, Type: "entity", Candidate: strPtr("白马坡")}}},
- }
- if !batched.offerContinuation() {
- t.Fatal("a session that wrote a batch is enumerating and must be offered the turn")
- }
-}
-
-// TestRouteAtTheFloorOffersTheModelTheDecision pins the routing wiring: the floor
-// routes to another model turn WITH the offer attached, and the cap routes to
-// finalize with nothing attached.
-func TestRouteAtTheFloorOffersTheModelTheDecision(t *testing.T) {
- s := enumerationSession(4, 90)
- if got := s.route(); got != routeRunAction {
- t.Fatalf("route at the floor = %v, want routeRunAction (the model decides)", got)
- }
- if len(s.Messages) != 1 {
- t.Fatalf("messages = %d, want the offer appended", len(s.Messages))
- }
-
- capped := enumerationSession(8, 90)
- if got := capped.route(); got != routeFinalize {
- t.Fatalf("route at the cap = %v, want routeFinalize", got)
- }
-
- // A terminal reply still ends the session immediately: the offer is for turns
- // that have not already concluded.
- done := enumerationSession(4, 90)
- done.Done = true
- if got := done.route(); got != routeEnd {
- t.Fatalf("route after a terminal reply = %v, want routeEnd", got)
- }
- // ...and a spent clock finalizes rather than offering.
- tight := enumerationSession(4, turnAskFloorS-1)
- if got := tight.route(); got != routeFinalize {
- t.Fatalf("route without clock = %v, want routeFinalize", got)
- }
-}
-
-func strPtr(s string) *string { return &s }
-
-// TestUnreadPoolExcerptShowsTextTheSessionHasNotSeen pins the pool read.
-//
-// The pool is text the round has already paid for, and a session only ever sees
-// the parts its own queries returned: everything else sits in hand, unread. That
-// is where members are lost without anyone noticing — measured (2026-09-15,
-// 三国演义/关羽): the passage naming 管亥 was fetched into the round's evidence and
-// no session ever named it, because nothing had shown it.
-func TestUnreadPoolExcerptShowsTextTheSessionHasNotSeen(t *testing.T) {
- kb := &Kbinfos{}
- kb.Admit(func(p *PoolAdmitter) {
- p.Add(map[string]any{"chunk_id": "seen01", "content": "关公温酒斩华雄,其酒尚温。"})
- p.Add(map[string]any{"chunk_id": "offtopic", "content": "那张角本是个不第秀才,因入山采药,遇一老人,碧眼童颜,手执藜杖,唤角至一洞中,以天书三卷授之。"})
- p.Add(map[string]any{"chunk_id": "unread01", "content": "关公大怒,拍马舞刀,直取管亥,管亥措手不及,被关公一刀劈于马下。"})
- })
- s := &SessionState{
- KB: kb,
- Direction: "关羽斩杀了哪些有名有姓的人物",
- SearchQueries: []string{"关公 斩 管亥"},
- RetrievedEvidenceIDs: []string{"seen01"},
- }
- got := s.unreadPoolExcerpt()
- if !strings.Contains(got, "unread01") {
- t.Fatalf("excerpt = %q, want the unread passage this session's own words point at", got)
- }
- if strings.Contains(got, "华雄") {
- t.Fatalf("excerpt = %q, want the passage this session has seen skipped", got)
- }
- // The passage that says the most the session has not seen is NOT the passage to
- // show: "most novel" and "about this question" are anti-correlated, and the
- // first version of this delivered nothing but chapter headings, 曹操's youth and
- // 张角 receiving the book. Only the session's own words tell the two apart.
- if strings.Contains(got, "张角") {
- t.Fatalf("excerpt = %q, want a passage the session's own words exclude", got)
- }
- // One excerpt per session: across the first two runs of this mechanism it
- // delivered twelve excerpts and none of them carried a member the record was
- // missing, so it stays a last resort rather than a per-turn routine.
- if again := s.unreadPoolExcerpt(); again != "" {
- t.Fatalf("a second excerpt was offered: %q", again)
- }
-}
-
-// TestCoverageFollowsThePlannersDeclaration pins the half of the gate that reads the
-// table: what the planner TYPED, never what a candidate looks like. A list-shaped
-// candidate under a scalar type is prose that happens to contain separators —
-// measured (2026-09-15, FRAMES) a slot typed "dataset" carried
-// `Grace's、High、Falls、Colonial、Creek`, ONE waterfall's name, and the permissive
-// reading of it is how a "how much shorter" record came to say "enumerated
-// members: 16" and how a `[count]` slot on a "how many times larger" question once
-// bought 24 continuation offers.
-func TestCoverageFollowsThePlannersDeclaration(t *testing.T) {
- value := State{State: []Variable{{ID: 0, Type: "entity", Candidate: strPtr("白马坡")}}}
- if CoverageOf(value).Set {
- t.Error("a single-value table asks for no set")
- }
- counted := State{State: []Variable{{ID: 0, Type: "count", Candidate: strPtr("10")}}}
- if !CoverageOf(counted).Set {
- t.Error("a count slot asks for a set")
- }
- quantity := State{State: []Variable{{ID: 0, Type: "number", Candidate: strPtr("2452 feet")}}}
- if CoverageOf(quantity).Set {
- t.Error("a number slot is a value, not a set")
- }
- prose := State{State: []Variable{{ID: 0, Type: "dataset", Candidate: strPtr("Grace's、High、Falls")}}}
- if CoverageOf(prose).Set {
- t.Error("the table must not be read through a candidate's separators")
- }
- // The actor's declared forms are what the corpus is read with, and the recall list is
- // one entry per operand — never one per (actor, act) pair.
- shaped := State{State: []Variable{
- {ID: 0, Type: "count", Terms: []string{"斩", "杀", "斩"}, Subject: "关羽|云长"},
- {ID: 1, Type: "dataset"},
- }}
- cov := CoverageOf(shaped)
- if !cov.Ok() {
- t.Fatalf("coverage = %+v, want an enumeration", cov)
- }
- if actors := cov.Actors(); len(actors) != 2 || actors[0] != "关羽" || actors[1] != "云长" {
- t.Errorf("actors = %v, want the two declared forms", actors)
- }
- if ops := cov.Operands(); len(ops) != 4 {
- t.Errorf("operands = %v, want 2 actor forms + 2 deduped act words", ops)
- }
-}
-
-// TestEnumerationIsSeededWithTheMethod pins WHERE the enumeration method is delivered — in the
-// seed, before the first turn — and, just as important, WHO gets it: only a table that declared the
-// whole enumeration (a count/set/list slot, a NAME-carrying slot and the act words, see
-// Coverage.Ok).
-//
-// Its first instruction — propose more candidates than you expect — is a decision taken before the
-// first query: measured (2026-09-16, 三国/关羽) the same question answered eighteen members with the
-// method in the seed of its `[count]` table and fourteen when it arrived a turn later. But the same
-// instruction on a question whose answer is ONE value sends the session looking for members it does
-// not need: measured (2026-09-16, FRAMES — 4 questions in flight, 300s deadline) the shape-only
-// gate seeded the value questions that merely contain a count and the run finished 0.833 with two
-// timeouts against 0.875 with none.
-//
-// What the seed carries alongside the method is the windows the enumeration FOUND, never the
-// queries to make (see CoverageSet.Render): a query list is advice the model did not follow, a
-// window is evidence with the chunk id a member is cited by.
-func TestEnumerationIsSeededWithTheMethod(t *testing.T) {
- loader := StringPromptLoader{"action_set": "SET / COUNT directions — the member list IS the work"}
- method := "SET / COUNT directions — the member list IS the work"
-
- declared := State{State: []Variable{
- {ID: 0, Type: "count", Candidate: strPtr("18"), Terms: []string{"斩", "杀"}, Subject: "关羽|云长"},
- {ID: 1, Type: "dataset"},
- }}
- seed := "## The enumeration already ran for this direction\n\n- chunk_id=c1 \"云长手起刀落,斩孔秀于马下\"\n"
- got := enumerationSeed(declared, loader, seed)
- if !strings.Contains(got, method) || !strings.Contains(got, "斩孔秀于马下") {
- t.Errorf("seed = %q, want the method AND the windows the enumeration found", got)
- }
- if strings.Contains(got, "one call per line") {
- t.Errorf("seed = %q still lists queries to make", got)
- }
- // No enumeration (no clock left, no executor): the method travels alone. The seed never
- // carries queries to make.
- if got := enumerationSeed(declared, loader, ""); got != method {
- t.Errorf("seed without an enumeration = %q, want the method alone", got)
- }
-
- // The ways a table FAILS to be an enumeration, each of which must leave a value question
- // alone: a count of EVENTS (the planner declares act words for those too), one named thing
- // with no count/set/list, a count with nothing declared to enumerate, a measured quantity, a
- // list-shaped candidate under a scalar type, and a single value.
- for _, tc := range []struct {
- name string
- table State
- }{
- {"a count of events", State{State: []Variable{{ID: 0, Type: "count", Candidate: strPtr("5"), Terms: []string{"won", "trophy"}, Subject: "Brazil"}}}},
- {"one named thing", State{State: []Variable{{ID: 0, Type: "person", Terms: []string{"wrote"}, Subject: "the writer"}, {ID: 1, Type: "date"}}}},
- {"a count with no act words", State{State: []Variable{{ID: 0, Type: "count", Candidate: strPtr("18")}}}},
- {"a measured quantity", State{State: []Variable{{ID: 0, Type: "number", Candidate: strPtr("14")}}}},
- {"a list-shaped candidate", State{State: []Variable{{ID: 1, Type: "entity", Candidate: strPtr("孔秀、孟坦")}}}},
- {"a single value", State{State: []Variable{{ID: 0, Type: "date", Candidate: strPtr("1858")}}}},
- } {
- if got := enumerationSeed(tc.table, loader, seed); got != "" {
- t.Errorf("%s must not be seeded with the member method, got %q", tc.name, got)
- }
- }
-
- // A loader that predates the template still travels with the evidence rather than
- // panicking (see loadOptionalPrompt): the windows are the part the model cannot get back.
- if got := enumerationSeed(declared, StringPromptLoader{}, seed); got != strings.TrimSpace(seed) {
- t.Errorf("a loader without action_set must yield the evidence alone, got %q", got)
- }
-}
-
-// TestUnseededSetDirectionIsHandedTheMethodOnItsFirstBatch pins the second delivery:
-// the CALLER's own batch, appended once, mid-session.
-//
-// It is the path for a set direction whose table was typed `number` rather than
-// `count` — and the signal has no measured false positives: over one FRAMES run of 20
-// questions the caller wrote zero batches, while one 三国 question wrote eighteen.
-func TestUnseededSetDirectionIsHandedTheMethodOnItsFirstBatch(t *testing.T) {
- method := "SET / COUNT directions — the member list IS the work"
- newSession := func(queries ...string) *SessionState {
- return &SessionState{
- SearchQueries: queries,
- EnumerationProtocol: method,
- Messages: []schema.Message{*schema.ToolMessage(`{"passages": []}`, "call_1")},
- }
- }
-
- // An English question never writes a batch, so it is never handed the method.
- english := newSession("What was the age difference between Mike Tyson and Trevor Berbick")
- english.appendBatchProtocol(true)
- if strings.Contains(english.Messages[0].Content, method) {
- t.Fatalf("tool message = %q, want no method on a question that wrote no batch", english.Messages[0].Content)
- }
- if english.BatchProtocolShown {
- t.Fatal("the method must not be marked shown when it was not appended")
- }
-
- // A caller-written batch IS the tell (space-separated CJK, the shape the model
- // actually writes — see callerBatch).
- chinese := newSession("关羽 斩 杀 颜良 文丑 华雄 蔡阳")
- chinese.appendBatchProtocol(true)
- if !strings.Contains(chinese.Messages[0].Content, method) {
- t.Fatalf("tool message = %q, want the method appended", chinese.Messages[0].Content)
- }
- // Once per session: method repeated is prompt noise.
- before := chinese.Messages[0].Content
- chinese.appendBatchProtocol(true)
- if chinese.Messages[0].Content != before {
- t.Fatal("the method must be appended at most once")
- }
- // A turn that ran no tool call cannot carry it either.
- quiet := newSession("关羽 斩 颜良")
- quiet.appendBatchProtocol(false)
- if quiet.BatchProtocolShown {
- t.Fatal("a turn that ran no tool call must not carry the method")
- }
-}
-
-// TestAppendRecordLineSkipsValueDirections pins the line's gate: it carries a set's
-// to-do list, and on a value question every field of it is empty (`members=0 |
-// probed-reached=0`) while it still costs a recomputation and a line of prompt on
-// every turn. Measured (2026-09-15, FRAMES): 214 such lines across 20 questions,
-// on a benchmark whose baseline run carried none.
-func TestAppendRecordLineSkipsValueDirections(t *testing.T) {
- kb := &Kbinfos{}
- kb.Admit(func(p *PoolAdmitter) {
- p.Add(map[string]any{"chunk_id": "c1", "content": "prose"})
- })
- payload := `{"passages": []}`
- s := &SessionState{
- KB: kb,
- ParentState: State{State: []Variable{
- {ID: 0, Type: "date", Candidate: strPtr("1858")},
- {ID: 1, Type: "number", Candidate: strPtr("2452")},
- }},
- Messages: []schema.Message{*schema.ToolMessage(payload, "call_1")},
- }
- s.appendRecordLine(true)
- if got := s.Messages[len(s.Messages)-1].Content; got != payload {
- t.Fatalf("tool message = %q, want it untouched on a value direction", got)
- }
- // The record itself is still computed and kept: the continuation ask reads it,
- // and only the model-facing line is skipped.
- if s.Record.Pool != 1 {
- t.Fatalf("record pool = %d, want the record still computed", s.Record.Pool)
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/stats.go b/internal/rag/agentic-rag/runtime/stats.go
deleted file mode 100644
index c1f88985a9..0000000000
--- a/internal/rag/agentic-rag/runtime/stats.go
+++ /dev/null
@@ -1,631 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "fmt"
- "log"
- "sort"
- "sync"
- "time"
-
- "gorm.io/gorm"
-
- "ragflow/internal/agent/chat"
-)
-
-// LLM-call instrumentation for the retrieval.
-//
-// Every phase of the pipeline (route / planner / orchestrator / direct / sufficiency /
-// finalize, ...) drives the LLM, and each call is attributed to the phase that was
-// executing. Phase wall-clock is measured here rather than summing LLM latency, which is
-// meaningless when calls run in parallel.
-//
-// The current phase and the active stats are carried on the context.Context, which is
-// already per-request (and per-goroutine when a phase fans out) — that is what gives
-// parallel work independent accounting.
-
-// Canonical pipeline order for the per-phase usage table. Phases are listed in
-// execution order so the log reads top-to-bottom like the actual flow,
-// regardless of which phase first touched the counters. Phases not in this list
-// are appended afterwards, alphabetically.
-// This list is the canonical order: it omits dynamic/draft/sca/rewrite (real phases, but
-// not part of the canonical list) and compute is a this-file-only phase
-// (arithmetic.Compute). The snapshot/Log logic below re-appends any phase not present here
-// alphabetically, so all of them still appear in the same relative position.
-var phaseOrder = []string{
- "formalize",
- "route",
- "planner",
- "decompose",
- "direct",
- "orchestrator",
- "claim_research",
- "sufficiency",
- "grounded",
- "finalize",
-}
-
-// Phase names used by the retrieval.
-const (
- PhaseFormalize = "formalize"
- PhaseRoute = "route"
- PhasePlanner = "planner"
- PhaseDecompose = "decompose"
- PhaseDynamic = "dynamic"
- PhaseDirect = "direct"
- PhaseOrchestrator = "orchestrator"
- PhaseClaimResearch = "claim_research"
- PhaseDraft = "draft"
- PhaseSufficiency = "sufficiency"
- PhaseSCA = "sca"
- PhaseRewrite = "rewrite"
- PhaseCompute = "compute"
- PhaseGrounded = "grounded"
- PhaseFinalize = "finalize"
-)
-
-// phaseRoundKey identifies a phase within one orchestrator round.
-type phaseRoundKey struct {
- phase string
- round int
-}
-
-// LLMUsageStats holds per-phase LLM call, wall-clock, and token counters for
-// one runtime run.
-type LLMUsageStats struct {
- mu sync.Mutex
-
- calls map[string]int
- failed map[string]int
- phaseTimeMs map[string]float64
- promptTokens map[string]int
- completionTokens map[string]int
- totalTokens map[string]int
- rounds map[string]int
- roundTimes map[string][]float64
- // roundPhaseTimesMs is the phase wall-clock split per orchestrator round
- // (index 0 = round 1). A phase that runs several times inside one round
- // accumulates into that round.
- roundPhaseTimesMs map[string][]float64
- roundClaimCounts map[string][]int
-
- roundStarts map[string]time.Time
- currentRound int
- phaseActive map[string]int
- phaseStarts map[string]time.Time
- roundPhaseAct map[phaseRoundKey]int
- roundPhaseSt map[phaseRoundKey]time.Time
-}
-
-// NewLLMUsageStats builds an empty counter set.
-func NewLLMUsageStats() *LLMUsageStats {
- return &LLMUsageStats{
- calls: map[string]int{},
- failed: map[string]int{},
- phaseTimeMs: map[string]float64{},
- promptTokens: map[string]int{},
- completionTokens: map[string]int{},
- totalTokens: map[string]int{},
- rounds: map[string]int{},
- roundTimes: map[string][]float64{},
- roundPhaseTimesMs: map[string][]float64{},
- roundClaimCounts: map[string][]int{},
- roundStarts: map[string]time.Time{},
- phaseActive: map[string]int{},
- phaseStarts: map[string]time.Time{},
- roundPhaseAct: map[phaseRoundKey]int{},
- roundPhaseSt: map[phaseRoundKey]time.Time{},
- }
-}
-
-// elapsedMs returns a monotonic-aware wall duration in milliseconds. Go's time.Time carries
-// a monotonic reading as long as the struct is kept (time.Now().UnixNano() would strip it
-// and expose wall-clock step-backs), so the stored start time is subtracted directly. That
-// guarantees phase/round durations never go negative when the system clock is stepped
-// backwards.
-func elapsedMs(now, start time.Time) float64 {
- return now.Sub(start).Seconds() * 1000.0
-}
-
-// CurrentRound is the 1-based index of the orchestrator round executing, 0 outside.
-func (s *LLMUsageStats) CurrentRound() int {
- s.mu.Lock()
- defer s.mu.Unlock()
- return s.currentRound
-}
-
-// RecordCall counts one LLM call in phaseName.
-func (s *LLMUsageStats) RecordCall(phaseName string) {
- s.mu.Lock()
- defer s.mu.Unlock()
- s.calls[phaseName]++
-}
-
-// RecordFailed counts one failed LLM call in phaseName.
-func (s *LLMUsageStats) RecordFailed(phaseName string) {
- s.mu.Lock()
- defer s.mu.Unlock()
- s.failed[phaseName]++
-}
-
-// RecordUsage adds provider-reported token usage to phaseName.
-func (s *LLMUsageStats) RecordUsage(phaseName string, prompt, completion, total int) {
- s.mu.Lock()
- defer s.mu.Unlock()
- s.promptTokens[phaseName] += prompt
- s.completionTokens[phaseName] += completion
- s.totalTokens[phaseName] += total
-}
-
-// RecordRound counts one iteration of a looping phase (e.g. an orchestrator
-// cycle) and closes the previous iteration's wall-clock.
-func (s *LLMUsageStats) RecordRound(phaseName string) {
- s.mu.Lock()
- defer s.mu.Unlock()
- s.rounds[phaseName]++
- s.currentRound = s.rounds[phaseName]
- now := time.Now()
- if prev, ok := s.roundStarts[phaseName]; ok {
- s.roundTimes[phaseName] = append(s.roundTimes[phaseName], elapsedMs(now, prev))
- }
- s.roundStarts[phaseName] = now
-}
-
-// RecordRoundClaims records how many claim-level tasks ran in the current round.
-func (s *LLMUsageStats) RecordRoundClaims(phaseName string, count int) {
- s.mu.Lock()
- defer s.mu.Unlock()
- if s.currentRound <= 0 {
- return
- }
- counts := s.roundClaimCounts[phaseName]
- for len(counts) < s.currentRound {
- counts = append(counts, 0)
- }
- counts[s.currentRound-1] += count
- s.roundClaimCounts[phaseName] = counts
-}
-
-// notePhaseEnter / notePhaseExit implement the re-entrancy rule: the same phase may be
-// wrapped several times along one call path, and only the outermost interval is timed.
-func (s *LLMUsageStats) notePhaseEnter(phaseName string, entryRound int) {
- s.mu.Lock()
- defer s.mu.Unlock()
- now := time.Now()
- s.phaseActive[phaseName]++
- if s.phaseActive[phaseName] == 1 {
- s.phaseStarts[phaseName] = now
- }
- if entryRound > 0 {
- k := phaseRoundKey{phaseName, entryRound}
- s.roundPhaseAct[k]++
- if s.roundPhaseAct[k] == 1 {
- s.roundPhaseSt[k] = now
- }
- }
-}
-
-func (s *LLMUsageStats) notePhaseExit(phaseName string, entryRound int) {
- s.mu.Lock()
- defer s.mu.Unlock()
- now := time.Now()
- if active := s.phaseActive[phaseName]; active > 0 {
- active--
- if active == 0 {
- start, ok := s.phaseStarts[phaseName]
- if !ok {
- start = now
- }
- delete(s.phaseStarts, phaseName)
- delete(s.phaseActive, phaseName)
- s.phaseTimeMs[phaseName] += elapsedMs(now, start)
- pending := s.rounds[phaseName] - len(s.roundTimes[phaseName])
- if pending > 0 {
- settled := 0.0
- for _, t := range s.roundTimes[phaseName] {
- settled += t
- }
- s.roundTimes[phaseName] = append(s.roundTimes[phaseName], max(0.0, s.phaseTimeMs[phaseName]-settled))
- delete(s.roundStarts, phaseName)
- }
- if phaseName == PhaseOrchestrator {
- s.currentRound = 0
- }
- } else {
- s.phaseActive[phaseName] = active
- }
- }
- if entryRound > 0 {
- k := phaseRoundKey{phaseName, entryRound}
- if active := s.roundPhaseAct[k]; active > 0 {
- active--
- if active == 0 {
- start, ok := s.roundPhaseSt[k]
- if !ok {
- start = now
- }
- delete(s.roundPhaseSt, k)
- delete(s.roundPhaseAct, k)
- times := s.roundPhaseTimesMs[phaseName]
- for len(times) < entryRound {
- times = append(times, 0.0)
- }
- times[entryRound-1] += elapsedMs(now, start)
- s.roundPhaseTimesMs[phaseName] = times
- } else {
- s.roundPhaseAct[k] = active
- }
- }
- }
-}
-
-// Snapshot: rows keyed by phase in
-// canonical pipeline order.
-func (s *LLMUsageStats) Snapshot() map[string]map[string]any {
- s.mu.Lock()
- defer s.mu.Unlock()
- known := map[string]struct{}{}
- for k := range s.calls {
- known[k] = struct{}{}
- }
- for k := range s.failed {
- known[k] = struct{}{}
- }
- for k := range s.totalTokens {
- known[k] = struct{}{}
- }
- for k := range s.phaseTimeMs {
- known[k] = struct{}{}
- }
- for k := range s.rounds {
- known[k] = struct{}{}
- }
- phases := make([]string, 0, len(known))
- for _, p := range phaseOrder {
- if _, ok := known[p]; ok {
- phases = append(phases, p)
- delete(known, p)
- }
- }
- for p := range known {
- phases = append(phases, p)
- }
- sort.Strings(phases[len(phases)-len(known):])
-
- rows := make(map[string]map[string]any, len(phases))
- for _, p := range phases {
- perRound := s.roundPhaseTimesMs[p]
- rounds, roundTimes := s.rounds[p], append([]float64(nil), s.roundTimes[p]...)
- if len(perRound) > 0 {
- rounds, roundTimes = len(perRound), append([]float64(nil), perRound...)
- }
- rows[p] = map[string]any{
- "calls": s.calls[p],
- "failed": s.failed[p],
- "phase_time_ms": s.phaseTimeMs[p],
- "prompt_tokens": s.promptTokens[p],
- "completion_tokens": s.completionTokens[p],
- "total_tokens": s.totalTokens[p],
- "rounds": rounds,
- "round_times": roundTimes,
- "round_claim_counts": append([]int(nil), s.roundClaimCounts[p]...),
- "round_phase_times_ms": append([]float64(nil), s.roundPhaseTimesMs[p]...),
- }
- }
- return rows
-}
-
-// Context plumbing
-
-type statsCtxKey struct{}
-type phaseCtxKey struct{}
-
-// WithStats binds stats to ctx so every LLM call beneath it is attributed there.
-func WithStats(ctx context.Context, stats *LLMUsageStats) context.Context {
- return context.WithValue(ctx, statsCtxKey{}, stats)
-}
-
-// CurrentStats returns the stats bound to ctx, or nil.
-func CurrentStats(ctx context.Context) *LLMUsageStats {
- if s, ok := ctx.Value(statsCtxKey{}).(*LLMUsageStats); ok {
- return s
- }
- return nil
-}
-
-// CurrentPhase returns the phase executing in ctx ("unknown" when none set).
-func CurrentPhase(ctx context.Context) string {
- if p, ok := ctx.Value(phaseCtxKey{}).(string); ok && p != "" {
- return p
- }
- return "unknown"
-}
-
-// Phase marks the enclosed block as executing `name` and accrues its wall-clock into the
-// bound stats.
-//
-// The returned func MUST be called when the block ends (defer it). Nesting the
-// same phase name is supported: only the outermost interval is timed, so the
-// time is not counted two or three times along a call path.
-func Phase(ctx context.Context, name string) (context.Context, func()) {
- stats := CurrentStats(ctx)
- round := 0
- if stats != nil {
- round = stats.CurrentRound()
- stats.notePhaseEnter(name, round)
- }
- child := context.WithValue(ctx, phaseCtxKey{}, name)
- return child, func() {
- if stats != nil {
- stats.notePhaseExit(name, round)
- }
- }
-}
-
-// InPhase runs fn inside Phase(name).
-func InPhase(ctx context.Context, name string, fn func(context.Context) error) error {
- ctx, done := Phase(ctx, name)
- defer done()
- return fn(ctx)
-}
-
-// RecordExternalResponse: records a raw
-// completion response that bypasses CountingInvoker, including token usage.
-// action_session calls this for the two raw model calls it makes directly
-// (action_session.py:_acompletion/1086).
-func RecordExternalResponse(ctx context.Context, resp *chat.Response) {
- stats := CurrentStats(ctx)
- if stats == nil {
- return
- }
- phase := CurrentPhase(ctx)
- stats.RecordCall(phase)
- if resp != nil {
- recordResponseUsage(stats, phase, resp.Usage, resp.Tokens)
- }
-}
-
-// recordResponseUsage attributes one call's token usage, honouring the split
-// when it is present and falling back to the single total counter otherwise.
-func recordResponseUsage(stats *LLMUsageStats, phase string, usage *chat.Usage, totalOnly int) {
- if usage != nil {
- stats.RecordUsage(phase, usage.PromptTokens, usage.CompletionTokens, usage.TotalTokens)
- return
- }
- if totalOnly > 0 {
- stats.RecordUsage(phase, 0, 0, totalOnly)
- }
-}
-
-// RecordRound
-func RecordRound(ctx context.Context, name string) {
- if s := CurrentStats(ctx); s != nil {
- s.RecordRound(name)
- }
-}
-
-// RecordRoundClaims
-func RecordRoundClaims(ctx context.Context, name string, count int) {
- if s := CurrentStats(ctx); s != nil {
- s.RecordRoundClaims(name, count)
- }
-}
-
-// Log emits the per-phase usage table.
-//
-// With orchestrator-round data it expands hierarchically: each round repeats its
-// "orchestrator" row with the nested sub-phases (claim_research / sufficiency / grounded)
-// indented underneath. Phases outside the loop (route / planner / finalize) are listed flat.
-func (s *LLMUsageStats) Log(logger *log.Logger) {
- rows := s.Snapshot()
- if len(rows) == 0 {
- // No LLM activity (e.g. a cache hit). Still emit a line so every
- // completed run is accounted for instead of silently disappearing.
- if logger != nil {
- logger.Println("[Agentic RAG] LLM usage by phase: (cached / no LLM calls)")
- }
- return
- }
- totalCalls, totalTokens := 0, 0
- for _, r := range rows {
- totalCalls += r["calls"].(int)
- totalTokens += r["total_tokens"].(int)
- }
-
- // Canonical order: phases in phaseOrder first, then any leftover sorted.
- seen := map[string]bool{}
- phases := make([]string, 0, len(rows))
- for _, p := range phaseOrder {
- if _, ok := rows[p]; ok {
- phases = append(phases, p)
- seen[p] = true
- }
- }
- rest := make([]string, 0, len(rows)-len(phases))
- for p := range rows {
- if !seen[p] {
- rest = append(rest, p)
- }
- }
- sort.Strings(rest)
- phases = append(phases, rest...)
-
- // Per-round structure, read from the locked snapshot.
- orchRT := []float64{}
- if orchRow, ok := rows[PhaseOrchestrator]; ok {
- if v, ok := orchRow["round_times"].([]float64); ok {
- orchRT = v
- }
- }
- perRound := map[string][]float64{}
- for p, r := range rows {
- if v, ok := r["round_phase_times_ms"].([]float64); ok && len(v) > 0 {
- perRound[p] = v
- }
- }
- nRounds := len(orchRT)
- for _, v := range perRound {
- if len(v) > nRounds {
- nRounds = len(v)
- }
- }
-
- lines := []string{
- "[Agentic RAG] LLM usage by phase:",
- fmt.Sprintf(" %-16s %7s %10s %12s %10s %10s", "phase", "llm_calls", "prompt_tok", "output_tok", "total_tok", "time(s)"),
- }
-
- // phaseLabel: claim_research gets a "(N)" suffix with the round's claim count when
- // known.
- phaseLabel := func(p string, r map[string]any, roundIdx int) string {
- label := p
- if p == PhaseClaimResearch {
- if counts, ok := r["round_claim_counts"].([]int); ok && roundIdx < len(counts) && counts[roundIdx] > 0 {
- label = fmt.Sprintf("%s (%d)", p, counts[roundIdx])
- }
- }
- return label
- }
- // row: the label is printed verbatim (orchestrator round headers pass a custom label),
- // token columns come from rows[p].
- row := func(indent, label, p string, tMs float64) string {
- r := rows[p]
- return fmt.Sprintf("%s%-16s %7d %10d %12d %10d %10.1f",
- indent, label, r["calls"].(int), r["prompt_tokens"].(int), r["completion_tokens"].(int), r["total_tokens"].(int), tMs/1000.0)
- }
-
- inRounds := make(map[string]bool, len(perRound))
- for k := range perRound {
- inRounds[k] = true
- }
-
- for _, p := range phases {
- if p == PhaseOrchestrator && nRounds > 0 {
- for i := 0; i < nRounds; i++ {
- orchT := 0.0
- if i < len(orchRT) {
- orchT = orchRT[i]
- } else {
- orchT = rows[p]["phase_time_ms"].(float64)
- }
- lines = append(lines, row(" ", fmt.Sprintf("orchestrator round %d", i+1), p, orchT))
- for _, sub := range phases {
- if sub == PhaseOrchestrator {
- continue
- }
- v, ok := perRound[sub]
- if !ok || i >= len(v) {
- continue
- }
- lines = append(lines, row(" ", phaseLabel(sub, rows[sub], i), sub, v[i]))
- }
- }
- } else if inRounds[p] && nRounds > 0 {
- // Already printed as a nested sub-phase of each orchestrator round.
- continue
- } else {
- lines = append(lines, row(" ", p, p, rows[p]["phase_time_ms"].(float64)))
- }
- }
- lines = append(lines, fmt.Sprintf(" total: %d LLM calls, %d tokens", totalCalls, totalTokens))
- if logger != nil {
- logger.Println(joinLines(lines))
- }
-}
-
-func joinLines(lines []string) string {
- out := ""
- for i, l := range lines {
- if i > 0 {
- out += "\n"
- }
- out += l
- }
- return out
-}
-
-// CountingInvoker: it wraps a chat.Invoker and
-// records calls / failures / token usage against the phase carried on ctx.
-//
-// Unlike a bundle-wide fallback, this records only when stats are bound to the context,
-// which is what gives the innermost active stats their accounting.
-type CountingInvoker struct {
- Inner chat.Invoker
- Stats *LLMUsageStats
-}
-
-// Invoke implements chat.Invoker.
-func (c *CountingInvoker) Invoke(ctx context.Context, db *gorm.DB, req chat.Request) (*chat.Response, error) {
- stats := CurrentStats(ctx)
- if stats == nil {
- stats = c.Stats
- }
- phase := CurrentPhase(ctx)
- if stats != nil {
- stats.RecordCall(phase)
- }
- resp, err := c.Inner.Invoke(ctx, db, req)
- if err != nil {
- if stats != nil {
- stats.RecordFailed(phase)
- }
- return nil, err
- }
- if stats != nil && resp != nil {
- recordResponseUsage(stats, phase, resp.Usage, resp.Tokens)
- }
- return resp, nil
-}
-
-// Stream implements chat.StreamingInvoker so a wrapped invoker keeps streaming
-// AND is counted. Without this, StreamComplete's type assertion
-// (m.Invoker.(chat.StreamingInvoker)) fails on the wrapper and the caller falls
-// back to a one-shot (non-streaming) Invoke — losing both the stream and the
-// accounting. Usage is recorded even when the inner model raised (no usage is reported
-// then, so the record step is a no-op — the same outcome as the success-only branch below).
-//
-// Usage is taken from the returned *Response. If the inner invoker is not itself a
-// StreamingInvoker, we decline rather than silently downgrade to a blocking
-// Invoke, so callers keep their existing "fall back to the one-shot call"
-// behaviour.
-func (c *CountingInvoker) Stream(ctx context.Context, db *gorm.DB, req chat.Request, onDelta func(delta string, isThink bool) error) (*chat.Response, error) {
- stats := CurrentStats(ctx)
- if stats == nil {
- stats = c.Stats
- }
- phase := CurrentPhase(ctx)
- if stats != nil {
- stats.RecordCall(phase)
- }
- inner, ok := c.Inner.(chat.StreamingInvoker)
- if !ok {
- return nil, fmt.Errorf("runtime: chat invoker %T does not support streaming", c.Inner)
- }
- resp, err := inner.Stream(ctx, db, req, onDelta)
- if err != nil {
- if stats != nil {
- stats.RecordFailed(phase)
- }
- return nil, err
- }
- if stats != nil && resp != nil {
- recordResponseUsage(stats, phase, resp.Usage, resp.Tokens)
- }
- return resp, nil
-}
diff --git a/internal/rag/agentic-rag/runtime/stats_test.go b/internal/rag/agentic-rag/runtime/stats_test.go
deleted file mode 100644
index 4bf4009f69..0000000000
--- a/internal/rag/agentic-rag/runtime/stats_test.go
+++ /dev/null
@@ -1,137 +0,0 @@
-package runtime
-
-import (
- "bytes"
- "context"
- "log"
- "strings"
- "testing"
-
- "ragflow/internal/agent/chat"
-)
-
-// TestLogHierarchicalRounds verifies the log shape: the orchestrator phase expands into one
-// "orchestrator round N" row per round, with the in-loop sub-phases (claim_research /
-// sufficiency) nested underneath, while the out-of-loop phases (route / planner / finalize)
-// stay flat.
-//
-// NOTE: this test cannot run while the unrelated pre-existing break in
-// internal/agent/tool/retrieval_nlp.go (RankFeature) blocks compilation of the
-// runtime test binary. It will run once that is resolved.
-func TestLogHierarchicalRounds(t *testing.T) {
- s := NewLLMUsageStats()
- ctx := WithStats(context.Background(), s)
-
- // Flat phases before the orchestrator loop.
- func() {
- _, done := Phase(ctx, PhaseRoute)
- defer done()
- s.RecordCall(PhaseRoute)
- }()
- func() {
- _, done := Phase(ctx, PhasePlanner)
- defer done()
- s.RecordCall(PhasePlanner)
- }()
-
- // Orchestrator loop: two rounds, each doing claim_research + sufficiency.
- func() {
- c, done := Phase(ctx, PhaseOrchestrator)
- defer done()
-
- RecordRound(c, PhaseOrchestrator)
- func() {
- _, d := Phase(c, PhaseClaimResearch)
- defer d()
- s.RecordCall(PhaseClaimResearch)
- s.RecordUsage(PhaseClaimResearch, 0, 0, 120)
- }()
- func() {
- _, d := Phase(c, PhaseSufficiency)
- defer d()
- s.RecordCall(PhaseSufficiency)
- }()
-
- RecordRound(c, PhaseOrchestrator)
- func() {
- _, d := Phase(c, PhaseClaimResearch)
- defer d()
- s.RecordCall(PhaseClaimResearch)
- s.RecordUsage(PhaseClaimResearch, 0, 0, 80)
- }()
- }()
-
- // Flat phase after the loop.
- func() {
- _, done := Phase(ctx, PhaseFinalize)
- defer done()
- s.RecordCall(PhaseFinalize)
- }()
-
- var buf bytes.Buffer
- s.Log(log.New(&buf, "", 0))
- out := buf.String()
-
- for _, want := range []string{
- "orchestrator round 1",
- "orchestrator round 2",
- "route",
- "planner",
- "finalize",
- } {
- if !strings.Contains(out, want) {
- t.Errorf("log output missing %q\n---\n%s", want, out)
- }
- }
- if !strings.Contains(out, "sufficiency") {
- t.Errorf("log output missing sufficiency\n---\n%s", out)
- }
- // claim_research runs inside every round, so it must appear at least once
- // per round as a nested (indented) row, not just as a flat top-level row.
- if n := strings.Count(out, "claim_research"); n < 2 {
- t.Errorf("expected claim_research at least twice (per round), got %d\n---\n%s", n, out)
- }
-}
-
-// TestCountingInvokerCountsStreaming mirrors Python CountingChatModel
-// async_chat_streamly / async_chat_streamly_delta: a wrapped invoker that DOES
-// stream must keep streaming AND be counted. Before CountingInvoker gained a
-// Stream method, StreamComplete's type assertion (m.Invoker.(chat.StreamingInvoker))
-// failed on the wrapper, so the whole pipeline fell back to a one-shot Invoke and
-// the stream was lost — the streaming branch was effectively uncredited.
-func TestCountingInvokerCountsStreaming(t *testing.T) {
- stats := NewLLMUsageStats()
- wrapped := &CountingInvoker{
- Inner: &streamingInvoker{pieces: []piece{{text: "hi", isThink: false}}},
- Stats: stats,
- }
- ctx, done := Phase(context.Background(), "finalize")
- defer done()
-
- resp, err := wrapped.Stream(ctx, nil, chat.Request{}, func(string, bool) error { return nil })
- if err != nil {
- t.Fatalf("Stream failed: %v", err)
- }
- if resp.Content != "hi" {
- t.Fatalf("Content = %q, want %q", resp.Content, "hi")
- }
- calls, _ := stats.Snapshot()["finalize"]["calls"].(int)
- if calls != 1 {
- t.Fatalf("streaming call not counted: calls = %v, want 1", stats.Snapshot()["finalize"]["calls"])
- }
-}
-
-// TestCountingInvokerStreamDeclinesWithoutStreamingInner pins the invariant that the wrapper
-// only streams when the inner model can: a non-streaming inner returns an explicit error rather
-// than silently downgrading to a blocking Invoke (which would look like a one-shot call in the
-// stats).
-func TestCountingInvokerStreamDeclinesWithoutStreamingInner(t *testing.T) {
- wrapped := &CountingInvoker{Inner: &plainInvoker{}}
- if _, err := wrapped.Stream(context.Background(), nil, chat.Request{}, nil); err == nil {
- t.Fatal("want an error when the inner invoker cannot stream")
- }
-}
-
-// capturingInvoker records the last Request and returns a native tool call so we can verify
-// the seam forwards the declared tools and reads the model's calls back out of the native
-// tool_calls field.
diff --git a/internal/rag/agentic-rag/runtime/structure_qa.go b/internal/rag/agentic-rag/runtime/structure_qa.go
deleted file mode 100644
index 6ed154a60e..0000000000
--- a/internal/rag/agentic-rag/runtime/structure_qa.go
+++ /dev/null
@@ -1,209 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "fmt"
- "strings"
- "unicode"
-
- "github.com/cloudwego/eino/schema"
-
- "ragflow/internal/agent/chat"
-)
-
-// Ask the chat model to answer a question from a compiled-structure outline.
-//
-// Both navigation paths that read compiled rows render entities + relations into the same
-// compact outline, ask the model whether that outline alone answers the question, and use the
-// returned relevant_entities to pull the underlying source chunks even when the outline is NOT
-// sufficient (so evidence still flows back to the caller).
-//
-// Lives apart from navigation.go because it is pure prompt/render work with no
-// store access, and because both callers would otherwise import it from a file
-// that in turn imports them.
-
-// structureQATemperature pins the temperature for the outline-verdict call.
-const structureQATemperature = 0.2
-
-// RenderStructure renders a compiled structure (entities + relations) as a
-// compact outline for the prompt:
-//
-// Entities:
-// - name (type): description
-// (blank line)
-// Relations:
-// - src -[type]-> tgt
-//
-// Both lists are capped at maxStructureEntities / maxStructureRelations; empty
-// names and empty relation endpoints are dropped. Entities with no type fall
-// back to "other"; relations with no type fall back to "related".
-func RenderStructure(entities, relations []map[string]any) string {
- var lines []string
- if len(entities) > 0 {
- lines = append(lines, "Entities:")
- n := len(entities)
- if n > maxStructureEntities {
- n = maxStructureEntities
- }
- for _, e := range entities[:n] {
- name := strings.TrimSpace(strAny(e["name"]))
- if name == "" {
- continue
- }
- typ := strings.TrimSpace(orStr(strAny(e["type"]), "other"))
- desc := strings.Join(strings.Fields(strAny(e["description"])), " ")
- line := "- " + name + " (" + typ + ")"
- if desc != "" {
- line += ": " + desc
- }
- lines = append(lines, line)
- }
- }
- if len(relations) > 0 {
- lines = append(lines, "\nRelations:")
- n := len(relations)
- if n > maxStructureRelations {
- n = maxStructureRelations
- }
- for _, r := range relations[:n] {
- src := strings.TrimSpace(strAny(r["from"]))
- tgt := strings.TrimSpace(strAny(r["to"]))
- if src == "" || tgt == "" {
- continue
- }
- rt := strings.TrimSpace(orStr(strAny(r["type"]), "related"))
- lines = append(lines, "- "+src+" -["+rt+"]-> "+tgt)
- }
- }
- return strings.Join(lines, "\n")
-}
-
-// AskStructure asks the chat model to answer `topic` from the rendered outline.
-//
-// Returns (answer, relevant_entity_names): answer is empty unless the model judged the
-// outline sufficient; the names are always returned so the caller can pull the underlying
-// source chunks. `noun`
-// is the display noun ("catalog" / "mindmap" / "knowledge graph"); `label` is
-// the log tag. `model` is the request-scoped chat model; a nil model means no model is
-// available and the call skips. Never raises: on any chat/parse failure both results are
-// empty.
-func AskStructure(ctx context.Context, model SessionModel, topic, noun, label string, entities, relations []map[string]any) (string, []string) {
- if model == nil {
- _LOG.Printf("[%s] structure QA skipped (no chat model)", label)
- return "", nil
- }
- system := strings.ReplaceAll(navSystemPrompt, "{noun}", "the "+noun)
- rendered := RenderStructure(entities, relations)
- user := fmt.Sprintf("Question:\n%s\n\n%s:\n%s\n\nOutput JSON:", topic, capitalizeWord(noun), rendered)
-
- // The context length is exposed via ContextLengthModel; when absent, the 8192 default
- // (chat.EffectiveContextLength) applies, i.e. when the model config omits it.
- budget := 0
- if cl, ok := model.(ContextLengthModel); ok {
- budget = cl.ContextLength()
- }
- if budget <= 0 {
- budget = chat.EffectiveContextLength(0)
- }
- fitted, fitErr := chat.FitMessages(system, []schema.Message{
- *schema.UserMessage(user),
- }, budget)
- if fitErr != "" {
- _LOG.Printf("[%s] prompt fitting failed: %s", label, fitErr)
- return "", nil
- }
- // FitMessages may prepend/trim a system message; re-extract it so the model
- // call is exactly [system, user...].
- if len(fitted) > 0 && fitted[0].Role == schema.System {
- system = fitted[0].Content
- fitted = fitted[1:]
- }
- msgs := make([]schema.Message, 0, 1+len(fitted))
- msgs = append(msgs, *schema.SystemMessage(system))
- msgs = append(msgs, fitted...)
- // The temperature is pinned to 0.2 for this node: a stable verdict on a mechanical
- // render, so it must not sample hot. The
- // value is a pinned constant: every production carrier implements
- // TemperatureModel (compile-time assertion on InvokerSessionModel), so the
- // temperature is always sent; a carrier without per-call temperature
- // support falls back to its own default (Go-only provider limitation).
- var resp *ModelReply
- var err error
- if tm, ok := model.(TemperatureModel); ok {
- resp, err = tm.CompleteWithTemperature(ctx, msgs, nil, structureQATemperature)
- } else {
- _LOG.Printf("[%s] model %T cannot carry per-call temperature; using its default (wanted %v)", label, model, structureQATemperature)
- resp, err = model.Complete(ctx, msgs, nil)
- }
- if err != nil {
- _LOG.Printf("[%s] could not read the outline with the model: %v", label, err)
- return "", nil
- }
-
- var verdict structureNavVerdict
- if err := UnmarshalModelJSON(resp.Content, &verdict); err != nil {
- _LOG.Printf("[%s] could not parse the outline verdict: %v", label, err)
- return "", nil
- }
-
- answer := ""
- if verdict.IsSufficient {
- answer = strings.TrimSpace(verdict.Answer)
- }
- relevant := make([]string, 0, len(verdict.RelevantEntities))
- for _, n := range verdict.RelevantEntities {
- if s := strings.TrimSpace(n); s != "" {
- relevant = append(relevant, s)
- }
- }
- outcome := "does not fully answer"
- if verdict.IsSufficient {
- outcome = "answers"
- }
- _LOG.Printf("[%s] the %s %s the question; %d relevant entity(ies)", label, noun, outcome, len(relevant))
- return answer, relevant
-}
-
-// strAny reads a map field as its plain string form.
-func strAny(v any) string {
- switch x := v.(type) {
- case string:
- return x
- case fmt.Stringer:
- return x.String()
- case nil:
- return ""
- default:
- return fmt.Sprint(x)
- }
-}
-
-// capitalizeWord uppercases the first rune and lowercases every remaining rune, so e.g.
-// "hELLo" -> "Hello"; leaving the tail untouched produced "HELLO", which diverged.
-func capitalizeWord(s string) string {
- if s == "" {
- return s
- }
- rs := []rune(s)
- rs[0] = unicode.ToUpper(rs[0])
- for i := 1; i < len(rs); i++ {
- rs[i] = unicode.ToLower(rs[i])
- }
- return string(rs)
-}
diff --git a/internal/rag/agentic-rag/runtime/structure_qa_test.go b/internal/rag/agentic-rag/runtime/structure_qa_test.go
deleted file mode 100644
index 36847f3ed3..0000000000
--- a/internal/rag/agentic-rag/runtime/structure_qa_test.go
+++ /dev/null
@@ -1,170 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "strings"
- "testing"
-)
-
-// entity map helper (the rendering inputs).
-func saEntity(name, typ, desc string) map[string]any {
- return map[string]any{"name": name, "type": typ, "description": desc}
-}
-
-func saRelation(from, to, typ string) map[string]any {
- return map[string]any{"from": from, "to": to, "type": typ}
-}
-
-// RenderStructure feeds the LLM prompt in graph_explore (via AskStructure); it must render
-// exactly "Name (Type): Description" for entities (Type defaulting to "other") and
-// "From -[Type]-> To" for relations (Type defaulting to "related").
-func TestRenderStructure(t *testing.T) {
- entities := []map[string]any{
- saEntity("OmiyaSoft", "company", "game dev"),
- saEntity("NoType", "", " "),
- {"name": " "}, // blank name dropped
- }
- relations := []map[string]any{
- saRelation("OmiyaSoft", "Culdcept", "founded"),
- saRelation("A", "B", ""), // missing type -> related
- }
- out := RenderStructure(entities, relations)
-
- if !strings.Contains(out, "Entities:") || !strings.Contains(out, "Relations:") {
- t.Fatalf("missing section headers: %q", out)
- }
- // Whitespace in descriptions is collapsed.
- if !strings.Contains(out, "- OmiyaSoft (company): game dev") {
- t.Errorf("entity with description not rendered: %q", out)
- }
- // No type -> "other"; blank description -> no trailing colon.
- if !strings.Contains(out, "- NoType (other)") {
- t.Errorf("entity default type missing: %q", out)
- }
- if strings.Contains(out, "- NoType (other):") {
- t.Errorf("blank description must not render a colon: %q", out)
- }
- if strings.Contains(out, "- (") {
- t.Errorf("blank-name entity must be dropped: %q", out)
- }
- if !strings.Contains(out, "- OmiyaSoft -[founded]-> Culdcept") {
- t.Errorf("relation not rendered: %q", out)
- }
- // Missing relation type -> "related".
- if !strings.Contains(out, "- A -[related]-> B") {
- t.Errorf("relation default type missing: %q", out)
- }
-}
-
-func TestRenderStructureCapsAtLimits(t *testing.T) {
- entities := make([]map[string]any, maxStructureEntities+5)
- for i := range entities {
- entities[i] = saEntity("E", "t", "")
- }
- relations := make([]map[string]any, maxStructureRelations+5)
- for i := range relations {
- relations[i] = saRelation("A", "B", "r")
- }
- out := RenderStructure(entities, relations)
- // Each entity bullet is "\n- E (t)"; caps at maxStructureEntities.
- if n := strings.Count(out, "\n- E (t)"); n != maxStructureEntities {
- t.Errorf("entity bullets = %d, want cap %d", n, maxStructureEntities)
- }
- if n := strings.Count(out, "\n- A -[r]-> B"); n != maxStructureRelations {
- t.Errorf("relation bullets = %d, want cap %d", n, maxStructureRelations)
- }
-}
-
-// An empty list is joined to "", so both-empty input renders "".
-func TestRenderStructureEmpty(t *testing.T) {
- if out := RenderStructure(nil, nil); out != "" {
- t.Errorf("empty structure must render empty string, got %q", out)
- }
-}
-
-// capitalizeWord: uppercase the first rune AND lowercase every remaining rune. Leaving the
-// tail untouched produced "HELLO" instead of "Hello".
-func TestCapitalizeWord(t *testing.T) {
- cases := []struct {
- in, want string
- }{
- {"", ""},
- {"knowledge graph", "Knowledge graph"},
- {"hELLo", "Hello"},
- {"WORLD", "World"},
- {"already", "Already"},
- {"123abc", "123abc"}, // first char non-letter: untouched, rest lowercased
- {"ÜBER", "Über"}, // unicode: first upper, rest lower
- }
- for _, c := range cases {
- if got := capitalizeWord(c.in); got != c.want {
- t.Errorf("capitalizeWord(%q) = %q, want %q", c.in, got, c.want)
- }
- }
-}
-
-// AskStructure: request-scoped model + temperature 0.2.
-
-// TestAskStructureNilModelSkips verifies a nil model short-circuits to an empty verdict
-// instead of reaching the global invoker (no model is available).
-func TestAskStructureNilModelSkips(t *testing.T) {
- answer, relevant := AskStructure(context.Background(), nil, "Q", "knowledge graph", "Graph exploration", nil, nil)
- if answer != "" || len(relevant) != 0 {
- t.Errorf("nil model -> (%q, %v), want empty", answer, relevant)
- }
-}
-
-// TestAskStructureUsesTemperatureTwoTenths verifies the structure verdict is a
-// mechanical rewrite drawn at 0.2.
-func TestAskStructureUsesTemperatureTwoTenths(t *testing.T) {
- mdl := &tempRecordingModel{replies: []*ModelReply{{
- Content: `{"is_sufficient": true, "answer": "founded in 1984", "relevant_entities": ["OmiyaSoft"]}`,
- }}}
- answer, relevant := AskStructure(context.Background(), mdl, "who founded Culdcept?",
- "knowledge graph", "Graph exploration",
- []map[string]any{saEntity("OmiyaSoft", "company", "")}, nil)
- if mdl.temperature == nil {
- t.Fatal("CompleteWithTemperature was not called")
- }
- if *mdl.temperature != 0.2 {
- t.Errorf("temperature = %v, want 0.2", *mdl.temperature)
- }
- if answer != "founded in 1984" {
- t.Errorf("answer = %q, want the sufficient verdict answer", answer)
- }
- if len(relevant) != 1 || relevant[0] != "OmiyaSoft" {
- t.Errorf("relevant = %v, want [OmiyaSoft]", relevant)
- }
-}
-
-// TestAskStructureInsufficientKeepsRelevant verifies an insufficient verdict
-// still returns the relevant entities (evidence pulled from the source chunks).
-func TestAskStructureInsufficientKeepsRelevant(t *testing.T) {
- mdl := &tempRecordingModel{replies: []*ModelReply{{
- Content: `{"is_sufficient": false, "answer": "", "relevant_entities": ["Culdcept"]}`,
- }}}
- answer, relevant := AskStructure(context.Background(), mdl, "Q", "mindmap", "Mindmap exploration",
- []map[string]any{saEntity("Culdcept", "game", "")}, nil)
- if answer != "" {
- t.Errorf("insufficient answer = %q, want empty", answer)
- }
- if len(relevant) != 1 || relevant[0] != "Culdcept" {
- t.Errorf("relevant = %v, want [Culdcept]", relevant)
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/table_view.go b/internal/rag/agentic-rag/runtime/table_view.go
deleted file mode 100644
index b6491fb4bd..0000000000
--- a/internal/rag/agentic-rag/runtime/table_view.go
+++ /dev/null
@@ -1,274 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "fmt"
- "strings"
-
- "golang.org/x/net/html"
-)
-
-// Render HTML tables into an LLM-friendly Markdown view.
-//
-// The compiled chunks of a Wikipedia-sourced KB carry machine-generated HTML tables.
-// Handing them over raw is both expensive and hard to read: a person infobox is 1-2K
-// chars of <table>/<td> markup, and a question like "how many children did all ten
-// nominees have" needs ten of them at once. The 2026-09-16 FRAMES attribution showed
-// the values ARE in the KB while the model still answers "cannot be determined" — the
-// numbers were never surfaced in a form it could aggregate.
-//
-// Two shapes, following the established practice for tables in RAG:
-//
-// - a two-column table (Wikipedia infobox) -> Markdown-KV lines: "Children: 3";
-// - any other table (ranked list, election results, timeline) -> a Markdown pipe
-// table carrying the header and EVERY row, because rank/order/completeness decide
-// those answers and row-window narrowing silently drops the answer row.
-//
-// Values are kept VERBATIM, entities/whitespace are collapsed, all-empty rows are
-// dropped. The raw HTML is never destroyed by this file: callers keep the original
-// chunk in the evidence pool (for citation) and use the view only for the
-// model-visible text.
-
-// tableViewMaxRows / tableViewMaxCharsPerTable are generous ceilings. A Wikipedia
-// election or standings table can run to 50K+ raw chars across 100+ rows; capping at
-// 4K chars / 60 rows silently dropped the answer row for exactly the
-// "large / multi-column" tables this renderer exists to serve (e.g. a 14.7K-char
-// standings table whose rank-19 row sat at ~62% of the table). The ceiling exists to
-// stop ONE pathological table from eating the whole evidence budget, not to truncate
-// normal tables — the caller still holds the raw chunk and can re-read it on demand.
-const (
- tableViewMaxRows = 400
- tableViewMaxCharsPerTable = 20000
-)
-
-// tableViewUnitTokens are the tokens of a Wikipedia table's "units" row — the row
-// right under the header that reads e.g. "No. | % | No. | %". It carries no data and
-// only pollutes a Markdown pipe table, so it is dropped before rendering.
-var tableViewUnitTokens = map[string]struct{}{
- "": {}, "no.": {}, "no": {}, "n": {}, "%": {}, "#": {},
- "—": {}, "-": {}, "•": {}, "·": {},
-}
-
-// RenderTables replaces every HTML table in text with a Markdown view.
-//
-// ok is false when text holds no renderable table, so callers keep their previous
-// behaviour untouched. A parse failure falls back to ok=false; this never panics and
-// never returns a partial view.
-func RenderTables(text string) (string, bool) {
- if text == "" || !strings.Contains(strings.ToLower(text), "<table") {
- return "", false
- }
- doc, err := html.Parse(strings.NewReader(text))
- if err != nil {
- return "", false
- }
- var views []string
- var walk func(*html.Node)
- walk = func(n *html.Node) {
- if n.Type == html.ElementNode && n.Data == "table" {
- if view := renderOneTable(n); view != "" {
- views = append(views, view)
- }
- }
- // Nested tables are visited too: the selector is recursive.
- for c := n.FirstChild; c != nil; c = c.NextSibling {
- walk(c)
- }
- }
- walk(doc)
- if len(views) == 0 {
- return "", false
- }
- return strings.Join(views, "\n\n"), true
-}
-
-// TableViewOrRaw is RenderTables when it produces something, else the text unchanged.
-//
-// Small convenience for the call sites that only want "the best available
-// model-visible form of this chunk".
-func TableViewOrRaw(text string) string {
- if view, ok := RenderTables(text); ok {
- return view
- }
- return text
-}
-
-// renderOneTable renders a single <table>, or "" when it holds no renderable row.
-func renderOneTable(table *html.Node) string {
- rows := tableRows(table)
- if len(rows) == 0 {
- return ""
- }
- // Infobox: two columns, a real key in the first cell and a value in the second (the
- // photo-caption row has an empty second cell and is dropped here).
- pairs := make([][2]string, 0, len(rows))
- for _, r := range rows {
- if len(r) == 2 && r[0] != "" && r[1] != "" {
- pairs = append(pairs, [2]string{r[0], r[1]})
- }
- }
- if len(pairs) >= 3 {
- limit := min(len(pairs), tableViewMaxRows)
- lines := make([]string, 0, limit)
- for _, p := range pairs[:limit] {
- lines = append(lines, p[0]+": "+p[1])
- }
- return truncateRunes(strings.Join(lines, "\n"), tableViewMaxCharsPerTable)
- }
-
- width := 0
- for _, r := range rows {
- width = max(width, len(r))
- }
- header := rows[0]
- // Drop header-repeat / units rows so the pipe body starts at real data (otherwise a
- // wide election table leads with a "No. | % | No. | %" junk row).
- body := make([][]string, 0, len(rows))
- for _, r := range rows[1:min(len(rows), tableViewMaxRows+1)] {
- if !isUnitsRow(r) {
- body = append(body, r)
- }
- }
- if len(body) == 0 {
- return ""
- }
- line := func(cells []string) string {
- padded := make([]string, width)
- for i := 0; i < width; i++ {
- if i < len(cells) {
- padded[i] = mdCell(cells[i])
- }
- }
- return "| " + strings.Join(padded, " | ") + " |"
- }
-
- out := make([]string, 0, len(body)+3)
- if caption := tableCaption(table); caption != "" {
- out = append(out, "**Table: "+caption+"**")
- }
- out = append(out, line(header), "|"+strings.Repeat("---|", width))
- for _, r := range body {
- out = append(out, line(r))
- }
- if omitted := len(rows) - 1 - len(body); omitted > 0 {
- out = append(out, fmt.Sprintf("... (%d more row(s) omitted)", omitted))
- }
- return truncateRunes(strings.Join(out, "\n"), tableViewMaxCharsPerTable)
-}
-
-// tableRows returns the non-empty rows of table as lists of cell strings.
-func tableRows(table *html.Node) [][]string {
- var rows [][]string
- for _, tr := range descendants(table, "tr") {
- cells := descendants(tr, "td", "th")
- if len(cells) == 0 {
- continue
- }
- vals := make([]string, 0, len(cells))
- nonEmpty := false
- for _, cell := range cells {
- v := cellText(cell)
- if v != "" {
- nonEmpty = true
- }
- vals = append(vals, v)
- }
- if nonEmpty {
- rows = append(rows, vals)
- }
- }
- return rows
-}
-
-// tableCaption returns the first <caption>'s text, "" when there is none.
-func tableCaption(table *html.Node) string {
- captions := descendants(table, "caption")
- if len(captions) == 0 {
- return ""
- }
- return cellText(captions[0])
-}
-
-// cellText is the whitespace-collapsed cell text (tags stripped, entities resolved).
-//
-// Text pieces are joined with a single space and every whitespace run collapses to one
-// space, so a multi-line cell reads as one value. Comments carry no text and are
-// skipped.
-func cellText(cell *html.Node) string {
- parts := make([]string, 0, 4)
- var walk func(*html.Node)
- walk = func(n *html.Node) {
- switch n.Type {
- case html.TextNode:
- if s := strings.TrimSpace(n.Data); s != "" {
- parts = append(parts, s)
- }
- return
- case html.CommentNode, html.DoctypeNode:
- return
- }
- for c := n.FirstChild; c != nil; c = c.NextSibling {
- walk(c)
- }
- }
- for c := cell.FirstChild; c != nil; c = c.NextSibling {
- walk(c)
- }
- return strings.Join(strings.Fields(strings.Join(parts, " ")), " ")
-}
-
-// descendants lists the descendant elements named in names, in document order.
-func descendants(n *html.Node, names ...string) []*html.Node {
- want := make(map[string]struct{}, len(names))
- for _, name := range names {
- want[name] = struct{}{}
- }
- var out []*html.Node
- var walk func(*html.Node)
- walk = func(x *html.Node) {
- for c := x.FirstChild; c != nil; c = c.NextSibling {
- if c.Type == html.ElementNode {
- if _, ok := want[c.Data]; ok {
- out = append(out, c)
- }
- }
- walk(c)
- }
- }
- walk(n)
- return out
-}
-
-// isUnitsRow reports whether every cell of a row is a unit token — a row that carries
-// no data.
-func isUnitsRow(vals []string) bool {
- if len(vals) == 0 {
- return false
- }
- for _, v := range vals {
- if _, ok := tableViewUnitTokens[strings.ToLower(strings.TrimSpace(v))]; !ok {
- return false
- }
- }
- return true
-}
-
-// mdCell escapes a value for a Markdown pipe cell (the delimiter must stay intact).
-func mdCell(value string) string {
- return strings.NewReplacer(`\`, `\\`, "|", `\|`).Replace(value)
-}
diff --git a/internal/rag/agentic-rag/runtime/table_view_test.go b/internal/rag/agentic-rag/runtime/table_view_test.go
deleted file mode 100644
index 1387ea251c..0000000000
--- a/internal/rag/agentic-rag/runtime/table_view_test.go
+++ /dev/null
@@ -1,219 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "fmt"
- "strings"
- "testing"
-)
-
-// TestTableViewInfoboxBecomesKeyValue pins the two-column shape: a Wikipedia infobox
-// renders as "key: value" lines, which is the form a field lookup reads best.
-func TestTableViewInfoboxBecomesKeyValue(t *testing.T) {
- raw := `<table>
- <tr><th>Born</th><td>1961</td></tr>
- <tr><th>Children</th><td>3</td></tr>
- <tr><th>Spouse</th><td>Jane Doe</td></tr>
- </table>`
- view, ok := RenderTables(raw)
- if !ok {
- t.Fatalf("RenderTables() reported no view for an infobox")
- }
- want := "Born: 1961\nChildren: 3\nSpouse: Jane Doe"
- if view != want {
- t.Errorf("infobox view = %q, want %q", view, want)
- }
-}
-
-// TestTableViewRankedTableKeepsEveryRow pins the pipe shape: rank/order/completeness
-// decide ranked-table answers, so every data row must survive verbatim.
-func TestTableViewRankedTableKeepsEveryRow(t *testing.T) {
- raw := `<table>
- <tr><th>Rank</th><th>Name</th><th>Points</th></tr>
- <tr><td>1</td><td>Alice</td><td>90</td></tr>
- <tr><td>2</td><td>Bob</td><td>88</td></tr>
- <tr><td>3</td><td>Carol</td><td>71</td></tr>
- </table>`
- view, ok := RenderTables(raw)
- if !ok {
- t.Fatalf("RenderTables() reported no view for a ranked table")
- }
- want := "| Rank | Name | Points |\n" +
- "|---|---|---|\n" +
- "| 1 | Alice | 90 |\n" +
- "| 2 | Bob | 88 |\n" +
- "| 3 | Carol | 71 |"
- if view != want {
- t.Errorf("ranked view = %q, want %q", view, want)
- }
-}
-
-// TestTableViewCaptionAndUnitsRow covers the caption header and the units row a
-// Wikipedia table carries right under its header ("No. | % | No. | %"): the row holds
-// no data and would only pollute the pipe body.
-func TestTableViewCaptionAndUnitsRow(t *testing.T) {
- raw := `<table>
- <caption>Final standings</caption>
- <tr><th>Rank</th><th>Rider</th><th>Points</th></tr>
- <tr><td>No.</td><td>%</td><td>#</td></tr>
- <tr><td>19</td><td>Danilo</td><td>62</td></tr>
- </table>`
- view, ok := RenderTables(raw)
- if !ok {
- t.Fatalf("RenderTables() reported no view")
- }
- if !strings.HasPrefix(view, "**Table: Final standings**\n") {
- t.Errorf("view = %q, want the caption as a bold header line", view)
- }
- if !strings.Contains(view, "| 19 | Danilo | 62 |") {
- t.Errorf("view = %q, want the rank-19 row kept", view)
- }
- if strings.Contains(view, "No.") || strings.Contains(view, "| % |") {
- t.Errorf("view = %q, want the units row dropped", view)
- }
-}
-
-// TestTableViewEscapesPipeAndBackslash pins the cell escaping: an unescaped "|" would
-// split the row into extra columns.
-func TestTableViewEscapesPipeAndBackslash(t *testing.T) {
- raw := `<table>
- <tr><th>Rank</th><th>Name</th><th>Note</th></tr>
- <tr><td>1</td><td>Alice</td><td>a|b</td></tr>
- <tr><td>2</td><td>Bob</td><td>c\d</td></tr>
- </table>`
- view, ok := RenderTables(raw)
- if !ok {
- t.Fatalf("RenderTables() reported no view")
- }
- if !strings.Contains(view, `| a\|b |`) {
- t.Errorf("view = %q, want the pipe escaped", view)
- }
- if !strings.Contains(view, `| c\\d |`) {
- t.Errorf("view = %q, want the backslash escaped", view)
- }
-}
-
-// TestTableViewNoopForNonTable pins the no-op contract: plain text, markdown pipe
-// tables and table-less HTML must come back untouched so a caller can route every
-// chunk through TableViewOrRaw.
-func TestTableViewNoopForNonTable(t *testing.T) {
- for _, raw := range []string{
- "",
- "plain prose with no markup at all",
- "| a | b |\n|---|---|\n| 1 | 2 |\n| 3 | 4 |",
- "<p>html without a table</p>",
- } {
- if view, ok := RenderTables(raw); ok {
- t.Errorf("RenderTables(%q) = (%q, true), want no view", raw, view)
- }
- if got := TableViewOrRaw(raw); got != raw {
- t.Errorf("TableViewOrRaw(%q) = %q, want the raw text", raw, got)
- }
- }
-}
-
-// TestTableViewNoopForEmptyTable pins the all-empty path: a table whose rows carry no
-// text has nothing to render, so the caller keeps the raw chunk instead of receiving an
-// empty string.
-func TestTableViewNoopForEmptyTable(t *testing.T) {
- for _, raw := range []string{
- "<table></table>",
- "<table><tr><td></td><td> </td></tr></table>",
- "<table><tr><th>Rank</th></tr></table>",
- } {
- if view, ok := RenderTables(raw); ok {
- t.Errorf("RenderTables(%q) = (%q, true), want no view", raw, view)
- }
- }
-}
-
-// TestTableViewTruncatedFragmentDoesNotPanic pins the fragment case the table
-// exemption exists for: a text window that cut the table mid-markup must never panic
-// and must never emit a partial pipe dump.
-func TestTableViewTruncatedFragmentDoesNotPanic(t *testing.T) {
- for _, raw := range []string{
- `<table><tr><td>only a fragment`,
- `row text then a stray <table`,
- `<tr><td>a</td></tr><tr><td>b</td></tr>`,
- } {
- if view, ok := RenderTables(raw); ok && strings.Contains(view, "\n") && !strings.Contains(raw, "<table") {
- t.Errorf("RenderTables(%q) = %q: a table-less fragment must not render", raw, view)
- }
- if got := TableViewOrRaw(raw); got == "" && raw != "" {
- t.Errorf("TableViewOrRaw(%q) returned an empty string", raw)
- }
- }
-}
-
-// TestTableViewJoinsMultipleTables pins the multi-table shape: a chunk that carries ten
-// nominee infoboxes hands the model ten blocks, separated by a blank line.
-func TestTableViewJoinsMultipleTables(t *testing.T) {
- raw := `<table>
- <tr><th>Born</th><td>1961</td></tr>
- <tr><th>Children</th><td>3</td></tr>
- <tr><th>Spouse</th><td>Jane</td></tr>
- </table>
- <table>
- <tr><th>Born</th><td>1954</td></tr>
- <tr><th>Children</th><td>2</td></tr>
- <tr><th>Spouse</th><td>John</td></tr>
- </table>`
- view, ok := RenderTables(raw)
- if !ok {
- t.Fatalf("RenderTables() reported no view")
- }
- blocks := strings.Split(view, "\n\n")
- if len(blocks) != 2 {
- t.Fatalf("view = %q, want 2 blocks", view)
- }
- for i, block := range blocks {
- if !strings.Contains(block, "Children: ") {
- t.Errorf("block %d = %q, want a key-value view", i, block)
- }
- }
-}
-
-// TestTableViewCapsRowsAndReportsOmission pins the ceiling: an oversized table keeps
-// tableViewMaxRows data rows and SAYS how many were dropped, so a truncated table is
-// never read as a complete one.
-func TestTableViewCapsRowsAndReportsOmission(t *testing.T) {
- var b strings.Builder
- b.WriteString("<table><tr><th>Rank</th><th>Rider</th><th>Team</th></tr>")
- for i := 1; i <= tableViewMaxRows+2; i++ {
- fmt.Fprintf(&b, "<tr><td>%d</td><td>Rider %d</td><td>Team %d</td></tr>", i, i, i)
- }
- b.WriteString("</table>")
-
- view, ok := RenderTables(b.String())
- if !ok {
- t.Fatalf("RenderTables() reported no view")
- }
- rows := 0
- for _, line := range strings.Split(view, "\n") {
- if strings.HasPrefix(line, "| ") {
- rows++
- }
- }
- // The header is a pipe line too, so the data rows are one fewer.
- if rows != tableViewMaxRows+1 {
- t.Errorf("rendered %d pipe lines, want %d data rows + 1 header", rows, tableViewMaxRows)
- }
- if !strings.HasSuffix(view, "... (2 more row(s) omitted)") {
- t.Errorf("view tail = %q, want the omission marker", view[len(view)-40:])
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/think_event.go b/internal/rag/agentic-rag/runtime/think_event.go
deleted file mode 100644
index b52d8bf607..0000000000
--- a/internal/rag/agentic-rag/runtime/think_event.go
+++ /dev/null
@@ -1,263 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-// Reasoning-step reporting for the agentic RAG run.
-//
-// A step is a thing a PRODUCER decides to report — there is no allowlist of log
-// tags and nothing intercepts the logger. Each stage declares its own step where
-// it happens (StepReporter.Stage), and a tool declares both halves of its call
-// (StepReporter.Emit). The developer log and the user-visible trace are separate
-// channels fed from that one call, so they cannot drift, and nothing becomes
-// visible (or invisible) because a log line was reworded.
-package runtime
-
-import (
- "context"
- "fmt"
- "log"
- "strings"
-
- "ragflow/internal/service"
-)
-
-// ThinkLineBreak separates two steps inside the think block.
-//
-// The block is delivered to the client as inline HTML (the chat UI wraps it in
-// <details class="think"> and renders the message as markdown), where a newline
-// is just whitespace. Mirrors Python think_log.py:69 ("<br>").
-const ThinkLineBreak = "<br>"
-
-// StepIndentUnit and StepMaxDepth shape the think block's nesting.
-//
-// A step is INDENTED when it is reported beneath the step that launched it: a
-// prefetch's search legs under the "[Prefetch] … up front." line that opened them,
-// a tool's internal searches under the tool call, an action session's tools under
-// the research round, and (when an outer react loop is wired) a whole research graph
-// under the "Running the rag tool …" line that started it. The depth is carried on
-// the context (Nested) because those layers are entered by ordinary calls, so the
-// indent costs each producer nothing.
-//
-// The unit is &nbsp; entities, not spaces: the block is delivered as inline HTML
-// (the chat UI wraps it in <details> and renders it as markdown), where a run of
-// leading spaces collapses away.
-//
-// The cap sits above the deepest nesting that actually occurs, so it clips nothing
-// real. Counted from the outermost line a run can start at, with an outer react loop
-// wired:
-//
-// level 0 "Running the rag tool with …" (outer react loop only)
-// level 1 the graph's own steps — Starting research / Planner / Prefetch /
-// RAGAgent Round N / Draft / SCA / Finalize
-// level 2 a tool call the round launches, and a prefetch's search legs
-// level 3 the search leg a tool runs (grep / bm25 / hybrid under `retrieve`)
-//
-// A leg that DELEGATES adds no level of its own: GrepSearch hands the pool build to
-// BM25Search with the same context, so "[BM25 search]" prints beside the
-// "[Grep search]" line that called it rather than under it. Without an outer loop —
-// the direct-graph path — every level shifts up by one: the graph's own steps sit at
-// 0, a tool call or a prefetch leg at 1, and the deepest line (a tool's search leg)
-// at 2. The cap exists so the indent (and the stored text) cannot grow without bound
-// if a future layer nests deeper or a path recurses.
-const (
- StepIndentUnit = "&nbsp;&nbsp;&nbsp;&nbsp;"
- StepMaxDepth = 5
-)
-
-// stepsDepthKey carries the current step nesting depth.
-type stepsDepthKey struct{}
-
-// Nested returns a ctx whose steps are reported one level deeper. A layer calls it
-// around the work it launches after reporting its own step, so everything that
-// layer produces reads as belonging to it.
-func Nested(ctx context.Context) context.Context {
- return context.WithValue(ctx, stepsDepthKey{}, stepDepth(ctx)+1)
-}
-
-// stepDepth reports the nesting depth bound to ctx, 0 at the top level.
-func stepDepth(ctx context.Context) int {
- d, _ := ctx.Value(stepsDepthKey{}).(int)
- return d
-}
-
-// indentStep prefixes line with depth indentation units, capped at StepMaxDepth.
-// Only the TEXT projection is indented: the log stays flush (it is grepped, and
-// its lines double as the Python-parity record) and the event Summary stays clean
-// (a client renders structure, not entities).
-func indentStep(line string, depth int) string {
- if depth <= 0 {
- return line
- }
- if depth > StepMaxDepth {
- depth = StepMaxDepth
- }
- return strings.Repeat(StepIndentUnit, depth) + line
-}
-
-// ThinkEvent is ONE reasoning step, structured. The definition lives in
-// internal/service, which owns the wire shape it travels in
-// (AsyncChatResult.think_event); this is an ALIAS of it.
-//
-// The definition cannot live here: service carries the step over SSE and cannot
-// import this package, while this one already imports service for its
-// exploration providers (tool_exploration.go). Two structs, one on each side,
-// bridged field by field in cmd, compiled for as long as one stayed a subset —
-// and silently dropped any field added on the other. An alias has no second copy
-// to fall out of step.
-type ThinkEvent = service.ThinkEvent
-
-// ThinkEvent kinds.
-const (
- // ThinkKindStage is one narrative pipeline stage.
- ThinkKindStage = "stage"
- // ThinkKindToolCall is a tool about to run, with its arguments.
- ThinkKindToolCall = "tool_call"
- // ThinkKindToolResult is a tool's outcome, with its result metadata.
- ThinkKindToolResult = "tool_result"
-)
-
-// ThinkMaxSources caps the evidence anchors a tool_result event carries: a step
-// is a pointer into the evidence, not a dump of it.
-const ThinkMaxSources = 12
-
-// ThinkSink receives one ThinkEvent per reasoning step. The sink is per request,
-// so concurrent requests stay isolated. Nil means no client asked for structured
-// steps: emission is a no-op and only the human text is produced.
-type ThinkSink func(ThinkEvent)
-
-// StepReporter delivers one reasoning step to its two projections.
-//
-// The caller fills one in once per run (RAGTools.Steps); Rag binds it on ctx and
-// every stage and tool reports through StepsFrom(ctx) — the same per-request
-// shape Python's ContextVar had, which the async task tree inherited. A
-// projection the caller left nil is skipped, so a run without a think block
-// still logs, and a run without a logger still narrates.
-type StepReporter struct {
- // Text is the human-sentence projection (the <think> block). Nil = no block.
- Text func(line string)
- // Events is the structured projection. Nil = no structured client.
- Events ThinkSink
- // depth is the nesting level this copy reports at. It is stamped by StepsFrom
- // from the caller's context (Nested) and never set by the caller, so the two
- // projections stay independent: a run that indents its text does not indent its
- // structured steps.
- depth int
-}
-
-// Enabled reports whether any projection is wired.
-func (r StepReporter) Enabled() bool { return r.Text != nil || r.Events != nil }
-
-// Stage reports one narrative stage: "[stage] msg" is written to the developer
-// log, the same sentence goes to the think block, and a stage event goes to
-// structured clients.
-//
-// The log write lives here (rather than at each call site) because a stage line
-// has no other developer-facing home: it IS the record.
-func (r StepReporter) Stage(log *log.Logger, stage, format string, args ...any) {
- r.StageLine(log, stage, fmt.Sprintf(format, args...))
-}
-
-// StageLine is Stage for an already-rendered sentence, so a caller that built
-// its message elsewhere (a helper that formats the whole line) does not have to
-// route it through a "%s" format verb.
-func (r StepReporter) StageLine(log *log.Logger, stage, message string) {
- r.StageLineDetail(log, stage, message, message)
-}
-
-// StageLineDetail is StageLine for the steps whose two audiences need different
-// words: `summary` is what the run REPORTS (the think block and the stage event)
-// and `detail` is what the developer log RECORDS.
-//
-// It exists for identifiers that are precise for a developer and noise for a
-// reader: a 32-hex document id, or the per-document chunk/char breakdown a search
-// leg computes. The log keeps them — it is the greppable record, and Python's
-// line-for-line parity lives there — while the think block names the same fact in
-// words ("3 passages from 2 documents" instead of "d1:2chunk(14chars)").
-//
-// Both strings describe ONE call: the two halves are rendered at the same call
-// site from the same chunks, so a reader and a developer can never be told
-// different stories about what a step saw. A step whose two audiences want the
-// same sentence uses StageLine, which is this function with one message.
-func (r StepReporter) StageLineDetail(log *log.Logger, stage, summary, detail string) {
- if log != nil {
- log.Printf("[%s] %s", stage, detail)
- }
- r.Emit(ThinkEvent{Kind: ThinkKindStage, Stage: stage, Summary: "[" + stage + "] " + summary})
-}
-
-// Emit reports a fully-formed step. The developer log is deliberately NOT
-// written here: the tool executor already logs its own two sentences (call and
-// result, Python tool_decorator parity), and writing the summary again would
-// duplicate every tool call in the log.
-//
-// The text projection is indented by this copy's depth (see StepIndentUnit); the
-// event keeps the sentence unindented, because a structured client renders the
-// nesting from its own step tree rather than from entities it would have to strip.
-func (r StepReporter) Emit(ev ThinkEvent) {
- if ev.Summary != "" && r.Text != nil {
- text := r.Text
- line := indentStep(ev.Summary, r.depth)
- func() {
- // A broken sink must never break the run (Python wrapped its sink
- // call in a bare try/except for the same reason).
- defer func() { _ = recover() }()
- text(line + ThinkLineBreak)
- }()
- }
- EmitThink(r.Events, ev)
-}
-
-// EmitThink delivers one event to sink. A nil sink and a panicking sink are both
-// tolerated: think narration must never break the request or the run.
-func EmitThink(sink ThinkSink, event ThinkEvent) {
- if sink == nil {
- return
- }
- defer func() { _ = recover() }()
- sink(event)
-}
-
-// stepsCtxKey carries the per-request StepReporter.
-type stepsCtxKey struct{}
-
-// WithSteps binds the run's step reporter to ctx. Every stage and tool beneath
-// reads it with StepsFrom, so a step is reported where it happens without
-// threading a sink through every node signature — the same shape as Python's
-// per-request ContextVar (rag/advanced_rag/think_log.py), which the async task
-// tree inherited. A sink bound here is per request, so concurrent requests stay
-// isolated.
-func WithSteps(ctx context.Context, steps StepReporter) context.Context {
- if !steps.Enabled() {
- return ctx
- }
- return context.WithValue(ctx, stepsCtxKey{}, steps)
-}
-
-// StepsFrom returns the step reporter bound to ctx, or the zero StepReporter
-// when none is bound — in which case every report is a no-op and the run simply
-// has no user-visible trace (a caller that wired neither projection).
-//
-// The returned copy carries ctx's nesting depth (Nested), which is what indents the
-// text projection of a step reported beneath another one. The bound reporter itself
-// stays depth-free, so the same run can report at different depths from different
-// contexts — which is exactly how a nested layer works.
-func StepsFrom(ctx context.Context) StepReporter {
- depth := stepDepth(ctx)
- if r, ok := ctx.Value(stepsCtxKey{}).(StepReporter); ok {
- r.depth = depth
- return r
- }
- return StepReporter{depth: depth}
-}
diff --git a/internal/rag/agentic-rag/runtime/think_event_test.go b/internal/rag/agentic-rag/runtime/think_event_test.go
deleted file mode 100644
index 9fa755278c..0000000000
--- a/internal/rag/agentic-rag/runtime/think_event_test.go
+++ /dev/null
@@ -1,251 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "bytes"
- "context"
- "log"
- "strings"
- "testing"
-)
-
-// TestStepReporterStageDeliversBothProjections is the core contract of the
-// explicit-step design: one call writes the developer log line AND delivers the
-// same sentence to the think block and to structured clients.
-func TestStepReporterStageDeliversBothProjections(t *testing.T) {
- var logged bytes.Buffer
- var text []string
- var events []ThinkEvent
- r := StepReporter{
- Text: func(line string) { text = append(text, line) },
- Events: func(ev ThinkEvent) { events = append(events, ev) },
- }
-
- r.Stage(log.New(&logged, "", 0), "Planner", "decomposed into %d fan-out(s)", 3)
-
- if got := logged.String(); !strings.Contains(got, "[Planner] decomposed into 3 fan-out(s)") {
- t.Errorf("developer log = %q, want the stage line", got)
- }
- if len(text) != 1 || text[0] != "[Planner] decomposed into 3 fan-out(s)"+ThinkLineBreak {
- t.Errorf("text projection = %#v, want the sentence with a block separator", text)
- }
- if len(events) != 1 {
- t.Fatalf("events = %#v, want one stage event", events)
- }
- if ev := events[0]; ev.Kind != ThinkKindStage || ev.Stage != "Planner" ||
- ev.Summary != "[Planner] decomposed into 3 fan-out(s)" {
- t.Errorf("event = %#v", ev)
- }
-}
-
-// TestStepReporterStageDetailSplitsTheTwoAudiences pins StageLineDetail, the
-// escape hatch the search legs use: the log records `detail` (identifiers kept)
-// while the think block and the stage event carry `summary`. Both strings are
-// passed in from ONE call site, so the two audiences can never be told different
-// things about what a step saw.
-func TestStepReporterStageDetailSplitsTheTwoAudiences(t *testing.T) {
- const docID = "95a7aee3f11143e69dc9fa5b7bad3a14"
- var logged bytes.Buffer
- var text []string
- var events []ThinkEvent
- r := StepReporter{
- Text: func(line string) { text = append(text, line) },
- Events: func(ev ThinkEvent) { events = append(events, ev) },
- }
-
- r.StageLineDetail(log.New(&logged, "", 0), "Hybrid search",
- `Found 5 passages in 1 document for "曹操".`,
- `"曹操" -> 5 chunk(s): `+docID+`:5chunk(4963chars)`)
-
- if got := logged.String(); !strings.Contains(got, docID+":5chunk(4963chars)") {
- t.Errorf("developer log = %q, want the per-document breakdown", got)
- }
- want := `[Hybrid search] Found 5 passages in 1 document for "曹操".`
- if len(text) != 1 || text[0] != want+ThinkLineBreak {
- t.Errorf("text projection = %#v, want %q", text, want)
- }
- if len(events) != 1 || events[0].Summary != want {
- t.Errorf("event = %#v, want the reader's sentence", events)
- }
- if strings.Contains(text[0], docID) || strings.Contains(logged.String(), "Found 5 passages") {
- t.Error("the two projections must not swap their sentences")
- }
-
- // StageLine is this same function with one message: its projections cannot
- // diverge either.
- logged.Reset()
- text = nil
- r.StageLine(log.New(&logged, "", 0), "Planner", "one sentence")
- if logged.String() != "[Planner] one sentence\n" ||
- len(text) != 1 || text[0] != "[Planner] one sentence"+ThinkLineBreak {
- t.Errorf("StageLine split its audiences: log=%q text=%#v", logged.String(), text)
- }
-}
-
-// TestStepReporterIndentsNestedSteps pins the nesting the trace shows: a step
-// reported beneath another one is indented in the TEXT projection, while the log
-// (the greppable, Python-parity record) and the event Summary (a structured client
-// gets a step tree, not entities) stay clean.
-//
-// The depth rides on the context, so a layer opts in by calling Nested around the
-// work it launches after reporting its own step.
-func TestStepReporterIndentsNestedSteps(t *testing.T) {
- var logged bytes.Buffer
- var text []string
- var events []ThinkEvent
- r := StepReporter{
- Text: func(line string) { text = append(text, line) },
- Events: func(ev ThinkEvent) { events = append(events, ev) },
- }
- ctx := WithSteps(context.Background(), r)
-
- StepsFrom(ctx).StageLine(log.New(&logged, "", 0), "Prefetch", "The plan lists 5 sub-questions; searching 3 opening queries up front.")
- StepsFrom(Nested(ctx)).StageLine(log.New(&logged, "", 0), "BM25 search", `Found 5 passages in 1 document for "q".`)
-
- wantTop := "[Prefetch] The plan lists 5 sub-questions; searching 3 opening queries up front." + ThinkLineBreak
- wantNested := StepIndentUnit + `[BM25 search] Found 5 passages in 1 document for "q".` + ThinkLineBreak
- if len(text) != 2 || text[0] != wantTop || text[1] != wantNested {
- t.Errorf("text = %#v, want %q then %q", text, wantTop, wantNested)
- }
- if got := logged.String(); !strings.Contains(got, "\n[BM25 search] Found 5 passages in 1 document") {
- t.Errorf("the log must stay flush:\n%s", got)
- }
- for _, ev := range events {
- if strings.Contains(ev.Summary, "&nbsp;") {
- t.Errorf("event summary carries indentation entities: %q", ev.Summary)
- }
- }
-
- // The cap: a five-level nesting (outer rag → graph → round → tool → leg) must not
- // walk off the right edge, so past StepMaxDepth the indent stops growing.
- deep := ctx
- for i := 0; i < StepMaxDepth+3; i++ {
- deep = Nested(deep)
- }
- StepsFrom(deep).StageLine(log.New(&logged, "", 0), "Deep", "x")
- if got, want := text[len(text)-1], strings.Repeat(StepIndentUnit, StepMaxDepth)+"[Deep] x"+ThinkLineBreak; got != want {
- t.Errorf("deep text = %q, want %q", got, want)
- }
-}
-
-// TestStepReporterEmitLeavesTheLogToTheProducer pins the split for tools: the
-// executor logs its own two sentences (Python tool_decorator parity), so Emit
-// must not write them again — but it still delivers both user projections.
-func TestStepReporterEmitLeavesTheLogToTheProducer(t *testing.T) {
- var logged bytes.Buffer
- var text []string
- var events []ThinkEvent
- r := StepReporter{
- Text: func(line string) { text = append(text, line) },
- Events: func(ev ThinkEvent) { events = append(events, ev) },
- }
-
- r.Emit(ThinkEvent{
- Kind: ThinkKindToolResult, Stage: "Function tool", Tool: "retrieve",
- Status: StatusOK, Results: 3, Documents: 2, Sources: []string{"c1", "c2"},
- DurationMS: 12, Cause: "index not found",
- Summary: "[Function tool] The retrieve tool returned 3 results from 2 documents.",
- })
-
- if logged.Len() != 0 {
- t.Errorf("Emit must not write the developer log, got %q", logged.String())
- }
- if len(text) != 1 || !strings.HasSuffix(text[0], ThinkLineBreak) {
- t.Errorf("text projection = %#v", text)
- }
- ev := events[0]
- if ev.Tool != "retrieve" || ev.Status != StatusOK || ev.Results != 3 || ev.Documents != 2 ||
- ev.DurationMS != 12 || ev.Cause != "index not found" || len(ev.Sources) != 2 {
- t.Errorf("event lost fields: %#v", ev)
- }
-}
-
-// TestStepReporterOptionalProjections pins that either projection may be absent:
-// a run with no think block still logs, a run with no logger still narrates, and
-// the zero reporter is inert rather than a crash.
-func TestStepReporterOptionalProjections(t *testing.T) {
- var events []ThinkEvent
- eventsOnly := StepReporter{Events: func(ev ThinkEvent) { events = append(events, ev) }}
- eventsOnly.Stage(nil, "SCA", "verdict=%s", "SUFFICIENT") // nil logger must be tolerated
- eventsOnly.Emit(ThinkEvent{Summary: "x"})
- if len(events) != 2 {
- t.Errorf("events = %#v, want both steps", events)
- }
-
- var text []string
- logOnly := StepReporter{Text: func(line string) { text = append(text, line) }}
- logOnly.Emit(ThinkEvent{Summary: "y"})
- if len(text) != 1 {
- t.Errorf("text = %#v, want the step", text)
- }
-
- var zero StepReporter
- if zero.Enabled() {
- t.Error("the zero reporter must report itself disabled")
- }
- zero.Stage(log.New(&bytes.Buffer{}, "", 0), "Planner", "x") // must not panic
- zero.Emit(ThinkEvent{Summary: "x"}) // must not panic
-}
-
-// TestStepReporterSurvivesPanickingText mirrors Python's bare try/except around
-// the sink: a broken consumer must never break the run, and the structured
-// projection must still be delivered.
-func TestStepReporterSurvivesPanickingText(t *testing.T) {
- var events []ThinkEvent
- r := StepReporter{
- Text: func(string) { panic("think block exploded") },
- Events: func(ev ThinkEvent) { events = append(events, ev) },
- }
-
- r.Stage(nil, "Planner", "x") // must not panic
- r.Emit(ThinkEvent{Summary: "y"})
-
- if len(events) != 2 {
- t.Errorf("events = %#v, want both steps despite the broken text sink", events)
- }
-}
-
-// TestStepsFromCtx pins the per-request carrier: a bound reporter comes back,
-// an unbound ctx yields the inert zero value (so a caller that wired nothing
-// simply gets no trace instead of a nil dereference).
-func TestStepsFromCtx(t *testing.T) {
- if StepsFrom(context.Background()).Enabled() {
- t.Fatal("an unbound ctx must yield the zero reporter")
- }
-
- var got []string
- want := StepReporter{Text: func(line string) { got = append(got, line) }}
- ctx := WithSteps(context.Background(), want)
- StepsFrom(ctx).Stage(nil, "Planner", "x")
-
- if len(got) != 1 || got[0] != "[Planner] x"+ThinkLineBreak {
- t.Errorf("reporter from ctx = %#v, want the bound one", got)
- }
-
- // A disabled reporter is not worth binding: the ctx stays untouched.
- plain := WithSteps(context.Background(), StepReporter{})
- if StepsFrom(plain).Enabled() {
- t.Error("binding a disabled reporter must be a no-op")
- }
-}
-
-// TestEmitThinkToleratesNilAndPanickingSink guards the event channel on its own.
-func TestEmitThinkToleratesNilAndPanickingSink(t *testing.T) {
- EmitThink(nil, ThinkEvent{Summary: "x"}) // must not panic
- EmitThink(func(ThinkEvent) { panic("boom") }, ThinkEvent{Summary: "x"})
-}
diff --git a/internal/rag/agentic-rag/runtime/tool_claim_router.go b/internal/rag/agentic-rag/runtime/tool_claim_router.go
deleted file mode 100644
index c6a49f0a8c..0000000000
--- a/internal/rag/agentic-rag/runtime/tool_claim_router.go
+++ /dev/null
@@ -1,176 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "math"
- "sort"
- "strings"
-
- "ragflow/internal/service/nav"
-)
-
-// The claim_agg navigation-tree router.
-//
-// The claim leg runs first and decides the ranking outright when it hits — a
-// claim is an atomic proposition carrying its own vector and verbatim
-// evidence, so matching one means the document asserts that fact. No claim
-// rows, or an empty claim leg, falls back to the chunk leg (PageIndex style:
-// hybrid chunk recall rolled up per document).
-//
-// Like chunk_agg (internal/service/nav), the algorithm consumes the agentic
-// claim-recall capability through the NavTreeRouter interface and keeps the
-// summarizer injected, so the runtime stays free of the service layer.
-
-// claimAggPool: the per-leg store limit for the
-// claim routing leg, wide enough for a document to collect several claim hits
-// before the per-document roll-up (the aggregation rewards multi-hit docs).
-const claimAggPool = 256
-
-// claimDocScore mirrors _nav_doc_score (PageIndex): hit scores summed, damped
-// by hit count. The sum rewards a document matching several claims; the
-// square-root denominator damps it so a document cannot win on volume alone.
-func claimDocScore(total float64, hits int) float64 {
- return total / math.Sqrt(float64(hits)+1)
-}
-
-// ClaimAggRouter is a NavTreeRouter that routes documents through the
-// compilers' claim rows first, falling back to the chunk_agg router when the
-// claim leg comes back empty (a raptor-less or pre-claim dataset is a
-// legitimate empty leg, not a failure).
-type ClaimAggRouter struct {
- // Deps carries the claim-recall wiring (doc engine, embedder, index).
- // Route overrides only the per-call tenant/dataset.
- Deps SearchDeps
- // Fallback is the chunk_agg router used when the claim leg is empty.
- Fallback NavTreeRouter
- // Summarize loads the nav_doc summary per routed document. Nil leaves the summaries empty.
- Summarize nav.DocSummarizer
-}
-
-// claimAggBucket is the per-document roll-up of claim hits: total, best and hit count per doc.
-type claimAggBucket struct {
- docID string
- total float64
- best float64
- hits int
-}
-
-// Route implements the agentic NavTreeRouter contract with the claim-first
-// strategy. It returns (nil, nil) when the fallback router reports no compiled
-// tree, an empty non-nil slice when a structure exists but nothing routed, and
-// a non-nil error when the fallback's retrieval itself failed — the claim leg
-// never errors (recall failures read as an empty leg, which is exactly the
-// fallback trigger).
-func (r *ClaimAggRouter) Route(ctx context.Context, tenantID, kbID, query string, docScope []string, topK int) ([][2]string, error) {
- fallback := func() ([][2]string, error) {
- if r.Fallback == nil {
- return nil, nil
- }
- return r.Fallback.Route(ctx, tenantID, kbID, query, docScope, topK)
- }
- deps := r.Deps
- deps.TenantID = tenantID
- deps.KbIDs = []string{kbID}
- // The two compilers are queried SEPARATELY, never as one mixed condition: tree first — it is
- // the compiled benchmark path — then page_index only when the tree pass came back empty. Each
- // pass pins compile_kwd and row_types=("claim",).
- claims := recallDatasetClaimsFiltered(ctx, deps, query, claimAggPool, []string{"tree"}, []string{"claim"})
- if len(claims) == 0 {
- claims = recallDatasetClaimsFiltered(ctx, deps, query, claimAggPool, []string{"page_index", "pageindex"}, []string{"claim"})
- }
- if len(claims) == 0 {
- // No claim rows (or the recall failed): a legitimate empty leg — the
- // chunk leg decides.
- return fallback()
- }
-
- // Roll the claim hits up per document:
- // the doc score lives on each claim's similarity.
- agg := map[string]*claimAggBucket{}
- for _, c := range claims {
- did := strings.TrimSpace(c.DocID)
- if did == "" {
- continue
- }
- b, ok := agg[did]
- if !ok {
- agg[did] = &claimAggBucket{docID: did, total: c.Score, best: c.Score, hits: 1}
- continue
- }
- b.total += c.Score
- b.hits++
- if c.Score > b.best {
- b.best = c.Score
- }
- }
-
- // doc_scope: the store condition is filtered before the recall; post-filtering the rolled-up
- // buckets is equivalent here because the recall is threshold-free.
- scope := make(map[string]bool, len(docScope))
- for _, d := range docScope {
- if d = strings.TrimSpace(d); d != "" {
- scope[d] = true
- }
- }
- ranked := make([]*claimAggBucket, 0, len(agg))
- for _, b := range agg {
- if len(scope) > 0 && !scope[b.docID] {
- continue
- }
- ranked = append(ranked, b)
- }
- if len(ranked) == 0 {
- // A scoped claim leg that reaches nothing is still an empty leg — the
- // chunk leg (which applies the same scope) may reach further.
- return fallback()
- }
- // Order by the damped DocScore, best similarity as the tie-break.
- sort.SliceStable(ranked, func(i, j int) bool {
- si := claimDocScore(ranked[i].total, ranked[i].hits)
- sj := claimDocScore(ranked[j].total, ranked[j].hits)
- if si != sj {
- return si > sj
- }
- return ranked[i].best > ranked[j].best
- })
- if topK > 0 && len(ranked) > topK {
- ranked = ranked[:topK]
- }
- // Same focus cap as the sibling routers (3):
- // routing to many documents makes the agent carry that many times the
- // evidence in every later round.
- const navDocFocusLimit = 3
- if navDocFocusLimit > 0 && len(ranked) > navDocFocusLimit {
- ranked = ranked[:navDocFocusLimit]
- }
-
- ids := make([]string, 0, len(ranked))
- for _, b := range ranked {
- ids = append(ids, b.docID)
- }
- summaries := map[string]string{}
- if r.Summarize != nil {
- summaries = r.Summarize(ctx, tenantID, kbID, ids)
- }
- routed := make([][2]string, 0, len(ranked))
- for _, b := range ranked {
- routed = append(routed, [2]string{b.docID, strings.TrimSpace(summaries[b.docID])})
- }
- return routed, nil
-}
diff --git a/internal/rag/agentic-rag/runtime/tool_claims.go b/internal/rag/agentic-rag/runtime/tool_claims.go
deleted file mode 100644
index 49109a12f4..0000000000
--- a/internal/rag/agentic-rag/runtime/tool_claims.go
+++ /dev/null
@@ -1,751 +0,0 @@
-// Package-level claim recall for the agentic runtime: a KB-wide FLAT search over
-// claim/evidence rows (entity_type_kwd="claim"), plus its prefetch wiring.
-//
-// Two legs per search — BM25 over content_ltks/content_sm_ltks and KNN over
-// the claim rows' q_<dim>_vec — fused by reciprocal rank. No similarity
-// threshold anywhere: the store ranks, we never scan. The BM25 leg alone
-// keeps recall alive without an embedder, so claim recall can only ever ADD
-// to a search result.
-
-package runtime
-
-import (
- "context"
- "crypto/md5"
- "encoding/json"
- "fmt"
- "sort"
- "strings"
- "sync"
- "time"
-
- "ragflow/internal/engine"
- "ragflow/internal/engine/types"
- nlp "ragflow/internal/service/nlp"
-)
-
-const (
- // ClaimPrefetchTopN
- ClaimPrefetchTopN = 6
- // ClaimEvidenceChars caps the rendered verbatim quote. 1200 over 600: the quote must be
- // self-sufficient in ONE shot — a thin quote makes the SCA judge the
- // context insufficient, and one extra re-search turn costs a full
- // 10k+-token prompt.
- ClaimEvidenceChars = 1200
- // claimRecallLegFloor is the per-leg store limit floor (max(top_n, 32)).
- claimRecallLegFloor = 32
- // claimPrefetchCacheTTL / claimPrefetchCacheCap: the agent re-issues near-identical
- // queries across turns and each miss costs one KNN round-trip per KB.
- claimPrefetchCacheTTL = 300.0
- claimPrefetchCacheCap = 64
- // EvidenceQuoteChars caps the verbatim quote in the GRAPH fan-out's channel-0 pseudo
- // chunks. The action-session prefetch uses the longer ClaimEvidenceChars (1200) — the two
- // conventions are deliberately different and must not be conflated.
- EvidenceQuoteChars = 400
-)
-
-// claimRowTypes lists the entity_type_kwd values the agentic claim search matches:
-// ("claim",) for every compile kind (page_index's pre-rename fact/conclusion spellings are
-// not searched — a recompile retypes them).
-var _ = []string{"claim", "fact", "conclusion"} // legacy spellings, API-side only
-
-// claimRowTypes lists the entity_type_kwd values that carry a claim: every compiler writes
-// "claim" now; the legacy spellings keep pre-rename rows visible.
-var claimRowTypes = []string{"claim"}
-
-// compilationKwds: the compile_kwd values
-// the knowledge-compilation paths write, used by the has-compilation probe.
-var compilationKwds = []string{"tree", "page_index", "pageindex", "timeline", "dataset_nav"}
-
-// ClaimHit is one recalled claim.
-type ClaimHit struct {
- Name string
- Description string
- Quote string
- DocID string
- ChunkIDs []string
- Score float64
- Rank int
-}
-
-type claimCacheEntry struct {
- at time.Time
- hits []*ClaimHit
-}
-
-var (
- claimRecallMu sync.Mutex
- claimRecallCache = map[string]claimCacheEntry{}
- compilationMu sync.Mutex
- compilationCache = map[string]compilationProbe{}
-)
-
-type compilationProbe struct {
- at time.Time
- has bool
-}
-
-// claimEngine resolves the doc engine: the deps' handle first, the process
-// singleton as fallback (mirrors runSearch's DocEngine resolution).
-func claimEngine(deps SearchDeps) engine.DocEngine {
- if deps.DocEngine != nil {
- return deps.DocEngine
- }
- return engine.Get()
-}
-
-// claimIndexName resolves the tenant index (deps override, then the default).
-func claimIndexName(deps SearchDeps) string {
- if v := strings.TrimSpace(deps.IndexName); v != "" {
- return v
- }
- return fmt.Sprintf("ragflow_%s", deps.TenantID)
-}
-
-// DatasetHasCompilation reports whether any in-scope KB carries compiled rows: one cheap
-// probe per KB set, TTL-cached. Probe failures FAIL OPEN (true) so a glitch never silently
-// disables claim recall.
-func DatasetHasCompilation(ctx context.Context, deps SearchDeps) bool {
- de := claimEngine(deps)
- if de == nil || len(deps.KbIDs) == 0 {
- return false
- }
- key := deps.TenantID + "|" + strings.Join(deps.KbIDs, ",")
- now := time.Now()
-
- compilationMu.Lock()
- if p, ok := compilationCache[key]; ok && now.Sub(p.at).Seconds() < claimPrefetchCacheTTL {
- compilationMu.Unlock()
- return p.has
- }
- compilationMu.Unlock()
-
- has := false
- res, err := de.Search(ctx, &types.SearchRequest{
- IndexNames: []string{claimIndexName(deps)},
- KbIDs: deps.KbIDs,
- Offset: 0,
- Limit: 1,
- SelectFields: []string{"id"},
- Filter: map[string]interface{}{"compile_kwd": compilationKwds},
- })
- if err != nil {
- has = true // fail open
- } else {
- has = res != nil && len(res.Chunks) > 0
- }
-
- compilationMu.Lock()
- compilationCache[key] = compilationProbe{at: now, has: has}
- compilationMu.Unlock()
- return has
-}
-
-// RecallDatasetClaims runs the two-leg hybrid claim recall and returns up to
-// topN fused hits. Empty (never an error) on any failure — the prefetch can
-// only ever add to a search result.
-func RecallDatasetClaims(ctx context.Context, deps SearchDeps, query string, topN int) []*ClaimHit {
- return recallDatasetClaimsFiltered(ctx, deps, query, topN, nil, nil)
-}
-
-// recallDatasetClaimsFiltered is the recall core with optional compile_kwd /
-// entity_type_kwd pinning: the pinning queries ("tree","claim") and ("page_index","claim")
-// run as two separate passes, while the session-level recall passes nil/nil and matches
-// row_types=("claim",) unconditionally.
-func recallDatasetClaimsFiltered(ctx context.Context, deps SearchDeps, query string, topN int, compileKwds, rowTypes []string) []*ClaimHit {
- de := claimEngine(deps)
- if de == nil || deps.TenantID == "" || len(deps.KbIDs) == 0 {
- return nil
- }
- query = strings.TrimSpace(query)
- if query == "" || topN <= 0 {
- return nil
- }
- key := deps.TenantID + "|" + strings.Join(deps.KbIDs, ",") + "|" + strings.Join(compileKwds, ",") + "|" + strings.ToLower(query)
- now := time.Now()
-
- claimRecallMu.Lock()
- if cached, ok := claimRecallCache[key]; ok && now.Sub(cached.at).Seconds() < claimPrefetchCacheTTL {
- claimRecallMu.Unlock()
- return cached.hits
- }
- if len(claimRecallCache) >= claimPrefetchCacheCap {
- claimRecallCache = map[string]claimCacheEntry{}
- }
- claimRecallMu.Unlock()
-
- limit := topN
- if limit < claimRecallLegFloor {
- limit = claimRecallLegFloor
- }
- rowTypesEffective := rowTypes
- if len(rowTypesEffective) == 0 {
- rowTypesEffective = claimRowTypes
- }
- filter := map[string]interface{}{
- "entity_type_kwd": rowTypesEffective,
- "scope_kwd": []string{"doc"},
- }
- if len(compileKwds) > 0 {
- filter["compile_kwd"] = compileKwds
- }
- fields := []string{"content_with_weight", "source_chunk_ids", "doc_id"}
-
- // KNN leg (needs an embedder); BM25 leg always runs.
- var denseLeg, textLeg []*ClaimHit
- if deps.HasEmbedder && deps.Embedder != nil {
- if vecs := mustEncodeQueries(deps, ctx, query); len(vecs) > 0 && len(vecs[0]) > 0 {
- qvec := vecs[0]
- dense := make([]float64, len(qvec))
- for i, v := range qvec {
- dense[i] = float64(v)
- }
- exprs := []interface{}{&types.MatchDenseExpr{
- VectorColumnName: fmt.Sprintf("q_%d_vec", len(qvec)),
- EmbeddingData: dense,
- EmbeddingDataType: "float",
- DistanceType: "cosine",
- TopN: limit,
- }}
- denseLeg = claimSearchLeg(ctx, de, deps, filter, fields, exprs, limit)
- }
- }
- exprs := []interface{}{&types.MatchTextExpr{
- Fields: []string{"content_ltks", "content_sm_ltks"},
- MatchingText: query,
- TopN: limit,
- }}
- textLeg = claimSearchLeg(ctx, de, deps, filter, fields, exprs, limit)
-
- if len(denseLeg) == 0 && len(textLeg) == 0 {
- return nil
- }
- out := rrfFuseClaims(denseLeg, textLeg)
- if len(out) > topN {
- out = out[:topN]
- }
-
- claimRecallMu.Lock()
- claimRecallCache[key] = claimCacheEntry{at: now, hits: out}
- claimRecallMu.Unlock()
- return out
-}
-
-// claimSearchLeg runs one store search and parses the claim hits.
-func claimSearchLeg(ctx context.Context, de engine.DocEngine, deps SearchDeps, filter map[string]interface{}, fields []string, exprs []interface{}, limit int) []*ClaimHit {
- res, err := de.Search(ctx, &types.SearchRequest{
- IndexNames: []string{claimIndexName(deps)},
- KbIDs: deps.KbIDs,
- Offset: 0,
- Limit: limit,
- SelectFields: fields,
- Filter: filter,
- MatchExprs: exprs,
- })
- if err != nil || res == nil {
- return nil
- }
- hits := make([]*ClaimHit, 0, len(res.Chunks))
- for _, row := range res.Chunks {
- if hit := parseClaimHit(row); hit != nil {
- hits = append(hits, hit)
- }
- }
- return hits
-}
-
-// parseClaimHit extracts one claim hit from a store row. Nil when the row is
-// not a usable claim (bad payload, unnamed, or no chunk pointer).
-func parseClaimHit(row map[string]interface{}) *ClaimHit {
- raw, _ := row["content_with_weight"].(string)
- var payload map[string]interface{}
- if err := json.Unmarshal([]byte(raw), &payload); err != nil || payload == nil {
- return nil
- }
- name := strings.TrimSpace(claimStr(payload["name"]))
- if name == "" {
- return nil
- }
- chunkIDs := claimChunkIDs(row["source_chunk_ids"], payload["source_chunk_ids"])
- if len(chunkIDs) == 0 {
- return nil
- }
- quote := ""
- if ev, ok := payload["evidence"].([]interface{}); ok {
- for _, item := range ev {
- m, ok := item.(map[string]interface{})
- if !ok {
- continue
- }
- if q := strings.TrimSpace(claimStr(m["quote"])); q != "" {
- quote = q
- break
- }
- }
- }
- hit := &ClaimHit{
- Name: name,
- Description: strings.TrimSpace(claimStr(payload["description"])),
- Quote: quote,
- DocID: strings.TrimSpace(claimStr(row["doc_id"])),
- ChunkIDs: chunkIDs,
- }
- if v, ok := row["similarity"].(float64); ok {
- hit.Score = v
- }
- return hit
-}
-
-// rrfFuseClaims fuses the retrieval legs by reciprocal rank (k=60): each fused hit keeps
-// its best leg score and gains a 1-based rank.
-func rrfFuseClaims(legs ...[]*ClaimHit) []*ClaimHit {
- const k = 60
- type key struct {
- docID string
- name string
- }
- best := map[key]*ClaimHit{}
- fused := map[key]float64{}
- for _, leg := range legs {
- for rank, hit := range leg {
- hk := key{docID: hit.DocID, name: hit.Name}
- fused[hk] += 1.0 / float64(k+rank+1)
- if cur := best[hk]; cur == nil || hit.Score > cur.Score {
- best[hk] = hit
- }
- }
- }
- out := make([]*ClaimHit, 0, len(best))
- for hk, hit := range best {
- hit.Rank = 0 // set below, after sorting
- _ = fused[hk]
- out = append(out, hit)
- }
- sort.Slice(out, func(i, j int) bool {
- ki := key{docID: out[i].DocID, name: out[i].Name}
- kj := key{docID: out[j].DocID, name: out[j].Name}
- return fused[ki] > fused[kj]
- })
- for i, hit := range out {
- hit.Rank = i + 1
- }
- return out
-}
-
-// ClaimPseudoChunks renders recalled claims as the pool-shaped pseudo chunks the GRAPH
-// fan-out's channel 0 admits: the row leads with the literal "[evidence]" prefix, the quote
-// is a LITERAL quoted span capped at EvidenceQuoteChars (400), and the whole content is
-// capped at 1200. source_chunk_ids ride along so the directional top-up and the slot prefill
-// can find the underlying chunks.
-//
-// This is NOT the action-session prefetch's format — that one renders "[claim #rank]" with a
-// 1200-char quote and is built inline by ClaimPrefetch. The two formats are deliberately
-// distinct.
-func ClaimPseudoChunks(claims []*ClaimHit) []map[string]interface{} {
- out := make([]map[string]interface{}, 0, len(claims))
- for _, c := range claims {
- cid := claimHitID(c)
- content := "[evidence] " + c.Name
- if c.Description != "" && c.Description != c.Name {
- content += " — " + c.Description
- }
- if c.Quote != "" {
- // The cap counts CODE POINTS.
- quote := truncateRunes(c.Quote, EvidenceQuoteChars)
- content += "\nEvidence (verbatim): \"" + quote + "\""
- }
- // The content cap counts CODE POINTS, not bytes.
- content = truncateRunes(content, 1200)
- out = append(out, map[string]interface{}{
- "chunk_id": cid,
- "content_with_weight": content,
- "doc_id": c.DocID,
- "source_chunk_ids": c.ChunkIDs,
- })
- }
- return out
-}
-
-// claimHitID is the pool id: "claim_" + md5(doc_id+":"+name)[:12].
-func claimHitID(c *ClaimHit) string {
- return "claim_" + fmt.Sprintf("%x", md5.Sum([]byte(c.DocID+":"+c.Name)))[:12]
-}
-
-// ClaimPrefetch runs the gated prefetch for one corpus search: the has-compilation gate,
-// the recall, and the exclusive passage build. ok is false when the caller must fall through
-// to the plain chunk search.
-func ClaimPrefetch(ctx context.Context, deps SearchDeps, query string, seen map[string]bool) ([]map[string]any, []string, []map[string]interface{}, bool) {
- if strings.TrimSpace(query) == "" {
- return nil, nil, nil, false
- }
- if !DatasetHasCompilation(ctx, deps) {
- return nil, nil, nil, false
- }
- claims := RecallDatasetClaims(ctx, deps, query, ClaimPrefetchTopN)
- if len(claims) == 0 {
- return nil, nil, nil, false
- }
- var payload []map[string]any
- var ids []string
- var pseudo []map[string]interface{}
- for _, c := range claims {
- cid := claimHitID(c)
- if seen[cid] {
- continue
- }
- content := fmt.Sprintf("[claim #%d] %s", c.Rank, c.Name)
- if c.Description != "" && c.Description != c.Name {
- content += " — " + c.Description
- }
- if c.Quote != "" {
- // The cap counts CODE POINTS.
- quote := truncateRunes(c.Quote, ClaimEvidenceChars)
- content += fmt.Sprintf("\nEvidence (verbatim): %q", quote)
- }
- // The content cap counts CODE POINTS, not bytes.
- content = truncateRunes(content, 1200)
- docID := c.DocID
- payload = append(payload, map[string]any{"id": cid, "content": content, "doc_id": docID})
- ids = append(ids, cid)
- pseudo = append(pseudo, map[string]interface{}{
- "chunk_id": cid,
- "content_with_weight": content,
- "doc_id": docID,
- "source_chunk_ids": c.ChunkIDs,
- })
- }
- if len(payload) == 0 {
- return nil, nil, nil, false
- }
- return payload, ids, pseudo, true
-}
-
-// claimChunkIDs merges the row column and the payload's own ids.
-func claimChunkIDs(values ...interface{}) []string {
- for _, v := range values {
- switch x := v.(type) {
- case []string:
- if out := claimTrimIDs(x); len(out) > 0 {
- return out
- }
- case []interface{}:
- out := make([]string, 0, len(x))
- for _, item := range x {
- if s := strings.TrimSpace(claimStr(item)); s != "" {
- out = append(out, s)
- }
- }
- if len(out) > 0 {
- return out
- }
- case string:
- if s := strings.TrimSpace(x); s != "" {
- return []string{s}
- }
- }
- }
- return nil
-}
-
-func claimTrimIDs(ids []string) []string {
- out := make([]string, 0, len(ids))
- for _, id := range ids {
- if s := strings.TrimSpace(id); s != "" {
- out = append(out, s)
- }
- }
- return out
-}
-
-// claimStr coerces a JSON value to its trimmed string form.
-func claimStr(v interface{}) string {
- switch x := v.(type) {
- case string:
- return x
- case fmt.Stringer:
- return x.String()
- default:
- if v == nil {
- return ""
- }
- return fmt.Sprintf("%v", v)
- }
-}
-
-// Document-level claim recall.
-
-// DocClaimHit is one claim recalled for a SINGLE document: the source chunk pointer, the
-// fused rank/score, the statement and its verbatim evidence ([{"quote": ...}] when the claim
-// carries one, else empty).
-type DocClaimHit struct {
- ChunkID string
- Score float64
- Rank int
- Name string
- Description string
- Evidence []map[string]any
-}
-
-// evidenceRowTypes: the entity_type_kwd
-// values carrying evidence for the given compile kinds. Every compiler writes
-// its evidence rows as "claim", so any known kind yields ["claim"]; with no
-// kinds the drill searches ["claim"] outright. Unknown kinds contribute
-// nothing (an unknown kind matches no row type).
-func evidenceRowTypes(kinds []string) []string {
- if len(kinds) == 0 {
- return []string{"claim"}
- }
- for _, k := range kinds {
- switch k {
- case "tree", "raptor", "page_index":
- return []string{"claim"}
- }
- }
- return nil
-}
-
-// docClaimCacheEntry memoizes one (doc_id, query) claim recall. The drill calls
-// it once per routed doc and the model re-issues navigate_structure, so the
-// same pair recurs within a session; each miss costs two store round-trips.
-type docClaimCacheEntry struct {
- at time.Time
- hits []DocClaimHit
-}
-
-var (
- docClaimMu sync.Mutex
- docClaimCache = map[string]docClaimCacheEntry{}
-)
-
-// RecallDocClaimHits runs the two-leg hybrid claim recall for ONE document and
-// returns up to topN fused hits:
-// a KNN leg over the claim rows' q_<dim>_vec (when qvec is available) and a
-// BM25 leg over content_ltks/content_sm_ltks, fused by reciprocal rank. No
-// similarity threshold: the store ranks, the top-N ARE the hit set. Empty —
-// never an error — when the document has no compiled claims or the store
-// fails; a failed recall simply leaves the drill-down in charge.
-func RecallDocClaimHits(ctx context.Context, deps SearchDeps, query, docID string, kinds []string, qvec []float64, topN int) []DocClaimHit {
- if docID == "" || topN <= 0 {
- return nil
- }
- de := claimEngine(deps)
- if de == nil || deps.TenantID == "" {
- return nil
- }
- query = strings.TrimSpace(query)
- key := docID + "|" + strings.ToLower(query)
- now := time.Now()
-
- docClaimMu.Lock()
- if cached, ok := docClaimCache[key]; ok && now.Sub(cached.at).Seconds() < claimPrefetchCacheTTL {
- docClaimMu.Unlock()
- return cached.hits
- }
- if len(docClaimCache) >= claimPrefetchCacheCap {
- docClaimCache = map[string]docClaimCacheEntry{}
- }
- docClaimMu.Unlock()
-
- limit := topN
- if limit < claimRecallLegFloor {
- limit = claimRecallLegFloor
- }
- rowTypes := evidenceRowTypes(kinds)
- filter := map[string]interface{}{
- "doc_id": []string{docID},
- "entity_type_kwd": rowTypes,
- // Doc-scope rows only: the Build button's KB-wide merged rows carry a
- // doc_id too, and without this filter they would answer for a single
- // document. Mirrors the dataset-level claim leg.
- "scope_kwd": []string{"doc"},
- }
- if len(kinds) > 0 {
- sorted := make([]string, 0, len(kinds))
- for _, k := range kinds {
- if k = strings.TrimSpace(k); k != "" {
- sorted = append(sorted, k)
- }
- }
- sort.Strings(sorted)
- filter["compile_kwd"] = sorted
- }
- fields := []string{"content_with_weight", "source_chunk_ids", "entity_type_kwd", "compile_kwd"}
-
- // KNN leg (needs a query vector); BM25 leg always runs.
- var denseLeg, textLeg []*ClaimHit
- if len(qvec) > 0 {
- exprs := []interface{}{&types.MatchDenseExpr{
- VectorColumnName: fmt.Sprintf("q_%d_vec", len(qvec)),
- EmbeddingData: qvec,
- EmbeddingDataType: "float",
- DistanceType: "cosine",
- TopN: limit,
- }}
- denseLeg = claimSearchLeg(ctx, de, deps, filter, fields, exprs, limit)
- }
- exprs := []interface{}{&types.MatchTextExpr{
- Fields: []string{"content_ltks", "content_sm_ltks"},
- MatchingText: query,
- TopN: limit,
- }}
- textLeg = claimSearchLeg(ctx, de, deps, filter, fields, exprs, limit)
-
- if len(denseLeg) == 0 && len(textLeg) == 0 {
- return nil
- }
- fused := rrfFuseClaims(denseLeg, textLeg)
- if len(fused) > topN {
- fused = fused[:topN]
- }
- out := make([]DocClaimHit, 0, len(fused))
- for _, h := range fused {
- hit := DocClaimHit{
- ChunkID: "",
- Score: h.Score,
- Rank: h.Rank,
- Name: h.Name,
- Description: h.Description,
- Evidence: []map[string]any{},
- }
- if len(h.ChunkIDs) > 0 {
- hit.ChunkID = h.ChunkIDs[0]
- }
- // Fall back to the name when the description is empty.
- if hit.Description == "" {
- hit.Description = h.Name
- }
- if h.Quote != "" {
- hit.Evidence = []map[string]any{{"quote": h.Quote}}
- }
- out = append(out, hit)
- }
-
- docClaimMu.Lock()
- docClaimCache[key] = docClaimCacheEntry{at: now, hits: out}
- docClaimMu.Unlock()
- return out
-}
-
-// docClaimQuote returns the first non-empty verbatim quote of a hit's evidence
-// list.
-func docClaimQuote(h DocClaimHit) string {
- for _, ev := range h.Evidence {
- if q, ok := ev["quote"].(string); ok {
- if s := strings.TrimSpace(q); s != "" {
- return s
- }
- }
- }
- return ""
-}
-
-// publishClaimHits mirrors the rendered claims of ONE document into the shared evidence
-// pool: the SCA, the slot prefill and the final compose all read ONLY the pool, so a claim
-// that
-// already states the fact must land there. Pure addition: no retrieval is
-// suppressed, entries dedup by the SAME "claim_"+md5(doc_id:name) id the
-// session prefetch writes, so a claim found by either path is one entry.
-// Returns how many NEW entries the pool gained.
-func publishClaimHits(deps SearchDeps, hits []DocClaimHit, docID string) int {
- if deps.KB == nil || len(hits) == 0 {
- return 0
- }
- added := 0
- deps.KB.Admit(func(p *PoolAdmitter) {
- for _, h := range hits {
- name := strings.TrimSpace(h.Name)
- if name == "" {
- continue
- }
- cid := claimHitID(&ClaimHit{DocID: docID, Name: name})
- rank := "?"
- if h.Rank > 0 {
- rank = fmt.Sprint(h.Rank)
- }
- content := fmt.Sprintf("[claim #%s] %s", rank, name)
- if desc := strings.TrimSpace(h.Description); desc != "" && desc != name {
- content += " — " + desc
- }
- if quote := docClaimQuote(h); quote != "" {
- // The cap counts CODE POINTS.
- quote := truncateRunes(docClaimQuote(h), ClaimEvidenceChars)
- content += "\nEvidence (verbatim): \"" + quote + "\""
- }
- // The content cap counts CODE POINTS.
- content = truncateRunes(content, 1200)
- src := []string{}
- if h.ChunkID != "" {
- src = []string{h.ChunkID}
- }
- if p.Add(map[string]interface{}{
- "chunk_id": cid,
- "content_with_weight": content,
- "doc_id": docID,
- "source_chunk_ids": src,
- }) {
- added++
- }
- }
- })
- if added > 0 {
- _LOG.Printf("[navigate_structure] published %d claim(s) to the evidence pool (doc=%s)", added, docID)
- }
- return added
-}
-
-// LoadChunksForIDs fetches source chunks by id, any owning document. Zero LLM, zero recall
-// — a directed fetch used by the fan-out evidence top-up to pull the passages an admitted
-// claim row cites. Empty on any failure: best effort by contract.
-func LoadChunksForIDs(ctx context.Context, deps SearchDeps, ids []string) []map[string]any {
- ids = claimTrimIDs(ids)
- if len(ids) == 0 {
- return nil
- }
- de := claimEngine(deps)
- if de == nil {
- return nil
- }
- res, err := de.Search(ctx, &types.SearchRequest{
- IndexNames: []string{claimIndexName(deps)},
- KbIDs: deps.KbIDs,
- Offset: 0,
- Limit: len(ids),
- SelectFields: []string{"content_with_weight", "docnm_kwd", "doc_id"},
- Filter: map[string]interface{}{"id": ids},
- })
- if err != nil || res == nil {
- return nil
- }
- out := make([]map[string]any, 0, len(res.Chunks))
- for _, row := range res.Chunks {
- out = append(out, map[string]any{
- "chunk_id": row["id"],
- "content_with_weight": row["content_with_weight"],
- "docnm_kwd": row["docnm_kwd"],
- "doc_id": row["doc_id"],
- })
- }
- return out
-}
-
-// mustEncodeQueries embeds one search query: query-side encoding when the
-// embedder is asymmetric (NavQueryEmbedder for Cohere/Voyage/Jina/NVIDIA), plain Encode
-// otherwise. Returns nil on
-// failure — the BM25 leg alone keeps claim recall alive.
-func mustEncodeQueries(deps SearchDeps, ctx context.Context, query string) [][]float32 {
- if deps.Embedder == nil {
- return nil
- }
- if qe, ok := deps.Embedder.(nlp.NavQueryEmbedder); ok {
- vecs, err := qe.EncodeQueries(ctx, deps.TenantID, []string{query})
- if err != nil {
- return nil
- }
- return vecs
- }
- vecs, err := deps.Embedder.Encode(ctx, deps.TenantID, []string{query})
- if err != nil {
- return nil
- }
- return vecs
-}
diff --git a/internal/rag/agentic-rag/runtime/tool_claims_test.go b/internal/rag/agentic-rag/runtime/tool_claims_test.go
deleted file mode 100644
index 0b288811d8..0000000000
--- a/internal/rag/agentic-rag/runtime/tool_claims_test.go
+++ /dev/null
@@ -1,392 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "encoding/json"
- "reflect"
- "strings"
- "testing"
-
- "ragflow/internal/engine"
- "ragflow/internal/engine/types"
-)
-
-// recordingClaimEngine implements engine.DocEngine by serving Search from a
-// fixed row set and recording every request; the rest of the interface is
-// promoted from a nil field and never called by these tests.
-type recordingClaimEngine struct {
- engine.DocEngine
- requests []*types.SearchRequest
- rows []map[string]interface{}
-}
-
-func (e *recordingClaimEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
- e.requests = append(e.requests, req)
- return &types.SearchResult{Chunks: e.rows}, nil
-}
-
-// claimRow builds one claim store row in the shape the compilers write.
-func claimRow(docID, name, description, quote string, similarity float64, chunkIDs []string) map[string]interface{} {
- payload := map[string]any{"name": name, "type": "claim"}
- if description != "" {
- payload["description"] = description
- }
- if quote != "" {
- payload["evidence"] = []any{map[string]any{"quote": quote}}
- }
- if chunkIDs != nil {
- payload["source_chunk_ids"] = chunkIDs
- }
- raw, _ := json.Marshal(payload)
- return map[string]interface{}{
- "content_with_weight": string(raw),
- "source_chunk_ids": chunkIDs,
- "doc_id": docID,
- "similarity": similarity,
- }
-}
-
-func resetClaimCaches() {
- claimRecallMu.Lock()
- claimRecallCache = map[string]claimCacheEntry{}
- claimRecallMu.Unlock()
- docClaimMu.Lock()
- docClaimCache = map[string]docClaimCacheEntry{}
- docClaimMu.Unlock()
- compilationMu.Lock()
- compilationCache = map[string]compilationProbe{}
- compilationMu.Unlock()
-}
-
-// TestRecallDocClaimHitsFiltersAndMaps pins the store contract and the output mapping of the
-// per-document claim leg: doc-scope + claim-row filters, sorted compile_kwd when kinds are
-// given, and hits shaped as {chunk_id, score, rank, name, description-or-name, evidence}.
-func TestRecallDocClaimHitsFiltersAndMaps(t *testing.T) {
- resetClaimCaches()
- de := &recordingClaimEngine{rows: []map[string]interface{}{
- claimRow("doc-1", "The tower opened in 1889", "", "The tower opened in 1889.", 0.9, []string{"c1", "c2"}),
- }}
- deps := SearchDeps{TenantID: "tenant-1", IndexName: "idx", DocEngine: de, KbIDs: []string{"kb-1"}}
-
- hits := RecallDocClaimHits(context.Background(), deps, "When did the tower open?", "doc-1", []string{"tree", "raptor"}, nil, 8)
- if len(hits) != 1 {
- t.Fatalf("hits = %d, want 1", len(hits))
- }
- h := hits[0]
- if h.ChunkID != "c1" {
- t.Errorf("ChunkID = %q, want the first source chunk c1", h.ChunkID)
- }
- if h.Score != 0.9 || h.Rank != 1 {
- t.Errorf("Score/Rank = %v/%d, want 0.9/1", h.Score, h.Rank)
- }
- if h.Name != "The tower opened in 1889" {
- t.Errorf("Name = %q", h.Name)
- }
- // The description falls back to the name.
- if h.Description != h.Name {
- t.Errorf("Description = %q, want the name fallback", h.Description)
- }
- wantEv := []map[string]any{{"quote": "The tower opened in 1889."}}
- if !reflect.DeepEqual(h.Evidence, wantEv) {
- t.Errorf("Evidence = %v, want %v", h.Evidence, wantEv)
- }
-
- // Store contract: one BM25 leg (no qvec), doc-scope claim filters, sorted
- // compile_kwd, the wide field list, limit floored at 32.
- if len(de.requests) != 1 {
- t.Fatalf("store calls = %d, want 1 (BM25 leg only)", len(de.requests))
- }
- req := de.requests[0]
- wantFilter := map[string]interface{}{
- "doc_id": []string{"doc-1"},
- "entity_type_kwd": []string{"claim"},
- "scope_kwd": []string{"doc"},
- "compile_kwd": []string{"raptor", "tree"},
- }
- if !reflect.DeepEqual(req.Filter, wantFilter) {
- t.Errorf("Filter = %v, want %v", req.Filter, wantFilter)
- }
- for _, f := range []string{"content_with_weight", "source_chunk_ids", "entity_type_kwd", "compile_kwd"} {
- if !sliceContains(req.SelectFields, f) {
- t.Errorf("SelectFields missing %q: %v", f, req.SelectFields)
- }
- }
- if req.Limit != claimRecallLegFloor {
- t.Errorf("Limit = %d, want the %d floor", req.Limit, claimRecallLegFloor)
- }
- if len(req.MatchExprs) != 1 {
- t.Fatalf("MatchExprs = %d, want the single BM25 leg", len(req.MatchExprs))
- }
- text, ok := req.MatchExprs[0].(*types.MatchTextExpr)
- if !ok {
- t.Fatalf("MatchExprs[0] is %T, want *types.MatchTextExpr", req.MatchExprs[0])
- }
- if text.MatchingText != "When did the tower open?" {
- t.Errorf("BM25 leg text = %q, want the raw query", text.MatchingText)
- }
-}
-
-// TestRecallDocClaimHitsDenseLeg pins the KNN leg: a query vector turns into
-// one MatchDenseExpr over q_<dim>_vec ahead of the BM25 leg.
-func TestRecallDocClaimHitsDenseLeg(t *testing.T) {
- resetClaimCaches()
- de := &recordingClaimEngine{}
- deps := SearchDeps{TenantID: "tenant-1", DocEngine: de}
- qvec := []float64{0.1, 0.2, 0.3}
- RecallDocClaimHits(context.Background(), deps, "q", "doc-1", nil, qvec, 8)
- if len(de.requests) != 2 {
- t.Fatalf("store calls = %d, want 2 (KNN + BM25 legs)", len(de.requests))
- }
- dense, ok := de.requests[0].MatchExprs[0].(*types.MatchDenseExpr)
- if !ok {
- t.Fatalf("first leg is %T, want *types.MatchDenseExpr", de.requests[0].MatchExprs[0])
- }
- if dense.VectorColumnName != "q_3_vec" {
- t.Errorf("vector column = %q, want q_3_vec", dense.VectorColumnName)
- }
- if !reflect.DeepEqual(dense.EmbeddingData, qvec) {
- t.Errorf("embedding data = %v, want %v", dense.EmbeddingData, qvec)
- }
- // No kinds: no compile_kwd filter, row types default to claim.
- if _, ok := de.requests[0].Filter["compile_kwd"]; ok {
- t.Errorf("compile_kwd filter present without kinds: %v", de.requests[0].Filter)
- }
- if !reflect.DeepEqual(de.requests[0].Filter["entity_type_kwd"], []string{"claim"}) {
- t.Errorf("entity_type_kwd = %v, want [claim]", de.requests[0].Filter["entity_type_kwd"])
- }
-}
-
-// TestRecallDocClaimHitsMemo pins the (doc_id, query) memo: the drill re-issues
-// navigate_structure, so the same pair must not cost two more store
-// round-trips, while a different query or doc must miss. Empty results are NOT cached (the
-// return happens before the cache write).
-func TestRecallDocClaimHitsMemo(t *testing.T) {
- resetClaimCaches()
- de := &recordingClaimEngine{rows: []map[string]interface{}{
- claimRow("doc-1", "claim a", "", "", 0.5, []string{"c1"}),
- }}
- deps := SearchDeps{TenantID: "tenant-1", DocEngine: de}
- ctx := context.Background()
-
- RecallDocClaimHits(ctx, deps, "same query", "doc-1", nil, nil, 8)
- RecallDocClaimHits(ctx, deps, "Same Query ", "doc-1", nil, nil, 8)
- if len(de.requests) != 1 {
- t.Fatalf("store calls after repeat = %d, want 1 (memo hit)", len(de.requests))
- }
- RecallDocClaimHits(ctx, deps, "other query", "doc-1", nil, nil, 8)
- if len(de.requests) != 2 {
- t.Fatalf("store calls after new query = %d, want 2", len(de.requests))
- }
- RecallDocClaimHits(ctx, deps, "same query", "doc-2", nil, nil, 8)
- if len(de.requests) != 3 {
- t.Fatalf("store calls after new doc = %d, want 3", len(de.requests))
- }
-
- empty := &recordingClaimEngine{}
- emptyDeps := SearchDeps{TenantID: "tenant-1", DocEngine: empty}
- RecallDocClaimHits(ctx, emptyDeps, "nothing here", "doc-9", nil, nil, 8)
- RecallDocClaimHits(ctx, emptyDeps, "nothing here", "doc-9", nil, nil, 8)
- if len(empty.requests) != 2 {
- t.Fatalf("empty result must not be memoized; store calls = %d, want 2", len(empty.requests))
- }
-}
-
-// TestClaimAggRouterClaimsDecide pins the claim-first routing order: documents
-// ranked by the damped claim DocScore with the best claim similarity as the
-// reported tie-break, nav summaries attached, and the chunk_agg fallback NEVER
-// consulted while the claim leg hits.
-func TestClaimAggRouterClaimsDecide(t *testing.T) {
- resetClaimCaches()
- de := &recordingClaimEngine{rows: []map[string]interface{}{
- // doc-a: one strong claim -> DocScore 0.9/sqrt(2) ≈ 0.636.
- claimRow("doc-a", "claim a", "", "quote a", 0.9, []string{"c1"}),
- // doc-b: two weaker claims -> DocScore 0.9/sqrt(3) ≈ 0.520.
- claimRow("doc-b", "claim b1", "", "", 0.5, []string{"c2"}),
- claimRow("doc-b", "claim b2", "", "", 0.4, []string{"c3"}),
- }}
- fallback := &fallbackRouter{}
- summaries := func(_ context.Context, _, _ string, docIDs []string) map[string]string {
- out := map[string]string{}
- for _, d := range docIDs {
- out[d] = "summary of " + d
- }
- return out
- }
- r := &ClaimAggRouter{
- Deps: SearchDeps{TenantID: "tenant-1", DocEngine: de},
- Fallback: fallback,
- Summarize: summaries,
- }
- routed, err := r.Route(context.Background(), "tenant-1", "kb-1", "tower height", nil, 8)
- if err != nil {
- t.Fatalf("Route: %v", err)
- }
- if fallback.called {
- t.Fatal("chunk_agg fallback must not run while the claim leg hits")
- }
- if len(routed) != 2 {
- t.Fatalf("routed = %v, want 2 docs", routed)
- }
- if routed[0][0] != "doc-a" || routed[1][0] != "doc-b" {
- t.Errorf("order = %v, want [doc-a doc-b] (damped DocScore)", routed)
- }
- if routed[0][1] != "summary of doc-a" {
- t.Errorf("summary = %q, want the nav_doc summary", routed[0][1])
- }
-}
-
-// TestClaimAggRouterDocScope pins the doc_scope filter: out-of-scope documents never surface,
-// and a scope that empties the claim leg falls back to the chunk leg (the store condition is
-// filtered, so a scoped miss is an empty leg, not a routing verdict).
-func TestClaimAggRouterDocScope(t *testing.T) {
- resetClaimCaches()
- de := &recordingClaimEngine{rows: []map[string]interface{}{
- claimRow("doc-a", "claim a", "", "", 0.9, []string{"c1"}),
- claimRow("doc-b", "claim b", "", "", 0.8, []string{"c2"}),
- }}
- fallback := &fallbackRouter{next: [][2]string{{"doc-fallback", ""}}}
- r := &ClaimAggRouter{
- Deps: SearchDeps{TenantID: "tenant-1", DocEngine: de},
- Fallback: fallback,
- }
- routed, err := r.Route(context.Background(), "tenant-1", "kb-1", "q", []string{"doc-b"}, 8)
- if err != nil {
- t.Fatalf("Route: %v", err)
- }
- if len(routed) != 1 || routed[0][0] != "doc-b" {
- t.Errorf("routed = %v, want only doc-b", routed)
- }
- if fallback.called {
- t.Fatal("fallback must not run when the scoped claim leg still hits")
- }
-
- fallback2 := &fallbackRouter{next: [][2]string{{"doc-fallback", ""}}}
- r2 := &ClaimAggRouter{Deps: SearchDeps{TenantID: "tenant-1", DocEngine: de}, Fallback: fallback2}
- routed2, err := r2.Route(context.Background(), "tenant-1", "kb-1", "q", []string{"doc-elsewhere"}, 8)
- if err != nil {
- t.Fatalf("Route: %v", err)
- }
- if !fallback2.called || len(routed2) != 1 || routed2[0][0] != "doc-fallback" {
- t.Errorf("emptied scoped leg must fall back, got routed=%v called=%v", routed2, fallback2.called)
- }
-}
-
-// TestClaimAggRouterFallsBackOnEmpty pins the empty-leg contract: no claim
-// rows (a raptor-less or pre-claim dataset) is a legitimate empty leg, so the
-// chunk_agg fallback decides.
-func TestClaimAggRouterFallsBackOnEmpty(t *testing.T) {
- resetClaimCaches()
- de := &recordingClaimEngine{}
- fallback := &fallbackRouter{next: [][2]string{{"doc-chunk", "chunk summary"}}}
- r := &ClaimAggRouter{
- Deps: SearchDeps{TenantID: "tenant-1", DocEngine: de},
- Fallback: fallback,
- }
- routed, err := r.Route(context.Background(), "tenant-1", "kb-1", "q", nil, 8)
- if err != nil {
- t.Fatalf("Route: %v", err)
- }
- if !fallback.called {
- t.Fatal("empty claim leg must consult the chunk_agg fallback")
- }
- if len(routed) != 1 || routed[0][0] != "doc-chunk" {
- t.Errorf("routed = %v, want the fallback's result", routed)
- }
-}
-
-// fallbackRouter stands in for the chunk_agg router.
-type fallbackRouter struct {
- called bool
- next [][2]string
-}
-
-func (r *fallbackRouter) Route(_ context.Context, _, _, _ string, _ []string, _ int) ([][2]string, error) {
- r.called = true
- return r.next, nil
-}
-
-// TestPublishClaimHits pins the evidence-pool publish: one pseudo chunk per claim under the
-// SAME "claim_"+md5(doc:name) id the session prefetch writes, verbatim evidence capped at
-// ClaimEvidenceChars, the claim's own chunk pointer as source_chunk_ids, deduped against the
-// live pool.
-func TestPublishClaimHits(t *testing.T) {
- kb := &Kbinfos{}
- deps := SearchDeps{KB: kb}
- hits := []DocClaimHit{
- {ChunkID: "c1", Rank: 2, Name: "Claim A", Description: "Claim A", Evidence: []map[string]any{{"quote": strings.Repeat("q", 2000)}}},
- {ChunkID: "", Rank: 3, Name: "Claim B", Description: "a different description"},
- }
- if got := publishClaimHits(deps, hits, "doc-1"); got != 2 {
- t.Fatalf("published = %d, want 2", got)
- }
- if got := publishClaimHits(deps, hits, "doc-1"); got != 0 {
- t.Fatalf("re-publish must dedup to 0, got %d", got)
- }
- if len(kb.Chunks) != 2 {
- t.Fatalf("pool = %d entries, want 2", len(kb.Chunks))
- }
- a := kb.Chunks[0]
- if a["chunk_id"] != claimHitID(&ClaimHit{DocID: "doc-1", Name: "Claim A"}) {
- t.Errorf("chunk_id = %v, want the shared claim id scheme", a["chunk_id"])
- }
- content, _ := a["content_with_weight"].(string)
- if !strings.HasPrefix(content, "[claim #2] Claim A") {
- t.Errorf("content = %q, want the [claim #rank] prefix", content)
- }
- if !strings.Contains(content, "\nEvidence (verbatim): \"") {
- t.Errorf("content missing the verbatim evidence line:\n%s", content)
- }
- if len(content) > 1200 {
- t.Errorf("content length = %d, want the 1200 cap", len(content))
- }
- if !reflect.DeepEqual(a["source_chunk_ids"], []string{"c1"}) {
- t.Errorf("source_chunk_ids = %v, want [c1]", a["source_chunk_ids"])
- }
- // Rank-3 claim with no chunk pointer and no quote: empty source list,
- // description rendered when it differs from the name.
- b := kb.Chunks[1]
- if !reflect.DeepEqual(b["source_chunk_ids"], []string{}) {
- t.Errorf("source_chunk_ids = %v, want []", b["source_chunk_ids"])
- }
- if !strings.Contains(b["content_with_weight"].(string), "Claim B — a different description") {
- t.Errorf("content = %q, want the description dash form", b["content_with_weight"])
- }
-}
-
-// TestLoadChunksForIDs pins the directed source-chunk fetch: an id-filtered store read that
-// maps rows back to pool-shaped chunks.
-func TestLoadChunksForIDs(t *testing.T) {
- de := &recordingClaimEngine{rows: []map[string]interface{}{
- {"id": "c1", "content_with_weight": "full text one", "doc_id": "doc-1", "docnm_kwd": "a.pdf"},
- }}
- deps := SearchDeps{TenantID: "tenant-1", IndexName: "idx", DocEngine: de, KbIDs: []string{"kb-1"}}
- out := LoadChunksForIDs(context.Background(), deps, []string{"c1", " "})
- if len(out) != 1 {
- t.Fatalf("fetched = %d, want 1", len(out))
- }
- if out[0]["chunk_id"] != "c1" || out[0]["content_with_weight"] != "full text one" {
- t.Errorf("fetched row = %v", out[0])
- }
- if len(de.requests) != 1 || !reflect.DeepEqual(de.requests[0].Filter["id"], []string{"c1"}) {
- t.Errorf("request = %+v, want one id-filtered read for [c1]", de.requests[0])
- }
- if LoadChunksForIDs(context.Background(), deps, nil) != nil {
- t.Error("empty ids must fetch nothing")
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/tool_compiled_expansion.go b/internal/rag/agentic-rag/runtime/tool_compiled_expansion.go
deleted file mode 100644
index e81a47d488..0000000000
--- a/internal/rag/agentic-rag/runtime/tool_compiled_expansion.go
+++ /dev/null
@@ -1,896 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "crypto/md5"
- "encoding/json"
- "fmt"
- "math"
- enginetypes "ragflow/internal/engine/types"
- "sort"
- "strings"
-
- "ragflow/internal/entity"
- "ragflow/internal/tokenizer"
-)
-
-// Dense-match parameters for compiled-row search. When a SearchCompiled implementation
-// answers matchText with a dense (ANN) leg it MUST pass num_candidates =
-// max(topN, vectorNumCandidates) and similarity = vectorSimilarity as named options: a
-// positional mix-up here (0.1 read as num_candidates/ef_search) collapses the HNSW
-// candidate list and silently destroys recall. Dense options are expressed as a named map
-// (engine MatchDenseExpr.ExtraOptions), so the pitfall is avoided as long as the
-// implementation uses these constants.
-const (
- vectorNumCandidates = 256
- vectorSimilarity = 0.1
-)
-
-// CompiledStore is the seam over the doc-store compiled-structure rows that the
-// runtime must not own directly (it would pull in the retrieval backend and
-// break unit-tier `go test ./...`). It mirrors the three primitives
-// compiled_expansion.py reaches for:
-//
-// - settings.docStoreConn.search -> SearchCompiled
-// - source_chunk_ids loading -> LoadChunks
-// - settings.retriever.get_vector -> Vectorize (kept for callers that need it)
-//
-// The production implementation lives in the integration layer and is injected
-// through SearchDeps.Expand (see NewCompiledExpander).
-type CompiledStore interface {
- // SearchCompiled returns compiled rows (entity / relation / synthesis page)
- // matching the given filters and free-text match, scoped to (kbID, tenantID,
- // docIDs). docIDs is applied to the doc store as the scope (as a "doc_id" filter for
- // entity/relation rows and "source_doc_ids" for synthesis pages; the exact key is the
- // implementation's concern). A row
- // carries at least content_with_weight / source_chunk_ids / doc_id / docnm_kwd
- // / from_entity_kwd / to_entity_kwd / name_kwd.
- //
- // filters keys mirror the condition dict: "knowledge_graph_kwd"
- // (kind: entity/relation), "compile_kwd" (tree/wiki_page/artifact_page/essence),
- // "compilation_template_kind_kwd", plus exact-list lookups ("name_kwd",
- // "from_entity_kwd", "to_entity_kwd") and "available_int". Each value is an
- // OR-list; filters are AND-ed.
- //
- // When matchText is answered with a dense (ANN) leg the implementation must
- // set its num_candidates = max(topN, vectorNumCandidates) and its similarity
- // threshold to vectorSimilarity (see the consts above).
- SearchCompiled(ctx context.Context, kbID, tenantID string, docIDs []string, filters map[string][]string, matchText string, topN int) ([]map[string]any, error)
- // LoadChunks returns the raw chunks for the given source_chunk_ids.
- LoadChunks(ctx context.Context, kbID, tenantID string, chunkIDs []string) ([]map[string]any, error)
- // Vectorize returns the embedding of text. Kept for the interface contract;
- // a faithful port assigns chunks a similarity and blends them by that field,
- // so expansion itself does not call Vectorize.
- Vectorize(ctx context.Context, text string) ([]float64, error)
-}
-
-// compiledScope is one (kb_id, tenant_id, doc_ids) triple — exactly the list produced for
-// a (possibly empty) doc_scope.
-type compiledScope struct {
- kbID string
- tenantID string
- docIDs []string
-}
-
-// Template-kind labels for the entity 1-hop strategies, in the order they are applied.
-var compiledEntityKinds = []struct {
- label string
- kind string
-}{
- {"knowledge_graph", "knowledge_graph"},
- {"mind_map", "mind_map"},
- {"timeline", "timeline"},
- {"page_index", "page_index"},
-}
-
-// Synthesis compile keywords, in the order they are applied.
-var compiledSynthesisKinds = []struct {
- label string
- kind string
-}{
- {"wiki_page", "wiki_page"},
- {"artifact_page", "artifact_page"},
- {"essence", "essence"},
-}
-
-// CompiledScopeConfig carries the scope-resolution inputs of one search session: the bound
-// datasets (each with its OWN owner tenant), the request tenant used as a fallback, and the
-// optional scoped-doc-ids / doc→(kb,tenant) hooks.
-type CompiledScopeConfig struct {
- // DatasetIDs are the bound dataset ids.
- DatasetIDs []string
- // TenantID is the request tenant, used only when a bound dataset carries no
- // tenant id of its own.
- TenantID string
- // KBs are the resolved dataset objects; their (ID, TenantID) pairs give each
- // bound dataset its own compiled-row index scope.
- KBs []*entity.Knowledgebase
- // DocTenantResolver resolves a document id to its owning (kb, tenant) so a
- // doc scope is grouped by real owner. Nil keeps the fallback: each bound dataset
- // searched with the whole doc scope.
- DocTenantResolver DocTenantResolver
-}
-
-// compiledExpander is the real implementation of CompiledExpander. It ports
-// tools/compiled_expansion.py::_expand_with_compiled and its strategy helpers,
-// with the doc-store side delegated to CompiledStore.
-type compiledExpander struct {
- store CompiledStore
- cfg CompiledScopeConfig
-}
-
-// NewCompiledExpander builds a CompiledExpander for one search session. A nil
-// store, no bound dataset, or no fallback tenant yields a nil expander
-// (expansion disabled), matching SearchDeps.Expand == nil.
-func NewCompiledExpander(store CompiledStore, cfg CompiledScopeConfig) CompiledExpander {
- if store == nil || len(cfg.DatasetIDs) == 0 || strings.TrimSpace(cfg.TenantID) == "" {
- return nil
- }
- return &compiledExpander{store: store, cfg: cfg}
-}
-
-// Expand: a zero-LLM enrichment that layers the dataset's compiled products on top of the
-// retrieved chunks. For each bound KB it runs the per-template 1-hop entity expansion
-// (knowledge_graph / mind_map / timeline / page_index), the tree structure graph
-// (compile_kwd="tree"), and the three synthesis-page expansions (wiki_page /
-// artifact_page / essence) — each capped at max_chunks=5 — then re-sorts the whole chunk
-// pool by similarity so compiled chunks blend by relevance with the regular ones.
-//
-// A doc-tenant resolution failure aborts the expansion and is returned rather than
-// swallowed: the caller reports it (tool_search.go, "compiled expansion failed") while the
-// already-retrieved chunks stay in place.
-func (e *compiledExpander) Expand(ctx context.Context, kb *Kbinfos, query, keywords string, docScope []string) error {
- if e == nil || e.store == nil || kb == nil {
- return nil
- }
- // Counted from what was actually pooled, not from a len(before)/len(after)
- // delta: another session's appends land in the same pool and would be
- // counted here as this expansion's contribution.
- expanded := 0
- seen := make(map[string]bool, len(kb.Chunks))
- for _, c := range kb.Chunks {
- if id := ChunkIDOf(c); id != "" {
- seen[id] = true
- }
- }
- match := strings.TrimSpace(query)
- if match == "" {
- match = strings.TrimSpace(keywords)
- }
-
- // The hybrid hits themselves — the claim-neighbour leg keys on them (the
- // passages hybrid_search deemed relevant), so capture them BEFORE any
- // expansion appends its own rows.
- hitChunks := make([]hitChunk, 0, len(kb.Chunks))
- for _, c := range kb.Chunks {
- hitChunks = append(hitChunks, hitChunk{
- id: ChunkIDOf(c),
- sim: similarityOf(c),
- docID: DocIDOf(c),
- })
- }
-
- // Admit one expansion batch. UNCAPPED (expansion appends straight to the pool, which is
- // what pushes it past the cap), but under the pool lock and deduped against the LIVE
- // pool — the per-call `seen` snapshot it is also gated on cannot see a concurrent
- // session's appends. Returns how many chunks were actually new.
- admitExpansion := func(chunks []map[string]any) int {
- added := 0
- kb.Admit(func(p *PoolAdmitter) {
- for _, c := range chunks {
- if p.Add(c) {
- added++
- }
- }
- })
- expanded += added
- return added
- }
-
- scopes := e.kgScopes(docScope)
- for _, sc := range scopes {
- // 1-hop entity-graph expansion, per template kind:
- // compile_kwd empty, template_kind selects the template.
- for _, tk := range compiledEntityKinds {
- chunks, err := e.expandCompiledStrategy(ctx, sc, match, seen, "", tk.kind, 5)
- if err != nil {
- return err
- }
- if n := admitExpansion(chunks); n > 0 {
- _LOG.Printf("[Compiled expand] %s: +%d chunks", tk.label, n)
- }
- }
- // Tree structure graph, selected by compile_kwd:
- // template_kind empty, compile_kwd="tree". When the per-row strategy
- // comes back empty, fall back to the LEGACY graph-blob strategy —
- // documents compiled before the per-row migration persist ONE
- // knowledge_graph_kwd="graph" blob per document and no raw rows; it disappears
- // once
- // those docs are recompiled.
- chunks, err := e.expandCompiledStrategy(ctx, sc, match, seen, "tree", "", 5)
- if err != nil {
- return err
- }
- if len(chunks) == 0 {
- chunks = e.expandTreeBlobStrategy(ctx, sc, match, seen, 5)
- }
- if n := admitExpansion(chunks); n > 0 {
- _LOG.Printf("[Compiled expand] tree: +%d chunks", n)
- }
- // Synthesis pages — standalone rendered articles, searched directly
- // (synthesis pages are searched directly).
- for _, ck := range compiledSynthesisKinds {
- chunks, err := e.expandWikiPageStrategy(ctx, sc, match, seen, ck.kind, 5)
- if err != nil {
- return err
- }
- if n := admitExpansion(chunks); n > 0 {
- _LOG.Printf("[Compiled expand] %s: +%d chunks", ck.label, n)
- }
- }
- // Claims adjacent to the passages the hybrid leg just hit. Non-redundant with the
- // query-keyed claim prefetch: when the prefetch hits, the exclusive
- // takeover means this expansion never runs; when it misses for the
- // QUERY, chunk hits are a different key and their sibling claims may
- // still match.
- if n := admitExpansion(e.expandClaimNeighborStrategy(ctx, sc, hitChunks, seen, 6)); n > 0 {
- _LOG.Printf("[Compiled expand] claim neighbours: +%d chunks", n)
- }
- }
-
- // Re-sort so compiled-expansion chunks blend by similarity with regular ones
- // Under the pool lock: this PERMUTES the shared slice, so
- // it must not run while another session reads or appends it.
- kb.Admit(func(*PoolAdmitter) {
- if len(kb.Chunks) > 0 {
- sort.SliceStable(kb.Chunks, func(i, j int) bool {
- return similarityOf(kb.Chunks[i]) > similarityOf(kb.Chunks[j])
- })
- }
- })
-
- _LOG.Printf("[Hybrid search] Compiled expansion added %d chunks.", expanded)
- return nil
-}
-
-// kgScopes: by delegating to the
-// same resolver graph_explore uses: with a doc scope the documents are grouped
-// by their real owning (kb, tenant) — which may lie OUTSIDE the bound datasets —
-// and without one each bound dataset is scanned under its OWN tenant. The doc
-// scope arrives already ceilinged by resolveDocScope.
-func (e *compiledExpander) kgScopes(docScope []string) []compiledScope {
- scopes := resolveKGScope(SearchDeps{
- KbIDs: e.cfg.DatasetIDs,
- TenantID: e.cfg.TenantID,
- KBs: e.cfg.KBs,
- DocTenantResolver: e.cfg.DocTenantResolver,
- }, docScope, e.cfg.DatasetIDs)
- out := make([]compiledScope, 0, len(scopes))
- for _, sc := range scopes {
- out = append(out, compiledScope{kbID: sc.KBID, tenantID: sc.TenantID, docIDs: sc.Docs})
- }
- return out
-}
-
-// searchCompiledRows: for a single scope:
-// a compiled-row search scoped to one (kb_id, tenant_id, doc_ids) with the given
-// OR-list filters (including exact name_kwd / from_entity_kwd / to_entity_kwd
-// lookups) and an optional free-text match.
-func (e *compiledExpander) searchCompiledRows(ctx context.Context, sc compiledScope, matchText string, topN int, filters map[string][]string) []map[string]any {
- rows, err := e.store.SearchCompiled(ctx, sc.kbID, sc.tenantID, sc.docIDs, filters, matchText, topN)
- if err != nil {
- // The failed search is logged and carried on with no rows: the expansion is an
- // enrichment, so a missing compiled leg must not fail the regular retrieval.
- _LOG.Printf("[Compiled expand] compiled-row search failed (kb=%s tenant=%s); treating as no rows: %v", sc.kbID, sc.tenantID, err)
- return nil
- }
- return rows
-}
-
-// expandCompiledStrategy: within one
-// scope, embedding-match seed entities of a given template_kind/compile_kwd,
-// hop 1-hop through adjacent relations (forward + backward), collect neighbour
-// entity names, look up their source_chunk_ids by exact name_kwd, then load the
-// referenced source chunks, deduped and capped at maxChunks.
-func (e *compiledExpander) expandCompiledStrategy(ctx context.Context, sc compiledScope, match string, seen map[string]bool, compileKwd, templateKwd string, maxChunks int) ([]map[string]any, error) {
- // -- 1. Seed entities (embedding match, top 5) --
- seedRows := e.searchCompiledRows(ctx, sc, match, 5, seedFilters("entity", compileKwd, templateKwd))
- if len(seedRows) == 0 {
- return nil, nil
- }
- seedNames := make(map[string]bool)
- for _, r := range seedRows {
- if name := seedNameFromRow(r); name != "" {
- seedNames[name] = true
- }
- }
- if len(seedNames) == 0 {
- return nil, nil
- }
-
- // -- 2. Adjacent relations (outgoing + incoming), top 50 each --
- // Provide both original and lowercased names — dataset_structure_merger
- // lowercases merged-row endpoints while per-doc rows keep original case.
- seedList := lowerUnionSorted(seedNames)
- var allRels []map[string]any
- for _, relFilter := range []map[string][]string{
- {"from_entity_kwd": seedList},
- {"to_entity_kwd": seedList},
- } {
- f := seedFilters("relation", compileKwd, templateKwd)
- for k, v := range relFilter {
- f[k] = v
- }
- allRels = append(allRels, e.searchCompiledRows(ctx, sc, "", 50, f)...)
- }
-
- // -- 3. Neighbour names (1-hop, exclude seeds) --
- seedLower := make(map[string]bool, len(seedNames))
- for n := range seedNames {
- seedLower[strings.ToLower(n)] = true
- }
- neighbourNames := make(map[string]bool)
- for _, rel := range allRels {
- frm := strings.TrimSpace(compiledRelationEndpoint(rel, "from_entity_kwd"))
- to := strings.TrimSpace(compiledRelationEndpoint(rel, "to_entity_kwd"))
- if frm != "" && seedLower[strings.ToLower(frm)] && to != "" && !seedLower[strings.ToLower(to)] {
- neighbourNames[to] = true
- }
- if to != "" && seedLower[strings.ToLower(to)] && frm != "" && !seedLower[strings.ToLower(frm)] {
- neighbourNames[frm] = true
- }
- }
- if len(neighbourNames) == 0 {
- return nil, nil
- }
-
- // -- 4. Neighbour entity source_chunk_ids (exact name_kwd lookup) --
- neighList := lowerUnionSorted(neighbourNames)
- if len(neighList) > 100 {
- neighList = neighList[:100] // reasonable cap for name_kwd search
- }
- neighFilter := seedFilters("entity", compileKwd, templateKwd)
- neighFilter["name_kwd"] = neighList // exact name_kwd lookup
- neighRows := e.searchCompiledRows(ctx, sc, "", len(neighList), neighFilter)
-
- // -- 5. Group chunk ids by doc, load, dedupe, cap --
- byDoc := map[string][]string{}
- order := []string{}
- for _, r := range neighRows {
- docID := compiledDocID(r)
- for _, cid := range compiledSourceChunkIDs(r) {
- if cid == "" || seen[cid] {
- continue
- }
- if _, ok := byDoc[docID]; !ok {
- order = append(order, docID)
- }
- byDoc[docID] = append(byDoc[docID], cid)
- }
- }
- return e.loadByDoc(ctx, sc, order, byDoc, seen, maxChunks)
-}
-
-// expandWikiPageStrategy: search the
-// synthesis-compiled pages of one compile_kwd directly, collect their
-// source_chunk_ids, then load the referenced source chunks. Pages rank high, so
-// each loaded chunk is assigned similarity=0.9 unless it already carries one.
-func (e *compiledExpander) expandWikiPageStrategy(ctx context.Context, sc compiledScope, match string, seen map[string]bool, compileKwd string, maxChunks int) ([]map[string]any, error) {
- // -- 1. Search synthesis pages (no knowledge_graph_kwd filter) --
- rows := e.searchSynthesisPages(ctx, sc, match, compileKwd, 5)
- if len(rows) == 0 {
- return nil, nil
- }
-
- // -- 2. Collect source_chunk_ids from matching pages --
- byDoc := map[string][]string{}
- order := []string{}
- for _, r := range rows {
- docID := compiledDocID(r)
- for _, cid := range compiledSourceChunkIDs(r) {
- if cid == "" || seen[cid] {
- continue
- }
- if _, ok := byDoc[docID]; !ok {
- order = append(order, docID)
- }
- byDoc[docID] = append(byDoc[docID], cid)
- }
- }
- if len(order) == 0 {
- return nil, nil
- }
-
- // -- 3. Load chunks, assign high similarity for priority ranking --
- loaded, err := e.loadByDoc(ctx, sc, order, byDoc, seen, maxChunks)
- if err != nil {
- return nil, err
- }
- for _, c := range loaded {
- if _, has := c["similarity"]; !has {
- c["similarity"] = 0.9 // wiki pages rank high
- }
- }
- return loaded, nil
-}
-
-// searchSynthesisPages: find synthesis
-// page rows whose title/topic/content matches the query, filtered by compile_kwd
-// and available_int=1. Synthesis pages are standalone articles that do NOT carry
-// the knowledge_graph_kwd field, so no entity/relation filter is applied.
-func (e *compiledExpander) searchSynthesisPages(ctx context.Context, sc compiledScope, matchText, compileKwd string, topN int) []map[string]any {
- return e.searchCompiledRows(ctx, sc, matchText, topN, map[string][]string{
- "compile_kwd": {compileKwd},
- "available_int": {"1"},
- })
-}
-
-// loadByDoc loads source chunks grouped per doc within one scope, honoring a
-// global maxChunks cap and the seen-id dedup set. Returns newly added chunks in
-// doc order, with the same max_chunks break/limit accounting as the per-doc loop.
-//
-// A doc whose owner cannot be resolved is dropped (the documented per-row rule), but a
-// doc-tenant resolution FAILURE is returned: it is a transport/database error, not "no row
-// resolved".
-func (e *compiledExpander) loadByDoc(ctx context.Context, sc compiledScope, order []string, byDoc map[string][]string, seen map[string]bool, maxChunks int) ([]map[string]any, error) {
- // EACH row's doc_id is resolved to its owning (kb, tenant) — not the searched scope —
- // and nothing is loaded when that resolution fails (merged dataset rows carry a
- // pseudo/empty doc_id). Loading from the scope instead would surface chunks that are
- // deliberately dropped, and would query the wrong dataset for a row whose doc belongs
- // elsewhere.
- owners := map[string]DocTenant{}
- if e.cfg.DocTenantResolver != nil {
- ids := make([]string, 0, len(order))
- for _, docID := range order {
- if docID != "" {
- ids = append(ids, docID)
- }
- }
- resolved, err := e.cfg.DocTenantResolver.ResolveDocTenants(ctx, ids)
- if err != nil {
- // A lookup failure aborts the load: nothing is loaded, and the failure is loud.
- // Return it so the caller reports it instead of letting a transport error
- // masquerade as the documented per-row drop below.
- return nil, fmt.Errorf("doc-tenant resolution failed for %d doc(s): %w", len(ids), err)
- }
- owners = resolved
- }
- var out []map[string]any
- for _, docID := range order {
- if len(out) >= maxChunks {
- break
- }
- kbID, tenantID := sc.kbID, sc.tenantID
- if e.cfg.DocTenantResolver != nil {
- owner, ok := owners[docID]
- if docID == "" || !ok || owner.KBID == "" {
- continue
- }
- kbID, tenantID = owner.KBID, owner.TenantID
- }
- cids := byDoc[docID]
- remaining := maxChunks - len(out)
- if len(cids) > remaining {
- cids = cids[:remaining]
- }
- rows, err := e.store.LoadChunks(ctx, kbID, tenantID, cids)
- if err != nil {
- // The failed load is logged per doc and only that doc is dropped; the remaining
- // docs still load.
- _LOG.Printf("[Compiled expand] failed to load chunks for doc_id=%s: %v", docID, err)
- continue
- }
- for _, c := range rows {
- id := ChunkIDOf(c)
- if id == "" || seen[id] {
- continue
- }
- seen[id] = true
- out = append(out, c)
- }
- }
- return out, nil
-}
-
-// seedNameFromRow: 300: parse content_with_weight as JSON and
-// read payload.name or payload.title (stripped).
-func seedNameFromRow(row map[string]any) string {
- cww, _ := row["content_with_weight"].(string)
- if cww == "" {
- return ""
- }
- var payload map[string]any
- if err := json.Unmarshal([]byte(cww), &payload); err != nil {
- return ""
- }
- name := strings.TrimSpace(asString(payload["name"]))
- if name == "" {
- name = strings.TrimSpace(asString(payload["title"]))
- }
- return name
-}
-
-// lowerUnionSorted returns the sorted union of the names and their lowercased forms.
-func lowerUnionSorted(names map[string]bool) []string {
- seen := make(map[string]bool, len(names)*2)
- for n := range names {
- if n == "" {
- continue
- }
- seen[n] = true
- seen[strings.ToLower(n)] = true
- }
- out := make([]string, 0, len(seen))
- for n := range seen {
- out = append(out, n)
- }
- sort.Strings(out)
- return out
-}
-
-func compiledDocID(row map[string]any) string {
- return asString(row["doc_id"])
-}
-
-// compiledRelationEndpoint reads a relation's from_entity_kwd/to_entity_kwd
-// (which may be a bare string or a single-element list).
-func compiledRelationEndpoint(row map[string]any, key string) string {
- if v, ok := row[key].(string); ok {
- return v
- }
- return asString(row[key])
-}
-
-func compiledSourceChunkIDs(row map[string]any) []string {
- switch v := row["source_chunk_ids"].(type) {
- case []string:
- return v
- case []any:
- out := make([]string, 0, len(v))
- for _, x := range v {
- if s, ok := x.(string); ok {
- out = append(out, s)
- }
- }
- return out
- case string:
- return []string{v}
- }
- return nil
-}
-
-// similarityOf returns a chunk's similarity score as a float64; nil/absent counts as 0.
-func similarityOf(c map[string]any) float64 {
- switch v := c["similarity"].(type) {
- case float64:
- return v
- case float32:
- return float64(v)
- case int:
- return float64(v)
- case int64:
- return float64(v)
- }
- return 0.0
-}
-
-func asString(v any) string {
- if s, ok := v.(string); ok {
- return s
- }
- return ""
-}
-
-// seedFilters builds the entity/relation condition: {"knowledge_graph_kwd": [kind]} plus an
-// optional canonical compilation-kind selector.
-func seedFilters(kind, compileKwd, templateKwd string) map[string][]string {
- f := map[string][]string{"knowledge_graph_kwd": {kind}}
- if compileKwd != "" {
- f["compile_kwd"] = []string{compileKwd}
- }
- if templateKwd != "" {
- f["compile_kwd"] = []string{enginetypes.CanonicalCompilationKind(templateKwd)}
- }
- return f
-}
-
-// cosine returns the cosine similarity of two equal-length vectors. Shared with
-// tool_navigation.go (navigation.py _node_score). Returns 0 on empty/mismatched
-// input or when either vector is zero.
-func cosine(a, b []float64) float64 {
- if len(a) == 0 || len(b) == 0 || len(a) != len(b) {
- return 0
- }
- var dot, na, nb float64
- for i := range a {
- dot += a[i] * b[i]
- na += a[i] * a[i]
- nb += b[i] * b[i]
- }
- if na == 0 || nb == 0 {
- return 0
- }
- return dot / (math.Sqrt(na) * math.Sqrt(nb))
-}
-
-// Legacy tree-blob expansion and the claim-neighbour expansion.
-
-// hitChunk is one hybrid_search hit the claim-neighbour leg keys on
-// (chunk_id, similarity, doc_id) triples.
-type hitChunk struct {
- id string
- sim float64
- docID string
-}
-
-// expandTreeBlobStrategy: expand a
-// tree-compiled document from its pre-migration graph blob, in-process. Same
-// contract as expandCompiledStrategy — seed entities, 1-hop neighbours, and
-// the neighbours' source_chunk_ids loaded back as real chunks — except seeds,
-// relations, and addressing all come from the blob's JSON, so the walk costs
-// zero extra store round-trips. Entity payloads are compiled summaries: only
-// the loaded chunks reach the model.
-func (e *compiledExpander) expandTreeBlobStrategy(ctx context.Context, sc compiledScope, match string, seen map[string]bool, maxChunks int) []map[string]any {
- // Filters: compile_kwd=["tree"] + knowledge_graph_kwd=["graph"], payload-only, capped at
- // 16 rows.
- filters := map[string][]string{"compile_kwd": {"tree"}, "knowledge_graph_kwd": {"graph"}}
- rows := e.searchCompiledRows(ctx, sc, "", 16, filters)
- if len(rows) == 0 {
- return nil
- }
-
- byDoc := map[string][]string{}
- var order []string
- add := func(docID, cid string) {
- for _, existing := range byDoc[docID] {
- if existing == cid {
- return
- }
- }
- byDoc[docID] = append(byDoc[docID], cid)
- kept := false
- for _, d := range order {
- if d == docID {
- kept = true
- break
- }
- }
- if !kept {
- order = append(order, docID)
- }
- }
- for _, row := range rows {
- docID := compiledDocID(row)
- if docID == "" {
- continue
- }
- graph, ok := decodeJSONObject(asString(row["content_with_weight"]))
- if !ok {
- continue
- }
- // Seeds: query-matched entities, then 1-hop neighbours through the
- // blob's own relations (parent/child edges) — no store round-trips.
- scored := scoreTreeEntities(graph, match)
- if len(scored) == 0 {
- continue
- }
- if len(scored) > 5 {
- scored = scored[:5]
- }
- seedNames := map[string]bool{}
- for _, ent := range scored {
- if name := strings.TrimSpace(asString(ent["name"])); name != "" {
- seedNames[name] = true
- }
- }
- neighbours := map[string]bool{}
- for _, rel := range objectList(graph["relations"]) {
- from := strings.TrimSpace(compiledRelationEndpoint(rel, "from"))
- to := strings.TrimSpace(compiledRelationEndpoint(rel, "to"))
- if from != "" && seedNames[from] && to != "" {
- neighbours[to] = true
- }
- if to != "" && seedNames[to] && from != "" {
- neighbours[from] = true
- }
- }
- // Address chunks: every entity at or adjacent to a seed contributes
- // its source_chunk_ids (the same contract the raw-row strategy applies
- // to neighbour entities).
- wanted := func(name string) bool { return seedNames[name] || neighbours[name] }
- for _, ent := range objectList(graph["entities"]) {
- if !wanted(strings.TrimSpace(asString(ent["name"]))) {
- continue
- }
- for _, cid := range compiledSourceChunkIDs(ent) {
- if cid != "" && !seen[cid] {
- add(docID, cid)
- }
- }
- }
- }
- loaded, err := e.loadByDoc(ctx, sc, order, byDoc, seen, maxChunks)
- if err != nil {
- // The store error is reported the same way Expand reports it for the other
- // strategies.
- _LOG.Printf("[Compiled expand] tree blob chunk load failed: %v", err)
- return nil
- }
- return loaded
-}
-
-// scoreTreeEntities: rank blob entities by
-// query-term overlap over name + description. Lexical on purpose — the raw-row
-// strategy seeds the same way (BM25 over content_ltks), and the blob's vectors
-// are one shared row vector with nothing per-entity to cosine against.
-func scoreTreeEntities(graph map[string]any, query string) []map[string]any {
- terms := treeEntityTerms(query)
- if len(terms) == 0 {
- return nil
- }
- type scored struct {
- ent map[string]any
- score int
- }
- var out []scored
- for _, ent := range objectList(graph["entities"]) {
- text := strings.ToLower(asString(ent["name"]) + " " + asString(ent["description"]))
- if strings.TrimSpace(text) == "" {
- continue
- }
- score := 0
- for _, t := range terms {
- if strings.Contains(text, t) {
- score++
- }
- }
- if score > 0 {
- out = append(out, scored{ent: ent, score: score})
- }
- }
- // Stable sort: ties keep the blob's own entity order.
- sort.SliceStable(out, func(i, j int) bool { return out[i].score > out[j].score })
- ents := make([]map[string]any, 0, len(out))
- for _, s := range out {
- ents = append(ents, s.ent)
- }
- return ents
-}
-
-// treeEntityTerms uses the coarse RAGFlow tokenizer, lowercased for the containment
-// match.
-func treeEntityTerms(query string) []string {
- toks, err := tokenizer.Tokenize(query)
- if err != nil {
- return nil
- }
- var out []string
- seen := map[string]bool{}
- for _, t := range strings.Fields(strings.ToLower(toks)) {
- if t != "" && !seen[t] {
- seen[t] = true
- out = append(out, t)
- }
- }
- return out
-}
-
-// expandClaimNeighborStrategy
-// inject claims adjacent to the passages hybrid_search just hit. A claim whose
-// source_chunk_ids intersect a hit chunk is a verified atomic statement about a
-// passage the retriever already deemed relevant — even when the QUERY never
-// matched it. Emitted as pseudo-chunks (name + verbatim quote, "[claim] "
-// prefix, ASCII " -- " description separator, no per-quote cap beyond the
-// overall 1200) with the hit chunk's similarity, so they blend into the
-// ranking. Deliberately NOT 1-hop expanded and deliberately keyed on the HIT
-// chunks, not the query.
-func (e *compiledExpander) expandClaimNeighborStrategy(ctx context.Context, sc compiledScope, hits []hitChunk, seen map[string]bool, maxChunks int) []map[string]any {
- live := make([]hitChunk, 0, len(hits))
- for _, h := range hits {
- if h.id != "" && !strings.HasPrefix(h.id, "claim_") {
- live = append(live, h)
- }
- }
- if len(live) == 0 {
- return nil
- }
- hitIDs := make(map[string]bool, len(live))
- hitDocs := []string{}
- docSeen := map[string]bool{}
- for _, h := range live {
- hitIDs[h.id] = true
- if h.docID != "" && !docSeen[h.docID] {
- docSeen[h.docID] = true
- hitDocs = append(hitDocs, h.docID)
- }
- }
- sort.Strings(hitDocs)
-
- // claim rows of the hit documents, payload-only, 256 rows.
- filters := map[string][]string{"entity_type_kwd": {"claim"}, "scope_kwd": {"doc"}}
- if len(hitDocs) > 0 {
- filters["doc_id"] = hitDocs
- }
- rows := e.searchCompiledRows(ctx, sc, "", 256, filters)
-
- out := make([]map[string]any, 0, maxChunks)
- for _, row := range rows {
- payload, ok := decodeJSONObject(asString(row["content_with_weight"]))
- if !ok {
- continue
- }
- name := strings.TrimSpace(asString(payload["name"]))
- if name == "" {
- continue
- }
- srcIDs := compiledSourceChunkIDs(row)
- if len(srcIDs) == 0 {
- srcIDs = compiledSourceChunkIDs(payload)
- }
- overlap := map[string]bool{}
- for _, cid := range srcIDs {
- if hitIDs[cid] {
- overlap[cid] = true
- }
- }
- if len(overlap) == 0 {
- continue
- }
- rowDoc := compiledDocID(row)
- cid := "claim_" + fmt.Sprintf("%x", md5.Sum([]byte(rowDoc+":"+name)))[:12]
- if seen[cid] {
- continue
- }
- description := strings.TrimSpace(asString(payload["description"]))
- quote := ""
- for _, ev := range objectList(payload["evidence"]) {
- if q := strings.TrimSpace(asString(ev["quote"])); q != "" {
- quote = q
- break
- }
- }
- content := "[claim] " + name
- if description != "" && description != name {
- content += " -- " + description
- }
- if quote != "" {
- content += "\nEvidence (verbatim): \"" + quote + "\""
- }
- // Inherit the best-matching hit's similarity so the pseudo-chunk lands
- // beside the passage that spawned it in the final ranking.
- similarity := 0.0
- for _, h := range live {
- if overlap[h.id] && h.sim > similarity {
- similarity = h.sim
- }
- }
- seen[cid] = true
- out = append(out, map[string]any{
- "chunk_id": cid,
- "content_with_weight": truncateRunes(content, 1200),
- "doc_id": rowDoc,
- "source_chunk_ids": srcIDs,
- "similarity": similarity,
- })
- if len(out) >= maxChunks {
- break
- }
- }
- return out
-}
diff --git a/internal/rag/agentic-rag/runtime/tool_compiled_expansion_test.go b/internal/rag/agentic-rag/runtime/tool_compiled_expansion_test.go
deleted file mode 100644
index 691654b663..0000000000
--- a/internal/rag/agentic-rag/runtime/tool_compiled_expansion_test.go
+++ /dev/null
@@ -1,552 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "bytes"
- "context"
- "errors"
- "log"
- enginetypes "ragflow/internal/engine/types"
- "strings"
- "testing"
-
- "ragflow/internal/entity"
-)
-
-// stubCompiledStore is an in-memory CompiledStore so the expander logic can be
-// exercised in the unit tier. It behaves like the doc store:
-//
-// - rows carry knowledge_graph_kwd / compile_kwd / available_int / from_entity_kwd
-// / to_entity_kwd / name_kwd / content_with_weight / source_chunk_ids / doc_id.
-// - filters are AND-ed, each value an OR-set (a row matches if its field's
-// string form equals a member).
-// - matchText selects rows whose name/content contains it (stand-in for the
-// dense+keyword retrieval the store normally does).
-type stubCompiledStore struct {
- rows []map[string]any
- chunks map[string]string
- // loadErrByChunk makes LoadChunks fail when one of the requested ids is
- // listed, standing in for a per-doc store error.
- loadErrByChunk map[string]error
-}
-
-func (s *stubCompiledStore) SearchCompiled(_ context.Context, _, _ string, _ []string, filters map[string][]string, matchText string, topN int) ([]map[string]any, error) {
- var out []map[string]any
- for _, r := range s.rows {
- if !rowMatchesFilters(r, filters) {
- continue
- }
- if matchText != "" {
- hay := strings.ToLower(asString(r["name_kwd"]) + " " + asString(r["content_with_weight"]))
- if !strings.Contains(hay, strings.ToLower(matchText)) {
- continue
- }
- }
- out = append(out, r)
- if topN > 0 && len(out) >= topN {
- break
- }
- }
- return out, nil
-}
-
-func (s *stubCompiledStore) LoadChunks(_ context.Context, _, _ string, chunkIDs []string) ([]map[string]any, error) {
- for _, id := range chunkIDs {
- if err, ok := s.loadErrByChunk[id]; ok {
- return nil, err
- }
- }
- var out []map[string]any
- for _, id := range chunkIDs {
- if content, ok := s.chunks[id]; ok {
- out = append(out, map[string]any{"id": id, "content": content, "doc_id": "d1"})
- }
- }
- return out, nil
-}
-
-func (s *stubCompiledStore) Vectorize(_ context.Context, _ string) ([]float64, error) {
- return nil, nil
-}
-
-// scopeRecordingStore records the (kb, tenant, docs) triple of every
-// SearchCompiled call and returns no rows, so only scope resolution is under
-// test.
-type scopeRecordingStore struct {
- calls []scopeCall
-}
-
-type scopeCall struct {
- kb, tenant string
- docs []string
-}
-
-func (s *scopeRecordingStore) SearchCompiled(_ context.Context, kbID, tenantID string, docIDs []string, _ map[string][]string, _ string, _ int) ([]map[string]any, error) {
- s.calls = append(s.calls, scopeCall{kb: kbID, tenant: tenantID, docs: append([]string(nil), docIDs...)})
- return nil, nil
-}
-
-func (s *scopeRecordingStore) LoadChunks(context.Context, string, string, []string) ([]map[string]any, error) {
- return nil, nil
-}
-
-func (s *scopeRecordingStore) Vectorize(context.Context, string) ([]float64, error) {
- return nil, nil
-}
-
-// ownerRecordingStore wraps stubCompiledStore to record the owner each
-// LoadChunks call is issued against.
-type ownerRecordingStore struct {
- *stubCompiledStore
- loads []scopeCall
-}
-
-func (s *ownerRecordingStore) LoadChunks(ctx context.Context, kbID, tenantID string, chunkIDs []string) ([]map[string]any, error) {
- s.loads = append(s.loads, scopeCall{kb: kbID, tenant: tenantID, docs: append([]string(nil), chunkIDs...)})
- return s.stubCompiledStore.LoadChunks(ctx, kbID, tenantID, chunkIDs)
-}
-
-// TestCompiledExpanderLoadsFromEachDocsOwnerAndSkipsUnresolvable: a neighbour row's source
-// chunks are fetched from THAT row's doc owner (not the searched scope), and a row whose doc id
-// cannot be resolved (empty / merged pseudo doc) contributes nothing.
-func TestCompiledExpanderLoadsFromEachDocsOwnerAndSkipsUnresolvable(t *testing.T) {
- store := &ownerRecordingStore{stubCompiledStore: &stubCompiledStore{
- rows: []map[string]any{
- {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "knowledge_graph",
- "name_kwd": "Culdcept", "content_with_weight": `{"name":"Culdcept"}`, "doc_id": "seedDoc"},
- {"knowledge_graph_kwd": "relation", "compilation_template_kind_kwd": "knowledge_graph",
- "from_entity_kwd": "Culdcept", "to_entity_kwd": "OmiyaSoft", "doc_id": "seedDoc"},
- {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "knowledge_graph",
- "name_kwd": "OmiyaSoft", "content_with_weight": `{"name":"OmiyaSoft"}`,
- "source_chunk_ids": []string{"s1"}, "doc_id": "docA"},
- // No doc id -> the owner cannot be resolved, so its source chunks are dropped.
- {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "knowledge_graph",
- "name_kwd": "OmiyaSoft", "content_with_weight": `{"name":"OmiyaSoft"}`,
- "source_chunk_ids": []string{"s2"}, "doc_id": ""},
- },
- chunks: map[string]string{"s1": "a", "s2": "b"},
- }}
- exp := NewCompiledExpander(store, CompiledScopeConfig{
- DatasetIDs: []string{"kb1"},
- TenantID: "t1",
- KBs: []*entity.Knowledgebase{{ID: "kb1", TenantID: "t1"}},
- DocTenantResolver: stubTenantResolver{owner: DocTenant{KBID: "kbOwner", TenantID: "tOwner"}},
- }).(*compiledExpander)
-
- kb := &Kbinfos{}
- if err := exp.Expand(context.Background(), kb, "culdcept", "", nil); err != nil {
- t.Fatalf("Expand: %v", err)
- }
- if len(store.loads) != 1 {
- t.Fatalf("LoadChunks calls = %d (%+v), want 1 (the unresolvable doc must be skipped)", len(store.loads), store.loads)
- }
- if store.loads[0].kb != "kbOwner" || store.loads[0].tenant != "tOwner" {
- t.Errorf("load owner = %+v, want the doc's own owner kbOwner/tOwner", store.loads[0])
- }
- got := compiledIDs(kb.Chunks)
- if len(got) != 1 || got[0] != "s1" {
- t.Errorf("expanded = %v, want only s1", got)
- }
-}
-
-// TestNewCompiledExpanderGuard pins the fail-closed cases: a nil store, no bound
-// dataset, or no fallback tenant leaves expansion disabled rather than searching
-// an unscoped/empty index.
-func TestNewCompiledExpanderGuard(t *testing.T) {
- store := &scopeRecordingStore{}
- if exp := NewCompiledExpander(nil, CompiledScopeConfig{DatasetIDs: []string{"kb1"}, TenantID: "t1"}); exp != nil {
- t.Error("nil store must disable expansion")
- }
- if exp := NewCompiledExpander(store, CompiledScopeConfig{TenantID: "t1"}); exp != nil {
- t.Error("no dataset must disable expansion")
- }
- if exp := NewCompiledExpander(store, CompiledScopeConfig{DatasetIDs: []string{"kb1"}}); exp != nil {
- t.Error("no tenant must disable expansion")
- }
- if exp := NewCompiledExpander(store, CompiledScopeConfig{DatasetIDs: []string{"kb1"}, TenantID: "t1"}); exp == nil {
- t.Error("store + dataset + tenant must enable expansion")
- }
-}
-
-// TestCompiledExpanderScansEachDatasetUnderItsOwnTenant: with no doc scope each bound dataset
-// is scanned under ITS OWN owner tenant, not the request tenant.
-func TestCompiledExpanderScansEachDatasetUnderItsOwnTenant(t *testing.T) {
- store := &scopeRecordingStore{}
- exp := NewCompiledExpander(store, CompiledScopeConfig{
- DatasetIDs: []string{"kb1", "kb2"},
- TenantID: "request-tenant",
- KBs: []*entity.Knowledgebase{
- {ID: "kb1", TenantID: "t1"},
- {ID: "kb2", TenantID: "t2"},
- },
- })
- if err := exp.Expand(context.Background(), &Kbinfos{}, "q", "", nil); err != nil {
- t.Fatalf("Expand: %v", err)
- }
- if len(store.calls) == 0 {
- t.Fatal("no compiled search was performed")
- }
- tenants := map[string]string{}
- for _, c := range store.calls {
- tenants[c.kb] = c.tenant
- if len(c.docs) != 0 {
- t.Errorf("no doc scope must be sent for %s, got %v", c.kb, c.docs)
- }
- if c.tenant == "request-tenant" {
- t.Errorf("request tenant leaked into scope %+v; each dataset must use its owner tenant", c)
- }
- }
- if tenants["kb1"] != "t1" || tenants["kb2"] != "t2" {
- t.Errorf("per-dataset tenants = %v, want kb1=t1 kb2=t2", tenants)
- }
-}
-
-// TestCompiledExpanderGroupsDocScopeByRealOwner covers the doc-scope branch: documents are
-// grouped by their real owning (kb, tenant) — which may lie OUTSIDE the bound datasets — and
-// the bound list is not scanned.
-func TestCompiledExpanderGroupsDocScopeByRealOwner(t *testing.T) {
- store := &scopeRecordingStore{}
- exp := NewCompiledExpander(store, CompiledScopeConfig{
- DatasetIDs: []string{"kb1", "kb2"},
- TenantID: "request-tenant",
- KBs: []*entity.Knowledgebase{
- {ID: "kb1", TenantID: "t1"},
- {ID: "kb2", TenantID: "t2"},
- },
- DocTenantResolver: stubTenantResolver{owner: DocTenant{KBID: "kbX", TenantID: "tX"}},
- })
- if err := exp.Expand(context.Background(), &Kbinfos{}, "q", "", []string{"d1"}); err != nil {
- t.Fatalf("Expand: %v", err)
- }
- if len(store.calls) == 0 {
- t.Fatal("no compiled search was performed")
- }
- for _, c := range store.calls {
- if c.kb != "kbX" || c.tenant != "tX" {
- t.Errorf("scope = %+v, want the document's real owner kbX/tX", c)
- }
- if len(c.docs) != 1 || c.docs[0] != "d1" {
- t.Errorf("docs = %v, want [d1]", c.docs)
- }
- }
-}
-
-func rowMatchesFilters(r map[string]any, filters map[string][]string) bool {
- for k, vals := range filters {
- rv := asString(r[k])
- if k == "compile_kwd" {
- rv = enginetypes.CompilationKind(r)
- if len(vals) > 0 && strings.HasPrefix(vals[0], "wiki_") {
- rv = enginetypes.CompilationRowType(r)
- }
- }
- matched := false
- for _, v := range vals {
- if rv == v {
- matched = true
- break
- }
- }
- if !matched {
- return false
- }
- }
- return true
-}
-
-func compiledIDs(chunks []map[string]any) []string {
- out := make([]string, 0, len(chunks))
- for _, c := range chunks {
- out = append(out, asString(c["id"]))
- }
- return out
-}
-
-// TestSeedNameFromRowParsesContentWithWeight: 300: the seed
-// name comes from content_with_weight JSON (name, else title), never from name_kwd.
-func TestSeedNameFromRowParsesContentWithWeight(t *testing.T) {
- if got := seedNameFromRow(map[string]any{"content_with_weight": `{"name":"Culdcept","title":"Game"}`}); got != "Culdcept" {
- t.Errorf("name: got %q", got)
- }
- if got := seedNameFromRow(map[string]any{"content_with_weight": `{"title":"Released 1984"}`}); got != "Released 1984" {
- t.Errorf("title fallback: got %q", got)
- }
- if got := seedNameFromRow(map[string]any{"content_with_weight": `not json`}); got != "" {
- t.Errorf("bad json: got %q", got)
- }
- // name_kwd is not consulted — only the JSON payload is read.
- if got := seedNameFromRow(map[string]any{"content_with_weight": `{}`, "name_kwd": "X"}); got != "" {
- t.Errorf("name_kwd must be ignored, got %q", got)
- }
-}
-
-func TestLowerUnionSorted(t *testing.T) {
- got := lowerUnionSorted(map[string]bool{"OmiyaSoft": true, "X": true})
- // Byte-order ascending over the original + lowercased forms.
- want := []string{"OmiyaSoft", "X", "omiyasoft", "x"}
- if len(got) != len(want) {
- t.Fatalf("lowerUnionSorted = %v", got)
- }
- for i := range want {
- if got[i] != want[i] {
- t.Fatalf("lowerUnionSorted[%d] = %q, want %q (%v)", i, got[i], want[i], got)
- }
- }
-}
-
-// TestExpandEntityStrategyOneHop verifies the 1-hop entity strategy: a seed
-// entity matching the query (found via its template kind) walks a forward
-// relation to a neighbour, resolves the neighbour by exact name_kwd, and loads
-// its source chunks.
-func TestExpandEntityStrategyOneHop(t *testing.T) {
- store := &stubCompiledStore{
- rows: []map[string]any{
- {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "knowledge_graph",
- "name_kwd": "Culdcept", "content_with_weight": `{"name":"Culdcept"}`},
- {"knowledge_graph_kwd": "relation", "compilation_template_kind_kwd": "knowledge_graph",
- "from_entity_kwd": "Culdcept", "to_entity_kwd": "OmiyaSoft"},
- {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "knowledge_graph",
- "name_kwd": "OmiyaSoft", "content_with_weight": `{"name":"OmiyaSoft"}`,
- "source_chunk_ids": []string{"s1", "s2"}},
- },
- chunks: map[string]string{"s1": "a", "s2": "b"},
- }
- exp := NewCompiledExpander(store, CompiledScopeConfig{DatasetIDs: []string{"kb1"}, TenantID: "t1"}).(*compiledExpander)
- kb := &Kbinfos{}
- if err := exp.Expand(context.Background(), kb, "culdcept", "", nil); err != nil {
- t.Fatalf("Expand: %v", err)
- }
- if len(kb.Chunks) != 2 {
- t.Fatalf("expanded = %d (%v), want 2 neighbour source chunks", len(kb.Chunks), compiledIDs(kb.Chunks))
- }
-}
-
-// TestExpandBackwardRelation confirms the strategy also walks incoming (to_entity_kwd)
-// relations — the forward + backward merge.
-func TestExpandBackwardRelation(t *testing.T) {
- store := &stubCompiledStore{
- rows: []map[string]any{
- {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "mind_map",
- "name_kwd": "Culdcept", "content_with_weight": `{"name":"Culdcept"}`},
- // Relation where the seed is the TARGET, not the source.
- {"knowledge_graph_kwd": "relation", "compilation_template_kind_kwd": "mind_map",
- "from_entity_kwd": "OmiyaSoft", "to_entity_kwd": "Culdcept"},
- {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "mind_map",
- "name_kwd": "OmiyaSoft", "content_with_weight": `{"name":"OmiyaSoft"}`,
- "source_chunk_ids": []string{"o1"}},
- },
- chunks: map[string]string{"o1": "from"},
- }
- exp := NewCompiledExpander(store, CompiledScopeConfig{DatasetIDs: []string{"kb1"}, TenantID: "t1"}).(*compiledExpander)
- kb := &Kbinfos{}
- if err := exp.Expand(context.Background(), kb, "culdcept", "", nil); err != nil {
- t.Fatalf("Expand: %v", err)
- }
- if len(kb.Chunks) != 1 {
- t.Fatalf("backward relation gave %d (%v), want 1", len(kb.Chunks), compiledIDs(kb.Chunks))
- }
-}
-
-// TestExpandWikiSetsSimilarityAndBlendsByIt verifies wiki synthesis pages load
-// their source chunks with a similarity boost (0.9) and the final sort puts the
-// higher-similarity wiki chunk ahead of the entity (0-similarity) chunk.
-func TestExpandWikiSetsSimilarityAndBlendsByIt(t *testing.T) {
- store := &stubCompiledStore{
- rows: []map[string]any{
- // knowledge_graph entity 1-hop -> chunk e1 (similarity 0 by default).
- {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "knowledge_graph",
- "name_kwd": "Culdcept", "content_with_weight": `{"name":"Culdcept"}`},
- {"knowledge_graph_kwd": "relation", "compilation_template_kind_kwd": "knowledge_graph",
- "from_entity_kwd": "Culdcept", "to_entity_kwd": "OmiyaSoft"},
- {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "knowledge_graph",
- "name_kwd": "OmiyaSoft", "content_with_weight": `{"name":"OmiyaSoft"}`,
- "source_chunk_ids": []string{"e1"}},
- // wiki synthesis page matching the query -> chunk w1.
- {"compile_kwd": "wiki_page", "available_int": "1", "doc_id": "d2",
- "name_kwd": "Culdcept", "content_with_weight": `{"title":"Culdcept article"}`,
- "source_chunk_ids": []string{"w1"}},
- },
- chunks: map[string]string{"e1": "entity", "w1": "wiki"},
- }
- exp := NewCompiledExpander(store, CompiledScopeConfig{DatasetIDs: []string{"kb1"}, TenantID: "t1"}).(*compiledExpander)
- kb := &Kbinfos{}
- if err := exp.Expand(context.Background(), kb, "culdcept", "", nil); err != nil {
- t.Fatalf("Expand: %v", err)
- }
- if len(kb.Chunks) != 2 {
- t.Fatalf("expanded = %d (%v), want 2", len(kb.Chunks), compiledIDs(kb.Chunks))
- }
- var wSim, eSim float64
- for _, c := range kb.Chunks {
- switch asString(c["id"]) {
- case "w1":
- wSim = similarityOf(c)
- case "e1":
- eSim = similarityOf(c)
- }
- }
- if wSim != 0.9 {
- t.Errorf("wiki similarity = %v, want 0.9", wSim)
- }
- if eSim != 0.0 {
- t.Errorf("entity chunk similarity = %v, want 0.0", eSim)
- }
- // Final global sort: wiki (0.9) must rank before entity (0).
- if compiledIDs(kb.Chunks)[0] != "w1" {
- t.Errorf("final sort should put wiki (0.9) first, got %v", compiledIDs(kb.Chunks))
- }
-}
-
-// TestExpandCapsMaxChunksAndSkipsSeen confirms a strategy stops adding at 5 and
-// chunks already in kbinfos are not duplicated.
-func TestExpandCapsMaxChunksAndSkipsSeen(t *testing.T) {
- store := &stubCompiledStore{
- rows: []map[string]any{
- {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "knowledge_graph",
- "name_kwd": "A", "content_with_weight": `{"name":"A"}`},
- {"knowledge_graph_kwd": "relation", "compilation_template_kind_kwd": "knowledge_graph",
- "from_entity_kwd": "A", "to_entity_kwd": "B"},
- {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "knowledge_graph",
- "name_kwd": "B", "content_with_weight": `{"name":"B"}`,
- "source_chunk_ids": []string{"b1", "b2", "b3", "b4", "b5", "b6", "b7"}},
- },
- chunks: map[string]string{"b1": "1", "b2": "2", "b3": "3", "b4": "4", "b5": "5", "b6": "6", "b7": "7"},
- }
- exp := NewCompiledExpander(store, CompiledScopeConfig{DatasetIDs: []string{"kb1"}, TenantID: "t1"}).(*compiledExpander)
- // b1 is already present in the working set; it must not come back.
- kb := &Kbinfos{Chunks: []map[string]any{{"id": "b1", "content": "pre-existing"}}}
- if err := exp.Expand(context.Background(), kb, "a", "", nil); err != nil {
- t.Fatalf("Expand: %v", err)
- }
- // 5 fresh (max_chunks) minus the seen b1 that was skipped within the first
- // load batch -> at most 5 new.
- if len(kb.Chunks) > 6 { // 1 pre-existing + up to 5 new
- t.Fatalf("exceeded max_chunks=5 new chunks: got %d total (%v)", len(kb.Chunks), compiledIDs(kb.Chunks))
- }
- for _, c := range kb.Chunks {
- if asString(c["id"]) == "b1" && asString(c["content"]) != "pre-existing" {
- t.Error("seen chunk b1 was duplicated")
- }
- }
-}
-
-// mapTenantResolver resolves each doc id to its own owner and can fail the whole batch,
-// standing in for the batched DB lookup.
-type mapTenantResolver struct {
- owners map[string]DocTenant
- err error
-}
-
-func (m mapTenantResolver) ResolveDocTenants(_ context.Context, docIDs []string) (map[string]DocTenant, error) {
- if m.err != nil {
- return nil, m.err
- }
- out := make(map[string]DocTenant, len(docIDs))
- for _, d := range docIDs {
- if o, ok := m.owners[d]; ok {
- out[d] = o
- }
- }
- return out, nil
-}
-
-// TestCompiledExpanderReportsDocTenantResolutionFailure pins the two rules apart: a row whose
-// owner cannot be resolved is dropped silently (see the test above), but a FAILED batched
-// lookup is a transport error — nothing is loaded and the failure is reported instead of
-// masquerading as "no row resolved".
-func TestCompiledExpanderReportsDocTenantResolutionFailure(t *testing.T) {
- store := &ownerRecordingStore{stubCompiledStore: &stubCompiledStore{
- rows: []map[string]any{
- {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "knowledge_graph",
- "name_kwd": "Culdcept", "content_with_weight": `{"name":"Culdcept"}`, "doc_id": "seedDoc"},
- {"knowledge_graph_kwd": "relation", "compilation_template_kind_kwd": "knowledge_graph",
- "from_entity_kwd": "Culdcept", "to_entity_kwd": "OmiyaSoft", "doc_id": "seedDoc"},
- {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "knowledge_graph",
- "name_kwd": "OmiyaSoft", "content_with_weight": `{"name":"OmiyaSoft"}`,
- "source_chunk_ids": []string{"s1"}, "doc_id": "docA"},
- },
- chunks: map[string]string{"s1": "a"},
- }}
- exp := NewCompiledExpander(store, CompiledScopeConfig{
- DatasetIDs: []string{"kb1"},
- TenantID: "t1",
- KBs: []*entity.Knowledgebase{{ID: "kb1", TenantID: "t1"}},
- DocTenantResolver: mapTenantResolver{err: errors.New("db unavailable")},
- }).(*compiledExpander)
-
- kb := &Kbinfos{}
- err := exp.Expand(context.Background(), kb, "culdcept", "", nil)
- if err == nil {
- t.Fatal("Expand must report the resolver failure instead of silently loading nothing")
- }
- if !strings.Contains(err.Error(), "doc-tenant resolution failed for 1 doc(s)") {
- t.Errorf("error = %q, want it to name the failed batch and its size", err)
- }
- if len(store.loads) != 0 {
- t.Errorf("LoadChunks calls = %+v, want none after a resolver failure", store.loads)
- }
- if len(kb.Chunks) != 0 {
- t.Errorf("chunks = %v, want none", compiledIDs(kb.Chunks))
- }
-}
-
-// TestCompiledExpanderLogsAndKeepsOtherDocsWhenOneLoadFails covers the per-doc load: the
-// failing doc is logged and dropped, while the remaining docs still contribute.
-func TestCompiledExpanderLogsAndKeepsOtherDocsWhenOneLoadFails(t *testing.T) {
- var buf bytes.Buffer
- prevLog := _LOG
- _LOG = log.New(&buf, "", 0)
- t.Cleanup(func() { _LOG = prevLog })
-
- store := &ownerRecordingStore{stubCompiledStore: &stubCompiledStore{
- rows: []map[string]any{
- {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "knowledge_graph",
- "name_kwd": "Culdcept", "content_with_weight": `{"name":"Culdcept"}`, "doc_id": "seedDoc"},
- {"knowledge_graph_kwd": "relation", "compilation_template_kind_kwd": "knowledge_graph",
- "from_entity_kwd": "Culdcept", "to_entity_kwd": "OmiyaSoft", "doc_id": "seedDoc"},
- {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "knowledge_graph",
- "name_kwd": "OmiyaSoft", "content_with_weight": `{"name":"OmiyaSoft"}`,
- "source_chunk_ids": []string{"s1"}, "doc_id": "docA"},
- {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "knowledge_graph",
- "name_kwd": "OmiyaSoft", "content_with_weight": `{"name":"OmiyaSoft"}`,
- "source_chunk_ids": []string{"s2"}, "doc_id": "docB"},
- },
- chunks: map[string]string{"s1": "a", "s2": "b"},
- loadErrByChunk: map[string]error{"s1": errors.New("store unavailable")},
- }}
- exp := NewCompiledExpander(store, CompiledScopeConfig{
- DatasetIDs: []string{"kb1"},
- TenantID: "t1",
- KBs: []*entity.Knowledgebase{{ID: "kb1", TenantID: "t1"}},
- DocTenantResolver: mapTenantResolver{owners: map[string]DocTenant{
- "docA": {KBID: "kbA", TenantID: "tA"},
- "docB": {KBID: "kbB", TenantID: "tB"},
- }},
- }).(*compiledExpander)
-
- kb := &Kbinfos{}
- if err := exp.Expand(context.Background(), kb, "culdcept", "", nil); err != nil {
- t.Fatalf("Expand: %v", err)
- }
- if got := compiledIDs(kb.Chunks); len(got) != 1 || got[0] != "s2" {
- t.Errorf("expanded = %v, want only s2 (docA's load failed)", got)
- }
- if !strings.Contains(buf.String(), "failed to load chunks for doc_id=docA") {
- t.Errorf("log = %q, want the failing doc named", buf.String())
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/tool_executor.go b/internal/rag/agentic-rag/runtime/tool_executor.go
deleted file mode 100644
index 32517e2dca..0000000000
--- a/internal/rag/agentic-rag/runtime/tool_executor.go
+++ /dev/null
@@ -1,2109 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-// Package runtime holds the low-level agentic-RAG capability library: the
-// retrieval backend, the action-session runtime, the tool executor, and the
-// compiled-structure / knowledge-graph navigation. These are the leaf building
-// blocks the RAGTools methods in the parent `agent` package (Run, ComposeAnswer,
-// ComposeNaiveAnswer, Formalize, ...) orchestrate.
-//
-// tool_executor.go is the tool-executor adapter: its searchExecutor type and the Execute
-// dispatch are the search / navigation / calculation / web tools the action session calls.
-// They are pulled out into this file so the runtime package need not import the
-// agentic_rag package.
-package runtime
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "sort"
- "strings"
- "time"
- "unicode/utf8"
-
- "ragflow/internal/agent/runtime"
- "ragflow/internal/common"
- "ragflow/internal/service/nav"
-)
-
-// RunRequest is one runtime run. It lives in the runtime package (not the agent
-// package) because searchExecutor needs it to carry the per-run query, and the
-// agentic_rag package must not be imported from here (import-cycle rule). RAGTools.Run
-// in the agentic_rag package uses it via the runtime package.
-type RunRequest struct {
- // Question is the user's question.
- Question string
- // ThinkingMode selects the ModeSpec ("low"/"medium"/"high"/"ultra").
- // An unrecognised value degrades to NAIVE.
- ThinkingMode string
- // Keywords narrow retrieved chunks to the sentences mentioning them.
- Keywords string
- // DatasetIDs restricts retrieval. Empty falls back to the canvas context.
- DatasetIDs []string
- // TenantID scopes retrieval. Empty falls back to the canvas context.
- TenantID string
- // UseCompiled enables compiled-structure expansion during retrieval.
- UseCompiled bool
- // TopN is the per-search result count. <=0 selects the default.
- TopN int
- // DeadlineLeft is the wall-clock budget in seconds. <=0 selects the default.
- DeadlineLeft float64
- // MaxLength is the chat model's context window. It bounds evidence and document-level
- // reads.
- // <=0 selects the file-level defaults.
- MaxLength int
- // SessionID identifies the conversation this call belongs to. It is carried
- // for plumbing (e.g. future session-scoped state) but is not read by the
- // near-duplicate answer cache: that cache is per-request, i.e. per turn.
- SessionID string
- // Images are vision-gated base64 data URIs. The
- // outer react loop turns them into multimodal content blocks on the last
- // user message so a vision model sees them (agentic_rag.Rag assembles the
- // message from these).
- Images []string
- // TextAttachments is the joined text-file content, appended to the question so the
- // model reads attached documents in the reasoning path.
- TextAttachments string
-}
-
-// searchExecutor adapts SearchDeps to ToolExecutor, so the retriever and
-// compiled expander are what the session's retrieval tools call.
-type searchExecutor struct {
- deps SearchDeps
- req RunRequest
-}
-
-// NewSearchExecutor builds the retrieval/navigation ToolExecutor used by both
-// the single-session path and the agentic loop's programmatic fan-out fetches.
-//
-// Exported so the agentic_rag package (which cannot be imported from here) can reuse
-// the same evidence handling instead of duplicating it.
-func NewSearchExecutor(deps SearchDeps, req RunRequest) ToolExecutor {
- return &searchExecutor{deps: deps, req: req}
-}
-
-// CoverageRunner is the tool layer's enumeration seam: the runtime asks the EXECUTOR to
-// run the direction's own enumeration over the corpus, so the step belongs to the graph
-// rather than to a prompt.
-//
-// It is a runner rather than a caller-side helper for one reason: the runtime has to be
-// able to ask the corpus itself. Queries handed to the model as a list are advice and may
-// simply not be run, so the enumeration is a step of the graph, and the only thing it needs
-// from the tool layer is a search.
-type CoverageRunner interface {
- EnumerateCoverage(ctx context.Context, cov Coverage, kb *Kbinfos) CoverageSet
-}
-
-// EnumerateCoverage runs the direction's OWN enumeration over the corpus: one recall per
-// operand (the actor's declared forms and the act words), then the windows where the deed
-// is stated (see EnumerateCoverage in coverage_enumerate.go).
-func (e *searchExecutor) EnumerateCoverage(ctx context.Context, cov Coverage, kb *Kbinfos) CoverageSet {
- deps := e.deps
- if len(deps.KbIDs) == 0 && len(e.req.DatasetIDs) > 0 {
- deps.KbIDs = e.req.DatasetIDs
- }
- return EnumerateCoverage(ctx, deps, cov, kb)
-}
-
-// Execute implements ToolExecutor for the wired tools. Tools whose port has not
-// landed are classified, not errored: they report MISS (the tool is valid, this
-// call reached nothing) so the model falls back to a different tool instead of
-// stalling.
-//
-// One tool step is reported as its two halves: the invocation before dispatch
-// and the outcome after it, each to the developer log AND as a step (think text +
-// structured event). Python logs only the invocation (tool_decorator.py:311) and
-// hands the raw payload to the model, which leaves the visible trace one-sided —
-// a call that reached nothing reads exactly like one that found the answer, and
-// the reader cannot tell which evidence the answer was built from.
-func (e *searchExecutor) Execute(ctx context.Context, name string, args map[string]any) (ToolOutcome, error) {
- logger := e.deps.Logger
- renderedArgs := RenderToolArgs(args)
- // "[Function tool] Running the {name} tool with: {args}" — one of the namespaces the
- // thinking block shows. The log line and the step below are the same call site on
- // purpose: a tool step is a step because this method says so, not because a tagged
- // line matched a pattern.
- //
- // The two projections then part: the log and the event's Args keep the argument
- // object Python logs and a developer greps, while the step reports the query
- // (ToolCallLine) — see there for why.
- if logger != nil && name != "" {
- logger.Printf("[Function tool] Running the %s tool with: %s", name, renderedArgs)
- }
- if name != "" {
- StepsFrom(ctx).Emit(ThinkEvent{
- Kind: ThinkKindToolCall,
- Stage: thinkToolStage,
- Tool: name,
- Args: renderedArgs,
- Summary: fmt.Sprintf("[%s] %s", thinkToolStage, ToolCallLine(name, args)),
- })
- }
-
- started := time.Now()
- // The tool's own work reports one level deeper: the search legs it runs (and
- // anything they run) belong to the call line above, not to the caller's level.
- // The result line below stays at the OUTER depth — it closes the call line, so
- // it is a sibling of it, not a child.
- outcome, err := e.dispatch(Nested(ctx), name, args)
- // One sentence template, two labels: the query names the call for both
- // audiences, and the fallback for a document-scoped call (no query argument)
- // exists only on the developer side — the think copy says nothing rather than
- // print an id a reader cannot use.
- label := ArgsLabel(args)
- human := renderToolOutcome(name, label, outcome, err)
- dev := human
- if label == "" {
- if devLabel := docIDLabel(args); devLabel != "" {
- dev = renderToolOutcome(name, devLabel, outcome, err)
- }
- }
- if logger != nil && name != "" {
- logger.Printf("[Function tool] %s", dev)
- }
- if name != "" {
- status := outcome.Status
- if err != nil && status == "" {
- // A transport failure is an error even when the executor returned a
- // zero ToolOutcome (its contract is to set one; this keeps a client
- // from seeing an unset status).
- status = StatusError
- }
- cause := outcome.Diagnostic
- if cause == "" && err != nil {
- cause = err.Error()
- }
- StepsFrom(ctx).Emit(ThinkEvent{
- Kind: ThinkKindToolResult,
- Stage: thinkToolStage,
- Tool: name,
- Args: renderedArgs,
- Status: status,
- Reason: outcome.Reason,
- Cause: cause,
- Results: len(outcome.Payload),
- Documents: len(payloadDocIDs(outcome.Payload)),
- Sources: eventSources(outcome, ThinkMaxSources),
- DurationMS: time.Since(started).Milliseconds(),
- Summary: fmt.Sprintf("[%s] %s", thinkToolStage, human),
- })
- }
- return outcome, err
-}
-
-// thinkToolStage is the stage every tool step is reported under. The two
-// sentences below are logged AND reported as steps here, so no other producer
-// synthesizes a stage event for them.
-const thinkToolStage = "Function tool"
-
-// dispatch routes one tool call to its implementation.
-func (e *searchExecutor) dispatch(ctx context.Context, name string, args map[string]any) (ToolOutcome, error) {
- switch name {
- case "retrieve", "search_chunks", "grep_search", "grep_chunks":
- return e.search(ctx, name, args)
- case "metadata_search":
- return e.metadataSearch(ctx, args)
- case "navigate_tree":
- return e.navigateTree(ctx, args)
- case "navigate_structure":
- return e.navigateStructure(ctx, args)
- case "list_chunks":
- return e.listChunks(ctx, args)
- case "fetch_full_document":
- return e.fetchFullDocument(ctx, args)
- case "summarize_document":
- return e.summarizeDocument(ctx, args)
- case "calculate":
- return e.calculate(ctx, args)
- case "graph_explore":
- return e.graphExplore(ctx, args)
- case "web_search":
- return WebSearchTool(ctx, e.deps, args)
- case "wiki_query":
- return e.wikiQuery(ctx, args)
- }
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": name,
- "note": fmt.Sprintf("%s is not wired in this deployment yet. Use retrieve, search_chunks, navigate_tree or navigate_structure.", name),
- }},
- Status: StatusMiss,
- Reason: ReasonUnwired,
- Metrics: map[string]any{},
- }, nil
-}
-
-// RenderToolArgs renders a tool's arguments for the "[Function tool]" think-log
-// line and the event's Args field. The args dict is marshalled to JSON so nested
-// values (scopes, tag maps) stay readable. Empty args render as "{}" rather than an
-// empty string.
-//
-// String values are capped (capForThink): this field is rendered for display, and
-// one uncapped slot-evidence query turns a single call into a paragraph repeated
-// on every line of its round.
-//
-// Exported because the outer react loop (agentic_rag.outerReactSession) logs the
-// same line for the tools it dispatches itself (rag / summarize_document).
-func RenderToolArgs(args map[string]any) string {
- if len(args) == 0 {
- return "{}"
- }
- b, err := json.Marshal(capArgs(args))
- if err != nil {
- return "{}"
- }
- return string(b)
-}
-
-// QueryLabel renders a tool call's query argument(s) as bare quoted text
-// ("\"曹操是谁\"" or "\"曹操的逝世日期\", \"曹操是谁\""), or "" when the call carries
-// none. It is the ONE place the query-ish keys are listed, so a tool that renames
-// its question field cannot make the think block and the result labels disagree
-// about which calls have a query.
-func QueryLabel(args map[string]any) string {
- for _, key := range []string{"query", "queries", "question"} {
- if label := quoteQueries(args[key]); label != "" {
- return label
- }
- }
- return ""
-}
-
-// ToolCallLine renders the think block's call step for a tool invocation:
-// "Running the retrieve tool with \"曹操是谁\"." — a sentence, not a label.
-//
-// It carries the query and NOTHING else. The arguments the model emitted are a
-// developer's record — document ids, nav hints, kind switches, scope lists — and
-// printing them put raw JSON in front of a reader who only needs to know what was
-// asked. A call with no query names only the tool ("Running the summarize_document
-// tool.") rather than trailing an empty "with" or a document id, and the log keeps
-// the full argument object for the developer who needs to correlate it.
-//
-// No colon after "with": the log line's colon (`with: {json}`, Python
-// tool_decorator.py:311) introduces a dump, and this introduces a name — the
-// sentence reads "running the tool with X". The log keeps its own punctuation.
-//
-// Exported because the outer react loop dispatches its own tools (rag /
-// summarize_document) and reports the same line.
-func ToolCallLine(name string, args map[string]any) string {
- if q := QueryLabel(args); q != "" {
- return fmt.Sprintf("Running the %s tool with %s.", name, q)
- }
- return fmt.Sprintf("Running the %s tool.", name)
-}
-
-// capArgs returns a copy of args with every string capped, walking the shapes a
-// tool argument actually takes: a string, a list of them (a query list), or a
-// nested object.
-func capArgs(args map[string]any) map[string]any {
- out := make(map[string]any, len(args))
- for k, v := range args {
- out[k] = capArgValue(v)
- }
- return out
-}
-
-// capArgValue caps one argument value, recursing through the container shapes.
-func capArgValue(v any) any {
- switch val := v.(type) {
- case string:
- return capForThink(val)
- case []string:
- out := make([]string, len(val))
- for i, s := range val {
- out[i] = capForThink(s)
- }
- return out
- case []any:
- out := make([]any, len(val))
- for i, item := range val {
- out[i] = capArgValue(item)
- }
- return out
- case map[string]any:
- return capArgs(val)
- }
- return v
-}
-
-// renderToolOutcome renders one tool call's result as a sentence for the think
-// block: the outcome status, how many results came back and — for passage-shaped
-// results — how many documents they came from. It closes the "[Function tool]"
-// narration opened by Execute, so one tool step reads as a whole.
-//
-// The sentence names the call's OWN arguments (the caller passes ArgsLabel, or
-// docIDLabel for the developer copy) because a round runs several tool calls
-// concurrently: with fan-out goals in flight, "Running navigate_tree with …" and
-// "The navigate_tree tool returned …" lines interleave, and a result that does not
-// say WHICH query it answers cannot be paired back to the call that produced it —
-// two goals that both returned 1 result from 1 document read identically, which is
-// exactly what a reader must be able to tell apart.
-//
-// The label is the ONLY thing that varies between the two audiences, so the
-// wording above is written once: the same template renders the think sentence and
-// the developer line, and they can never drift.
-//
-// The machine-readable reason is NOT spelled into the sentence. It stays in the
-// tool_result event (Reason/Cause): "no_structure" is a routing token for the
-// session's strike logic, while the sentence says what happened in words and, for
-// failures, carries the producer's own diagnostic via causeSuffix.
-func renderToolOutcome(name, label string, oc ToolOutcome, err error) string {
- if err != nil {
- return fmt.Sprintf("The %s tool failed%s: %v.", name, label, err)
- }
- results := CountOf(len(oc.Payload), "result")
- switch oc.Status {
- case StatusOK:
- return fmt.Sprintf("The %s tool returned %s%s%s.", name, results, documentSuffix(oc.Payload), label)
- case StatusRedundant:
- return fmt.Sprintf("The %s tool returned %s%s, already in the evidence pool.", name, results, label)
- case StatusMiss:
- if oc.Reason == ReasonUnwired {
- return fmt.Sprintf("The %s tool is not wired in this deployment, so nothing ran%s.", name, label)
- }
- return fmt.Sprintf("The %s tool matched nothing%s.", name, orForThisQuery(label))
- case StatusEmpty:
- if oc.Reason == ReasonNoStructure {
- return fmt.Sprintf("The %s tool has no compiled structure to read%s%s.", name, label, causeSuffix(oc.Diagnostic))
- }
- return fmt.Sprintf("The %s tool found nothing available%s%s.", name, label, causeSuffix(oc.Diagnostic))
- case StatusPoor:
- return fmt.Sprintf("The %s tool produced a result too weak to use%s%s.", name, label, causeSuffix(oc.Diagnostic))
- case StatusError:
- return fmt.Sprintf("The %s tool could not run%s%s.", name, label, causeSuffix(oc.Diagnostic))
- }
- return fmt.Sprintf("The %s tool returned %s%s.", name, results, label)
-}
-
-// ArgsLabel renders a tool call's identifying arguments as a trailing clause
-// (" for \"曹操是谁\""), or "" when the call carries nothing that names it. It is
-// what pairs a result line with its call line: the query (or question) is what
-// distinguishes concurrent calls, and it is also the half a reader understands.
-// Exact pairing stays possible in every case because the tool_result event
-// carries the FULL arguments in Args.
-//
-// A document id is deliberately NOT a fallback here: it names nothing a reader
-// knows. Where a document-scoped call has no query, the human sentence simply
-// goes unlabelled and the developer copy uses docIDLabel.
-func ArgsLabel(args map[string]any) string {
- if q := QueryLabel(args); q != "" {
- return " for " + q
- }
- return ""
-}
-
-// docIDLabel is the developer-log fallback ArgsLabel refuses: " for document
-// 95a7aee3…" for a call that named a document and nothing else. It rides the log
-// copy of a sentence only (never the think block), so a developer can pair a
-// document-scoped call with the file it touched.
-func docIDLabel(args map[string]any) string {
- if id, ok := args["doc_id"].(string); ok {
- if id = strings.TrimSpace(id); id != "" {
- return " for document " + id
- }
- }
- return ""
-}
-
-// quoteQueries renders a query argument as a quoted label: a single string, or a
-// list of them rendered as a list. Each entry is capped (capForThink) because the
-// slot-research driver searches a paragraph of evidence as its "query". It returns
-// "" when the argument holds no text, so the caller falls back to the next
-// identifying field.
-func quoteQueries(raw any) string {
- var items []string
- switch v := raw.(type) {
- case string:
- items = []string{v}
- case []string:
- items = v
- case []any:
- for _, item := range v {
- if s, ok := item.(string); ok {
- items = append(items, s)
- }
- }
- default:
- return ""
- }
- quoted := make([]string, 0, len(items))
- for _, s := range items {
- if strings.TrimSpace(s) != "" {
- quoted = append(quoted, fmt.Sprintf("%q", capForThink(s)))
- }
- }
- if len(quoted) == 0 {
- return ""
- }
- return strings.Join(quoted, ", ")
-}
-
-// orForThisQuery keeps a sentence from ending on a dangling preposition when the
-// call named no query: "matched nothing for \"q\"" once labelled, "matched nothing
-// for this query" otherwise.
-func orForThisQuery(label string) string {
- if label == "" {
- return " for this query"
- }
- return label
-}
-
-// causeSuffix renders a tool's own failure explanation as a trailing clause.
-// "reason: infra" answers WHICH bucket the failure fell in; this answers what
-// actually happened, which is the part a reader can act on.
-func causeSuffix(diagnostic string) string {
- if strings.TrimSpace(diagnostic) == "" {
- return ""
- }
- return ": " + diagnostic
-}
-
-// CountOf renders a count with a singular/plural noun ("1 result", "3 results").
-// Shared with the outer react loop's narration (agentic_rag.outerReactSession)
-// so both layers word their "[Function tool]" result lines identically.
-func CountOf(n int, noun string) string {
- if n == 1 {
- return "1 " + noun
- }
- return fmt.Sprintf("%d %s", n, pluralOf(noun))
-}
-
-// pluralOf pluralizes the LAST word of noun ("passage" → "passages"), including
-// the consonant+y rule: appending a bare "s" is what printed "3 first-hop querys"
-// and "3 targeted querys" in the trace.
-func pluralOf(noun string) string {
- if len(noun) >= 2 && noun[len(noun)-1] == 'y' {
- switch noun[len(noun)-2] {
- case 'a', 'e', 'i', 'o', 'u':
- default:
- return noun[:len(noun)-1] + "ies"
- }
- }
- return noun + "s"
-}
-
-// ThinkLabelMaxRunes caps the piece of ONE argument a step's sentence and a tool
-// call's rendering may carry.
-//
-// A tool's "query" is not always a question: the slot-research driver searches a
-// paragraph of slot evidence, and an uncapped label put the same 200 characters
-// on six lines of a single round (running / matched nothing / locate / searching
-// / returned …) until the round was unreadable. The cap is per VALUE, so the
-// sentence stays a sentence.
-const ThinkLabelMaxRunes = 60
-
-// capForThink caps one string for the think block: rune-safe (it never cuts a
-// character in half) and marked with an ellipsis when it was cut.
-func capForThink(s string) string {
- if utf8.RuneCountInString(s) <= ThinkLabelMaxRunes {
- return s
- }
- return truncateRunes(s, ThinkLabelMaxRunes) + "…"
-}
-
-// documentSuffix renders " from N document(s)" for the distinct doc ids a
-// payload carries, or "" when it names none (a computed number or a note has no
-// documents to name).
-func documentSuffix(payload []any) string {
- n := len(payloadDocIDs(payload))
- if n == 0 {
- return ""
- }
- return " from " + CountOf(n, "document")
-}
-
-// payloadDocIDs lists the distinct documents a payload names, in first-seen
-// order. Retrieval payloads carry one doc id per passage ("doc_id");
-// navigate_tree's routed payload names the documents it routed to as a list
-// ("doc_ids"), which is the only thing that outcome reports about the corpus.
-// The structured event and the human sentence are both built from this, so the
-// count a client shows always matches the count in the sentence.
-func payloadDocIDs(payload []any) []string {
- seen := make(map[string]struct{}, len(payload))
- out := make([]string, 0, len(payload))
- add := func(id string) {
- if id == "" {
- return
- }
- if _, dup := seen[id]; dup {
- return
- }
- seen[id] = struct{}{}
- out = append(out, id)
- }
- for _, item := range payload {
- m, ok := item.(map[string]any)
- if !ok {
- continue
- }
- if id, _ := m["doc_id"].(string); id != "" {
- add(id)
- }
- for _, id := range payloadStringList(m["doc_ids"]) {
- add(id)
- }
- }
- return out
-}
-
-// eventSources lists the evidence anchors a tool outcome produced: the chunk ids
-// it admitted, or — for a tool that returns no passages (navigate_tree routes
-// documents, it does not quote them) — the documents it reached. Capped at
-// limit.
-func eventSources(oc ToolOutcome, limit int) []string {
- seen := make(map[string]struct{}, len(oc.EvidenceIDs))
- sources := make([]string, 0, len(oc.EvidenceIDs))
- for _, s := range oc.EvidenceIDs {
- if s == "" {
- continue
- }
- if _, dup := seen[s]; dup {
- continue
- }
- seen[s] = struct{}{}
- sources = append(sources, s)
- }
- if len(sources) == 0 {
- sources = payloadDocIDs(oc.Payload)
- }
- if limit > 0 && len(sources) > limit {
- sources = sources[:limit]
- }
- return sources
-}
-
-// payloadStringList reads a string-list FIELD of a tool payload: the executor
-// builds it as []string, but a payload that made a round trip through a decoded
-// message carries []any. Unlike toolStringList (an ARG reader that also accepts a
-// lone string and coerces scalars), this is a payload read: only real strings
-// count, so a malformed entry is skipped instead of being stringified.
-func payloadStringList(raw any) []string {
- switch v := raw.(type) {
- case []string:
- return v
- case []any:
- out := make([]string, 0, len(v))
- for _, item := range v {
- if s, ok := item.(string); ok {
- out = append(out, s)
- }
- }
- return out
- }
- return nil
-}
-
-func (e *searchExecutor) fetchFullDocument(ctx context.Context, args map[string]any) (ToolOutcome, error) {
- docID := argString(args, "doc_id")
- if docID == "" {
- return ToolOutcome{Payload: []any{}, Status: StatusError, Reason: ReasonBadArgs}, nil
- }
- if e.deps.DocChunks == nil {
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": "fetch_full_document",
- "doc_id": docID,
- "note": "Whole-document reading is not available for this session.",
- }},
- Status: StatusEmpty,
- Reason: ReasonNoDoc,
- }, nil
- }
- chunks, aggs := fetchFullDocument(ctx, e.deps, docID, e.req.MaxLength)
- if len(chunks) == 0 {
- return ToolOutcome{Payload: []any{}, Status: StatusEmpty, Reason: ReasonNoDoc}, nil
- }
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": "fetch_full_document",
- "doc_id": docID,
- "count": len(chunks),
- "chunks": chunks,
- "doc_aggs": aggs,
- "doc_names": []string{docID},
- }},
- Status: StatusOK,
- Metrics: map[string]any{"chunks": len(chunks)},
- }, nil
-}
-
-// summarizeDocument loads the document and hands the model the freshly rendered evidence
-// blocks.
-func (e *searchExecutor) summarizeDocument(ctx context.Context, args map[string]any) (ToolOutcome, error) {
- docID := argString(args, "doc_id")
- if docID == "" {
- return ToolOutcome{Payload: []any{}, Status: StatusError, Reason: ReasonBadArgs}, nil
- }
- if e.deps.DocChunks == nil {
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": "summarize_document",
- "doc_id": docID,
- "note": "Whole-document reading is not available for this session.",
- }},
- Status: StatusEmpty,
- Reason: ReasonNoDoc,
- }, nil
- }
- blocks := summarizeDocument(ctx, e.deps, docID, e.req.MaxLength)
- if len(blocks) == 0 {
- return ToolOutcome{Payload: []any{}, Status: StatusEmpty, Reason: ReasonNoDoc}, nil
- }
- content := ""
- for _, b := range blocks {
- content += b + "\n\n"
- }
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": "summarize_document",
- "doc_id": docID,
- "content": content,
- }},
- Status: StatusOK,
- Metrics: map[string]any{"blocks": len(blocks)},
- }, nil
-}
-
-// navigateTree routes to the top-n documents by descending the dataset's compiled
-// navigation tree, reporting three outcomes for the ladder to act on:
-// - no router configured → EMPTY/no_structure (disable the tool, dataset-level);
-// - tree exists but this query routed to nothing → MISS (fall through to
-// `global`, do NOT disable);
-// - routed → OK with the doc ids and their summaries.
-func (e *searchExecutor) navigateTree(ctx context.Context, args map[string]any) (ToolOutcome, error) {
- query := argString(args, "query")
- if query == "" {
- return ToolOutcome{Payload: []any{}, Status: StatusError, Reason: ReasonBadArgs}, nil
- }
- router := e.navRouter(ctx)
- // The tool argument is NOT threaded (navigate_tree takes query + keywords only), but
- // the SESSION scope still applies: the router ceilings its scope with the session
- // doc_scope. The router must therefore receive the session ceiling, never the tool
- // argument.
- res := NavigateTree(ctx, router, NavTreeInput{
- Query: query,
- // threads ONLY the tool argument's keywords (usually absent, so "") — passing the
- // run-level request keywords re-biased every tree routing toward the original
- // question keywords.
- Keywords: argString(args, "keywords"),
- DocScope: e.deps.DocScope,
- TenantID: e.deps.TenantID,
- KbIDs: e.deps.KbIDs,
- })
-
- switch res.EmptyReason {
- case ReasonNoStructure:
- return ToolOutcome{
- Payload: []any{map[string]any{
- // The dataset_has_compilation gate: the payload carries kind+note only.
- "kind": "navigate_tree",
- "note": "This dataset has no compiled document-navigation structure; use search_chunks / retrieve instead.",
- }},
- Status: StatusEmpty,
- Reason: ReasonNoStructure,
- }, nil
- case ReasonNoDoc:
- // Structure exists, this query reached nothing — a MISS, not an EMPTY.
- // note verbatim; the payload carries kind+note only.
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": "navigate_tree",
- "note": "navigate_tree routed to no document for this query. Rephrase the query, or use search_chunks / retrieve instead.",
- }},
- Status: StatusMiss,
- Reason: ReasonNoDoc,
- }, nil
- case ReasonInfra, ReasonBadArgs:
- return ToolOutcome{
- Payload: []any{},
- Status: ReasonStatus(res.EmptyReason),
- Reason: res.EmptyReason,
- // Carry WHY the descent failed: the reason bucket alone ("infra")
- // leaves a reader with nothing to act on.
- Diagnostic: res.Diagnostic,
- }, nil
- }
-
- // Routed: expose the summary-bearing payload the ladder consumes, capped at 8000
- // chars.
- content := res.Text
- if len(content) > 8000 {
- content = content[:8000]
- }
- payload := []any{map[string]any{
- "kind": "navigate_tree",
- "content": content,
- "doc_ids": res.DocIDs,
- }}
- return ToolOutcome{
- Payload: payload,
- EvidenceIDs: nil, // routing only — no passages retrieved
- Status: StatusOK,
- Reason: ReasonNone,
- Metrics: map[string]any{
- "docs": len(res.DocIDs),
- "routed_docs": res.RoutedDocs,
- },
- }, nil
-}
-
-// chunkAggRetrieveFrom adapts a runtime Retriever into the chunk_agg retrieve
-// leg. It scopes the search to the dataset (kbID) and the caller's document
-// scope and pulls the wide pool chunk_agg needs; the backend is expected to
-// exclude compiled rows (the production retrieval filters available_int=1).
-//
-// It stays in the runtime package because it depends on the agentic Retriever /
-// RetrieveRequest; the routing algorithm itself lives in internal/service/nav.
-func chunkAggRetrieveFrom(r Retriever) nav.ChunkRetriever {
- return func(ctx context.Context, tenantID, kbID, query string, docScope []string, topN int, _ float64) ([]map[string]any, error) {
- chunks, err := r.Retrieve(ctx, RetrieveRequest{
- Query: query,
- DatasetIDs: []string{kbID},
- TenantID: tenantID,
- DocScope: docScope,
- TopN: topN,
- // page_size here IS the pool (Python passes `pool` positionally as
- // page_size, dataset_api_service.py:4106-4129), and the retrieval
- // service refuses `page * page_size > rerank_candidates_count`
- // (nlp/retrieval.go:126, mirroring rag/nlp/search.py:745). Python
- // therefore raises the candidate count to the same pool for this
- // leg (`rerank_candidates_count=pool`); omitting it left the default
- // 64 against a 256-wide pool, so EVERY nav-tree descent failed with
- // "rerank_candidates_count(64) must be greater than or equal to
- // page(1) * page_size(256)" and reported infra.
- RerankCandidatesCount: topN,
- // `must_not={"exists": "compile_kwd"}` (_NAV_CHUNK_AGG_EXCLUDE_COMPILED
- // = True): this leg aggregates ORIGINAL chunks per document, and a
- // compiled product row would otherwise be attributed to a document
- // as if it were a passage.
- ExcludeCompiled: true,
- })
- if err != nil {
- // Surface the failure instead of folding it into an empty result:
- // the router reports it as an error so the orchestrator takes its
- // fallback rather than telling the model the tree routed nothing.
- return nil, err
- }
- return chunks, nil
- }
-}
-
-// navigateStructure pinpoints passages inside ONE document using its compiled
-// structure (heading tree / concept mindmap).
-//
-// It delegates to the navigation package's NavigateStructure, which loads the
-// document's compiled rows, asks the model which entities answer the query, and
-// returns the underlying source chunks.
-//
-// NOTE on shapes: the Go reader (navigation.loadStructureEntities) currently
-// reads only the compact graph-blob rows, whereas BOTH row shapes (graph blob AND
-// per-entity/relation rows) should be merged. ParseCompiledStructure in navtools.go
-// already implements the merged parsing; wiring it into the reader is the remaining
-// step.
-func (e *searchExecutor) navigateStructure(ctx context.Context, args map[string]any) (ToolOutcome, error) {
- query := argString(args, "query")
- docID := argString(args, "doc_id")
- if query == "" {
- query = e.req.Question
- }
- if query == "" {
- return ToolOutcome{Payload: []any{}, Status: StatusError, Reason: ReasonBadArgs}, nil
- }
- // No scope-as-doc-pin: only args["doc_id"] is read — args["doc_scope"] is never
- // consulted.
- kind := argString(args, "kind")
- if kind == "" {
- kind = "catalog"
- }
-
- // Resolve the document set. When the caller omits doc_id, route to the documents by
- // descending the compiled navigation tree (the exact seam navigate_tree uses), not by
- // refusing the call. The routed set is capped at navTreeMaxDocs.
- var docIDs []string
- if docID != "" {
- docIDs = []string{docID}
- } else {
- // DocScope is the session ceiling, not the tool argument (see the note in
- // navigateTree): args["doc_scope"] is never threaded, but the routing done when
- // doc_id is absent applies the session document scope.
- res := NavigateTree(ctx, e.navRouter(ctx), NavTreeInput{
- Query: query,
- Keywords: e.req.Keywords,
- DocScope: e.deps.DocScope,
- TenantID: e.deps.TenantID,
- KbIDs: e.deps.KbIDs,
- })
- if res.EmptyReason == ReasonInfra {
- return ToolOutcome{Payload: []any{}, Status: ReasonStatus(res.EmptyReason), Reason: res.EmptyReason, Diagnostic: res.Diagnostic}, nil
- }
- docIDs = res.DocIDs
- if len(docIDs) > navTreeMaxDocs {
- docIDs = docIDs[:navTreeMaxDocs]
- }
- if len(docIDs) == 0 {
- // Routing reached no document (empty_reason="no_doc"). The SAME note is emitted
- // for every empty_reason.
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": "navigate_structure",
- "doc_id": docID,
- "note": fmt.Sprintf("No compiled structure of kind='%s' reachable for this document. Try another doc_id or kind, or use search_chunks / retrieve / list_chunks.", kind),
- }},
- Status: StatusMiss,
- Reason: ReasonNoDoc,
- }, nil
- }
- }
-
- // Zero-LLM vector-beam drill-down: read each document's compiled structure of the
- // requested kind, drill toward the query, and hand the model the merged
- // <structure_navigation> outline with chunk-pointer anchors.
- res, drill := navigateStructures(ctx, e.deps.TenantID, query, docIDs, kind, e.navRouter(ctx), e.deps)
- metrics := map[string]any{
- "entities": drill.nodes,
- "chunk_ptrs": drill.chunkPtrs,
- "top_score": drill.topScore,
- "chunk_paths": drill.chunkPaths,
- "claim_hits": drill.claimHits,
- }
- if res.EmptyReason != "" {
- // one note for every empty_reason; kind renders quoted.
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": "navigate_structure",
- "doc_id": docID,
- "note": fmt.Sprintf("No compiled structure of kind='%s' reachable for this document. Try another doc_id or kind, or use search_chunks / retrieve / list_chunks.", kind),
- }},
- Status: ReasonStatus(res.EmptyReason),
- Reason: res.EmptyReason,
- Metrics: metrics,
- }, nil
- }
- // Reached structures but drilled to nothing usable: not an error, just weak —
- // the orchestrator falls back to retrieval on status == poor. Claim-first
- // hits are the exception: the claims carry the answer material and load no
- // chunk snippets, so a zero chunk_ptr count with matched claims is a success.
- status := StatusOK
- if drill.chunkPtrs == 0 && drill.claimHits == 0 {
- status = StatusPoor
- }
- content := res.Text
- if len(content) > 8000 {
- content = content[:8000]
- }
- evidenceIDs := res.DocIDs
- if len(evidenceIDs) == 0 && len(docIDs) > 0 {
- evidenceIDs = docIDs
- }
- return ToolOutcome{
- Payload: []any{map[string]any{"kind": "navigate_structure", "doc_id": firstOr(docIDs, ""), "content": content}},
- EvidenceIDs: evidenceIDs,
- Status: status,
- Reason: ReasonNone,
- Metrics: metrics,
- }, nil
-}
-
-// navRouter returns the navigation-tree router navigateTree and navigateStructure
-// share, decided in this order:
-//
-// 1. The dataset has a COMPILATION. Navigation routes over compiled structure, so a
-// dataset carrying no compiled rows has nothing to route on and gets
-// noCompilationRouter — the tool then reports no_structure and the session is told to
-// use search_chunks / retrieve, i.e. plain search. Without this gate the route stood
-// behind an uncompiled dataset too, because the chunk leg is a plain similarity
-// ranking that needs no compilation: raw retrieval came back dressed as a structure
-// descent, and the rounds went to "navigation" results that name no structure.
-// 2. It follows the agentic router choice: the navigation-tree route asks for
-// router="claim_agg" — the claim leg runs first and decides the ranking when it hits,
-// and raw-chunk aggregation (chunk_agg) is the fallback. A nil retrieval backend
-// degrades to the nav-row router, which needs no backend.
-//
-// A caller-installed NavRouter is an explicit override and skips both.
-func (e *searchExecutor) navRouter(ctx context.Context) NavTreeRouter {
- if e.deps.NavRouter != nil {
- return e.deps.NavRouter
- }
- if !DatasetHasCompilation(ctx, e.deps) {
- return noCompilationRouter{}
- }
- return defaultNavRouter(e.deps)
-}
-
-// noCompilationRouter is what an UNCOMPILED dataset gets: Route answers (nil, nil), the
-// NavTreeRouter contract's "this dataset has no compiled tree", so NavigateTree reports
-// no_structure (dataset-level, and its tool note says to use search_chunks / retrieve)
-// instead of routing the query through raw-chunk aggregation.
-type noCompilationRouter struct{}
-
-// Route implements NavTreeRouter: no compiled structure, nothing to route, not an error.
-func (noCompilationRouter) Route(context.Context, string, string, string, []string, int) ([][2]string, error) {
- return nil, nil
-}
-
-// defaultNavRouter builds the claim_agg router over the chunk_agg fallback.
-// See ClaimAggRouter and nav.NewChunkAggRouter.
-func defaultNavRouter(deps SearchDeps) NavTreeRouter {
- if deps.Backend == nil {
- return nav.NewNavServiceRouter()
- }
- return &ClaimAggRouter{
- Deps: deps,
- Fallback: nav.NewChunkAggRouter(chunkAggRetrieveFrom(deps.Backend), nav.ChunkAggSummarize()),
- Summarize: nav.ChunkAggSummarize(),
- }
-}
-
-func firstOr(xs []string, def string) string {
- if len(xs) > 0 {
- return xs[0]
- }
- return def
-}
-
-// argString reads a string arg, treating an absent key or an explicit JSON null
-// as empty. fmt.Sprint(nil) yields "<nil>", so the presence check must come
-// first — otherwise a missing arg silently becomes the literal "<nil>".
-func argString(args map[string]any, key string) string {
- raw, ok := args[key]
- if !ok || raw == nil {
- return ""
- }
- s := strings.TrimSpace(fmt.Sprint(raw))
- if s == "<nil>" {
- return ""
- }
- return s
-}
-
-// evidencePoolCap is the hard cap on the shared evidence pool. It is deliberately LARGER
-// than the SCA view cap (60) so storage and review stay DECOUPLED — the pool accumulates
-// while the SCA reads a ranked top-60 view. Coupling them at 60 starved the raw-evidence
-// channel in 42% of rounds (every admit rejected -> status REDUNDANT -> the model
-// re-searched for nothing).
-//
-// Claim pseudo-chunks BYPASS this cap: they are appended directly to the pool, and the cap
-// only guards regular chunk admission.
-//
-// The cap is 200 (the old 120 was sized for a consumer that no longer exists) —
-// the round-level sweep that rendered the WHOLE pool into one prompt, where the cap
-// and that prompt's budget were the same number. Nothing renders the pool whole any
-// more (the SCA reads a ranked 60-chunk view, the session seed injects a bounded
-// digest, the draft is bounded), so the cap is a storage-discipline number again —
-// and on an enumeration it is the NEXT ceiling rather than a prompt limit: what it
-// refuses is the passage that would have reached the last members, which are named
-// late in the pool's order.
-const evidencePoolCap = 200
-
-// The cap check and its "pool FULL" line now live on PoolAdmitter.Full, where
-// the pool lock is held (see kbinfos.go): the check must not read len(Chunks)
-// while another session appends.
-
-// probeTerms returns the terms of a PROBE query — the caller's own alternation,
-// "荀正|管亥|车胄" — and nil for every other query shape.
-//
-// An alternation is the one place the model states explicitly WHICH individuals
-// it is asking about, which makes its per-term result the batch's answer and not
-// just another search: a name that comes back empty is a name this corpus does
-// not carry, and a name that comes back with a window is a member. Both facts are
-// destroyed by a cap that drops the window (see PoolAdmitter.Novelty), so a probe
-// is exempt while a topic query is not — a topic query names no individuals, so
-// its hits compete for the cap like everything else.
-func probeTerms(q string) []string {
- if !strings.Contains(q, "|") {
- return nil
- }
- return GrepTermsFromQuery(q)
-}
-
-// namedTermsOf is the call's own statement of WHICH individuals it asked about:
-// the terms of the queries it carried, deduped and capped.
-//
-// Every shape the model uses lands here — an alternation ("A|B|C"), a
-// space-separated list inside one string, or a list of query strings — because
-// GrepTermsFromQuery already reads all three. That is the point: the seat
-// mechanism is attached to the FACT that the call named terms, not to a syntax
-// the model may never write — a `|`-triggered mechanism would fire never.
-//
-// The cap (GrepTermsMax) bounds the seat pass, which runs one cheap keyword
-// search per unreached term; the caller logs how many named terms were dropped
-// so the ceiling is visible in the run.
-//
-// The terms are the caller's OWN WORDS (GrepWordsFromQuery), not the CJK windows
-// the locate step derives from them: a window is our guess at where a name can be
-// found, and probing one spends a retrieval on a fragment nobody asked about.
-func namedTermsOf(queries []string) []string {
- var out []string
- seen := make(map[string]bool, len(queries)*2)
- for _, q := range queries {
- for _, t := range GrepWordsFromQuery(q) {
- // A PIECE OF A PATTERN is a phrase, not a name: an alternation carrying
- // operators asks how a deed is written, and probing its pieces as names
- // spends a retrieval on a word nobody proposed as a member. An alternation
- // of PLAIN words is exactly what the seat exists for, so the test is pattern
- // OPERATORS, not "|".
- if strings.ContainsAny(t, ".*+?()[]{}^$\\") {
- continue
- }
- low := strings.ToLower(strings.TrimSpace(t))
- if low == "" || seen[low] {
- continue
- }
- seen[low] = true
- out = append(out, t)
- if len(out) >= GrepTermsMax {
- return out
- }
- }
- }
- return out
-}
-
-// search runs one retrieval call for a tool invocation.
-//
-// The two retrieval tools share this body, differing only in whether compiled expansion
-// runs (search_chunks expands, retrieve does not) and in the accepted query count.
-func (e *searchExecutor) search(ctx context.Context, name string, args map[string]any) (ToolOutcome, error) {
- queries := toolQueries(args)
- if len(queries) == 0 {
- // A missing/blank query yields no queries and simply admits nothing: the outcome is
- // MISS/no_doc, NOT a bad-args error.
- return ToolOutcome{
- Payload: []any{},
- EvidenceIDs: nil,
- Status: StatusMiss,
- Reason: ReasonNoDoc,
- Metrics: map[string]any{"hits": 0, "new_evidence": 0},
- }, nil
- }
- // The pool is CREATED when absent, so every search has a (possibly empty) pool to admit
- // into: the search runs and its outcome is decided by what it actually admitted.
- if e.deps.KB == nil {
- e.deps.KB = &Kbinfos{}
- }
- logger := searchLogger(e.deps)
-
- // Max queries per tool call: retrieve=3, search_chunks=2. grep_search/grep_chunks are
- // Go-internal tools, so they are
- // left uncapped.
- //
- // A direction assembling a SET/COUNT raises the ceiling. There the call's
- // queries are facets of one list rather than rephrasings of one question, so
- // a dropped query is not a spared repeat — it is a member nobody searched.
- // The flag is set by the same gate that hands the model the set method (see
- // Kbinfos.MarkSetDirection), so a VALUE direction keeps the small cap it had.
- // Calls ask several queries apiece, and a name the answer later turns out to be
- // missing was often named only in a dropped one.
- maxQ := len(queries)
- switch name {
- case "retrieve":
- maxQ = 3
- case "search_chunks":
- maxQ = 2
- }
- if e.deps.KB.IsSetDirection() {
- switch name {
- case "retrieve":
- maxQ = 6
- case "search_chunks":
- maxQ = 4
- }
- }
- // The full list is kept: maxQ bounds how many EXPENSIVE queries run, but the
- // terms of the dropped ones still get their own cheap seat below. Dropping a
- // list item silently (as this cut used to) drops the individuals it named
- // before any retrieval happens, and those individuals can be the only mentions of
- // the very names the answer is missing.
- allQueries := queries
- dropped := 0
- if len(queries) > maxQ {
- queries = queries[:maxQ]
- dropped = len(allQueries) - maxQ
- logger.Printf("[Action Session] %s asked %d quer(ies); running %d and taking named-term seats for the rest.", name, len(allQueries), maxQ)
- }
-
- var payload []any
- var evidenceIDs []string
- newChunks := 0
- // Per-call admittance state: `seen` dedups chunks ACROSS the queries of this one call.
- // The pool-side dedup is Kbinfos.Admit's job, against the LIVE pool — a per-call
- // snapshot of it is exact only while nothing can interleave, and another session
- // appending makes it stale.
- seen := map[string]bool{}
- // reached accumulates every candidate the call's own searches returned. It is
- // what the seat pass below asks "which of the named terms did this retrieval
- // NOT reach?" — the question whose answer is a search of its own.
- var reached []map[string]any
- // Per-call reach notes: what each query reached term by term, and which of its
- // terms nothing reached (see ToolOutcome.Note / GrepReachLine). A batch of
- // names is exactly where this matters — the passages alone cannot say whether
- // a member was missing or simply never asked about.
- var reachNotes []string
- // Claim-first, MUTUALLY EXCLUSIVE: when claim rows hit, their verbatim evidence IS the
- // answer material — chunk snippets on top would echo the same passages and burn tokens.
- // Claims carry chunk pointers, so deep-reading stays one list_chunks away. No hits →
- // the chunk search runs exactly as before. Applies to the whole retrieve family +
- // search_chunks. Best effort: any failure falls through without claims.
- if name == "retrieve" || name == "search_chunks" || strings.HasPrefix(name, "grep") {
- if len(queries) > 0 {
- if claimPayload, _, claimPseudo, ok := ClaimPrefetch(ctx, e.deps, queries[0], seen); ok {
- newEvidence := 0
- e.deps.KB.Admit(func(p *PoolAdmitter) {
- for _, pc := range claimPseudo {
- // Claim pseudo-chunks BYPASS the pool cap: they are appended directly to the
- // pool and the cap only guards regular chunk admits — the verbatim
- // evidence must land in the pool even when it is FULL.
- cid := ChunkIDOf(pc)
- if seen[cid] {
- continue
- }
- seen[cid] = true
- evidenceIDs = append(evidenceIDs, cid)
- payload = append(payload, passageFromChunk(pc))
- if p.Add(pc) {
- newEvidence++
- }
- }
- })
- status := StatusOK
- if newEvidence == 0 {
- status = StatusRedundant
- }
- return ToolOutcome{
- Payload: payload,
- EvidenceIDs: evidenceIDs,
- Status: status,
- Reason: ReasonNone,
- Metrics: map[string]any{"hits": len(claimPayload), "new_evidence": newEvidence, "claims": len(claimPayload)},
- }, nil
- }
- }
- }
- for _, q := range queries {
- // Per-tool top_n: retrieve=10, search_chunks=20. Collapsing both onto e.req.TopN
- // made search_chunks return far fewer candidates.
- topN := 10
- if name == "search_chunks" {
- topN = 20
- }
- // These tools dispatch to genuinely different search functions: retrieve/grep_* →
- // grep_search (keyword-only), search_chunks → hybrid_search (vector leg when an
- // embedder is configured). Each is its own function so a single global switch can no
- // longer disable the semantic leg.
- //
- // search_chunks ALWAYS enables compiled-structure expansion, in ALL modes, not just
- // high; retrieve/grep_search never do. So compiled is on for search_chunks and off for
- // every other retrieve-family tool, independent of e.req.UseCompiled (which gates the
- // L1 direct retrieve, not the action-session tool loop).
- var searchFn func(context.Context, SearchDeps, SearchParams) ([]map[string]any, []map[string]any)
- useCompiled := name == "search_chunks"
- if useCompiled {
- searchFn = HybridSearch
- } else {
- searchFn = GrepSearch
- }
- // Keywords differ per tool:
- // - retrieve → grep_search(keywords=nav_hint or None). The nav hint is an explicit
- // PARAMETER, passed ONLY by the navigation ladder; the model's own retrieve
- // dispatch passes none, in which case GrepSearch falls back to the query's own
- // extracted terms and turns them into the BM25 hint. The session run keywords
- // (req.Keywords) are never forwarded.
- // - search_chunks takes no keywords at all, so hybrid_search's narrowing is a
- // no-op.
- var kws string
- if !useCompiled {
- kws = argString(args, "nav_hint")
- }
- // doc_scope is honoured by the retrieve family only: retrieve reads
- // args["doc_scope"], while search_chunks takes no doc_scope at all and its
- // hybrid_search call is unscoped. Gating on useCompiled keeps that split.
- var docScope []string
- if !useCompiled {
- docScope = toolDocScope(args)
- }
- // The aggregations the search body would otherwise drop are what the answer's
- // document/reference list is built from — this tool is their only writer.
- chunks, aggs := searchFn(ctx, e.deps, SearchParams{
- Question: q,
- Keywords: kws,
- UseCompiled: useCompiled,
- TopN: topN,
- KbIDs: e.req.DatasetIDs,
- DocScope: docScope,
- })
- if len(chunks) == 0 {
- continue
- }
- // Computed on the LEG's candidates, not on the payload below: the payload is
- // truncated to the per-query snippet cap, and a term whose window was cut is
- // still a term this query reached.
- if line := GrepReachLine(q, chunks, ReachTermsOf(q)); line != "" {
- reachNotes = append(reachNotes, line)
- }
- // Only the first snippetsPerQueryFor(mode) hits of each query are considered
- // — EXCEPT for a query that NAMES
- // terms, which keeps one candidate per named term first. The locate step
- // hands back one window per term, and a flat cut is what turns a six-name
- // call into "the names that matched most": the rarest lose their seat to the
- // ones the ranking already preferred.
- limit := snippetsPerQueryFor(e.req.ThinkingMode)
- if n := len(namedTermsOf([]string{q})); n > limit {
- limit = n
- }
- if len(chunks) > limit {
- chunks = chunks[:limit]
- }
- reached = append(reached, chunks...)
- // Admittance mirrors _admit_evidence exactly: per-call dedup by chunk
- // id, the chunk ID as the evidence reference (ids, not pool positions), and only
- // chunks NEW to the shared pool appended to it — so REDUNDANT means "nothing new",
- // not "nothing returned".
- //
- // ONE query's batch is one critical section: the per-query loop has no await, so
- // two sessions' batches cannot interleave. Locking per chunk would let them
- // interleave into pool orders that should not happen.
- e.deps.KB.Admit(func(p *PoolAdmitter) {
- // The claim-covered set is computed from the LIVE pool once per batch, under the
- // same critical section.
- covered := p.ClaimCoveredIDs()
- // The cap exemption for this batch, derived from the probe's own terms
- // (see probeTerms / PoolAdmitter.Novelty): a full pool still takes the
- // window that answers a name nothing in the pool has reached yet,
- // because that window is the batch's result rather than one more
- // passage.
- novel := p.Novelty(probeTerms(q))
- for _, c := range chunks {
- // Admission early-stops once the shared pool reaches the cap, BEFORE the
- // per-call dedup — except for the probe window above, which IS the answer the
- // caller asked for.
- if p.Full() && !novel.Admits(c) {
- continue
- }
- cid := ChunkIDOf(c)
- if seen[cid] {
- continue
- }
- // already quoted verbatim by a pooled claim →
- // skip the full passage (table chunks exempt: their answer rows
- // survive only in full text). Not pooled, not passed to the model.
- if p.CoveredByClaim(cid, covered, IsTableChunk(c)) {
- continue
- }
- seen[cid] = true
- evidenceIDs = append(evidenceIDs, cid)
- payload = append(payload, passageFromChunk(c))
- // Pool identity uses chunkKey (a stable key): an address-based key would never
- // match an equivalent re-retrieved chunk, and it would be appended again.
- if p.Add(c) {
- newChunks++
- }
- }
- })
- // Pool this search's doc_aggs (skipped when it returned no chunks, like
- // _merge_kbinfos).
- e.deps.KB.MergeDocAggs(aggs)
- }
-
- // ── Named-term seats ─────────────────────────────────────────────────────
- // The individuals this call named are honored IN FULL, independent of the
- // expensive leg's own limits: one cheap keyword search per term the call's
- // retrievals did not reach, each keeping the window that carries it (see
- // TermSeat). The terms of list items maxQ dropped are included — "the model
- // already said the name" should mean the name was looked for, whatever
- // syntax carried it and whichever leg ran.
- //
- // A term that reaches nothing is RECORDED, not retried (Kbinfos.RecordProbedAbsent):
- // "this corpus has no 荀正" is the answer to a probe, and it is what the
- // rewrite reads before choosing its next angle.
- if named := namedTermsOf(allQueries); len(named) > 0 {
- seatScope := []string(nil)
- if name != "search_chunks" {
- // doc_scope is a retrieve-family argument; search_chunks takes none
- // (see the query loop above).
- seatScope = toolDocScope(args)
- }
- unreached := termsNotCarried(reached, named)
- // Every named term is PROBED (recall: its own window enters the pool), but
- // only the ones the call proposed as ITEMS are RECORDED — the ledger is read
- // back as the session's to-do list, and a question's words are not members
- // it could record (see probeItemsOf).
- proposed := probeItemsOf(allQueries)
- seats, absent := 0, 0
- for _, term := range unreached {
- record := proposed[strings.ToLower(term)]
- seat, found := TermSeat(ctx, e.deps, SearchParams{
- KbIDs: e.req.DatasetIDs,
- DocScope: seatScope,
- }, term)
- if !found {
- absent++
- if record {
- e.deps.KB.RecordProbedAbsent(term)
- }
- continue
- }
- // The seat's ids are collected here and recorded AFTER the batch: the
- // ledger has its own lock, so writing it inside the critical section
- // would be safe, but keeping the pool lock to pool work costs nothing
- // and leaves the locking order (pool → ledger) exercised in one place
- // only.
- var seatedIDs []string
- e.deps.KB.Admit(func(p *PoolAdmitter) {
- // A seat is a probe's answer, so it is exempt from the pool cap
- // on the same grounds as the weave above (see Novelty).
- novel := p.Novelty([]string{term})
- for _, c := range seat {
- if p.Full() && !novel.Admits(c) {
- continue
- }
- cid := ChunkIDOf(c)
- if cid != "" && seen[cid] {
- continue
- }
- if cid != "" {
- seen[cid] = true
- }
- evidenceIDs = append(evidenceIDs, cid)
- payload = append(payload, passageFromChunk(c))
- if p.Add(c) {
- newChunks++
- }
- if cid != "" {
- seatedIDs = append(seatedIDs, cid)
- }
- seats++
- }
- })
- // A seat IS the proof that this name is a member: record the pair
- // (term, passage) so the round's record holds the members with their
- // evidence, not just the count — again only for a proposed item.
- if record {
- for _, cid := range seatedIDs {
- e.deps.KB.RecordReachedTerm(term, cid)
- }
- }
- }
- if len(unreached) > 0 {
- logger.Printf("[Action Session] named-term seats: %d named, %d unreached, %d seat(s) admitted, %d absent.",
- len(named), len(unreached), seats, absent)
- }
- }
-
- // A cut is a fact about the SEARCH, not about the corpus, and the model cannot
- // read it off the passages: every dropped query's terms were seated above, so
- // a name it asked about still came back, while the query's own wording never
- // ran. Say so, so a dropped facet is re-asked instead of forgotten.
- if dropped > 0 {
- reachNotes = append(reachNotes, fmt.Sprintf(
- "only %d of this call's %d queries were searched — %d were dropped, and a dropped query's own wording was never run (the terms it named were each probed on their own, so a term that came back is proven to occur). Re-ask a dropped one directly if you need its wording.",
- maxQ, len(allQueries), dropped))
- }
- if len(payload) == 0 {
- return ToolOutcome{
- Payload: []any{},
- EvidenceIDs: nil,
- Status: StatusMiss,
- Reason: ReasonNoDoc,
- Metrics: map[string]any{"hits": 0, "new_evidence": 0},
- Note: strings.Join(reachNotes, "\n"),
- }, nil
- }
- // Zero new evidence (every hit was already in the pool) is REDUNDANT, not
- // OK. The FULL payload is still returned here — the model sees the passages AND a
- // redundant status, so it knows the ground is already covered. The session tool node
- // appends the "ALREADY in your evidence" note on StatusRedundant, which is what stops
- // the re-issue; dropping the payload (as an earlier version did) hid the evidence the
- // model needs.
- if newChunks == 0 {
- return ToolOutcome{
- Payload: payload,
- EvidenceIDs: evidenceIDs,
- Status: StatusRedundant,
- Reason: ReasonNone,
- Metrics: map[string]any{"hits": len(payload), "new_evidence": 0},
- Note: strings.Join(reachNotes, "\n"),
- }, nil
- }
- return ToolOutcome{
- Payload: payload,
- EvidenceIDs: evidenceIDs,
- Status: StatusOK,
- Reason: ReasonNone,
- Metrics: map[string]any{"hits": len(payload), "new_evidence": newChunks},
- Note: strings.Join(reachNotes, "\n"),
- }, nil
-}
-
-// metadataKeysHintMax caps the available-keys list echoed back on a bad key.
-const metadataKeysHintMax = 30
-
-// metadataSearch is the metadata_search tool: a document-metadata SELECTOR. It resolves the
-// documents a filter matches and returns their doc_ids — it runs NO retrieval of its own.
-//
-// The ids are the whole output on purpose: a document handle can be spent on several
-// follow-up calls (list_chunks, navigate_structure, retrieve's doc_scope, and in ultra
-// mode graph_explore's doc_scope), whereas the passages an in-tool retrieval used to return
-// could only be read once. Keeping the selector free of retrieval is also what keeps its
-// answer "which documents match" independent of what the model meant to search for.
-//
-// The statuses are set explicitly rather than derived (see ReasonStatus): a missing filter
-// or a metadata key the dataset does not carry is MISS/bad_args — the model should change
-// the filter or switch tools, not read it as an infrastructure failure — a filter matching
-// no document is MISS/no_doc, and only a metadata-index read failure is ERROR/infra.
-// (Python folds a keys-query exception into "this dataset has no metadata", which tells the
-// model the wrong thing about the corpus.)
-func (e *searchExecutor) metadataSearch(ctx context.Context, args map[string]any) (ToolOutcome, error) {
- // searchLogger falls back to the package logger: Logger is optional on SearchDeps,
- // and a nil *log.Logger panics on the first Printf.
- logger := searchLogger(e.deps)
- badArgs := func(note string) (ToolOutcome, error) {
- return ToolOutcome{
- Payload: []any{map[string]any{"kind": "metadata_search", "note": note}},
- Status: StatusMiss,
- Reason: ReasonBadArgs,
- Metrics: map[string]any{"docs": 0},
- }, nil
- }
- filters := metadataFiltersOf(args)
- if len(filters) == 0 {
- return badArgs("No metadata conditions given. metadata_search needs at least one {key, value, op} filter — otherwise use search_chunks / retrieve.")
- }
- if e.deps.MetadataResolver == nil {
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": "metadata_search",
- "note": "metadata_search is unavailable in this deployment (no metadata resolver is wired). Use search_chunks / retrieve.",
- }},
- Status: StatusError,
- Reason: ReasonInfra,
- Metrics: map[string]any{"docs": 0},
- }, nil
- }
- targetIDs := metadataTargetIDs(e.deps)
- if len(targetIDs) == 0 {
- return ToolOutcome{
- Payload: []any{},
- Status: StatusError,
- Reason: ReasonInfra,
- Metrics: map[string]any{"docs": 0},
- }, nil
- }
-
- // 1) Key validation against the dataset's OFFERED metadata fields — the same union of
- // declared and indexed fields the tool schema advertises (see resolveMetadataFields).
- // Validating against anything else would let the schema offer a key the tool then
- // rejects; a dataset without the requested key must degrade to a hint, never to an
- // empty retrieval the model retries forever.
- known, _, declared, err := resolveMetadataFields(ctx, e.deps)
- if err != nil {
- logger.Printf("[Metadata search] metadata index read failed: %v", err)
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": "metadata_search",
- "note": "The document-metadata index could not be read (infrastructure failure — NOT a statement about the dataset). Fall back to search_chunks / retrieve.",
- }},
- Status: StatusError,
- Reason: ReasonInfra,
- Metrics: map[string]any{"docs": 0},
- }, nil
- }
- knownSet := make(map[string]bool, len(known))
- for _, k := range known {
- knownSet[k] = true
- }
- var bad []string
- for _, f := range filters {
- if key := asString(f["key"]); !knownSet[key] {
- bad = append(bad, key)
- }
- }
- if len(bad) > 0 {
- available := strings.Join(capStrings(known, metadataKeysHintMax), ", ")
- if available == "" {
- available = "NONE — this dataset has no metadata; use search_chunks / retrieve"
- }
- logger.Printf("[Metadata search] bad key(s) %v — not in the dataset's metadata (available: %s)", bad, available)
- return badArgs(fmt.Sprintf("Metadata key(s) %v do not exist in this dataset. Available: %s.", bad, available))
- }
-
- // 2) Normalize the filter values BEFORE they reach the push-down / in-memory filter, so
- // a model that emits a list where one keyword is expected can never silently produce
- // a no-match.
- normalized := make([]map[string]any, 0, len(filters))
- for _, f := range filters {
- nf := make(map[string]any, len(f)+1)
- for k, v := range f {
- nf[k] = v
- }
- nf["value"] = NormalizeMetadataValue(f["value"], asString(f["op"]))
- normalized = append(normalized, nf)
- }
- logic := argString(args, "logic")
- if logic != "or" {
- logic = "and"
- }
-
- // 3) Resolve the document set. This IS the tool's output: the ids are returned to the
- // model so it can spend them on the search tools that take a document handle, and the
- // filter itself is query-independent (it never needed a query).
- docIDs, ok := MetadataDocIDs(ctx, e.deps, normalized, logic)
- if !ok {
- logger.Printf("[Metadata search] no documents matched filters=%v logic=%s", normalized, logic)
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": "metadata_search",
- "note": "No documents match the given metadata conditions. Loosen or change the filters (try a shorter substring, or the space form instead of underscores), or use search_chunks / retrieve for unfiltered search.",
- }},
- Status: StatusMiss,
- Reason: ReasonNoDoc,
- Metrics: map[string]any{"docs": 0},
- }, nil
- }
-
- // 4) Context block: the filter that was actually applied plus each matched document's
- // own metadata values.
- //
- // WHY the values travel with the ids. The ids alone are a handle, not an answer, and
- // the values are exactly what the filter matched ON: a model that must call
- // list_chunks on every selected document just to learn its title, file name or
- // timestamp spends a whole round re-reading what the selection already knew. Carrying
- // them here is what makes the selection readable in one step — "these 3 documents were
- // updated on 2026-09-20, and here are their titles and file names" — instead of
- // N round-trips that each cost context.
- //
- // WHY it is context and NOT an interface. No other tool consumes these values: the
- // machine-readable part of the result stays doc_ids (the argument retrieve's doc_scope,
- // list_chunks and navigate_structure take). Keeping the values read-only matters
- // because they are free-form text — a title or a file name spliced into a later
- // search argument would let a value the model merely READ silently re-scope the
- // search, while the document selection is the one thing the filter actually
- // guarantees. The values therefore explain the selection; they never become input to
- // the next one.
- //
- // The two halves are also reported separately for exactly this reason: `doc_ids` is
- // complete (every match), while `documents` is capped (see metadataContextDocsMax) —
- // truncating context must never truncate the result the model acts with.
- item := map[string]any{
- "kind": "metadata_search",
- "doc_ids": docIDs,
- "filters": normalized,
- }
- perDoc, ctxErr := e.deps.MetadataResolver.MetadataForDocIDs(ctx, targetIDs, docIDs)
- if ctxErr != nil {
- logger.Printf("[Metadata search] metadata context read degraded: %v", ctxErr)
- }
- if docs := metadataContextDocs(docIDs, perDoc, metadataContextKeys(normalized, declared)); len(docs) > 0 {
- item["documents"] = docs
- }
-
- logger.Printf("[Metadata search] %d doc(s) selected via filters=%v logic=%s", len(docIDs), normalized, logic)
- return ToolOutcome{
- Payload: []any{item},
- // No passages are retrieved here, so there is no evidence to admit: the ids reach
- // the answer only through the tool the model spends them on.
- EvidenceIDs: nil,
- Status: StatusOK,
- Reason: ReasonNone,
- Metrics: map[string]any{"docs": len(docIDs)},
- }, nil
-}
-
-// Metadata-search context caps: how many matched documents carry their metadata values in
-// the result, and how long one value may be.
-const (
- metadataContextDocsMax = 20
- metadataContextValueRunes = 200
-)
-
-// metadataContextKeys picks the fields the context block carries per document: first the
-// dataset's DECLARED fields (its curated schema — a title, a file name, a timestamp), then
-// any field the filter itself named, so the model can see WHY each document matched.
-//
-// Observed-only keys are deliberately left out: the metadata index also carries extraction
-// and annotation noise (a judge's rationale, a verdict) that the dataset never declared as
-// metadata, and dumping it into every result would spend context on fields no one can act on.
-func metadataContextKeys(filters []map[string]any, declared map[string]common.MetadataFieldDef) []string {
- out := make([]string, 0, len(declared)+len(filters))
- seen := map[string]bool{}
- keys := make([]string, 0, len(declared))
- for k := range declared {
- keys = append(keys, k)
- }
- sort.Strings(keys) // map iteration is randomised; the payload must not be
- for _, k := range keys {
- if k != "" && !seen[k] {
- seen[k] = true
- out = append(out, k)
- }
- }
- for _, f := range filters {
- k := asString(f["key"])
- if k != "" && !seen[k] {
- seen[k] = true
- out = append(out, k)
- }
- }
- return out
-}
-
-// metadataContextDocs renders the per-document context block: one {doc_id, metadata} entry
-// per matched document, in the selection's own order, capped at metadataContextDocsMax.
-// Missing and blank values are skipped, so the block never advertises a field as present
-// when the document does not carry it. A nil perDoc (the context read failed or no
-// resolver answered) yields nil — the ids alone are still a complete result.
-func metadataContextDocs(docIDs []string, perDoc map[string]map[string]any, keys []string) []any {
- if len(perDoc) == 0 || len(keys) == 0 {
- return nil
- }
- out := make([]any, 0, min(len(docIDs), metadataContextDocsMax))
- for _, docID := range docIDs {
- if len(out) >= metadataContextDocsMax {
- break
- }
- fields, ok := perDoc[docID]
- if !ok {
- continue
- }
- values := make(map[string]any, len(keys))
- for _, k := range keys {
- if metadataCatalogExcluded(k) {
- continue
- }
- v, exists := fields[k]
- if !exists || v == nil {
- continue
- }
- rendered := strings.TrimSpace(metadataValueString(v))
- if rendered == "" {
- continue
- }
- values[k] = Snippet(rendered, metadataContextValueRunes)
- }
- if len(values) == 0 {
- continue
- }
- out = append(out, map[string]any{"doc_id": docID, "metadata": values})
- }
- return out
-}
-
-// metadataValueString renders one metadata value as text. The doc-metadata index merges a
-// document's per-chunk values into a list, so a list joins with ", " rather than reaching
-// the model as Go's "[a b]" rendering; a scalar renders as-is.
-func metadataValueString(v any) string {
- switch t := v.(type) {
- case nil:
- return ""
- case string:
- return t
- case []string:
- return strings.Join(t, ", ")
- case []any:
- parts := make([]string, 0, len(t))
- for _, item := range t {
- if s := strings.TrimSpace(fmt.Sprint(item)); s != "" {
- parts = append(parts, s)
- }
- }
- return strings.Join(parts, ", ")
- default:
- return fmt.Sprint(t)
- }
-}
-
-// metadataFiltersOf reads the filters argument: a list of {key, value, op} objects.
-// Entries without a key or an op are dropped, so a malformed entry cannot ride along into
-// the push-down as a condition that matches everything.
-func metadataFiltersOf(args map[string]any) []map[string]any {
- raw, ok := args["filters"]
- if !ok || raw == nil {
- return nil
- }
- items, ok := raw.([]any)
- if !ok {
- return nil
- }
- out := make([]map[string]any, 0, len(items))
- for _, item := range items {
- m, ok := item.(map[string]any)
- if !ok {
- continue
- }
- key := strings.TrimSpace(asString(m["key"]))
- op := strings.TrimSpace(asString(m["op"]))
- if key == "" || op == "" {
- continue
- }
- out = append(out, map[string]any{"key": key, "value": m["value"], "op": op})
- }
- return out
-}
-
-// NormalizeMetadataValue coerces a model-supplied filter value to the single value a
-// metadata filter expects. The string ops (contains / = / start with / end with /
-// not contains) take ONE keyword: a list that sneaks in collapses to its first non-empty
-// element (one condition = one keyword) and is logged, so the caller can re-issue per
-// keyword. 'in' / 'not in' keep their list (the value SET); empty / not empty take no value.
-//
-// Exported because the two callers that accept model-written metadata conditions — the
-// metadata_search tool and the pre-search fan-out channel — must coerce them identically.
-func NormalizeMetadataValue(value any, op string) any {
- if op == "in" || op == "not in" {
- return value
- }
- items, ok := value.([]any)
- if !ok {
- return value
- }
- flat := make([]string, 0, len(items))
- for _, item := range items {
- if item == nil {
- continue
- }
- if s := strings.TrimSpace(fmt.Sprint(item)); s != "" {
- flat = append(flat, s)
- }
- }
- if len(flat) == 0 {
- return nil
- }
- if len(flat) > 1 {
- _LOG.Printf("[Metadata search] value list collapsed to the single keyword %q (ignored: %v); to match all, issue one condition per keyword", flat[0], flat[1:])
- }
- return flat[0]
-}
-
-// capStrings returns the first n elements of list (n <= 0 means "no cap").
-func capStrings(list []string, n int) []string {
- if n <= 0 || len(list) <= n {
- return list
- }
- return list[:n]
-}
-
-// calculate derives a number the evidence does not state outright, by having the model
-// write ONE
-// expression and evaluating it against the AST whitelist (see arithmetic.go).
-//
-// The computed value is returned as evidence, so a later answer step can cite it
-// without re-deriving. Nothing derivable is POOR/no_doc, not an error — the model
-// then answers from the facts it already has.
-func (e *searchExecutor) calculate(ctx context.Context, args map[string]any) (ToolOutcome, error) {
- // NOTHING is validated here: an absent question or fact list flows into
- // compute_from_facts, whose own `if not question or not facts` guard returns None — a
- // POOR/no_doc "nothing derivable", never a bad-args error. There is no fallback to the
- // run question either.
- question := argString(args, "question")
- facts := make([]string, 0, 8)
- for _, f := range toolStringList(args, "facts") {
- if s := strings.TrimSpace(f); s != "" {
- facts = append(facts, s)
- }
- }
- if e.deps.Model == nil {
- // no chat model is an INFRA failure, checked BEFORE any derivation.
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": "calculate",
- "error": "no model",
- }},
- Status: StatusError,
- Reason: ReasonInfra,
- Metrics: map[string]any{},
- }, nil
- }
- cf := ComputeFromFacts(ctx, e.deps.Model, question, facts, 0)
- if cf == nil {
- // Nothing derivable is POOR/no_doc.
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": "calculate",
- "expression": nil,
- "note": "no numeric answer derivable from given facts; answer directly or retrieve more numbers.",
- }},
- Status: StatusPoor,
- Reason: ReasonNoDoc,
- Metrics: map[string]any{},
- }, nil
- }
- // the success payload is exactly {"kind","expression","result"};
- // the label/uses the model returned are deliberately NOT echoed back.
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": "calculate",
- "expression": cf.Expression,
- "result": cf.Value,
- }},
- Status: StatusOK,
- Reason: ReasonNone,
- Metrics: map[string]any{},
- }, nil
-}
-
-// toolStringList reads a string-list arg, tolerating []any, []string, and a
-// single string (models emit all three).
-func toolStringList(args map[string]any, key string) []string {
- raw, ok := args[key]
- if !ok || raw == nil {
- return nil
- }
- switch v := raw.(type) {
- case []string:
- return v
- case []any:
- out := make([]string, 0, len(v))
- for _, item := range v {
- if item == nil {
- continue
- }
- out = append(out, fmt.Sprint(item))
- }
- return out
- case string:
- return []string{v}
- }
- return nil
-}
-func toolQueries(args map[string]any) []string {
- raw, ok := args["query"]
- if !ok || raw == nil {
- // Absent (or an explicit null, which JSON models emit) falls back to the
- // `q` alias before giving up.
- if q, ok := args["q"].(string); ok && strings.TrimSpace(q) != "" {
- return []string{strings.TrimSpace(q)}
- }
- return nil
- }
- switch v := raw.(type) {
- case string:
- if strings.TrimSpace(v) == "" {
- return nil
- }
- return []string{v}
- case []any:
- out := make([]string, 0, len(v))
- for _, item := range v {
- if s := strings.TrimSpace(fmt.Sprint(item)); s != "" {
- out = append(out, s)
- }
- }
- return out
- case []string:
- out := make([]string, 0, len(v))
- for _, s := range v {
- if strings.TrimSpace(s) != "" {
- out = append(out, s)
- }
- }
- return out
- }
- return nil
-}
-
-// toolDocScope reads the optional doc_scope restriction — parsed in exactly two
-// dispatchers: retrieve and graph_explore, as
-//
-// [str(d) for d in (args.get("doc_scope") or []) if str(d).strip()]
-//
-// so ONLY the "doc_scope" key is honoured: there is deliberately NO "doc_ids" alias and
-// NO bare-string coercion (a string would be iterated char-wise; no tool schema advertises
-// a doc_ids alias or a bare string, so both cases are unreachable). The other tools that
-// used to call this — search_chunks, navigate_tree, navigate_structure — must NOT read a
-// scope: none of them threads args["doc_scope"] into its impl.
-func toolDocScope(args map[string]any) []string {
- raw, ok := args["doc_scope"]
- if !ok {
- return nil
- }
- var items []string
- switch v := raw.(type) {
- case []string:
- items = v
- case []any:
- items = make([]string, 0, len(v))
- for _, item := range v {
- items = append(items, fmt.Sprint(item))
- }
- default:
- return nil
- }
- out := make([]string, 0, len(items))
- for _, s := range items {
- if s = strings.TrimSpace(s); s != "" {
- out = append(out, s)
- }
- }
- return out
-}
-
-// passageFromChunk renders one chunk as the passage dict the model sees: {"id",
-// "content", "doc_id"} — no title and no query. The id must stay under "id": the drill
-// merge reads it as entry["id"],
-// so a "chunk_id" key silently disables the drill's structure_path attachment.
-func passageFromChunk(c map[string]any) map[string]any {
- // Table chunks are shown to the model as a Markdown view (key-value for infoboxes,
- // a full-row pipe table for ranked/result tables) instead of raw <table> markup:
- // same rows, a fraction of the tokens, and a form the model can aggregate. They
- // also pass through un-truncated: the 1200-char cap would hide rows mid/late in a
- // long standings table. The shared pool keeps the RAW chunk for citation.
- var content string
- if IsTableChunk(c) {
- content = TableViewOrRaw(ChunkTextOf(c))
- } else {
- // Content is a plain slice at 1200 chars (no trim, no ellipsis).
- content = truncateRunes(ChunkTextOf(c), 1200)
- }
- return map[string]any{
- "id": ChunkIDOf(c),
- "content": content,
- "doc_id": DocIDOf(c),
- }
-}
-
-// PublishReferences writes the accumulated evidence into the canvas state so the
-// agent's post-stream citation grounding can read it.
-//
-// Chunk and aggregation shapes match runtime's referenceChunksFromRetrieval /
-// referenceDocAggsFromRetrieval (which dual-write both field-name styles), so no
-// translation is needed by the consumer. Exported because the
-// caller sits in the parent agentic_rag package (agentic_rag.go), not
-// inside retrieval.
-func PublishReferences(ctx context.Context, kb *Kbinfos) {
- state, err := runtime.GetStateFromContext(ctx)
- if err != nil || state == nil {
- return
- }
- chunks := make([]map[string]any, 0, len(kb.Chunks))
- for idx, c := range kb.Chunks {
- chunkID := ChunkIDOf(c)
- docID := DocIDOf(c)
- name := DocTitleOf(c)
- content := ChunkTextOf(c)
- datasetID := DatasetIDOf(c)
- chunks = append(chunks, map[string]any{
- "id": fmt.Sprint(idx),
- "chunk_id": chunkID,
- "content": content,
- "content_with_weight": content,
- "document_id": docID,
- "doc_id": docID,
- "document_name": name,
- "docnm_kwd": name,
- "dataset_id": datasetID,
- "kb_id": datasetID,
- })
- }
- state.SetRetrievalReferences(chunks, kb.DocAggs)
-}
-
-// RuntimeRetriever adapts runtime.GetRetrievalService() to the runtime Retriever
-// interface. The service is read on every call (not captured at construction)
-// because the server installs it during boot, which may happen after a runtime
-// component was built.
-//
-// This is the only place in the runtime that knows about internal/agent/runtime;
-// it lives in the runtime root package (not a sub-package) so the runtime
-// dependency does not leak into the runtime's testable core.
-// RuntimeRetriever is the production runtime retriever: it runs the backend search and
-// returns the normalised chunks. Child-fragment promotion (retrieval_by_children) is NOT
-// done here — it is entry-point specific (only the hybrid and retrieve legs do it), so it
-// lives in runSearch, gated by searchOpts.promoteChildren, and this Backend stays
-// caller-agnostic.
-type RuntimeRetriever struct{}
-
-func (r *RuntimeRetriever) Retrieve(ctx context.Context, req RetrieveRequest) ([]map[string]any, error) {
- svc := runtime.GetRetrievalService()
- chunks, err := svc.Search(ctx, nil, runtime.RetrievalRequest{
- Query: req.Query,
- DatasetIDs: req.DatasetIDs,
- DocScope: req.DocScope,
- TopN: req.TopN,
- TopK: req.TopK,
- RerankCandidatesCount: req.RerankCandidatesCount,
- // Passed through as pointers: nil (the caller did not supply one) must
- // stay nil so the retrieval service keeps its own default, while a zero
- // is a real override. Taking the address of a zero value here forced
- // "threshold 0 / full vector weight" onto every caller that omitted them.
- SimilarityThreshold: req.SimilarityThreshold,
- KeywordsSimilarityWeight: req.KeywordsSimilarityWeight,
- TenantID: req.TenantID,
- MetaDataFilter: req.MetaDataFilter,
- RankFeature: &req.RankFeature,
- // Python hybrid_search always excludes compile_kwd rows from the base
- // retrieval; use_compiled controls a separate expansion step.
- ExcludeCompiled: req.ExcludeCompiled,
- })
- if err != nil {
- return nil, err
- }
- return chunksToMaps(chunks), nil
-}
-
-// chunksToMaps normalises runtime.RetrievalChunk values into the runtime chunk
-// shape (the keys the rest of the runtime expects: chunk_id / content / doc_id
-// / docnm_kwd / ...).
-func chunksToMaps(chunks []runtime.RetrievalChunk) []map[string]any {
- out := make([]map[string]any, 0, len(chunks))
- for _, c := range chunks {
- out = append(out, map[string]any{
- "chunk_id": c.ID,
- "content": c.Content,
- // content_with_weight is the engine/storage spelling of the body.
- // Carried as an alias because the shared reference builder
- // (chunksFormat → getValue("content_with_weight", "content")) and
- // the citation prompt read the storage name; without it those
- // readers get an empty body from an agentic chunk.
- "content_with_weight": c.Content,
- "doc_id": c.DocumentID,
- "docnm_kwd": c.DocumentName,
- "dataset_id": c.DatasetID,
- "kb_id": c.DatasetID,
- // doc_type_kwd is what marks a chunk as an image (or table); the
- // reference card renders it and drops the marker without it.
- "doc_type_kwd": c.DocType,
- "mom_id": c.MomID,
- "similarity": c.Score,
- "image_id": c.ImageID,
- "url": c.URL,
- "positions": c.Positions,
- "chunk_order_int": c.ChunkIndex,
- "page_num_int": c.PageNum,
- "score": c.Score,
- "term_similarity": c.TermSimilarity,
- "vector_similarity": c.VectorSimilarity,
- })
- }
- return out
-}
-
-// TenantIDFromContext returns the tenant id bound on the canvas context, used as
-// a fallback for SearchDeps.TenantID when the caller's SearchRequest leaves it
-// empty. The runtime stays independent of the tool registry, so it reads only
-// what CanvasState.GetVar exposes. Exported so the agentic_rag package's Run can call
-// it.
-func TenantIDFromContext(ctx context.Context) string {
- state, err := runtime.GetStateFromContext(ctx)
- if err != nil || state == nil {
- return ""
- }
- if tid, err := state.GetVar("tenant_id"); err == nil {
- if s, _ := tid.(string); s != "" {
- return s
- }
- }
- return ""
-}
diff --git a/internal/rag/agentic-rag/runtime/tool_executor_test.go b/internal/rag/agentic-rag/runtime/tool_executor_test.go
deleted file mode 100644
index b74c5879a5..0000000000
--- a/internal/rag/agentic-rag/runtime/tool_executor_test.go
+++ /dev/null
@@ -1,1986 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package runtime
-
-import (
- "bytes"
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "log"
- "strings"
- "testing"
- "unicode/utf8"
-
- "ragflow/internal/agent/runtime"
- "ragflow/internal/common"
- "ragflow/internal/engine"
- enginetypes "ragflow/internal/engine/types"
-
- "gorm.io/gorm"
-)
-
-// stubRetrievalService returns a fixed set of child chunks so the runtime
-// retireval path can be exercised without a live search backend. lastReq, when
-// set, receives the request the runtime handed down.
-type stubRetrievalService struct {
- chunks []runtime.RetrievalChunk
- lastReq *runtime.RetrievalRequest
-}
-
-func (s stubRetrievalService) Search(_ context.Context, _ *gorm.DB, req runtime.RetrievalRequest) ([]runtime.RetrievalChunk, error) {
- if s.lastReq != nil {
- *s.lastReq = req
- }
- return s.chunks, nil
-}
-
-// stubDocEngine serves the document-scoped Search used by retrieval_by_children;
-// the rest of the (large) interface is promoted from a nil field and never
-// called by this test.
-type stubDocEngine struct {
- engine.DocEngine
- parents map[string]map[string]any
-}
-
-func (s stubDocEngine) Search(_ context.Context, req *enginetypes.SearchRequest) (*enginetypes.SearchResult, error) {
- parentID, _ := req.Filter["id"].(string)
- docID, _ := req.Filter["doc_id"].(string)
- if p, ok := s.parents[parentID]; ok && p["doc_id"] == docID {
- return &enginetypes.SearchResult{Chunks: []map[string]interface{}{p}}, nil
- }
- return &enginetypes.SearchResult{}, nil
-}
-
-// TestRuntimeRetrieverPreservesUnsetControls pins the presence semantics of the
-// retrieval controls: an omitted threshold/weight must reach the retrieval
-// service as nil so it keeps its own default, while an explicit zero is a real
-// override. Taking the address of a zero value used to force "no threshold
-// floor + the vector leg at full weight" onto every caller that omitted them.
-func TestRuntimeRetrieverPreservesUnsetControls(t *testing.T) {
- prev := runtime.GetRetrievalService()
- var got runtime.RetrievalRequest
- runtime.SetRetrievalService(stubRetrievalService{lastReq: &got})
- t.Cleanup(func() { runtime.SetRetrievalService(prev) })
-
- r := &RuntimeRetriever{}
- if _, err := r.Retrieve(context.Background(), RetrieveRequest{Query: "q", DatasetIDs: []string{"kb-1"}}); err != nil {
- t.Fatalf("Retrieve: %v", err)
- }
- if got.SimilarityThreshold != nil || got.KeywordsSimilarityWeight != nil {
- t.Errorf("omitted controls = %v / %v, want nil so the service keeps its defaults",
- got.SimilarityThreshold, got.KeywordsSimilarityWeight)
- }
-
- threshold, keywordsWeight := 0.35, 0.7
- if _, err := r.Retrieve(context.Background(), RetrieveRequest{
- Query: "q",
- SimilarityThreshold: &threshold,
- KeywordsSimilarityWeight: &keywordsWeight,
- }); err != nil {
- t.Fatalf("Retrieve: %v", err)
- }
- if got.SimilarityThreshold == nil || *got.SimilarityThreshold != threshold {
- t.Errorf("SimilarityThreshold = %v, want %v", got.SimilarityThreshold, threshold)
- }
- if got.KeywordsSimilarityWeight == nil || *got.KeywordsSimilarityWeight != keywordsWeight {
- t.Errorf("KeywordsSimilarityWeight = %v, want %v", got.KeywordsSimilarityWeight, keywordsWeight)
- }
-}
-
-// TestChunkAggRetrieveLeavesControlsUnset pins the caller side: the chunk-agg
-// retriever has no threshold/weight of its own, so it must omit both rather
-// than pass zero overrides.
-func TestChunkAggRetrieveLeavesControlsUnset(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{{"chunk_id": "c1"}}}
-
- chunks, err := chunkAggRetrieveFrom(r)(context.Background(), "t1", "kb-1", "q", nil, 5, 0.9)
- if err != nil {
- t.Fatalf("chunkAggRetrieveFrom: %v", err)
- }
- if len(chunks) != 1 {
- t.Fatalf("chunks = %d, want the retrieved chunk", len(chunks))
- }
- req := r.lastReq(t)
- if req.SimilarityThreshold != nil || req.KeywordsSimilarityWeight != nil {
- t.Errorf("controls = %v / %v, want nil (zero is a valid value, not an unset marker)",
- req.SimilarityThreshold, req.KeywordsSimilarityWeight)
- }
-}
-
-// TestChunkAggRetrievePropagatesError pins the adapter half of the router
-// contract: a retrieval failure must reach the router as an error, not as an
-// empty chunk list (which the router would report as a successful empty route).
-func TestChunkAggRetrievePropagatesError(t *testing.T) {
- wantErr := errors.New("retrieval down")
- r := &stubRetriever{err: wantErr}
-
- chunks, err := chunkAggRetrieveFrom(r)(context.Background(), "t1", "kb-1", "q", nil, 5, 0.9)
- if !errors.Is(err, wantErr) {
- t.Fatalf("err = %v, want %v", err, wantErr)
- }
- if chunks != nil {
- t.Errorf("chunks = %v, want nil alongside the error", chunks)
- }
-}
-
-// TestRuntimeRetrieverPromotesChildrenToParent verifies that
-// RuntimeRetriever.Retrieve threads child chunks through retrieval_by_children:
-// two child fragments sharing a mom_id collapse into the single parent chunk.
-func TestRuntimeRetrieverPromotesChildrenToParent(t *testing.T) {
- prev := runtime.GetRetrievalService()
- runtime.SetRetrievalService(stubRetrievalService{chunks: []runtime.RetrievalChunk{
- {ID: "child-1", Content: "frag one", DocumentID: "doc-1", DatasetID: "kb-1", MomID: "parent-1", Score: 0.6},
- {ID: "child-2", Content: "frag two", DocumentID: "doc-1", DatasetID: "kb-1", MomID: "parent-1", Score: 0.8},
- {ID: "top-1", Content: "standalone", DatasetID: "kb-1", Score: 0.9},
- }})
- t.Cleanup(func() { runtime.SetRetrievalService(prev) })
-
- de := stubDocEngine{parents: map[string]map[string]any{
- "parent-1": {
- "chunk_id": "parent-1",
- "content_with_weight": "the full parent chunk text",
- "doc_id": "doc-1",
- "docnm_kwd": "doc.pdf",
- "kb_id": "kb-1",
- "doc_type_kwd": "pdf",
- },
- }}
-
- deps := SearchDeps{
- Backend: &RuntimeRetriever{},
- DocEngine: de,
- TenantID: "tenant-1",
- }
- out, aggs := HybridSearch(context.Background(), deps, SearchParams{
- Question: "q",
- KbIDs: []string{"kb-1"},
- })
- _ = aggs
- if len(out) == 0 {
- t.Fatal("HybridSearch returned no chunks")
- }
-
- // parent-1 (from child-1 + child-2) + top-1 = 2 chunks.
- if len(out) != 2 {
- t.Fatalf("expected 2 aggregated chunks, got %d: %v", len(out), out)
- }
- var parent, top map[string]any
- for _, c := range out {
- if c["chunk_id"] == "parent-1" {
- parent = c
- }
- if c["chunk_id"] == "top-1" {
- top = c
- }
- }
- if parent == nil {
- t.Fatalf("parent chunk missing from result: %v", out)
- }
- if top == nil {
- t.Fatalf("top-level chunk missing from result: %v", out)
- }
- // similarity is the average of the two child scores (0.6, 0.8) -> 0.7.
- if sim, ok := parent["similarity"].(float64); !ok || sim != 0.7 {
- t.Fatalf("expected parent similarity 0.7, got %v", parent["similarity"])
- }
- if pw, _ := parent["content_with_weight"].(string); pw != "the full parent chunk text" {
- t.Fatalf("expected parent content_with_weight, got %v", parent["content_with_weight"])
- }
- if v, ok := parent["mom_id"]; ok && v != "" {
- t.Fatalf("aggregated parent should no longer carry a mom_id, got %v", v)
- }
-}
-
-// TestHybridSearchSkipsWhenNoChildren verifies that chunks without a mom_id
-// pass through unchanged and retrieval_by_children is a no-op for them.
-func TestHybridSearchSkipsWhenNoChildren(t *testing.T) {
- prev := runtime.GetRetrievalService()
- runtime.SetRetrievalService(stubRetrievalService{chunks: []runtime.RetrievalChunk{
- {ID: "top-1", Content: "a", DatasetID: "kb-1", Score: 0.5},
- }})
- t.Cleanup(func() { runtime.SetRetrievalService(prev) })
-
- deps := SearchDeps{Backend: &RuntimeRetriever{}, TenantID: "tenant-1"}
- out, _ := HybridSearch(context.Background(), deps, SearchParams{Question: "q", KbIDs: []string{"kb-1"}})
- if len(out) != 1 || out[0]["chunk_id"] != "top-1" {
- t.Fatalf("expected unchanged single chunk, got %v", out)
- }
-}
-
-// TestBM25SearchDoesPromoteChildren verifies that bm25_search (like every other entry
-// point) runs retrieval_by_children: a child fragment is lifted to its parent chunk, because
-// the normaliser calls retrieval_by_children.
-func TestBM25SearchDoesPromoteChildren(t *testing.T) {
- prev := runtime.GetRetrievalService()
- runtime.SetRetrievalService(stubRetrievalService{chunks: []runtime.RetrievalChunk{
- {ID: "child-1", Content: "frag", DocumentID: "doc-1", DatasetID: "kb-1", MomID: "parent-1", Score: 0.6},
- }})
- t.Cleanup(func() { runtime.SetRetrievalService(prev) })
-
- // DocEngine is set, so BM25Search promotes the child to its parent.
- deps := SearchDeps{Backend: &RuntimeRetriever{}, DocEngine: stubDocEngine{parents: map[string]map[string]any{
- "parent-1": {"chunk_id": "parent-1", "content_with_weight": "the full parent chunk text", "doc_id": "doc-1"},
- }}, TenantID: "tenant-1"}
- out, _ := BM25Search(context.Background(), deps, SearchParams{Question: "q", KbIDs: []string{"kb-1"}})
- if len(out) != 1 || out[0]["chunk_id"] != "parent-1" {
- t.Fatalf("bm25_search must promote the child to its parent, got %v", out)
- }
-}
-
-// TestExecuteLogsFunctionToolLine covers the "[Function tool] Running the {name} tool
-// with: {args}" line, one of the namespaces forwarded into the think block.
-func TestExecuteLogsFunctionToolLine(t *testing.T) {
- var buf bytes.Buffer
- deps := SearchDeps{Logger: log.New(&buf, "", 0)}
- ex := NewSearchExecutor(deps, RunRequest{})
-
- // "calculate" is a local tool; no backend is configured, so the call may
- // fail. The think-log line must be emitted regardless of the outcome.
- _, _ = ex.Execute(context.Background(), "calculate", map[string]any{"expr": "1+1"})
-
- out := buf.String()
- if !strings.Contains(out, "[Function tool] Running the calculate tool with:") {
- t.Fatalf("missing [Function tool] line, got:\n%s", out)
- }
- if !strings.Contains(out, `"expr":"1+1"`) {
- t.Errorf("args not rendered, got:\n%s", out)
- }
-}
-
-// TestExecuteNoLoggerIsSafe guards the nil-Logger path (Logger is optional).
-func TestExecuteNoLoggerIsSafe(t *testing.T) {
- ex := NewSearchExecutor(SearchDeps{}, RunRequest{})
- if _, err := ex.Execute(context.Background(), "calculate", map[string]any{"expr": "2+2"}); err == nil {
- t.Log("call succeeded; nil logger must not panic")
- }
-}
-
-// TestSearchChunksAlwaysUsesCompiled pins fix #2: the search_chunks tool ALWAYS enables
-// compiled-structure expansion; retrieve never does. The executor must NOT gate it on
-// RunRequest.UseCompiled (which controls the L1 direct retrieve).
-func TestSearchChunksAlwaysUsesCompiled(t *testing.T) {
- // search_chunks must expand (independent of req.UseCompiled).
- deps, _ := newTestSearchDeps(&stubRetriever{chunks: []map[string]any{{"content": "hit", "chunk_id": "c1"}}})
- deps.Expand = &stubExpander{}
- ex := NewSearchExecutor(deps, RunRequest{DatasetIDs: []string{"kb1"}})
- if _, err := ex.Execute(context.Background(), "search_chunks", map[string]any{"query": "q"}); err != nil {
- t.Fatalf("search_chunks: %v", err)
- }
- if deps.Expand.(*stubExpander).calls != 1 {
- t.Errorf("search_chunks compiled expansion calls = %d, want 1", deps.Expand.(*stubExpander).calls)
- }
-
- // retrieve must NOT expand.
- deps2, _ := newTestSearchDeps(&stubRetriever{chunks: []map[string]any{{"content": "hit", "chunk_id": "c1"}}})
- deps2.Expand = &stubExpander{}
- ex2 := NewSearchExecutor(deps2, RunRequest{DatasetIDs: []string{"kb1"}})
- if _, err := ex2.Execute(context.Background(), "retrieve", map[string]any{"query": "q"}); err != nil {
- t.Fatalf("retrieve: %v", err)
- }
- if deps2.Expand.(*stubExpander).calls != 0 {
- t.Errorf("retrieve compiled expansion calls = %d, want 0", deps2.Expand.(*stubExpander).calls)
- }
-
- // An explicit req.UseCompiled=false must NOT disable it for search_chunks.
- deps3, _ := newTestSearchDeps(&stubRetriever{chunks: []map[string]any{{"content": "hit", "chunk_id": "c1"}}})
- deps3.Expand = &stubExpander{}
- exOff := NewSearchExecutor(deps3, RunRequest{DatasetIDs: []string{"kb1"}, UseCompiled: false})
- if _, err := exOff.Execute(context.Background(), "search_chunks", map[string]any{"query": "q"}); err != nil {
- t.Fatalf("search_chunks (UseCompiled=false): %v", err)
- }
- if deps3.Expand.(*stubExpander).calls != 1 {
- t.Errorf("search_chunks with req.UseCompiled=false must still expand, calls = %d, want 1", deps3.Expand.(*stubExpander).calls)
- }
-}
-
-func TestRenderToolArgs(t *testing.T) {
- if got := RenderToolArgs(nil); got != "{}" {
- t.Errorf("RenderToolArgs(nil) = %q, want {}", got)
- }
- if got := RenderToolArgs(map[string]any{}); got != "{}" {
- t.Errorf("RenderToolArgs(map{}) = %q, want {}", got)
- }
- got := RenderToolArgs(map[string]any{"q": "hi", "n": 3})
- if !strings.Contains(got, `"q":"hi"`) || !strings.Contains(got, `"n":3`) {
- t.Errorf("RenderToolArgs = %q, want both keys", got)
- }
- // Unmarshalable values must degrade to "{}", not blow up the log line.
- if got := RenderToolArgs(map[string]any{"f": func() {}}); got != "{}" {
- t.Errorf("RenderToolArgs(func value) = %q, want {}", got)
- }
-
- // A long value is CAPPED: the slot-research driver passes a paragraph of
- // evidence as its "query", and the uncapped rendering repeated the same 200
- // characters on every line of its round.
- long := strings.Repeat("曹", 200)
- got = RenderToolArgs(map[string]any{"query": []any{long}})
- if strings.Contains(got, long) || !strings.Contains(got, "…") {
- t.Errorf("long args = %q, want the value capped with an ellipsis", got)
- }
- if n := utf8.RuneCountInString(got); n > ThinkLabelMaxRunes+16 {
- t.Errorf("rendered args = %d runes, want one capped value", n)
- }
- // Capping must not produce invalid UTF-8 (it cuts on rune boundaries)...
- if !utf8.ValidString(got) {
- t.Errorf("capped args are not valid UTF-8: %q", got)
- }
- // ...nor invalid JSON: the capped value still parses.
- var decoded map[string]any
- if err := json.Unmarshal([]byte(got), &decoded); err != nil {
- t.Errorf("capped args must stay valid JSON: %v", err)
- }
- // Nested shapes are capped too (a tool argument can be an object).
- if got := RenderToolArgs(map[string]any{"scope": map[string]any{"q": long}}); !strings.Contains(got, "…") {
- t.Errorf("nested args = %q, want the nested value capped", got)
- }
- // A list of short queries is left alone: capping caps VALUES, it does not
- // truncate the call.
- if got := RenderToolArgs(map[string]any{"query": []any{"a", "b"}}); got != `{"query":["a","b"]}` {
- t.Errorf("short args = %q, want them untouched", got)
- }
-}
-
-// TestToolCallLine pins the think block's call step: the tool and its query, and
-// nothing else. The argument object is a developer's record — the document id, the
-// kind switch, the nav hint and the empty scope list are all detail a reader would
-// have to parse — and a call that carries no query names only the tool instead of
-// trailing an empty "with:" or falling back to an id.
-func TestToolCallLine(t *testing.T) {
- const docID = "95a7aee3f11143e69dc9fa5b7bad3a14"
- cases := []struct {
- name string
- tool string
- args map[string]any
- want string
- }{
- // The shape the reader actually meets: a document-scoped call that also
- // carries the query it was made for. Only the query survives.
- {"query-with-plumbing", "navigate_structure",
- map[string]any{"doc_id": docID, "kind": "catalog", "query": "曹操历史地位"},
- `Running the navigate_structure tool with "曹操历史地位".`},
- {"query-list", "retrieve", map[string]any{"nav_hint": "", "query": []any{"曹操的逝世日期", "曹操是谁"}},
- `Running the retrieve tool with "曹操的逝世日期", "曹操是谁".`},
- {"question", "calculate", map[string]any{"question": "how many people"},
- `Running the calculate tool with "how many people".`},
- {"doc-only", "summarize_document", map[string]any{"doc_id": docID},
- "Running the summarize_document tool."},
- {"no-args", "summarize_document", nil, "Running the summarize_document tool."},
- }
- for _, tc := range cases {
- if got := ToolCallLine(tc.tool, tc.args); got != tc.want {
- t.Errorf("%s: ToolCallLine = %q, want %q", tc.name, got, tc.want)
- }
- }
-}
-
-// TestCountOfPluralizes pins the two rules the trace needs: a regular noun takes
-// "s", and a consonant+y noun takes "ies". Appending a bare "s" is what printed
-// "3 first-hop querys" and "3 targeted querys".
-func TestCountOfPluralizes(t *testing.T) {
- cases := []struct {
- n int
- noun string
- want string
- }{
- {1, "query", "1 query"},
- {3, "query", "3 queries"},
- {2, "first-hop query", "2 first-hop queries"},
- {2, "targeted query", "2 targeted queries"},
- {2, "passage", "2 passages"},
- {2, "new passage", "2 new passages"},
- {1, "sub-question", "1 sub-question"},
- {2, "sub-question", "2 sub-questions"},
- {2, "evidence block", "2 evidence blocks"},
- {2, "key", "2 keys"},
- {0, "gathered passage", "0 gathered passages"},
- }
- for _, tc := range cases {
- if got := CountOf(tc.n, tc.noun); got != tc.want {
- t.Errorf("CountOf(%d, %q) = %q, want %q", tc.n, tc.noun, got, tc.want)
- }
- }
-}
-
-// TestExecuteNarratesToolOutcome pins the RESULT half of the "[Function tool]"
-// think-block narration: every call reports what came back (status, result
-// count, source documents), not only that it ran. Without it the trace shows a
-// tool was invoked but never whether it found anything.
-//
-// It also pins the STRUCTURED twin of both halves: a client rendering steps
-// gets the tool name, arguments, status, result/document counts and evidence
-// anchors without parsing the sentences.
-func TestExecuteNarratesToolOutcome(t *testing.T) {
- var buf bytes.Buffer
- deps, _ := newTestSearchDeps(&stubRetriever{chunks: []map[string]any{
- {"chunk_id": "c1", "doc_id": "d1", "content": "hit one"},
- {"chunk_id": "c2", "doc_id": "d1", "content": "hit two"},
- {"chunk_id": "c3", "doc_id": "d2", "content": "hit three"},
- }})
- deps.Logger = log.New(&buf, "", 0)
- var events []ThinkEvent
- ctx := WithSteps(context.Background(), StepReporter{Events: func(ev ThinkEvent) { events = append(events, ev) }})
- // search_chunks never narrows (no keywords), so the stub's hits reach the
- // outcome verbatim.
- ex := NewSearchExecutor(deps, RunRequest{DatasetIDs: []string{"kb1"}})
-
- if _, err := ex.Execute(ctx, "search_chunks", map[string]any{"query": "q"}); err != nil {
- t.Fatalf("search_chunks: %v", err)
- }
- out := buf.String()
- for _, want := range []string{
- "[Function tool] Running the search_chunks tool with: ",
- `[Function tool] The search_chunks tool returned 3 results from 2 documents for "q".`,
- } {
- if !strings.Contains(out, want) {
- t.Errorf("think-log narration missing %q:\n%s", want, out)
- }
- }
-
- // The tool's two halves PLUS the search legs underneath it: the legs report
- // their own steps (which leg ran, what it searched, what it found), so the
- // think block answers "what did it actually search".
- var call, result ThinkEvent
- var legLines []string
- for _, ev := range events {
- switch ev.Kind {
- case ThinkKindToolCall:
- call = ev
- case ThinkKindToolResult:
- result = ev
- case ThinkKindStage:
- legLines = append(legLines, ev.Summary)
- }
- }
- legs := strings.Join(legLines, "\n")
- for _, want := range []string{
- // The leg's two steps as the READER gets them: sentences in the family every
- // leg shares (the method is named inside the sentence, because the
- // "[Hybrid search]" prefix is a developer's label), counts in words, no ids —
- // and no "the knowledge base", which every leg has in common.
- `[Hybrid search] Searching by meaning and keyword for "q".`,
- `[Hybrid search] Found 3 passages in 2 documents for "q".`,
- } {
- if !strings.Contains(legs, want) {
- t.Errorf("search-leg step missing %q; legs:\n%s", want, legs)
- }
- }
- // The log keeps Python's own wording (search.py:122): it names the corpus, not
- // the method, and ends without a period.
- if !strings.Contains(out, `[Hybrid search] Searching the knowledge base for "q"`) {
- t.Errorf("the log must keep its searching line:\n%s", out)
- }
- // ...and the log form must not be what the block got: the two sentences differ
- // by design (the log names the corpus, the block names the method, and only the
- // block ends in a period).
- if strings.Contains(legs, `Searching the knowledge base for`) {
- t.Errorf("the think block got the log line instead of the sentence:\n%s", legs)
- }
- // The per-document breakdown the sentence is derived from stays in the
- // developer log, keyed by id: that is the half a developer greps for.
- if !strings.Contains(out, `"q" -> 3 chunk(s): d1:2chunk(14chars); d2:1chunk(9chars)`) {
- t.Errorf("the log must keep the per-document breakdown:\n%s", out)
- }
- if strings.Contains(legs, "d1:2chunk") {
- t.Errorf("document ids must not reach the think block; legs:\n%s", legs)
- }
- if call.Kind != ThinkKindToolCall || call.Tool != "search_chunks" || call.Stage != "Function tool" {
- t.Errorf("call event = %#v", call)
- }
- // The call step names the tool and its query — nothing else — while the log and
- // the event's Args keep the argument object the model emitted.
- if want := `[Function tool] Running the search_chunks tool with "q".`; call.Summary != want {
- t.Errorf("call summary = %q, want %q", call.Summary, want)
- }
- if !strings.Contains(out, `[Function tool] Running the search_chunks tool with: {"query":"q"}`) {
- t.Errorf("the log must keep the argument object:\n%s", out)
- }
- if !strings.Contains(call.Args, `"query":"q"`) {
- t.Errorf("call args = %q, want the rendered arguments", call.Args)
- }
-
- if result.Kind != ThinkKindToolResult || result.Status != StatusOK {
- t.Errorf("result event = %#v, want an ok result", result)
- }
- if result.Results != 3 || result.Documents != 2 {
- t.Errorf("result counts = %d/%d, want 3 results from 2 documents", result.Results, result.Documents)
- }
- // The evidence anchors are the admitted chunk ids — what the sentence's
- // "3 results" points at.
- if len(result.Sources) != 3 {
- t.Errorf("sources = %#v, want the 3 admitted chunk ids", result.Sources)
- }
- // The result carries the call's arguments too, and its sentence names them:
- // with concurrent calls interleaving their lines, that is what pairs a result
- // back to the call that produced it.
- if result.Args != call.Args {
- t.Errorf("result args = %q, want the call's arguments %q", result.Args, call.Args)
- }
- if !strings.Contains(result.Summary, `for "q"`) {
- t.Errorf("result summary = %q, want it to name the query", result.Summary)
- }
- // The structured counts must agree with the sentence shown to the user.
- if !strings.Contains(result.Summary, "3 results from 2 documents") {
- t.Errorf("result summary = %q, want it to carry the same counts", result.Summary)
- }
-}
-
-// TestExecuteHidesDocumentIDsFromThink pins the split the two projections exist
-// for: a 32-hex document id is exact for a developer and meaningless for a reader,
-// so it belongs in the developer log and in the machine-readable Args — and not in
-// the think block, neither in the call's arguments nor in the outcome sentence.
-//
-// The call targets a tool name this deployment has no binding for on purpose: it
-// exercises the whole path (call line, outcome line, label fallback) without
-// needing a retriever.
-func TestExecuteHidesDocumentIDsFromThink(t *testing.T) {
- const docID = "95a7aee3f11143e69dc9fa5b7bad3a14"
- var logBuf, think strings.Builder
- deps, _ := newTestSearchDeps(&stubRetriever{})
- deps.Logger = log.New(&logBuf, "", 0)
- var events []ThinkEvent
- ctx := WithSteps(context.Background(), StepReporter{
- Text: func(line string) { think.WriteString(line) },
- Events: func(ev ThinkEvent) { events = append(events, ev) },
- })
- ex := NewSearchExecutor(deps, RunRequest{DatasetIDs: []string{"kb1"}})
-
- if _, err := ex.Execute(ctx, "time_travel", map[string]any{"doc_id": docID}); err != nil {
- t.Fatalf("Execute: %v", err)
- }
-
- // A doc-scoped call has no query, so the call step names only the tool: no
- // argument object, and therefore no id.
- if !strings.Contains(think.String(), "[Function tool] Running the time_travel tool.") {
- t.Errorf("think block should show the call without arguments:\n%s", think.String())
- }
- if strings.Contains(think.String(), docID) {
- t.Errorf("the think block must not carry a document id:\n%s", think.String())
- }
- // A document-scoped call has no human label: it goes unlabelled rather than
- // naming a document the reader cannot resolve.
- if !strings.Contains(think.String(),
- "[Function tool] The time_travel tool is not wired in this deployment, so nothing ran.") {
- t.Errorf("think block missing the outcome sentence:\n%s", think.String())
- }
-
- // The structured summaries follow the same rule as the text (a client renders
- // them), while the event's Args keep the exact call: that is what a client
- // correlates a step on.
- for _, ev := range events {
- if strings.Contains(ev.Summary, docID) {
- t.Errorf("event summary %q carries the document id", ev.Summary)
- }
- }
- if last := events[len(events)-1]; !strings.Contains(last.Args, docID) {
- t.Errorf("event Args = %q, want the document id kept", last.Args)
- }
-
- // The developer log keeps the id on both halves of the call.
- if !strings.Contains(logBuf.String(), `"doc_id":"`+docID+`"`) {
- t.Errorf("the log's call line must keep the document id:\n%s", logBuf.String())
- }
- if !strings.Contains(logBuf.String(), "so nothing ran for document "+docID) {
- t.Errorf("the log's outcome line must name the document by id:\n%s", logBuf.String())
- }
-}
-
-// TestExecuteNarratesEmptyOutcome pins the other end: a tool that reached
-// nothing says so, instead of leaving that indistinguishable from a hit.
-func TestExecuteNarratesEmptyOutcome(t *testing.T) {
- var buf bytes.Buffer
- deps, _ := newTestSearchDeps(&stubRetriever{})
- deps.Logger = log.New(&buf, "", 0)
- ex := NewSearchExecutor(deps, RunRequest{DatasetIDs: []string{"kb1"}})
-
- if _, err := ex.Execute(context.Background(), "retrieve", map[string]any{"query": "q"}); err != nil {
- t.Fatalf("retrieve: %v", err)
- }
- if want := `[Function tool] The retrieve tool matched nothing for "q".`; !strings.Contains(buf.String(), want) {
- t.Errorf("think-log narration missing %q:\n%s", want, buf.String())
- }
-}
-
-// TestChunkAggLegFitsTheCandidateCountToItsPool pins the nav-tree chunk-agg
-// leg's retrieval parameters against the retrieval service's own invariant
-// (`page * page_size <= rerank_candidates_count`, nlp/retrieval.go:126 =
-// rag/nlp/search.py:745).
-//
-// Python passes the pool as page_size AND raises rerank_candidates_count to the
-// same pool for this leg (dataset_api_service.py:4106-4129, plus the
-// must_not={"exists":"compile_kwd"} of _NAV_CHUNK_AGG_EXCLUDE_COMPILED=True).
-// Go dropped both, so page_size 256 met the default candidate count 64 and every
-// nav-tree descent failed with "rerank_candidates_count(64) must be greater than
-// or equal to page(1) * page_size(256)" — reported as an infra tool failure, and
-// the navigation ladder silently degraded to plain retrieval on every round.
-func TestChunkAggLegFitsTheCandidateCountToItsPool(t *testing.T) {
- const pool = 256 // nav chunkAggPool / Python _NAV_CHUNK_AGG_POOL
- r := &stubRetriever{chunks: []map[string]any{{"chunk_id": "c1"}}}
- retrieve := chunkAggRetrieveFrom(r)
-
- chunks, err := retrieve(context.Background(), "tenant1", "kb1", "q", []string{"d1"}, pool, 0.3)
- if err != nil {
- t.Fatalf("chunkAggRetrieveFrom: %v", err)
- }
- if len(chunks) != 1 {
- t.Fatalf("chunks = %d, want the backend's hit", len(chunks))
- }
- req := r.lastReq(t)
- if req.TopN != pool {
- t.Errorf("TopN = %d, want %d (page_size IS the pool)", req.TopN, pool)
- }
- if req.RerankCandidatesCount < req.TopN {
- t.Errorf("RerankCandidatesCount = %d < page_size %d: the retrieval service rejects this combination",
- req.RerankCandidatesCount, req.TopN)
- }
- if !req.ExcludeCompiled {
- t.Error("ExcludeCompiled = false: this leg aggregates ORIGINAL chunks per document, so compiled rows must be filtered out")
- }
- if len(req.DocScope) != 1 || req.DocScope[0] != "d1" {
- t.Errorf("DocScope = %v, want the caller's document scope", req.DocScope)
- }
-}
-
-// TestRenderToolOutcomeCoversEveryStatus pins one written sentence per status,
-// so an unmapped status can never silently fall back to a bare result count.
-//
-// Two properties are pinned alongside the status wording, because both are what
-// makes the sentence usable: it names the call's query (parallel calls interleave
-// their lines, so an unlabelled result cannot be paired with its call) and it
-// never spells the machine-readable reason into prose.
-//
-// The label is the ONLY input that differs between the two audiences (the think
-// sentence passes ArgsLabel, the developer line docIDLabel), so this test passes
-// labels rather than arguments: the wording above is written once and cannot
-// drift between them. TestExecuteHidesDocumentIDsFromThink pins the split itself.
-func TestRenderToolOutcomeCoversEveryStatus(t *testing.T) {
- cases := []struct {
- name string
- tool string
- label string
- oc ToolOutcome
- want string
- }{
- {"ok-no-docs", "calculate", "", ToolOutcome{Status: StatusOK, Payload: []any{map[string]any{"id": "c1"}}},
- "The calculate tool returned 1 result."},
- {"ok-with-docs", "retrieve", ` for "曹操是谁"`, ToolOutcome{Status: StatusOK, Payload: []any{
- map[string]any{"doc_id": "d1"}, map[string]any{"doc_id": "d2"}}},
- `The retrieve tool returned 2 results from 2 documents for "曹操是谁".`},
- {"ok-multi-query", "retrieve", ` for "曹操的逝世日期", "曹操是谁"`, ToolOutcome{Status: StatusOK,
- Payload: []any{map[string]any{"doc_id": "d1"}}},
- `The retrieve tool returned 1 result from 1 document for "曹操的逝世日期", "曹操是谁".`},
- // navigate_tree reports routing, not passages: its single payload entry
- // names the documents it routed to as doc_ids.
- {"ok-routed-docs", "navigate_tree", ` for "曹操是谁"`, ToolOutcome{Status: StatusOK, Payload: []any{
- map[string]any{"kind": "navigate_tree", "doc_ids": []string{"d1", "d2", "d3"}}}},
- `The navigate_tree tool returned 1 result from 3 documents for "曹操是谁".`},
- // A document-scoped call has NO label a reader can use: the id names
- // nothing they know, so the sentence goes unlabelled while the developer
- // copy carries " for document d9" (docIDLabel — pinned by TestQuoteQueries).
- {"ok-doc-scoped", "list_chunks", "", ToolOutcome{Status: StatusOK,
- Payload: []any{map[string]any{"id": "c1"}}},
- "The list_chunks tool returned 1 result."},
- // Singular "1 result" must not be followed by "all of them".
- {"redundant", "retrieve", ` for "q"`, ToolOutcome{Status: StatusRedundant,
- Payload: []any{map[string]any{"id": "c1"}}},
- `The retrieve tool returned 1 result for "q", already in the evidence pool.`},
- {"miss", "retrieve", ` for "q"`, ToolOutcome{Status: StatusMiss, Reason: ReasonNoDoc},
- `The retrieve tool matched nothing for "q".`},
- // Without a query argument the sentence still has to name what was
- // matched against, or it ends on a dangling preposition.
- {"miss-unlabelled", "retrieve", "", ToolOutcome{Status: StatusMiss, Reason: ReasonNoDoc},
- "The retrieve tool matched nothing for this query."},
- // A hallucinated tool name is not a failure of this deployment's tools.
- {"unwired", "time_travel", ` for "q"`, ToolOutcome{Status: StatusMiss, Reason: ReasonUnwired},
- `The time_travel tool is not wired in this deployment, so nothing ran for "q".`},
- {"empty-no-structure", "navigate_structure", ` for "曹操是谁"`,
- ToolOutcome{Status: StatusEmpty, Reason: ReasonNoStructure},
- `The navigate_structure tool has no compiled structure to read for "曹操是谁".`},
- {"poor", "calculate", ` for "how many people"`, ToolOutcome{Status: StatusPoor, Reason: ReasonNoDoc},
- `The calculate tool produced a result too weak to use for "how many people".`},
- // The producer's own diagnostic is the actionable half of a failure; the
- // reason token stays in the event.
- {"error-with-cause", "navigate_tree", "", ToolOutcome{Status: StatusError, Reason: ReasonInfra,
- Diagnostic: "nav-tree descent failed for kb=kb1"},
- "The navigate_tree tool could not run: nav-tree descent failed for kb=kb1."},
- }
- for _, tc := range cases {
- if got := renderToolOutcome(tc.tool, tc.label, tc.oc, nil); got != tc.want {
- t.Errorf("%s: renderToolOutcome = %q, want %q", tc.name, got, tc.want)
- }
- }
-
- got := renderToolOutcome("retrieve", ` for "q"`, ToolOutcome{Status: StatusOK}, errors.New("backend down"))
- if !strings.Contains(got, "backend down") || !strings.Contains(got, `for "q"`) {
- t.Errorf("a transport error must be reported against its call: %q", got)
- }
-}
-
-// TestQuoteQueries pins the query label's two shapes and its refusal to label an
-// argument that holds no text (a fall-through, not a blank label).
-func TestQuoteQueries(t *testing.T) {
- cases := []struct {
- name string
- raw any
- want string
- }{
- {"string", "曹操是谁", `"曹操是谁"`},
- {"string-list", []string{"a", "b"}, `"a", "b"`},
- {"any-list", []any{"a", 3, "b"}, `"a", "b"`},
- {"blank", " ", ""},
- {"blank-list", []any{"", " "}, ""},
- {"wrong-type", 42, ""},
- {"nil", nil, ""},
- }
- for _, tc := range cases {
- if got := quoteQueries(tc.raw); got != tc.want {
- t.Errorf("%s: quoteQueries = %q, want %q", tc.name, got, tc.want)
- }
- }
- // A blank query does NOT fall back to the document id on the human side: the
- // id names nothing a reader knows. The developer copy is what carries it.
- if got := ArgsLabel(map[string]any{"query": " ", "doc_id": "d1"}); got != "" {
- t.Errorf("ArgsLabel = %q, want no label for a blank query", got)
- }
- if got := docIDLabel(map[string]any{"query": " ", "doc_id": "d1"}); got != " for document d1" {
- t.Errorf("docIDLabel = %q, want the doc_id fallback for the developer log", got)
- }
- if got := docIDLabel(map[string]any{"doc_id": " "}); got != "" {
- t.Errorf("docIDLabel = %q, want no label for a blank id", got)
- }
- if got := ArgsLabel(map[string]any{"nav_hint": "x"}); got != "" {
- t.Errorf("ArgsLabel = %q, want no label", got)
- }
-
- // A paragraph-long "query" (the slot-research driver searches slot evidence)
- // is capped, and capped on a rune boundary: the label rides the result
- // sentence, so an uncapped one repeated the same 200 characters on every line
- // of the round.
- label := quoteQueries(strings.Repeat("曹", 200))
- if !strings.Contains(label, "…") {
- t.Errorf("quoteQueries(long) = %q, want it capped", label)
- }
- if n := utf8.RuneCountInString(label); n > ThinkLabelMaxRunes+4 {
- t.Errorf("quoteQueries(long) = %d runes, want a capped label", n)
- }
- if !utf8.ValidString(label) {
- t.Errorf("quoteQueries(long) = %q, not valid UTF-8", label)
- }
-}
-
-// TestSearchNilKBIsSeeded pins the seed-on-demand semantics: a nil deps.KB is not a caller
-// mistake — the executor creates the (empty) pool on first use, so the search RUNS and its
-// outcome is decided by what it admitted (OK here), never a forced MISS. The executor must
-// also not panic on the nil pool.
-func TestSearchNilKBIsSeeded(t *testing.T) {
- deps, _ := newTestSearchDeps(&stubRetriever{chunks: []map[string]any{{"content": "hit", "chunk_id": "c1"}}})
- deps.KB = nil
- ex := NewSearchExecutor(deps, RunRequest{DatasetIDs: []string{"kb1"}})
- oc, err := ex.Execute(context.Background(), "retrieve", map[string]any{"query": "q"})
- if err != nil {
- t.Fatalf("nil KB must not error: %v", err)
- }
- if oc.Status != StatusOK {
- t.Errorf("nil KB retrieve status = %s, want %s", oc.Status, StatusOK)
- }
- if len(oc.Payload) != 1 {
- t.Errorf("nil KB retrieve payload = %v, want 1 passage", oc.Payload)
- }
-}
-
-// TestListChunksNilKBIsMiss pins the semantics: a nil deps.KB is seeded on first use, and
-// with no doc-store reader wired the read yields nothing, which is MISS/no_doc (there is no
-// "graceful OK empty" branch). It must never error or panic.
-func TestListChunksNilKBIsMiss(t *testing.T) {
- deps, _ := newTestSearchDeps(&stubRetriever{})
- deps.KB = nil
- ex := &searchExecutor{deps: deps, req: RunRequest{DatasetIDs: []string{"kb1"}}}
- oc, err := ex.listChunks(context.Background(), map[string]any{"doc_id": "doc-a"})
- if err != nil {
- t.Fatalf("nil KB list_chunks must not error: %v", err)
- }
- if oc.Status != StatusMiss || oc.Reason != ReasonNoDoc {
- t.Errorf("nil KB list_chunks = (%s,%s), want (miss,no_doc)", oc.Status, oc.Reason)
- }
- if len(oc.Payload) != 0 {
- t.Errorf("nil KB list_chunks payload = %v, want empty", oc.Payload)
- }
-}
-
-// TestListChunksDeepReadsDocStore pins the deep read: list_chunks reads the document off the
-// chunk store (deps.DocChunks, the same reader fetch_full_document uses) even when NONE of
-// its chunks are in the evidence pool yet, and admits them into the shared pool the model can
-// cite.
-func TestListChunksDeepReadsDocStore(t *testing.T) {
- deps, kb := newTestSearchDeps(&stubRetriever{})
- deps.DocChunks = docChunksFor("doc-a", 2) // only doc-a is readable
- ex := &searchExecutor{deps: deps, req: RunRequest{DatasetIDs: []string{"kb1"}, MaxLength: 8192}}
-
- oc, err := ex.listChunks(context.Background(), map[string]any{"doc_id": "doc-a"})
- if err != nil {
- t.Fatalf("list_chunks: %v", err)
- }
- if oc.Status != StatusOK {
- t.Errorf("status = %s, want %s", oc.Status, StatusOK)
- }
- if len(oc.Payload) != 2 {
- t.Fatalf("payload = %d, want 2 (only doc-a's chunks)", len(oc.Payload))
- }
- // Deep-read admitted into the shared evidence pool.
- if len(kb.Chunks) != 2 {
- t.Errorf("kb.Chunks = %d, want 2 admitted", len(kb.Chunks))
- }
- if len(oc.EvidenceIDs) != 2 {
- t.Errorf("evidenceIDs = %d, want 2", len(oc.EvidenceIDs))
- }
- // Passage shape mirrors _admit_evidence(include_doc_id=False): id + content.
- for _, p := range oc.Payload {
- m, ok := p.(map[string]any)
- if !ok || m["id"] == nil || m["content"] == nil {
- t.Fatalf("passage missing id/content: %#v", p)
- }
- if _, hasDoc := m["doc_id"]; hasDoc {
- t.Errorf("list_chunks passage must not carry doc_id: %#v", m)
- }
- }
-}
-
-// TestListChunksCapsDeepReadAndOutput pins the two caps: the deep read may fetch up to
-// listChunksMaxDeep (80) chunks, but list_chunks admits only the first listChunksMaxOut (30)
-// into the shared pool and shows the model the same 30 — so the pool holds 30, not 80.
-func TestListChunksCapsDeepReadAndOutput(t *testing.T) {
- deps, kb := newTestSearchDeps(&stubRetriever{})
- deps.DocChunks = docChunksFor("doc-a", 100)
- ex := &searchExecutor{deps: deps, req: RunRequest{DatasetIDs: []string{"kb1"}, MaxLength: 1 << 20}}
-
- oc, err := ex.listChunks(context.Background(), map[string]any{"doc_id": "doc-a"})
- if err != nil {
- t.Fatalf("list_chunks: %v", err)
- }
- if len(kb.Chunks) != listChunksMaxOut {
- t.Errorf("kb.Chunks = %d, want admitted cap %d", len(kb.Chunks), listChunksMaxOut)
- }
- if len(oc.Payload) != listChunksMaxOut {
- t.Errorf("payload = %d, want output cap %d", len(oc.Payload), listChunksMaxOut)
- }
-}
-
-// TestEvidencePoolCapStopsAdmitting pins the PR's _EVIDENCE_POOL_CAP early-stop:
-// once the shared pool reaches the cap, a search admits no further chunk, so its
-// outcome collapses to MISS (nothing new was admitted) — a saturated session
-// stops bloating the pool beyond what the SCA view can read.
-func TestEvidencePoolCapStopsAdmitting(t *testing.T) {
- pre := make([]map[string]any, 0, evidencePoolCap)
- for i := 0; i < evidencePoolCap; i++ {
- pre = append(pre, map[string]any{"chunk_id": fmt.Sprintf("pre-%d", i), "content": "old"})
- }
- deps, kb := newTestSearchDeps(&stubRetriever{chunks: []map[string]any{{"content": "hit", "chunk_id": "c1"}}})
- kb.Chunks = pre
- ex := NewSearchExecutor(deps, RunRequest{DatasetIDs: []string{"kb1"}})
- oc, err := ex.Execute(context.Background(), "retrieve", map[string]any{"query": "q"})
- if err != nil {
- t.Fatalf("retrieve: %v", err)
- }
- if oc.Status != StatusMiss {
- t.Errorf("status = %s, want %s (a full pool admits nothing)", oc.Status, StatusMiss)
- }
- if len(kb.Chunks) != evidencePoolCap {
- t.Errorf("kb.Chunks = %d, want the pool to stay at the cap %d", len(kb.Chunks), evidencePoolCap)
- }
- // Dropping one chunk frees a slot and admission resumes.
- kb.Chunks = kb.Chunks[:evidencePoolCap-1]
- oc, err = ex.Execute(context.Background(), "retrieve", map[string]any{"query": "q"})
- if err != nil {
- t.Fatalf("retrieve after freeing a slot: %v", err)
- }
- if oc.Status != StatusOK {
- t.Errorf("status after freeing a slot = %s, want %s", oc.Status, StatusOK)
- }
-}
-
-// TestEvidencePoolCapExemptsTheProbeWindow pins the cap EXEMPTION at the tool
-// boundary: a FULL pool still takes the window that answers a name the pool has
-// not reached, because that window is the probe's own RESULT.
-//
-// The contrasting case is the test above: the same full pool, a query that is not
-// a probe, and nothing is admitted. Both behaviours are needed — the exemption is
-// what keeps a name batch from turning a found member into "nothing new", and the
-// cap is what keeps everything else from bloating storage.
-func TestEvidencePoolCapExemptsTheProbeWindow(t *testing.T) {
- pre := make([]map[string]any, 0, evidencePoolCap)
- for i := 0; i < evidencePoolCap; i++ {
- pre = append(pre, map[string]any{"chunk_id": fmt.Sprintf("pre-%d", i), "content": "already pooled prose"})
- }
- deps, kb := newTestSearchDeps(&stubRetriever{chunks: []map[string]any{
- {"chunk_id": "c1", "content": "荀正 被关公一刀斩于马下"},
- }})
- kb.Chunks = pre
- ex := NewSearchExecutor(deps, RunRequest{DatasetIDs: []string{"kb1"}})
-
- oc, err := ex.Execute(context.Background(), "retrieve", map[string]any{"query": "车胄|荀正|管亥"})
- if err != nil {
- t.Fatalf("retrieve: %v", err)
- }
- if oc.Status != StatusOK {
- t.Errorf("status = %s, want %s (the probe window answered an unanswered name)", oc.Status, StatusOK)
- }
- if len(kb.Chunks) != evidencePoolCap+1 {
- t.Errorf("kb.Chunks = %d, want %d (one seat for the unanswered name)", len(kb.Chunks), evidencePoolCap+1)
- }
- if len(oc.Payload) != 1 {
- t.Errorf("payload = %d, want the probe's window", len(oc.Payload))
- }
-
- // The same probe again: the pool now carries the window, so the cap is back
- // in charge and the outcome is REDUNDANT, not another seat.
- oc, err = ex.Execute(context.Background(), "retrieve", map[string]any{"query": "车胄|荀正|管亥"})
- if err != nil {
- t.Fatalf("retrieve (second): %v", err)
- }
- if len(kb.Chunks) != evidencePoolCap+1 {
- t.Errorf("kb.Chunks = %d, want the pool to stay at %d", len(kb.Chunks), evidencePoolCap+1)
- }
- if oc.Status != StatusMiss && oc.Status != StatusRedundant {
- t.Errorf("status = %s, want MISS/REDUNDANT on the repeated probe", oc.Status)
- }
-}
-
-// TestWebSearchAdmitsToPool pins the web_search parity fix: web results merge into the SAME
-// shared evidence pool as corpus hits, so downstream formalize/compose can cite them, and
-// the outcome is REDUNDANT/OK/MISS like the corpus tools.
-func TestWebSearchAdmitsToPool(t *testing.T) {
- deps, kb := newTestSearchDeps(&stubRetriever{})
- deps.WebSearch = stubWebSearch{results: []string{"web answer one", "web answer two", "web answer three"}}
- // The query list is read from args["query"].
- oc, err := WebSearchTool(context.Background(), deps, map[string]any{"query": []any{"q1", "q2"}})
- if err != nil {
- t.Fatalf("web_search: %v", err)
- }
- if oc.Status != StatusOK {
- t.Errorf("status = %s, want %s", oc.Status, StatusOK)
- }
- if len(oc.Payload) != 3 {
- t.Errorf("payload = %d, want 3", len(oc.Payload))
- }
- if len(kb.Chunks) != 3 {
- t.Errorf("kb.Chunks = %d, want 3 admitted to pool", len(kb.Chunks))
- }
- if len(oc.EvidenceIDs) != 3 {
- t.Errorf("evidenceIDs = %d, want 3", len(oc.EvidenceIDs))
- }
- for _, p := range oc.Payload {
- m, ok := p.(map[string]any)
- if !ok || m["id"] == nil || m["content"] == nil {
- t.Fatalf("passage missing id/content: %#v", p)
- }
- }
-}
-
-// TestWebSearchDedupsAcrossQueries pins the dedup: a passage that both queries return is
-// admitted/shown only once (dedup by chunk_id across the queries via a shared `seen` set).
-func TestWebSearchDedupsAcrossQueries(t *testing.T) {
- deps, kb := newTestSearchDeps(&stubRetriever{})
- deps.WebSearch = stubWebSearch{results: []string{"dup passage", "unique one", "dup passage"}}
- // Same args["query"] contract as above.
- oc, err := WebSearchTool(context.Background(), deps, map[string]any{"query": []any{"q1", "q2"}})
- if err != nil {
- t.Fatalf("web_search: %v", err)
- }
- if len(oc.Payload) != 2 {
- t.Errorf("payload = %d, want 2 (duplicate collapsed)", len(oc.Payload))
- }
- if len(kb.Chunks) != 2 {
- t.Errorf("kb.Chunks = %d, want 2 admitted (duplicate not double-merged)", len(kb.Chunks))
- }
-}
-
-// TestListChunksSkipsEmptyChunkID pins the empty-id skip: a deep-read chunk with no
-// chunk_id is skipped — neither admitted to the pool nor shown to the model.
-func TestListChunksSkipsEmptyChunkID(t *testing.T) {
- deps, kb := newTestSearchDeps(&stubRetriever{})
- deps.DocChunks = docChunksWithIDs("doc-a", []string{"", "c1"}) // first has empty chunk_id
- ex := &searchExecutor{deps: deps, req: RunRequest{DatasetIDs: []string{"kb1"}, MaxLength: 8192}}
-
- oc, err := ex.listChunks(context.Background(), map[string]any{"doc_id": "doc-a"})
- if err != nil {
- t.Fatalf("list_chunks: %v", err)
- }
- if len(kb.Chunks) != 1 {
- t.Errorf("kb.Chunks = %d, want 1 (empty-id chunk skipped)", len(kb.Chunks))
- }
- if len(oc.Payload) != 1 {
- t.Fatalf("payload = %d, want 1", len(oc.Payload))
- }
- if m, ok := oc.Payload[0].(map[string]any); !ok || m["id"] == "" {
- t.Errorf("payload passage must not have empty id: %#v", oc.Payload[0])
- }
-}
-
-// docChunksFor returns a DocChunkLister that serves n chunks for the given doc id
-// (filtering by doc id, like the production chunk-store reader).
-func docChunksFor(docID string, n int) DocChunkLister {
- return docChunksStub{docID: docID, n: n}
-}
-
-type docChunksStub struct {
- docID string
- n int
-}
-
-func (s docChunksStub) DocChunks(_ context.Context, req DocChunksRequest) ([]map[string]any, error) {
- if req.DocID != s.docID {
- return nil, nil
- }
- out := make([]map[string]any, 0, s.n)
- for i := 0; i < s.n; i++ {
- out = append(out, map[string]any{
- "chunk_id": fmt.Sprintf("c%d", i),
- "doc_id": s.docID,
- "content": "x",
- })
- }
- return out, nil
-}
-
-// docChunksWithIDs serves one chunk per id (empty strings stay empty), used to
-// exercise the empty-chunk_id skip.
-func docChunksWithIDs(docID string, ids []string) DocChunkLister {
- return docChunksIDsStub{docID: docID, ids: ids}
-}
-
-type docChunksIDsStub struct {
- docID string
- ids []string
-}
-
-func (s docChunksIDsStub) DocChunks(_ context.Context, req DocChunksRequest) ([]map[string]any, error) {
- if req.DocID != s.docID {
- return nil, nil
- }
- out := make([]map[string]any, 0, len(s.ids))
- for _, id := range s.ids {
- out = append(out, map[string]any{
- "chunk_id": id,
- "doc_id": s.docID,
- "content": "x",
- })
- }
- return out, nil
-}
-
-type stubWebSearch struct{ results []string }
-
-func (s stubWebSearch) Search(_ context.Context, _ []string) ([]string, error) {
- return s.results, nil
-}
-
-// TestSearchRedundantReturnsFullPayload pins fix #5: when every hit is already in the
-// evidence pool the result is StatusRedundant — but the FULL passages are still returned so
-// the model sees what it already has. Dropping the payload to empty hid the evidence.
-func TestSearchRedundantReturnsFullPayload(t *testing.T) {
- chunk := map[string]any{"content": "already known passage", "chunk_id": "c1"}
- deps, _ := newTestSearchDeps(&stubRetriever{chunks: []map[string]any{chunk}})
- // Pre-seed the pool with the same chunk so the merge adds nothing new.
- deps.KB.Chunks = append(deps.KB.Chunks, chunk)
- ex := NewSearchExecutor(deps, RunRequest{DatasetIDs: []string{"kb1"}})
- oc, err := ex.Execute(context.Background(), "retrieve", map[string]any{"query": "q"})
- if err != nil {
- t.Fatalf("retrieve: %v", err)
- }
- if oc.Status != StatusRedundant {
- t.Fatalf("status = %s, want %s", oc.Status, StatusRedundant)
- }
- if len(oc.Payload) != 1 {
- t.Fatalf("REDUNDANT payload = %v, want 1 full passage", oc.Payload)
- }
- p, ok := oc.Payload[0].(map[string]any)
- if !ok || p["content"] != "already known passage" {
- t.Errorf("REDUNDANT payload[0] = %#v, want the known passage", oc.Payload[0])
- }
-}
-
-func TestNavigateToolsRouteWithinSessionDocScope(t *testing.T) {
- for _, tool := range []struct {
- name string
- args map[string]any
- }{
- {"navigate_tree", map[string]any{"query": "topic"}},
- {"navigate_structure", map[string]any{"query": "topic", "kind": "catalog"}},
- } {
- router := &stubNavRouter{docs: [][2]string{{"d1", "summary"}}}
- ex := &searchExecutor{deps: SearchDeps{
- NavRouter: router,
- DocScope: []string{"sess1"},
- KbIDs: []string{"kb1"},
- TenantID: "t1",
- }}
- if _, err := ex.Execute(context.Background(), tool.name, tool.args); err != nil {
- t.Fatalf("%s: %v", tool.name, err)
- }
- if len(router.scope) != 1 || router.scope[0] != "sess1" {
- t.Errorf("%s: router doc scope = %v, want [sess1] (session ceiling)", tool.name, router.scope)
- }
- }
-}
-
-// TestDocInDatasetsRejectsForeignDocumentViaSubsetVerifier reproduces the
-// production DocIDLookup.KnownDocIDs shape: a foreign doc is reported via an
-// EMPTY known map (not a false entry). docInDatasets must still reject it
-// (an unresolvable owner), not fail open.
-
-// End-to-end wiring tests: one Run call, with a stubbed retriever and a
-// scripted model, must move evidence from the backend into Kbinfos and (when a
-// canvas state is attached) into the citation store.
-
-// corpusRetriever answers every query from a fixed corpus.
-type corpusRetriever struct{ calls []string }
-
-func (c *corpusRetriever) Retrieve(_ context.Context, req RetrieveRequest) ([]map[string]any, error) {
- c.calls = append(c.calls, req.Query)
- return []map[string]any{{
- "chunk_id": "c1",
- "content": "Culdcept was created by OmiyaSoft and released in 1999.",
- "doc_id": "doc-culdcept",
- "doc_name": "Culdcept History",
- "dataset_id": "kb1",
- }}, nil
-}
-
-func TestSearchExecutorReportsMissAndRedundancy(t *testing.T) {
- kb := &Kbinfos{}
- sd := SearchDeps{Backend: &corpusRetriever{}, KbIDs: []string{"kb1"}, KB: kb}
- ex := &searchExecutor{deps: sd, req: RunRequest{}}
-
- // First call: OK with new evidence.
- oc, err := ex.Execute(context.Background(), "retrieve", map[string]any{"query": "q1"})
- if err != nil {
- t.Fatalf("Execute: %v", err)
- }
- if oc.Status != StatusOK {
- t.Fatalf("first call status = %s, want ok", oc.Status)
- }
- if len(oc.EvidenceIDs) != 1 {
- t.Errorf("evidence ids = %v, want 1", oc.EvidenceIDs)
- }
-
- // Same query again: every hit is already in the pool -> REDUNDANT, not ok.
- oc, _ = ex.Execute(context.Background(), "retrieve", map[string]any{"query": "q1"})
- if oc.Status != StatusRedundant {
- t.Errorf("repeat call status = %s, want redundant", oc.Status)
- }
-
- // Missing query -> MISS/no_doc, NOT a bad-args error: an empty query list is simply a
- // miss.
- oc, _ = ex.Execute(context.Background(), "retrieve", map[string]any{})
- if oc.Status != StatusMiss || oc.Reason != ReasonNoDoc {
- t.Errorf("missing query: got (%s,%s), want (miss,no_doc)", oc.Status, oc.Reason)
- }
-
- // A genuinely unwired tool -> miss (valid tool, nothing reached), never an
- // error, so the model falls back instead of stalling.
- oc, _ = ex.Execute(context.Background(), "wiki_query", map[string]any{"query": "topic"})
- if oc.Status != StatusMiss {
- t.Errorf("unwired tool status = %s, want miss", oc.Status)
- }
-}
-
-// TestSearchRecordsDocAggsForReferences: the search tools are the only writer of
-// kbinfos["doc_aggs"], which the chat pipeline turns into the answer's
-// document/reference list.
-func TestSearchRecordsDocAggsForReferences(t *testing.T) {
- kb := &Kbinfos{}
- ex := &searchExecutor{deps: SearchDeps{Backend: &corpusRetriever{}, KbIDs: []string{"kb1"}, KB: kb}, req: RunRequest{}}
-
- if _, err := ex.Execute(context.Background(), "retrieve", map[string]any{"query": "Culdcept"}); err != nil {
- t.Fatalf("Execute: %v", err)
- }
- if len(kb.DocAggs) != 1 {
- t.Fatalf("kb.DocAggs = %v, want the searched document recorded", kb.DocAggs)
- }
- if got := kb.DocAggs[0]["doc_id"]; got != "doc-culdcept" {
- t.Errorf("doc_agg doc_id = %v, want doc-culdcept", got)
- }
-
- // A REDUNDANT repeat (nothing new admitted) must not duplicate the agg.
- if _, err := ex.Execute(context.Background(), "retrieve", map[string]any{"query": "Culdcept"}); err != nil {
- t.Fatalf("repeat: %v", err)
- }
- if len(kb.DocAggs) != 1 {
- t.Errorf("kb.DocAggs = %v, want 1 (deduped by doc_id)", kb.DocAggs)
- }
-}
-
-// fakeWikiRetriever returns a fixed compiled wiki page.
-type fakeWikiRetriever struct{}
-
-func (f *fakeWikiRetriever) SearchWiki(_ context.Context, question string, keywords []string, topN int) ([]WikiPage, error) {
- return []WikiPage{{
- ChunkID: "w1", DocID: "wdoc1", DocName: "Synthesis", Title: "Culdcept Overview",
- Content: "Culdcept is a board game by OmiyaSoft.", Score: 0.9,
- }}, nil
-}
-
-func TestWikiQueryWiredReturnsPages(t *testing.T) {
- // When a WikiRetriever is wired, wiki_query returns the parsed page payload in the shape
- // the action layer consumes.
- ex := &searchExecutor{deps: SearchDeps{WikiRetriever: &fakeWikiRetriever{}}, req: RunRequest{}}
-
- oc, err := ex.Execute(context.Background(), "wiki_query", map[string]any{"query": "Culdcept overview"})
- if err != nil {
- t.Fatalf("Execute: %v", err)
- }
- if oc.Status != StatusOK {
- t.Fatalf("status = %s, want ok", oc.Status)
- }
- if n, _ := common.ToInt(oc.Metrics["n_hits"]); n != 1 {
- t.Errorf("n_hits = %v, want 1", oc.Metrics["n_hits"])
- }
- hit, ok := oc.Payload[0].(map[string]any)
- if !ok {
- t.Fatalf("payload[0] not a map: %T", oc.Payload[0])
- }
- for _, key := range []string{"chunk_id", "doc_id", "docnm_kwd", "title", "content", "query"} {
- if _, present := hit[key]; !present {
- t.Errorf("payload missing key %q", key)
- }
- }
- if hit["content"] != "Culdcept is a board game by OmiyaSoft." {
- t.Errorf("content = %v", hit["content"])
- }
-}
-
-func TestWikiQuerySchemaIsUnplugged(t *testing.T) {
- // wiki_query is an UNPLUGGED extension seam: the handler exists
- // (TestWikiQueryWiredReturnsPages), but it is NOT part of any mode's active tool set
- // (removed from allTools) and no SearchWiki backend is wired in production, so it must
- // never appear in the advertised surface.
- for _, mode := range []string{"medium", "high", "ultra"} {
- ts := &Toolset{ThinkingMode: mode}
- if spec, ok := findSpec(ts.ActiveToolSpecs(), "wiki_query"); ok {
- t.Errorf("wiki_query surfaced in mode %q: %+v", mode, spec)
- }
- }
-}
-
-// findSpec is a test helper mirroring ActiveToolSpecs lookup.
-func findSpec(specs []ToolSpec, name string) (ToolSpec, bool) {
- for _, s := range specs {
- if s.Function.Name == name {
- return s, true
- }
- }
- return ToolSpec{}, false
-}
-
-func TestListChunksReadsFromEvidencePool(t *testing.T) {
- // list_chunks deep-reads a document. With no doc-store reader wired it scans the
- // accumulated evidence pool; an unknown/missing doc_id is a query-level MISS, never a
- // dataset EMPTY and never a hard error.
- kb := &Kbinfos{Chunks: []map[string]any{
- {"chunk_id": "c1", "doc_id": "doc-a", "content": "first"},
- {"chunk_id": "c2", "doc_id": "doc-b", "content": "other"},
- {"chunk_id": "c3", "doc_id": "doc-a", "content": "second"},
- }}
- ex := &searchExecutor{deps: SearchDeps{KB: kb}}
-
- oc, _ := ex.Execute(context.Background(), "list_chunks", map[string]any{"doc_id": "doc-a"})
- // Already in evidence → REDUNDANT (nothing new admitted), but the passages
- // are still returned.
- if oc.Status != StatusRedundant {
- t.Fatalf("status = %s, want redundant", oc.Status)
- }
- if len(oc.Payload) != 2 {
- t.Errorf("payload = %d, want 2 chunks of doc-a", len(oc.Payload))
- }
- // Evidence ids are the CHUNK ids, not pool positions.
- if len(oc.EvidenceIDs) != 2 || oc.EvidenceIDs[0] != "c1" || oc.EvidenceIDs[1] != "c3" {
- t.Errorf("evidence ids = %v, want [c1 c3]", oc.EvidenceIDs)
- }
-
- oc, _ = ex.Execute(context.Background(), "list_chunks", map[string]any{"doc_id": "doc-zz"})
- if oc.Status != StatusMiss {
- t.Errorf("unknown doc status = %s, want miss", oc.Status)
- }
- // Missing doc_id → MISS.
- oc, _ = ex.Execute(context.Background(), "list_chunks", map[string]any{})
- if oc.Status != StatusMiss {
- t.Errorf("no doc_id: got %s, want miss", oc.Status)
- }
-}
-
-func TestToolQueriesAcceptsBothShapes(t *testing.T) {
- // Models emit `query` as a string OR a list, per the advertised schema.
- if got := toolQueries(map[string]any{"query": "single"}); len(got) != 1 || got[0] != "single" {
- t.Errorf("string form = %v", got)
- }
- if got := toolQueries(map[string]any{"query": []any{"a", "", "b"}}); len(got) != 2 {
- t.Errorf("list form = %v, want blanks dropped", got)
- }
- if got := toolQueries(map[string]any{"query": []string{"a", "b"}}); len(got) != 2 {
- t.Errorf("[]string form = %v", got)
- }
- if got := toolQueries(map[string]any{}); got != nil {
- t.Errorf("absent query = %v, want nil", got)
- }
- // The `q` alias some models emit.
- if got := toolQueries(map[string]any{"q": "alias"}); len(got) != 1 || got[0] != "alias" {
- t.Errorf("q alias = %v", got)
- }
-}
-
-func TestPublishReferencesSkipsWithoutCanvasState(t *testing.T) {
- // No canvas state attached: publishing is a silent no-op (best-effort),
- // which is what happens in unit tests and non-canvas callers.
- kb := &Kbinfos{Chunks: []map[string]any{{"content": "x", "doc_id": "d1"}}}
- PublishReferences(context.Background(), kb) // must not panic
-}
-
-func TestPassageFromChunkTruncatesContent(t *testing.T) {
- long := strings.Repeat("word ", 500)
- p := passageFromChunk(map[string]any{
- "chunk_id": "c1", "doc_id": "d1", "docnm_kwd": "Title", "content": long,
- })
- content, _ := p["content"].(string)
- // Non-table chunk is cut to 1200 code points as a plain slice — so NO trailing ellipsis
- // is added.
- if n := len([]rune(content)); n != 1200 {
- t.Errorf("content length = %d runes, want 1200", n)
- }
- if strings.HasSuffix(content, "...") {
- t.Errorf("the 1200-code-point cut adds no ellipsis marker: %q", content)
- }
- // Keys: exactly: {"id","content","doc_id"} — the
- // id must live under "id", which is what the drill merge reads back.
- if p["doc_id"] != "d1" || p["id"] != "c1" {
- t.Errorf("passage = %v", p)
- }
-}
-
-// TestMetadataSearchToolBadArgsAndKeyHints pins the two "the model should change what it
-// asked for" outcomes: a call without filters, and a filter key the dataset does not
-// carry. Both are MISS/bad_args — a query-level miss, never a claim about the tool — and
-// the key case names the fields the dataset really has.
-func TestMetadataSearchToolBadArgsAndKeyHints(t *testing.T) {
- deps, _ := newTestSearchDeps(&stubRetriever{})
- deps.MetadataResolver = &stubMetadataResolver{
- pushdownOK: true,
- metas: common.MetaData{"author": {"Alice": {"d1"}}},
- }
- exec := NewSearchExecutor(deps, RunRequest{ThinkingMode: "high"})
- ctx := context.Background()
-
- out, err := exec.Execute(ctx, "metadata_search", map[string]any{})
- if err != nil {
- t.Fatalf("execute: %v", err)
- }
- if out.Status != StatusMiss || out.Reason != ReasonBadArgs {
- t.Errorf("status/reason = %s/%s, want miss/bad_args for a call without filters", out.Status, out.Reason)
- }
- if note := toolNote(out); !strings.Contains(note, "search_chunks") {
- t.Errorf("note = %q, want the fallback tools named", note)
- }
-
- out, err = exec.Execute(ctx, "metadata_search", map[string]any{
- "filters": []any{map[string]any{"key": "title", "op": "contains", "value": "New York"}},
- })
- if err != nil {
- t.Fatalf("execute: %v", err)
- }
- if out.Status != StatusMiss || out.Reason != ReasonBadArgs {
- t.Errorf("status/reason = %s/%s, want miss/bad_args for a missing key", out.Status, out.Reason)
- }
- if note := toolNote(out); !strings.Contains(note, "author") {
- t.Errorf("note = %q, want the dataset's REAL keys listed", note)
- }
-}
-
-// TestMetadataSearchToolInfraWhenIndexUnreadable pins the semantic split: a metadata-index
-// failure is ERROR/infra and must NOT be phrased as "this dataset has no metadata" — the
-// model would then give up on a filter that a working index would have honoured.
-func TestMetadataSearchToolInfraWhenIndexUnreadable(t *testing.T) {
- deps, _ := newTestSearchDeps(&stubRetriever{})
- deps.MetadataResolver = &stubMetadataResolver{flattenErr: errors.New("es down")}
- exec := NewSearchExecutor(deps, RunRequest{ThinkingMode: "high"})
-
- out, err := exec.Execute(context.Background(), "metadata_search", map[string]any{
- "filters": []any{map[string]any{"key": "title", "op": "contains", "value": "New York"}},
- })
- if err != nil {
- t.Fatalf("execute: %v", err)
- }
- if out.Status != StatusError || out.Reason != ReasonInfra {
- t.Errorf("status/reason = %s/%s, want error/infra", out.Status, out.Reason)
- }
- if note := toolNote(out); !strings.Contains(note, "NOT a statement about the dataset") {
- t.Errorf("note = %q, want an explicit infra framing", note)
- }
-}
-
-// TestMetadataSearchToolReturnsMatchedDocIDsOnly pins the selector contract: the tool
-// resolves the matching documents and returns their doc_ids — it runs NO retrieval, admits
-// nothing to the shared pool and returns no passages. The ids are the whole output, because
-// that is what lets the model spend one filter on several follow-up tools.
-//
-// The result also carries the applied filter and each matched document's metadata values —
-// context the model reads, never an argument to hand to another tool (the ids are that
-// argument). Values are drawn from the dataset's DECLARED fields, so annotation noise the
-// dataset never declared cannot leak into every result.
-func TestMetadataSearchToolReturnsMatchedDocIDsOnly(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{
- {"id": "c1", "doc_id": "d1", "content": "Culdcept was released in 1999 by OmiyaSoft."},
- }}
- deps, kb := newTestSearchDeps(r)
- deps.MetadataResolver = &stubMetadataResolver{
- pushdownOK: true,
- pushdownIDs: []string{"d1", "d2"},
- metas: common.MetaData{"title": {"Culdcept": {"d1", "d2"}}},
- docMeta: map[string]map[string]any{
- "d1": {
- "title": "Culdcept",
- "file_name": "culdcept.pdf",
- "update_time": "2026-09-20 13:55:03",
- "verdict": "incorrect", // observed-only noise: must not be carried
- },
- "d2": {"title": []any{"Culdcept", "Culdcept II"}},
- },
- }
- deps.DeclaredMetadata = &stubDeclaredMetadata{defs: []common.MetadataFieldDef{
- {Key: "title", Type: "string"},
- {Key: "file_name", Type: "string"},
- {Key: "update_time", Type: "time"},
- }}
- exec := NewSearchExecutor(deps, RunRequest{ThinkingMode: "high"})
-
- out, err := exec.Execute(context.Background(), "metadata_search", map[string]any{
- "filters": []any{map[string]any{"key": "title", "op": "contains", "value": "Culdcept"}},
- "logic": "and",
- })
- if err != nil {
- t.Fatalf("execute: %v", err)
- }
- if out.Status != StatusOK {
- t.Fatalf("status = %s (reason %s), want ok", out.Status, out.Reason)
- }
- // A selector retrieves nothing: the retriever must not be called at all.
- if len(r.requests) != 0 {
- t.Errorf("retriever calls = %d, want 0 (metadata_search must not search)", len(r.requests))
- }
- if len(kb.Chunks) != 0 {
- t.Errorf("pool chunks = %d, want none admitted by a selector", len(kb.Chunks))
- }
- if len(out.EvidenceIDs) != 0 {
- t.Errorf("evidence ids = %v, want none", out.EvidenceIDs)
- }
- if len(out.Payload) != 1 {
- t.Fatalf("payload = %v, want one doc_ids object", out.Payload)
- }
- m, ok := out.Payload[0].(map[string]any)
- if !ok {
- t.Fatalf("payload[0] = %#v, want a map", out.Payload[0])
- }
- if m["kind"] != "metadata_search" {
- t.Errorf("payload kind = %v, want metadata_search", m["kind"])
- }
- ids, _ := m["doc_ids"].([]string)
- if len(ids) != 2 || ids[0] != "d1" || ids[1] != "d2" {
- t.Errorf("doc_ids = %v, want the matched documents", m["doc_ids"])
- }
- if docs, _ := out.Metrics["docs"].(int); docs != 2 {
- t.Errorf("metrics docs = %v, want 2", out.Metrics["docs"])
- }
-
- // The applied filter travels with the result.
- applied, _ := m["filters"].([]map[string]any)
- if len(applied) != 1 || applied[0]["key"] != "title" || applied[0]["op"] != "contains" {
- t.Errorf("payload filters = %v, want the applied condition", m["filters"])
- }
-
- // Per-document context: declared fields only, lists joined, noise excluded.
- docsList, _ := m["documents"].([]any)
- if len(docsList) != 2 {
- t.Fatalf("payload documents = %v, want one entry per matched document", m["documents"])
- }
- first, _ := docsList[0].(map[string]any)
- if first["doc_id"] != "d1" {
- t.Errorf("documents[0].doc_id = %v, want d1", first["doc_id"])
- }
- fields, _ := first["metadata"].(map[string]any)
- if fields["file_name"] != "culdcept.pdf" || fields["update_time"] != "2026-09-20 13:55:03" {
- t.Errorf("documents[0].metadata = %v, want the document's declared values", fields)
- }
- if _, leaked := fields["verdict"]; leaked {
- t.Errorf("documents[0].metadata carries an undeclared observed key: %v", fields)
- }
- second, _ := docsList[1].(map[string]any)
- fields2, _ := second["metadata"].(map[string]any)
- if fields2["title"] != "Culdcept, Culdcept II" {
- t.Errorf("documents[1].metadata[title] = %v, want the list joined with \", \"", fields2["title"])
- }
-}
-
-// TestMetadataSearchToolContextReadFailureKeepsTheSelection pins the best-effort contract of
-// the context block: the documents were already resolved, so a failed metadata read must
-// still return them (with the filter) rather than turning the call into an error.
-func TestMetadataSearchToolContextReadFailureKeepsTheSelection(t *testing.T) {
- deps, _ := newTestSearchDeps(&stubRetriever{})
- deps.MetadataResolver = &stubMetadataResolver{
- pushdownOK: true,
- pushdownIDs: []string{"d1"},
- metas: common.MetaData{"title": {"Culdcept": {"d1"}}},
- docMetaErr: errors.New("metadata index down"),
- }
- exec := NewSearchExecutor(deps, RunRequest{ThinkingMode: "high"})
-
- out, err := exec.Execute(context.Background(), "metadata_search", map[string]any{
- "filters": []any{map[string]any{"key": "title", "op": "contains", "value": "Culdcept"}},
- })
- if err != nil {
- t.Fatalf("execute: %v", err)
- }
- if out.Status != StatusOK {
- t.Fatalf("status = %s (reason %s), want ok — the ids resolved, only the context is missing", out.Status, out.Reason)
- }
- m, _ := out.Payload[0].(map[string]any)
- if got := payloadDocIDs(out.Payload); len(got) != 1 || got[0] != "d1" {
- t.Errorf("doc_ids = %v, want the matched document", got)
- }
- if _, present := m["documents"]; present {
- t.Errorf("payload documents = %v, want the block omitted when nothing could be read", m["documents"])
- }
-}
-
-// TestMetadataSearchToolAcceptsCatalogFieldKey pins the non-title path end to end at the
-// executor: a field the dataset really carries is accepted and the push-down resolves the
-// documents. Before the catalog the schema never let a model emit this filter, so the
-// executor's field-agnosticism was unreachable in practice.
-func TestMetadataSearchToolAcceptsCatalogFieldKey(t *testing.T) {
- deps, _ := newTestSearchDeps(&stubRetriever{})
- resolver := &stubMetadataResolver{
- pushdownOK: true,
- pushdownIDs: []string{"d1"},
- metas: common.MetaData{"author": {"Alice": {"d1"}}},
- }
- deps.MetadataResolver = resolver
- exec := NewSearchExecutor(deps, RunRequest{ThinkingMode: "high"})
-
- out, err := exec.Execute(context.Background(), "metadata_search", map[string]any{
- "filters": []any{map[string]any{"key": "author", "op": "contains", "value": "Alice"}},
- "logic": "and",
- })
- if err != nil {
- t.Fatalf("execute: %v", err)
- }
- if out.Status != StatusOK {
- t.Fatalf("status = %s (reason %s), want ok — a field the dataset carries must not be rejected", out.Status, out.Reason)
- }
- if len(resolver.gotFilters) != 1 || resolver.gotFilters[0]["key"] != "author" {
- t.Errorf("resolver filters = %v, want the author condition", resolver.gotFilters)
- }
- if got := payloadDocIDs(out.Payload); len(got) != 1 || got[0] != "d1" {
- t.Errorf("doc_ids = %v, want the author-matched document", got)
- }
-}
-
-// TestMetadataSearchToolWithoutMetadataIsMissNotError pins the metadata-free outcome: a
-// call against a dataset whose metadata index carries NOTHING is a query-level bad_args
-// miss with the fallback named — never an ERROR, which would tell the model the
-// infrastructure failed and it should give up on the whole line of search.
-func TestMetadataSearchToolWithoutMetadataIsMissNotError(t *testing.T) {
- deps, _ := newTestSearchDeps(&stubRetriever{})
- deps.MetadataResolver = &stubMetadataResolver{pushdownOK: true, metas: common.MetaData{}}
- exec := NewSearchExecutor(deps, RunRequest{ThinkingMode: "high"})
-
- out, err := exec.Execute(context.Background(), "metadata_search", map[string]any{
- "filters": []any{map[string]any{"key": "title", "op": "contains", "value": "New York"}},
- })
- if err != nil {
- t.Fatalf("execute: %v", err)
- }
- if out.Status == StatusError {
- t.Fatalf("status = error (reason %s); a metadata-free dataset is NOT an infrastructure failure", out.Reason)
- }
- if out.Status != StatusMiss || out.Reason != ReasonBadArgs {
- t.Errorf("status/reason = %s/%s, want miss/bad_args", out.Status, out.Reason)
- }
- note := toolNote(out)
- if !strings.Contains(note, "NONE") {
- t.Errorf("note = %q, want the explicit no-metadata hint", note)
- }
- if !strings.Contains(note, "search_chunks") {
- t.Errorf("note = %q, want the fallback tools named", note)
- }
-}
-
-// TestMetadataSearchToolAcceptsDeclaredOnlyField pins the consistency the declarative source
-// forces: a field the dataset DECLARES but has not indexed yet is advertised by the schema, so
-// the executor — which validates against the same union — must accept it rather than answer
-// "no such key" to the very filter it offered.
-func TestMetadataSearchToolAcceptsDeclaredOnlyField(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{
- {"id": "c1", "doc_id": "d1", "content": "A report about towers."},
- }}
- deps, _ := newTestSearchDeps(r)
- deps.MetadataResolver = &stubMetadataResolver{pushdownOK: true, pushdownIDs: []string{"d1"}, metas: common.MetaData{}}
- deps.DeclaredMetadata = &stubDeclaredMetadata{defs: []common.MetadataFieldDef{
- {Key: "doc_type", Description: "kind of document", Enum: []string{"report"}},
- }}
- exec := NewSearchExecutor(deps, RunRequest{ThinkingMode: "high"})
-
- out, err := exec.Execute(context.Background(), "metadata_search", map[string]any{
- "filters": []any{map[string]any{"key": "doc_type", "op": "=", "value": "report"}},
- })
- if err != nil {
- t.Fatalf("execute: %v", err)
- }
- if out.Status != StatusOK {
- t.Fatalf("status = %s (reason %s); a declared field must be accepted, not answered with bad_args", out.Status, out.Reason)
- }
- if got := payloadDocIDs(out.Payload); len(got) != 1 || got[0] != "d1" {
- t.Errorf("doc_ids = %v, want the declared-field match", got)
- }
-}
-
-// TestMetadataSearchToolRejectsSystemKey pins the blacklist at the enforcement point: a key
-// the catalog refuses to advertise (benchmark labels, connector fields) is ALSO refused when
-// the model names it anyway, so the leak cannot be reached by guessing.
-func TestMetadataSearchToolRejectsSystemKey(t *testing.T) {
- deps, _ := newTestSearchDeps(&stubRetriever{})
- deps.MetadataResolver = &stubMetadataResolver{pushdownOK: true, metas: common.MetaData{
- "title": {"Culdcept": {"d1"}},
- "question_id": {"444": {"d1"}},
- }}
- exec := NewSearchExecutor(deps, RunRequest{ThinkingMode: "high"})
-
- out, err := exec.Execute(context.Background(), "metadata_search", map[string]any{
- "filters": []any{map[string]any{"key": "question_id", "op": "=", "value": "444"}},
- })
- if err != nil {
- t.Fatalf("execute: %v", err)
- }
- if out.Status != StatusMiss || out.Reason != ReasonBadArgs {
- t.Fatalf("status/reason = %s/%s, want miss/bad_args for a blacklisted key", out.Status, out.Reason)
- }
- note := toolNote(out)
- if !strings.Contains(note, "Available: title") {
- t.Errorf("note = %q, want the offered fields only (no question_id)", note)
- }
- if strings.Contains(note, "444") {
- t.Errorf("note = %q, must not echo a blacklisted key's value", note)
- }
-}
-
-// TestNormalizeMetadataValue pins the value coercion: a list where one keyword is expected
-// collapses to its first non-empty element (one call = one keyword), 'in' keeps its list,
-// and an all-empty list becomes no value at all.
-func TestNormalizeMetadataValue(t *testing.T) {
- cases := []struct {
- value any
- op string
- want any
- }{
- {[]any{"New York", "Boston"}, "contains", "New York"},
- {[]any{"", nil, "Boston"}, "contains", "Boston"},
- {[]any{"", nil}, "contains", nil},
- {[]any{"a", "b"}, "in", []any{"a", "b"}},
- {"New York", "contains", "New York"},
- }
- for _, c := range cases {
- if got := NormalizeMetadataValue(c.value, c.op); fmt.Sprint(got) != fmt.Sprint(c.want) {
- t.Errorf("NormalizeMetadataValue(%v, %q) = %v, want %v", c.value, c.op, got, c.want)
- }
- }
-}
-
-// TestMetadataSearchOneShotGuardBlocksSecondCall pins the one-shot policy end to end at
-// the tool node: the first call runs, the second within the same session is answered with
-// a nudge and never executed.
-func TestMetadataSearchOneShotGuardBlocksSecondCall(t *testing.T) {
- exec := &ladderExec{emptyFor: map[string]bool{}}
- st := &SessionState{
- Tools: &Toolset{Exec: exec, ThinkingMode: "high"},
- DeadlineLeft: 60,
- ToolCache: NewToolCache(),
- PendingCalls: []ToolCall{{ID: "call-1", Name: "metadata_search"}},
- }
- if err := st.toolNode(context.Background()); err != nil {
- t.Fatalf("toolNode: %v", err)
- }
- if len(exec.calls) != 1 || exec.calls[0] != "metadata_search" {
- t.Fatalf("calls = %v, want the first metadata_search executed", exec.calls)
- }
- if !st.MetadataSearchUsed {
- t.Error("MetadataSearchUsed must be set after the first call")
- }
-
- st.PendingCalls = []ToolCall{{ID: "call-2", Name: "metadata_search"}}
- if err := st.toolNode(context.Background()); err != nil {
- t.Fatalf("toolNode (second turn): %v", err)
- }
- if len(exec.calls) != 1 {
- t.Errorf("calls = %v, want the second metadata_search blocked", exec.calls)
- }
- last := st.Messages[len(st.Messages)-1]
- if !strings.Contains(last.Content, "ONE-SHOT") {
- t.Errorf("second call's tool message = %q, want the one-shot nudge", last.Content)
- }
-}
-
-// toolNote reads the note off a tool outcome whose payload is a single note object.
-func toolNote(out ToolOutcome) string {
- if len(out.Payload) == 0 {
- return ""
- }
- m, ok := out.Payload[0].(map[string]any)
- if !ok {
- return ""
- }
- note, _ := m["note"].(string)
- return note
-}
-
-// TestPassageFromChunkRendersTablesAsMarkdown pins the action-session table shape: the
-// model sees a Markdown view (same rows, a fraction of the tokens) and never the raw
-// <table>/<td> markup. The chunk in the shared pool stays raw for citation.
-func TestPassageFromChunkRendersTablesAsMarkdown(t *testing.T) {
- table := "<table><tr><th>Rank</th><th>Rider</th><th>Points</th></tr>" +
- "<tr><td>19</td><td>Danilo</td><td>62</td></tr>" +
- "<tr><td>20</td><td>Erik</td><td>61</td></tr></table>"
- p := passageFromChunk(map[string]any{
- "chunk_id": "c1", "doc_id": "d1", "content": table,
- })
- content, _ := p["content"].(string)
- if strings.Contains(content, "<td>") || strings.Contains(content, "<table") {
- t.Errorf("the model-visible passage still carries raw HTML: %q", content)
- }
- for _, want := range []string{"| 19 |", "Danilo", "| 20 |", "Erik"} {
- if !strings.Contains(content, want) {
- t.Errorf("passage lost %q: %q", want, content)
- }
- }
- if p["doc_id"] != "d1" || p["id"] != "c1" {
- t.Errorf("passage = %v", p)
- }
-}
-
-// seatRetriever answers a search by its query string, normalized to the DISTINCT
-// tokens it carries: the engine receives "question keywords" (a seat search sends
-// the same term twice), so a fixture must not depend on how that string is
-// assembled.
-type seatRetriever struct {
- byQuery map[string][]map[string]any
- calls []string
-}
-
-func (s *seatRetriever) Retrieve(_ context.Context, req RetrieveRequest) ([]map[string]any, error) {
- s.calls = append(s.calls, req.Query)
- seen := map[string]bool{}
- var tokens []string
- for _, tok := range strings.Fields(req.Query) {
- if seen[tok] {
- continue
- }
- seen[tok] = true
- tokens = append(tokens, tok)
- }
- return s.byQuery[strings.Join(tokens, " ")], nil
-}
-
-// TestNamedTermSeatsReachTermsThePhraseSearchMissed pins the seat pass: every
-// individual a call NAMES gets its own cheap keyword search, so the names the
-// call's own phrase queries cannot reach still arrive with a passage.
-//
-// Three things are asserted, and each one is a measured loss from the 2026-09-15
-// run: (a) a name the phrase query's ranking stranded (荀正) still gets a
-// passage; (b) a name in a list item maxQ DROPPED (杨龄 — the run left 8 of 29
-// named queries unexecuted) still gets one; (c) a name nothing reaches is
-// recorded as probed-and-absent rather than silently dropped (庞德).
-func TestNamedTermSeatsReachTermsThePhraseSearchMissed(t *testing.T) {
- famous := map[string]any{"chunk_id": "c-famous", "content": "关羽 斩华雄 于马下。"}
- rare := map[string]any{"chunk_id": "c-rare", "content": "荀正 引军来战,关羽一刀斩之。"}
- predicate := map[string]any{"chunk_id": "c-pred", "content": "云长 斩颜良 于白马,文丑心怯。"}
- third := map[string]any{"chunk_id": "c-third", "content": "杨龄 出马,关羽手起刀落。"}
- r := &seatRetriever{byQuery: map[string][]map[string]any{
- // The phrase batch returns ONLY the passage matching several names at
- // once — the ranking that strands the rare ones.
- "关羽 斩华雄 荀正": {famous},
- "关羽 斩颜良": {},
- "关羽": {famous},
- "斩华雄": {famous},
- "荀正": {rare},
- "斩颜良": {predicate},
- "杨龄": {third},
- // 庞德 reaches nothing anywhere: the corpus does not carry it.
- "庞德": {},
- }}
- deps, kb := newTestSearchDeps(r)
- ex := NewSearchExecutor(deps, RunRequest{DatasetIDs: []string{"kb1"}})
-
- oc, err := ex.Execute(context.Background(), "retrieve", map[string]any{
- "query": []any{"关羽 斩华雄 荀正", "关羽 斩颜良", "杨龄", "庞德"},
- })
- if err != nil {
- t.Fatalf("retrieve: %v", err)
- }
- pool := map[string]bool{}
- for _, c := range kb.Chunks {
- pool[ChunkIDOf(c)] = true
- }
- for _, want := range []string{"c-famous", "c-rare", "c-pred", "c-third"} {
- if !pool[want] {
- t.Errorf("pool lacks %s: %v — a named term lost its seat (phrase ranking, maxQ cut, or the flat per-query cap)", want, pool)
- }
- }
- if oc.Status != StatusOK {
- t.Errorf("status = %s, want %s (the seats are new evidence)", oc.Status, StatusOK)
- }
- absent := kb.ProbedAbsentTerms()
- if !containsString(absent, "庞德") {
- t.Errorf("ProbedAbsent = %v, want 庞德 recorded: a probe that reaches nothing is a fact about the corpus, not a failed lookup", absent)
- }
- if containsString(absent, "荀正") || containsString(absent, "杨龄") {
- t.Errorf("ProbedAbsent = %v: a term that got a seat must not be recorded as unreached", absent)
- }
-}
-
-func containsString(list []string, want string) bool {
- for _, s := range list {
- if s == want {
- return true
- }
- }
- return false
-}
-
-// TestSearchOutcomeCarriesTheReachLine pins the delivery: the per-term reach the
-// grep leg computes reaches the MODEL through ToolOutcome.Note (rendered behind
-// the payload by the session tool node), not only the log.
-//
-// This is what makes a batch probe iterable: "华雄|荀正|管亥" either answers all
-// three or says which of them nothing reached, and the next call can be aimed.
-func TestSearchOutcomeCarriesTheReachLine(t *testing.T) {
- deps, _ := newTestSearchDeps(&stubRetriever{chunks: []map[string]any{
- {"chunk_id": "c1", "content": "云长手起一刀,斩华雄于马下"},
- }})
- ex := NewSearchExecutor(deps, RunRequest{DatasetIDs: []string{"kb1"}})
-
- oc, err := ex.Execute(context.Background(), "retrieve", map[string]any{"query": "华雄|荀正|管亥"})
- if err != nil {
- t.Fatalf("retrieve: %v", err)
- }
- if oc.Note == "" {
- t.Fatal("Note is empty: the reach of a batch probe must reach the model")
- }
- for _, want := range []string{"华雄(1)", "荀正", "管亥", "NOT reached by this query"} {
- if !strings.Contains(oc.Note, want) {
- t.Errorf("Note %q missing %q", oc.Note, want)
- }
- }
-
- // A query with nothing to report on carries no note at all.
- plain, err := ex.Execute(context.Background(), "retrieve", map[string]any{"query": "云长"})
- if err != nil {
- t.Fatalf("retrieve (plain): %v", err)
- }
- if strings.HasPrefix(plain.Note, "[reach]") && !strings.Contains(plain.Note, "carry:") {
- t.Errorf("Note = %q, want no half-formed reach line", plain.Note)
- }
-}
-
-// TestSetDirectionWidensTheQueryBudget pins the recall rule a SET/COUNT direction
-// buys, and it is deliberately written with NEUTRAL queries: nothing here depends
-// on a corpus, a language, or a name.
-//
-// A direction assembling a SET asks the corpus about one facet per query, so a
-// dropped query is a member nobody searched rather than a spared repeat. The
-// executor therefore runs more of them once the direction has declared itself
-// (Kbinfos.MarkSetDirection — set by the same gate that hands the model the set
-// method), and when it still cannot run them all it SAYS so in the tool result,
-// because a silent cut is invisible in the passages.
-//
-// Measured (2026-09-16, medium mode): every call asked 3-5 queries against caps of
-// 2 (search_chunks) / 3 (retrieve), nine calls were cut, and the members the run
-// then failed to record had been named only in the dropped ones.
-func TestSetDirectionWidensTheQueryBudget(t *testing.T) {
- queries := []string{"term one", "term two", "term three", "term four"}
-
- // A value direction keeps the small cap — and reports the cut.
- valueDeps, _ := newTestSearchDeps(&stubRetriever{chunks: []map[string]any{{"content": "hit", "chunk_id": "c1"}}})
- value := &stubRetriever{chunks: []map[string]any{{"content": "hit", "chunk_id": "c1"}}}
- valueDeps.Backend = value
- valueEx := NewSearchExecutor(valueDeps, RunRequest{DatasetIDs: []string{"kb1"}})
- oc, err := valueEx.Execute(context.Background(), "retrieve", map[string]any{"query": queries})
- if err != nil {
- t.Fatalf("retrieve: %v", err)
- }
- if !strings.Contains(oc.Note, "only 3 of this call's 4 queries were searched") {
- t.Errorf("Note = %q, want it to name the dropped query", oc.Note)
- }
-
- // The same call on a SET direction runs every query and reports no cut.
- setDeps, setKB := newTestSearchDeps(&stubRetriever{chunks: []map[string]any{{"content": "hit", "chunk_id": "c1"}}})
- dropped := &stubRetriever{chunks: []map[string]any{{"content": "hit", "chunk_id": "c1"}}}
- setDeps.Backend = dropped
- setKB.MarkSetDirection()
- if !setKB.IsSetDirection() {
- t.Fatal("MarkSetDirection did not declare the direction")
- }
- setEx := NewSearchExecutor(setDeps, RunRequest{DatasetIDs: []string{"kb1"}})
- oc2, err := setEx.Execute(context.Background(), "retrieve", map[string]any{"query": queries})
- if err != nil {
- t.Fatalf("retrieve (set): %v", err)
- }
- if strings.Contains(oc2.Note, "queries were searched") {
- t.Errorf("Note = %q, want no cut on a set direction", oc2.Note)
- }
- if len(dropped.requests) <= len(value.requests) {
- t.Errorf("set direction searched %d backend request(s), value direction %d: the set direction must search MORE of the caller's own queries",
- len(dropped.requests), len(value.requests))
- }
-
- // A nil pool is not a set direction, and asking is safe.
- var noPool *Kbinfos
- if noPool.IsSetDirection() {
- t.Error("a nil pool must not report a set direction")
- }
- noPool.MarkSetDirection()
-}
-
-// TestChunksToMapsCarriesReferenceFields pins the fields the answer reference
-// card and the shared citation readers expect on an agentic chunk:
-// doc_type_kwd (the image/table marker) and the content_with_weight spelling of
-// the body. Without them an image chunk reached the response as an untyped,
-// bodyless reference while the naive path rendered it fine.
-func TestChunksToMapsCarriesReferenceFields(t *testing.T) {
- got := chunksToMaps([]runtime.RetrievalChunk{{
- ID: "c1",
- Content: "wolf",
- ImageID: "kb-doc",
- DocType: "image",
- }})
- if len(got) != 1 {
- t.Fatalf("len = %d, want 1", len(got))
- }
- if got[0]["doc_type_kwd"] != "image" {
- t.Errorf("doc_type_kwd = %v, want image", got[0]["doc_type_kwd"])
- }
- if got[0]["content_with_weight"] != "wolf" || got[0]["content"] != "wolf" {
- t.Errorf("content fields = %v/%v, want wolf/wolf",
- got[0]["content_with_weight"], got[0]["content"])
- }
- if got[0]["image_id"] != "kb-doc" {
- t.Errorf("image_id = %v, want kb-doc", got[0]["image_id"])
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/tool_exploration.go b/internal/rag/agentic-rag/runtime/tool_exploration.go
deleted file mode 100644
index 34ea10cb48..0000000000
--- a/internal/rag/agentic-rag/runtime/tool_exploration.go
+++ /dev/null
@@ -1,941 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "ragflow/internal/common"
- "sort"
- "strings"
-
- "ragflow/internal/dao"
- "ragflow/internal/engine"
- "ragflow/internal/engine/types"
- "ragflow/internal/service"
- "ragflow/internal/service/nlp"
-)
-
-// Exploration providers: knowledge-graph walks (graph_explore) and wiki page drill-downs
-// (wiki_query), plus the open-web search provider seam.
-//
-// graph_explore walks the compiled knowledge graph (see ExploreGraph): it seeds entities by
-// dense similarity, hops out over relations, then asks the model whether the subgraph
-// answers the question. wiki_query searches the compiled wiki_page_draft rows.
-//
-// Search seams stay independent of the search backend so the runtime core is
-// testable without one; the bridge wires concrete implementations (wiki_page_draft
-// rows for SearchWiki, an HTTP client for WebSearcher).
-
-// WebSearcher runs an open-web search and returns result strings.
-type WebSearcher interface {
- Search(ctx context.Context, queries []string) ([]string, error)
-}
-
-// WikiPage is one compiled wiki page returned by WikiRetriever: the parsed page markdown
-// plus the
-// originating row identity so the runtime can cite/load it.
-type WikiPage struct {
- ChunkID string
- DocID string
- DocName string
- Title string
- // Content is the synthesised page markdown.
- Content string
- // Score is the retrieval relevance (higher = better).
- Score float64
-}
-
-// WikiRetriever searches the compiled wiki_page_draft rows for a question and returns the
-// most relevant pages. It is a seam, so the runtime core stays independent of the search
-// backend and remains testable without one.
-type WikiRetriever interface {
- // SearchWiki returns up to topN compiled wiki pages for the question,
- // biased toward the optional keywords.
- SearchWiki(ctx context.Context, question string, keywords []string, topN int) ([]WikiPage, error)
-}
-
-// wikiQuery implements the `wiki_query` tool.
-//
-// It searches the compiled "wiki_page_draft" rows for the question, parses each
-// hit's synthesised page markdown out of the row, and returns them in the same
-// payload shape the action layer consumes: per hit
-//
-// {chunk_id, doc_id, docnm_kwd, title, content, query}
-//
-// plus n_hits in Metrics. When no wiki retriever is wired (or the dataset has no wiki
-// compilation), it returns a clean MISS so the model falls back to corpus tools.
-func (e *searchExecutor) wikiQuery(ctx context.Context, args map[string]any) (ToolOutcome, error) {
- if e.deps.WikiRetriever == nil {
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": "wiki_query",
- "note": "wiki_query is not available for this dataset (no compiled wiki). Use corpus tools (retrieve/search_chunks) for in-document facts.",
- }},
- Status: StatusMiss,
- Reason: ReasonNoDoc,
- Metrics: map[string]any{},
- }, nil
- }
-
- // The wiki_query schema presents a 1-2 element array: [question, keywords?].
- queries := toolQueries(args)
- question := strings.TrimSpace(argString(args, "query"))
- keywords := toolStringList(args, "keywords")
- if question == "" && len(queries) > 0 {
- question = queries[0]
- if len(queries) > 1 {
- keywords = append(keywords, splitComma(queries[1])...)
- }
- }
- if question == "" {
- return ToolOutcome{Payload: []any{}, Status: StatusError, Reason: ReasonBadArgs}, nil
- }
-
- topN, _ := common.ToInt(args["top_n"])
- if topN <= 0 {
- topN = 12
- }
- if topN > 30 {
- topN = 30
- }
-
- pages, err := e.deps.WikiRetriever.SearchWiki(ctx, question, keywords, topN)
- if err != nil || len(pages) == 0 {
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": "wiki_query",
- "note": "No compiled wiki page matched the question.",
- }},
- Status: StatusMiss,
- Reason: ReasonNoDoc,
- Metrics: map[string]any{},
- }, nil
- }
-
- passages := make([]any, 0, len(pages))
- for _, p := range pages {
- passages = append(passages, map[string]any{
- "chunk_id": p.ChunkID,
- "doc_id": p.DocID,
- "docnm_kwd": p.DocName,
- "title": p.Title,
- "content": p.Content,
- "query": question,
- })
- }
- return ToolOutcome{
- Payload: passages,
- Status: StatusOK,
- Reason: ReasonNone,
- Metrics: map[string]any{"n_hits": len(passages)},
- }, nil
-}
-
-// splitComma splits a comma-separated keyword string into a clean slice,
-// tolerating spaces.
-func splitComma(s string) []string {
- if s == "" {
- return nil
- }
- parts := strings.Split(s, ",")
- out := make([]string, 0, len(parts))
- for _, p := range parts {
- if t := strings.TrimSpace(p); t != "" {
- out = append(out, t)
- }
- }
- return out
-}
-
-// graphExplore caps: at most 6 passages of 1500 chars.
-const (
- graphExploreMaxChunks = 6
- graphExplorePassageChars = 1500
-)
-
-// graphExplore explores the compiled knowledge graph for a relational / multi-hop answer
-// (ultra only), delegating to ExploreGraph: seed entities for the query, hop along their
-// relations, and return either a direct answer or the source passages behind the relevant
-// entities.
-//
-// The outcome shape: a single-element payload tagged kind="graph_explore" whose body is
-// exactly one of answer / note / chunks, at most 6 passages of 1500 chars, and NO pool
-// merge.
-//
-// Deliberate deviation from the original design: reading c["id"] / c["content"] while the
-// loader emits chunk_id / content_with_weight yields an always-empty snippet with zero
-// evidence ids — a key-mismatch bug. This reads the keys the loader produces.
-func (e *searchExecutor) graphExplore(ctx context.Context, args map[string]any) (ToolOutcome, error) {
- query := argString(args, "query")
- if query == "" {
- query = e.req.Question
- }
- if query == "" {
- return ToolOutcome{Payload: []any{}, Status: StatusError, Reason: ReasonBadArgs}, nil
- }
- // No keywords: graph_explore is called with keywords empty, so the graph's own
- // narrowing is a no-op. e.req.Keywords must not be injected.
- docScope := toolDocScope(args)
- if docScope != nil && len(docScope) == 0 {
- // An explicitly empty list is falsy here: it reads as "no scope" and scans the
- // bound datasets. (The non-nil empty "match nothing" signal only comes from
- // resolveDocScope, downstream of here.)
- docScope = nil
- }
- res, err := ExploreGraph(ctx, e.deps, e.deps.TenantID, e.deps.KbIDs, query, "", docScope)
- if err != nil {
- // The failure is swallowed (res = {}) and falls into
- // the empty branch below — an infra failure here is reported as a
- // dataset-level EMPTY/no_structure, never an ERROR.
- _LOG.Printf("[graph_explore] failed: %v", err)
- res = ExploreResult{}
- }
- answer := strings.TrimSpace(res.Answer)
- if answer != "" {
- // a direct answer short-circuits: no passages.
- return ToolOutcome{
- Payload: []any{map[string]any{"kind": "graph_explore", "answer": answer}},
- Status: StatusOK,
- Reason: ReasonNone,
- Metrics: map[string]any{"hits": 0},
- }, nil
- }
- if len(res.Chunks) == 0 {
- // no compiled KG in scope is a DATASET-level dead end
- // (EMPTY/no_structure), the one class of failure that may disable the tool.
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": "graph_explore",
- "note": "This dataset has NO compiled knowledge graph (or none in the given scope). graph_explore is unavailable; use search_chunks / navigate_structure / retrieve instead.",
- }},
- Status: StatusEmpty,
- Reason: ReasonNoStructure,
- Metrics: map[string]any{},
- }, nil
- }
- chunks := res.Chunks
- if len(chunks) > graphExploreMaxChunks {
- chunks = chunks[:graphExploreMaxChunks]
- }
- snippet := make([]any, 0, len(chunks))
- for _, c := range chunks {
- snippet = append(snippet, map[string]any{
- "id": ChunkIDOf(c),
- "content": truncateRunes(ChunkTextOf(c), graphExplorePassageChars),
- })
- }
- // No EvidenceIDs and no pool merge: the loaded passages are not admitted to kbinfos.
- return ToolOutcome{
- Payload: []any{map[string]any{"kind": "graph_explore", "chunks": snippet}},
- Status: StatusOK,
- Reason: ReasonNone,
- Metrics: map[string]any{"hits": 0},
- }, nil
-}
-
-// graph_explore (knowledge-graph walk) — mirrors exploration.py::graph_explore
-
-// ExploreGraph walks the compiled knowledge graph: seed entities for the query
-// by dense similarity, hop kgHops out over their relations, then ask the model
-// whether the resulting subgraph answers the question directly. When it does the
-// answer is returned; when it doesn't, the source passages behind the relevant
-// nodes are returned so the caller can keep researching.
-//
-// Seeds use the tenant embedding model via service.NavEmbedder (dense KNN,
-// similarity >= kgSeedSim, re-ranked by mention_count_int desc); when the embedding model
-// is unavailable it degrades to keyword match.
-const (
- kgScopeDataset = "dataset"
- kgScopeDoc = "doc"
-
- kgSeeds = 2 // top-N entities matched directly to the question
- kgSeedPool = 64 // KNN candidate pool before the mention_count_int re-sort
- kgSeedSim = 0.8 // dense seed similarity floor
- kgHops = 2 // relation hops out from the seeds
- kgNeighbors = 128 // cap on neighbour entity rows resolved per hop
- kgRelLimit = 32 // relations fetched per endpoint filter
-)
-
-// kgScope is one (kb, tenant, docs) search group. When Docs is non-empty the group
-// searches
-// within those documents (kgScopeDoc); when nil it searches the whole dataset
-// (kgScopeDataset).
-type kgScope struct {
- KBID string
- TenantID string
- Docs []string
-}
-
-// resolveKGScope builds the (kb, tenant, docs) search groups. The session doc_scope
-// ceilings the caller's scope first. With no scope
-// it returns one group per bound dataset (docs=nil => whole-dataset search).
-// With a scope and a DocTenantResolver it groups documents by their real owning
-// (kb, tenant), which may surface knowledge bases outside datasetIDs; documents
-// whose owner cannot be resolved fall back to the caller's datasets. Without a
-// resolver it keeps the pre-existing behaviour: each bound dataset is searched
-// with the whole scope.
-func resolveKGScope(deps SearchDeps, docScope, datasetIDs []string) []kgScope {
- // A non-nil empty scope is resolveDocScope's "match nothing" signal (the session
- // ceiling removed every requested id). A falsy-empty check
- // would fall through to whole-dataset scopes; Go keeps the ceiling absolute
- // and expands nothing.
- if docScope != nil && len(docScope) == 0 {
- return nil
- }
- docScope = scopedDocIDs(deps.DocScope, docScope)
-
- // Each bound dataset is scanned under its OWN owner tenant; the request tenant is
- // only a fallback for a dataset that carries no tenant id.
- tenantByKB := make(map[string]string, len(deps.KBs))
- for _, kb := range deps.KBs {
- if kb != nil && kb.ID != "" && kb.TenantID != "" {
- tenantByKB[kb.ID] = kb.TenantID
- }
- }
- boundTenant := func(kbID string) string {
- if t := tenantByKB[kbID]; t != "" {
- return t
- }
- return deps.TenantID
- }
-
- if len(docScope) == 0 {
- out := make([]kgScope, 0, len(datasetIDs))
- for _, kbID := range datasetIDs {
- out = append(out, kgScope{KBID: kbID, TenantID: boundTenant(kbID), Docs: nil})
- }
- return out
- }
-
- byOwner := map[DocTenant][]string{}
- if deps.DocTenantResolver != nil {
- if owners, err := deps.DocTenantResolver.ResolveDocTenants(context.Background(), docScope); err == nil {
- // Walk docScope (not the owners map): Go randomises map iteration, so
- // grouping from the map made both the doc order inside a scope and the
- // scope order itself vary run to run.
- seen := map[string]bool{}
- for _, docID := range docScope {
- owner, ok := owners[docID]
- if !ok || seen[docID] {
- continue
- }
- seen[docID] = true
- key := DocTenant{KBID: owner.KBID, TenantID: owner.TenantID}
- byOwner[key] = append(byOwner[key], docID)
- }
- } else {
- // The per-doc lookup failed; the bound datasets are used instead, so the lost
- // owner grouping must not be silent.
- _LOG.Printf("[Graph explore] doc-tenant resolution failed for %d doc(s); falling back to the bound datasets: %v", len(docScope), err)
- }
- }
-
- if len(byOwner) == 0 {
- // No resolver, or it failed: keep the old behaviour — every bound
- // dataset searched with the whole DocScope.
- out := make([]kgScope, 0, len(datasetIDs))
- for _, kbID := range datasetIDs {
- out = append(out, kgScope{KBID: kbID, TenantID: boundTenant(kbID), Docs: docScope})
- }
- return out
- }
-
- // Group by real owner, including knowledge bases beyond datasetIDs.
- out := make([]kgScope, 0, len(byOwner))
- for owner, docs := range byOwner {
- out = append(out, kgScope{KBID: owner.KBID, TenantID: owner.TenantID, Docs: docs})
- }
- // Stable scope order too — same reason as above.
- sort.Slice(out, func(i, j int) bool {
- if out[i].KBID != out[j].KBID {
- return out[i].KBID < out[j].KBID
- }
- return out[i].TenantID < out[j].TenantID
- })
- return out
-}
-
-type kgEntity struct {
- Name string `json:"name"`
- Type string `json:"type"`
- Description string `json:"description"`
- Aliases []string `json:"aliases"`
- SourceChunkIDs []string `json:"source_chunk_ids"`
- DocID string `json:"doc_id"`
-}
-
-type kgRelation struct {
- ID string `json:"id"`
- From string `json:"from"`
- To string `json:"to"`
- Type string `json:"type"`
- SourceChunkIDs []string `json:"source_chunk_ids"`
- DocID string `json:"doc_id"`
-}
-
-// ExploreResult is the graph_explore output: exactly one of Answer / Chunks is populated,
-// and DocAggs always reflects the returned set.
-type ExploreResult struct {
- Answer string
- Chunks []map[string]interface{}
- DocAggs []map[string]interface{}
-}
-
-// ExploreGraph implements graph_explore.
-func ExploreGraph(ctx context.Context, deps SearchDeps, tenantID string, datasetIDs []string, query, keywords string, docScope []string) (ExploreResult, error) {
- empty := ExploreResult{}
- text := strings.TrimSpace(query + " " + keywords)
- if text == "" || len(datasetIDs) == 0 {
- return empty, nil
- }
- de := engine.Get()
- if de == nil {
- return empty, fmt.Errorf("graph_explore: engine not configured")
- }
-
- // Build the (kb, tenant, docs) search groups: group documents by their real owner
- // (which may include KBs
- // outside datasetIDs), or fall back to whole-dataset search.
- scopes := resolveKGScope(deps, docScope, datasetIDs)
-
- var entities []kgEntity
- var relations []kgRelation
- seenNames := map[string]bool{}
-
- addEntities := func(new []kgEntity, scopeKey string) []string {
- var added []string
- for _, e := range new {
- key := scopeKey + ":" + strings.ToLower(e.Name)
- if seenNames[key] {
- continue
- }
- seenNames[key] = true
- entities = append(entities, e)
- added = append(added, e.Name)
- }
- return added
- }
-
- // Encode the seed text ONCE (not per dataset): a single embedding request
- // serves every KB. A nil vector means the embedding model is unavailable and
- // the seed search falls back to keyword match.
- seedVec := encodeSeedVector(ctx, deps, tenantID, text)
-
- for _, sc := range scopes {
- // A scope with Docs searches within those documents; one without
- // searches the whole dataset (docs is nil for the whole-dataset case).
- scopeKwd := kgScopeDataset
- if len(sc.Docs) > 0 {
- scopeKwd = kgScopeDoc
- }
- // (1) Seeds: dense KNN (similarity>=_KG_SEED_SIM) over the scoped entity
- // rows, re-ranked by mention_count_int desc; falls back to keyword match
- // when the embedding model is unavailable.
- seedRows := kgSeedSearch(ctx, de, sc.TenantID, sc.KBID, sc.Docs, text, scopeKwd, seedVec, deps.IndexName)
- var seeds []kgEntity
- for _, r := range seedRows {
- if e, ok := kgParseEntity(r); ok {
- seeds = append(seeds, e)
- }
- }
- frontier := addEntities(seeds, sc.KBID)
-
- // (2) Expand kgHops out, collecting relations and neighbour entities.
- for hop := 0; hop < kgHops; hop++ {
- if len(frontier) == 0 {
- break
- }
- terms := endpointTerms(frontier)
- relRows := kgSearch(ctx, de, sc.TenantID, sc.KBID, sc.Docs, "relation", "", kgRelLimit, scopeKwd,
- map[string]interface{}{"from_entity_kwd": terms}, "", 0, deps.IndexName)
- relRows = append(relRows, kgSearch(ctx, de, sc.TenantID, sc.KBID, sc.Docs, "relation", "", kgRelLimit, scopeKwd,
- map[string]interface{}{"to_entity_kwd": terms}, "", 0, deps.IndexName)...)
- // Dedup keys: a relation is kept unless its source row id is already seen, so the
- // same endpoint pair from *different* rows survives. Prefer the row id; only fall
- // back to the from|to|type triple when the engine returned no id (so exact
- // duplicates still collapse).
- seenRel := map[string]bool{}
- var hopRelations []kgRelation
- for _, r := range relRows {
- rel, ok := kgParseRelation(r)
- if !ok {
- continue
- }
- k := rel.ID
- if k == "" {
- k = rel.From + "|" + rel.To + "|" + rel.Type
- }
- if seenRel[k] {
- continue
- }
- seenRel[k] = true
- hopRelations = append(hopRelations, rel)
- }
- relations = append(relations, hopRelations...)
-
- // Neighbour entity names not yet visited.
- seenLower := map[string]bool{}
- for k := range seenNames {
- if strings.HasPrefix(k, sc.KBID+":") {
- seenLower[strings.TrimPrefix(k, sc.KBID+":")] = true
- }
- }
- neighSet := map[string]string{}
- for _, r := range hopRelations {
- for _, n := range []string{r.From, r.To} {
- n = strings.TrimSpace(n)
- if n == "" || seenLower[strings.ToLower(n)] {
- continue
- }
- neighSet[strings.ToLower(n)] = n
- }
- }
- if len(neighSet) == 0 {
- break
- }
- neighFiltered := make([]string, 0, len(neighSet))
- for _, n := range neighSet {
- neighFiltered = append(neighFiltered, n)
- }
- limit := kgNeighbors
- if len(neighFiltered) < limit {
- limit = len(neighFiltered)
- }
- neighRows := kgSearch(ctx, de, sc.TenantID, sc.KBID, sc.Docs, "entity", "", limit, scopeKwd,
- map[string]interface{}{"name_kwd": endpointTerms(neighFiltered)}, "", 0, deps.IndexName)
- var neighbours []kgEntity
- for _, r := range neighRows {
- if e, ok := kgParseEntity(r); ok {
- neighbours = append(neighbours, e)
- }
- }
- frontier = addEntities(neighbours, sc.KBID)
- }
- }
-
- if len(entities) == 0 && len(relations) == 0 {
- return empty, nil
- }
-
- // (3) Does the subgraph answer the question? The request-scoped model (deps.Model) is
- // threaded through so the verdict is drawn by the same tenant model as the rest of the
- // retrieval.
- answer, relevant := askStructureAnswer(ctx, deps.Model, query, entities, relations)
- // (4a) Sufficient — return the answer, no chunks.
- if answer != "" {
- return ExploreResult{Answer: answer, DocAggs: []map[string]interface{}{}}, nil
- }
-
- // (4b) Insufficient — return source passages behind the relevant nodes, narrowed to the
- // sentences that carry the keywords. NarrowByKeywords keeps the whole set when keywords
- // is empty, so the strict narrow is gated on a non-empty keyword string to avoid
- // emptying the evidence. doc_aggs is computed from the resulting set. The evidence
- // groups are iterated in FIRST-SEEN order (the order each doc was first reached), so
- // the executor's chunks[:6] picks the same passages each run.
- evidence := collectEvidenceIDs(entities, relations, relevant)
- var chunks []map[string]interface{}
- for _, ev := range evidence {
- if ev.DocID != "" && len(ev.IDs) > 0 {
- chunks = append(chunks, loadChunksByIDs(ctx, indexNameFor(deps.TenantID, deps.IndexName), ev.IDs)...)
- }
- }
- if strings.TrimSpace(keywords) != "" {
- chunks = NarrowByKeywords(chunks, keywords)
- }
- return ExploreResult{Chunks: chunks, DocAggs: DocAggs(chunks)}, nil
-}
-
-// seedEncoder encodes a seed text into a dense vector. It is a package seam so
-// the seed search can be exercised without the tenant embedding service, which
-// is backed by the database and therefore unavailable in unit tests.
-//
-// Returning nil is always valid: every caller falls back to keyword matching.
-var seedEncoder = defaultSeedEncoder
-
-// encodeSeedVector encodes the seed text once for the whole ExploreGraph call.
-// Returns nil when the tenant embedding model is unavailable (or encoding fails).
-//
-// An embedder passed in via SearchDeps.Embedder (the external embed handle) is used when
-// present; otherwise the call falls back to the package's internal resolver
-// (defaultSeedEncoder), which needs a database and therefore stays self-contained for
-// callers that cannot or do not want to supply one.
-func encodeSeedVector(ctx context.Context, deps SearchDeps, tenantID, text string) []float64 {
- // Without a
- // wired database there is no tenant default embedding model, so skip encoding
- // and let the caller fall back to keyword matching rather than panicking.
- if dao.GetDB() == nil {
- return nil
- }
- if deps.Embedder != nil {
- // The external handle may itself panic on a misconfigured model service
- // (e.g. nil-DB); degrade to keyword matching, never crash the request.
- vecs, err := safeEncodeEmbedder(deps.Embedder, ctx, tenantID, text)
- if err != nil || len(vecs) == 0 || len(vecs[0]) == 0 {
- return nil
- }
- out := make([]float64, len(vecs[0]))
- for i, v := range vecs[0] {
- out[i] = float64(v)
- }
- return out
- }
- return seedEncoder(ctx, tenantID, text)
-}
-
-// safeEncodeEmbedder calls the embedder and recovers from any panic so a
-// misconfigured embedding service degrades to keyword matching instead of
-// crashing the request.
-func safeEncodeEmbedder(emb nlp.NavEmbedder, ctx context.Context, tenantID, text string) ([][]float32, error) {
- defer func() {
- if r := recover(); r != nil {
- _LOG.Printf("[graph_explore] seed encode panicked; falling back to keyword: %v", r)
- }
- }()
- // Seed encoding is a QUERY, not a document: graph_explore and navigation seed through
- // query encoding. Prefer the asymmetric query encoding when the embedder exposes it.
- if q, ok := emb.(nlp.NavQueryEmbedder); ok {
- return q.EncodeQueries(ctx, tenantID, []string{text})
- }
- return emb.Encode(ctx, tenantID, []string{text})
-}
-
-// SetSeedEncoder overrides the seed embedding used by graph exploration. It is
-// a testing seam: the default encoder reaches the tenant embedding service,
-// which needs a database that unit tests do not have. Pass nil to restore it.
-// Returning nil is always valid — callers fall back to keyword matching.
-func SetSeedEncoder(fn func(ctx context.Context, tenantID, text string) []float64) {
- if fn == nil {
- seedEncoder = defaultSeedEncoder
- return
- }
- seedEncoder = fn
-}
-
-func defaultSeedEncoder(ctx context.Context, tenantID, text string) []float64 {
- // With no database (and thus no tenant default embedding model) wired up, skip encoding
- // entirely and fall back to keyword matching rather than panicking on a nil DB inside
- // the model-provider layer.
- if dao.GetDB() == nil {
- return nil
- }
- embedder := service.NewNavEmbedder(service.NewModelFactory(), "")
- // Query-side encoding: the seed is the user's query, not an indexed document.
- vecs, err := embedder.EncodeQueries(ctx, tenantID, []string{text})
- if err != nil || len(vecs) == 0 || len(vecs[0]) == 0 {
- return nil
- }
- vec := make([]float64, len(vecs[0]))
- for i, v := range vecs[0] {
- vec[i] = float64(v)
- }
- return vec
-}
-
-// kgSeedSearch searches the compiled KG entity rows for seeds: dense KNN over name_kwd with
-// similarity>=0.8, re-ranked by mention_count_int desc, top kgSeeds. Falls back to keyword
-// match when seedVec is nil (embedding model unavailable). indexName is the caller's
-// configured index override (deps.IndexName); empty keeps ragflow_<tenantID>.
-func kgSeedSearch(ctx context.Context, de engine.DocEngine, tenantID, kbID string, docIDs []string, text, scopeKwd string, seedVec []float64, indexName string) []map[string]interface{} {
- if seedVec != nil {
- dense := &types.MatchDenseExpr{
- VectorColumnName: fmt.Sprintf("q_%d_vec", len(seedVec)),
- EmbeddingData: seedVec,
- EmbeddingDataType: "float",
- DistanceType: "cosine",
- TopN: kgSeedPool,
- ExtraOptions: map[string]interface{}{"similarity": kgSeedSim},
- }
- rows := kgSearchRaw(ctx, de, tenantID, kbID, docIDs, "entity", scopeKwd, nil, []interface{}{dense}, "mention_count_int", kgSeedPool, indexName)
- return topMentionCount(rows, kgSeeds)
- }
- // Text fallback (no embedding model available).
- return kgSearch(ctx, de, tenantID, kbID, docIDs, "entity", text, kgSeeds, scopeKwd, nil, "mention_count_int", kgSeedPool, indexName)
-}
-
-// topMentionCount re-ranks rows by mention_count_int desc and returns topN.
-func topMentionCount(rows []map[string]interface{}, topN int) []map[string]interface{} {
- sort.SliceStable(rows, func(i, j int) bool {
- return mentionCount(rows[i]) > mentionCount(rows[j])
- })
- if len(rows) > topN {
- rows = rows[:topN]
- }
- return rows
-}
-
-func mentionCount(row map[string]interface{}) int {
- switch v := row["mention_count_int"].(type) {
- case int:
- return v
- case int64:
- return int(v)
- case float64:
- return int(v)
- case float32:
- return int(v)
- }
- return 0
-}
-
-// kgSearchRaw is the low-level KG row search with explicit match exprs and any
-// extra filter keys (e.g. from_entity_kwd/to_entity_kwd/name_kwd).
-//
-// indexName is the caller's configured index override (deps.IndexName): the KG
-// rows are written to the same index the chunk loads read (ExploreGraph loads
-// its evidence with indexNameFor(deps.TenantID, deps.IndexName)), so a tenant
-// that overrides the name must have it honoured here too — otherwise the walk
-// queries one index and loads passages from another. Empty falls back to
-// ragflow_<tenantID>.
-func kgSearchRaw(ctx context.Context, de engine.DocEngine, tenantID, kbID string, docIDs []string, kind, scopeKwd string, extra map[string]interface{}, matchExprs []interface{}, orderDesc string, limit int, indexName string) []map[string]interface{} {
- idx := indexNameFor(tenantID, indexName)
- condition := map[string]interface{}{"knowledge_graph_kwd": kind}
- if scopeKwd != "" {
- condition["scope_kwd"] = scopeKwd
- }
- if len(docIDs) > 0 {
- condition["doc_id"] = docIDs
- }
- for k, v := range extra {
- condition[k] = v
- }
- fields := []string{"id", "content_with_weight", "source_chunk_ids", "doc_id", "docnm_kwd", "name_kwd", "mention_count_int", "from_entity_kwd", "to_entity_kwd"}
- req := &types.SearchRequest{
- IndexNames: []string{idx},
- KbIDs: []string{kbID},
- SelectFields: fields,
- Filter: condition,
- Limit: limit,
- MatchExprs: matchExprs,
- }
- if orderDesc != "" {
- req.OrderBy = &types.OrderByExpr{}
- req.OrderBy.Desc(orderDesc)
- }
- res, err := de.Search(ctx, req)
- if err != nil {
- return nil
- }
- return res.Chunks
-}
-
-// kgSearch searches the compiled KG rows of one KB,
-// using keyword match. It only builds the MatchTextExpr (with the pool-based
-// TopN) and delegates the request construction to kgSearchRaw.
-func kgSearch(ctx context.Context, de engine.DocEngine, tenantID, kbID string, docIDs []string, kind, text string, topN int, scopeKwd string, extra map[string]interface{}, orderDesc string, pool int, indexName string) []map[string]interface{} {
- var matchExprs []interface{}
- if text != "" {
- knnTopN := topN
- if pool > knnTopN {
- knnTopN = pool
- }
- matchExprs = []interface{}{&types.MatchTextExpr{
- Fields: []string{"content_ltks", "content_sm_ltks"},
- MatchingText: text,
- TopN: knnTopN,
- }}
- }
- return kgSearchRaw(ctx, de, tenantID, kbID, docIDs, kind, scopeKwd, extra, matchExprs, orderDesc, topN, indexName)
-}
-
-func kgParseEntity(row map[string]interface{}) (kgEntity, bool) {
- name := ""
- payload := map[string]interface{}{}
- if s, ok := row["content_with_weight"].(string); ok {
- _ = json.Unmarshal([]byte(s), &payload)
- }
- if v, ok := payload["name"].(string); ok && v != "" {
- name = v
- } else if v, ok := payload["term"].(string); ok && v != "" {
- name = v
- } else if v, ok := payload["title"].(string); ok && v != "" {
- name = v
- }
- name = strings.TrimSpace(name)
- if name == "" {
- return kgEntity{}, false
- }
- e := kgEntity{
- Name: name,
- Type: strOr(payload["type"], "other"),
- Description: strOr(payload["description"], ""),
- SourceChunkIDs: strSliceField(row["source_chunk_ids"]),
- DocID: strOr(row["doc_id"], ""),
- }
- if aliases, ok := payload["aliases"].([]interface{}); ok {
- for _, a := range aliases {
- if s, ok := a.(string); ok && strings.TrimSpace(s) != "" {
- e.Aliases = append(e.Aliases, strings.TrimSpace(s))
- }
- }
- }
- return e, true
-}
-
-func kgParseRelation(row map[string]interface{}) (kgRelation, bool) {
- src := strings.TrimSpace(strOr(row["from_entity_kwd"], ""))
- tgt := strings.TrimSpace(strOr(row["to_entity_kwd"], ""))
- if src == "" || tgt == "" {
- return kgRelation{}, false
- }
- typ := "related"
- if payload, ok := row["content_with_weight"].(string); ok {
- var p map[string]interface{}
- if json.Unmarshal([]byte(payload), &p) == nil {
- if t, ok := p["type"].(string); ok && t != "" {
- typ = t
- } else if t, ok := p["relation"].(string); ok && t != "" {
- typ = t
- }
- }
- }
- return kgRelation{
- ID: strOr(row["id"], ""),
- From: src, To: tgt, Type: typ,
- SourceChunkIDs: strSliceField(row["source_chunk_ids"]),
- DocID: strOr(row["doc_id"], ""),
- }, true
-}
-
-// endpointTerms: original + lowercased forms, so
-// hop queries match both merged (lowercased) and per-doc (original-case)
-// endpoint fields.
-func endpointTerms(names []string) []string {
- set := map[string]bool{}
- for _, n := range names {
- n = strings.TrimSpace(n)
- if n == "" {
- continue
- }
- set[n] = true
- set[strings.ToLower(n)] = true
- }
- out := make([]string, 0, len(set))
- for n := range set {
- out = append(out, n)
- }
- sort.Strings(out)
- return out
-}
-
-// kgEvidence is one document's relevant source chunk ids, in FIRST-SEEN order.
-// Insertion-ordered, hence deterministic: a plain map would randomise the order, and the
-// executor caps the loaded passages at chunks[:6], so the SAME query could surface
-// different passages on every run.
-type kgEvidence struct {
- DocID string
- IDs []string
-}
-
-// collectEvidenceIDs groups the source chunk ids of relevant entities AND relations by
-// doc, preserving the doc order in which each doc is first reached.
-func collectEvidenceIDs(entities []kgEntity, relations []kgRelation, relevantNames []string) []kgEvidence {
- wanted := map[string]bool{}
- for _, n := range relevantNames {
- if s := strings.TrimSpace(n); s != "" {
- wanted[strings.ToLower(s)] = true
- }
- }
- byDoc := make([]kgEvidence, 0, 4)
- index := map[string]int{}
- seen := map[string]bool{}
- add := func(docID string, ids []string) {
- for _, cid := range ids {
- if cid == "" {
- continue
- }
- key := docID + "|" + cid
- if seen[key] {
- continue
- }
- seen[key] = true
- i, ok := index[docID]
- if !ok {
- i = len(byDoc)
- index[docID] = i
- byDoc = append(byDoc, kgEvidence{DocID: docID})
- }
- byDoc[i].IDs = append(byDoc[i].IDs, cid)
- }
- }
- for _, e := range entities {
- names := map[string]bool{strings.ToLower(e.Name): true}
- for _, a := range e.Aliases {
- names[strings.ToLower(a)] = true
- }
- if intersects(names, wanted) {
- add(e.DocID, e.SourceChunkIDs)
- }
- }
- for _, r := range relations {
- if wanted[strings.ToLower(r.From)] || wanted[strings.ToLower(r.To)] {
- add(r.DocID, r.SourceChunkIDs)
- }
- }
- return byDoc
-}
-
-func intersects(a, b map[string]bool) bool {
- for k := range a {
- if b[k] {
- return true
- }
- }
- return false
-}
-
-// askStructureAnswer asks the chat model whether the subgraph answers the query,
-// returning (answer, relevant_names). Delegates to AskStructure (structure_qa.go).
-// `model` is the request-scoped chat model; a nil model simply skips the verdict and
-// yields no answer.
-func askStructureAnswer(ctx context.Context, model SessionModel, query string, entities []kgEntity, relations []kgRelation) (string, []string) {
- ems := make([]map[string]any, 0, len(entities))
- for _, e := range entities {
- ems = append(ems, map[string]any{
- "name": e.Name, "type": e.Type, "description": e.Description,
- })
- }
- rms := make([]map[string]any, 0, len(relations))
- for _, r := range relations {
- rms = append(rms, map[string]any{"from": r.From, "to": r.To, "type": r.Type})
- }
- return AskStructure(ctx, model, query, "knowledge graph", "Graph exploration", ems, rms)
-}
-
-func strOr(v interface{}, def string) string {
- if s, ok := v.(string); ok && s != "" {
- return s
- }
- return def
-}
-
-func strSliceField(v interface{}) []string {
- switch x := v.(type) {
- case []string:
- return x
- case []interface{}:
- var out []string
- for _, item := range x {
- if s, ok := item.(string); ok {
- out = append(out, s)
- }
- }
- return out
- }
- return nil
-}
diff --git a/internal/rag/agentic-rag/runtime/tool_exploration_test.go b/internal/rag/agentic-rag/runtime/tool_exploration_test.go
deleted file mode 100644
index e42538e0d3..0000000000
--- a/internal/rag/agentic-rag/runtime/tool_exploration_test.go
+++ /dev/null
@@ -1,84 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "testing"
-
- "ragflow/internal/engine"
- "ragflow/internal/engine/types"
-)
-
-// kgIndexCapturingEngine records the index names of every search it serves.
-type kgIndexCapturingEngine struct {
- engine.DocEngine
- indexNames []string
-}
-
-func (e *kgIndexCapturingEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
- e.indexNames = append(e.indexNames, req.IndexNames...)
- return &types.SearchResult{}, nil
-}
-
-// TestKGSearchUsesConfiguredIndexName pins that the knowledge-graph walk searches
-// the SAME index the rest of the request reads: ExploreGraph loads its evidence
-// with indexNameFor(deps.TenantID, deps.IndexName), so a tenant that overrides
-// the index name must have it honoured by the KG rows too. Hardcoding
-// ragflow_<tenantID> made the walk query one index and return passages loaded
-// from another. Empty keeps the default index name.
-func TestKGSearchUsesConfiguredIndexName(t *testing.T) {
- ctx := context.Background()
- for _, tc := range []struct {
- name string
- indexName string
- want string
- }{
- {"configured override", "custom_idx", "custom_idx"},
- {"default", "", "ragflow_tenantA"},
- } {
- t.Run(tc.name, func(t *testing.T) {
- for _, call := range []struct {
- name string
- fn func(de engine.DocEngine)
- }{
- {"kgSearchRaw", func(de engine.DocEngine) {
- kgSearchRaw(ctx, de, "tenantA", "kb1", nil, "entity", "dataset", nil, nil, "", 5, tc.indexName)
- }},
- {"kgSearch", func(de engine.DocEngine) {
- kgSearch(ctx, de, "tenantA", "kb1", nil, "entity", "q", 5, "dataset", nil, "", 0, tc.indexName)
- }},
- {"kgSeedSearch", func(de engine.DocEngine) {
- kgSeedSearch(ctx, de, "tenantA", "kb1", nil, "q", "dataset", nil, tc.indexName)
- }},
- } {
- t.Run(call.name, func(t *testing.T) {
- de := &kgIndexCapturingEngine{}
- call.fn(de)
- if len(de.indexNames) == 0 {
- t.Fatal("no search was issued")
- }
- for _, got := range de.indexNames {
- if got != tc.want {
- t.Errorf("index = %q, want %q", got, tc.want)
- }
- }
- })
- }
- })
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/tool_navigation.go b/internal/rag/agentic-rag/runtime/tool_navigation.go
deleted file mode 100644
index 3f1f933111..0000000000
--- a/internal/rag/agentic-rag/runtime/tool_navigation.go
+++ /dev/null
@@ -1,1828 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "fmt"
- "sort"
- "strconv"
- "strings"
-
- "ragflow/internal/engine"
- "ragflow/internal/engine/types"
-)
-
-// Compiled-structure navigation: ontology_navigate / mindmap_navigate /
-// navigate_tree / wiki_query.
-//
-// Structure navigation, the nav-tree router, the compiled-structure reader and
-// dataset-tree LLM selection. The knowledge-graph walk (graph_explore) lives in
-// exploration.go; the shared seams (verdict struct, prompt, load helpers) stay here and
-// are reached by those files through the runtime package. These were previously spread
-// across several files (navigation / navtools / navservice / datasetnav / kg_explore);
-// they are consolidated here.
-
-// Structure navigation (ontology_navigate / mindmap_navigate)
-
-// catalogKinds: {"tree", "timeline", "page_index", "pageindex"}. raptor is NOT a catalog
-// kind — upper RAPTOR clusters carry no per-node vectors and drilling into them is the
-// claim leg's job.
-var catalogKinds = map[string]bool{"tree": true, "timeline": true, "page_index": true, "pageindex": true}
-var mindmapKinds = map[string]bool{"mindmap": true, "mind_map": true}
-
-const navSystemPrompt = `You are given the {noun} of one or more documents — an outline of entities and their relations — and a question.
-
-Decide whether that outline alone already answers the question.
-
-Rules:
-1. Answer ONLY from the outline below. Do not invent facts.
-2. Set "is_sufficient" to true only when the outline genuinely answers the question; otherwise false with an empty answer.
-3. Always fill "relevant_entities" with the exact ` + "`name`" + ` values of the entities most related to the question (up to 10), even when the outline is not sufficient — they are used to pull the underlying source text.
-
-Output ONLY JSON, no prose, no code fences:
-{"is_sufficient": true/false, "answer": "<answer, or empty>", "relevant_entities": ["<entity name>", ...]}`
-
-const (
- maxStructureEntities = 300
- maxStructureRelations = 300
- maxEvidenceChunks = 24
-)
-
-type structureEntity struct {
- Name string `json:"name"`
- Type string `json:"type"`
- Description string `json:"description"`
- SourceChunkIDs []string `json:"source_chunk_ids"`
- DocID string `json:"-"`
- Vec []float64 `json:"-"` // q_<dim>_vec row embedding, when the row carries one
-}
-
-type structureNavVerdict struct {
- IsSufficient bool `json:"is_sufficient"`
- Answer string `json:"answer"`
- RelevantEntities []string `json:"relevant_entities"`
-}
-
-// shapeKwds are the knowledge_graph_kwd ROW SHAPES a compiled structure can be written as.
-// Reading BOTH the compact "graph" blob and the per-entity / per-relation rows is
-// required: otherwise navigation silently returns EMPTY for datasets compiled into
-// per-entity/relation rows rather than a graph blob.
-var shapeKwds = []string{"graph", "entity", "relation"}
-
-// loadStructureGraph reads a document's compiled structure rows and splits them into both
-// the entity list and the parent->child relations: the graph-blob and the
-// per-entity/per-relation row shapes are merged, and both entities and relations feed the
-// TOC builders. Navigation must carry relations through or the tree hierarchy (children,
-// parents, roots, ancestors) collapses.
-func loadStructureGraph(ctx context.Context, indexName, docID string, kinds map[string]bool, vecField string) ([]structureEntity, []structureRel) {
- de := engine.Get()
- if de == nil {
- return nil, nil
- }
- idx := indexName
- selectFields := []string{"content_with_weight", "compile_kwd", "compilation_template_kind_kwd", "type_kwd", "knowledge_graph_kwd"}
- if vecField != "" {
- selectFields = append(selectFields, vecField)
- }
- req := &types.SearchRequest{
- IndexNames: []string{idx},
- Filter: map[string]interface{}{"doc_id": []string{docID}, "knowledge_graph_kwd": shapeKwds},
- SelectFields: selectFields,
- Limit: 3000,
- }
- res, err := de.Search(ctx, req)
- if err != nil {
- return nil, nil
- }
- rows := make([]StructureRow, 0, len(res.Chunks))
- for _, row := range res.Chunks {
- kg, _ := row["type_kwd"].(string)
- if kg == "" {
- kg, _ = row["knowledge_graph_kwd"].(string)
- }
- sr := StructureRow{
- CompileKwd: fmt.Sprint(row["compile_kwd"]),
- TemplateKind: types.CompilationKind(row),
- KnowledgeGraphKwd: kg,
- Content: fmt.Sprint(row["content_with_weight"]),
- }
- if vecField != "" {
- if vec, ok := parseFloats(row[vecField]); ok {
- sr.Vec = vec
- }
- }
- rows = append(rows, sr)
- }
- kindList := make([]string, 0, len(kinds))
- for k := range kinds {
- kindList = append(kindList, k)
- }
- rawEntities, rawRels := ParseCompiledStructure(rows, kindList)
- return structureGraphFromRaw(rawEntities, rawRels)
-}
-
-// structureGraphFromRaw maps what ParseCompiledStructure returns onto the typed
-// entities/relations the drill-down consumes.
-//
-// A field the compiled payload omits must read as ABSENT, not as its Go
-// rendering: fmt.Sprint(nil) is the non-empty string "<nil>", which survives
-// navTypeOr's empty check and reaches the model as "- Name (<nil>): <nil>" with
-// a phantom edge to "<nil>". The intended rendering is "- Name (other)" and no
-// description, because a missing key is falsy — and a relation with a missing endpoint is
-// DROPPED instead of kept. Leaving the field empty is also what lets the renderers apply
-// their defaults: "other" for a type, "related_to" for a relation type.
-func structureGraphFromRaw(rawEntities, rawRels []map[string]any) ([]structureEntity, []structureRel) {
- var out []structureEntity
- for _, e := range rawEntities {
- name, _ := e["name"].(string)
- if strings.TrimSpace(name) == "" {
- continue
- }
- typ, _ := e["type"].(string)
- desc, _ := e["description"].(string)
- se := structureEntity{
- Name: name,
- Type: typ,
- Description: desc,
- }
- if v, ok := e["_vec"].([]float64); ok {
- se.Vec = v
- }
- if ids, ok := e["source_chunk_ids"].([]interface{}); ok {
- for _, id := range ids {
- if s, ok := id.(string); ok && s != "" {
- se.SourceChunkIDs = append(se.SourceChunkIDs, s)
- }
- }
- }
- out = append(out, se)
- }
- var rels []structureRel
- for _, r := range rawRels {
- pRaw, _ := r["from"].(string)
- cRaw, _ := r["to"].(string)
- p := strings.TrimSpace(pRaw)
- c := strings.TrimSpace(cRaw)
- // Empty or self-loop relations are skipped.
- if p == "" || c == "" || p == c {
- continue
- }
- relType, _ := r["type"].(string)
- rels = append(rels, structureRel{
- from: p,
- to: c,
- relType: relType,
- })
- }
- return out, rels
-}
-
-// parseFloats normalizes an engine vector cell — a []float64, or a []any of
-// numbers (engine decoders may return either) — into []float64.
-func parseFloats(v any) ([]float64, bool) {
- switch t := v.(type) {
- case []float64:
- return t, true
- case []any:
- out := make([]float64, 0, len(t))
- for _, item := range t {
- f, ok := item.(float64)
- if !ok {
- return nil, false
- }
- out = append(out, f)
- }
- return out, true
- }
- return nil, false
-}
-
-func normalizeKind(row map[string]interface{}) string {
- kind := types.CompilationKind(row)
- if kind == "page_index" || kind == "graph" {
- return "timeline"
- }
- if kind == "mind_map" {
- return "mindmap"
- }
- return kind
-}
-
-func loadChunksByIDs(ctx context.Context, indexName string, ids []string) []map[string]interface{} {
- if len(ids) == 0 {
- return nil
- }
- de := engine.Get()
- if de == nil {
- return nil
- }
- idx := indexName
- limit := maxEvidenceChunks
- if len(ids) < limit {
- limit = len(ids)
- }
- req := &types.SearchRequest{
- IndexNames: []string{idx},
- Filter: map[string]interface{}{"id": ids},
- SelectFields: []string{"content_with_weight", "docnm_kwd", "doc_id"},
- Limit: limit,
- }
- res, err := de.Search(ctx, req)
- if err != nil {
- return nil
- }
- var out []map[string]interface{}
- for _, row := range res.Chunks {
- out = append(out, map[string]interface{}{
- "chunk_id": row["id"], "content_with_weight": row["content_with_weight"],
- "docnm_kwd": row["docnm_kwd"], "doc_id": row["doc_id"],
- })
- }
- return out
-}
-
-func dedupStrings(in []string) []string {
- seen := map[string]bool{}
- var out []string
- for _, s := range in {
- if s != "" && !seen[s] {
- seen[s] = true
- out = append(out, s)
- }
- }
- return out
-}
-
-func orStr(v, def string) string {
- if v == "" {
- return def
- }
- return v
-}
-
-// indexNameFor resolves the document/structure search index: the dataset's configured
-// index name, or the RAGFlow default "ragflow_<tenant_id>" — behaviour is identical
-// unless a tenant overrides index_name.
-func indexNameFor(tenantID, configured string) string {
- if configured != "" {
- return configured
- }
- return "ragflow_" + tenantID
-}
-
-// The routing seam + NavResult payload
-
-// Compiled-navigation tools: locate (dataset tree routing) and drill
-// (in-document structure pinpointing).
-//
-// Covers the tree walk, the titled search, the NavResult payload, the compiled-structure
-// reader and the kind normalization.
-//
-// These are the payload builders; the tool dispatch lives in tool_executor.go.
-
-// Navigation constants.
-const (
- // navSearchMaxDocs is how many documents the hybrid nav search routes to.
- navSearchMaxDocs = 12
- // navMinDocScore drops documents below this score.
- navMinDocScore = 0.2
- // navTreeMaxDocs caps the documents surfaced by navigate_tree.
- navTreeMaxDocs = 8
-)
-
-// NavResult: the structured outcome of ONE
-// compiled-navigation call.
-//
-// Text is what the MODEL sees (XML, unchanged). The remaining fields are the
-// signals the ORCHESTRATOR routes on: does this dataset have the structure at
-// all, did THIS query reach anything, and was the result worth using? Without
-// them the caller can only regex the XML — which is how a `count="1"
-// entities="0"` empty shell used to read as a successful hit.
-type NavResult struct {
- // Text is the XML the model consumes.
- Text string
- // DocIDs are the documents reached (the routing result).
- DocIDs []string
- // RoutedDocs are (doc_id, summary) pairs — the routed documents WITH their
- // overall summaries. The summary is the "hint" fed back into retrieval: nav
- // is a hint, not a constraint, so these become a soft boost/rerank signal
- // instead of a hard doc_scope filter.
- RoutedDocs [][2]string
- // Entities are compiled entities discovered across those documents.
- Entities []map[string]any
- // Relations are compiled relations discovered across those documents.
- Relations []map[string]any
- // ChunkPaths maps chunk_id -> root→chunk structure path.
- ChunkPaths map[string]string
- // EmptyReason is "" when the call succeeded, else the machine-readable cause:
- // "infra" (no backend), "bad_args" (missing query), "no_structure"
- // (dataset-level: no compiled structure of this kind here), "no_doc"
- // (query-level: structure exists but THIS query reached nothing).
- EmptyReason string
- // Diagnostic is the underlying failure message behind EmptyReason (Go-only,
- // mirroring ToolOutcome.Diagnostic). "infra" alone does not say what broke,
- // so the router's own error is carried here instead of only in a log line.
- // Never part of Text: that keeps Python's error="..." attribute intact.
- Diagnostic string
-}
-
-// HasStructure reports whether the dataset has the compiled structure at all.
-// A query-level miss is not a structure absence.
-func (n NavResult) HasStructure() bool { return n.EmptyReason != ReasonNoStructure }
-
-// navEmpty returns an empty NavResult carrying the <tree_navigation> text:
-// count="0" plus an error="<label>" attribute, omitted when label is empty —
-// exactly the shapes navigation.py builds (:877 "no retriever", :880 "query is
-// required", :899 no attribute at all for a query-level miss). That text is not
-// what the model reads for an empty result — both sides substitute their own
-// note for every empty_reason (, tool_executor.go:402)
-// but the NavResult must still carry it. The status is derived from the reason
-// by ReasonStatus.
-func navEmpty(reason, label string) NavResult {
- attr := ""
- if label != "" {
- attr = ` error="` + XMLEscape(label) + `"`
- }
- return NavResult{
- EmptyReason: reason,
- Text: "<tree_navigation count=\"0\"" + attr + ">\n</tree_navigation>",
- }
-}
-
-// The routing seam
-
-// NavTreeRouter descends a dataset's compiled navigation tree and returns the
-// routed documents.
-//
-// A hybrid BFS beam descent (vector + BM25) from the root clusters down to the nav_doc
-// leaves.
-//
-// The Go side exposes this through internal/service/nav's NavService.Search,
-// which is the same KNN-over-nav-rows capability. It is an interface here (not a
-// direct call) for two reasons: the runtime must not import the service layer,
-// and a dataset without a compiled tree must be distinguishable from a query
-// that routed to nothing.
-type NavTreeRouter interface {
- // Route descends the nav tree for one dataset and returns the routed
- // documents with their summaries, ordered by descending score.
- // Returns (nil, nil) when the dataset has no compiled tree at all.
- Route(ctx context.Context, tenantID, kbID, query string, docScope []string, topK int) ([][2]string, error)
-}
-
-// NavTreeInput is one navigate_tree call.
-type NavTreeInput struct {
- Query string
- Keywords string
- DocScope []string
- // TenantID and KbIDs bound the datasets to route over.
- TenantID string
- KbIDs []string
-}
-
-// NavigateTree: locate the document(s) most
-// likely to hold the answer by descending the compiled navigation tree.
-//
-// This tool ROUTES, it does not retrieve. It deliberately does NOT fetch
-// document content: loading full documents here cost ~79 ES queries per document
-// only to render a snippet that navigate_structure supersedes — under per-pass
-// fan-out that alone was enough to knock ES over.
-//
-// Each nav row already carries the document's overall summary, so labelling the
-// route costs ZERO extra queries.
-//
-// There is deliberately NO chunk-retrieval fallback: when routing misses, the
-// caller falls back to retrieve/search_chunks — the same work, owned by the
-// orchestrator instead of hidden inside a "route" call.
-func NavigateTree(ctx context.Context, router NavTreeRouter, in NavTreeInput) NavResult {
- query := strings.TrimSpace(in.Query)
- if query == "" {
- return navEmpty(ReasonBadArgs, "query is required")
- }
- if router == nil {
- // A missing retriever is the infra case: the router IS the retrieval backend, and
- // the error label is verbatim.
- //
- // The label is also carried as the Diagnostic: "infra" alone tells a client
- // nothing about what actually broke.
- res := navEmpty(ReasonInfra, "no retriever")
- res.Diagnostic = "no retriever"
- return res
- }
- // Descend on the topic + keywords: routing descends on similarity and does
- // not require keyword hits, so keywords only enrich the query.
- full := strings.TrimSpace(strings.TrimSpace(in.Query) + " " + strings.TrimSpace(in.Keywords))
-
- var (
- // ordered keeps the router's ranking: it returns documents in DESCENDING
- // score order, and that ordering IS the routing result. Re-sorting by
- // doc_id would discard the relevance the descent computed.
- ordered [][2]string
- seen = map[string]bool{}
- anyTree bool
- anyError error
- )
- for _, kbID := range in.KbIDs {
- routed, err := router.Route(ctx, in.TenantID, kbID, full, in.DocScope, navSearchMaxDocs)
- if err != nil {
- anyError = err
- _LOG.Printf("[Dataset navigation search] nav-tree descent failed for kb=%s: %v", kbID, err)
- continue
- }
- if routed == nil {
- // No compiled tree for this dataset (distinguished from "routed to
- // nothing" by the router returning an empty non-nil slice).
- continue
- }
- anyTree = true
- for _, pair := range routed {
- did := strings.TrimSpace(pair[0])
- if did == "" || seen[did] {
- continue
- }
- seen[did] = true
- ordered = append(ordered, [2]string{did, strings.TrimSpace(pair[1])})
- }
- }
- if !anyTree {
- if anyError != nil {
- // A backend failure is an INFRASTRUCTURE error, not a dataset verdict:
- // no read succeeded, so "this dataset has no tree" would be a verdict
- // drawn from zero evidence — and Route() documents that contract
- // itself ("(nil, nil) when the dataset has no compiled tree at all",
- // so a non-nil error is a failure, not an absence). A failed descent never
- // produces no_structure either: the nav-tree route yields only infra / bad_args /
- // no_doc, while no_structure belongs to the STRUCTURE path, where a successful
- // read found zero entities.
- //
- // The error travels as the Diagnostic: "infra" alone does not say what broke.
- res := navEmpty(ReasonInfra, "nav tree descent failed")
- res.Diagnostic = anyError.Error()
- return res
- }
- // No compiled tree is a DATASET-level fact: there is deliberately NO fallback here
- // (re-inventing dataset_nav's work through the generic retriever was the most
- // expensive thing that path could do). When routing misses, the CALLER falls back to
- // retrieve/search_chunks — the same work, done once and owned by the
- // orchestrator instead of hidden inside a "route" call. The tool's empty
- // verdict lets the session disable the tool.
- // Every dataset lacks a compiled tree — a DATASET-level fact, so the
- // caller may disable the tool for the session.
- return navEmpty(ReasonNoStructure, "no compiled navigation tree")
- }
- if len(ordered) == 0 {
- // Structure exists but THIS query reached nothing: a query-level miss.
- // The dataset may still have a tree a better-formed query would hit.
- // This miss is deliberately NOT backed by a retrieval fallback: it returns
- // count="0" / no_doc and leaves the fallback to the caller's retrieve/search_chunks.
- // No error attribute on the empty verdict: a query-level miss carries none, unlike
- // infra/bad_args.
- return navEmpty(ReasonNoDoc, "")
- }
-
- // Cap after dedup, preserving the score order.
- if len(ordered) > navTreeMaxDocs {
- ordered = ordered[:navTreeMaxDocs]
- }
- docs := make([]string, 0, len(ordered))
- for _, pair := range ordered {
- docs = append(docs, pair[0])
- }
-
- var parts []string
- parts = append(parts, fmt.Sprintf(`<tree_navigation count="%d" query="%s">`, len(ordered), XMLEscape(query)))
- for i, pair := range ordered {
- if pair[1] != "" {
- parts = append(parts,
- fmt.Sprintf(` <doc rank="%d" doc_id="%s">`, i+1, XMLEscape(pair[0])),
- fmt.Sprintf(" <summary>%s</summary>", XMLEscape(pair[1])),
- " </doc>")
- } else {
- parts = append(parts, fmt.Sprintf(` <doc rank="%d" doc_id="%s"/>`, i+1, XMLEscape(pair[0])))
- }
- }
- parts = append(parts, "</tree_navigation>")
-
- return NavResult{
- Text: strings.Join(parts, "\n"),
- DocIDs: docs,
- RoutedDocs: ordered,
- }
-}
-
-// Compiled-structure reading (in-document)
-
-// StructureRow is one compiled-structure row: the doc-store fields the structure reader
-// reads.
-type StructureRow struct {
- // CompileKwd distinguishes the COMPILE TYPE (tree / page_index / timeline /
- // ...). NOT knowledge_graph_kwd.
- CompileKwd string
- // TemplateKind is the template-authored kind (compilation_template_kind_kwd).
- TemplateKind string
- // KnowledgeGraphKwd selects the ROW SHAPE:
- // "entity" → one row per node, content is a single entity dict
- // "relation" → one row per edge, content is a single relation dict
- // (written by page_index, tree, and the pipeline Compiler)
- // "graph" → legacy: one compact blob {"entities":[], "relations":[]};
- // pre-migration tree rows only, nothing writes it anymore
- KnowledgeGraphKwd string
- // Content is the row's JSON payload.
- Content string
- // Vec is the row's q_<dim>_vec embedding, when the reader selected it. A
- // graph blob row carries the one vector its nested nodes share (legacy); a
- // per-entity row carries that node's own vector.
- Vec []float64
-}
-
-// StructureReader reads a document's compiled structure rows.
-//
-// It issues one doc-store query over the per-entity/relation rows and merges the matching
-// buckets (the graph blob and the raptor_graph projection are gone from the storage
-// model).
-//
-// Go has no doc-store structured-query interface (see runtime.RetrievalService,
-// which only exposes Search), so this is a seam: the runtime ships the full
-// parsing/merging logic below, and the reader is injected. Until a Go-side
-// implementation is wired, navigate_structure reports EMPTY/no_structure and the
-// navigation ladder falls through to `global` — the same behaviour as a dataset
-// with no compiled structures.
-type StructureReader interface {
- // ReadStructure returns the compiled rows for a document. Returns nil when
- // the backend has no compiled-structure support at all.
- ReadStructure(ctx context.Context, tenantID, kbID, docID string) ([]StructureRow, error)
-}
-
-// normalizeKind is defined above; rowKind projects a StructureRow into that shape, applying
-// the API's kind normalization (page_index / knowledge_graph → timeline).
-func rowKind(row StructureRow) string {
- return normalizeKind(map[string]any{
- "compile_kwd": row.CompileKwd,
- "compilation_template_kind_kwd": row.TemplateKind,
- })
-}
-
-// ParseCompiledStructure: merge step
-// (navigation.py:_query): keep rows whose compile TYPE is in kinds, then split
-// by row shape into entities / relations.
-//
-// Two row shapes coexist for compiled structures, and a given compile type may
-// produce either:
-//
-// graph blob (knowledge_graph_kwd="graph"): nested entities/relations
-// per-entity (knowledge_graph_kwd="entity"): one node per row
-// per-relation (knowledge_graph_kwd="relation"): one edge per row
-//
-// Reading BOTH and merging is what makes navigation work regardless of which
-// shape a compile type produced.
-func ParseCompiledStructure(rows []StructureRow, kinds []string) ([]map[string]any, []map[string]any) {
- want := map[string]bool{}
- for _, k := range kinds {
- if k != "" {
- want[k] = true
- }
- }
- var entities, relations []map[string]any
- for _, row := range rows {
- kind := rowKind(row)
- if len(want) > 0 && !want[kind] {
- continue
- }
- graph, ok := decodeJSONObject(row.Content)
- if !ok {
- continue
- }
- attachVec := func(list []map[string]any) {
- if len(row.Vec) == 0 {
- return
- }
- for _, m := range list {
- m["_vec"] = row.Vec
- }
- }
- switch row.KnowledgeGraphKwd {
- case "graph":
- es := objectList(graph["entities"])
- rs := objectList(graph["relations"])
- attachVec(es)
- attachVec(rs)
- entities = append(entities, es...)
- relations = append(relations, rs...)
- case "entity":
- attachVec([]map[string]any{graph})
- entities = append(entities, graph)
- case "relation":
- attachVec([]map[string]any{graph})
- relations = append(relations, graph)
- }
- }
- return entities, relations
-}
-
-func decodeJSONObject(s string) (map[string]any, bool) {
- v := ExtractJSON(s)
- m, ok := v.(map[string]any)
- return m, ok
-}
-
-func objectList(v any) []map[string]any {
- list, _ := v.([]any)
- out := make([]map[string]any, 0, len(list))
- for _, item := range list {
- if m, ok := item.(map[string]any); ok {
- out = append(out, m)
- }
- }
- return out
-}
-
-// In-document structure drill-down (mirrors navigation.py, distributed helpers)
-//
-// These functions reuse primitives imported from elsewhere: cosine / appendUnique
-// (compiled_expansion.go, action_session.go), XMLEscape / Snippet / ChunkTextOf
-// (chunk_utils.go). They stay here so navigation.go owns the navigate-tree /
-// navigate-structure algorithm while importing its primitives.
-
-// Structure drill-down bounds.
-const (
- structMaxDepth = 3 // _STRUCT_MAX_DEPTH: max TOC levels drilled
- structMaxNodes = 10 // _STRUCT_MAX_NODES: cap on nodes rendered in the outline
- structBranchK = 2 // _STRUCT_BRANCH_K: keep top-K most relevant nodes per level
- structVecBeamRatio = 0.5 // _STRUCT_VEC_BEAM_RATIO: score floor relative to best
- structRelevanceMin = 1 // a node must match at least this many query terms to descend
- structDescSnippet = 180 // _STRUCT_DESC_SNIPPET: cap on a node's description snippet
- structMaxChunks = 4 // _STRUCT_MAX_CHUNKS: cap on chunk snippets returned in the outline
- structCatalogKinds = "tree_node|page_index|section" // kinds eligible for catalog outline
-
- // structTocMaxDepth bounds the ancestor walk in renderTocDrilldown's chunk-retrieval and
- // llm_toc branches. The remaining _STRUCT_TOC_* limits gate the whole-TOC LLM pass,
- // which is off by default and not mirrored here (it would be unreachable).
- structTocMaxDepth = 6
-
- // Chunk-index selection bounds for structures that cannot score themselves
- // (RAPTOR blob path). Mirror _STRUCT_RECALL_TOP_N and _STRUCT_MAX_CHUNK_HITS.
- structRecallTopN = 24 // _STRUCT_RECALL_TOP_N: hybrid chunk hits recalled per document
- structMaxChunkHits = 8 // _STRUCT_MAX_CHUNK_HITS: retrieved chunks shown per drilled doc
-
- // Claim/evidence leg for the drill-down.
- // Claim rows are scored FLAT against the query (hybrid BM25+KNN, RRF), so a
- // bad early pick cannot prune the subtree holding the answer. Claim-first:
- // when claims hit they REPLACE the drill-down — a claim is an atomic
- // proposition carrying its own verbatim evidence, so a hit already is the
- // answer material; the tree only labels where each claim sits.
- structClaimLeg = true // _STRUCT_CLAIM_LEG
- structClaimTopN = 8 // _STRUCT_CLAIM_TOP_N: claims rendered per document
- structClaimFirst = true // _STRUCT_CLAIM_FIRST
- structClaimFirstMinHits = 1 // _STRUCT_CLAIM_FIRST_MIN_HITS
- // structClaimSufficientHits: enough distinct claims matching the query means the model
- // may cite them without deep-reading chunks. The drillout emits <claims_sufficient/>
- // when a document's claim hits reach it.
- structClaimSufficientHits = 3
-
- // Per-node-vector flat selection: page_index spreads facts across TOC levels, so
- // descending level by level can prune the branch holding the answer before it is ever
- // scored. Scoring EVERY node against the query at once costs nothing extra (their
- // embeddings are already loaded) and cannot prune — this is the DEFAULT primary path
- // for page_index structures.
- structTocLLMSelect = false // the one-shot LLM TOC selection is OFF — not ported
- structFlatNodeSelect = true // flat per-node selection is the primary path
- structFlatTopN = 10 // _STRUCT_FLAT_TOP_N (navigation.py:1158)
-)
-
-// queryTerms mirrors navigation.py's regex tokenizer: lowercase coarse word
-// tokens of length >= 2 (the BM25-ish keyword signal).
-func queryTerms(q string) []string {
- if q == "" {
- return nil
- }
- lower := strings.ToLower(q)
- var out []string
- var cur []byte
- for i := 0; i < len(lower); i++ {
- c := lower[i]
- if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '_' {
- cur = append(cur, c)
- continue
- }
- if len(cur) >= 2 {
- out = append(out, string(cur))
- }
- cur = cur[:0]
- }
- if len(cur) >= 2 {
- out = append(out, string(cur))
- }
- return out
-}
-
-// nodeRelevance mirrors navigation.py _node_relevance: how many query terms occur
-// in a TOC node's name+description.
-func nodeRelevance(terms []string, name, desc string) int {
- if len(terms) == 0 {
- return 0
- }
- text := strings.ToLower(name + " " + desc)
- n := 0
- for _, t := range terms {
- if strings.Contains(text, t) {
- n++
- }
- }
- return n
-}
-
-// structureNode is a node of a compiled structure hierarchy during drill-down:
-// the entity plus its optional embedding vector (for cosine scoring).
-type structureNode struct {
- name string
- nodeType string
- desc string
- sourceChunkIDs []string
- vec []float64
-}
-
-// structureRel is a parent->child relation of a compiled structure hierarchy.
-type structureRel struct {
- from string
- to string
- relType string
-}
-
-// buildTocTree mirrors navigation.py _build_toc_tree: a name->node map, the
-// parent->children map, the child->parent map (single parent, last-wins) and the
-// root names (tree nodes with no parent; isolated nodes are their own roots).
-func buildTocTree(nodes []structureNode, rels []structureRel) (map[string]structureNode, map[string][]string, map[string]string, []string) {
- byName := map[string]structureNode{}
- for _, n := range nodes {
- if n.name != "" {
- byName[n.name] = n
- }
- }
- children := map[string][]string{}
- parents := map[string]string{}
- for _, r := range rels {
- p, c := r.from, r.to
- if p == "" || c == "" || p == c {
- continue
- }
- if _, ok := children[p]; !ok {
- children[p] = []string{}
- }
- dup := false
- for _, x := range children[p] {
- if x == c {
- dup = true
- break
- }
- }
- if !dup {
- children[p] = append(children[p], c)
- }
- parents[c] = p
- }
- var roots []string
- for n := range byName {
- if _, has := parents[n]; !has {
- roots = append(roots, n)
- }
- }
- if len(roots) == 0 {
- for n := range byName {
- if _, has := children[n]; !has {
- roots = append(roots, n)
- }
- }
- }
- if len(roots) == 0 {
- for n := range byName {
- roots = append(roots, n)
- }
- }
- return byName, children, parents, roots
-}
-
-// chunkPtrs mirrors navigation.py _chunk_ptrs: a bounded, deduped comma-joined
-// slice of an item's source chunk ids (<=8, the anchors the model sees).
-func chunkPtrs(ids []string) string {
- if len(ids) == 0 {
- return ""
- }
- var seen []string
- for _, c := range ids {
- if c == "" {
- continue
- }
- if !sliceContains(seen, c) {
- seen = append(seen, c)
- if len(seen) >= 8 {
- break
- }
- }
- }
- return strings.Join(seen, ",")
-}
-
-func sliceContains(s []string, v string) bool {
- for _, x := range s {
- if x == v {
- return true
- }
- }
- return false
-}
-
-// collectChunkIDs mirrors navigation.py _collect_chunk_ids: a deduped, capped
-// union of source chunk ids across a set of nodes.
-func collectChunkIDs(nodes []structureNode, capN int) []string {
- if capN <= 0 {
- capN = 32
- }
- var out []string
- for _, n := range nodes {
- for _, c := range n.sourceChunkIDs {
- if c == "" {
- continue
- }
- if sliceContains(out, c) {
- continue
- }
- out = append(out, c)
- if len(out) >= capN {
- return out
- }
- }
- }
- return out
-}
-
-// nodeScore mirrors navigation.py _node_score: cosine when a query vector and the
-// node embedding both exist, else keyword relevance.
-func nodeScore(qvec []float64, terms []string, n structureNode) float64 {
- if len(qvec) > 0 && len(n.vec) > 0 {
- return cosine(qvec, n.vec)
- }
- return float64(nodeRelevance(terms, n.name, n.desc))
-}
-
-// vecsEqual reports whether two node embeddings are element-wise equal, mirroring
-// navigation.py _vecs_equal. Zero-length or differently-sized vectors are unequal.
-func vecsEqual(a, b []float64) bool {
- if len(a) != len(b) {
- return false
- }
- for i := range a {
- if a[i] != b[i] {
- return false
- }
- }
- return true
-}
-
-// hasDistinctNodeVectors reports whether the nodes carry per-node embeddings
-// rather than one shared vector. Mirrors navigation.py _has_distinct_node_vectors:
-// a page_index / per-node tree has distinct node vectors, while a RAPTOR blob
-// inherits the blob's one vector for every node. Returns false when fewer than
-// two nodes carry a vector or all share the same one.
-func hasDistinctNodeVectors(nodes []structureNode) bool {
- var first []float64
- haveFirst := false
- for _, n := range nodes {
- if len(n.vec) == 0 {
- continue
- }
- if !haveFirst {
- first = n.vec
- haveFirst = true
- continue
- }
- if !vecsEqual(first, n.vec) {
- return true
- }
- }
- return false
-}
-
-// flatSelectNodes mirrors navigation.py _flat_select_nodes: score EVERY node
-// against the query at once and return the top-N names. page_index spreads
-// facts across TOC levels, so descending level by level can prune the branch
-// holding the answer before it is ever scored; scoring all nodes costs nothing
-// extra (their embeddings are already loaded) and cannot prune. Returns nil
-// when no query vector is available: the caller then falls back to the beam
-// descent, which also works from keyword overlap.
-func flatSelectNodes(qvec []float64, nodes []structureNode, topN int) []string {
- if len(qvec) == 0 || len(nodes) == 0 || topN <= 0 {
- return nil
- }
- type scored struct {
- name string
- score float64
- }
- var candidates []scored
- for _, n := range nodes {
- name := strings.TrimSpace(n.name)
- if name == "" || len(n.vec) == 0 {
- continue
- }
- candidates = append(candidates, scored{name: name, score: cosine(qvec, n.vec)})
- }
- if len(candidates) == 0 {
- return nil
- }
- sort.Slice(candidates, func(i, j int) bool { return candidates[i].score > candidates[j].score })
- if len(candidates) > topN {
- candidates = candidates[:topN]
- }
- out := make([]string, 0, len(candidates))
- for _, c := range candidates {
- out = append(out, c.name)
- }
- return out
-}
-
-// nodesCoveringChunks returns the names of nodes whose source_chunk_ids cover any
-// of chunkIDs, mirroring navigation.py _nodes_covering_chunks. Used only to give
-// retrieved chunks a structural home; a chunk no node covers is still shown.
-func nodesCoveringChunks(nodes []structureNode, chunkIDs []string) []string {
- if len(chunkIDs) == 0 {
- return nil
- }
- want := make(map[string]bool, len(chunkIDs))
- for _, c := range chunkIDs {
- if c != "" {
- want[c] = true
- }
- }
- var names []string
- for _, n := range nodes {
- name := strings.TrimSpace(n.name)
- if name == "" {
- continue
- }
- for _, c := range n.sourceChunkIDs {
- if c != "" && want[c] {
- names = append(names, name)
- break
- }
- }
- }
- return names
-}
-
-// recallChunkIDsInDoc mirrors navigation.py _recall_chunk_ids_in_doc: for a
-// structure whose nodes share one blob vector (RAPTOR) it asks the retrieval
-// backend for the document's most relevant ORIGINAL chunks (the backend already
-// excludes compiled rows), and returns them as (id, score) hits. A nil
-// retriever, empty query, or an error yields no hits.
-func recallChunkIDsInDoc(ctx context.Context, retriever Retriever, query, docID, tenantID string, datasetIDs []string, topN int) []chunkHit {
- if retriever == nil || query == "" || docID == "" {
- return nil
- }
- if topN <= 0 {
- topN = structRecallTopN
- }
- chunks, err := retriever.Retrieve(ctx, RetrieveRequest{
- Query: query,
- DatasetIDs: datasetIDs,
- DocScope: []string{docID},
- TopN: topN,
- TenantID: tenantID,
- })
- if err != nil || len(chunks) == 0 {
- return nil
- }
- hits := make([]chunkHit, 0, len(chunks))
- for _, c := range chunks {
- id, _ := c["chunk_id"].(string)
- if id == "" {
- continue
- }
- score := 0.0
- if s, ok := c["similarity"].(float64); ok {
- score = s
- }
- hits = append(hits, chunkHit{id: id, score: score})
- }
- return hits
-}
-
-// drillKeptNodes mirrors navigation.py _drill_kept_nodes: vector-beam BFS descent
-// from the TOC roots, keeping top-K children per level by cosine (relative beam
-// threshold) or by keyword relevance (absolute minimum), returning the kept
-// nodes, the single-parent map, the kept-name set and the overall best score.
-func drillKeptNodes(qvec []float64, terms []string, nodes []structureNode, rels []structureRel) ([]structureNode, map[string]string, map[string]bool, float64) {
- byName, children, parents, roots := buildTocTree(nodes, rels)
- if len(roots) == 0 {
- return nil, nil, nil, 0.0
- }
- frontier := roots
- keptNames := map[string]bool{}
- depth := 0
- bestOverall := 0.0
- for len(frontier) > 0 && depth <= structMaxDepth {
- type scored struct {
- score float64
- name string
- }
- var scoredList []scored
- for _, n := range frontier {
- e, ok := byName[n]
- if ok {
- scoredList = append(scoredList, scored{nodeScore(qvec, terms, e), n})
- }
- }
- if len(scoredList) == 0 {
- break
- }
- sort.SliceStable(scoredList, func(i, j int) bool { return scoredList[i].score > scoredList[j].score })
- best := scoredList[0].score
- if best > bestOverall {
- bestOverall = best
- }
- if len(qvec) == 0 && best < float64(structRelevanceMin) {
- break
- }
- var top []scored
- if len(qvec) > 0 {
- for _, s := range scoredList {
- if s.score >= best*structVecBeamRatio {
- top = append(top, s)
- if len(top) >= structBranchK {
- break
- }
- }
- }
- } else {
- for _, s := range scoredList {
- if s.score >= float64(structRelevanceMin) {
- top = append(top, s)
- if len(top) >= structBranchK {
- break
- }
- }
- }
- }
- if len(top) == 0 {
- break
- }
- var next []string
- for _, s := range top {
- if !keptNames[s.name] {
- keptNames[s.name] = true
- }
- next = append(next, children[s.name]...)
- }
- frontier = next
- depth++
- }
- // Order-independent (see drillWithAncestors): the ancestor closure is the same
- // SET whatever order the names are visited in.
- for name := range keptNames {
- cur := parents[name]
- guard := 0
- for cur != "" && !keptNames[cur] && guard < structMaxDepth {
- keptNames[cur] = true
- cur = parents[cur]
- guard++
- }
- }
- var kept []structureNode
- for _, n := range sortedKeptNames(keptNames) {
- if e, ok := byName[n]; ok {
- kept = append(kept, e)
- }
- }
- return kept, parents, keptNames, bestOverall
-}
-
-// outlineStats mirrors navigation.py _outline_stats: the entity / chunk-pointer
-// counts for the flat fallback outline (no drill happened => top_score 0).
-func outlineStats(nodes []structureNode) (nodeCount, ptrCount int) {
- count := len(nodes)
- if count > structMaxNodes {
- count = structMaxNodes
- }
- ptrs := 0
- for _, n := range nodes[:count] {
- ptrs += len(collectChunkIDs([]structureNode{n}, 32))
- }
- return count, ptrs
-}
-
-// navTypeOr returns a node type, defaulting to "other".
-func navTypeOr(t string) string {
- if t == "" {
- return "other"
- }
- return t
-}
-
-// navOutlineLine renders one "- name (type): desc [chunks: c1,c2]" line.
-func navOutlineLine(indent, name, nodeType, desc string, chunks []string) string {
- line := indent + "- " + name + " (" + navTypeOr(nodeType) + ")"
- if desc != "" {
- line += ": " + Snippet(desc, structDescSnippet)
- }
- if c := chunkPtrs(chunks); c != "" {
- line += " [chunks: " + c + "]"
- }
- return line
-}
-
-// renderOutline mirrors navigation.py _render_outline: a compact flat outline of
-// entities then relations (fallback when no query terms are usable).
-func renderOutline(nodes []structureNode, rels []structureRel) string {
- var lines []string
- capE, capR := 40, 40
- for i, n := range nodes {
- if i >= capE || n.name == "" {
- continue
- }
- lines = append(lines, navOutlineLine("", n.name, n.nodeType, n.desc, n.sourceChunkIDs))
- }
- for i, r := range rels {
- if i >= capR || r.from == "" || r.to == "" {
- continue
- }
- rt := r.relType
- if rt == "" {
- rt = "related_to"
- }
- lines = append(lines, "- "+r.from+" -["+rt+"]-> "+r.to)
- }
- return strings.Join(lines, "\n")
-}
-
-// rankChunksByTerms mirrors navigation.py _rank_chunks_by_terms: rank candidate
-// chunks by how many query terms overlap with their text. Zero-LLM keyword
-// relevance for the precise chunk_ids path. The accessor abstracts reading a
-// chunk's text so the runtime stays store-agnostic and testable.
-func rankChunksByTerms(candidates []chunkWithText, queries []string) []chunkWithText {
- terms := make([]string, 0, 8)
- for _, q := range queries {
- for _, tok := range queryTerms(q) {
- if !sliceContains(terms, tok) {
- terms = append(terms, tok)
- }
- }
- }
- if len(terms) == 0 {
- return candidates
- }
- type scored struct {
- hits int
- c chunkWithText
- }
- var scoredList []scored
- for _, c := range candidates {
- text := strings.ToLower(c.text)
- hits := 0
- for _, t := range terms {
- if strings.Contains(text, t) {
- hits++
- }
- }
- if hits > 0 {
- scoredList = append(scoredList, scored{hits, c})
- }
- }
- sort.SliceStable(scoredList, func(i, j int) bool { return scoredList[i].hits > scoredList[j].hits })
- out := make([]chunkWithText, 0, len(scoredList))
- for _, s := range scoredList {
- out = append(out, s.c)
- }
- return out
-}
-
-// chunkWithText pairs a chunk id with its text for structure snippet ranking.
-type chunkWithText struct {
- id string
- text string
-}
-
-// structureDrillout is the per-document result of the TOC drill-down. selector records
-// which strategy chose the nodes: llm_toc / chunk_retrieval / beam.
-type structureDrillout struct {
- outline string
- nodes int
- chunkPtrs int
- topScore float64
- chunkPaths map[string]string
- selector string
- // claimHits is the count of claim rows that matched the query for this drill, set by
- // the caller when non-empty.
- claimHits int
-}
-
-// chunkHit is a retrieved chunk id with its retrieval score — the input shape of the
-// chunk_retrieval strategy.
-type chunkHit struct {
- id string
- score float64
-}
-
-// sortedKeptNames returns a kept-name set as a sorted slice.
-//
-// The kept set is a map because it is used for membership tests, but three
-// order-sensitive outputs are materialised from it: the rendered outline
-// (kept[:structMaxNodes]), chunkPaths (first writer wins for a chunk covered by
-// several nodes) and collectChunkIDs(kept, 32) — the chunk list whose snippets
-// reach the model. Iterating the kept set gives an arbitrary order that is not even
-// reproducible across runs (string hashing is randomised per process), so there is no
-// canonical order to inherit. Go randomises map iteration per range statement, so leaving
-// it to the map would vary those three outputs even between two calls in one process;
-// sorting gives a stable, machine-independent order instead.
-func sortedKeptNames(names map[string]bool) []string {
- out := make([]string, 0, len(names))
- for n := range names {
- out = append(out, n)
- }
- sort.Strings(out)
- return out
-}
-
-// drillWithAncestors pulls the ancestors of names into the kept set so a path
-// renders as root -> ... -> node, mirroring _with_ancestors in _render_toc_drilldown.
-func drillWithAncestors(names map[string]bool, parents map[string]string) map[string]bool {
- // Ranging a map while adding keys visits the new ones in an unspecified order,
- // but the result is order-independent: every start walks up to the root (or the
- // depth guard), so the SET is the same whatever order the names are visited in
- // the same closure built from a `list(names)` snapshot.
- for name := range names {
- cur := parents[name]
- guard := 0
- for cur != "" && !names[cur] && guard < structTocMaxDepth {
- names[cur] = true
- cur = parents[cur]
- guard++
- }
- }
- return names
-}
-
-// renderTocDrilldown mirrors navigation.py _render_toc_drilldown: it renders a
-// query-focused outline of one document's compiled structure, choosing the nodes
-// to show by one of four strategies in priority order:
-//
-// - claimHits (claim-first, non-empty and >= structClaimFirstMinHits): claims
-// matched the query directly. They are rendered as statement + verbatim
-// evidence and take over the selection outright — nothing is drilled, no
-// chunk snippets are loaded, and the tree only labels where each claim sits
-//
-// selector "claim".
-// - selected (non-empty): node names picked by the whole-TOC LLM pass, kept
-// whole with their ancestors — selector "llm_toc".
-// - chunkHits (non-empty): [(id, score)] from chunk retrieval (RAPTOR blob path);
-// the chunks are shown as retrieved and the tree only labels where they sit —
-// selector "chunk_retrieval".
-// - neither: VECTOR BEAM descent toward the query, falling back to keyword
-// overlap when no vectors are available — selector "beam".
-//
-// When no terms/vector and neither selection is provided it falls back to the flat
-// outline. loader, when non-nil, fetches chunk texts so short snippets can be
-// appended; a nil loader skips the snippet pass.
-func renderTocDrilldown(query string, qvec []float64, nodes []structureNode, rels []structureRel, loader func(ids []string) []chunkWithText, chunkHits []chunkHit, selected []string, claimHits []DocClaimHit) structureDrillout {
- flat := func() structureDrillout {
- out := renderOutline(nodes, rels)
- n, p := outlineStats(nodes)
- return structureDrillout{outline: out, nodes: n, chunkPtrs: p, topScore: 0.0, chunkPaths: map[string]string{}}
- }
- terms := queryTerms(query)
- // Pinned nodes (chunk retrieval / whole-TOC) stand even when the query yields
- // no usable terms or vector; bailing out here would discard a valid selection.
- if len(chunkHits) == 0 && len(selected) == 0 && len(terms) == 0 && len(qvec) == 0 {
- return flat()
- }
-
- byName, _, parents, _ := buildTocTree(nodes, rels)
-
- var kept []structureNode
- var keptNames map[string]bool
- var selectedIDs []string
- best := 0.0
- selector := "beam"
- claimFirst := structClaimFirst && len(claimHits) >= structClaimFirstMinHits
-
- switch {
- case claimFirst:
- // Claims already carry the answer material (statement + verbatim
- // evidence), so nothing is drilled: the tree's only remaining job is to
- // show where each claim sits. Descending would re-derive the same facts
- // and risk pruning past them.
- var claimCIDs []string
- for _, h := range claimHits {
- if cid := strings.TrimSpace(h.ChunkID); cid != "" && !sliceContains(claimCIDs, cid) {
- claimCIDs = append(claimCIDs, cid)
- }
- }
- covering := nodesCoveringChunks(nodes, claimCIDs)
- names := make(map[string]bool)
- for _, n := range covering {
- if _, ok := byName[n]; ok {
- names[n] = true
- }
- }
- keptNames = drillWithAncestors(names, parents)
- for _, n := range sortedKeptNames(keptNames) {
- if e, ok := byName[n]; ok {
- kept = append(kept, e)
- }
- }
- for _, h := range claimHits {
- if h.Score > best {
- best = h.Score
- }
- }
- selector = "claim"
- case len(selected) > 0:
- names := make(map[string]bool)
- for _, n := range selected {
- if _, ok := byName[n]; ok {
- names[n] = true
- }
- }
- keptNames = drillWithAncestors(names, parents)
- for _, n := range sortedKeptNames(keptNames) {
- if e, ok := byName[n]; ok {
- kept = append(kept, e)
- }
- }
- best = 0.0
- selector = "llm_toc"
- case len(chunkHits) > 0:
- seen := make(map[string]bool)
- for _, h := range chunkHits {
- if h.id != "" && !seen[h.id] {
- seen[h.id] = true
- selectedIDs = append(selectedIDs, h.id)
- }
- }
- covering := nodesCoveringChunks(nodes, selectedIDs)
- names := make(map[string]bool)
- for _, n := range covering {
- if _, ok := byName[n]; ok {
- names[n] = true
- }
- }
- keptNames = drillWithAncestors(names, parents)
- for _, n := range sortedKeptNames(keptNames) {
- if e, ok := byName[n]; ok {
- kept = append(kept, e)
- }
- }
- for _, h := range chunkHits {
- if h.score > best {
- best = h.score
- }
- }
- selector = "chunk_retrieval"
- default:
- kept, parents, keptNames, best = drillKeptNodes(qvec, terms, nodes, rels)
- selector = "beam"
- }
- if len(kept) == 0 && len(selectedIDs) == 0 && !(structClaimFirst && len(claimHits) > 0) {
- // A strategy was chosen but kept nothing; still report which one ran.
- // Claim-first is the exception: upper
- // RAPTOR clusters carry no source_chunk_ids, so no node may cover a hit
- // claim — bailing out here would throw the claims away.
- out := flat()
- out.selector = selector
- return out
- }
- // depth_of: node -> number of kept ancestors (rendering indentation). The
- // iteration order does not matter here: each entry is a pure function of
- // (node, parents, keptNames), so the map comes out the same either way.
- depthOf := map[string]int{}
- for n := range keptNames {
- d := 0
- cur := parents[n]
- for cur != "" && keptNames[cur] {
- d++
- cur = parents[cur]
- }
- depthOf[n] = d
- }
- // root -> ... -> node path for each kept node, mapped onto its chunks.
- chunkPaths := map[string]string{}
- for _, e := range kept {
- name := strings.TrimSpace(e.name)
- if name == "" {
- continue
- }
- var segs []string
- segs = append(segs, name)
- cur := parents[name]
- guard := 0
- for cur != "" && guard < structMaxDepth {
- segs = append(segs, cur)
- cur = parents[cur]
- guard++
- }
- for i, j := 0, len(segs)-1; i < j; i, j = i+1, j-1 {
- segs[i], segs[j] = segs[j], segs[i]
- }
- path := strings.Join(segs, " -> ")
- for _, cid := range e.sourceChunkIDs {
- if cid != "" {
- if _, seen := chunkPaths[cid]; !seen {
- chunkPaths[cid] = path
- }
- }
- }
- }
- var lines []string
- count := len(kept)
- if count > structMaxNodes {
- count = structMaxNodes
- }
- for _, e := range kept[:count] {
- name := strings.TrimSpace(e.name)
- if name == "" {
- continue
- }
- indent := strings.Repeat(" ", depthOf[name])
- lines = append(lines, navOutlineLine(indent, name, e.nodeType, e.desc, e.sourceChunkIDs))
- }
- // Claim/evidence lines come before the chunk snippets: a gated claim is a verified
- // atomic fact carrying its verbatim quote, so it reads better than a truncated chunk
- // and needs no
- // chunk load at all. Its chunk is dropped from the snippet list below so the
- // same content is not sent twice.
- claimed := map[string]bool{}
- for _, hit := range claimHits {
- cid := strings.TrimSpace(hit.ChunkID)
- if cid == "" || claimed[cid] {
- continue
- }
- claimed[cid] = true
- line := "- [claim] " + strings.TrimSpace(hit.Name)
- if quote := docClaimQuote(hit); quote != "" {
- line += " | Evidence: \"" + Snippet(quote, ClaimEvidenceChars) + "\""
- }
- line += " [chunks: " + cid + "]"
- lines = append(lines, line)
- }
- // A claim's chunk still deserves a structural label — annotate it from the
- // node that covers it, falling back to nothing when no node does (upper
- // RAPTOR clusters carry no source_chunk_ids, so this stays honest rather
- // than inventing a path).
- for cid := range claimed {
- if _, ok := chunkPaths[cid]; ok {
- continue
- }
- if owners := nodesCoveringChunks(nodes, []string{cid}); len(owners) > 0 {
- chunkPaths[cid] = owners[0]
- }
- }
- // Chunk snippets. When chunk retrieval did the selecting, the hits are shown
- // as-is (filtering them back through the nodes would drop chunks no node
- // covers); otherwise take the chunks behind the drilled nodes. Under
- // claim-first a matched claim already renders its verbatim evidence, so
- // nothing is loaded — loading the chunk behind it would echo the same
- // passage a second time.
- var wanted []string
- if claimFirst {
- wanted = nil
- } else if len(selectedIDs) > 0 {
- wanted = selectedIDs
- } else {
- wanted = collectChunkIDs(kept, 32)
- }
- if len(wanted) > 0 && len(claimed) > 0 {
- filtered := make([]string, 0, len(wanted))
- for _, c := range wanted {
- if !claimed[c] {
- filtered = append(filtered, c)
- }
- }
- wanted = filtered
- }
- if len(wanted) > 0 && loader != nil {
- chunks := loader(wanted)
- if len(chunks) > 0 {
- var ranked []chunkWithText
- limit := structMaxChunks
- if len(selectedIDs) > 0 {
- // Retrieval order is the relevance order; re-ranking by terms would
- // discard the score that selected them.
- pos := make(map[string]int, len(selectedIDs))
- for i, id := range selectedIDs {
- pos[id] = i
- }
- ranked = append(ranked, chunks...)
- sort.SliceStable(ranked, func(i, j int) bool { return pos[ranked[i].id] < pos[ranked[j].id] })
- limit = structMaxChunkHits
- } else {
- ranked = rankChunksByTerms(chunks, []string{query})
- }
- capN := len(ranked)
- if capN > limit {
- capN = limit
- }
- for _, c := range ranked[:capN] {
- text := strings.TrimSpace(c.text)
- if text == "" {
- continue
- }
- lines = append(lines, "- [chunk "+c.id+"]: "+Snippet(text, 300))
- }
- }
- }
- return structureDrillout{
- outline: strings.Join(lines, "\n"),
- nodes: len(kept),
- chunkPtrs: len(wanted),
- topScore: best,
- chunkPaths: chunkPaths,
- selector: selector,
- claimHits: len(claimHits),
- }
-}
-
-// navigate_structure (zero-LLM vector-beam drill-down; mirrors _navigate_structure_impl)
-
-// structureKindsFor maps a navigate_structure kind string to the compiled-kinds
-// set to read, mirroring navigation.py _structure_kinds_for. Defaults to catalog.
-func structureKindsFor(kind string) map[string]bool {
- k := strings.TrimSpace(strings.ToLower(kind))
- if k == "" {
- k = "catalog"
- }
- switch k {
- case "mindmap", "mind_map", "concept":
- out := make(map[string]bool, len(mindmapKinds))
- for kk := range mindmapKinds {
- out[kk] = true
- }
- return out
- case "graph", "kg", "entity", "ontology":
- return map[string]bool{"graph": true, "ontology": true, "entity": true, "raptor": true}
- default:
- out := make(map[string]bool, len(catalogKinds))
- for kk := range catalogKinds {
- out[kk] = true
- }
- return out
- }
-}
-
-// structureNodesFromEntities converts loaded structure entities into the
-// drill-down node shape (no vectors unless filled in by a vector pass).
-func structureNodesFromEntities(es []structureEntity) []structureNode {
- if len(es) == 0 {
- return nil
- }
- nodes := make([]structureNode, 0, len(es))
- for _, e := range es {
- if strings.TrimSpace(e.Name) == "" {
- continue
- }
- nodes = append(nodes, structureNode{
- name: strings.TrimSpace(e.Name),
- nodeType: e.Type,
- desc: e.Description,
- sourceChunkIDs: append([]string(nil), e.SourceChunkIDs...),
- vec: e.Vec,
- })
- }
- return nodes
-}
-
-// structureNodeLoader fetches chunk texts by id for the drill-down snippet pass,
-// mirroring the chunk loader _render_toc_drilldown uses. Returns nil when the
-// store/engine is not available (outline still renders, minus snippets).
-func structureNodeLoader(ctx context.Context, indexName string, ids []string) []chunkWithText {
- raw := loadChunksByIDs(ctx, indexName, ids)
- if len(raw) == 0 {
- return nil
- }
- out := make([]chunkWithText, 0, len(raw))
- for _, c := range raw {
- id := ChunkIDOf(c)
- if id == "" {
- continue
- }
- out = append(out, chunkWithText{id: id, text: ChunkTextOf(c)})
- }
- return out
-}
-
-// readStructureDocCore reads ONE document's compiled structure of the requested
-// kind and vector-beam drills it toward the query. It returns the per-document
-// drill data (entities, relations, outline, chunk pointers) rather than the XML,
-// so both the single-doc wrapper and the multi-doc router render the same doc as
-// a <doc> segment. emptyReason == "" means the structure was read.
-func readStructureDocCore(ctx context.Context, tenantID, query, docID, kind string, deps SearchDeps) (structureDrillout, []structureNode, []structureRel, string) {
- none := structureDrillout{chunkPaths: map[string]string{}}
- switch {
- case query == "":
- return none, nil, nil, ReasonBadArgs
- case docID == "":
- return none, nil, nil, ReasonNoDoc
- }
- // A document that is not in the bound datasets yields an empty structure rather than an
- // unscoped read, so a stale doc_id cannot pull in another dataset's outline.
- if belongs, verified := docInDatasets(ctx, deps, docID); verified && !belongs {
- return none, nil, nil, ReasonNoStructure
- }
- kinds := structureKindsFor(kind)
- indexName := indexNameFor(tenantID, deps.IndexName)
- // Vector-aware load mirrors _read_structures: encode the query first so the
- // structure rows can be read with the q_<dim>_vec field, then judge whether
- // the nodes carry per-node embeddings.
- var qvec []float64
- var vecField string
- if vec := encodeSeedVector(ctx, deps, tenantID, query); vec != nil {
- qvec = vec
- vecField = "q_" + strconv.Itoa(len(vec)) + "_vec"
- }
- entities, rels := loadStructureGraph(ctx, indexName, docID, kinds, vecField)
- nodes := structureNodesFromEntities(entities)
- if len(nodes) == 0 {
- return none, nil, nil, ReasonNoStructure
- }
- loader := func(ids []string) []chunkWithText { return structureNodeLoader(ctx, indexName, ids) }
-
- // Claim leg: flat hybrid claim recall
- // for this document. Selection priority: claims when they hit, then
- // whatever the tree's own shape supports.
- var claimHits []DocClaimHit
- if structClaimLeg {
- kindList := make([]string, 0, len(kinds))
- for k := range kinds {
- kindList = append(kindList, k)
- }
- sort.Strings(kindList)
- claimHits = RecallDocClaimHits(ctx, deps, query, docID, kindList, qvec, structClaimTopN)
- }
- // Claim-first: the claims decide, so nothing is drilled and no chunks are
- // recalled to choose.
- claimFirst := structClaimFirst && len(claimHits) >= structClaimFirstMinHits
-
- // RAPTOR / shared-vector blobs cannot score their nodes against the query, so
- // they defer to chunk retrieval for the drill. Per-node-vector structures take the FLAT
- // node
- // selection instead of the beam drill.
- var selected []string
- var chunkHits []chunkHit
- if !claimFirst && hasDistinctNodeVectors(nodes) {
- // _STRUCT_TOC_LLM_SELECT is False,
- // so the one-shot LLM TOC selection never runs and the flat vector scoring
- // is the default primary path. It renders under selector="llm_toc"
- // (renderTocDrilldown's selected branch); an empty result (no query vector)
- // falls back to the beam descent, which also works from keyword overlap.
- if structFlatNodeSelect {
- selected = flatSelectNodes(qvec, nodes, structFlatTopN)
- }
- } else if !claimFirst {
- chunkHits = recallChunkIDsInDoc(ctx, deps.Backend, query, docID, tenantID, deps.KbIDs, 0)
- }
- drill := renderTocDrilldown(query, qvec, nodes, rels, loader, chunkHits, selected, claimHits)
- if len(claimHits) > 0 {
- // Mirror the rendered claims into the shared evidence pool: the SCA, the slot prefill
- // and the final compose all read ONLY the pool, so a claim that already states the
- // fact must land there.
- drill.claimHits = len(claimHits)
- publishClaimHits(deps, claimHits, docID)
- }
- return drill, nodes, rels, ""
-}
-
-// structureDocSegment renders one <doc> element (doc_id / doc_title="" / entities /
-// relations plus the <structure> outline); doc_title is always empty.
-func structureDocSegment(docID, query, kind string, rank int, nodes []structureNode, rels []structureRel, drill structureDrillout) string {
- esc := func(s string) string { return XMLEscape(s) }
- var b strings.Builder
- fmt.Fprintf(&b, ` <doc rank="%d" doc_id="%s" doc_title="" entities="%d" relations="%d">`, rank, esc(docID), len(nodes), len(rels))
- if drill.outline != "" {
- b.WriteString("\n <structure>" + esc(drill.outline) + "</structure>")
- }
- // Enough matched claims mark the document's evidence as citable without a list_chunks
- // deep-read.
- if drill.claimHits >= structClaimSufficientHits {
- b.WriteString("\n <claims_sufficient/>")
- }
- b.WriteString("\n </doc>")
- return b.String()
-}
-
-// emptyReasonLabel maps an empty-reason constant to the XML error label written into
-// <structure_navigation error="...">.
-func emptyReasonLabel(reason string) string {
- switch reason {
- case ReasonBadArgs:
- return "query is required"
- case ReasonNoDoc:
- return "no document located"
- default:
- return "no structure"
- }
-}
-
-// navigateStructures renders the compiled structures of MULTIPLE documents into a single
-// <structure_navigation> with one <doc> per readable structure: each document is read and
-// drilled independently, documents without a compiled structure of the requested kind are
-// skipped, and the resulting outline(s) are merged with per-doc ranks.
-// The aggregate drillout carries summed node/pointer counts, the max top score,
-// and the union of chunk paths across the documents.
-// navigateStructures drills the requested compiled-structure kind across the
-// given documents. When docIDs is empty it vector-routes the documents FIRST (the nav
-// tree) and then drills those — so a direct call need not rely on its caller pre-routing.
-// The navigate_structure TOOL already routes before calling this, so in the tool path
-// docIDs is non-empty and routing here is a no-op.
-func navigateStructures(ctx context.Context, tenantID, query string, docIDs []string, kind string, router NavTreeRouter, deps SearchDeps) (NavResult, structureDrillout) {
- agg := structureDrillout{chunkPaths: map[string]string{}}
- var segs []string
- var docIDsSeen []string
- var entities []map[string]any
- if len(docIDs) == 0 {
- // With an empty doc scope the documents are vector-routed first; only if that also
- // reaches nothing is it a MISS.
- if router != nil {
- // DocScope: the session ceiling applied when doc_ids is empty.
- routed := NavigateTree(ctx, router, NavTreeInput{
- Query: query,
- KbIDs: deps.KbIDs,
- TenantID: tenantID,
- DocScope: deps.DocScope,
- })
- docIDs = dedupStrings(routed.DocIDs)
- }
- if len(docIDs) == 0 {
- return NavResult{
- Text: `<structure_navigation count="0" error="no document located">` + "\n</structure_navigation>",
- EmptyReason: ReasonNoDoc,
- }, agg
- }
- }
- for _, did := range docIDs {
- drill, nodes, rels, empty := readStructureDocCore(ctx, tenantID, query, did, kind, deps)
- if empty != "" {
- // No compiled structure of this kind — the per-doc read skips it.
- continue
- }
- segs = append(segs, structureDocSegment(did, query, kind, len(segs)+1, nodes, rels, drill))
- docIDsSeen = append(docIDsSeen, did)
- entities = append(entities, entityMaps(nodes)...)
- agg.nodes += drill.nodes
- agg.chunkPtrs += drill.chunkPtrs
- agg.claimHits += drill.claimHits
- if drill.topScore > agg.topScore {
- agg.topScore = drill.topScore
- }
- for cid, p := range drill.chunkPaths {
- if _, ok := agg.chunkPaths[cid]; !ok {
- agg.chunkPaths[cid] = p
- }
- }
- }
- if len(segs) == 0 {
- // doc_ids were given but none carried a compiled structure of this kind:
- // empty_reason is "no_structure" when total_entities == 0 (the count="0"
- // <structure_navigation> carries no <doc> elements).
- return NavResult{
- Text: `<structure_navigation count="0" error="no structure">` + "\n</structure_navigation>",
- EmptyReason: ReasonNoStructure,
- }, agg
- }
- esc := func(s string) string { return XMLEscape(s) }
- parts := append([]string{
- fmt.Sprintf(`<structure_navigation count="%d" query="%s" kind="%s">`, len(segs), esc(query), esc(kind)),
- }, segs...)
- parts = append(parts, "</structure_navigation>")
- return NavResult{
- Text: strings.Join(parts, "\n"),
- DocIDs: docIDsSeen,
- Entities: entities,
- ChunkPaths: agg.chunkPaths,
- EmptyReason: "",
- }, agg
-}
-
-// entityMaps renders drill nodes back to the generic maps the orchestrator's
-// NavResult.Entities slice carries.
-func entityMaps(nodes []structureNode) []map[string]any {
- out := make([]map[string]any, 0, len(nodes))
- for _, n := range nodes {
- out = append(out, map[string]any{
- "name": n.name,
- "type": n.nodeType,
- "description": n.desc,
- "source_chunk_ids": n.sourceChunkIDs,
- })
- }
- return out
-}
diff --git a/internal/rag/agentic-rag/runtime/tool_navigation_test.go b/internal/rag/agentic-rag/runtime/tool_navigation_test.go
deleted file mode 100644
index 99e59cb050..0000000000
--- a/internal/rag/agentic-rag/runtime/tool_navigation_test.go
+++ /dev/null
@@ -1,1149 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "errors"
- "reflect"
- "strings"
- "testing"
-)
-
-// Compiled-structure rendering / normalization
-
-func TestNormalizeKind(t *testing.T) {
- cases := []struct {
- row map[string]interface{}
- want string
- }{
- {map[string]interface{}{"compilation_template_kind_kwd": "page_index"}, "timeline"},
- {map[string]interface{}{"compilation_template_kind_kwd": "knowledge_graph"}, "timeline"},
- {map[string]interface{}{"compilation_template_kind_kwd": "mindmap"}, "mindmap"},
- {map[string]interface{}{"compile_kwd": "tree"}, "tree"},
- }
- for _, c := range cases {
- if got := normalizeKind(c.row); got != c.want {
- t.Errorf("normalizeKind(%v) = %q, want %q", c.row, got, c.want)
- }
- }
-}
-
-func containsAll(s string, subs ...string) bool {
- for _, sub := range subs {
- if !strings.Contains(s, sub) {
- return false
- }
- }
- return true
-}
-
-// navigate_tree routing
-
-type stubRouter struct {
- table [][2]string
- nilResult bool
- emptyResult bool
- err error
- gotQuery string
- gotDocScope []string
- gotTopK int
-}
-
-func (s *stubRouter) Route(_ context.Context, _, _, query string, docScope []string, topK int) ([][2]string, error) {
- s.gotQuery = query
- s.gotDocScope = docScope
- s.gotTopK = topK
- if s.err != nil {
- return nil, s.err
- }
- if s.nilResult {
- return nil, nil
- }
- if s.emptyResult {
- return [][2]string{}, nil
- }
- return s.table, nil
-}
-
-func TestNavigateTreeRoutesAndRendersXML(t *testing.T) {
- router := &stubRouter{table: [][2]string{
- {"doc-c", "Culdcept history and design"},
- {"doc-a", "OmiyaSoft company profile"},
- {"doc-b", ""},
- }}
- got := NavigateTree(context.Background(), router, NavTreeInput{
- Query: "Who created Culdcept?",
- Keywords: "Culdcept, creator",
- KbIDs: []string{"kb1"},
- })
- if got.EmptyReason != "" {
- t.Fatalf("empty_reason = %q, want a successful route", got.EmptyReason)
- }
- if len(got.DocIDs) != 3 {
- t.Fatalf("doc_ids = %v, want 3", got.DocIDs)
- }
- if len(got.RoutedDocs) != 3 {
- t.Fatalf("routed_docs = %v, want 3", got.RoutedDocs)
- }
- if got.RoutedDocs[0][1] != "Culdcept history and design" {
- t.Errorf("routed summary = %q", got.RoutedDocs[0][1])
- }
- if !strings.Contains(router.gotQuery, "Culdcept, creator") {
- t.Errorf("routing query = %q, want topic + keywords", router.gotQuery)
- }
- if !strings.Contains(got.Text, `count="3"`) {
- t.Errorf("XML missing the count: %s", got.Text)
- }
- if !strings.Contains(got.Text, `doc_id="doc-c"`) {
- t.Errorf("XML missing doc ids: %s", got.Text)
- }
- if !strings.Contains(got.Text, "<summary>Culdcept history and design</summary>") {
- t.Errorf("XML missing the summary: %s", got.Text)
- }
- if !strings.Contains(got.Text, `doc_id="doc-b"/>`) {
- t.Errorf("summary-less doc must be self-closing: %s", got.Text)
- }
- if !strings.HasSuffix(strings.TrimSpace(got.Text), "</tree_navigation>") {
- t.Errorf("XML not closed: %s", got.Text)
- }
-}
-
-func TestNavigateTreeDistinguishesNoStructureFromNoDoc(t *testing.T) {
- got := NavigateTree(context.Background(), &stubRouter{nilResult: true}, NavTreeInput{
- Query: "q", KbIDs: []string{"kb1"},
- })
- if got.EmptyReason != ReasonNoStructure {
- t.Errorf("no tree: reason = %q, want %q", got.EmptyReason, ReasonNoStructure)
- }
- if got.HasStructure() {
- t.Error("no tree must report HasStructure=false")
- }
- got = NavigateTree(context.Background(), &stubRouter{emptyResult: true}, NavTreeInput{
- Query: "q", KbIDs: []string{"kb1"},
- })
- if got.EmptyReason != ReasonNoDoc {
- t.Errorf("routed to nothing: reason = %q, want %q", got.EmptyReason, ReasonNoDoc)
- }
- if !got.HasStructure() {
- t.Error("a query-level miss must still report HasStructure=true")
- }
-}
-
-func TestNavigateTreeBadArgsAndInfra(t *testing.T) {
- if got := NavigateTree(context.Background(), &stubRouter{}, NavTreeInput{}); got.EmptyReason != ReasonBadArgs {
- t.Errorf("no query: reason = %q, want bad_args", got.EmptyReason)
- }
- if got := NavigateTree(context.Background(), nil, NavTreeInput{Query: "q"}); got.EmptyReason != ReasonInfra {
- t.Errorf("no router: reason = %q, want infra", got.EmptyReason)
- }
- // Every configured dataset failed to answer: infra, NOT no_structure. Route()
- // reports "no compiled tree" as (nil, nil), so a non-nil error is a failure —
- // a dataset-level verdict drawn from zero successful reads would mislabel an
- // outage, and no_structure is the verdict the session strikes a tool out on.
- got := NavigateTree(context.Background(), &stubRouter{err: errors.New("ES down")}, NavTreeInput{
- Query: "q", KbIDs: []string{"kb1"},
- })
- if got.EmptyReason != ReasonInfra {
- t.Errorf("backend error: reason = %q, want infra", got.EmptyReason)
- }
- if !got.HasStructure() {
- t.Error("an outage must not read as a structure absence (HasStructure must stay true)")
- }
-}
-
-// perKBRouter answers each dataset differently, so a partially failed fan-out can
-// be exercised; stubRouter answers the same for every dataset.
-type perKBRouter struct {
- results map[string][][2]string
- errs map[string]error
-}
-
-func (r *perKBRouter) Route(_ context.Context, _, kbID, _ string, _ []string, _ int) ([][2]string, error) {
- if err := r.errs[kbID]; err != nil {
- return nil, err
- }
- return r.results[kbID], nil
-}
-
-// TestNavigateTreePartialFailureIsInfra pins the mixed fan-out: one dataset fails
-// while another legitimately reports no compiled tree. The failed read leaves the
-// datasets' state unknown, so the answer must stay infra — concluding
-// no_structure there would end the tool on evidence that was never read.
-func TestNavigateTreePartialFailureIsInfra(t *testing.T) {
- got := NavigateTree(context.Background(), &perKBRouter{
- results: map[string][][2]string{"kb2": nil}, // kb2 answered: no compiled tree
- errs: map[string]error{"kb1": errors.New("ES down")},
- }, NavTreeInput{Query: "q", KbIDs: []string{"kb1", "kb2"}})
- if got.EmptyReason != ReasonInfra {
- t.Errorf("partial failure: reason = %q, want infra", got.EmptyReason)
- }
- // A failed descent has no upstream counterpart to copy — the exception escapes upstream
- // and there is no empty_reason for it — so this label is this file's own wording; only
- // the XML shape is fixed.
- if want := "<tree_navigation count=\"0\" error=\"nav tree descent failed\">\n</tree_navigation>"; got.Text != want {
- t.Errorf("partial failure: text = %q, want %q", got.Text, want)
- }
-
- // A dataset that answered AND routed still wins: one failed read must not
- // discard the documents another dataset returned.
- got = NavigateTree(context.Background(), &perKBRouter{
- results: map[string][][2]string{"kb2": [][2]string{{"doc-a", "summary"}}},
- errs: map[string]error{"kb1": errors.New("ES down")},
- }, NavTreeInput{Query: "q", KbIDs: []string{"kb1", "kb2"}})
- if got.EmptyReason != "" || len(got.DocIDs) != 1 || got.DocIDs[0] != "doc-a" {
- t.Errorf("partial success: reason = %q, doc_ids = %v, want doc-a from kb2", got.EmptyReason, got.DocIDs)
- }
-}
-
-// TestNavigateTreeEmptyXML pins the text each empty NavResult carries: `count="0"` plus an
-// `error="..."` attribute that is OMITTED for a query-level miss (present for "no retriever"
-// and "query is required"). The model does not read it — every empty_reason is replaced with
-// the tool's own note (tool_executor.go) — but the NavResult value must match.
-func TestNavigateTreeEmptyXML(t *testing.T) {
- cases := []struct {
- name string
- got NavResult
- want string
- }{
- {
- "bad_args",
- NavigateTree(context.Background(), &stubRouter{}, NavTreeInput{}),
- "<tree_navigation count=\"0\" error=\"query is required\">\n</tree_navigation>",
- },
- {
- "infra: no router",
- NavigateTree(context.Background(), nil, NavTreeInput{Query: "q"}),
- "<tree_navigation count=\"0\" error=\"no retriever\">\n</tree_navigation>",
- },
- {
- "no_doc: no attribute",
- NavigateTree(context.Background(), &stubRouter{emptyResult: true}, NavTreeInput{
- Query: "q", KbIDs: []string{"kb1"},
- }),
- "<tree_navigation count=\"0\">\n</tree_navigation>",
- },
- {
- // A state with no upstream equivalent: the nav-tree route has no
- // no_structure verdict, so this label is this file's own wording.
- "no_structure",
- NavigateTree(context.Background(), &stubRouter{nilResult: true}, NavTreeInput{
- Query: "q", KbIDs: []string{"kb1"},
- }),
- "<tree_navigation count=\"0\" error=\"no compiled navigation tree\">\n</tree_navigation>",
- },
- }
- for _, c := range cases {
- if c.got.Text != c.want {
- t.Errorf("%s: text = %q, want %q", c.name, c.got.Text, c.want)
- }
- }
-}
-
-func TestNavigateTreeCachesBestPerDocAcrossDatasets(t *testing.T) {
- router := &stubRouter{table: [][2]string{
- {"doc-a", "summary from kb1"},
- {"doc-a", "summary from kb2"},
- {"doc-b", "b"},
- }}
- got := NavigateTree(context.Background(), router, NavTreeInput{
- Query: "q", KbIDs: []string{"kb1", "kb2"},
- })
- if len(got.DocIDs) != 2 {
- t.Fatalf("doc_ids = %v, want 2 (deduped)", got.DocIDs)
- }
-}
-
-func TestNavigateTreeCapsAt8(t *testing.T) {
- var table [][2]string
- for i := 0; i < 20; i++ {
- table = append(table, [2]string{string(rune('a'+i%26)) + string(rune('0'+i/26)), "s"})
- }
- got := NavigateTree(context.Background(), &stubRouter{table: table}, NavTreeInput{
- Query: "q", KbIDs: []string{"kb1"},
- })
- if len(got.DocIDs) != navTreeMaxDocs {
- t.Errorf("doc_ids = %d, want capped at %d", len(got.DocIDs), navTreeMaxDocs)
- }
-}
-
-func TestNavigateTreeEscapesXML(t *testing.T) {
- router := &stubRouter{table: [][2]string{{"d&<>\"'1", "a < b & c"}}}
- got := NavigateTree(context.Background(), router, NavTreeInput{
- Query: `who "made" <it> & that`, KbIDs: []string{"kb1"},
- })
- if strings.Contains(got.Text, "<it>") {
- t.Errorf("query not escaped: %s", got.Text)
- }
- if !strings.Contains(got.Text, "&lt;it&gt;") {
- t.Errorf("query not escaped: %s", got.Text)
- }
- if strings.Contains(got.Text, `doc_id="d&<>\"'1"`) {
- t.Errorf("doc_id not escaped: %s", got.Text)
- }
-}
-
-// Compiled-structure parsing
-
-func TestParseCompiledStructureGraphBlob(t *testing.T) {
- rows := []StructureRow{{
- CompileKwd: "tree",
- KnowledgeGraphKwd: "graph",
- Content: `{"entities": [{"name": "OmiyaSoft"}, {"name": "Culdcept"}], "relations": [{"src": "OmiyaSoft", "dst": "Culdcept"}]}`,
- }}
- ents, rels := ParseCompiledStructure(rows, []string{"tree"})
- if len(ents) != 2 || len(rels) != 1 {
- t.Fatalf("blob: entities=%d relations=%d, want 2/1", len(ents), len(rels))
- }
-}
-
-func TestParseCompiledStructurePerEntityRows(t *testing.T) {
- rows := []StructureRow{
- {CompileKwd: "page_index", KnowledgeGraphKwd: "entity", Content: `{"name": "OmiyaSoft"}`},
- {CompileKwd: "page_index", KnowledgeGraphKwd: "entity", Content: `{"name": "Culdcept"}`},
- {CompileKwd: "page_index", KnowledgeGraphKwd: "relation", Content: `{"src": "OmiyaSoft", "dst": "Culdcept"}`},
- }
- ents, rels := ParseCompiledStructure(rows, []string{"timeline"})
- if len(ents) != 2 || len(rels) != 1 {
- t.Fatalf("per-row: entities=%d relations=%d, want 2/1", len(ents), len(rels))
- }
-}
-
-func TestParseCompiledStructureKindFilterAndNormalization(t *testing.T) {
- rows := []StructureRow{
- {CompileKwd: "page_index", KnowledgeGraphKwd: "entity", Content: `{"name": "A"}`},
- {CompileKwd: "knowledge_graph", KnowledgeGraphKwd: "entity", Content: `{"name": "B"}`},
- {CompileKwd: "tree", KnowledgeGraphKwd: "entity", Content: `{"name": "C"}`},
- }
- ents, _ := ParseCompiledStructure(rows, []string{"timeline"})
- if len(ents) != 2 {
- t.Errorf("timeline filter = %d entities, want 2 (page_index + knowledge_graph)", len(ents))
- }
- ents, _ = ParseCompiledStructure([]StructureRow{
- {CompileKwd: "", TemplateKind: "tree", KnowledgeGraphKwd: "entity", Content: `{"name": "E"}`},
- }, []string{"tree"})
- if len(ents) != 1 {
- t.Errorf("template_kind fallback = %d, want 1", len(ents))
- }
-}
-
-func TestParseCompiledStructureSkipsMalformed(t *testing.T) {
- rows := []StructureRow{
- {CompileKwd: "tree", KnowledgeGraphKwd: "entity", Content: "not json"},
- {CompileKwd: "tree", KnowledgeGraphKwd: "entity", Content: `[1,2,3]`},
- {CompileKwd: "tree", KnowledgeGraphKwd: "unknown_shape", Content: `{"name":"X"}`},
- {CompileKwd: "tree", KnowledgeGraphKwd: "entity", Content: `{"name": "ok"}`},
- }
- ents, _ := ParseCompiledStructure(rows, []string{"tree"})
- if len(ents) != 1 {
- t.Errorf("entities = %d, want 1 (malformed rows skipped)", len(ents))
- }
-}
-
-func TestParseCompiledStructureEmptyKindsMatchesAll(t *testing.T) {
- rows := []StructureRow{
- {CompileKwd: "tree", KnowledgeGraphKwd: "entity", Content: `{"name": "A"}`},
- {CompileKwd: "other", KnowledgeGraphKwd: "entity", Content: `{"name": "B"}`},
- }
- ents, _ := ParseCompiledStructure(rows, nil)
- if len(ents) != 2 {
- t.Errorf("no filter = %d entities, want 2", len(ents))
- }
-}
-
-// Knowledge-graph exploration
-
-// TestExploreGraphShortCircuitsOnEmptyInput verifies the documented contract:
-// no query text or no bound datasets yields an empty ExploreResult (no engine
-// call). The engine-backed happy path needs a live Infinity backend and is not
-// exercised in the unit tier.
-func TestExploreGraphShortCircuitsOnEmptyInput(t *testing.T) {
- // No query text.
- if res, err := ExploreGraph(context.Background(), SearchDeps{}, "t1", []string{"kb1"}, "", "", nil); err != nil || res.Answer != "" || len(res.Chunks) != 0 {
- t.Errorf("empty query -> (%+v, %v), want empty result", res, err)
- }
- // No bound datasets.
- if res, err := ExploreGraph(context.Background(), SearchDeps{}, "t1", nil, "OmiyaSoft", "", nil); err != nil || res.Answer != "" || len(res.Chunks) != 0 {
- t.Errorf("no datasets -> (%+v, %v), want empty result", res, err)
- }
-}
-
-// TestExploreGraphUnconfiguredEngineErrors verifies a missing engine is
-// surfaced rather than silently returning empty.
-func TestExploreGraphUnconfiguredEngineErrors(t *testing.T) {
- if _, err := ExploreGraph(context.Background(), SearchDeps{}, "t1", []string{"kb1"}, "OmiyaSoft", "", nil); err == nil {
- t.Error("expected an error when the engine is not configured")
- }
-}
-
-// stubTenantResolver is a DocTenantResolver test double that assigns a fixed
-// (kb, tenant) to every document id it is asked about.
-type stubTenantResolver struct {
- owner DocTenant
-}
-
-func (s stubTenantResolver) ResolveDocTenants(ctx context.Context, docIDs []string) (map[string]DocTenant, error) {
- out := make(map[string]DocTenant, len(docIDs))
- for _, d := range docIDs {
- out[d] = s.owner
- }
- return out, nil
-}
-
-// TestResolveKGScopeGroupsByOwner: documents are
-// grouped by their real owning (kb, tenant) — which may be a KB outside the
-// caller's datasetIDs — rather than every bound dataset being searched with
-// the whole DocScope.
-func TestResolveKGScopeGroupsByOwner(t *testing.T) {
- // No DocScope: one whole-dataset group per bound dataset.
- got := resolveKGScope(SearchDeps{TenantID: "t1"}, nil, []string{"kb1", "kb2"})
- if len(got) != 2 {
- t.Fatalf("no-docScope scopes = %d, want 2", len(got))
- }
- for _, s := range got {
- if len(s.Docs) != 0 {
- t.Errorf("scope %s has Docs=%v, want nil (whole-dataset)", s.KBID, s.Docs)
- }
- }
-
- // DocScope + resolver that maps every doc to a NEW kb outside datasetIDs.
- got = resolveKGScope(SearchDeps{
- TenantID: "t1",
- DocTenantResolver: stubTenantResolver{owner: DocTenant{KBID: "otherKB", TenantID: "t9"}},
- }, []string{"d1", "d2"}, []string{"kb1"})
- if len(got) != 1 {
- t.Fatalf("resolver scopes = %d, want 1 (new kb from resolver)", len(got))
- }
- if got[0].KBID != "otherKB" || got[0].TenantID != "t9" {
- t.Errorf("scope = %+v, want KBID=otherKB TenantID=t9", got[0])
- }
- if len(got[0].Docs) != 2 {
- t.Errorf("scope Docs = %v, want [d1 d2]", got[0].Docs)
- }
-
- // DocScope but NO resolver: fall back to old behaviour (each bound dataset
- // searched with the whole DocScope).
- got = resolveKGScope(SearchDeps{TenantID: "t1"}, []string{"d1"}, []string{"kb1", "kb2"})
- if len(got) != 2 {
- t.Fatalf("no-resolver scopes = %d, want 2", len(got))
- }
- for _, s := range got {
- if len(s.Docs) != 1 || s.Docs[0] != "d1" {
- t.Errorf("scope %s Docs=%v, want [d1]", s.KBID, s.Docs)
- }
- }
-}
-
-// TestResolveKGScopeAppliesSessionDocScopeCeiling verifies the session ceiling: the session
-// doc_scope restricts whatever scope the caller passed before ownership resolution.
-func TestResolveKGScopeAppliesSessionDocScopeCeiling(t *testing.T) {
- got := resolveKGScope(SearchDeps{
- TenantID: "t1",
- DocScope: []string{"d1", "d2"}, // session ceiling
- DocTenantResolver: stubTenantResolver{owner: DocTenant{KBID: "kb1", TenantID: "t1"}},
- }, []string{"d1", "drop", "d2"}, []string{"kb1"})
- if len(got) != 1 || len(got[0].Docs) != 2 {
- t.Fatalf("after ceiling scopes = %+v, want 1 scope with [d1 d2]", got)
- }
- if got[0].Docs[0] != "d1" || got[0].Docs[1] != "d2" {
- t.Errorf("docs = %v, want [d1 d2] (drop is outside the session scope)", got[0].Docs)
- }
-}
-
-// TestResolveKGScopeNonNilEmptyExpandsNothing pins the resolveDocScope "match
-// nothing" signal: a non-nil empty scope must yield no search groups at all
-// (not a whole-dataset scan).
-func TestResolveKGScopeNonNilEmptyExpandsNothing(t *testing.T) {
- if got := resolveKGScope(SearchDeps{TenantID: "t1"}, []string{}, []string{"kb1", "kb2"}); len(got) != 0 {
- t.Fatalf("scopes = %+v, want none for an explicit empty (match nothing) scope", got)
- }
-}
-
-// Knowledge-graph row parsing
-
-func TestKgParseEntity_NameFallbacks(t *testing.T) {
- // name wins.
- if e, ok := kgParseEntity(map[string]interface{}{"content_with_weight": `{"name":"OmiyaSoft"}`}); !ok || e.Name != "OmiyaSoft" {
- t.Fatalf("name parse = %+v ok=%v", e, ok)
- }
- // term fallback.
- if e, ok := kgParseEntity(map[string]interface{}{"content_with_weight": `{"term":"Culdcept"}`}); !ok || e.Name != "Culdcept" {
- t.Fatalf("term fallback = %+v ok=%v", e, ok)
- }
- // title fallback.
- if e, ok := kgParseEntity(map[string]interface{}{"content_with_weight": `{"title":"Rocket"}`}); !ok || e.Name != "Rocket" {
- t.Fatalf("title fallback = %+v ok=%v", e, ok)
- }
- // aliases parsed + doc_id carried.
- row := map[string]interface{}{
- "content_with_weight": `{"name":"OmiyaSoft","type":"company","description":"dev","aliases":["Omiya","OS"]}`,
- "doc_id": "d1",
- "source_chunk_ids": []interface{}{"c1", "c2"},
- }
- e, ok := kgParseEntity(row)
- if !ok || e.Name != "OmiyaSoft" || e.Type != "company" || e.DocID != "d1" {
- t.Fatalf("entity = %+v ok=%v", e, ok)
- }
- if len(e.Aliases) != 2 || e.Aliases[0] != "Omiya" {
- t.Errorf("aliases = %v, want [Omiya OS]", e.Aliases)
- }
- if len(e.SourceChunkIDs) != 2 {
- t.Errorf("source_chunk_ids = %v, want [c1 c2]", e.SourceChunkIDs)
- }
-}
-
-func TestKgParseEntityRejectsEmptyName(t *testing.T) {
- cases := []string{
- `{"type":"x"}`, // no name
- `{"name":""}`, // empty name
- `{"name":null}`, // null name
- `not json`, // unparsable
- `[1,2,3]`, // not an object
- }
- for _, c := range cases {
- if _, ok := kgParseEntity(map[string]interface{}{"content_with_weight": c}); ok {
- t.Errorf("content %q should not parse to an entity", c)
- }
- }
-}
-
-func TestKgParseRelationEndpointsRequired(t *testing.T) {
- // Missing from endpoint.
- if _, ok := kgParseRelation(map[string]interface{}{"to_entity_kwd": "B"}); ok {
- t.Error("relation without from endpoint must be rejected")
- }
- // Missing to endpoint.
- if _, ok := kgParseRelation(map[string]interface{}{"from_entity_kwd": "A"}); ok {
- t.Error("relation without to endpoint must be rejected")
- }
- // Valid: type from content payload takes precedence over default.
- row := map[string]interface{}{
- "from_entity_kwd": "A",
- "to_entity_kwd": "B",
- "content_with_weight": `{"type":"founded"}`,
- "doc_id": "d1",
- }
- r, ok := kgParseRelation(row)
- if !ok || r.From != "A" || r.To != "B" || r.Type != "founded" || r.DocID != "d1" {
- t.Fatalf("relation = %+v ok=%v", r, ok)
- }
- // Default type when payload has neither type nor relation.
- r2, ok := kgParseRelation(map[string]interface{}{
- "from_entity_kwd": "A", "to_entity_kwd": "B",
- "content_with_weight": `{}`,
- })
- if !ok || r2.Type != "related" {
- t.Errorf("default relation type = %q, want related", r2.Type)
- }
-}
-
-// endpoint terms (original + lowercased, deduped)
-
-func TestEndpointTerms(t *testing.T) {
- got := endpointTerms([]string{"OmiyaSoft", "omiya", "", "Culdcept"})
- want := map[string]bool{
- "OmiyaSoft": true, "omiyasoft": true, "omiya": true, "Culdcept": true, "culdcept": true,
- }
- if len(got) != len(want) {
- t.Fatalf("endpointTerms = %v, want %v", got, want)
- }
- for _, g := range got {
- if !want[g] {
- t.Errorf("unexpected term %q", g)
- }
- }
-}
-
-// Evidence collection: relevant entities + relations grouped by doc
-
-func TestCollectEvidenceIDsByDocAndAlias(t *testing.T) {
- entities := []kgEntity{
- {Name: "OmiyaSoft", Aliases: []string{"Omiya"}, DocID: "d1", SourceChunkIDs: []string{"c1", "c2"}},
- {Name: "Culdcept", DocID: "d2", SourceChunkIDs: []string{"c3"}},
- }
- relations := []kgRelation{
- {From: "OmiyaSoft", To: "Culdcept", DocID: "d1", SourceChunkIDs: []string{"c4"}},
- }
- // Relevant set includes an alias ("Omiya") and the relation target ("Culdcept").
- byDoc := collectEvidenceIDs(entities, relations, []string{"Omiya", "Culdcept"})
- // Doc order is FIRST-SEEN (the order the docs were reached), d1 before d2.
- if len(byDoc) != 2 || byDoc[0].DocID != "d1" || byDoc[1].DocID != "d2" {
- t.Fatalf("doc order = %v, want [d1 d2]", byDoc)
- }
- if got := byDoc[0].IDs; len(got) != 3 || got[0] != "c1" || got[1] != "c2" || got[2] != "c4" {
- t.Errorf("d1 chunks = %v, want [c1 c2 c4]", got)
- }
- if got := byDoc[1].IDs; len(got) != 1 || got[0] != "c3" {
- t.Errorf("d2 chunks = %v, want [c3]", got)
- }
- // A non-relevant entity must contribute nothing.
- byDoc = collectEvidenceIDs(entities, relations, []string{"nobody"})
- if len(byDoc) != 0 {
- t.Errorf("non-relevant selection = %v, want empty", byDoc)
- }
-}
-
-func TestCollectEvidenceIDsDeduplicatesChunks(t *testing.T) {
- entities := []kgEntity{
- {Name: "A", DocID: "d1", SourceChunkIDs: []string{"c1", "c1"}},
- }
- rel := []kgRelation{
- {From: "A", To: "B", DocID: "d1", SourceChunkIDs: []string{"c1"}},
- }
- byDoc := collectEvidenceIDs(entities, rel, []string{"A", "B"})
- if len(byDoc) != 1 || byDoc[0].DocID != "d1" {
- t.Fatalf("byDoc = %v, want a single d1 group", byDoc)
- }
- seen := map[string]int{}
- for _, c := range byDoc[0].IDs {
- seen[c]++
- }
- if seen["c1"] != 1 {
- t.Errorf("chunk c1 counted %d times, want 1 (deduped)", seen["c1"])
- }
-}
-
-// mention_count re-ranking (the dense-seed re-sort)
-
-func TestMentionCountTypeCoercion(t *testing.T) {
- cases := []struct {
- in interface{}
- want int
- }{
- {int(5), 5},
- {int64(7), 7},
- {float64(9), 9},
- {float32(3), 3},
- {nil, 0},
- {"x", 0},
- }
- for _, c := range cases {
- if got := mentionCount(map[string]interface{}{"mention_count_int": c.in}); got != c.want {
- t.Errorf("mentionCount(%T(%v)) = %d, want %d", c.in, c.in, got, c.want)
- }
- }
-}
-
-func TestTopMentionCountSortsAndCaps(t *testing.T) {
- rows := []map[string]interface{}{
- {"mention_count_int": 1, "name": "a"},
- {"mention_count_int": 10, "name": "b"},
- {"mention_count_int": 5, "name": "c"},
- }
- got := topMentionCount(rows, 2)
- if len(got) != 2 || mentionCount(got[0]) != 10 || mentionCount(got[1]) != 5 {
- t.Fatalf("topMentionCount = %v, want [10 5]", got)
- }
-}
-
-// String-field helpers
-
-func TestStrSliceField(t *testing.T) {
- if got := strSliceField([]string{"a", "b"}); len(got) != 2 {
- t.Errorf("[]string path lost elements: %v", got)
- }
- if got := strSliceField([]interface{}{"a", 2, "b"}); len(got) != 2 {
- t.Errorf("[]interface{} path must skip non-strings: %v", got)
- }
- if got := strSliceField(42); got != nil {
- t.Errorf("scalar path must yield nil, got %v", got)
- }
-}
-
-func TestStrOr(t *testing.T) {
- if strOr("x", "def") != "x" {
- t.Error("non-empty value must win")
- }
- if strOr("", "def") != "def" {
- t.Error("empty value must fall back")
- }
- if strOr(nil, "def") != "def" {
- t.Error("non-string must fall back")
- }
-}
-
-// TestBuildTocTreeHierarchyFromRelations guards the navigate_structure relations
-// wiring: when relations are loaded the tree must expose the parent→child hierarchy and a
-// single root, instead of collapsing to isolated roots the way an empty/nil relation list
-// does.
-func TestBuildTocTreeHierarchyFromRelations(t *testing.T) {
- nodes := []structureNode{
- {name: "root"},
- {name: "A"},
- {name: "A1"},
- {name: "A2"},
- }
- rels := []structureRel{
- {from: "root", to: "A"},
- {from: "A", to: "A1"},
- {from: "A", to: "A2"},
- }
- byName, children, parents, roots := buildTocTree(nodes, rels)
- if len(byName) != 4 {
- t.Fatalf("byName = %d, want 4", len(byName))
- }
- if len(roots) != 1 || roots[0] != "root" {
- t.Fatalf("roots = %v, want single root \"root\"", roots)
- }
- if parents["A"] != "root" || parents["A1"] != "A" || parents["A2"] != "A" {
- t.Fatalf("parents = %v, want A->root and A1/A2->A", parents)
- }
- if len(children["A"]) != 2 {
- t.Fatalf("children[A] = %v, want [A1 A2]", children["A"])
- }
-}
-
-// TestBuildTocTreeNoRelationsYieldsIsolatedRoots documents the pre-fix shape: a
-// nil/empty relation list leaves every node without a parent, so each node is its
-// own root and no parent→child hierarchy can be descended.
-func TestBuildTocTreeNoRelationsYieldsIsolatedRoots(t *testing.T) {
- nodes := []structureNode{
- {name: "root"},
- {name: "A"},
- {name: "A1"},
- }
- _, _, _, roots := buildTocTree(nodes, nil)
- if len(roots) != 3 {
- t.Fatalf("roots without relations = %v, want all 3 isolated", roots)
- }
-}
-
-// TestStructureGraphFromRawMissingFieldsStayEmpty pins the compiled-payload
-// defaults: a field a compiled row simply does not carry must read as absent, so
-// the renderers apply their defaults ("other" for a type, "related_to" for a
-// relation type) and a relation with a missing endpoint is DROPPED. fmt.Sprint(nil)
-// returns the non-empty string "<nil>", which defeated all three: the model read
-// "- Name (<nil>): <nil>" and the tree grew a phantom edge to "<nil>".
-func TestStructureGraphFromRawMissingFieldsStayEmpty(t *testing.T) {
- rawEntities := []map[string]any{
- {"name": "Bare"}, // no type, no description
- {"name": "Typed", "type": "PERSON"}, // no description
- {"name": " ", "type": "PERSON"}, // nameless rows are dropped
- }
- rawRels := []map[string]any{
- {"from": "A", "to": "B"}, // no type
- {"from": "A"}, // missing endpoint -> dropped
- {"to": "B"}, // missing endpoint -> dropped
- {"from": "A", "to": "A"}, // self-loop -> dropped
- }
- nodes, rels := structureGraphFromRaw(rawEntities, rawRels)
-
- if len(nodes) != 2 {
- t.Fatalf("entities = %+v, want 2 (the nameless row dropped)", nodes)
- }
- if nodes[0].Type != "" || nodes[0].Description != "" {
- t.Errorf("missing fields = %q/%q, want empty so the renderer defaults the type to \"other\"",
- nodes[0].Type, nodes[0].Description)
- }
- if nodes[1].Type != "PERSON" || nodes[1].Description != "" {
- t.Errorf("entity = %+v, want PERSON with an empty description", nodes[1])
- }
- if len(rels) != 1 {
- t.Errorf("relations = %+v, want only the complete one", rels)
- }
- if rels[0].relType != "" {
- t.Errorf("relation type = %q, want empty so the renderer defaults it to \"related_to\"", rels[0].relType)
- }
-
- // End to end: the outline the model reads must show the defaults, not "<nil>" and not
- // the dropped relations.
- outline := renderOutline(structureNodesFromEntities(nodes), rels)
- if strings.Contains(outline, "<nil>") {
- t.Fatalf("outline leaks <nil>:\n%s", outline)
- }
- if !strings.Contains(outline, "- Bare (other)") {
- t.Errorf("outline = %q, want the missing type defaulted to \"other\"", outline)
- }
- if !strings.Contains(outline, "- A -[related_to]-> B") {
- t.Errorf("outline = %q, want the missing relation type defaulted to \"related_to\"", outline)
- }
- if got := strings.Count(outline, "->"); got != 1 {
- t.Errorf("outline has %d relation line(s), want 1 (missing endpoints and self-loops dropped):\n%s", got, outline)
- }
-}
-
-// TestStructureDocSegmentFormat locks the per-doc XML shape navigateStructures renders for
-// each routed/requested document: the <doc rank doc_id doc_title="" entities relations>
-// element, where doc_title is always empty.
-func TestStructureDocSegmentFormat(t *testing.T) {
- nodes := []structureNode{{name: "N1"}, {name: "N2"}}
- rels := []structureRel{{from: "N0", to: "N1"}}
- drill := structureDrillout{outline: "· N1 [chunks: c1]"}
-
- seg := structureDocSegment("docA", "q", "catalog", 2, nodes, rels, drill)
- want := ` <doc rank="2" doc_id="docA" doc_title="" entities="2" relations="1">`
- if !strings.Contains(seg, want) {
- t.Fatalf("segment missing header %q:\n%s", want, seg)
- }
- if !strings.Contains(seg, "\n <structure>· N1 [chunks: c1]</structure>") {
- t.Fatalf("segment missing structure outline:\n%s", seg)
- }
- if !strings.Contains(seg, "\n </doc>") {
- t.Fatalf("segment missing </doc> close:\n%s", seg)
- }
-}
-
-// TestEmptyReasonLabel verifies the empty_reason→XML error label mapping used by
-// the single- and multi-document renderers.
-func TestEmptyReasonLabel(t *testing.T) {
- cases := map[string]string{
- ReasonBadArgs: "query is required",
- ReasonNoDoc: "no document located",
- ReasonNoStructure: "no structure",
- }
- for in, want := range cases {
- if got := emptyReasonLabel(in); got != want {
- t.Errorf("emptyReasonLabel(%q) = %q, want %q", in, got, want)
- }
- }
-}
-
-func TestVecsEqual(t *testing.T) {
- if !vecsEqual([]float64{1, 2, 3}, []float64{1, 2, 3}) {
- t.Error("identical vectors should be equal")
- }
- if vecsEqual([]float64{1, 2, 3}, []float64{1, 2, 4}) {
- t.Error("differing vectors should not be equal")
- }
- if vecsEqual([]float64{1, 2}, []float64{1, 2, 3}) {
- t.Error("differently-sized vectors should not be equal")
- }
- if vecsEqual(nil, []float64{1}) {
- t.Error("empty vs non-empty should not be equal")
- }
-}
-
-func TestHasDistinctNodeVectors(t *testing.T) {
- if hasDistinctNodeVectors([]structureNode{{name: "a"}, {name: "b"}}) {
- t.Error("no vectors at all -> not distinct")
- }
- if hasDistinctNodeVectors([]structureNode{{name: "a", vec: []float64{1, 2}}}) {
- t.Error("a single vector-bearing node -> not distinct")
- }
- // RAPTOR: every node inherits the same blob vector.
- if hasDistinctNodeVectors([]structureNode{
- {name: "a", vec: []float64{1, 2}},
- {name: "b", vec: []float64{1, 2}},
- }) {
- t.Error("all nodes sharing one vector -> not distinct")
- }
- // page_index: per-node vectors differ.
- if !hasDistinctNodeVectors([]structureNode{
- {name: "a", vec: []float64{1, 2}},
- {name: "b", vec: []float64{3, 4}},
- }) {
- t.Error("distinct per-node vectors -> distinct")
- }
- // One vector-bearing among several is treated as not-distinct (fewer than two distinct).
- if hasDistinctNodeVectors([]structureNode{
- {name: "a", vec: []float64{1, 2}},
- {name: "b"},
- }) {
- t.Error("one vector-bearing among two -> not distinct")
- }
-}
-
-func TestNodesCoveringChunks(t *testing.T) {
- nodes := []structureNode{
- {name: "A", sourceChunkIDs: []string{"c1", "c2"}},
- {name: "B", sourceChunkIDs: []string{"c3"}},
- {name: "C"},
- }
- got := nodesCoveringChunks(nodes, []string{"c2", "c9"})
- if len(got) != 1 || got[0] != "A" {
- t.Errorf("covering = %v, want [A]", got)
- }
- if got := nodesCoveringChunks(nodes, nil); got != nil {
- t.Errorf("nil chunk list -> nil, got %v", got)
- }
-}
-
-func renderDrillNodes() ([]structureNode, []structureRel) {
- nodes := []structureNode{
- {name: "Root", nodeType: "tree_node", desc: "root entity", sourceChunkIDs: []string{"r1"}},
- {name: "Leaf", nodeType: "tree_node", desc: "leaf entity", sourceChunkIDs: []string{"l1"}},
- }
- rels := []structureRel{{from: "Root", to: "Leaf", relType: "contains"}}
- return nodes, rels
-}
-
-func drillLoader(ids []string) []chunkWithText {
- var out []chunkWithText
- for _, id := range ids {
- out = append(out, chunkWithText{id: id, text: "chunk content for " + id})
- }
- return out
-}
-
-func TestRenderTocDrilloutLLMSelect(t *testing.T) {
- nodes, rels := renderDrillNodes()
- out := renderTocDrilldown("some query", nil, nodes, rels, drillLoader, nil, []string{"Leaf"}, nil)
- if out.selector != "llm_toc" {
- t.Errorf("selector = %q, want llm_toc", out.selector)
- }
- if out.outline == "" {
- t.Fatal("outline should be non-empty for a selected leaf")
- }
- // The selected leaf's parent (Root) must be pulled in as an ancestor.
- if !strings.Contains(out.outline, "Root") {
- t.Errorf("ancestor Root missing from outline:\n%s", out.outline)
- }
-}
-
-func TestRenderTocDrilloutChunkRetrieval(t *testing.T) {
- nodes, rels := renderDrillNodes()
- hits := []chunkHit{{id: "r1", score: 0.9}, {id: "l1", score: 0.7}}
- out := renderTocDrilldown("q", nil, nodes, rels, drillLoader, hits, nil, nil)
- if out.selector != "chunk_retrieval" {
- t.Errorf("selector = %q, want chunk_retrieval", out.selector)
- }
- if out.topScore != 0.9 {
- t.Errorf("topScore = %v, want 0.9", out.topScore)
- }
- // Retrieved chunks are surfaced even when they are not a selected node's anchor.
- if !strings.Contains(out.outline, "r1") || !strings.Contains(out.outline, "l1") {
- t.Errorf("retrieved chunks missing from outline:\n%s", out.outline)
- }
-}
-
-func TestRenderTocDrilloutBeamFallback(t *testing.T) {
- nodes, rels := renderDrillNodes()
- // No selection and no hits, but a query: beam picks by keyword relevance. The
- // blank query yields no terms, so it must still return a (flat) outline, not panic.
- out := renderTocDrilldown("leaf", nil, nodes, rels, drillLoader, nil, nil, nil)
- if out.outline == "" {
- t.Fatal("beam path produced an empty outline")
- }
- if out.selector != "beam" {
- t.Errorf("selector = %q, want beam", out.selector)
- }
-}
-
-// stableDrillout runs call repeatedly and fails if any kept-node-derived output
-// differs between runs (Go randomises map iteration per range statement).
-func stableDrillout(t *testing.T, call func() structureDrillout) structureDrillout {
- t.Helper()
- first := call()
- for i := 1; i < 64; i++ {
- got := call()
- if got.outline != first.outline || got.chunkPtrs != first.chunkPtrs || got.nodes != first.nodes ||
- !reflect.DeepEqual(got.chunkPaths, first.chunkPaths) {
- t.Fatalf("run %d differs from run 0:\n--- run 0\n%s\n--- run %d\n%s",
- i, first.outline, i, got.outline)
- }
- }
- return first
-}
-
-// assertKeptNodeOrder requires the outline's node lines to appear in the sorted
-// order the kept names are materialised in.
-func assertKeptNodeOrder(t *testing.T, outline string) {
- t.Helper()
- prev := -1
- for _, name := range []string{"Alpha", "Beta", "Gamma", "Root"} {
- at := strings.Index(outline, "- "+name+" (")
- if at < 0 {
- continue // this strategy did not keep the node
- }
- if at < prev {
- t.Errorf("node line %q is out of sorted order:\n%s", name, outline)
- }
- prev = at
- }
-}
-
-// TestTocDrilldownKeptOrderIsDeterministic pins the kept-node ordering across all
-// three selection strategies. kept drives the rendered outline, chunkPaths (first
-// writer wins for a chunk covered by several nodes) and the capped chunk list
-// whose snippets reach the model, so an arbitrary order hands back a different
-// outline AND a different chunk subset on every run. Iterating the kept set gives an
-// arbitrary order that is not even reproducible across runs (hash-randomised per process),
-// so there is no canonical order to inherit; the names are sorted instead.
-func TestTocDrilldownKeptOrderIsDeterministic(t *testing.T) {
- nodes := []structureNode{
- {name: "Root", nodeType: "tree_node", desc: "root node", sourceChunkIDs: []string{"r1", "shared"}},
- {name: "Beta", nodeType: "tree_node", desc: "beta node", sourceChunkIDs: []string{"b1", "b2"}},
- {name: "Alpha", nodeType: "tree_node", desc: "alpha node", sourceChunkIDs: []string{"a1", "a2"}},
- {name: "Gamma", nodeType: "tree_node", desc: "gamma node", sourceChunkIDs: []string{"g1", "shared"}},
- }
- rels := []structureRel{
- // Deliberately not the order the node names sort in.
- {from: "Root", to: "Beta"},
- {from: "Root", to: "Alpha"},
- {from: "Root", to: "Gamma"},
- }
-
- t.Run("llm_toc", func(t *testing.T) {
- // The model's picks arrive in ITS order; it must not leak into the output.
- got := stableDrillout(t, func() structureDrillout {
- return renderTocDrilldown("q", nil, nodes, rels, drillLoader, nil, []string{"Gamma", "Alpha", "Beta"}, nil)
- })
- assertKeptNodeOrder(t, got.outline)
- // chunkPaths follows the same order: "shared" is covered by both Gamma and
- // Root, and sorted order visits Gamma first, so Gamma's path wins.
- want := map[string]string{
- "a1": "Root -> Alpha", "a2": "Root -> Alpha",
- "b1": "Root -> Beta", "b2": "Root -> Beta",
- "g1": "Root -> Gamma", "shared": "Root -> Gamma",
- "r1": "Root",
- }
- if !reflect.DeepEqual(got.chunkPaths, want) {
- t.Errorf("chunkPaths = %v, want %v", got.chunkPaths, want)
- }
- // Snippets are capped at structMaxChunks (=4)
- // in kept order, so "g1" (Gamma) and "r1" (Root sorts last) sit outside
- // the stable, repeatable window.
- if !strings.Contains(got.outline, "[chunk a1]") || strings.Contains(got.outline, "[chunk g1]") || strings.Contains(got.outline, "[chunk r1]") {
- t.Errorf("outline's snippet window is not the first %d chunks in kept order:\n%s",
- structMaxChunks, got.outline)
- }
- })
-
- t.Run("chunk_retrieval", func(t *testing.T) {
- got := stableDrillout(t, func() structureDrillout {
- return renderTocDrilldown("q", nil, nodes, rels, drillLoader,
- []chunkHit{{id: "g1", score: 0.9}, {id: "a1", score: 0.7}}, nil, nil)
- })
- assertKeptNodeOrder(t, got.outline)
- })
-
- t.Run("beam", func(t *testing.T) {
- got := stableDrillout(t, func() structureDrillout {
- return renderTocDrilldown("alpha beta gamma root", nil, nodes, rels, drillLoader, nil, nil, nil)
- })
- assertKeptNodeOrder(t, got.outline)
- })
-}
-
-func TestParseCompiledStructureCarriesVec(t *testing.T) {
- rows := []StructureRow{{
- CompileKwd: "tree",
- KnowledgeGraphKwd: "graph",
- Content: `{"entities": [{"name": "A"}, {"name": "B"}], "relations": []}`,
- Vec: []float64{1, 2},
- }}
- ents, _ := ParseCompiledStructure(rows, []string{"tree"})
- if len(ents) != 2 {
- t.Fatalf("entities = %d, want 2", len(ents))
- }
- // Both nested entities of the RAPTOR blob inherit its single vector.
- for _, e := range ents {
- v, _ := e["_vec"].([]float64)
- if len(v) != 2 || v[0] != 1 {
- t.Errorf("entity %v did not inherit the blob vector", e["name"])
- }
- }
-}
-
-func TestNavigationToolsReportEmptyWhenUncompiled(t *testing.T) {
- // The Go port has no compiled navigation structures, so navigate_tree /
- // navigate_structure report EMPTY/no_structure — the DATASET-level signal
- // that lets the ladder fall through to `global` and (after the strikes)
- // disable the tool, instead of looping on a rung that cannot succeed.
- kb := &Kbinfos{}
- ex := &searchExecutor{deps: SearchDeps{KB: kb}}
- // No database is wired in this unit test, so defaultSeedEncoder's nil-DB
- // guard returns nil and structure navigation falls back to keyword matching
- // instead of panicking.
-
- oc, _ := ex.Execute(context.Background(), "navigate_tree", map[string]any{"query": "topic"})
- if oc.Status != StatusEmpty || oc.Reason != ReasonNoStructure {
- t.Errorf("navigate_tree: got (%s,%s), want (empty,no_structure)", oc.Status, oc.Reason)
- }
- // A note must accompany it, so the model knows to switch tools.
- if len(oc.Payload) == 0 {
- t.Error("navigate_tree must return an actionable note, not an empty payload")
- }
-
- oc, _ = ex.Execute(context.Background(), "navigate_structure", map[string]any{"doc_id": "d1", "query": "topic"})
- if oc.Status != StatusEmpty || oc.Reason != ReasonNoStructure {
- t.Errorf("navigate_structure: got (%s,%s), want (empty,no_structure)", oc.Status, oc.Reason)
- }
-}
-
-func TestNavigateStructureDocRejectsForeignDocument(t *testing.T) {
- // A doc_id outside the bound datasets yields no structure, so nothing from the other
- // dataset is ever read.
- res, _ := navigateStructures(context.Background(), "t", "q", []string{"foreign-doc"}, "catalog", nil, SearchDeps{
- KbIDs: []string{"kb"},
- DocIDVerifier: stubVerifier{known: map[string]bool{"mine": true}},
- })
-
- if res.EmptyReason != ReasonNoStructure {
- t.Fatalf("EmptyReason = %q, want %q", res.EmptyReason, ReasonNoStructure)
- }
- if len(res.DocIDs) != 0 {
- t.Fatalf("DocIDs = %v, want none", res.DocIDs)
- }
-}
-
-// TestIndexNameForFallsBackToRagflowPrefix pins the index-name seam: the configured search
-// index name is used when set; otherwise the default is "ragflow_<tenant_id>", so an empty
-// configured name falls back to that, while a tenant override is honoured verbatim.
-func TestIndexNameForFallsBackToRagflowPrefix(t *testing.T) {
- if got := indexNameFor("tenantA", ""); got != "ragflow_tenantA" {
- t.Errorf("empty configured = %q, want ragflow_tenantA", got)
- }
- if got := indexNameFor("tenantA", "custom_idx"); got != "custom_idx" {
- t.Errorf("configured = %q, want custom_idx", got)
- }
-}
-
-// TestNavigateStructuresEmptyDocIDsRoutesViaRouter pins the empty-doc_id vector routing: a
-// direct call with no doc_ids routes through the NavTreeRouter first instead of immediately
-// returning MISS. The tool path still passes non-empty doc_ids and must NOT consult the
-// router.
-func TestNavigateStructuresEmptyDocIDsRoutesViaRouter(t *testing.T) {
- router := &stubNavRouter{docs: [][2]string{{"d1", "summary"}}}
- // Empty doc_ids + router => router consulted (it then returns ReasonNoDoc
- // because no engine-backed structure surfaces in the test env, identical to
- // the post-route empty result).
- if _, _ = navigateStructures(context.Background(), "t", "q", []string{}, "catalog", router, SearchDeps{KbIDs: []string{"kb"}}); !router.called {
- t.Fatal("empty docIDs should route via NavTreeRouter, but router was not called")
- }
-
- // Non-empty doc_ids => router must NOT be consulted (no double routing).
- router2 := &stubNavRouter{docs: [][2]string{{"d1", "summary"}}}
- navigateStructures(context.Background(), "t", "q", []string{"foreign-doc"}, "catalog", router2, SearchDeps{
- KbIDs: []string{"kb"},
- DocIDVerifier: stubVerifier{known: map[string]bool{"mine": true}},
- })
- if router2.called {
- t.Error("non-empty docIDs should not consult the router")
- }
-}
-
-// TestNavigateTreeRoutesWithinSessionDocScope pins that the nav-tree route runs
-// inside the session document scope. args["doc_scope"] is not threaded into the tree route,
-// but the routing ceilings with the session doc_scope — dropping it routed over the whole
-// dataset.
-func TestNavigateTreeRoutesWithinSessionDocScope(t *testing.T) {
- router := &stubNavRouter{docs: [][2]string{{"d1", "summary"}}}
- NavigateTree(context.Background(), router, NavTreeInput{
- Query: "topic",
- TenantID: "t1",
- KbIDs: []string{"kb1"},
- DocScope: []string{"d1", "d2"},
- })
- if len(router.scope) != 2 || router.scope[0] != "d1" || router.scope[1] != "d2" {
- t.Fatalf("router doc scope = %v, want [d1 d2]", router.scope)
- }
-}
-
-// TestNavigateToolsRouteWithinSessionDocScope pins the executor end of the same
-// contract: both navigation tools hand the router the SESSION scope
-// (deps.DocScope), never the tool argument.
diff --git a/internal/rag/agentic-rag/runtime/tool_search.go b/internal/rag/agentic-rag/runtime/tool_search.go
deleted file mode 100644
index 19d9e855d9..0000000000
--- a/internal/rag/agentic-rag/runtime/tool_search.go
+++ /dev/null
@@ -1,2255 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "crypto/sha256"
- "encoding/json"
- "fmt"
- "log"
- "regexp"
- "sort"
- "strings"
- "sync"
- "unicode/utf8"
-
- "ragflow/internal/common"
- "ragflow/internal/engine"
- "ragflow/internal/entity"
- "ragflow/internal/service"
- "ragflow/internal/service/nlp"
-)
-
-// Search tools: the retrieval legs plus the narrowing that runs on top.
-//
-// One call carries both legs — the vector / BM25 split is a weight on the same call —
-// so the three entry points below differ only in the defaults they pass.
-
-// Retrieval defaults: callers that supply no configuration get exactly these values.
-const (
- DefaultSimilarityThreshold = 0.2
- DefaultTopN = 12
- DefaultRerankCandidatesCount = 64
- DefaultTopK = 1024
- // maxEffectiveQueryChars caps the expanded query in CODE POINTS. A byte cap would cut
- // a multi-byte rune in half and, for CJK, would apply a ~133-character cap instead of
- // 400.
- maxEffectiveQueryChars = 400
- // maxQueryTerms caps the terms derived from a query.
- maxQueryTerms = 16
-)
-
-// Term-splitting pattern for QueryToTerms.
-var (
- // queryMetaRe matches the regex metacharacters blanked out before splitting a query
- // into its literal terms. The split that follows is on WHITE SPACE, never on \w+:
- // Go's RE2 \w is ASCII-only, so tokenizing a CJK phrase with it yields nothing at all
- // and the fan-out BM25 leg loses the keyed terms it needs to weight the
- // discriminating entity.
- queryMetaRe = regexp.MustCompile(`[.*+?^$()\[\]{}]`)
-
- // DefaultAgenticVectorWeight is the vector leg's weight when agentic
- // retrieval runs keyword-only: zero.
- //
- // The agentic loop runs on keyword matches plus reranking, and nothing turns the
- // embedded leg on: SearchDeps.UsingEmbedding defaults to false, so the weight
- // forwarded is 0.0. Keeping that default is what keeps recall on keyword matches.
- DefaultAgenticVectorWeight = 0.0
-
- // DefaultHybridVectorWeight is the vector leg's weight when embedding IS used
- // (SearchDeps.UsingEmbedding true): 0.7, the configured default when a caller turns
- // embedding on.
- DefaultHybridVectorWeight = 0.7
-
- // HybridSearchDefaultVectorWeight is the vector leg's weight for the standalone
- // hybrid_search tool: 0.3 — NOT 0.7 — because that tool reads its own configured
- // default rather than the retrieve channel's.
- HybridSearchDefaultVectorWeight = 0.3
-
- // VectorSearchDefaultSimilarityThreshold is the engine similarity floor for
- // the pure-vector entry point (search.py:vector_search, 0.2).
- VectorSearchDefaultSimilarityThreshold = 0.2
- // BM25SearchDefaultSimilarityThreshold is the engine similarity floor for
- // bm25_search / grep_search (/395: 0.0).
- BM25SearchDefaultSimilarityThreshold = 0.0
-)
-
-// Retriever is the retrieval backend the runtime searches through.
-//
-// It is an interface so the session and the orchestrator can be tested without a search
-// backend, and so the concrete adapter (the runtime root package's RuntimeRetriever)
-// stays the only place that knows about the runtime.
-type Retriever interface {
- // Retrieve runs one search and returns raw chunk maps. The chunks carry at
- // least chunk_id / content / doc_id / docnm_kwd so the runtime accessors
- // (ChunkIDOf, ChunkTextOf, DocIDOf, ...) can read them.
- Retrieve(ctx context.Context, req RetrieveRequest) ([]map[string]any, error)
-}
-
-// DocChunkLister returns a document's chunks in reading order, which is what the
-// document-level tools need (position-sorted, strings only).
-//
-// Retrieval cannot express it: RetrieveRequest ranks by relevance to a query,
-// while these tools want every chunk of one document, page-ordered, paged.
-// Like DocIDVerifier it is injected, so this package needs no document-store
-// dependency of its own. Nil means the tools that need it stay unavailable.
-type DocChunkLister interface {
- // DocChunks returns up to Limit chunks of DocID starting at Offset, ordered
- // by reading position. A short page means the document is exhausted.
- DocChunks(ctx context.Context, req DocChunksRequest) ([]map[string]any, error)
-}
-
-// DocChunksRequest is one page of one document's chunks.
-type DocChunksRequest struct {
- DocID string
- DatasetIDs []string
- TenantID string
- Offset int
- Limit int
-}
-
-// DocIDVerifier reports which of a candidate document id set are known to the session's
-// datasets. The lookup is deliberately left to the caller so this package stays free of a
-// database dependency.
-type DocIDVerifier interface {
- // KnownDocIDs returns the subset of candidates that exist in the given
- // datasets. An error or an unavailable backend yields an empty set, which
- // the caller treats as "verification unavailable" rather than "no match".
- KnownDocIDs(ctx context.Context, datasetIDs, candidates []string) (map[string]bool, error)
-}
-
-// DocTenant is the owning (knowledge base, tenant) pair for one document, as returned by
-// DocTenantResolver.
-type DocTenant struct {
- KBID string
- TenantID string
-}
-
-// DocTenantResolver maps document ids to their owning (kb, tenant). Unlike
-// DocIDVerifier, the returned KB/tenant may lie OUTSIDE the caller's
-// datasetIDs, so graph_explore can group documents by their real owner and
-// search knowledge bases that were not in the original search set. The lookup
-// is left to the caller (this package stays database-free); an error or a nil
-// resolver makes ExploreGraph fall back to the pre-existing behaviour of
-// searching each bound dataset with the whole DocScope.
-type DocTenantResolver interface {
- ResolveDocTenants(ctx context.Context, docIDs []string) (map[string]DocTenant, error)
-}
-
-// MetadataResolver resolves document sets from document metadata for the metadata_search
-// tool. Like DocIDVerifier / DocTenantResolver the lookup is left to the caller, so the
-// search legs hold no metadata-index dependency of their own; the production
-// implementation is internal/service.MetadataService. Nil makes the metadata channel
-// unavailable (the tool reports a clean miss, the pre-search channel is skipped).
-//
-// Filters are the tool's own {key, value, op} condition maps, kept as maps so the
-// implementation needs no import of this package.
-type MetadataResolver interface {
- // FilterDocIDsByMetaPushdown resolves the documents whose metadata matches filters
- // (logic = "and" | "or") by pushing the predicate into the document-metadata index.
- //
- // ok=false means the push-down is NOT viable or errored and the caller must fall back
- // to GetFlattedMetaByKBs + the in-memory filter. ok=true with an empty slice is the
- // definitive "no document matches".
- FilterDocIDsByMetaPushdown(ctx context.Context, kbIDs []string, filters []map[string]any, logic string) ([]string, bool)
- // GetFlattedMetaByKBs returns field → value → doc_ids for the given datasets. It is
- // the in-memory filter's input and the source of the "which fields exist" hint that
- // turns "this dataset has no title" into advice instead of an empty retrieval the
- // model retries forever.
- GetFlattedMetaByKBs(ctx context.Context, kbIDs []string) (common.MetaData, error)
- // MetadataForDocIDs returns doc_id → its metadata fields, reading ONLY the given
- // documents. It backs the metadata_search result's context block: the ids stay the
- // tool's machine-consumable output, while these values let the model answer FROM the
- // selection (its titles, file names, timestamps) without spending another tool call.
- //
- // Best effort by contract: a dataset that cannot be read contributes nothing and the
- // returned error says why, but the caller has already resolved the document ids and
- // must not fail a successful selection over a missing context block.
- MetadataForDocIDs(ctx context.Context, kbIDs []string, docIDs []string) (map[string]map[string]any, error)
-}
-
-// DeclaredMetadataResolver reads the metadata fields a dataset DECLARES in its
-// parser_config — the {key, type, description, enum} definitions a user configured for
-// extraction. Optional: nil leaves the catalog with only what the metadata index carries.
-//
-// It is separate from MetadataResolver because it answers a different question (what the
-// dataset is CONFIGURED to filter on, before anything is indexed) from a different store
-// (the KB row, not the doc-metadata index), and because the declarative read is cheap enough
-// to run per metadata_search call while the observational one scans documents.
-type DeclaredMetadataResolver interface {
- // DeclaredMetadataFields returns the declared fields of the given datasets, in
- // configuration order. An error means "no declarative source available"; callers degrade
- // to the metadata index alone rather than failing.
- DeclaredMetadataFields(ctx context.Context, kbIDs []string) ([]common.MetadataFieldDef, error)
-}
-
-// RetrieveRequest is one retrieval call.
-//
-// Weight is the keyword-similarity weight: 0.7 hybrid (default), 0.0 vector-only,
-// 1.0 keyword-only (BM25). This is how Python's three search entry points
-// (hybrid_search / vector_search / bm25_search) differ.
-type RetrieveRequest struct {
- Query string
- DatasetIDs []string
- DocScope []string
- TopN int
- TopK int
- RerankCandidatesCount int
- // SimilarityThreshold is a pointer because ZERO is a valid explicit setting
- // (threshold 0 = no floor); nil means "not supplied", so the retriever keeps
- // its own default instead of being handed a zero override.
- SimilarityThreshold *float64
- // KeywordsSimilarityWeight is the keyword leg's weight. The retrieval
- // adapter derives the complementary vector weight and pure-mode behavior.
- KeywordsSimilarityWeight *float64
- TenantID string
- // MetaDataFilter restricts retrieval to chunks whose metadata matches
- // Nil means no filtering.
- MetaDataFilter map[string]any
- // RankFeature: `rank_feature` argument
- // (agentic_rag.py:retrieve): question-type tags produced by
- // label_question(question, self.kbs) that the retriever uses to boost
- // matching chunks. The Go engine consumes it as a tag → weight map (matching
- // internal/engine/types and the chat pipeline), so it is map[string]float64,
- // not a bare list. Nil means no rank feature.
- RankFeature map[string]float64
- // ExcludeCompiled excludes compiled-product rows from plain retrieval (the
- // must_not={"exists": "compile_kwd"} filter the hybrid leg applies). Compiled products
- // have their own expansion step, so the base retrieval here should surface ordinary
- // document chunks only. False is what the retrieve channel passes: it does not exclude
- // them.
- ExcludeCompiled bool
-}
-
-// intOrDef returns v when set, else fallback.
-func intOrDef(v, fallback int) int {
- if v > 0 {
- return v
- }
- return fallback
-}
-
-// floatOrDef returns v when set, else fallback. Kept separate from the pointer
-// form below so both "unset" conventions stay explicit at the call site.
-func floatOrDef(v, fallback float64) float64 {
- if v > 0 {
- return v
- }
- return fallback
-}
-
-// floatPtrOrDef returns *v when configured, else fallback. A pointer is used so
-// a configured 0 is honoured instead of looking like "unset".
-func floatPtrOrDef(v *float64, fallback float64) float64 {
- if v != nil {
- return *v
- }
- return fallback
-}
-
-// SearchChannel identifies which search entry point a call mirrors. The three channels
-// are genuinely different search functions rather than one with a flag; collapsing them
-// let a single global UsingEmbedding switch silently disable the vector leg for a tool
-// that never consulted it.
-type SearchChannel int
-
-const (
- // ChannelGrep: the `retrieve`, `grep_search` and `grep_chunks` session tools. It is
- // keyword-only: the grep leg has NO vector leg at all, so the weight is
- // unconditionally 0 and UsingEmbedding must not be consulted.
- ChannelGrep SearchChannel = iota
- // ChannelHybrid is the standalone hybrid search — the `search_chunks` session tool.
- // Its vector leg is gated on an embedder being available (the configured weight when
- // one is, else 0), NOT on UsingEmbedding; production always supplies an embed model,
- // so it runs. Default weight 0.3.
- ChannelHybrid
- // ChannelRetrieve is used by the low/naive direct passes. THIS is the only channel
- // that honours UsingEmbedding (false by default). Default weight 0.7 when embedding is
- // on.
- ChannelRetrieve
-)
-
-// resolveKeywordsSimilarityWeight mirrors the weight each Python entry point computes. The
-// gate differs per channel — that is the whole point:
-//
-// - ChannelGrep: always 1 (grep_search has no vector leg).
-// - ChannelHybrid: 1 when no embedder is configured (Python's `if embd_mdl`),
-// otherwise KeywordsSimilarityWeight ?? 0.7. UsingEmbedding is IRRELEVANT
-// here: Python's hybrid_search has no such parameter.
-// - ChannelRetrieve: 1 unless UsingEmbedding (Python's using_embedding),
-// otherwise KeywordsSimilarityWeight ?? 0.3.
-func resolveKeywordsSimilarityWeight(deps SearchDeps, ch SearchChannel) float64 {
- switch ch {
- case ChannelGrep:
- return 1
- case ChannelHybrid:
- // The configured weight when an embedder is available, else 0. HasEmbedder says
- // only that one is available — the retrieval service resolves the actual model.
- if !deps.HasEmbedder {
- return 1
- }
- return floatPtrOrDef(deps.KeywordsSimilarityWeight, 1-HybridSearchDefaultVectorWeight)
- default:
- if !deps.UsingEmbedding {
- return 1
- }
- return floatPtrOrDef(deps.KeywordsSimilarityWeight, 1-DefaultHybridVectorWeight)
- }
-}
-
-// SearchDeps are the dependencies of HybridSearch.
-type SearchDeps struct {
- // Backend is the retrieval service.
- Backend Retriever
- // KbIDs are the session's bound dataset ids.
- KbIDs []string
- // SQLKBs are the session's structured (SQL-backed) datasets. The hybrid channel
- // merges them into the target id list (the bound KB ids plus every SQL-KB id), so the
- // keyword/vector search spans the structured tables too. They stay separate on
- // SearchDeps and are folded into targetIDs in runSearch.
- SQLKBs []string
- // TenantID scopes the retrieval.
- TenantID string
- // IndexName is the search index that holds the dataset's document/structure rows (the
- // compiled-structure read is routed through it). Empty falls back to
- // "ragflow_<TenantID>", the
- // RAGFlow default — but a tenant that overrides index_name must surface it
- // here or structure/doc reads would hit the wrong index. The earlier Go port
- // hardcoded "ragflow_<tenantID>" and ignored per-tenant overrides.
- IndexName string
- // DocEngine fetches parent chunks during retrieval_by_children (child
- // fragments are promoted to their parent chunk). Nil falls back to
- // engine.Get(). Kept here so runSearch can promote children for the
- // hybrid/retrieve entry points without the Backend guessing which caller
- // wants promotion.
- DocEngine engine.DocEngine
- // KB receives the raw chunks into the lossless memory store BEFORE
- // narrowing, and carries the per-request search cache.
- KB *Kbinfos
- // Expand runs the compiled-structure expansion when UseCompiled is set.
- // Nil disables compiled expansion.
- Expand CompiledExpander
- // NavRouter descends the dataset's compiled navigation tree for the
- // navigate_tree tool. Nil falls back to nav.NewNavServiceRouter() (the
- // internal/service/nav singleton).
- NavRouter NavTreeRouter
- // Model is the request-scoped chat model. It drives
- // the calculate tool's expression-writing call and the graph_explore
- // structure verdict (AskStructure). Required for `calculate`; nil makes the
- // model-backed tools report a miss.
- Model SessionModel
- // Logger is optional; nil uses the default logger. It is the DEVELOPER log:
- // internal diagnostics (search legs, narrowing, compiled expansion) stay
- // here and are not steps. The tool call and its result ARE steps, reported
- // through the per-request reporter StepsFrom(ctx) carries.
- Logger *log.Logger
- // DocIDVerifier resolves which of the caller-supplied document ids actually belong to
- // the session's datasets. Nil means "no verification available": the doc scope is
- // passed through to the retriever unchanged, which is the pre-existing behaviour.
- //
- // It is injected rather than queried here because resolving document
- // ownership is a database lookup this package deliberately does not depend
- // on; the caller (internal/rag/agentic_rag) supplies the implementation.
- DocIDVerifier DocIDVerifier
- // DocChunks pages through one document's chunks for the document-level
- // tools (fetch_full_document / summarize_document). Nil leaves both
- // unavailable.
- DocChunks DocChunkLister
- // Embedder is the external embedding handle used by graph/structure seed encoding.
- // Nil falls back to this package's internal tenant-default resolver, which needs a
- // database and therefore stays self-contained for callers that cannot or do not want
- // to supply one.
- Embedder nlp.NavEmbedder
- // Retrieval tuning (Python RAGTools.retrieve: _setting(self, "top_n"),
- // similarity_threshold, keywords_similarity_weight, rerank_candidates_count,
- // top_k). Zero falls back to this package's Default* constants.
- TopN int
- SimilarityThreshold float64
- // KeywordsSimilarityWeight is a pointer so an explicit 0 (vector-only) is
- // distinguishable from "not configured", which falls back to the channel's
- // default keyword weight.
- KeywordsSimilarityWeight *float64
- // UsingEmbedding is the Go spelling of Python RAGTools.retrieve's
- // `using_embedding: bool = False` (agentic_rag.py:retrieve). When false (the
- // agentic default), the vector leg is disabled and retrieval is keyword-only
- // (vector weight 0) — exactly Python's `embd_mdl = None; vector_weight = 0`.
- // When true, the embedder is engaged and the vector weight is applied:
- // KeywordsSimilarityWeight if set, else 0.3 (the complement of Python's
- // default vector weight of 0.7). Unlike Python, the
- // query embedder is supplied by the runtime retrieval service, so Go carries
- // the flag rather than an embd_mdl handle on the retrieve call itself.
- //
- // SCOPE: this flag governs ChannelRetrieve ONLY. The hybrid channel (search_chunks)
- // has no such parameter, so gating it here is what silently disabled the semantic
- // leg.
- UsingEmbedding bool
- // HasEmbedder reports whether an embedding handle is available — the gate the hybrid
- // channel applies to its vector leg (the configured weight when one is, else 0).
- // Production always supplies one, so the semantic leg of search_chunks runs. False
- // keeps callers that have no embedding model on keyword-only retrieval.
- HasEmbedder bool
- RerankCandidatesCount int
- TopK int
- // MetaDataFilter restricts retrieval to matching chunk metadata. Nil means no
- // filtering.
- MetaDataFilter map[string]any
- // DocScope is the session-wide document restriction. It is a CEILING applied by
- // scopedDocIDs before any search: an explicit caller scope is intersected with it.
- // Empty means "search everything".
- DocScope []string
- // DocTenantResolver maps document ids to their owning (kb, tenant). Unlike
- // DocIDVerifier (which only answers "belongs to the bound datasets?"), this may
- // return KB/tenant pairs OUTSIDE the caller's datasetIDs, so graph_explore can search
- // documents that belong to other knowledge bases. Nil falls back to the pre-existing
- // behaviour: each bound dataset is searched with the whole DocScope (no per-document
- // re-grouping).
- DocTenantResolver DocTenantResolver
- // MetadataResolver resolves document sets from document metadata, backing the
- // metadata_search tool and the pre-search metadata channel. Nil leaves both
- // unavailable: the tool reports a clean miss and the channel is skipped.
- MetadataResolver MetadataResolver
- // DeclaredMetadata reads the metadata fields the datasets DECLARE in their
- // parser_config (description/enum included), which is what lets the metadata_search
- // catalog say what a field MEANS and not just that it exists. Nil leaves the catalog
- // with the metadata index alone.
- DeclaredMetadata DeclaredMetadataResolver
- // DoRefer: when true, summarize_document
- // prefixes the citation rules so the model cites the blocks it summarises.
- DoRefer bool
- // CiteRules is the optional user-defined citation template. Empty falls back to the
- // embedded citation_prompt.md. summarize_document passes it to the citation header
- // verbatim.
- CiteRules string
- // WebSearch is the optional open-web provider for the `web_search` tool. Nil HIDES
- // the tool from the session surface — Toolset.HasWebSearch is set from
- // mode.HasTool("web_search") && WebSearch != nil. Should a call reach the handler
- // anyway, it reports StatusError/ReasonInfra with a do-not-retry note, never a
- // query-level MISS.
- WebSearch WebSearcher
- // WikiRetriever is the optional provider for the `wiki_query` tool. The tool is
- // advertised in NO mode: its spec is not in the tool map either (see
- // action_session.go). The handler is kept as an unplugged seam and reports a clean
- // MISS (StatusMiss/ReasonNoDoc) when no retriever is wired.
- WikiRetriever WikiRetriever
- // KBs: the resolved Knowledgebase objects (carrying parser_config / tenant_id) the
- // agentic tools run over. The rank feature is derived from these — the retrieve
- // channel labels the question against them. Projected from RAGTools.KBs when the
- // search deps are built.
- KBs []*entity.Knowledgebase
- // Tagger classifies the query into question-type tags the retriever boosts on. Nil
- // means no tag boost. The implementation lives in internal/service
- // (MetadataService.LabelQuestion), which queries the tag service.
- Tagger QuestionLabeler
-}
-
-// QuestionLabeler labels a question against the KB objects: given the query and the KBs
-// it returns a map of question-type tag → weight the retriever uses to rank results. Go has
-// no tag-library dataset: the labeler matches the query against the KBs' tag source files
-// (parser_config tags.tag_file_id), and the extractor (extractor_tag.go) writes only tag_feas
-// (per-tag weights) onto each chunk, which the retriever ranks with. The production
-// implementation is internal/service.MetadataService.LabelQuestion; tests supply a stub.
-type QuestionLabeler interface {
- LabelQuestion(ctx context.Context, question string, kbs []*entity.Knowledgebase) map[string]float64
-}
-
-// CompiledExpander enriches a search result with the dataset's compiled
-// structure (page index / tree / knowledge graph / wiki synthesis pages).
-//
-// The compiled expansion the hybrid leg runs when use_compiled is set. It is a seam
-// rather than a direct call so the search leg stays independent of the compiled-structure
-// machinery (and testable without a knowledge graph).
-type CompiledExpander interface {
- Expand(ctx context.Context, kb *Kbinfos, query, keywords string, docScope []string) error
-}
-
-// QueryToTerms: derive the
-// literal terms a query carries. The alternation's segments are split out (capped
-// at maxQueryTerms) so the regex locate and the term-based context window fire on
-// the same broad set of names; a plain query simply splits on white space. Terms
-// shorter than two characters and duplicates are dropped.
-//
-// Case is PRESERVED (it is never lowercased): the downstream consumers are
-// case-insensitive — NarrowByTerms compiles with "(?i)" (grep_sed_narrow.go) and the
-// BM25 keyword leg is tokenized server-side — so lowercasing here would only change the
-// text handed to them.
-func QueryToTerms(query string) []string {
- q := strings.TrimSpace(query)
- if q == "" {
- return nil
- }
- // Strip the regex decorations an inline query may carry: a flag prefix plus \b and
- // (?i) markers.
- q = strings.TrimPrefix(q, "(?i)")
- q = strings.ReplaceAll(q, `\b`, "")
- q = strings.ReplaceAll(q, "(?i)", "")
-
- var (
- out []string
- seen = make(map[string]bool, len(q)/2)
- )
- for _, part := range strings.Split(q, "|") {
- part = strings.TrimSpace(queryMetaRe.ReplaceAllString(part, " "))
- if part == "" {
- continue
- }
- for _, tok := range strings.Fields(part) {
- if utf8.RuneCountInString(tok) < 2 || seen[tok] {
- continue
- }
- seen[tok] = true
- out = append(out, tok)
- }
- if len(out) >= maxQueryTerms {
- break
- }
- }
- if len(out) > maxQueryTerms {
- out = out[:maxQueryTerms]
- }
- return out
-}
-
-// FanoutKeyedTerms is the fan-out term filter: the subset of terms worth keying a BM25
-// round on. Short stopwords and bare numbers are dropped, but a long number (a year, a
-// specimen ID) is discriminative and survives.
-func FanoutKeyedTerms(terms []string) []string {
- var out []string
- for _, t := range terms {
- digits := isAllDigits(t)
- // The stopword set is lowercase while the terms themselves keep their case, so
- // "The" must be dropped like "the".
- if (utf8.RuneCountInString(t) >= 3 && !IsStopword(strings.ToLower(t)) && !digits) ||
- (utf8.RuneCountInString(t) >= 4 && digits) {
- out = append(out, t)
- }
- }
- return out
-}
-
-// searchOpts carries the per-entry-point differences. Each entry point is its own
-// function that builds a searchOpts and delegates to runSearch, so the vector-weight gate,
-// similarity threshold and compiled-row exclusion are chosen by the CALLER, never by a
-// single shared "channel flag" that a global switch could silently corrupt.
-type searchOpts struct {
- // keywordsSimilarityWeight is the keyword leg's similarity weight.
- keywordsSimilarityWeight float64
- // threshold is the engine similarity floor.
- threshold float64
- // excludeCompiled: must_not={"exists":"compile_kwd"}.
- excludeCompiled bool
- // promoteChildren lifts child fragments to their parent chunk after search. EVERY
- // entry point sets it true (hybrid, vector, bm25 and grep — the grep leg builds on the
- // bm25 leg); only a future non-promoting path would leave it false. An empty tenant_id
- // still skips promotion.
- promoteChildren bool
- // logLabel / logVerb / logKeywords describe the entry point's own
- // "searching" line — the LOG form, terse and grep-friendly, with the keywords
- // suffix and no period:
- //
- // hybrid → [Hybrid search] Searching the knowledge base for "q" (keywords: k)
- // vector → [Vector search] Searching by meaning for "q" (keywords: k)
- // bm25 → [BM25 search] Searching by keyword for "q" (keywords: k)
- // grep → [Grep search] Keyword-first locate for "q" (NO keywords)
- // retrieve → [Retrieve] Searching for "q" (keywords: k) (Go-only tag)
- //
- // Every verb is Python's own (search.py:122/:216/:254/:421), so a log diff
- // against Python lines up — retrieve is the one Go-only tag, and it no longer
- // carries "the knowledge base" (2026-09-16), which every leg searches and which
- // therefore distinguishes nothing a reader can act on. The shape is otherwise
- // unchanged — same tag, same keywords suffix, same `"%q -> N chunk(s): …"`
- // result line.
- //
- // The think block gets thinkVerb through searchThinkLine instead — one sentence
- // family for every leg, with the method named and the keyword list left out. The
- // two sentences come from the same call (runSearch / GrepSearch), so they can
- // never describe different searches.
- //
- // The RESULT line is the other half of a leg (reportSearchResult): every leg
- // emits one, on every exit, including an empty pool. In the log it keeps Python's
- // `"%q -> N chunk(s): <per-doc>"` shape where Python has one (hybrid :203, grep
- // :471); the vector and bm25 legs get it in Go only.
- //
- // The "(keywords: …)" suffix is printed only when non-empty: a keyword-less leg is a
- // deliberate state (see the fan-out's narrow bypass) and search_chunks takes no keywords
- // at all, so a bare "(keywords: )" would read like a dropped argument. logKeywords also
- // keeps the grep leg from growing a suffix it never carries.
- logLabel string
- logVerb string
- logKeywords bool
- // thinkVerb is the same leg's verb for the think block, which reports a
- // SENTENCE: it names the method ("by meaning and keyword") because the
- // "[Hybrid search]" prefix is a developer's label, and it is the reason the two
- // projections differ at all. Every leg sets it explicitly — so the log side can
- // be reworded (the Go-only retrieve leg dropped "the knowledge base", above)
- // without the trace moving with it; the fallback to logVerb only covers a future
- // leg that forgets to set one, where an identical sentence beats an empty one.
- thinkVerb string
- // cache enables the per-request search cache (SearchCacheLoad/Store). Only the hybrid
- // leg touches that cache; the vector / bm25 / grep / retrieve legs never consult it.
- // The cache lives on the shared deps.KB, so without this gate a keyword-only leg could
- // be served a
- // hybrid result (different vector weight, threshold and compiled policy)
- // and vice versa — Go's "enhanced" typing did not make it hybrid-scoped.
- cache bool
- // narrowLabel tags the narrowing line. The forms are deliberately UNEVEN: the hybrid
- // leg passes the snake_case "hybrid_search", while the vector and bm25 legs pass their
- // display labels "Vector search" / "BM25 search". Both forms are reproduced verbatim.
- narrowLabel string
- // rankFeature opts the entry point into the question-type tag boost
- // (rank_feature). ONLY the retrieve channel passes the question-type tags; the three
- // search legs call the retriever WITHOUT them (the grep leg delegates to bm25). Before
- // this gate every leg shipped the boost, which re-ranked search_chunks and retrieve
- // results the same way.
- rankFeature bool
-}
-
-// searchLogLine renders the "what this leg is searching for" message, WITHOUT
-// the stage prefix: the line is a STEP now (StepReporter.StageLine), and that
-// call writes "[<label>] " itself from the same string — so the developer log
-// stays byte-identical to Python's line (search.py:hybrid_search/215/252/418)
-// while the same sentence reaches the think block.
-//
-// Both the verb and the label vary by entry point. The keywords suffix is
-// appended by the caller, because Python prints it on hybrid/vector/bm25 only —
-// grep_search's locate line never carries one.
-func searchLogLine(verb, question string) string {
- return fmt.Sprintf("%s %q", verb, question)
-}
-
-// searchThinkLine renders the same leg's "searching" sentence for the think block:
-// the same verb with the query in quotes, plus the period a sentence needs (the
-// log line has none — Python's does not) and a length cap (the slot-research
-// driver searches a paragraph of evidence as its "query").
-//
-// It is the log line's sibling, not a replacement: StageLineDetail reports this
-// one and logs searchLogLine's, from the same call.
-func searchThinkLine(verb, question string) string {
- return fmt.Sprintf("%s %q.", verb, trunc(question, 80))
-}
-
-// searchLogger returns the run's logger, falling back to the package logger.
-func searchLogger(deps SearchDeps) *log.Logger {
- if deps.Logger != nil {
- return deps.Logger
- }
- return _LOG
-}
-
-// runSearch is the shared body of every search entry point. It is deliberately dumb
-// about WHICH entry point it is serving: that is encoded by opts, supplied by the
-// calling function.
-func runSearch(ctx context.Context, deps SearchDeps, p SearchParams, opts searchOpts) ([]map[string]any, []map[string]any) {
- logger := searchLogger(deps)
- // An explicit argument wins, then the caller's configuration, then this package's own
- // defaults.
- topN := p.TopN
- if topN <= 0 {
- topN = deps.TopN
- }
- if topN <= 0 {
- topN = DefaultTopN
- }
- // The structured (SQL) datasets are folded into the target id list. The explicit
- // argument still wins; otherwise the session's kb ids and sql ids are merged, so the
- // vector/keyword search spans the structured tables.
- targetIDs := p.KbIDs
- if len(targetIDs) == 0 {
- // The merge DE-DUPLICATES, so a KB present in both KbIDs and SQLKBs is not searched
- // twice (and the KbIDs themselves are deduped the same way).
- merged := append(append([]string{}, deps.KbIDs...), deps.SQLKBs...)
- seen := make(map[string]struct{}, len(merged))
- targetIDs = targetIDs[:0]
- for _, id := range merged {
- if _, ok := seen[id]; ok {
- continue
- }
- seen[id] = struct{}{}
- targetIDs = append(targetIDs, id)
- }
- }
- if len(targetIDs) == 0 {
- return nil, nil
- }
- if deps.Backend == nil {
- return nil, nil
- }
- docScope := resolveDocScope(ctx, deps, p.DocScope, targetIDs, logger)
-
- // 2. Effective query.
- //
- // The 400-char cap applies ONLY to the retrieval_query branch (the entity-weighted
- // expansion appended to the question). The keywords and bare-question branches are NOT
- // truncated.
- var effectiveQuery string
- if strings.TrimSpace(p.RetrievalQuery) != "" {
- // The cap slices code points: a byte slice would both cut a multi-byte rune in
- // half — handing the retriever invalid UTF-8 — and, for CJK, stop at ~133
- // characters, silently dropping two thirds of the expansion terms the fan-out leg
- // weighs on.
- effectiveQuery = truncateRunes(strings.TrimSpace(fmt.Sprintf("%s %s", p.Question, p.RetrievalQuery)), maxEffectiveQueryChars)
- } else if strings.TrimSpace(p.Keywords) != "" {
- effectiveQuery = strings.TrimSpace(fmt.Sprintf("%s %s", p.Question, p.Keywords))
- } else {
- effectiveQuery = p.Question
- }
-
- // The per-leg searching line: BOTH the bracket label and the verb vary by entry
- // point — "[Hybrid search] Searching the knowledge base for", "[Vector search]
- // Searching by meaning for", "[BM25 search] Searching by keyword for", "[Grep
- // search] Keyword-first locate for" — and only the three keyword-carrying legs
- // append the keywords (see searchOpts.logLabel / logVerb / logKeywords for the
- // Python source of each verb, and for the one Go-only tag that dropped "the
- // knowledge base").
- // The suffix is printed only when non-empty: a keyword-less leg is a
- // deliberate state (fan-out Channel B is a narrow bypass,
- // agentic_rag_graph.py:_search_one; search_chunks takes no keywords at all,
- // action_session.py:execute_tool), so a bare "(keywords: )" reads like a dropped
- // argument rather than an intentional empty.
- searchLine := searchLogLine(opts.logVerb, p.Question)
- if kws := strings.TrimSpace(p.Keywords); opts.logKeywords && kws != "" {
- searchLine += fmt.Sprintf(" (keywords: %s)", kws)
- }
- // A STEP, not only a log line: which leg ran and what it searched for is
- // exactly what the think block has to show to answer "what did it actually
- // search".
- //
- // The two audiences get different sentences from this one call site. The log
- // keeps Python's shape — label, keyword list and result line, plus Python's verb
- // on every leg that has one (only the Go-only retrieve verb deliberately dropped
- // "the knowledge base"; see logVerb above for what a log diff can still be
- // compared against). The think block gets the sentence family every leg
- // shares, which names the method inside the sentence because "[BM25 search]" is
- // a developer's label, and leaves out the keyword list: that appears only on the
- // legs that carry keywords, so in the block it reads as random noise, and it is
- // a detail a reader would have to parse rather than read.
- thinkVerb := opts.thinkVerb
- if thinkVerb == "" {
- thinkVerb = opts.logVerb
- }
- StepsFrom(ctx).StageLineDetail(logger, opts.logLabel,
- searchThinkLine(thinkVerb, p.Question), searchLine)
-
- // 3. Per-request dedup: an identical query+scope is retrieved at most once,
- // so e.g. pre_search and a claim search asking the same question do not
- // repeat the round-trip, child fetch and narrowing. The cache is hybrid-only — both the
- // lookup and the store belong to that leg — hence opts.cache.
- if deps.KB != nil && opts.cache {
- if chunks, aggs, ok := deps.KB.SearchCacheLoad(SearchCacheKey(effectiveQuery, targetIDs, topN, docScope)); ok {
- logger.Printf("[%s] Already searched this — reusing the %d passage(s) found earlier.", opts.logLabel, len(chunks))
- // A cache hit still hands a pool back, so it still reports one: the
- // dedup line explains WHY nothing was retrieved, this says WHAT the leg
- // contributed (Go-only: Python returns here silently on the result side).
- reportSearchResult(ctx, logger, opts.logLabel, p.Question, chunks)
- return chunks, aggs
- }
- }
-
- // 4. Retrieve.
- // rank_feature ONLY on the retrieve leg. The hybrid/vector/bm25 legs never pass it, so
- // the other entry points leave the request's RankFeature nil. It is computed from the
- // KB objects via the injected Tagger; a nil Tagger ⇒ no boost.
- var rankFeature map[string]float64
- if opts.rankFeature && deps.Tagger != nil {
- rankFeature = deps.Tagger.LabelQuestion(ctx, effectiveQuery, deps.KBs)
- }
- // Neither of the leg's two lines is emitted here: the searching line is above
- // and the result line at the end (reportSearchResult). Emitting either one here
- // as well would show the same search twice in the reasoning block.
- // D5: a retrieval that FAILS is retried once before the query is written off. The
- // failure this exists for is the upstream one: measured (2026-09-16) two queries of a
- // run answered `SILICONFLOW API error: 503 Service Unavailable … Model service
- // overloaded`, and the passages those queries would have returned were simply gone
- // from a question whose whole work is coverage. One retry costs one round trip on the
- // failure path only, and it is a retry of the same request (no re-planning, nothing
- // cached, nothing narrowed yet).
- retrieve := func() ([]map[string]any, error) {
- return deps.Backend.Retrieve(ctx, RetrieveRequest{
- Query: effectiveQuery,
- DatasetIDs: targetIDs,
- DocScope: docScope,
- TopN: topN,
- TopK: intOrDef(deps.TopK, DefaultTopK),
- RerankCandidatesCount: max(intOrDef(deps.RerankCandidatesCount, DefaultRerankCandidatesCount), topN),
- SimilarityThreshold: &opts.threshold,
- KeywordsSimilarityWeight: &opts.keywordsSimilarityWeight,
- TenantID: deps.TenantID,
- MetaDataFilter: deps.MetaDataFilter,
- RankFeature: rankFeature,
- ExcludeCompiled: opts.excludeCompiled,
- })
- }
- chunks, err := retrieve()
- if err != nil {
- // Go-only line: the retriever's failure is reported here instead of propagating.
- logger.Printf("[%s] retrieval failed: %v — retrying once.", opts.logLabel, err)
- chunks, err = retrieve()
- }
- if err != nil {
- logger.Printf("[%s] retrieval failed twice: %v", opts.logLabel, err)
- return nil, nil
- }
-
- // 4b. retrieval_by_children: child fragments are promoted to their parent chunk right
- // after retrieval. ALL entry points do this, so every runSearch caller sets
- // promoteChildren: true. The promotion lives here (gated by opts.promoteChildren)
- // rather than in the shared Backend so a future non-promoting path can opt out; an
- // empty tenant_id skips it.
- if opts.promoteChildren {
- de := deps.DocEngine
- if de == nil {
- de = engine.Get()
- }
- if de != nil && len(chunks) > 0 {
- chunks = nlp.RetrievalByChildren(chunks, []string{deps.TenantID}, de, ctx)
- }
- }
-
- // 5. doc_aggs from the FULL retrieved candidate set: computed here and left untouched
- // by the narrowing / compiled-expansion steps below, which only ever replace the chunk
- // list.
- aggs := DocAggs(chunks)
-
- // Memory BEFORE narrowing: the raw corpus may hold a fact the narrowing
- // drops, and a gap-driven grep over memory recovers it without re-querying.
- if deps.KB != nil {
- MemoryAdd(deps.KB, chunks)
- }
-
- // 6. Narrow-or-keep (chunks only; doc_aggs stays as retrieved).
- chunks = NarrowOrKeep(ctx, chunks, p.Keywords, opts.narrowLabel, logger)
-
- // 7. Compiled expansion.
- if p.UseCompiled && len(chunks) > 0 && deps.Expand != nil {
- logger.Printf("[%s] Compiled expansion enabled — enriching with page_index/tree/KG navigation.", opts.logLabel)
- if err := deps.Expand.Expand(ctx, deps.KB, p.Question, p.Keywords, docScope); err != nil {
- logger.Printf("[%s] compiled expansion failed: %v", opts.logLabel, err)
- }
- }
-
- // What the leg FOUND, reported once, on every path (see reportSearchResult).
- reportSearchResult(ctx, logger, opts.logLabel, p.Question, chunks)
-
- // 8. Cache the result — hybrid leg only.
- if deps.KB != nil && opts.cache {
- deps.KB.SearchCacheStore(SearchCacheKey(effectiveQuery, targetIDs, topN, docScope), chunks, aggs)
- }
- return chunks, aggs
-}
-
-// HybridSearch: the vector leg is gated on an embedder being configured (the configured
-// weight when one is, else 0); default weight 0.3. Compiled rows are excluded.
-func HybridSearch(ctx context.Context, deps SearchDeps, p SearchParams) ([]map[string]any, []map[string]any) {
- return runSearch(ctx, deps, p, searchOpts{
- keywordsSimilarityWeight: resolveKeywordsSimilarityWeight(deps, ChannelHybrid),
- threshold: floatOrDef(deps.SimilarityThreshold, DefaultSimilarityThreshold),
- excludeCompiled: true,
- promoteChildren: true,
- logLabel: "Hybrid search",
- logVerb: "Searching the knowledge base for",
- logKeywords: true,
- cache: true, // Python's search_cache is read/written here only (:136/:207)
- thinkVerb: "Searching by meaning and keyword for",
- // Python passes the SNAKE_CASE tag to _narrow_or_keep on this leg
- // (search.py:hybrid_search) although its own lines say "Hybrid search" — the
- // inconsistency is Python's and is reproduced verbatim.
- narrowLabel: "hybrid_search",
- })
-}
-
-// MetadataDocIDs resolves the documents a metadata filter matches: metadata-index
-// push-down (ES / Infinity) -> in-memory filter when the push-down is not viable ->
-// intersect with the session document scope.
-//
-// ok=false means NO document is eligible — nothing matched, or the session scope removed
-// every match. That is a normal empty result, never an error: the caller reports a
-// query-level miss. The resolution is query-independent, so callers resolve once and then
-// search inside the returned documents.
-func MetadataDocIDs(ctx context.Context, deps SearchDeps, filters []map[string]any, logic string) ([]string, bool) {
- logger := searchLogger(deps)
- if len(filters) == 0 || deps.MetadataResolver == nil {
- return nil, false
- }
- targetIDs := metadataTargetIDs(deps)
- if len(targetIDs) == 0 {
- return nil, false
- }
- docIDs, pushdownOK := deps.MetadataResolver.FilterDocIDsByMetaPushdown(ctx, targetIDs, filters, logic)
- if !pushdownOK {
- metas, err := deps.MetadataResolver.GetFlattedMetaByKBs(ctx, targetIDs)
- if err != nil {
- logger.Printf("[Metadata search] push-down unavailable and the in-memory fallback failed: %v", err)
- return nil, false
- }
- docIDs = metaFilterDocIDs(metas, filters, logic)
- }
- if len(docIDs) == 0 {
- logger.Printf("[Metadata search] no documents matched filters=%v logic=%s", filters, logic)
- return nil, false
- }
- // The session's document scope is a ceiling: a document outside it stays unreachable
- // even when its metadata matches.
- scoped := scopedDocIDs(deps.DocScope, docIDs)
- if len(scoped) == 0 {
- logger.Printf("[Metadata search] 0 documents after the doc-scope intersection")
- return nil, false
- }
- return scoped, true
-}
-
-// MetadataSearch: hybrid retrieval restricted to the documents a metadata filter matches
-// (MetadataDocIDs + HybridSearch), the leg the pre-search metadata channel runs.
-//
-// Compiled expansion stays OFF: it would pull in out-of-scope chunks and break the "only
-// these documents" contract.
-func MetadataSearch(ctx context.Context, deps SearchDeps, p SearchParams, filters []map[string]any, logic string) ([]map[string]any, []map[string]any) {
- docIDs, ok := MetadataDocIDs(ctx, deps, filters, logic)
- if !ok {
- return nil, nil
- }
- searchLogger(deps).Printf("[Metadata search] %q searching %d matched doc(s) via filters=%v", trunc(p.Question, 80), len(docIDs), filters)
- return HybridSearch(ctx, deps, SearchParams{
- Question: p.Question,
- Keywords: p.Keywords,
- DocScope: docIDs,
- TopN: p.TopN,
- UseCompiled: false,
- })
-}
-
-// metadataTargetIDs is the deduplicated dataset id list a metadata lookup spans: the
-// session's bound datasets plus its structured (SQL-backed) ones.
-func metadataTargetIDs(deps SearchDeps) []string {
- merged := append(append([]string{}, deps.KbIDs...), deps.SQLKBs...)
- out := make([]string, 0, len(merged))
- seen := make(map[string]struct{}, len(merged))
- for _, id := range merged {
- if id == "" {
- continue
- }
- if _, dup := seen[id]; dup {
- continue
- }
- seen[id] = struct{}{}
- out = append(out, id)
- }
- return out
-}
-
-// Metadata-search catalog caps: fields listed, values shown per field, and the rendered
-// block's length in CODE POINTS (a byte cap would cut CJK fields early).
-const (
- metadataCatalogKeysMax = 20
- metadataCatalogSamplesPerKey = 8
- metadataCatalogRenderMax = 2000
-)
-
-// metadataCatalogSystemKeys are metadata keys that exist in the doc-metadata index but
-// must never be offered as filters: unique identifiers and benchmark annotations let a
-// model shrink retrieval to the answer's own documents (the FRAMES `question_id` stamps
-// every document of one question with the same id), and the PDF/connector-derived fields
-// describe the file rather than its content.
-var metadataCatalogSystemKeys = map[string]bool{
- "question_id": true,
- "source_uri": true,
- "pageid": true,
- "outline": true,
-}
-
-// MetadataSample is one value of a metadata field plus how many documents carry it.
-type MetadataSample struct {
- Value string `json:"value"`
- Docs int `json:"docs"`
-}
-
-// MetadataCatalog is what a dataset offers as metadata_search filters, drawn from BOTH
-// sources it has:
-//
-// - the DECLARATIVE one (Fields): the {key, type, description, enum} definitions stored in
-// the dataset's parser_config. They exist BEFORE anything is indexed, and they carry what
-// the observational source cannot: what a field MEANS and which values it accepts;
-// - the OBSERVATIONAL one (Samples): the values the doc-metadata index actually carries,
-// with document counts.
-//
-// It is rendered into three places: the metadata_search tool schema (the `key` enum), the
-// action-session seed (the AVAILABLE METADATA block) and the pre-search fan-out channel's
-// field vocabulary. Nothing is baked in as a fallback — no field is advertised, filtered on or
-// named in a prompt unless the dataset's catalog carries it. The tool's own key check reads the
-// same two sources (see resolveMetadataFields), so what is advertised is what is accepted.
-type MetadataCatalog struct {
- Keys []string
- Samples map[string][]MetadataSample
- // Fields holds a key's DECLARED definition when the dataset declares one. A key present
- // only here is declared but not yet indexed: still a legitimate filter, just with no value
- // sample to show for it.
- Fields map[string]common.MetadataFieldDef
-}
-
-// Empty reports whether the catalog carries no usable field. An empty catalog means
-// "leave the tool spec exactly as it ships" — never "offer a field list that is empty",
-// because some providers reject an empty enum.
-func (c MetadataCatalog) Empty() bool { return len(c.Keys) == 0 }
-
-// Render builds the seed block. Empty catalogs render as "" so the caller can append
-// unconditionally. Each field carries what is known about it — its declared meaning, the
-// values it accepts, and the values seen in the index — so the model fills a filter instead
-// of guessing. The result is capped at metadataCatalogRenderMax code points.
-func (c MetadataCatalog) Render() string {
- if c.Empty() {
- return ""
- }
- var b strings.Builder
- b.WriteString("AVAILABLE METADATA (dataset fields usable as metadata_search filters):")
- for _, k := range c.Keys {
- b.WriteString("\n- ")
- b.WriteString(k)
- b.WriteString(describeMetadataField(c.Fields[k]))
- if samples := c.Samples[k]; len(samples) > 0 {
- b.WriteString("; values seen: ")
- for i, s := range samples {
- if i > 0 {
- b.WriteString(", ")
- }
- fmt.Fprintf(&b, "%q (%d doc(s))", s.Value, s.Docs)
- }
- }
- }
- b.WriteString("\nFields NOT listed here are unusable as filters — never invent one.")
- return truncateRunes(b.String(), metadataCatalogRenderMax)
-}
-
-// describeMetadataField renders a field's declared meaning and allowed values, shared by
-// Render and FieldList so the two can never describe the same field differently.
-func describeMetadataField(def common.MetadataFieldDef) string {
- var b strings.Builder
- if def.Description != "" {
- b.WriteString(" — ")
- b.WriteString(def.Description)
- }
- if len(def.Enum) > 0 {
- b.WriteString(" (one of: ")
- b.WriteString(strings.Join(def.Enum, " / "))
- b.WriteString(")")
- }
- return b.String()
-}
-
-// MetadataCatalogFor reads the metadata fields the session's datasets offer, for the
-// metadata_search schema and the session seed.
-//
-// It is DELIBERATELY error-free: the observational read is the same GetFlattedMetaByKBs the
-// executor already performs for its key check, and a nil resolver, an unreadable index or an
-// empty KB set all degrade to whatever the other half provides — "keep today's behaviour"
-// rather than failing the session. (A resolver failure is still reported to the model by the
-// tool itself as ERROR/infra if a call reaches the executor.)
-func MetadataCatalogFor(ctx context.Context, deps SearchDeps) MetadataCatalog {
- keys, metas, declared, err := resolveMetadataFields(ctx, deps)
- if err != nil || len(keys) == 0 {
- return MetadataCatalog{}
- }
- cat := MetadataCatalog{
- Keys: keys,
- Fields: declared,
- Samples: make(map[string][]MetadataSample, len(keys)),
- }
- for _, k := range keys {
- if samples := topMetadataSamples(metas[k], metadataCatalogSamplesPerKey); len(samples) > 0 {
- cat.Samples[k] = samples
- }
- }
- return cat
-}
-
-// MetadataCatalogPtr is MetadataCatalogFor returning nil for an empty catalog, so a
-// Toolset carries "no catalog" as nil and every render path skips it by construction.
-func MetadataCatalogPtr(ctx context.Context, deps SearchDeps) *MetadataCatalog {
- cat := MetadataCatalogFor(ctx, deps)
- if cat.Empty() {
- return nil
- }
- return &cat
-}
-
-// resolveMetadataFields is the single resolution both the catalog (what to advertise) and the
-// executor (what to accept) run. It returns the offered keys in sorted order, the flattened
-// observational view (for value samples), the declared definitions by key, and — unlike the
-// catalog's own view — the metadata-index read error, which the executor reports as infra
-// rather than folding into "this dataset has no metadata".
-//
-// Both halves are best-effort and independent: no declared resolver or a failing parser_config
-// read leaves the observational half alone, and vice versa. A DECLARED key is offered even
-// when the index has no value for it yet (it is a real filter that currently matches nothing);
-// an OBSERVED key is offered only when it carries at least one usable value.
-func resolveMetadataFields(ctx context.Context, deps SearchDeps) ([]string, common.MetaData, map[string]common.MetadataFieldDef, error) {
- targetIDs := metadataTargetIDs(deps)
- if len(targetIDs) == 0 {
- return nil, nil, nil, nil
- }
-
- declared := map[string]common.MetadataFieldDef{}
- if deps.DeclaredMetadata != nil {
- if defs, err := deps.DeclaredMetadata.DeclaredMetadataFields(ctx, targetIDs); err == nil {
- for _, def := range defs {
- if metadataCatalogExcluded(def.Key) {
- continue
- }
- if _, dup := declared[def.Key]; dup {
- continue
- }
- declared[def.Key] = def
- }
- }
- }
-
- metas := common.MetaData{}
- if deps.MetadataResolver != nil {
- got, err := deps.MetadataResolver.GetFlattedMetaByKBs(ctx, targetIDs)
- if err != nil {
- return nil, nil, declared, err
- }
- metas = got
- }
-
- seen := make(map[string]bool, len(declared)+len(metas))
- keys := make([]string, 0, len(declared)+len(metas))
- for k := range declared {
- seen[k] = true
- keys = append(keys, k)
- }
- for k := range metas {
- if seen[k] || metadataCatalogExcluded(k) {
- continue
- }
- if len(topMetadataSamples(metas[k], 1)) == 0 {
- // No usable value: offering the key would only invite a call that cannot match.
- continue
- }
- seen[k] = true
- keys = append(keys, k)
- }
- sort.Strings(keys)
- if len(keys) > metadataCatalogKeysMax {
- keys = keys[:metadataCatalogKeysMax]
- }
- if len(declared) == 0 {
- declared = nil
- }
- return keys, metas, declared, nil
-}
-
-// metadataCatalogExcluded reports whether a metadata key must stay out of the catalog:
-// blank, a connector/derived field (leading underscore is how they are namespaced), or a
-// system/label field (see metadataCatalogSystemKeys).
-func metadataCatalogExcluded(key string) bool {
- k := strings.TrimSpace(key)
- if k == "" || strings.HasPrefix(k, "_") {
- return true
- }
- return metadataCatalogSystemKeys[k]
-}
-
-// topMetadataSamples orders one field's values by document count (descending) then value,
-// so the rendered sample is deterministic, and caps them at n (n <= 0 means "no cap").
-func topMetadataSamples(values common.MetaValueDocs, n int) []MetadataSample {
- out := make([]MetadataSample, 0, len(values))
- for v, docs := range values {
- if strings.TrimSpace(v) == "" || len(docs) == 0 {
- continue
- }
- out = append(out, MetadataSample{Value: v, Docs: len(docs)})
- }
- sort.Slice(out, func(i, j int) bool {
- if out[i].Docs != out[j].Docs {
- return out[i].Docs > out[j].Docs
- }
- return out[i].Value < out[j].Value
- })
- if n > 0 && len(out) > n {
- out = out[:n]
- }
- return out
-}
-
-// metaFilterDocIDs is the in-memory metadata filter used when the index push-down is not
-// viable. The condition conversion and the operator normalisation stay in
-// internal/service, where the chat pipeline's own filter shares them.
-func metaFilterDocIDs(metas common.MetaData, filters []map[string]any, logic string) []string {
- if len(metas) == 0 || len(filters) == 0 {
- return nil
- }
- conditions := make([]service.MetaFilterCondition, 0, len(filters))
- for _, f := range filters {
- if f == nil {
- continue
- }
- conditions = append(conditions, service.MetaFilterCondition{
- Key: asString(f["key"]),
- Value: f["value"],
- Op: asString(f["op"]),
- })
- }
- if len(conditions) == 0 {
- return nil
- }
- return service.ApplyMetaFilter(metas, conditions, logic)
-}
-
-// VectorSearch: the pure vector entry point. With no embedder it returns nothing,
-// otherwise the vector weight is 1.0 and compiled rows are excluded.
-func VectorSearch(ctx context.Context, deps SearchDeps, p SearchParams) ([]map[string]any, []map[string]any) {
- if !deps.HasEmbedder {
- return nil, nil
- }
- return runSearch(ctx, deps, p, searchOpts{
- keywordsSimilarityWeight: 0,
- threshold: VectorSearchDefaultSimilarityThreshold,
- excludeCompiled: true,
- promoteChildren: true,
- logLabel: "Vector search",
- logVerb: "Searching by meaning for",
- logKeywords: true,
- thinkVerb: "Searching by meaning for",
- narrowLabel: "Vector search",
- })
-}
-
-// BM25Search: Keyword-only: the
-// vector weight is unconditionally 0, the similarity floor is 0.0, and compiled
-// rows are excluded.
-func BM25Search(ctx context.Context, deps SearchDeps, p SearchParams) ([]map[string]any, []map[string]any) {
- return runSearch(ctx, deps, p, searchOpts{
- keywordsSimilarityWeight: 1,
- threshold: BM25SearchDefaultSimilarityThreshold,
- excludeCompiled: true,
- promoteChildren: true,
- logLabel: "BM25 search",
- logVerb: "Searching by keyword for",
- logKeywords: true,
- thinkVerb: "Searching by keyword for",
- narrowLabel: "BM25 search",
- })
-}
-
-// patternRecallTopN is how wide ONE operand's recall goes when the query is a
-// STRUCTURAL pattern — `关公.*斩`, a question about how a thing is written rather
-// than about a name.
-//
-// A pattern is a LOCATOR, and a locator is only as good as the ground it is given:
-// recalling ten passages answers "does this pattern occur in those ten", not
-// "where does it occur". Recalling only a small topN per operand lets a pattern match
-// inside a handful of passages of a corpus where the subject alone occurs in a large part
-// of the text. That is the one retrieval path that does not depend on the model already
-// knowing the name, and a narrow recall has it looking through a keyhole.
-//
-// It is the widest per-operand number the pipeline already uses elsewhere
-// (SCAViewCap), and only the MATCHED windows travel onwards — the grep output cap
-// (GrepOutTotalChars) still bounds what the model pays for.
-//
-// 200, because the binding constraint is the recall bound rather than the corpus: an
-// operand's own match count runs well past a small topN, so a pattern's recall stops before
-// the corpus does and the passages past a ranking's head are never matched against the
-// pattern at all — and those are exactly the windows that hold the members nobody has
-// named. The number is a RECALL bound on candidates that are only matched and narrowed (the
-// model sees the matched windows, capped by GrepOutTotalChars), so widening it costs
-// retrieval, not prompt — and a pattern whose operand hits the bound is logged rather than
-// silently truncated (see retrieveGrepCandidates).
-const patternRecallTopN = 200
-
-// isStructuralPattern reports whether a query asks for structure (any-wildcard
-// pattern) rather than listing terms: `关公.*斩` does, `华雄|颜良` does not.
-func isStructuralPattern(query string) bool {
- return strings.Contains(query, ".*") || strings.Contains(query, ".+")
-}
-
-// callerBatch reports whether the caller wrote a BATCH — several terms in one
-// query — rather than a sentence.
-//
-// Three shapes count, and they are the three the model actually writes: an
-// alternation, a pattern (whose operands are the terms), and whitespace-separated CJK
-// pieces. The last one is the reason this predicate exists rather than a `|` test: the model
-// writes that batch shape and never writes a `|`, so a `|`-only gate would keep the
-// per-term seat allocation permanently off.
-//
-// A sentence is excluded on purpose, in both languages: an English question
-// splits on whitespace but carries no CJK, and a Chinese question has no
-// whitespace to split on. Those are the questions that are not enumerating
-// anything, and they must not pay for per-term searches.
-func callerBatch(query string) bool {
- if strings.Contains(query, "|") || grepPatternOf(query) != nil {
- return true
- }
- pieces := 0
- for _, field := range strings.Fields(query) {
- if !hasCJK(field) {
- continue
- }
- pieces++
- if pieces >= 2 {
- return true
- }
- }
- return false
-}
-
-// retrieveGrepCandidates fetches the candidate set the locate step then narrows.
-//
-// One ranked search answers "what does this query match", which is the wrong question for
-// an alternation. A batch of names asks WHICH of them the corpus carries, and a single
-// top-N hands nearly every seat to the passages that match many of the terms at once: the
-// rarest names — the ones the probe was made for — never reach the pool, and the round
-// answers short while the probe itself had done its job.
-//
-// So an alternation searches each term on its own, and the results are woven
-// term-by-term, best hit first. The caller's per-query cap then keeps ONE hit
-// per term before it keeps a second hit for any single term — which is the
-// shape an enumeration needs: one passage per name, and a name that comes back
-// empty is a name the corpus does not carry. That empty answer is the point of
-// the probe, so it is returned, not filled with the hits the other terms found.
-//
-// The weave is bounded by the terms (GrepTermsMax) and by each search's own
-// TopN, and the narrowing stage's char budget still decides how much of the
-// woven set survives.
-//
-// The trigger is the CALLER's BATCH, not the `|` character: the model writes batches of
-// names separated by spaces and no `|`, so a `|`-only trigger would leave this whole
-// function dead code while the batches it was written for went to a single ranked top-N.
-// A batch is an alternation, a pattern, or two
-// or more CJK pieces separated by whitespace; a sentence in either language is
-// none of those, which is what keeps the extra searches off the questions that
-// are not enumerating anything.
-func retrieveGrepCandidates(
- ctx context.Context,
- deps SearchDeps,
- bp SearchParams,
- query string,
- terms []string,
-) ([]map[string]any, []map[string]any) {
- if !callerBatch(query) || len(terms) < 2 {
- return BM25Search(ctx, deps, bp)
- }
-
- perTerm := make([][]map[string]any, 0, len(terms))
- aggs := make([]map[string]any, 0, len(terms))
- perTopN := bp.TopN
- if isStructuralPattern(query) && perTopN < patternRecallTopN {
- perTopN = patternRecallTopN
- }
- for _, term := range terms {
- // The term alone is the query: on a keyword leg one rare token is the
- // strongest query there is, and the rest of the alternation can only
- // dilute it.
- sub := bp
- sub.Question = term
- sub.Keywords = term
- sub.TopN = perTopN
- chunks, docAggs := BM25Search(ctx, deps, sub)
- if isStructuralPattern(query) && len(chunks) >= perTopN {
- // A pattern is a LOCATOR over what the keyword leg recalled, so a full page means
- // it was matched against a TRUNCATED candidate set: matches past this bound are not
- // "absent from the corpus", they were never looked at. Said out loud because the
- // difference decides whether a member the pattern did not show is a corpus fact or
- // a ceiling (see patternRecallTopN for the measurement).
- searchLogger(deps).Printf("[Grep search] operand %q filled its recall bound (%d passage(s)); matches beyond it were not matched against the pattern.", term, perTopN)
- }
- perTerm = append(perTerm, chunks)
- aggs = append(aggs, docAggs...)
- // A term searched ON ITS OWN is the caller's probe of ONE individual, so
- // a passage that carries it is a confirmed member with its evidence —
- // recorded as such, because the round needs the members (and the passages
- // behind them), not just the number of chunks it holds.
- //
- // Except when the caller IS the runtime: the completeness pass searches the
- // actor and the act words, which are not names and must not enter the record's
- // to-do list (see SearchParams.SkipReachLedger).
- if !bp.SkipReachLedger {
- for _, c := range chunks {
- if strings.Contains(strings.ToLower(ChunkTextOf(c)), strings.ToLower(term)) {
- deps.KB.RecordReachedTerm(term, ChunkIDOf(c))
- break
- }
- }
- }
- }
-
- seen := make(map[string]bool)
- out := make([]map[string]any, 0, len(terms))
- for depth := 0; ; depth++ {
- any := false
- for _, chunks := range perTerm {
- if depth >= len(chunks) {
- continue
- }
- any = true
- c := chunks[depth]
- if id := ChunkIDOf(c); id != "" {
- if seen[id] {
- continue
- }
- seen[id] = true
- }
- out = append(out, c)
- }
- if !any {
- break
- }
- }
- // One line, only for a batch — this is the fingerprint that says the per-term
- // seats were actually allocated, so a run can be read for whether the trigger
- // fired at all (the failure mode this replaced was a mechanism that never ran).
- searchLogger(deps).Printf("[Grep search] batch of %d term(s): each searched on its own, %d candidate(s) woven",
- len(terms), len(out))
- return out, aggs
-}
-
-// ProbeSeatTopN bounds how many candidates one term's seat search takes before
-// the window is picked: a seat exists to carry the name, not to rank it.
-const ProbeSeatTopN = 3
-
-// TermSeat runs ONE cheap keyword search for a single named term and returns the
-// passage that carries it, narrowed to that term's own window, or (nil, false)
-// when nothing reached it.
-//
-// This is the retrieval unit an enumeration needs, and it is deliberately not a
-// query SYNTAX: the caller's terms may arrive as an alternation ("A|B|C"), as a
-// space-separated list inside one string, or as a list of query strings, and the
-// seat is the same thing in all three cases. What it replaces is asking for
-// several individuals at once: one ranked search hands its seats to the passages
-// that match MANY of the named terms, so the rarest name — the reason the call
-// was made — is the one that loses: most queries in a call are cut by the per-query cap,
-// and the candidates that did come back were flattened to one hit each, so the rarest names
-// are missing from an answer whose retrieval had returned tens of candidates per query.
-//
-// The term alone is the query, on the keyword leg only: no vector leg, no
-// compiled expansion, no model call — a few hundred milliseconds, which is what
-// lets the caller afford one per named term.
-//
-// The boolean is the OTHER half of the answer: false means this corpus reached
-// nothing for that term, which is a fact about the corpus the run must keep
-// (Kbinfos.RecordProbedAbsent) rather than a failed lookup to retry.
-func TermSeat(ctx context.Context, deps SearchDeps, base SearchParams, term string) ([]map[string]any, bool) {
- term = strings.TrimSpace(term)
- if term == "" {
- return nil, false
- }
- sub := base
- sub.Question = term
- sub.Keywords = term
- sub.TopN = ProbeSeatTopN
- sub.UseCompiled = false
- chunks, _ := BM25Search(ctx, deps, sub)
- if len(chunks) == 0 {
- return nil, false
- }
- res := NarrowByTerms(chunks, []string{term}, nil, term,
- NarrowContext{Before: 1, After: 0}, GrepOutCharsPerChunk, GrepOutTotalChars)
- if len(res.Kept) > 0 {
- return res.Kept[:1], true
- }
- // The keyword leg returned candidates that do not carry the term: on a
- // keyword leg that is the corpus answering "not here", so the seat is empty
- // rather than filled with the nearest passages.
- return nil, false
-}
-
-// GrepSearch: a keyword-first locate that runs the bm25 leg and then narrows the prose
-// candidates to the term-grep window (regex locate + short line-context).
-//
-// Behaviour:
-// - BM25 candidate pool built from the query's extracted terms — or the explicit
-// keywords hint when one is supplied — never a
-// hard filter: the pool still spans the corpus.
-// - Table chunks pass through UN-narrowed (full text): the grep window would
-// truncate them to a header-only snippet and hide mid/late-table answer rows.
-// - Prose chunks are narrowed via NarrowByTerms with context {before:1,
-// after:0}, per-chunk 700-char and total 8000-char caps.
-// - When grep matches nothing, the raw BM25 candidates are returned unchanged so
-// evidence is never dropped (enumeration / multi-hop must not lose candidates).
-func GrepSearch(ctx context.Context, deps SearchDeps, p SearchParams) ([]map[string]any, []map[string]any) {
- logger := searchLogger(deps)
- query := strings.TrimSpace(p.Question)
- // Python logs the locate line BEFORE extracting the terms and before the
- // empty-query bail-out (search.py:grep_search).
- //
- // Same split as runSearch: the log keeps Python's phrasing ("Keyword-first
- // locate for", search.py:418) and the think block says what the leg does — find
- // these exact words — in the family the other legs use. "Keyword-first locate"
- // was a noun phrase naming the implementation, not an action a reader could read.
- StepsFrom(ctx).StageLineDetail(logger, "Grep search",
- searchThinkLine("Searching for the exact words", query),
- searchLogLine("Keyword-first locate for", query))
- // Python prints its result line at the END of grep_search, reading whatever the
- // function is about to return (`_g = res.get("chunks"); if _g:`, :471-477) — so
- // every early return below gets one too. Reporting through this closure is what
- // keeps that true: the count is always of the RETURNED set, which for the
- // fallbacks below is the raw candidate pool or the table chunks, not the pieces
- // the term window happened to keep.
- report := func(returned []map[string]any) {
- reportSearchResult(ctx, logger, "Grep search", query, returned)
- }
- if query == "" {
- // Nothing was searched, so there is no result to report: the searching line
- // above already says the query was empty, and "Found nothing for \"\"" would
- // read like a search that ran and came back empty.
- return nil, nil
- }
- terms := GrepTermsFromQuery(query)
- pattern := grepPatternOf(query)
- if pattern != nil {
- // The operands of a pattern ARE its recall terms (see GrepPatternOperands):
- // the phrase reading of the same string would treat "关公.*斩" as one clause
- // and lose half of it.
- terms = GrepPatternOperands(query)
- }
- // The WEAVE searches the caller's own WORDS; `terms` above is the LOCATOR's
- // list, and for an unbroken CJK clause it is a set of two-rune windows (see
- // GrepWordsFromQuery).
- //
- // A window searched on its own spends a retrieval on a fragment nobody
- // proposed — and, because a term searched on its own records what it reached,
- // it also entered the reach ledger. The session then reads that ledger back as
- // its to-do list: the windows of an unbroken clause are recorded as if they were probes,
- // so the ledger fills with fragments the question is made of, while the names the round is
- // actually missing are not on it at all.
- //
- // A pattern keeps its operands: those are already the caller's words, and its
- // recall needs the long ones (`关公.*斩` searches 关公 and 斩).
- weave := terms
- if pattern == nil {
- weave = GrepWordsFromQuery(query)
- }
- // The grep leg then delegates to the bm25 leg with an explicit keywords hint: the
- // caller's keywords when supplied, else the extracted terms joined. A long question
- // buries its proper nouns under stopwords; without the hint the noun chunk never enters
- // the candidate pool and grep has nothing to locate. It is a SOFT boost — the pool
- // still spans the corpus. Delegating (rather than calling runSearch) is also what
- // produces the log sequence: the bm25 leg logs its searching line and narrows under
- // that tag, then this leg logs its own two lines.
- hint := strings.TrimSpace(p.Keywords)
- if hint == "" {
- hint = strings.Join(terms, " ")
- }
- bp := p
- if grepPatternOf(query) != nil {
- // A pattern is an expression for the MATCHER, not for the engine: handed
- // over, `|` and `.*` become either literals to escape or syntax of their
- // own, and a batch probe that the engine echoes back empty is then read as
- // "the corpus does not carry it". So recall gets the OPERANDS, which is
- // what a keyword leg can actually search for, and the pattern stays here
- // and decides which of the candidates are evidence.
- bp.Question = strings.Join(terms, " ")
- bp.Keywords = bp.Question
- } else {
- bp.Question = query
- bp.Keywords = hint
- }
- chunks, docAggs := retrieveGrepCandidates(ctx, deps, bp, query, weave)
- // No candidates or no terms: there is nothing to locate.
- if len(chunks) == 0 || len(terms) == 0 {
- report(chunks)
- return chunks, docAggs
- }
-
- var table, prose []map[string]any
- for _, c := range chunks {
- if IsTableChunk(c) {
- table = append(table, c)
- } else {
- prose = append(prose, c)
- }
- }
- if len(prose) == 0 {
- // All tables: nothing to locate, and the tables ARE the evidence.
- report(chunks)
- return chunks, docAggs
- }
-
- // A query that carries pattern syntax is read as a PATTERN and matched against
- // the candidates the keyword leg returned (see matchGrepPattern): `A|B` is how
- // a batch of names is asked about, `A.*B` is how the way a thing was done is
- // asked about, and neither can be expressed by locating terms one at a time.
- // The pattern itself never leaves this function — the engine only ever saw the
- // operands — so `|` and `.*` are operators here rather than characters somebody
- // has to escape.
- if pattern != nil {
- kept, matched := matchGrepPattern(prose, pattern, contextCharBudget, GrepOutTotalChars)
- if matched == 0 {
- // The pattern matched nothing in what was reached: keep the raw
- // candidates so evidence is not dropped, and SAY what happened — a
- // silent return is how "the pattern did not occur here" gets read as
- // "the corpus does not carry it".
- logGrepReach(logger, query, chunks, terms)
- return chunks, docAggs
- }
- out := make([]map[string]any, 0, len(table)+len(kept))
- out = append(out, table...)
- out = append(out, kept...)
- chars := 0
- for _, c := range out {
- chars += len(ChunkTextOf(c))
- }
- logger.Printf("[Grep search] pattern %q matched %d/%d candidate(s) -> %d chunk(s), %.1fK chars.",
- trunc(query, 80), matched, len(prose), len(out), float64(chars)/1000.0)
- logGrepReach(logger, query, chunks, terms)
- return out, docAggs
- }
-
- res := NarrowByTerms(prose, terms, nil, query, NarrowContext{Before: 1, After: 0},
- GrepOutCharsPerChunk, GrepOutTotalChars)
- kept := res.Kept
- if len(kept) == 0 {
- // Nothing matched: keep the raw BM25 candidates so evidence is not dropped
- // — and report THOSE, because they are what the caller receives.
- report(chunks)
- return chunks, docAggs
- }
- out := make([]map[string]any, 0, len(table)+len(kept))
- out = append(out, table...)
- out = append(out, kept...)
- // Python logs the post-narrow size and the per-doc breakdown for the
- // combined table+prose set (search.py:grep_search / :469) — the grep leg's own
- // line, which only exists on this path (the five above return before it). It is
- // a LOG line, not a step: the think block's result sentence comes from report
- // below, like every other leg's.
- chars := 0
- for _, c := range out {
- chars += len(ChunkTextOf(c))
- }
- logger.Printf("[Grep search] narrowed %d->%d chunk(s), %.1fK chars.", len(chunks), len(out), float64(chars)/1000.0)
- report(out)
- return out, docAggs
-}
-
-// RetrieveSearch: the low/naive direct pass. Unlike HybridSearch it honours
-// UsingEmbedding and does NOT exclude compiled rows.
-func RetrieveSearch(ctx context.Context, deps SearchDeps, p SearchParams) ([]map[string]any, []map[string]any) {
- return runSearch(ctx, deps, p, searchOpts{
- keywordsSimilarityWeight: resolveKeywordsSimilarityWeight(deps, ChannelRetrieve),
- threshold: floatOrDef(deps.SimilarityThreshold, DefaultSimilarityThreshold),
- excludeCompiled: false,
- promoteChildren: true,
- // Go-only identity: Python's L1 RAGTools.retrieve
- // logs nothing of its own, so there is no Python string to mirror. The
- // tag stays human-readable and the extra lines stay on, because this is
- // the only trace the low-mode direct pass leaves.
- logLabel: "Retrieve",
- logVerb: "Searching for",
- logKeywords: true,
- thinkVerb: "Searching for",
- narrowLabel: "retrieve",
- // The ONLY entry point that passes the question-type tag boost.
- rankFeature: true,
- })
-}
-
-// webSearchMaxChunks is the hard ceiling on admitted web passages per call: each query
-// is capped at 8 chunks and at most two queries run, so 16 is the maximum.
-const webSearchMaxChunks = 16
-
-// WebSearchTool: the `web_search` tool. Its schema takes a 1-2 element array of query
-// strings (see webSearchToolSpec), so args is the positional query list, not a key/value
-// object. Results share the RAGFlow chunk shape so they merge into the SAME shared
-// evidence pool as corpus hits, and the model sees the same REDUNDANT/OK/MISS outcome as
-// the corpus tools.
-func WebSearchTool(ctx context.Context, deps SearchDeps, args map[string]any) (ToolOutcome, error) {
- if deps.WebSearch == nil {
- // An absent provider is an infra ERROR with an explicit do-not-retry note (returning
- // MISS would let the model retry the same dead tool and burn turns), never a
- // query-level miss.
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": "web_search",
- "note": "Web search is NOT configured for this session. Do not use this tool again; use the corpus tools (retrieve / search_chunks / navigate_*) instead.",
- }},
- Status: StatusError,
- Reason: ReasonInfra,
- Metrics: map[string]any{},
- }, nil
- }
- // The query list is read from args["query"]. The old ""-key read never found anything,
- // so every call with a proper {"query": [...]} payload was rejected as bad_args.
- queries := toolStringList(args, "query")
- if len(queries) == 0 {
- return ToolOutcome{Payload: []any{}, Status: StatusError, Reason: ReasonBadArgs}, nil
- }
- // Cap at two queries.
- if len(queries) > 2 {
- queries = queries[:2]
- }
- results, err := deps.WebSearch.Search(ctx, queries)
- if err != nil || len(results) == 0 {
- return ToolOutcome{
- Payload: []any{map[string]any{
- "kind": "web_search",
- "note": "No web result returned.",
- }},
- Status: StatusMiss,
- Reason: ReasonNoDoc,
- Metrics: map[string]any{},
- }, nil
- }
- // Each query's chunks are capped at 8; with up to two queries that is a hard ceiling of
- // 16 web passages per tool call.
- if len(results) > webSearchMaxChunks {
- results = results[:webSearchMaxChunks]
- }
-
- // Build RAGFlow-shaped chunk maps and admit them to the shared evidence pool, mirroring
- // the corpus tools' merge + outcome. Corpus passages are de-duplicated across the (up to
- // two) queries by chunk_id; the web provider surfaces raw strings, so de-dup by content
- // to keep the same "each distinct passage appears once" behaviour.
- var payload []any
- var evidenceIDs []string
- newChunks := 0
- seen := make(map[string]bool, len(results))
- // The whole batch is ONE critical section (Kbinfos.Admit): the admit loop has no await
- // (the retrieval itself is awaited above it), so two sessions can never interleave
- // here.
- deps.KB.Admit(func(p *PoolAdmitter) {
- for _, r := range results {
- if r == "" || seen[r] {
- continue
- }
- // Admission early-stops at the pool cap, BEFORE the chunk is recorded as seen, so
- // a rejected passage is retried once room frees.
- if p.Full() {
- continue
- }
- // Stable content identity deduplicates repeated passages across calls
- // while preserving distinct results from concurrent sessions.
- chunkID := fmt.Sprintf("web_%x", sha256.Sum256([]byte(r)))
- seen[r] = true
- c := map[string]any{
- "chunk_id": chunkID,
- "content": r,
- "doc_id": "web",
- }
- // p.Add, not Merge: Merge takes this same pool lock and would deadlock
- // inside the critical section. It also answers "new to the pool" per
- // chunk, so the count no longer has to be inferred from a length delta
- // that other sessions' appends inflate.
- if p.Add(c) {
- newChunks++
- }
- // Evidence references the chunk id, not a pool position.
- evidenceIDs = append(evidenceIDs, chunkID)
- // Passage shape: {"id","content"} (no doc_id), non-table text cut to 1200 code
- // points (plain slice, no ellipsis).
- content := r
- if !IsTableChunk(c) {
- content = truncateRunes(content, 1200)
- }
- payload = append(payload, map[string]any{"id": chunkID, "content": content})
- }
- })
-
- if len(payload) == 0 {
- return ToolOutcome{Payload: []any{}, Status: StatusMiss, Reason: ReasonNoDoc, Metrics: map[string]any{"hits": 0, "new_evidence": 0}}, nil
- }
- status := StatusOK
- if newChunks == 0 {
- status = StatusRedundant
- }
- return ToolOutcome{
- Payload: payload,
- EvidenceIDs: evidenceIDs,
- Status: status,
- Metrics: map[string]any{"hits": len(payload), "new_evidence": newChunks},
- }, nil
-}
-
-// list_chunks deep-read caps.
-const (
- // listChunksMaxDeep caps the deep-read pool admission.
- listChunksMaxDeep = 80
- // listChunksMaxOut caps the model-facing passage list.
- listChunksMaxOut = 30
-)
-
-// listChunks deep-reads one document's FULL text and admits it to the shared evidence
-// pool.
-//
-// The read goes through deps.DocChunks (the same reader fetch_full_document /
-// summarize_document use), not the evidence pool, so navigate_tree can route a
-// doc_id whose chunks were never retrieved; when DocChunks is unwired it falls
-// back to the pool so the tool still works in unit tests.
-//
-// Outcome semantics mirror _search_outcome: an empty result (blank doc_id, unknown
-// doc, or nothing read) is a MISS — not an error; a non-empty result is REDUNDANT
-// when nothing new entered the pool, else OK. The deep read is capped at
-// listChunksMaxDeep and the model-facing output at listChunksMaxOut.
-func (e *searchExecutor) listChunks(ctx context.Context, args map[string]any) (ToolOutcome, error) {
- docID := argString(args, "doc_id")
- if docID == "" {
- // An empty doc_id is a query-level MISS.
- return ToolOutcome{Payload: []any{}, Status: StatusMiss, Reason: ReasonNoDoc, Metrics: map[string]any{"hits": 0}}, nil
- }
-
- var chunks []map[string]any
- if e.deps.DocChunks != nil {
- // Real doc-store deep read, budgeted by the model window.
- deep, _ := fetchFullDocument(ctx, e.deps, docID, e.req.MaxLength)
- if len(deep) > listChunksMaxDeep {
- deep = deep[:listChunksMaxDeep]
- }
- chunks = deep
- } else if e.deps.KB != nil {
- // Fallback: scan the in-memory evidence pool (unit-test / unwired path).
- for _, c := range e.deps.KB.Chunks {
- if DocIDOf(c) == docID {
- chunks = append(chunks, c)
- }
- }
- }
-
- if len(chunks) == 0 {
- // Unknown doc_id / out-of-scope doc, or nothing readable: a query-level MISS. There
- // is no "graceful OK empty" branch.
- return ToolOutcome{Payload: []any{}, Status: StatusMiss, Reason: ReasonNoDoc, Metrics: map[string]any{"hits": 0, "new_evidence": 0}}, nil
- }
-
- // The pool is CREATED when absent, so the admittance below always has somewhere to
- // write.
- if e.deps.KB == nil {
- e.deps.KB = &Kbinfos{}
- }
-
- // Only the first listChunksMaxOut (30) of the listChunksMaxDeep (80) fetched passages
- // are admitted: no-id chunks and in-call duplicates are skipped, evidence references
- // the chunk id, and only chunks new to the shared pool are appended.
- admit := chunks
- if len(admit) > listChunksMaxOut {
- admit = admit[:listChunksMaxOut]
- }
- seen := map[string]bool{}
- var payload []any
- var evidenceIDs []string
- newChunks := 0
- // The batch is ONE critical section (Kbinfos.Admit): the admit stretch has no await, so
- // two sessions can never interleave here, and the pool-side dedup must see the LIVE
- // pool rather than a snapshot taken upfront.
- e.deps.KB.Admit(func(p *PoolAdmitter) {
- covered := p.ClaimCoveredIDs()
- for _, c := range admit {
- cid := ChunkIDOf(c)
- if cid == "" {
- // A blank cid is skipped in the caller, BEFORE admission.
- continue
- }
- // Admission early-stops at the pool cap, BEFORE the per-call dedup.
- if p.Full() {
- continue
- }
- if seen[cid] {
- continue
- }
- // already quoted verbatim by a pooled claim →
- // skip the full passage (table chunks exempt).
- if p.CoveredByClaim(cid, covered, IsTableChunk(c)) {
- continue
- }
- seen[cid] = true
- evidenceIDs = append(evidenceIDs, cid)
- // Pool shape: {"id","content"} (no doc_id), with non-table text cut to 1200 code
- // points (a plain slice, no ellipsis).
- content := ChunkTextOf(c)
- if !IsTableChunk(c) {
- content = truncateRunes(content, 1200)
- }
- payload = append(payload, map[string]any{"id": cid, "content": content})
- if p.Add(c) {
- newChunks++
- }
- }
- })
-
- if len(payload) == 0 {
- return ToolOutcome{Payload: []any{}, Status: StatusMiss, Reason: ReasonNoDoc, Metrics: map[string]any{"hits": 0, "new_evidence": 0}}, nil
- }
- // A deep read COVERS its claims: once the full chunk text is in the pool, the claim's
- // 1200-char quote of the same passage is duplicated tokens in every later prompt.
- // Retire claim pseudo-chunks whose source chunk was just read.
- if retired := e.deps.KB.RetireClaimsCoveredBy(evidenceIDs); retired > 0 {
- searchLogger(e.deps).Printf("[list_chunks] retired %d covered claim(s) from the evidence pool", retired)
- }
- status := StatusOK
- if newChunks == 0 {
- status = StatusRedundant
- }
- return ToolOutcome{
- Payload: payload,
- EvidenceIDs: evidenceIDs,
- Status: status,
- Reason: ReasonNone,
- Metrics: map[string]any{"hits": len(payload), "new_evidence": newChunks},
- }, nil
-}
-
-// SearchCacheKey: the tuple of what actually
-// determines a retrieval result. Scope and limits are part of the key so only a
-// genuinely identical query is served from cache.
-func SearchCacheKey(effectiveQuery string, targetIDs []string, topN int, docScope []string) string {
- query := normalizeSpace(effectiveQuery)
- ids := append([]string(nil), targetIDs...)
- sort.Strings(ids)
- docs := append([]string(nil), docScope...)
- sort.Strings(docs)
- return fmt.Sprintf("%s|%v|%d|%v", query, ids, topN, docs)
-}
-
-func normalizeSpace(s string) string {
- return strings.ToLower(strings.Join(strings.Fields(s), " "))
-}
-
-// docInDatasets reports whether docID belongs to the datasets being searched.
-//
-// The second result is false only when ownership could NOT be determined — no
-// verifier injected, or the lookup errored. It is true when the lookup
-// completed, including the "document absent from the bound datasets" case:
-// KnownDocIDs returns a known-subset map, so a foreign/stale doc_id yields an
-// empty map (never an error), and is reported as belongs=false / verified=true so the
-// caller rejects it rather than fetching it unscoped.
-func docInDatasets(ctx context.Context, deps SearchDeps, docID string) (belongs, verified bool) {
- if deps.DocIDVerifier == nil || docID == "" {
- return true, false
- }
- known, err := deps.DocIDVerifier.KnownDocIDs(ctx, deps.KbIDs, []string{docID})
- if err != nil {
- // Fail open only on an actual lookup failure; never on "not found".
- return true, false
- }
- return known[docID], true
-}
-
-// scopedDocIDs
-// the ceiling every search entry point applies before touching the doc store:
-// with no session scope the caller's scope passes through; with a session scope
-// an absent caller scope falls back to it and an explicit one is INTERSECTED
-// with it, so a model-generated doc_scope can never escape the session's
-// document restriction.
-func scopedDocIDs(sessionScope, scope []string) []string {
- if len(sessionScope) == 0 {
- return scope
- }
- if len(scope) == 0 {
- return append([]string(nil), sessionScope...)
- }
- allowed := make(map[string]bool, len(sessionScope))
- for _, d := range sessionScope {
- allowed[d] = true
- }
- out := make([]string, 0, len(scope))
- for _, d := range scope {
- if allowed[d] {
- out = append(out, d)
- }
- }
- return out
-}
-
-// resolveDocScope: apply the session doc_scope ceiling, then keep only the requested
-// document ids that actually belong to the session's datasets.
-//
-// The ceiling: a None request scope falls back to the session's fixed doc_scope
-// (deps.DocScope), and an explicit request scope is intersected with it, so a session
-// bound to a fixed document set keeps searching within it whichever scope the tool
-// passes.
-//
-// Three outcomes are deliberate, because the document scope is supplied by
-// model-generated tool arguments and must never silently change what is searched:
-//
-// - verification unavailable (no verifier injected, or it errored): the scope is
-// passed through unchanged (fail open);
-// - every candidate unknown AND a fixed session base scope: return an EMPTY result,
-// never unfiltered retrieval;
-// - every candidate unknown with NO session base scope: drop the scope and search
-// UNFILTERED.
-//
-// The empty-vs-unfiltered distinction is encoded as a non-nil empty slice vs
-// nil: the backend treats a nil DocScope as "search everything" and a non-nil
-// empty DocScope as "match nothing".
-func resolveDocScope(ctx context.Context, deps SearchDeps, scope, kbIDs []string, logger *log.Logger) []string {
- scope = scopedDocIDs(deps.DocScope, scope)
- if len(scope) == 0 && len(deps.DocScope) > 0 {
- // The session ceiling removed every requested id. Passing the empty scope through
- // would be read as "no filter" and search unfiltered; Go deliberately keeps the
- // safer "match nothing" instead of letting a model-supplied scope escape the
- // session's document restriction.
- return []string{}
- }
- if deps.DocIDVerifier == nil {
- // No verifier: trust the scope as given (fail open).
- return scope
- }
- candidates := make([]string, 0, len(scope))
- for _, d := range scope {
- if d != "" {
- candidates = append(candidates, d)
- }
- }
- if len(candidates) == 0 {
- // No usable candidate ids (request scope None and no session base scope, or only
- // empty strings): search unfiltered. Return nil, not an empty set, so the backend
- // does not filter to zero documents.
- return nil
- }
- known, err := deps.DocIDVerifier.KnownDocIDs(ctx, kbIDs, candidates)
- if err != nil {
- logger.Printf("[Hybrid search] document-id verification unavailable (%d candidate(s)); searching the scope as given", len(candidates))
- return scope
- }
- valid := make([]string, 0, len(candidates))
- for _, d := range candidates {
- if known[d] {
- valid = append(valid, d)
- }
- }
- if len(valid) == 0 {
- // Every candidate is unknown. Two outcomes are distinguished by the SESSION base
- // scope:
- // - a fixed session doc_scope → return an empty result (no documents match);
- // - no session base scope → fall back to UNFILTERED retrieval.
- if len(deps.DocScope) > 0 {
- return []string{}
- }
- logger.Printf("[Hybrid search] every supplied doc ID was unknown; falling back to unfiltered retrieval")
- return nil
- }
- return valid
-}
-
-// DocAggs builds the per-document aggregation from a chunk list: each entry carries
-// doc_id (the key the reporting layer dedups on) plus the counts it reads. The Retriever
-// interface returns only chunks, so the aggregations are derived here.
-func DocAggs(chunks []map[string]any) []map[string]any {
- if len(chunks) == 0 {
- return nil
- }
- type stat struct {
- count int
- chars int
- name string
- }
- order := make([]string, 0, 8)
- stats := map[string]*stat{}
- for _, c := range chunks {
- if c == nil {
- continue
- }
- id := DocIDOf(c)
- if id == "" {
- id = DocTitleOf(c)
- }
- if id == "" {
- id = "?"
- }
- s, ok := stats[id]
- if !ok {
- s = &stat{name: DocTitleOf(c)}
- stats[id] = s
- order = append(order, id)
- }
- s.count++
- s.chars += len(ChunkTextOf(c))
- }
- out := make([]map[string]any, 0, len(order))
- for _, id := range order {
- s := stats[id]
- // Field names match runtime's referenceDocAggsFromRetrieval
- // (doc_id / doc_name / count) so these aggregations can be handed
- // straight to CanvasState.SetRetrievalReferences without translation.
- out = append(out, map[string]any{
- "doc_id": id,
- "doc_name": s.name,
- "count": s.count,
- "char_count": s.chars,
- })
- }
- return out
-}
-
-// searchResultLine is the think-block form of a search leg's result: how many
-// passages came back and how many documents they came from, as a sentence. The
-// query comes last (that is where a reader looks after the numbers) and there is
-// no arrow: "->" is log shorthand, not English, and this line is read by users.
-// An empty result says so in words ("Found nothing for …") rather than counting
-// zero documents.
-//
-// It is deliberately NOT docStatsLine — that one is keyed by 32-hex document id
-// (a developer's handle, and the line Python prints), which is exactly what a
-// reader cannot use. Both are emitted by the same step (StageLineDetail), from
-// the same chunks, so the sentence and the id breakdown always describe the same
-// call.
-func searchResultLine(query string, chunks []map[string]any) string {
- if len(chunks) == 0 {
- return fmt.Sprintf("Found nothing for %q.", trunc(query, 80))
- }
- return fmt.Sprintf("Found %s in %s for %q.", CountOf(len(chunks), "passage"),
- CountOf(len(DocAggs(chunks)), "document"), trunc(query, 80))
-}
-
-// searchResultLogLine renders the same result in the log's shape:
-// `"%q -> N chunk(s): <per-document breakdown>"` (Python search.py:203 for hybrid,
-// :471 for grep). The breakdown is dropped when the leg returned nothing, so a
-// zero-result line does not end on a dangling colon.
-func searchResultLogLine(query string, chunks []map[string]any) string {
- line := fmt.Sprintf("%q -> %d chunk(s)", trunc(query, 80), len(chunks))
- if stats := docStatsLine(chunks); stats != "" {
- line += ": " + stats
- }
- return line
-}
-
-// reportSearchResult is how a leg ends: it reports what the leg found, to both
-// audiences, from one place.
-//
-// It runs on EVERY exit — an empty result and a cached one included — because a leg
-// with no result line cannot be told apart from a leg that found nothing, and that
-// is the one question the block exists to answer.
-//
-// The log keeps Python's line shape where Python has one. The vector and bm25 legs
-// are the exception: Python logs nothing there beyond the searching line
-// (search.py:215/252), and Go deliberately adds the result line, because those legs
-// are the ones whose pool size a reader most needs.
-func reportSearchResult(ctx context.Context, logger *log.Logger, label, query string, chunks []map[string]any) {
- StepsFrom(ctx).StageLineDetail(logger, label,
- searchResultLine(query, chunks), searchResultLogLine(query, chunks))
-}
-
-// docStatsLine renders the per-document breakdown used by the search log line.
-func docStatsLine(chunks []map[string]any) string {
- parts := make([]string, 0, 8)
- for _, a := range DocAggs(chunks) {
- parts = append(parts, fmt.Sprintf("%s:%vchunk(%vchars)",
- a["doc_id"], a["count"], a["char_count"]))
- }
- return strings.Join(parts, "; ")
-}
-
-// Normalisation of web-search payloads.
-//
-// Web results are shaped so they enter kbinfos with the same field names every other
-// chunk carries.
-
-// normalizeWebResults converts a raw web-search JSON payload into chunk maps.
-// Accepts either {"chunks": [...]} or {"results": [...]}; entries without both a
-// URL and content are dropped.
-func normalizeWebResults(raw []byte) []map[string]any {
- var res struct {
- Chunks []map[string]any `json:"chunks"`
- Results []map[string]any `json:"results"`
- }
- if err := json.Unmarshal(raw, &res); err != nil {
- return nil
- }
- src := res.Chunks
- if len(src) == 0 {
- src = res.Results
- }
- out := make([]map[string]any, 0, len(src))
- for i, c := range src {
- url := firstNonEmpty(anyString(c["url"]), anyString(c["link"]), anyString(c["source"]))
- if url == "" {
- continue
- }
- content := firstNonEmpty(anyString(c["content"]), anyString(c["raw_content"]), anyString(c["text"]))
- if content == "" {
- continue
- }
- docID := anyString(c["doc_id"])
- if docID == "" {
- docID = url
- }
- // doc_id stays the source URL; chunk_id must be UNIQUE per snippet so
- // Kbinfos.Merge (dedup by chunkKey) does not collapse several snippets
- // from the same URL, or the same URL across two retrieval rounds.
- chunkID := fmt.Sprintf("%s#%d", docID, i)
- out = append(out, map[string]any{
- "chunk_id": chunkID,
- "content_with_weight": content,
- "doc_id": docID,
- "docnm_kwd": firstNonEmpty(anyString(c["title"]), anyString(c["source"])),
- "dataset_id": anyString(c["dataset_id"]),
- "url": url,
- "source": "web",
- })
- }
- return out
-}
-
-// firstNonEmpty returns the first argument that is not blank.
-func firstNonEmpty(ss ...string) string {
- for _, s := range ss {
- if strings.TrimSpace(s) != "" {
- return s
- }
- }
- return ""
-}
-
-// Per-request retrieval cache.
-//
-// The cache is keyed by the search parameters; an identical (query, scope, limits)
-// search is served from cache instead of hitting the retriever again. It is attached to
-// Kbinfos and exposed as methods here, so the search tool and its cache live in one
-// file.
-
-// searchCache is the per-request retrieval cache.
-type searchCache struct {
- mu sync.Mutex
- entries map[string]cachedSearch
-}
-
-// cachedSearch is one cache entry.
-type cachedSearch struct {
- chunks []map[string]any
- aggs []map[string]any
-}
-
-// SearchCacheLoad returns a cached retrieval for key, or (nil, nil, false) when
-// the key is absent.
-func (k *Kbinfos) SearchCacheLoad(key string) ([]map[string]any, []map[string]any, bool) {
- c := k.ensureCache()
- c.mu.Lock()
- defer c.mu.Unlock()
- e, ok := c.entries[key]
- if !ok {
- return nil, nil, false
- }
- return e.chunks, e.aggs, true
-}
-
-// SearchCacheStore records a retrieval under key.
-func (k *Kbinfos) SearchCacheStore(key string, chunks, aggs []map[string]any) {
- c := k.ensureCache()
- c.mu.Lock()
- defer c.mu.Unlock()
- if c.entries == nil {
- c.entries = make(map[string]cachedSearch)
- }
- c.entries[key] = cachedSearch{chunks: chunks, aggs: aggs}
-}
-
-// ensureCache lazily initialises the per-request cache so a zero-value Kbinfos
-// is usable without an explicit setup step.
-func (k *Kbinfos) ensureCache() *searchCache {
- k.cacheOnce.Do(func() {
- if k.cache == nil {
- k.cache = &searchCache{}
- }
- })
- return k.cache
-}
diff --git a/internal/rag/agentic-rag/runtime/tool_search_test.go b/internal/rag/agentic-rag/runtime/tool_search_test.go
deleted file mode 100644
index 7850eba493..0000000000
--- a/internal/rag/agentic-rag/runtime/tool_search_test.go
+++ /dev/null
@@ -1,2145 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "bytes"
- "context"
- "encoding/json"
- "errors"
- "io"
- "log"
- "net/http"
- "net/http/httptest"
- "reflect"
- "strings"
- "testing"
- "unicode/utf8"
-
- "ragflow/internal/common"
- "ragflow/internal/entity"
-)
-
-// stubRetriever returns a fixed result and records the requests it received.
-type stubRetriever struct {
- chunks []map[string]any
- err error
- requests []RetrieveRequest
-}
-
-func (s *stubRetriever) Retrieve(_ context.Context, req RetrieveRequest) ([]map[string]any, error) {
- s.requests = append(s.requests, req)
- if s.err != nil {
- return nil, s.err
- }
- return s.chunks, nil
-}
-
-func newTestSearchDeps(r Retriever) (SearchDeps, *Kbinfos) {
- kb := &Kbinfos{}
- return SearchDeps{
- Backend: r,
- KbIDs: []string{"kb1"},
- TenantID: "tenant1",
- KB: kb,
- }, kb
-}
-
-func TestHybridSearchBailsWithoutDatasetsOrBackend(t *testing.T) {
- // No bound datasets -> no search at all (empty kbinfos).
- got, aggs := HybridSearch(context.Background(), SearchDeps{
- Backend: &stubRetriever{chunks: []map[string]any{{"content": "x"}}},
- }, SearchParams{Question: "q"})
- if got != nil || aggs != nil {
- t.Error("no datasets must short-circuit to empty")
- }
- // No backend -> empty.
- deps, _ := newTestSearchDeps(nil)
- if got, _ := HybridSearch(context.Background(), deps, SearchParams{Question: "q"}); got != nil {
- t.Error("no backend must return empty")
- }
-}
-
-func TestHybridSearchBuildsEffectiveQuery(t *testing.T) {
- // Weighted query wins, and the result is capped at 400 chars.
- r := &stubRetriever{chunks: []map[string]any{{"content": "hit"}}}
- deps, _ := newTestSearchDeps(r)
- long := strings.Repeat("weighted ", 100)
- HybridSearch(context.Background(), deps, SearchParams{
- Question: "who made it?", RetrievalQuery: long,
- })
- if len(r.requests) != 1 {
- t.Fatalf("requests = %d, want 1", len(r.requests))
- }
- q := r.requests[0].Query
- if !strings.HasPrefix(q, "who made it? weighted") {
- t.Errorf("query = %q, want question + weighted query", q)
- }
- if len(q) != maxEffectiveQueryChars {
- t.Errorf("query length = %d, want capped at %d", len(q), maxEffectiveQueryChars)
- }
-
- // No weighted query -> keywords fall into the query text.
- r = &stubRetriever{chunks: []map[string]any{{"content": "hit"}}}
- deps, _ = newTestSearchDeps(r)
- HybridSearch(context.Background(), deps, SearchParams{Question: "who made it?", Keywords: "culdcept"})
- if r.requests[0].Query != "who made it? culdcept" {
- t.Errorf("query = %q, want keywords appended", r.requests[0].Query)
- }
-
- // Neither -> the bare question.
- r = &stubRetriever{chunks: []map[string]any{{"content": "hit"}}}
- deps, _ = newTestSearchDeps(r)
- HybridSearch(context.Background(), deps, SearchParams{Question: "who made it?"})
- if r.requests[0].Query != "who made it?" {
- t.Errorf("query = %q, want the bare question", r.requests[0].Query)
- }
-}
-
-// TestHybridSearchEffectiveQueryCapsCodePoints pins the expanded-query cap to code points,
-// not bytes: a byte slice both splits a multi-byte rune — handing the retriever invalid
-// UTF-8 — and caps a CJK query at ~133 characters, dropping expansion terms the fan-out
-// leg weighs on. The ASCII case above cannot tell the two apart (bytes == runes there).
-func TestHybridSearchEffectiveQueryCapsCodePoints(t *testing.T) {
- // "who made it" + " " is 12 bytes, so byte 400 lands one byte inside a CJK
- // rune (400-12 = 388 = 3*129 + 1): the byte slice is not even valid UTF-8.
- expansion := strings.Repeat("知识", 300) // 600 runes, 1800 bytes
- r := &stubRetriever{chunks: []map[string]any{{"content": "hit"}}}
- deps, _ := newTestSearchDeps(r)
- HybridSearch(context.Background(), deps, SearchParams{
- Question: "who made it", RetrievalQuery: expansion,
- })
-
- if len(r.requests) != 1 {
- t.Fatalf("requests = %d, want 1", len(r.requests))
- }
- q := r.requests[0].Query
- if !utf8.ValidString(q) {
- t.Fatalf("query is not valid UTF-8 — a byte slice cut a rune in half: %q", q)
- }
- if got := utf8.RuneCountInString(q); got != maxEffectiveQueryChars {
- t.Errorf("query runes = %d, want the cap at %d code points (%d bytes)", got, maxEffectiveQueryChars, len(q))
- }
- full := "who made it " + expansion
- if want := string([]rune(full)[:maxEffectiveQueryChars]); q != want {
- t.Errorf("query = %q, want the first %d code points %q", q, maxEffectiveQueryChars, want)
- }
- // 400 code points of mostly-CJK text is far more than 400 bytes: the cap is a
- // character budget, not a byte budget.
- if len(q) <= maxEffectiveQueryChars {
- t.Errorf("query bytes = %d, want > %d (the cap counts code points)", len(q), maxEffectiveQueryChars)
- }
-}
-
-func TestRankFeatureOnlyOnRetrieveLeg(t *testing.T) {
- // rank_feature is passed ONLY by the retrieve channel; the three search legs call the
- // retriever WITHOUT it (the grep leg delegates to bm25), so those requests must stay
- // nil even with a Tagger.
- r := &stubRetriever{chunks: []map[string]any{{"content": "hit"}}}
- deps, _ := newTestSearchDeps(r)
- deps.KBs = []*entity.Knowledgebase{{}}
- deps.Tagger = stubTagger{t: t}
-
- HybridSearch(context.Background(), deps, SearchParams{Question: "who made it?"})
- VectorSearch(context.Background(), deps, SearchParams{Question: "who made it?"})
- BM25Search(context.Background(), deps, SearchParams{Question: "who made it?"})
- GrepSearch(context.Background(), deps, SearchParams{Question: "who made it?"})
-
- want := map[string]float64{"definition": 1.0, "entity": 1.0}
- for i, req := range r.requests {
- if req.RankFeature != nil {
- t.Errorf("request %d (%s) RankFeature = %v, want nil (search.py legs never pass rank_feature)", i, req.Query, req.RankFeature)
- }
- }
-
- // RAGTools.retrieve is the only leg that carries the label_question boost.
- r2 := &stubRetriever{chunks: []map[string]any{{"content": "hit"}}}
- deps2, _ := newTestSearchDeps(r2)
- deps2.KBs = []*entity.Knowledgebase{{}}
- deps2.Tagger = stubTagger{t: t}
- deps2.UsingEmbedding = true
- RetrieveSearch(context.Background(), deps2, SearchParams{Question: "who made it?"})
- if len(r2.requests) != 1 {
- t.Fatalf("requests = %d, want 1", len(r2.requests))
- }
- if got := r2.requests[0].RankFeature; !reflect.DeepEqual(got, want) {
- t.Errorf("RetrieveSearch RankFeature = %v, want %v", got, want)
- }
-
- // Nil Tagger -> empty rank feature even on the retrieve leg.
- r3 := &stubRetriever{chunks: []map[string]any{{"content": "hit"}}}
- deps3, _ := newTestSearchDeps(r3)
- deps3.UsingEmbedding = true
- RetrieveSearch(context.Background(), deps3, SearchParams{Question: "unique query no rf"})
- if len(r3.requests) != 1 {
- t.Fatalf("requests = %d, want 1", len(r3.requests))
- }
- if len(r3.requests[0].RankFeature) != 0 {
- t.Errorf("RankFeature = %v, want empty for nil Tagger", r3.requests[0].RankFeature)
- }
-}
-
-// stubTagger implements QuestionLabeler for TestHybridSearchPassesRankFeature.
-type stubTagger struct{ t *testing.T }
-
-func (s stubTagger) LabelQuestion(_ context.Context, question string, kbs []*entity.Knowledgebase) map[string]float64 {
- if question != "who made it?" {
- s.t.Errorf("LabelQuestion called with question=%q", question)
- }
- if len(kbs) != 1 {
- s.t.Errorf("LabelQuestion called with %d kbs, want 1", len(kbs))
- }
- return map[string]float64{"definition": 1.0, "entity": 1.0}
-}
-
-func TestHybridSearchCachesIdenticalQuery(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{{"content": "hit"}}}
- deps, _ := newTestSearchDeps(r)
- p := SearchParams{Question: "same question"}
- HybridSearch(context.Background(), deps, p)
- HybridSearch(context.Background(), deps, p)
- if len(r.requests) != 1 {
- t.Errorf("backend calls = %d, want 1 (second served from cache)", len(r.requests))
- }
- // A different scope is a different retrieval.
- HybridSearch(context.Background(), deps, SearchParams{Question: "same question", DocScope: []string{"doc9"}})
- if len(r.requests) != 2 {
- t.Errorf("backend calls = %d, want 2 (doc_scope is part of the key)", len(r.requests))
- }
- // A different TopN is a different retrieval.
- HybridSearch(context.Background(), deps, SearchParams{Question: "same question", TopN: 5})
- if len(r.requests) != 3 {
- t.Errorf("backend calls = %d, want 3 (top_n is part of the key)", len(r.requests))
- }
-}
-
-func TestSearchCacheKeyIgnoresOrderAndWhitespace(t *testing.T) {
- a := SearchCacheKey(" Who Made It ", []string{"b", "a"}, 12, []string{"d2", "d1"})
- b := SearchCacheKey("who made it", []string{"a", "b"}, 12, []string{"d1", "d2"})
- if a != b {
- t.Errorf("cache key not canonical:\n%q\n%q", a, b)
- }
-}
-
-func TestHybridSearchStoresMemoryBeforeNarrowing(t *testing.T) {
- // The raw chunk must reach memory even when narrowing rewrites the copy
- // handed downstream.
- r := &stubRetriever{chunks: []map[string]any{{
- "chunk_id": "c1",
- "content": "Alpha sentence. Culdcept was made by OmiyaSoft in 1999. Omega sentence.",
- "doc_id": "d1",
- }}}
- deps, kb := newTestSearchDeps(r)
- HybridSearch(context.Background(), deps, SearchParams{
- Question: "who made Culdcept?", Keywords: "culdcept",
- })
- if MemorySize(kb) != 1 {
- t.Fatalf("memory size = %d, want 1", MemorySize(kb))
- }
- if strings.Contains(ChunkTextOf(kb.Memory[0]), "*Culdcept*") {
- t.Error("memory must hold the RAW chunk, not the narrowed/highlighted copy")
- }
-}
-
-func TestHybridSearchReturnsEmptyOnBackendError(t *testing.T) {
- deps, _ := newTestSearchDeps(&stubRetriever{err: errors.New("ES down")})
- got, aggs := HybridSearch(context.Background(), deps, SearchParams{Question: "q"})
- if len(got) != 0 || len(aggs) != 0 {
- t.Errorf("a backend failure must yield empty results, got %d chunks", len(got))
- }
-}
-
-// Narrowing
-
-func TestNarrowOrKeepIsAllOrNothing(t *testing.T) {
- var lines []string
- ctx := WithSteps(context.Background(), StepReporter{
- Text: func(line string) { lines = append(lines, line) },
- })
- var logged bytes.Buffer
- chunks := []map[string]any{
- {"content": "Culdcept was made by OmiyaSoft."},
- {"content": "It was released in 1999."},
- }
- // Keywords hit -> narrowed subset (the non-matching chunk is dropped).
- got := NarrowOrKeep(ctx, chunks, "culdcept", "test", log.New(&logged, "", 0))
- if len(got) != 1 || !strings.Contains(ChunkTextOf(got[0]), "Culdcept") {
- t.Errorf("narrowed = %v, want only the matching chunk", got)
- }
- // How the filter resized the pool is a developer's diagnostic: it goes to the
- // log with Python's own wording (text_processing.py:464) and NOT to the think
- // block, where the leg's result line already reports what came back.
- if !strings.Contains(logged.String(), "[test] Kept 1 of 2 passage(s) that actually mention the keywords.") {
- t.Errorf("narrowed log line = %q", logged.String())
- }
- if len(lines) != 0 {
- t.Errorf("narrowing must not reach the think block: %q", strings.Join(lines, "\n"))
- }
-
- // No keyword overlap -> ALL chunks kept (this is the whole point: the
- // retriever already ranked them, and dropping everything produced empty
- // results and unverified claims).
- logged.Reset()
- got = NarrowOrKeep(ctx, chunks, "zzz-no-match", "test", log.New(&logged, "", 0))
- if len(got) != len(chunks) {
- t.Errorf("no-match kept %d, want all %d", len(got), len(chunks))
- }
- if !strings.Contains(logged.String(),
- "[test] Keyword narrowing matched nothing — keeping all 2 retrieved passage(s).") {
- t.Errorf("no-match log line = %q", logged.String())
- }
- if len(lines) != 0 {
- t.Errorf("the no-match line must not reach the think block either: %q", strings.Join(lines, "\n"))
- }
-
- // Empty keywords -> untouched, and silent: nothing was narrowed, so there is
- // nothing to report.
- logged.Reset()
- if got := NarrowOrKeep(ctx, chunks, "", "test", log.New(&logged, "", 0)); len(got) != len(chunks) {
- t.Error("empty keywords must return chunks unchanged")
- }
- if logged.Len() != 0 || len(lines) != 0 {
- t.Errorf("empty keywords must report nothing; log = %q, think = %v", logged.String(), lines)
- }
-}
-
-func TestNarrowContentKeepsNeighboursAndHighlights(t *testing.T) {
- content := "Alpha filler. Culdcept was made by OmiyaSoft. Omega filler."
- got, ok := NarrowContent(content, []string{"culdcept"})
- if !ok {
- t.Fatal("NarrowContent must match")
- }
- if !strings.Contains(got, "*Culdcept*") {
- t.Errorf("keyword not highlighted: %q", got)
- }
- // +/-1 neighbour window: Alpha and Omega should be present.
- if !strings.Contains(got, "Alpha") || !strings.Contains(got, "Omega") {
- t.Errorf("neighbour sentences missing: %q", got)
- }
- if !strings.HasPrefix(got, "...") || !strings.HasSuffix(got, "...") {
- t.Errorf("narrowed text must be wrapped in ellipses: %q", got)
- }
- if _, ok := NarrowContent("nothing relevant here", []string{"culdcept"}); ok {
- t.Error("no match must return false")
- }
-}
-
-// TestNarrowContentRendersHTMLTablesAsMarkdown pins the table branch: an HTML table is
-// serialized to a Markdown view before the model sees it (raw <table>/<td> markup is the
-// expensive and least readable form), and the row set is not pruned.
-func TestNarrowContentRendersHTMLTablesAsMarkdown(t *testing.T) {
- content := "<table><tr><th>Rank</th><th>Rider</th><th>Points</th></tr>" +
- "<tr><td>19</td><td>Danilo</td><td>62</td></tr>" +
- "<tr><td>20</td><td>Erik</td><td>61</td></tr></table>"
- got, ok := NarrowContent(content, []string{"danilo"})
- if !ok {
- t.Fatal("NarrowContent must keep a table whole")
- }
- if strings.Contains(got, "<td>") || strings.Contains(got, "<table") {
- t.Errorf("narrowed table still carries raw HTML: %q", got)
- }
- for _, want := range []string{"| 19 |", "Danilo", "| 20 |", "Erik"} {
- if !strings.Contains(got, want) {
- t.Errorf("narrowed table lost %q: %q", want, got)
- }
- }
- if !strings.HasPrefix(got, "...") || !strings.HasSuffix(got, "...") {
- t.Errorf("narrowed text must be wrapped in ellipses: %q", got)
- }
-}
-
-func TestSplitKeywordsFallsBackToBigrams(t *testing.T) {
- // >=3 comma terms -> used as-is, lower-cased.
- got := SplitKeywords("Alpha, Beta, Gamma")
- if len(got) != 3 || got[0] != "alpha" {
- t.Errorf("comma split = %v", got)
- }
- // <3 terms -> bigrams are more discriminative than single words.
- got = SplitKeywords("finale run time")
- if len(got) != 2 || got[0] != "finale run" || got[1] != "run time" {
- t.Errorf("bigram fallback = %v, want [finale run, run time]", got)
- }
- if SplitKeywords(" ") != nil {
- t.Error("blank keywords must yield no terms")
- }
-}
-
-func TestHighlightKeywordsPrefersLongestTerm(t *testing.T) {
- // "new york" must win over "york" so the shorter term cannot split it.
- got := HighlightKeywords("welcome to New York city", []string{"york", "new york"})
- // The star marker, and ONE span for the multi-word entity — never "*New* *York*".
- if !strings.Contains(got, "*New York*") {
- t.Errorf("highlight = %q, want the longest term applied", got)
- }
- if strings.Contains(got, "*york*") {
- t.Errorf("shorter term must not win: %q", got)
- }
-}
-
-// TestHighlightKeywordsFoldsWithoutByteOffsets pins the matching to rune space.
-// `strings.ToLower` is not byte-length-preserving: "İ" (U+0130) is 2 bytes and folds to
-// the 1-byte "i" (Go applies the simple 1:1 case mapping, the opposite direction from a
-// full Unicode fold, which expands it to 3 bytes). So a byte
-// offset taken from the original indexes the folded string at a different
-// position: the loop then runs past its end (panic: slice bounds out of range
-// [10:9]) or cuts a rune in half (invalid UTF-8). Corpus text reaches this via
-// NarrowContent, e.g. a Turkish document.
-func TestHighlightKeywordsFoldsWithoutByteOffsets(t *testing.T) {
- // "İstanbul" is 9 bytes but folds to the 8-byte "istanbul", so the old byte
- // loop wrote text[0:8] — one byte short, ending mid-word ("İstanbu").
- if got := HighlightKeywords("İstanbul is here", []string{"istanbul"}); got != "*İstanbul* is here" {
- t.Errorf("highlight = %q, want the whole word wrapped", got)
- }
-
- // Two shrunken runes push the byte index one past the end of the folded
- // string while the original still has a byte left: the old loop panicked.
- got := HighlightKeywords("İİstanbul", []string{"istanbul"})
- if !utf8.ValidString(got) {
- t.Fatalf("highlight is not valid UTF-8 — a span split a rune: %q", got)
- }
- if !strings.Contains(got, "*İstanbul*") {
- t.Errorf("highlight = %q, want the fold-matched runes wrapped in place", got)
- }
-}
-
-// TestHighlightKeywordsFoldsUppercaseKeywords pins that a keyword's OWN casing is
-// folded the way the haystack is: terms are matched against the lowercased text (`lows`),
-// and the phrase list is built with a strip+lower pass plus case-insensitive matching. A
-// caller-supplied "Rocket"
-// used to be compared verbatim, so a capitalised keyword never matched and the
-// span was silently left unstarred.
-func TestHighlightKeywordsFoldsUppercaseKeywords(t *testing.T) {
- for _, tc := range []struct {
- name string
- text string
- kwds []string
- want string
- }{
- {"single word", "the Rocket launched", []string{"Rocket"}, "the *Rocket* launched"},
- {"multi-word phrase stays one span", "welcome to New York city", []string{"New York"}, "welcome to *New York* city"},
- {"source casing is preserved", "ROCKET is loud", []string{"Rocket"}, "*ROCKET* is loud"},
- {"surrounding spaces are trimmed", "the Rocket launched", []string{" Rocket "}, "the *Rocket* launched"},
- } {
- t.Run(tc.name, func(t *testing.T) {
- if got := HighlightKeywords(tc.text, tc.kwds); got != tc.want {
- t.Errorf("HighlightKeywords(%q, %v) = %q, want %q", tc.text, tc.kwds, got, tc.want)
- }
- })
- }
-}
-
-// TestHighlightKeywordsKeepsPhrasePartsWhole pins the phrase guard: a stem-matched word
-// that already occurs inside a keyword phrase is NOT added as a term of its own, so a
-// standalone "Braves" is
-// left alone and only the "Atlanta Braves" span is starred.
-func TestHighlightKeywordsKeepsPhrasePartsWhole(t *testing.T) {
- got := HighlightKeywords("Braves lost. Atlanta Braves won.", []string{"Atlanta Braves"})
- if want := "Braves lost. *Atlanta Braves* won."; got != want {
- t.Errorf("highlight = %q, want %q", got, want)
- }
-}
-
-// TestHighlightKeywordsStemMatchesCapitalisedWords pins the word scan: every `[A-Za-z]+`
-// word is stemmed, so a capitalised inflected word still contributes its stem term. The
-// shared lowercase pattern matched only the
-// fragment after the capital ("Nominated" -> "ominated"), so nothing was starred.
-func TestHighlightKeywordsStemMatchesCapitalisedWords(t *testing.T) {
- got := HighlightKeywords("Nominated twice.", []string{"nominations"})
- if want := "*Nominated* twice."; got != want {
- t.Errorf("highlight = %q, want %q", got, want)
- }
-}
-
-// Doc aggregations
-
-func TestDocAggsGroupsByDocument(t *testing.T) {
- chunks := []map[string]any{
- {"doc_id": "d1", "docnm_kwd": "Doc One", "content": "aaaa"},
- {"doc_id": "d1", "docnm_kwd": "Doc One", "content": "bb"},
- {"doc_id": "d2", "docnm_kwd": "Doc Two", "content": "c"},
- }
- aggs := DocAggs(chunks)
- if len(aggs) != 2 {
- t.Fatalf("aggs = %d, want 2 documents", len(aggs))
- }
- // Order follows first appearance, so the log line is stable.
- if aggs[0]["doc_id"] != "d1" || aggs[0]["count"] != 2 {
- t.Errorf("d1 agg = %v, want count 2", aggs[0])
- }
- if aggs[0]["char_count"] != 6 {
- t.Errorf("d1 char_count = %v, want 6 (4+2)", aggs[0]["char_count"])
- }
- if aggs[1]["doc_id"] != "d2" || aggs[1]["count"] != 1 {
- t.Errorf("d2 agg = %v", aggs[1])
- }
- if DocAggs(nil) != nil {
- t.Error("no chunks must yield no aggregations")
- }
-}
-
-func TestHybridSearchAggregatesAndRespectsScope(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{
- {"doc_id": "d1", "content": "a"},
- {"doc_id": "d2", "content": "b"},
- }}
- deps, _ := newTestSearchDeps(r)
- chunks, aggs := HybridSearch(context.Background(), deps, SearchParams{
- Question: "q", DocScope: []string{"d1"},
- })
- if len(chunks) != 2 || len(aggs) != 2 {
- t.Fatalf("chunks=%d aggs=%d", len(chunks), len(aggs))
- }
- // doc_scope must reach the backend.
- if len(r.requests[0].DocScope) != 1 || r.requests[0].DocScope[0] != "d1" {
- t.Errorf("doc_scope not forwarded: %v", r.requests[0].DocScope)
- }
- // Explicit kb_ids override the session default.
- r.requests = nil
- HybridSearch(context.Background(), deps, SearchParams{Question: "q", KbIDs: []string{"kb9"}})
- if len(r.requests[0].DatasetIDs) != 1 || r.requests[0].DatasetIDs[0] != "kb9" {
- t.Errorf("kb_ids not honoured: %v", r.requests[0].DatasetIDs)
- }
-}
-
-func TestHybridSearchAggsCoverFullRetrievedBeforeNarrowing(t *testing.T) {
- // doc_aggs must reflect the FULL pre-narrow retrieved set: the aggregations come
- // straight from the retriever and narrowing only replaces the chunk list. Here the
- // keyword narrows the chunks to doc d1,
- // but the aggregation still counts both retrieved documents.
- r := &stubRetriever{chunks: []map[string]any{
- {"doc_id": "d1", "docnm_kwd": "one", "content": "needle buried here", "id": "c1"},
- {"doc_id": "d2", "docnm_kwd": "two", "content": "nothing relevant", "id": "c2"},
- }}
- deps, _ := newTestSearchDeps(r)
- chunks, aggs := HybridSearch(context.Background(), deps, SearchParams{
- Question: "q", Keywords: "needle",
- })
- if len(chunks) != 1 {
- t.Fatalf("narrowed chunks = %d, want 1 (d1 only)", len(chunks))
- }
- if len(aggs) != 2 {
- t.Fatalf("aggs = %d, want 2 (full pre-narrow set: d1 and d2)", len(aggs))
- }
-}
-
-func TestHybridSearchInvokesCompiledExpansion(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{{"content": "hit"}}}
- deps, _ := newTestSearchDeps(r)
- exp := &stubExpander{}
- deps.Expand = exp
- HybridSearch(context.Background(), deps, SearchParams{Question: "q", UseCompiled: true})
- if exp.calls != 1 {
- t.Errorf("compiled expansion calls = %d, want 1", exp.calls)
- }
- // Without UseCompiled it must not run.
- exp.calls = 0
- HybridSearch(context.Background(), deps, SearchParams{Question: "other", UseCompiled: false})
- if exp.calls != 0 {
- t.Error("compiled expansion must be opt-in")
- }
-}
-
-type stubExpander struct{ calls int }
-
-// stubMetadataResolver is a scripted MetadataResolver: the push-down answer, the
-// flattened view and the call log are all set per test.
-type stubMetadataResolver struct {
- pushdownIDs []string
- pushdownOK bool
- metas common.MetaData
- flattenErr error
- pushdownCalls int
- flattenCalls int
- gotKbIDs []string
- gotFilters []map[string]any
- gotLogic string
- // docMeta is the per-document metadata the context block is built from; docMetaErr lets
- // a test pin that a failed context read still returns the selected ids.
- docMeta map[string]map[string]any
- docMetaErr error
-}
-
-func (s *stubMetadataResolver) FilterDocIDsByMetaPushdown(_ context.Context, kbIDs []string, filters []map[string]any, logic string) ([]string, bool) {
- s.pushdownCalls++
- s.gotKbIDs = kbIDs
- s.gotFilters = filters
- s.gotLogic = logic
- return s.pushdownIDs, s.pushdownOK
-}
-
-func (s *stubMetadataResolver) GetFlattedMetaByKBs(context.Context, []string) (common.MetaData, error) {
- s.flattenCalls++
- return s.metas, s.flattenErr
-}
-
-func (s *stubMetadataResolver) MetadataForDocIDs(context.Context, []string, []string) (map[string]map[string]any, error) {
- return s.docMeta, s.docMetaErr
-}
-
-// TestMetadataSearchScopesHybridToMatchedDocuments pins the retrieval leg: the metadata
-// match decides the document set, the hybrid search runs inside it (compiled expansion
-// OFF, so nothing outside the set can be pulled in), and the push-down hit means the
-// flattened view is never read.
-func TestMetadataSearchScopesHybridToMatchedDocuments(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{{"id": "c1", "doc_id": "d1", "content": "hit"}}}
- deps, _ := newTestSearchDeps(r)
- res := &stubMetadataResolver{pushdownOK: true, pushdownIDs: []string{"d1", "d2"}}
- deps.MetadataResolver = res
-
- filters := []map[string]any{{"key": "title", "op": "contains", "value": "New York"}}
- chunks, _ := MetadataSearch(context.Background(), deps, SearchParams{Question: "how many?", TopN: 20}, filters, "and")
-
- if len(chunks) != 1 {
- t.Fatalf("chunks = %d, want 1", len(chunks))
- }
- if res.flattenCalls != 0 {
- t.Errorf("flatten calls = %d: a push-down hit must not read the flattened view", res.flattenCalls)
- }
- if res.gotLogic != "and" || len(res.gotFilters) != 1 {
- t.Errorf("resolver got filters=%v logic=%q", res.gotFilters, res.gotLogic)
- }
- if len(res.gotKbIDs) != 1 || res.gotKbIDs[0] != "kb1" {
- t.Errorf("resolver kbIDs = %v, want the session's datasets", res.gotKbIDs)
- }
- if len(r.requests) != 1 {
- t.Fatalf("requests = %d, want 1", len(r.requests))
- }
- scope := r.requests[0].DocScope
- if len(scope) != 2 || scope[0] != "d1" || scope[1] != "d2" {
- t.Errorf("doc_scope = %v, want the matched documents", scope)
- }
-}
-
-// TestMetadataSearchFallsBackToInMemoryFilter pins the fallback: when the push-down is
-// not viable, the flattened metadata is filtered in memory with the same conditions.
-func TestMetadataSearchFallsBackToInMemoryFilter(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{{"id": "c1", "doc_id": "d1", "content": "hit"}}}
- deps, _ := newTestSearchDeps(r)
- res := &stubMetadataResolver{
- pushdownOK: false,
- metas: common.MetaData{
- "title": {"New York City": {"d1"}, "Boston": {"d2"}},
- },
- }
- deps.MetadataResolver = res
-
- chunks, _ := MetadataSearch(context.Background(), deps, SearchParams{Question: "q"},
- []map[string]any{{"key": "title", "op": "contains", "value": "New York"}}, "and")
-
- if len(chunks) != 1 {
- t.Fatalf("chunks = %d, want the in-memory hit", len(chunks))
- }
- if res.flattenCalls != 1 {
- t.Errorf("flatten calls = %d, want 1", res.flattenCalls)
- }
- if scope := r.requests[0].DocScope; len(scope) != 1 || scope[0] != "d1" {
- t.Errorf("doc_scope = %v, want the in-memory match", scope)
- }
-}
-
-// TestMetadataSearchEmptyResultIsNotASearch pins the miss path: an empty match (definitive
-// from the push-down, or after the session ceiling) must reach the retriever zero times,
-// and must not be reported as an error.
-func TestMetadataSearchEmptyResultIsNotASearch(t *testing.T) {
- filters := []map[string]any{{"key": "title", "op": "contains", "value": "nowhere"}}
-
- // Push-down says: definitively no match.
- r := &stubRetriever{chunks: []map[string]any{{"content": "x"}}}
- deps, _ := newTestSearchDeps(r)
- deps.MetadataResolver = &stubMetadataResolver{pushdownOK: true}
- if chunks, _ := MetadataSearch(context.Background(), deps, SearchParams{Question: "q"}, filters, "and"); chunks != nil {
- t.Errorf("chunks = %v, want nil", chunks)
- }
- if len(r.requests) != 0 {
- t.Errorf("requests = %d: an empty match must not search", len(r.requests))
- }
-
- // The session ceiling removes every matched document.
- r2 := &stubRetriever{chunks: []map[string]any{{"content": "x"}}}
- deps2, _ := newTestSearchDeps(r2)
- deps2.DocScope = []string{"d9"}
- deps2.MetadataResolver = &stubMetadataResolver{pushdownOK: true, pushdownIDs: []string{"d1"}}
- if chunks, _ := MetadataSearch(context.Background(), deps2, SearchParams{Question: "q"}, filters, "and"); chunks != nil {
- t.Errorf("chunks = %v, want nil", chunks)
- }
- if len(r2.requests) != 0 {
- t.Errorf("requests = %d: a scope that excludes everything must not search", len(r2.requests))
- }
-}
-
-// TestMetadataSearchIntersectsTheSessionScope pins the ceiling: the metadata set cannot
-// escape the session's document restriction.
-func TestMetadataSearchIntersectsTheSessionScope(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{{"id": "c1", "doc_id": "d2", "content": "hit"}}}
- deps, _ := newTestSearchDeps(r)
- deps.DocScope = []string{"d2", "d3"}
- deps.MetadataResolver = &stubMetadataResolver{pushdownOK: true, pushdownIDs: []string{"d1", "d2", "d3"}}
-
- filters := []map[string]any{{"key": "title", "op": "contains", "value": "x"}}
- if chunks, _ := MetadataSearch(context.Background(), deps, SearchParams{Question: "q"}, filters, "and"); len(chunks) != 1 {
- t.Fatalf("chunks = %d, want 1", len(chunks))
- }
- scope := r.requests[0].DocScope
- if len(scope) != 2 || scope[0] != "d2" || scope[1] != "d3" {
- t.Errorf("doc_scope = %v, want the intersection with the session scope", scope)
- }
-}
-
-// TestMetadataSearchWithoutResolverIsInert pins the unwired seam: no resolver (or no
-// filters) means the leg does nothing at all, never a failed search.
-func TestMetadataSearchWithoutResolverIsInert(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{{"content": "x"}}}
- deps, _ := newTestSearchDeps(r)
- filters := []map[string]any{{"key": "title", "op": "contains", "value": "x"}}
-
- if chunks, _ := MetadataSearch(context.Background(), deps, SearchParams{Question: "q"}, filters, "and"); chunks != nil {
- t.Errorf("chunks = %v, want nil without a resolver", chunks)
- }
- deps.MetadataResolver = &stubMetadataResolver{pushdownOK: true, pushdownIDs: []string{"d1"}}
- if chunks, _ := MetadataSearch(context.Background(), deps, SearchParams{Question: "q"}, nil, "and"); chunks != nil {
- t.Errorf("chunks = %v, want nil without filters", chunks)
- }
- if len(r.requests) != 0 {
- t.Errorf("requests = %d, want none", len(r.requests))
- }
-}
-
-// ===================== metadata catalog =====================
-
-// TestMetadataCatalogForBuildsSortedKeysWithSamples pins what the model gets to see: the
-// dataset's real fields, deterministically ordered, each with its strongest values.
-func TestMetadataCatalogForBuildsSortedKeysWithSamples(t *testing.T) {
- deps, _ := newTestSearchDeps(&stubRetriever{})
- deps.MetadataResolver = &stubMetadataResolver{metas: common.MetaData{
- "title": {"Boston": {"d1"}, "New York City": {"d2", "d3", "d4"}},
- "author": {"Alice": {"d1"}, "Bob": {"d2"}},
- }}
-
- cat := MetadataCatalogFor(context.Background(), deps)
- if got := strings.Join(cat.Keys, ","); got != "author,title" {
- t.Fatalf("keys = %q, want author,title (sorted, deterministic)", got)
- }
- samples := cat.Samples["title"]
- if len(samples) != 2 {
- t.Fatalf("title samples = %v, want 2", samples)
- }
- // Ordered by document count: the value most documents carry is the useful one to show.
- if samples[0].Value != "New York City" || samples[0].Docs != 3 {
- t.Errorf("first title sample = %+v, want New York City (3 docs)", samples[0])
- }
-
- render := cat.Render()
- for _, want := range []string{"AVAILABLE METADATA", "title", "author", "New York City"} {
- if !strings.Contains(render, want) {
- t.Errorf("render missing %q:\n%s", want, render)
- }
- }
- if n := utf8.RuneCountInString(render); n > metadataCatalogRenderMax {
- t.Errorf("render = %d runes, cap is %d", n, metadataCatalogRenderMax)
- }
-}
-
-// TestMetadataCatalogForDropsSystemAndLabelKeys pins the blacklist: identifiers and
-// benchmark annotations must never become filters — a `question_id` filter would let the
-// model shrink retrieval to the very documents that answer the question.
-func TestMetadataCatalogForDropsSystemAndLabelKeys(t *testing.T) {
- deps, _ := newTestSearchDeps(&stubRetriever{})
- deps.MetadataResolver = &stubMetadataResolver{metas: common.MetaData{
- "title": {"A": {"d1"}},
- "question_id": {"444": {"d1"}},
- "source_uri": {"s3://x": {"d1"}},
- "pageid": {"1": {"d1"}},
- "outline": {"intro": {"d1"}},
- "_version": {"2": {"d1"}},
- }}
-
- cat := MetadataCatalogFor(context.Background(), deps)
- if got := strings.Join(cat.Keys, ","); got != "title" {
- t.Fatalf("keys = %q, want only title", got)
- }
- if render := cat.Render(); strings.Contains(render, "question_id") {
- t.Errorf("the benchmark-annotation key leaked into the catalog:\n%s", render)
- }
-}
-
-// TestMetadataCatalogForDegradesToEmpty pins the failure contract every "no metadata" path
-// depends on: a nil resolver, an unreadable index, a metadata-free dataset or no bound
-// datasets all yield the EMPTY catalog — never an error and never a panic. An empty
-// catalog is what makes the session keep the shipped title-only schema.
-func TestMetadataCatalogForDegradesToEmpty(t *testing.T) {
- ctx := context.Background()
- base, _ := newTestSearchDeps(&stubRetriever{})
-
- // No resolver wired (deployment without the metadata link).
- if cat := MetadataCatalogFor(ctx, base); !cat.Empty() {
- t.Errorf("keys = %v, want empty without a resolver", cat.Keys)
- }
- if ptr := MetadataCatalogPtr(ctx, base); ptr != nil {
- t.Error("MetadataCatalogPtr must be nil for an empty catalog")
- }
-
- // Index unreadable.
- broken := base
- broken.MetadataResolver = &stubMetadataResolver{flattenErr: errors.New("es down")}
- if cat := MetadataCatalogFor(ctx, broken); !cat.Empty() {
- t.Errorf("keys = %v, want empty when the index is unreadable", cat.Keys)
- }
-
- // Dataset carries metadata rows but no usable field.
- blank := base
- blank.MetadataResolver = &stubMetadataResolver{metas: common.MetaData{
- "_version": {"2": {"d1"}}, // hidden by the blacklist
- "empty": {}, // no value to match
- }}
- if cat := MetadataCatalogFor(ctx, blank); !cat.Empty() {
- t.Errorf("keys = %v, want empty when no field is usable", cat.Keys)
- }
-
- // No bound datasets at all.
- unbound, _ := newTestSearchDeps(&stubRetriever{})
- unbound.KbIDs = nil
- unbound.MetadataResolver = &stubMetadataResolver{metas: common.MetaData{"title": {"A": {"d1"}}}}
- if cat := MetadataCatalogFor(ctx, unbound); !cat.Empty() {
- t.Errorf("keys = %v, want empty without datasets", cat.Keys)
- }
-}
-
-// TestActiveToolSpecsRendersCatalogKeyEnum pins the payoff: the advertised `key` enum is
-// the dataset's real fields, so a model can name one. Before this it was ["title"] and no
-// other field could be asked for however plainly the question named it.
-func TestActiveToolSpecsRendersCatalogKeyEnum(t *testing.T) {
- cat := &MetadataCatalog{
- Keys: []string{"author", "title"},
- Samples: map[string][]MetadataSample{"author": {{Value: "Alice", Docs: 1}}},
- }
- ts := &Toolset{ThinkingMode: "high", MetadataFields: cat}
- spec, ok := findSpec(ts.ActiveToolSpecs(), "metadata_search")
- if !ok {
- t.Fatal("metadata_search missing from the high-mode surface")
- }
-
- if got := strings.Join(metadataKeyEnum(spec), ","); got != "author,title" {
- t.Errorf("key enum = %q, want the catalog's fields", got)
- }
- desc := spec.Function.Description
- for _, want := range []string{"WHEN TO CALL", "DO NOT CALL", "ARGUMENTS", "OUTPUT", "IF IT FAILS", "AVAILABLE METADATA"} {
- if !strings.Contains(desc, want) {
- t.Errorf("description missing %q:\n%s", want, desc)
- }
- }
- // The key parameter's description is dataset-independent — it points at AVAILABLE METADATA
- // and names no field, so a dataset's field list can never leak into another session's view
- // of the tool; the fields themselves reach the model through the enum asserted above and
- // through the seed's block.
- const wantKeyDesc = "one of this dataset's metadata fields (see AVAILABLE METADATA)"
- if got, _ := metadataKeyParam(spec)["description"].(string); got != wantKeyDesc {
- t.Errorf("key description = %q, want %q", got, wantKeyDesc)
- }
- if n := utf8.RuneCountInString(desc); n > maxToolDescriptionRunes {
- t.Errorf("description = %d runes, cap = %d", n, maxToolDescriptionRunes)
- }
-}
-
-// TestActiveToolSpecsKeepsStaticSpecWithoutCatalog pins the "no metadata = no field
-// advertised" contract: without a catalog the shipped spec is returned untouched — same
-// struct — and it names NO field, so a model cannot be told `title` is filterable on a
-// dataset that never said so. That is also how an empty enum (which some providers reject)
-// is avoided.
-func TestActiveToolSpecsKeepsStaticSpecWithoutCatalog(t *testing.T) {
- for _, cat := range []*MetadataCatalog{nil, &MetadataCatalog{}} {
- ts := &Toolset{ThinkingMode: "high", MetadataFields: cat}
- spec, ok := findSpec(ts.ActiveToolSpecs(), "metadata_search")
- if !ok {
- t.Fatal("metadata_search missing from the high-mode surface")
- }
- if !reflect.DeepEqual(spec, ToolMap["metadata_search"]) {
- t.Errorf("catalog %v: an empty catalog must leave the shipped spec untouched", cat)
- }
- if got := metadataKeyEnum(spec); len(got) != 0 {
- t.Errorf("key enum = %v, want NO advertised field without a catalog", got)
- }
- key, _ := metadataKeyParam(spec)["enum"]
- if key != nil {
- t.Errorf("key enum key = %v, want it absent (an empty enum is rejected by some providers)", key)
- }
- if strings.Contains(spec.Function.Description, "'title'") {
- t.Errorf("the shipped description still names a field:\n%s", spec.Function.Description)
- }
- }
-}
-
-// metadataKeyParam reads the metadata_search `key` parameter object off a spec.
-func metadataKeyParam(spec ToolSpec) map[string]any {
- props, _ := spec.Function.Parameters["properties"].(map[string]any)
- filters, _ := props["filters"].(map[string]any)
- items, _ := filters["items"].(map[string]any)
- itemProps, _ := items["properties"].(map[string]any)
- key, _ := itemProps["key"].(map[string]any)
- return key
-}
-
-// stubDeclaredMetadata is a scripted DeclaredMetadataResolver.
-type stubDeclaredMetadata struct {
- defs []common.MetadataFieldDef
- err error
-}
-
-func (s *stubDeclaredMetadata) DeclaredMetadataFields(context.Context, []string) ([]common.MetadataFieldDef, error) {
- return s.defs, s.err
-}
-
-// TestMetadataCatalogForIncludesDeclaredFields pins the declarative half: a dataset that
-// declares its fields is described with what each field MEANS and which values it accepts,
-// which is what the metadata index alone cannot supply. It also pins that the blacklist
-// applies to the declarative source too.
-func TestMetadataCatalogForIncludesDeclaredFields(t *testing.T) {
- deps, _ := newTestSearchDeps(&stubRetriever{})
- deps.MetadataResolver = &stubMetadataResolver{metas: common.MetaData{
- "author": {"Alice": {"d1", "d2"}},
- }}
- deps.DeclaredMetadata = &stubDeclaredMetadata{defs: []common.MetadataFieldDef{
- {Key: "author", Type: "string", Description: "who wrote it"},
- {Key: "doc_type", Type: "string", Description: "kind of document", Enum: []string{"report", "paper"}},
- {Key: "question_id", Description: "benchmark label"}, // blacklisted
- }}
-
- cat := MetadataCatalogFor(context.Background(), deps)
- if got := strings.Join(cat.Keys, ","); got != "author,doc_type" {
- t.Fatalf("keys = %q, want the declared fields (minus the blacklist), sorted", got)
- }
- if cat.Fields["author"].Description != "who wrote it" {
- t.Errorf("declared definition lost: %+v", cat.Fields["author"])
- }
-
- render := cat.Render()
- for _, want := range []string{
- "author — who wrote it",
- "doc_type — kind of document",
- "one of: report / paper",
- `values seen: "Alice" (2 doc(s))`,
- } {
- if !strings.Contains(render, want) {
- t.Errorf("render missing %q:\n%s", want, render)
- }
- }
- if strings.Contains(render, "question_id") {
- t.Errorf("the blacklist must cover the declarative source too:\n%s", render)
- }
-}
-
-// TestMetadataCatalogForOffersDeclaredFieldWithoutIndexedValues pins the case the
-// observational source cannot see at all: a field declared but not yet indexed is still a
-// legitimate filter, so it is offered (with its description) and advertised in the schema.
-func TestMetadataCatalogForOffersDeclaredFieldWithoutIndexedValues(t *testing.T) {
- deps, _ := newTestSearchDeps(&stubRetriever{})
- deps.MetadataResolver = &stubMetadataResolver{metas: common.MetaData{}}
- deps.DeclaredMetadata = &stubDeclaredMetadata{defs: []common.MetadataFieldDef{
- {Key: "doc_type", Description: "kind of document", Enum: []string{"report"}},
- }}
-
- cat := MetadataCatalogFor(context.Background(), deps)
- if got := strings.Join(cat.Keys, ","); got != "doc_type" {
- t.Fatalf("keys = %q, want the declared field even with no indexed value", got)
- }
- if len(cat.Samples["doc_type"]) != 0 {
- t.Errorf("samples = %v, want none: the index carries no value yet", cat.Samples["doc_type"])
- }
-
- ts := &Toolset{ThinkingMode: "high", MetadataFields: &cat}
- spec, ok := findSpec(ts.ActiveToolSpecs(), "metadata_search")
- if !ok {
- t.Fatal("metadata_search missing from the high-mode surface")
- }
- if got := strings.Join(metadataKeyEnum(spec), ","); got != "doc_type" {
- t.Errorf("key enum = %q, want the declared field advertised", got)
- }
-}
-
-// TestMetadataCatalogForDegradesWhenDeclaredReadFails pins the independence of the two
-// halves: an unreadable parser_config leaves the observational source untouched, and vice
-// versa (see TestMetadataCatalogForDegradesToEmpty for the index failure).
-func TestMetadataCatalogForDegradesWhenDeclaredReadFails(t *testing.T) {
- deps, _ := newTestSearchDeps(&stubRetriever{})
- deps.MetadataResolver = &stubMetadataResolver{metas: common.MetaData{"title": {"A": {"d1"}}}}
- deps.DeclaredMetadata = &stubDeclaredMetadata{err: errors.New("kb row unreadable")}
-
- cat := MetadataCatalogFor(context.Background(), deps)
- if got := strings.Join(cat.Keys, ","); got != "title" {
- t.Errorf("keys = %q, want the observational half alone", got)
- }
-}
-
-// metadataKeyEnum reads the metadata_search `key` enum off a rendered spec.
-func metadataKeyEnum(spec ToolSpec) []string {
- props, _ := spec.Function.Parameters["properties"].(map[string]any)
- filters, _ := props["filters"].(map[string]any)
- items, _ := filters["items"].(map[string]any)
- itemProps, _ := items["properties"].(map[string]any)
- key, _ := itemProps["key"].(map[string]any)
- out, _ := key["enum"].([]string)
- return out
-}
-
-func (s *stubExpander) Expand(_ context.Context, _ *Kbinfos, _, _ string, _ []string) error {
- s.calls++
- return nil
-}
-
-// lastReq returns the most recent RetrieveRequest the stub recorded, failing if
-// none was issued.
-func (r *stubRetriever) lastReq(t *testing.T) RetrieveRequest {
- t.Helper()
- if len(r.requests) == 0 {
- t.Fatal("expected a retrieve request, got none")
- }
- return r.requests[len(r.requests)-1]
-}
-
-// ptrFloat dereferences a control pointer. nil means "the caller supplied
-// nothing, so the retriever keeps its own default"; the search-leg tests below
-// assert the opposite (an explicit value is passed), so nil is a failure here.
-func ptrFloat(t *testing.T, p *float64) float64 {
- t.Helper()
- if p == nil {
- t.Fatal("control pointer = nil, want an explicit value")
- }
- return *p
-}
-
-// TestVectorSearchBailsWithoutEmbedder: with no embedder configured it returns nothing,
-// and must NOT even hit the backend.
-func TestVectorSearchBailsWithoutEmbedder(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{{"content": "x"}}}
- deps, _ := newTestSearchDeps(r)
- if got, _ := VectorSearch(context.Background(), deps, SearchParams{Question: "q"}); got != nil {
- t.Error("vector search without embedder must be empty")
- }
- if len(r.requests) != 0 {
- t.Errorf("vector search without embedder must not call the backend, got %d calls", len(r.requests))
- }
-}
-
-// TestVectorSearchUsesZeroKeywordWeight mirrors Python vector_search: the
-// pure-vector leg carries keyword weight 0 and excludes compiled rows.
-func TestVectorSearchUsesZeroKeywordWeight(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{{"content": "x"}}}
- deps, _ := newTestSearchDeps(r)
- deps.HasEmbedder = true
- VectorSearch(context.Background(), deps, SearchParams{Question: "q"})
- req := r.lastReq(t)
- if got := ptrFloat(t, req.KeywordsSimilarityWeight); got != 0 {
- t.Errorf("vector search keyword weight = %v, want 0", got)
- }
- if !req.ExcludeCompiled {
- t.Error("vector search must exclude compiled rows")
- }
-}
-
-// TestBM25SearchUsesFullKeywordWeight mirrors Python bm25_search: keyword-only,
-// keyword weight 1, threshold 0.0, excludes compiled rows.
-func TestBM25SearchUsesFullKeywordWeight(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{{"content": "x"}}}
- deps, _ := newTestSearchDeps(r)
- BM25Search(context.Background(), deps, SearchParams{Question: "q"})
- req := r.lastReq(t)
- if got := ptrFloat(t, req.KeywordsSimilarityWeight); got != 1 {
- t.Errorf("bm25 search keyword weight = %v, want 1", got)
- }
- if got := ptrFloat(t, req.SimilarityThreshold); got != 0 {
- t.Errorf("bm25 search threshold = %v, want 0", got)
- }
- if !req.ExcludeCompiled {
- t.Error("bm25 search must exclude compiled rows")
- }
-}
-
-// TestGrepSearchDelegatesToBM25 mirrors Python grep_search: it is bm25_search
-// with a keyword-only (weight 1) leg and compiled-row exclusion.
-func TestGrepSearchDelegatesToBM25(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{{"content": "x"}}}
- deps, _ := newTestSearchDeps(r)
- GrepSearch(context.Background(), deps, SearchParams{Question: "q", Keywords: "kw"})
- req := r.lastReq(t)
- if got := ptrFloat(t, req.KeywordsSimilarityWeight); got != 1 {
- t.Errorf("grep search keyword weight = %v, want 1", got)
- }
- if !req.ExcludeCompiled {
- t.Error("grep search must exclude compiled rows")
- }
-}
-
-// TestGrepSearchSearchingLineSplitsAudiences pins the split every leg now makes:
-// the think block says what the leg DOES ("Searching for the exact words …", in the
-// family the other legs use, without the phrase every leg shares — "the knowledge
-// base" — that separates nothing a reader can act on), while the log keeps Python's
-// own phrasing ("Keyword-first locate for …", search.py:418), so a log diff against
-// Python still lines up.
-func TestGrepSearchSearchingLineSplitsAudiences(t *testing.T) {
- var logged bytes.Buffer
- deps, _ := newTestSearchDeps(&stubRetriever{chunks: []map[string]any{{"content": "x"}}})
- deps.Logger = log.New(&logged, "", 0)
- var text []string
- ctx := WithSteps(context.Background(), StepReporter{Text: func(line string) { text = append(text, line) }})
-
- GrepSearch(ctx, deps, SearchParams{Question: "曹操是谁"})
-
- think := strings.Join(text, "")
- if !strings.Contains(think, `[Grep search] Searching for the exact words "曹操是谁".`) {
- t.Errorf("think text = %q, want the reader's sentence", think)
- }
- if strings.Contains(think, "Keyword-first") {
- t.Errorf("the implementation phrasing must not reach the think block: %q", think)
- }
- if !strings.Contains(logged.String(), `[Grep search] Keyword-first locate for "曹操是谁"`) {
- t.Errorf("log = %q, want Python's line kept verbatim", logged.String())
- }
-}
-
-// TestSearchLegsShareOneThinkSentenceFamily pins the wording of all five legs'
-// "searching" step in ONE place. They land in a single think block, so they have to
-// scan as one family ("Searching … for \"q\".") while still saying which leg
-// searched — the "[Hybrid search]" prefix is a developer's label, so the difference
-// has to be IN the sentence.
-//
-// "the knowledge base" is deliberately absent from every one of them: all five legs
-// search it, so the phrase distinguishes nothing a reader can act on. The LOG side
-// keeps Python's own wording (including that phrase where Python has it) — pinned by
-// TestGrepSearchSearchingLineSplitsAudiences and the log-diff tests, which is why it
-// is not repeated here.
-func TestSearchLegsShareOneThinkSentenceFamily(t *testing.T) {
- cases := []struct {
- name string
- run func(ctx context.Context, deps SearchDeps)
- want string
- }{
- {"hybrid", func(ctx context.Context, d SearchDeps) {
- HybridSearch(ctx, d, SearchParams{Question: "q"})
- },
- `[Hybrid search] Searching by meaning and keyword for "q".`},
- {"vector", func(ctx context.Context, d SearchDeps) {
- VectorSearch(ctx, d, SearchParams{Question: "q"})
- },
- `[Vector search] Searching by meaning for "q".`},
- {"bm25", func(ctx context.Context, d SearchDeps) {
- BM25Search(ctx, d, SearchParams{Question: "q"})
- },
- `[BM25 search] Searching by keyword for "q".`},
- {"retrieve", func(ctx context.Context, d SearchDeps) {
- RetrieveSearch(ctx, d, SearchParams{Question: "q"})
- },
- `[Retrieve] Searching for "q".`},
- {"grep", func(ctx context.Context, d SearchDeps) {
- GrepSearch(ctx, d, SearchParams{Question: "q"})
- },
- `[Grep search] Searching for the exact words "q".`},
- }
- for _, tc := range cases {
- deps, _ := newTestSearchDeps(&stubRetriever{chunks: []map[string]any{{"content": "x"}}})
- // VectorSearch is gated on a configured embedder (Python bm25/vector
- // asymmetry): without this it returns before reporting anything.
- deps.HasEmbedder = true
- var text []string
- ctx := WithSteps(context.Background(), StepReporter{Text: func(line string) { text = append(text, line) }})
-
- tc.run(ctx, deps)
-
- if got := strings.Join(text, ""); !strings.Contains(got, tc.want) {
- t.Errorf("%s: think = %q, want it to carry %q", tc.name, got, tc.want)
- }
- }
-}
-
-// captureLeg runs one search leg and returns what each audience saw: the developer
-// log and the think text.
-func captureLeg(chunks []map[string]any, run func(ctx context.Context, deps SearchDeps)) (logged, think string) {
- var logBuf bytes.Buffer
- deps, _ := newTestSearchDeps(&stubRetriever{chunks: chunks})
- deps.Logger = log.New(&logBuf, "", 0)
- // VectorSearch is gated on a configured embedder; set for every leg, harmless
- // for the others.
- deps.HasEmbedder = true
- var text []string
- ctx := WithSteps(context.Background(), StepReporter{Text: func(line string) { text = append(text, line) }})
-
- run(ctx, deps)
- return logBuf.String(), strings.Join(text, "")
-}
-
-// TestSearchLegResultLinesReportEveryOutcome pins the second half of a leg's step:
-// what it FOUND. Every exit reports one — an empty pool, an all-table pool and a term
-// window that matched nothing included — because a leg with no result line cannot be
-// told apart from a leg that found nothing, which is the one question the block is
-// read for.
-//
-// The fallbacks count the RETURNED set, not what the term window kept: when grep's
-// locate matches nothing it hands back the raw BM25 candidates, and that is what the
-// line says.
-func TestSearchLegResultLinesReportEveryOutcome(t *testing.T) {
- const q = "曹操生平简介"
- hit := []map[string]any{{"chunk_id": "c1", "doc_id": "d1", "content": "曹操,字孟德。"}}
- // A >=3-row pipe table: IsTableChunk keeps it whole, so grep has no prose to
- // locate in and returns it as-is.
- table := []map[string]any{{"chunk_id": "t1", "doc_id": "d1",
- "content": "a | b | c\nd | e | f\ng | h | i"}}
- proseNoMatch := []map[string]any{{"chunk_id": "p1", "doc_id": "d1",
- "content": "A completely unrelated sentence about weather and clouds."}}
-
- cases := []struct {
- name string
- chunks []map[string]any
- run func(ctx context.Context, deps SearchDeps)
- wantThink string
- wantLog string
- }{
- {
- name: "bm25-hit", chunks: hit,
- run: func(ctx context.Context, d SearchDeps) { BM25Search(ctx, d, SearchParams{Question: q}) },
- wantThink: `[BM25 search] Found 1 passage in 1 document for "` + q + `".`,
- // The log keeps the hybrid/grep shape; on the bm25 leg this line is Go-only.
- wantLog: `[BM25 search] "` + q + `" -> 1 chunk(s): d1:1chunk(`,
- },
- {
- // The leg ran and came back empty: it says so in both projections, and the
- // log line does not dangle a colon.
- name: "bm25-empty", chunks: nil,
- run: func(ctx context.Context, d SearchDeps) { BM25Search(ctx, d, SearchParams{Question: q}) },
- wantThink: `[BM25 search] Found nothing for "` + q + `".`,
- wantLog: `[BM25 search] "` + q + `" -> 0 chunk(s)`,
- },
- {
- name: "vector-hit", chunks: hit,
- run: func(ctx context.Context, d SearchDeps) { VectorSearch(ctx, d, SearchParams{Question: q}) },
- wantThink: `[Vector search] Found 1 passage in 1 document for "` + q + `".`,
- wantLog: `[Vector search] "` + q + `" -> 1 chunk(s): d1:1chunk(`,
- },
- {
- // grep delegated to bm25 (which reported its own pool) and then found no
- // prose to locate in: the tables ARE the evidence, so the line counts them.
- name: "grep-all-tables", chunks: table,
- run: func(ctx context.Context, d SearchDeps) {
- GrepSearch(ctx, d, SearchParams{Question: "who made Culdcept?"})
- },
- wantThink: `[Grep search] Found 1 passage in 1 document for "who made Culdcept?".`,
- wantLog: `[Grep search] "who made Culdcept?" -> 1 chunk(s): d1:1chunk(`,
- },
- {
- // The term window matched nothing, so grep returns the raw BM25 candidates
- // — and reports THAT, not "nothing".
- name: "grep-no-match", chunks: proseNoMatch,
- run: func(ctx context.Context, d SearchDeps) {
- GrepSearch(ctx, d, SearchParams{Question: "who made Culdcept?"})
- },
- wantThink: `[Grep search] Found 1 passage in 1 document for "who made Culdcept?".`,
- wantLog: `[Grep search] "who made Culdcept?" -> 1 chunk(s): d1:1chunk(`,
- },
- {
- // Nothing to locate in and nothing returned: still reported.
- name: "grep-empty-pool", chunks: nil,
- run: func(ctx context.Context, d SearchDeps) {
- GrepSearch(ctx, d, SearchParams{Question: "who made Culdcept?"})
- },
- wantThink: `[Grep search] Found nothing for "who made Culdcept?".`,
- wantLog: `[Grep search] "who made Culdcept?" -> 0 chunk(s)`,
- },
- }
- for _, tc := range cases {
- logged, think := captureLeg(tc.chunks, tc.run)
- if !strings.Contains(think, tc.wantThink) {
- t.Errorf("%s: think = %q, want it to carry %q", tc.name, think, tc.wantThink)
- }
- if !strings.Contains(logged, tc.wantLog) {
- t.Errorf("%s: log = %q, want it to carry %q", tc.name, logged, tc.wantLog)
- }
- if strings.Contains(logged, "chunk(s): \n") {
- t.Errorf("%s: a zero/empty result must not dangle a colon: %q", tc.name, logged)
- }
- }
-}
-
-// TestGrepTermsFromQuery mirrors Python _grep_terms_from_query:
-// bare alnum words of length>=2, deduped (order-preserving) and capped at 10.
-func TestGrepTermsFromQuery(t *testing.T) {
- // Proper nouns preserved; stopwords/dupes dropped; bare single chars skipped.
- got := GrepTermsFromQuery("Where was Culdcept Saga made? culdcept was made by OmiyaSoft.")
- want := []string{"Where", "was", "Culdcept", "Saga", "made", "by", "OmiyaSoft"}
- if !reflect.DeepEqual(got, want) {
- t.Errorf("terms = %v, want %v", got, want)
- }
- // Leading/trailing punctuation trimmed from each token.
- got = GrepTermsFromQuery("apollo.-. 13 mission")
- if !reflect.DeepEqual(got, []string{"apollo", "13", "mission"}) {
- t.Errorf("terms = %v, want [apollo 13 mission]", got)
- }
- // Capped at 10.
- got = GrepTermsFromQuery("a1 a2 a3 a4 a5 a6 a7 a8 a9 a10 a11 a12")
- if len(got) != GrepTermsMax {
- t.Errorf("terms = %d, want capped at %d", len(got), GrepTermsMax)
- }
- // Empty/blank -> nil.
- if GrepTermsFromQuery("") != nil || GrepTermsFromQuery(" ") != nil {
- t.Error("blank query must yield nil terms")
- }
-}
-
-// TestGrepSearchNarrowsProseViaTermWindow: two
-// narrowing stages: bm25_search narrows by the keywords hint first
-// (search.py:grep_search via _narrow_or_keep), then the prose candidates alone are
-// narrowed to the term-grep window (search.py:grep_search) while table chunks pass
-// through UN-narrowed.
-func TestGrepSearchNarrowsProseViaTermWindow(t *testing.T) {
- // A long multi-line prose chunk: only the line carrying the term should be
- // kept (the long unmatched tail is trimmed by the grep window).
- prose := map[string]any{
- "chunk_id": "p1",
- "content": "Alpha filler sentence about unrelated weather and clouds.\n" +
- "Culdcept was made by OmiyaSoft in 1999 after the original board game.\n" +
- strings.Repeat("Omega filler unrelated text about the sea and the sky. ", 20),
- }
- // A table chunk: >=3 pipe rows must NOT be term-narrowed.
- table := map[string]any{
- "chunk_id": "t1",
- "content": "team | pts | rank\nA | 10 | 1\nB | 8 | 2\nC | 5 | 3\nD | 2 | 4",
- }
- r := &stubRetriever{chunks: []map[string]any{prose, table}}
- deps, _ := newTestSearchDeps(r)
- // The query's own terms become the BM25 keywords hint (there is none here),
- // so the prose only survives the FIRST stage if it carries those terms:
- // "Culdcept was made" yields the bigrams "culdcept was"/"was made", which
- // the prose sentence does. (With "who made Culdcept?" the hint matches the
- // table alone and the prose is dropped outright — pinned by
- // TestGrepSearchDerivesKeywordsHint.)
- chunks, _ := GrepSearch(context.Background(), deps, SearchParams{Question: "Culdcept was made"})
-
- // Order: table chunks first, then narrowed prose (table + kept prose).
- if len(chunks) != 2 {
- t.Fatalf("chunks = %d, want 2 (table + narrowed prose)", len(chunks))
- }
- if chunks[0]["chunk_id"] != "t1" {
- t.Errorf("first chunk = %v, want table chunk t1", chunks[0]["chunk_id"])
- }
- // The table chunk must be returned WHOLE, every row intact: the pipe-table branch wraps
- // the highlighted content, so the body is unchanged and only the wrapper differs from the
- // raw chunk.
- if got := ChunkTextOf(chunks[0]); !strings.Contains(got, "team | pts | rank") || !strings.Contains(got, "D | 2 | 4") {
- t.Errorf("table chunk was truncated; got %q", got)
- }
- // The prose chunk must be narrowed to the Culdcept term window.
- got := ChunkTextOf(chunks[1])
- if !strings.Contains(got, "Culdcept") {
- t.Errorf("prose not narrowed to term: %q", got)
- }
- if strings.Contains(got, "Omega filler") {
- t.Errorf("prose tail should be trimmed by the grep window: %q", got)
- }
- if len(got) >= len(ChunkTextOf(prose)) {
- t.Errorf("prose was not truncated by the grep window: len=%d", len(got))
- }
-}
-
-// TestGrepSearchKeepsRawCandidatesWhenNoMatch: when
-// the term-grep matches nothing, the raw BM25 candidates are returned unchanged
-// so evidence is never dropped.
-func TestGrepSearchKeepsRawCandidatesWhenNoMatch(t *testing.T) {
- prose := map[string]any{
- "chunk_id": "p1",
- "content": "A completely unrelated sentence about weather and clouds.",
- }
- r := &stubRetriever{chunks: []map[string]any{prose}}
- deps, _ := newTestSearchDeps(r)
- chunks, _ := GrepSearch(context.Background(), deps, SearchParams{Question: "who made Culdcept?"})
- if len(chunks) != 1 || chunks[0]["chunk_id"] != "p1" {
- t.Fatalf("chunks = %v, want the raw candidate preserved", chunks)
- }
- if ChunkTextOf(chunks[0]) != ChunkTextOf(prose) {
- t.Error("unmatched prose must be returned whole, not narrowed")
- }
-}
-
-// TestGrepSearchDerivesKeywordsHint: with no
-// explicit hint the BM25 pool is built from "query + the query's own extracted
-// terms", so a long question's proper nouns stop hiding under stopwords and the
-// keyword-narrowing stage runs even without a nav hint. An explicit hint (the
-// nav-routed docs' summary) wins — "nav is a hint, not a constraint".
-func TestGrepSearchDerivesKeywordsHint(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{{"content": "hit"}}}
- deps, _ := newTestSearchDeps(r)
- GrepSearch(context.Background(), deps, SearchParams{Question: "who made Culdcept?"})
- req := r.lastReq(t)
- if want := "who made Culdcept? who made Culdcept"; req.Query != want {
- t.Errorf("query = %q, want %q (question + derived terms)", req.Query, want)
- }
- if ptrFloat(t, req.KeywordsSimilarityWeight) != 1 || !req.ExcludeCompiled {
- t.Error("grep must stay keyword-only (weight 1) and exclude compiled rows")
- }
- // The derived hint also drives the narrowing stage: a prose candidate whose
- // sentences miss those terms is dropped, while a >=3-row pipe table is kept
- // whole (the narrowing's table branch).
- prose := map[string]any{"chunk_id": "p1", "content": "Unrelated sentence about weather."}
- table := map[string]any{"chunk_id": "t1", "content": "a | b | c\nd | e | f\ng | h | i"}
- r2 := &stubRetriever{chunks: []map[string]any{prose, table}}
- deps2, _ := newTestSearchDeps(r2)
- got, _ := GrepSearch(context.Background(), deps2, SearchParams{Question: "who made Culdcept?"})
- if len(got) != 1 || got[0]["chunk_id"] != "t1" {
- t.Errorf("chunks = %v, want only the table chunk", got)
- }
-
- // An explicit hint wins over the derived terms.
- r3 := &stubRetriever{chunks: []map[string]any{{"content": "hit"}}}
- deps3, _ := newTestSearchDeps(r3)
- GrepSearch(context.Background(), deps3, SearchParams{Question: "who made Culdcept?", Keywords: "nav summary"})
- if got := r3.lastReq(t).Query; got != "who made Culdcept? nav summary" {
- t.Errorf("query = %q, want the explicit hint appended", got)
- }
-}
-
-// TestSearchCacheIsHybridOnly: the search cache is read and written by the hybrid leg
-// alone. A keyword-only leg must neither serve nor be served by it — the key carries no
-// weight, threshold or compiled policy, so a shared cache would hand a hybrid result
-// (vector weight 0.3) to a BM25/grep call and vice versa.
-func TestSearchCacheIsHybridOnly(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{{"content": "hit"}}}
- deps, _ := newTestSearchDeps(r)
- p := SearchParams{Question: "same query"}
- HybridSearch(context.Background(), deps, p) // fills the cache
- BM25Search(context.Background(), deps, p) // must not be served from it
- GrepSearch(context.Background(), deps, p) // nor must this one
- if len(r.requests) != 3 {
- t.Fatalf("backend calls = %d, want 3 (the search cache is hybrid-only)", len(r.requests))
- }
- // The hybrid leg still dedups its own repeats.
- HybridSearch(context.Background(), deps, p)
- if len(r.requests) != 3 {
- t.Errorf("backend calls = %d, want 3 (hybrid repeat served from cache)", len(r.requests))
- }
-}
-
-// TestHybridSearchExcludesCompiledAndUsesSevenTenthsKeywordWeight mirrors Python
-// hybrid_search: keyword weight 0.7 when an embedder is configured, compiled
-// rows excluded.
-func TestHybridSearchExcludesCompiledAndUsesSevenTenthsKeywordWeight(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{{"content": "x"}}}
- deps, _ := newTestSearchDeps(r)
- deps.HasEmbedder = true
- HybridSearch(context.Background(), deps, SearchParams{Question: "q"})
- req := r.lastReq(t)
- if got := ptrFloat(t, req.KeywordsSimilarityWeight); got != 1-HybridSearchDefaultVectorWeight {
- t.Errorf("hybrid search keyword weight = %v, want %v", got, 1-HybridSearchDefaultVectorWeight)
- }
- if !req.ExcludeCompiled {
- t.Error("hybrid search must exclude compiled rows")
- }
-}
-
-// TestRetrieveSearchDoesNotExcludeCompiled
-// unlike hybrid_search it does NOT exclude compiled rows, and honours
-// UsingEmbedding (weight 0.7 when on).
-func TestRetrieveSearchDoesNotExcludeCompiled(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{{"content": "x"}}}
- deps, _ := newTestSearchDeps(r)
- deps.UsingEmbedding = true
- RetrieveSearch(context.Background(), deps, SearchParams{Question: "q"})
- req := r.lastReq(t)
- if req.ExcludeCompiled {
- t.Error("retrieve search must NOT exclude compiled rows")
- }
- if got := ptrFloat(t, req.KeywordsSimilarityWeight); got != 1-DefaultHybridVectorWeight {
- t.Errorf("retrieve search keyword weight = %v, want %v", got, 1-DefaultHybridVectorWeight)
- }
-}
-
-// TestHybridSearchMergesSQLKBs verifies that the hybrid leg folds the session's structured
-// (SQL) datasets into the target id list.
-func TestHybridSearchMergesSQLKBs(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{{"content": "x"}}}
- deps, _ := newTestSearchDeps(r)
- deps.SQLKBs = []string{"sqlkb1", "sqlkb2"}
- HybridSearch(context.Background(), deps, SearchParams{Question: "q"})
- req := r.lastReq(t)
- want := map[string]bool{"kb1": true, "sqlkb1": true, "sqlkb2": true}
- if len(req.DatasetIDs) != len(want) {
- t.Fatalf("expected %d dataset ids, got %v", len(want), req.DatasetIDs)
- }
- for _, id := range req.DatasetIDs {
- if !want[id] {
- t.Errorf("unexpected dataset id %q in %v", id, req.DatasetIDs)
- }
- }
-}
-
-// TestNormalizeWebResults verifies the web-search payload shaper drops entries
-// without a URL or content and keeps chunk_id unique per snippet.
-func TestNormalizeWebResults(t *testing.T) {
- srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- var in map[string]any
- _ = json.NewDecoder(r.Body).Decode(&in)
- out := map[string]any{
- "chunks": []map[string]any{
- {"url": "u1", "content": "c1", "title": "t1"},
- {"url": "", "content": "c2"}, // missing url -> dropped
- {"url": "u2", "content": ""}, // missing content -> dropped
- {"url": "u3", "content": "c3"}, // kept
- {"url": "u1", "content": "c4"}, // same url as u1 -> must keep (unique chunk_id)
- },
- }
- _ = json.NewEncoder(w).Encode(out)
- }))
- defer srv.Close()
-
- resp, _ := http.Post(srv.URL, "application/json", bytes.NewReader([]byte(`{}`)))
- raw, _ := io.ReadAll(resp.Body)
- _ = resp.Body.Close()
- got := normalizeWebResults(raw)
- if len(got) != 3 {
- t.Fatalf("got %d chunks, want 3 (u1, u3, u1-second-snippet)", len(got))
- }
-
- seen := map[string]int{}
- for _, c := range got {
- id, _ := c["chunk_id"].(string)
- seen[id]++
- }
- for id, n := range seen {
- if n != 1 {
- t.Errorf("chunk_id %q appeared %d times, must be unique", id, n)
- }
- }
-
- // Filtering of bad entries.
- if c := normalizeWebResults([]byte(`{"chunks":[{"content":"no-url"}]}`)); len(c) != 0 {
- t.Errorf("entry without url kept: %v", c)
- }
- if c := normalizeWebResults([]byte(`{"chunks":[{"url":"x"}]}`)); len(c) != 0 {
- t.Errorf("entry without content kept: %v", c)
- }
-}
-
-// TestQueryToTerms pins the tokenizer on the shapes the fan-out prefetch feeds it. The CJK
-// case is the regression guard: Go's RE2 \w is ASCII-only, so tokenizing a Chinese
-// fan-out with \w+ produced NO terms, leaving the BM25 leg without the keyed terms that
-// give the discriminating entity its own score mass — the log line then read
-// "keywords: ".
-func TestQueryToTerms(t *testing.T) {
- for _, c := range []struct {
- name string
- in string
- want []string
- }{
- {"cjk sentence", "关羽 斩杀 有姓名 人物 名单", []string{"关羽", "斩杀", "有姓名", "人物", "名单"}},
- // Case is preserved (downstream matching is case-insensitive).
- {"ascii sentence", "Culdcept Saga release date", []string{"Culdcept", "Saga", "release", "date"}},
- // Dedup is case-sensitive, so both survive.
- {"case-sensitive dedup", "Saga saga", []string{"Saga", "saga"}},
- {"regex alternation", `(?i)\b关羽|张飞\b`, []string{"关羽", "张飞"}},
- {"tokens below two runes dropped", "a b cd", []string{"cd"}},
- {"empty query", " ", nil},
- } {
- t.Run(c.name, func(t *testing.T) {
- if got := QueryToTerms(c.in); !reflect.DeepEqual(got, c.want) {
- t.Errorf("QueryToTerms(%q) = %v, want %v", c.in, got, c.want)
- }
- })
- }
-
- // The fan-out's keyed subset must survive for CJK too: without it the BM25
- // leg runs the bare sentence.
- if got := FanoutKeyedTerms(QueryToTerms("关羽 斩杀 有姓名 人物")); len(got) != 1 || got[0] != "有姓名" {
- t.Errorf("FanoutKeyedTerms = %v, want [有姓名] (>=3 runes, non-stopword)", got)
- }
- // Stopwords are compared case-insensitively, so a capitalized one is dropped
- // while the discriminative terms keep their case.
- if got := FanoutKeyedTerms(QueryToTerms("The Culdcept Saga")); !reflect.DeepEqual(got, []string{"Culdcept", "Saga"}) {
- t.Errorf("FanoutKeyedTerms = %v, want [Culdcept Saga]", got)
- }
-
- // The exact fan-out queries of a real high-mode run ("关羽杀了多少有姓名的人?"),
- // with the keywords the implementation produces for them (verified by
- // running rag.advanced_rag.harness.tools.search._query_to_terms plus the keyed
- // filter of agentic_rag_graph.py:_search_one). Note the surviving term is a generic
- // long word, not the entity: the len>=3 rule drops every two-character
- // Chinese name (关羽/华雄/颜良/文丑/蔡阳), so this is parity, not a Go bug.
- for _, c := range []struct{ in, want string }{
- {"关羽 斩杀 有姓名 人物 名单", "有姓名"},
- {"关羽 斩将 记录 三国演义", "三国演义"},
- {"关羽 杀 将领 三国志", "三国志"},
- } {
- if got := strings.Join(FanoutKeyedTerms(QueryToTerms(c.in)), " "); got != c.want {
- t.Errorf("fan-out keywords for %q = %q, want %q", c.in, got, c.want)
- }
- }
-}
-
-func TestAgenticVectorWeightDefaultsToZero(t *testing.T) {
- // The agentic retrieve runs keyword-only: UsingEmbedding defaults to
- // False and no caller passes True (agentic_rag.py:retrieve, 643-646).
- got := floatPtrOrDef(nil, DefaultAgenticVectorWeight)
- if got != 0 {
- t.Fatalf("default vector weight = %v, want 0 (keyword-only)", got)
- }
- if DefaultAgenticVectorWeight != 0 {
- t.Fatalf("DefaultAgenticVectorWeight = %v, want 0", DefaultAgenticVectorWeight)
- }
-}
-
-func TestVectorWeightHonoursExplicitZero(t *testing.T) {
- // A pointer keeps "configured 0" distinct from "unset", so keyword-only can
- // be requested explicitly rather than only by omission.
- zero := 0.0
- if got := floatPtrOrDef(&zero, DefaultAgenticVectorWeight); got != 0 {
- t.Fatalf("got %v, want 0", got)
- }
-
- // Fallback must apply only when unset.
- if DefaultAgenticVectorWeight != 0 {
- t.Fatal("the fallback itself must stay 0")
- }
-}
-
-func TestVectorWeightCanEnableHybrid(t *testing.T) {
- // Hybrid retrieval stays available, just not by default.
- hybrid := 0.3
- if got := floatPtrOrDef(&hybrid, DefaultAgenticVectorWeight); got != 0.3 {
- t.Fatalf("got %v, want 0.3 when explicitly configured", got)
- }
-}
-
-func TestRetrievalDefaultsUseIntOrDef(t *testing.T) {
- if got := intOrDef(0, 12); got != 12 {
- t.Fatalf("intOrDef(0) = %d, want the fallback 12", got)
- }
- if got := intOrDef(5, 12); got != 5 {
- t.Fatalf("intOrDef(5) = %d, want 5", got)
- }
-}
-
-func TestResolveKeywordsSimilarityWeightRetrieveMirrorsUsingEmbedding(t *testing.T) {
- // Python RAGTools.retrieve(using_embedding: bool = False).
- // Off → keyword-only (weight 1); on → 0.3 default or the configured override.
- if got := resolveKeywordsSimilarityWeight(SearchDeps{UsingEmbedding: false}, ChannelRetrieve); got != 1 {
- t.Fatalf("using_embedding=false → %v, want 1 (keyword-only)", got)
- }
- if got := resolveKeywordsSimilarityWeight(SearchDeps{UsingEmbedding: true}, ChannelRetrieve); got != 1-DefaultHybridVectorWeight {
- t.Fatalf("using_embedding=true → %v, want %v", got, 1-DefaultHybridVectorWeight)
- }
- override := 0.5
- if got := resolveKeywordsSimilarityWeight(SearchDeps{UsingEmbedding: true, KeywordsSimilarityWeight: &override}, ChannelRetrieve); got != 0.5 {
- t.Fatalf("using_embedding=true with override → %v, want 0.5", got)
- }
-}
-
-func TestResolveKeywordsSimilarityWeightHybridDefaultsToSevenTenths(t *testing.T) {
- // Python hybrid_search defaults the vector weight to 0.3, hence its
- // keyword weight is 0.7.
- // (_DEFAULT_HYBRID_VECTOR_WEIGHT,), unlike RAGTools.retrieve's 0.7.
- if got := resolveKeywordsSimilarityWeight(SearchDeps{HasEmbedder: true}, ChannelHybrid); got != 1-HybridSearchDefaultVectorWeight {
- t.Fatalf("hybrid → %v, want %v", got, 1-HybridSearchDefaultVectorWeight)
- }
- if got := resolveKeywordsSimilarityWeight(SearchDeps{HasEmbedder: false}, ChannelHybrid); got != 1 {
- t.Fatalf("hybrid with no embedder → %v, want 1 (Python: `if embd_mdl`)", got)
- }
-}
-
-// TestResolveKeywordsSimilarityWeightHybridIgnoresUsingEmbedding is the regression guard for
-// the channel-granularity bug: Python's hybrid_search has NO using_embedding
-// parameter (search.py:hybrid_search, :143-145), so gating it on that flag disabled the
-// semantic leg for search_chunks — losing recall of passages sharing no surface
-// words. The gate for this channel is the embedder, nothing else.
-func TestResolveKeywordsSimilarityWeightHybridIgnoresUsingEmbedding(t *testing.T) {
- if got := resolveKeywordsSimilarityWeight(SearchDeps{UsingEmbedding: false, HasEmbedder: true}, ChannelHybrid); got != 1-HybridSearchDefaultVectorWeight {
- t.Fatalf("hybrid with using_embedding=false → %v, want %v: the vector leg "+
- "must NOT depend on using_embedding", got, 1-HybridSearchDefaultVectorWeight)
- }
-}
-
-// TestResolveKeywordsSimilarityWeightGrepIsAlwaysOne pins Python grep_search: the retrieve
-// and grep_* session tools are keyword-only and have no vector leg at all, so no
-// flag can turn one on.
-func TestResolveKeywordsSimilarityWeightGrepIsAlwaysOne(t *testing.T) {
- override := 0.1
- if got := resolveKeywordsSimilarityWeight(SearchDeps{UsingEmbedding: true, HasEmbedder: true}, ChannelGrep); got != 1 {
- t.Fatalf("grep → %v, want 1 (grep_search has no vector leg)", got)
- }
- if got := resolveKeywordsSimilarityWeight(SearchDeps{UsingEmbedding: true, HasEmbedder: true, KeywordsSimilarityWeight: &override}, ChannelGrep); got != 1 {
- t.Fatalf("grep with override → %v, want 1 (grep_search has no vector leg)", got)
- }
-}
-
-// stubVerifier reports a fixed known set, or fails when err is set.
-type stubVerifier struct {
- known map[string]bool
- err error
-}
-
-func (s stubVerifier) KnownDocIDs(_ context.Context, _, _ []string) (map[string]bool, error) {
- return s.known, s.err
-}
-
-// subsetVerifier mimics the production DocIDLookup.KnownDocIDs: it returns only
-// the candidates that are in knownDocs, so an unknown/foreign document yields
-// an empty map rather than a false entry. The fixed-set stubVerifier above
-// cannot reproduce that empty-map path, which is exactly where the foreign-doc
-// rejection used to regress.
-type subsetVerifier struct {
- knownDocs map[string]bool
-}
-
-func (s subsetVerifier) KnownDocIDs(_ context.Context, _ []string, candidates []string) (map[string]bool, error) {
- out := make(map[string]bool, len(candidates))
- for _, c := range candidates {
- if s.knownDocs[c] {
- out[c] = true
- }
- }
- return out, nil
-}
-
-func TestResolveDocScopeKeepsKnownIDs(t *testing.T) {
- got := resolveDocScope(context.Background(), SearchDeps{
- DocIDVerifier: stubVerifier{known: map[string]bool{"a": true}},
- }, []string{"a", "b"}, []string{"kb"}, _LOG)
-
- if len(got) != 1 || got[0] != "a" {
- t.Fatalf("scope = %v, want [a]", got)
- }
-}
-
-func TestResolveDocScopeDropsScopeWhenAllUnknown(t *testing.T) {
- // An entirely bogus scope falls back to unfiltered retrieval instead of returning
- // nothing.
- got := resolveDocScope(context.Background(), SearchDeps{
- DocIDVerifier: stubVerifier{known: map[string]bool{"z": true}},
- }, []string{"a", "b"}, []string{"kb"}, _LOG)
-
- if got != nil {
- t.Fatalf("scope = %v, want nil (unfiltered)", got)
- }
-}
-
-func TestResolveDocScopePassthroughWithoutVerifier(t *testing.T) {
- scope := []string{"a", "b"}
- got := resolveDocScope(context.Background(), SearchDeps{}, scope, []string{"kb"}, _LOG)
-
- if len(got) != len(scope) {
- t.Fatalf("scope = %v, want %v unchanged", got, scope)
- }
-}
-
-func TestResolveDocScopePassthroughOnVerifierError(t *testing.T) {
- scope := []string{"a"}
- got := resolveDocScope(context.Background(), SearchDeps{
- DocIDVerifier: stubVerifier{err: errors.New("db down")},
- }, scope, []string{"kb"}, _LOG)
-
- if len(got) != 1 {
- t.Fatalf("scope = %v, want %v (verification unavailable)", got, scope)
- }
-}
-
-func TestDocInDatasetsRejectsForeignDocument(t *testing.T) {
- belongs, verified := docInDatasets(context.Background(), SearchDeps{
- DocIDVerifier: stubVerifier{known: map[string]bool{"other": true}},
- }, "doc")
-
- if !verified || belongs {
- t.Fatalf("belongs=%v verified=%v, want false/true (doc outside the datasets)", belongs, verified)
- }
-}
-
-func TestDocInDatasetsAcceptsOwnDocument(t *testing.T) {
- belongs, verified := docInDatasets(context.Background(), SearchDeps{
- DocIDVerifier: stubVerifier{known: map[string]bool{"doc": true}},
- }, "doc")
-
- if !verified || !belongs {
- t.Fatalf("belongs=%v verified=%v, want true/true", belongs, verified)
- }
-}
-
-func TestDocInDatasetsUnverifiedWithoutVerifier(t *testing.T) {
- // Without a verifier the caller must keep the previous behaviour instead of
- // rejecting the document.
- belongs, verified := docInDatasets(context.Background(), SearchDeps{}, "doc")
-
- if !belongs || verified {
- t.Fatalf("belongs=%v verified=%v, want true/false", belongs, verified)
- }
-}
-
-func TestDocInDatasetsUnverifiedOnError(t *testing.T) {
- belongs, verified := docInDatasets(context.Background(), SearchDeps{
- DocIDVerifier: stubVerifier{err: errors.New("db down")},
- }, "doc")
-
- if !belongs || verified {
- t.Fatalf("belongs=%v verified=%v, want true/false (lookup failed)", belongs, verified)
- }
-}
-
-type stubNavRouter struct {
- called bool
- docs [][2]string
- // scope records the doc scope the router was asked to route within.
- scope []string
-}
-
-func (s *stubNavRouter) Route(_ context.Context, _, _, _ string, docScope []string, _ int) ([][2]string, error) {
- s.called = true
- s.scope = append([]string(nil), docScope...)
- return s.docs, nil
-}
-
-func TestDocInDatasetsRejectsForeignDocumentViaSubsetVerifier(t *testing.T) {
- belongs, verified := docInDatasets(context.Background(), SearchDeps{
- DocIDVerifier: subsetVerifier{knownDocs: map[string]bool{"other": true}},
- KbIDs: []string{"kb"},
- }, "doc")
-
- if !verified || belongs {
- t.Fatalf("belongs=%v verified=%v, want false/true (foreign doc rejected even with empty known map)", belongs, verified)
- }
-}
-
-// TestResolveDocScopeDropsScopeWhenAllUnknownViaSubsetVerifier exercises the
-// production empty-map path: when every candidate is foreign the verifier
-// returns an empty map (no error). The scope must be dropped (unfiltered retrieval), not
-// passed through.
-func TestResolveDocScopeDropsScopeWhenAllUnknownViaSubsetVerifier(t *testing.T) {
- got := resolveDocScope(context.Background(), SearchDeps{
- DocIDVerifier: subsetVerifier{knownDocs: map[string]bool{"z": true}},
- KbIDs: []string{"kb"},
- }, []string{"a", "b"}, []string{"kb"}, _LOG)
-
- if got != nil {
- t.Fatalf("scope = %v, want nil (unfiltered)", got)
- }
-}
-
-// TestResolveDocScopeEmptyWhenCeilingRemovesEveryID pins the ceiling's hard
-// edge: when the session doc_scope removes every requested id the result is
-// EMPTY (match nothing), never unfiltered. A falsy-empty check would fall through to
-// unfiltered retrieval here; the ceiling is deliberately kept absolute.
-func TestResolveDocScopeEmptyWhenCeilingRemovesEveryID(t *testing.T) {
- got := resolveDocScope(context.Background(), SearchDeps{
- DocIDVerifier: subsetVerifier{knownDocs: map[string]bool{"sess1": true}},
- KbIDs: []string{"kb"},
- DocScope: []string{"sess1"}, // fixed session base scope
- }, []string{"a", "b"}, []string{"kb"}, _LOG)
-
- if got == nil || len(got) != 0 {
- t.Fatalf("scope = %v, want non-nil empty (ceiling removed every requested id)", got)
- }
-}
-
-// TestResolveDocScopeEmptyWhenSessionScopeSetAndAllUnknown: when the session has a fixed
-// base doc_scope and the
-// requested ids survive the ceiling but do not resolve, the result is EMPTY,
-// never unfiltered. Encoded as a non-nil empty slice.
-func TestResolveDocScopeEmptyWhenSessionScopeSetAndAllUnknown(t *testing.T) {
- got := resolveDocScope(context.Background(), SearchDeps{
- DocIDVerifier: subsetVerifier{knownDocs: map[string]bool{"elsewhere": true}},
- KbIDs: []string{"kb"},
- DocScope: []string{"sess1"}, // fixed session base scope
- }, []string{"sess1", "sess2"}, []string{"kb"}, _LOG)
-
- if got == nil || len(got) != 0 {
- t.Fatalf("scope = %v, want non-nil empty (match nothing, not unfiltered)", got)
- }
-}
-
-// TestResolveDocScopeFallsBackToSessionScopeWhenRequestNone: a nil request scope falls
-// back to the
-// session's fixed doc_scope before ownership verification.
-func TestResolveDocScopeFallsBackToSessionScopeWhenRequestNone(t *testing.T) {
- got := resolveDocScope(context.Background(), SearchDeps{
- DocIDVerifier: subsetVerifier{knownDocs: map[string]bool{"sess1": true}},
- KbIDs: []string{"kb"},
- DocScope: []string{"sess1"}, // fixed session base scope
- }, nil, []string{"kb"}, _LOG) // request scope None
-
- if len(got) != 1 || got[0] != "sess1" {
- t.Fatalf("scope = %v, want [sess1] (request None falls back to session scope)", got)
- }
-}
-
-// TestResolveDocScopeUnfilteredWhenNoScopeAtAll pins that a nil request scope
-// AND no session base scope searches unfiltered (doc_scope = None), encoded as
-// nil — never a non-nil empty slice, which the backend would read as
-// "match nothing".
-func TestResolveDocScopeUnfilteredWhenNoScopeAtAll(t *testing.T) {
- got := resolveDocScope(context.Background(), SearchDeps{
- DocIDVerifier: subsetVerifier{knownDocs: map[string]bool{"z": true}},
- KbIDs: []string{"kb"},
- }, nil, []string{"kb"}, _LOG)
-
- if got != nil {
- t.Fatalf("scope = %v, want nil (unfiltered)", got)
- }
-}
-
-// TestScopedDocIDsCeiling: no session scope → the caller's scope as-is; a session scope
-// with no caller scope → the session scope; both → the intersection.
-func TestScopedDocIDsCeiling(t *testing.T) {
- cases := []struct {
- name string
- session []string
- scope []string
- want []string
- }{
- {"no session scope passes through", nil, []string{"a", "b"}, []string{"a", "b"}},
- {"session scope with no caller scope", []string{"s1"}, nil, []string{"s1"}},
- {"explicit scope is intersected", []string{"s1", "s2"}, []string{"s2", "x"}, []string{"s2"}},
- {"disjoint ceiling empties the scope", []string{"s1"}, []string{"x"}, []string{}},
- }
- for _, c := range cases {
- got := scopedDocIDs(c.session, c.scope)
- if len(got) != len(c.want) {
- t.Fatalf("%s: scopedDocIDs = %v, want %v", c.name, got, c.want)
- }
- for i := range got {
- if got[i] != c.want[i] {
- t.Fatalf("%s: scopedDocIDs = %v, want %v", c.name, got, c.want)
- }
- }
- }
-}
-
-// TestResolveDocScopeIntersectsExplicitScopeWithSession pins that a
-// model-generated scope can never escape the session's document ceiling: it is
-// intersected with it, not used as-is.
-func TestResolveDocScopeIntersectsExplicitScopeWithSession(t *testing.T) {
- got := resolveDocScope(context.Background(), SearchDeps{
- DocIDVerifier: subsetVerifier{knownDocs: map[string]bool{"sess1": true, "other": true}},
- KbIDs: []string{"kb"},
- DocScope: []string{"sess1"}, // session ceiling
- }, []string{"other", "sess1"}, []string{"kb"}, _LOG)
-
- if len(got) != 1 || got[0] != "sess1" {
- t.Fatalf("scope = %v, want [sess1] (explicit scope must be ceilinged by the session scope)", got)
- }
-}
-
-// TestGrepSearchMatchesPatternOverKeywordCandidates pins the integration: the
-// keyword leg samples the corpus, and the PATTERN decides which of those
-// candidates are evidence — each returned as the clause around its match.
-//
-// The two properties a term-locate pass cannot give: a candidate the pattern does
-// not match is not evidence whatever its retrieval score (c3), and the match is
-// reported with the sentence that carries it rather than the whole chunk.
-func TestGrepSearchMatchesPatternOverKeywordCandidates(t *testing.T) {
- long := strings.Repeat("前情提要。", 80) + "华雄出马,关公温酒斩之。" + strings.Repeat("余者不表。", 80)
- r := &stubRetriever{chunks: []map[string]any{
- {"chunk_id": "c1", "content": long},
- {"chunk_id": "c2", "content": "荀正 引军来战,被云长一刀斩于马下。"},
- {"chunk_id": "c3", "content": "曹操引军回许都,不在话下。"},
- }}
- deps, _ := newTestSearchDeps(r)
-
- got, _ := GrepSearch(context.Background(), deps, SearchParams{Question: "华雄|荀正|管亥"})
- if len(got) != 2 {
- t.Fatalf("got %d chunk(s), want 2: only the candidates the pattern matches are evidence", len(got))
- }
- ids := map[string]bool{}
- for _, c := range got {
- ids[ChunkIDOf(c)] = true
- }
- if !ids["c1"] || !ids["c2"] || ids["c3"] {
- t.Errorf("kept %v, want c1+c2 and NOT c3 (a candidate the pattern does not match is not evidence)", ids)
- }
- for _, c := range got {
- if ChunkIDOf(c) != "c1" {
- continue
- }
- if n := len([]rune(ChunkTextOf(c))); n > 200 {
- t.Errorf("c1 window = %d runes, want the clause around the match, not the whole %d-rune chunk", n, len([]rune(long)))
- }
- }
-
- // Nothing matched: the raw candidates are returned so evidence is never
- // dropped (and the reach line says what happened).
- none, _ := GrepSearch(context.Background(), deps, SearchParams{Question: "杨龄|夏侯存"})
- if len(none) != 3 {
- t.Errorf("no-match grep returned %d chunk(s), want all 3 raw candidates kept", len(none))
- }
-}
-
-// TestPatternNeverReachesTheEngine pins the invariant that makes `|` and `.*`
-// usable at all: recall is given the pattern's OPERANDS, and the pattern itself is
-// applied here.
-//
-// Handing the pattern to a keyword leg is what turned a batch probe into a silent
-// "the corpus does not carry it": the engine either escapes `|`/`.*` into
-// literals or reads them as its own syntax, and an empty result is then
-// indistinguishable from an absent fact.
-func TestPatternNeverReachesTheEngine(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{
- {"chunk_id": "c1", "content": "关公马快,早赶上文丑,脑后一刀,斩于马下。"},
- }}
- deps, _ := newTestSearchDeps(r)
-
- for _, q := range []string{"华雄|荀正|管亥", "关公.*斩"} {
- GrepSearch(context.Background(), deps, SearchParams{Question: q})
- for _, req := range r.requests {
- for _, bad := range []string{"|", ".*", ".+"} {
- if strings.Contains(req.Query, bad) {
- t.Errorf("query %q handed %q to the engine (request %q): pattern syntax must stay on this side", q, bad, req.Query)
- }
- }
- }
- }
- // And the recall that WAS issued is the operands, so the engine is still
- // asked about every alternative.
- joined := ""
- for _, req := range r.requests {
- joined += " " + req.Query
- }
- for _, want := range []string{"华雄", "荀正", "管亥", "关公", "斩"} {
- if !strings.Contains(joined, want) {
- t.Errorf("no request carried %q; recall must be issued per operand (%q)", want, joined)
- }
- }
-}
-
-// TestPerTermSearchRecordsConfirmedMembers pins that the weave's per-term search
-// doubles as the member record: a term searched ON ITS OWN that comes back with a
-// passage is a confirmed member with its evidence.
-func TestPerTermSearchRecordsConfirmedMembers(t *testing.T) {
- r := &stubRetriever{chunks: []map[string]any{
- {"chunk_id": "c-yan", "content": "荀正 引军来战,被云长一刀斩于马下。"},
- }}
- deps, kb := newTestSearchDeps(r)
-
- GrepSearch(context.Background(), deps, SearchParams{Question: "荀正|管亥"})
- got := kb.ReachedTerms()
- if len(got) != 1 || got[0].Term != "荀正" {
- t.Fatalf("ReachedTerms = %+v, want 荀正 recorded with the passage that carries it", got)
- }
- if got[0].ChunkID != "c-yan" {
- t.Errorf("recorded chunk = %q, want the passage the term's own search returned", got[0].ChunkID)
- }
-}
-
-// TestPatternRecallIsPerOperandAndWide pins the LOCATOR fix: a structural pattern
-// ("关公.*斩") asks the engine about each of its operands, at a width that gives
-// the pattern ground to match in.
-//
-// The pattern used to be only a FILTER: the legs' own topN (10 for `retrieve`)
-// decided what it could ever see, so a query about how a deed is written could
-// match inside ten passages, and reported per-term reach over those ten. That is
-// the one retrieval path that does not depend on the model already knowing the
-// name it is looking for — and it was looking through a keyhole.
-func TestPatternRecallIsPerOperandAndWide(t *testing.T) {
- answer := []map[string]any{{"chunk_id": "c1", "content": "关公勒马,一刀斩之"}}
-
- // Structural pattern: one search per operand, each WIDE.
- wide := &stubRetriever{chunks: answer}
- deps, _ := newTestSearchDeps(wide)
- GrepSearch(context.Background(), deps, SearchParams{
- Question: "关公.*斩|云长.*斩", TopN: 10, KbIDs: []string{"kb1"},
- })
- if len(wide.requests) < 2 {
- t.Fatalf("requests = %d, want one per operand", len(wide.requests))
- }
- for _, r := range wide.requests {
- if r.TopN < patternRecallTopN {
- t.Errorf("operand %q recalled TopN=%d, want >= %d", r.Query, r.TopN, patternRecallTopN)
- }
- }
-
- // An alternation of NAMES keeps the leg's own topN: a rare name's top ten is
- // enough, and widening there would only cost the engine.
- narrow := &stubRetriever{chunks: answer}
- depsNames, _ := newTestSearchDeps(narrow)
- GrepSearch(context.Background(), depsNames, SearchParams{
- Question: "华雄|荀正", TopN: 10, KbIDs: []string{"kb1"},
- })
- if len(narrow.requests) == 0 {
- t.Fatal("no requests: the alternation must still search per name")
- }
- for _, r := range narrow.requests {
- if r.TopN != 10 {
- t.Errorf("name probe %q recalled TopN=%d, want the leg's own 10", r.Query, r.TopN)
- }
- }
-}
-
-// TestCallerBatchRecognisesTheBatchTheModelWrites pins the weave's trigger.
-//
-// The per-term seat allocation exists for the batches the model writes, and over
-// the runs of 2026-09-15 it wrote them with SPACES: `关羽 斩 华雄 颜良 文丑 蔡阳`,
-// never once with `|`. A trigger keyed on `|` therefore left the whole mechanism
-// dead code while precisely those batches went to a single ranked top-N — the
-// ranking that hands nearly every seat to the passages matching the most terms at
-// once, which is how four names the corpus carries came back with nothing.
-func TestCallerBatchRecognisesTheBatchTheModelWrites(t *testing.T) {
- for _, q := range []string{
- "关羽 斩 华雄 颜良 文丑 蔡阳",
- "华雄|颜良|文丑",
- "关公.*斩|云长.*斩",
- } {
- if !callerBatch(q) {
- t.Errorf("callerBatch(%q) = false, want the batch recognised", q)
- }
- }
- // A sentence is not a batch, in either language: that is what keeps the extra
- // searches off the questions that are not enumerating anything.
- for _, q := range []string{
- "三国演义中,关羽杀了多少有姓名的人物?",
- "What was the outcome of the battle at Red Cliffs?",
- "关羽",
- } {
- if callerBatch(q) {
- t.Errorf("callerBatch(%q) = true, want a sentence left on the single search", q)
- }
- }
-}
-
-// TestProbeItemsAreTheCallersOwnWords pins the reach ledger's reading of a call.
-//
-// The ledger is read back as the session's to-do list ("probed, came back with a
-// passage, not recorded"), so it may only hold what the call PROPOSED as items —
-// the pieces of a batch, or a query that is one word. Measured (2026-09-15): the
-// line read `FOUND BUT NOT RECORDED=三国、演义、关羽、五关…+15` in a run whose
-// sessions were missing six members, none of which was on the list, because the
-// windows an unbroken clause decomposes into had been probed AND recorded.
-func TestProbeItemsAreTheCallersOwnWords(t *testing.T) {
- got := probeItemsOf([]string{"关羽 古城 蔡阳 斩 颜良 文丑 华雄 庞德 荀正", "韩福"})
- for _, want := range []string{"蔡阳", "颜良", "华雄", "荀正", "韩福"} {
- if !got[strings.ToLower(want)] {
- t.Errorf("probeItemsOf missed the proposed item %q", want)
- }
- }
- // A one-rune verb is stripped as a term edge, not proposed as an item.
- if got["斩"] {
- t.Error("a single-rune fragment must not count as a proposed item")
- }
-
- // A question is not a proposal, and an unbroken clause yields no item either.
- if items := probeItemsOf([]string{"三国演义中关羽一共杀死多少有姓名的人物"}); len(items) != 0 {
- t.Errorf("a sentence proposed %v as items, want nothing", items)
- }
-
- // The batch case that produced the junk: the windows of 关羽过五关斩六将 must
- // not appear, while the caller's own words do.
- batch := probeItemsOf([]string{"三国演义 关羽过五关斩六将 六将姓名"})
- for _, window := range []string{"国演", "演义", "羽过", "过五", "关斩", "斩六"} {
- if batch[window] {
- t.Errorf("window %q must never reach the ledger", window)
- }
- }
- if !batch["三国演义"] {
- t.Error("the caller's own word 三国演义 must be a candidate item")
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/tool_text_processing.go b/internal/rag/agentic-rag/runtime/tool_text_processing.go
deleted file mode 100644
index 63ad93acb2..0000000000
--- a/internal/rag/agentic-rag/runtime/tool_text_processing.go
+++ /dev/null
@@ -1,698 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "context"
- "crypto/md5"
- "fmt"
- "log"
- "regexp"
- "slices"
- "sort"
- "strings"
-)
-
-// Text processing: keyword narrowing of retrieved chunks.
-//
-// This covers the narrowing family (keep-or-narrow, keyword narrowing, content
-// narrowing, term narrowing) and the compiled-structure grep_sed narrowing. It lives next
-// to search.go so the only two consumers (HybridSearch and the structure-nav grepper)
-// both reach it.
-
-// NarrowOrKeep: narrow chunks to
-// keyword-bearing sentences, but keep the originals when narrowing would drop
-// everything.
-//
-// The all-or-nothing behaviour is the point: no keyword overlap does NOT mean
-// irrelevant. The retriever already ranked these chunks, and a sub-question's
-// wording need not contain the parent question's keywords. Dropping them all
-// produced empty results, unverified claims and pointless retry cycles.
-//
-// Both outcomes are reported to the DEVELOPER log only (Python parity), never to
-// the think block: the resize ratio is pool bookkeeping, and the leg's own result
-// line already reports what came back.
-func NarrowOrKeep(ctx context.Context, chunks []map[string]any, keywords, label string, logger *log.Logger) []map[string]any {
- if strings.TrimSpace(keywords) == "" || len(chunks) == 0 {
- return chunks
- }
- if logger == nil {
- logger = _LOG
- }
- // LOG-ONLY, and worded exactly as Python words it (text_processing.py:464/:466):
- // how the keyword filter resized the candidate pool is a developer's diagnostic,
- // not something a reader acts on — the leg's own result line already reports what
- // came back, including the narrowed count. It used to be a step (and, before
- // that, a rewritten sentence), which put pool bookkeeping in front of the user.
- //
- // ctx is unused here on purpose: the signature stays uniform with the other
- // narrowing entry points, and a future step would have it available.
- narrowed := NarrowByKeywords(chunks, keywords)
- if len(narrowed) > 0 {
- logger.Printf("[%s] Kept %d of %d passage(s) that actually mention the keywords.",
- label, len(narrowed), len(chunks))
- return narrowed
- }
- logger.Printf("[%s] Keyword narrowing matched nothing — keeping all %d retrieved passage(s).",
- label, len(chunks))
- return chunks
-}
-
-// NarrowByKeywords narrows each chunk to the sentences mentioning any keyword
-// (+/-1 neighbour) and drops keyword-less chunks.
-//
-// Unlike NarrowOrKeep this is the strict form: it may return an empty slice, and callers
-// that must not lose evidence should use NarrowOrKeep instead.
-func NarrowByKeywords(chunks []map[string]any, keywords string) []map[string]any {
- kwds := SplitKeywords(keywords)
- // The input is returned unchanged when there is nothing to narrow on. A nil return
- // would wipe the whole evidence pool, so return the original chunks verbatim instead.
- if len(kwds) == 0 || len(chunks) == 0 {
- return chunks
- }
- out := make([]map[string]any, 0, len(chunks))
- seen := make(map[string]bool, len(chunks))
- for _, c := range chunks {
- if c == nil {
- continue
- }
- narrowed, ok := NarrowContent(ChunkTextOf(c), kwds)
- if !ok {
- continue
- }
- // Dedup identical narrowed passages (chunks whose narrowed text hashes the same).
- h := md5.Sum([]byte(narrowed))
- key := fmt.Sprintf("%x", h)
- if seen[key] {
- continue
- }
- seen[key] = true
- cp := make(map[string]any, len(c)+1)
- for k, v := range c {
- cp[k] = v
- }
- // content_with_weight is
- // always overwritten with the narrowed text, "content" is mirrored ONLY
- // when the original chunk already carried a "content" key, and the
- // pre-narrow "highlight" spans are dropped because they no longer apply.
- // (withNarrowedText in grep_sed_narrow.go applies the identical rule for
- // the term path; keep the two in lock-step.)
- cp["content_with_weight"] = narrowed
- if _, ok := cp["content"]; ok {
- cp["content"] = narrowed
- }
- delete(cp, "highlight")
- out = append(out, cp)
- }
- return out
-}
-
-// SplitKeywords normalizes a keyword string into search terms. When fewer than
-// 3 comma terms exist, falls back to space-split bigrams — a bare keyword blob
-// ("finale run time") is more discriminative as bigrams than as single words.
-// This is the term construction used by NarrowByKeywords.
-func SplitKeywords(keywords string) []string {
- if strings.TrimSpace(keywords) == "" {
- return nil
- }
- kwds := make([]string, 0, 8)
- for _, k := range strings.Split(keywords, ",") {
- if k = strings.TrimSpace(k); k != "" {
- kwds = append(kwds, strings.ToLower(k))
- }
- }
- if len(kwds) < 3 {
- words := make([]string, 0, 8)
- for _, w := range strings.Fields(keywords) {
- words = append(words, strings.ToLower(w))
- }
- bigrams := make([]string, 0, len(words))
- for i := 0; i+1 < len(words); i++ {
- bigrams = append(bigrams, words[i]+" "+words[i+1])
- }
- if len(bigrams) > 0 {
- return bigrams
- }
- }
- return kwds
-}
-
-// Stem-aware keyword matching: the stemmed forms of a keyword and of the text's words are
-// matched, so e.g. "nominated" is highlighted for the keyword "nominations".
-
-var wordRe = regexp.MustCompile("[a-z0-9]+")
-
-// wordLetterRe finds the words to stem-match. It is deliberately NOT the shared lowercase
-// wordRe, which on capitalized text matches only fragments ("New" -> "ew", "Nominated" ->
-// "ominated") and therefore never yields the stem term those words should contribute.
-var wordLetterRe = regexp.MustCompile("[A-Za-z]+")
-
-// containedInPhrase reports whether low occurs inside any keyword phrase.
-func containedInPhrase(low string, phrases map[string]struct{}) bool {
- for p := range phrases {
- if strings.Contains(p, low) {
- return true
- }
- }
- return false
-}
-
-func isAlphaOnly(s string) bool {
- for _, r := range s {
- if !((r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z')) {
- return false
- }
- }
- return true
-}
-
-// stemmable: len>=4 and purely ASCII letters.
-func stemmable(token string) bool { return len(token) >= 4 && isAlphaOnly(token) }
-
-// stem is the stemmer used throughout: porterStem is a faithful port of nltk's
-// PorterStemmer (NLTK_EXTENSIONS mode), so the stems match word-for-word. The old
-// suffix-stripping fallback is gone: it diverged on exactly the words that matter for
-// keyword narrowing.
-func stem(word string) string { return porterStem(word) }
-
-// keywordForms: verbatim keeps forms containing
-// any non-stemmable token (matched by substring); stemmed holds all-ASCII-letter
-// keyword forms as stem tuples (matched by a contiguous stem sequence).
-func keywordForms(kwds []string) (verbatim []string, stemmed [][]string) {
- for _, kw := range kwds {
- k := strings.ToLower(strings.TrimSpace(kw))
- if k == "" {
- continue
- }
- tokens := wordRe.FindAllString(k, -1)
- if len(tokens) > 0 && allStemmable(tokens) {
- seq := make([]string, len(tokens))
- for i, t := range tokens {
- seq[i] = stem(t)
- }
- stemmed = append(stemmed, seq)
- } else {
- verbatim = append(verbatim, k)
- }
- }
- return
-}
-
-func allStemmable(tokens []string) bool {
- for _, t := range tokens {
- if !stemmable(t) {
- return false
- }
- }
- return true
-}
-
-// sentenceStems
-func sentenceStems(sentence string) []string {
- tokens := wordRe.FindAllString(strings.ToLower(sentence), -1)
- out := make([]string, len(tokens))
- for i, t := range tokens {
- if stemmable(t) {
- out[i] = stem(t)
- } else {
- out[i] = t
- }
- }
- return out
-}
-
-// sentenceMatches: any verbatim substring OR a
-// contiguous stemmed sequence.
-func sentenceMatches(low string, stems, verbatim []string, stemmed [][]string) bool {
- for _, v := range verbatim {
- if strings.Contains(low, v) {
- return true
- }
- }
- for _, seq := range stemmed {
- width := len(seq)
- if width == 0 || width > len(stems) {
- continue
- }
- for start := 0; start+width <= len(stems); start++ {
- if slices.Equal(stems[start:start+width], seq) {
- return true
- }
- }
- }
- return false
-}
-
-// NarrowContent returns the keyword-bearing sentences (+/-2 neighbours) with
-// the keywords highlighted, or ("", false) when no keyword occurs.
-// Keyword sentences are kept within a +/-2 window, AND fact-dense sentences (numbers /
-// years / percentages / proper nouns) within a +/-1 window even without a keyword hit, so
-// numeric or named-entity answers survive narrowing. Block-level tables and markdown
-// pipe-tables (>=3 rows) are returned whole — keyword-window narrowing would otherwise
-// truncate them.
-func NarrowContent(content string, kwds []string) (string, bool) {
- if strings.TrimSpace(content) == "" || len(kwds) == 0 {
- return "", false
- }
- lowContent := strings.ToLower(content)
- if strings.Contains(lowContent, "<table") || strings.Contains(lowContent, "<tr") || strings.Contains(lowContent, "<td") {
- // Serialize the table to Markdown before the model sees it: the format
- // comparison over 11 serializations ranks Markdown-KV first for field lookups
- // (key: value beats header/position alignment) and a Markdown table as the
- // cost/accuracy compromise; raw HTML is the expensive and least readable
- // option. The row set is NOT pruned — rank/order and completeness decide table
- // answers. Falls back to the raw text when nothing renders.
- return "..." + HighlightKeywords(TableViewOrRaw(content), kwds) + "...", true
- }
- pipeRows := 0
- for _, line := range strings.Split(content, "\n") {
- if strings.Count(line, "|") >= 2 {
- pipeRows++
- }
- }
- if pipeRows >= 3 {
- return "..." + HighlightKeywords(content, kwds) + "...", true
- }
- sents := SplitSentences(content)
- if len(sents) == 0 {
- return "", false
- }
- verbatim, stemmed := keywordForms(kwds)
- keep := make(map[int]bool, len(sents))
- matched := false
- for i, s := range sents {
- low := strings.ToLower(s)
- hit := sentenceMatches(low, sentenceStems(s), verbatim, stemmed)
- if hit {
- matched = true
- for j := max(0, i-2); j < min(len(sents), i+3); j++ {
- keep[j] = true
- }
- } else if IsFactDenseSentence(s) {
- // Keep fact-dense sentences even without a keyword hit so the answer
- // value (a bare figure, a date, a proper noun) is never lost.
- for j := max(0, i-1); j < min(len(sents), i+2); j++ {
- keep[j] = true
- }
- }
- }
- if !matched {
- return "", false
- }
- var b strings.Builder
- for i := range sents {
- if keep[i] {
- b.WriteString(sents[i])
- }
- }
- return "..." + HighlightKeywords(b.String(), kwds) + "...", true
-}
-
-// HighlightKeywords stars keyword occurrences, longest term first so a longer keyword is
-// not partially consumed by a shorter one. The marker is a STAR, not an XML tag — a
-// multi-word entity must stay ONE contiguous span
-// ("*Atlanta Braves*", never "*Atlanta* *Braves*") for the downstream
-// entity cross-check. The <em> tags elsewhere in this port are the ENGINE's
-// highlight markup (rag/utils/*_conn.py, agentic_search.go), a different layer.
-func HighlightKeywords(text string, kwds []string) string {
- if len(kwds) == 0 {
- return text
- }
- terms := append([]string(nil), kwds...)
- // The phrase set: keywords trimmed, lowercased and deduplicated. It guards the stem
- // terms added just below.
- phrases := make(map[string]struct{}, len(kwds))
- for _, kw := range kwds {
- if p := strings.ToLower(strings.TrimSpace(kw)); p != "" {
- phrases[p] = struct{}{}
- }
- }
- // Stem-based highlight terms: a stemmed form that matches in the text is wrapped too,
- // so e.g. "nominated" is highlighted for keyword "nominations".
- _, stemmed := keywordForms(kwds)
- if len(stemmed) > 0 {
- stemSet := make(map[string]bool, 8)
- for _, seq := range stemmed {
- for _, s := range seq {
- stemSet[s] = true
- }
- }
- for _, word := range wordLetterRe.FindAllString(text, -1) {
- low := strings.ToLower(word)
- // A stem-matched word is added only when it is NOT already inside a keyword phrase:
- // "nominated" is starred for "nominations", while the "Braves" of "Atlanta Braves"
- // is left to the phrase's own span instead of being starred on its own elsewhere.
- if stemmable(low) && stemSet[stem(low)] && !containedInPhrase(low, phrases) {
- terms = append(terms, low)
- }
- }
- }
- // Match in RUNE space. `strings.ToLower` is not byte-length-preserving: "İ"
- // is 2 bytes and folds to the 1-byte "i", so a byte offset taken from the
- // original indexes the folded string at a different position. The loop then
- // slices past the end of the folded string (panic: slice bounds out of
- // range) or cuts a rune in half and emits invalid UTF-8. Go's case mapping is
- // 1:1 per RUNE, so a rune index is valid in both strings. (A regex-based rewrite is
- // immune for a different reason: it re-emits the matched group from the ORIGINAL text
- // instead of re-slicing it.)
- rs := []rune(text)
- lows := []rune(strings.ToLower(text))
- if len(lows) != len(rs) {
- // Unreachable while the fold stays rune-for-rune; kept so a future switch
- // to a full case fold (which does change the rune count) degrades to
- // plain text instead of misaligned spans.
- return text
- }
- // Longest term first, compared by rune count — i.e. by code points.
- termRunes := make([][]rune, 0, len(terms))
- for _, t := range terms {
- // Fold the TERM the same way the haystack was folded: the match below
- // compares against `lows`, and only the stem-derived terms appended above
- // were already lowercase — a caller-supplied "Rocket" or "New York" kept
- // its casing and therefore never matched, silently dropping the highlight.
- // The phrase list is built with a strip+lower pass and matched case-insensitively, so
- // the trim and the case fold both belong here.
- if t = strings.ToLower(strings.TrimSpace(t)); t != "" {
- termRunes = append(termRunes, []rune(t))
- }
- }
- sort.SliceStable(termRunes, func(i, j int) bool { return len(termRunes[i]) > len(termRunes[j]) })
- var b strings.Builder
- for i := 0; i < len(rs); {
- bestLen := 0
- for _, t := range termRunes {
- if len(t) > bestLen && runesHavePrefix(lows[i:], t) {
- bestLen = len(t)
- }
- }
- if bestLen == 0 {
- b.WriteRune(rs[i])
- i++
- continue
- }
- // Emit the ORIGINAL runes, so the highlight keeps the source casing
- // wrapped in the star marker.
- b.WriteString("*")
- b.WriteString(string(rs[i : i+bestLen]))
- b.WriteString("*")
- i += bestLen
- }
- return b.String()
-}
-
-// runesHavePrefix reports whether hay starts with needle.
-func runesHavePrefix(hay, needle []rune) bool {
- if len(needle) == 0 || len(needle) > len(hay) {
- return false
- }
- for i, r := range needle {
- if hay[i] != r {
- return false
- }
- }
- return true
-}
-
-// IsFactDenseSentence reports whether a sentence carries a fact-bearing signal: a
-// number / year / percentage / magnitude word or a proper noun that is not part of an
-// abbreviation run. It keeps only informative sentences when narrowing / grepping, so a
-// numeric or entity answer is never dropped just because it lacks the query keywords.
-// Deliberately NO quoted-span or ≥6-token rule, and no bare digit counts as a fact
-// signal — those widened the gate far beyond the strict definition.
-func IsFactDenseSentence(s string) bool {
- s = strings.TrimSpace(s)
- if s == "" {
- return false
- }
- // The regex is case-insensitive, so a single (?i) search covers both the original and
- // the lowercased form.
- if factPattern.MatchString(s) {
- return true
- }
- if hasProperNoun(s) {
- return true
- }
- return false
-}
-
-var (
- // factPattern: a number that
- // may carry an ordinal suffix (st/nd/rd/th) or a percent sign and may use
- // comma/dot group separators, a 1900-2099 four-digit year, or a magnitude
- // word (percent/million/billion/thousand/km/km2/sq km/m above/m). re.IGNORECASE
- // makes the words case-insensitive.
- factPattern = regexp.MustCompile(`(?i)(?:\d[\d,\.]*(?:st|nd|rd|th)?%?)|(?:19|20)\d{2}|\b(?:percent|percentage|million|billion|thousand|km|km2|sq\s*km|m\s*above|m)\b`)
-
- // properNounPattern: word pattern: a
- // capitalized word of at least three letters, normally wrapped in the negative lookbehind
- // (?<![.!?]\.); RE2 has no lookbehind, so that abbreviation guard is applied by
- // hasProperNoun instead of in the regex.
- properNounPattern = regexp.MustCompile(`\b[A-Z][a-z]{2,}\b`)
-)
-
-// hasProperNoun reports whether s contains a capitalized proper noun that is not
-// preceded by an abbreviation point. The pattern is
-// (?<![.!?]\.)\b[A-Z][a-z]{2,}\b: the guard rejects a match whose start sits right
-// after the two characters "X." where X ∈ {., !, ?} (e.g. a "U.S."-style run).
-func hasProperNoun(s string) bool {
- for _, loc := range properNounPattern.FindAllStringIndex(s, -1) {
- idx := loc[0]
- // The negative lookbehind (?<![.!?]\.) fails when s[idx-2] ∈ {. ! ?} and
- // s[idx-1] == '.'. A match at idx < 2 cannot be guarded, so keep it.
- if idx >= 2 && (s[idx-2] == '.' || s[idx-2] == '!' || s[idx-2] == '?') && s[idx-1] == '.' {
- continue
- }
- return true
- }
- return false
-}
-
-// Sentence segmentation.
-//
-// The foundation for every narrowing / highlighting / fact-density step in the
-// retrieval. Two properties matter:
-//
-// 1. terminators (。!?;!?; and a digit-guarded English period) are KEPT on
-// their sentence, so "3.14" and "v1.2" do not split;
-// 2. block-level HTML elements (table/div/p/ul/li/... — see htmlBlockTags) and
-// markdown tables are ATOMIC and never split internally, so a whole table /
-// list / block counts as ONE "sentence" for keyword matching (a keyword
-// inside one keeps the whole block). The scanning is nesting-aware, not a bare
-// <table> regex.
-//
-// Go's RE2 lacks lookbehind, so the digit guard is a manual scan rather than a
-// regex assertion.
-
-// htmlBlockTags are the block-level HTML containers kept atomic during sentence
-// splitting. Inline tags like <b>/<i>/<em> are deliberately excluded, so ordinary prose
-// still splits.
-var htmlBlockTags = map[string]bool{
- "table": true, "thead": true, "tbody": true, "tfoot": true, "tr": true,
- "td": true, "th": true, "caption": true, "colgroup": true, "ul": true,
- "ol": true, "li": true, "dl": true, "dt": true, "dd": true, "div": true,
- "p": true, "pre": true, "blockquote": true, "section": true, "article": true,
- "aside": true, "nav": true, "main": true, "figure": true, "figcaption": true,
- "header": true, "footer": true, "address": true, "details": true, "summary": true,
- "form": true, "fieldset": true, "h1": true, "h2": true, "h3": true, "h4": true,
- "h5": true, "h6": true,
-}
-
-var htmlTagRe = regexp.MustCompile(`(?i)<(/?)([a-zA-Z][a-zA-Z0-9]*)\b([^>]*)>`)
-
-// mdTableRe: header row with a pipe, a separator row
-// of dashes/colons/pipes, then zero+ body rows with a pipe.
-var mdTableRe = regexp.MustCompile("(?m)^[ \t]*\\|?[^\n]*\\|[\n][ \t]*\\|?[ \t]*:?-{1,}:?[ \t]*(?:\\|[ \t]*:?-{1,}:?[ \t]*)+\\|?[ \t]*\r?\n(?:[ \t]*\\|?[^\n]*\\|[^\n]*\r?\n?)*")
-
-// htmlBlockSpans returns outermost balanced block-level HTML element spans
-// (nesting-aware) via a tag stack.
-func htmlBlockSpans(text string) [][2]int {
- type stackItem struct {
- name string
- start int
- }
- var spans [][2]int
- var stack []stackItem
- for _, m := range htmlTagRe.FindAllStringSubmatchIndex(text, -1) {
- name := strings.ToLower(text[m[4]:m[5]])
- if !htmlBlockTags[name] {
- continue
- }
- closing := m[2] != -1 && m[2] != m[3] // group 1 captured => closing tag
- if closing {
- for i := len(stack) - 1; i >= 0; i-- {
- if stack[i].name == name {
- start := stack[i].start
- stack = stack[:i]
- if len(stack) == 0 { // closed an outermost block
- spans = append(spans, [2]int{start, m[1]})
- }
- break
- }
- }
- } else {
- attrs := text[m[6]:m[7]]
- if strings.HasSuffix(strings.TrimSpace(attrs), "/") {
- continue // self-closing
- }
- stack = append(stack, stackItem{name: name, start: m[2]})
- }
- }
- return spans
-}
-
-// protectedSpans returns non-overlapping atomic (start, end) spans in order, covering
-// block-level HTML elements and markdown tables (overlaps are unioned).
-func protectedSpans(text string) [][2]int {
- spans := htmlBlockSpans(text)
- for _, m := range mdTableRe.FindAllStringIndex(text, -1) {
- spans = append(spans, [2]int{m[0], m[1]})
- }
- sort.Slice(spans, func(i, j int) bool { return spans[i][0] < spans[j][0] })
- var merged [][2]int
- lastEnd := -1
- for _, s := range spans {
- if s[0] < lastEnd { // overlaps an already-kept span -> union it in
- if s[1] > lastEnd {
- merged[len(merged)-1][1] = s[1]
- lastEnd = s[1]
- }
- continue
- }
- merged = append(merged, s)
- lastEnd = s[1]
- }
- return merged
-}
-
-// SplitSentences splits text into sentences, treating each block-level HTML
-// element and markdown table as one atomic unit.
-func SplitSentences(text string) []string {
- if text == "" {
- return nil
- }
- spans := protectedSpans(text)
- if len(spans) == 0 {
- return splitPlainSentences(text)
- }
- var sents []string
- pos := 0
- for _, m := range spans {
- if m[0] > pos {
- sents = append(sents, splitPlainSentences(text[pos:m[0]])...)
- }
- if block := text[m[0]:m[1]]; strings.TrimSpace(block) != "" {
- sents = append(sents, block)
- }
- pos = m[1]
- }
- if pos < len(text) {
- sents = append(sents, splitPlainSentences(text[pos:])...)
- }
- return sents
-}
-
-// splitPlainSentences splits plain text (no table/block spans) into sentences,
-// keeping each terminator attached and guarding decimal periods. Operates on
-// runes; rune indices == byte indices for the ASCII terminators we emit.
-//
-// The split is LOSSLESS: inter-sentence whitespace is kept as the prefix of the
-// FOLLOWING sentence (only whitespace-only segments are dropped), so
-// `"".join(sents)` reproduces the input, and the content narrowing depends on that
-// property — it rejoins the kept sentences with "", and trimming each sentence would
-// collapse a multi-line chunk into one line, which the grep term-window (line-based) then
-// matches wholesale instead of line by line. Observed as the grep leg keeping an entire
-// trailing paragraph.
-func splitPlainSentences(text string) []string {
- rs := []rune(text)
- var sents []string
- start := 0
- for i := 0; i < len(rs); i++ {
- r := rs[i]
- if !isSentTerminator(r) {
- continue
- }
- // ASCII period guarded against decimals (digit on BOTH sides).
- if r == '.' && i > 0 && i+1 < len(rs) && isASCIIDigit(rs[i-1]) && isASCIIDigit(rs[i+1]) {
- continue
- }
- // Consume a run of terminators (e.g. "。!?" or "...").
- j := i + 1
- for j < len(rs) && isSentTerminator(rs[j]) && rs[j] != '.' {
- j++
- }
- seg := string(rs[start:j])
- if strings.TrimSpace(seg) != "" {
- sents = append(sents, seg)
- }
- start = j
- i = j - 1
- }
- if start < len(rs) {
- if tail := string(rs[start:]); strings.TrimSpace(tail) != "" {
- sents = append(sents, tail)
- }
- }
- return sents
-}
-
-func isSentTerminator(r rune) bool {
- switch r {
- case '。', '!', '?', ';', '!', '?', ';', '.':
- return true
- }
- return false
-}
-
-func isASCIIDigit(r rune) bool { return r >= '0' && r <= '9' }
-
-// Keyword compaction.
-//
-// Post-processing for keyword extraction: dedupe (preserving order)
-// and cap the compacted keyword string at compactMaxKeywords terms. The
-// extraction prompt asks for 3-10 terms PLUS 2-3 synonyms each, which models
-// answer with a 40-60 word redundant synonym run; appending that whole run onto
-// the query diluted the vector leg and dragged BM25 onto unrelated docs. This
-// keeps the recall terms but drops the redundancy, so keywords stay a compact
-// hint instead of a pollution source.
-//
-// It lives here because it is a text-processing primitive of the runtime, not an
-// agentic-pipe stage.
-
-// compactMaxKeywords caps the compacted keyword string.
-const compactMaxKeywords = 15
-
-// CompactKeywords dedupes (preserving order) and caps at compactMaxKeywords.
-//
-// Accepts both space- and comma-separated input (single-turn extract_keywords
-// emits spaces; multi-turn formalize emits commas).
-func CompactKeywords(kw string) string {
- if strings.TrimSpace(kw) == "" {
- return ""
- }
- var seen []string
- for _, t := range regexp.MustCompile(`[,\s]+`).Split(strings.TrimSpace(kw), -1) {
- t = strings.TrimSpace(t)
- if t == "" || containsStr(seen, t) {
- continue
- }
- seen = append(seen, t)
- if len(seen) >= compactMaxKeywords {
- break
- }
- }
- return strings.Join(seen, " ")
-}
diff --git a/internal/rag/agentic-rag/runtime/tool_text_processing_test.go b/internal/rag/agentic-rag/runtime/tool_text_processing_test.go
deleted file mode 100644
index 5c5c6d78e9..0000000000
--- a/internal/rag/agentic-rag/runtime/tool_text_processing_test.go
+++ /dev/null
@@ -1,275 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package runtime
-
-import (
- "regexp"
- "strings"
- "testing"
- "unicode/utf8"
-)
-
-func TestEscapeTermAsciiWord(t *testing.T) {
- got := EscapeTerm("cat")
- if got != `\bcat\b` {
- t.Errorf("EscapeTerm(cat) = %q, want \\bcat\\b (case-insensitivity is added by TermsToPatterns via (?i))", got)
- }
- if _, err := regexp.Compile(got); err != nil {
- t.Errorf("bad regexp: %v", err)
- }
-}
-
-func TestEscapeTermCJK(t *testing.T) {
- got := EscapeTerm("机器")
- if got != "机器" {
- t.Errorf("EscapeTerm(机器) = %q, want bare CJK", got)
- }
- if _, err := regexp.Compile(got); err != nil {
- t.Errorf("bad regexp: %v", err)
- }
-}
-
-func TestEscapeTermSymbols(t *testing.T) {
- got := EscapeTerm("c++")
- // + is regex-special -> escaped to "\+".
- if got != `c\+\+` {
- t.Errorf("EscapeTerm(c++) = %q, want c\\+\\+", got)
- }
- re, err := regexp.Compile(got)
- if err != nil {
- t.Errorf("bad regexp: %v", err)
- }
- if !re.MatchString("c++") {
- t.Error("pattern should match literal c++")
- }
- if re.MatchString("c--") {
- t.Error("pattern should NOT match c--")
- }
-}
-
-func TestEscapeTermEmpty(t *testing.T) {
- if EscapeTerm("") != "" {
- t.Error("empty term must stay empty so TermsToPatterns can skip it")
- }
-}
-
-func TestTermsToPatternsCapsAndSkipsEmpty(t *testing.T) {
- // 40 terms -> capped at maxGrepTerms (16); empties skipped.
- terms := make([]string, 40)
- for i := range terms {
- terms[i] = "t"
- }
- pats := TermsToPatterns(terms)
- if len(pats) != maxGrepTerms {
- t.Errorf("len = %d, want capped at %d", len(pats), maxGrepTerms)
- }
- // Mixed empties (whitespace) are skipped.
- pats = TermsToPatterns([]string{"", " ", ",,,", "cat"})
- if len(pats) != 1 {
- t.Errorf("len = %d, want 1 (only 'cat' compiles)", len(pats))
- }
-}
-
-// mustNarrow is a convenience for the full NarrowByTerms signature with the
-// default context and caps used by the term path.
-func mustNarrow(chunks []map[string]any, terms, fallbackTerms []string, keywords string) NarrowResult {
- return NarrowByTerms(chunks, terms, fallbackTerms, keywords, NarrowContext{Before: 2, After: 2}, defaultOutCharsPerChunk, defaultOutTotalChars)
-}
-
-func TestNarrowByTermsKeepsAllChunks(t *testing.T) {
- // NarrowByTerms NEVER drops a chunk: short chunks are kept whole, long ones
- // are narrowed to the hit window. Only Matched / the returned text vary.
- chunks := []map[string]any{
- {"content": "Culdcept was made by OmiyaSoft in 1999."},
- {"content": "The weather is nice today."},
- {"content": "OmiyaSoft released Culdcept."},
- }
- res := mustNarrow(chunks, []string{"culdcept", "omiya"}, nil, "")
- if len(res.Kept) != len(chunks) {
- t.Fatalf("Kept len = %d, want all %d kept", len(res.Kept), len(chunks))
- }
- if !res.Stats.Matched {
- t.Error("Matched = false, want true (terms hit)")
- }
- if res.Stats.ChunksKpt != len(chunks) {
- t.Errorf("ChunksKpt = %d, want %d", res.Stats.ChunksKpt, len(chunks))
- }
-}
-
-func TestNarrowByTermsFallbackTermsTriedOnNoPrimaryHit(t *testing.T) {
- // Primary terms hit nothing -> the fallback term list is tried mechanically.
- chunks := []map[string]any{
- {"content": "Culdcept was made by OmiyaSoft."},
- {"content": "The weather is nice today."},
- }
- res := mustNarrow(chunks, []string{"zzzz"}, []string{"culdcept"}, "")
- if !res.Stats.Matched {
- t.Error("Matched = false, want true after the fallback term path hit")
- }
- if len(res.Kept) != len(chunks) {
- t.Errorf("Kept len = %d, want both retained", len(res.Kept))
- }
-}
-
-func TestNarrowWithFallbackKeywordAndOriginals(t *testing.T) {
- chunks := []map[string]any{
- {"content": "Culdcept was made by OmiyaSoft."},
- {"content": "The weather is nice today."},
- }
- // Keyword narrowing matches -> narrowed subset.
- if got := NarrowWithFallbackKeyword(chunks, "culdcept"); len(got) != 1 {
- t.Errorf("keyword fallback kept %d, want 1 (only the matching chunk)", len(got))
- }
- // No keyword -> originals kept (never drop).
- if got := NarrowWithFallbackKeyword(chunks, ""); len(got) != len(chunks) {
- t.Errorf("len = %d, want all %d on empty keyword", len(got), len(chunks))
- }
-}
-
-func TestIsFactDenseSentence(t *testing.T) {
- cases := []struct {
- s string
- want bool
- }{
- {"Culdcept was released in 1999 by OmiyaSoft with 50 cards.", true}, // year + digit + proper nouns
- {"Revenue grew by 5% this quarter.", true}, // percent
- {"A 3.14 version was shipped.", true}, // decimal number
- {"The project spans over two million users.", true}, // magnitude word (case-insensitive)
- {"Culdcept was released by OmiyaSoft.", true}, // proper noun
- // A quoted span or a ≥6-token clause is NOT a fact signal on its own.
- {"He said \"hello\" to the group about the meeting.", false}, // quoted only
- {"the cat sat on the mat and looked at the bird outside.", false}, // >=6 tokens only
- {"the cat sat", false}, // short, no signal
- // Abbreviation guard: a proper noun right after ".", "!" or "?" followed by
- // a dot is not counted (the abbreviation guard). The lead "It" is
- // only one lowercase letter so it is not a proper noun either; Atlas is the
- // sole candidate and it is excluded.
- {"It was wonderful!!..Atlas shrugged", false},
- }
- for _, c := range cases {
- if got := IsFactDenseSentence(c.s); got != c.want {
- t.Errorf("IsFactDenseSentence(%q) = %v, want %v", c.s, got, c.want)
- }
- }
-}
-
-func TestNarrowByTermsNilInput(t *testing.T) {
- res := mustNarrow(nil, nil, nil, "")
- if res.Kept != nil || res.Stats.ChunksIn != 0 {
- t.Errorf("nil input -> Kept len %d, ChunksIn %d, want (nil, 0)", len(res.Kept), res.Stats.ChunksIn)
- }
-}
-
-// TestNarrowByKeywordsEmptyReturnsChunks: with nothing to narrow on, the input chunks are
-// returned unchanged — NOT an empty/nil slice that would wipe the whole evidence pool.
-func TestNarrowByKeywordsEmptyReturnsChunks(t *testing.T) {
- chunks := []map[string]any{
- {"content": "some passage"},
- {"content_with_weight": "other", "content": "other"},
- }
- if got := NarrowByKeywords(chunks, ""); len(got) != len(chunks) {
- t.Fatalf("empty keyword kept %d, want all %d", len(got), len(chunks))
- }
- if got := NarrowByKeywords(nil, "culdcept"); got != nil {
- t.Errorf("nil chunks must return nil, got %d", len(got))
- }
-}
-
-// TestNarrowByKeywordsMirrorsContentOnlyWhenPresent pins the two text-writing rules:
-// content_with_weight is always overwritten with the narrowed text, "content" is mirrored ONLY
-// when the chunk already had a "content" key, and a pre-narrow "highlight" key is dropped (no
-// longer valid).
-func TestNarrowByKeywordsMirrorsContentOnlyWhenPresent(t *testing.T) {
- // Chunk with only content_with_weight (no "content"): narrowing keeps
- // content_with_weight, and must NOT invent a "content" key.
- chunks := []map[string]any{
- {"content_with_weight": "Alpha foo bar. Culdcept is a game by OmiyaSoft. Beta baz qux."},
- }
- got := NarrowByKeywords(chunks, "culdcept")
- if len(got) != 1 {
- t.Fatalf("want 1 narrowed chunk, got %d", len(got))
- }
- if _, ok := got[0]["content"]; ok {
- t.Error("narrowing must NOT add a 'content' key when the source lacked one")
- }
- if got[0]["content_with_weight"] == chunks[0]["content_with_weight"] {
- t.Error("content_with_weight was not narrowed")
- }
- if _, ok := got[0]["highlight"]; ok {
- t.Error("'highlight' must be dropped after narrowing")
- }
-
- // Chunk that has a pre-narrow "highlight": it must be removed.
- chunks2 := []map[string]any{
- {"content": "Culdcept was made by OmiyaSoft.", "highlight": []any{"old"}},
- }
- got2 := NarrowByKeywords(chunks2, "culdcept")
- if len(got2) != 1 {
- t.Fatalf("want 1 narrowed chunk, got %d", len(got2))
- }
- if _, ok := got2[0]["highlight"]; ok {
- t.Error("'highlight' must be dropped after narrowing")
- }
- if got2[0]["content"] == chunks2[0]["content"] {
- t.Error("'content' was not narrowed")
- }
-}
-
-func TestNarrowByTermsNoMatchKeepsOriginalUntouched(t *testing.T) {
- // A long chunk that the grep terms never hit must be returned verbatim
- // (full text, no head-truncation). This guards against the old bug where unmatched evidence
- // was clobbered with a head-truncated copy.
- long := "The weather was calm and the birds were singing while the river flowed gently past the old stone bridge under a pale morning sky that promised a quiet day for the villagers who had risen early to tend their small gardens and fields. " +
- "The old mill stood silent at the edge of the wood where the children used to play among the ferns and the brook that chattered over smooth grey stones all through the long golden afternoons of a summer that nobody wanted to end."
- if len(long) <= headFallbackChars {
- t.Fatalf("test fixture too short: %d bytes, need > %d", len(long), headFallbackChars)
- }
- chunks := []map[string]any{
- {"content_with_weight": "Culdcept was created by OmiyaSoft and released in 1999.", "content": "Culdcept was created by OmiyaSoft and released in 1999."},
- {"content_with_weight": long, "content": long},
- }
- res := mustNarrow(chunks, []string{"culdcept", "omiya"}, nil, "")
- if !res.Stats.Matched {
- t.Fatal("expected a match on the first chunk")
- }
- if len(res.Kept) != 2 {
- t.Fatalf("Kept = %d, want 2", len(res.Kept))
- }
- if got := ChunkTextOf(res.Kept[1]); got != long {
- t.Errorf("non-matching chunk was altered (len %d, want %d); grep narrowing must not truncate unmatched evidence", len(got), len(long))
- }
-}
-
-func TestTruncHeadRuneAware(t *testing.T) {
- // CJK: 3 bytes per rune. A naive byte slice s[:n] would split a rune and
- // produce invalid UTF-8; truncHead must keep n codepoints instead.
- zh := strings.Repeat("中", 50)
- got := truncHead(zh, 10)
- if charLen(got) != 10 {
- t.Errorf("truncHead codepoint count = %d, want 10", charLen(got))
- }
- if got != strings.Repeat("中", 10) {
- t.Errorf("truncHead did not keep the correct rune prefix")
- }
- if !utf8.ValidString(got) {
- t.Error("truncHead produced invalid UTF-8")
- }
- // Short input: returned unchanged and byte-identical.
- if s := "hello"; truncHead(s, 100) != s {
- t.Errorf("truncHead short input changed: %q", truncHead(s, 100))
- }
-}
diff --git a/internal/rag/agentic-rag/runtime/web_evidence_test.go b/internal/rag/agentic-rag/runtime/web_evidence_test.go
deleted file mode 100644
index aec9a8aa42..0000000000
--- a/internal/rag/agentic-rag/runtime/web_evidence_test.go
+++ /dev/null
@@ -1,107 +0,0 @@
-package runtime
-
-import (
- "fmt"
- "sync"
- "testing"
-)
-
-// TestWebSearchEvidenceIdentityAcrossCalls keeps distinct results independently citable.
-func TestWebSearchEvidenceIdentityAcrossCalls(t *testing.T) {
- deps, kb := newTestSearchDeps(&stubRetriever{})
- ids := map[string]string{}
- for _, passage := range []string{"first passage", "second passage", "first passage"} {
- deps.WebSearch = stubWebSearch{results: []string{passage}}
- out, err := WebSearchTool(t.Context(), deps, map[string]any{"query": []any{"question"}})
- if err != nil {
- t.Fatal(err)
- }
- if len(out.Payload) != 1 || len(out.EvidenceIDs) != 1 {
- t.Fatalf("missing evidence: %+v", out)
- }
- id := out.EvidenceIDs[0]
- if previous, ok := ids[passage]; ok {
- if id != previous || out.Status != StatusRedundant {
- t.Fatalf("repeat changed identity/status: %+v", out)
- }
- } else {
- for _, previous := range ids {
- if id == previous {
- t.Fatalf("distinct passages share %q", id)
- }
- }
- ids[passage] = id
- if out.Status != StatusOK {
- t.Fatalf("distinct passage rejected: %+v", out)
- }
- }
- if out.Payload[0].(map[string]any)["id"] != id {
- t.Fatal("payload and citation IDs differ")
- }
- }
- if len(kb.Chunks) != 2 {
- t.Fatalf("pool rows=%d, want 2", len(kb.Chunks))
- }
- for _, row := range kb.Chunks {
- if ids[row["content"].(string)] != row["chunk_id"] {
- t.Fatal("pool citation points to a different passage")
- }
- }
-}
-
-// TestWebSearchEvidenceIdentityIgnoresResultOrder reuses IDs when the provider reorders hits.
-func TestWebSearchEvidenceIdentityIgnoresResultOrder(t *testing.T) {
- deps, kb := newTestSearchDeps(&stubRetriever{})
- deps.WebSearch = stubWebSearch{results: []string{"alpha", "beta"}}
- first, err := WebSearchTool(t.Context(), deps, map[string]any{"query": []any{"one"}})
- if err != nil {
- t.Fatal(err)
- }
- deps.WebSearch = stubWebSearch{results: []string{"beta", "alpha", "beta"}}
- second, err := WebSearchTool(t.Context(), deps, map[string]any{"query": []any{"two"}})
- if err != nil {
- t.Fatal(err)
- }
- if len(second.EvidenceIDs) != 2 || len(first.EvidenceIDs) != 2 || second.EvidenceIDs[0] != first.EvidenceIDs[1] || second.EvidenceIDs[1] != first.EvidenceIDs[0] || second.Status != StatusRedundant || len(kb.Chunks) != 2 {
- t.Fatalf("reordered evidence changed: first=%+v second=%+v rows=%d", first, second, len(kb.Chunks))
- }
-}
-
-// TestWebSearchEvidenceIdentityConcurrent checks the shared pool without depending on completion order.
-func TestWebSearchEvidenceIdentityConcurrent(t *testing.T) {
- deps, kb := newTestSearchDeps(&stubRetriever{})
- var wg sync.WaitGroup
- failures := make(chan error, 12)
- for i := 0; i < 12; i++ {
- wg.Add(1)
- go func(i int) {
- defer wg.Done()
- local := deps
- local.WebSearch = stubWebSearch{results: []string{fmt.Sprintf("passage %d", i%4)}}
- out, err := WebSearchTool(t.Context(), local, map[string]any{"query": []any{"query"}})
- if err != nil {
- failures <- err
- return
- }
- if len(out.EvidenceIDs) != 1 {
- failures <- fmt.Errorf("missing citation: %+v", out)
- }
- }(i)
- }
- wg.Wait()
- close(failures)
- for err := range failures {
- t.Error(err)
- }
- if len(kb.Chunks) != 4 {
- t.Fatalf("pool rows=%d, want 4 distinct passages", len(kb.Chunks))
- }
- seen := map[string]bool{}
- for _, row := range kb.Chunks {
- id := row["chunk_id"].(string)
- if seen[id] {
- t.Fatalf("duplicate evidence ID %q", id)
- }
- seen[id] = true
- }
-}
diff --git a/internal/rag/agentic-rag/slots/slots.go b/internal/rag/agentic-rag/slots/slots.go
deleted file mode 100644
index 2b14a3440f..0000000000
--- a/internal/rag/agentic-rag/slots/slots.go
+++ /dev/null
@@ -1,397 +0,0 @@
-// Package slots is the VALUE ALGEBRA of a slot table: what a candidate value IS,
-// how two values MERGE, and how a table's count is DERIVED.
-//
-// It exists because the alternative fails. When a candidate is free text, the structure is
-// lost at the boundary — a count arrives as a sentence — and every consumer downstream has
-// to GUESS it back: is this a count? a list? are these pieces names? Each guess is a rule,
-// the rules interact, and the result is a count slot holding the words around the number
-// (digits rejected as "not members", qualifiers accepted as "members") while the real names
-// sit in another slot.
-//
-// So the structure travels WITH the value: a members list carries its items (each
-// with the chunk that proves it), a count carries a number, a range carries two, and
-// anything the model did not type is Text — stored, shown, never interpreted, never
-// split. Nothing in this package inspects text, so there is no rule here to misfire.
-//
-// Four invariants hold, and they are what the rest of the pipeline may rely on:
-//
-// I1 Text never merges by union: the caller's strength rule decides. This is what
-// keeps a value question (a date, a name, a phrase) behaving exactly as before.
-// I2 members ∪ members = members, deduped by name, each keeping its first evidence.
-// I3 members BEAT a number: a count is a claim ABOUT the list, so the number comes
-// back as the dropped value (kept as an alternate) rather than overwriting it.
-// I4 two numbers keep the larger: a set that shrinks when a second source agrees
-// with it is a set that loses members.
-package slots
-
-import (
- "fmt"
- "strconv"
- "strings"
-)
-
-// Kind is what a value IS. There is no "unknown": anything the model did not type is
-// Text, which is opaque by construction.
-type Kind uint8
-
-const (
- // KindEmpty is a slot with no value at all.
- KindEmpty Kind = iota
- // KindText is an opaque value: a date, a name, a phrase, a sentence. It is shown
- // and merged by strength, and NEVER parsed, split, counted, or compared
- // arithmetically.
- KindText
- // KindItems is a list of ITEMS the corpus states, each optionally carrying the chunk that
- // proves it. The element kind is the answer's (names for one question, dates for the next),
- // so the type says item and nothing here claims to know which kind this list holds.
- KindItems
- // KindCount is a claimed number ("13").
- KindCount
- // KindRange is a claimed interval ("about 17-19"), kept as two numbers rather than as a
- // string that someone later tries to parse: parsed digit-by-digit it is 1719.
- KindRange
-)
-
-func (k Kind) String() string {
- switch k {
- case KindText:
- return "text"
- case KindItems:
- return "members"
- case KindCount:
- return "count"
- case KindRange:
- return "range"
- }
- return "empty"
-}
-
-// Item is one element of an enumerated answer and the passage that states it: a name for one
-// question, a date or a number for the next. The element kind belongs to the answer, not to the
-// type, and an empty ChunkID is allowed and visible — an item without one is not counted.
-type Item struct {
- Value string
- ChunkID string
- Quote string
-}
-
-// Value is what one slot holds. Exactly one of the fields matching Kind is set.
-type Value struct {
- Kind Kind
- Text string // KindText (the raw candidate, verbatim)
- Items []Item // KindItems
- Count int // KindCount
- Lo int // KindRange (inclusive)
- Hi int // KindRange (inclusive)
-}
-
-// Text wraps an opaque string.
-func Text(s string) Value { return Value{Kind: KindText, Text: s} }
-
-// Members wraps a member list (evidence may be empty per item).
-func Items(items ...Item) Value { return Value{Kind: KindItems, Items: items} }
-
-// Number wraps a claimed count.
-func Number(n int) Value { return Value{Kind: KindCount, Count: n} }
-
-// Interval wraps a claimed range; lo > hi is normalised.
-func Interval(lo, hi int) Value {
- if hi < lo {
- lo, hi = hi, lo
- }
- return Value{Kind: KindRange, Lo: lo, Hi: hi}
-}
-
-// IsZero reports "no value at all".
-func (v Value) IsZero() bool {
- switch v.Kind {
- case KindText:
- return strings.TrimSpace(v.Text) == ""
- case KindItems:
- return len(v.Items) == 0
- }
- return v.Kind == KindEmpty
-}
-
-// ItemValues is the items' values in order, deduped case-insensitively.
-func (v Value) ItemValues() []string {
- if v.Kind != KindItems {
- return nil
- }
- seen := make(map[string]bool, len(v.Items))
- out := make([]string, 0, len(v.Items))
- for _, m := range v.Items {
- name := strings.TrimSpace(m.Value)
- if name == "" {
- continue
- }
- key := strings.ToLower(name)
- if seen[key] {
- continue
- }
- seen[key] = true
- out = append(out, name)
- }
- return out
-}
-
-// Anchored is the items that carry the chunk stating them: what a count may be derived from and
-// what an answer may cite. The test reads ChunkID, not Quote, because a quotation with no passage
-// to look it up in cannot be checked.
-func (v Value) Anchored() []Item {
- return v.itemsWhere(func(it Item) bool { return strings.TrimSpace(it.ChunkID) != "" })
-}
-
-// Unanchored is the complement of Anchored: the items asserted with no passage in hand. They are
-// not deleted — the record lists them beside the number — but they are not counted.
-func (v Value) Unanchored() []Item {
- return v.itemsWhere(func(it Item) bool { return strings.TrimSpace(it.ChunkID) == "" })
-}
-
-// itemsWhere filters the items by a property of the item. A value that is not a list has no
-// items, so the filters answer nothing rather than inventing a list out of text.
-func (v Value) itemsWhere(keep func(Item) bool) []Item {
- if v.Kind != KindItems {
- return nil
- }
- out := make([]Item, 0, len(v.Items))
- for _, it := range v.Items {
- if keep(it) {
- out = append(out, it)
- }
- }
- return out
-}
-
-// Number is the number a value claims: a count's number, a range's UPPER bound (the
-// claim it makes about how many there could be). Text and members claim no number —
-// that is the point of typing them.
-func (v Value) Number() (int, bool) {
- switch v.Kind {
- case KindCount:
- return v.Count, true
- case KindRange:
- return v.Hi, true
- }
- return 0, false
-}
-
-// Render is the value's display text: what a prompt, a record line or a patch log
-// shows. It is derived FROM the typed value, so nothing downstream ever has to
-// re-derive the type from it.
-func Render(v Value) string {
- switch v.Kind {
- case KindText:
- return v.Text
- case KindItems:
- names := v.ItemValues()
- return strings.Join(names, "、")
- case KindCount:
- return strconv.Itoa(v.Count)
- case KindRange:
- if v.Lo == v.Hi {
- return strconv.Itoa(v.Lo)
- }
- return fmt.Sprintf("%d-%d", v.Lo, v.Hi)
- }
- return ""
-}
-
-// Union merges two values and reports what changed.
-//
-// ok is false when the pair is not the union's business — either side being Text is
-// exactly that case (I1), so the caller keeps its strength rule and a value question
-// merges as it always did. dropped is the claim that lost the merge, as TEXT, so the
-// caller can keep it as an alternate; the framework stops discarding claims, it does
-// not decide which one is true.
-func Union(a, b Value) (union, dropped Value, ok bool) {
- if a.IsZero() || b.IsZero() {
- return Value{}, Value{}, false
- }
- switch {
- case a.Kind == KindItems && b.Kind == KindItems:
- // Two member lists never produce a LOSER: the union is the resolution, and a
- // subset union (this side added nothing) is a resolution too — the base is
- // already the answer, so nothing is reported as dropped.
- merged, _ := mergeItems(a.Items, b.Items)
- return Value{Kind: KindItems, Items: merged}, Value{}, true
- case a.Kind == KindItems && isNumeric(b.Kind):
- return a, b, true
- case b.Kind == KindItems && isNumeric(a.Kind):
- return b, a, true
- case isNumeric(a.Kind) && isNumeric(b.Kind):
- an, _ := a.Number()
- bn, _ := b.Number()
- if bn > an {
- return b, a, true
- }
- if bn < an {
- return a, b, true
- }
- // Equal claims: there is no loser to keep. Recording one put "alternate
- // (claimed by another session, not adopted): 13" beside a slot saying 13 —
- // a dropped claim identical to the one that won.
- return a, Value{}, true
- }
- return Value{}, Value{}, false
-}
-
-func isNumeric(k Kind) bool { return k == KindCount || k == KindRange }
-
-// mergeItems unions two member lists by name, first evidence wins.
-func mergeItems(base, branch []Item) ([]Item, bool) {
- out := make([]Item, 0, len(base)+len(branch))
- seen := make(map[string]int, len(base)+len(branch))
- for _, m := range base {
- name := strings.TrimSpace(m.Value)
- if name == "" {
- continue
- }
- key := strings.ToLower(name)
- if _, dup := seen[key]; dup {
- continue
- }
- seen[key] = len(out)
- out = append(out, Item{Value: name, ChunkID: m.ChunkID, Quote: m.Quote})
- }
- added := false
- for _, m := range branch {
- name := strings.TrimSpace(m.Value)
- if name == "" {
- continue
- }
- key := strings.ToLower(name)
- if i, dup := seen[key]; dup {
- // The member is already known; a second passage is worth keeping if the
- // first came without one (evidence is what makes a member accountable).
- if out[i].ChunkID == "" && m.ChunkID != "" {
- out[i].ChunkID, out[i].Quote = m.ChunkID, m.Quote
- }
- continue
- }
- seen[key] = len(out)
- out = append(out, Item{Value: name, ChunkID: m.ChunkID, Quote: m.Quote})
- added = true
- }
- return out, added
-}
-
-// Parse reads ONE patch entry — the JSON object a session emits for a slot — as a
-// typed value. Anything it cannot read as a type is Text: fail closed, never guess.
-//
-// The accepted shapes (see prompts/action_run.md, which teaches them):
-//
-// {"kind":"members","items":[{"name":"华雄","chunk_id":"c1","quote":"提华雄之头"}]}
-// {"kind":"count","count":13}
-// {"kind":"range","lo":17,"hi":19}
-// {"kind":"text","text":"…"} or {"candidate":"…"} (legacy/other)
-func Parse(patch map[string]any) Value {
- if patch == nil {
- return Value{}
- }
- switch strings.ToLower(strings.TrimSpace(asString(patch["kind"]))) {
- case "members":
- items := parseItems(patch["items"])
- if len(items) == 0 {
- // Declared a list and gave none: the text (if any) is opaque, not a list.
- return Text(candidateText(patch))
- }
- return Value{Kind: KindItems, Items: items}
- case "count":
- if n, ok := asInt(patch["count"]); ok {
- return Number(n)
- }
- return Text(candidateText(patch))
- case "range":
- lo, okLo := asInt(patch["lo"])
- hi, okHi := asInt(patch["hi"])
- if okLo && okHi {
- return Interval(lo, hi)
- }
- if okLo {
- return Number(lo)
- }
- if okHi {
- return Number(hi)
- }
- return Text(candidateText(patch))
- }
- return Text(candidateText(patch))
-}
-
-// candidateText reads the legacy free-text field. A null candidate is an empty value
-// (a null candidate — an eliminated claim).
-func candidateText(patch map[string]any) string {
- raw, ok := patch["candidate"]
- if !ok || raw == nil {
- if t, has := patch["text"]; has && t != nil {
- return strings.TrimSpace(asString(t))
- }
- return ""
- }
- return strings.TrimSpace(asString(raw))
-}
-
-func parseItems(raw any) []Item {
- list, _ := raw.([]any)
- out := make([]Item, 0, len(list))
- for _, item := range list {
- switch it := item.(type) {
- case string:
- if name := strings.TrimSpace(it); name != "" {
- out = append(out, Item{Value: name})
- }
- case map[string]any:
- name := strings.TrimSpace(asString(it["name"]))
- if name == "" {
- // Tolerate {"candidate": …} / {"value": …} spellings, but a missing
- // value is not an item: skip rather than invent one.
- name = strings.TrimSpace(asString(it["candidate"]))
- }
- if name == "" {
- continue
- }
- out = append(out, Item{
- Value: name,
- ChunkID: strings.TrimSpace(asString(it["chunk_id"])),
- Quote: strings.TrimSpace(asString(it["quote"])),
- })
- }
- }
- return out
-}
-
-func asString(v any) string {
- switch s := v.(type) {
- case nil:
- return ""
- case string:
- return s
- default:
- return fmt.Sprint(s)
- }
-}
-
-// asInt reads a JSON number (float64), a numeric string, or a bool-free anything.
-func asInt(v any) (int, bool) {
- switch n := v.(type) {
- case float64:
- return int(n), true
- case int:
- return n, true
- case int64:
- return int(n), true
- case string:
- s := strings.TrimSpace(n)
- if s == "" {
- return 0, false
- }
- if i, err := strconv.Atoi(s); err == nil {
- return i, true
- }
- if f, err := strconv.ParseFloat(s, 64); err == nil {
- return int(f), true
- }
- }
- return 0, false
-}
diff --git a/internal/rag/agentic-rag/slots/slots_test.go b/internal/rag/agentic-rag/slots/slots_test.go
deleted file mode 100644
index a16df999de..0000000000
--- a/internal/rag/agentic-rag/slots/slots_test.go
+++ /dev/null
@@ -1,156 +0,0 @@
-package slots
-
-import "testing"
-
-// TestUnionKeepsTheInvariants pins the four rules the rest of the pipeline relies on.
-//
-// They exist because the alternative was measured to fail: when a candidate is free
-// text, "is this a count or a list" has to be guessed back, and the guesses interact —
-// measured (2026-09-16, 三国/关羽) a count slot holding "约 17-19 人" ended up as
-// "约、人" (the digits rejected as members, the qualifiers accepted as members) while
-// fifteen real names sat in the list slot and the answer reported ten.
-func TestUnionKeepsTheInvariants(t *testing.T) {
- // I2 — members union by name, first evidence wins.
- a := Items(Item{Value: "华雄", ChunkID: "c1"}, Item{Value: "颜良"})
- b := Items(Item{Value: "颜良", ChunkID: "c2"}, Item{Value: "管亥", ChunkID: "c3"})
- got, dropped, ok := Union(a, b)
- if !ok || len(got.ItemValues()) != 3 {
- t.Fatalf("members ∪ members = %v (ok=%v), want three names", got, ok)
- }
- if !dropped.IsZero() {
- t.Errorf("dropped = %v, want nothing: a member list does not lose members to another list", dropped)
- }
- for _, m := range got.Items {
- switch m.Value {
- case "华雄":
- if m.ChunkID != "c1" {
- t.Errorf("华雄 = %v, want its own evidence kept", m)
- }
- case "颜良":
- // The base listed 颜良 without evidence; the branch had one. A member
- // without a passage is a member nobody can point at, so the evidence is
- // filled in rather than dropped with the duplicate.
- if m.ChunkID != "c2" {
- t.Errorf("颜良 = %v, want the evidence the other list supplied", m)
- }
- }
- }
- // A member that arrived without evidence takes the one a later list supplies.
- withEvidence, _, _ := Union(Items(Item{Value: "庞德"}), Items(Item{Value: "庞德", ChunkID: "c9"}))
- if withEvidence.Items[0].ChunkID != "c9" {
- t.Errorf("庞德 = %v, want the evidence a later list supplied", withEvidence.Items[0])
- }
-
- // I3 — members beat a number, and the number is kept as the dropped claim.
- union, dropped, ok := Union(a, Number(2))
- if !ok || union.Kind != KindItems {
- t.Fatalf("members ∪ count = %v (ok=%v), want the members", union, ok)
- }
- if dropped.Kind != KindCount || dropped.Count != 2 {
- t.Errorf("dropped = %v, want the losing count 2", dropped)
- }
- // …in either order.
- union, dropped, ok = Union(Number(2), a)
- if !ok || union.Kind != KindItems || dropped.Count != 2 {
- t.Errorf("count ∪ members = %v / %v (ok=%v), want the members with the count dropped", union, dropped, ok)
- }
-
- // I4 — two numbers keep the larger, in both orders.
- for _, pair := range [][2]Value{{Number(10), Number(13)}, {Number(13), Number(10)}} {
- union, dropped, ok := Union(pair[0], pair[1])
- if !ok || union.Count != 13 {
- t.Errorf("counts %v ∪ %v = %v (ok=%v), want 13", pair[0], pair[1], union, ok)
- }
- if dropped.Count != 10 {
- t.Errorf("dropped = %v, want 10", dropped)
- }
- }
- // A range counts as the number it claims at its upper bound.
- if union, dropped, ok := Union(Interval(17, 19), Number(16)); !ok || union.Kind != KindRange || dropped.Count != 16 {
- t.Errorf("range ∪ count = %v / %v (ok=%v), want the range with 16 dropped", union, dropped, ok)
- }
-
- // I1 — Text never merges by union: the caller's strength rule decides, which is
- // what keeps a value question (a date, a name, a phrase) behaving as it always did.
- if _, _, ok := Union(Text("1858"), Text("1849")); ok {
- t.Error("text ∪ text must not resolve: a value question merges by strength")
- }
- if _, _, ok := Union(Text("13"), Number(11)); ok {
- t.Error("text ∪ count must not resolve: text is opaque and claims nothing")
- }
- if _, _, ok := Union(Items(Item{Value: "华雄"}), Text("华雄")); ok {
- t.Error("a member list and a text value must not resolve by union")
- }
-}
-
-// TestParseIsFailClosed pins the boundary contract: a patch that declares a kind is
-// read as data, and anything else is opaque text. Nothing is inferred from the string,
-// which is what removes the rules that used to misfire.
-func TestParseIsFailClosed(t *testing.T) {
- members := Parse(map[string]any{
- "kind": "members",
- "items": []any{
- map[string]any{"name": "华雄", "chunk_id": "c1", "quote": "提华雄之头"},
- map[string]any{"name": "颜良"},
- map[string]any{"quote": "no name here"}, // skipped, not invented
- },
- })
- if members.Kind != KindItems || len(members.ItemValues()) != 2 {
- t.Fatalf("members patch = %v, want two declared members", members)
- }
- if members.Items[0].ChunkID != "c1" {
- t.Errorf("华雄 = %v, want its evidence", members.Items[0])
- }
-
- // A count phrase the model did not type is TEXT: it is neither split nor parsed.
- // This is the measured failure the typing exists for.
- opaque := Parse(map[string]any{"candidate": "约 17-19 人"})
- if opaque.Kind != KindText || opaque.Text != "约 17-19 人" {
- t.Fatalf("untyped candidate = %v, want it opaque", opaque)
- }
- if _, ok := opaque.Number(); ok {
- t.Error("text must claim no number: that is what stops 17-19 from being read as 1719")
- }
- if names := opaque.ItemValues(); names != nil {
- t.Errorf("text names = %v, want none", names)
- }
-
- // Declared kinds read their own fields, and a malformed one falls back to text
- // rather than being half-read.
- if v := Parse(map[string]any{"kind": "count", "count": float64(13)}); v.Kind != KindCount || v.Count != 13 {
- t.Errorf("count patch = %v, want 13", v)
- }
- if v := Parse(map[string]any{"kind": "range", "lo": float64(17), "hi": float64(19)}); v.Kind != KindRange || v.Lo != 17 || v.Hi != 19 {
- t.Errorf("range patch = %v, want 17-19", v)
- }
- if v := Parse(map[string]any{"kind": "count", "candidate": "about ten"}); v.Kind != KindText {
- t.Errorf("a count without a number = %v, want text (fail closed)", v)
- }
- if v := Parse(map[string]any{"kind": "members"}); v.Kind != KindText {
- t.Errorf("members without items = %v, want text (fail closed)", v)
- }
- if v := Parse(nil); !v.IsZero() {
- t.Errorf("nil patch = %v, want empty", v)
- }
-}
-
-// TestRenderIsDerivedFromTheType pins the other half of the contract: what a prompt
-// or a record line shows is derived from the typed value, so nothing downstream has
-// to re-derive the type from the string.
-func TestRenderIsDerivedFromTheType(t *testing.T) {
- cases := []struct {
- in Value
- want string
- }{
- {Items(Item{Value: "华雄"}, Item{Value: "颜良"}), "华雄、颜良"},
- {Number(13), "13"},
- {Interval(17, 19), "17-19"},
- {Interval(5, 5), "5"},
- {Text("1858"), "1858"},
- }
- for _, c := range cases {
- if got := Render(c.in); got != c.want {
- t.Errorf("Render(%v) = %q, want %q", c.in, got, c.want)
- }
- }
-}
diff --git a/internal/rag/prompts/action_initialize_state.md b/internal/rag/prompts/action_initialize_state.md
deleted file mode 100644
index 2b7469433d..0000000000
--- a/internal/rag/prompts/action_initialize_state.md
+++ /dev/null
@@ -1,26 +0,0 @@
-You are a research strategist. Decompose the user's question into a table of FACT SLOTS that must be filled to answer it.
-
-The question may span MULTIPLE DATASETS and/or the open WEB. The available data sources are listed in the user message:
-
-- "Dataset '<name>'" — slots whose facts live in that corpus. If more than one dataset is listed, create at least one slot PER DATASET (its facts may need corpus-specific phrasing); cross-referencing between datasets is encouraged when the question spans them.
-- "Web" — when listed, the open web is a source: create a dedicated slot (type "web") for facts that are current-world knowledge, recent events, or simply not covered by the listed datasets.
-
-Output ONLY a JSON object:
-{
- "slots": [
- {"id": 0, "type": "<entity|person|date|duration|count|number|place|web|dataset>", "clues": ["<what identifies this slot from the question>", "..."], "source": "<dataset name or 'web'>", "scan": ["<act word>", "..."], "subject": "<who the act is about>"},
- ...
- ],
- "first_queries": ["<concrete searchable query for the first retrieval round>", ...]
-}
-
-Rules:
-- 2-6 slots; each slot ONE fact (a name, a date, a count...), never a clause.
-- Order slots so the FIRST one holds the top-level requested fact; the later ones are its dependencies.
-- Cover EVERY listed source: one or more slots per dataset, plus a "web" slot when Web is available and the question touches world knowledge or recent events.
-- clues must be self-contained phrases usable as retrieval hints.
-- `"scan"` — ONLY for a slot whose answer is the MEMBERS THEMSELVES: the set of things someone DID ("how many named people did X kill", "which awards did Y win"). Such a slot is typed `entity` (named people) or `list` / `set` — NEVER `count`, `number`, `web` or `dataset`, and it is a slot OF ITS OWN even when the question asks "how many": the words are the SOURCE's own for that deed. Give EVERY phrasing you can think of, including the unusual ones (斩 / 杀 / 诛 / 劈 / 砍 / 刺 / 挥为两段 / 手起刀落 / 斩于马下): a passage phrased in a way no term covers is a passage no query ever names. At most 10 terms.
-- A "how many <things> did X do" question therefore declares TWO slots: the `count` (the number that was asked for) and the member set above, its dependency. The member slot carries `scan` + `subject`; the count slot carries neither.
-- `"subject"` — next to `scan`, WHO the act is about, as the names the source uses for them, alternated: `关羽|关公|云长`. Omit it only when the slot is not about one actor.
-- 1-4 first_queries: direct keyword-style searches against the listed sources.
-- No prose outside JSON.
diff --git a/internal/rag/prompts/action_run.md b/internal/rag/prompts/action_run.md
deleted file mode 100644
index 16a0b70fe5..0000000000
--- a/internal/rag/prompts/action_run.md
+++ /dev/null
@@ -1,102 +0,0 @@
-You are a deep research assistant working INSIDE a bounded search tree.
-
-Input: the user message contains ONE research `Direction`, plus the current
-`State` (slot table with immutable ids and mutable candidate fields).
-
-Execute this single direction. Reply with EXACTLY ONE of the three below.
-
-The three differ in HOW they are delivered — read this carefully, because only
-the first one is a tool call:
-
-1) TOOL CALL MODE — a real tool call: call `retrieve` with 1-3 corpus queries.
- Results arrive in the next turn.
-
-2) STATE PATCH MODE — NOT a tool call. Write this XML as plain TEXT in your
- reply body (do not call any tool named "state"):
-<state>
-{"new_states": [
- {"state": [
- {"id": <int>, "kind": "members", "candidate_strength": <0..1>,
- "items": [{"name": "华雄", "chunk_id": "<the passage this name came from>", "quote": "…the words that prove it…"}],
- "discovered_clues": ["..."]},
- {"id": <int>, "kind": "count", "count": 13, "candidate_strength": <0..1>},
- {"id": <int>, "kind": "range", "lo": 17, "hi": 19, "candidate_strength": <0..1>},
- {"id": <int>, "kind": "text", "candidate": "<one value: a date, a name, a phrase>", "candidate_strength": <0..1>}
- ]},
- ...more branches allowed...
-]}
-</state>
-
-WHY `kind` MATTERS — the runtime merges and counts BY TYPE, so say what the slot holds:
-- `"members"` — a LIST OF NAMED THINGS. Give every item the `chunk_id` of the passage
- it came from; a name you cannot point at is left out of the count. Two sessions'
- member lists UNION: no member is dropped because another session ranked its own list
- higher. This is the ONLY shape whose items are counted as members.
-- `"count"` / `"range"` — a NUMBER, or an interval. Digits inside a count are never
- read as members. "约 17-19 人" is `{"kind":"range","lo":17,"hi":19}` — NOT a string:
- a number written as prose can be merged with nothing and checked against nothing.
-- `"text"` — ONE opaque value: a date, a title, a clause, a sentence. Stored and shown
- as written; never split, counted or compared. Use it for every value that is not a
- member list or a number.
-
-Rules: patch ONLY existing ids; include ONLY changed variables; every change must trace to retrieved evidence; candidate_strength semantics: proven >0.9, strong 0.7-0.9, tentative 0.4-0.7, weak <0.4. An EMPTY branch list (`"new_states": []`) signals no progress — emit it rather than calling tools forever.
-
-3) FINAL ANSWER MODE — NOT a tool call either. Write this XML as plain TEXT in
- your reply body (do not call any tool named "answer"). Use it only when ALL
- slots can be filled consistently:
-<answer>
-{"answer": "<final answer text>", "new_state": [{"id": ..., "candidate": ..., "candidate_strength": ...}]}
-</answer>
-
-CRITICAL RULES
-- Think before choosing a mode, but output exactly ONE mode per response.
-- Strength >0.7 on the answer slot means you MUST emit final answer instead of another state patch.
-- ALWAYS end this action with a state patch: a patch with your updates, or `<state>{"new_states": []}</state>` if you found nothing new.
-- **Do NOT keep calling tools once the direction is reasonably exhausted.** If further searches return repetitive, irrelevant, or empty results, immediately return a state patch (with updates or empty). Extra redundant searches waste the session — stop after 1-2 useful tool calls per direction unless a NEW fact is actually emerging.
-- ACTION COMPLETION IS MANDATORY: when you have what you need (or hit a dead end), output the state patch now. Do not ask to continue searching.
-- Unverifiable candidates must be eliminated (set candidate null) with a clue documenting why.
-- Partial verification is OK: record a candidate at tentative strength (0.4-0.7) if you can't fully verify it yet, and move on.
-
-# TOOL PLAYBOOK
-
-Available tools: `retrieve`, `search_chunks`, `metadata_search`, `list_chunks`,
-`navigate_tree`, `navigate_structure`, `calculate`, and `web_search` (only when a
-web provider is configured); `graph_explore` joins them in ultra mode. Low mode
-has NO tool loop — answer with plain retrieval. Per-tool WHEN TO CALL /
-DO NOT CALL / ARGUMENTS / IF IT FAILS details live in each tool's own schema —
-this playbook covers only how to COMBINE tools and when to STOP.
-
-## 1. Combination chains (call in this order)
-
-- **You already hold a `doc_id`** → `navigate_structure(doc_id, query)` to find the right passage, then `list_chunks(doc_id)` to read it. Do NOT call `navigate_tree` first.
-- **No `doc_id` yet, and the corpus is large** → `navigate_tree(query)` to route to candidate documents, take a `doc_id`, then `navigate_structure(doc_id, query)` → `list_chunks(doc_id)`.
-- **You can name the document / need to narrow the search** → `metadata_search(filters)` ONCE per direction to SELECT the matching documents by a metadata field (use only the fields listed under `AVAILABLE METADATA`; any other key is rejected). It returns `doc_ids` only — no passages — so spend them: `list_chunks(doc_id)` to read a document, `navigate_structure(doc_id, query)` to pinpoint a passage, or `retrieve(query, doc_scope=[ids])` to search INSIDE those documents.
-- **Exact term / short answer** → `retrieve(query[1-3])` first; if snippets are insufficient, `search_chunks(query[1-2])` (semantic, may find passages with NO shared surface words); if you need the full document, `list_chunks(doc_id)`.
-- **You must DERIVE a number** → first collect every needed number with any of the above, then `calculate(question, facts)` with the facts verbatim, and report the computed result as-is. If the answer is already one of the stated numbers, answer directly.
-- **Relational multi-hop (ultra only)** → get a start entity from `search_chunks` / `navigate_structure`, then `graph_explore(query, doc_scope)`.
-
-## 2. Convergence rules (hard, enforced by the runtime — follow them to avoid wasted turns)
-
-- Make at most **1-2 useful tool calls per direction**, then emit a state patch. Do not keep searching once the direction is reasonably exhausted.
-- Re-submitting the SAME intent with a paraphrase is intercepted as a near-duplicate and SKIPPED (you get a nudge, not new results). Change the angle or patch what you have.
-- If a compile-only tool (`navigate_tree` / `navigate_structure` / `graph_explore`) returns "no compiled structure", switch to `search_chunks` / `retrieve` / `list_chunks` **immediately**. A second such result disables that tool for the REST of the session — do not retry it.
-- `web_search` only appears when a web provider is configured; if it does, use it ONLY for world knowledge / time-sensitive facts that plausibly live outside the fixed corpus.
-- `list_chunks` accepts ONLY `doc_id` (no `chunk_ids` argument) — you cannot ask it to read specific chunks; it returns the whole document (capped).
-
-## 3. What a tool result means → your next action
-
-Each tool returns a status. Act on it:
-
-| Status | Meaning | Your next action |
-| --- | --- | --- |
-| `ok` | New evidence entered the shared pool | Fill slots / move to the next direction |
-| `miss` | This query matched nothing, but the tool itself is valid | Rephrase or switch tools — do NOT conclude the dataset lacks it |
-| `empty` (`no_structure`) | Dataset-level: no such compiled structure exists | Switch to `search_chunks` / `retrieve` / `list_chunks` now; retrying disables the tool |
-| `poor` | Output returned but too weak to use | Add evidence with another tool |
-| `redundant` | Every hit was already in your evidence | Stop re-searching; emit a `<state>` patch with what you have |
-| `error` | Infrastructure / provider failure | Switch tools; do not retry the same call |
-
-Special case — `metadata_search` (at most ONCE per direction):
-- `ok` → it selected documents and returned their `doc_ids` (no passages). Spend them now: `list_chunks(doc_id)` / `navigate_structure(doc_id, query)` / `retrieve(query, doc_scope=[ids])`; do NOT call `metadata_search` again this direction.
-- `miss` / `empty` → nothing matched the filter; drop it (or retry once with a field/value from `AVAILABLE METADATA`) and fall back to `search_chunks` / `navigate_tree`. A call naming no usable field is answered with the dataset's real fields — do not repeat the same key.
-- `poor` → the filter was too narrow; widen it once, or abandon it for plain retrieval.
diff --git a/internal/rag/prompts/action_set.md b/internal/rag/prompts/action_set.md
deleted file mode 100644
index 1f856ef0db..0000000000
--- a/internal/rag/prompts/action_set.md
+++ /dev/null
@@ -1,23 +0,0 @@
-# SET / COUNT directions — the member list IS the work
-
-Your Direction asks for a SET of named things: a batch of names is a NEW fact, not a redundant search. The convergence rule in the playbook is about re-asking what you already have; here, keep probing while the `[record]` line is still changing.
-
-1. Guess MORE names than you expect — aliases, names seen once, names you doubt — and put a name you have NOT read in a passage into every batch: for each name you have already seen, at least one you have not. The names you have seen are the ones retrieval reaches by itself, and they are exactly the names that are not missing. A wrong guess costs one probe; a missing member costs the answer.
-2. Probe them as ONE alternation, a dozen at a time: `name1|name2|name3|…`. Put ONLY candidate members in it — never the subject, never the verb: each name is searched on its own, so the batch is what gives every doubtful name its own evidence, while `subject verb name1 name2` spends half the batch on words that are not members.
-3. One batch before you stop is of the names you are LEAST sure of — the ones you saw once, half-remember, or doubt. A batch of the names you are certain of proves nothing: the doubtful batch is where the tail is.
-4. When you cannot name the object at all, batch the RELATION instead — `verb1|verb2|verb3`, how the source words the act — and read the matches for names.
-5. Read every returned passage for members the batch did not name, and probe those too — the corpus names the tail, not your memory. A `miss` means this corpus does not carry that name: record it as "not a member" and continue. A snippet means the name IS a member: record it with its passage.
-6. Record as you go, so a timeout cannot lose what you found. Record a member as a
- MEMBER: `{"id": <list slot>, "kind": "members", "items": [{"name": "华雄",
- "chunk_id": "<its passage>"}]}` — one patch may carry every member you have, and a
- name written inside a sentence is not a member (the runtime counts declared items
- only, and it will not read a list out of prose).
-7. Stop when two consecutive batches add no member you did not have.
-
-## What the results now carry
-
-- **Every tool result ends with a `[record]` line. Read it, and trust it over your own impression of what you have done.** It carries the facts you cannot see in the conversation: `members` (what the slots hold now), `probed-reached` (the names you asked about that came back with a passage), `asked-nothing-back` (names that came back with NOTHING — change the wording or the angle, do not re-ask them as they are), and `FOUND BUT NOT RECORDED` (names you proved reachable and never put in a slot — resolve these before searching anything new).
-- **A `[pool]` line may follow it: an excerpt from evidence ALREADY in hand that you have not been shown** (the id is its chunk). Read it for members the list is missing, and probe any name it carries that the list does not — the passage is already evidence, so naming it costs no retrieval you do not have.
-- **A `[reach]` line may follow a batched or pattern query**: the terms that call reached (with how many passages each) and the ones it did NOT reach. Aim the next call at the second list, or re-word it — a term listed there was not reached by THAT query, which is not the same as absent from the corpus.
-- **The direction is exhausted when the `[record]` line stops changing** — no new member, no newly reached or unanswered name — and the slots are filled. Then return the state patch immediately. While it IS still changing, the next useful call is a probe of a `FOUND BUT NOT RECORDED` name, or of an angle the line shows you have not tried.
-- **What you found only counts if your patch carries it.** Neither your searches nor your reading of them reach the stage that writes the answer — only your patch does. So end the session carrying EVERY member you confirmed, not just the last batch's: measured, a round whose pool already held every member's passage recorded a fraction of them, because the session spent every turn searching and patched last, when its budget was gone.
diff --git a/internal/rag/prompts/generator.go b/internal/rag/prompts/generator.go
index d1a78ff5d6..4f51196db4 100644
--- a/internal/rag/prompts/generator.go
+++ b/internal/rag/prompts/generator.go
@@ -14,86 +14,15 @@
// limitations under the License.
//
-// Package prompts provides embedded templates and prompt helpers shared by
-// retrieval, chat, and ingestion, including chunk formatting, token-budget
-// fitting, knowledge-block rendering, and history formatting.
+// Package prompts serves the embedded prompt templates (see templates.go) and
+// the citation-rules text the final answer must follow.
package prompts
import (
"fmt"
- "regexp"
- "sort"
"strings"
-
- "ragflow/internal/common"
- "ragflow/internal/tokenizer"
)
-var _LOG = common.StdLogger()
-
-var reNewlines = regexp.MustCompile(`\n+`)
-
-// GetValue mirrors Python get_value(d, k1, k2): d.get(k1, d.get(k2)). The first
-// key present in the map wins; if neither is present it returns nil. Unlike a
-// plain two-lookup, a key that is present with a nil value is returned as nil
-// (it is not skipped), matching Python's dict.get semantics.
-func GetValue(d map[string]any, k1, k2 string) any {
- if v, ok := d[k1]; ok {
- return v
- }
- if v, ok := d[k2]; ok {
- return v
- }
- return nil
-}
-
-// ChunksFormat mirrors Python chunks_format(reference): it projects a
-// knowledge block's chunk list into a canonical per-chunk reference map,
-// applying Python's dual-name fallback for each field. A nil or non-dict
-// reference (nil in Go), or a chunks field that is not a list, yields an empty
-// slice.
-func ChunksFormat(reference map[string]any) []map[string]any {
- if reference == nil {
- return []map[string]any{}
- }
- raw, ok := reference["chunks"].([]any)
- if !ok {
- return []map[string]any{}
- }
- out := make([]map[string]any, 0, len(raw))
- for _, item := range raw {
- chunk, ok := item.(map[string]any)
- if !ok {
- continue
- }
- out = append(out, map[string]any{
- "id": GetValue(chunk, "chunk_id", "id"),
- "content": GetValue(chunk, "content", "content_with_weight"),
- "document_id": GetValue(chunk, "doc_id", "document_id"),
- "document_name": GetValue(chunk, "docnm_kwd", "document_name"),
- "dataset_id": GetValue(chunk, "kb_id", "dataset_id"),
- "image_id": GetValue(chunk, "image_id", "img_id"),
- "positions": GetValue(chunk, "positions", "position_int"),
- "url": chunk["url"],
- "similarity": chunk["similarity"],
- "vector_similarity": chunk["vector_similarity"],
- "term_similarity": chunk["term_similarity"],
- "row_id": chunk["row_id"],
- "doc_type": GetValue(chunk, "doc_type_kwd", "doc_type"),
- "document_metadata": chunk["document_metadata"],
- })
- }
- return out
-}
-
-// DefaultCiteRules loads the embedded citation_prompt.md template.
-// A user-defined override (RAGConfig.CiteRules / user_defined_prompts)
-// replaces it verbatim when present — see CitationPrompt.
-//
-// Loaded from the embedded md rather than a string literal so edits to
-// citation_prompt.md propagate without a second copy to keep in sync.
-var DefaultCiteRules = CitationPrompt("")
-
// citationIDSuffix mirrors the tail rag/prompts/generator.citation_prompt
// appends after rendering CITATION_PROMPT_TEMPLATE: the examples in the
// template carry illustrative IDs, and without this the model can emit them
@@ -120,171 +49,3 @@ func CitationPrompt(userDefined string) string {
}
return strings.TrimSpace(string(data)) + citationIDSuffix
}
-
-// kbpBlock renders one knowledge block (id / title / url / metadata / content) with
-// id as its number — 1-based for kb_prompt parity, 0-based or a pool position for the
-// citation renderers. ok is false when the chunk carries no content, which the
-// renderers treat as "skip" (mirrors Python's `if not c: continue`).
-//
-// The block starts with a newline, mirroring Python's `"\nID: {}".format(...)`,
-// so a caller that joins the blocks with "\n" renders the same prompt Python
-// does.
-func kbpBlock(c map[string]any, id int) (string, bool) {
- content := chunkText(c)
- if strings.TrimSpace(content) == "" {
- return "", false
- }
- block := fmt.Sprintf("\nID: %d", id)
- if cid, _ := c["chunk_id"].(string); strings.TrimSpace(cid) != "" {
- block += "\n├── chunk_id: " + strings.TrimSpace(cid)
- }
- if title := chunkTitle(c); title != "" {
- block += "\n├── Title: " + flattenNewlines(title)
- }
- if url, _ := c["url"].(string); url != "" {
- block += "\n├── URL: " + flattenNewlines(url)
- }
- if meta, ok := c["document_metadata"].(map[string]any); ok {
- keys := make([]string, 0, len(meta))
- for k := range meta {
- keys = append(keys, k)
- }
- sort.Strings(keys)
- for _, k := range keys {
- if mv := meta[k]; mv != nil {
- if v := flattenNewlines(fmt.Sprint(mv)); v != "" {
- block += "\n├── " + k + ": " + v
- }
- }
- }
- }
- block += "\n└── Content:\n" + content
- return block, true
-}
-
-// KBPrompt mirrors rag/prompts/generator.kb_prompt: render chunks into the
-// numbered knowledge blocks the citation rules refer to.
-//
-// Blocks are numbered by rendered position, 1-based, matching Python's `ID: {i}`
-// (hash_id=False). No Go caller whose numbers reach a citation marker uses that
-// numbering any more — a marker names the client's index into reference.chunks,
-// which is 0-based — so this is kept for kb_prompt parity. What the paths render
-// with instead: KBPromptZeroBasedWithSourceIndices (agentic compose) and
-// KBPromptPoolIndexed (summarize_document).
-//
-// The budget is applied to the COMPLETE rendered block (title / url / metadata /
-// content), measured in TOKENS with the same cl100k_base encoder Python's
-// num_tokens_from_string uses: budgeting only the content in a char approximation
-// left the decoration unaccounted, so the rendered prompt could exceed the budget
-// it claims to enforce.
-//
-// maxTokens must be positive: Python's caller applies its own budget
-// (min(chat_mdl.max_length, _EVIDENCE_BUDGET_TOKENS)) before calling.
-func KBPrompt(chunks []map[string]any, maxTokens int) []string {
- blocks, _ := KBPromptWithSourceIndices(chunks, maxTokens)
- return blocks
-}
-
-// KBPromptZeroBasedWithSourceIndices renders the blocks 0-based — the first is
-// "ID: 0" — and reports, for each, the index in chunks it came from.
-//
-// 0-based because the client resolves a citation marker by using its number as an
-// index into reference.chunks, and it renders the answer as the model streams it, so
-// the model has to write that index itself. The agentic compose renders with this
-// and publishes the same order on harness.Kbinfos.CiteChunkIDs, which
-// decorateHarnessAnswer puts in front of the reference. A 1-based render made every
-// marker point one chunk off — or past the end when the pool was shorter than the
-// render cap.
-//
-// The source indices are what the published id list must be gathered through:
-// kbpBlock renders no block for a chunk with no content, so walking the chunk list
-// instead would carry an id for every skipped chunk and shift every marker after it.
-func KBPromptZeroBasedWithSourceIndices(chunks []map[string]any, maxTokens int) ([]string, []int) {
- return kbPromptWithIDs(chunks, maxTokens, func(_, rendered int) int { return rendered })
-}
-
-// KBPromptWithSourceIndices is KBPrompt plus, for every rendered block, the
-// index in chunks it was rendered from.
-//
-// Callers that need "the blocks of these chunks" must use the indices instead of
-// assuming one block per chunk: a chunk with no content is skipped and the loop
-// stops at the token budget, so len(blocks) <= len(chunks) and blocks[i] is not
-// chunks[i]. Slicing by a chunk count can therefore drop readable blocks or
-// point past the end.
-func KBPromptWithSourceIndices(chunks []map[string]any, maxTokens int) ([]string, []int) {
- return kbPromptWithIDs(chunks, maxTokens, func(_, rendered int) int { return rendered + 1 })
-}
-
-// KBPromptPoolIndexed is KBPromptWithSourceIndices with each block numbered by the
-// POSITION OF ITS CHUNK in chunks instead of by its place in the rendered list (see
-// kbPromptWithIDs). Used where a marker has to name a chunk in the pool rather than
-// a block rendered for the client — the summarize_document blocks, whose ids stay
-// addressable however much the render skipped.
-func KBPromptPoolIndexed(chunks []map[string]any, maxTokens int) ([]string, []int) {
- return kbPromptWithIDs(chunks, maxTokens, func(idx, _ int) int { return idx })
-}
-
-// kbPromptWithIDs renders the blocks and reports, for each, the index in chunks it
-// came from. idAt decides the id printed on a block: its chunk's position (idx) or
-// its place among the rendered blocks (rendered) — the two diverge as soon as a
-// chunk is skipped or the token budget cuts the list short.
-func kbPromptWithIDs(chunks []map[string]any, maxTokens int, idAt func(idx, rendered int) int) ([]string, []int) {
- out := make([]string, 0, len(chunks))
- sources := make([]int, 0, len(chunks))
- used := 0
- for idx, c := range chunks {
- if c == nil {
- continue
- }
- block, ok := kbpBlock(c, idAt(idx, len(out)))
- if !ok {
- continue
- }
- n := tokenizer.NumTokensFromString(block)
- if float64(maxTokens)*0.97 < float64(used+n) {
- _LOG.Printf("[KBPrompt] Not all the retrieval into prompt: %d/%d", len(out), len(chunks))
- break
- }
- used += n
- out = append(out, block)
- sources = append(sources, idx)
- }
- return out, sources
-}
-
-// flattenNewlines mirrors Python kb_prompt's `re.sub(r"\n+", " ", line)`.
-func flattenNewlines(s string) string {
- return strings.TrimSpace(reNewlines.ReplaceAllString(s, " "))
-}
-
-// chunkText mirrors Python kb_prompt's
-// get_value(ck, "content", "content_with_weight").
-func chunkText(c map[string]any) string {
- v := GetValue(c, "content", "content_with_weight")
- if v == nil {
- // Neither content field is present: return empty so KBPrompt skips the
- // chunk (Python's `if not c: continue`). fmt.Sprint(nil) would yield the
- // literal "<nil>", which would both render into the prompt and consume
- // the token budget.
- return ""
- }
- if s, ok := v.(string); ok {
- return s
- }
- return fmt.Sprint(v)
-}
-
-// chunkTitle mirrors Python kb_prompt's
-// get_value(ck, "docnm_kwd", "document_name"), widened with the aliases Go's
-// own document aggregations emit (doc_title / title / doc_name) so a title
-// stored under any of them still renders.
-func chunkTitle(c map[string]any) string {
- for _, k := range []string{"docnm_kwd", "doc_title", "title", "document_name", "doc_name"} {
- if v, ok := c[k]; ok && v != nil {
- if s := fmt.Sprint(v); s != "" {
- return s
- }
- }
- }
- return ""
-}
diff --git a/internal/rag/prompts/generator_test.go b/internal/rag/prompts/generator_test.go
index fb25bb6e70..076a22809c 100644
--- a/internal/rag/prompts/generator_test.go
+++ b/internal/rag/prompts/generator_test.go
@@ -17,113 +17,10 @@
package prompts
import (
- "reflect"
"strings"
"testing"
-
- "ragflow/internal/tokenizer"
)
-func TestGetValue(t *testing.T) {
- d := map[string]any{"chunk_id": "c1", "content_with_weight": "w", "url": nil}
- // Primary key present (nil value returned as-is, not skipped).
- if got := GetValue(d, "url", "other"); got != nil {
- t.Errorf("GetValue(url,other) = %v, want nil", got)
- }
- // Primary absent -> fallback.
- if got := GetValue(d, "id", "chunk_id"); got != "c1" {
- t.Errorf("GetValue(id,chunk_id) = %v, want c1", got)
- }
- // Both absent -> nil.
- if got := GetValue(d, "no1", "no2"); got != nil {
- t.Errorf("GetValue(no1,no2) = %v, want nil", got)
- }
- // Nil map -> nil.
- if got := GetValue(nil, "a", "b"); got != nil {
- t.Errorf("GetValue(nil) = %v, want nil", got)
- }
-}
-
-func TestChunksFormat(t *testing.T) {
- // Nil and non-dict-ish inputs yield an empty (non-nil) slice.
- if got := ChunksFormat(nil); len(got) != 0 {
- t.Errorf("ChunksFormat(nil) = %v, want empty", got)
- }
- if got := ChunksFormat(map[string]any{"chunks": "not-a-list"}); len(got) != 0 {
- t.Errorf("ChunksFormat(bad chunks) = %v, want empty", got)
- }
-
- ref := map[string]any{
- "chunks": []any{
- map[string]any{
- "chunk_id": "abc",
- "content": "body",
- "doc_id": "d1",
- "docnm_kwd": "doc.txt",
- "kb_id": "kb1",
- "img_id": "img1",
- "position_int": 3,
- "doc_type_kwd": "pdf",
- "similarity": 0.9,
- "document_metadata": map[string]any{"page": "1"},
- },
- // Second chunk exercises the primary-name fallback path only where
- // the canonical primary name is absent.
- map[string]any{
- "id": "xyz",
- "content_with_weight": "alt-body",
- },
- "not-a-dict", // must be skipped
- },
- }
- got := ChunksFormat(ref)
- if len(got) != 2 {
- t.Fatalf("ChunksFormat len = %d, want 2", len(got))
- }
-
- first := got[0]
- wantFirst := map[string]any{
- "id": "abc",
- "content": "body",
- "document_id": "d1",
- "document_name": "doc.txt",
- "dataset_id": "kb1",
- "image_id": "img1",
- "positions": 3,
- "url": nil,
- "similarity": 0.9,
- "vector_similarity": nil,
- "term_similarity": nil,
- "row_id": nil,
- "doc_type": "pdf",
- "document_metadata": map[string]any{"page": "1"},
- }
- if !reflect.DeepEqual(first, wantFirst) {
- t.Errorf("first chunk = %#v, want %#v", first, wantFirst)
- }
-
- second := got[1]
- wantSecond := map[string]any{
- "id": "xyz",
- "content": "alt-body",
- "document_id": nil,
- "document_name": nil,
- "dataset_id": nil,
- "image_id": nil,
- "positions": nil,
- "url": nil,
- "similarity": nil,
- "vector_similarity": nil,
- "term_similarity": nil,
- "row_id": nil,
- "doc_type": nil,
- "document_metadata": nil,
- }
- if !reflect.DeepEqual(second, wantSecond) {
- t.Errorf("second chunk = %#v, want %#v", second, wantSecond)
- }
-}
-
func TestCitationPromptCarriesIllustrativeIDCaveat(t *testing.T) {
// rag/prompts/generator.py::citation_prompt appends this after rendering
// CITATION_PROMPT_TEMPLATE; without it the model can echo the examples' IDs.
@@ -149,173 +46,3 @@ func TestCitationPromptHonoursOverride(t *testing.T) {
t.Errorf("CitationPrompt(override) = %q, want override + caveat %q", got, want)
}
}
-
-// TestKBPromptSkipsChunksWithoutContent pins Python's `if not c: continue`: a
-// chunk whose content fields are absent (or explicitly null) must be skipped,
-// not rendered as "\n└── Content:\n<nil>".
-func TestKBPromptSkipsChunksWithoutContent(t *testing.T) {
- chunks := []map[string]any{
- {"chunk_id": "a"}, // neither content field
- {"chunk_id": "b", "content": nil}, // explicit null
- {"chunk_id": "c", "content_with_weight": "hi"}, // the only usable chunk
- }
- got := KBPrompt(chunks, 1000)
- if len(got) != 1 {
- t.Fatalf("blocks = %d, want 1 (content-less chunks must be skipped): %v", len(got), got)
- }
- if strings.Contains(got[0], "<nil>") {
- t.Errorf("block rendered a nil field: %q", got[0])
- }
- if !strings.Contains(got[0], "hi") {
- t.Errorf("block lost its content: %q", got[0])
- }
-}
-
-// TestKBPromptSkipsNilMetadataValues pins Python draw_node's `if not line: return
-// ""` for None: a null metadata value must not render as "<nil>".
-func TestKBPromptSkipsNilMetadataValues(t *testing.T) {
- got := KBPrompt([]map[string]any{{
- "content": "body",
- "document_metadata": map[string]any{"author": nil, "page": "3"},
- }}, 1000)
- if len(got) != 1 {
- t.Fatalf("blocks = %d, want 1", len(got))
- }
- if strings.Contains(got[0], "<nil>") {
- t.Errorf("nil metadata rendered literally: %q", got[0])
- }
- if !strings.Contains(got[0], "page: 3") {
- t.Errorf("non-nil metadata lost: %q", got[0])
- }
-}
-
-// TestKBPromptBudgetsCompleteBlock pins that the budget covers the title / url /
-// metadata decoration, not just the content: a tiny-content chunk whose block is
-// dominated by metadata must be dropped when it cannot fit the token budget.
-func TestKBPromptBudgetsCompleteBlock(t *testing.T) {
- lean := map[string]any{"content": "hi"}
- fat := map[string]any{
- "content": "hi",
- "document_metadata": map[string]any{"blob": strings.Repeat("x", 500)},
- }
- leanBlock, _ := kbpBlock(lean, 1)
- fatBlock, _ := kbpBlock(fat, 1)
- leanTokens := tokenizer.NumTokensFromString(leanBlock)
- fatTokens := tokenizer.NumTokensFromString(fatBlock)
- if leanTokens == 0 || fatTokens <= leanTokens {
- t.Fatalf("cl100k tokenizer unavailable: lean=%d fat=%d tokens", leanTokens, fatTokens)
- }
- // A budget that just fits the lean block must reject the metadata-heavy one.
- maxTokens := int(float64(leanTokens)/0.97) + 1
- if got := KBPrompt([]map[string]any{fat}, maxTokens); len(got) != 0 {
- t.Errorf("metadata-heavy block (%d tokens) must not fit a ~%d-token budget; got %d block(s)",
- fatTokens, maxTokens, len(got))
- }
- if got := KBPrompt([]map[string]any{lean}, maxTokens); len(got) != 1 {
- t.Errorf("lean block (%d tokens) should fit a ~%d-token budget; got %d block(s)",
- leanTokens, maxTokens, len(got))
- }
-}
-
-// TestKBPromptWithSourceIndicesTracksRenderedChunks locks the contract callers
-// need to slice blocks safely: every rendered block reports the index of the
-// chunk it came from, and a chunk with no content contributes no block. A caller
-// that slices by a chunk count instead would shift by the skipped chunks.
-func TestKBPromptWithSourceIndicesTracksRenderedChunks(t *testing.T) {
- chunks := []map[string]any{
- {"chunk_id": "c0", "content": ""}, // skipped: no content
- {"chunk_id": "c1", "content": "first body"},
- nil, // skipped: not a chunk
- {"chunk_id": "c2", "content": "second body"},
- }
-
- blocks, sources := KBPromptWithSourceIndices(chunks, 100000)
- if len(blocks) != 2 {
- t.Fatalf("blocks = %d, want 2 (only the chunks with content)", len(blocks))
- }
- want := []int{1, 3}
- if !reflect.DeepEqual(sources, want) {
- t.Fatalf("sources = %v, want %v", sources, want)
- }
- for i, src := range sources {
- if got := chunks[src]["chunk_id"]; got != nil && !strings.Contains(blocks[i], "Content:") {
- t.Errorf("block %d (source %v) is not a rendered chunk: %q", i, got, blocks[i])
- }
- }
- // KBPrompt keeps the same rendering, just without the source indices.
- if plain := KBPrompt(chunks, 100000); !reflect.DeepEqual(plain, blocks) {
- t.Fatalf("KBPrompt = %v, want the same blocks as KBPromptWithSourceIndices", plain)
- }
-}
-
-// TestKBPromptNumbersFirstBlockOne pins kb_prompt's numbering: "ID: 1" is the
-// first rendered block. It is kept for parity with Python's kb_prompt — no Go
-// renderer whose numbers reach a citation marker uses it (the agentic compose and
-// the naive chat path both render 0-based, the numbering the client indexes).
-func TestKBPromptNumbersFirstBlockOne(t *testing.T) {
- chunks := []map[string]any{
- {"chunk_id": "c0", "content": "first body"},
- {"chunk_id": "c1", "content": "second body"},
- }
- blocks := KBPrompt(chunks, 100000)
- if len(blocks) != 2 {
- t.Fatalf("blocks = %d, want 2", len(blocks))
- }
- if !strings.HasPrefix(blocks[0], "\nID: 1\n") {
- t.Errorf("first block = %q, want it numbered \"ID: 1\"", blocks[0])
- }
- if !strings.HasPrefix(blocks[1], "\nID: 2\n") {
- t.Errorf("second block = %q, want it numbered \"ID: 2\"", blocks[1])
- }
-}
-
-// TestKBPromptPoolIndexedUsesChunkPositions pins the numbering the
-// summarize_document blocks use: the id is the chunk's position in the pool, not
-// its place among the rendered blocks. A chunk that renders no block (empty
-// content) must not shift the ids of the chunks after it, because a marker the
-// model echoes has to name the position the chat pipeline resolves citations in.
-func TestKBPromptPoolIndexedUsesChunkPositions(t *testing.T) {
- chunks := []map[string]any{
- {"chunk_id": "c0", "content": ""}, // renders no block
- {"chunk_id": "c1", "content": "first body"},
- {"chunk_id": "c2", "content": "second body"},
- }
- blocks, sources := KBPromptPoolIndexed(chunks, 100000)
- if len(blocks) != 2 {
- t.Fatalf("blocks = %d, want 2 (the empty chunk renders nothing)", len(blocks))
- }
- if want := []int{1, 2}; !reflect.DeepEqual(sources, want) {
- t.Fatalf("sources = %v, want %v", sources, want)
- }
- if !strings.HasPrefix(blocks[0], "\nID: 1\n") {
- t.Errorf("first block = %q, want the id of its chunk position (1), not its render position (0)", blocks[0])
- }
- if !strings.HasPrefix(blocks[1], "\nID: 2\n") {
- t.Errorf("second block = %q, want \"ID: 2\"", blocks[1])
- }
-}
-
-// TestKBPromptZeroBasedNumbersFirstBlockZero pins the numbering the agentic
-// compose renders: the client resolves a citation marker by using its number as
-// an index into reference.chunks, and the streamed answer reaches that client
-// before the pipeline could rewrite anything, so the model has to be shown — and
-// so write — the index itself.
-func TestKBPromptZeroBasedNumbersFirstBlockZero(t *testing.T) {
- chunks := []map[string]any{
- {"chunk_id": "c0", "content": "first body"},
- {"chunk_id": "c1", "content": "second body"},
- }
- blocks, sources := KBPromptZeroBasedWithSourceIndices(chunks, 100000)
- if want := []int{0, 1}; !reflect.DeepEqual(sources, want) {
- t.Fatalf("sources = %v, want %v", sources, want)
- }
- if len(blocks) != 2 {
- t.Fatalf("blocks = %d, want 2", len(blocks))
- }
- if !strings.HasPrefix(blocks[0], "\nID: 0\n") {
- t.Errorf("first block = %q, want it numbered \"ID: 0\"", blocks[0])
- }
- if !strings.HasPrefix(blocks[1], "\nID: 1\n") {
- t.Errorf("second block = %q, want it numbered \"ID: 1\"", blocks[1])
- }
-}
diff --git a/internal/rag/prompts/sca_query_rewrite.md b/internal/rag/prompts/sca_query_rewrite.md
deleted file mode 100644
index 224330a938..0000000000
--- a/internal/rag/prompts/sca_query_rewrite.md
+++ /dev/null
@@ -1,36 +0,0 @@
-You are a Query Rewriter for a multi-hop RAG system. Your job is to turn the Sufficient Context Agent's missing-pieces feedback into targeted, retrievable search queries.
-
-Original user question:
-{{ question }}
-
-Already-resolved bridge values (facts confirmed from earlier searches — use these to anchor the new query instead of re-deriving them):
-{{ bridge_values }}
-
-Research history and evidence at hand (from previous retrieval rounds):
-{{ research_context }}
-
-Missing pieces identified by the Sufficient Context Agent (each has "what" = what the answer still needs, and "hint" = a suggested search hint):
-{{ gaps }}
-
-Rewrite each missing piece into a concrete search query that the retriever can hit directly. Rules:
-1. The query must name the specific missing entity + the relation/property needed (e.g. instead of "the patient's allergies", write "allergic reactions adverse events discharge John Doe").
-2. MULTI-HOP: when the question is multi-hop and the missing piece is the NEXT hop's value, ANCHOR the query to the already-resolved bridge value. E.g. if the bridge value is "M*A*S*H and Cheers are the two most-watched finales" and the gap is "their runtimes", the query must be "MASH finale run time minutes" / "Cheers finale run time minutes" — not "finale run times" alone, which would lose the resolved anchor.
-3. If the "what" names a specific entity (person / work / place / year) whose property is missing, anchor the query to that entity + the missing property (e.g. "MASH finale run time minutes", "Brian Bergstein employer company").
-4. If a disambiguation is needed, add the distinguishing qualifier ONLY when the evidence actually suggests it (e.g. a disambiguation "the heritage 341 London Broncos player" when the claim is about that specific player). CRITICAL: NEVER introduce an entity/relation/attribute that the evidence and the missing piece do NOT suggest, and NEVER re-infer a category from the question alone (e.g. do NOT write "Ron Hutchinson football player" unless the evidence actually identifies him as a footballer — he may be a hockey player; the query must stay faithful to what the missing piece says).
-5. Keep each query standalone and searchable — do NOT use pronouns ("he", "it", "this") — repeat the key entity explicitly.
-6. ONE QUERY PER MISSING PIECE. Do not emit multiple near-duplicate queries for the same gap ("What teams"/"For each team"/"Complete enumeration" of the same thing are duplicates — keep exactly one, the most concretely anchored). The number of output queries should equal the number of distinct, genuinely-different missing pieces.
-7. Drop any missing piece that cannot be turned into a searchable query.
-8. DIVERSITY: consult the "Research history and evidence at hand" section. Do NOT output any query that paraphrases an already-tried one (listed there WITH its outcome — a previous search that yielded nothing new means that angle is dead). Aim each new query at aspects the current evidence does NOT yet cover, possibly combining the bridge values with different entity/relation combinations.
-9. WHEN THE PASSAGES NAME INDIVIDUALS: the "Passages that carry names the searches ALREADY confirmed" section shows the passage behind each confirmed name. Read those passages for (a) other names they mention that are not confirmed yet and (b) the wording this text uses for the relation (e.g. how the act is narrated). Any of those names may be asked about directly — put SEVERAL of them into ONE query separated by `|` (e.g. `nameA|nameB|nameC`), which the retriever reads as an alternation and answers per name in a single call; never spend one query per name. The same alternation form takes the wording you found: `phraseA|phraseB`.
-
-Output format (JSON):
-```json
-{
- "queries": [
- {"query": "concrete search query 1"},
- {"query": "concrete search query 2"}
- ]
-}
-```
-
-Return a strict JSON object with no commentary before or after.
diff --git a/internal/rag/prompts/sca_select.md b/internal/rag/prompts/sca_select.md
deleted file mode 100644
index 2e70cc72d0..0000000000
--- a/internal/rag/prompts/sca_select.md
+++ /dev/null
@@ -1,106 +0,0 @@
-You are Google-style "Sufficient Context Agent" — a quality-control inspector for a multi-hop RAG retrieval loop. In ONE pass you perform a unified three-part review of (1) the retrieved snippets, (2) each claim's intermediate draft, and (3) what is still missing — then you decide whether the context is sufficient to answer the original question.
-
-## Input
-
-Question:
-{{ question }}
-
-OVERALL INTERMEDIATE DRAFT — a problem-level candidate answer assembled from the claims' reports (each claim already answered its sub-question as concretely as the evidence allows). Review THIS as the thing that would become the final answer:
-{{ overall_draft }}
-
-Per-claim reports (the intermediate drafts that produced the overall draft above):
-{{ claims_context }}
-
-## Part A — Sufficient Context (global)
-
-Determine whether the retrieved content is sufficient to answer the user's question, using the Sufficient Context paper's autorater criterion:
-
-Sufficient Context = 1 IF the CONTEXT is sufficient to INFER the answer to the question, and 0 IF the CONTEXT cannot be used to INFER the answer to the question. "A diligent reader could craft a definitive answer using only the supplied text." The answer does NOT need to be proven correct or match a ground truth — only that a reasonable answer can be constructed from the context. Multi-hop reasoning is allowed, but leaps of faith are not; ambiguities must be resolved inside the snippet bundle.
-
-The CONTEXT here is the OVERALL INTERMEDIATE DRAFT (the problem-level candidate answer assembled from the claims) plus the per-claim reports. Judge whether THIS draft lets a diligent reader construct the answer to the ORIGINAL question end-to-end — including any cross-claim synthesis (e.g. two claims that must be COMBINED into one derived answer, or an enumeration the claims only partially cover). If the draft answers the question (even partially), mark sufficient; if the question cannot be answered from the draft at all, mark insufficient and say exactly what is missing (Part C).
-
-MANDATORY three-stage reasoning (this is what makes the judgement reliable — follow it strictly):
-1. FIRST, write down the STEP-BY-STEP SUB-QUESTIONS you would need to answer in order to arrive at the label. Make sure to include questions about any ASSUMPTIONS implicit in the QUESTION, and include questions about any MATHEMATICAL CALCULATIONS or ARITHMETIC that would be required.
-2. THEN, answer each of those sub-questions step by step, working through any required mathematical calculations or arithmetic explicitly.
-3. FINALLY, use these answers to evaluate the criterion and decide `is_sufficient`. If the sub-question answers let you construct a definitive answer from the context, it is sufficient — even if the answer is not perfectly complete. Only when the sub-questions cannot be answered from the context (so NO plausible answer can be inferred) is it insufficient, and then you must fill `missing_information` with exactly what is missing (Part C).
-
-Computed answers: when a sub-question requires arithmetic (e.g. difference, count, "how many years"), WORK THE CALCULATION yourself from the values in the context and only mark sufficient if it resolves to a definite result with the RIGHT operands. If it cannot be completed, mark insufficient and list what is missing.
-
-The STEP-BY-STEP SUB-QUESTIONS from stage 1 MUST ALSO be emitted in a structured `sub_queries` array (one entry per sub-question), because they are the precise "what is missing" signal the next search round consumes. Decomposition rules (mutually-exclusive + complete coverage — every sub-question is an atomic fact needed to answer the ORIGINAL question, sub-questions do not overlap, and together they fully cover the question):
-- MULTI-STEP / multi-hop: emit ONE sub-query per logical hop, and the LAST hop is the entity/role the question actually asks for. E.g. Q="Who is the president of the team whose name was inspired by the Boston Braves?" -> [sub-query "Which team's name was inspired by the Boston Braves?", sub-query "Who is that team's president?"].
-- MULTI-ASPECT / enumeration+aggregate: emit ONE sub-query for the enumeration (values of EVERY member) and ONE for the aggregate (the combination over those members). E.g. Q="average left-field distance of every stadium" -> [sub-query "left-field distance of every stadium", sub-query "average of those left-field distances"].
-- ARITHMETIC / temporal: the arithmetic itself is a sub-query, and every OPERAND must have a value. If an operand is absent, that sub-query is `satisfied: false` and `missing_fact` names the absent operand specifically. E.g. Q="how many days between the two deaths?" -> [sub-query "McLean death date" (satisfied), sub-query "Meyer death date" (satisfied:false, missing_fact:"Eugene Meyer's exact death date", search_hint:"Eugene Meyer Washington Post died date"), sub-query "days between them"].
-
-Each entry in `sub_queries` is `{"sub_query": "...", "satisfied": true/false}`; when `satisfied` is false it MUST additionally carry `missing_fact` (the concrete absent fact) and `search_hint` (a searchable query anchoring entity+attribute+time). `satisfied: false` marks exactly WHAT is missing and WHERE to search next.
-
-## Part B — Claim-draft groundedness (per claim)
-
-For each claim's report (the intermediate draft), determine whether it is grounded and self-consistent. Each report was generated from that claim's cited evidence by the search agent, so the main risk is NOT absent-evidence hallucination but (a) prior-knowledge padding sneaking past the "saw it in evidence" rule, (b) internal contradictions between a claim's own numbers/entities, and (c) over-claiming a derived value.
-
-A report is GROUNDED only if it reads as a faithful, internally consistent, evidence-backed finding. Flag as UNGROUNDED anything that:
-- asserts a specific number/entity/relation but contradicts another part of the same report (internal conflict);
-- over-claims a computed/derived result beyond what the underlying values support;
-- reads like prior-knowledge padding rather than an evidence-backed finding.
-
-For each claim, classify the report as GROUNDED or UNGROUNDED. List each ungrounded assertion with a one-line reason.
-
-Derived/computed assertions — APPLY ONLY IF the question asks for a DERIVED result AND the report states that computed result. Then VERIFY THE CALCULATION YOURSELF from the values in the evidence: every operand must have an explicit value in the evidence AND be the RIGHT entity; recompute and check the report's stated result matches. If any operand's figure is absent, or the operands are the wrong entities, or the recomputed result disagrees, the computed assertion is UNGROUNDED (name the missing/wrong operand or the recomputed vs stated value).
-
-## Part C — Missing pieces (forward gap)
-
-`missing_information` = what the answer STILL needs but the evidence does NOT contain, with a searchable hint. This is the targeted re-search signal (Google Query Rewriter input). It is distinct from `ungrounded_assertions`:
-- `ungrounded_assertions` = the draft asserted something the evidence does NOT support -> drop/correct it.
-- `missing_information` = the answer REQUIRES a fact/entity/value the evidence lacks -> search for it.
-
-Populate `missing_information` when, even if every draft is grounded, the evidence does NOT cover a part the question asks for. Examples:
-- the question asks for a specific property (age / year / employer / distance / count) and the evidence has the entity but not that property;
-- a disambiguation is needed (evidence has a same-named but wrong entity);
-- the evidence covers only part of an enumeration (some list members present, others absent).
-
-Each `missing_information` item MUST carry a concrete `search_hint`: a searchable query (keywords / qualifiers / the specific entity+relation) that would retrieve the missing fact. Leave empty when the evidence is complete.
-
-## Output format (JSON) — ONE object, no commentary before/after
-
-```json
-{
- "is_sufficient": true,
- "confidence": 0.0,
- "required_entities": ["Entity 1"],
- "contradictions": ["conflicting figures if any"],
- "reasoning": "step-by-step sufficiency judgment",
- "sub_queries": [
- {
- "sub_query": "Which team's name was inspired by the Boston Braves?",
- "satisfied": true
- },
- {
- "sub_query": "Who is that team's president?",
- "satisfied": false,
- "missing_fact": "the team's current president",
- "search_hint": "Washington Commanders president 2024"
- }
- ],
- "claims": [
- {
- "claim_id": "c1",
- "grounded": true,
- "ungrounded_assertions": [],
- "missing_information": [
- {"what": "the specific missing fact the answer needs", "search_hint": "searchable query to retrieve it"}
- ]
- }
- ]
-}
-```
-
-## Requirements
-
-0. **The TOP-LEVEL output is ONE single JSON object.** `claims` is an array FIELD inside that object — never wrap the whole output in an array, and never output just the `claims` array. The output must start with `{` and end with `}`.
-1. `is_sufficient` true iff a plausible answer can be inferred from the context (Part A). Use the paper's SUFFICIENT-CONTEXT criterion: sufficient means a plausible answer can be constructed from the context — it does NOT need to be provably correct or complete. **A missing detail that a rough/partial answer can still accommodate does NOT make the context insufficient** — if the context lets you infer ANY plausible answer, mark sufficient (the presentation layer decides whether to caveat). Only when the context cannot support ANY plausible answer is it insufficient, and then your job is to say exactly what is missing (Part C).
-2. `confidence` (0-1): how confident you are in the sufficiency decision. 0.9-1.0 if clearly sufficient/fails; 0.5-0.7 if partial/ambiguous; below 0.5 if you cannot tell.
-3. `contradictions`: list internally conflicting figures/statements that make a single answer ambiguous. Empty array when none.
-4. `claims` is MANDATORY and must NEVER be empty when there is at least one claim draft. Include EVERY claim_id exactly once. For each claim you must output `grounded`, `ungrounded_assertions`, AND `missing_information`. Do NOT omit any claim or any field.
-5. `missing_information` is the MOST IMPORTANT field when `is_sufficient` is false. It is what drives the next search. When the context is insufficient, you MUST list, per claim, the concrete facts/entities/relations that are absent but required — each with a searchable `search_hint`. An empty `missing_information` is only acceptable when the claim is fully covered. **Never output an empty `claims` array while also saying `is_sufficient: false`** — that is a contradiction and aborts the whole search. If you judge the context insufficient, your primary job is to say exactly WHAT is missing and WHERE to search next.
-6. `sub_queries` is MANDATORY (may be an empty array only when there is nothing to decompose). Emit the full step-by-step sub-query set from Part A stage 1, each marked `satisfied`. A `satisfied: false` entry MUST carry `missing_fact` + `search_hint` and is the precise "what to search next" signal — the Query Rewriter turns exactly those into the next round's queries. Keep sub-queries mutually-exclusive and fully-covering (no overlap, no gap beyond the unsatisfied ones).
-7. `ungrounded_assertions` empty when grounded; otherwise list each with a one-line `reason`.
-8. `reasoning` should be concise and clear.
diff --git a/internal/rag/prompts/templates.go b/internal/rag/prompts/templates.go
index 3a4bab4d51..8323d6e078 100644
--- a/internal/rag/prompts/templates.go
+++ b/internal/rag/prompts/templates.go
@@ -19,20 +19,13 @@ package prompts
import (
"embed"
"fmt"
- "regexp"
"strings"
)
// Prompt templates for retrieval, chat, and ingestion are embedded here so
// callers can load them independently of the working directory.
-//
-// action_set is the ONE template that is not part of a system prompt: the harness
-// appends it to a session's seed message, and only for a direction that is a SET
-// (see enumerates). Strategy text that is always in the prompt is paid for by
-// every question, including the ones it cannot help.
-//go:embed action_run.md action_set.md action_initialize_state.md sca_select.md sca_query_rewrite.md citation_prompt.md
-//go:embed ask_summary.md keyword_prompt.md cross_languages_sys_prompt.md cross_languages_user_prompt.md meta_filter.md related_question.md vision_llm_describe_prompt.md vision_llm_figure_describe_prompt.md
+//go:embed citation_prompt.md ask_summary.md keyword_prompt.md cross_languages_sys_prompt.md cross_languages_user_prompt.md meta_filter.md related_question.md vision_llm_describe_prompt.md vision_llm_figure_describe_prompt.md
var templatesFS embed.FS
// EmbeddedPromptLoader serves the bundled .md templates by name. Load returns
@@ -40,8 +33,7 @@ var templatesFS embed.FS
// engages exactly as with the previous per-package loaders.
type EmbeddedPromptLoader struct{}
-// Load implements the harness PromptLoader contract: Load(name string)
-// (string, error), reading "<name>.md" and returning its content trimmed.
+// Load reads "<name>.md" and returns its content trimmed.
func (EmbeddedPromptLoader) Load(name string) (string, error) {
data, err := templatesFS.ReadFile(name + ".md")
if err != nil {
@@ -49,39 +41,3 @@ func (EmbeddedPromptLoader) Load(name string) (string, error) {
}
return strings.TrimSpace(string(data)), nil
}
-
-// Loader is the minimal template-source contract used by Render. It mirrors the
-// harness PromptLoader shape (Load(name string) (string, error)) so any loader
-// wired into the harness (including EmbeddedPromptLoader and the in-memory test
-// loader) satisfies it without the prompts package importing harness.
-type Loader interface {
- Load(name string) (string, error)
-}
-
-// leftoverVarRE matches a Jinja variable placeholder that no provided var
-// filled in. Python renders these through PROMPT_JINJA_ENV's default Undefined
-// (SandboxedEnvironment without undefined=StrictUndefined), which interpolates
-// as the empty string; the regex mirrors that instead of leaving {{ var }}
-// verbatim in the prompt.
-var leftoverVarRE = regexp.MustCompile(`\{\{\s*[A-Za-z_][A-Za-z0-9_.]*\s*\}\}`)
-
-// Render loads `name` and substitutes the {{var}} placeholders. A nil loader
-// defaults to the embedded templates; a loader that cannot resolve `name`
-// falls back to `fallback`. Unprovided variables render as the empty string
-// (Jinja default Undefined), not as a literal placeholder.
-func Render(loader Loader, name, fallback string, vars map[string]string) string {
- if loader == nil {
- loader = EmbeddedPromptLoader{}
- }
- tmpl := fallback
- if t, err := loader.Load(name); err == nil && strings.TrimSpace(t) != "" {
- tmpl = t
- }
- for k, v := range vars {
- // Jinja tolerates both `{{var}}` and `{{ var }}`; the project's
- // templates use the spaced form, so substitute both.
- tmpl = strings.ReplaceAll(tmpl, "{{"+k+"}}", v)
- tmpl = strings.ReplaceAll(tmpl, "{{ "+k+" }}", v)
- }
- return leftoverVarRE.ReplaceAllString(tmpl, "")
-}
diff --git a/internal/rag/prompts/templates_test.go b/internal/rag/prompts/templates_test.go
index 2bdde02820..195ec9edc5 100644
--- a/internal/rag/prompts/templates_test.go
+++ b/internal/rag/prompts/templates_test.go
@@ -26,11 +26,6 @@ import (
func TestEmbeddedLoaderCanonicalTemplates(t *testing.T) {
t.Chdir(t.TempDir())
names := []string{
- "action_run",
- "action_set",
- "action_initialize_state",
- "sca_select",
- "sca_query_rewrite",
"citation_prompt",
"ask_summary",
"keyword_prompt",
@@ -62,177 +57,3 @@ func TestEmbeddedLoaderMissingName(t *testing.T) {
t.Fatal("Load(no_such_template) succeeded, want error")
}
}
-
-// TestRenderPromptNilLoaderUsesEmbeddedCanonical locks in that an unset loader
-// resolves the embedded sca_select.md
-// rather than the condensed constant — the whole point of the orchestration
-// parity fix.
-func TestRenderPromptNilLoaderUsesEmbeddedCanonical(t *testing.T) {
- vars := map[string]string{"question": "Q?", "overall_draft": "D", "claims_context": "C"}
- got := Render(nil, "sca_select", "", vars)
-
- for _, frag := range []string{
- `Google-style "Sufficient Context Agent"`, // canonical sca_select.md only
- "Q?",
- "D",
- "C",
- } {
- if !strings.Contains(got, frag) {
- t.Errorf("canonical sca_select render missing %q", frag)
- }
- }
- if strings.Contains(got, "{{") {
- t.Errorf("render left an unresolved placeholder: %q", got)
- }
-}
-
-// TestRenderPromptNilLoaderQueryRewriteCanonical is the rewrite-side twin:
-// nil loader must resolve the full embedded sca_query_rewrite.md, not the
-// condensed constant.
-
-// TestRenderPromptNilLoaderQueryRewriteCanonical is the rewrite-side twin:
-// nil loader must resolve the full embedded sca_query_rewrite.md, not the
-// condensed constant.
-func TestRenderPromptNilLoaderQueryRewriteCanonical(t *testing.T) {
- vars := map[string]string{
- "question": "Q?",
- "gaps": "G",
- "bridge_values": "B",
- "research_context": "R",
- }
- got := Render(nil, "sca_query_rewrite", "", vars)
-
- // Rule 1 wording exists only in the canonical template.
- if !strings.Contains(got, "The query must name the specific missing entity") {
- t.Errorf("canonical sca_query_rewrite render not used; got:\n%s", got)
- }
- for _, frag := range []string{"Q?", "G", "B", "R"} {
- if !strings.Contains(got, frag) {
- t.Errorf("canonical sca_query_rewrite render missing %q", frag)
- }
- }
- if strings.Contains(got, "{{") {
- t.Errorf("render left an unresolved placeholder: %q", got)
- }
-}
-
-// TestRenderUnknownNameWithEmptyFallbackReturnsEmpty mirrors Python's
-// fail-fast loading: Go has no condensed fallback constant (unlike the earlier
-// prototype), so a loader that cannot resolve the name with fallback="" yields
-// the empty string rather than a degraded prompt.
-
-// TestRenderUnknownNameWithEmptyFallbackReturnsEmpty mirrors Python's
-// fail-fast loading: Go has no condensed fallback constant (unlike the earlier
-// prototype), so a loader that cannot resolve the name with fallback="" yields
-// the empty string rather than a degraded prompt.
-func TestRenderUnknownNameWithEmptyFallbackReturnsEmpty(t *testing.T) {
- got := Render(stubLoader{}, "no_such_template", "",
- map[string]string{"question": "Q?", "gaps": "G"})
- if got != "" {
- t.Errorf("expected empty string for unknown template with empty fallback, got:\n%s", got)
- }
-}
-
-// TestRenderPromptMissingVarInterpolatesEmpty mirrors Python's default Jinja
-// Undefined: an unprovided variable renders as the empty string, not as a
-// literal {{ var }}.
-
-// TestRenderPromptMissingVarInterpolatesEmpty mirrors Python's default Jinja
-// Undefined: an unprovided variable renders as the empty string, not as a
-// literal {{ var }}.
-func TestRenderPromptMissingVarInterpolatesEmpty(t *testing.T) {
- loader := stubLoader{text: "A={{present}} B={{ absent }} C={{absent}}"}
- got := Render(loader, "t", "fallback", map[string]string{"present": "x"})
- if want := "A=x B= C="; got != want {
- t.Errorf("render = %q, want %q", got, want)
- }
-}
-
-// TestRenderPromptSubstitutesBothPlaceholderSpellings: Jinja tolerates the
-// spaced and unspaced forms; the renderer must handle both.
-
-// TestRenderPromptSubstitutesBothPlaceholderSpellings: Jinja tolerates the
-// spaced and unspaced forms; the renderer must handle both.
-func TestRenderPromptSubstitutesBothPlaceholderSpellings(t *testing.T) {
- loader := stubLoader{text: "{{v}}|{{ v }}"}
- got := Render(loader, "t", "fallback", map[string]string{"v": "x"})
- if want := "x|x"; got != want {
- t.Errorf("render = %q, want %q", got, want)
- }
-}
-
-// stubLoader serves a canned template, mirroring how tests wire
-// harness.StringPromptLoader.
-type stubLoader struct{ text string }
-
-func (s stubLoader) Load(string) (string, error) { return s.text, nil }
-
-// TestRenderPromptNilLoaderUsesEmbeddedCanonical locks in that an unset loader
-// resolves the embedded sca_select.md
-// rather than the condensed constant — the whole point of the orchestration
-// parity fix.
-
-// TestEnumerationProtocolIsNotInTheSystemPrompt pins the split: the enumeration
-// protocol lives in its own template, which the harness appends to the turn of a
-// session that has SHOWN it is enumerating (see appendBatchProtocol) — never to the
-// system prompt, and never to the seed.
-//
-// It was briefly a section of action_run.md, and it did its job there (the answer
-// went from ten-to-thirteen members to sixteen), but a system prompt is paid for
-// by EVERY question: a strategy that only helps an enumeration is a tax on the
-// single-value questions it cannot help. It then rode the seed, gated on the shape
-// of the planner's table, and that gate misfired too — measured (2026-09-15) 11 of
-// 20 FRAMES sessions were handed it for questions whose answer is one number.
-//
-// The FORM of the batch matters as much as the instruction to write one, which is
-// why the alternation is pinned here. Measured (2026-09-14, the recall run): every
-// name the answer needed that later runs lost — 程远志、荀正、杨龄、夏侯存 — entered
-// the corpus as the MODEL'S OWN query term, in an alternation of the names it
-// doubted (`关羽斩杨龄|荀正|夏侯存`, `关公斩庞德|成何`, `关羽斩黄巾|程远志|管亥|张宝|张梁`),
-// with the runtime then searching each name on its own. Then the template was
-// rewritten to say "batches of four to six, space-separated" and the same question
-// came back with thirteen members: the batches became `关羽 斩 颜良 文丑` — four
-// fields of which two are the subject and the verb — and the doubtful names were
-// never asked about at all (grep of that run for 荀正/杨龄/夏侯存: zero).
-//
-// The alternation instruction ALONE did not restore them, which is why the demand
-// for unseen names is pinned too. Measured (2026-09-16): a run whose seed carried
-// this template (3416 chars — the version with items 2 and 3 written as the
-// alternation and the doubtful batch, but without item 1's "at least one you have
-// not") wrote the alternation ZERO times and answered the same thirteen members.
-// The template that had produced seventeen members (2509ccb96) differed in exactly
-// two places, and both demanded NAMES rather than a FORM: it asked for a long list
-// ("fifteen to twenty-five candidates") and it said what to leave out — "Do NOT
-// stop at the names you have already seen in a passage: those are the ones
-// retrieval reaches by itself, and they are exactly the ones that are not
-// missing". A number is a corpus-independent way of saying "long", but the ratio
-// in item 1 says the same thing without pretending the answer's size is known, and
-// it keeps the second, load-bearing half: the names you have already read are not
-// the missing ones.
-func TestEnumerationProtocolIsNotInTheSystemPrompt(t *testing.T) {
- run, err := (EmbeddedPromptLoader{}).Load("action_run")
- if err != nil {
- t.Fatalf("Load(action_run): %v", err)
- }
- if strings.Contains(run, "SET / COUNT directions") {
- t.Error("the enumeration protocol must not be part of the system prompt")
- }
-
- set, err := (EmbeddedPromptLoader{}).Load("action_set")
- if err != nil {
- t.Fatalf("Load(action_set): %v", err)
- }
- for _, want := range []string{"SET / COUNT directions", "name1|name2|name3", "at least one you have not", "names you are LEAST sure of", "two consecutive batches"} {
- if !strings.Contains(set, want) {
- t.Errorf("action_set is missing %q", want)
- }
- }
- // The budget exception is GONE, and this is the assertion that keeps it gone.
- // "the 1-2-call budget does NOT apply" turned a method into a licence: the run
- // that carried it in every set-shaped seed went 60 → 92 rounds and 1 → 5
- // zero-score questions. A batch is a new fact ON a set direction; the global call
- // budget is not suspended by this template.
- if strings.Contains(set, "does NOT apply") {
- t.Error("action_set must not suspend the global call budget")
- }
-}
diff --git a/internal/service/chat_harness_prompt_test.go b/internal/service/chat_harness_prompt_test.go
deleted file mode 100644
index df9ca93bbd..0000000000
--- a/internal/service/chat_harness_prompt_test.go
+++ /dev/null
@@ -1,91 +0,0 @@
-package service
-
-import (
- "strings"
- "testing"
-
- "ragflow/internal/entity"
-)
-
-// Regression lock for the first-turn empty dataset binding: the agentic
-// (reasoning) system prompt used to render {knowledge} as "" — the web UI's
-// default template then read "derived solely from this dataset: ``" and the
-// outer model answered the canned "not found in the dataset!" line without
-// calling the terminal `rag` tool (logs/api_server_9384.log 2026-09-14
-// 13:50:14: answer_chars=59, zero graph LLM calls).
-
-func TestHarnessBoundDatasetNames(t *testing.T) {
- cases := []struct {
- name string
- kbs []*entity.Knowledgebase
- want string
- }{
- {"nil", nil, ""},
- {"empty", []*entity.Knowledgebase{}, ""},
- {
- "filters nil and unnamed",
- []*entity.Knowledgebase{nil, {ID: "kb1"}, {ID: "kb2", Name: "三国演义"}},
- "三国演义",
- },
- {
- "joins multiple names",
- []*entity.Knowledgebase{{Name: "三国演义"}, {Name: "三国志"}},
- "三国演义, 三国志",
- },
- }
- for _, c := range cases {
- t.Run(c.name, func(t *testing.T) {
- if got := harnessBoundDatasetNames(c.kbs); got != c.want {
- t.Fatalf("harnessBoundDatasetNames(%v) = %q, want %q", c.kbs, got, c.want)
- }
- })
- }
-}
-
-// TestHarnessSystemPromptFirstTurnNonEmpty pins the end-to-end first-turn
-// render: with datasets bound, the dialog system prompt's {knowledge}
-// placeholder (backtick-wrapped in the web UI default) must carry the bound
-// dataset names — never an empty “ binding.
-func TestHarnessSystemPromptFirstTurnNonEmpty(t *testing.T) {
- // The web UI default template (web/src/locales en.ts systemInitialValue):
- // the placeholder is wrapped in backticks.
- template := "You are an intelligent assistant. Your primary function is to answer questions based strictly on the provided knowledge base.\n\n**Essential Rules:**\n - Your answer must be derived **solely** from this dataset: `{knowledge}`.\n"
-
- s := &ChatPipelineService{}
- kws := map[string]interface{}{
- "date": "2026-09-14 06:00:00",
- }
- if _, ok := kws["knowledge"]; !ok {
- kws["knowledge"] = harnessBoundDatasetNames([]*entity.Knowledgebase{
- {ID: "kb1", Name: "三国演义"},
- })
- }
- rendered := s.formatPrompt(template, kws)
-
- if strings.Contains(rendered, "``") {
- t.Fatalf("first-turn system prompt rendered an empty dataset binding:\n%s", rendered)
- }
- if !strings.Contains(rendered, "三国演义") {
- t.Fatalf("first-turn system prompt does not carry the bound dataset name:\n%s", rendered)
- }
- if strings.Contains(rendered, "{knowledge}") {
- t.Fatalf("{knowledge} placeholder was not substituted:\n%s", rendered)
- }
-}
-
-// TestHarnessSystemPromptCallerSuppliedKnowledgeWins: a caller-supplied
-// knowledge value must not be overwritten by the dataset-name default.
-func TestHarnessSystemPromptCallerSuppliedKnowledgeWins(t *testing.T) {
- s := &ChatPipelineService{}
- kws := map[string]interface{}{"knowledge": "caller evidence"}
- if _, ok := kws["knowledge"]; !ok {
- kws["knowledge"] = harnessBoundDatasetNames([]*entity.Knowledgebase{{Name: "三国演义"}})
- }
- rendered := s.formatPrompt("Context: {knowledge}", kws)
- if !strings.Contains(rendered, "caller evidence") {
- t.Fatalf("caller-supplied knowledge was overwritten:\n%s", rendered)
- }
- if strings.Contains(rendered, "三国演义") {
- t.Fatalf("dataset-name default leaked over the caller-supplied value:\n%s", rendered)
- }
-}
diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.go
index 987a859ac1..f4dc141a45 100644
--- a/internal/service/chat_pipeline.go
+++ b/internal/service/chat_pipeline.go
@@ -34,7 +34,6 @@ import (
"regexp"
"sort"
"strings"
- "sync"
"time"
"unicode/utf8"
@@ -110,17 +109,15 @@ type AsyncChatResult struct {
accumulatedAnswer string
}
-// ThinkEvent is one structured agentic-RAG reasoning step as it goes over the
-// wire (the SSE `think_event` field). This is the ONE definition: the harness
-// that produces the step aliases it (harness.ThinkEvent), because the dependency
-// runs this way only — the harness imports this package for its exploration
-// providers, so this package cannot import the harness back.
+// ThinkEvent is one structured agentic reasoning step as it goes over the
+// wire (the SSE `think_event` field). This is the ONE definition, so a
+// producer that needs structured steps aliases this type instead of carrying
+// a second copy bridged field by field — that older shape compiled for as
+// long as one side stayed a subset, and a field added on the producer side
+// then reached the client as nothing at all.
//
-// The two sides used to carry two identical structs bridged field by field in
-// cmd. That shape compiles for as long as one side stays a subset, so a field
-// added on the harness side and not mirrored here reached the client as nothing
-// at all. The JSON tags below ARE the client contract; TestThinkEventWireKeys
-// pins them.
+// The JSON tags below ARE the client contract; TestThinkEventWireKeys pins
+// them.
type ThinkEvent struct {
Kind string `json:"kind"`
// Stage is the bracketed tag the step belongs to ("Function tool",
@@ -130,7 +127,7 @@ type ThinkEvent struct {
Tool string `json:"tool,omitempty"`
// Args is the tool call's arguments, rendered for display.
Args string `json:"args,omitempty"`
- // Status and Reason mirror the harness ToolOutcome.
+ // Status and Reason describe a tool step's outcome.
Status string `json:"status,omitempty"`
Reason string `json:"reason,omitempty"`
// Cause is the tool's own explanation of a failure: a bare reason such as
@@ -672,23 +669,13 @@ func (s *ChatPipelineService) AsyncChat(
timer.Exit(common.PhaseQueryRefinement)
// === Phase 9: Retrieval ===
- // reasoning is an integer level: 0 = off (regular RAG via async_chat),
- // 1..4 = low/medium/high/ultra (harness agentic). It comes from the
- // request kwargs first, then prompt_config. reasoningLevelAgentic never
- // reaches here: the AsyncChat dispatch already routed it to the
- // smart-reasoning agent, which returns before this goroutine starts.
- //
- // Python rag_agent also refuses the agentic loop when the model cannot
- // call tools, and routes those requests to async_chat.
- reasoningLevel := resolveReasoningLevel(kwargs, map[string]interface{}(chat.PromptConfig))
- // The chat model's tool capability rides on the config Phase 2 resolved,
- // so the guard costs nothing here and judges the model the request
- // actually runs on.
- useReasoning := reasoningNeedsAgenticGraph(chat, llmModelConfig, reasoningLevel)
+ // The retrieval strategy was chosen before this goroutine started:
+ // AsyncChat hands reasoning level reasoningLevelAgentic to the
+ // smart-reasoning agent, which runs its own retrieval and never returns
+ // here. Every other level — including a stale 1..4 sent by an older
+ // client — retrieves through the standard path below.
common.Info("Phase 9: Retrieval",
- zap.Bool("has_knowledge_param", hasKnowledgeParam),
- zap.Int("reasoning_level", reasoningLevel),
- zap.Bool("reasoning", useReasoning))
+ zap.Bool("has_knowledge_param", hasKnowledgeParam))
timer.Enter(common.PhaseRetrieval)
var kbinfos map[string]interface{}
@@ -703,323 +690,150 @@ func (s *ChatPipelineService) AsyncChat(
rerankCandidatesCount = 64
}
- // When hasKnowledgeParam is true, runs (mutually exclusive):
- // a) If reasoning is enabled: DeepResearcher replaces vector retrieval.
- // b) Otherwise: standard retrieval, then:
- // - TOC enhancement (if toc_enhance is enabled).
- // - Child chunk retrieval.
- // - Web search provider (if internet is enabled).
- // - Knowledge graph retrieval (if use_kg is enabled).
+ // When hasKnowledgeParam is true, runs standard retrieval, then:
+ // - TOC enhancement (if toc_enhance is enabled).
+ // - Child chunk retrieval.
+ // - Web search provider (if internet is enabled).
+ // - Knowledge graph retrieval (if use_kg is enabled).
// Populates kbinfos (chunks + doc_aggs) and knowledges.
// When false, the entire block is skipped.
if hasKnowledgeParam {
- if useReasoning && chatModel != nil && len(kbs) > 0 {
- // The harness collects evidence and composes the answer for
- // reasoning levels 1..4. Apply citation visibility to its stream
- // and final answer. If no answer is returned, keep the evidence
- // and continue through the regular generation path.
- thinkingMode := harnessModeForLevel(reasoningLevel)
- question := strings.Join(questions, " ")
- // Stream the answer as the harness composes it (Python
- // tools.answer_sink). The final event below still carries the
- // complete answer plus references, matching how the non-agentic
- // path streams deltas and then re-sends the full answer
- // (dialog_service.py:807).
- //
- // isThink marks pieces of the harness's hidden reasoning block
- // (the final-answer model's native reasoning). Frame them like
- // the tool path above (ChatStreamlyWithTools): StartToThink →
- // Reasoning → EndToThink, so every SSE consumer (chat_session,
- // bot_completion, openai_chat) sees the standard think framing.
- // Reasoning text is deliberately kept on the Reasoning field
- // (not Answer) so OpenAI-compat maps it to reasoning_content.
- harnessThinking := false
- // answerStreamed records that an answer delta already reached the
- // client, which makes any LATER think delta unrenderable: the UI
- // would show a stray "Thought" block after the finished answer.
- //
- // That is what the loop narration around the single harness call
- // would otherwise do: "…produced the final answer, done." is emitted
- // after retrieveViaHarness returns, but the answer itself streams
- // DURING it. Python logs that line immediately before yielding the
- // terminal tool's answer (chat_model.py:799-802), so its "done" line
- // still belongs to the thinking phase. Same for late engine-progress
- // lines (concurrent research slots); a thought block after the answer
- // is never useful.
- answerStreamed := false
- // Engine progress (B) is pushed from concurrent research-slot
- // goroutines while the final compose runs on the main goroutine, so
- // serialize the think-framing state machine and the out<-send.
- var answerFilter, reasoningFilter citationStreamFilter
- var sinkMu sync.Mutex
- send := func(ev AsyncChatResult) {
- select {
- case out <- ev:
- case <-ctx.Done():
+ // Regular RAG mirroring Python's async_chat — native docStore
+ // retrieval, then the TOC / child chunk / web search / KG
+ // enhancements below.
+ searchQuestion := strings.Join(questions, " ")
+ if embModel != nil {
+ rankFeature := s.MetadataSvc.LabelQuestion(ctx, searchQuestion, kbs)
+ {
+ tenantIDs := make([]string, 0)
+ kbIDs := make([]string, 0)
+ for _, kb := range kbs {
+ tenantIDs = append(tenantIDs, kb.TenantID)
+ kbIDs = append(kbIDs, kb.ID)
}
- }
- sink := func(delta string, isThink bool) {
- sinkMu.Lock()
- defer sinkMu.Unlock()
- if isThink {
- if answerStreamed {
- // Never reopen a thought block after the answer: drop the
- // late narration/progress line instead.
- return
- }
- if !harnessThinking {
- harnessThinking = true
- send(AsyncChatResult{
- Reference: map[string]interface{}{},
- CreatedAt: float64(time.Now().Unix()),
- Final: false,
- StartToThink: true,
- })
- }
- if !quote {
- delta = reasoningFilter.write(delta)
- }
- if delta != "" {
- send(AsyncChatResult{
- Reasoning: delta,
- Reference: map[string]interface{}{},
- CreatedAt: float64(time.Now().Unix()),
- Final: false,
- })
- }
- return
- }
- if harnessThinking {
- if text := reasoningFilter.flush(); text != "" {
- send(AsyncChatResult{Reasoning: text, Reference: map[string]interface{}{}, CreatedAt: float64(time.Now().Unix())})
- }
- harnessThinking = false
- send(AsyncChatResult{
- Reference: map[string]interface{}{},
- CreatedAt: float64(time.Now().Unix()),
- Final: false,
- EndToThink: true,
- })
- }
- if delta != "" {
- answerStreamed = true
- }
- if !quote {
- delta = answerFilter.write(delta)
- }
- if delta != "" {
- send(AsyncChatResult{
- Answer: delta,
- Reference: map[string]interface{}{},
- CreatedAt: float64(time.Now().Unix()),
- Final: false,
- })
- }
- }
- thinkSink := harnessThinkSink(ctx, out)
- // Python dialog_service.py:2077 — the web provider is handed to
- // RAGTools only when the internet flag enables web search;
- // otherwise web_search stays off the agentic tool surface.
- var webSearch func(context.Context, []string) ([]string, error)
- if s.shouldUseWebSearch(chat, kwargs["internet"]) {
- webSearch = s.harnessWebSearchFunc(chat.PromptConfig)
- }
- // Python passes system_prompt=_render_reasoning_system_prompt(
- // dialog, prompt_config, kwargs) to RAGTools
- // (dialog_service.py:2084): the dialog system prompt rendered
- // with the caller kwargs, a UTC date, and {knowledge} defaulted
- // to the BOUND DATASET NAMES — the agentic graph supplies the
- // evidence itself, but an empty binding made the outer model
- // read the prompt as "the dataset is empty" and answer the
- // canned "not found in the dataset!" line without calling the
- // terminal `rag` tool (first-turn short-circuit, observed
- // 2026-09-14: answer_chars=59, zero graph LLM calls).
- harnessSystemPrompt := ""
- if sp, ok := chat.PromptConfig["system"].(string); ok && sp != "" {
- kws := make(map[string]interface{}, len(kwargs)+2)
- for k, v := range kwargs {
- kws[k] = v
- }
- kws["date"] = time.Now().UTC().Format(time.RFC3339)
- if _, ok := kws["knowledge"]; !ok {
- kws["knowledge"] = harnessBoundDatasetNames(kbs)
- }
- harnessSystemPrompt = s.formatPrompt(sp, kws)
- }
- harnessImages := append(historyImageFiles, imageFiles...)
- var history []map[string]interface{}
- if kwargs["store_history_messages"] == false {
- history = messages
- }
- hk, slotCites, citeChunkIDs, harnessAnswer, hErr := s.retrieveViaHarness(ctx, question, kbs, docIDs, harnessImages, attachments, thinkingMode, chat.TenantID, chat.LLMID, chat.ID, HarnessRetrieval{
- TopN: int(chat.TopN),
- SimilarityThreshold: chat.SimilarityThreshold,
- VectorSimilarityWeight: chat.VectorSimilarityWeight,
- RerankCandidatesCount: int(chat.RerankCandidatesCount),
- }, webSearch, sink, thinkSink, harnessSystemPrompt, history)
- // The harness streams think-then-answer inside ONE compose call.
- // Close the block here, once that call (and its trailing
- // narration line) has returned: a reasoning-only run would
- // otherwise keep it open until the goroutine exits, emitting
- // EndToThink after the Phase 10/11 answer or the Final below.
- // No-op when no block is open.
- sink("", false)
- sinkMu.Lock()
- if text := answerFilter.flush(); text != "" {
- send(AsyncChatResult{Answer: text, Reference: map[string]interface{}{}, CreatedAt: float64(time.Now().Unix())})
- }
- sinkMu.Unlock()
- if hErr != nil {
- common.Warn("harness retrieval failed", zap.Error(hErr))
- } else {
- kbinfos = hk
- if harnessAnswer != "" {
- common.Info("harness produced final cited answer; short-circuiting",
- zap.Int("answer_chars", len(harnessAnswer)))
- final := s.decorateHarnessAnswer(harnessAnswer, kbinfos, slotCites, citeChunkIDs, quote)
- final.Final = true
- out <- final
- return
+
+ docEngine := engine.Get()
+ documentDAO := dao.NewDocumentDAO()
+ retrievalSvc := nlp.NewRetrievalService(docEngine, documentDAO)
+
+ top := int(chat.TopK)
+ threshold := chat.SimilarityThreshold
+ vsw := chat.VectorSimilarityWeight
+ topN := int(chat.TopN)
+
+ req := &nlp.RetrievalRequest{
+ Question: searchQuestion,
+ TenantIDs: tenantIDs,
+ KbIDs: kbIDs,
+ DocIDs: docIDs,
+ Page: 1,
+ PageSize: topN,
+ RerankCandidatesCount: &rerankCandidatesCount,
+ KNNTopK: &top,
+ SimilarityThreshold: &threshold,
+ VectorSimilarityWeight: &vsw,
+ RankFeature: &rankFeature,
+ RerankModel: rerankModel,
+ EmbeddingModel: embModel,
+ Aggs: func() *bool { v := true; return &v }(),
}
- }
- } else {
- // Non-reasoning chat (level 0): regular RAG mirroring Python's
- // async_chat — native docStore retrieval, then the TOC / child
- // chunk / web search / KG enhancements below.
- searchQuestion := strings.Join(questions, " ")
- if embModel != nil {
- rankFeature := s.MetadataSvc.LabelQuestion(ctx, searchQuestion, kbs)
- {
- tenantIDs := make([]string, 0)
- kbIDs := make([]string, 0)
- for _, kb := range kbs {
- tenantIDs = append(tenantIDs, kb.TenantID)
- kbIDs = append(kbIDs, kb.ID)
- }
- docEngine := engine.Get()
- documentDAO := dao.NewDocumentDAO()
- retrievalSvc := nlp.NewRetrievalService(docEngine, documentDAO)
-
- top := int(chat.TopK)
- threshold := chat.SimilarityThreshold
- vsw := chat.VectorSimilarityWeight
- topN := int(chat.TopN)
-
- req := &nlp.RetrievalRequest{
- Question: searchQuestion,
- TenantIDs: tenantIDs,
- KbIDs: kbIDs,
- DocIDs: docIDs,
- Page: 1,
- PageSize: topN,
- RerankCandidatesCount: &rerankCandidatesCount,
- KNNTopK: &top,
- SimilarityThreshold: &threshold,
- VectorSimilarityWeight: &vsw,
- RankFeature: &rankFeature,
- RerankModel: rerankModel,
- EmbeddingModel: embModel,
- Aggs: func() *bool { v := true; return &v }(),
+ result, retErr := retrievalSvc.Retrieval(ctx, req)
+ if retErr != nil {
+ kbinfos = map[string]interface{}{
+ "total": 0,
+ "chunks": []map[string]interface{}{},
+ "doc_aggs": []interface{}{},
}
-
- result, retErr := retrievalSvc.Retrieval(ctx, req)
- if retErr != nil {
- kbinfos = map[string]interface{}{
- "total": 0,
- "chunks": []map[string]interface{}{},
- "doc_aggs": []interface{}{},
- }
- err = retErr
- } else {
- docAggs := make([]interface{}, len(result.DocAggs))
- for i, da := range result.DocAggs {
- docAggs[i] = da
- }
-
- kbinfos = map[string]interface{}{
- "total": len(result.Chunks),
- "chunks": result.Chunks,
- "doc_aggs": docAggs,
- }
+ err = retErr
+ } else {
+ docAggs := make([]interface{}, len(result.DocAggs))
+ for i, da := range result.DocAggs {
+ docAggs[i] = da
}
- }
- if err != nil {
- common.Warn("Retrieval failed", zap.Error(err))
- out <- AsyncChatResult{
- Answer: fmt.Sprintf("**ERROR**: %s", err.Error()),
- Final: true,
+
+ kbinfos = map[string]interface{}{
+ "total": len(result.Chunks),
+ "chunks": result.Chunks,
+ "doc_aggs": docAggs,
}
- return
}
}
-
- // TOC enhancement
- if useTOC, _ := chat.PromptConfig["toc_enhance"].(bool); useTOC && chatModel != nil && len(kbs) > 0 {
- enhancer := NewTOCEnhancer(
- engine.Get(),
- chatModel,
- kbTenantIDStrings(kbs),
- kbIDStrings(kbs),
- searchQuestion,
- int(chat.TopN),
- )
- if added, err := enhancer.Enhance(ctx, kbinfos); err != nil {
- common.Warn("TOC enhance failed", zap.Error(err))
- } else if added > 0 {
- common.Debug("TOC enhance added chunks", zap.Int("added", added))
+ if err != nil {
+ common.Warn("Retrieval failed", zap.Error(err))
+ out <- AsyncChatResult{
+ Answer: fmt.Sprintf("**ERROR**: %s", err.Error()),
+ Final: true,
}
+ return
}
+ }
- // Child chunk retrieval
- if existingChunks, ok := kbinfos["chunks"].([]map[string]interface{}); ok && len(existingChunks) > 0 {
- kbinfos["chunks"] = nlp.RetrievalByChildren(existingChunks, kbTenantIDStrings(kbs), engine.Get(), ctx)
+ // TOC enhancement
+ if useTOC, _ := chat.PromptConfig["toc_enhance"].(bool); useTOC && chatModel != nil && len(kbs) > 0 {
+ enhancer := NewTOCEnhancer(
+ engine.Get(),
+ chatModel,
+ kbTenantIDStrings(kbs),
+ kbIDStrings(kbs),
+ searchQuestion,
+ int(chat.TopN),
+ )
+ if added, err := enhancer.Enhance(ctx, kbinfos); err != nil {
+ common.Warn("TOC enhance failed", zap.Error(err))
+ } else if added > 0 {
+ common.Debug("TOC enhance added chunks", zap.Int("added", added))
}
+ }
- // Web search
- if s.shouldUseWebSearch(chat, kwargs["internet"]) {
- provider := resolveWebSearchProvider(chat.PromptConfig)
- webResult, webErr := s.retrieveWebSearch(ctx, provider, searchQuestion)
- if webErr != nil {
- common.Warn("Web search failed", zap.Error(webErr))
- } else {
- // Extend chunks and doc_aggs with web search results.
- if existingChunks, ok := kbinfos["chunks"].([]map[string]interface{}); ok {
- if newChunks, ok := webResult["chunks"].([]map[string]interface{}); ok {
- kbinfos["chunks"] = append(existingChunks, newChunks...)
- }
+ // Child chunk retrieval
+ if existingChunks, ok := kbinfos["chunks"].([]map[string]interface{}); ok && len(existingChunks) > 0 {
+ kbinfos["chunks"] = nlp.RetrievalByChildren(existingChunks, kbTenantIDStrings(kbs), engine.Get(), ctx)
+ }
+
+ // Web search
+ if s.shouldUseWebSearch(chat, kwargs["internet"]) {
+ provider := resolveWebSearchProvider(chat.PromptConfig)
+ webResult, webErr := s.retrieveWebSearch(ctx, provider, searchQuestion)
+ if webErr != nil {
+ common.Warn("Web search failed", zap.Error(webErr))
+ } else {
+ // Extend chunks and doc_aggs with web search results.
+ if existingChunks, ok := kbinfos["chunks"].([]map[string]interface{}); ok {
+ if newChunks, ok := webResult["chunks"].([]map[string]interface{}); ok {
+ kbinfos["chunks"] = append(existingChunks, newChunks...)
}
- if existingAggs, ok := kbinfos["doc_aggs"].([]interface{}); ok {
- if newAggs, ok := webResult["doc_aggs"].([]interface{}); ok {
- kbinfos["doc_aggs"] = append(existingAggs, newAggs...)
- }
+ }
+ if existingAggs, ok := kbinfos["doc_aggs"].([]interface{}); ok {
+ if newAggs, ok := webResult["doc_aggs"].([]interface{}); ok {
+ kbinfos["doc_aggs"] = append(existingAggs, newAggs...)
}
}
}
+ }
- // Knowledge Graph retrieval
- if useKG, _ := chat.PromptConfig["use_kg"].(bool); useKG && chatModel != nil && len(kbs) > 0 {
- kgIDs := kbIDStrings(kbs)
- if len(kgIDs) > 0 {
- kgPipeline := graph.NewPipeline(engine.Get(), kgIDs, kbTenantIDStrings(kbs), searchQuestion)
- kgPipeline.SetChatModel(chatModel)
- if embModel != nil {
- kgPipeline.SetEmbModel(embModel)
- }
- kgChunk, kgErr := kgPipeline.Retrieval(ctx)
- if kgErr != nil {
- common.Warn("KG retrieval failed; falling through to vector-only",
- zap.Error(kgErr))
- } else if kgChunk != nil {
- if _, hasContent := kgChunk["content_with_weight"]; hasContent {
- if existingChunks, ok := kbinfos["chunks"].([]map[string]interface{}); ok {
- newChunks := make([]map[string]interface{}, 0, len(existingChunks)+1)
- newChunks = append(newChunks, kgChunk)
- newChunks = append(newChunks, existingChunks...)
- kbinfos["chunks"] = newChunks
- common.Debug("KG chunk prepended",
- zap.Int("total_chunks", len(newChunks)))
- }
+ // Knowledge Graph retrieval
+ if useKG, _ := chat.PromptConfig["use_kg"].(bool); useKG && chatModel != nil && len(kbs) > 0 {
+ kgIDs := kbIDStrings(kbs)
+ if len(kgIDs) > 0 {
+ kgPipeline := graph.NewPipeline(engine.Get(), kgIDs, kbTenantIDStrings(kbs), searchQuestion)
+ kgPipeline.SetChatModel(chatModel)
+ if embModel != nil {
+ kgPipeline.SetEmbModel(embModel)
+ }
+ kgChunk, kgErr := kgPipeline.Retrieval(ctx)
+ if kgErr != nil {
+ common.Warn("KG retrieval failed; falling through to vector-only",
+ zap.Error(kgErr))
+ } else if kgChunk != nil {
+ if _, hasContent := kgChunk["content_with_weight"]; hasContent {
+ if existingChunks, ok := kbinfos["chunks"].([]map[string]interface{}); ok {
+ newChunks := make([]map[string]interface{}, 0, len(existingChunks)+1)
+ newChunks = append(newChunks, kgChunk)
+ newChunks = append(newChunks, existingChunks...)
+ kbinfos["chunks"] = newChunks
+ common.Debug("KG chunk prepended",
+ zap.Int("total_chunks", len(newChunks)))
}
}
}
@@ -2036,39 +1850,6 @@ func (s *ChatPipelineService) shouldUseWebSearch(chat *entity.Chat, internet int
return normalized != nil && *normalized
}
-// harnessWebSearchFunc builds the callback the agentic harness' web_search tool
-// invokes (Python RAGTools(web_search=...)). It resolves the provider once and
-// runs one provider query per requested query, flattening each result's chunk
-// text. A per-query failure is logged and skipped — mirroring Python
-// _exec_web_search, where one bad query must not fail the whole tool call.
-// Returns nil when no provider is configured so the tool stays hidden.
-func (s *ChatPipelineService) harnessWebSearchFunc(promptConfig map[string]interface{}) func(context.Context, []string) ([]string, error) {
- provider := resolveWebSearchProvider(promptConfig)
- if provider == nil {
- return nil
- }
- return func(ctx context.Context, queries []string) ([]string, error) {
- out := make([]string, 0, len(queries)*6)
- for _, q := range queries {
- if strings.TrimSpace(q) == "" {
- continue
- }
- res, err := s.retrieveWebSearch(ctx, provider, q)
- if err != nil {
- common.Warn("harness web search failed for a query", zap.Error(err))
- continue
- }
- chunks, _ := res["chunks"].([]map[string]interface{})
- for _, c := range chunks {
- if text := getMapString(c, "content_with_weight", "content"); text != "" {
- out = append(out, text)
- }
- }
- }
- return out, nil
- }
-}
-
// tavilyRetrieve calls the Tavily API and returns results in the same chunk
// format used by performRetrieval. Mirrors Python's Tavily.retrieve_chunks()
// in rag/utils/tavily_conn.py.
@@ -2201,36 +1982,6 @@ func (s *ChatPipelineService) getLLMModelConfig(ctx context.Context, chat *entit
return cfg, modelName, factoryName, baseURL, nil
}
-// reasoningNeedsAgenticGraph gates the agentic path on reasoning being enabled
-// and the resolved chat model supporting tool calls.
-func reasoningNeedsAgenticGraph(chat *entity.Chat, cfg map[string]interface{}, reasoningLevel int) bool {
- if chat == nil || reasoningLevel <= 0 {
- return false
- }
- if chatConfigSupportsTools(cfg) {
- return true
- }
- common.Info("LLM does not support tool calls; falling back to regular RAG chat",
- zap.Int("reasoning_level", reasoningLevel))
- return false
-}
-
-func chatConfigSupportsTools(cfg map[string]interface{}) bool {
- if cfg == nil {
- return false
- }
- switch v := cfg["is_tools"].(type) {
- case bool:
- return v
- case string:
- return strings.EqualFold(strings.TrimSpace(v), "true")
- case float64:
- return v != 0
- default:
- return false
- }
-}
-
// buildLLMModelConfig returns only the public metadata needed by RAG's token
// fitting and attachment dispatch. The persisted legacy max_tokens value is
// the input/context budget; generated output limits are enforced by ChatModel
@@ -3349,284 +3100,6 @@ func (s *ChatPipelineService) decorateAnswer(
}
}
-// chunksOnlyReference is the reference of an answer whose citations were
-// disabled: the retrieved evidence, in the client-facing shape, without
-// doc_aggs. The evidence still records what the answer was grounded on; only
-// the source cards go, since there is no citation marker left in the answer to
-// open them from.
-func chunksOnlyReference(kbinfos map[string]interface{}) map[string]interface{} {
- refs := map[string]interface{}{}
- if chunksRaw, ok := kbinfos["chunks"].([]map[string]interface{}); ok {
- refs["chunks"] = chunksFormat(referenceChunks(nil, chunksRaw))
- }
- return refs
-}
-
-// decorateHarnessAnswer formats the reasoning chat's final answer. The harness
-// 'rag' tool already composed the answer WITH its own [ID:N] citations, so —
-// unlike decorateAnswer for the native async_chat path — we never run
-// insert_citations here. We only resolve the existing markers, repair bad
-// formats, filter doc_aggs to the cited docs, and build the reference from the
-// harness citation pool (chunks stripped of their vectors). Prompt stays empty,
-// matching Python.
-//
-// The answer's [ID:n] markers index the evidence list the compose rendered for
-// the model — harness.Kbinfos.CiteChunkIDs, passed in as citeChunkIDs. That list
-// is similarity-ranked and capped, and it is numbered 0-based: the client resolves
-// a marker by using its number as an index into reference.chunks, and the text it
-// already streamed carries the model's own numbers (this function repairs and drops
-// markers, and rewrites the slot-table ones, but never renumbers a citation). The
-// reference is therefore built with that list in front, in the rendered order, so
-// marker n is the n-th rendered passage — and the rest of the pool follows so no
-// source is lost. Canonical "[ID:n]" markers that name nothing are dropped (a bare
-// "[5]" is left alone: in prose it may be a footnote or a year). The reference is
-// returned whenever the pool is non-empty, so an answer the model composed without
-// markers still carries its sources.
-//
-// When quote is false the citations are removed entirely and the reference is
-// cleared (see stripCitations).
-func (s *ChatPipelineService) decorateHarnessAnswer(answer string, kbinfos map[string]interface{}, slotCitations map[string][]string, citeChunkIDs []string, quote bool) AsyncChatResult {
- think := ""
- ans := answer
- if strings.Contains(answer, "</think>") {
- if parts := strings.Split(answer, "</think>"); len(parts) == 2 {
- think = parts[0] + "</think>"
- ans = strings.TrimSpace(parts[1])
- }
- }
-
- // Invalid-key hint (dialog_service.py:2134-2135).
- if strings.Contains(strings.ToLower(ans), "invalid key") ||
- strings.Contains(strings.ToLower(ans), "invalid api") {
- ans += " Please set LLM API-Key in 'User Setting -> Model providers -> API-Key'"
- }
-
- if !quote {
- return AsyncChatResult{
- Answer: stripCitations(think + ans),
- Reference: chunksOnlyReference(kbinfos),
- CreatedAt: float64(time.Now().Unix()),
- }
- }
-
- chunksRaw, _ := kbinfos["chunks"].([]map[string]interface{})
- // The markers exactly as the model wrote them (before any repair), for the
- // citation observability log below.
- rawMarkers := rawCitationMarkers(think+ans, 8)
-
- // The evidence list the model was shown, as pool positions: citeIdx[i] is the
- // pool chunk behind the i-th rendered block. Empty ids fall back to pool
- // order, which is the pre-CiteChunkIDs behavior.
- citeIdx := citePoolIdx(chunksRaw, citeChunkIDs)
- citeChunks := citeChunksByIdx(citeIdx, chunksRaw)
-
- // Slot-table markers ("[ID:Slot 0]") are internal references the compose
- // model leaks from the Research Summary's slot draft — they index the slot
- // table, not any chunk the user can open. Rewrite them into citations of
- // the chunk that filled the slot (or drop them) BEFORE the marker scan, so
- // they resolve like any other citation. The rewrite emits the evidence
- // block's 0-based index in the rendered list, the numbering the client
- // indexes.
- ans = RepairSlotCitations(ans, slotCitations, citeChunks)
-
- // Range-merged citations ("[ID:1-3]") are the model compressing
- // consecutive individual citations on its own; expand them back so every
- // marker resolves to exactly one chunk (out-of-range ranges are dropped).
- // Bounds are 0-based indexes in the rendered list.
- ans = ExpandRangeCitations(ans, len(citeChunks))
-
- // repair_bad_citation_formats (dialog_service.py:2109), then resolve the
- // markers: each is the 0-based index of the rendered block it cites.
- // Canonical markers that name no block are dropped; the resolved pool
- // positions drive doc_aggs filtering.
- think = RepairBadCitationFormats(think)
- ans = RepairBadCitationFormats(ans)
- think, thinkCitedIdx := ResolveCitationMarkers(think, citeIdx)
- ans, answerCitedIdx := ResolveCitationMarkers(ans, citeIdx)
- citedIdx := append(thinkCitedIdx, answerCitedIdx...)
-
- // Citation observability: the markers the model wrote against the blocks it was
- // shown, so a numbering mismatch is visible in the log (`raw_markers` is what
- // the model actually emitted, before any repair). No judgement is attached —
- // see zeroBasedEvidenceRule for the prompt-side guard on the numbering.
- if len(citeIdx) > 0 {
- common.Info("agentic citation markers",
- zap.Int("rendered_blocks", len(citeIdx)),
- zap.Ints("raw_markers", rawMarkers),
- zap.Int("resolved_blocks", len(citedIdx)),
- // Which figure points at which chunk, and whether that chunk can show a
- // picture at all: "why is there no image under Fig. n" is asked far more
- // often than "the numbering was off", and from the page the two look the
- // same. See citationAudit.
- zap.String("cited", citationAudit(citeIdx, citedIdx, chunksRaw)),
- )
- }
-
- // Map cited chunks to doc_ids (dialog_service.py:2111-2122).
- citedDocIDs := make(map[string]struct{})
- for _, ci := range citedIdx {
- if ci >= 0 && ci < len(chunksRaw) {
- if docID, ok := chunksRaw[ci]["doc_id"].(string); ok && docID != "" {
- citedDocIDs[docID] = struct{}{}
- }
- }
- }
-
- // Keep only documents referenced by a citation. An uncited evidence pool is
- // not a document reference and must not be returned to the client.
- if len(citedDocIDs) > 0 {
- if docAggsRaw, ok := kbinfos["doc_aggs"].([]interface{}); ok {
- filtered := make([]interface{}, 0, len(docAggsRaw))
- for _, da := range docAggsRaw {
- if dam, ok := da.(map[string]interface{}); ok {
- if docID, ok := dam["doc_id"].(string); ok {
- if _, cited := citedDocIDs[docID]; cited {
- filtered = append(filtered, da)
- }
- }
- }
- }
- kbinfos["doc_aggs"] = filtered
- }
- }
-
- // The rendered evidence list comes first, in the order the model saw it, so a
- // marker's number is that entry's index; the remaining pool chunks follow so
- // no retrieved passage is lost. chunksFormat builds the client-facing shape
- // (content, document_name, dataset_id, ...) in NEW maps, so the engine keys
- // and the per-chunk vector never reach the reference and the shared chunks
- // stay intact. The evidence ships even when the final answer cited none of
- // it; doc_aggs, absent here, is what stays gated on `quote`.
- refs := make(map[string]interface{}, len(kbinfos))
- for k, v := range kbinfos {
- refs[k] = v
- }
- refs["chunks"] = chunksFormat(referenceChunks(citeIdx, chunksRaw))
-
- return AsyncChatResult{
- Answer: think + ans,
- Reference: refs,
- Prompt: "",
- CreatedAt: float64(time.Now().Unix()),
- }
-}
-
-// citePoolIdx maps the ordered evidence list the compose rendered for the model
-// (harness.Kbinfos.CiteChunkIDs) onto the pool: entry i is the index in pool of
-// the i-th rendered block. A block whose id is empty or unknown stays in the list
-// as -1 so the blocks after it keep the number the model saw; its citations simply
-// cannot be resolved. An empty id list — a harness that published none — falls
-// back to pool order, where block i is pool[i].
-func citePoolIdx(pool []map[string]interface{}, ids []string) []int {
- if len(ids) == 0 {
- out := make([]int, len(pool))
- for i := range pool {
- out[i] = i
- }
- return out
- }
- byID := make(map[string]int, len(pool))
- for i, c := range pool {
- if id := chunkCitationID(c); id != "" {
- if _, dup := byID[id]; !dup {
- byID[id] = i
- }
- }
- }
- out := make([]int, len(ids))
- for i, id := range ids {
- idx, found := byID[id]
- if id == "" || !found {
- idx = -1
- }
- out[i] = idx
- }
- return out
-}
-
-// citeChunksByIdx gathers the rendered evidence list from its pool positions,
-// keeping -1 entries as nil so positions stay aligned with the numbering the model
-// was shown (a nil chunk matches no slot-citation evidence id and resolves no
-// marker, but it must not shift the blocks after it).
-func citeChunksByIdx(citeIdx []int, pool []map[string]interface{}) []map[string]interface{} {
- out := make([]map[string]interface{}, len(citeIdx))
- for i, pi := range citeIdx {
- if pi >= 0 && pi < len(pool) {
- out[i] = pool[pi]
- }
- }
- return out
-}
-
-// referenceChunks orders the reference payload: the evidence list the compose
-// rendered first, in exactly that order, then the rest of the pool. The client
-// resolves a citation marker by indexing this list with the number the model
-// wrote, so the rendered order has to come first — and one entry per RENDERED
-// block, at the block's own index. A block whose chunk cannot be located in the
-// pool (or a repeat of one already placed) still occupies its slot, as a nil
-// chunk: dropping it would slide every later marker one position down, onto a
-// passage the model never cited. Such a block's marker is dropped from the answer
-// upstream, so the slot is a gap in the sources panel, not a broken citation.
-//
-// The trailing chunks keep every retrieved passage reachable from the sources
-// panel.
-func referenceChunks(citeIdx []int, pool []map[string]interface{}) []map[string]interface{} {
- if len(citeIdx) == 0 {
- return pool
- }
- placed := make(map[int]struct{}, len(citeIdx))
- out := make([]map[string]interface{}, 0, len(pool)+len(citeIdx))
- for _, pi := range citeIdx {
- if pi < 0 || pi >= len(pool) {
- out = append(out, nil)
- continue
- }
- if _, dup := placed[pi]; dup {
- out = append(out, nil)
- continue
- }
- placed[pi] = struct{}{}
- out = append(out, pool[pi])
- }
- for i, c := range pool {
- if _, dup := placed[i]; dup {
- continue
- }
- out = append(out, c)
- }
- return out
-}
-
-// citationAudit renders the resolved citations for the observability log: for every
-// cited pool chunk, the figure number(s) that point at it and whether the chunk
-// carries a picture.
-//
-// Fig. n is the client's label for rendered block n-1 (the marker's number plus
-// one), and the client draws a picture under a figure only when that chunk has an
-// image_id. So "Fig. n shows no image" has two very different explanations — the
-// marker resolved to a text-only chunk (expected, nothing to show) or the chunk
-// behind that figure is not the one the model cited (a resolution bug) — and they
-// look identical on the page. This line tells them apart without a re-run.
-func citationAudit(citeIdx, citedIdx []int, chunksRaw []map[string]interface{}) string {
- cited := make(map[int]struct{}, len(citedIdx))
- for _, ci := range citedIdx {
- cited[ci] = struct{}{}
- }
- parts := make([]string, 0, len(citedIdx))
- for block, ci := range citeIdx {
- if _, ok := cited[ci]; !ok {
- continue
- }
- if ci < 0 || ci >= len(chunksRaw) {
- continue
- }
- imageID, _ := getChunkValue(chunksRaw[ci], "image_id", "img_id").(string)
- parts = append(parts, fmt.Sprintf("Fig.%d=%s image=%t",
- block+1, chunkCitationID(chunksRaw[ci]), imageID != ""))
- }
- return strings.Join(parts, " ")
-}
-
// langfuseExtractTimeElapsed extracts the time-elapsed + token-usage
// block from the prompt and applies the \n → " \n" substitution.
// Mirrors dialog_service.py:853-854:
@@ -5078,219 +4551,6 @@ func getChunkValue(chunk map[string]interface{}, k1, k2 string) interface{} {
return chunk[k2]
}
-// harnessBoundDatasetNames renders the {knowledge} default for the agentic
-// (reasoning) system prompt: the comma-joined names of the chat's bound
-// datasets. Python parity: dialog_service._render_reasoning_system_prompt /
-// _bound_dataset_names. The agentic graph supplies the retrieved evidence
-// itself, but the prompt must still NAME the bound datasets — defaulting the
-// placeholder to "" left the web UI's default template rendering as
-// "derived solely from this dataset: “", which the outer model read as an
-// empty dataset and answered the canned "not found in the dataset!" line
-// without ever calling the terminal `rag` tool.
-func harnessBoundDatasetNames(kbs []*entity.Knowledgebase) string {
- names := make([]string, 0, len(kbs))
- for _, kb := range kbs {
- if kb != nil && kb.Name != "" {
- names = append(names, kb.Name)
- }
- }
- return strings.Join(names, ", ")
-}
-
-// HarnessRetrieval is the dialog-level retrieval tuning the chat pipeline hands
-// to the agentic harness. Without it the harness falls back to its package
-// defaults (top_n=12), silently overriding the dialog's configured top_n and
-// making a reasoning run retrieve a different passage set than the standard
-// path — a two-document comparison could come back with one document's chunks
-// filling the whole budget.
-type HarnessRetrieval struct {
- TopN int
- SimilarityThreshold float64
- VectorSimilarityWeight float64
- RerankCandidatesCount int
-}
-
-// HarnessRequest carries the minimal inputs the chat pipeline hands to the
-// agentic-RAG harness for evidence collection.
-type HarnessRequest struct {
- Question string
- DatasetIDs []string
- Messages []map[string]interface{}
- // DocIDs scopes the agentic search to these document ids (Python
- // dialog_service.py doc_scope). chat_pipeline folds the chat-level
- // meta_data_filter into docIDs before calling, so forwarding this alone
- // restores both doc_ids and meta_data_filter scoping without double-filtering.
- DocIDs []string
- ThinkingMode string
- TenantID string
- // ModelID is the tenant-scoped chat model id (dialog llm_id). The harness
- // driver uses it as the default model name for agentic LLM turns.
- ModelID string
- // SessionID scopes cross-turn state such as the near-duplicate answer cache
- // (Python RAGTools._rag_cache lives on an instance spanning the dialog).
- SessionID string
- // AnswerSink receives the answer as the model produces it (Python
- // tools.answer_sink), so the caller can stream instead of waiting for the
- // whole answer. Nil disables streaming; the full answer is still returned.
- AnswerSink func(delta string, isThink bool)
- // ThinkSink receives the structured form of each reasoning step (the twin
- // of what AnswerSink narrates as think text). Nil disables it; the text
- // narration is unaffected.
- ThinkSink func(ThinkEvent)
- // Images are vision-gated base64 data URIs (Python image_attachments) that
- // survive gateImageAttachments. They ride the last user message into the
- // outer react loop so a vision model can see them. chat_pipeline drops them
- // for text-only models, so only surviving URIs reach here.
- Images []string
- // TextAttachments is the joined text-file content (Python
- // text_attachments_content); it is appended to the last user message so the
- // model reads attached documents even in the reasoning path.
- TextAttachments string
- // WebSearch runs an open-web search for the harness' web_search tool
- // (Python RAGTools(web_search=create_web_search_provider(...))). Nil hides
- // the tool from the agentic surface, mirroring Python's provider gate
- // (action_session.py:463 discards web_search when tools.web_search is None).
- WebSearch func(ctx context.Context, queries []string) ([]string, error)
- // SystemPrompt is the dialog-level system prompt rendered the way Python's
- // _render_reasoning_system_prompt (dialog_service.py:1887-1917) renders it
- // for RAGTools(system_prompt=...): caller kwargs + a UTC date, with
- // {knowledge} defaulted to "" (the agentic graph supplies evidence through
- // its own evidence block). Empty when the dialog configures none — Python
- // then composes without the "# Assistant configuration" block.
- SystemPrompt string
- // Retrieval is the dialog's own retrieval tuning, so the harness searches
- // with the same budget as the standard path.
- Retrieval HarnessRetrieval
-}
-
-// HarnessResult is the evidence the harness returns, normalized to the map
-// shape chat_pipeline's downstream phases already consume.
-type HarnessResult struct {
- Chunks []map[string]any
- DocAggs []map[string]any
- Memory []map[string]any
- PreSummary string
- // Answer is the harness's own composed final cited answer (RunResponse.Answer),
- // populated when a model was available. When non-empty, the chat pipeline uses
- // it directly as the reply instead of re-generating via a second model pass —
- // mirroring Python's terminal `rag` tool.
- Answer string
- // SlotCitations maps a slot-table id ("0", ...) to the evidence chunk ids
- // that filled it (RunResponse.SlotCitations). The citation decoration uses
- // it to rewrite leaked "[ID:Slot N]" markers into locatable chunk citations.
- SlotCitations map[string][]string
- // CiteChunkIDs is the ordered evidence list the compose rendered for the
- // model (RunResponse.CiteChunkIDs): similarity-ranked and capped, so it is
- // NOT Chunks in pool order. The answer's "[ID:n]" markers index THIS list —
- // resolving them against Chunks by position is what lost citations.
- CiteChunkIDs []string
-}
-
-// harnessRetriever is installed at server bootstrap using
-// retrievalbridge.NewHarnessRetriever. The bridge imports internal/service,
-// so this package receives the callback instead of importing the bridge.
-// When nil, retrieveViaHarness reports an error and the pipeline continues
-// with empty kbinfos.
-var harnessRetriever func(ctx context.Context, req HarnessRequest) (HarnessResult, error)
-
-// SetHarnessRetriever injects the agentic-RAG harness driver. Call once at
-// server bootstrap.
-func SetHarnessRetriever(fn func(ctx context.Context, req HarnessRequest) (HarnessResult, error)) {
- harnessRetriever = fn
-}
-
-// retrieveViaHarness collects evidence chunks for the chat question by driving
-// the agentic-RAG harness, mirroring Python's dialog_service → RAGTools →
-// harness/* path. It returns the same map shape the downstream phases
-// (TOC/child-chunk/web/KG/enrich/kbPrompt) already consume, so the rest of the
-// chat pipeline is unaffected by where the chunks came from.
-//
-// thinkingMode is "naive" (reasoning disabled) or one of the agentic levels
-// ("low"/"medium"/"high"/"ultra", reasoning enabled).
-//
-// answerSink and thinkSink are the request's two narration sinks, both optional:
-// answerSink receives the streamed answer plus think TEXT, thinkSink the structured
-// step events. They are parameters rather than something built in here because only
-// the streaming caller owns the channel they write to.
-//
-// The reasoning NARRATION ("[Tool loop] Deciding what to do next ...", the tool
-// calls and their results, the research stages) is not produced here: the
-// harness reports each step from where it happens, over the sinks this request
-// carries (AnswerSink for the think text, ThinkSink for the structured form).
-// The pipeline used to synthesize the "[Tool loop]" lines around its single
-// harness call, which made every non-naive run read as if an outer tool loop had
-// run — including the ones the direct graph answered.
-//
-// harnessThinkSink forwards the harness' STRUCTURED reasoning steps (the twin of
-// the think text) onto the pipeline's own channel, so the client can render
-// steps — tool, status, reason, counts, sources, duration — instead of
-// paragraphs.
-//
-// Unlike think TEXT, a late event is not dropped after the answer: text would
-// reopen a stray "Thought" block, while an event is a record a client may still
-// want. No mutex is needed either: this path touches neither the think-framing
-// state machine nor anything but the channel, which is safe for the concurrent
-// research goroutines that emit most of the events.
-func harnessThinkSink(ctx context.Context, out chan<- AsyncChatResult) func(ThinkEvent) {
- return func(ev ThinkEvent) {
- e := ev
- select {
- case out <- AsyncChatResult{
- ThinkEvent: &e,
- Reference: map[string]interface{}{},
- CreatedAt: float64(time.Now().Unix()),
- Final: false,
- }:
- case <-ctx.Done():
- }
- }
-}
-
-func (s *ChatPipelineService) retrieveViaHarness(ctx context.Context, question string, kbs []*entity.Knowledgebase, docIDs []string, images []string, textAttachments string, thinkingMode, tenantID, modelID, sessionID string, retrieval HarnessRetrieval, webSearch func(context.Context, []string) ([]string, error), answerSink func(delta string, isThink bool), thinkSink func(ThinkEvent), dialogSystemPrompt string, messages []map[string]interface{}) (map[string]interface{}, map[string][]string, []string, string, error) {
- if harnessRetriever == nil {
- return nil, nil, nil, "", fmt.Errorf("harness retriever not wired at bootstrap")
- }
- kbIDs := kbIDStrings(kbs)
- // The "[Tool loop]" narration is NOT synthesized here. The outer react loop
- // reports its own steps from where it runs (advanced_rag.runOuterReact*), so
- // the trace describes a loop only when one actually ran — the pipeline used
- // to emit those lines for every non-naive run, including the ones the direct
- // graph answered, which read as a loop that had not happened.
- res, err := harnessRetriever(ctx, HarnessRequest{
- Question: question,
- Messages: messages,
- DatasetIDs: kbIDs,
- DocIDs: docIDs,
- ThinkingMode: thinkingMode,
- TenantID: tenantID,
- ModelID: modelID,
- SessionID: sessionID,
- AnswerSink: answerSink,
- ThinkSink: thinkSink,
- Images: images,
- TextAttachments: textAttachments,
- WebSearch: webSearch,
- SystemPrompt: dialogSystemPrompt,
- Retrieval: retrieval,
- })
- if err != nil {
- return nil, nil, nil, "", err
- }
-
- kbinfos := map[string]interface{}{
- "total": len(res.Chunks),
- "chunks": res.Chunks,
- "doc_aggs": toAnySlice(res.DocAggs),
- }
- if res.Memory != nil {
- kbinfos["memory"] = res.Memory
- }
- if res.PreSummary != "" {
- kbinfos["pre_summary"] = res.PreSummary
- }
- return kbinfos, res.SlotCitations, res.CiteChunkIDs, res.Answer, nil
-}
-
// toAnySlice widens a []map[string]any to []interface{} so the doc_aggs field
// keeps the element type the downstream phases expect from the native pipeline.
func toAnySlice(in []map[string]any) []interface{} {
@@ -5351,17 +4611,15 @@ func quoteEnabled(kwargs map[string]interface{}, promptConfig map[string]interfa
// smart-reasoning agent (internal/agentic_rag) instead of running the pipeline's
// own retrieval phases.
//
-// It is deliberately NOT one of harnessModeForLevel's levels. 1..4 pick a depth
-// within the harness graph; this level picks a different engine outright, so
-// harnessModeForLevel must never see it — its `level >= 4` case would silently
-// answer "ultra" for a level that is no longer part of that domain.
+// It is the only level that means anything beyond "retrieve normally": every
+// other value — including a stale 1..4 from a client still sending one of the
+// removed thinking modes — runs the standard retrieval path below.
const reasoningLevelAgentic = 5
-// resolveReasoningLevel mirrors Python's rag_agent: the requesting reasoning
-// level is taken from the request kwargs first, falling back to the chat
-// prompt_config, and is an integer 0..4 (0 = off, 1..4 = low/medium/high/
-// ultra), or reasoningLevelAgentic for the smart-reasoning agent. Frontend
-// sends Number(getThinkingLevel()), so the raw value is numeric, not a bool.
+// resolveReasoningLevel reads the requesting reasoning level from the request
+// kwargs first, falling back to the chat prompt_config. Only
+// reasoningLevelAgentic selects the smart-reasoning agent. Frontend sends
+// Number(getThinkingLevel()), so the raw value is numeric, not a bool.
func resolveReasoningLevel(kwargs map[string]interface{}, promptConfig map[string]interface{}) int {
if kwargs != nil {
if v, ok := kwargs["reasoning"]; ok {
@@ -5379,25 +4637,3 @@ func resolveReasoningLevel(kwargs map[string]interface{}, promptConfig map[strin
}
return 0
}
-
-// harnessModeForLevel maps a Python-style reasoning level to the harness
-// thinking mode. Python uses THINKING_MODES = [low, medium, high, ultra] and
-// falls back to "medium" when n is out of range.
-//
-// Its domain is levels 1..4 only. reasoningLevelAgentic never arrives here — the
-// AsyncChat dispatch hands that turn to the smart-reasoning agent before Phase 9
-// runs — so the `level >= 4` case below does not need to exclude it.
-func harnessModeForLevel(level int) string {
- switch {
- case level >= 4:
- return "ultra"
- case level == 3:
- return "high"
- case level == 2:
- return "medium"
- case level == 1:
- return "low"
- default:
- return "medium"
- }
-}
diff --git a/internal/service/chat_pipeline_test.go b/internal/service/chat_pipeline_test.go
index d0c7ec568f..74fad9c6b0 100644
--- a/internal/service/chat_pipeline_test.go
+++ b/internal/service/chat_pipeline_test.go
@@ -1672,47 +1672,6 @@ func TestClampChatConfigMaxTokensRejectsExhaustedCapacity(t *testing.T) {
}
}
-// stubHarness installs a fake harnessRetriever returning the given answer and
-// restores the previous one when the test ends.
-func stubHarness(t *testing.T, answer string) {
- t.Helper()
- prev := harnessRetriever
- t.Cleanup(func() { harnessRetriever = prev })
- harnessRetriever = func(ctx context.Context, req HarnessRequest) (HarnessResult, error) {
- return HarnessResult{Answer: answer}, nil
- }
-}
-
-// TestRetrieveViaHarnessForwardsRetrievalTuning pins the fix for a harness that
-// ignored the dialog's retrieval settings: without them it falls back to its
-// package default top_n=12, so a dialog configured with top_n=8 retrieved a
-// different passage set than the standard path — the failure that answered a
-// two-document comparison from one document's chunks because that document's
-// nine chunks filled the widened budget.
-func TestRetrieveViaHarnessForwardsRetrievalTuning(t *testing.T) {
- prev := harnessRetriever
- t.Cleanup(func() { harnessRetriever = prev })
- var got HarnessRequest
- harnessRetriever = func(_ context.Context, req HarnessRequest) (HarnessResult, error) {
- got = req
- return HarnessResult{}, nil
- }
-
- s := &ChatPipelineService{}
- want := HarnessRetrieval{
- TopN: 8,
- SimilarityThreshold: 0.2,
- VectorSimilarityWeight: 0.3,
- RerankCandidatesCount: 64,
- }
- if _, _, _, _, err := s.retrieveViaHarness(t.Context(), "q", nil, nil, nil, "", "low", "t", "m", "sess", want, nil, nil, nil, "", nil); err != nil {
- t.Fatalf("retrieveViaHarness: %v", err)
- }
- if got.Retrieval != want {
- t.Errorf("harness retrieval = %+v, want %+v", got.Retrieval, want)
- }
-}
-
// collectSink records every delta with its isThink flag.
func collectSink(got *[]string, thinks *[]bool) func(string, bool) {
return func(delta string, isThink bool) {
@@ -1721,354 +1680,6 @@ func collectSink(got *[]string, thinks *[]bool) func(string, bool) {
}
}
-// collectThinkSink is collectSink's structured twin: the harness' step events in
-// order, so a test can assert what reached the request's ThinkSink.
-func collectThinkSink(events *[]ThinkEvent) func(ThinkEvent) {
- return func(ev ThinkEvent) { *events = append(*events, ev) }
-}
-
-// TestRetrieveViaHarnessDoesNotSynthesizeLoopLines pins the fix for a trace that
-// described a loop which had not run: the pipeline used to emit
-// "[Tool loop] Deciding what to do next ..." / "Step 1: running rag..." around
-// EVERY non-naive harness call, whether or not the outer react loop was the
-// thing driving research. Those steps are now reported by the loop itself
-// (advanced_rag.runOuterReact*), so a harness stub — which by definition runs no
-// outer loop — must produce no narration at all.
-func TestRetrieveViaHarnessDoesNotSynthesizeLoopLines(t *testing.T) {
- stubHarness(t, "the final cited answer")
- var receivedHistory []map[string]interface{}
- retriever := harnessRetriever
- harnessRetriever = func(ctx context.Context, req HarnessRequest) (HarnessResult, error) {
- receivedHistory = req.Messages
- return retriever(ctx, req)
- }
- history := []map[string]interface{}{{"role": "user", "content": "earlier question"}, {"role": "assistant", "content": "earlier answer"}, {"role": "user", "content": "q"}}
-
- var got []string
- var thinks []bool
- var events []ThinkEvent
- s := &ChatPipelineService{}
- _, _, _, answer, err := s.retrieveViaHarness(t.Context(), "q", nil, nil, nil, "", "high", "t", "m", "sess", HarnessRetrieval{}, nil, collectSink(&got, &thinks), collectThinkSink(&events), "", history)
- if err != nil {
- t.Fatalf("retrieveViaHarness: %v", err)
- }
- if !reflect.DeepEqual(receivedHistory, history) {
- t.Fatalf("harness history = %#v, want %#v", receivedHistory, history)
- }
- if answer != "the final cited answer" {
- t.Fatalf("answer = %q", answer)
- }
- if len(got) != 0 {
- t.Errorf("the pipeline synthesized %d think delta(s), want none: %#v", len(got), got)
- }
- if len(events) != 0 {
- t.Errorf("the pipeline synthesized %d step event(s), want none: %#v", len(events), events)
- }
-}
-
-// TestRetrieveViaHarnessForwardsThinkSink pins the structured-step channel: the
-// sink the pipeline is given reaches the harness request, and the events the
-// harness emits arrive with their fields intact.
-func TestRetrieveViaHarnessForwardsThinkSink(t *testing.T) {
- prev := harnessRetriever
- t.Cleanup(func() { harnessRetriever = prev })
- harnessRetriever = func(ctx context.Context, req HarnessRequest) (HarnessResult, error) {
- if req.ThinkSink == nil {
- return HarnessResult{}, fmt.Errorf("harness request carries no ThinkSink")
- }
- req.ThinkSink(ThinkEvent{
- Kind: "tool_result", Stage: "Function tool", Tool: "retrieve",
- Status: "ok", Reason: "no_doc", Cause: "index not found", Results: 3, Documents: 2,
- Sources: []string{"c1", "c2"}, DurationMS: 12,
- Summary: "[Function tool] The retrieve tool returned 3 results.",
- })
- return HarnessResult{Answer: "a"}, nil
- }
-
- // The text sink is beside the point here (this test is about the structured
- // channel) and the local `got` below is the EVENT, so leave answerSink nil.
- var events []ThinkEvent
- s := &ChatPipelineService{}
- if _, _, _, _, err := s.retrieveViaHarness(t.Context(), "q", nil, nil, nil, "", "high", "t", "m", "sess", HarnessRetrieval{}, nil, nil, collectThinkSink(&events), "", nil); err != nil {
- t.Fatalf("retrieveViaHarness: %v", err)
- }
- // Only what the harness reported: the pipeline adds nothing of its own.
- var got *ThinkEvent
- for i := range events {
- if events[i].Kind == "tool_result" {
- got = &events[i]
- break
- }
- }
- if got == nil {
- t.Fatalf("no tool_result among %#v", events)
- }
- if got.Tool != "retrieve" || got.Status != "ok" || got.Cause != "index not found" ||
- got.Results != 3 || got.Documents != 2 || got.DurationMS != 12 ||
- len(got.Sources) != 2 || got.Summary == "" {
- t.Errorf("event lost fields on the way through: %#v", *got)
- }
- if len(events) != 1 {
- t.Errorf("events = %#v, want only what the harness reported", events)
- }
-}
-
-// TestHarnessThinkSinkForwardsEvent covers the pipeline-side half: one event in,
-// one chunk out, carrying the event and no answer delta.
-func TestHarnessThinkSinkForwardsEvent(t *testing.T) {
- out := make(chan AsyncChatResult, 1)
- harnessThinkSink(context.Background(), out)(ThinkEvent{Kind: "stage", Summary: "[Planner] x"})
-
- select {
- case chunk := <-out:
- if chunk.ThinkEvent == nil || chunk.ThinkEvent.Kind != "stage" || chunk.ThinkEvent.Summary != "[Planner] x" {
- t.Fatalf("chunk = %#v, want the event", chunk)
- }
- if chunk.Answer != "" || chunk.Final {
- t.Errorf("an event chunk must carry no answer and not be final: %#v", chunk)
- }
- default:
- t.Fatal("event was not forwarded")
- }
-
- // A cancelled request must not block the producer.
- ctx, cancel := context.WithCancel(context.Background())
- cancel()
- harnessThinkSink(ctx, make(chan AsyncChatResult))(ThinkEvent{Summary: "y"})
-}
-
-// TestRetrieveViaHarnessNaiveEmitsNothing guards the naive path: no agentic loop
-// runs, so nothing is narrated — and with the pipeline no longer synthesizing
-// its own lines, nothing is emitted at all.
-func TestRetrieveViaHarnessNaiveEmitsNothing(t *testing.T) {
- stubHarness(t, "x")
-
- var got []string
- var thinks []bool
- s := &ChatPipelineService{}
- if _, _, _, _, err := s.retrieveViaHarness(t.Context(), "q", nil, nil, nil, "", "naive", "t", "m", "sess", HarnessRetrieval{}, nil, collectSink(&got, &thinks), nil, "", nil); err != nil {
- t.Fatalf("retrieveViaHarness: %v", err)
- }
- if len(got) != 0 {
- t.Errorf("naive mode emitted %d deltas, want 0: %#v", len(got), got)
- }
-}
-
-// TestDecorateHarnessAnswerReferenceUsesClientChunkShape pins that the reasoning
-// path hands consumers the same reference shape as the non-reasoning one
-// (content / document_name / dataset_id — not the engine keys), and that the
-// source chunks are left untouched (no in-place vector stripping).
-func TestDecorateHarnessAnswerReferenceUsesClientChunkShape(t *testing.T) {
- src := map[string]interface{}{
- "chunk_id": "c1",
- "content_with_weight": "hello",
- "docnm_kwd": "Doc One",
- "doc_id": "d1",
- "kb_id": "kb1",
- "vector": []float64{0.1, 0.2},
- }
- kbinfos := map[string]interface{}{
- "chunks": []map[string]interface{}{src},
- "doc_aggs": []interface{}{map[string]interface{}{"doc_id": "d1", "doc_name": "Doc One"}},
- }
-
- s := &ChatPipelineService{}
- res := s.decorateHarnessAnswer("The answer [ID:0]", kbinfos, nil, nil, true)
- if res.Reference == nil {
- t.Fatal("a cited answer must carry a reference")
- }
- chunks, _ := res.Reference["chunks"].([]map[string]interface{})
- if len(chunks) != 1 {
- t.Fatalf("reference chunks = %#v, want 1", res.Reference["chunks"])
- }
- if chunks[0]["content"] != "hello" || chunks[0]["document_name"] != "Doc One" || chunks[0]["dataset_id"] != "kb1" {
- t.Errorf("reference chunk = %#v, want the client-facing keys populated", chunks[0])
- }
- for _, engineKey := range []string{"content_with_weight", "docnm_kwd", "kb_id", "vector"} {
- if _, has := chunks[0][engineKey]; has {
- t.Errorf("reference chunk leaked engine key %q", engineKey)
- }
- }
- if _, has := src["vector"]; !has {
- t.Error("the shared source chunk lost its vector: the reference must not mutate it")
- }
-}
-
-// TestReferenceChunksKeepsRenderedSlots pins the index contract of the reference
-// payload: the client resolves a marker by indexing reference.chunks with the
-// number the model wrote, so entry i has to be the chunk of rendered block i. A
-// block whose chunk cannot be located keeps its slot — dropping it would slide
-// every later marker onto a passage the model never cited.
-func TestReferenceChunksKeepsRenderedSlots(t *testing.T) {
- pool := []map[string]interface{}{
- {"chunk_id": "p0", "content": "zero"},
- {"chunk_id": "p1", "content": "one"},
- {"chunk_id": "p2", "content": "two"},
- }
- // Rendered blocks: pool 2, one that cannot be located, pool 0. The remaining
- // pool chunk follows so nothing retrieved is lost.
- got := referenceChunks([]int{2, -1, 0}, pool)
- if len(got) != 4 {
- t.Fatalf("reference = %d entries (%v), want 3 rendered slots plus the pool remainder", len(got), got)
- }
- if got[0]["chunk_id"] != "p2" || got[1] != nil || got[2]["chunk_id"] != "p0" {
- t.Fatalf("reference = %v, want p2 at block 0, an empty slot at block 1, p0 at block 2", got)
- }
- if got[3]["chunk_id"] != "p1" {
- t.Errorf("reference tail = %v, want the remaining pool chunk p1", got[3])
- }
-}
-
-// TestReferenceChunksFallsBackToPoolOrder keeps the no-CiteChunkIDs behavior: the
-// pool order is then the numbering, so the reference is the pool untouched.
-func TestReferenceChunksFallsBackToPoolOrder(t *testing.T) {
- pool := []map[string]interface{}{{"chunk_id": "p0"}, {"chunk_id": "p1"}}
- got := referenceChunks(nil, pool)
- if len(got) != 2 || got[0]["chunk_id"] != "p0" || got[1]["chunk_id"] != "p1" {
- t.Fatalf("reference = %v, want the pool in order", got)
- }
-}
-
-// TestCitationAuditReportsFigureChunkAndImage pins the observability line that
-// answers "why does Fig. n show no image": it names the chunk behind each cited
-// figure (Fig. n = rendered block n-1) and whether that chunk has an image, so a
-// text-only chunk (nothing to draw) is distinguishable from a marker that resolved
-// to the wrong chunk.
-func TestCitationAuditReportsFigureChunkAndImage(t *testing.T) {
- chunks := []map[string]interface{}{
- {"chunk_id": "c0", "image_id": "img-0"},
- {"chunk_id": "c1"},
- {"chunk_id": "c2", "img_id": "img-2"},
- }
- // Rendered blocks: pool 2, pool 0, pool 1 (Fig. 1..3). The model cited blocks 0
- // and 2, so pool 2 (image) and pool 1 (no image) come back resolved.
- got := citationAudit([]int{2, 0, 1}, []int{2, 1}, chunks)
- want := "Fig.1=c2 image=true Fig.3=c1 image=false"
- if got != want {
- t.Errorf("citationAudit = %q, want %q", got, want)
- }
-}
-
-// TestDecorateHarnessAnswerRewritesSlotCitations is the end-to-end lock for
-// the leaked "[ID:Slot 0]" bug: the harness answer arrives citing the internal
-// slot table, and the final answer must cite the chunk that filled the slot
-// instead — with that chunk present in the reference payload.
-func TestDecorateHarnessAnswerRewritesSlotCitations(t *testing.T) {
- kbinfos := map[string]interface{}{
- "chunks": []map[string]interface{}{
- {"chunk_id": "c1", "content_with_weight": "unrelated", "doc_id": "d0", "docnm_kwd": "Other"},
- {"chunk_id": "c9", "content_with_weight": "eiffel", "doc_id": "d1", "docnm_kwd": "Doc One"},
- },
- "doc_aggs": []interface{}{
- map[string]interface{}{"doc_id": "d1", "doc_name": "Doc One"},
- },
- }
- s := &ChatPipelineService{}
- res := s.decorateHarnessAnswer(
- "The tower opened in 1889 [ID:Slot 0].",
- kbinfos,
- map[string][]string{"0": {"c9"}},
- nil,
- true,
- )
- if strings.Contains(res.Answer, "[ID:Slot") {
- t.Fatalf("final answer still carries the internal slot citation: %q", res.Answer)
- }
- if !strings.Contains(res.Answer, "[ID:1]") {
- t.Fatalf("final answer must cite the evidence chunk pool index, got %q", res.Answer)
- }
- chunks, _ := res.Reference["chunks"].([]map[string]interface{})
- // The reference keeps the whole citation pool (Python parity); it must at
- // least carry the slot's evidence chunk so [ID:1] is openable.
- found := false
- for _, c := range chunks {
- if c["content"] == "eiffel" {
- found = true
- }
- }
- if !found {
- t.Fatalf("reference must carry the slot's evidence chunk, got %#v", res.Reference["chunks"])
- }
-}
-
-func TestAsyncChatHarnessQuote(t *testing.T) {
- db := setupChatPipelineVisionTestDB(t)
- seedChatPipelineVisionTenant(t, db, "glm-4-flash@ZHIPU-AI")
- if err := db.AutoMigrate(&entity.Knowledgebase{}); err != nil {
- t.Fatal(err)
- }
- if err := db.Create(&entity.Knowledgebase{ID: "kb-1", TenantID: "tenant-1", Name: "Knowledge"}).Error; err != nil {
- t.Fatal(err)
- }
- previousHarness := harnessRetriever
- t.Cleanup(func() { harnessRetriever = previousHarness })
- harnessRetriever = func(_ context.Context, req HarnessRequest) (HarnessResult, error) {
- req.AnswerSink("Research[ID:", true)
- req.AnswerSink("0] done", true)
- for _, delta := range []string{"Answer[", "ID:0", "] end"} {
- req.AnswerSink(delta, false)
- }
- return HarnessResult{
- Answer: "Answer[ID:0] end",
- Chunks: []map[string]interface{}{{"chunk_id": "c1", "content_with_weight": "Evidence", "doc_id": "d1"}},
- DocAggs: []map[string]interface{}{{"doc_id": "d1", "doc_name": "Source"}},
- }, nil
- }
- for _, tt := range []struct {
- name string
- config, request any
- wantQuote bool
- }{
- {"default enabled", nil, nil, true},
- {"chat disabled", false, true, false},
- {"request disabled", true, false, false},
- } {
- t.Run(tt.name, func(t *testing.T) {
- chat := dialForTest("glm-4-flash@ZHIPU-AI")
- chat.KBIDs = entity.JSONSlice{"kb-1"}
- chat.PromptConfig["system"] = "Answer using {knowledge}"
- chat.PromptConfig["quote"] = tt.config
- results, err := NewChatPipelineService().AsyncChat(t.Context(), "user-1", chat,
- []map[string]interface{}{{"role": "user", "content": "Question"}}, true,
- map[string]interface{}{"reasoning": 1, "quote": tt.request})
- if err != nil {
- t.Fatal(err)
- }
- var streamed, reasoning strings.Builder
- var final AsyncChatResult
- want := "Answer end"
- if tt.wantQuote {
- want = "Answer[ID:0] end"
- }
- for result := range results {
- reasoning.WriteString(result.Reasoning)
- if result.Final {
- final = result
- } else {
- streamed.WriteString(result.Answer)
- if !strings.HasPrefix(want, streamed.String()) {
- t.Fatalf("stream leaked citation text: %q", streamed.String())
- }
- }
- }
- if !final.Final || final.Answer != want || streamed.String() != want {
- t.Fatalf("stream=%q final=%+v, want %q", streamed.String(), final, want)
- }
- // The evidence ships either way; only doc_aggs follows the cite switch.
- chunks, _ := final.Reference["chunks"].([]map[string]interface{})
- if len(chunks) != 1 {
- t.Fatalf("reference chunks = %#v, want the harness evidence", final.Reference["chunks"])
- }
- if _, hasDocAggs := final.Reference["doc_aggs"]; hasDocAggs != tt.wantQuote {
- t.Fatalf("doc_aggs presence = %v, quote=%v: %#v", hasDocAggs, tt.wantQuote, final.Reference)
- }
- if !tt.wantQuote && strings.Contains(reasoning.String(), "[ID:") {
- t.Fatalf("disabled citations must be removed from reasoning, got %q", reasoning.String())
- }
- })
- }
-}
-
func TestAsyncChatEmptyResponseQuoteDisabled(t *testing.T) {
db := setupChatPipelineVisionTestDB(t)
seedChatPipelineVisionTenant(t, db, "glm-4-flash@ZHIPU-AI")
@@ -2108,10 +1719,10 @@ func TestAsyncChatEmptyResponseQuoteDisabled(t *testing.T) {
// the client contract (the SSE `think_event` field), so renaming or dropping a tag
// breaks every client while neither the compiler nor any Go caller notices.
//
-// It is also the only assertion on this struct's shape, which is why the harness
-// ALIASES it (harness.ThinkEvent) instead of carrying a second copy bridged field
-// by field in cmd — that copy compiled for as long as it stayed a subset, and
-// silently dropped any field added on the other side.
+// It is also the only assertion on this struct's shape, which is why a producer
+// ALIASES this type instead of carrying a second copy bridged field by field —
+// that copy compiled for as long as it stayed a subset, and silently dropped
+// any field added on the other side.
func TestThinkEventWireKeys(t *testing.T) {
// Every field populated: with `omitempty` on all but Kind and Summary, that is
// what makes the whole key set appear.
@@ -2165,225 +1776,6 @@ func thinkEventWireKeys(t *testing.T, ev ThinkEvent) []string {
return keys
}
-// TestDecorateHarnessAnswerUncitedKeepsChunks pins that an agentic answer using
-// no citation marker still carries the evidence it was grounded on: the pool is
-// the provenance of the answer, and the cite switch governs the doc_aggs cards
-// rather than the passages themselves.
-func TestDecorateHarnessAnswerUncitedKeepsChunks(t *testing.T) {
- kbinfos := map[string]interface{}{
- "chunks": []map[string]interface{}{
- {
- "chunk_id": "c1",
- "content_with_weight": "the wolf is grey",
- "doc_id": "d1",
- "docnm_kwd": "wolf.jpg",
- "kb_id": "kb1",
- },
- },
- "doc_aggs": []interface{}{
- map[string]interface{}{"doc_id": "d1", "doc_name": "wolf.jpg"},
- },
- }
-
- s := &ChatPipelineService{}
- res := s.decorateHarnessAnswer("小狼的颜色是灰色的。", kbinfos, nil, nil, true)
- chunks, _ := res.Reference["chunks"].([]map[string]interface{})
- if len(chunks) != 1 || chunks[0]["id"] != "c1" {
- t.Fatalf("reference chunks = %#v, want the evidence pool", res.Reference["chunks"])
- }
-}
-
-// TestDecorateHarnessAnswerCitesThinkBlock ensures citations emitted inside the
-// hidden thinking block still resolve to client-visible source documents.
-func TestDecorateHarnessAnswerCitesThinkBlock(t *testing.T) {
- kbinfos := map[string]interface{}{
- "chunks": []map[string]interface{}{
- {
- "chunk_id": "c1",
- "content_with_weight": "the wolf is grey",
- "doc_id": "d1",
- "docnm_kwd": "wolf.jpg",
- },
- },
- "doc_aggs": []interface{}{
- map[string]interface{}{"doc_id": "d1", "doc_name": "wolf.jpg"},
- },
- }
-
- s := &ChatPipelineService{}
- res := s.decorateHarnessAnswer(" thinkingSupported by [ID:0]. response小狼的颜色是灰色的。", kbinfos, nil, []string{"c1"}, true)
- if res.Reference == nil {
- t.Fatal("a think-block citation must carry a reference")
- }
- if aggs, _ := res.Reference["doc_aggs"].([]interface{}); len(aggs) != 1 || aggs[0].(map[string]interface{})["doc_id"] != "d1" {
- t.Fatalf("reference doc_aggs = %#v, want the cited document", res.Reference["doc_aggs"])
- }
-}
-
-// TestDecorateHarnessAnswerResolvesRenderedPosition is the regression lock for
-// the one-passage agentic answer that came back with no reference. The compose
-// numbers its evidence 0-based and the model writes that number, so the marker
-// and the reference line up: the passage it names is openable, and a marker that
-// names nothing is removed instead of being handed to the client.
-func TestDecorateHarnessAnswerResolvesRenderedPosition(t *testing.T) {
- kbinfos := map[string]interface{}{
- "chunks": []map[string]interface{}{
- {
- "chunk_id": "c1",
- "content_with_weight": "the wolf is grey",
- "doc_id": "d1",
- "docnm_kwd": "wolf.jpg",
- },
- },
- "doc_aggs": []interface{}{
- map[string]interface{}{"doc_id": "d1", "doc_name": "wolf.jpg"},
- },
- }
-
- s := &ChatPipelineService{}
- res := s.decorateHarnessAnswer("小狼的颜色是灰色的 [ID:0]。", kbinfos, nil, []string{"c1"}, true)
- if !strings.Contains(res.Answer, "[ID:0]") {
- t.Fatalf("the marker is the client's index and must survive, got %q", res.Answer)
- }
- if res.Reference == nil {
- t.Fatal("a one-passage agentic answer must carry a reference")
- }
- chunks, _ := res.Reference["chunks"].([]map[string]interface{})
- if len(chunks) != 1 || chunks[0]["document_name"] != "wolf.jpg" {
- t.Fatalf("reference chunks = %#v, want the cited passage", res.Reference["chunks"])
- }
-
- // The old numbering (a 1-based position) names no entry in a one-passage
- // reference, so it must be dropped rather than shipped as a dead marker.
- res = s.decorateHarnessAnswer("小狼的颜色是灰色的 [ID:1]。", kbinfos, nil, []string{"c1"}, true)
- if strings.Contains(res.Answer, "[ID:1]") {
- t.Fatalf("an unresolvable marker must be dropped, got %q", res.Answer)
- }
- // The unresolvable marker is dropped, but the evidence it was drawn from
- // still ships.
- remaining, _ := res.Reference["chunks"].([]map[string]interface{})
- if len(remaining) != 1 || remaining[0]["document_name"] != "wolf.jpg" {
- t.Fatalf("reference chunks = %#v, want the evidence passage", res.Reference["chunks"])
- }
-}
-
-// TestDecorateHarnessAnswerOrdersReferenceByRenderedOrder pins the ordering half
-// of the contract: the compose renders the evidence similarity-ranked (and
-// capped), so rendered position 0 is not pool[0]. The reference must put the
-// rendered list first — a marker's index is its position there — and append the
-// rest of the pool.
-func TestDecorateHarnessAnswerOrdersReferenceByRenderedOrder(t *testing.T) {
- kbinfos := map[string]interface{}{
- "chunks": []map[string]interface{}{
- {"chunk_id": "c1", "content_with_weight": "unrelated", "doc_id": "d1", "docnm_kwd": "Doc One"},
- {"chunk_id": "c2", "content_with_weight": "unrelated too", "doc_id": "d1", "docnm_kwd": "Doc One"},
- {"chunk_id": "c3", "content_with_weight": "eiffel", "doc_id": "d2", "docnm_kwd": "Doc Two"},
- },
- "doc_aggs": []interface{}{
- map[string]interface{}{"doc_id": "d1", "doc_name": "Doc One"},
- map[string]interface{}{"doc_id": "d2", "doc_name": "Doc Two"},
- },
- }
-
- s := &ChatPipelineService{}
- // The compose ranked c3 first, so the model's [ID:0] is c3.
- res := s.decorateHarnessAnswer(
- "The tower opened in 1889 [ID:0].",
- kbinfos,
- nil,
- []string{"c3", "c1"},
- true,
- )
- chunks, _ := res.Reference["chunks"].([]map[string]interface{})
- if len(chunks) != 3 {
- t.Fatalf("reference chunks = %#v, want the rendered list plus the remaining pool", res.Reference["chunks"])
- }
- if chunks[0]["content"] != "eiffel" || chunks[0]["id"] != "c3" {
- t.Fatalf("reference[0] = %#v, want the rendered first passage c3", chunks[0])
- }
- if chunks[1]["id"] != "c1" || chunks[2]["id"] != "c2" {
- t.Fatalf("reference order = %#v, want [c3 c1 c2]", chunks)
- }
- // Only Doc Two (c3) was cited, so the other document is filtered out.
- aggs, _ := res.Reference["doc_aggs"].([]interface{})
- if len(aggs) != 1 {
- t.Fatalf("reference doc_aggs = %#v, want only the cited document", res.Reference["doc_aggs"])
- }
- if dam, _ := aggs[0].(map[string]interface{}); dam["doc_id"] != "d2" {
- t.Fatalf("reference doc_aggs = %#v, want d2", aggs[0])
- }
-}
-
-// TestDecorateHarnessAnswerDropsOnlyCanonicalMarkers pins the text-safety rule: a
-// canonical citation naming no rendered passage is removed (the user could never
-// open it), while a bare bracketed number in prose is left alone — "[2024]" is far
-// more likely to be ordinary text than a citation, and deleting user-visible text
-// is worse than a chip that cannot open.
-func TestDecorateHarnessAnswerDropsOnlyCanonicalMarkers(t *testing.T) {
- kbinfos := map[string]interface{}{
- "chunks": []map[string]interface{}{
- {"chunk_id": "c1", "content_with_weight": "one", "doc_id": "d1", "docnm_kwd": "Doc One"},
- },
- "doc_aggs": []interface{}{map[string]interface{}{"doc_id": "d1", "doc_name": "Doc One"}},
- }
-
- s := &ChatPipelineService{}
- res := s.decorateHarnessAnswer(
- "发布于 [2024] 年 [ID:0],并在 [ID:7] 修订。",
- kbinfos,
- nil,
- []string{"c1"},
- true,
- )
- if !strings.Contains(res.Answer, "[2024]") {
- t.Errorf("prose brackets must survive, got %q", res.Answer)
- }
- if !strings.Contains(res.Answer, "[ID:0]") {
- t.Errorf("a resolvable citation must survive, got %q", res.Answer)
- }
- if strings.Contains(res.Answer, "[ID:7]") {
- t.Errorf("an unresolvable canonical marker must be dropped, got %q", res.Answer)
- }
-}
-
-// TestDecorateHarnessAnswerKeepsPositionsAfterUnknownBlock: when the harness
-// publishes an id the pool does not carry, that block keeps its slot so the blocks
-// after it keep the numbers the model saw. A marker naming a later block must
-// still resolve to the right chunk — and to the right document for doc_aggs.
-func TestDecorateHarnessAnswerKeepsPositionsAfterUnknownBlock(t *testing.T) {
- kbinfos := map[string]interface{}{
- "chunks": []map[string]interface{}{
- {"chunk_id": "c1", "content_with_weight": "one", "doc_id": "d1", "docnm_kwd": "Doc One"},
- {"chunk_id": "c2", "content_with_weight": "two", "doc_id": "d1", "docnm_kwd": "Doc One"},
- {"chunk_id": "c3", "content_with_weight": "eiffel", "doc_id": "d2", "docnm_kwd": "Doc Two"},
- },
- "doc_aggs": []interface{}{
- map[string]interface{}{"doc_id": "d1", "doc_name": "Doc One"},
- map[string]interface{}{"doc_id": "d2", "doc_name": "Doc Two"},
- },
- }
-
- s := &ChatPipelineService{}
- // The harness rendered c1, an unknown chunk, then c3 — so [ID:2] is c3.
- res := s.decorateHarnessAnswer(
- "The tower opened in 1889 [ID:2].",
- kbinfos,
- nil,
- []string{"c1", "gone", "c3"},
- true,
- )
- if !strings.Contains(res.Answer, "[ID:2]") {
- t.Fatalf("a marker past the unknown block must survive, got %q", res.Answer)
- }
- aggs, _ := res.Reference["doc_aggs"].([]interface{})
- if len(aggs) != 1 {
- t.Fatalf("reference doc_aggs = %#v, want only the cited document", res.Reference["doc_aggs"])
- }
- if dam, _ := aggs[0].(map[string]interface{}); dam["doc_id"] != "d2" {
- t.Fatalf("reference doc_aggs = %#v, want d2 (c3)", aggs[0])
- }
-}
-
// setupChatPipelineToolSupportTestDB seeds an in-memory scope whose models differ
// only in their persisted tool-calling flag, so the probe runs without the provider
// catalog:
@@ -2448,6 +1840,13 @@ func seedChatPipelineToolSupportTenant(t *testing.T, db *gorm.DB, defaultLLMID s
}
}
+// configTools reports the is_tools flag getLLMModelConfig carries, read the way
+// a consumer of that config map does.
+func configTools(cfg map[string]interface{}) bool {
+ v, _ := cfg["is_tools"].(bool)
+ return v
+}
+
// TestGetLLMModelConfigCarriesToolSupport pins that the resolution reports the
// capability of the row it actually loaded — the config the pipeline then carries
// through the request, which is where Python reads chat_mdl.is_tools from too.
@@ -2469,7 +1868,7 @@ func TestGetLLMModelConfigCarriesToolSupport(t *testing.T) {
if err != nil {
t.Fatalf("getLLMModelConfig(%s) failed: %v", tc.llm, err)
}
- if got := chatConfigSupportsTools(cfg); got != tc.wantTools {
+ if got := configTools(cfg); got != tc.wantTools {
t.Errorf("is_tools for %s = %v, want %v", tc.llm, got, tc.wantTools)
}
if got, _ := cfg["model_type"].(string); got != tc.wantType {
@@ -2513,7 +1912,7 @@ func TestGetLLMModelConfigRejectsUnusableModels(t *testing.T) {
} {
t.Run(tc.name, func(t *testing.T) {
cfg, _, _, _, err := svc.getLLMModelConfig(t.Context(), &entity.Chat{TenantID: "tenant-1", LLMID: tc.llm})
- if err == nil && chatConfigSupportsTools(cfg) {
+ if err == nil && configTools(cfg) {
t.Errorf("unusable model %s reported tool support", tc.llm)
}
})
@@ -2532,7 +1931,7 @@ func TestGetLLMModelConfigFallsBackToTenantDefault(t *testing.T) {
if err != nil {
t.Fatalf("getLLMModelConfig(tenant default) failed: %v", err)
}
- if !chatConfigSupportsTools(cfg) {
+ if !configTools(cfg) {
t.Error("tenant default with is_tools=true reported no tool support, want true")
}
}
@@ -2548,66 +1947,16 @@ func TestGetLLMModelConfigTenantDefaultWithoutToolSupport(t *testing.T) {
if err != nil {
t.Fatalf("getLLMModelConfig(tenant default) failed: %v", err)
}
- if chatConfigSupportsTools(cfg) {
+ if configTools(cfg) {
t.Error("tenant default with is_tools=false reported tool support, want false")
}
}
-// TestChatConfigSupportsToolsFailClosed is the guard case for the config read: a
-// missing config, a config without the flag, and a flag carrying anything but a
-// truthy boolean all mean "no tool support", the same outcome as Python's default.
-func TestChatConfigSupportsToolsFailClosed(t *testing.T) {
- if chatConfigSupportsTools(nil) {
- t.Error("nil config reported tool support, want false")
- }
- if chatConfigSupportsTools(map[string]interface{}{"model_type": "chat"}) {
- t.Error("config without an is_tools flag reported tool support, want false")
- }
- for _, v := range []interface{}{false, "false", "", 0, nil} {
- if chatConfigSupportsTools(map[string]interface{}{"is_tools": v}) {
- t.Errorf("is_tools=%#v reported tool support, want false", v)
- }
- }
- if !chatConfigSupportsTools(map[string]interface{}{"is_tools": true}) {
- t.Error("is_tools=true reported no tool support, want true")
- }
-}
-
-// TestReasoningNeedsAgenticGraphGatesOnToolSupport pins the gate Python
-// rag_agent applies before building its agentic loop: reasoning level 0 turns it
-// off, and so does a chat model that cannot call tools. Only reasoning enabled
-// together with tool support drives the agentic graph.
-func TestReasoningNeedsAgenticGraphGatesOnToolSupport(t *testing.T) {
- capable := map[string]interface{}{"model_type": "chat", "is_tools": true}
- incapable := map[string]interface{}{"model_type": "chat", "is_tools": false}
- chat := &entity.Chat{TenantID: "tenant-1", LLMID: "model-tools-on"}
-
- for _, tc := range []struct {
- name string
- chat *entity.Chat
- cfg map[string]interface{}
- reasoningLevel int
- want bool
- }{
- {"reasoning off", chat, capable, 0, false},
- {"reasoning on and tool-capable", chat, capable, 2, true},
- {"reasoning on but no tool support", chat, incapable, 2, false},
- {"reasoning on, unresolvable model", chat, nil, 4, false},
- {"reasoning on, nil dialog", nil, capable, 2, false},
- } {
- t.Run(tc.name, func(t *testing.T) {
- if got := reasoningNeedsAgenticGraph(tc.chat, tc.cfg, tc.reasoningLevel); got != tc.want {
- t.Errorf("reasoningNeedsAgenticGraph(level=%d) = %v, want %v", tc.reasoningLevel, got, tc.want)
- }
- })
- }
-}
-
// TestResolveReasoningLevelReadsAgenticLevel pins where the agentic RAG mode is
// selected from. The level arrives as a number (frontend sends
// Number(getThinkingLevel())), from the request first and the dialog's
-// prompt_config second, and reasoningLevelAgentic is a level of its own rather
-// than a harness depth.
+// prompt_config second, and reasoningLevelAgentic is the one level that means
+// anything beyond "retrieve normally".
func TestResolveReasoningLevelReadsAgenticLevel(t *testing.T) {
for _, tc := range []struct {
name string
@@ -2617,7 +1966,7 @@ func TestResolveReasoningLevelReadsAgenticLevel(t *testing.T) {
}{
{"absent everywhere", nil, nil, 0},
{"naive from request", map[string]interface{}{"reasoning": 0}, nil, 0},
- {"harness depth from request", map[string]interface{}{"reasoning": 3}, nil, 3},
+ {"stale level from request", map[string]interface{}{"reasoning": 3}, nil, 3},
{"agentic from request", map[string]interface{}{"reasoning": 5}, nil, reasoningLevelAgentic},
{
"request float from JSON",
@@ -2651,30 +2000,6 @@ func TestResolveReasoningLevelReadsAgenticLevel(t *testing.T) {
}
}
-// TestAgenticLevelIsNotAHarnessMode pins the boundary between the two engines.
-// harnessModeForLevel's domain is the harness depths 1..4; reasoningLevelAgentic
-// belongs to a different engine and is meant to be dispatched to
-// smart-reasoning before any harness mode is derived.
-//
-// harnessModeForLevel itself is NOT asserted to reject the agentic level: it
-// takes an int and has no level vocabulary of its own, so any such assertion
-// would only pin an implementation detail. The contract these tests protect is
-// the one callers rely on — the level constant the selector sends, and a harness
-// mode for each depth that is supposed to have one.
-func TestAgenticLevelIsNotAHarnessMode(t *testing.T) {
- // The selector's option value; a mismatch means the UI and this package
- // disagree on which number means agentic.
- if reasoningLevelAgentic != 5 {
- t.Fatalf("reasoningLevelAgentic = %d, want 5 (the level message-input/next.tsx sends)", reasoningLevelAgentic)
- }
- want := map[int]string{1: "low", 2: "medium", 3: "high", 4: "ultra"}
- for level, mode := range want {
- if got := harnessModeForLevel(level); got != mode {
- t.Errorf("harnessModeForLevel(%d) = %q, want %q", level, got, mode)
- }
- }
-}
-
// TestQuoteEnabledNeedsBothSourcesToAgree pins the effective citation setting:
// the request and the dialog each get a veto, and an absent setting on either
// side leaves citations on.
@@ -2756,7 +2081,7 @@ func TestGetLLMModelConfigReadsToolSupportPerResolution(t *testing.T) {
if err != nil {
t.Fatalf("getLLMModelConfig failed: %v", err)
}
- if !chatConfigSupportsTools(cfg) {
+ if !configTools(cfg) {
t.Fatal("tool-capable model reported no support")
}
@@ -2769,7 +2094,7 @@ func TestGetLLMModelConfigReadsToolSupportPerResolution(t *testing.T) {
if err != nil {
t.Fatalf("getLLMModelConfig after the flag flip failed: %v", err)
}
- if chatConfigSupportsTools(cfg) {
+ if configTools(cfg) {
t.Error("a flipped is_tools flag was not seen by the next resolution")
}
}
diff --git a/internal/service/citation.go b/internal/service/citation.go
index 0c4a26429b..c218990106 100644
--- a/internal/service/citation.go
+++ b/internal/service/citation.go
@@ -384,99 +384,12 @@ func RepairBadCitationFormats(answer string) string {
// citations of the chunk the slot was filled from, or dropped.
var slotCitationPattern = regexp.MustCompile(`(?i)\[\s*(?:ID\s*[:: ]*\s*)?slot\s*([0-9]+)\s*\]`)
-// RepairSlotCitations rewrites leaked slot-table citations into locatable
-// chunk citations. slotCitations maps a slot-table id ("0", "1", ...) to the
-// evidence chunk ids that produced the slot's candidate; chunks is the ordered
-// evidence list the compose rendered for the model, whose block ids are 0-based
-// ("[ID:N]" where N is the block's index in that list — the index the client
-// resolves against the reference). A marker whose slot has no evidence chunk in
-// the list is dropped outright — an internal marker the user cannot resolve must
-// not reach the answer. Python parity note: the same leak exists upstream
-// (agentic_rag.py:885 only repairs "(ID:5)" shapes; the pre_summary slot
-// draft reaches Python's compose prompt too), so this repair is a Go-side
-// decoration-layer fix under the "every citation must locate a chunk" rule.
-func RepairSlotCitations(answer string, slotCitations map[string][]string, chunks []map[string]interface{}) string {
- if answer == "" || len(slotCitations) == 0 || !slotCitationPattern.MatchString(answer) {
- return answer
- }
- posByID := make(map[string]int, len(chunks))
- for i, c := range chunks {
- if id := chunkCitationID(c); id != "" {
- if _, dup := posByID[id]; !dup {
- posByID[id] = i
- }
- }
- }
- var b strings.Builder
- b.Grow(len(answer))
- last := 0
- for _, m := range slotCitationPattern.FindAllStringSubmatchIndex(answer, -1) {
- slotID := answer[m[2]:m[3]]
- repl := ""
- for _, eid := range slotCitations[slotID] {
- if pos, ok := posByID[eid]; ok {
- repl = "[ID:" + strconv.Itoa(pos) + "]"
- break
- }
- }
- b.WriteString(answer[last:m[0]])
- b.WriteString(repl)
- last = m[1]
- }
- b.WriteString(answer[last:])
- return b.String()
-}
-
-// chunkCitationID returns a pool chunk's id under the keys the harness uses
-// (chunk_id, then id — harness.ChunkIDOf's order).
-func chunkCitationID(c map[string]interface{}) string {
- for _, k := range []string{"chunk_id", "id"} {
- if s, ok := c[k].(string); ok && s != "" {
- return s
- }
- }
- return ""
-}
-
// rangeCitationPattern matches range-merged citations: "[ID:1-3]", "[ID: 1 - 3]",
// "[ID:1~3]". Models sometimes compress consecutive individual citations
// ([ID:1][ID:2][ID:3]) into such a range on their own — no code path produces
// it, and no chunk resolves through it, so it must be normalized defensively.
var rangeCitationPattern = regexp.MustCompile(`(?i)\[\s*ID\s*[:: ]*\s*([0-9]+)\s*[-–—~~]\s*([0-9]+)\s*\]`)
-// ExpandRangeCitations expands range-merged citations back into individual
-// ones: [ID:1-3] → [ID:1][ID:2][ID:3]. A range expands only when BOTH bounds
-// name valid evidence positions (0-based, bounded by poolSize — the numbering
-// the compose renders and the client indexes); reversed bounds are swapped
-// first. Anything out of range is dropped outright — an unresolvable marker must
-// not reach the user's answer, matching the slot-citation repair above. poolSize
-// is the length of the ordered evidence list the compose rendered
-// (harness.Kbinfos.CiteChunkIDs).
-func ExpandRangeCitations(answer string, poolSize int) string {
- if answer == "" || !rangeCitationPattern.MatchString(answer) {
- return answer
- }
- var b strings.Builder
- b.Grow(len(answer) * 2)
- last := 0
- for _, m := range rangeCitationPattern.FindAllStringSubmatchIndex(answer, -1) {
- a, _ := strconv.Atoi(normalizeArabicDigits(answer[m[2]:m[3]]))
- c, _ := strconv.Atoi(normalizeArabicDigits(answer[m[4]:m[5]]))
- if a > c {
- a, c = c, a
- }
- b.WriteString(answer[last:m[0]])
- if a >= 0 && c < poolSize {
- for i := a; i <= c; i++ {
- b.WriteString("[ID:" + strconv.Itoa(i) + "]")
- }
- }
- last = m[1]
- }
- b.WriteString(answer[last:])
- return b.String()
-}
-
// Keep an unfinished citation until its closing delimiter arrives. Retaining
// the trailing word also preserves the word boundaries used by refN markers.
var citationStreamTailPattern = regexp.MustCompile(`(?i)[\[(【][\s*idslot::0-9\x{0660}-\x{0669}\x{06F0}-\x{06F9}–—~~-]*$|#{1,2}(?:[0-9]+\$?)?$|\bref\s*[0-9\x{0660}-\x{0669}\x{06F0}-\x{06F9}]*$|\w+$`)
@@ -513,101 +426,8 @@ func (f *citationStreamFilter) flush() string {
return text
}
-// ResolveCitationMarkers drops the answer's canonical [ID:n] markers that name
-// nothing and returns the pool positions of the markers that do resolve.
-//
-// citeIdx maps a rendered evidence position to its chunk's index in the pool the
-// reference is built from (see citePoolIdx); -1 is a block whose chunk could not
-// be located. The rendered blocks are numbered 0-based, so a marker's number IS
-// the client's index into the reference — resolvable markers therefore keep the
-// number the model wrote.
-//
-// Only a marker carrying the literal "ID:" prefix is removed when it cannot be
-// resolved: the compose's citation rules prescribe that form, while a bare "[5]"
-// in prose is as likely to be a footnote, a version or a year, and deleting
-// user-visible text is worse than leaving a marker the client cannot open.
-//
-// The returned positions are pool indexes, de-duplicated, in first-seen order.
-func ResolveCitationMarkers(answer string, citeIdx []int) (string, []int) {
- if answer == "" || len(citeIdx) == 0 {
- return answer, nil
- }
- var b strings.Builder
- b.Grow(len(answer))
- last := 0
- seen := make(map[int]struct{})
- var out []int
- for _, m := range CitationMarkerPattern.FindAllStringSubmatchIndex(answer, -1) {
- n, ok := markerNumber(answer[m[2]:m[3]])
- resolved := -1
- if ok && n >= 0 && n < len(citeIdx) {
- resolved = citeIdx[n]
- }
- if resolved >= 0 {
- if _, dup := seen[resolved]; !dup {
- seen[resolved] = struct{}{}
- out = append(out, resolved)
- }
- continue
- }
- if canonicalIDMarkerPattern.MatchString(answer[m[0]:m[1]]) {
- // A canonical citation that names no rendered block: the user could
- // never open it, so it must not reach the answer.
- b.WriteString(answer[last:m[0]])
- last = m[1]
- }
- }
- b.WriteString(answer[last:])
- return b.String(), out
-}
-
// canonicalIDMarkerPattern matches a citation marker that carries the literal
// "ID:" prefix — the form the citation rules prescribe, and the only one that can
// be removed from the answer without risking ordinary text like "[2024]".
var canonicalIDMarkerPattern = regexp.MustCompile(
`(?i)\[\s*ID\s*[:: ]*\s*[0-9\x{0660}-\x{0669}\x{06F0}-\x{06F9}]+\s*\]`)
-
-// rawCitationMarkers returns the citation numbers the model wrote, in order and
-// including duplicates — the repetition is what the citation observability log in
-// decorateHarnessAnswer looks at — capped at limit entries. They are read before
-// any repair, so they are what the model actually emitted.
-func rawCitationMarkers(answer string, limit int) []int {
- if answer == "" || limit <= 0 {
- return nil
- }
- out := make([]int, 0, limit)
- for _, m := range CitationMarkerPattern.FindAllStringSubmatch(answer, -1) {
- if len(m) < 2 {
- continue
- }
- n, ok := markerNumber(m[1])
- if !ok {
- continue
- }
- out = append(out, n)
- if len(out) >= limit {
- break
- }
- }
- return out
-}
-
-// markerNumber parses a citation marker's digits (Arabic-Indic digits accepted)
-// into a position. ok is false for an empty or non-numeric capture.
-func markerNumber(digits string) (int, bool) {
- s := normalizeArabicDigits(digits)
- if s == "" {
- return 0, false
- }
- n := 0
- for _, r := range s {
- if r < '0' || r > '9' {
- return 0, false
- }
- n = n*10 + int(r-'0')
- if n > 1<<30 {
- return 0, false
- }
- }
- return n, true
-}
diff --git a/internal/service/citation_test.go b/internal/service/citation_test.go
index 916702a4f2..5c29d931e4 100644
--- a/internal/service/citation_test.go
+++ b/internal/service/citation_test.go
@@ -375,182 +375,6 @@ func TestRepairBadCitationFormats(t *testing.T) {
}
// -----------------------------------------------------------------------
-// RepairSlotCitations — leaked "[ID:Slot N]" markers (Bug: slot-table
-// citations in the final answer, unlocatable by the user).
-// -----------------------------------------------------------------------
-
-func TestRepairSlotCitations_MapsSlotToEvidenceChunk(t *testing.T) {
- chunks := []map[string]interface{}{
- {"chunk_id": "c1", "content": "one"},
- {"chunk_id": "c2", "content": "two"},
- {"id": "c9", "content": "slot evidence"},
- }
- answer := "Gustave Eiffel designed it [ID:Slot 0]. It opened in 1889."
- got := RepairSlotCitations(answer, map[string][]string{"0": {"c9", "missing"}}, chunks)
- if strings.Contains(got, "Slot") {
- t.Fatalf("slot marker leaked: %q", got)
- }
- if !strings.Contains(got, "[ID:2]") {
- t.Fatalf("slot marker not mapped to the evidence chunk position: %q", got)
- }
- if got != "Gustave Eiffel designed it [ID:2]. It opened in 1889." {
- t.Fatalf("unexpected rewrite: %q", got)
- }
-}
-
-func TestRepairSlotCitations_DropsUnresolvable(t *testing.T) {
- chunks := []map[string]interface{}{{"chunk_id": "c1", "content": "one"}}
- answer := "Fact one [ID:Slot 0]. Fact two [Slot 1]."
- got := RepairSlotCitations(answer, map[string][]string{"0": {"not-in-pool"}, "1": {"also-missing"}}, chunks)
- if strings.Contains(strings.ToLower(got), "slot") {
- t.Fatalf("unresolvable slot markers must be dropped, got %q", got)
- }
- if got != "Fact one . Fact two ." {
- t.Fatalf("unexpected rewrite: %q", got)
- }
-}
-
-func TestRepairSlotCitations_MarkerVariants(t *testing.T) {
- chunks := []map[string]interface{}{{"chunk_id": "a"}, {"id": "b"}}
- cases := []struct {
- marker string
- want string
- }{
- {"[ID:Slot 1]", "[ID:1]"},
- {"[ID: slot 1]", "[ID:1]"},
- {"[Slot 1]", "[ID:1]"},
- {"[slot 1]", "[ID:1]"},
- }
- for _, c := range cases {
- got := RepairSlotCitations("x "+c.marker+" y", map[string][]string{"1": {"b"}}, chunks)
- if !strings.Contains(got, c.want) {
- t.Errorf("marker %q: got %q, want it rewritten to %q", c.marker, got, c.want)
- }
- }
-}
-
-func TestRepairSlotCitations_NoopWithoutSlotMarkersOrMap(t *testing.T) {
- answer := "Plain answer [ID:0]."
- if got := RepairSlotCitations(answer, nil, nil); got != answer {
- t.Fatalf("nil map must be a no-op, got %q", got)
- }
- if got := RepairSlotCitations(answer, map[string][]string{"0": {"c"}}, nil); got != answer {
- t.Fatalf("no slot markers must be a no-op, got %q", got)
- }
-}
-
-// -----------------------------------------------------------------------
-// ExpandRangeCitations — range-merged citations ("[ID:1-3]") the model
-// produces on its own (no code path emits them).
-// -----------------------------------------------------------------------
-
-func TestExpandRangeCitations(t *testing.T) {
- cases := []struct {
- name string
- answer string
- poolSize int
- want string
- }{
- {
- "plain range",
- "Fact [ID:1-3].",
- 5,
- "Fact [ID:1][ID:2][ID:3].",
- },
- {
- "spaced dash variant",
- "Fact [ID: 1 - 3].",
- 5,
- "Fact [ID:1][ID:2][ID:3].",
- },
- {
- "tilde variant",
- "Fact [ID:1~3].",
- 5,
- "Fact [ID:1][ID:2][ID:3].",
- },
- {
- "reversed bounds swap",
- "Fact [ID:3-1].",
- 5,
- "Fact [ID:1][ID:2][ID:3].",
- },
- {
- "single-bound range",
- "Fact [ID:2-2].",
- 5,
- "Fact [ID:2].",
- },
- {
- "out of bounds dropped",
- "Fact [ID:99-102].",
- 5,
- "Fact .",
- },
- {
- "partially out of bounds dropped",
- "Fact [ID:2-99].",
- 5,
- "Fact .",
- },
- {
- "empty pool dropped",
- "Fact [ID:1-3].",
- 0,
- "Fact .",
- },
- {
- "no ranges untouched",
- "Fact [ID:1] and [ID:2].",
- 5,
- "Fact [ID:1] and [ID:2].",
- },
- }
- for _, c := range cases {
- t.Run(c.name, func(t *testing.T) {
- if got := ExpandRangeCitations(c.answer, c.poolSize); got != c.want {
- t.Fatalf("ExpandRangeCitations(%q, %d) = %q, want %q", c.answer, c.poolSize, got, c.want)
- }
- })
- }
-}
-
-// TestDecorateHarnessAnswerExpandsRangeCitations pins the end-to-end fix for
-// the [ID:1-3] report: the final answer carries only individual citations,
-// each resolving to a pooled chunk, and only the cited doc reaches the
-// reference.
-func TestDecorateHarnessAnswerExpandsRangeCitations(t *testing.T) {
- kbinfos := map[string]interface{}{
- "chunks": []map[string]interface{}{
- {"chunk_id": "c0", "content_with_weight": "a", "doc_id": "d1", "docnm_kwd": "Doc One"},
- {"chunk_id": "c1", "content_with_weight": "b", "doc_id": "d1", "docnm_kwd": "Doc One"},
- {"chunk_id": "c2", "content_with_weight": "c", "doc_id": "d1", "docnm_kwd": "Doc One"},
- {"chunk_id": "c3", "content_with_weight": "d", "doc_id": "d1", "docnm_kwd": "Doc One"},
- },
- "doc_aggs": []interface{}{
- map[string]interface{}{"doc_id": "d1", "doc_name": "Doc One"},
- map[string]interface{}{"doc_id": "d2", "doc_name": "Doc Two"},
- },
- }
- s := &ChatPipelineService{}
- res := s.decorateHarnessAnswer("The range claim holds [ID:1-3].", kbinfos, nil, nil, true)
-
- if strings.Contains(res.Answer, "1-3") {
- t.Fatalf("final answer still carries the range citation: %q", res.Answer)
- }
- // Expanded markers keep the 0-based indexes the client resolves against
- // reference.chunks.
- for _, want := range []string{"[ID:1]", "[ID:2]", "[ID:3]"} {
- if !strings.Contains(res.Answer, want) {
- t.Fatalf("final answer missing expanded citation %s: %q", want, res.Answer)
- }
- }
- // Every cited chunk belongs to Doc One, so the uncited Doc Two must be
- // filtered out of the reference (recall_docs = cited docs).
- if aggs, _ := res.Reference["doc_aggs"].([]interface{}); len(aggs) != 1 {
- t.Fatalf("reference doc_aggs = %#v, want the single cited doc", res.Reference["doc_aggs"])
- }
-}
func TestCitationStreamFilter(t *testing.T) {
var filter citationStreamFilter
@@ -567,32 +391,3 @@ func TestCitationStreamFilter(t *testing.T) {
t.Fatalf("filtered answer=%q, want %q", got, want)
}
}
-
-// TestDecorateHarnessAnswerKeepsCitationsForNotFoundAnswer pins that an answer
-// reporting no result is decorated like any other answer: its resolvable [ID:n]
-// markers and the referenced doc_aggs still reach the client. The citation
-// switch (quote) is the only thing that removes them.
-func TestDecorateHarnessAnswerKeepsCitationsForNotFoundAnswer(t *testing.T) {
- kbinfos := map[string]interface{}{
- "chunks": []map[string]interface{}{
- {"chunk_id": "c0", "content_with_weight": "a", "doc_id": "d1", "docnm_kwd": "Doc One"},
- },
- "doc_aggs": []interface{}{
- map[string]interface{}{"doc_id": "d1", "doc_name": "Doc One"},
- },
- }
- answer := "知识库中未找到您要的答案!因此无法统计得克萨斯州和加利福尼亚州 [ID:0] 选出同一候选人的次数。"
- s := &ChatPipelineService{}
- res := s.decorateHarnessAnswer(answer, kbinfos, nil, nil, true)
-
- if !strings.Contains(res.Answer, "[ID:0]") {
- t.Fatalf("answer lost its citation markers: %q", res.Answer)
- }
- if _, has := res.Reference["doc_aggs"]; !has {
- t.Fatalf("answer lost its doc_aggs: %#v", res.Reference)
- }
- chunks, _ := res.Reference["chunks"].([]map[string]interface{})
- if len(chunks) != 1 || chunks[0]["id"] != "c0" {
- t.Fatalf("reference chunks = %#v, want the evidence passage", res.Reference["chunks"])
- }
-}
diff --git a/internal/service/web_search_provider_test.go b/internal/service/web_search_provider_test.go
index 490aa7a769..0bd2870ab6 100644
--- a/internal/service/web_search_provider_test.go
+++ b/internal/service/web_search_provider_test.go
@@ -25,23 +25,6 @@ import (
"testing"
)
-// TestHarnessWebSearchFuncGatedByProvider pins that the harness web_search seam
-// is produced only when a provider is configured (mirroring Python's
-// RAGTools.web_search provider gate). A nil callback is what keeps the tool off
-// the agentic surface.
-func TestHarnessWebSearchFuncGatedByProvider(t *testing.T) {
- s := &ChatPipelineService{}
- if fn := s.harnessWebSearchFunc(nil); fn != nil {
- t.Error("nil prompt config must yield no web-search callback")
- }
- if fn := s.harnessWebSearchFunc(map[string]interface{}{}); fn != nil {
- t.Error("empty prompt config must yield no web-search callback")
- }
- if fn := s.harnessWebSearchFunc(map[string]interface{}{"tavily_api_key": "tvly-test"}); fn == nil {
- t.Error("configured provider must yield a web-search callback")
- }
-}
-
func TestWebSearchFuncGatedByProvider(t *testing.T) {
s := &ChatPipelineService{}
for _, config := range []map[string]interface{}{nil, {}, {"tavily_api_key": " "}} {
diff --git a/web/src/locales/az.ts b/web/src/locales/az.ts
index 968076f5aa..926054d7b4 100644
--- a/web/src/locales/az.ts
+++ b/web/src/locales/az.ts
@@ -1197,14 +1197,6 @@ Bu avtomatik etiketləmə funksiyası mövcud verilənlər dəstinə sahəyə ai
thinkingEnabled: 'Aktivdir',
thinkingDisabled: 'Deaktiv',
thinkingLevelNone: 'Sadə',
- thinkingLevelLow: 'Aşağı',
- thinkingLevelLowDescription: 'Sürətli cavablar',
- thinkingLevelMedium: 'Orta',
- thinkingLevelMediumDescription: 'Sürət və dərinlik arasında tarazlıq',
- thinkingLevelHigh: 'Yüksək',
- thinkingLevelHighDescription: 'Dərin mühakimə',
- thinkingLevelUltra: 'Ən yüksək',
- thinkingLevelUltraDescription: 'Maksimum idrak səyi',
thinkingTip:
'Yalnız rəsmi Qwen, Kimi və GLM model provayderləri üçün düşünmə rejimini idarə edir. Sistemin standart dəyəri uzun çəkən tapşırıqların qarşısını almaq üçün Qwen modelinin düşünmə rejimini söndürür.',
quote: 'İstinadları göstər',
diff --git a/web/src/locales/ja.ts b/web/src/locales/ja.ts
index f04482d8f8..371333d397 100644
--- a/web/src/locales/ja.ts
+++ b/web/src/locales/ja.ts
@@ -993,10 +993,6 @@ export default {
thinkingDefault: 'システムデフォルト',
thinkingEnabled: '有効',
thinkingDisabled: '無効',
- thinkingLevelLow: '低',
- thinkingLevelMedium: '中',
- thinkingLevelHigh: '高',
- thinkingLevelUltra: '最高',
thinkingTip:
'Qwen、Kimi、GLMの公式モデルプロバイダーの思考モードのみを制御します。システムデフォルトでは、処理の長時間化を防ぐためQwenの思考を無効にします。',
published: '公開済み',
diff --git a/web/src/locales/tr.ts b/web/src/locales/tr.ts
index 5300cc49f6..a255ea5d87 100644
--- a/web/src/locales/tr.ts
+++ b/web/src/locales/tr.ts
@@ -1163,14 +1163,6 @@ Bu otomatik etiketleme özelliği, mevcut datasete alanına özgü bilgi katman
thinkingEnabled: 'Etkin',
thinkingDisabled: 'Devre dışı',
thinkingLevelNone: 'Yalın',
- thinkingLevelLow: 'Düşük',
- thinkingLevelLowDescription: 'Hızlı yanıtlar',
- thinkingLevelMedium: 'Orta',
- thinkingLevelMediumDescription: 'Dengeli hız ve derinlik',
- thinkingLevelHigh: 'Yüksek',
- thinkingLevelHighDescription: 'Derin akıl yürütme',
- thinkingLevelUltra: 'Ultra',
- thinkingLevelUltraDescription: 'Maksimum bilişsel çaba',
thinkingTip:
'Yalnızca resmi Qwen, Kimi ve GLM model sağlayıcıları için düşünme modunu denetler. Sistem varsayılanı, uzun süren görevlerden kaçınmak için Qwen düşünmesini devre dışı bırakır.',
quote: 'Alıntı göster',
diff --git a/web/src/locales/uz.ts b/web/src/locales/uz.ts
index 1ab312ba01..5594c6d095 100644
--- a/web/src/locales/uz.ts
+++ b/web/src/locales/uz.ts
@@ -1086,14 +1086,6 @@ export default {
thinkingEnabled: 'Yoqilgan',
thinkingDisabled: 'O‘chirilgan',
thinkingLevelNone: 'Oddiy',
- thinkingLevelLow: 'Past',
- thinkingLevelLowDescription: 'Tez javoblar',
- thinkingLevelMedium: 'O‘rta',
- thinkingLevelMediumDescription: 'Tezlik va chuqurlik muvozanati',
- thinkingLevelHigh: 'Yuqori',
- thinkingLevelHighDescription: 'Chuqur fikrlash',
- thinkingLevelUltra: 'Eng yuqori',
- thinkingLevelUltraDescription: 'Maksimal aqliy kuch-g‘ayrat',
thinkingTip:
'Faqat rasmiy Qwen, Kimi va GLM model provayderlarining fikrlash rejimini boshqaradi. Tizim standarti uzoq vazifalardan qochish uchun Qwen fikrlashini o‘chiradi.',
quote: 'Iqtiboslarni ko‘rsatish',