Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(harness): extract retrieval wiring from server startup (#20012)

Sbaaoui Idriss2d9acf7762b2
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go
index 2b11f0299e..33468b7a7f 100644
--- a/cmd/ragflow_server.go
+++ b/cmd/ragflow_server.go
@@ -40,7 +40,6 @@ import (
"ragflow/internal/ingestion/knowledge_compile"
ingestion "ragflow/internal/ingestion/service"
"ragflow/internal/rag/agentic-rag"
- agenticruntime "ragflow/internal/rag/agentic-rag/runtime"
"ragflow/internal/router"
"ragflow/internal/server/local"
"ragflow/internal/service"
@@ -69,7 +68,6 @@ import (
"ragflow/internal/deepdoc/parser/pdf/inference/native_analyzer"
"ragflow/internal/engine"
"ragflow/internal/engine/kvrocks"
- et "ragflow/internal/engine/types"
"ragflow/internal/entity"
_ "ragflow/internal/ingestion/wire"
"ragflow/internal/server"
@@ -92,94 +90,6 @@ type serverArgs struct {
}
-// engineDocEngine is the small slice of the engine surface the doc-chunk pager
-// needs, kept as a local interface so it is trivially unit-testable without a
-// live engine (the production value is engine.Get()).
-type engineDocEngine interface {
- Search(ctx context.Context, req *et.SearchRequest) (*et.SearchResult, error)
-}
-
-// docChunkPager is the production agenticruntime.DocChunkLister. It pages one
-// document's chunks in reading order straight off the chunk index, mirroring
-// Python settings.retriever.chunk_list with sort_by_position=True
-// (rag/nlp/search.py:824). This is what the document-level tools
-// (summarize_document / fetch_full_document) consume.
-//
-// The index name is derived from the tenant id (ragflow_<tenantID>), exactly as
-// retrieval does, so it needs no user-scope resolution: DocChunksRequest
-// already carries the tenant and the bound datasets.
-type docChunkPager struct {
- docEngine engineDocEngine
-}
-
-// DocChunks implements agenticruntime.DocChunkLister.
-func (p *docChunkPager) DocChunks(ctx context.Context, req agenticruntime.DocChunksRequest) ([]map[string]any, error) {
- if p == nil || p.docEngine == nil {
- return nil, nil
- }
- if req.DocID == "" {
- return nil, nil
- }
- orderBy := (&et.OrderByExpr{}).
- Asc("chunk_order_int").
- Asc("page_num_int").
- Asc("top_int").
- Desc("create_timestamp_flt")
-
- resp, err := p.docEngine.Search(ctx, &et.SearchRequest{
- IndexNames: []string{tenantChunkIndexName(req.TenantID)},
- KbIDs: req.DatasetIDs,
- Offset: req.Offset,
- Limit: req.Limit,
- OrderBy: orderBy,
- SelectFields: []string{
- "id", "doc_id", "docnm", "content_with_weight", "content", "available_int",
- },
- Filter: map[string]interface{}{
- "doc_id": req.DocID,
- "available_int": 1,
- "must_not": map[string]interface{}{"exists": "compile_kwd"},
- },
- })
- if err != nil {
- return nil, err
- }
- if resp == nil {
- return nil, nil
- }
- rows := make([]map[string]any, 0, len(resp.Chunks))
- for _, ck := range resp.Chunks {
- row := make(map[string]any, 8)
- // The runtime chunk rows need the canonical keys its citation pipeline
- // reads: chunk_id (dedup), docnm_kwd (doc title), doc_id and the text
- // under content/content_with_weight (chunkText prefers content first).
- if id, ok := ck["id"].(string); ok && id != "" {
- row["chunk_id"] = id
- }
- if d, ok := ck["doc_id"]; ok {
- row["doc_id"] = d
- }
- if d, ok := ck["docnm"]; ok {
- row["docnm_kwd"] = d
- }
- if c, ok := ck["content"]; ok {
- row["content"] = c
- }
- if c, ok := ck["content_with_weight"]; ok {
- row["content_with_weight"] = c
- }
- // Keep whatever else the engine returned (e.g. position_int, img) so the
- // merged Kbinfos rows stay rich, as retrieval does.
- for k, v := range ck {
- if _, seen := row[k]; !seen {
- row[k] = v
- }
- }
- rows = append(rows, row)
- }
- return rows, nil
-}
-
func parseArgs() (*serverArgs, error) {
args := &serverArgs{}
@@ -1118,209 +1028,7 @@ func startServer(ctx context.Context) error {
// reasoning chats.
runtime.SetRetrievalService(retrievalbridge.NewRuntimeAdapter(retrievalAdapter))
agentic_rag.SetAgenticLoop(agentic_rag.NewAgenticLoop())
- service.SetHarnessRetriever(func(ctx context.Context, req service.HarnessRequest) (service.HarnessResult, error) {
- // Resolve the tenant's actual chat model, mirroring Python where RAGTools
- // receives a fully-resolved LLMBundle: chat.LLMID may be a UUID/tenant_model
- // id that the default invoker cannot split into a provider, so resolving here
- // avoids falling through to a dummy driver. If resolution fails we leave model
- // nil so the runtime degrades to a direct search (no dummy fallback).
- var model agenticruntime.SessionModel
- // outerModel mirrors Python RAGTools.chat_mdl: the fully-resolved chat
- // model that drives the outer rag_agent react loop (tools=[rag,
- // summarize_document], terminal_tools={"rag"}). Wired into RAGTools.Outer
- // so Rag() can run that loop instead of the direct graph.
- var outerModel *modelModule.ChatModel
- // resolvedModelName is the resolved chat model's own name (e.g.
- // "gpt-4o"), i.e. Python chat_mdl.llm_name. It is the gen_json reply-cache
- // key's model component; empty disables that cache.
- resolvedModelName := ""
- // The resolution goes through the same entry point the pipeline's own
- // capability probe uses, so the model decided there is the model wired
- // here: req.ModelID may be empty (a dialog without an llm_id), and both
- // layers must then fall back to the tenant default rather than resolving
- // nothing here and judging capability there.
- if target, mErr := modelProviderService.ResolveChatModelTarget(ctx, req.TenantID, req.ModelID); mErr == nil {
- if inv := component.NewResolvedInvoker(target.Driver, target.ModelName, target.APIConfig); inv != nil {
- // MaxLength mirrors Python LLMBundle.max_length (the model's
- // context window in tokens); message-fitting nodes (calculate,
- // structure_qa) use it as their chat.FitMessages budget.
- model = &agenticruntime.InvokerSessionModel{Invoker: inv, DB: dao.DB, MaxLength: target.ContextLength}
- resolvedModelName = target.ModelName
- // Reuse the same resolved driver/name/api as the invoker so
- // the outer loop and the inner tool calls share one model.
- outerModel = modelModule.NewChatModel(target.Driver, &target.ModelName, target.APIConfig)
- }
- } else {
- common.Warn("runtime: failed to resolve chat model for reasoning; runtime will degrade to direct search", zap.Error(mErr))
- }
-
- // Load the KB objects (mirroring Python RAGTools' self.kbs via
- // KnowledgebaseService.get_by_ids(kb_ids)) so the agentic tool can
- // derive rank features. The Go tag extractor (extractor_tag.go) writes
- // both tag_kwd (the list of tag names) and tag_feas (per-tag weights)
- // onto each chunk at parse time; the labeler aggregates tag_kwd to build
- // the tag vocabulary and the retriever ranks with tag_feas. Best-effort:
- // a load failure leaves KBs empty and the adapter resolves them itself.
- var kbs []*entity.Knowledgebase
- if len(req.DatasetIDs) > 0 {
- if loaded, lErr := dao.NewKnowledgebaseDAO().GetByIDs(ctx, dao.DB, req.DatasetIDs); lErr == nil {
- kbs = loaded
- }
- }
- // validate_dataset_embedding_models runs FIRST upstream
- // (dialog_service.py:358-360) and mixing is a hard failure there, not a
- // fallback to keyword-only retrieval.
- if err := validateDatasetEmbeddingModels(ctx, kbs); err != nil {
- return service.HarnessResult{}, err
- }
- // HasEmbedder mirrors Python `embd_mdl = ... if kbs and kbs[0].embd_id else
- // None` (dialog_service.py:362): the gate is the FIRST dataset's embd_id. It
- // is what hybrid_search applies to its vector leg (search.py:143), so
- // getting it wrong silently degrades search_chunks to keyword-only.
- hasEmbedder := hasEmbedderFor(kbs)
- // Retrieval tuning mirrors the dialog's own settings, so the agentic
- // searches run with the same budget as the standard path. Without it the
- // harness falls back to its package defaults (top_n=12) and the dialog's
- // configured top_n is silently ignored.
- //
- // The runtime carries the KEYWORD leg's weight and derives the vector leg
- // as its complement (runtime.resolveKeywordsSimilarityWeight, and the
- // RetrieveRequest contract: 0.7 hybrid / 0.0 vector-only / 1.0
- // keyword-only). The dialog configures the VECTOR weight, so it is inverted
- // here; the dialog's default 0.3 therefore yields the runtime's own 0.7.
- keywordsWeight := 1 - req.Retrieval.VectorSimilarityWeight
- deps := agentic_rag.RAGTools{
- Model: model,
- ModelName: resolvedModelName,
- Outer: outerModel,
- TopN: req.Retrieval.TopN,
- SimilarityThreshold: req.Retrieval.SimilarityThreshold,
- KeywordsSimilarityWeight: &keywordsWeight,
- RerankCandidatesCount: req.Retrieval.RerankCandidatesCount,
- // OriginalQuestion mirrors Python RAGTools(original_user_question=...):
- // the user's own, unrewritten question as received from the chat
- // layer. The outer model's `rag(question=...)` argument is
- // model-generated and often compresses a multi-hop question to its
- // first hop; resolveEffectiveQuestion (agentic_rag.py:865) prefers
- // this original over that rewrite when both describe the same turn.
- OriginalQuestion: req.Question,
- // DocScope mirrors Python RAGTools(doc_scope=...): the narrowed doc_ids
- // (chat-level doc_ids + meta_data_filter) restrict every agentic
- // retrieval to the user-selected documents instead of the whole kb.
- DocScope: req.DocIDs,
- DocIDVerifier: agentic_rag.NewDocIDLookup(),
- // MetadataResolver backs the metadata_search tool and the pre-search
- // metadata channel (Python DocMetadataService push-down + meta_filter
- // fallback). metadataService already carries the doc engine and the DAO
- // the metadata index reads need.
- MetadataResolver: metadataService,
- // DeclaredMetadata lets the metadata_search catalog describe each field
- // (meaning + allowed values) from the dataset's own parser_config, not just
- // name it. Same service: it also carries the KB DAO that read needs.
- DeclaredMetadata: metadataService,
- // DocChunks pages one document's chunks in reading order off the
- // chunk index (Python retriever.chunk_list), backing the
- // document-level tools (summarize_document / fetch_full_document).
- DocChunks: &docChunkPager{docEngine: docEngine},
- // Expand runs search_chunks' compiled-structure expansion
- // (Python hybrid_search use_compiled=True → _expand_with_compiled).
- // Backed by the same document engine the chunk reads use, with the
- // dense seed leg wired to the bound dataset's own embedding model.
- // The scope config lets each dataset be scanned under its own tenant
- // and a doc scope be grouped by real owner, like graph_explore.
- Expand: agenticruntime.NewCompiledExpander(
- newDatasetCompiledStore(docEngine, modelProviderService, kbs),
- agenticruntime.CompiledScopeConfig{
- DatasetIDs: req.DatasetIDs,
- TenantID: req.TenantID,
- KBs: kbs,
- DocTenantResolver: agentic_rag.NewDocTenantResolver(),
- },
- ),
- // WebSearch backs the web_search tool (Python RAGTools.web_search).
- // The pipeline supplies a callback only when the chat enables
- // internet web search; a nil one also keeps the tool off the surface.
- WebSearch: harnessWebSearcher(req.WebSearch),
- KBs: kbs,
- HasEmbedder: hasEmbedder,
- // Embedder backs claim recall's KNN leg (Python
- // recall_dataset_claims, navigation.py:1837-1909, which embeds the
- // query with tools.embed_mdl — the FIRST dataset's embedding model,
- // dialog_service.py:362-366) and the structure-drill seed vector.
- // Without it the claim leg silently degrades to BM25-only and
- // paraphrase-phrased claims are never recalled. Bound to
- // kbs[0].EmbdID (validateDatasetEmbeddingModels above guarantees
- // every bound dataset shares it), query-side encoded.
- Embedder: embedderForDatasets(kbs, modelProviderService),
- // Tagger is the Go equivalent of Python's label_question
- // (agentic_rag.py:668): classifies the query into question-type
- // tags the retriever boosts on. metadataService implements it
- // (service.MetadataService.LabelQuestion).
- Tagger: metadataService,
- // SystemPrompt mirrors Python RAGTools(system_prompt=
- // _render_reasoning_system_prompt(dialog, prompt_config, kwargs),
- // dialog_service.py:2084) — the dialog-level UI configuration the
- // final-answer compose appends after the agentic contract
- // (agentic_rag_graph.py:923-933).
- SystemPrompt: req.SystemPrompt,
- // CiteRules and EvidenceMaxTokens deliberately stay unset: Python's
- // dialog path constructs RAGTools WITHOUT user_defined_prompts
- // (dialog_service.py:2072-2090), so citation_prompt defaults apply,
- // and Python has no evidence-token override (the compose always
- // uses min(chat_mdl.max_length, _EVIDENCE_BUDGET_TOKENS=8000),
- // which EvidenceMaxTokens<=0 reproduces).
- }
- // Diagnose WHY compiled expansion is disabled: NewCompiledExpander
- // returns nil for three reasons (store==nil / no datasets / no tenant)
- // and RAGTools.Expand==nil silences the whole channel with no other
- // trace — the observed "Compiled expansion enabled = 0" was invisible.
- if deps.Expand == nil {
- switch {
- case docEngine == nil:
- common.Warn("compiled expansion disabled: document engine unavailable (store==nil)")
- case len(req.DatasetIDs) == 0:
- common.Warn("compiled expansion disabled: no bound dataset (DatasetIDs empty)")
- case strings.TrimSpace(req.TenantID) == "":
- common.Warn("compiled expansion disabled: no tenant (TenantID empty)")
- default:
- common.Warn("compiled expansion disabled: unknown reason")
- }
- }
- // The two projections of one reasoning step: the sentence the chat UI
- // appends to its think block, and the structured event a step-rendering
- // client consumes. Steps.Stage/Emit feeds both from one call, so the
- // trace cannot drift from its structured twin. Each step is an
- // isThink=true delta, i.e. think-block content rather than answer text.
- if req.AnswerSink != nil {
- answerSink := req.AnswerSink
- deps.AnswerSink = &agentic_rag.AnswerSink{
- OnDelta: req.AnswerSink,
- }
- deps.Steps.Text = func(line string) { answerSink(line, true) }
- }
- if req.ThinkSink != nil {
- // One type on both sides — agenticruntime.ThinkEvent is an alias of
- // service.ThinkEvent — so the sink passes straight through: there is
- // nothing to copy field by field, and no way to forget a field that
- // was added on one side only.
- deps.Steps.Events = req.ThinkSink
- }
- r := agentic_rag.Rag(ctx, deps, agenticruntime.RunRequest{
- Question: req.Question,
- ThinkingMode: req.ThinkingMode,
- DatasetIDs: req.DatasetIDs,
- TenantID: req.TenantID,
- SessionID: req.SessionID,
- Images: req.Images,
- TextAttachments: req.TextAttachments,
- })
- res := service.HarnessResult{Chunks: r.Chunks, DocAggs: r.DocAggs, Answer: r.Answer, SlotCitations: r.SlotCitations, CiteChunkIDs: r.CiteChunkIDs}
- if r.Kbinfos != nil {
- res.Memory = r.Kbinfos.Memory
- res.PreSummary = r.Kbinfos.PreSummary
- }
- return res, nil
- })
+ service.SetHarnessRetriever(retrievalbridge.NewHarnessRetriever(modelProviderService, metadataService, docEngine))
common.Info("agent: runtime chat retriever wired (runtime retrieval + agentic loop)")
// Initialize handler layer
@@ -1806,395 +1514,3 @@ func resolveDeepDocDropScore() float64 {
func dirHasModels(dir string) bool {
return common.HasModelFiles(dir)
}
-
-// hasEmbedderFor mirrors Python `embd_mdl = LLMBundle(...) if kbs and
-// kbs[0].embd_id else None` (dialog_service.py:362). The gate is the FIRST bound
-// dataset's embd_id, not "any dataset has one": a dialog whose first dataset
-// carries no embedding model yields embd_mdl=None upstream even if later ones do.
-//
-// Callers must run validateDatasetEmbeddingModels first — with validation
-// passing, all datasets share one model and this equals "all have one".
-func hasEmbedderFor(kbs []*entity.Knowledgebase) bool {
- return len(kbs) > 0 && kbs[0] != nil && kbs[0].EmbdID != ""
-}
-
-// embedderForDatasets builds the embedding handle the agentic runtime uses for
-// query-side encoding outside the main retrieval leg: claim recall's KNN leg
-// (Python recall_dataset_claims, navigation.py:1837-1909, embedding with
-// tools.embed_mdl) and the structure-drill seed vector. The model is the FIRST
-// dataset's embedding (Python dialog_service.py:362-366 resolves
-// kbs[0].embd_id under kbs[0].tenant_id; validateDatasetEmbeddingModels
-// guarantees the rest share it). Nil when no dataset carries an embedding
-// model — the claim leg then degrades to BM25-only, matching a Python run
-// without embed_mdl.
-func embedderForDatasets(kbs []*entity.Knowledgebase, modelSvc *service.ModelProviderService) nlp.NavEmbedder {
- if len(kbs) == 0 || kbs[0] == nil || kbs[0].EmbdID == "" {
- return nil
- }
- return service.NewNavEmbedder(modelSvc, kbs[0].EmbdID)
-}
-
-// validateDatasetEmbeddingModels mirrors Python validate_dataset_embedding_models
-// (knowledgebase_service.py:62-94): every bound dataset must use the same embedding
-// model, or none at all. Upstream this runs before embd_mdl is resolved
-// (dialog_service.py:358-360) and a violation RAISES — it is not a "no embedding
-// model" fallback: treating mixing as HasEmbedder=false would silently serve
-// keyword-only results where Python errors.
-func validateDatasetEmbeddingModels(ctx context.Context, kbs []*entity.Knowledgebase) error {
- var withEmbd int
- for _, kb := range kbs {
- if kb != nil && kb.EmbdID != "" {
- withEmbd++
- }
- }
- hasEmbd := withEmbd > 0
- if hasEmbd && withEmbd != len(kbs) {
- return errors.New("cannot search across datasets where some have embedding models and others do not")
- }
- if !hasEmbd {
- return nil
- }
- // Mirror Python's grouping key exactly:
- //
- // ref = tenant_embd_id or (embd_id when it is a raw tenant_model id)
- // composite = resolved_names.get(ref) # "model@instance@provider"
- // key = _base_model_name(composite) # == tenant_model.model_name
- //
- // The @instance@provider suffix is stripped by _base_model_name
- // (knowledgebase_service.py:33), so only the MODEL NAME is compared — the
- // instance and provider tables are not consulted at all.
- refs := make([]string, 0, len(kbs))
- for _, kb := range kbs {
- if kb == nil || kb.EmbdID == "" {
- continue
- }
- ref := ""
- if kb.TenantEmbdID != nil {
- ref = strings.TrimSpace(*kb.TenantEmbdID)
- }
- if ref == "" && !strings.Contains(kb.EmbdID, "@") {
- ref = strings.TrimSpace(kb.EmbdID)
- }
- if ref != "" {
- refs = append(refs, ref)
- }
- }
-
- // Python resolves the refs with one batched lookup
- // (tenant_model_service.py:562 get_by_ids) and silently skips ids that no
- // longer resolve — a dangling id must not fail the request.
- resolved := make(map[string]string, len(refs))
- // A nil/absent DB or any query error must not fail the request: Python skips
- // unresolvable ids (tenant_model_service.py:557) and falls back to the raw
- // id, so validation degrades to a stricter but safe comparison.
- if len(refs) > 0 && dao.DB != nil {
- if models, err := dao.NewTenantModelDAO().GetByIDs(ctx, dao.DB, refs); err == nil {
- for _, m := range models {
- if m != nil {
- resolved[m.ID] = m.ModelName
- }
- }
- }
- }
-
- keys := make(map[string]struct{}, len(kbs))
- for _, kb := range kbs {
- if kb == nil || kb.EmbdID == "" {
- continue
- }
- embdID := strings.TrimSpace(kb.EmbdID)
- ref := ""
- if kb.TenantEmbdID != nil {
- ref = strings.TrimSpace(*kb.TenantEmbdID)
- }
- if ref == "" && !strings.Contains(embdID, "@") {
- ref = embdID
- }
- key := ""
- if name, ok := resolved[ref]; ok && name != "" {
- key = name
- } else if embdID != "" && embdID != ref {
- key = baseModelName(embdID)
- } else {
- key = ref
- }
- if key != "" {
- keys[key] = struct{}{}
- }
- }
- if len(keys) > 1 {
- return fmt.Errorf("datasets use different embedding models")
- }
- return nil
-}
-
-// baseModelName mirrors Python _base_model_name
-// (knowledgebase_service.py:33): strip the @instance@provider suffix, keeping the
-// model name. Python uses rsplit("@", 2)[0], which for the purposes of taking
-// the leading segment is the same as splitting once on "@".
-func baseModelName(embdID string) string {
- name, _, _ := strings.Cut(embdID, "@")
- return name
-}
-
-// ---------------------------------------------------------------------------
-// Harness seams (compiled expansion + open-web search)
-// ---------------------------------------------------------------------------
-
-// engineCompiledStore implements agenticruntime.CompiledStore over the document
-// engine, backing search_chunks' compiled-structure expansion (Python
-// tools/compiled_expansion.py). The runtime owns the expansion strategy; this
-// adapter only maps its (scope, filters, free text) request onto one engine
-// search, mirroring Python's settings.docStoreConn.search calls.
-type engineCompiledStore struct {
- engine engineDocEngine
- // embed encodes the seed query for the dense leg. It must be the QUERY
- // encoding of the dataset's own embedding model (Python
- // _search_compiled_rows → _dense_expr → settings.retriever.get_vector →
- // emb_mdl.encode_queries). Nil leaves the keyword leg only.
- embed compiledQueryEncoder
- // embedTenantID is the tenant owning the embedder's model (Python
- // embd_owner_tenant_id = kbs[0].tenant_id). It is distinct from the searched
- // scope's tenant, because Python builds ONE embedder from the leading bound
- // dataset and reuses it for every scope.
- embedTenantID string
-}
-
-var _ agenticruntime.CompiledStore = (*engineCompiledStore)(nil)
-
-// compiledQueryEncoder is the query-side embedding seam used for the compiled
-// dense seed leg. *service.NavEmbedder implements it via EncodeQueries.
-type compiledQueryEncoder interface {
- EncodeQueries(ctx context.Context, tenantID string, texts []string) ([][]float32, error)
-}
-
-// Mirror Python compiled_expansion._VECTOR_NUM_CANDIDATES / _VECTOR_SIMILARITY.
-// The HNSW ef_search floor (num_candidates) must be >= topN or the ANN search is
-// bounded by the candidate list instead of by relevance.
-const (
- compiledVectorNumCandidates = 256
- compiledVectorSimilarity = 0.1
-)
-
-// newEngineCompiledStore returns nil when no engine is available so
-// agenticruntime.NewCompiledExpander disables expansion instead of searching nowhere.
-// A nil embed (or empty embedTenantID) keeps the keyword-only leg.
-func newEngineCompiledStore(e engineDocEngine, embed compiledQueryEncoder, embedTenantID string) agenticruntime.CompiledStore {
- if e == nil {
- return nil
- }
- return &engineCompiledStore{engine: e, embed: embed, embedTenantID: embedTenantID}
-}
-
-// newDatasetCompiledStore builds the per-request compiled-row store, wiring the
-// dense seed leg to the FIRST bound dataset's own embedding model. Python builds
-// embd_mdl from kbs[0] (dialog_service.py:362-365) and reuses it for the whole
-// expansion; using the tenant-default model instead would compare the query
-// against rows embedded by a different model — different vector spaces, so the
-// dense hits would be noise.
-func newDatasetCompiledStore(e engineDocEngine, modelSvc *service.ModelProviderService, kbs []*entity.Knowledgebase) agenticruntime.CompiledStore {
- if e == nil {
- return nil
- }
- if hasEmbedderFor(kbs) {
- return newEngineCompiledStore(e, service.NewNavEmbedder(modelSvc, kbs[0].EmbdID), kbs[0].TenantID)
- }
- return newEngineCompiledStore(e, nil, "")
-}
-
-const tenantChunkIndexPrefix = "ragflow_"
-
-func tenantChunkIndexName(tenantID string) string {
- return tenantChunkIndexPrefix + tenantID
-}
-
-// compiledSynthesisKinds are the standalone synthesis-page compile_kwd values.
-// Python scopes those rows by source_doc_ids, while entity/relation rows are
-// scoped by doc_id (compiled_expansion.py _search_compiled_rows /
-// _search_synthesis_pages).
-var compiledSynthesisKinds = map[string]bool{
- "wiki_page": true,
- "artifact_page": true,
- "essence": true,
-}
-
-// compiledRowSelectFields is the projection the expander reads off a compiled
-// row: content_with_weight (seed name parsing), the entity/relation endpoints,
-// exact name lookups, and provenance back to source chunks.
-var compiledRowSelectFields = []string{
- "id", "kb_id", "doc_id", "docnm_kwd",
- "content_with_weight", "summary_with_weight", "source_chunk_ids",
- "name_kwd", "from_entity_kwd", "to_entity_kwd", "available_int",
-}
-
-// compiledChunkSelectFields is the projection for source chunks promoted into
-// evidence by compiled expansion.
-var compiledChunkSelectFields = []string{
- "id", "kb_id", "doc_id", "docnm_kwd",
- "content_with_weight", "source_chunk_ids", "similarity",
-}
-
-// SearchCompiled implements agenticruntime.CompiledStore.
-func (s *engineCompiledStore) SearchCompiled(ctx context.Context, kbID, tenantID string, docIDs []string, filters map[string][]string, matchText string, topN int) ([]map[string]any, error) {
- if s == nil || s.engine == nil || kbID == "" || strings.TrimSpace(tenantID) == "" {
- return nil, nil
- }
- if topN <= 0 {
- topN = 1
- }
- filter := compiledEngineFilter(filters)
- if len(docIDs) > 0 {
- filter[compiledDocScopeKey(filters)] = docIDs
- }
- req := &et.SearchRequest{
- IndexNames: []string{tenantChunkIndexName(tenantID)},
- KbIDs: []string{kbID},
- Limit: topN,
- SelectFields: compiledRowSelectFields,
- Filter: filter,
- }
- // Python prefers a dense seed leg and only falls back to a keyword match
- // when no embedder is available or the vector build fails
- // (compiled_expansion.py _search_compiled_rows:180-194 /
- // _search_synthesis_pages:423-438). Exactly ONE expr is appended.
- if text := strings.TrimSpace(matchText); text != "" {
- if dense := s.denseExpr(ctx, text, topN); dense != nil {
- req.MatchExprs = []interface{}{dense}
- } else {
- req.MatchExprs = []interface{}{&et.MatchTextExpr{
- Fields: []string{"content_ltks", "content_sm_ltks"},
- MatchingText: text,
- TopN: topN,
- }}
- }
- }
- res, err := s.engine.Search(ctx, req)
- if err != nil || res == nil {
- return nil, err
- }
- return res.Chunks, nil
-}
-
-// denseExpr builds the compiled-row dense match for the seed text, mirroring
-// Python _dense_expr (compiled_expansion.py:31-47): the query is embedded with
-// the dataset model's QUERY encoding, and the match carries the HNSW
-// num_candidates floor and similarity threshold. Returns nil when no embedder is
-// wired, the encode fails, or it panics — every caller then uses the keyword leg,
-// exactly like Python's `except Exception` around get_vector.
-func (s *engineCompiledStore) denseExpr(ctx context.Context, text string, topN int) *et.MatchDenseExpr {
- if s == nil || s.embed == nil || strings.TrimSpace(s.embedTenantID) == "" {
- return nil
- }
- var (
- vecs [][]float32
- err error
- )
- func() {
- defer func() {
- if r := recover(); r != nil {
- common.Warn("compiled expansion: seed encode panicked; falling back to keyword match")
- }
- }()
- vecs, err = s.embed.EncodeQueries(ctx, s.embedTenantID, []string{text})
- }()
- if err != nil || len(vecs) == 0 || len(vecs[0]) == 0 {
- return nil
- }
- vec := make([]float64, len(vecs[0]))
- for i, v := range vecs[0] {
- vec[i] = float64(v)
- }
- return &et.MatchDenseExpr{
- VectorColumnName: fmt.Sprintf("q_%d_vec", len(vec)),
- EmbeddingData: vec,
- EmbeddingDataType: "float",
- DistanceType: "cosine",
- TopN: topN,
- ExtraOptions: map[string]interface{}{
- "similarity": compiledVectorSimilarity,
- "num_candidates": max(topN, compiledVectorNumCandidates),
- },
- }
-}
-
-// LoadChunks implements agenticruntime.CompiledStore: fetch the referenced source
-// chunks by id (Python compiled_expansion.py:_load_chunks_for_doc searches the
-// tenant index with an {"id": chunk_ids} condition).
-func (s *engineCompiledStore) LoadChunks(ctx context.Context, kbID, tenantID string, chunkIDs []string) ([]map[string]any, error) {
- if s == nil || s.engine == nil || kbID == "" || strings.TrimSpace(tenantID) == "" || len(chunkIDs) == 0 {
- return nil, nil
- }
- res, err := s.engine.Search(ctx, &et.SearchRequest{
- IndexNames: []string{tenantChunkIndexName(tenantID)},
- KbIDs: []string{kbID},
- Limit: len(chunkIDs),
- SelectFields: compiledChunkSelectFields,
- Filter: map[string]interface{}{"id": chunkIDs},
- })
- if err != nil || res == nil {
- return nil, err
- }
- return res.Chunks, nil
-}
-
-// Vectorize implements agenticruntime.CompiledStore. The ported expander assigns
-// compiled chunks the similarity stored on the row and blends by that field, so
-// expansion itself never needs a query embedding.
-func (s *engineCompiledStore) Vectorize(context.Context, string) ([]float64, error) {
- return nil, fmt.Errorf("compiled store: vectorize is not wired")
-}
-
-// compiledDocScopeKey mirrors Python's doc-scope column choice: synthesis pages
-// are scoped by source_doc_ids, entity/relation rows by doc_id.
-func compiledDocScopeKey(filters map[string][]string) string {
- for _, ck := range filters["compile_kwd"] {
- if compiledSynthesisKinds[strings.ToLower(strings.TrimSpace(ck))] {
- return "source_doc_ids"
- }
- }
- return "doc_id"
-}
-
-// compiledEngineFilter maps the runtime' OR-list filters onto the engine filter
-// shape. available_int is a numeric column, so its string values are coerced to
-// ints (Python passes the literal 1).
-func compiledEngineFilter(filters map[string][]string) map[string]interface{} {
- out := make(map[string]interface{}, len(filters))
- for k, vals := range filters {
- if k == "available_int" {
- ints := make([]int, 0, len(vals))
- for _, v := range vals {
- if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil {
- ints = append(ints, n)
- }
- }
- if len(ints) > 0 {
- out[k] = ints
- }
- continue
- }
- out[k] = vals
- }
- return out
-}
-
-// harnessWebSearcherFunc adapts the chat pipeline's web-search callback to the
-// agenticruntime.WebSearcher seam consumed by the web_search tool. The callback is a
-// plain func so internal/service does not have to depend on the runtime package.
-type harnessWebSearcherFunc func(ctx context.Context, queries []string) ([]string, error)
-
-// Search implements agenticruntime.WebSearcher.
-func (f harnessWebSearcherFunc) Search(ctx context.Context, queries []string) ([]string, error) {
- return f(ctx, queries)
-}
-
-var _ agenticruntime.WebSearcher = harnessWebSearcherFunc(nil)
-
-// harnessWebSearcher wraps a non-nil callback, returning a nil WebSearcher when
-// the pipeline did not supply one — which is what hides the web_search tool.
-func harnessWebSearcher(fn func(context.Context, []string) ([]string, error)) agenticruntime.WebSearcher {
- if fn == nil {
- return nil
- }
- return harnessWebSearcherFunc(fn)
-}
diff --git a/cmd/ragflow_server_test.go b/cmd/ragflow_server_test.go
index 288831c96a..4413c701ac 100644
--- a/cmd/ragflow_server_test.go
+++ b/cmd/ragflow_server_test.go
@@ -18,23 +18,14 @@ package main
import (
"context"
- "errors"
"os"
+ "os/exec"
+ "path/filepath"
+ "strings"
"testing"
-
- "ragflow/internal/engine/types"
- "ragflow/internal/entity"
- "ragflow/internal/service"
+ "time"
)
-func kb(embdID string, tenantEmbdID string) *entity.Knowledgebase {
- kb := &entity.Knowledgebase{EmbdID: embdID}
- if tenantEmbdID != "" {
- kb.TenantEmbdID = &tenantEmbdID
- }
- return kb
-}
-
func parseArgsForTest(t *testing.T, argv ...string) (*serverArgs, error) {
t.Helper()
orig := os.Args
@@ -43,122 +34,6 @@ func parseArgsForTest(t *testing.T, argv ...string) (*serverArgs, error) {
return parseArgs()
}
-// TestHasEmbedderForMirrorsPython pins dialog_service.py:362 —
-// `embd_mdl = LLMBundle(...) if kbs and kbs[0].embd_id else None`. The gate is
-// the FIRST dataset's embd_id, so a dialog whose leading dataset has no model
-// yields no embedding model even when a later one does.
-func TestHasEmbedderForMirrorsPython(t *testing.T) {
- cases := []struct {
- name string
- kbs []*entity.Knowledgebase
- want bool
- }{
- {"no datasets", nil, false},
- {"single dataset with model", []*entity.Knowledgebase{kb("bge-m3", "")}, true},
- {"single dataset without model", []*entity.Knowledgebase{kb("", "")}, false},
- // Python reads kbs[0].embd_id only; it does not scan for any dataset
- // carrying a model.
- {"first without, later with", []*entity.Knowledgebase{kb("", ""), kb("bge-m3", "")}, false},
- {"all with the same model", []*entity.Knowledgebase{kb("bge-m3", ""), kb("bge-m3", "")}, true},
- }
- for _, c := range cases {
- if got := hasEmbedderFor(c.kbs); got != c.want {
- t.Errorf("%s: hasEmbedderFor = %v, want %v", c.name, got, c.want)
- }
- }
-}
-
-// TestValidateDatasetEmbeddingModels pins knowledgebase_service.py:62-94.
-// Mixing is a hard error upstream (get_models raises at dialog_service.py:358) —
-// NOT a silent fallback to keyword-only retrieval.
-func TestValidateDatasetEmbeddingModels(t *testing.T) {
- t.Run("all without a model is allowed", func(t *testing.T) {
- if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{
- kb("", ""), kb("", ""),
- }); err != nil {
- t.Errorf("all-without must be allowed, got %v", err)
- }
- })
-
- t.Run("all same model is allowed", func(t *testing.T) {
- if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{
- kb("bge-m3", ""), kb("bge-m3", ""),
- }); err != nil {
- t.Errorf("same model must be allowed, got %v", err)
- }
- })
-
- t.Run("some with some without is an error", func(t *testing.T) {
- if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{
- kb("bge-m3", ""), kb("", ""),
- }); err == nil {
- t.Error("mixing present/absent embedding models must error (Python raises)")
- }
- })
-
- t.Run("different models is an error", func(t *testing.T) {
- if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{
- kb("bge-m3", ""), kb("text-embedding-3", ""),
- }); err == nil {
- t.Error("different embedding models must error (Python raises)")
- }
- })
-
- t.Run("dangling tenant_model id falls back to embd_id base name", func(t *testing.T) {
- // Python silently skips ids that no longer resolve
- // (tenant_model_service.py:557) and falls back to
- // _base_model_name(embd_id) — it must not fail the request.
- if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{
- kb("bge-m3@inst@prov", "gone-1"), kb("bge-m3@inst2@prov2", "gone-2"),
- }); err != nil {
- t.Errorf("two dangling refs with the same base name must be allowed, got %v", err)
- }
- })
-}
-
-// TestBaseModelNameMirrorsPython pins knowledgebase_service.py:33:
-// rsplit("@", 2)[0] — the model name, with @instance@provider stripped.
-func TestBaseModelNameMirrorsPython(t *testing.T) {
- cases := []struct{ in, want string }{
- {"bge-m3@inst@prov", "bge-m3"},
- {"bge-m3@inst", "bge-m3"},
- {"bge-m3", "bge-m3"},
- {"", ""},
- }
- for _, c := range cases {
- if got := baseModelName(c.in); got != c.want {
- t.Errorf("baseModelName(%q) = %q, want %q", c.in, got, c.want)
- }
- }
-}
-
-// TestHarnessWebSearcherNilAndWrapped pins the adapter: a nil pipeline callback
-// must yield a nil harness.WebSearcher (which hides the web_search tool), and a
-// non-nil one must be callable through the seam.
-func TestHarnessWebSearcherNilAndWrapped(t *testing.T) {
- if got := harnessWebSearcher(nil); got != nil {
- t.Error("nil callback must yield a nil WebSearcher")
- }
- called := false
- searcher := harnessWebSearcher(func(_ context.Context, queries []string) ([]string, error) {
- called = true
- return queries, nil
- })
- if searcher == nil {
- t.Fatal("non-nil callback must yield a WebSearcher")
- }
- got, err := searcher.Search(context.Background(), []string{"q"})
- if err != nil {
- t.Fatalf("Search: %v", err)
- }
- if !called {
- t.Error("callback was not invoked")
- }
- if len(got) != 1 || got[0] != "q" {
- t.Errorf("Search = %v, want [q]", got)
- }
-}
-
func TestParseArgsMigrateIsStandalone(t *testing.T) {
args, err := parseArgsForTest(t, "--migrate")
if err != nil {
@@ -254,249 +129,20 @@ func TestParseArgsKeepsMCPOutOfCLI(t *testing.T) {
}
}
-// errStubEncode stands in for a provider-side embedding failure.
-var errStubEncode = errors.New("stub encode failure")
-
-// recordingEngine captures the SearchRequest so the tests can pin the adapter's
-// mapping without a live document engine.
-type recordingEngine struct {
- last *types.SearchRequest
- res *types.SearchResult
-}
-
-func (e *recordingEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
- e.last = req
- if e.res != nil {
- return e.res, nil
- }
- return &types.SearchResult{}, nil
-}
-
-// TestEngineCompiledStoreSearchCompiledMapsRequest pins the mapping from the
-// harness' compiled-row request onto the engine search: tenant index, kb scope,
-// numeric available_int, the synthesis-page doc-scope column (source_doc_ids),
-// and the keyword match leg over content_ltks/content_sm_ltks.
-func TestEngineCompiledStoreSearchCompiledMapsRequest(t *testing.T) {
- eng := &recordingEngine{}
- // No embedder wired → the keyword leg stands alone.
- store := newEngineCompiledStore(eng, nil, "")
- _, err := store.SearchCompiled(context.Background(), "kb1", "t1", []string{"d1"}, map[string][]string{
- "compile_kwd": {"wiki_page"},
- "available_int": {"1"},
- }, "culdcept", 7)
- if err != nil {
- t.Fatalf("SearchCompiled: %v", err)
- }
- req := eng.last
- if req == nil {
- t.Fatal("engine was not called")
- }
- if len(req.IndexNames) != 1 || req.IndexNames[0] != "ragflow_t1" {
- t.Errorf("IndexNames = %v, want [ragflow_t1]", req.IndexNames)
- }
- if len(req.KbIDs) != 1 || req.KbIDs[0] != "kb1" {
- t.Errorf("KbIDs = %v, want [kb1]", req.KbIDs)
- }
- if req.Limit != 7 {
- t.Errorf("Limit = %d, want 7", req.Limit)
- }
- // available_int is a numeric column: "1" must reach the engine as an int.
- if got, ok := req.Filter["available_int"].([]int); !ok || len(got) != 1 || got[0] != 1 {
- t.Errorf("available_int = %#v, want []int{1}", req.Filter["available_int"])
- }
- if got, ok := req.Filter["compile_kwd"].([]string); !ok || len(got) != 1 || got[0] != "wiki_page" {
- t.Errorf("compile_kwd = %#v, want []string{wiki_page}", req.Filter["compile_kwd"])
- }
- // Synthesis pages are scoped by source_doc_ids, not doc_id.
- if _, ok := req.Filter["source_doc_ids"]; !ok {
- t.Errorf("doc scope key = %v, want source_doc_ids present", req.Filter)
- }
- if _, ok := req.Filter["doc_id"]; ok {
- t.Errorf("synthesis rows must not use doc_id scope: %v", req.Filter)
- }
- if len(req.MatchExprs) != 1 {
- t.Fatalf("MatchExprs = %#v, want one keyword expr", req.MatchExprs)
+func TestMainMigrateReachesConfiguration(t *testing.T) {
+ if os.Getenv("RAGFLOW_TEST_MIGRATE_MAIN") == "1" {
+ os.Args = append([]string{os.Args[0]}, os.Args[len(os.Args)-3:]...)
+ main()
+ return
}
- mt, ok := req.MatchExprs[0].(*types.MatchTextExpr)
- if !ok {
- t.Fatalf("MatchExprs[0] = %T, want *types.MatchTextExpr", req.MatchExprs[0])
- }
- if mt.MatchingText != "culdcept" || mt.TopN != 7 {
- t.Errorf("keyword expr = %+v, want text=culdcept topN=7", mt)
- }
- if len(mt.Fields) != 2 || mt.Fields[0] != "content_ltks" || mt.Fields[1] != "content_sm_ltks" {
- t.Errorf("keyword fields = %v, want [content_ltks content_sm_ltks]", mt.Fields)
- }
-}
-
-// TestCompiledDocScopeKey pins Python's column choice: entity/relation rows use
-// doc_id; the three synthesis kinds use source_doc_ids.
-func TestCompiledDocScopeKey(t *testing.T) {
- cases := []struct {
- name string
- filters map[string][]string
- want string
- }{
- {"entity rows", map[string][]string{"knowledge_graph_kwd": {"entity"}}, "doc_id"},
- {"tree rows", map[string][]string{"knowledge_graph_kwd": {"entity"}, "compile_kwd": {"tree"}}, "doc_id"},
- {"wiki pages", map[string][]string{"compile_kwd": {"wiki_page"}}, "source_doc_ids"},
- {"artifact pages", map[string][]string{"compile_kwd": {"artifact_page"}}, "source_doc_ids"},
- {"essence", map[string][]string{"compile_kwd": {"essence"}}, "source_doc_ids"},
- }
- for _, c := range cases {
- if got := compiledDocScopeKey(c.filters); got != c.want {
- t.Errorf("%s: compiledDocScopeKey = %q, want %q", c.name, got, c.want)
- }
- }
-}
-
-// TestEngineCompiledStoreLoadChunks pins the id-scoped fetch used to promote
-// source chunks into evidence.
-func TestEngineCompiledStoreLoadChunks(t *testing.T) {
- eng := &recordingEngine{res: &types.SearchResult{Chunks: []map[string]interface{}{{"id": "c1"}}}}
- store := newEngineCompiledStore(eng, nil, "")
- rows, err := store.LoadChunks(context.Background(), "kb1", "t1", []string{"c1", "c2"})
- if err != nil {
- t.Fatalf("LoadChunks: %v", err)
- }
- if len(rows) != 1 {
- t.Fatalf("rows = %d, want 1", len(rows))
- }
- if eng.last.Limit != 2 {
- t.Errorf("Limit = %d, want 2", eng.last.Limit)
- }
- ids, ok := eng.last.Filter["id"].([]string)
- if !ok || len(ids) != 2 || ids[0] != "c1" || ids[1] != "c2" {
- t.Errorf("id filter = %#v, want [c1 c2]", eng.last.Filter["id"])
- }
-}
-
-// TestNewDatasetCompiledStore pins the store wiring: no engine yields no store;
-// datasets without an embedding model yield a keyword-only store; a dataset
-// embedding model wires the query encoder with THAT dataset's model and owner
-// tenant (not a tenant default or the request tenant).
-func TestNewDatasetCompiledStore(t *testing.T) {
- eng := &recordingEngine{}
- if got := newDatasetCompiledStore(nil, nil, nil); got != nil {
- t.Error("nil engine must disable the compiled store")
- }
- if got := newDatasetCompiledStore(eng, nil, []*entity.Knowledgebase{{ID: "kb1", TenantID: "t1"}}); got == nil {
- t.Error("engine without an embedder must still build a keyword-only store")
- }
- store := newDatasetCompiledStore(eng, service.NewModelProviderService(),
- []*entity.Knowledgebase{{ID: "kb1", EmbdID: "bge-m3@inst@prov", TenantID: "owner-tenant"}})
- es, ok := store.(*engineCompiledStore)
- if !ok {
- t.Fatalf("store = %T, want *engineCompiledStore", store)
- }
- if es.embed == nil {
- t.Fatal("the dataset's embedding model must wire the query encoder")
- }
- if es.embedTenantID != "owner-tenant" {
- t.Errorf("embedTenantID = %q, want owner-tenant (the dataset owner, not the request tenant)", es.embedTenantID)
- }
-}
-
-// stubQueryEncoder records the query-encoding calls the dense leg makes.
-type stubQueryEncoder struct {
- vecs [][]float32
- err error
- calls int
- lastTenant string
-}
-
-func (s *stubQueryEncoder) EncodeQueries(_ context.Context, tenantID string, _ []string) ([][]float32, error) {
- s.calls++
- s.lastTenant = tenantID
- return s.vecs, s.err
-}
-
-// TestEngineCompiledStoreUsesDenseSeedLeg pins the parity fix: when an embedder
-// is wired, the compiled-row search goes out as a DENSE expr on the query
-// vector's q_<dim>_vec column with Python's num_candidates floor and similarity
-// threshold — and carries NO keyword expr alongside it (Python appends exactly
-// one). The encoding runs under the embedder's OWN tenant (Python
-// embd_owner_tenant_id = kbs[0].tenant_id), not the searched scope's tenant.
-func TestEngineCompiledStoreUsesDenseSeedLeg(t *testing.T) {
- eng := &recordingEngine{}
- enc := &stubQueryEncoder{vecs: [][]float32{{1, 2, 3}}}
- // Scope tenant differs from the embedder's owner tenant on purpose.
- store := newEngineCompiledStore(eng, enc, "owner-tenant")
-
- if _, err := store.SearchCompiled(context.Background(), "kb1", "scope-tenant", nil,
- map[string][]string{"knowledge_graph_kwd": {"entity"}}, "culdcept", 5); err != nil {
- t.Fatalf("SearchCompiled: %v", err)
- }
- if enc.calls != 1 {
- t.Fatalf("encode calls = %d, want 1", enc.calls)
- }
- if enc.lastTenant != "owner-tenant" {
- t.Errorf("encode tenant = %q, want the embedder's owner tenant owner-tenant", enc.lastTenant)
- }
- if len(eng.last.MatchExprs) != 1 {
- t.Fatalf("MatchExprs = %#v, want exactly one dense expr", eng.last.MatchExprs)
- }
- dense, ok := eng.last.MatchExprs[0].(*types.MatchDenseExpr)
- if !ok {
- t.Fatalf("MatchExprs[0] = %T, want *types.MatchDenseExpr", eng.last.MatchExprs[0])
- }
- if dense.VectorColumnName != "q_3_vec" {
- t.Errorf("VectorColumnName = %q, want q_3_vec", dense.VectorColumnName)
- }
- if dense.EmbeddingDataType != "float" || dense.DistanceType != "cosine" || dense.TopN != 5 {
- t.Errorf("dense expr = %+v, want float/cosine/topN=5", dense)
- }
- if dense.ExtraOptions["similarity"] != 0.1 {
- t.Errorf("similarity = %v, want 0.1", dense.ExtraOptions["similarity"])
- }
- // max(topN, 256) — a small topN is floored to the HNSW candidate count.
- if dense.ExtraOptions["num_candidates"] != 256 {
- t.Errorf("num_candidates = %v, want 256 (max(5,256))", dense.ExtraOptions["num_candidates"])
- }
-
- // A topN above the floor is passed through unchanged.
- if _, err := store.SearchCompiled(context.Background(), "kb1", "scope-tenant", nil,
- map[string][]string{"knowledge_graph_kwd": {"entity"}}, "culdcept", 300); err != nil {
- t.Fatalf("SearchCompiled(300): %v", err)
- }
- dense300 := eng.last.MatchExprs[0].(*types.MatchDenseExpr)
- if dense300.ExtraOptions["num_candidates"] != 300 {
- t.Errorf("num_candidates = %v, want 300", dense300.ExtraOptions["num_candidates"])
- }
-}
-
-// TestEngineCompiledStoreDenseFallsBackToKeyword pins Python's `except Exception`
-// around get_vector: an encode failure (or an empty vector) must degrade to the
-// keyword leg rather than dropping the expansion.
-func TestEngineCompiledStoreDenseFallsBackToKeyword(t *testing.T) {
- for _, c := range []struct {
- name string
- enc *stubQueryEncoder
- }{
- {"encode error", &stubQueryEncoder{err: errStubEncode}},
- {"empty vector", &stubQueryEncoder{vecs: [][]float32{{}}}},
- {"no vectors", &stubQueryEncoder{vecs: nil}},
- } {
- t.Run(c.name, func(t *testing.T) {
- eng := &recordingEngine{}
- store := newEngineCompiledStore(eng, c.enc, "t1")
- if _, err := store.SearchCompiled(context.Background(), "kb1", "t1", nil,
- map[string][]string{"knowledge_graph_kwd": {"entity"}}, "culdcept", 5); err != nil {
- t.Fatalf("SearchCompiled: %v", err)
- }
- if c.enc.calls != 1 {
- t.Errorf("encode calls = %d, want 1", c.enc.calls)
- }
- if len(eng.last.MatchExprs) != 1 {
- t.Fatalf("MatchExprs = %#v, want one keyword expr", eng.last.MatchExprs)
- }
- mt, ok := eng.last.MatchExprs[0].(*types.MatchTextExpr)
- if !ok {
- t.Fatalf("MatchExprs[0] = %T, want *types.MatchTextExpr keyword fallback", eng.last.MatchExprs[0])
- }
- if mt.MatchingText != "culdcept" {
- t.Errorf("keyword text = %q, want culdcept", mt.MatchingText)
- }
- })
+ ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second)
+ defer cancel()
+ dir := t.TempDir()
+ cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestMainMigrateReachesConfiguration$", "--", "--migrate", "--config", filepath.Join(dir, "missing.yaml"))
+ cmd.Dir = dir
+ cmd.Env = append(os.Environ(), "RAGFLOW_TEST_MIGRATE_MAIN=1")
+ output, err := cmd.CombinedOutput()
+ if err == nil || !strings.Contains(string(output), "initialize configuration") || strings.Contains(string(output), "Usage:") {
+ t.Fatalf("migration did not reach config: %v\n%s", err, output)
}
}
diff --git a/internal/agent/retrievalbridge/harness.go b/internal/agent/retrievalbridge/harness.go
new file mode 100644
index 0000000000..0bf8d6f777
--- /dev/null
+++ b/internal/agent/retrievalbridge/harness.go
@@ -0,0 +1,745 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package retrievalbridge
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strconv"
+ "strings"
+
+ "ragflow/internal/agent/component"
+ "ragflow/internal/common"
+ "ragflow/internal/dao"
+ et "ragflow/internal/engine/types"
+ "ragflow/internal/entity"
+ modelModule "ragflow/internal/entity/models"
+ "ragflow/internal/rag/agentic-rag"
+ agenticruntime "ragflow/internal/rag/agentic-rag/runtime"
+ "ragflow/internal/service"
+ "ragflow/internal/service/nlp"
+
+ "github.com/cloudwego/eino/schema"
+ "go.uber.org/zap"
+)
+
+// NewHarnessRetriever connects chat requests to the agentic runtime using the
+// shared model, metadata and document services initialized by the API server.
+func NewHarnessRetriever(modelProviderService *service.ModelProviderService, metadataService *service.MetadataService, docEngine engineDocEngine) func(context.Context, service.HarnessRequest) (service.HarnessResult, error) {
+ return func(ctx context.Context, req service.HarnessRequest) (service.HarnessResult, error) {
+ // Resolve the tenant's actual chat model, mirroring Python where RAGTools
+ // receives a fully-resolved LLMBundle: chat.LLMID may be a UUID/tenant_model
+ // id that the default invoker cannot split into a provider, so resolving here
+ // avoids falling through to a dummy driver. If resolution fails we leave model
+ // nil so the runtime degrades to a direct search (no dummy fallback).
+ var model agenticruntime.SessionModel
+ // outerModel mirrors Python RAGTools.chat_mdl: the fully-resolved chat
+ // model that drives the outer rag_agent react loop (tools=[rag,
+ // summarize_document], terminal_tools={"rag"}). Wired into RAGTools.Outer
+ // so Rag() can run that loop instead of the direct graph.
+ var outerModel *modelModule.ChatModel
+ // resolvedModelName is the resolved chat model's own name (e.g.
+ // "gpt-4o"), i.e. Python chat_mdl.llm_name. It is the gen_json reply-cache
+ // key's model component; empty disables that cache.
+ resolvedModelName := ""
+ // The resolution goes through the same entry point the pipeline's own
+ // capability probe uses, so the model decided there is the model wired
+ // here: req.ModelID may be empty (a dialog without an llm_id), and both
+ // layers must then fall back to the tenant default rather than resolving
+ // nothing here and judging capability there.
+ if target, mErr := modelProviderService.ResolveChatModelTarget(ctx, req.TenantID, req.ModelID); mErr == nil {
+ if inv := component.NewResolvedInvoker(target.Driver, target.ModelName, target.APIConfig); inv != nil {
+ // MaxLength mirrors Python LLMBundle.max_length (the model's
+ // context window in tokens); message-fitting nodes (calculate,
+ // structure_qa) use it as their chat.FitMessages budget.
+ model = &agenticruntime.InvokerSessionModel{Invoker: inv, DB: dao.DB, MaxLength: target.ContextLength}
+ resolvedModelName = target.ModelName
+ // Reuse the same resolved driver/name/api as the invoker so
+ // the outer loop and the inner tool calls share one model.
+ outerModel = modelModule.NewChatModel(target.Driver, &target.ModelName, target.APIConfig)
+ }
+ } else {
+ common.Warn("runtime: failed to resolve chat model for reasoning; runtime will degrade to direct search", zap.Error(mErr))
+ }
+
+ // Load the KB objects (mirroring Python RAGTools' self.kbs via
+ // KnowledgebaseService.get_by_ids(kb_ids)) so the agentic tool can
+ // derive rank features. The Go tag extractor (extractor_tag.go) writes
+ // both tag_kwd (the list of tag names) and tag_feas (per-tag weights)
+ // onto each chunk at parse time; the labeler aggregates tag_kwd to build
+ // the tag vocabulary and the retriever ranks with tag_feas.
+ var kbs []*entity.Knowledgebase
+ if len(req.DatasetIDs) > 0 {
+ var err error
+ kbs, err = dao.NewKnowledgebaseDAO().GetByIDs(ctx, dao.DB, req.DatasetIDs)
+ if err != nil {
+ return service.HarnessResult{}, err
+ }
+ if err := validateLoadedDatasets(req.DatasetIDs, kbs); err != nil {
+ return service.HarnessResult{}, err
+ }
+ }
+ // validate_dataset_embedding_models runs FIRST upstream
+ // (dialog_service.py:358-360) and mixing is a hard failure there, not a
+ // fallback to keyword-only retrieval.
+ if err := validateDatasetEmbeddingModels(ctx, kbs); err != nil {
+ return service.HarnessResult{}, err
+ }
+ // HasEmbedder mirrors Python `embd_mdl = ... if kbs and kbs[0].embd_id else
+ // None` (dialog_service.py:362): the gate is the FIRST dataset's embd_id. It
+ // is what hybrid_search applies to its vector leg (search.py:143), so
+ // getting it wrong silently degrades search_chunks to keyword-only.
+ hasEmbedder := hasEmbedderFor(kbs)
+ // Retrieval tuning mirrors the dialog's own settings, so the agentic
+ // searches run with the same budget as the standard path. Without it the
+ // harness falls back to its package defaults (top_n=12) and the dialog's
+ // configured top_n is silently ignored.
+ //
+ // The runtime carries the KEYWORD leg's weight and derives the vector leg
+ // as its complement (runtime.resolveKeywordsSimilarityWeight, and the
+ // RetrieveRequest contract: 0.7 hybrid / 0.0 vector-only / 1.0
+ // keyword-only). The dialog configures the VECTOR weight, so it is inverted
+ // here; the dialog's default 0.3 therefore yields the runtime's own 0.7.
+ keywordsWeight := 1 - req.Retrieval.VectorSimilarityWeight
+ deps := agentic_rag.RAGTools{
+ Model: model,
+ ModelName: resolvedModelName,
+ Outer: outerModel,
+ TopN: req.Retrieval.TopN,
+ SimilarityThreshold: req.Retrieval.SimilarityThreshold,
+ KeywordsSimilarityWeight: &keywordsWeight,
+ RerankCandidatesCount: req.Retrieval.RerankCandidatesCount,
+ // OriginalQuestion mirrors Python RAGTools(original_user_question=...):
+ // the user's own, unrewritten question as received from the chat
+ // layer. The outer model's `rag(question=...)` argument is
+ // model-generated and often compresses a multi-hop question to its
+ // first hop; resolveEffectiveQuestion (agentic_rag.py:865) prefers
+ // this original over that rewrite when both describe the same turn.
+ OriginalQuestion: req.Question,
+ // DocScope mirrors Python RAGTools(doc_scope=...): the narrowed doc_ids
+ // (chat-level doc_ids + meta_data_filter) restrict every agentic
+ // retrieval to the user-selected documents instead of the whole kb.
+ DocScope: req.DocIDs,
+ DocIDVerifier: agentic_rag.NewDocIDLookup(),
+ // MetadataResolver backs the metadata_search tool and the pre-search
+ // metadata channel (Python DocMetadataService push-down + meta_filter
+ // fallback). metadataService already carries the doc engine and the DAO
+ // the metadata index reads need.
+ MetadataResolver: metadataService,
+ // DeclaredMetadata lets the metadata_search catalog describe each field
+ // (meaning + allowed values) from the dataset's own parser_config, not just
+ // name it. Same service: it also carries the KB DAO that read needs.
+ DeclaredMetadata: metadataService,
+ // DocChunks pages one document's chunks in reading order off the
+ // chunk index (Python retriever.chunk_list), backing the
+ // document-level tools (summarize_document / fetch_full_document).
+ DocChunks: &docChunkPager{docEngine: docEngine},
+ // Expand runs search_chunks' compiled-structure expansion
+ // (Python hybrid_search use_compiled=True → _expand_with_compiled).
+ // Backed by the same document engine the chunk reads use, with the
+ // dense seed leg wired to the bound dataset's own embedding model.
+ // The scope config lets each dataset be scanned under its own tenant
+ // and a doc scope be grouped by real owner, like graph_explore.
+ Expand: agenticruntime.NewCompiledExpander(
+ newDatasetCompiledStore(docEngine, modelProviderService, kbs),
+ agenticruntime.CompiledScopeConfig{
+ DatasetIDs: req.DatasetIDs,
+ TenantID: req.TenantID,
+ KBs: kbs,
+ DocTenantResolver: agentic_rag.NewDocTenantResolver(),
+ },
+ ),
+ // WebSearch backs the web_search tool (Python RAGTools.web_search).
+ // The pipeline supplies a callback only when the chat enables
+ // internet web search; a nil one also keeps the tool off the surface.
+ WebSearch: harnessWebSearcher(req.WebSearch),
+ KBs: kbs,
+ HasEmbedder: hasEmbedder,
+ // Embedder backs claim recall's KNN leg (Python
+ // recall_dataset_claims, navigation.py:1837-1909, which embeds the
+ // query with tools.embed_mdl — the FIRST dataset's embedding model,
+ // dialog_service.py:362-366) and the structure-drill seed vector.
+ // Without it the claim leg silently degrades to BM25-only and
+ // paraphrase-phrased claims are never recalled. Bound to
+ // kbs[0].EmbdID (validateDatasetEmbeddingModels above guarantees
+ // every bound dataset shares it), query-side encoded.
+ Embedder: embedderForDatasets(kbs, modelProviderService),
+ // Tagger is the Go equivalent of Python's label_question
+ // (agentic_rag.py:668): classifies the query into question-type
+ // tags the retriever boosts on. metadataService implements it
+ // (service.MetadataService.LabelQuestion).
+ Tagger: metadataService,
+ // SystemPrompt mirrors Python RAGTools(system_prompt=
+ // _render_reasoning_system_prompt(dialog, prompt_config, kwargs),
+ // dialog_service.py:2084) — the dialog-level UI configuration the
+ // final-answer compose appends after the agentic contract
+ // (agentic_rag_graph.py:923-933).
+ SystemPrompt: req.SystemPrompt,
+ // CiteRules and EvidenceMaxTokens deliberately stay unset: Python's
+ // dialog path constructs RAGTools WITHOUT user_defined_prompts
+ // (dialog_service.py:2072-2090), so citation_prompt defaults apply,
+ // and Python has no evidence-token override (the compose always
+ // uses min(chat_mdl.max_length, _EVIDENCE_BUDGET_TOKENS=8000),
+ // which EvidenceMaxTokens<=0 reproduces).
+ }
+ for _, message := range req.Messages {
+ content, err := service.NormalizeOpenAIMessageContent(message["content"])
+ if err != nil {
+ return service.HarnessResult{}, err
+ }
+ role, _ := message["role"].(string)
+ deps.Messages = append(deps.Messages, schema.Message{Role: schema.RoleType(role), Content: content})
+ }
+ // Diagnose WHY compiled expansion is disabled: NewCompiledExpander
+ // returns nil for three reasons (store==nil / no datasets / no tenant)
+ // and RAGTools.Expand==nil silences the whole channel with no other
+ // trace — the observed "Compiled expansion enabled = 0" was invisible.
+ if deps.Expand == nil {
+ switch {
+ case docEngine == nil:
+ common.Warn("compiled expansion disabled: document engine unavailable (store==nil)")
+ case len(req.DatasetIDs) == 0:
+ common.Warn("compiled expansion disabled: no bound dataset (DatasetIDs empty)")
+ case strings.TrimSpace(req.TenantID) == "":
+ common.Warn("compiled expansion disabled: no tenant (TenantID empty)")
+ default:
+ common.Warn("compiled expansion disabled: unknown reason")
+ }
+ }
+ // The two projections of one reasoning step: the sentence the chat UI
+ // appends to its think block, and the structured event a step-rendering
+ // client consumes. Steps.Stage/Emit feeds both from one call, so the
+ // trace cannot drift from its structured twin. Each step is an
+ // isThink=true delta, i.e. think-block content rather than answer text.
+ if req.AnswerSink != nil {
+ answerSink := req.AnswerSink
+ deps.AnswerSink = &agentic_rag.AnswerSink{
+ OnDelta: req.AnswerSink,
+ }
+ deps.Steps.Text = func(line string) { answerSink(line, true) }
+ }
+ if req.ThinkSink != nil {
+ // One type on both sides — agenticruntime.ThinkEvent is an alias of
+ // service.ThinkEvent — so the sink passes straight through: there is
+ // nothing to copy field by field, and no way to forget a field that
+ // was added on one side only.
+ deps.Steps.Events = req.ThinkSink
+ }
+ r := agentic_rag.Rag(ctx, deps, agenticruntime.RunRequest{
+ Question: req.Question,
+ ThinkingMode: req.ThinkingMode,
+ DatasetIDs: req.DatasetIDs,
+ TenantID: req.TenantID,
+ SessionID: req.SessionID,
+ Images: req.Images,
+ TextAttachments: req.TextAttachments,
+ })
+ res := service.HarnessResult{Chunks: r.Chunks, DocAggs: r.DocAggs, Answer: r.Answer, SlotCitations: r.SlotCitations, CiteChunkIDs: r.CiteChunkIDs}
+ if r.Kbinfos != nil {
+ res.Memory = r.Kbinfos.Memory
+ res.PreSummary = r.Kbinfos.PreSummary
+ }
+ return res, nil
+ }
+}
+
+// engineDocEngine is the small slice of the engine surface the doc-chunk pager
+// needs, kept as a local interface so it is trivially unit-testable without a
+// live engine (the production value is engine.Get()).
+type engineDocEngine interface {
+ Search(ctx context.Context, req *et.SearchRequest) (*et.SearchResult, error)
+}
+
+// docChunkPager is the production agenticruntime.DocChunkLister. It pages one
+// document's chunks in reading order straight off the chunk index, mirroring
+// Python settings.retriever.chunk_list with sort_by_position=True
+// (rag/nlp/search.py:824). This is what the document-level tools
+// (summarize_document / fetch_full_document) consume.
+//
+// The index name is derived from the tenant id (ragflow_<tenantID>), exactly as
+// retrieval does, so it needs no user-scope resolution: DocChunksRequest
+// already carries the tenant and the bound datasets.
+type docChunkPager struct {
+ docEngine engineDocEngine
+}
+
+// DocChunks implements agenticruntime.DocChunkLister.
+func (p *docChunkPager) DocChunks(ctx context.Context, req agenticruntime.DocChunksRequest) ([]map[string]any, error) {
+ if p == nil || p.docEngine == nil {
+ return nil, nil
+ }
+ if req.DocID == "" {
+ return nil, nil
+ }
+ orderBy := (&et.OrderByExpr{}).
+ Asc("chunk_order_int").
+ Asc("page_num_int").
+ Asc("top_int").
+ Desc("create_timestamp_flt")
+
+ resp, err := p.docEngine.Search(ctx, &et.SearchRequest{
+ IndexNames: []string{tenantChunkIndexName(req.TenantID)},
+ KbIDs: req.DatasetIDs,
+ Offset: req.Offset,
+ Limit: req.Limit,
+ OrderBy: orderBy,
+ SelectFields: []string{
+ "id", "doc_id", "docnm", "content_with_weight", "content", "available_int",
+ },
+ Filter: map[string]interface{}{
+ "doc_id": req.DocID,
+ "available_int": 1,
+ "must_not": map[string]interface{}{"exists": "compile_kwd"},
+ },
+ })
+ if err != nil {
+ return nil, err
+ }
+ if resp == nil {
+ return nil, nil
+ }
+ rows := make([]map[string]any, 0, len(resp.Chunks))
+ for _, ck := range resp.Chunks {
+ row := make(map[string]any, 8)
+ // The runtime chunk rows need the canonical keys its citation pipeline
+ // reads: chunk_id (dedup), docnm_kwd (doc title), doc_id and the text
+ // under content/content_with_weight (chunkText prefers content first).
+ if id, ok := ck["id"].(string); ok && id != "" {
+ row["chunk_id"] = id
+ }
+ if d, ok := ck["doc_id"]; ok {
+ row["doc_id"] = d
+ }
+ if d, ok := ck["docnm"]; ok {
+ row["docnm_kwd"] = d
+ }
+ if c, ok := ck["content"]; ok {
+ row["content"] = c
+ }
+ if c, ok := ck["content_with_weight"]; ok {
+ row["content_with_weight"] = c
+ }
+ // Keep whatever else the engine returned (e.g. position_int, img) so the
+ // merged Kbinfos rows stay rich, as retrieval does.
+ for k, v := range ck {
+ if _, seen := row[k]; !seen {
+ row[k] = v
+ }
+ }
+ rows = append(rows, row)
+ }
+ return rows, nil
+}
+
+// hasEmbedderFor mirrors Python `embd_mdl = LLMBundle(...) if kbs and
+// kbs[0].embd_id else None` (dialog_service.py:362). The gate is the FIRST bound
+// dataset's embd_id, not "any dataset has one": a dialog whose first dataset
+// carries no embedding model yields embd_mdl=None upstream even if later ones do.
+//
+// Callers must run validateDatasetEmbeddingModels first — with validation
+// passing, all datasets share one model and this equals "all have one".
+func hasEmbedderFor(kbs []*entity.Knowledgebase) bool {
+ return len(kbs) > 0 && kbs[0] != nil && kbs[0].EmbdID != ""
+}
+
+// embedderForDatasets builds the embedding handle the agentic runtime uses for
+// query-side encoding outside the main retrieval leg: claim recall's KNN leg
+// (Python recall_dataset_claims, navigation.py:1837-1909, embedding with
+// tools.embed_mdl) and the structure-drill seed vector. The model is the FIRST
+// dataset's embedding (Python dialog_service.py:362-366 resolves
+// kbs[0].embd_id under kbs[0].tenant_id; validateDatasetEmbeddingModels
+// guarantees the rest share it). Nil when no dataset carries an embedding
+// model — the claim leg then degrades to BM25-only, matching a Python run
+// without embed_mdl.
+func embedderForDatasets(kbs []*entity.Knowledgebase, modelSvc *service.ModelProviderService) nlp.NavEmbedder {
+ if len(kbs) == 0 || kbs[0] == nil || kbs[0].EmbdID == "" {
+ return nil
+ }
+ return service.NewNavEmbedder(modelSvc, kbs[0].EmbdID)
+}
+
+func validateLoadedDatasets(datasetIDs []string, kbs []*entity.Knowledgebase) error {
+ loaded := make(map[string]struct{}, len(kbs))
+ for _, kb := range kbs {
+ if kb != nil {
+ loaded[kb.ID] = struct{}{}
+ }
+ }
+ for _, id := range datasetIDs {
+ if _, ok := loaded[id]; !ok {
+ return fmt.Errorf("dataset %q was not found", id)
+ }
+ }
+ return nil
+}
+
+// validateDatasetEmbeddingModels mirrors Python validate_dataset_embedding_models
+// (knowledgebase_service.py:62-94): every bound dataset must use the same embedding
+// model, or none at all. Upstream this runs before embd_mdl is resolved
+// (dialog_service.py:358-360) and a violation RAISES — it is not a "no embedding
+// model" fallback: treating mixing as HasEmbedder=false would silently serve
+// keyword-only results where Python errors.
+func validateDatasetEmbeddingModels(ctx context.Context, kbs []*entity.Knowledgebase) error {
+ var withEmbd int
+ for _, kb := range kbs {
+ if kb != nil && kb.EmbdID != "" {
+ withEmbd++
+ }
+ }
+ hasEmbd := withEmbd > 0
+ if hasEmbd && withEmbd != len(kbs) {
+ return errors.New("cannot search across datasets where some have embedding models and others do not")
+ }
+ if !hasEmbd {
+ return nil
+ }
+ // Mirror Python's grouping key exactly:
+ //
+ // ref = tenant_embd_id or (embd_id when it is a raw tenant_model id)
+ // composite = resolved_names.get(ref) # "model@instance@provider"
+ // key = _base_model_name(composite) # == tenant_model.model_name
+ //
+ // The @instance@provider suffix is stripped by _base_model_name
+ // (knowledgebase_service.py:33), so only the MODEL NAME is compared — the
+ // instance and provider tables are not consulted at all.
+ refs := make([]string, 0, len(kbs))
+ for _, kb := range kbs {
+ if kb == nil || kb.EmbdID == "" {
+ continue
+ }
+ ref := ""
+ if kb.TenantEmbdID != nil {
+ ref = strings.TrimSpace(*kb.TenantEmbdID)
+ }
+ if ref == "" && !strings.Contains(kb.EmbdID, "@") {
+ ref = strings.TrimSpace(kb.EmbdID)
+ }
+ if ref != "" {
+ refs = append(refs, ref)
+ }
+ }
+
+ // Python resolves the refs with one batched lookup
+ // (tenant_model_service.py:562 get_by_ids) and silently skips ids that no
+ // longer resolve — a dangling id must not fail the request.
+ resolved := make(map[string]string, len(refs))
+ // A nil/absent DB or any query error must not fail the request: Python skips
+ // unresolvable ids (tenant_model_service.py:557) and falls back to the raw
+ // id, so validation degrades to a stricter but safe comparison.
+ if len(refs) > 0 && dao.DB != nil {
+ if models, err := dao.NewTenantModelDAO().GetByIDs(ctx, dao.DB, refs); err == nil {
+ for _, m := range models {
+ if m != nil {
+ resolved[m.ID] = m.ModelName
+ }
+ }
+ }
+ }
+
+ keys := make(map[string]struct{}, len(kbs))
+ for _, kb := range kbs {
+ if kb == nil || kb.EmbdID == "" {
+ continue
+ }
+ embdID := strings.TrimSpace(kb.EmbdID)
+ ref := ""
+ if kb.TenantEmbdID != nil {
+ ref = strings.TrimSpace(*kb.TenantEmbdID)
+ }
+ if ref == "" && !strings.Contains(embdID, "@") {
+ ref = embdID
+ }
+ key := ""
+ if name, ok := resolved[ref]; ok && name != "" {
+ key = name
+ } else if embdID != "" && embdID != ref {
+ key = common.BaseModelName(embdID)
+ } else {
+ key = ref
+ }
+ if key != "" {
+ keys[key] = struct{}{}
+ }
+ }
+ if len(keys) > 1 {
+ return fmt.Errorf("datasets use different embedding models")
+ }
+ return nil
+}
+
+// ---------------------------------------------------------------------------
+// Harness seams (compiled expansion + open-web search)
+// ---------------------------------------------------------------------------
+
+// engineCompiledStore implements agenticruntime.CompiledStore over the document
+// engine, backing search_chunks' compiled-structure expansion (Python
+// tools/compiled_expansion.py). The runtime owns the expansion strategy; this
+// adapter only maps its (scope, filters, free text) request onto one engine
+// search, mirroring Python's settings.docStoreConn.search calls.
+type engineCompiledStore struct {
+ engine engineDocEngine
+ // embed encodes the seed query for the dense leg. It must be the QUERY
+ // encoding of the dataset's own embedding model (Python
+ // _search_compiled_rows → _dense_expr → settings.retriever.get_vector →
+ // emb_mdl.encode_queries). Nil leaves the keyword leg only.
+ embed compiledQueryEncoder
+ // embedTenantID is the tenant owning the embedder's model (Python
+ // embd_owner_tenant_id = kbs[0].tenant_id). It is distinct from the searched
+ // scope's tenant, because Python builds ONE embedder from the leading bound
+ // dataset and reuses it for every scope.
+ embedTenantID string
+}
+
+var _ agenticruntime.CompiledStore = (*engineCompiledStore)(nil)
+
+// compiledQueryEncoder is the query-side embedding seam used for the compiled
+// dense seed leg. *service.NavEmbedder implements it via EncodeQueries.
+type compiledQueryEncoder interface {
+ EncodeQueries(ctx context.Context, tenantID string, texts []string) ([][]float32, error)
+}
+
+// Mirror Python compiled_expansion._VECTOR_NUM_CANDIDATES / _VECTOR_SIMILARITY.
+// The HNSW ef_search floor (num_candidates) must be >= topN or the ANN search is
+// bounded by the candidate list instead of by relevance.
+const (
+ compiledVectorNumCandidates = 256
+ compiledVectorSimilarity = 0.1
+)
+
+// newEngineCompiledStore returns nil when no engine is available so
+// agenticruntime.NewCompiledExpander disables expansion instead of searching nowhere.
+// A nil embed (or empty embedTenantID) keeps the keyword-only leg.
+func newEngineCompiledStore(e engineDocEngine, embed compiledQueryEncoder, embedTenantID string) agenticruntime.CompiledStore {
+ if e == nil {
+ return nil
+ }
+ return &engineCompiledStore{engine: e, embed: embed, embedTenantID: embedTenantID}
+}
+
+// newDatasetCompiledStore builds the per-request compiled-row store, wiring the
+// dense seed leg to the FIRST bound dataset's own embedding model. Python builds
+// embd_mdl from kbs[0] (dialog_service.py:362-365) and reuses it for the whole
+// expansion; using the tenant-default model instead would compare the query
+// against rows embedded by a different model — different vector spaces, so the
+// dense hits would be noise.
+func newDatasetCompiledStore(e engineDocEngine, modelSvc *service.ModelProviderService, kbs []*entity.Knowledgebase) agenticruntime.CompiledStore {
+ if e == nil {
+ return nil
+ }
+ if hasEmbedderFor(kbs) {
+ return newEngineCompiledStore(e, service.NewNavEmbedder(modelSvc, kbs[0].EmbdID), kbs[0].TenantID)
+ }
+ return newEngineCompiledStore(e, nil, "")
+}
+
+const tenantChunkIndexPrefix = "ragflow_"
+
+func tenantChunkIndexName(tenantID string) string {
+ return tenantChunkIndexPrefix + tenantID
+}
+
+// compiledSynthesisKinds are the standalone synthesis-page compile_kwd values.
+// Python scopes those rows by source_doc_ids, while entity/relation rows are
+// scoped by doc_id (compiled_expansion.py _search_compiled_rows /
+// _search_synthesis_pages).
+var compiledSynthesisKinds = map[string]bool{
+ "wiki_page": true,
+ "artifact_page": true,
+ "essence": true,
+}
+
+// compiledRowSelectFields is the projection the expander reads off a compiled
+// row: content_with_weight (seed name parsing), the entity/relation endpoints,
+// exact name lookups, and provenance back to source chunks.
+var compiledRowSelectFields = []string{
+ "id", "kb_id", "doc_id", "docnm_kwd",
+ "content_with_weight", "summary_with_weight", "source_chunk_ids",
+ "name_kwd", "from_entity_kwd", "to_entity_kwd", "available_int",
+}
+
+// compiledChunkSelectFields is the projection for source chunks promoted into
+// evidence by compiled expansion.
+var compiledChunkSelectFields = []string{
+ "id", "kb_id", "doc_id", "docnm_kwd",
+ "content_with_weight", "source_chunk_ids", "similarity",
+}
+
+// SearchCompiled implements agenticruntime.CompiledStore.
+func (s *engineCompiledStore) SearchCompiled(ctx context.Context, kbID, tenantID string, docIDs []string, filters map[string][]string, matchText string, topN int) ([]map[string]any, error) {
+ if s == nil || s.engine == nil || kbID == "" || strings.TrimSpace(tenantID) == "" {
+ return nil, nil
+ }
+ if topN <= 0 {
+ topN = 1
+ }
+ filter := compiledEngineFilter(filters)
+ if len(docIDs) > 0 {
+ filter[compiledDocScopeKey(filters)] = docIDs
+ }
+ req := &et.SearchRequest{
+ IndexNames: []string{tenantChunkIndexName(tenantID)},
+ KbIDs: []string{kbID},
+ Limit: topN,
+ SelectFields: compiledRowSelectFields,
+ Filter: filter,
+ }
+ // Python prefers a dense seed leg and only falls back to a keyword match
+ // when no embedder is available or the vector build fails
+ // (compiled_expansion.py _search_compiled_rows:180-194 /
+ // _search_synthesis_pages:423-438). Exactly ONE expr is appended.
+ if text := strings.TrimSpace(matchText); text != "" {
+ if dense := s.denseExpr(ctx, text, topN); dense != nil {
+ req.MatchExprs = []interface{}{dense}
+ } else {
+ req.MatchExprs = []interface{}{&et.MatchTextExpr{
+ Fields: []string{"content_ltks", "content_sm_ltks"},
+ MatchingText: text,
+ TopN: topN,
+ }}
+ }
+ }
+ res, err := s.engine.Search(ctx, req)
+ if err != nil || res == nil {
+ return nil, err
+ }
+ return res.Chunks, nil
+}
+
+// denseExpr builds the compiled-row dense match for the seed text, mirroring
+// Python _dense_expr (compiled_expansion.py:31-47): the query is embedded with
+// the dataset model's QUERY encoding, and the match carries the HNSW
+// num_candidates floor and similarity threshold. Returns nil when no embedder is
+// wired, the encode fails, or it panics — every caller then uses the keyword leg,
+// exactly like Python's `except Exception` around get_vector.
+func (s *engineCompiledStore) denseExpr(ctx context.Context, text string, topN int) *et.MatchDenseExpr {
+ if s == nil || s.embed == nil || strings.TrimSpace(s.embedTenantID) == "" {
+ return nil
+ }
+ var (
+ vecs [][]float32
+ err error
+ )
+ func() {
+ defer func() {
+ if r := recover(); r != nil {
+ common.Warn("compiled expansion: seed encode panicked; falling back to keyword match")
+ }
+ }()
+ vecs, err = s.embed.EncodeQueries(ctx, s.embedTenantID, []string{text})
+ }()
+ if err != nil || len(vecs) == 0 || len(vecs[0]) == 0 {
+ return nil
+ }
+ vec := make([]float64, len(vecs[0]))
+ for i, v := range vecs[0] {
+ vec[i] = float64(v)
+ }
+ return &et.MatchDenseExpr{
+ VectorColumnName: fmt.Sprintf("q_%d_vec", len(vec)),
+ EmbeddingData: vec,
+ EmbeddingDataType: "float",
+ DistanceType: "cosine",
+ TopN: topN,
+ ExtraOptions: map[string]interface{}{
+ "similarity": compiledVectorSimilarity,
+ "num_candidates": max(topN, compiledVectorNumCandidates),
+ },
+ }
+}
+
+// LoadChunks implements agenticruntime.CompiledStore: fetch the referenced source
+// chunks by id (Python compiled_expansion.py:_load_chunks_for_doc searches the
+// tenant index with an {"id": chunk_ids} condition).
+func (s *engineCompiledStore) LoadChunks(ctx context.Context, kbID, tenantID string, chunkIDs []string) ([]map[string]any, error) {
+ if s == nil || s.engine == nil || kbID == "" || strings.TrimSpace(tenantID) == "" || len(chunkIDs) == 0 {
+ return nil, nil
+ }
+ res, err := s.engine.Search(ctx, &et.SearchRequest{
+ IndexNames: []string{tenantChunkIndexName(tenantID)},
+ KbIDs: []string{kbID},
+ Limit: len(chunkIDs),
+ SelectFields: compiledChunkSelectFields,
+ Filter: map[string]interface{}{"id": chunkIDs},
+ })
+ if err != nil || res == nil {
+ return nil, err
+ }
+ return res.Chunks, nil
+}
+
+// Vectorize implements agenticruntime.CompiledStore. The ported expander assigns
+// compiled chunks the similarity stored on the row and blends by that field, so
+// expansion itself never needs a query embedding.
+func (s *engineCompiledStore) Vectorize(context.Context, string) ([]float64, error) {
+ return nil, fmt.Errorf("compiled store: vectorize is not wired")
+}
+
+// compiledDocScopeKey mirrors Python's doc-scope column choice: synthesis pages
+// are scoped by source_doc_ids, entity/relation rows by doc_id.
+func compiledDocScopeKey(filters map[string][]string) string {
+ for _, ck := range filters["compile_kwd"] {
+ if compiledSynthesisKinds[strings.ToLower(strings.TrimSpace(ck))] {
+ return "source_doc_ids"
+ }
+ }
+ return "doc_id"
+}
+
+// compiledEngineFilter maps the runtime' OR-list filters onto the engine filter
+// shape. available_int is a numeric column, so its string values are coerced to
+// ints (Python passes the literal 1).
+func compiledEngineFilter(filters map[string][]string) map[string]interface{} {
+ out := make(map[string]interface{}, len(filters))
+ for k, vals := range filters {
+ if k == "available_int" {
+ ints := make([]int, 0, len(vals))
+ for _, v := range vals {
+ if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil {
+ ints = append(ints, n)
+ }
+ }
+ if len(ints) > 0 {
+ out[k] = ints
+ }
+ continue
+ }
+ out[k] = vals
+ }
+ return out
+}
+
+// harnessWebSearcherFunc adapts the chat pipeline's web-search callback to the
+// agenticruntime.WebSearcher seam consumed by the web_search tool. The callback is a
+// plain func so internal/service does not have to depend on the runtime package.
+type harnessWebSearcherFunc func(ctx context.Context, queries []string) ([]string, error)
+
+// Search implements agenticruntime.WebSearcher.
+func (f harnessWebSearcherFunc) Search(ctx context.Context, queries []string) ([]string, error) {
+ return f(ctx, queries)
+}
+
+var _ agenticruntime.WebSearcher = harnessWebSearcherFunc(nil)
+
+// harnessWebSearcher wraps a non-nil callback, returning a nil WebSearcher when
+// the pipeline did not supply one — which is what hides the web_search tool.
+func harnessWebSearcher(fn func(context.Context, []string) ([]string, error)) agenticruntime.WebSearcher {
+ if fn == nil {
+ return nil
+ }
+ return harnessWebSearcherFunc(fn)
+}
diff --git a/internal/agent/retrievalbridge/harness_test.go b/internal/agent/retrievalbridge/harness_test.go
new file mode 100644
index 0000000000..ac4f61e7a6
--- /dev/null
+++ b/internal/agent/retrievalbridge/harness_test.go
@@ -0,0 +1,462 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package retrievalbridge
+
+import (
+ "context"
+ "errors"
+ "github.com/cloudwego/eino/schema"
+ "log"
+ "ragflow/internal/common"
+ "ragflow/internal/engine/types"
+ "ragflow/internal/entity"
+ "ragflow/internal/rag/agentic-rag"
+ agenticruntime "ragflow/internal/rag/agentic-rag/runtime"
+ "ragflow/internal/service"
+ "reflect"
+ "strings"
+ "testing"
+)
+
+func kb(embdID string, tenantEmbdID string) *entity.Knowledgebase {
+ kb := &entity.Knowledgebase{EmbdID: embdID}
+ if tenantEmbdID != "" {
+ kb.TenantEmbdID = &tenantEmbdID
+ }
+ return kb
+}
+
+// TestHasEmbedderForMirrorsPython pins dialog_service.py:362 —
+// `embd_mdl = LLMBundle(...) if kbs and kbs[0].embd_id else None`. The gate is
+// the FIRST dataset's embd_id, so a dialog whose leading dataset has no model
+// yields no embedding model even when a later one does.
+func TestHasEmbedderForMirrorsPython(t *testing.T) {
+ cases := []struct {
+ name string
+ kbs []*entity.Knowledgebase
+ want bool
+ }{
+ {"no datasets", nil, false},
+ {"single dataset with model", []*entity.Knowledgebase{kb("bge-m3", "")}, true},
+ {"single dataset without model", []*entity.Knowledgebase{kb("", "")}, false},
+ // Python reads kbs[0].embd_id only; it does not scan for any dataset
+ // carrying a model.
+ {"first without, later with", []*entity.Knowledgebase{kb("", ""), kb("bge-m3", "")}, false},
+ {"all with the same model", []*entity.Knowledgebase{kb("bge-m3", ""), kb("bge-m3", "")}, true},
+ }
+ for _, c := range cases {
+ if got := hasEmbedderFor(c.kbs); got != c.want {
+ t.Errorf("%s: hasEmbedderFor = %v, want %v", c.name, got, c.want)
+ }
+ }
+}
+
+// TestValidateDatasetEmbeddingModels pins knowledgebase_service.py:62-94.
+// Mixing is a hard error upstream (get_models raises at dialog_service.py:358) —
+// NOT a silent fallback to keyword-only retrieval.
+func TestValidateDatasetEmbeddingModels(t *testing.T) {
+ t.Run("all without a model is allowed", func(t *testing.T) {
+ if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{
+ kb("", ""), kb("", ""),
+ }); err != nil {
+ t.Errorf("all-without must be allowed, got %v", err)
+ }
+ })
+
+ t.Run("all same model is allowed", func(t *testing.T) {
+ if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{
+ kb("bge-m3", ""), kb("bge-m3", ""),
+ }); err != nil {
+ t.Errorf("same model must be allowed, got %v", err)
+ }
+ })
+
+ t.Run("some with some without is an error", func(t *testing.T) {
+ if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{
+ kb("bge-m3", ""), kb("", ""),
+ }); err == nil {
+ t.Error("mixing present/absent embedding models must error (Python raises)")
+ }
+ })
+
+ t.Run("different models is an error", func(t *testing.T) {
+ if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{
+ kb("bge-m3", ""), kb("text-embedding-3", ""),
+ }); err == nil {
+ t.Error("different embedding models must error (Python raises)")
+ }
+ })
+
+ t.Run("dangling tenant_model id falls back to embd_id base name", func(t *testing.T) {
+ // Python silently skips ids that no longer resolve
+ // (tenant_model_service.py:557) and falls back to
+ // _base_model_name(embd_id) — it must not fail the request.
+ if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{
+ kb("bge-m3@inst@prov", "gone-1"), kb("bge-m3@inst2@prov2", "gone-2"),
+ }); err != nil {
+ t.Errorf("two dangling refs with the same base name must be allowed, got %v", err)
+ }
+ })
+}
+
+func TestValidateLoadedDatasetsRejectsMissing(t *testing.T) {
+ err := validateLoadedDatasets([]string{"kb-1", "kb-2"}, []*entity.Knowledgebase{{ID: "kb-1"}})
+ if err == nil || !strings.Contains(err.Error(), `dataset "kb-2" was not found`) {
+ t.Fatalf("validateLoadedDatasets error = %v, want missing kb-2", err)
+ }
+}
+
+// TestBaseModelNameMirrorsPython pins knowledgebase_service.py:33:
+// rsplit("@", 2)[0] — the model name, with @instance@provider stripped.
+func TestBaseModelNameMirrorsPython(t *testing.T) {
+ cases := []struct{ in, want string }{
+ {"bge-m3@inst@prov", "bge-m3"},
+ {"bge-m3@inst", "bge-m3"},
+ {"bge-m3", "bge-m3"},
+ {"", ""},
+ }
+ for _, c := range cases {
+ if got := common.BaseModelName(c.in); got != c.want {
+ t.Errorf("baseModelName(%q) = %q, want %q", c.in, got, c.want)
+ }
+ }
+}
+
+// TestHarnessWebSearcherNilAndWrapped pins the adapter: a nil pipeline callback
+// must yield a nil harness.WebSearcher (which hides the web_search tool), and a
+// non-nil one must be callable through the seam.
+func TestHarnessWebSearcherNilAndWrapped(t *testing.T) {
+ if got := harnessWebSearcher(nil); got != nil {
+ t.Error("nil callback must yield a nil WebSearcher")
+ }
+ called := false
+ searcher := harnessWebSearcher(func(_ context.Context, queries []string) ([]string, error) {
+ called = true
+ return queries, nil
+ })
+ if searcher == nil {
+ t.Fatal("non-nil callback must yield a WebSearcher")
+ }
+ got, err := searcher.Search(context.Background(), []string{"q"})
+ if err != nil {
+ t.Fatalf("Search: %v", err)
+ }
+ if !called {
+ t.Error("callback was not invoked")
+ }
+ if len(got) != 1 || got[0] != "q" {
+ t.Errorf("Search = %v, want [q]", got)
+ }
+}
+
+// errStubEncode stands in for a provider-side embedding failure.
+var errStubEncode = errors.New("stub encode failure")
+
+// recordingEngine captures the SearchRequest so the tests can pin the adapter's
+// mapping without a live document engine.
+type recordingEngine struct {
+ last *types.SearchRequest
+ res *types.SearchResult
+}
+
+func (e *recordingEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
+ e.last = req
+ if e.res != nil {
+ return e.res, nil
+ }
+ return &types.SearchResult{}, nil
+}
+
+// TestEngineCompiledStoreSearchCompiledMapsRequest pins the mapping from the
+// harness' compiled-row request onto the engine search: tenant index, kb scope,
+// numeric available_int, the synthesis-page doc-scope column (source_doc_ids),
+// and the keyword match leg over content_ltks/content_sm_ltks.
+func TestEngineCompiledStoreSearchCompiledMapsRequest(t *testing.T) {
+ eng := &recordingEngine{}
+ // No embedder wired → the keyword leg stands alone.
+ store := newEngineCompiledStore(eng, nil, "")
+ _, err := store.SearchCompiled(context.Background(), "kb1", "t1", []string{"d1"}, map[string][]string{
+ "compile_kwd": {"wiki_page"},
+ "available_int": {"1"},
+ }, "culdcept", 7)
+ if err != nil {
+ t.Fatalf("SearchCompiled: %v", err)
+ }
+ req := eng.last
+ if req == nil {
+ t.Fatal("engine was not called")
+ }
+ if len(req.IndexNames) != 1 || req.IndexNames[0] != "ragflow_t1" {
+ t.Errorf("IndexNames = %v, want [ragflow_t1]", req.IndexNames)
+ }
+ if len(req.KbIDs) != 1 || req.KbIDs[0] != "kb1" {
+ t.Errorf("KbIDs = %v, want [kb1]", req.KbIDs)
+ }
+ if req.Limit != 7 {
+ t.Errorf("Limit = %d, want 7", req.Limit)
+ }
+ // available_int is a numeric column: "1" must reach the engine as an int.
+ if got, ok := req.Filter["available_int"].([]int); !ok || len(got) != 1 || got[0] != 1 {
+ t.Errorf("available_int = %#v, want []int{1}", req.Filter["available_int"])
+ }
+ if got, ok := req.Filter["compile_kwd"].([]string); !ok || len(got) != 1 || got[0] != "wiki_page" {
+ t.Errorf("compile_kwd = %#v, want []string{wiki_page}", req.Filter["compile_kwd"])
+ }
+ // Synthesis pages are scoped by source_doc_ids, not doc_id.
+ if _, ok := req.Filter["source_doc_ids"]; !ok {
+ t.Errorf("doc scope key = %v, want source_doc_ids present", req.Filter)
+ }
+ if _, ok := req.Filter["doc_id"]; ok {
+ t.Errorf("synthesis rows must not use doc_id scope: %v", req.Filter)
+ }
+ if len(req.MatchExprs) != 1 {
+ t.Fatalf("MatchExprs = %#v, want one keyword expr", req.MatchExprs)
+ }
+ mt, ok := req.MatchExprs[0].(*types.MatchTextExpr)
+ if !ok {
+ t.Fatalf("MatchExprs[0] = %T, want *types.MatchTextExpr", req.MatchExprs[0])
+ }
+ if mt.MatchingText != "culdcept" || mt.TopN != 7 {
+ t.Errorf("keyword expr = %+v, want text=culdcept topN=7", mt)
+ }
+ if len(mt.Fields) != 2 || mt.Fields[0] != "content_ltks" || mt.Fields[1] != "content_sm_ltks" {
+ t.Errorf("keyword fields = %v, want [content_ltks content_sm_ltks]", mt.Fields)
+ }
+}
+
+// TestCompiledDocScopeKey pins Python's column choice: entity/relation rows use
+// doc_id; the three synthesis kinds use source_doc_ids.
+func TestCompiledDocScopeKey(t *testing.T) {
+ cases := []struct {
+ name string
+ filters map[string][]string
+ want string
+ }{
+ {"entity rows", map[string][]string{"knowledge_graph_kwd": {"entity"}}, "doc_id"},
+ {"tree rows", map[string][]string{"knowledge_graph_kwd": {"entity"}, "compile_kwd": {"tree"}}, "doc_id"},
+ {"wiki pages", map[string][]string{"compile_kwd": {"wiki_page"}}, "source_doc_ids"},
+ {"artifact pages", map[string][]string{"compile_kwd": {"artifact_page"}}, "source_doc_ids"},
+ {"essence", map[string][]string{"compile_kwd": {"essence"}}, "source_doc_ids"},
+ }
+ for _, c := range cases {
+ if got := compiledDocScopeKey(c.filters); got != c.want {
+ t.Errorf("%s: compiledDocScopeKey = %q, want %q", c.name, got, c.want)
+ }
+ }
+}
+
+// TestEngineCompiledStoreLoadChunks pins the id-scoped fetch used to promote
+// source chunks into evidence.
+func TestEngineCompiledStoreLoadChunks(t *testing.T) {
+ eng := &recordingEngine{res: &types.SearchResult{Chunks: []map[string]interface{}{{"id": "c1"}}}}
+ store := newEngineCompiledStore(eng, nil, "")
+ rows, err := store.LoadChunks(context.Background(), "kb1", "t1", []string{"c1", "c2"})
+ if err != nil {
+ t.Fatalf("LoadChunks: %v", err)
+ }
+ if len(rows) != 1 {
+ t.Fatalf("rows = %d, want 1", len(rows))
+ }
+ if eng.last.Limit != 2 {
+ t.Errorf("Limit = %d, want 2", eng.last.Limit)
+ }
+ ids, ok := eng.last.Filter["id"].([]string)
+ if !ok || len(ids) != 2 || ids[0] != "c1" || ids[1] != "c2" {
+ t.Errorf("id filter = %#v, want [c1 c2]", eng.last.Filter["id"])
+ }
+}
+
+// TestNewDatasetCompiledStore pins the store wiring: no engine yields no store;
+// datasets without an embedding model yield a keyword-only store; a dataset
+// embedding model wires the query encoder with THAT dataset's model and owner
+// tenant (not a tenant default or the request tenant).
+func TestNewDatasetCompiledStore(t *testing.T) {
+ eng := &recordingEngine{}
+ if got := newDatasetCompiledStore(nil, nil, nil); got != nil {
+ t.Error("nil engine must disable the compiled store")
+ }
+ if got := newDatasetCompiledStore(eng, nil, []*entity.Knowledgebase{{ID: "kb1", TenantID: "t1"}}); got == nil {
+ t.Error("engine without an embedder must still build a keyword-only store")
+ }
+ store := newDatasetCompiledStore(eng, service.NewModelProviderService(),
+ []*entity.Knowledgebase{{ID: "kb1", EmbdID: "bge-m3@inst@prov", TenantID: "owner-tenant"}})
+ es, ok := store.(*engineCompiledStore)
+ if !ok {
+ t.Fatalf("store = %T, want *engineCompiledStore", store)
+ }
+ if es.embed == nil {
+ t.Fatal("the dataset's embedding model must wire the query encoder")
+ }
+ if es.embedTenantID != "owner-tenant" {
+ t.Errorf("embedTenantID = %q, want owner-tenant (the dataset owner, not the request tenant)", es.embedTenantID)
+ }
+}
+
+// stubQueryEncoder records the query-encoding calls the dense leg makes.
+type stubQueryEncoder struct {
+ vecs [][]float32
+ err error
+ calls int
+ lastTenant string
+}
+
+func (s *stubQueryEncoder) EncodeQueries(_ context.Context, tenantID string, _ []string) ([][]float32, error) {
+ s.calls++
+ s.lastTenant = tenantID
+ return s.vecs, s.err
+}
+
+// TestEngineCompiledStoreUsesDenseSeedLeg pins the parity fix: when an embedder
+// is wired, the compiled-row search goes out as a DENSE expr on the query
+// vector's q_<dim>_vec column with Python's num_candidates floor and similarity
+// threshold — and carries NO keyword expr alongside it (Python appends exactly
+// one). The encoding runs under the embedder's OWN tenant (Python
+// embd_owner_tenant_id = kbs[0].tenant_id), not the searched scope's tenant.
+func TestEngineCompiledStoreUsesDenseSeedLeg(t *testing.T) {
+ eng := &recordingEngine{}
+ enc := &stubQueryEncoder{vecs: [][]float32{{1, 2, 3}}}
+ // Scope tenant differs from the embedder's owner tenant on purpose.
+ store := newEngineCompiledStore(eng, enc, "owner-tenant")
+
+ if _, err := store.SearchCompiled(context.Background(), "kb1", "scope-tenant", nil,
+ map[string][]string{"knowledge_graph_kwd": {"entity"}}, "culdcept", 5); err != nil {
+ t.Fatalf("SearchCompiled: %v", err)
+ }
+ if enc.calls != 1 {
+ t.Fatalf("encode calls = %d, want 1", enc.calls)
+ }
+ if enc.lastTenant != "owner-tenant" {
+ t.Errorf("encode tenant = %q, want the embedder's owner tenant owner-tenant", enc.lastTenant)
+ }
+ if len(eng.last.MatchExprs) != 1 {
+ t.Fatalf("MatchExprs = %#v, want exactly one dense expr", eng.last.MatchExprs)
+ }
+ dense, ok := eng.last.MatchExprs[0].(*types.MatchDenseExpr)
+ if !ok {
+ t.Fatalf("MatchExprs[0] = %T, want *types.MatchDenseExpr", eng.last.MatchExprs[0])
+ }
+ if dense.VectorColumnName != "q_3_vec" {
+ t.Errorf("VectorColumnName = %q, want q_3_vec", dense.VectorColumnName)
+ }
+ if dense.EmbeddingDataType != "float" || dense.DistanceType != "cosine" || dense.TopN != 5 {
+ t.Errorf("dense expr = %+v, want float/cosine/topN=5", dense)
+ }
+ if dense.ExtraOptions["similarity"] != 0.1 {
+ t.Errorf("similarity = %v, want 0.1", dense.ExtraOptions["similarity"])
+ }
+ // max(topN, 256) — a small topN is floored to the HNSW candidate count.
+ if dense.ExtraOptions["num_candidates"] != 256 {
+ t.Errorf("num_candidates = %v, want 256 (max(5,256))", dense.ExtraOptions["num_candidates"])
+ }
+
+ // A topN above the floor is passed through unchanged.
+ if _, err := store.SearchCompiled(context.Background(), "kb1", "scope-tenant", nil,
+ map[string][]string{"knowledge_graph_kwd": {"entity"}}, "culdcept", 300); err != nil {
+ t.Fatalf("SearchCompiled(300): %v", err)
+ }
+ dense300 := eng.last.MatchExprs[0].(*types.MatchDenseExpr)
+ if dense300.ExtraOptions["num_candidates"] != 300 {
+ t.Errorf("num_candidates = %v, want 300", dense300.ExtraOptions["num_candidates"])
+ }
+}
+
+// TestEngineCompiledStoreDenseFallsBackToKeyword pins Python's `except Exception`
+// around get_vector: an encode failure (or an empty vector) must degrade to the
+// keyword leg rather than dropping the expansion.
+func TestEngineCompiledStoreDenseFallsBackToKeyword(t *testing.T) {
+ for _, c := range []struct {
+ name string
+ enc *stubQueryEncoder
+ }{
+ {"encode error", &stubQueryEncoder{err: errStubEncode}},
+ {"empty vector", &stubQueryEncoder{vecs: [][]float32{{}}}},
+ {"no vectors", &stubQueryEncoder{vecs: nil}},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ eng := &recordingEngine{}
+ store := newEngineCompiledStore(eng, c.enc, "t1")
+ if _, err := store.SearchCompiled(context.Background(), "kb1", "t1", nil,
+ map[string][]string{"knowledge_graph_kwd": {"entity"}}, "culdcept", 5); err != nil {
+ t.Fatalf("SearchCompiled: %v", err)
+ }
+ if c.enc.calls != 1 {
+ t.Errorf("encode calls = %d, want 1", c.enc.calls)
+ }
+ if len(eng.last.MatchExprs) != 1 {
+ t.Fatalf("MatchExprs = %#v, want one keyword expr", eng.last.MatchExprs)
+ }
+ mt, ok := eng.last.MatchExprs[0].(*types.MatchTextExpr)
+ if !ok {
+ t.Fatalf("MatchExprs[0] = %T, want *types.MatchTextExpr keyword fallback", eng.last.MatchExprs[0])
+ }
+ if mt.MatchingText != "culdcept" {
+ t.Errorf("keyword text = %q, want culdcept", mt.MatchingText)
+ }
+ })
+ }
+}
+
+// Exercise the production bridge: history must reach the graph and its
+// formalizer rather than being replaced by only the latest question.
+func TestHarnessRetrieverForwardsConversation(t *testing.T) {
+ model := &historyModel{}
+ called := false
+ agentic_rag.SetAgenticLoop(func(ctx context.Context, deps agentic_rag.RAGTools, req agenticruntime.RunRequest, _ *agenticruntime.Kbinfos, resp *agentic_rag.RunResponse, _ *log.Logger) {
+ called = true
+ want := []schema.Message{{Role: schema.User, Content: "Tell me about Culdcept."}, {Role: schema.Assistant, Content: "Culdcept was made by OmiyaSoft."}, {Role: schema.User, Content: "When was it released?"}}
+ if !reflect.DeepEqual(deps.Messages, want) {
+ t.Fatalf("history lost: %#v", deps.Messages)
+ }
+ resp.Answer, _ = agentic_rag.Formalize(ctx, agenticruntime.SessionDeps{Model: model}, deps.Messages, 8192)
+ })
+ t.Cleanup(func() { agentic_rag.SetAgenticLoop(nil) })
+ req := service.HarnessRequest{Question: "When was it released?", ThinkingMode: "high", Messages: []map[string]interface{}{
+ {"role": "user", "content": "Tell me about Culdcept."},
+ {"role": "assistant", "content": "Culdcept was made by OmiyaSoft."},
+ {"role": "user", "content": []interface{}{map[string]interface{}{"type": "text", "text": "When was it released?"}}},
+ }}
+ got, err := NewHarnessRetriever(nil, nil, nil)(t.Context(), req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !called || model.calls != 1 || got.Answer != "When was Culdcept released?" || !strings.Contains(model.transcript, "OmiyaSoft") {
+ t.Fatalf("formalization lost conversation: result=%+v model=%+v", got, model)
+ }
+ req.Messages[0]["content"] = 17
+ if _, err := NewHarnessRetriever(nil, nil, nil)(t.Context(), req); err == nil {
+ t.Fatal("invalid history content accepted")
+ }
+}
+
+type historyModel struct {
+ transcript string
+ calls int
+}
+
+func (m *historyModel) Complete(_ context.Context, messages []schema.Message, _ []agenticruntime.ToolSpec) (*agenticruntime.ModelReply, error) {
+ m.calls++
+ for _, msg := range messages {
+ m.transcript += msg.Content
+ }
+ return &agenticruntime.ModelReply{Content: `{"question":"When was Culdcept released?","keywords":"Culdcept released"}`}, nil
+}
+
+func TestEmbeddingValidationPreservesAtInModelNames(t *testing.T) {
+ if err := validateDatasetEmbeddingModels(t.Context(), []*entity.Knowledgebase{kb("model@q4@instance@provider", ""), kb("model@q8@instance@provider", "")}); err == nil {
+ t.Fatal("different quantized models were conflated")
+ }
+}
diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.go
index 02e14fbd7c..3968d4b66f 100644
--- a/internal/service/chat_pipeline.go
+++ b/internal/service/chat_pipeline.go
@@ -5155,12 +5155,11 @@ type HarnessResult struct {
CiteChunkIDs []string
}
-// harnessRetriever is wired at server bootstrap (cmd/ragflow_server.go:889) to
-// the agentic-RAG entry point advanced_rag.Rag (:1022). The advanced_rag package
-// imports internal/service (e.g. harness/tool_exploration.go), so the service
-// layer cannot import it back without an import cycle; the function is injected
-// instead. When nil, retrieveViaHarness reports an error and the pipeline
-// continues with empty kbinfos.
+// harnessRetriever is installed at server bootstrap using
+// retrievalbridge.NewHarnessRetriever. The bridge imports internal/service,
+// so this package receives the callback instead of importing the bridge.
+// When nil, retrieveViaHarness reports an error and the pipeline continues
+// with empty kbinfos.
var harnessRetriever func(ctx context.Context, req HarnessRequest) (HarnessResult, error)
// SetHarnessRetriever injects the agentic-RAG harness driver. Call once at