RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(harness): extract retrieval wiring from server startup (#20012)
Sbaaoui Idriss2d9acf7762b2
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.goindex 2b11f0299e..33468b7a7f 100644--- a/cmd/ragflow_server.go+++ b/cmd/ragflow_server.go@@ -40,7 +40,6 @@ import ("ragflow/internal/ingestion/knowledge_compile"ingestion "ragflow/internal/ingestion/service""ragflow/internal/rag/agentic-rag"- agenticruntime "ragflow/internal/rag/agentic-rag/runtime""ragflow/internal/router""ragflow/internal/server/local""ragflow/internal/service"@@ -69,7 +68,6 @@ import ("ragflow/internal/deepdoc/parser/pdf/inference/native_analyzer""ragflow/internal/engine""ragflow/internal/engine/kvrocks"- et "ragflow/internal/engine/types""ragflow/internal/entity"_ "ragflow/internal/ingestion/wire""ragflow/internal/server"@@ -92,94 +90,6 @@ type serverArgs struct {}-// engineDocEngine is the small slice of the engine surface the doc-chunk pager-// needs, kept as a local interface so it is trivially unit-testable without a-// live engine (the production value is engine.Get()).-type engineDocEngine interface {- Search(ctx context.Context, req *et.SearchRequest) (*et.SearchResult, error)-}--// docChunkPager is the production agenticruntime.DocChunkLister. It pages one-// document's chunks in reading order straight off the chunk index, mirroring-// Python settings.retriever.chunk_list with sort_by_position=True-// (rag/nlp/search.py:824). This is what the document-level tools-// (summarize_document / fetch_full_document) consume.-//-// The index name is derived from the tenant id (ragflow_<tenantID>), exactly as-// retrieval does, so it needs no user-scope resolution: DocChunksRequest-// already carries the tenant and the bound datasets.-type docChunkPager struct {- docEngine engineDocEngine-}--// DocChunks implements agenticruntime.DocChunkLister.-func (p *docChunkPager) DocChunks(ctx context.Context, req agenticruntime.DocChunksRequest) ([]map[string]any, error) {- if p == nil || p.docEngine == nil {- return nil, nil- }- if req.DocID == "" {- return nil, nil- }- orderBy := (&et.OrderByExpr{}).- Asc("chunk_order_int").- Asc("page_num_int").- Asc("top_int").- Desc("create_timestamp_flt")-- resp, err := p.docEngine.Search(ctx, &et.SearchRequest{- IndexNames: []string{tenantChunkIndexName(req.TenantID)},- KbIDs: req.DatasetIDs,- Offset: req.Offset,- Limit: req.Limit,- OrderBy: orderBy,- SelectFields: []string{- "id", "doc_id", "docnm", "content_with_weight", "content", "available_int",- },- Filter: map[string]interface{}{- "doc_id": req.DocID,- "available_int": 1,- "must_not": map[string]interface{}{"exists": "compile_kwd"},- },- })- if err != nil {- return nil, err- }- if resp == nil {- return nil, nil- }- rows := make([]map[string]any, 0, len(resp.Chunks))- for _, ck := range resp.Chunks {- row := make(map[string]any, 8)- // The runtime chunk rows need the canonical keys its citation pipeline- // reads: chunk_id (dedup), docnm_kwd (doc title), doc_id and the text- // under content/content_with_weight (chunkText prefers content first).- if id, ok := ck["id"].(string); ok && id != "" {- row["chunk_id"] = id- }- if d, ok := ck["doc_id"]; ok {- row["doc_id"] = d- }- if d, ok := ck["docnm"]; ok {- row["docnm_kwd"] = d- }- if c, ok := ck["content"]; ok {- row["content"] = c- }- if c, ok := ck["content_with_weight"]; ok {- row["content_with_weight"] = c- }- // Keep whatever else the engine returned (e.g. position_int, img) so the- // merged Kbinfos rows stay rich, as retrieval does.- for k, v := range ck {- if _, seen := row[k]; !seen {- row[k] = v- }- }- rows = append(rows, row)- }- return rows, nil-}-func parseArgs() (*serverArgs, error) {args := &serverArgs{}@@ -1118,209 +1028,7 @@ func startServer(ctx context.Context) error {// reasoning chats.runtime.SetRetrievalService(retrievalbridge.NewRuntimeAdapter(retrievalAdapter))agentic_rag.SetAgenticLoop(agentic_rag.NewAgenticLoop())- service.SetHarnessRetriever(func(ctx context.Context, req service.HarnessRequest) (service.HarnessResult, error) {- // Resolve the tenant's actual chat model, mirroring Python where RAGTools- // receives a fully-resolved LLMBundle: chat.LLMID may be a UUID/tenant_model- // id that the default invoker cannot split into a provider, so resolving here- // avoids falling through to a dummy driver. If resolution fails we leave model- // nil so the runtime degrades to a direct search (no dummy fallback).- var model agenticruntime.SessionModel- // outerModel mirrors Python RAGTools.chat_mdl: the fully-resolved chat- // model that drives the outer rag_agent react loop (tools=[rag,- // summarize_document], terminal_tools={"rag"}). Wired into RAGTools.Outer- // so Rag() can run that loop instead of the direct graph.- var outerModel *modelModule.ChatModel- // resolvedModelName is the resolved chat model's own name (e.g.- // "gpt-4o"), i.e. Python chat_mdl.llm_name. It is the gen_json reply-cache- // key's model component; empty disables that cache.- resolvedModelName := ""- // The resolution goes through the same entry point the pipeline's own- // capability probe uses, so the model decided there is the model wired- // here: req.ModelID may be empty (a dialog without an llm_id), and both- // layers must then fall back to the tenant default rather than resolving- // nothing here and judging capability there.- if target, mErr := modelProviderService.ResolveChatModelTarget(ctx, req.TenantID, req.ModelID); mErr == nil {- if inv := component.NewResolvedInvoker(target.Driver, target.ModelName, target.APIConfig); inv != nil {- // MaxLength mirrors Python LLMBundle.max_length (the model's- // context window in tokens); message-fitting nodes (calculate,- // structure_qa) use it as their chat.FitMessages budget.- model = &agenticruntime.InvokerSessionModel{Invoker: inv, DB: dao.DB, MaxLength: target.ContextLength}- resolvedModelName = target.ModelName- // Reuse the same resolved driver/name/api as the invoker so- // the outer loop and the inner tool calls share one model.- outerModel = modelModule.NewChatModel(target.Driver, &target.ModelName, target.APIConfig)- }- } else {- common.Warn("runtime: failed to resolve chat model for reasoning; runtime will degrade to direct search", zap.Error(mErr))- }-- // Load the KB objects (mirroring Python RAGTools' self.kbs via- // KnowledgebaseService.get_by_ids(kb_ids)) so the agentic tool can- // derive rank features. The Go tag extractor (extractor_tag.go) writes- // both tag_kwd (the list of tag names) and tag_feas (per-tag weights)- // onto each chunk at parse time; the labeler aggregates tag_kwd to build- // the tag vocabulary and the retriever ranks with tag_feas. Best-effort:- // a load failure leaves KBs empty and the adapter resolves them itself.- var kbs []*entity.Knowledgebase- if len(req.DatasetIDs) > 0 {- if loaded, lErr := dao.NewKnowledgebaseDAO().GetByIDs(ctx, dao.DB, req.DatasetIDs); lErr == nil {- kbs = loaded- }- }- // validate_dataset_embedding_models runs FIRST upstream- // (dialog_service.py:358-360) and mixing is a hard failure there, not a- // fallback to keyword-only retrieval.- if err := validateDatasetEmbeddingModels(ctx, kbs); err != nil {- return service.HarnessResult{}, err- }- // HasEmbedder mirrors Python `embd_mdl = ... if kbs and kbs[0].embd_id else- // None` (dialog_service.py:362): the gate is the FIRST dataset's embd_id. It- // is what hybrid_search applies to its vector leg (search.py:143), so- // getting it wrong silently degrades search_chunks to keyword-only.- hasEmbedder := hasEmbedderFor(kbs)- // Retrieval tuning mirrors the dialog's own settings, so the agentic- // searches run with the same budget as the standard path. Without it the- // harness falls back to its package defaults (top_n=12) and the dialog's- // configured top_n is silently ignored.- //- // The runtime carries the KEYWORD leg's weight and derives the vector leg- // as its complement (runtime.resolveKeywordsSimilarityWeight, and the- // RetrieveRequest contract: 0.7 hybrid / 0.0 vector-only / 1.0- // keyword-only). The dialog configures the VECTOR weight, so it is inverted- // here; the dialog's default 0.3 therefore yields the runtime's own 0.7.- keywordsWeight := 1 - req.Retrieval.VectorSimilarityWeight- deps := agentic_rag.RAGTools{- Model: model,- ModelName: resolvedModelName,- Outer: outerModel,- TopN: req.Retrieval.TopN,- SimilarityThreshold: req.Retrieval.SimilarityThreshold,- KeywordsSimilarityWeight: &keywordsWeight,- RerankCandidatesCount: req.Retrieval.RerankCandidatesCount,- // OriginalQuestion mirrors Python RAGTools(original_user_question=...):- // the user's own, unrewritten question as received from the chat- // layer. The outer model's `rag(question=...)` argument is- // model-generated and often compresses a multi-hop question to its- // first hop; resolveEffectiveQuestion (agentic_rag.py:865) prefers- // this original over that rewrite when both describe the same turn.- OriginalQuestion: req.Question,- // DocScope mirrors Python RAGTools(doc_scope=...): the narrowed doc_ids- // (chat-level doc_ids + meta_data_filter) restrict every agentic- // retrieval to the user-selected documents instead of the whole kb.- DocScope: req.DocIDs,- DocIDVerifier: agentic_rag.NewDocIDLookup(),- // MetadataResolver backs the metadata_search tool and the pre-search- // metadata channel (Python DocMetadataService push-down + meta_filter- // fallback). metadataService already carries the doc engine and the DAO- // the metadata index reads need.- MetadataResolver: metadataService,- // DeclaredMetadata lets the metadata_search catalog describe each field- // (meaning + allowed values) from the dataset's own parser_config, not just- // name it. Same service: it also carries the KB DAO that read needs.- DeclaredMetadata: metadataService,- // DocChunks pages one document's chunks in reading order off the- // chunk index (Python retriever.chunk_list), backing the- // document-level tools (summarize_document / fetch_full_document).- DocChunks: &docChunkPager{docEngine: docEngine},- // Expand runs search_chunks' compiled-structure expansion- // (Python hybrid_search use_compiled=True → _expand_with_compiled).- // Backed by the same document engine the chunk reads use, with the- // dense seed leg wired to the bound dataset's own embedding model.- // The scope config lets each dataset be scanned under its own tenant- // and a doc scope be grouped by real owner, like graph_explore.- Expand: agenticruntime.NewCompiledExpander(- newDatasetCompiledStore(docEngine, modelProviderService, kbs),- agenticruntime.CompiledScopeConfig{- DatasetIDs: req.DatasetIDs,- TenantID: req.TenantID,- KBs: kbs,- DocTenantResolver: agentic_rag.NewDocTenantResolver(),- },- ),- // WebSearch backs the web_search tool (Python RAGTools.web_search).- // The pipeline supplies a callback only when the chat enables- // internet web search; a nil one also keeps the tool off the surface.- WebSearch: harnessWebSearcher(req.WebSearch),- KBs: kbs,- HasEmbedder: hasEmbedder,- // Embedder backs claim recall's KNN leg (Python- // recall_dataset_claims, navigation.py:1837-1909, which embeds the- // query with tools.embed_mdl — the FIRST dataset's embedding model,- // dialog_service.py:362-366) and the structure-drill seed vector.- // Without it the claim leg silently degrades to BM25-only and- // paraphrase-phrased claims are never recalled. Bound to- // kbs[0].EmbdID (validateDatasetEmbeddingModels above guarantees- // every bound dataset shares it), query-side encoded.- Embedder: embedderForDatasets(kbs, modelProviderService),- // Tagger is the Go equivalent of Python's label_question- // (agentic_rag.py:668): classifies the query into question-type- // tags the retriever boosts on. metadataService implements it- // (service.MetadataService.LabelQuestion).- Tagger: metadataService,- // SystemPrompt mirrors Python RAGTools(system_prompt=- // _render_reasoning_system_prompt(dialog, prompt_config, kwargs),- // dialog_service.py:2084) — the dialog-level UI configuration the- // final-answer compose appends after the agentic contract- // (agentic_rag_graph.py:923-933).- SystemPrompt: req.SystemPrompt,- // CiteRules and EvidenceMaxTokens deliberately stay unset: Python's- // dialog path constructs RAGTools WITHOUT user_defined_prompts- // (dialog_service.py:2072-2090), so citation_prompt defaults apply,- // and Python has no evidence-token override (the compose always- // uses min(chat_mdl.max_length, _EVIDENCE_BUDGET_TOKENS=8000),- // which EvidenceMaxTokens<=0 reproduces).- }- // Diagnose WHY compiled expansion is disabled: NewCompiledExpander- // returns nil for three reasons (store==nil / no datasets / no tenant)- // and RAGTools.Expand==nil silences the whole channel with no other- // trace — the observed "Compiled expansion enabled = 0" was invisible.- if deps.Expand == nil {- switch {- case docEngine == nil:- common.Warn("compiled expansion disabled: document engine unavailable (store==nil)")- case len(req.DatasetIDs) == 0:- common.Warn("compiled expansion disabled: no bound dataset (DatasetIDs empty)")- case strings.TrimSpace(req.TenantID) == "":- common.Warn("compiled expansion disabled: no tenant (TenantID empty)")- default:- common.Warn("compiled expansion disabled: unknown reason")- }- }- // The two projections of one reasoning step: the sentence the chat UI- // appends to its think block, and the structured event a step-rendering- // client consumes. Steps.Stage/Emit feeds both from one call, so the- // trace cannot drift from its structured twin. Each step is an- // isThink=true delta, i.e. think-block content rather than answer text.- if req.AnswerSink != nil {- answerSink := req.AnswerSink- deps.AnswerSink = &agentic_rag.AnswerSink{- OnDelta: req.AnswerSink,- }- deps.Steps.Text = func(line string) { answerSink(line, true) }- }- if req.ThinkSink != nil {- // One type on both sides — agenticruntime.ThinkEvent is an alias of- // service.ThinkEvent — so the sink passes straight through: there is- // nothing to copy field by field, and no way to forget a field that- // was added on one side only.- deps.Steps.Events = req.ThinkSink- }- r := agentic_rag.Rag(ctx, deps, agenticruntime.RunRequest{- Question: req.Question,- ThinkingMode: req.ThinkingMode,- DatasetIDs: req.DatasetIDs,- TenantID: req.TenantID,- SessionID: req.SessionID,- Images: req.Images,- TextAttachments: req.TextAttachments,- })- res := service.HarnessResult{Chunks: r.Chunks, DocAggs: r.DocAggs, Answer: r.Answer, SlotCitations: r.SlotCitations, CiteChunkIDs: r.CiteChunkIDs}- if r.Kbinfos != nil {- res.Memory = r.Kbinfos.Memory- res.PreSummary = r.Kbinfos.PreSummary- }- return res, nil- })+ service.SetHarnessRetriever(retrievalbridge.NewHarnessRetriever(modelProviderService, metadataService, docEngine))common.Info("agent: runtime chat retriever wired (runtime retrieval + agentic loop)")// Initialize handler layer@@ -1806,395 +1514,3 @@ func resolveDeepDocDropScore() float64 {func dirHasModels(dir string) bool {return common.HasModelFiles(dir)}--// hasEmbedderFor mirrors Python `embd_mdl = LLMBundle(...) if kbs and-// kbs[0].embd_id else None` (dialog_service.py:362). The gate is the FIRST bound-// dataset's embd_id, not "any dataset has one": a dialog whose first dataset-// carries no embedding model yields embd_mdl=None upstream even if later ones do.-//-// Callers must run validateDatasetEmbeddingModels first — with validation-// passing, all datasets share one model and this equals "all have one".-func hasEmbedderFor(kbs []*entity.Knowledgebase) bool {- return len(kbs) > 0 && kbs[0] != nil && kbs[0].EmbdID != ""-}--// embedderForDatasets builds the embedding handle the agentic runtime uses for-// query-side encoding outside the main retrieval leg: claim recall's KNN leg-// (Python recall_dataset_claims, navigation.py:1837-1909, embedding with-// tools.embed_mdl) and the structure-drill seed vector. The model is the FIRST-// dataset's embedding (Python dialog_service.py:362-366 resolves-// kbs[0].embd_id under kbs[0].tenant_id; validateDatasetEmbeddingModels-// guarantees the rest share it). Nil when no dataset carries an embedding-// model — the claim leg then degrades to BM25-only, matching a Python run-// without embed_mdl.-func embedderForDatasets(kbs []*entity.Knowledgebase, modelSvc *service.ModelProviderService) nlp.NavEmbedder {- if len(kbs) == 0 || kbs[0] == nil || kbs[0].EmbdID == "" {- return nil- }- return service.NewNavEmbedder(modelSvc, kbs[0].EmbdID)-}--// validateDatasetEmbeddingModels mirrors Python validate_dataset_embedding_models-// (knowledgebase_service.py:62-94): every bound dataset must use the same embedding-// model, or none at all. Upstream this runs before embd_mdl is resolved-// (dialog_service.py:358-360) and a violation RAISES — it is not a "no embedding-// model" fallback: treating mixing as HasEmbedder=false would silently serve-// keyword-only results where Python errors.-func validateDatasetEmbeddingModels(ctx context.Context, kbs []*entity.Knowledgebase) error {- var withEmbd int- for _, kb := range kbs {- if kb != nil && kb.EmbdID != "" {- withEmbd++- }- }- hasEmbd := withEmbd > 0- if hasEmbd && withEmbd != len(kbs) {- return errors.New("cannot search across datasets where some have embedding models and others do not")- }- if !hasEmbd {- return nil- }- // Mirror Python's grouping key exactly:- //- // ref = tenant_embd_id or (embd_id when it is a raw tenant_model id)- // composite = resolved_names.get(ref) # "model@instance@provider"- // key = _base_model_name(composite) # == tenant_model.model_name- //- // The @instance@provider suffix is stripped by _base_model_name- // (knowledgebase_service.py:33), so only the MODEL NAME is compared — the- // instance and provider tables are not consulted at all.- refs := make([]string, 0, len(kbs))- for _, kb := range kbs {- if kb == nil || kb.EmbdID == "" {- continue- }- ref := ""- if kb.TenantEmbdID != nil {- ref = strings.TrimSpace(*kb.TenantEmbdID)- }- if ref == "" && !strings.Contains(kb.EmbdID, "@") {- ref = strings.TrimSpace(kb.EmbdID)- }- if ref != "" {- refs = append(refs, ref)- }- }-- // Python resolves the refs with one batched lookup- // (tenant_model_service.py:562 get_by_ids) and silently skips ids that no- // longer resolve — a dangling id must not fail the request.- resolved := make(map[string]string, len(refs))- // A nil/absent DB or any query error must not fail the request: Python skips- // unresolvable ids (tenant_model_service.py:557) and falls back to the raw- // id, so validation degrades to a stricter but safe comparison.- if len(refs) > 0 && dao.DB != nil {- if models, err := dao.NewTenantModelDAO().GetByIDs(ctx, dao.DB, refs); err == nil {- for _, m := range models {- if m != nil {- resolved[m.ID] = m.ModelName- }- }- }- }-- keys := make(map[string]struct{}, len(kbs))- for _, kb := range kbs {- if kb == nil || kb.EmbdID == "" {- continue- }- embdID := strings.TrimSpace(kb.EmbdID)- ref := ""- if kb.TenantEmbdID != nil {- ref = strings.TrimSpace(*kb.TenantEmbdID)- }- if ref == "" && !strings.Contains(embdID, "@") {- ref = embdID- }- key := ""- if name, ok := resolved[ref]; ok && name != "" {- key = name- } else if embdID != "" && embdID != ref {- key = baseModelName(embdID)- } else {- key = ref- }- if key != "" {- keys[key] = struct{}{}- }- }- if len(keys) > 1 {- return fmt.Errorf("datasets use different embedding models")- }- return nil-}--// baseModelName mirrors Python _base_model_name-// (knowledgebase_service.py:33): strip the @instance@provider suffix, keeping the-// model name. Python uses rsplit("@", 2)[0], which for the purposes of taking-// the leading segment is the same as splitting once on "@".-func baseModelName(embdID string) string {- name, _, _ := strings.Cut(embdID, "@")- return name-}--// ----------------------------------------------------------------------------// Harness seams (compiled expansion + open-web search)-// -----------------------------------------------------------------------------// engineCompiledStore implements agenticruntime.CompiledStore over the document-// engine, backing search_chunks' compiled-structure expansion (Python-// tools/compiled_expansion.py). The runtime owns the expansion strategy; this-// adapter only maps its (scope, filters, free text) request onto one engine-// search, mirroring Python's settings.docStoreConn.search calls.-type engineCompiledStore struct {- engine engineDocEngine- // embed encodes the seed query for the dense leg. It must be the QUERY- // encoding of the dataset's own embedding model (Python- // _search_compiled_rows → _dense_expr → settings.retriever.get_vector →- // emb_mdl.encode_queries). Nil leaves the keyword leg only.- embed compiledQueryEncoder- // embedTenantID is the tenant owning the embedder's model (Python- // embd_owner_tenant_id = kbs[0].tenant_id). It is distinct from the searched- // scope's tenant, because Python builds ONE embedder from the leading bound- // dataset and reuses it for every scope.- embedTenantID string-}--var _ agenticruntime.CompiledStore = (*engineCompiledStore)(nil)--// compiledQueryEncoder is the query-side embedding seam used for the compiled-// dense seed leg. *service.NavEmbedder implements it via EncodeQueries.-type compiledQueryEncoder interface {- EncodeQueries(ctx context.Context, tenantID string, texts []string) ([][]float32, error)-}--// Mirror Python compiled_expansion._VECTOR_NUM_CANDIDATES / _VECTOR_SIMILARITY.-// The HNSW ef_search floor (num_candidates) must be >= topN or the ANN search is-// bounded by the candidate list instead of by relevance.-const (- compiledVectorNumCandidates = 256- compiledVectorSimilarity = 0.1-)--// newEngineCompiledStore returns nil when no engine is available so-// agenticruntime.NewCompiledExpander disables expansion instead of searching nowhere.-// A nil embed (or empty embedTenantID) keeps the keyword-only leg.-func newEngineCompiledStore(e engineDocEngine, embed compiledQueryEncoder, embedTenantID string) agenticruntime.CompiledStore {- if e == nil {- return nil- }- return &engineCompiledStore{engine: e, embed: embed, embedTenantID: embedTenantID}-}--// newDatasetCompiledStore builds the per-request compiled-row store, wiring the-// dense seed leg to the FIRST bound dataset's own embedding model. Python builds-// embd_mdl from kbs[0] (dialog_service.py:362-365) and reuses it for the whole-// expansion; using the tenant-default model instead would compare the query-// against rows embedded by a different model — different vector spaces, so the-// dense hits would be noise.-func newDatasetCompiledStore(e engineDocEngine, modelSvc *service.ModelProviderService, kbs []*entity.Knowledgebase) agenticruntime.CompiledStore {- if e == nil {- return nil- }- if hasEmbedderFor(kbs) {- return newEngineCompiledStore(e, service.NewNavEmbedder(modelSvc, kbs[0].EmbdID), kbs[0].TenantID)- }- return newEngineCompiledStore(e, nil, "")-}--const tenantChunkIndexPrefix = "ragflow_"--func tenantChunkIndexName(tenantID string) string {- return tenantChunkIndexPrefix + tenantID-}--// compiledSynthesisKinds are the standalone synthesis-page compile_kwd values.-// Python scopes those rows by source_doc_ids, while entity/relation rows are-// scoped by doc_id (compiled_expansion.py _search_compiled_rows /-// _search_synthesis_pages).-var compiledSynthesisKinds = map[string]bool{- "wiki_page": true,- "artifact_page": true,- "essence": true,-}--// compiledRowSelectFields is the projection the expander reads off a compiled-// row: content_with_weight (seed name parsing), the entity/relation endpoints,-// exact name lookups, and provenance back to source chunks.-var compiledRowSelectFields = []string{- "id", "kb_id", "doc_id", "docnm_kwd",- "content_with_weight", "summary_with_weight", "source_chunk_ids",- "name_kwd", "from_entity_kwd", "to_entity_kwd", "available_int",-}--// compiledChunkSelectFields is the projection for source chunks promoted into-// evidence by compiled expansion.-var compiledChunkSelectFields = []string{- "id", "kb_id", "doc_id", "docnm_kwd",- "content_with_weight", "source_chunk_ids", "similarity",-}--// SearchCompiled implements agenticruntime.CompiledStore.-func (s *engineCompiledStore) SearchCompiled(ctx context.Context, kbID, tenantID string, docIDs []string, filters map[string][]string, matchText string, topN int) ([]map[string]any, error) {- if s == nil || s.engine == nil || kbID == "" || strings.TrimSpace(tenantID) == "" {- return nil, nil- }- if topN <= 0 {- topN = 1- }- filter := compiledEngineFilter(filters)- if len(docIDs) > 0 {- filter[compiledDocScopeKey(filters)] = docIDs- }- req := &et.SearchRequest{- IndexNames: []string{tenantChunkIndexName(tenantID)},- KbIDs: []string{kbID},- Limit: topN,- SelectFields: compiledRowSelectFields,- Filter: filter,- }- // Python prefers a dense seed leg and only falls back to a keyword match- // when no embedder is available or the vector build fails- // (compiled_expansion.py _search_compiled_rows:180-194 /- // _search_synthesis_pages:423-438). Exactly ONE expr is appended.- if text := strings.TrimSpace(matchText); text != "" {- if dense := s.denseExpr(ctx, text, topN); dense != nil {- req.MatchExprs = []interface{}{dense}- } else {- req.MatchExprs = []interface{}{&et.MatchTextExpr{- Fields: []string{"content_ltks", "content_sm_ltks"},- MatchingText: text,- TopN: topN,- }}- }- }- res, err := s.engine.Search(ctx, req)- if err != nil || res == nil {- return nil, err- }- return res.Chunks, nil-}--// denseExpr builds the compiled-row dense match for the seed text, mirroring-// Python _dense_expr (compiled_expansion.py:31-47): the query is embedded with-// the dataset model's QUERY encoding, and the match carries the HNSW-// num_candidates floor and similarity threshold. Returns nil when no embedder is-// wired, the encode fails, or it panics — every caller then uses the keyword leg,-// exactly like Python's `except Exception` around get_vector.-func (s *engineCompiledStore) denseExpr(ctx context.Context, text string, topN int) *et.MatchDenseExpr {- if s == nil || s.embed == nil || strings.TrimSpace(s.embedTenantID) == "" {- return nil- }- var (- vecs [][]float32- err error- )- func() {- defer func() {- if r := recover(); r != nil {- common.Warn("compiled expansion: seed encode panicked; falling back to keyword match")- }- }()- vecs, err = s.embed.EncodeQueries(ctx, s.embedTenantID, []string{text})- }()- if err != nil || len(vecs) == 0 || len(vecs[0]) == 0 {- return nil- }- vec := make([]float64, len(vecs[0]))- for i, v := range vecs[0] {- vec[i] = float64(v)- }- return &et.MatchDenseExpr{- VectorColumnName: fmt.Sprintf("q_%d_vec", len(vec)),- EmbeddingData: vec,- EmbeddingDataType: "float",- DistanceType: "cosine",- TopN: topN,- ExtraOptions: map[string]interface{}{- "similarity": compiledVectorSimilarity,- "num_candidates": max(topN, compiledVectorNumCandidates),- },- }-}--// LoadChunks implements agenticruntime.CompiledStore: fetch the referenced source-// chunks by id (Python compiled_expansion.py:_load_chunks_for_doc searches the-// tenant index with an {"id": chunk_ids} condition).-func (s *engineCompiledStore) LoadChunks(ctx context.Context, kbID, tenantID string, chunkIDs []string) ([]map[string]any, error) {- if s == nil || s.engine == nil || kbID == "" || strings.TrimSpace(tenantID) == "" || len(chunkIDs) == 0 {- return nil, nil- }- res, err := s.engine.Search(ctx, &et.SearchRequest{- IndexNames: []string{tenantChunkIndexName(tenantID)},- KbIDs: []string{kbID},- Limit: len(chunkIDs),- SelectFields: compiledChunkSelectFields,- Filter: map[string]interface{}{"id": chunkIDs},- })- if err != nil || res == nil {- return nil, err- }- return res.Chunks, nil-}--// Vectorize implements agenticruntime.CompiledStore. The ported expander assigns-// compiled chunks the similarity stored on the row and blends by that field, so-// expansion itself never needs a query embedding.-func (s *engineCompiledStore) Vectorize(context.Context, string) ([]float64, error) {- return nil, fmt.Errorf("compiled store: vectorize is not wired")-}--// compiledDocScopeKey mirrors Python's doc-scope column choice: synthesis pages-// are scoped by source_doc_ids, entity/relation rows by doc_id.-func compiledDocScopeKey(filters map[string][]string) string {- for _, ck := range filters["compile_kwd"] {- if compiledSynthesisKinds[strings.ToLower(strings.TrimSpace(ck))] {- return "source_doc_ids"- }- }- return "doc_id"-}--// compiledEngineFilter maps the runtime' OR-list filters onto the engine filter-// shape. available_int is a numeric column, so its string values are coerced to-// ints (Python passes the literal 1).-func compiledEngineFilter(filters map[string][]string) map[string]interface{} {- out := make(map[string]interface{}, len(filters))- for k, vals := range filters {- if k == "available_int" {- ints := make([]int, 0, len(vals))- for _, v := range vals {- if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil {- ints = append(ints, n)- }- }- if len(ints) > 0 {- out[k] = ints- }- continue- }- out[k] = vals- }- return out-}--// harnessWebSearcherFunc adapts the chat pipeline's web-search callback to the-// agenticruntime.WebSearcher seam consumed by the web_search tool. The callback is a-// plain func so internal/service does not have to depend on the runtime package.-type harnessWebSearcherFunc func(ctx context.Context, queries []string) ([]string, error)--// Search implements agenticruntime.WebSearcher.-func (f harnessWebSearcherFunc) Search(ctx context.Context, queries []string) ([]string, error) {- return f(ctx, queries)-}--var _ agenticruntime.WebSearcher = harnessWebSearcherFunc(nil)--// harnessWebSearcher wraps a non-nil callback, returning a nil WebSearcher when-// the pipeline did not supply one — which is what hides the web_search tool.-func harnessWebSearcher(fn func(context.Context, []string) ([]string, error)) agenticruntime.WebSearcher {- if fn == nil {- return nil- }- return harnessWebSearcherFunc(fn)-}diff --git a/cmd/ragflow_server_test.go b/cmd/ragflow_server_test.goindex 288831c96a..4413c701ac 100644--- a/cmd/ragflow_server_test.go+++ b/cmd/ragflow_server_test.go@@ -18,23 +18,14 @@ package mainimport ("context"- "errors""os"+ "os/exec"+ "path/filepath"+ "strings""testing"-- "ragflow/internal/engine/types"- "ragflow/internal/entity"- "ragflow/internal/service"+ "time")-func kb(embdID string, tenantEmbdID string) *entity.Knowledgebase {- kb := &entity.Knowledgebase{EmbdID: embdID}- if tenantEmbdID != "" {- kb.TenantEmbdID = &tenantEmbdID- }- return kb-}-func parseArgsForTest(t *testing.T, argv ...string) (*serverArgs, error) {t.Helper()orig := os.Args@@ -43,122 +34,6 @@ func parseArgsForTest(t *testing.T, argv ...string) (*serverArgs, error) {return parseArgs()}-// TestHasEmbedderForMirrorsPython pins dialog_service.py:362 —-// `embd_mdl = LLMBundle(...) if kbs and kbs[0].embd_id else None`. The gate is-// the FIRST dataset's embd_id, so a dialog whose leading dataset has no model-// yields no embedding model even when a later one does.-func TestHasEmbedderForMirrorsPython(t *testing.T) {- cases := []struct {- name string- kbs []*entity.Knowledgebase- want bool- }{- {"no datasets", nil, false},- {"single dataset with model", []*entity.Knowledgebase{kb("bge-m3", "")}, true},- {"single dataset without model", []*entity.Knowledgebase{kb("", "")}, false},- // Python reads kbs[0].embd_id only; it does not scan for any dataset- // carrying a model.- {"first without, later with", []*entity.Knowledgebase{kb("", ""), kb("bge-m3", "")}, false},- {"all with the same model", []*entity.Knowledgebase{kb("bge-m3", ""), kb("bge-m3", "")}, true},- }- for _, c := range cases {- if got := hasEmbedderFor(c.kbs); got != c.want {- t.Errorf("%s: hasEmbedderFor = %v, want %v", c.name, got, c.want)- }- }-}--// TestValidateDatasetEmbeddingModels pins knowledgebase_service.py:62-94.-// Mixing is a hard error upstream (get_models raises at dialog_service.py:358) —-// NOT a silent fallback to keyword-only retrieval.-func TestValidateDatasetEmbeddingModels(t *testing.T) {- t.Run("all without a model is allowed", func(t *testing.T) {- if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{- kb("", ""), kb("", ""),- }); err != nil {- t.Errorf("all-without must be allowed, got %v", err)- }- })-- t.Run("all same model is allowed", func(t *testing.T) {- if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{- kb("bge-m3", ""), kb("bge-m3", ""),- }); err != nil {- t.Errorf("same model must be allowed, got %v", err)- }- })-- t.Run("some with some without is an error", func(t *testing.T) {- if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{- kb("bge-m3", ""), kb("", ""),- }); err == nil {- t.Error("mixing present/absent embedding models must error (Python raises)")- }- })-- t.Run("different models is an error", func(t *testing.T) {- if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{- kb("bge-m3", ""), kb("text-embedding-3", ""),- }); err == nil {- t.Error("different embedding models must error (Python raises)")- }- })-- t.Run("dangling tenant_model id falls back to embd_id base name", func(t *testing.T) {- // Python silently skips ids that no longer resolve- // (tenant_model_service.py:557) and falls back to- // _base_model_name(embd_id) — it must not fail the request.- if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{- kb("bge-m3@inst@prov", "gone-1"), kb("bge-m3@inst2@prov2", "gone-2"),- }); err != nil {- t.Errorf("two dangling refs with the same base name must be allowed, got %v", err)- }- })-}--// TestBaseModelNameMirrorsPython pins knowledgebase_service.py:33:-// rsplit("@", 2)[0] — the model name, with @instance@provider stripped.-func TestBaseModelNameMirrorsPython(t *testing.T) {- cases := []struct{ in, want string }{- {"bge-m3@inst@prov", "bge-m3"},- {"bge-m3@inst", "bge-m3"},- {"bge-m3", "bge-m3"},- {"", ""},- }- for _, c := range cases {- if got := baseModelName(c.in); got != c.want {- t.Errorf("baseModelName(%q) = %q, want %q", c.in, got, c.want)- }- }-}--// TestHarnessWebSearcherNilAndWrapped pins the adapter: a nil pipeline callback-// must yield a nil harness.WebSearcher (which hides the web_search tool), and a-// non-nil one must be callable through the seam.-func TestHarnessWebSearcherNilAndWrapped(t *testing.T) {- if got := harnessWebSearcher(nil); got != nil {- t.Error("nil callback must yield a nil WebSearcher")- }- called := false- searcher := harnessWebSearcher(func(_ context.Context, queries []string) ([]string, error) {- called = true- return queries, nil- })- if searcher == nil {- t.Fatal("non-nil callback must yield a WebSearcher")- }- got, err := searcher.Search(context.Background(), []string{"q"})- if err != nil {- t.Fatalf("Search: %v", err)- }- if !called {- t.Error("callback was not invoked")- }- if len(got) != 1 || got[0] != "q" {- t.Errorf("Search = %v, want [q]", got)- }-}-func TestParseArgsMigrateIsStandalone(t *testing.T) {args, err := parseArgsForTest(t, "--migrate")if err != nil {@@ -254,249 +129,20 @@ func TestParseArgsKeepsMCPOutOfCLI(t *testing.T) {}}-// errStubEncode stands in for a provider-side embedding failure.-var errStubEncode = errors.New("stub encode failure")--// recordingEngine captures the SearchRequest so the tests can pin the adapter's-// mapping without a live document engine.-type recordingEngine struct {- last *types.SearchRequest- res *types.SearchResult-}--func (e *recordingEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {- e.last = req- if e.res != nil {- return e.res, nil- }- return &types.SearchResult{}, nil-}--// TestEngineCompiledStoreSearchCompiledMapsRequest pins the mapping from the-// harness' compiled-row request onto the engine search: tenant index, kb scope,-// numeric available_int, the synthesis-page doc-scope column (source_doc_ids),-// and the keyword match leg over content_ltks/content_sm_ltks.-func TestEngineCompiledStoreSearchCompiledMapsRequest(t *testing.T) {- eng := &recordingEngine{}- // No embedder wired → the keyword leg stands alone.- store := newEngineCompiledStore(eng, nil, "")- _, err := store.SearchCompiled(context.Background(), "kb1", "t1", []string{"d1"}, map[string][]string{- "compile_kwd": {"wiki_page"},- "available_int": {"1"},- }, "culdcept", 7)- if err != nil {- t.Fatalf("SearchCompiled: %v", err)- }- req := eng.last- if req == nil {- t.Fatal("engine was not called")- }- if len(req.IndexNames) != 1 || req.IndexNames[0] != "ragflow_t1" {- t.Errorf("IndexNames = %v, want [ragflow_t1]", req.IndexNames)- }- if len(req.KbIDs) != 1 || req.KbIDs[0] != "kb1" {- t.Errorf("KbIDs = %v, want [kb1]", req.KbIDs)- }- if req.Limit != 7 {- t.Errorf("Limit = %d, want 7", req.Limit)- }- // available_int is a numeric column: "1" must reach the engine as an int.- if got, ok := req.Filter["available_int"].([]int); !ok || len(got) != 1 || got[0] != 1 {- t.Errorf("available_int = %#v, want []int{1}", req.Filter["available_int"])- }- if got, ok := req.Filter["compile_kwd"].([]string); !ok || len(got) != 1 || got[0] != "wiki_page" {- t.Errorf("compile_kwd = %#v, want []string{wiki_page}", req.Filter["compile_kwd"])- }- // Synthesis pages are scoped by source_doc_ids, not doc_id.- if _, ok := req.Filter["source_doc_ids"]; !ok {- t.Errorf("doc scope key = %v, want source_doc_ids present", req.Filter)- }- if _, ok := req.Filter["doc_id"]; ok {- t.Errorf("synthesis rows must not use doc_id scope: %v", req.Filter)- }- if len(req.MatchExprs) != 1 {- t.Fatalf("MatchExprs = %#v, want one keyword expr", req.MatchExprs)+func TestMainMigrateReachesConfiguration(t *testing.T) {+ if os.Getenv("RAGFLOW_TEST_MIGRATE_MAIN") == "1" {+ os.Args = append([]string{os.Args[0]}, os.Args[len(os.Args)-3:]...)+ main()+ return}- mt, ok := req.MatchExprs[0].(*types.MatchTextExpr)- if !ok {- t.Fatalf("MatchExprs[0] = %T, want *types.MatchTextExpr", req.MatchExprs[0])- }- if mt.MatchingText != "culdcept" || mt.TopN != 7 {- t.Errorf("keyword expr = %+v, want text=culdcept topN=7", mt)- }- if len(mt.Fields) != 2 || mt.Fields[0] != "content_ltks" || mt.Fields[1] != "content_sm_ltks" {- t.Errorf("keyword fields = %v, want [content_ltks content_sm_ltks]", mt.Fields)- }-}--// TestCompiledDocScopeKey pins Python's column choice: entity/relation rows use-// doc_id; the three synthesis kinds use source_doc_ids.-func TestCompiledDocScopeKey(t *testing.T) {- cases := []struct {- name string- filters map[string][]string- want string- }{- {"entity rows", map[string][]string{"knowledge_graph_kwd": {"entity"}}, "doc_id"},- {"tree rows", map[string][]string{"knowledge_graph_kwd": {"entity"}, "compile_kwd": {"tree"}}, "doc_id"},- {"wiki pages", map[string][]string{"compile_kwd": {"wiki_page"}}, "source_doc_ids"},- {"artifact pages", map[string][]string{"compile_kwd": {"artifact_page"}}, "source_doc_ids"},- {"essence", map[string][]string{"compile_kwd": {"essence"}}, "source_doc_ids"},- }- for _, c := range cases {- if got := compiledDocScopeKey(c.filters); got != c.want {- t.Errorf("%s: compiledDocScopeKey = %q, want %q", c.name, got, c.want)- }- }-}--// TestEngineCompiledStoreLoadChunks pins the id-scoped fetch used to promote-// source chunks into evidence.-func TestEngineCompiledStoreLoadChunks(t *testing.T) {- eng := &recordingEngine{res: &types.SearchResult{Chunks: []map[string]interface{}{{"id": "c1"}}}}- store := newEngineCompiledStore(eng, nil, "")- rows, err := store.LoadChunks(context.Background(), "kb1", "t1", []string{"c1", "c2"})- if err != nil {- t.Fatalf("LoadChunks: %v", err)- }- if len(rows) != 1 {- t.Fatalf("rows = %d, want 1", len(rows))- }- if eng.last.Limit != 2 {- t.Errorf("Limit = %d, want 2", eng.last.Limit)- }- ids, ok := eng.last.Filter["id"].([]string)- if !ok || len(ids) != 2 || ids[0] != "c1" || ids[1] != "c2" {- t.Errorf("id filter = %#v, want [c1 c2]", eng.last.Filter["id"])- }-}--// TestNewDatasetCompiledStore pins the store wiring: no engine yields no store;-// datasets without an embedding model yield a keyword-only store; a dataset-// embedding model wires the query encoder with THAT dataset's model and owner-// tenant (not a tenant default or the request tenant).-func TestNewDatasetCompiledStore(t *testing.T) {- eng := &recordingEngine{}- if got := newDatasetCompiledStore(nil, nil, nil); got != nil {- t.Error("nil engine must disable the compiled store")- }- if got := newDatasetCompiledStore(eng, nil, []*entity.Knowledgebase{{ID: "kb1", TenantID: "t1"}}); got == nil {- t.Error("engine without an embedder must still build a keyword-only store")- }- store := newDatasetCompiledStore(eng, service.NewModelProviderService(),- []*entity.Knowledgebase{{ID: "kb1", EmbdID: "bge-m3@inst@prov", TenantID: "owner-tenant"}})- es, ok := store.(*engineCompiledStore)- if !ok {- t.Fatalf("store = %T, want *engineCompiledStore", store)- }- if es.embed == nil {- t.Fatal("the dataset's embedding model must wire the query encoder")- }- if es.embedTenantID != "owner-tenant" {- t.Errorf("embedTenantID = %q, want owner-tenant (the dataset owner, not the request tenant)", es.embedTenantID)- }-}--// stubQueryEncoder records the query-encoding calls the dense leg makes.-type stubQueryEncoder struct {- vecs [][]float32- err error- calls int- lastTenant string-}--func (s *stubQueryEncoder) EncodeQueries(_ context.Context, tenantID string, _ []string) ([][]float32, error) {- s.calls++- s.lastTenant = tenantID- return s.vecs, s.err-}--// TestEngineCompiledStoreUsesDenseSeedLeg pins the parity fix: when an embedder-// is wired, the compiled-row search goes out as a DENSE expr on the query-// vector's q_<dim>_vec column with Python's num_candidates floor and similarity-// threshold — and carries NO keyword expr alongside it (Python appends exactly-// one). The encoding runs under the embedder's OWN tenant (Python-// embd_owner_tenant_id = kbs[0].tenant_id), not the searched scope's tenant.-func TestEngineCompiledStoreUsesDenseSeedLeg(t *testing.T) {- eng := &recordingEngine{}- enc := &stubQueryEncoder{vecs: [][]float32{{1, 2, 3}}}- // Scope tenant differs from the embedder's owner tenant on purpose.- store := newEngineCompiledStore(eng, enc, "owner-tenant")-- if _, err := store.SearchCompiled(context.Background(), "kb1", "scope-tenant", nil,- map[string][]string{"knowledge_graph_kwd": {"entity"}}, "culdcept", 5); err != nil {- t.Fatalf("SearchCompiled: %v", err)- }- if enc.calls != 1 {- t.Fatalf("encode calls = %d, want 1", enc.calls)- }- if enc.lastTenant != "owner-tenant" {- t.Errorf("encode tenant = %q, want the embedder's owner tenant owner-tenant", enc.lastTenant)- }- if len(eng.last.MatchExprs) != 1 {- t.Fatalf("MatchExprs = %#v, want exactly one dense expr", eng.last.MatchExprs)- }- dense, ok := eng.last.MatchExprs[0].(*types.MatchDenseExpr)- if !ok {- t.Fatalf("MatchExprs[0] = %T, want *types.MatchDenseExpr", eng.last.MatchExprs[0])- }- if dense.VectorColumnName != "q_3_vec" {- t.Errorf("VectorColumnName = %q, want q_3_vec", dense.VectorColumnName)- }- if dense.EmbeddingDataType != "float" || dense.DistanceType != "cosine" || dense.TopN != 5 {- t.Errorf("dense expr = %+v, want float/cosine/topN=5", dense)- }- if dense.ExtraOptions["similarity"] != 0.1 {- t.Errorf("similarity = %v, want 0.1", dense.ExtraOptions["similarity"])- }- // max(topN, 256) — a small topN is floored to the HNSW candidate count.- if dense.ExtraOptions["num_candidates"] != 256 {- t.Errorf("num_candidates = %v, want 256 (max(5,256))", dense.ExtraOptions["num_candidates"])- }-- // A topN above the floor is passed through unchanged.- if _, err := store.SearchCompiled(context.Background(), "kb1", "scope-tenant", nil,- map[string][]string{"knowledge_graph_kwd": {"entity"}}, "culdcept", 300); err != nil {- t.Fatalf("SearchCompiled(300): %v", err)- }- dense300 := eng.last.MatchExprs[0].(*types.MatchDenseExpr)- if dense300.ExtraOptions["num_candidates"] != 300 {- t.Errorf("num_candidates = %v, want 300", dense300.ExtraOptions["num_candidates"])- }-}--// TestEngineCompiledStoreDenseFallsBackToKeyword pins Python's `except Exception`-// around get_vector: an encode failure (or an empty vector) must degrade to the-// keyword leg rather than dropping the expansion.-func TestEngineCompiledStoreDenseFallsBackToKeyword(t *testing.T) {- for _, c := range []struct {- name string- enc *stubQueryEncoder- }{- {"encode error", &stubQueryEncoder{err: errStubEncode}},- {"empty vector", &stubQueryEncoder{vecs: [][]float32{{}}}},- {"no vectors", &stubQueryEncoder{vecs: nil}},- } {- t.Run(c.name, func(t *testing.T) {- eng := &recordingEngine{}- store := newEngineCompiledStore(eng, c.enc, "t1")- if _, err := store.SearchCompiled(context.Background(), "kb1", "t1", nil,- map[string][]string{"knowledge_graph_kwd": {"entity"}}, "culdcept", 5); err != nil {- t.Fatalf("SearchCompiled: %v", err)- }- if c.enc.calls != 1 {- t.Errorf("encode calls = %d, want 1", c.enc.calls)- }- if len(eng.last.MatchExprs) != 1 {- t.Fatalf("MatchExprs = %#v, want one keyword expr", eng.last.MatchExprs)- }- mt, ok := eng.last.MatchExprs[0].(*types.MatchTextExpr)- if !ok {- t.Fatalf("MatchExprs[0] = %T, want *types.MatchTextExpr keyword fallback", eng.last.MatchExprs[0])- }- if mt.MatchingText != "culdcept" {- t.Errorf("keyword text = %q, want culdcept", mt.MatchingText)- }- })+ ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second)+ defer cancel()+ dir := t.TempDir()+ cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestMainMigrateReachesConfiguration$", "--", "--migrate", "--config", filepath.Join(dir, "missing.yaml"))+ cmd.Dir = dir+ cmd.Env = append(os.Environ(), "RAGFLOW_TEST_MIGRATE_MAIN=1")+ output, err := cmd.CombinedOutput()+ if err == nil || !strings.Contains(string(output), "initialize configuration") || strings.Contains(string(output), "Usage:") {+ t.Fatalf("migration did not reach config: %v\n%s", err, output)}}diff --git a/internal/agent/retrievalbridge/harness.go b/internal/agent/retrievalbridge/harness.gonew file mode 100644index 0000000000..0bf8d6f777--- /dev/null+++ b/internal/agent/retrievalbridge/harness.go@@ -0,0 +1,745 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package retrievalbridge++import (+ "context"+ "errors"+ "fmt"+ "strconv"+ "strings"++ "ragflow/internal/agent/component"+ "ragflow/internal/common"+ "ragflow/internal/dao"+ et "ragflow/internal/engine/types"+ "ragflow/internal/entity"+ modelModule "ragflow/internal/entity/models"+ "ragflow/internal/rag/agentic-rag"+ agenticruntime "ragflow/internal/rag/agentic-rag/runtime"+ "ragflow/internal/service"+ "ragflow/internal/service/nlp"++ "github.com/cloudwego/eino/schema"+ "go.uber.org/zap"+)++// NewHarnessRetriever connects chat requests to the agentic runtime using the+// shared model, metadata and document services initialized by the API server.+func NewHarnessRetriever(modelProviderService *service.ModelProviderService, metadataService *service.MetadataService, docEngine engineDocEngine) func(context.Context, service.HarnessRequest) (service.HarnessResult, error) {+ return func(ctx context.Context, req service.HarnessRequest) (service.HarnessResult, error) {+ // Resolve the tenant's actual chat model, mirroring Python where RAGTools+ // receives a fully-resolved LLMBundle: chat.LLMID may be a UUID/tenant_model+ // id that the default invoker cannot split into a provider, so resolving here+ // avoids falling through to a dummy driver. If resolution fails we leave model+ // nil so the runtime degrades to a direct search (no dummy fallback).+ var model agenticruntime.SessionModel+ // outerModel mirrors Python RAGTools.chat_mdl: the fully-resolved chat+ // model that drives the outer rag_agent react loop (tools=[rag,+ // summarize_document], terminal_tools={"rag"}). Wired into RAGTools.Outer+ // so Rag() can run that loop instead of the direct graph.+ var outerModel *modelModule.ChatModel+ // resolvedModelName is the resolved chat model's own name (e.g.+ // "gpt-4o"), i.e. Python chat_mdl.llm_name. It is the gen_json reply-cache+ // key's model component; empty disables that cache.+ resolvedModelName := ""+ // The resolution goes through the same entry point the pipeline's own+ // capability probe uses, so the model decided there is the model wired+ // here: req.ModelID may be empty (a dialog without an llm_id), and both+ // layers must then fall back to the tenant default rather than resolving+ // nothing here and judging capability there.+ if target, mErr := modelProviderService.ResolveChatModelTarget(ctx, req.TenantID, req.ModelID); mErr == nil {+ if inv := component.NewResolvedInvoker(target.Driver, target.ModelName, target.APIConfig); inv != nil {+ // MaxLength mirrors Python LLMBundle.max_length (the model's+ // context window in tokens); message-fitting nodes (calculate,+ // structure_qa) use it as their chat.FitMessages budget.+ model = &agenticruntime.InvokerSessionModel{Invoker: inv, DB: dao.DB, MaxLength: target.ContextLength}+ resolvedModelName = target.ModelName+ // Reuse the same resolved driver/name/api as the invoker so+ // the outer loop and the inner tool calls share one model.+ outerModel = modelModule.NewChatModel(target.Driver, &target.ModelName, target.APIConfig)+ }+ } else {+ common.Warn("runtime: failed to resolve chat model for reasoning; runtime will degrade to direct search", zap.Error(mErr))+ }++ // Load the KB objects (mirroring Python RAGTools' self.kbs via+ // KnowledgebaseService.get_by_ids(kb_ids)) so the agentic tool can+ // derive rank features. The Go tag extractor (extractor_tag.go) writes+ // both tag_kwd (the list of tag names) and tag_feas (per-tag weights)+ // onto each chunk at parse time; the labeler aggregates tag_kwd to build+ // the tag vocabulary and the retriever ranks with tag_feas.+ var kbs []*entity.Knowledgebase+ if len(req.DatasetIDs) > 0 {+ var err error+ kbs, err = dao.NewKnowledgebaseDAO().GetByIDs(ctx, dao.DB, req.DatasetIDs)+ if err != nil {+ return service.HarnessResult{}, err+ }+ if err := validateLoadedDatasets(req.DatasetIDs, kbs); err != nil {+ return service.HarnessResult{}, err+ }+ }+ // validate_dataset_embedding_models runs FIRST upstream+ // (dialog_service.py:358-360) and mixing is a hard failure there, not a+ // fallback to keyword-only retrieval.+ if err := validateDatasetEmbeddingModels(ctx, kbs); err != nil {+ return service.HarnessResult{}, err+ }+ // HasEmbedder mirrors Python `embd_mdl = ... if kbs and kbs[0].embd_id else+ // None` (dialog_service.py:362): the gate is the FIRST dataset's embd_id. It+ // is what hybrid_search applies to its vector leg (search.py:143), so+ // getting it wrong silently degrades search_chunks to keyword-only.+ hasEmbedder := hasEmbedderFor(kbs)+ // Retrieval tuning mirrors the dialog's own settings, so the agentic+ // searches run with the same budget as the standard path. Without it the+ // harness falls back to its package defaults (top_n=12) and the dialog's+ // configured top_n is silently ignored.+ //+ // The runtime carries the KEYWORD leg's weight and derives the vector leg+ // as its complement (runtime.resolveKeywordsSimilarityWeight, and the+ // RetrieveRequest contract: 0.7 hybrid / 0.0 vector-only / 1.0+ // keyword-only). The dialog configures the VECTOR weight, so it is inverted+ // here; the dialog's default 0.3 therefore yields the runtime's own 0.7.+ keywordsWeight := 1 - req.Retrieval.VectorSimilarityWeight+ deps := agentic_rag.RAGTools{+ Model: model,+ ModelName: resolvedModelName,+ Outer: outerModel,+ TopN: req.Retrieval.TopN,+ SimilarityThreshold: req.Retrieval.SimilarityThreshold,+ KeywordsSimilarityWeight: &keywordsWeight,+ RerankCandidatesCount: req.Retrieval.RerankCandidatesCount,+ // OriginalQuestion mirrors Python RAGTools(original_user_question=...):+ // the user's own, unrewritten question as received from the chat+ // layer. The outer model's `rag(question=...)` argument is+ // model-generated and often compresses a multi-hop question to its+ // first hop; resolveEffectiveQuestion (agentic_rag.py:865) prefers+ // this original over that rewrite when both describe the same turn.+ OriginalQuestion: req.Question,+ // DocScope mirrors Python RAGTools(doc_scope=...): the narrowed doc_ids+ // (chat-level doc_ids + meta_data_filter) restrict every agentic+ // retrieval to the user-selected documents instead of the whole kb.+ DocScope: req.DocIDs,+ DocIDVerifier: agentic_rag.NewDocIDLookup(),+ // MetadataResolver backs the metadata_search tool and the pre-search+ // metadata channel (Python DocMetadataService push-down + meta_filter+ // fallback). metadataService already carries the doc engine and the DAO+ // the metadata index reads need.+ MetadataResolver: metadataService,+ // DeclaredMetadata lets the metadata_search catalog describe each field+ // (meaning + allowed values) from the dataset's own parser_config, not just+ // name it. Same service: it also carries the KB DAO that read needs.+ DeclaredMetadata: metadataService,+ // DocChunks pages one document's chunks in reading order off the+ // chunk index (Python retriever.chunk_list), backing the+ // document-level tools (summarize_document / fetch_full_document).+ DocChunks: &docChunkPager{docEngine: docEngine},+ // Expand runs search_chunks' compiled-structure expansion+ // (Python hybrid_search use_compiled=True → _expand_with_compiled).+ // Backed by the same document engine the chunk reads use, with the+ // dense seed leg wired to the bound dataset's own embedding model.+ // The scope config lets each dataset be scanned under its own tenant+ // and a doc scope be grouped by real owner, like graph_explore.+ Expand: agenticruntime.NewCompiledExpander(+ newDatasetCompiledStore(docEngine, modelProviderService, kbs),+ agenticruntime.CompiledScopeConfig{+ DatasetIDs: req.DatasetIDs,+ TenantID: req.TenantID,+ KBs: kbs,+ DocTenantResolver: agentic_rag.NewDocTenantResolver(),+ },+ ),+ // WebSearch backs the web_search tool (Python RAGTools.web_search).+ // The pipeline supplies a callback only when the chat enables+ // internet web search; a nil one also keeps the tool off the surface.+ WebSearch: harnessWebSearcher(req.WebSearch),+ KBs: kbs,+ HasEmbedder: hasEmbedder,+ // Embedder backs claim recall's KNN leg (Python+ // recall_dataset_claims, navigation.py:1837-1909, which embeds the+ // query with tools.embed_mdl — the FIRST dataset's embedding model,+ // dialog_service.py:362-366) and the structure-drill seed vector.+ // Without it the claim leg silently degrades to BM25-only and+ // paraphrase-phrased claims are never recalled. Bound to+ // kbs[0].EmbdID (validateDatasetEmbeddingModels above guarantees+ // every bound dataset shares it), query-side encoded.+ Embedder: embedderForDatasets(kbs, modelProviderService),+ // Tagger is the Go equivalent of Python's label_question+ // (agentic_rag.py:668): classifies the query into question-type+ // tags the retriever boosts on. metadataService implements it+ // (service.MetadataService.LabelQuestion).+ Tagger: metadataService,+ // SystemPrompt mirrors Python RAGTools(system_prompt=+ // _render_reasoning_system_prompt(dialog, prompt_config, kwargs),+ // dialog_service.py:2084) — the dialog-level UI configuration the+ // final-answer compose appends after the agentic contract+ // (agentic_rag_graph.py:923-933).+ SystemPrompt: req.SystemPrompt,+ // CiteRules and EvidenceMaxTokens deliberately stay unset: Python's+ // dialog path constructs RAGTools WITHOUT user_defined_prompts+ // (dialog_service.py:2072-2090), so citation_prompt defaults apply,+ // and Python has no evidence-token override (the compose always+ // uses min(chat_mdl.max_length, _EVIDENCE_BUDGET_TOKENS=8000),+ // which EvidenceMaxTokens<=0 reproduces).+ }+ for _, message := range req.Messages {+ content, err := service.NormalizeOpenAIMessageContent(message["content"])+ if err != nil {+ return service.HarnessResult{}, err+ }+ role, _ := message["role"].(string)+ deps.Messages = append(deps.Messages, schema.Message{Role: schema.RoleType(role), Content: content})+ }+ // Diagnose WHY compiled expansion is disabled: NewCompiledExpander+ // returns nil for three reasons (store==nil / no datasets / no tenant)+ // and RAGTools.Expand==nil silences the whole channel with no other+ // trace — the observed "Compiled expansion enabled = 0" was invisible.+ if deps.Expand == nil {+ switch {+ case docEngine == nil:+ common.Warn("compiled expansion disabled: document engine unavailable (store==nil)")+ case len(req.DatasetIDs) == 0:+ common.Warn("compiled expansion disabled: no bound dataset (DatasetIDs empty)")+ case strings.TrimSpace(req.TenantID) == "":+ common.Warn("compiled expansion disabled: no tenant (TenantID empty)")+ default:+ common.Warn("compiled expansion disabled: unknown reason")+ }+ }+ // The two projections of one reasoning step: the sentence the chat UI+ // appends to its think block, and the structured event a step-rendering+ // client consumes. Steps.Stage/Emit feeds both from one call, so the+ // trace cannot drift from its structured twin. Each step is an+ // isThink=true delta, i.e. think-block content rather than answer text.+ if req.AnswerSink != nil {+ answerSink := req.AnswerSink+ deps.AnswerSink = &agentic_rag.AnswerSink{+ OnDelta: req.AnswerSink,+ }+ deps.Steps.Text = func(line string) { answerSink(line, true) }+ }+ if req.ThinkSink != nil {+ // One type on both sides — agenticruntime.ThinkEvent is an alias of+ // service.ThinkEvent — so the sink passes straight through: there is+ // nothing to copy field by field, and no way to forget a field that+ // was added on one side only.+ deps.Steps.Events = req.ThinkSink+ }+ r := agentic_rag.Rag(ctx, deps, agenticruntime.RunRequest{+ Question: req.Question,+ ThinkingMode: req.ThinkingMode,+ DatasetIDs: req.DatasetIDs,+ TenantID: req.TenantID,+ SessionID: req.SessionID,+ Images: req.Images,+ TextAttachments: req.TextAttachments,+ })+ res := service.HarnessResult{Chunks: r.Chunks, DocAggs: r.DocAggs, Answer: r.Answer, SlotCitations: r.SlotCitations, CiteChunkIDs: r.CiteChunkIDs}+ if r.Kbinfos != nil {+ res.Memory = r.Kbinfos.Memory+ res.PreSummary = r.Kbinfos.PreSummary+ }+ return res, nil+ }+}++// engineDocEngine is the small slice of the engine surface the doc-chunk pager+// needs, kept as a local interface so it is trivially unit-testable without a+// live engine (the production value is engine.Get()).+type engineDocEngine interface {+ Search(ctx context.Context, req *et.SearchRequest) (*et.SearchResult, error)+}++// docChunkPager is the production agenticruntime.DocChunkLister. It pages one+// document's chunks in reading order straight off the chunk index, mirroring+// Python settings.retriever.chunk_list with sort_by_position=True+// (rag/nlp/search.py:824). This is what the document-level tools+// (summarize_document / fetch_full_document) consume.+//+// The index name is derived from the tenant id (ragflow_<tenantID>), exactly as+// retrieval does, so it needs no user-scope resolution: DocChunksRequest+// already carries the tenant and the bound datasets.+type docChunkPager struct {+ docEngine engineDocEngine+}++// DocChunks implements agenticruntime.DocChunkLister.+func (p *docChunkPager) DocChunks(ctx context.Context, req agenticruntime.DocChunksRequest) ([]map[string]any, error) {+ if p == nil || p.docEngine == nil {+ return nil, nil+ }+ if req.DocID == "" {+ return nil, nil+ }+ orderBy := (&et.OrderByExpr{}).+ Asc("chunk_order_int").+ Asc("page_num_int").+ Asc("top_int").+ Desc("create_timestamp_flt")++ resp, err := p.docEngine.Search(ctx, &et.SearchRequest{+ IndexNames: []string{tenantChunkIndexName(req.TenantID)},+ KbIDs: req.DatasetIDs,+ Offset: req.Offset,+ Limit: req.Limit,+ OrderBy: orderBy,+ SelectFields: []string{+ "id", "doc_id", "docnm", "content_with_weight", "content", "available_int",+ },+ Filter: map[string]interface{}{+ "doc_id": req.DocID,+ "available_int": 1,+ "must_not": map[string]interface{}{"exists": "compile_kwd"},+ },+ })+ if err != nil {+ return nil, err+ }+ if resp == nil {+ return nil, nil+ }+ rows := make([]map[string]any, 0, len(resp.Chunks))+ for _, ck := range resp.Chunks {+ row := make(map[string]any, 8)+ // The runtime chunk rows need the canonical keys its citation pipeline+ // reads: chunk_id (dedup), docnm_kwd (doc title), doc_id and the text+ // under content/content_with_weight (chunkText prefers content first).+ if id, ok := ck["id"].(string); ok && id != "" {+ row["chunk_id"] = id+ }+ if d, ok := ck["doc_id"]; ok {+ row["doc_id"] = d+ }+ if d, ok := ck["docnm"]; ok {+ row["docnm_kwd"] = d+ }+ if c, ok := ck["content"]; ok {+ row["content"] = c+ }+ if c, ok := ck["content_with_weight"]; ok {+ row["content_with_weight"] = c+ }+ // Keep whatever else the engine returned (e.g. position_int, img) so the+ // merged Kbinfos rows stay rich, as retrieval does.+ for k, v := range ck {+ if _, seen := row[k]; !seen {+ row[k] = v+ }+ }+ rows = append(rows, row)+ }+ return rows, nil+}++// hasEmbedderFor mirrors Python `embd_mdl = LLMBundle(...) if kbs and+// kbs[0].embd_id else None` (dialog_service.py:362). The gate is the FIRST bound+// dataset's embd_id, not "any dataset has one": a dialog whose first dataset+// carries no embedding model yields embd_mdl=None upstream even if later ones do.+//+// Callers must run validateDatasetEmbeddingModels first — with validation+// passing, all datasets share one model and this equals "all have one".+func hasEmbedderFor(kbs []*entity.Knowledgebase) bool {+ return len(kbs) > 0 && kbs[0] != nil && kbs[0].EmbdID != ""+}++// embedderForDatasets builds the embedding handle the agentic runtime uses for+// query-side encoding outside the main retrieval leg: claim recall's KNN leg+// (Python recall_dataset_claims, navigation.py:1837-1909, embedding with+// tools.embed_mdl) and the structure-drill seed vector. The model is the FIRST+// dataset's embedding (Python dialog_service.py:362-366 resolves+// kbs[0].embd_id under kbs[0].tenant_id; validateDatasetEmbeddingModels+// guarantees the rest share it). Nil when no dataset carries an embedding+// model — the claim leg then degrades to BM25-only, matching a Python run+// without embed_mdl.+func embedderForDatasets(kbs []*entity.Knowledgebase, modelSvc *service.ModelProviderService) nlp.NavEmbedder {+ if len(kbs) == 0 || kbs[0] == nil || kbs[0].EmbdID == "" {+ return nil+ }+ return service.NewNavEmbedder(modelSvc, kbs[0].EmbdID)+}++func validateLoadedDatasets(datasetIDs []string, kbs []*entity.Knowledgebase) error {+ loaded := make(map[string]struct{}, len(kbs))+ for _, kb := range kbs {+ if kb != nil {+ loaded[kb.ID] = struct{}{}+ }+ }+ for _, id := range datasetIDs {+ if _, ok := loaded[id]; !ok {+ return fmt.Errorf("dataset %q was not found", id)+ }+ }+ return nil+}++// validateDatasetEmbeddingModels mirrors Python validate_dataset_embedding_models+// (knowledgebase_service.py:62-94): every bound dataset must use the same embedding+// model, or none at all. Upstream this runs before embd_mdl is resolved+// (dialog_service.py:358-360) and a violation RAISES — it is not a "no embedding+// model" fallback: treating mixing as HasEmbedder=false would silently serve+// keyword-only results where Python errors.+func validateDatasetEmbeddingModels(ctx context.Context, kbs []*entity.Knowledgebase) error {+ var withEmbd int+ for _, kb := range kbs {+ if kb != nil && kb.EmbdID != "" {+ withEmbd+++ }+ }+ hasEmbd := withEmbd > 0+ if hasEmbd && withEmbd != len(kbs) {+ return errors.New("cannot search across datasets where some have embedding models and others do not")+ }+ if !hasEmbd {+ return nil+ }+ // Mirror Python's grouping key exactly:+ //+ // ref = tenant_embd_id or (embd_id when it is a raw tenant_model id)+ // composite = resolved_names.get(ref) # "model@instance@provider"+ // key = _base_model_name(composite) # == tenant_model.model_name+ //+ // The @instance@provider suffix is stripped by _base_model_name+ // (knowledgebase_service.py:33), so only the MODEL NAME is compared — the+ // instance and provider tables are not consulted at all.+ refs := make([]string, 0, len(kbs))+ for _, kb := range kbs {+ if kb == nil || kb.EmbdID == "" {+ continue+ }+ ref := ""+ if kb.TenantEmbdID != nil {+ ref = strings.TrimSpace(*kb.TenantEmbdID)+ }+ if ref == "" && !strings.Contains(kb.EmbdID, "@") {+ ref = strings.TrimSpace(kb.EmbdID)+ }+ if ref != "" {+ refs = append(refs, ref)+ }+ }++ // Python resolves the refs with one batched lookup+ // (tenant_model_service.py:562 get_by_ids) and silently skips ids that no+ // longer resolve — a dangling id must not fail the request.+ resolved := make(map[string]string, len(refs))+ // A nil/absent DB or any query error must not fail the request: Python skips+ // unresolvable ids (tenant_model_service.py:557) and falls back to the raw+ // id, so validation degrades to a stricter but safe comparison.+ if len(refs) > 0 && dao.DB != nil {+ if models, err := dao.NewTenantModelDAO().GetByIDs(ctx, dao.DB, refs); err == nil {+ for _, m := range models {+ if m != nil {+ resolved[m.ID] = m.ModelName+ }+ }+ }+ }++ keys := make(map[string]struct{}, len(kbs))+ for _, kb := range kbs {+ if kb == nil || kb.EmbdID == "" {+ continue+ }+ embdID := strings.TrimSpace(kb.EmbdID)+ ref := ""+ if kb.TenantEmbdID != nil {+ ref = strings.TrimSpace(*kb.TenantEmbdID)+ }+ if ref == "" && !strings.Contains(embdID, "@") {+ ref = embdID+ }+ key := ""+ if name, ok := resolved[ref]; ok && name != "" {+ key = name+ } else if embdID != "" && embdID != ref {+ key = common.BaseModelName(embdID)+ } else {+ key = ref+ }+ if key != "" {+ keys[key] = struct{}{}+ }+ }+ if len(keys) > 1 {+ return fmt.Errorf("datasets use different embedding models")+ }+ return nil+}++// ---------------------------------------------------------------------------+// Harness seams (compiled expansion + open-web search)+// ---------------------------------------------------------------------------++// engineCompiledStore implements agenticruntime.CompiledStore over the document+// engine, backing search_chunks' compiled-structure expansion (Python+// tools/compiled_expansion.py). The runtime owns the expansion strategy; this+// adapter only maps its (scope, filters, free text) request onto one engine+// search, mirroring Python's settings.docStoreConn.search calls.+type engineCompiledStore struct {+ engine engineDocEngine+ // embed encodes the seed query for the dense leg. It must be the QUERY+ // encoding of the dataset's own embedding model (Python+ // _search_compiled_rows → _dense_expr → settings.retriever.get_vector →+ // emb_mdl.encode_queries). Nil leaves the keyword leg only.+ embed compiledQueryEncoder+ // embedTenantID is the tenant owning the embedder's model (Python+ // embd_owner_tenant_id = kbs[0].tenant_id). It is distinct from the searched+ // scope's tenant, because Python builds ONE embedder from the leading bound+ // dataset and reuses it for every scope.+ embedTenantID string+}++var _ agenticruntime.CompiledStore = (*engineCompiledStore)(nil)++// compiledQueryEncoder is the query-side embedding seam used for the compiled+// dense seed leg. *service.NavEmbedder implements it via EncodeQueries.+type compiledQueryEncoder interface {+ EncodeQueries(ctx context.Context, tenantID string, texts []string) ([][]float32, error)+}++// Mirror Python compiled_expansion._VECTOR_NUM_CANDIDATES / _VECTOR_SIMILARITY.+// The HNSW ef_search floor (num_candidates) must be >= topN or the ANN search is+// bounded by the candidate list instead of by relevance.+const (+ compiledVectorNumCandidates = 256+ compiledVectorSimilarity = 0.1+)++// newEngineCompiledStore returns nil when no engine is available so+// agenticruntime.NewCompiledExpander disables expansion instead of searching nowhere.+// A nil embed (or empty embedTenantID) keeps the keyword-only leg.+func newEngineCompiledStore(e engineDocEngine, embed compiledQueryEncoder, embedTenantID string) agenticruntime.CompiledStore {+ if e == nil {+ return nil+ }+ return &engineCompiledStore{engine: e, embed: embed, embedTenantID: embedTenantID}+}++// newDatasetCompiledStore builds the per-request compiled-row store, wiring the+// dense seed leg to the FIRST bound dataset's own embedding model. Python builds+// embd_mdl from kbs[0] (dialog_service.py:362-365) and reuses it for the whole+// expansion; using the tenant-default model instead would compare the query+// against rows embedded by a different model — different vector spaces, so the+// dense hits would be noise.+func newDatasetCompiledStore(e engineDocEngine, modelSvc *service.ModelProviderService, kbs []*entity.Knowledgebase) agenticruntime.CompiledStore {+ if e == nil {+ return nil+ }+ if hasEmbedderFor(kbs) {+ return newEngineCompiledStore(e, service.NewNavEmbedder(modelSvc, kbs[0].EmbdID), kbs[0].TenantID)+ }+ return newEngineCompiledStore(e, nil, "")+}++const tenantChunkIndexPrefix = "ragflow_"++func tenantChunkIndexName(tenantID string) string {+ return tenantChunkIndexPrefix + tenantID+}++// compiledSynthesisKinds are the standalone synthesis-page compile_kwd values.+// Python scopes those rows by source_doc_ids, while entity/relation rows are+// scoped by doc_id (compiled_expansion.py _search_compiled_rows /+// _search_synthesis_pages).+var compiledSynthesisKinds = map[string]bool{+ "wiki_page": true,+ "artifact_page": true,+ "essence": true,+}++// compiledRowSelectFields is the projection the expander reads off a compiled+// row: content_with_weight (seed name parsing), the entity/relation endpoints,+// exact name lookups, and provenance back to source chunks.+var compiledRowSelectFields = []string{+ "id", "kb_id", "doc_id", "docnm_kwd",+ "content_with_weight", "summary_with_weight", "source_chunk_ids",+ "name_kwd", "from_entity_kwd", "to_entity_kwd", "available_int",+}++// compiledChunkSelectFields is the projection for source chunks promoted into+// evidence by compiled expansion.+var compiledChunkSelectFields = []string{+ "id", "kb_id", "doc_id", "docnm_kwd",+ "content_with_weight", "source_chunk_ids", "similarity",+}++// SearchCompiled implements agenticruntime.CompiledStore.+func (s *engineCompiledStore) SearchCompiled(ctx context.Context, kbID, tenantID string, docIDs []string, filters map[string][]string, matchText string, topN int) ([]map[string]any, error) {+ if s == nil || s.engine == nil || kbID == "" || strings.TrimSpace(tenantID) == "" {+ return nil, nil+ }+ if topN <= 0 {+ topN = 1+ }+ filter := compiledEngineFilter(filters)+ if len(docIDs) > 0 {+ filter[compiledDocScopeKey(filters)] = docIDs+ }+ req := &et.SearchRequest{+ IndexNames: []string{tenantChunkIndexName(tenantID)},+ KbIDs: []string{kbID},+ Limit: topN,+ SelectFields: compiledRowSelectFields,+ Filter: filter,+ }+ // Python prefers a dense seed leg and only falls back to a keyword match+ // when no embedder is available or the vector build fails+ // (compiled_expansion.py _search_compiled_rows:180-194 /+ // _search_synthesis_pages:423-438). Exactly ONE expr is appended.+ if text := strings.TrimSpace(matchText); text != "" {+ if dense := s.denseExpr(ctx, text, topN); dense != nil {+ req.MatchExprs = []interface{}{dense}+ } else {+ req.MatchExprs = []interface{}{&et.MatchTextExpr{+ Fields: []string{"content_ltks", "content_sm_ltks"},+ MatchingText: text,+ TopN: topN,+ }}+ }+ }+ res, err := s.engine.Search(ctx, req)+ if err != nil || res == nil {+ return nil, err+ }+ return res.Chunks, nil+}++// denseExpr builds the compiled-row dense match for the seed text, mirroring+// Python _dense_expr (compiled_expansion.py:31-47): the query is embedded with+// the dataset model's QUERY encoding, and the match carries the HNSW+// num_candidates floor and similarity threshold. Returns nil when no embedder is+// wired, the encode fails, or it panics — every caller then uses the keyword leg,+// exactly like Python's `except Exception` around get_vector.+func (s *engineCompiledStore) denseExpr(ctx context.Context, text string, topN int) *et.MatchDenseExpr {+ if s == nil || s.embed == nil || strings.TrimSpace(s.embedTenantID) == "" {+ return nil+ }+ var (+ vecs [][]float32+ err error+ )+ func() {+ defer func() {+ if r := recover(); r != nil {+ common.Warn("compiled expansion: seed encode panicked; falling back to keyword match")+ }+ }()+ vecs, err = s.embed.EncodeQueries(ctx, s.embedTenantID, []string{text})+ }()+ if err != nil || len(vecs) == 0 || len(vecs[0]) == 0 {+ return nil+ }+ vec := make([]float64, len(vecs[0]))+ for i, v := range vecs[0] {+ vec[i] = float64(v)+ }+ return &et.MatchDenseExpr{+ VectorColumnName: fmt.Sprintf("q_%d_vec", len(vec)),+ EmbeddingData: vec,+ EmbeddingDataType: "float",+ DistanceType: "cosine",+ TopN: topN,+ ExtraOptions: map[string]interface{}{+ "similarity": compiledVectorSimilarity,+ "num_candidates": max(topN, compiledVectorNumCandidates),+ },+ }+}++// LoadChunks implements agenticruntime.CompiledStore: fetch the referenced source+// chunks by id (Python compiled_expansion.py:_load_chunks_for_doc searches the+// tenant index with an {"id": chunk_ids} condition).+func (s *engineCompiledStore) LoadChunks(ctx context.Context, kbID, tenantID string, chunkIDs []string) ([]map[string]any, error) {+ if s == nil || s.engine == nil || kbID == "" || strings.TrimSpace(tenantID) == "" || len(chunkIDs) == 0 {+ return nil, nil+ }+ res, err := s.engine.Search(ctx, &et.SearchRequest{+ IndexNames: []string{tenantChunkIndexName(tenantID)},+ KbIDs: []string{kbID},+ Limit: len(chunkIDs),+ SelectFields: compiledChunkSelectFields,+ Filter: map[string]interface{}{"id": chunkIDs},+ })+ if err != nil || res == nil {+ return nil, err+ }+ return res.Chunks, nil+}++// Vectorize implements agenticruntime.CompiledStore. The ported expander assigns+// compiled chunks the similarity stored on the row and blends by that field, so+// expansion itself never needs a query embedding.+func (s *engineCompiledStore) Vectorize(context.Context, string) ([]float64, error) {+ return nil, fmt.Errorf("compiled store: vectorize is not wired")+}++// compiledDocScopeKey mirrors Python's doc-scope column choice: synthesis pages+// are scoped by source_doc_ids, entity/relation rows by doc_id.+func compiledDocScopeKey(filters map[string][]string) string {+ for _, ck := range filters["compile_kwd"] {+ if compiledSynthesisKinds[strings.ToLower(strings.TrimSpace(ck))] {+ return "source_doc_ids"+ }+ }+ return "doc_id"+}++// compiledEngineFilter maps the runtime' OR-list filters onto the engine filter+// shape. available_int is a numeric column, so its string values are coerced to+// ints (Python passes the literal 1).+func compiledEngineFilter(filters map[string][]string) map[string]interface{} {+ out := make(map[string]interface{}, len(filters))+ for k, vals := range filters {+ if k == "available_int" {+ ints := make([]int, 0, len(vals))+ for _, v := range vals {+ if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil {+ ints = append(ints, n)+ }+ }+ if len(ints) > 0 {+ out[k] = ints+ }+ continue+ }+ out[k] = vals+ }+ return out+}++// harnessWebSearcherFunc adapts the chat pipeline's web-search callback to the+// agenticruntime.WebSearcher seam consumed by the web_search tool. The callback is a+// plain func so internal/service does not have to depend on the runtime package.+type harnessWebSearcherFunc func(ctx context.Context, queries []string) ([]string, error)++// Search implements agenticruntime.WebSearcher.+func (f harnessWebSearcherFunc) Search(ctx context.Context, queries []string) ([]string, error) {+ return f(ctx, queries)+}++var _ agenticruntime.WebSearcher = harnessWebSearcherFunc(nil)++// harnessWebSearcher wraps a non-nil callback, returning a nil WebSearcher when+// the pipeline did not supply one — which is what hides the web_search tool.+func harnessWebSearcher(fn func(context.Context, []string) ([]string, error)) agenticruntime.WebSearcher {+ if fn == nil {+ return nil+ }+ return harnessWebSearcherFunc(fn)+}diff --git a/internal/agent/retrievalbridge/harness_test.go b/internal/agent/retrievalbridge/harness_test.gonew file mode 100644index 0000000000..ac4f61e7a6--- /dev/null+++ b/internal/agent/retrievalbridge/harness_test.go@@ -0,0 +1,462 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package retrievalbridge++import (+ "context"+ "errors"+ "github.com/cloudwego/eino/schema"+ "log"+ "ragflow/internal/common"+ "ragflow/internal/engine/types"+ "ragflow/internal/entity"+ "ragflow/internal/rag/agentic-rag"+ agenticruntime "ragflow/internal/rag/agentic-rag/runtime"+ "ragflow/internal/service"+ "reflect"+ "strings"+ "testing"+)++func kb(embdID string, tenantEmbdID string) *entity.Knowledgebase {+ kb := &entity.Knowledgebase{EmbdID: embdID}+ if tenantEmbdID != "" {+ kb.TenantEmbdID = &tenantEmbdID+ }+ return kb+}++// TestHasEmbedderForMirrorsPython pins dialog_service.py:362 —+// `embd_mdl = LLMBundle(...) if kbs and kbs[0].embd_id else None`. The gate is+// the FIRST dataset's embd_id, so a dialog whose leading dataset has no model+// yields no embedding model even when a later one does.+func TestHasEmbedderForMirrorsPython(t *testing.T) {+ cases := []struct {+ name string+ kbs []*entity.Knowledgebase+ want bool+ }{+ {"no datasets", nil, false},+ {"single dataset with model", []*entity.Knowledgebase{kb("bge-m3", "")}, true},+ {"single dataset without model", []*entity.Knowledgebase{kb("", "")}, false},+ // Python reads kbs[0].embd_id only; it does not scan for any dataset+ // carrying a model.+ {"first without, later with", []*entity.Knowledgebase{kb("", ""), kb("bge-m3", "")}, false},+ {"all with the same model", []*entity.Knowledgebase{kb("bge-m3", ""), kb("bge-m3", "")}, true},+ }+ for _, c := range cases {+ if got := hasEmbedderFor(c.kbs); got != c.want {+ t.Errorf("%s: hasEmbedderFor = %v, want %v", c.name, got, c.want)+ }+ }+}++// TestValidateDatasetEmbeddingModels pins knowledgebase_service.py:62-94.+// Mixing is a hard error upstream (get_models raises at dialog_service.py:358) —+// NOT a silent fallback to keyword-only retrieval.+func TestValidateDatasetEmbeddingModels(t *testing.T) {+ t.Run("all without a model is allowed", func(t *testing.T) {+ if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{+ kb("", ""), kb("", ""),+ }); err != nil {+ t.Errorf("all-without must be allowed, got %v", err)+ }+ })++ t.Run("all same model is allowed", func(t *testing.T) {+ if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{+ kb("bge-m3", ""), kb("bge-m3", ""),+ }); err != nil {+ t.Errorf("same model must be allowed, got %v", err)+ }+ })++ t.Run("some with some without is an error", func(t *testing.T) {+ if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{+ kb("bge-m3", ""), kb("", ""),+ }); err == nil {+ t.Error("mixing present/absent embedding models must error (Python raises)")+ }+ })++ t.Run("different models is an error", func(t *testing.T) {+ if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{+ kb("bge-m3", ""), kb("text-embedding-3", ""),+ }); err == nil {+ t.Error("different embedding models must error (Python raises)")+ }+ })++ t.Run("dangling tenant_model id falls back to embd_id base name", func(t *testing.T) {+ // Python silently skips ids that no longer resolve+ // (tenant_model_service.py:557) and falls back to+ // _base_model_name(embd_id) — it must not fail the request.+ if err := validateDatasetEmbeddingModels(context.Background(), []*entity.Knowledgebase{+ kb("bge-m3@inst@prov", "gone-1"), kb("bge-m3@inst2@prov2", "gone-2"),+ }); err != nil {+ t.Errorf("two dangling refs with the same base name must be allowed, got %v", err)+ }+ })+}++func TestValidateLoadedDatasetsRejectsMissing(t *testing.T) {+ err := validateLoadedDatasets([]string{"kb-1", "kb-2"}, []*entity.Knowledgebase{{ID: "kb-1"}})+ if err == nil || !strings.Contains(err.Error(), `dataset "kb-2" was not found`) {+ t.Fatalf("validateLoadedDatasets error = %v, want missing kb-2", err)+ }+}++// TestBaseModelNameMirrorsPython pins knowledgebase_service.py:33:+// rsplit("@", 2)[0] — the model name, with @instance@provider stripped.+func TestBaseModelNameMirrorsPython(t *testing.T) {+ cases := []struct{ in, want string }{+ {"bge-m3@inst@prov", "bge-m3"},+ {"bge-m3@inst", "bge-m3"},+ {"bge-m3", "bge-m3"},+ {"", ""},+ }+ for _, c := range cases {+ if got := common.BaseModelName(c.in); got != c.want {+ t.Errorf("baseModelName(%q) = %q, want %q", c.in, got, c.want)+ }+ }+}++// TestHarnessWebSearcherNilAndWrapped pins the adapter: a nil pipeline callback+// must yield a nil harness.WebSearcher (which hides the web_search tool), and a+// non-nil one must be callable through the seam.+func TestHarnessWebSearcherNilAndWrapped(t *testing.T) {+ if got := harnessWebSearcher(nil); got != nil {+ t.Error("nil callback must yield a nil WebSearcher")+ }+ called := false+ searcher := harnessWebSearcher(func(_ context.Context, queries []string) ([]string, error) {+ called = true+ return queries, nil+ })+ if searcher == nil {+ t.Fatal("non-nil callback must yield a WebSearcher")+ }+ got, err := searcher.Search(context.Background(), []string{"q"})+ if err != nil {+ t.Fatalf("Search: %v", err)+ }+ if !called {+ t.Error("callback was not invoked")+ }+ if len(got) != 1 || got[0] != "q" {+ t.Errorf("Search = %v, want [q]", got)+ }+}++// errStubEncode stands in for a provider-side embedding failure.+var errStubEncode = errors.New("stub encode failure")++// recordingEngine captures the SearchRequest so the tests can pin the adapter's+// mapping without a live document engine.+type recordingEngine struct {+ last *types.SearchRequest+ res *types.SearchResult+}++func (e *recordingEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {+ e.last = req+ if e.res != nil {+ return e.res, nil+ }+ return &types.SearchResult{}, nil+}++// TestEngineCompiledStoreSearchCompiledMapsRequest pins the mapping from the+// harness' compiled-row request onto the engine search: tenant index, kb scope,+// numeric available_int, the synthesis-page doc-scope column (source_doc_ids),+// and the keyword match leg over content_ltks/content_sm_ltks.+func TestEngineCompiledStoreSearchCompiledMapsRequest(t *testing.T) {+ eng := &recordingEngine{}+ // No embedder wired → the keyword leg stands alone.+ store := newEngineCompiledStore(eng, nil, "")+ _, err := store.SearchCompiled(context.Background(), "kb1", "t1", []string{"d1"}, map[string][]string{+ "compile_kwd": {"wiki_page"},+ "available_int": {"1"},+ }, "culdcept", 7)+ if err != nil {+ t.Fatalf("SearchCompiled: %v", err)+ }+ req := eng.last+ if req == nil {+ t.Fatal("engine was not called")+ }+ if len(req.IndexNames) != 1 || req.IndexNames[0] != "ragflow_t1" {+ t.Errorf("IndexNames = %v, want [ragflow_t1]", req.IndexNames)+ }+ if len(req.KbIDs) != 1 || req.KbIDs[0] != "kb1" {+ t.Errorf("KbIDs = %v, want [kb1]", req.KbIDs)+ }+ if req.Limit != 7 {+ t.Errorf("Limit = %d, want 7", req.Limit)+ }+ // available_int is a numeric column: "1" must reach the engine as an int.+ if got, ok := req.Filter["available_int"].([]int); !ok || len(got) != 1 || got[0] != 1 {+ t.Errorf("available_int = %#v, want []int{1}", req.Filter["available_int"])+ }+ if got, ok := req.Filter["compile_kwd"].([]string); !ok || len(got) != 1 || got[0] != "wiki_page" {+ t.Errorf("compile_kwd = %#v, want []string{wiki_page}", req.Filter["compile_kwd"])+ }+ // Synthesis pages are scoped by source_doc_ids, not doc_id.+ if _, ok := req.Filter["source_doc_ids"]; !ok {+ t.Errorf("doc scope key = %v, want source_doc_ids present", req.Filter)+ }+ if _, ok := req.Filter["doc_id"]; ok {+ t.Errorf("synthesis rows must not use doc_id scope: %v", req.Filter)+ }+ if len(req.MatchExprs) != 1 {+ t.Fatalf("MatchExprs = %#v, want one keyword expr", req.MatchExprs)+ }+ mt, ok := req.MatchExprs[0].(*types.MatchTextExpr)+ if !ok {+ t.Fatalf("MatchExprs[0] = %T, want *types.MatchTextExpr", req.MatchExprs[0])+ }+ if mt.MatchingText != "culdcept" || mt.TopN != 7 {+ t.Errorf("keyword expr = %+v, want text=culdcept topN=7", mt)+ }+ if len(mt.Fields) != 2 || mt.Fields[0] != "content_ltks" || mt.Fields[1] != "content_sm_ltks" {+ t.Errorf("keyword fields = %v, want [content_ltks content_sm_ltks]", mt.Fields)+ }+}++// TestCompiledDocScopeKey pins Python's column choice: entity/relation rows use+// doc_id; the three synthesis kinds use source_doc_ids.+func TestCompiledDocScopeKey(t *testing.T) {+ cases := []struct {+ name string+ filters map[string][]string+ want string+ }{+ {"entity rows", map[string][]string{"knowledge_graph_kwd": {"entity"}}, "doc_id"},+ {"tree rows", map[string][]string{"knowledge_graph_kwd": {"entity"}, "compile_kwd": {"tree"}}, "doc_id"},+ {"wiki pages", map[string][]string{"compile_kwd": {"wiki_page"}}, "source_doc_ids"},+ {"artifact pages", map[string][]string{"compile_kwd": {"artifact_page"}}, "source_doc_ids"},+ {"essence", map[string][]string{"compile_kwd": {"essence"}}, "source_doc_ids"},+ }+ for _, c := range cases {+ if got := compiledDocScopeKey(c.filters); got != c.want {+ t.Errorf("%s: compiledDocScopeKey = %q, want %q", c.name, got, c.want)+ }+ }+}++// TestEngineCompiledStoreLoadChunks pins the id-scoped fetch used to promote+// source chunks into evidence.+func TestEngineCompiledStoreLoadChunks(t *testing.T) {+ eng := &recordingEngine{res: &types.SearchResult{Chunks: []map[string]interface{}{{"id": "c1"}}}}+ store := newEngineCompiledStore(eng, nil, "")+ rows, err := store.LoadChunks(context.Background(), "kb1", "t1", []string{"c1", "c2"})+ if err != nil {+ t.Fatalf("LoadChunks: %v", err)+ }+ if len(rows) != 1 {+ t.Fatalf("rows = %d, want 1", len(rows))+ }+ if eng.last.Limit != 2 {+ t.Errorf("Limit = %d, want 2", eng.last.Limit)+ }+ ids, ok := eng.last.Filter["id"].([]string)+ if !ok || len(ids) != 2 || ids[0] != "c1" || ids[1] != "c2" {+ t.Errorf("id filter = %#v, want [c1 c2]", eng.last.Filter["id"])+ }+}++// TestNewDatasetCompiledStore pins the store wiring: no engine yields no store;+// datasets without an embedding model yield a keyword-only store; a dataset+// embedding model wires the query encoder with THAT dataset's model and owner+// tenant (not a tenant default or the request tenant).+func TestNewDatasetCompiledStore(t *testing.T) {+ eng := &recordingEngine{}+ if got := newDatasetCompiledStore(nil, nil, nil); got != nil {+ t.Error("nil engine must disable the compiled store")+ }+ if got := newDatasetCompiledStore(eng, nil, []*entity.Knowledgebase{{ID: "kb1", TenantID: "t1"}}); got == nil {+ t.Error("engine without an embedder must still build a keyword-only store")+ }+ store := newDatasetCompiledStore(eng, service.NewModelProviderService(),+ []*entity.Knowledgebase{{ID: "kb1", EmbdID: "bge-m3@inst@prov", TenantID: "owner-tenant"}})+ es, ok := store.(*engineCompiledStore)+ if !ok {+ t.Fatalf("store = %T, want *engineCompiledStore", store)+ }+ if es.embed == nil {+ t.Fatal("the dataset's embedding model must wire the query encoder")+ }+ if es.embedTenantID != "owner-tenant" {+ t.Errorf("embedTenantID = %q, want owner-tenant (the dataset owner, not the request tenant)", es.embedTenantID)+ }+}++// stubQueryEncoder records the query-encoding calls the dense leg makes.+type stubQueryEncoder struct {+ vecs [][]float32+ err error+ calls int+ lastTenant string+}++func (s *stubQueryEncoder) EncodeQueries(_ context.Context, tenantID string, _ []string) ([][]float32, error) {+ s.calls+++ s.lastTenant = tenantID+ return s.vecs, s.err+}++// TestEngineCompiledStoreUsesDenseSeedLeg pins the parity fix: when an embedder+// is wired, the compiled-row search goes out as a DENSE expr on the query+// vector's q_<dim>_vec column with Python's num_candidates floor and similarity+// threshold — and carries NO keyword expr alongside it (Python appends exactly+// one). The encoding runs under the embedder's OWN tenant (Python+// embd_owner_tenant_id = kbs[0].tenant_id), not the searched scope's tenant.+func TestEngineCompiledStoreUsesDenseSeedLeg(t *testing.T) {+ eng := &recordingEngine{}+ enc := &stubQueryEncoder{vecs: [][]float32{{1, 2, 3}}}+ // Scope tenant differs from the embedder's owner tenant on purpose.+ store := newEngineCompiledStore(eng, enc, "owner-tenant")++ if _, err := store.SearchCompiled(context.Background(), "kb1", "scope-tenant", nil,+ map[string][]string{"knowledge_graph_kwd": {"entity"}}, "culdcept", 5); err != nil {+ t.Fatalf("SearchCompiled: %v", err)+ }+ if enc.calls != 1 {+ t.Fatalf("encode calls = %d, want 1", enc.calls)+ }+ if enc.lastTenant != "owner-tenant" {+ t.Errorf("encode tenant = %q, want the embedder's owner tenant owner-tenant", enc.lastTenant)+ }+ if len(eng.last.MatchExprs) != 1 {+ t.Fatalf("MatchExprs = %#v, want exactly one dense expr", eng.last.MatchExprs)+ }+ dense, ok := eng.last.MatchExprs[0].(*types.MatchDenseExpr)+ if !ok {+ t.Fatalf("MatchExprs[0] = %T, want *types.MatchDenseExpr", eng.last.MatchExprs[0])+ }+ if dense.VectorColumnName != "q_3_vec" {+ t.Errorf("VectorColumnName = %q, want q_3_vec", dense.VectorColumnName)+ }+ if dense.EmbeddingDataType != "float" || dense.DistanceType != "cosine" || dense.TopN != 5 {+ t.Errorf("dense expr = %+v, want float/cosine/topN=5", dense)+ }+ if dense.ExtraOptions["similarity"] != 0.1 {+ t.Errorf("similarity = %v, want 0.1", dense.ExtraOptions["similarity"])+ }+ // max(topN, 256) — a small topN is floored to the HNSW candidate count.+ if dense.ExtraOptions["num_candidates"] != 256 {+ t.Errorf("num_candidates = %v, want 256 (max(5,256))", dense.ExtraOptions["num_candidates"])+ }++ // A topN above the floor is passed through unchanged.+ if _, err := store.SearchCompiled(context.Background(), "kb1", "scope-tenant", nil,+ map[string][]string{"knowledge_graph_kwd": {"entity"}}, "culdcept", 300); err != nil {+ t.Fatalf("SearchCompiled(300): %v", err)+ }+ dense300 := eng.last.MatchExprs[0].(*types.MatchDenseExpr)+ if dense300.ExtraOptions["num_candidates"] != 300 {+ t.Errorf("num_candidates = %v, want 300", dense300.ExtraOptions["num_candidates"])+ }+}++// TestEngineCompiledStoreDenseFallsBackToKeyword pins Python's `except Exception`+// around get_vector: an encode failure (or an empty vector) must degrade to the+// keyword leg rather than dropping the expansion.+func TestEngineCompiledStoreDenseFallsBackToKeyword(t *testing.T) {+ for _, c := range []struct {+ name string+ enc *stubQueryEncoder+ }{+ {"encode error", &stubQueryEncoder{err: errStubEncode}},+ {"empty vector", &stubQueryEncoder{vecs: [][]float32{{}}}},+ {"no vectors", &stubQueryEncoder{vecs: nil}},+ } {+ t.Run(c.name, func(t *testing.T) {+ eng := &recordingEngine{}+ store := newEngineCompiledStore(eng, c.enc, "t1")+ if _, err := store.SearchCompiled(context.Background(), "kb1", "t1", nil,+ map[string][]string{"knowledge_graph_kwd": {"entity"}}, "culdcept", 5); err != nil {+ t.Fatalf("SearchCompiled: %v", err)+ }+ if c.enc.calls != 1 {+ t.Errorf("encode calls = %d, want 1", c.enc.calls)+ }+ if len(eng.last.MatchExprs) != 1 {+ t.Fatalf("MatchExprs = %#v, want one keyword expr", eng.last.MatchExprs)+ }+ mt, ok := eng.last.MatchExprs[0].(*types.MatchTextExpr)+ if !ok {+ t.Fatalf("MatchExprs[0] = %T, want *types.MatchTextExpr keyword fallback", eng.last.MatchExprs[0])+ }+ if mt.MatchingText != "culdcept" {+ t.Errorf("keyword text = %q, want culdcept", mt.MatchingText)+ }+ })+ }+}++// Exercise the production bridge: history must reach the graph and its+// formalizer rather than being replaced by only the latest question.+func TestHarnessRetrieverForwardsConversation(t *testing.T) {+ model := &historyModel{}+ called := false+ agentic_rag.SetAgenticLoop(func(ctx context.Context, deps agentic_rag.RAGTools, req agenticruntime.RunRequest, _ *agenticruntime.Kbinfos, resp *agentic_rag.RunResponse, _ *log.Logger) {+ called = true+ want := []schema.Message{{Role: schema.User, Content: "Tell me about Culdcept."}, {Role: schema.Assistant, Content: "Culdcept was made by OmiyaSoft."}, {Role: schema.User, Content: "When was it released?"}}+ if !reflect.DeepEqual(deps.Messages, want) {+ t.Fatalf("history lost: %#v", deps.Messages)+ }+ resp.Answer, _ = agentic_rag.Formalize(ctx, agenticruntime.SessionDeps{Model: model}, deps.Messages, 8192)+ })+ t.Cleanup(func() { agentic_rag.SetAgenticLoop(nil) })+ req := service.HarnessRequest{Question: "When was it released?", ThinkingMode: "high", Messages: []map[string]interface{}{+ {"role": "user", "content": "Tell me about Culdcept."},+ {"role": "assistant", "content": "Culdcept was made by OmiyaSoft."},+ {"role": "user", "content": []interface{}{map[string]interface{}{"type": "text", "text": "When was it released?"}}},+ }}+ got, err := NewHarnessRetriever(nil, nil, nil)(t.Context(), req)+ if err != nil {+ t.Fatal(err)+ }+ if !called || model.calls != 1 || got.Answer != "When was Culdcept released?" || !strings.Contains(model.transcript, "OmiyaSoft") {+ t.Fatalf("formalization lost conversation: result=%+v model=%+v", got, model)+ }+ req.Messages[0]["content"] = 17+ if _, err := NewHarnessRetriever(nil, nil, nil)(t.Context(), req); err == nil {+ t.Fatal("invalid history content accepted")+ }+}++type historyModel struct {+ transcript string+ calls int+}++func (m *historyModel) Complete(_ context.Context, messages []schema.Message, _ []agenticruntime.ToolSpec) (*agenticruntime.ModelReply, error) {+ m.calls+++ for _, msg := range messages {+ m.transcript += msg.Content+ }+ return &agenticruntime.ModelReply{Content: `{"question":"When was Culdcept released?","keywords":"Culdcept released"}`}, nil+}++func TestEmbeddingValidationPreservesAtInModelNames(t *testing.T) {+ if err := validateDatasetEmbeddingModels(t.Context(), []*entity.Knowledgebase{kb("model@q4@instance@provider", ""), kb("model@q8@instance@provider", "")}); err == nil {+ t.Fatal("different quantized models were conflated")+ }+}diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.goindex 02e14fbd7c..3968d4b66f 100644--- a/internal/service/chat_pipeline.go+++ b/internal/service/chat_pipeline.go@@ -5155,12 +5155,11 @@ type HarnessResult struct {CiteChunkIDs []string}-// harnessRetriever is wired at server bootstrap (cmd/ragflow_server.go:889) to-// the agentic-RAG entry point advanced_rag.Rag (:1022). The advanced_rag package-// imports internal/service (e.g. harness/tool_exploration.go), so the service-// layer cannot import it back without an import cycle; the function is injected-// instead. When nil, retrieveViaHarness reports an error and the pipeline-// continues with empty kbinfos.+// harnessRetriever is installed at server bootstrap using+// retrievalbridge.NewHarnessRetriever. The bridge imports internal/service,+// so this package receives the callback instead of importing the bridge.+// When nil, retrieveViaHarness reports an error and the pipeline continues+// with empty kbinfos.var harnessRetriever func(ctx context.Context, req HarnessRequest) (HarnessResult, error)// SetHarnessRetriever injects the agentic-RAG harness driver. Call once at