RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(integration): register in-process DeepDoc backend + correct stale chunk/knn assertions (#19107)
xugangqiangb2670a7264ca
diff --git a/internal/ingestion/pipeline/real_storage_integration_test.go b/internal/ingestion/pipeline/real_storage_integration_test.goindex d226326acd..4ec571ad01 100644--- a/internal/ingestion/pipeline/real_storage_integration_test.go+++ b/internal/ingestion/pipeline/real_storage_integration_test.go@@ -18,6 +18,8 @@ import ("time""ragflow/internal/dao"+ "ragflow/internal/deepdoc/parser/pdf"+ doctype "ragflow/internal/deepdoc/parser/type""ragflow/internal/entity"componentpkg "ragflow/internal/ingestion/component"_ "ragflow/internal/ingestion/component/chunker"@@ -34,6 +36,16 @@ func TestPipelineRun_TemplateGeneral_RealMySQLMinIO_OutputShape(t *testing.T) {prepareTokenizerResourceForIntegration(t)RequireTokenizerPool(t)+ // The production parse path must never degrade to a mock; install a+ // test-only MockDocAnalyzer as the in-process DeepDoc backend via the+ // public factory seam so the pipeline runs without a real DeepDoc+ // service or ONNX Runtime models. Reset to nil on cleanup (this test+ // binary registers no real backend).+ t.Cleanup(func() { doctype.SetNativeDocAnalyzerFactory(nil) })+ doctype.SetNativeDocAnalyzerFactory(func() (doctype.DocAnalyzer, bool) {+ return &pdf.MockDocAnalyzer{Healthy: true}, true+ })+cfg := mustLoadRealIntegrationConfig(t)realDB := mustOpenRealMySQL(t, cfg)if err := realDB.AutoMigrate(@@ -109,7 +121,13 @@ func TestPipelineRun_TemplateGeneral_RealMySQLMinIO_OutputShape(t *testing.T) {if !ok {t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])}- wantChunkTexts := []string{"Alpha paragraph.", "Beta paragraph."}+ // The TokenChunker merges the two short paragraphs into one chunk under+ // the global token-size budget (chunk_token_size=512, no backtick+ // delimiter): this matches Python's _merge_text_chunks_by_token_size and+ // is the intended behaviour. The parser still emits two json items+ // (see the Parser state assertion below); only the downstream chunker+ // collapses them.+ wantChunkTexts := []string{"Alpha paragraph.\nBeta paragraph."}if len(chunks) != len(wantChunkTexts) {t.Fatalf("len(chunks) = %d, want %d", len(chunks), len(wantChunkTexts))}@@ -154,7 +172,10 @@ func TestPipelineRun_TemplateGeneral_RealMySQLMinIO_OutputShape(t *testing.T) {if !ok || len(jsonItems) != 2 {t.Fatalf("parser json = %T/%v, want 2 items", parserState["json"], parserState["json"])}- for i, wantText := range wantChunkTexts {+ // The parser emits one json item per paragraph (two items), which the+ // downstream TokenChunker later merges into a single chunk.+ wantParserTexts := []string{"Alpha paragraph.", "Beta paragraph."}+ for i, wantText := range wantParserTexts {if got := jsonItems[i]["text"]; got != wantText {t.Fatalf("parser json[%d].text = %v, want %q", i, got, wantText)}diff --git a/internal/ingestion/pipeline/template_integration_test.go b/internal/ingestion/pipeline/template_integration_test.goindex 7e87db04f8..d05aad94e3 100644--- a/internal/ingestion/pipeline/template_integration_test.go+++ b/internal/ingestion/pipeline/template_integration_test.go@@ -36,6 +36,8 @@ import ("testing""ragflow/internal/agent/runtime"+ "ragflow/internal/deepdoc/parser/pdf"+ doctype "ragflow/internal/deepdoc/parser/type"componentpkg "ragflow/internal/ingestion/component"_ "ragflow/internal/ingestion/component/chunker""ragflow/internal/ingestion/testutil"@@ -104,7 +106,7 @@ func TestPipelineRun_TemplateGeneral_RealComponents(t *testing.T) {if !ok {t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])}- wantChunkTexts := []string{"Alpha paragraph.", "Beta paragraph."}+ wantChunkTexts := []string{"Alpha paragraph.\nBeta paragraph."}if len(chunks) != len(wantChunkTexts) {t.Fatalf("len(chunks) = %d, want %d", len(chunks), len(wantChunkTexts))}@@ -769,6 +771,18 @@ func attachFixedEmbedderFactory(t *testing.T, pipe *Pipeline) {func withRealTemplateDeps(t *testing.T) storage.Storage {t.Helper()+ // The production parse path must never degrade to a mock; install a+ // test-only MockDocAnalyzer as the in-process DeepDoc backend via the+ // public factory seam so the pipeline runs without a real DeepDoc+ // service or ONNX Runtime models. Reset to nil on cleanup (this test+ // binary registers no real backend). Text content is extracted by pdfium+ // (the PDF text layer) and tokenized offline, independent of the analyzer,+ // so a mock preserves the chunking/embedding assertions under test.+ t.Cleanup(func() { doctype.SetNativeDocAnalyzerFactory(nil) })+ doctype.SetNativeDocAnalyzerFactory(func() (doctype.DocAnalyzer, bool) {+ return &pdf.MockDocAnalyzer{Healthy: true}, true+ })+origStorage := storage.GetStorageFactory().GetStorage()mem := storage.NewMemoryStorage()storage.GetStorageFactory().SetStorage(mem)diff --git a/internal/ingestion/task/debug_pages_integration_test.go b/internal/ingestion/task/debug_pages_integration_test.goindex d4c2694893..03423f3e9c 100644--- a/internal/ingestion/task/debug_pages_integration_test.go+++ b/internal/ingestion/task/debug_pages_integration_test.go@@ -31,6 +31,8 @@ import ("go.uber.org/zap/zaptest/observer""ragflow/internal/common""ragflow/internal/dao"+ "ragflow/internal/deepdoc/parser/pdf"+ doctype "ragflow/internal/deepdoc/parser/type""ragflow/internal/entity""ragflow/internal/ingestion/component""ragflow/internal/ingestion/pipeline"@@ -63,6 +65,18 @@ func TestExecute_DebugViaEntry_HonorsPagesCap_Integration(t *testing.T) {requireTokenizerPool(t)mustLoadTaskTestConfig(t)+ // The production parse path must never degrade to a mock; install a+ // test-only MockDocAnalyzer as the in-process DeepDoc backend via the+ // public factory seam so the PDF pipeline runs without a real DeepDoc+ // service or ONNX Runtime models. Reset to nil on cleanup (this test+ // binary registers no real backend). The page-cap assertion below relies+ // on pdfium reading the capped/uncapped number of pages — not on layout+ // inference — so a mock analyzer preserves the behaviour under test.+ t.Cleanup(func() { doctype.SetNativeDocAnalyzerFactory(nil) })+ doctype.SetNativeDocAnalyzerFactory(func() (doctype.DocAnalyzer, bool) {+ return &pdf.MockDocAnalyzer{Healthy: true}, true+ })+origDB := dao.DBrealDB := mustOpenTaskTestDB(t)dao.DB = realDBdiff --git a/internal/service/nlp/datasetnav_integration_test.go b/internal/service/nlp/datasetnav_integration_test.goindex c297b835fc..d1747c293f 100644--- a/internal/service/nlp/datasetnav_integration_test.go+++ b/internal/service/nlp/datasetnav_integration_test.go@@ -78,7 +78,7 @@ func TestDatasetNav_AvailableIntZero_Isolation(t *testing.T) {if err := server.Init(configPath); err != nil {t.Fatalf("init service config: %v", err)}- if err := engine.InitDocEngine(); err != nil {+ if err := engine.InitDocEngine(context.Background()); err != nil {t.Fatalf("init document engine: %v", err)}if engine.Get() == nil {diff --git a/internal/service/nlp/datasetnav_test.go b/internal/service/nlp/datasetnav_test.goindex 008f084029..67e4366d66 100644--- a/internal/service/nlp/datasetnav_test.go+++ b/internal/service/nlp/datasetnav_test.go@@ -241,7 +241,15 @@ type stubNavEmbedder struct{}func (stubNavEmbedder) Encode(_ context.Context, _ string, texts []string) ([][]float32, error) {out := make([][]float32, len(texts))for i, t := range texts {- dim := 8+ // The ES document index template (mapping.json, the ragflow_*+ // dynamic_templates) maps q_<dim>_vec to a dense_vector field only for+ // the standard embedding dimensions 512/768/1024/1536. The integration+ // test runs NavService.Search as a real knn query against that index,+ // so the synthetic vector must use one of those dimensions — otherwise+ // ES dynamically maps q_<dim>_vec as a plain float array and the knn+ // query fails with "[knn] queries are only supported on [dense_vector]+ // fields". 1024 is the canonical RAGFlow embedding size.+ dim := 1024v := make([]float32, dim)for d := 0; d < dim; d++ {v[d] = float32(int(t[0]) + d)