Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(integration): register in-process DeepDoc backend + correct stale chunk/knn assertions (#19107)

xugangqiangb2670a7264ca
diff --git a/internal/ingestion/pipeline/real_storage_integration_test.go b/internal/ingestion/pipeline/real_storage_integration_test.go
index d226326acd..4ec571ad01 100644
--- a/internal/ingestion/pipeline/real_storage_integration_test.go
+++ b/internal/ingestion/pipeline/real_storage_integration_test.go
@@ -18,6 +18,8 @@ import (
"time"
"ragflow/internal/dao"
+ "ragflow/internal/deepdoc/parser/pdf"
+ doctype "ragflow/internal/deepdoc/parser/type"
"ragflow/internal/entity"
componentpkg "ragflow/internal/ingestion/component"
_ "ragflow/internal/ingestion/component/chunker"
@@ -34,6 +36,16 @@ func TestPipelineRun_TemplateGeneral_RealMySQLMinIO_OutputShape(t *testing.T) {
prepareTokenizerResourceForIntegration(t)
RequireTokenizerPool(t)
+ // The production parse path must never degrade to a mock; install a
+ // test-only MockDocAnalyzer as the in-process DeepDoc backend via the
+ // public factory seam so the pipeline runs without a real DeepDoc
+ // service or ONNX Runtime models. Reset to nil on cleanup (this test
+ // binary registers no real backend).
+ t.Cleanup(func() { doctype.SetNativeDocAnalyzerFactory(nil) })
+ doctype.SetNativeDocAnalyzerFactory(func() (doctype.DocAnalyzer, bool) {
+ return &pdf.MockDocAnalyzer{Healthy: true}, true
+ })
+
cfg := mustLoadRealIntegrationConfig(t)
realDB := mustOpenRealMySQL(t, cfg)
if err := realDB.AutoMigrate(
@@ -109,7 +121,13 @@ func TestPipelineRun_TemplateGeneral_RealMySQLMinIO_OutputShape(t *testing.T) {
if !ok {
t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])
}
- wantChunkTexts := []string{"Alpha paragraph.", "Beta paragraph."}
+ // The TokenChunker merges the two short paragraphs into one chunk under
+ // the global token-size budget (chunk_token_size=512, no backtick
+ // delimiter): this matches Python's _merge_text_chunks_by_token_size and
+ // is the intended behaviour. The parser still emits two json items
+ // (see the Parser state assertion below); only the downstream chunker
+ // collapses them.
+ wantChunkTexts := []string{"Alpha paragraph.\nBeta paragraph."}
if len(chunks) != len(wantChunkTexts) {
t.Fatalf("len(chunks) = %d, want %d", len(chunks), len(wantChunkTexts))
}
@@ -154,7 +172,10 @@ func TestPipelineRun_TemplateGeneral_RealMySQLMinIO_OutputShape(t *testing.T) {
if !ok || len(jsonItems) != 2 {
t.Fatalf("parser json = %T/%v, want 2 items", parserState["json"], parserState["json"])
}
- for i, wantText := range wantChunkTexts {
+ // The parser emits one json item per paragraph (two items), which the
+ // downstream TokenChunker later merges into a single chunk.
+ wantParserTexts := []string{"Alpha paragraph.", "Beta paragraph."}
+ for i, wantText := range wantParserTexts {
if got := jsonItems[i]["text"]; got != wantText {
t.Fatalf("parser json[%d].text = %v, want %q", i, got, wantText)
}
diff --git a/internal/ingestion/pipeline/template_integration_test.go b/internal/ingestion/pipeline/template_integration_test.go
index 7e87db04f8..d05aad94e3 100644
--- a/internal/ingestion/pipeline/template_integration_test.go
+++ b/internal/ingestion/pipeline/template_integration_test.go
@@ -36,6 +36,8 @@ import (
"testing"
"ragflow/internal/agent/runtime"
+ "ragflow/internal/deepdoc/parser/pdf"
+ doctype "ragflow/internal/deepdoc/parser/type"
componentpkg "ragflow/internal/ingestion/component"
_ "ragflow/internal/ingestion/component/chunker"
"ragflow/internal/ingestion/testutil"
@@ -104,7 +106,7 @@ func TestPipelineRun_TemplateGeneral_RealComponents(t *testing.T) {
if !ok {
t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])
}
- wantChunkTexts := []string{"Alpha paragraph.", "Beta paragraph."}
+ wantChunkTexts := []string{"Alpha paragraph.\nBeta paragraph."}
if len(chunks) != len(wantChunkTexts) {
t.Fatalf("len(chunks) = %d, want %d", len(chunks), len(wantChunkTexts))
}
@@ -769,6 +771,18 @@ func attachFixedEmbedderFactory(t *testing.T, pipe *Pipeline) {
func withRealTemplateDeps(t *testing.T) storage.Storage {
t.Helper()
+ // The production parse path must never degrade to a mock; install a
+ // test-only MockDocAnalyzer as the in-process DeepDoc backend via the
+ // public factory seam so the pipeline runs without a real DeepDoc
+ // service or ONNX Runtime models. Reset to nil on cleanup (this test
+ // binary registers no real backend). Text content is extracted by pdfium
+ // (the PDF text layer) and tokenized offline, independent of the analyzer,
+ // so a mock preserves the chunking/embedding assertions under test.
+ t.Cleanup(func() { doctype.SetNativeDocAnalyzerFactory(nil) })
+ doctype.SetNativeDocAnalyzerFactory(func() (doctype.DocAnalyzer, bool) {
+ return &pdf.MockDocAnalyzer{Healthy: true}, true
+ })
+
origStorage := storage.GetStorageFactory().GetStorage()
mem := storage.NewMemoryStorage()
storage.GetStorageFactory().SetStorage(mem)
diff --git a/internal/ingestion/task/debug_pages_integration_test.go b/internal/ingestion/task/debug_pages_integration_test.go
index d4c2694893..03423f3e9c 100644
--- a/internal/ingestion/task/debug_pages_integration_test.go
+++ b/internal/ingestion/task/debug_pages_integration_test.go
@@ -31,6 +31,8 @@ import (
"go.uber.org/zap/zaptest/observer"
"ragflow/internal/common"
"ragflow/internal/dao"
+ "ragflow/internal/deepdoc/parser/pdf"
+ doctype "ragflow/internal/deepdoc/parser/type"
"ragflow/internal/entity"
"ragflow/internal/ingestion/component"
"ragflow/internal/ingestion/pipeline"
@@ -63,6 +65,18 @@ func TestExecute_DebugViaEntry_HonorsPagesCap_Integration(t *testing.T) {
requireTokenizerPool(t)
mustLoadTaskTestConfig(t)
+ // The production parse path must never degrade to a mock; install a
+ // test-only MockDocAnalyzer as the in-process DeepDoc backend via the
+ // public factory seam so the PDF pipeline runs without a real DeepDoc
+ // service or ONNX Runtime models. Reset to nil on cleanup (this test
+ // binary registers no real backend). The page-cap assertion below relies
+ // on pdfium reading the capped/uncapped number of pages — not on layout
+ // inference — so a mock analyzer preserves the behaviour under test.
+ t.Cleanup(func() { doctype.SetNativeDocAnalyzerFactory(nil) })
+ doctype.SetNativeDocAnalyzerFactory(func() (doctype.DocAnalyzer, bool) {
+ return &pdf.MockDocAnalyzer{Healthy: true}, true
+ })
+
origDB := dao.DB
realDB := mustOpenTaskTestDB(t)
dao.DB = realDB
diff --git a/internal/service/nlp/datasetnav_integration_test.go b/internal/service/nlp/datasetnav_integration_test.go
index c297b835fc..d1747c293f 100644
--- a/internal/service/nlp/datasetnav_integration_test.go
+++ b/internal/service/nlp/datasetnav_integration_test.go
@@ -78,7 +78,7 @@ func TestDatasetNav_AvailableIntZero_Isolation(t *testing.T) {
if err := server.Init(configPath); err != nil {
t.Fatalf("init service config: %v", err)
}
- if err := engine.InitDocEngine(); err != nil {
+ if err := engine.InitDocEngine(context.Background()); err != nil {
t.Fatalf("init document engine: %v", err)
}
if engine.Get() == nil {
diff --git a/internal/service/nlp/datasetnav_test.go b/internal/service/nlp/datasetnav_test.go
index 008f084029..67e4366d66 100644
--- a/internal/service/nlp/datasetnav_test.go
+++ b/internal/service/nlp/datasetnav_test.go
@@ -241,7 +241,15 @@ type stubNavEmbedder struct{}
func (stubNavEmbedder) Encode(_ context.Context, _ string, texts []string) ([][]float32, error) {
out := make([][]float32, len(texts))
for i, t := range texts {
- dim := 8
+ // The ES document index template (mapping.json, the ragflow_*
+ // dynamic_templates) maps q_<dim>_vec to a dense_vector field only for
+ // the standard embedding dimensions 512/768/1024/1536. The integration
+ // test runs NavService.Search as a real knn query against that index,
+ // so the synthetic vector must use one of those dimensions — otherwise
+ // ES dynamically maps q_<dim>_vec as a plain float array and the knn
+ // query fails with "[knn] queries are only supported on [dense_vector]
+ // fields". 1024 is the canonical RAGFlow embedding size.
+ dim := 1024
v := make([]float32, dim)
for d := 0; d < dim; d++ {
v[d] = float32(int(t[0]) + d)