RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(retrieval): align Go parent-child chunk behavior (#19828)
jay777218afebbb67069
diff --git a/internal/engine/elasticsearch/chunk_readback_test.go b/internal/engine/elasticsearch/chunk_readback_test.goindex 1dc70a0ffe..6c52510c02 100644--- a/internal/engine/elasticsearch/chunk_readback_test.go+++ b/internal/engine/elasticsearch/chunk_readback_test.go@@ -84,6 +84,8 @@ func TestInsertChunks_WritesIngestionShape(t *testing.T) {"important_kwd": []string{"k1"},"page_num_int": int(1),"position_int": int(2),+ "mom_id": "parent-1",+ "available_int": int(0),}if _, err := engine.InsertChunks(t.Context(), []map[string]interface{}{chunk}, "ragflow_chunk_readback_test", "kb-1"); err != nil {@@ -106,6 +108,8 @@ func TestInsertChunks_WritesIngestionShape(t *testing.T) {assertStoredField(t, doc, "important_kwd", []interface{}{"k1"})assertStoredField(t, doc, "page_num_int", float64(1))assertStoredField(t, doc, "position_int", float64(2))+ assertStoredField(t, doc, "mom_id", "parent-1")+ assertStoredField(t, doc, "available_int", float64(0))// InsertChunks overrides kb_id with datasetID on write.assertStoredField(t, doc, "kb_id", "kb-1")}diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.goindex 18f66e3618..ffd4865504 100644--- a/internal/engine/infinity/chunk.go+++ b/internal/engine/infinity/chunk.go@@ -848,10 +848,10 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.SfilterParts = append(filterParts, fmt.Sprintf("available_int=%v", availInt))} else if status, ok := req.Filter["status"]; ok {filterParts = append(filterParts, fmt.Sprintf("status='%s'", status))- } else if !isSkillIndex {+ } else if shouldDefaultAvailableFilter(req, isSkillIndex) {filterParts = append(filterParts, "available_int=1")}- } else if !isSkillIndex {+ } else if shouldDefaultAvailableFilter(req, isSkillIndex) {filterParts = append(filterParts, "available_int=1")}}@@ -1042,7 +1042,7 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S}denseFilterStr := filterStr- if denseFilterStr == "" && !isSkillIndex {+ if denseFilterStr == "" && shouldDefaultAvailableFilter(req, isSkillIndex) {denseFilterStr = "available_int=1"}@@ -1224,6 +1224,10 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S}, nil}+func shouldDefaultAvailableFilter(req *types.SearchRequest, isSkillIndex bool) bool {+ return !isSkillIndex && !req.IncludeUnavailable+}+// GetChunk gets a chunk by IDfunc (e *Engine) GetChunk(ctx context.Context, tableName, chunkID string, datasetIDs []string) (interface{}, error) {if e.client == nil || e.client.pool == nil {diff --git a/internal/engine/infinity/chunk_filter_test.go b/internal/engine/infinity/chunk_filter_test.gonew file mode 100644index 0000000000..84c6716d8c--- /dev/null+++ b/internal/engine/infinity/chunk_filter_test.go@@ -0,0 +1,19 @@+package infinity++import (+ "testing"++ "ragflow/internal/engine/types"+)++func TestShouldDefaultAvailableFilter(t *testing.T) {+ if !shouldDefaultAvailableFilter(&types.SearchRequest{}, false) {+ t.Fatal("ordinary retrieval must default to available_int=1")+ }+ if shouldDefaultAvailableFilter(&types.SearchRequest{IncludeUnavailable: true}, false) {+ t.Fatal("management list must not add the retrieval availability default")+ }+ if shouldDefaultAvailableFilter(&types.SearchRequest{}, true) {+ t.Fatal("skill indexes do not use chunk availability")+ }+}diff --git a/internal/engine/infinity/chunk_transform_test.go b/internal/engine/infinity/chunk_transform_test.goindex 5cab6154a3..9c92b573b5 100644--- a/internal/engine/infinity/chunk_transform_test.go+++ b/internal/engine/infinity/chunk_transform_test.go@@ -43,6 +43,8 @@ func TestTransformChunkFields_IngestionShape(t *testing.T) {"important_kwd": []interface{}{"k1"},"page_num_int": int(1),"position_int": int(2),+ "mom_id": "parent-1",+ "available_int": int(0),}got := transformChunkFields(chunk, nil)@@ -58,6 +60,8 @@ func TestTransformChunkFields_IngestionShape(t *testing.T) {"important_keywords": "k1","page_num_int": int(1),"position_int": int(2),+ "mom_id": "parent-1",+ "available_int": int(0),}for k, wv := range want {diff --git a/internal/engine/oceanbase/compatibility_test.go b/internal/engine/oceanbase/compatibility_test.goindex ce660853c5..d600da2794 100644--- a/internal/engine/oceanbase/compatibility_test.go+++ b/internal/engine/oceanbase/compatibility_test.go@@ -133,7 +133,7 @@ func TestPythonPhysicalTableNameContract(t *testing.T) {func TestLegacyChunkEncodingContract(t *testing.T) {document := map[string]interface{}{"id": "chunk-1", "kb_id": []string{"kb-1"}, "doc_id": "doc-1",- "available_int": nil, "removed_kwd": nil, "chunk_order_int": 7,+ "available_int": 0, "removed_kwd": nil, "chunk_order_int": 7, "mom_id": "parent-1","metadata": map[string]interface{}{"_group_id": "group-1", "_title": "renamed"},"important_kwd": []string{" alpha\t", "beta\n"},"q_3_vec": []float64{0.1, 0.2, 0.3},@@ -143,7 +143,7 @@ func TestLegacyChunkEncodingContract(t *testing.T) {if err != nil {t.Fatal(err)}- if got["kb_id"] != "kb-1" || got["available_int"] != 1 || got["removed_kwd"] != "N" || got["_order_id"] != 7 {+ if got["kb_id"] != "kb-1" || got["available_int"] != 0 || got["removed_kwd"] != "N" || got["_order_id"] != 7 || got["mom_id"] != "parent-1" {t.Fatalf("legacy scalar/default encoding changed: %#v", got)}if got["group_id"] != "group-1" || got["docnm_kwd"] != "renamed" {diff --git a/internal/engine/serenedb/search.go b/internal/engine/serenedb/search.goindex e032a2b941..d3c57e445e 100644--- a/internal/engine/serenedb/search.go+++ b/internal/engine/serenedb/search.go@@ -59,7 +59,7 @@ func (e *serenedbEngine) Search(ctx context.Context, req *types.SearchRequest) (outputFields := resolveOutputFields(req.SelectFields)fieldsExpr := strings.Join(outputFields, ", ")- filters := searchFilters(req.Filter, req.KbIDs, pm.hasText || pm.hasVector)+ filters := searchFilters(req.Filter, req.KbIDs, pm.hasText || pm.hasVector, req.IncludeUnavailable)where := filtersExpr(filters)offset := req.Offset@@ -155,9 +155,10 @@ func resolveOutputFields(selectFields []string) []string {}// searchFilters builds the SQL predicates for a search. kb_id scopes the query-// to the requested datasets within the shared tenant table, and scored queries-// default to available_int=1 when the caller did not set it.-func searchFilters(filter map[string]interface{}, kbIDs []string, scored bool) []string {+// to the requested datasets within the shared tenant table. Scored retrieval+// defaults to available_int=1, except when a management caller explicitly+// requests disabled chunks as well.+func searchFilters(filter map[string]interface{}, kbIDs []string, scored, includeUnavailable bool) []string {cond := map[string]interface{}{}for k, v := range filter {cond[k] = v@@ -165,7 +166,7 @@ func searchFilters(filter map[string]interface{}, kbIDs []string, scored bool) [if kbs := stringSlice(kbIDs); len(kbs) > 0 {cond["kb_id"] = kbs}- if scored {+ if scored && !includeUnavailable {_, hasAvail := cond["available_int"]_, hasStatus := cond["status"]if !hasAvail && !hasStatus {diff --git a/internal/engine/serenedb/serenedb_test.go b/internal/engine/serenedb/serenedb_test.goindex 2f81356639..e18e0e9cba 100644--- a/internal/engine/serenedb/serenedb_test.go+++ b/internal/engine/serenedb/serenedb_test.go@@ -118,6 +118,24 @@ func TestL2NormalizeUnitLength(t *testing.T) {}}+func TestPrepareChunkRowKeepsHiddenParentFields(t *testing.T) {+ cols, values := prepareChunkRow(map[string]interface{}{+ "id": "parent-1",+ "doc_id": "doc-1",+ "content_with_weight": "parent text",+ "mom_id": "parent-1",+ "available_int": 0,+ }, "kb-1")++ row := make(map[string]interface{}, len(cols))+ for i, col := range cols {+ row[col] = values[i]+ }+ if row["mom_id"] != "parent-1" || row["available_int"] != 0 || row["kb_id"] != "kb-1" {+ t.Fatalf("parent row = %#v, want hidden and dataset-scoped", row)+ }+}+func TestVectorLiteralTyped(t *testing.T) {got := vectorLiteral([]float64{3, 4})mustContain(t, got, "ARRAY[0.6,0.8]::FLOAT[2]")@@ -260,23 +278,27 @@ func TestResolveOutputFields(t *testing.T) {func TestSearchFiltersScoping(t *testing.T) {// KbIDs become an IN predicate over the shared tenant table.- scored := searchFilters(map[string]interface{}{}, []string{"kb1", "kb2"}, true)+ scored := searchFilters(map[string]interface{}{}, []string{"kb1", "kb2"}, true, false)joined := strings.Join(scored, " AND ")mustContain(t, joined, "kb_id IN ('kb1', 'kb2')")mustContain(t, joined, "available_int = 1")// No KbIDs and no match expr -> no predicates.- if got := searchFilters(map[string]interface{}{}, nil, false); len(got) != 0 {+ if got := searchFilters(map[string]interface{}{}, nil, false, false); len(got) != 0 {t.Errorf("expected no filters, got %v", got)}// A scored query with no available_int/status defaults available_int=1.- one := searchFilters(map[string]interface{}{}, nil, true)+ one := searchFilters(map[string]interface{}{}, nil, true, false)if len(one) != 1 || one[0] != "available_int = 1" {t.Errorf("scored default = %v", one)}// Blank dataset ids are dropped, so no empty IN () is emitted.- if got := searchFilters(map[string]interface{}{}, []string{""}, false); len(got) != 0 {+ if got := searchFilters(map[string]interface{}{}, []string{""}, false, false); len(got) != 0 {t.Errorf("blank kb ids should yield no filter, got %v", got)}+ // Management listing explicitly opts into disabled parent chunks.+ if got := searchFilters(map[string]interface{}{}, nil, true, true); len(got) != 0 {+ t.Errorf("include unavailable filters = %v, want no availability default", got)+ }}func TestFusionVectorWeight(t *testing.T) {diff --git a/internal/engine/types/types.go b/internal/engine/types/types.goindex ec6c53d493..f85f4565f8 100644--- a/internal/engine/types/types.go+++ b/internal/engine/types/types.go@@ -44,7 +44,8 @@ type SearchRequest struct {SelectFields []string // List of field names to return// Filtering- Filter map[string]interface{} // Filters for search+ Filter map[string]interface{} // Filters for search+ IncludeUnavailable bool // Do not add the retrieval-only available_int=1 default// Match expressionsMatchExprs []interface{} // List of match expressions: [matchText, matchDense, fusionExpr]diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.goindex 9a4ec49172..05e813c1a7 100644--- a/internal/ingestion/pipeline/pipeline_params.go+++ b/internal/ingestion/pipeline/pipeline_params.go@@ -287,6 +287,43 @@ func BuildParserConfig(dslJSON []byte, rawConfig map[string]interface{}) entity.return result}+// ApplyParentChildChunkerConfig derives runtime children_delimiters from the+// top-level parent_child setting. parent_child is the sole public source of+// truth; chunker fields are generated runtime parameters and must not be+// accepted as an independent dataset setting.+func ApplyParentChildChunkerConfig(componentConfig entity.JSONMap, rawConfig map[string]interface{}) {+ parentChild, ok := rawConfig["parent_child"].(map[string]interface{})+ if !ok {+ return+ }+ componentConfig["parent_child"] = parentChild+ useParentChild, _ := parentChild["use_parent_child"].(bool)+ childrenDelimiters := []string{}+ if useParentChild {+ if delimiter, ok := parentChild["children_delimiter"].(string); ok {+ childrenDelimiters = parserchunk.ParseDelimiterField(delimiter)+ }+ }++ for componentID, value := range componentConfig {+ if !IsChunkerComponent(componentID) {+ continue+ }+ params, ok := value.(map[string]interface{})+ if !ok {+ continue+ }+ params["children_delimiters"] = childrenDelimiters+ }+}++// IsChunkerComponent reports whether a pipeline component can split parent+// chunks into children.+func IsChunkerComponent(componentID string) bool {+ lowerID := strings.ToLower(componentID)+ return strings.HasPrefix(lowerID, "generalchunker:") || strings.HasPrefix(lowerID, "tokenchunker:")+}+// ResolveComponentParamsDefaults takes DSL JSON bytes and returns the// component params defaults as an entity.JSONMap {cpnID: {param: value}}.// This is a pure function — callers must load the DSL themselves.diff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.goindex 08dfff7867..4d49d57bcc 100644--- a/internal/ingestion/pipeline/pipeline_params_test.go+++ b/internal/ingestion/pipeline/pipeline_params_test.go@@ -436,6 +436,32 @@ func TestBuildParserConfig_BuiltinExtractorKeepsBuiltInMetadata(t *testing.T) {}}+func TestApplyParentChildChunkerConfig(t *testing.T) {+ config := entity.JSONMap{+ "GeneralChunker:one": map[string]interface{}{"chunk_token_num": 256, "children_delimiters": []string{"wrong"}},+ "TokenChunker:two": map[string]interface{}{"chunk_token_num": 128, "children_delimiters": []string{"wrong"}},+ "Extractor:three": map[string]interface{}{"llm_id": "llm-1"},+ }++ ApplyParentChildChunkerConfig(config, map[string]interface{}{+ "parent_child": map[string]interface{}{+ "use_parent_child": true,+ "children_delimiter": "|",+ },+ })++ for _, componentID := range []string{"GeneralChunker:one", "TokenChunker:two"} {+ params := config[componentID].(map[string]interface{})+ got, ok := params["children_delimiters"].([]string)+ if !ok || len(got) != 1 || got[0] != "|" {+ t.Fatalf("%s children_delimiters = %#v, want [|]", componentID, params["children_delimiters"])+ }+ }+ if _, ok := config["Extractor:three"].(map[string]interface{})["children_delimiters"]; ok {+ t.Fatal("non-chunker component was modified")+ }+}+func TestCleanComponentParams_KeepsModularExtractorParams(t *testing.T) {// Builtin template with empty Extractor params: {}dsl := map[string]any{diff --git a/internal/ingestion/task/chunk_index_writer.go b/internal/ingestion/task/chunk_index_writer.goindex f5ce30c372..bfdffab918 100644--- a/internal/ingestion/task/chunk_index_writer.go+++ b/internal/ingestion/task/chunk_index_writer.go@@ -16,7 +16,20 @@package task-import "context"+import (+ "context"+ "fmt"+ "time"++ "ragflow/internal/common"++ "go.uber.org/zap"+)++const (+ chunkInsertAttempts = 3+ chunkInsertRetryBaseDelay = 100 * time.Millisecond+)// InsertFunc is the signature of the chunk insertion backend (e.g. engine.InsertChunks).type InsertFunc func(ctx context.Context, chunks []map[string]any, baseName, datasetID string) ([]string, error)@@ -65,8 +78,38 @@ func (w *chunkIndexWriter) Write(ctx context.Context, chunks []map[string]any) eif err := ctx.Err(); err != nil {return err}- if _, err := w.insertFunc(ctx, chunks[b:end], w.baseName, w.datasetID); err != nil {- return err+ var err error+ for attempt := 1; attempt <= chunkInsertAttempts; attempt++ {+ _, err = w.insertFunc(ctx, chunks[b:end], w.baseName, w.datasetID)+ if err == nil {+ break+ }+ if ctxErr := ctx.Err(); ctxErr != nil {+ return ctxErr+ }+ if attempt == chunkInsertAttempts {+ continue+ }++ delay := chunkInsertRetryBaseDelay << (attempt - 1)+ common.Warn("retrying chunk index write",+ zap.Int("batch_start", b),+ zap.Int("batch_end", end),+ zap.Int("attempt", attempt+1),+ zap.Int("max_attempts", chunkInsertAttempts),+ zap.Duration("delay", delay),+ zap.Error(err),+ )+ timer := time.NewTimer(delay)+ select {+ case <-ctx.Done():+ timer.Stop()+ return ctx.Err()+ case <-timer.C:+ }+ }+ if err != nil {+ return fmt.Errorf("insert chunk batch %d-%d after %d attempts: %w", b, end, chunkInsertAttempts, err)}}return nildiff --git a/internal/ingestion/task/chunk_index_writer_test.go b/internal/ingestion/task/chunk_index_writer_test.goindex a4f5ce41b7..d3c05f485b 100644--- a/internal/ingestion/task/chunk_index_writer_test.go+++ b/internal/ingestion/task/chunk_index_writer_test.go@@ -18,7 +18,9 @@ package taskimport ("context"+ "errors""testing"+ "time")func TestChunkIndexWriter_EmptyChunks(t *testing.T) {@@ -114,3 +116,42 @@ func TestChunkIndexWriter_BulkSizeZero(t *testing.T) {t.Fatalf("batch size = %d, want 20 (bulkSize=0 should degrade to len(chunks))", lastBatchSize)}}++func TestChunkIndexWriterRetriesFailedBatch(t *testing.T) {+ attempts := 0+ writer := newChunkIndexWriter(func(_ context.Context, _ []map[string]any, _, _ string) ([]string, error) {+ attempts+++ if attempts == 1 {+ return nil, errors.New("temporary parent write failure")+ }+ return nil, nil+ }, "ragflow_tenant", "kb", 0)++ if err := writer.Write(t.Context(), []map[string]any{{"id": "child"}, {"id": "parent", "available_int": 0}}); err != nil {+ t.Fatalf("Write() error = %v", err)+ }+ if attempts != 2 {+ t.Fatalf("insert attempts = %d, want 2", attempts)+ }+}++func TestChunkIndexWriterWaitsBeforeRetryingFailedBatch(t *testing.T) {+ attempts := make([]time.Time, 0, 2)+ writer := newChunkIndexWriter(func(_ context.Context, _ []map[string]any, _, _ string) ([]string, error) {+ attempts = append(attempts, time.Now())+ if len(attempts) == 1 {+ return nil, errors.New("temporary write failure")+ }+ return nil, nil+ }, "ragflow_tenant", "kb", 0)++ if err := writer.Write(t.Context(), []map[string]any{{"id": "chunk"}}); err != nil {+ t.Fatalf("Write() error = %v", err)+ }+ if len(attempts) != 2 {+ t.Fatalf("insert attempts = %d, want 2", len(attempts))+ }+ if delay := attempts[1].Sub(attempts[0]); delay < 50*time.Millisecond {+ t.Fatalf("retry delay = %s, want at least 50ms", delay)+ }+}diff --git a/internal/ingestion/task/indexdoc/parent_child.go b/internal/ingestion/task/indexdoc/parent_child.gonew file mode 100644index 0000000000..67d97a40c9--- /dev/null+++ b/internal/ingestion/task/indexdoc/parent_child.go@@ -0,0 +1,75 @@+package indexdoc++import (+ "fmt"+ "strings"++ "github.com/cespare/xxhash/v2"+)++// MaterializeParentChunks turns the chunker's transient parent text into+// hidden index rows and links every child row to its parent. A parent ID is+// scoped to its dataset and document so equal parent text from separate+// documents or datasets cannot overwrite each other in a shared index.+func MaterializeParentChunks(datasetID string, chunks []map[string]any) []map[string]any {+ seen := make(map[string]struct{})+ parents := make([]map[string]any, 0)++ for _, chunk := range chunks {+ mom := parentText(chunk)+ delete(chunk, "mom")+ delete(chunk, "mom_with_weight")+ if mom == "" {+ continue+ }++ docID, _ := chunk["doc_id"].(string)+ if docID == "" {+ continue+ }+ parentID := parentChunkID(datasetID, docID, mom)+ chunk["mom_id"] = parentID+ if _, exists := seen[parentID]; exists {+ continue+ }++ parent := map[string]any{+ "id": parentID,+ "content_with_weight": mom,+ "available_int": 0,+ }+ for _, field := range []string{+ "doc_id", "docnm_kwd", "position_int", "create_timestamp_flt",+ "page_num_int", "top_int",+ } {+ if value, exists := chunk[field]; exists {+ parent[field] = value+ }+ }+ seen[parentID] = struct{}{}+ parents = append(parents, parent)+ }+ return parents+}++// parentChunkID returns the deterministic parent row identifier. NUL+// separators make the dataset, document, and parent-text boundaries+// unambiguous.+func parentChunkID(datasetID, docID, mom string) string {+ hasher := xxhash.New()+ _, _ = hasher.WriteString(datasetID)+ _, _ = hasher.Write([]byte{0})+ _, _ = hasher.WriteString(docID)+ _, _ = hasher.Write([]byte{0})+ _, _ = hasher.WriteString(mom)+ return fmt.Sprintf("%016x", hasher.Sum64())+}++func parentText(chunk map[string]any) string {+ for _, field := range []string{"mom", "mom_with_weight"} {+ if value, ok := chunk[field].(string); ok && strings.TrimSpace(value) != "" {+ return value+ }+ }+ return ""+}diff --git a/internal/ingestion/task/indexdoc/parent_child_test.go b/internal/ingestion/task/indexdoc/parent_child_test.gonew file mode 100644index 0000000000..df56f7877a--- /dev/null+++ b/internal/ingestion/task/indexdoc/parent_child_test.go@@ -0,0 +1,31 @@+package indexdoc++import "testing"++func TestParentChunkIDScopesDatasetAndDocument(t *testing.T) {+ if got, again := parentChunkID("kb-a", "doc-a", "same parent text"), parentChunkID("kb-a", "doc-a", "same parent text"); got != again {+ t.Fatalf("parentChunkID is not deterministic: %q != %q", got, again)+ }+ if got, otherDataset := parentChunkID("kb-a", "doc-a", "same parent text"), parentChunkID("kb-b", "doc-a", "same parent text"); got == otherDataset {+ t.Fatalf("parentChunkID collides across datasets: %q", got)+ }+}++func TestMaterializeParentChunksScopesSameParentTextToDocument(t *testing.T) {+ chunks := []map[string]any{+ {"id": "child-a", "doc_id": "doc-a", "mom": "shared parent"},+ {"id": "child-b", "doc_id": "doc-b", "mom": "shared parent"},+ }++ parents := MaterializeParentChunks("kb-a", chunks)++ if len(parents) != 2 {+ t.Fatalf("parent count = %d, want 2", len(parents))+ }+ if chunks[0]["mom_id"] == chunks[1]["mom_id"] {+ t.Fatalf("mom_id collision: %q", chunks[0]["mom_id"])+ }+ if parents[0]["doc_id"] == parents[1]["doc_id"] {+ t.Fatalf("parent docs = %#v, want one parent per document", parents)+ }+}diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex d7ff66805b..bfc224d4ce 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -69,10 +69,11 @@ type PipelineExecutor struct {canvasID stringdocBulkSize int- indexWriter *chunkIndexWriter- loadDSLFunc func(ctx context.Context, canvasID string) (string, string, error)- runPipelineFunc func(ctx context.Context, dsl string) (map[string]any, string, error)- progressSink pipelinepkg.ProgressSink+ indexWriter *chunkIndexWriter+ deleteChunksFunc DeleteChunksFunc+ loadDSLFunc func(ctx context.Context, canvasID string) (string, string, error)+ runPipelineFunc func(ctx context.Context, dsl string) (map[string]any, string, error)+ progressSink pipelinepkg.ProgressSink}func validateTaskContext(taskCtx *TaskContext) error {@@ -138,17 +139,29 @@ func NewPipelineExecutor(canvasID: canvasID,docBulkSize: docBulkSize,indexWriter: newChunkIndexWriter(insertChunksForIngestion(engine.Get()), fmt.Sprintf("ragflow_%s", taskCtx.Tenant.ID), taskCtx.Doc.KbID, docBulkSize),+ deleteChunksFunc: func(ctx context.Context, condition map[string]any, baseName, datasetID string) (int64, error) {+ return engine.Get().DeleteChunks(ctx, condition, baseName, datasetID)+ },}svc.loadDSLFunc = svc.loadDSLFromCanvassvc.runPipelineFunc = svc.runPipelineWithDSLreturn svc, nil}+// DeleteChunksFunc removes partially written rows after a failed index write.+type DeleteChunksFunc func(ctx context.Context, condition map[string]any, baseName, datasetID string) (int64, error)+func (s *PipelineExecutor) WithInsertFunc(f InsertFunc) *PipelineExecutor {s.indexWriter.insertFunc = freturn s}+// WithDeleteChunksFunc replaces index-write compensation. It is used by the+// in-memory pipeline tests; production uses the configured document engine.+func (s *PipelineExecutor) WithDeleteChunksFunc(f DeleteChunksFunc) *PipelineExecutor {+ s.deleteChunksFunc = f+ return s+}func (s *PipelineExecutor) WithLoadDSLFunc(f func(ctx context.Context, canvasID string) (string, string, error)) *PipelineExecutor {s.loadDSLFunc = freturn s@@ -258,6 +271,7 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput mapif err != nil {return nil, err}+ parentChunks := indexdoc.MaterializeParentChunks(s.taskCtx.Doc.KbID, chunks)tableMeta := indexdoc.AggregateTableDocMetadata(chunks, map[string]interface{}(s.taskCtx.Doc.ParserConfig))if tableMeta != nil {@@ -295,8 +309,12 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput mapif err != nil {return nil, err}- if err := s.indexWriter.Write(ctx, chunks); err != nil {- return nil, err+ indexChunks := append(chunks, parentChunks...)+ if err := s.indexWriter.Write(ctx, indexChunks); err != nil {+ if cleanupErr := s.compensateFailedIndexWrite(ctx, indexChunks); cleanupErr != nil {+ return nil, fmt.Errorf("write chunks: %w; compensate partial index write: %v", err, cleanupErr)+ }+ return nil, fmt.Errorf("write chunks: %w", err)}if err := s.reconcileDocumentCompiledProducts(ctx, oldCompiledProductIDs, chunks); err != nil {return nil, err@@ -363,6 +381,37 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map}, nil}+func (s *PipelineExecutor) compensateFailedIndexWrite(ctx context.Context, chunks []map[string]any) error {+ if s.deleteChunksFunc == nil {+ return nil+ }+ ids := make([]string, 0, len(chunks))+ seen := make(map[string]struct{}, len(chunks))+ for _, chunk := range chunks {+ id, _ := chunk["id"].(string)+ if id == "" {+ continue+ }+ if _, exists := seen[id]; exists {+ continue+ }+ seen[id] = struct{}{}+ ids = append(ids, id)+ }+ if len(ids) == 0 {+ return nil+ }+ cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)+ defer cancel()+ _, err := s.deleteChunksFunc(+ cleanupCtx,+ map[string]any{"id": ids, "kb_id": s.taskCtx.Doc.KbID},+ s.indexWriter.baseName,+ s.taskCtx.Doc.KbID,+ )+ return err+}+// terminalDuration measures the run from the document's process_begin_at —// the anchor PrepareValidatedRun stamped before the run and the one every// mid-run progress-sink duration write uses — so the terminal value applieddiff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.goindex 536fd2cc2a..4e6d6cf3f8 100644--- a/internal/ingestion/task/pipeline_executor_test.go+++ b/internal/ingestion/task/pipeline_executor_test.go@@ -791,6 +791,112 @@ func TestRunPipeline_FullFlow(t *testing.T) {}}+// TestPipelineExecutor_ProcessOutputMaterializesParentChunk verifies the+// persistence boundary turns chunker-level mom text into one hidden parent row+// and links every child to it. Removing that materialization would leave only+// independently retrievable child rows, which is the parent-child regression.+func TestPipelineExecutor_ProcessOutputMaterializesParentChunk(t *testing.T) {+ var inserted []map[string]any+ writeCalls := 0+ svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0).WithInsertFunc(+ func(_ context.Context, chunks []map[string]any, _, _ string) ([]string, error) {+ writeCalls+++ inserted = append(inserted, chunks...)+ return nil, nil+ },+ )++ _, err := svc.processOutput(t.Context(), map[string]any{+ "chunks": []map[string]any{+ {"text": "child one", "mom": "full parent passage"},+ {"text": "child two", "mom": "full parent passage"},+ },+ }, time.Now())+ if err != nil {+ t.Fatalf("processOutput: %v", err)+ }+ if len(inserted) != 3 {+ t.Fatalf("persisted rows = %d, want two children and one parent", len(inserted))+ }+ if writeCalls != 1 {+ t.Fatalf("write calls = %d, want children and parents in one write", writeCalls)+ }++ var parent map[string]any+ children := make([]map[string]any, 0, 2)+ for _, row := range inserted {+ if row["available_int"] == 0 && row["content_with_weight"] == "full parent passage" {+ parent = row+ continue+ }+ children = append(children, row)+ }+ if parent == nil {+ t.Fatal("hidden parent row was not persisted")+ }+ parentID, _ := parent["id"].(string)+ if parentID == "" {+ t.Fatalf("parent id = %#v, want deterministic non-empty id", parent["id"])+ }+ for _, child := range children {+ if child["mom_id"] != parentID {+ t.Errorf("child mom_id = %#v, want parent id %q", child["mom_id"], parentID)+ }+ if _, exists := child["mom"]; exists {+ t.Errorf("child retained pipeline-only mom field: %#v", child)+ }+ }+}++func TestPipelineExecutorCompensatesAfterExhaustingCombinedWriteRetries(t *testing.T) {+ var deleted map[string]any+ svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0).+ WithInsertFunc(func(context.Context, []map[string]any, string, string) ([]string, error) {+ return nil, errors.New("index unavailable")+ }).+ WithDeleteChunksFunc(func(_ context.Context, condition map[string]any, _, _ string) (int64, error) {+ deleted = condition+ return 0, nil+ })++ _, err := svc.processOutput(t.Context(), map[string]any{+ "chunks": []map[string]any{{"text": "child", "mom": "parent"}},+ }, time.Now())+ if err == nil {+ t.Fatal("processOutput succeeded after exhausted writes")+ }+ ids, ok := deleted["id"].([]string)+ if !ok || len(ids) != 2 {+ t.Fatalf("compensation ids = %#v, want child and parent IDs", deleted)+ }+}++func TestPipelineExecutorBoundsCompensationCleanup(t *testing.T) {+ var cleanupCtx context.Context+ svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0).+ WithInsertFunc(func(context.Context, []map[string]any, string, string) ([]string, error) {+ return nil, errors.New("index unavailable")+ }).+ WithDeleteChunksFunc(func(ctx context.Context, _ map[string]any, _, _ string) (int64, error) {+ cleanupCtx = ctx+ return 0, nil+ })++ _, err := svc.processOutput(t.Context(), map[string]any{+ "chunks": []map[string]any{{"text": "child", "mom": "parent"}},+ }, time.Now())+ if err == nil {+ t.Fatal("processOutput succeeded after exhausted writes")+ }+ deadline, ok := cleanupCtx.Deadline()+ if !ok {+ t.Fatal("compensation cleanup context has no deadline")+ }+ if remaining := time.Until(deadline); remaining <= 0 || remaining > 5*time.Second {+ t.Fatalf("compensation cleanup deadline remaining = %s, want (0, 5s]", remaining)+ }+}+func TestRunPipeline_AlreadyHasVectors(t *testing.T) {svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0).WithInsertFunc(func(ctx context.Context, chunks []map[string]any, baseName, datasetID string) ([]string, error) {diff --git a/internal/rag/agentic-rag/runtime/tool_executor_test.go b/internal/rag/agentic-rag/runtime/tool_executor_test.goindex f638ea2ae7..03a80da158 100644--- a/internal/rag/agentic-rag/runtime/tool_executor_test.go+++ b/internal/rag/agentic-rag/runtime/tool_executor_test.go@@ -28,6 +28,7 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/common""ragflow/internal/engine"+ enginetypes "ragflow/internal/engine/types""gorm.io/gorm")@@ -47,19 +48,21 @@ func (s stubRetrievalService) Search(_ context.Context, _ *gorm.DB, req runtime.return s.chunks, nil}-// stubDocEngine implements engine.DocEngine by only serving GetChunk; the rest-// of the (large) interface is promoted from a nil field and never called by-// this test.+// stubDocEngine serves the document-scoped Search used by retrieval_by_children;+// the rest of the (large) interface is promoted from a nil field and never+// called by this test.type stubDocEngine struct {engine.DocEngineparents map[string]map[string]any}-func (s stubDocEngine) GetChunk(_ context.Context, _, chunkID string, _ []string) (interface{}, error) {- if p, ok := s.parents[chunkID]; ok {- return p, nil+func (s stubDocEngine) Search(_ context.Context, req *enginetypes.SearchRequest) (*enginetypes.SearchResult, error) {+ parentID, _ := req.Filter["id"].(string)+ docID, _ := req.Filter["doc_id"].(string)+ if p, ok := s.parents[parentID]; ok && p["doc_id"] == docID {+ return &enginetypes.SearchResult{Chunks: []map[string]interface{}{p}}, nil}- return nil, fmt.Errorf("parent %s not found", chunkID)+ return &enginetypes.SearchResult{}, nil}// TestRuntimeRetrieverPreservesUnsetControls pins the presence semantics of the@@ -140,8 +143,8 @@ func TestChunkAggRetrievePropagatesError(t *testing.T) {func TestRuntimeRetrieverPromotesChildrenToParent(t *testing.T) {prev := runtime.GetRetrievalService()runtime.SetRetrievalService(stubRetrievalService{chunks: []runtime.RetrievalChunk{- {ID: "child-1", Content: "frag one", DatasetID: "kb-1", MomID: "parent-1", Score: 0.6},- {ID: "child-2", Content: "frag two", DatasetID: "kb-1", MomID: "parent-1", Score: 0.8},+ {ID: "child-1", Content: "frag one", DocumentID: "doc-1", DatasetID: "kb-1", MomID: "parent-1", Score: 0.6},+ {ID: "child-2", Content: "frag two", DocumentID: "doc-1", DatasetID: "kb-1", MomID: "parent-1", Score: 0.8},{ID: "top-1", Content: "standalone", DatasetID: "kb-1", Score: 0.9},}})t.Cleanup(func() { runtime.SetRetrievalService(prev) })@@ -224,7 +227,7 @@ func TestHybridSearchSkipsWhenNoChildren(t *testing.T) {func TestBM25SearchDoesPromoteChildren(t *testing.T) {prev := runtime.GetRetrievalService()runtime.SetRetrievalService(stubRetrievalService{chunks: []runtime.RetrievalChunk{- {ID: "child-1", Content: "frag", DatasetID: "kb-1", MomID: "parent-1", Score: 0.6},+ {ID: "child-1", Content: "frag", DocumentID: "doc-1", DatasetID: "kb-1", MomID: "parent-1", Score: 0.6},}})t.Cleanup(func() { runtime.SetRetrievalService(prev) })diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.goindex 021ffc5e16..b91ba33b25 100644--- a/internal/service/chunk/chunk.go+++ b/internal/service/chunk/chunk.go@@ -863,12 +863,13 @@ func (s *ChunkService) List(ctx context.Context, req *service.ListChunksRequest,// Build search request - same as retrieval test but filtered by doc_idsearchReq := &types.SearchRequest{- IndexNames: []string{indexName},- MatchExprs: matchExprs,- KbIDs: kbIDs,- Offset: (page - 1) * size,- Limit: size,- OrderBy: orderBy,+ IndexNames: []string{indexName},+ MatchExprs: matchExprs,+ KbIDs: kbIDs,+ Offset: (page - 1) * size,+ Limit: size,+ OrderBy: orderBy,+ IncludeUnavailable: req.AvailableInt == nil,SelectFields: []string{"id","content_with_weight",diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.goindex 8a7018332f..bbf700a95e 100644--- a/internal/service/chunk/chunk_test.go+++ b/internal/service/chunk/chunk_test.go@@ -334,6 +334,9 @@ func TestListSortsChunksByDocumentPosition(t *testing.T) {if engine.searchReq == nil {t.Fatal("expected Search to be called")}+ if !engine.searchReq.IncludeUnavailable {+ t.Fatal("management chunk list must include disabled parent chunks")+ }if engine.searchReq.OrderBy == nil {t.Fatal("expected OrderBy to be set")}diff --git a/internal/service/dataset/create_test.go b/internal/service/dataset/create_test.goindex 43a5c45ea1..618660c576 100644--- a/internal/service/dataset/create_test.go+++ b/internal/service/dataset/create_test.go@@ -63,6 +63,32 @@ func TestCreateDataset_NoComponentParams(t *testing.T) {}}+func TestCreateDataset_DefaultsParentChildConfig(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertCreateDatasetTenant(t, "tenant-1")++ result, code, err := testDatasetCreateService(t).CreateDataset(t.Context(), &service.CreateDatasetRequest{+ Name: "ds-default-parent-child",+ ParserConfig: map[string]interface{}{},+ }, "tenant-1")+ if err != nil || code != common.CodeSuccess {+ t.Fatalf("CreateDataset err=%v code=%d", err, code)+ }++ config, ok := result["parser_config"].(entity.JSONMap)+ if !ok {+ t.Fatalf("parser_config type = %T, want entity.JSONMap", result["parser_config"])+ }+ parentChild, ok := config["parent_child"].(map[string]interface{})+ if !ok {+ t.Fatalf("parent_child = %#v, want default map", config["parent_child"])+ }+ if parentChild["use_parent_child"] != false || parentChild["children_delimiter"] != "\n" {+ t.Fatalf("parent_child = %#v, want disabled defaults", parentChild)+ }+}+func TestCreateDataset_BuiltinParserDoesNotRequireParseType(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)@@ -111,6 +137,77 @@ func TestCreateDataset_ComponentParamsPopulated(t *testing.T) {}}+// TestCreateDataset_ParentChildConfigReachesGeneralChunker verifies the public+// parent-child setting controls the component that actually performs the+// secondary split. Dropping this mapping silently leaves parent-child disabled.+func TestCreateDataset_ParentChildConfigReachesGeneralChunker(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertCreateDatasetTenant(t, "tenant-1")++ parserID := "general"+ parseType := 1+ result, code, err := testDatasetCreateService(t).CreateDataset(t.Context(), &service.CreateDatasetRequest{+ Name: "ds-parent-child",+ ParserID: &parserID,+ ParseType: &parseType,+ ParserConfig: map[string]interface{}{+ "parent_child": map[string]interface{}{+ "use_parent_child": true,+ "children_delimiter": "|",+ },+ },+ }, "tenant-1")+ if err != nil || code != common.CodeSuccess {+ t.Fatalf("CreateDataset err=%v code=%d", err, code)+ }+ config, ok := result["parser_config"].(entity.JSONMap)+ if !ok {+ t.Fatalf("parser_config type = %T, want entity.JSONMap", result["parser_config"])+ }+ chunker, ok := config["GeneralChunker:SixApplesFall"].(map[string]interface{})+ if !ok {+ t.Fatalf("general chunker params = %#v", config["GeneralChunker:SixApplesFall"])+ }+ if got, ok := chunker["children_delimiters"].([]interface{}); !ok || len(got) != 1 || got[0] != "|" {+ t.Fatalf("children_delimiters = %#v, want [\"|\"]", chunker["children_delimiters"])+ }+}++func TestCreateDataset_PreservesChunkerComponentOverrides(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertCreateDatasetTenant(t, "tenant-1")++ parserID := "general"+ parseType := 1+ result, code, err := testDatasetCreateService(t).CreateDataset(t.Context(), &service.CreateDatasetRequest{+ Name: "ds-chunker-override",+ ParserID: &parserID,+ ParseType: &parseType,+ ParserConfig: map[string]interface{}{+ "GeneralChunker:SixApplesFall": map[string]interface{}{+ "chunk_token_size": float64(256),+ },+ },+ }, "tenant-1")+ if err != nil || code != common.CodeSuccess {+ t.Fatalf("CreateDataset err=%v code=%d", err, code)+ }++ config, ok := result["parser_config"].(entity.JSONMap)+ if !ok {+ t.Fatalf("parser_config type = %T, want entity.JSONMap", result["parser_config"])+ }+ chunker, ok := config["GeneralChunker:SixApplesFall"].(map[string]interface{})+ if !ok {+ t.Fatalf("general chunker params = %#v", config["GeneralChunker:SixApplesFall"])+ }+ if got := chunker["chunk_token_size"]; got != float64(256) {+ t.Fatalf("chunk_token_size = %#v, want 256", got)+ }+}+func TestCreateDataset_ParseTypeBuiltinClearsPipelineID(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)diff --git a/internal/service/dataset/crud.go b/internal/service/dataset/crud.goindex d3a390feaa..3ac413435f 100644--- a/internal/service/dataset/crud.go+++ b/internal/service/dataset/crud.go@@ -123,6 +123,25 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateDparserConfig = pipelinepkg.BuildParserConfig(dslJSON, req.ParserConfig)}+ // Preserve the public default shape when parser_config is empty. The+ // parent_child block remains the single source of truth; chunker+ // children_delimiters are derived below only when it is configured.+ var parentChild map[string]interface{}+ if req.ParserConfig != nil {+ if pc, ok := req.ParserConfig["parent_child"].(map[string]interface{}); ok {+ parentChild = pc+ }+ }+ if parentChild == nil {+ parentChild = map[string]interface{}{+ "use_parent_child": false,+ "children_delimiter": "\n",+ }+ }+ parserConfig["parent_child"] = parentChild++ pipelinepkg.ApplyParentChildChunkerConfig(parserConfig, map[string]interface{}(parserConfig))+var parserConfigMap map[string]interface{} = parserConfigembdID := tenant.EmbdIDdiff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.goindex 8c414eb0ee..eff2b8392d 100644--- a/internal/service/dataset/helpers.go+++ b/internal/service/dataset/helpers.go@@ -508,7 +508,7 @@ func datasetUpdateEmbeddingID(req service.UpdateDatasetRequest) (string, bool, ereturn embdID, true, nil}-func preserveDatasetParserConfigMetadata(next, existing entity.JSONMap, incoming map[string]interface{}) entity.JSONMap {+func preserveDatasetParserConfigState(next, existing entity.JSONMap, incoming map[string]interface{}) entity.JSONMap {if next == nil {next = entity.JSONMap{}}@@ -526,6 +526,24 @@ func preserveDatasetParserConfigMetadata(next, existing entity.JSONMap, incomingif mm != nil {next["metadata"] = mm}+ var parentChild map[string]any+ if incoming != nil {+ if value, ok := incoming["parent_child"].(map[string]any); ok {+ parentChild = value+ }+ }+ if parentChild == nil && existing != nil {+ if value, ok := existing["parent_child"].(map[string]any); ok {+ parentChild = value+ }+ }+ if parentChild != nil {+ next["parent_child"] = parentChild+ }+ // The persisted top-level setting is authoritative. Re-derive the+ // chunker's runtime delimiter after every config merge so parser/pipeline+ // switches cannot retain parent_child without retaining its behavior.+ pipelinepkg.ApplyParentChildChunkerConfig(next, map[string]interface{}(next))return next}diff --git a/internal/service/dataset/helpers_test.go b/internal/service/dataset/helpers_test.goindex 9bc74ad252..b112a42868 100644--- a/internal/service/dataset/helpers_test.go+++ b/internal/service/dataset/helpers_test.go@@ -232,6 +232,16 @@ func TestValidateDocumentParserConfig_AllowsUnknownFields(t *testing.T) {}}+func TestValidateParserConfigAcceptsFlatParentChildDelimiter(t *testing.T) {+ config := map[string]interface{}{"children_delimiter": "|"}+ if err := validateDatasetParserConfig(config); err != nil {+ t.Fatalf("flat children_delimiter should remain accepted for compatibility: %v", err)+ }+ if err := ValidateDocumentParserConfig(config); err != nil {+ t.Fatalf("document parser config should accept children_delimiter: %v", err)+ }+}+// --- normalizeDatasetID ---func TestNormalizeDatasetID_Invalid(t *testing.T) {@@ -356,7 +366,7 @@ func TestNormalizeMetadataConfigFields_TrimsKey(t *testing.T) {}}-func TestPreserveDatasetParserConfigMetadata_FallsBackWhenIncomingNotMap(t *testing.T) {+func TestPreserveDatasetParserConfigState_FallsBackWhenIncomingNotMap(t *testing.T) {existing := entity.JSONMap{"metadata": map[string]any{"enabled": true,@@ -371,7 +381,7 @@ func TestPreserveDatasetParserConfigMetadata_FallsBackWhenIncomingNotMap(t *testfor name, incomingMetadata := range cases {t.Run(name, func(t *testing.T) {incoming := map[string]interface{}{"metadata": incomingMetadata}- got := preserveDatasetParserConfigMetadata(entity.JSONMap{}, existing, incoming)+ got := preserveDatasetParserConfigState(entity.JSONMap{}, existing, incoming)meta, ok := got["metadata"].(map[string]any)if !ok {t.Fatalf("expected existing modular metadata preserved, got %#v", got["metadata"])@@ -384,7 +394,7 @@ func TestPreserveDatasetParserConfigMetadata_FallsBackWhenIncomingNotMap(t *test}}-func TestPreserveDatasetParserConfigMetadata_UsesValidIncomingMap(t *testing.T) {+func TestPreserveDatasetParserConfigState_UsesValidIncomingMap(t *testing.T) {existing := entity.JSONMap{"metadata": map[string]any{"enabled": false,@@ -399,7 +409,7 @@ func TestPreserveDatasetParserConfigMetadata_UsesValidIncomingMap(t *testing.T)"built_in_metadata": []any{},},}- got := preserveDatasetParserConfigMetadata(entity.JSONMap{}, existing, incoming)+ got := preserveDatasetParserConfigState(entity.JSONMap{}, existing, incoming)meta, ok := got["metadata"].(map[string]any)if !ok {t.Fatalf("expected modular metadata map, got %#v", got["metadata"])diff --git a/internal/service/dataset/update.go b/internal/service/dataset/update.goindex 3fb86e9b7b..b593d65799 100644--- a/internal/service/dataset/update.go+++ b/internal/service/dataset/update.go@@ -247,7 +247,7 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID}if dslJSON != nil {parserConfig := pipelinepkg.BuildParserConfig(dslJSON, map[string]interface{}(req.ParserConfig))- updates["parser_config"] = preserveDatasetParserConfigMetadata(parserConfig, lockedKB.ParserConfig, req.ParserConfig)+ updates["parser_config"] = preserveDatasetParserConfigState(parserConfig, lockedKB.ParserConfig, req.ParserConfig)}}if pagerankRequested && requestedPagerank != lockedKB.Pagerank {@@ -268,7 +268,7 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantIDcommon.Warn("failed to resolve component params defaults on parser_id switch",zap.String("parserID", parserID), zap.Error(cpErr))} else if resolved != nil {- updates["parser_config"] = preserveDatasetParserConfigMetadata(resolved, lockedKB.ParserConfig, req.ParserConfig)+ updates["parser_config"] = preserveDatasetParserConfigState(resolved, lockedKB.ParserConfig, req.ParserConfig)}}}@@ -288,7 +288,7 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantIDcommon.Warn("failed to resolve component params defaults on pipeline change",zap.String("parserID", cfgParserID), zap.Error(cpErr))} else if cpDefaults != nil {- updates["parser_config"] = preserveDatasetParserConfigMetadata(cpDefaults, lockedKB.ParserConfig, req.ParserConfig)+ updates["parser_config"] = preserveDatasetParserConfigState(cpDefaults, lockedKB.ParserConfig, req.ParserConfig)}}diff --git a/internal/service/dataset/update_test.go b/internal/service/dataset/update_test.goindex ca59493cd8..887f37fa28 100644--- a/internal/service/dataset/update_test.go+++ b/internal/service/dataset/update_test.go@@ -98,6 +98,77 @@ func TestDatasetServiceUpdateDatasetUpdatesFields(t *testing.T) {}}+// TestUpdateDataset_ParentChildConfigReachesGeneralChunker verifies an edit to+// the dataset setting updates the runtime chunker parameter, not only the UI+// payload retained in parser_config.+func TestUpdateDataset_ParentChildConfigReachesGeneralChunker(t *testing.T) {+ db := setupDatasetUpdateTestDB(t)+ pushServiceDB(t, db)+ insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Original")++ _, code, err := testDatasetUpdateService(t).UpdateDataset(t.Context(), "kb-1", "tenant-1", service.UpdateDatasetRequest{+ ParserConfig: map[string]interface{}{+ "parent_child": map[string]interface{}{+ "use_parent_child": true,+ "children_delimiter": "|",+ },+ },+ })+ if err != nil || code != common.CodeSuccess {+ t.Fatalf("UpdateDataset err=%v code=%d", err, code)+ }+ persisted, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")+ if err != nil {+ t.Fatalf("get updated dataset: %v", err)+ }+ chunker, ok := persisted.ParserConfig["GeneralChunker:SixApplesFall"].(map[string]interface{})+ if !ok {+ t.Fatalf("general chunker params = %#v", persisted.ParserConfig["GeneralChunker:SixApplesFall"])+ }+ if got, ok := chunker["children_delimiters"].([]interface{}); !ok || len(got) != 1 || got[0] != "|" {+ t.Fatalf("children_delimiters = %#v, want [\"|\"]", chunker["children_delimiters"])+ }+ parentChild, ok := persisted.ParserConfig["parent_child"].(map[string]interface{})+ if !ok || parentChild["use_parent_child"] != true || parentChild["children_delimiter"] != "|" {+ t.Fatalf("parent_child = %#v, want persisted public setting", persisted.ParserConfig["parent_child"])+ }+}++func TestUpdateDatasetPreservesParentChildChunkerRuntimeConfig(t *testing.T) {+ db := setupDatasetUpdateTestDB(t)+ pushServiceDB(t, db)+ insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Original")++ existingConfig := entity.JSONMap{+ "parent_child": map[string]interface{}{+ "use_parent_child": true,+ "children_delimiter": "|",+ },+ }+ if err := db.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", existingConfig).Error; err != nil {+ t.Fatalf("seed parent-child parser config: %v", err)+ }++ _, code, err := testDatasetUpdateService(t).UpdateDataset(t.Context(), "kb-1", "tenant-1", service.UpdateDatasetRequest{+ ParserConfig: map[string]interface{}{"chunk_token_num": float64(256)},+ })+ if err != nil || code != common.CodeSuccess {+ t.Fatalf("UpdateDataset err=%v code=%d", err, code)+ }++ persisted, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")+ if err != nil {+ t.Fatalf("load updated dataset: %v", err)+ }+ chunker, ok := persisted.ParserConfig["GeneralChunker:SixApplesFall"].(map[string]interface{})+ if !ok {+ t.Fatalf("general chunker params = %#v", persisted.ParserConfig["GeneralChunker:SixApplesFall"])+ }+ if got, ok := chunker["children_delimiters"].([]interface{}); !ok || len(got) != 1 || got[0] != "|" {+ t.Fatalf("children_delimiters = %#v, want [\"|\"]", chunker["children_delimiters"])+ }+}+func TestUpdateDataset_RejectsSimultaneousParserIDAndPipelineID(t *testing.T) {db := setupDatasetUpdateTestDB(t)pushServiceDB(t, db)diff --git a/internal/service/document/document_dataset_update.go b/internal/service/document/document_dataset_update.goindex 771fd9cfe0..8b091bbc2e 100644--- a/internal/service/document/document_dataset_update.go+++ b/internal/service/document/document_dataset_update.go@@ -162,6 +162,7 @@ func (s *DocumentService) UpdateDatasetDocument(ctx context.Context, userID, dat}} else {cleaned := pipelinepkg.BuildParserConfig(dslJSON, req.ParserConfig)+ pipelinepkg.ApplyParentChildChunkerConfig(cleaned, req.ParserConfig)tenant, tenantErr := dao.NewTenantDAO().GetByID(ctx, dao.DB, kb.TenantID)if tenantErr == nil && tenant != nil {cleaned = service.ApplyComponentScopedParserConfig(@@ -467,6 +468,7 @@ func (s *DocumentService) updateDocumentParserConfig(ctx context.Context, documeif _, ok := config["raptor"]; !ok {delete(merged, "raptor")}+ pipelinepkg.ApplyParentChildChunkerConfig(merged, merged)return s.documentDAO.UpdateByID(ctx, dao.DB, documentID, map[string]interface{}{"parser_config": entity.JSONMap(merged),diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.goindex 6ada2f35af..985718b885 100644--- a/internal/service/document/document_test.go+++ b/internal/service/document/document_test.go@@ -248,6 +248,7 @@ type rerunDeleteDocEngine struct {condition map[string]interface{}indexName stringdatasetID string+ search *types.SearchRequest}type failingDeleteDocEngine struct {@@ -286,11 +287,13 @@ func (e *rerunDeleteDocEngine) ChunkStoreExists(context.Context, string, string)return true, nil}-func (e *rerunDeleteDocEngine) Search(context.Context, *types.SearchRequest) (*types.SearchResult, error) {+func (e *rerunDeleteDocEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {+ e.search = reqreturn &types.SearchResult{Chunks: []map[string]interface{}{{"id": "source-1"},+ {"id": "parent-1", "available_int": 0},{"id": "wiki-1", "compile_kwd": "wiki_page"},- }, Total: 2}, nil+ }, Total: 3}, nil}func (e *rerunDeleteDocEngine) DeleteChunks(_ context.Context, condition map[string]interface{}, indexName string, datasetID string) (int64, error) {@@ -308,10 +311,12 @@ func (e *failingDeleteDocEngine) DeleteChunks(context.Context, map[string]interftype sourceAvailabilityDocEngine struct {fakeChatDocEngineupdateConditions []map[string]interface{}+ search *types.SearchRequestupdateValues []map[string]interface{}}-func (e *sourceAvailabilityDocEngine) Search(context.Context, *types.SearchRequest) (*types.SearchResult, error) {+func (e *sourceAvailabilityDocEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {+ e.search = reqreturn &types.SearchResult{Chunks: []map[string]interface{}{{"id": "source-1"},{"id": "tree-1", "compile_kwd": "tree"},@@ -2475,9 +2480,12 @@ func TestClearDocumentParseResultsClearsCountersTasksAndChunks(t *testing.T) {if engine.deleteCalls != 2 {t.Fatalf("deleteCalls = %d, want 2 (generated and source chunks)", engine.deleteCalls)}- if engine.indexName != "ragflow_tenant-1" || engine.datasetID != "kb-1" || !reflect.DeepEqual(engine.condition["id"], []string{"source-1"}) {+ if engine.indexName != "ragflow_tenant-1" || engine.datasetID != "kb-1" || !reflect.DeepEqual(engine.condition["id"], []string{"source-1", "parent-1"}) {t.Fatalf("unexpected delete call: index=%s dataset=%s condition=%v", engine.indexName, engine.datasetID, engine.condition)}+ if engine.search == nil || !engine.search.IncludeUnavailable {+ t.Fatalf("reparse search = %#v, want hidden parent rows included", engine.search)+ }}func TestClearDocumentParseResultsDeletesDocumentGeneratedChunks(t *testing.T) {@@ -2584,6 +2592,9 @@ func TestUpdateDocumentChunkAvailabilityTogglesFinalProducts(t *testing.T) {if got := docEngine.updateValues[0]["available_int"]; got != 0 {t.Fatalf("available_int = %#v, want 0", got)}+ if docEngine.search == nil || docEngine.search.IncludeUnavailable {+ t.Fatalf("availability search = %#v, must not include hidden parents", docEngine.search)+ }}func TestClearDocumentParseResultsIsIdempotentForStaleDocSnapshot(t *testing.T) {@@ -3405,6 +3416,43 @@ func TestUpdateDatasetDocumentParseTypePipelineIgnoresDirtyParserID(t *testing.T}}+func TestUpdateDatasetDocumentParentChildConfigSurvivesDSLFailure(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)+ insertNamedTestDoc(t, "doc-1", "kb-1", "doc.txt", 10, 5)+ if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("parser_config", entity.JSONMap{+ "GeneralChunker:SixApplesFall": map[string]any{+ "children_delimiters": []any{},+ },+ }).Error; err != nil {+ t.Fatalf("seed document parser config: %v", err)+ }++ parseType := 2+ pipelineID := "1234567890abcdef1234567890abcdef"+ resp, code, err := testDocumentService(t).UpdateDatasetDocument(t.Context(), "tenant-1", "kb-1", "doc-1", &UpdateDatasetDocumentRequest{+ ParseType: &parseType,+ PipelineID: &pipelineID,+ ParserConfig: map[string]any{+ "parent_child": map[string]any{+ "use_parent_child": true,+ "children_delimiter": "|",+ },+ },+ }, map[string]bool{"pipeline_id": true, "parse_type": true, "parser_config": true})+ if err != nil || code != common.CodeSuccess {+ t.Fatalf("UpdateDatasetDocument err=%v code=%d", err, code)+ }+ chunker, ok := resp.ParserConfig["GeneralChunker:SixApplesFall"].(map[string]interface{})+ if !ok {+ t.Fatalf("general chunker params = %#v", resp.ParserConfig["GeneralChunker:SixApplesFall"])+ }+ if got, ok := chunker["children_delimiters"].([]interface{}); !ok || len(got) != 1 || got[0] != "|" {+ t.Fatalf("children_delimiters = %#v, want [|]", chunker["children_delimiters"])+ }+}+// TestUpdateDatasetDocumentRejectsInvalidPages verifies the fail-fast contract// for the "pages" range: an invalid range (from<1) aborts the request with// CodeDataError instead of being silently dropped or persisted.@@ -3437,6 +3485,36 @@ func TestUpdateDatasetDocumentRejectsInvalidPages(t *testing.T) {}}+func TestUpdateDatasetDocumentParentChildConfigReachesGeneralChunker(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)+ insertNamedTestDoc(t, "doc-1", "kb-1", "doc.txt", 10, 5)++ resp, code, err := testDocumentService(t).UpdateDatasetDocument(t.Context(), "tenant-1", "kb-1", "doc-1", &UpdateDatasetDocumentRequest{+ ParserConfig: map[string]any{+ "parent_child": map[string]any{+ "use_parent_child": true,+ "children_delimiter": "|",+ },+ },+ }, map[string]bool{"parser_config": true})+ if err != nil || code != common.CodeSuccess {+ t.Fatalf("UpdateDatasetDocument err=%v code=%d", err, code)+ }+ chunker, ok := resp.ParserConfig["GeneralChunker:SixApplesFall"].(map[string]interface{})+ if !ok {+ t.Fatalf("general chunker params = %#v", resp.ParserConfig["GeneralChunker:SixApplesFall"])+ }+ if got, ok := chunker["children_delimiters"].([]interface{}); !ok || len(got) != 1 || got[0] != "|" {+ t.Fatalf("children_delimiters = %#v, want [|]", chunker["children_delimiters"])+ }+ parentChild, ok := resp.ParserConfig["parent_child"].(map[string]interface{})+ if !ok || parentChild["use_parent_child"] != true || parentChild["children_delimiter"] != "|" {+ t.Fatalf("parent_child = %#v, want persisted public setting", resp.ParserConfig["parent_child"])+ }+}+func TestUpdateDatasetDocumentEnabledUpdatesStatus(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)diff --git a/internal/service/document/wiki_refresh.go b/internal/service/document/wiki_refresh.goindex af320f7a38..a1fa55f6c9 100644--- a/internal/service/document/wiki_refresh.go+++ b/internal/service/document/wiki_refresh.go@@ -102,17 +102,18 @@ func (s *DocumentService) loadAvailabilityToggleChunkIDs(ctx context.Context, inreturn ids, nil}-func (s *DocumentService) loadSourceChunkIDs(ctx context.Context, indexName, datasetID, documentID string) ([]string, error) {+func (s *DocumentService) loadSourceChunkIDs(ctx context.Context, indexName, datasetID, documentID string, includeUnavailable bool) ([]string, error) {ids := make([]string, 0)for offset := 0; ; offset += sourceChunkAvailabilityBatchSize {searchCtx, cancel := context.WithTimeout(ctx, cleanupBatchTimeout)result, err := s.docEngine.Search(searchCtx, &enginetypes.SearchRequest{- IndexNames: []string{indexName},- KbIDs: []string{datasetID},- Offset: offset,- Limit: sourceChunkAvailabilityBatchSize,- SelectFields: []string{"id", "compile_kwd"},- Filter: map[string]any{"doc_id": []string{documentID}},+ IndexNames: []string{indexName},+ KbIDs: []string{datasetID},+ Offset: offset,+ Limit: sourceChunkAvailabilityBatchSize,+ SelectFields: []string{"id", "compile_kwd"},+ Filter: map[string]any{"doc_id": []string{documentID}},+ IncludeUnavailable: includeUnavailable,})cancel()if err != nil {@@ -141,7 +142,10 @@ func (s *DocumentService) deleteSourceChunks(ctx context.Context, tenantID, datareturn nil}indexName := fmt.Sprintf("ragflow_%s", tenantID)- ids, err := s.loadSourceChunkIDs(ctx, indexName, datasetID, documentID)+ // Reparse is the migration boundary for parent-child IDs. Include hidden+ // parent rows so old hash(mom) rows are removed with their children before+ // new document-scoped rows are written.+ ids, err := s.loadSourceChunkIDs(ctx, indexName, datasetID, documentID, true)if err != nil || len(ids) == 0 {return err}diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.goindex eecb202f04..58a4080bf1 100644--- a/internal/service/nlp/retrieval.go+++ b/internal/service/nlp/retrieval.go@@ -1034,12 +1034,18 @@ func RetrievalByChildren(chunks []map[string]interface{}, tenantIDs []string, doreturn chunks}- // Group child chunks by mom_id+ // Group child chunks by parent identity and document. The document scope is+ // required both for new parent IDs and for safe fallback when reading legacy+ // rows that used hash(mom) without document scope.type childChunk struct {chunk map[string]interface{}+ }+ type parentKey struct {+ momID string+ docID stringkbID string}- momChunks := make(map[string][]childChunk)+ momChunks := make(map[parentKey][]childChunk)remainingChunks := make([]map[string]interface{}, 0, len(chunks))for _, ck := range chunks {@@ -1049,7 +1055,13 @@ func RetrievalByChildren(chunks []map[string]interface{}, tenantIDs []string, docontinue}kbID, _ := ck["kb_id"].(string)- momChunks[momID] = append(momChunks[momID], childChunk{chunk: ck, kbID: kbID})+ docID, _ := ck["doc_id"].(string)+ if docID == "" {+ remainingChunks = append(remainingChunks, ck)+ continue+ }+ key := parentKey{momID: momID, docID: docID, kbID: kbID}+ momChunks[key] = append(momChunks[key], childChunk{chunk: ck})}if len(momChunks) == 0 {@@ -1059,26 +1071,32 @@ func RetrievalByChildren(chunks []map[string]interface{}, tenantIDs []string, do// Fetch parent chunks and aggregatevectorSize := 1024- for momID, childList := range momChunks {- kbIDs := make([]string, 0, len(childList))- for _, c := range childList {- if c.kbID != "" {- kbIDs = append(kbIDs, c.kbID)- }- }- if len(kbIDs) == 0 {- kbIDs = append(kbIDs, "")- }-- parent, err := docEngine.GetChunk(ctx, indexNames[0], momID, kbIDs)+ for key, childList := range momChunks {+ momID := key.momID+ parentResult, err := docEngine.Search(ctx, &types.SearchRequest{+ IndexNames: []string{indexNames[0]},+ KbIDs: []string{key.kbID},+ Limit: 1,+ IncludeUnavailable: true,+ Filter: map[string]interface{}{+ "id": momID,+ "doc_id": key.docID,+ },+ })if err != nil {common.Warn("Failed to get parent chunk", zap.String("momID", momID), zap.Error(err))+ for _, child := range childList {+ remainingChunks = append(remainingChunks, child.chunk)+ }continue}- parentMap, ok := parent.(map[string]interface{})- if !ok {+ if parentResult == nil || len(parentResult.Chunks) == 0 {+ for _, child := range childList {+ remainingChunks = append(remainingChunks, child.chunk)+ }continue}+ parentMap := parentResult.Chunks[0]// Calculate average similaritysimBuf := make([]float64, 0, len(childList))diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.goindex dbcdd7418d..76263f9c5e 100644--- a/internal/service/nlp/retrieval_test.go+++ b/internal/service/nlp/retrieval_test.go@@ -2,6 +2,7 @@ package nlpimport ("context"+ "errors""fmt""maps""slices"@@ -242,6 +243,76 @@ func (e *retrievalCountEngine) FilterDocIdsByMetaPushdown(context.Context, *gormreturn nil}+type parentChunkMissingEngine struct{ engine.DocEngine }++func (parentChunkMissingEngine) Search(context.Context, *types.SearchRequest) (*types.SearchResult, error) {+ return nil, errors.New("parent chunk missing")+}++type parentChunkScopeEngine struct {+ engine.DocEngine+ parentSearch *types.SearchRequest+}++func (e *parentChunkScopeEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {+ e.parentSearch = req+ return &types.SearchResult{}, nil+}++func TestRetrievalByChildrenScopesParentLookupToChildDocument(t *testing.T) {+ engine := &parentChunkScopeEngine{}+ child := map[string]interface{}{+ "chunk_id": "child-a",+ "mom_id": "legacy-shared-parent-id",+ "doc_id": "doc-a",+ "kb_id": "kb-1",+ "content_with_weight": "matching child text",+ "similarity": 0.8,+ }++ got := RetrievalByChildren([]map[string]interface{}{child}, []string{"tenant-1"}, engine, t.Context())++ if len(got) != 1 || got[0]["chunk_id"] != "child-a" {+ t.Fatalf("retrieval result = %#v, want fallback child", got)+ }+ if engine.parentSearch == nil {+ t.Fatal("parent lookup did not use a document-scoped search")+ }+ if engine.parentSearch.Filter["doc_id"] != "doc-a" {+ t.Fatalf("parent lookup filter = %#v, want doc_id doc-a", engine.parentSearch.Filter)+ }+ if !engine.parentSearch.IncludeUnavailable {+ t.Fatal("parent lookup must include hidden parent chunks")+ }+}++// TestRetrievalByChildrenKeepsChildWhenParentIsMissing verifies a partial+// parent-child write never turns a relevant child hit into an empty result.+// Removing the fallback is a retrieval data-loss bug.+func TestRetrievalByChildrenKeepsChildWhenParentIsMissing(t *testing.T) {+ child := map[string]interface{}{+ "chunk_id": "child-1",+ "mom_id": "parent-1",+ "doc_id": "doc-1",+ "kb_id": "kb-1",+ "content_with_weight": "matching child text",+ "similarity": 0.8,+ }++ got := RetrievalByChildren(+ []map[string]interface{}{child},+ []string{"tenant-1"},+ parentChunkMissingEngine{},+ t.Context(),+ )+ if len(got) != 1 {+ t.Fatalf("retrieval result count = %d, want fallback child", len(got))+ }+ if got[0]["chunk_id"] != "child-1" {+ t.Fatalf("fallback chunk_id = %#v, want child-1", got[0]["chunk_id"])+ }+}+func TestBuildInfinityFusionExprUsesVectorSimilarityWeight(t *testing.T) {tests := []struct {name string