Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(retrieval): align Go parent-child chunk behavior (#19828)

jay777218afebbb67069
diff --git a/internal/engine/elasticsearch/chunk_readback_test.go b/internal/engine/elasticsearch/chunk_readback_test.go
index 1dc70a0ffe..6c52510c02 100644
--- a/internal/engine/elasticsearch/chunk_readback_test.go
+++ b/internal/engine/elasticsearch/chunk_readback_test.go
@@ -84,6 +84,8 @@ func TestInsertChunks_WritesIngestionShape(t *testing.T) {
"important_kwd": []string{"k1"},
"page_num_int": int(1),
"position_int": int(2),
+ "mom_id": "parent-1",
+ "available_int": int(0),
}
if _, err := engine.InsertChunks(t.Context(), []map[string]interface{}{chunk}, "ragflow_chunk_readback_test", "kb-1"); err != nil {
@@ -106,6 +108,8 @@ func TestInsertChunks_WritesIngestionShape(t *testing.T) {
assertStoredField(t, doc, "important_kwd", []interface{}{"k1"})
assertStoredField(t, doc, "page_num_int", float64(1))
assertStoredField(t, doc, "position_int", float64(2))
+ assertStoredField(t, doc, "mom_id", "parent-1")
+ assertStoredField(t, doc, "available_int", float64(0))
// InsertChunks overrides kb_id with datasetID on write.
assertStoredField(t, doc, "kb_id", "kb-1")
}
diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.go
index 18f66e3618..ffd4865504 100644
--- a/internal/engine/infinity/chunk.go
+++ b/internal/engine/infinity/chunk.go
@@ -848,10 +848,10 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S
filterParts = append(filterParts, fmt.Sprintf("available_int=%v", availInt))
} else if status, ok := req.Filter["status"]; ok {
filterParts = append(filterParts, fmt.Sprintf("status='%s'", status))
- } else if !isSkillIndex {
+ } else if shouldDefaultAvailableFilter(req, isSkillIndex) {
filterParts = append(filterParts, "available_int=1")
}
- } else if !isSkillIndex {
+ } else if shouldDefaultAvailableFilter(req, isSkillIndex) {
filterParts = append(filterParts, "available_int=1")
}
}
@@ -1042,7 +1042,7 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S
}
denseFilterStr := filterStr
- if denseFilterStr == "" && !isSkillIndex {
+ if denseFilterStr == "" && shouldDefaultAvailableFilter(req, isSkillIndex) {
denseFilterStr = "available_int=1"
}
@@ -1224,6 +1224,10 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S
}, nil
}
+func shouldDefaultAvailableFilter(req *types.SearchRequest, isSkillIndex bool) bool {
+ return !isSkillIndex && !req.IncludeUnavailable
+}
+
// GetChunk gets a chunk by ID
func (e *Engine) GetChunk(ctx context.Context, tableName, chunkID string, datasetIDs []string) (interface{}, error) {
if e.client == nil || e.client.pool == nil {
diff --git a/internal/engine/infinity/chunk_filter_test.go b/internal/engine/infinity/chunk_filter_test.go
new file mode 100644
index 0000000000..84c6716d8c
--- /dev/null
+++ b/internal/engine/infinity/chunk_filter_test.go
@@ -0,0 +1,19 @@
+package infinity
+
+import (
+ "testing"
+
+ "ragflow/internal/engine/types"
+)
+
+func TestShouldDefaultAvailableFilter(t *testing.T) {
+ if !shouldDefaultAvailableFilter(&types.SearchRequest{}, false) {
+ t.Fatal("ordinary retrieval must default to available_int=1")
+ }
+ if shouldDefaultAvailableFilter(&types.SearchRequest{IncludeUnavailable: true}, false) {
+ t.Fatal("management list must not add the retrieval availability default")
+ }
+ if shouldDefaultAvailableFilter(&types.SearchRequest{}, true) {
+ t.Fatal("skill indexes do not use chunk availability")
+ }
+}
diff --git a/internal/engine/infinity/chunk_transform_test.go b/internal/engine/infinity/chunk_transform_test.go
index 5cab6154a3..9c92b573b5 100644
--- a/internal/engine/infinity/chunk_transform_test.go
+++ b/internal/engine/infinity/chunk_transform_test.go
@@ -43,6 +43,8 @@ func TestTransformChunkFields_IngestionShape(t *testing.T) {
"important_kwd": []interface{}{"k1"},
"page_num_int": int(1),
"position_int": int(2),
+ "mom_id": "parent-1",
+ "available_int": int(0),
}
got := transformChunkFields(chunk, nil)
@@ -58,6 +60,8 @@ func TestTransformChunkFields_IngestionShape(t *testing.T) {
"important_keywords": "k1",
"page_num_int": int(1),
"position_int": int(2),
+ "mom_id": "parent-1",
+ "available_int": int(0),
}
for k, wv := range want {
diff --git a/internal/engine/oceanbase/compatibility_test.go b/internal/engine/oceanbase/compatibility_test.go
index ce660853c5..d600da2794 100644
--- a/internal/engine/oceanbase/compatibility_test.go
+++ b/internal/engine/oceanbase/compatibility_test.go
@@ -133,7 +133,7 @@ func TestPythonPhysicalTableNameContract(t *testing.T) {
func TestLegacyChunkEncodingContract(t *testing.T) {
document := map[string]interface{}{
"id": "chunk-1", "kb_id": []string{"kb-1"}, "doc_id": "doc-1",
- "available_int": nil, "removed_kwd": nil, "chunk_order_int": 7,
+ "available_int": 0, "removed_kwd": nil, "chunk_order_int": 7, "mom_id": "parent-1",
"metadata": map[string]interface{}{"_group_id": "group-1", "_title": "renamed"},
"important_kwd": []string{" alpha\t", "beta\n"},
"q_3_vec": []float64{0.1, 0.2, 0.3},
@@ -143,7 +143,7 @@ func TestLegacyChunkEncodingContract(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if got["kb_id"] != "kb-1" || got["available_int"] != 1 || got["removed_kwd"] != "N" || got["_order_id"] != 7 {
+ if got["kb_id"] != "kb-1" || got["available_int"] != 0 || got["removed_kwd"] != "N" || got["_order_id"] != 7 || got["mom_id"] != "parent-1" {
t.Fatalf("legacy scalar/default encoding changed: %#v", got)
}
if got["group_id"] != "group-1" || got["docnm_kwd"] != "renamed" {
diff --git a/internal/engine/serenedb/search.go b/internal/engine/serenedb/search.go
index e032a2b941..d3c57e445e 100644
--- a/internal/engine/serenedb/search.go
+++ b/internal/engine/serenedb/search.go
@@ -59,7 +59,7 @@ func (e *serenedbEngine) Search(ctx context.Context, req *types.SearchRequest) (
outputFields := resolveOutputFields(req.SelectFields)
fieldsExpr := strings.Join(outputFields, ", ")
- filters := searchFilters(req.Filter, req.KbIDs, pm.hasText || pm.hasVector)
+ filters := searchFilters(req.Filter, req.KbIDs, pm.hasText || pm.hasVector, req.IncludeUnavailable)
where := filtersExpr(filters)
offset := req.Offset
@@ -155,9 +155,10 @@ func resolveOutputFields(selectFields []string) []string {
}
// searchFilters builds the SQL predicates for a search. kb_id scopes the query
-// to the requested datasets within the shared tenant table, and scored queries
-// default to available_int=1 when the caller did not set it.
-func searchFilters(filter map[string]interface{}, kbIDs []string, scored bool) []string {
+// to the requested datasets within the shared tenant table. Scored retrieval
+// defaults to available_int=1, except when a management caller explicitly
+// requests disabled chunks as well.
+func searchFilters(filter map[string]interface{}, kbIDs []string, scored, includeUnavailable bool) []string {
cond := map[string]interface{}{}
for k, v := range filter {
cond[k] = v
@@ -165,7 +166,7 @@ func searchFilters(filter map[string]interface{}, kbIDs []string, scored bool) [
if kbs := stringSlice(kbIDs); len(kbs) > 0 {
cond["kb_id"] = kbs
}
- if scored {
+ if scored && !includeUnavailable {
_, hasAvail := cond["available_int"]
_, hasStatus := cond["status"]
if !hasAvail && !hasStatus {
diff --git a/internal/engine/serenedb/serenedb_test.go b/internal/engine/serenedb/serenedb_test.go
index 2f81356639..e18e0e9cba 100644
--- a/internal/engine/serenedb/serenedb_test.go
+++ b/internal/engine/serenedb/serenedb_test.go
@@ -118,6 +118,24 @@ func TestL2NormalizeUnitLength(t *testing.T) {
}
}
+func TestPrepareChunkRowKeepsHiddenParentFields(t *testing.T) {
+ cols, values := prepareChunkRow(map[string]interface{}{
+ "id": "parent-1",
+ "doc_id": "doc-1",
+ "content_with_weight": "parent text",
+ "mom_id": "parent-1",
+ "available_int": 0,
+ }, "kb-1")
+
+ row := make(map[string]interface{}, len(cols))
+ for i, col := range cols {
+ row[col] = values[i]
+ }
+ if row["mom_id"] != "parent-1" || row["available_int"] != 0 || row["kb_id"] != "kb-1" {
+ t.Fatalf("parent row = %#v, want hidden and dataset-scoped", row)
+ }
+}
+
func TestVectorLiteralTyped(t *testing.T) {
got := vectorLiteral([]float64{3, 4})
mustContain(t, got, "ARRAY[0.6,0.8]::FLOAT[2]")
@@ -260,23 +278,27 @@ func TestResolveOutputFields(t *testing.T) {
func TestSearchFiltersScoping(t *testing.T) {
// KbIDs become an IN predicate over the shared tenant table.
- scored := searchFilters(map[string]interface{}{}, []string{"kb1", "kb2"}, true)
+ scored := searchFilters(map[string]interface{}{}, []string{"kb1", "kb2"}, true, false)
joined := strings.Join(scored, " AND ")
mustContain(t, joined, "kb_id IN ('kb1', 'kb2')")
mustContain(t, joined, "available_int = 1")
// No KbIDs and no match expr -> no predicates.
- if got := searchFilters(map[string]interface{}{}, nil, false); len(got) != 0 {
+ if got := searchFilters(map[string]interface{}{}, nil, false, false); len(got) != 0 {
t.Errorf("expected no filters, got %v", got)
}
// A scored query with no available_int/status defaults available_int=1.
- one := searchFilters(map[string]interface{}{}, nil, true)
+ one := searchFilters(map[string]interface{}{}, nil, true, false)
if len(one) != 1 || one[0] != "available_int = 1" {
t.Errorf("scored default = %v", one)
}
// Blank dataset ids are dropped, so no empty IN () is emitted.
- if got := searchFilters(map[string]interface{}{}, []string{""}, false); len(got) != 0 {
+ if got := searchFilters(map[string]interface{}{}, []string{""}, false, false); len(got) != 0 {
t.Errorf("blank kb ids should yield no filter, got %v", got)
}
+ // Management listing explicitly opts into disabled parent chunks.
+ if got := searchFilters(map[string]interface{}{}, nil, true, true); len(got) != 0 {
+ t.Errorf("include unavailable filters = %v, want no availability default", got)
+ }
}
func TestFusionVectorWeight(t *testing.T) {
diff --git a/internal/engine/types/types.go b/internal/engine/types/types.go
index ec6c53d493..f85f4565f8 100644
--- a/internal/engine/types/types.go
+++ b/internal/engine/types/types.go
@@ -44,7 +44,8 @@ type SearchRequest struct {
SelectFields []string // List of field names to return
// Filtering
- Filter map[string]interface{} // Filters for search
+ Filter map[string]interface{} // Filters for search
+ IncludeUnavailable bool // Do not add the retrieval-only available_int=1 default
// Match expressions
MatchExprs []interface{} // List of match expressions: [matchText, matchDense, fusionExpr]
diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.go
index 9a4ec49172..05e813c1a7 100644
--- a/internal/ingestion/pipeline/pipeline_params.go
+++ b/internal/ingestion/pipeline/pipeline_params.go
@@ -287,6 +287,43 @@ func BuildParserConfig(dslJSON []byte, rawConfig map[string]interface{}) entity.
return result
}
+// ApplyParentChildChunkerConfig derives runtime children_delimiters from the
+// top-level parent_child setting. parent_child is the sole public source of
+// truth; chunker fields are generated runtime parameters and must not be
+// accepted as an independent dataset setting.
+func ApplyParentChildChunkerConfig(componentConfig entity.JSONMap, rawConfig map[string]interface{}) {
+ parentChild, ok := rawConfig["parent_child"].(map[string]interface{})
+ if !ok {
+ return
+ }
+ componentConfig["parent_child"] = parentChild
+ useParentChild, _ := parentChild["use_parent_child"].(bool)
+ childrenDelimiters := []string{}
+ if useParentChild {
+ if delimiter, ok := parentChild["children_delimiter"].(string); ok {
+ childrenDelimiters = parserchunk.ParseDelimiterField(delimiter)
+ }
+ }
+
+ for componentID, value := range componentConfig {
+ if !IsChunkerComponent(componentID) {
+ continue
+ }
+ params, ok := value.(map[string]interface{})
+ if !ok {
+ continue
+ }
+ params["children_delimiters"] = childrenDelimiters
+ }
+}
+
+// IsChunkerComponent reports whether a pipeline component can split parent
+// chunks into children.
+func IsChunkerComponent(componentID string) bool {
+ lowerID := strings.ToLower(componentID)
+ return strings.HasPrefix(lowerID, "generalchunker:") || strings.HasPrefix(lowerID, "tokenchunker:")
+}
+
// ResolveComponentParamsDefaults takes DSL JSON bytes and returns the
// component params defaults as an entity.JSONMap {cpnID: {param: value}}.
// This is a pure function — callers must load the DSL themselves.
diff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.go
index 08dfff7867..4d49d57bcc 100644
--- a/internal/ingestion/pipeline/pipeline_params_test.go
+++ b/internal/ingestion/pipeline/pipeline_params_test.go
@@ -436,6 +436,32 @@ func TestBuildParserConfig_BuiltinExtractorKeepsBuiltInMetadata(t *testing.T) {
}
}
+func TestApplyParentChildChunkerConfig(t *testing.T) {
+ config := entity.JSONMap{
+ "GeneralChunker:one": map[string]interface{}{"chunk_token_num": 256, "children_delimiters": []string{"wrong"}},
+ "TokenChunker:two": map[string]interface{}{"chunk_token_num": 128, "children_delimiters": []string{"wrong"}},
+ "Extractor:three": map[string]interface{}{"llm_id": "llm-1"},
+ }
+
+ ApplyParentChildChunkerConfig(config, map[string]interface{}{
+ "parent_child": map[string]interface{}{
+ "use_parent_child": true,
+ "children_delimiter": "|",
+ },
+ })
+
+ for _, componentID := range []string{"GeneralChunker:one", "TokenChunker:two"} {
+ params := config[componentID].(map[string]interface{})
+ got, ok := params["children_delimiters"].([]string)
+ if !ok || len(got) != 1 || got[0] != "|" {
+ t.Fatalf("%s children_delimiters = %#v, want [|]", componentID, params["children_delimiters"])
+ }
+ }
+ if _, ok := config["Extractor:three"].(map[string]interface{})["children_delimiters"]; ok {
+ t.Fatal("non-chunker component was modified")
+ }
+}
+
func TestCleanComponentParams_KeepsModularExtractorParams(t *testing.T) {
// Builtin template with empty Extractor params: {}
dsl := map[string]any{
diff --git a/internal/ingestion/task/chunk_index_writer.go b/internal/ingestion/task/chunk_index_writer.go
index f5ce30c372..bfdffab918 100644
--- a/internal/ingestion/task/chunk_index_writer.go
+++ b/internal/ingestion/task/chunk_index_writer.go
@@ -16,7 +16,20 @@
package task
-import "context"
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "ragflow/internal/common"
+
+ "go.uber.org/zap"
+)
+
+const (
+ chunkInsertAttempts = 3
+ chunkInsertRetryBaseDelay = 100 * time.Millisecond
+)
// InsertFunc is the signature of the chunk insertion backend (e.g. engine.InsertChunks).
type InsertFunc func(ctx context.Context, chunks []map[string]any, baseName, datasetID string) ([]string, error)
@@ -65,8 +78,38 @@ func (w *chunkIndexWriter) Write(ctx context.Context, chunks []map[string]any) e
if err := ctx.Err(); err != nil {
return err
}
- if _, err := w.insertFunc(ctx, chunks[b:end], w.baseName, w.datasetID); err != nil {
- return err
+ var err error
+ for attempt := 1; attempt <= chunkInsertAttempts; attempt++ {
+ _, err = w.insertFunc(ctx, chunks[b:end], w.baseName, w.datasetID)
+ if err == nil {
+ break
+ }
+ if ctxErr := ctx.Err(); ctxErr != nil {
+ return ctxErr
+ }
+ if attempt == chunkInsertAttempts {
+ continue
+ }
+
+ delay := chunkInsertRetryBaseDelay << (attempt - 1)
+ common.Warn("retrying chunk index write",
+ zap.Int("batch_start", b),
+ zap.Int("batch_end", end),
+ zap.Int("attempt", attempt+1),
+ zap.Int("max_attempts", chunkInsertAttempts),
+ zap.Duration("delay", delay),
+ zap.Error(err),
+ )
+ timer := time.NewTimer(delay)
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+ return ctx.Err()
+ case <-timer.C:
+ }
+ }
+ if err != nil {
+ return fmt.Errorf("insert chunk batch %d-%d after %d attempts: %w", b, end, chunkInsertAttempts, err)
}
}
return nil
diff --git a/internal/ingestion/task/chunk_index_writer_test.go b/internal/ingestion/task/chunk_index_writer_test.go
index a4f5ce41b7..d3c05f485b 100644
--- a/internal/ingestion/task/chunk_index_writer_test.go
+++ b/internal/ingestion/task/chunk_index_writer_test.go
@@ -18,7 +18,9 @@ package task
import (
"context"
+ "errors"
"testing"
+ "time"
)
func TestChunkIndexWriter_EmptyChunks(t *testing.T) {
@@ -114,3 +116,42 @@ func TestChunkIndexWriter_BulkSizeZero(t *testing.T) {
t.Fatalf("batch size = %d, want 20 (bulkSize=0 should degrade to len(chunks))", lastBatchSize)
}
}
+
+func TestChunkIndexWriterRetriesFailedBatch(t *testing.T) {
+ attempts := 0
+ writer := newChunkIndexWriter(func(_ context.Context, _ []map[string]any, _, _ string) ([]string, error) {
+ attempts++
+ if attempts == 1 {
+ return nil, errors.New("temporary parent write failure")
+ }
+ return nil, nil
+ }, "ragflow_tenant", "kb", 0)
+
+ if err := writer.Write(t.Context(), []map[string]any{{"id": "child"}, {"id": "parent", "available_int": 0}}); err != nil {
+ t.Fatalf("Write() error = %v", err)
+ }
+ if attempts != 2 {
+ t.Fatalf("insert attempts = %d, want 2", attempts)
+ }
+}
+
+func TestChunkIndexWriterWaitsBeforeRetryingFailedBatch(t *testing.T) {
+ attempts := make([]time.Time, 0, 2)
+ writer := newChunkIndexWriter(func(_ context.Context, _ []map[string]any, _, _ string) ([]string, error) {
+ attempts = append(attempts, time.Now())
+ if len(attempts) == 1 {
+ return nil, errors.New("temporary write failure")
+ }
+ return nil, nil
+ }, "ragflow_tenant", "kb", 0)
+
+ if err := writer.Write(t.Context(), []map[string]any{{"id": "chunk"}}); err != nil {
+ t.Fatalf("Write() error = %v", err)
+ }
+ if len(attempts) != 2 {
+ t.Fatalf("insert attempts = %d, want 2", len(attempts))
+ }
+ if delay := attempts[1].Sub(attempts[0]); delay < 50*time.Millisecond {
+ t.Fatalf("retry delay = %s, want at least 50ms", delay)
+ }
+}
diff --git a/internal/ingestion/task/indexdoc/parent_child.go b/internal/ingestion/task/indexdoc/parent_child.go
new file mode 100644
index 0000000000..67d97a40c9
--- /dev/null
+++ b/internal/ingestion/task/indexdoc/parent_child.go
@@ -0,0 +1,75 @@
+package indexdoc
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/cespare/xxhash/v2"
+)
+
+// MaterializeParentChunks turns the chunker's transient parent text into
+// hidden index rows and links every child row to its parent. A parent ID is
+// scoped to its dataset and document so equal parent text from separate
+// documents or datasets cannot overwrite each other in a shared index.
+func MaterializeParentChunks(datasetID string, chunks []map[string]any) []map[string]any {
+ seen := make(map[string]struct{})
+ parents := make([]map[string]any, 0)
+
+ for _, chunk := range chunks {
+ mom := parentText(chunk)
+ delete(chunk, "mom")
+ delete(chunk, "mom_with_weight")
+ if mom == "" {
+ continue
+ }
+
+ docID, _ := chunk["doc_id"].(string)
+ if docID == "" {
+ continue
+ }
+ parentID := parentChunkID(datasetID, docID, mom)
+ chunk["mom_id"] = parentID
+ if _, exists := seen[parentID]; exists {
+ continue
+ }
+
+ parent := map[string]any{
+ "id": parentID,
+ "content_with_weight": mom,
+ "available_int": 0,
+ }
+ for _, field := range []string{
+ "doc_id", "docnm_kwd", "position_int", "create_timestamp_flt",
+ "page_num_int", "top_int",
+ } {
+ if value, exists := chunk[field]; exists {
+ parent[field] = value
+ }
+ }
+ seen[parentID] = struct{}{}
+ parents = append(parents, parent)
+ }
+ return parents
+}
+
+// parentChunkID returns the deterministic parent row identifier. NUL
+// separators make the dataset, document, and parent-text boundaries
+// unambiguous.
+func parentChunkID(datasetID, docID, mom string) string {
+ hasher := xxhash.New()
+ _, _ = hasher.WriteString(datasetID)
+ _, _ = hasher.Write([]byte{0})
+ _, _ = hasher.WriteString(docID)
+ _, _ = hasher.Write([]byte{0})
+ _, _ = hasher.WriteString(mom)
+ return fmt.Sprintf("%016x", hasher.Sum64())
+}
+
+func parentText(chunk map[string]any) string {
+ for _, field := range []string{"mom", "mom_with_weight"} {
+ if value, ok := chunk[field].(string); ok && strings.TrimSpace(value) != "" {
+ return value
+ }
+ }
+ return ""
+}
diff --git a/internal/ingestion/task/indexdoc/parent_child_test.go b/internal/ingestion/task/indexdoc/parent_child_test.go
new file mode 100644
index 0000000000..df56f7877a
--- /dev/null
+++ b/internal/ingestion/task/indexdoc/parent_child_test.go
@@ -0,0 +1,31 @@
+package indexdoc
+
+import "testing"
+
+func TestParentChunkIDScopesDatasetAndDocument(t *testing.T) {
+ if got, again := parentChunkID("kb-a", "doc-a", "same parent text"), parentChunkID("kb-a", "doc-a", "same parent text"); got != again {
+ t.Fatalf("parentChunkID is not deterministic: %q != %q", got, again)
+ }
+ if got, otherDataset := parentChunkID("kb-a", "doc-a", "same parent text"), parentChunkID("kb-b", "doc-a", "same parent text"); got == otherDataset {
+ t.Fatalf("parentChunkID collides across datasets: %q", got)
+ }
+}
+
+func TestMaterializeParentChunksScopesSameParentTextToDocument(t *testing.T) {
+ chunks := []map[string]any{
+ {"id": "child-a", "doc_id": "doc-a", "mom": "shared parent"},
+ {"id": "child-b", "doc_id": "doc-b", "mom": "shared parent"},
+ }
+
+ parents := MaterializeParentChunks("kb-a", chunks)
+
+ if len(parents) != 2 {
+ t.Fatalf("parent count = %d, want 2", len(parents))
+ }
+ if chunks[0]["mom_id"] == chunks[1]["mom_id"] {
+ t.Fatalf("mom_id collision: %q", chunks[0]["mom_id"])
+ }
+ if parents[0]["doc_id"] == parents[1]["doc_id"] {
+ t.Fatalf("parent docs = %#v, want one parent per document", parents)
+ }
+}
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index d7ff66805b..bfc224d4ce 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -69,10 +69,11 @@ type PipelineExecutor struct {
canvasID string
docBulkSize int
- indexWriter *chunkIndexWriter
- loadDSLFunc func(ctx context.Context, canvasID string) (string, string, error)
- runPipelineFunc func(ctx context.Context, dsl string) (map[string]any, string, error)
- progressSink pipelinepkg.ProgressSink
+ indexWriter *chunkIndexWriter
+ deleteChunksFunc DeleteChunksFunc
+ loadDSLFunc func(ctx context.Context, canvasID string) (string, string, error)
+ runPipelineFunc func(ctx context.Context, dsl string) (map[string]any, string, error)
+ progressSink pipelinepkg.ProgressSink
}
func validateTaskContext(taskCtx *TaskContext) error {
@@ -138,17 +139,29 @@ func NewPipelineExecutor(
canvasID: canvasID,
docBulkSize: docBulkSize,
indexWriter: newChunkIndexWriter(insertChunksForIngestion(engine.Get()), fmt.Sprintf("ragflow_%s", taskCtx.Tenant.ID), taskCtx.Doc.KbID, docBulkSize),
+ deleteChunksFunc: func(ctx context.Context, condition map[string]any, baseName, datasetID string) (int64, error) {
+ return engine.Get().DeleteChunks(ctx, condition, baseName, datasetID)
+ },
}
svc.loadDSLFunc = svc.loadDSLFromCanvas
svc.runPipelineFunc = svc.runPipelineWithDSL
return svc, nil
}
+// DeleteChunksFunc removes partially written rows after a failed index write.
+type DeleteChunksFunc func(ctx context.Context, condition map[string]any, baseName, datasetID string) (int64, error)
+
func (s *PipelineExecutor) WithInsertFunc(f InsertFunc) *PipelineExecutor {
s.indexWriter.insertFunc = f
return s
}
+// WithDeleteChunksFunc replaces index-write compensation. It is used by the
+// in-memory pipeline tests; production uses the configured document engine.
+func (s *PipelineExecutor) WithDeleteChunksFunc(f DeleteChunksFunc) *PipelineExecutor {
+ s.deleteChunksFunc = f
+ return s
+}
func (s *PipelineExecutor) WithLoadDSLFunc(f func(ctx context.Context, canvasID string) (string, string, error)) *PipelineExecutor {
s.loadDSLFunc = f
return s
@@ -258,6 +271,7 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
if err != nil {
return nil, err
}
+ parentChunks := indexdoc.MaterializeParentChunks(s.taskCtx.Doc.KbID, chunks)
tableMeta := indexdoc.AggregateTableDocMetadata(chunks, map[string]interface{}(s.taskCtx.Doc.ParserConfig))
if tableMeta != nil {
@@ -295,8 +309,12 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
if err != nil {
return nil, err
}
- if err := s.indexWriter.Write(ctx, chunks); err != nil {
- return nil, err
+ indexChunks := append(chunks, parentChunks...)
+ if err := s.indexWriter.Write(ctx, indexChunks); err != nil {
+ if cleanupErr := s.compensateFailedIndexWrite(ctx, indexChunks); cleanupErr != nil {
+ return nil, fmt.Errorf("write chunks: %w; compensate partial index write: %v", err, cleanupErr)
+ }
+ return nil, fmt.Errorf("write chunks: %w", err)
}
if err := s.reconcileDocumentCompiledProducts(ctx, oldCompiledProductIDs, chunks); err != nil {
return nil, err
@@ -363,6 +381,37 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
}, nil
}
+func (s *PipelineExecutor) compensateFailedIndexWrite(ctx context.Context, chunks []map[string]any) error {
+ if s.deleteChunksFunc == nil {
+ return nil
+ }
+ ids := make([]string, 0, len(chunks))
+ seen := make(map[string]struct{}, len(chunks))
+ for _, chunk := range chunks {
+ id, _ := chunk["id"].(string)
+ if id == "" {
+ continue
+ }
+ if _, exists := seen[id]; exists {
+ continue
+ }
+ seen[id] = struct{}{}
+ ids = append(ids, id)
+ }
+ if len(ids) == 0 {
+ return nil
+ }
+ cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
+ defer cancel()
+ _, err := s.deleteChunksFunc(
+ cleanupCtx,
+ map[string]any{"id": ids, "kb_id": s.taskCtx.Doc.KbID},
+ s.indexWriter.baseName,
+ s.taskCtx.Doc.KbID,
+ )
+ return err
+}
+
// terminalDuration measures the run from the document's process_begin_at —
// the anchor PrepareValidatedRun stamped before the run and the one every
// mid-run progress-sink duration write uses — so the terminal value applied
diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.go
index 536fd2cc2a..4e6d6cf3f8 100644
--- a/internal/ingestion/task/pipeline_executor_test.go
+++ b/internal/ingestion/task/pipeline_executor_test.go
@@ -791,6 +791,112 @@ func TestRunPipeline_FullFlow(t *testing.T) {
}
}
+// TestPipelineExecutor_ProcessOutputMaterializesParentChunk verifies the
+// persistence boundary turns chunker-level mom text into one hidden parent row
+// and links every child to it. Removing that materialization would leave only
+// independently retrievable child rows, which is the parent-child regression.
+func TestPipelineExecutor_ProcessOutputMaterializesParentChunk(t *testing.T) {
+ var inserted []map[string]any
+ writeCalls := 0
+ svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0).WithInsertFunc(
+ func(_ context.Context, chunks []map[string]any, _, _ string) ([]string, error) {
+ writeCalls++
+ inserted = append(inserted, chunks...)
+ return nil, nil
+ },
+ )
+
+ _, err := svc.processOutput(t.Context(), map[string]any{
+ "chunks": []map[string]any{
+ {"text": "child one", "mom": "full parent passage"},
+ {"text": "child two", "mom": "full parent passage"},
+ },
+ }, time.Now())
+ if err != nil {
+ t.Fatalf("processOutput: %v", err)
+ }
+ if len(inserted) != 3 {
+ t.Fatalf("persisted rows = %d, want two children and one parent", len(inserted))
+ }
+ if writeCalls != 1 {
+ t.Fatalf("write calls = %d, want children and parents in one write", writeCalls)
+ }
+
+ var parent map[string]any
+ children := make([]map[string]any, 0, 2)
+ for _, row := range inserted {
+ if row["available_int"] == 0 && row["content_with_weight"] == "full parent passage" {
+ parent = row
+ continue
+ }
+ children = append(children, row)
+ }
+ if parent == nil {
+ t.Fatal("hidden parent row was not persisted")
+ }
+ parentID, _ := parent["id"].(string)
+ if parentID == "" {
+ t.Fatalf("parent id = %#v, want deterministic non-empty id", parent["id"])
+ }
+ for _, child := range children {
+ if child["mom_id"] != parentID {
+ t.Errorf("child mom_id = %#v, want parent id %q", child["mom_id"], parentID)
+ }
+ if _, exists := child["mom"]; exists {
+ t.Errorf("child retained pipeline-only mom field: %#v", child)
+ }
+ }
+}
+
+func TestPipelineExecutorCompensatesAfterExhaustingCombinedWriteRetries(t *testing.T) {
+ var deleted map[string]any
+ svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0).
+ WithInsertFunc(func(context.Context, []map[string]any, string, string) ([]string, error) {
+ return nil, errors.New("index unavailable")
+ }).
+ WithDeleteChunksFunc(func(_ context.Context, condition map[string]any, _, _ string) (int64, error) {
+ deleted = condition
+ return 0, nil
+ })
+
+ _, err := svc.processOutput(t.Context(), map[string]any{
+ "chunks": []map[string]any{{"text": "child", "mom": "parent"}},
+ }, time.Now())
+ if err == nil {
+ t.Fatal("processOutput succeeded after exhausted writes")
+ }
+ ids, ok := deleted["id"].([]string)
+ if !ok || len(ids) != 2 {
+ t.Fatalf("compensation ids = %#v, want child and parent IDs", deleted)
+ }
+}
+
+func TestPipelineExecutorBoundsCompensationCleanup(t *testing.T) {
+ var cleanupCtx context.Context
+ svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0).
+ WithInsertFunc(func(context.Context, []map[string]any, string, string) ([]string, error) {
+ return nil, errors.New("index unavailable")
+ }).
+ WithDeleteChunksFunc(func(ctx context.Context, _ map[string]any, _, _ string) (int64, error) {
+ cleanupCtx = ctx
+ return 0, nil
+ })
+
+ _, err := svc.processOutput(t.Context(), map[string]any{
+ "chunks": []map[string]any{{"text": "child", "mom": "parent"}},
+ }, time.Now())
+ if err == nil {
+ t.Fatal("processOutput succeeded after exhausted writes")
+ }
+ deadline, ok := cleanupCtx.Deadline()
+ if !ok {
+ t.Fatal("compensation cleanup context has no deadline")
+ }
+ if remaining := time.Until(deadline); remaining <= 0 || remaining > 5*time.Second {
+ t.Fatalf("compensation cleanup deadline remaining = %s, want (0, 5s]", remaining)
+ }
+}
+
func TestRunPipeline_AlreadyHasVectors(t *testing.T) {
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0).
WithInsertFunc(func(ctx context.Context, chunks []map[string]any, baseName, datasetID string) ([]string, error) {
diff --git a/internal/rag/agentic-rag/runtime/tool_executor_test.go b/internal/rag/agentic-rag/runtime/tool_executor_test.go
index f638ea2ae7..03a80da158 100644
--- a/internal/rag/agentic-rag/runtime/tool_executor_test.go
+++ b/internal/rag/agentic-rag/runtime/tool_executor_test.go
@@ -28,6 +28,7 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
"ragflow/internal/engine"
+ enginetypes "ragflow/internal/engine/types"
"gorm.io/gorm"
)
@@ -47,19 +48,21 @@ func (s stubRetrievalService) Search(_ context.Context, _ *gorm.DB, req runtime.
return s.chunks, nil
}
-// stubDocEngine implements engine.DocEngine by only serving GetChunk; the rest
-// of the (large) interface is promoted from a nil field and never called by
-// this test.
+// stubDocEngine serves the document-scoped Search used by retrieval_by_children;
+// the rest of the (large) interface is promoted from a nil field and never
+// called by this test.
type stubDocEngine struct {
engine.DocEngine
parents map[string]map[string]any
}
-func (s stubDocEngine) GetChunk(_ context.Context, _, chunkID string, _ []string) (interface{}, error) {
- if p, ok := s.parents[chunkID]; ok {
- return p, nil
+func (s stubDocEngine) Search(_ context.Context, req *enginetypes.SearchRequest) (*enginetypes.SearchResult, error) {
+ parentID, _ := req.Filter["id"].(string)
+ docID, _ := req.Filter["doc_id"].(string)
+ if p, ok := s.parents[parentID]; ok && p["doc_id"] == docID {
+ return &enginetypes.SearchResult{Chunks: []map[string]interface{}{p}}, nil
}
- return nil, fmt.Errorf("parent %s not found", chunkID)
+ return &enginetypes.SearchResult{}, nil
}
// TestRuntimeRetrieverPreservesUnsetControls pins the presence semantics of the
@@ -140,8 +143,8 @@ func TestChunkAggRetrievePropagatesError(t *testing.T) {
func TestRuntimeRetrieverPromotesChildrenToParent(t *testing.T) {
prev := runtime.GetRetrievalService()
runtime.SetRetrievalService(stubRetrievalService{chunks: []runtime.RetrievalChunk{
- {ID: "child-1", Content: "frag one", DatasetID: "kb-1", MomID: "parent-1", Score: 0.6},
- {ID: "child-2", Content: "frag two", DatasetID: "kb-1", MomID: "parent-1", Score: 0.8},
+ {ID: "child-1", Content: "frag one", DocumentID: "doc-1", DatasetID: "kb-1", MomID: "parent-1", Score: 0.6},
+ {ID: "child-2", Content: "frag two", DocumentID: "doc-1", DatasetID: "kb-1", MomID: "parent-1", Score: 0.8},
{ID: "top-1", Content: "standalone", DatasetID: "kb-1", Score: 0.9},
}})
t.Cleanup(func() { runtime.SetRetrievalService(prev) })
@@ -224,7 +227,7 @@ func TestHybridSearchSkipsWhenNoChildren(t *testing.T) {
func TestBM25SearchDoesPromoteChildren(t *testing.T) {
prev := runtime.GetRetrievalService()
runtime.SetRetrievalService(stubRetrievalService{chunks: []runtime.RetrievalChunk{
- {ID: "child-1", Content: "frag", DatasetID: "kb-1", MomID: "parent-1", Score: 0.6},
+ {ID: "child-1", Content: "frag", DocumentID: "doc-1", DatasetID: "kb-1", MomID: "parent-1", Score: 0.6},
}})
t.Cleanup(func() { runtime.SetRetrievalService(prev) })
diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.go
index 021ffc5e16..b91ba33b25 100644
--- a/internal/service/chunk/chunk.go
+++ b/internal/service/chunk/chunk.go
@@ -863,12 +863,13 @@ func (s *ChunkService) List(ctx context.Context, req *service.ListChunksRequest,
// Build search request - same as retrieval test but filtered by doc_id
searchReq := &types.SearchRequest{
- IndexNames: []string{indexName},
- MatchExprs: matchExprs,
- KbIDs: kbIDs,
- Offset: (page - 1) * size,
- Limit: size,
- OrderBy: orderBy,
+ IndexNames: []string{indexName},
+ MatchExprs: matchExprs,
+ KbIDs: kbIDs,
+ Offset: (page - 1) * size,
+ Limit: size,
+ OrderBy: orderBy,
+ IncludeUnavailable: req.AvailableInt == nil,
SelectFields: []string{
"id",
"content_with_weight",
diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.go
index 8a7018332f..bbf700a95e 100644
--- a/internal/service/chunk/chunk_test.go
+++ b/internal/service/chunk/chunk_test.go
@@ -334,6 +334,9 @@ func TestListSortsChunksByDocumentPosition(t *testing.T) {
if engine.searchReq == nil {
t.Fatal("expected Search to be called")
}
+ if !engine.searchReq.IncludeUnavailable {
+ t.Fatal("management chunk list must include disabled parent chunks")
+ }
if engine.searchReq.OrderBy == nil {
t.Fatal("expected OrderBy to be set")
}
diff --git a/internal/service/dataset/create_test.go b/internal/service/dataset/create_test.go
index 43a5c45ea1..618660c576 100644
--- a/internal/service/dataset/create_test.go
+++ b/internal/service/dataset/create_test.go
@@ -63,6 +63,32 @@ func TestCreateDataset_NoComponentParams(t *testing.T) {
}
}
+func TestCreateDataset_DefaultsParentChildConfig(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertCreateDatasetTenant(t, "tenant-1")
+
+ result, code, err := testDatasetCreateService(t).CreateDataset(t.Context(), &service.CreateDatasetRequest{
+ Name: "ds-default-parent-child",
+ ParserConfig: map[string]interface{}{},
+ }, "tenant-1")
+ if err != nil || code != common.CodeSuccess {
+ t.Fatalf("CreateDataset err=%v code=%d", err, code)
+ }
+
+ config, ok := result["parser_config"].(entity.JSONMap)
+ if !ok {
+ t.Fatalf("parser_config type = %T, want entity.JSONMap", result["parser_config"])
+ }
+ parentChild, ok := config["parent_child"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("parent_child = %#v, want default map", config["parent_child"])
+ }
+ if parentChild["use_parent_child"] != false || parentChild["children_delimiter"] != "\n" {
+ t.Fatalf("parent_child = %#v, want disabled defaults", parentChild)
+ }
+}
+
func TestCreateDataset_BuiltinParserDoesNotRequireParseType(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
@@ -111,6 +137,77 @@ func TestCreateDataset_ComponentParamsPopulated(t *testing.T) {
}
}
+// TestCreateDataset_ParentChildConfigReachesGeneralChunker verifies the public
+// parent-child setting controls the component that actually performs the
+// secondary split. Dropping this mapping silently leaves parent-child disabled.
+func TestCreateDataset_ParentChildConfigReachesGeneralChunker(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertCreateDatasetTenant(t, "tenant-1")
+
+ parserID := "general"
+ parseType := 1
+ result, code, err := testDatasetCreateService(t).CreateDataset(t.Context(), &service.CreateDatasetRequest{
+ Name: "ds-parent-child",
+ ParserID: &parserID,
+ ParseType: &parseType,
+ ParserConfig: map[string]interface{}{
+ "parent_child": map[string]interface{}{
+ "use_parent_child": true,
+ "children_delimiter": "|",
+ },
+ },
+ }, "tenant-1")
+ if err != nil || code != common.CodeSuccess {
+ t.Fatalf("CreateDataset err=%v code=%d", err, code)
+ }
+ config, ok := result["parser_config"].(entity.JSONMap)
+ if !ok {
+ t.Fatalf("parser_config type = %T, want entity.JSONMap", result["parser_config"])
+ }
+ chunker, ok := config["GeneralChunker:SixApplesFall"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("general chunker params = %#v", config["GeneralChunker:SixApplesFall"])
+ }
+ if got, ok := chunker["children_delimiters"].([]interface{}); !ok || len(got) != 1 || got[0] != "|" {
+ t.Fatalf("children_delimiters = %#v, want [\"|\"]", chunker["children_delimiters"])
+ }
+}
+
+func TestCreateDataset_PreservesChunkerComponentOverrides(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertCreateDatasetTenant(t, "tenant-1")
+
+ parserID := "general"
+ parseType := 1
+ result, code, err := testDatasetCreateService(t).CreateDataset(t.Context(), &service.CreateDatasetRequest{
+ Name: "ds-chunker-override",
+ ParserID: &parserID,
+ ParseType: &parseType,
+ ParserConfig: map[string]interface{}{
+ "GeneralChunker:SixApplesFall": map[string]interface{}{
+ "chunk_token_size": float64(256),
+ },
+ },
+ }, "tenant-1")
+ if err != nil || code != common.CodeSuccess {
+ t.Fatalf("CreateDataset err=%v code=%d", err, code)
+ }
+
+ config, ok := result["parser_config"].(entity.JSONMap)
+ if !ok {
+ t.Fatalf("parser_config type = %T, want entity.JSONMap", result["parser_config"])
+ }
+ chunker, ok := config["GeneralChunker:SixApplesFall"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("general chunker params = %#v", config["GeneralChunker:SixApplesFall"])
+ }
+ if got := chunker["chunk_token_size"]; got != float64(256) {
+ t.Fatalf("chunk_token_size = %#v, want 256", got)
+ }
+}
+
func TestCreateDataset_ParseTypeBuiltinClearsPipelineID(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
diff --git a/internal/service/dataset/crud.go b/internal/service/dataset/crud.go
index d3a390feaa..3ac413435f 100644
--- a/internal/service/dataset/crud.go
+++ b/internal/service/dataset/crud.go
@@ -123,6 +123,25 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD
parserConfig = pipelinepkg.BuildParserConfig(dslJSON, req.ParserConfig)
}
+ // Preserve the public default shape when parser_config is empty. The
+ // parent_child block remains the single source of truth; chunker
+ // children_delimiters are derived below only when it is configured.
+ var parentChild map[string]interface{}
+ if req.ParserConfig != nil {
+ if pc, ok := req.ParserConfig["parent_child"].(map[string]interface{}); ok {
+ parentChild = pc
+ }
+ }
+ if parentChild == nil {
+ parentChild = map[string]interface{}{
+ "use_parent_child": false,
+ "children_delimiter": "\n",
+ }
+ }
+ parserConfig["parent_child"] = parentChild
+
+ pipelinepkg.ApplyParentChildChunkerConfig(parserConfig, map[string]interface{}(parserConfig))
+
var parserConfigMap map[string]interface{} = parserConfig
embdID := tenant.EmbdID
diff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.go
index 8c414eb0ee..eff2b8392d 100644
--- a/internal/service/dataset/helpers.go
+++ b/internal/service/dataset/helpers.go
@@ -508,7 +508,7 @@ func datasetUpdateEmbeddingID(req service.UpdateDatasetRequest) (string, bool, e
return embdID, true, nil
}
-func preserveDatasetParserConfigMetadata(next, existing entity.JSONMap, incoming map[string]interface{}) entity.JSONMap {
+func preserveDatasetParserConfigState(next, existing entity.JSONMap, incoming map[string]interface{}) entity.JSONMap {
if next == nil {
next = entity.JSONMap{}
}
@@ -526,6 +526,24 @@ func preserveDatasetParserConfigMetadata(next, existing entity.JSONMap, incoming
if mm != nil {
next["metadata"] = mm
}
+ var parentChild map[string]any
+ if incoming != nil {
+ if value, ok := incoming["parent_child"].(map[string]any); ok {
+ parentChild = value
+ }
+ }
+ if parentChild == nil && existing != nil {
+ if value, ok := existing["parent_child"].(map[string]any); ok {
+ parentChild = value
+ }
+ }
+ if parentChild != nil {
+ next["parent_child"] = parentChild
+ }
+ // The persisted top-level setting is authoritative. Re-derive the
+ // chunker's runtime delimiter after every config merge so parser/pipeline
+ // switches cannot retain parent_child without retaining its behavior.
+ pipelinepkg.ApplyParentChildChunkerConfig(next, map[string]interface{}(next))
return next
}
diff --git a/internal/service/dataset/helpers_test.go b/internal/service/dataset/helpers_test.go
index 9bc74ad252..b112a42868 100644
--- a/internal/service/dataset/helpers_test.go
+++ b/internal/service/dataset/helpers_test.go
@@ -232,6 +232,16 @@ func TestValidateDocumentParserConfig_AllowsUnknownFields(t *testing.T) {
}
}
+func TestValidateParserConfigAcceptsFlatParentChildDelimiter(t *testing.T) {
+ config := map[string]interface{}{"children_delimiter": "|"}
+ if err := validateDatasetParserConfig(config); err != nil {
+ t.Fatalf("flat children_delimiter should remain accepted for compatibility: %v", err)
+ }
+ if err := ValidateDocumentParserConfig(config); err != nil {
+ t.Fatalf("document parser config should accept children_delimiter: %v", err)
+ }
+}
+
// --- normalizeDatasetID ---
func TestNormalizeDatasetID_Invalid(t *testing.T) {
@@ -356,7 +366,7 @@ func TestNormalizeMetadataConfigFields_TrimsKey(t *testing.T) {
}
}
-func TestPreserveDatasetParserConfigMetadata_FallsBackWhenIncomingNotMap(t *testing.T) {
+func TestPreserveDatasetParserConfigState_FallsBackWhenIncomingNotMap(t *testing.T) {
existing := entity.JSONMap{
"metadata": map[string]any{
"enabled": true,
@@ -371,7 +381,7 @@ func TestPreserveDatasetParserConfigMetadata_FallsBackWhenIncomingNotMap(t *test
for name, incomingMetadata := range cases {
t.Run(name, func(t *testing.T) {
incoming := map[string]interface{}{"metadata": incomingMetadata}
- got := preserveDatasetParserConfigMetadata(entity.JSONMap{}, existing, incoming)
+ got := preserveDatasetParserConfigState(entity.JSONMap{}, existing, incoming)
meta, ok := got["metadata"].(map[string]any)
if !ok {
t.Fatalf("expected existing modular metadata preserved, got %#v", got["metadata"])
@@ -384,7 +394,7 @@ func TestPreserveDatasetParserConfigMetadata_FallsBackWhenIncomingNotMap(t *test
}
}
-func TestPreserveDatasetParserConfigMetadata_UsesValidIncomingMap(t *testing.T) {
+func TestPreserveDatasetParserConfigState_UsesValidIncomingMap(t *testing.T) {
existing := entity.JSONMap{
"metadata": map[string]any{
"enabled": false,
@@ -399,7 +409,7 @@ func TestPreserveDatasetParserConfigMetadata_UsesValidIncomingMap(t *testing.T)
"built_in_metadata": []any{},
},
}
- got := preserveDatasetParserConfigMetadata(entity.JSONMap{}, existing, incoming)
+ got := preserveDatasetParserConfigState(entity.JSONMap{}, existing, incoming)
meta, ok := got["metadata"].(map[string]any)
if !ok {
t.Fatalf("expected modular metadata map, got %#v", got["metadata"])
diff --git a/internal/service/dataset/update.go b/internal/service/dataset/update.go
index 3fb86e9b7b..b593d65799 100644
--- a/internal/service/dataset/update.go
+++ b/internal/service/dataset/update.go
@@ -247,7 +247,7 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID
}
if dslJSON != nil {
parserConfig := pipelinepkg.BuildParserConfig(dslJSON, map[string]interface{}(req.ParserConfig))
- updates["parser_config"] = preserveDatasetParserConfigMetadata(parserConfig, lockedKB.ParserConfig, req.ParserConfig)
+ updates["parser_config"] = preserveDatasetParserConfigState(parserConfig, lockedKB.ParserConfig, req.ParserConfig)
}
}
if pagerankRequested && requestedPagerank != lockedKB.Pagerank {
@@ -268,7 +268,7 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID
common.Warn("failed to resolve component params defaults on parser_id switch",
zap.String("parserID", parserID), zap.Error(cpErr))
} else if resolved != nil {
- updates["parser_config"] = preserveDatasetParserConfigMetadata(resolved, lockedKB.ParserConfig, req.ParserConfig)
+ updates["parser_config"] = preserveDatasetParserConfigState(resolved, lockedKB.ParserConfig, req.ParserConfig)
}
}
}
@@ -288,7 +288,7 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID
common.Warn("failed to resolve component params defaults on pipeline change",
zap.String("parserID", cfgParserID), zap.Error(cpErr))
} else if cpDefaults != nil {
- updates["parser_config"] = preserveDatasetParserConfigMetadata(cpDefaults, lockedKB.ParserConfig, req.ParserConfig)
+ updates["parser_config"] = preserveDatasetParserConfigState(cpDefaults, lockedKB.ParserConfig, req.ParserConfig)
}
}
diff --git a/internal/service/dataset/update_test.go b/internal/service/dataset/update_test.go
index ca59493cd8..887f37fa28 100644
--- a/internal/service/dataset/update_test.go
+++ b/internal/service/dataset/update_test.go
@@ -98,6 +98,77 @@ func TestDatasetServiceUpdateDatasetUpdatesFields(t *testing.T) {
}
}
+// TestUpdateDataset_ParentChildConfigReachesGeneralChunker verifies an edit to
+// the dataset setting updates the runtime chunker parameter, not only the UI
+// payload retained in parser_config.
+func TestUpdateDataset_ParentChildConfigReachesGeneralChunker(t *testing.T) {
+ db := setupDatasetUpdateTestDB(t)
+ pushServiceDB(t, db)
+ insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Original")
+
+ _, code, err := testDatasetUpdateService(t).UpdateDataset(t.Context(), "kb-1", "tenant-1", service.UpdateDatasetRequest{
+ ParserConfig: map[string]interface{}{
+ "parent_child": map[string]interface{}{
+ "use_parent_child": true,
+ "children_delimiter": "|",
+ },
+ },
+ })
+ if err != nil || code != common.CodeSuccess {
+ t.Fatalf("UpdateDataset err=%v code=%d", err, code)
+ }
+ persisted, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")
+ if err != nil {
+ t.Fatalf("get updated dataset: %v", err)
+ }
+ chunker, ok := persisted.ParserConfig["GeneralChunker:SixApplesFall"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("general chunker params = %#v", persisted.ParserConfig["GeneralChunker:SixApplesFall"])
+ }
+ if got, ok := chunker["children_delimiters"].([]interface{}); !ok || len(got) != 1 || got[0] != "|" {
+ t.Fatalf("children_delimiters = %#v, want [\"|\"]", chunker["children_delimiters"])
+ }
+ parentChild, ok := persisted.ParserConfig["parent_child"].(map[string]interface{})
+ if !ok || parentChild["use_parent_child"] != true || parentChild["children_delimiter"] != "|" {
+ t.Fatalf("parent_child = %#v, want persisted public setting", persisted.ParserConfig["parent_child"])
+ }
+}
+
+func TestUpdateDatasetPreservesParentChildChunkerRuntimeConfig(t *testing.T) {
+ db := setupDatasetUpdateTestDB(t)
+ pushServiceDB(t, db)
+ insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Original")
+
+ existingConfig := entity.JSONMap{
+ "parent_child": map[string]interface{}{
+ "use_parent_child": true,
+ "children_delimiter": "|",
+ },
+ }
+ if err := db.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", existingConfig).Error; err != nil {
+ t.Fatalf("seed parent-child parser config: %v", err)
+ }
+
+ _, code, err := testDatasetUpdateService(t).UpdateDataset(t.Context(), "kb-1", "tenant-1", service.UpdateDatasetRequest{
+ ParserConfig: map[string]interface{}{"chunk_token_num": float64(256)},
+ })
+ if err != nil || code != common.CodeSuccess {
+ t.Fatalf("UpdateDataset err=%v code=%d", err, code)
+ }
+
+ persisted, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")
+ if err != nil {
+ t.Fatalf("load updated dataset: %v", err)
+ }
+ chunker, ok := persisted.ParserConfig["GeneralChunker:SixApplesFall"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("general chunker params = %#v", persisted.ParserConfig["GeneralChunker:SixApplesFall"])
+ }
+ if got, ok := chunker["children_delimiters"].([]interface{}); !ok || len(got) != 1 || got[0] != "|" {
+ t.Fatalf("children_delimiters = %#v, want [\"|\"]", chunker["children_delimiters"])
+ }
+}
+
func TestUpdateDataset_RejectsSimultaneousParserIDAndPipelineID(t *testing.T) {
db := setupDatasetUpdateTestDB(t)
pushServiceDB(t, db)
diff --git a/internal/service/document/document_dataset_update.go b/internal/service/document/document_dataset_update.go
index 771fd9cfe0..8b091bbc2e 100644
--- a/internal/service/document/document_dataset_update.go
+++ b/internal/service/document/document_dataset_update.go
@@ -162,6 +162,7 @@ func (s *DocumentService) UpdateDatasetDocument(ctx context.Context, userID, dat
}
} else {
cleaned := pipelinepkg.BuildParserConfig(dslJSON, req.ParserConfig)
+ pipelinepkg.ApplyParentChildChunkerConfig(cleaned, req.ParserConfig)
tenant, tenantErr := dao.NewTenantDAO().GetByID(ctx, dao.DB, kb.TenantID)
if tenantErr == nil && tenant != nil {
cleaned = service.ApplyComponentScopedParserConfig(
@@ -467,6 +468,7 @@ func (s *DocumentService) updateDocumentParserConfig(ctx context.Context, docume
if _, ok := config["raptor"]; !ok {
delete(merged, "raptor")
}
+ pipelinepkg.ApplyParentChildChunkerConfig(merged, merged)
return s.documentDAO.UpdateByID(ctx, dao.DB, documentID, map[string]interface{}{
"parser_config": entity.JSONMap(merged),
diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go
index 6ada2f35af..985718b885 100644
--- a/internal/service/document/document_test.go
+++ b/internal/service/document/document_test.go
@@ -248,6 +248,7 @@ type rerunDeleteDocEngine struct {
condition map[string]interface{}
indexName string
datasetID string
+ search *types.SearchRequest
}
type failingDeleteDocEngine struct {
@@ -286,11 +287,13 @@ func (e *rerunDeleteDocEngine) ChunkStoreExists(context.Context, string, string)
return true, nil
}
-func (e *rerunDeleteDocEngine) Search(context.Context, *types.SearchRequest) (*types.SearchResult, error) {
+func (e *rerunDeleteDocEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
+ e.search = req
return &types.SearchResult{Chunks: []map[string]interface{}{
{"id": "source-1"},
+ {"id": "parent-1", "available_int": 0},
{"id": "wiki-1", "compile_kwd": "wiki_page"},
- }, Total: 2}, nil
+ }, Total: 3}, nil
}
func (e *rerunDeleteDocEngine) DeleteChunks(_ context.Context, condition map[string]interface{}, indexName string, datasetID string) (int64, error) {
@@ -308,10 +311,12 @@ func (e *failingDeleteDocEngine) DeleteChunks(context.Context, map[string]interf
type sourceAvailabilityDocEngine struct {
fakeChatDocEngine
updateConditions []map[string]interface{}
+ search *types.SearchRequest
updateValues []map[string]interface{}
}
-func (e *sourceAvailabilityDocEngine) Search(context.Context, *types.SearchRequest) (*types.SearchResult, error) {
+func (e *sourceAvailabilityDocEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
+ e.search = req
return &types.SearchResult{Chunks: []map[string]interface{}{
{"id": "source-1"},
{"id": "tree-1", "compile_kwd": "tree"},
@@ -2475,9 +2480,12 @@ func TestClearDocumentParseResultsClearsCountersTasksAndChunks(t *testing.T) {
if engine.deleteCalls != 2 {
t.Fatalf("deleteCalls = %d, want 2 (generated and source chunks)", engine.deleteCalls)
}
- if engine.indexName != "ragflow_tenant-1" || engine.datasetID != "kb-1" || !reflect.DeepEqual(engine.condition["id"], []string{"source-1"}) {
+ if engine.indexName != "ragflow_tenant-1" || engine.datasetID != "kb-1" || !reflect.DeepEqual(engine.condition["id"], []string{"source-1", "parent-1"}) {
t.Fatalf("unexpected delete call: index=%s dataset=%s condition=%v", engine.indexName, engine.datasetID, engine.condition)
}
+ if engine.search == nil || !engine.search.IncludeUnavailable {
+ t.Fatalf("reparse search = %#v, want hidden parent rows included", engine.search)
+ }
}
func TestClearDocumentParseResultsDeletesDocumentGeneratedChunks(t *testing.T) {
@@ -2584,6 +2592,9 @@ func TestUpdateDocumentChunkAvailabilityTogglesFinalProducts(t *testing.T) {
if got := docEngine.updateValues[0]["available_int"]; got != 0 {
t.Fatalf("available_int = %#v, want 0", got)
}
+ if docEngine.search == nil || docEngine.search.IncludeUnavailable {
+ t.Fatalf("availability search = %#v, must not include hidden parents", docEngine.search)
+ }
}
func TestClearDocumentParseResultsIsIdempotentForStaleDocSnapshot(t *testing.T) {
@@ -3405,6 +3416,43 @@ func TestUpdateDatasetDocumentParseTypePipelineIgnoresDirtyParserID(t *testing.T
}
}
+func TestUpdateDatasetDocumentParentChildConfigSurvivesDSLFailure(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
+ insertNamedTestDoc(t, "doc-1", "kb-1", "doc.txt", 10, 5)
+ if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("parser_config", entity.JSONMap{
+ "GeneralChunker:SixApplesFall": map[string]any{
+ "children_delimiters": []any{},
+ },
+ }).Error; err != nil {
+ t.Fatalf("seed document parser config: %v", err)
+ }
+
+ parseType := 2
+ pipelineID := "1234567890abcdef1234567890abcdef"
+ resp, code, err := testDocumentService(t).UpdateDatasetDocument(t.Context(), "tenant-1", "kb-1", "doc-1", &UpdateDatasetDocumentRequest{
+ ParseType: &parseType,
+ PipelineID: &pipelineID,
+ ParserConfig: map[string]any{
+ "parent_child": map[string]any{
+ "use_parent_child": true,
+ "children_delimiter": "|",
+ },
+ },
+ }, map[string]bool{"pipeline_id": true, "parse_type": true, "parser_config": true})
+ if err != nil || code != common.CodeSuccess {
+ t.Fatalf("UpdateDatasetDocument err=%v code=%d", err, code)
+ }
+ chunker, ok := resp.ParserConfig["GeneralChunker:SixApplesFall"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("general chunker params = %#v", resp.ParserConfig["GeneralChunker:SixApplesFall"])
+ }
+ if got, ok := chunker["children_delimiters"].([]interface{}); !ok || len(got) != 1 || got[0] != "|" {
+ t.Fatalf("children_delimiters = %#v, want [|]", chunker["children_delimiters"])
+ }
+}
+
// TestUpdateDatasetDocumentRejectsInvalidPages verifies the fail-fast contract
// for the "pages" range: an invalid range (from<1) aborts the request with
// CodeDataError instead of being silently dropped or persisted.
@@ -3437,6 +3485,36 @@ func TestUpdateDatasetDocumentRejectsInvalidPages(t *testing.T) {
}
}
+func TestUpdateDatasetDocumentParentChildConfigReachesGeneralChunker(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 10, 5)
+ insertNamedTestDoc(t, "doc-1", "kb-1", "doc.txt", 10, 5)
+
+ resp, code, err := testDocumentService(t).UpdateDatasetDocument(t.Context(), "tenant-1", "kb-1", "doc-1", &UpdateDatasetDocumentRequest{
+ ParserConfig: map[string]any{
+ "parent_child": map[string]any{
+ "use_parent_child": true,
+ "children_delimiter": "|",
+ },
+ },
+ }, map[string]bool{"parser_config": true})
+ if err != nil || code != common.CodeSuccess {
+ t.Fatalf("UpdateDatasetDocument err=%v code=%d", err, code)
+ }
+ chunker, ok := resp.ParserConfig["GeneralChunker:SixApplesFall"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("general chunker params = %#v", resp.ParserConfig["GeneralChunker:SixApplesFall"])
+ }
+ if got, ok := chunker["children_delimiters"].([]interface{}); !ok || len(got) != 1 || got[0] != "|" {
+ t.Fatalf("children_delimiters = %#v, want [|]", chunker["children_delimiters"])
+ }
+ parentChild, ok := resp.ParserConfig["parent_child"].(map[string]interface{})
+ if !ok || parentChild["use_parent_child"] != true || parentChild["children_delimiter"] != "|" {
+ t.Fatalf("parent_child = %#v, want persisted public setting", resp.ParserConfig["parent_child"])
+ }
+}
+
func TestUpdateDatasetDocumentEnabledUpdatesStatus(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
diff --git a/internal/service/document/wiki_refresh.go b/internal/service/document/wiki_refresh.go
index af320f7a38..a1fa55f6c9 100644
--- a/internal/service/document/wiki_refresh.go
+++ b/internal/service/document/wiki_refresh.go
@@ -102,17 +102,18 @@ func (s *DocumentService) loadAvailabilityToggleChunkIDs(ctx context.Context, in
return ids, nil
}
-func (s *DocumentService) loadSourceChunkIDs(ctx context.Context, indexName, datasetID, documentID string) ([]string, error) {
+func (s *DocumentService) loadSourceChunkIDs(ctx context.Context, indexName, datasetID, documentID string, includeUnavailable bool) ([]string, error) {
ids := make([]string, 0)
for offset := 0; ; offset += sourceChunkAvailabilityBatchSize {
searchCtx, cancel := context.WithTimeout(ctx, cleanupBatchTimeout)
result, err := s.docEngine.Search(searchCtx, &enginetypes.SearchRequest{
- IndexNames: []string{indexName},
- KbIDs: []string{datasetID},
- Offset: offset,
- Limit: sourceChunkAvailabilityBatchSize,
- SelectFields: []string{"id", "compile_kwd"},
- Filter: map[string]any{"doc_id": []string{documentID}},
+ IndexNames: []string{indexName},
+ KbIDs: []string{datasetID},
+ Offset: offset,
+ Limit: sourceChunkAvailabilityBatchSize,
+ SelectFields: []string{"id", "compile_kwd"},
+ Filter: map[string]any{"doc_id": []string{documentID}},
+ IncludeUnavailable: includeUnavailable,
})
cancel()
if err != nil {
@@ -141,7 +142,10 @@ func (s *DocumentService) deleteSourceChunks(ctx context.Context, tenantID, data
return nil
}
indexName := fmt.Sprintf("ragflow_%s", tenantID)
- ids, err := s.loadSourceChunkIDs(ctx, indexName, datasetID, documentID)
+ // Reparse is the migration boundary for parent-child IDs. Include hidden
+ // parent rows so old hash(mom) rows are removed with their children before
+ // new document-scoped rows are written.
+ ids, err := s.loadSourceChunkIDs(ctx, indexName, datasetID, documentID, true)
if err != nil || len(ids) == 0 {
return err
}
diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.go
index eecb202f04..58a4080bf1 100644
--- a/internal/service/nlp/retrieval.go
+++ b/internal/service/nlp/retrieval.go
@@ -1034,12 +1034,18 @@ func RetrievalByChildren(chunks []map[string]interface{}, tenantIDs []string, do
return chunks
}
- // Group child chunks by mom_id
+ // Group child chunks by parent identity and document. The document scope is
+ // required both for new parent IDs and for safe fallback when reading legacy
+ // rows that used hash(mom) without document scope.
type childChunk struct {
chunk map[string]interface{}
+ }
+ type parentKey struct {
+ momID string
+ docID string
kbID string
}
- momChunks := make(map[string][]childChunk)
+ momChunks := make(map[parentKey][]childChunk)
remainingChunks := make([]map[string]interface{}, 0, len(chunks))
for _, ck := range chunks {
@@ -1049,7 +1055,13 @@ func RetrievalByChildren(chunks []map[string]interface{}, tenantIDs []string, do
continue
}
kbID, _ := ck["kb_id"].(string)
- momChunks[momID] = append(momChunks[momID], childChunk{chunk: ck, kbID: kbID})
+ docID, _ := ck["doc_id"].(string)
+ if docID == "" {
+ remainingChunks = append(remainingChunks, ck)
+ continue
+ }
+ key := parentKey{momID: momID, docID: docID, kbID: kbID}
+ momChunks[key] = append(momChunks[key], childChunk{chunk: ck})
}
if len(momChunks) == 0 {
@@ -1059,26 +1071,32 @@ func RetrievalByChildren(chunks []map[string]interface{}, tenantIDs []string, do
// Fetch parent chunks and aggregate
vectorSize := 1024
- for momID, childList := range momChunks {
- kbIDs := make([]string, 0, len(childList))
- for _, c := range childList {
- if c.kbID != "" {
- kbIDs = append(kbIDs, c.kbID)
- }
- }
- if len(kbIDs) == 0 {
- kbIDs = append(kbIDs, "")
- }
-
- parent, err := docEngine.GetChunk(ctx, indexNames[0], momID, kbIDs)
+ for key, childList := range momChunks {
+ momID := key.momID
+ parentResult, err := docEngine.Search(ctx, &types.SearchRequest{
+ IndexNames: []string{indexNames[0]},
+ KbIDs: []string{key.kbID},
+ Limit: 1,
+ IncludeUnavailable: true,
+ Filter: map[string]interface{}{
+ "id": momID,
+ "doc_id": key.docID,
+ },
+ })
if err != nil {
common.Warn("Failed to get parent chunk", zap.String("momID", momID), zap.Error(err))
+ for _, child := range childList {
+ remainingChunks = append(remainingChunks, child.chunk)
+ }
continue
}
- parentMap, ok := parent.(map[string]interface{})
- if !ok {
+ if parentResult == nil || len(parentResult.Chunks) == 0 {
+ for _, child := range childList {
+ remainingChunks = append(remainingChunks, child.chunk)
+ }
continue
}
+ parentMap := parentResult.Chunks[0]
// Calculate average similarity
simBuf := make([]float64, 0, len(childList))
diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.go
index dbcdd7418d..76263f9c5e 100644
--- a/internal/service/nlp/retrieval_test.go
+++ b/internal/service/nlp/retrieval_test.go
@@ -2,6 +2,7 @@ package nlp
import (
"context"
+ "errors"
"fmt"
"maps"
"slices"
@@ -242,6 +243,76 @@ func (e *retrievalCountEngine) FilterDocIdsByMetaPushdown(context.Context, *gorm
return nil
}
+type parentChunkMissingEngine struct{ engine.DocEngine }
+
+func (parentChunkMissingEngine) Search(context.Context, *types.SearchRequest) (*types.SearchResult, error) {
+ return nil, errors.New("parent chunk missing")
+}
+
+type parentChunkScopeEngine struct {
+ engine.DocEngine
+ parentSearch *types.SearchRequest
+}
+
+func (e *parentChunkScopeEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
+ e.parentSearch = req
+ return &types.SearchResult{}, nil
+}
+
+func TestRetrievalByChildrenScopesParentLookupToChildDocument(t *testing.T) {
+ engine := &parentChunkScopeEngine{}
+ child := map[string]interface{}{
+ "chunk_id": "child-a",
+ "mom_id": "legacy-shared-parent-id",
+ "doc_id": "doc-a",
+ "kb_id": "kb-1",
+ "content_with_weight": "matching child text",
+ "similarity": 0.8,
+ }
+
+ got := RetrievalByChildren([]map[string]interface{}{child}, []string{"tenant-1"}, engine, t.Context())
+
+ if len(got) != 1 || got[0]["chunk_id"] != "child-a" {
+ t.Fatalf("retrieval result = %#v, want fallback child", got)
+ }
+ if engine.parentSearch == nil {
+ t.Fatal("parent lookup did not use a document-scoped search")
+ }
+ if engine.parentSearch.Filter["doc_id"] != "doc-a" {
+ t.Fatalf("parent lookup filter = %#v, want doc_id doc-a", engine.parentSearch.Filter)
+ }
+ if !engine.parentSearch.IncludeUnavailable {
+ t.Fatal("parent lookup must include hidden parent chunks")
+ }
+}
+
+// TestRetrievalByChildrenKeepsChildWhenParentIsMissing verifies a partial
+// parent-child write never turns a relevant child hit into an empty result.
+// Removing the fallback is a retrieval data-loss bug.
+func TestRetrievalByChildrenKeepsChildWhenParentIsMissing(t *testing.T) {
+ child := map[string]interface{}{
+ "chunk_id": "child-1",
+ "mom_id": "parent-1",
+ "doc_id": "doc-1",
+ "kb_id": "kb-1",
+ "content_with_weight": "matching child text",
+ "similarity": 0.8,
+ }
+
+ got := RetrievalByChildren(
+ []map[string]interface{}{child},
+ []string{"tenant-1"},
+ parentChunkMissingEngine{},
+ t.Context(),
+ )
+ if len(got) != 1 {
+ t.Fatalf("retrieval result count = %d, want fallback child", len(got))
+ }
+ if got[0]["chunk_id"] != "child-1" {
+ t.Fatalf("fallback chunk_id = %#v, want child-1", got[0]["chunk_id"])
+ }
+}
+
func TestBuildInfinityFusionExprUsesVectorSimilarityWeight(t *testing.T) {
tests := []struct {
name string