RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
test(chunker): pin current applyChildrenDelimText behavior (#17876 partial coverage) (#18315)
Taranum Wasu030006990bf4
diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.goindex 4d2ac8e206..93095b954f 100644--- a/internal/ingestion/component/chunker/token_test.go+++ b/internal/ingestion/component/chunker/token_test.go@@ -20,6 +20,7 @@ import ("context""math""reflect"+ "regexp""strings""testing"@@ -727,3 +728,99 @@ func TestMergeByTokenSize_OversizeDropsBlankLines(t *testing.T) {t.Errorf("blank line survived in oversize path (Python drops it): got chunk text %q, want no blank line (\\n\\n)", got)}}++// TestApplyChildrenDelimText_DefaultsMomToCurrentChunk verifies that when an+// incoming chunk has no Mom, the children fall back to the current chunk's+// text (the historical behaviour preserved by the fix for #17876).+func TestApplyChildrenDelimText_DefaultsMomToCurrentChunk(t *testing.T) {+ docs := []schema.ChunkDoc{+ {Text: "alpha. beta. gamma"},+ }+ pattern := regexp.MustCompile(`\. `)++ out := applyChildrenDelimText(docs, pattern)+ if len(out) != 3 {+ t.Fatalf("want 3 children, got %d", len(out))+ }+ for i, c := range out {+ if c.Mom != "alpha. beta. gamma" {+ t.Errorf("child %d: Mom=%q, want %q", i, c.Mom, "alpha. beta. gamma")+ }+ }+}++// TestApplyChildrenDelimText_OverwritesIncomingMom documents the+// CURRENT behaviour: when a chunk flowing into applyChildrenDelimText+// already has a non-empty Mom, the function OVERWRITES it with+// TrimPrefix(d.Text, "\n") of the current chunk's text. This is+// the divergence that #17876 item 2 (multi-chunk text-path Mom+// granularity) is tracking. Once the merge-granularity fix lands,+// this test should be updated (or the PreservesIncomingMom version+// added back).+func TestApplyChildrenDelimText_OverwritesIncomingMom(t *testing.T) {+ docs := []schema.ChunkDoc{+ {Text: "alpha. beta. gamma", Mom: "incoming-mom-from-upstream"},+ }+ pattern := regexp.MustCompile(`\. `)++ out := applyChildrenDelimText(docs, pattern)+ if len(out) != 3 {+ t.Fatalf("want 3 children, got %d", len(out))+ }+ for i, c := range out {+ // Children must NOT carry the incoming Mom; the function+ // overwrites it with TrimPrefix(d.Text, "\n") of the current+ // chunk's text. This pins the current behavior; a future+ // merge-granularity fix should update this test (or the test+ // itself flips to assert the new preserved Mom behavior).+ if c.Mom == "incoming-mom-from-upstream" {+ t.Errorf("child %d: Mom=%q, expected overwrite to %q (not preserved)",+ i, c.Mom, "alpha. beta. gamma")+ }+ if c.Mom != "alpha. beta. gamma" {+ t.Errorf("child %d: Mom=%q, want %q (TrimPrefix(d.Text, \"\\n\"))",+ i, c.Mom, "alpha. beta. gamma")+ }+ }+}++// TestApplyChildrenDelimText_NilPatternIsNoop verifies the early return so+// callers that pass a nil pattern don't accidentally clear Mom.+func TestApplyChildrenDelimText_NilPatternIsNoop(t *testing.T) {+ docs := []schema.ChunkDoc{+ {Text: "alpha. beta", Mom: "kept"},+ }+ out := applyChildrenDelimText(docs, nil)+ if len(out) != 1 {+ t.Fatalf("want 1 chunk unchanged, got %d", len(out))+ }+ if out[0].Mom != "kept" || out[0].Text != "alpha. beta" {+ t.Errorf("input mutated under nil pattern: %+v", out[0])+ }+}++// TestApplyChildrenDelimText_FallbackStripsLeadingNewline verifies that+// when no incoming Mom is set, the fallback Mom uses+// strings.TrimPrefix(t, "\n") — i.e. it strips a single leading newline+// from the current chunk's text. The historical default this PR+// preserves; if a child path forgets to strip, JSON-keyed SQL+// downstream could see an extra leading "\n" in the Mom value.+func TestApplyChildrenDelimText_FallbackStripsLeadingNewline(t *testing.T) {+ docs := []schema.ChunkDoc{+ {Text: "\nalpha. beta. gamma"},+ }+ pattern := regexp.MustCompile(`\. `)++ out := applyChildrenDelimText(docs, pattern)+ if len(out) != 3 {+ t.Fatalf("want 3 children, got %d", len(out))+ }+ for i, c := range out {+ // Each child Mom must be the text-path parent segment with+ // the leading "\n" stripped, NOT the raw text-with-newline.+ if c.Mom != "alpha. beta. gamma" {+ t.Errorf("child %d: Mom=%q, want %q (leading newline must be stripped)",+ i, c.Mom, "alpha. beta. gamma")+ }+ }+}