RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix ingestion pipeline child delimiter extra newline (#18490)
Wang Qi5f92323cedcf
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex d0b302f420..fd6b5bb148 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -1113,8 +1113,8 @@ func splitByChildren(chunks []schema.ChunkDoc, pattern *regexp.Regexp) []schema.out = append(out, ck)continue}- mom := ck.Text- parts := splitDroppingDelim(mom, pattern)+ mom := strings.TrimPrefix(ck.Text, "\n")+ parts := splitDroppingDelim(ck.Text, pattern)for _, p := range parts {if strings.TrimSpace(p) == "" {continue@@ -1160,7 +1160,7 @@ func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp) []scif strings.TrimSpace(child) == "" {continue}- out = append(out, schema.ChunkDoc{Text: child, Mom: t})+ out = append(out, schema.ChunkDoc{Text: child, Mom: strings.TrimPrefix(t, "\n")})}}return outdiff --git a/rag/flow/chunker/token_chunker.py b/rag/flow/chunker/token_chunker.pyindex 2038817053..6fb4b23da2 100644--- a/rag/flow/chunker/token_chunker.py+++ b/rag/flow/chunker/token_chunker.py@@ -373,7 +373,7 @@ def _split_chunk_docs_by_children(chunks, pattern):split_texts = _split_text_by_pattern(chunk.get("text", ""), pattern)- mom = chunk.get("text", "")+ mom = chunk.get("text", "").removeprefix("\n")for text in split_texts:if not text.strip():continue@@ -428,7 +428,7 @@ class TokenChunker(ProcessBase):for text in _split_text_by_pattern(c, custom_pattern):if not text.strip():continue- docs.append({"text": text, "mom": c})+ docs.append({"text": text, "mom": c.removeprefix("\n")})self.set_output("chunks", docs)else:self.set_output("chunks", [{"text": c.strip()} for c in cks if c.strip()])