Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix ingestion pipeline child delimiter extra newline (#18490)

Wang Qi5f92323cedcf
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index d0b302f420..fd6b5bb148 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -1113,8 +1113,8 @@ func splitByChildren(chunks []schema.ChunkDoc, pattern *regexp.Regexp) []schema.
out = append(out, ck)
continue
}
- mom := ck.Text
- parts := splitDroppingDelim(mom, pattern)
+ mom := strings.TrimPrefix(ck.Text, "\n")
+ parts := splitDroppingDelim(ck.Text, pattern)
for _, p := range parts {
if strings.TrimSpace(p) == "" {
continue
@@ -1160,7 +1160,7 @@ func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp) []sc
if strings.TrimSpace(child) == "" {
continue
}
- out = append(out, schema.ChunkDoc{Text: child, Mom: t})
+ out = append(out, schema.ChunkDoc{Text: child, Mom: strings.TrimPrefix(t, "\n")})
}
}
return out
diff --git a/rag/flow/chunker/token_chunker.py b/rag/flow/chunker/token_chunker.py
index 2038817053..6fb4b23da2 100644
--- a/rag/flow/chunker/token_chunker.py
+++ b/rag/flow/chunker/token_chunker.py
@@ -373,7 +373,7 @@ def _split_chunk_docs_by_children(chunks, pattern):
split_texts = _split_text_by_pattern(chunk.get("text", ""), pattern)
- mom = chunk.get("text", "")
+ mom = chunk.get("text", "").removeprefix("\n")
for text in split_texts:
if not text.strip():
continue
@@ -428,7 +428,7 @@ class TokenChunker(ProcessBase):
for text in _split_text_by_pattern(c, custom_pattern):
if not text.strip():
continue
- docs.append({"text": text, "mom": c})
+ docs.append({"text": text, "mom": c.removeprefix("\n")})
self.set_output("chunks", docs)
else:
self.set_output("chunks", [{"text": c.strip()} for c in cks if c.strip()])