Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(parser): align TextParser with Python _code delimiter split (#18128)

Jack7b2d052f8a09
diff --git a/internal/parser/parser/parse_with_result_test.go b/internal/parser/parser/parse_with_result_test.go
index 8ab285dd69..a33d274a99 100644
--- a/internal/parser/parser/parse_with_result_test.go
+++ b/internal/parser/parser/parse_with_result_test.go
@@ -33,6 +33,7 @@
package parser
import (
+ "os"
"strings"
"testing"
@@ -85,10 +86,12 @@ func TestTextParser_ParseWithResult_Empty(t *testing.T) {
}
}
-// TestTextParser_ParseWithResult_LongParagraphSlicing pins the
-// maxItemBytes boundary behaviour. A single paragraph longer
-// than 8192 bytes is sliced at the nearest line boundary.
-func TestTextParser_ParseWithResult_LongParagraphSlicing(t *testing.T) {
+// TestTextParser_ParseWithResult_NoSizeCap pins that the parser performs no
+// per-item byte slicing: a single continuous run longer than any prior cap
+// (here 9000 'a's with no delimiter) stays as one item whose full content is
+// preserved. Sizing is delegated to the chunker / embedding truncation, matching
+// python's parser_txt (which also does no size slicing).
+func TestTextParser_ParseWithResult_NoSizeCap(t *testing.T) {
ctx := t.Context()
p := NewTextParser()
long := strings.Repeat("a", 9000)
@@ -96,13 +99,11 @@ func TestTextParser_ParseWithResult_LongParagraphSlicing(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if len(res.JSON) < 2 {
- t.Errorf("JSON len = %d, want >=2 (sliced at maxItemBytes)", len(res.JSON))
+ if len(res.JSON) != 1 {
+ t.Fatalf("JSON len = %d, want 1 (no per-item size cap)", len(res.JSON))
}
- for i, it := range res.JSON {
- if txt, _ := it["text"].(string); len(txt) > 8192 {
- t.Errorf("JSON[%d].text len = %d, exceeds maxItemBytes=8192", i, len(txt))
- }
+ if txt, _ := res.JSON[0]["text"].(string); txt != long {
+ t.Errorf("text len = %d, want %d (full content preserved, not sliced)", len(txt), len(long))
}
}
@@ -282,3 +283,197 @@ func TestGetParser_RoutesTextAndCode(t *testing.T) {
t.Fatal("TextParser does not implement ParseResultProducer")
}
}
+
+// TestTextParser_ParseWithResult_DefaultDelimiter pins the alignment fix:
+// TextParser now splits on the flow parser's default delimiter set
+// ("\n!?;。;!?"), mirroring deepdoc TxtParser.parser_txt, instead of only on
+// blank lines. keep_delimiters=True (the flow _code path) keeps each trailing
+// delimiter attached, so sentence-ending punctuation survives the split.
+func TestTextParser_ParseWithResult_DefaultDelimiter(t *testing.T) {
+ ctx := t.Context()
+ p := NewTextParser()
+
+ // Single newlines now split too (previously only "\n\n" did).
+ src := []byte("First line.\nSecond line.\nThird line.")
+ res := p.ParseWithResult(ctx, "doc.txt", src)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if len(res.JSON) != 3 {
+ t.Fatalf("JSON len = %d, want 3 (single-newline split)", len(res.JSON))
+ }
+
+ // Sentence delimiters split and keep the delimiter attached. The period
+ // "." is NOT in the default set, so "Foo. Bar" stays joined until the ";".
+ // TrimSpace drops the incidental leading space before each delimiter (the
+ // package's established convention, also used by markdown leafText).
+ src = []byte("Hello! World? Foo. Bar; Baz。 Qux!")
+ res = p.ParseWithResult(ctx, "doc.txt", src)
+ want := []string{"Hello!", "World?", "Foo. Bar;", "Baz。", "Qux!"}
+ if len(res.JSON) != len(want) {
+ t.Fatalf("JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
+ }
+ for i, w := range want {
+ if got := res.JSON[i]["text"]; got != w {
+ t.Errorf("JSON[%d].text = %v, want %v", i, got, w)
+ }
+ }
+
+ // Chinese sentence delimiters split the same way.
+ src = []byte("这是第一句。这是第二句!第三句?结尾。")
+ res = p.ParseWithResult(ctx, "doc.txt", src)
+ if len(res.JSON) != 4 {
+ t.Fatalf("JSON len = %d, want 4 (CJK delimiter split)", len(res.JSON))
+ }
+}
+
+// TestTextParser_ParseWithResult_NewlineNormalization pins the
+// normalizeTextNewlines contract: CRLF ("\r\n") and lone-CR ("\r") line
+// endings fold to LF before splitting, so every variant of the same logical
+// content yields identical items. This mirrors rag/nlp/delim.
+// normalize_text_newlines, which is what Python splits on, so Windows-line
+// documents parse identically to Unix ones.
+func TestTextParser_ParseWithResult_NewlineNormalization(t *testing.T) {
+ ctx := t.Context()
+ p := NewTextParser()
+
+ // Same logical content expressed with LF, CRLF, and lone-CR line endings.
+ lf := "First line.\nSecond line! Third? Fourth."
+ crlf := strings.ReplaceAll(lf, "\n", "\r\n")
+ cr := strings.ReplaceAll(lf, "\n", "\r")
+
+ extract := func(src string) []string {
+ res := p.ParseWithResult(ctx, "doc.txt", []byte(src))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ out := make([]string, 0, len(res.JSON))
+ for _, it := range res.JSON {
+ if txt, _ := it["text"].(string); txt != "" {
+ out = append(out, txt)
+ }
+ }
+ return out
+ }
+
+ want := extract(lf)
+ if len(want) == 0 {
+ t.Fatal("LF baseline produced no items")
+ }
+ for _, variant := range []struct {
+ name string
+ src string
+ }{
+ {"crlf", crlf},
+ {"cr", cr},
+ } {
+ got := extract(variant.src)
+ if len(got) != len(want) {
+ t.Fatalf("%s: JSON len = %d, want %d: %#v", variant.name, len(got), len(want), got)
+ }
+ for i := range want {
+ if got[i] != want[i] {
+ t.Errorf("%s: JSON[%d].text = %q, want %q", variant.name, i, got[i], want[i])
+ }
+ }
+ }
+}
+
+// TestTextParser_AlignmentGolden verifies Go's ParseWithResult output is
+// content-equivalent to Python's _code on the shared sample, using the shared
+// concatenation-normalization alignment tool (align_test.go). Python applies
+// the OVER_CAP token merge (chunking ownership retained by the Go Chunker per
+// contract #17799), so item counts differ; the
+// comparison normalizes both (delimiters stripped, whitespace collapsed) and
+// joins on whitespace, so only CONTENT equivalence — not byte-exact layout — is
+// checked. The golden files are a NORMALIZED content baseline, not a verbatim
+// Python transcript: a fresh _code run at chunk_token_num=128 may merge into a
+// different item count/structure (e.g. the en sample collapses to one chunk while
+// the golden keeps prose and code as two items) and may collapse inter-sentence
+// newlines, so do not treat them as byte-exact.
+//
+// No generator script is committed. The baseline meta records generator, sample,
+// delimiter, keep_delimiters and chunk_token_num (see textcode.python.en/zh.golden.json);
+// sample, delimiter, keep_delimiters, chunk_token_num): call the python flow
+// _code on the sample with keep_delimiters=True and the default delimiter set,
+// then project each merged section to {"text": section[0], "doc_type_kwd": "text"}.
+func TestTextParser_AlignmentGolden(t *testing.T) {
+ ctx := t.Context()
+ p := NewTextParser()
+
+ cases := []struct {
+ name string
+ sample string
+ golden string
+ }{
+ {"en", "testdata/textcode.sample.en.txt", "testdata/textcode.python.en.golden.json"},
+ {"zh", "testdata/textcode.sample.zh.txt", "testdata/textcode.python.zh.golden.json"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ sample, err := os.ReadFile(tc.sample)
+ if err != nil {
+ t.Fatalf("read sample: %v", err)
+ }
+ res := p.ParseWithResult(ctx, tc.sample, sample)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+
+ gd := LoadGoldenDoc(t, tc.golden)
+ ignore := AcceptedDivergences(gd.Meta)
+
+ goText := FilterOutDocTypes(FilterByDocType(res.JSON, "text"), ignore)
+ pyText := FilterOutDocTypes(FilterByDocType(gd.Items, "text"), ignore)
+
+ if ok, diff := CompareAlignment(goText, pyText, TextCodeAlignOptions(DefaultTextCodeDelimiter)); !ok {
+ t.Fatalf("text&code parser not aligned with Python golden:%s", diff)
+ }
+ })
+ }
+}
+
+// TestTextParser_AdjacentDelimiters pins Go's behavior on adjacent
+// delimiters, confirming it matches Python's deepdoc TxtParser.parser_txt
+// delimiter-loop exactly (not a divergence). Both ports run the same
+// re.split(r"(%s)" % dels, txt) loop with keep_delimiters=True and merge a
+// run of adjacent delimiters into the preceding segment, so the standalone
+// second delimiter is dropped on both sides: "a!!b" → ["a!", "b"] (verified
+// against deepdoc/parser/txt_parser.py). The alignment test's delimiter-strip
+// normalization also reconciles this, but this test guards splitCapturingDelims
+// directly so a future silent change there is caught independently.
+func TestTextParser_AdjacentDelimiters(t *testing.T) {
+ ctx := t.Context()
+ p := NewTextParser()
+
+ // Two adjacent sentence delimiters: Go (and Python's parser_txt) merge
+ // them into the preceding segment and drop the standalone second delimiter.
+ src := []byte("a!!b")
+ res := p.ParseWithResult(ctx, "doc.txt", src)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ want := []string{"a!", "b"}
+ if len(res.JSON) != len(want) {
+ t.Fatalf("adjacent delimiters: JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
+ }
+ for i, w := range want {
+ if got := res.JSON[i]["text"]; got != w {
+ t.Errorf("adjacent delimiters: JSON[%d].text = %v, want %v", i, got, w)
+ }
+ }
+
+ // Delimiters separated by text each attach to their own segment (no merge
+ // across the gap).
+ src = []byte("x?y!z")
+ res = p.ParseWithResult(ctx, "doc.txt", src)
+ want = []string{"x?", "y!", "z"}
+ if len(res.JSON) != len(want) {
+ t.Fatalf("mixed delimiters: JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
+ }
+ for i, w := range want {
+ if got := res.JSON[i]["text"]; got != w {
+ t.Errorf("mixed delimiters: JSON[%d].text = %v, want %v", i, got, w)
+ }
+ }
+}
diff --git a/internal/parser/parser/testdata/textcode.python.en.golden.json b/internal/parser/parser/testdata/textcode.python.en.golden.json
new file mode 100644
index 0000000000..6cfda25b04
--- /dev/null
+++ b/internal/parser/parser/testdata/textcode.python.en.golden.json
@@ -0,0 +1,23 @@
+{
+ "meta": {
+ "generator": "rag/flow/parser/parser.py:_code",
+ "sample": "internal/parser/parser/testdata/textcode.sample.en.txt",
+ "delimiter": "\n!?;。;!?",
+ "chunk_token_num": 128,
+ "keep_delimiters": true,
+ "separate_tables": false,
+ "accepted_divergences": [],
+ "python_engine": "deepdoc.parser.txt_parser.RAGFlowTxtParser",
+ "note": "No generator script is committed. To regenerate: call _code on the sample with keep_delimiters=True (chunk_token_num=128, default delimiter set), project each merged section to {\"text\": section[0], \"doc_type_kwd\": \"text\"}, then dump {meta, items}. The golden is a NORMALIZED content baseline, not a verbatim Python transcript (see TestTextParser_AlignmentGolden): a fresh _code run at chunk_token_num=128 may merge into a different item count/structure and may collapse inter-sentence newlines, so this file is not byte-exact."
+ },
+ "items": [
+ {
+ "text": "RAGFlow parses plain text and source code through the text&code family. This is the first English sentence. Here is a second sentence with more detail!\n Is this a question that the parser should handle?\nA blank line separates paragraphs. Semicolons also act as delimiters;\n this clause stays attached to the previous one. The parser keeps the trailing punctuation so sentence boundaries survive the split.",
+ "doc_type_kwd": "text"
+ },
+ {
+ "text": "def greet(name):\n\n return f\"hello, {name}\"\n\ndef main():\n\n print(greet(\"world\"))\n\nLong code lines are kept as their own segments. The parser does not perform the token merge;\n the downstream chunker owns chunking per PARSER_ALIGNMENT_HANDOFF.md section 2.3.\n",
+ "doc_type_kwd": "text"
+ }
+ ]
+}
diff --git a/internal/parser/parser/testdata/textcode.python.zh.golden.json b/internal/parser/parser/testdata/textcode.python.zh.golden.json
new file mode 100644
index 0000000000..fc7b6e836b
--- /dev/null
+++ b/internal/parser/parser/testdata/textcode.python.zh.golden.json
@@ -0,0 +1,19 @@
+{
+ "meta": {
+ "generator": "rag/flow/parser/parser.py:_code",
+ "sample": "internal/parser/parser/testdata/textcode.sample.zh.txt",
+ "delimiter": "\n!?;。;!?",
+ "chunk_token_num": 128,
+ "keep_delimiters": true,
+ "separate_tables": false,
+ "accepted_divergences": [],
+ "python_engine": "deepdoc.parser.txt_parser.RAGFlowTxtParser",
+ "note": "No generator script is committed. To regenerate: call _code on the sample with keep_delimiters=True (chunk_token_num=128, default delimiter set), project each merged section to {\"text\": section[0], \"doc_type_kwd\": \"text\"}, then dump {meta, items}. The golden is a NORMALIZED content baseline, not a verbatim Python transcript (see TestTextParser_AlignmentGolden): a fresh _code run at chunk_token_num=128 may merge into a different item count/structure and may collapse inter-sentence newlines, so this file is not byte-exact."
+ },
+ "items": [
+ {
+ "text": "这是第一段中文。逗号不是分隔符,但句号是!第二句以感叹号结尾?第三句以问号结尾。中文段落同样按默认分隔符切分。",
+ "doc_type_kwd": "text"
+ }
+ ]
+}
diff --git a/internal/parser/parser/testdata/textcode.sample.en.txt b/internal/parser/parser/testdata/textcode.sample.en.txt
new file mode 100644
index 0000000000..8ed59c03ee
--- /dev/null
+++ b/internal/parser/parser/testdata/textcode.sample.en.txt
@@ -0,0 +1,11 @@
+RAGFlow parses plain text and source code through the text&code family. This is the first English sentence. Here is a second sentence with more detail! Is this a question that the parser should handle?
+
+A blank line separates paragraphs. Semicolons also act as delimiters; this clause stays attached to the previous one. The parser keeps the trailing punctuation so sentence boundaries survive the split.
+
+def greet(name):
+ return f"hello, {name}"
+
+def main():
+ print(greet("world"))
+
+Long code lines are kept as their own segments. The parser does not perform the token merge; the downstream chunker owns chunking per PARSER_ALIGNMENT_HANDOFF.md section 2.3.
diff --git a/internal/parser/parser/testdata/textcode.sample.zh.txt b/internal/parser/parser/testdata/textcode.sample.zh.txt
new file mode 100644
index 0000000000..59c5857e1b
--- /dev/null
+++ b/internal/parser/parser/testdata/textcode.sample.zh.txt
@@ -0,0 +1 @@
+这是第一段中文。逗号不是分隔符,但句号是!第二句以感叹号结尾?第三句以问号结尾。中文段落同样按默认分隔符切分。
diff --git a/internal/parser/parser/text_parser.go b/internal/parser/parser/text_parser.go
index 6f2f03769f..188f4a5bfb 100644
--- a/internal/parser/parser/text_parser.go
+++ b/internal/parser/parser/text_parser.go
@@ -22,36 +22,31 @@
// side needs a parser for these families so `text&code` resolves to a
// real ParseResultProducer.
//
-// TextParser fills that gap with a minimal but real implementation:
-// it splits the input into paragraph-sized items and emits the
-// python-compatible `{text, doc_type_kwd:"text"}` shape. The
-// python TxtParser additionally does layout-aware section
-// detection; the Go version is intentionally simpler because (a)
-// no production template currently relies on text&code for richer
-// structure than paragraph items.
+// TextParser fills that gap with a real implementation: it splits the
+// input into fine segments on the flow parser's default delimiter set and
+// emits the python-compatible `{text, doc_type_kwd:"text"}` shape. Block
+// boundaries converge to the Python flow TxtParser (which uses the same
+// delimiter set); the OVER_CAP token merge that Python applies afterwards is
+// intentionally NOT performed here — chunking ownership stays with the
+// downstream Chunker (contract #17799), so
+// item counts differ from Python's merged chunks and are reconciled by the
+// stitch-compare alignment test (align_test.go).
package parser
import (
- "bytes"
"context"
+ "regexp"
"strings"
)
// TextParser is the text&code family parser. It implements the
// structured ParseResultProducer contract directly.
-type TextParser struct {
- // maxItemBytes caps each emitted item's text length. The
- // python TxtParser uses similar paragraph-style chunking;
- // 8192 bytes is a conservative ceiling that prevents the
- // downstream chunker from receiving oversized inputs.
- maxItemBytes int
-}
+type TextParser struct{}
-// NewTextParser constructs a TextParser with the default
-// paragraph-sized chunking ceiling.
+// NewTextParser constructs a TextParser.
func NewTextParser() *TextParser {
- return &TextParser{maxItemBytes: 8192}
+ return &TextParser{}
}
// ParseWithResult emits one item per non-empty paragraph. The
@@ -65,7 +60,7 @@ func (p *TextParser) ParseWithResult(ctx context.Context, filename string, data
if !utf8Valid(data) {
return ParseResult{Err: errInvalidUTF8}
}
- items := textParserItems(data, p.maxItemBytes)
+ items := textParserItems(data)
if items == nil {
items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
}
@@ -140,32 +135,109 @@ func decodeRune(p []byte) (rune, int) {
return 0xFFFD, 1
}
-// textParserItems splits `data` into paragraph-sized chunks. The
-// split rule mirrors the python TxtParser: blank lines separate
-// paragraphs; long paragraphs are sliced at maxItemBytes boundaries.
-func textParserItems(data []byte, maxItemBytes int) []map[string]any {
+// defaultTextDelimiterPattern is the regexp alternation of the flow parser's
+// default delimiter set DefaultTextCodeDelimiter (rag/flow/parser/parser.py:_code
+// → deepdoc TxtParser default "\n!?;。;!?"), each rune re.escape'd to mirror
+// rag/nlp/delim.compile_delimiter_pattern. The Parser component has no user-facing
+// delimiter config entry, so this default
+// is exactly what the python flow always splits on. The shared DefaultTextCodeDelimiter
+// const lives in delimiter.go so production and the alignment tests use one source
+// of truth. Go's regexp.Split drops captured delimiters, so splitCapturingDelims
+// walks the match indexes manually to reproduce python's re.split(r"(%s)" % pattern, txt)
+// interleaving.
+var (
+ defaultTextDelimiterPattern = buildDelimiterPattern(DefaultTextCodeDelimiter)
+ textDelimiterSplitRe = regexp.MustCompile(defaultTextDelimiterPattern)
+ textDelimiterExactRe = regexp.MustCompile("^(?:" + defaultTextDelimiterPattern + ")$")
+)
+
+// buildDelimiterPattern builds an alternation of re.escape'd delimiter runes
+// (longest-first is a no-op here: every delimiter in the default set is a
+// single rune, so insertion order is preserved like python's stable sort).
+func buildDelimiterPattern(delims string) string {
+ parts := make([]string, 0, len(delims))
+ for _, r := range delims {
+ parts = append(parts, regexp.QuoteMeta(string(r)))
+ }
+ return strings.Join(parts, "|")
+}
+
+// normalizeTextNewlines folds CRLF and standalone CR to LF, mirroring
+// rag/nlp/delim.normalize_text_newlines so Windows-line-ending documents split
+// identically to Unix ones.
+func normalizeTextNewlines(s string) string {
+ if s == "" {
+ return s
+ }
+ s = strings.ReplaceAll(s, "\r\n", "\n")
+ return strings.ReplaceAll(s, "\r", "\n")
+}
+
+// splitCapturingDelims reproduces python re.split(r"(%s)" % pattern, s): it
+// splits on the regexp and includes each matched delimiter as its own element
+// (with empty strings between adjacent delimiters) so callers can keep or drop
+// them. Go's regexp.Split discards captured groups, hence the manual walk.
+func splitCapturingDelims(s string, re *regexp.Regexp) []string {
+ locs := re.FindAllStringIndex(s, -1)
+ if len(locs) == 0 {
+ return []string{s}
+ }
+ out := make([]string, 0, 2*len(locs)+1)
+ prev := 0
+ for _, loc := range locs {
+ out = append(out, s[prev:loc[0]])
+ out = append(out, s[loc[0]:loc[1]])
+ prev = loc[1]
+ }
+ out = append(out, s[prev:])
+ return out
+}
+
+// textParserItems splits data into fine segments on the flow parser's default
+// delimiter set, mirroring deepdoc.parser.txt_parser.TxtParser.parser_txt up to
+// (but not including) the OVER_CAP token merge. The token merge is intentionally
+// NOT performed here — chunking ownership stays with the downstream Chunker per
+// contract #17799 — so item counts differ from the
+// python flow's merged chunks and are reconciled by the stitch-compare alignment
+// test (align_test.go).
+//
+// Unlike the python signature default keep_delimiters=False, the flow _code
+// path calls TxtParser with keep_delimiters=True, so each segment keeps its
+// trailing delimiter attached (sentence-ending punctuation preserved for code
+// and prose).
+//
+// No per-item byte cap is applied: the parser is a pure delimiter splitter, just
+// like python's parser_txt (which also does no size slicing). Sizing belongs to
+// the chunker and the embedding truncation step, so a continuous run longer than
+// the embedding token budget (e.g. a minified / no-newline file) is kept as one
+// item here and collapsed to one chunk downstream — matching python's behaviour
+// rather than diverging from it.
+func textParserItems(data []byte) []map[string]any {
+ txt := normalizeTextNewlines(string(data))
+ secs := splitCapturingDelims(txt, textDelimiterSplitRe)
+
+ var paras []string
+ for i, sec := range secs {
+ if textDelimiterExactRe.MatchString(sec) {
+ continue
+ }
+ if sec == "" {
+ continue
+ }
+ // keep_delimiters=True: append the delimiter to the segment it
+ // follows, mirroring python's parser_txt.
+ if i+1 < len(secs) && textDelimiterExactRe.MatchString(secs[i+1]) {
+ sec += secs[i+1]
+ }
+ paras = append(paras, sec)
+ }
+
var items []map[string]any
- for _, raw := range bytes.Split(data, []byte("\n\n")) {
- text := strings.TrimSpace(string(raw))
+ for _, para := range paras {
+ text := strings.TrimSpace(para)
if text == "" {
continue
}
- if maxItemBytes > 0 && len(text) > maxItemBytes {
- // Slice at the nearest newline below maxItemBytes;
- // falls back to a hard slice when no newline exists.
- cut := strings.LastIndex(text[:maxItemBytes], "\n")
- if cut <= 0 {
- cut = maxItemBytes
- }
- items = append(items, map[string]any{
- "text": strings.TrimSpace(text[:cut]),
- "doc_type_kwd": "text",
- })
- text = strings.TrimSpace(text[cut:])
- if text == "" {
- continue
- }
- }
items = append(items, map[string]any{
"text": text,
"doc_type_kwd": "text",