RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(parser): align TextParser with Python _code delimiter split (#18128)
Jack7b2d052f8a09
diff --git a/internal/parser/parser/parse_with_result_test.go b/internal/parser/parser/parse_with_result_test.goindex 8ab285dd69..a33d274a99 100644--- a/internal/parser/parser/parse_with_result_test.go+++ b/internal/parser/parser/parse_with_result_test.go@@ -33,6 +33,7 @@package parserimport (+ "os""strings""testing"@@ -85,10 +86,12 @@ func TestTextParser_ParseWithResult_Empty(t *testing.T) {}}-// TestTextParser_ParseWithResult_LongParagraphSlicing pins the-// maxItemBytes boundary behaviour. A single paragraph longer-// than 8192 bytes is sliced at the nearest line boundary.-func TestTextParser_ParseWithResult_LongParagraphSlicing(t *testing.T) {+// TestTextParser_ParseWithResult_NoSizeCap pins that the parser performs no+// per-item byte slicing: a single continuous run longer than any prior cap+// (here 9000 'a's with no delimiter) stays as one item whose full content is+// preserved. Sizing is delegated to the chunker / embedding truncation, matching+// python's parser_txt (which also does no size slicing).+func TestTextParser_ParseWithResult_NoSizeCap(t *testing.T) {ctx := t.Context()p := NewTextParser()long := strings.Repeat("a", 9000)@@ -96,13 +99,11 @@ func TestTextParser_ParseWithResult_LongParagraphSlicing(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if len(res.JSON) < 2 {- t.Errorf("JSON len = %d, want >=2 (sliced at maxItemBytes)", len(res.JSON))+ if len(res.JSON) != 1 {+ t.Fatalf("JSON len = %d, want 1 (no per-item size cap)", len(res.JSON))}- for i, it := range res.JSON {- if txt, _ := it["text"].(string); len(txt) > 8192 {- t.Errorf("JSON[%d].text len = %d, exceeds maxItemBytes=8192", i, len(txt))- }+ if txt, _ := res.JSON[0]["text"].(string); txt != long {+ t.Errorf("text len = %d, want %d (full content preserved, not sliced)", len(txt), len(long))}}@@ -282,3 +283,197 @@ func TestGetParser_RoutesTextAndCode(t *testing.T) {t.Fatal("TextParser does not implement ParseResultProducer")}}++// TestTextParser_ParseWithResult_DefaultDelimiter pins the alignment fix:+// TextParser now splits on the flow parser's default delimiter set+// ("\n!?;。;!?"), mirroring deepdoc TxtParser.parser_txt, instead of only on+// blank lines. keep_delimiters=True (the flow _code path) keeps each trailing+// delimiter attached, so sentence-ending punctuation survives the split.+func TestTextParser_ParseWithResult_DefaultDelimiter(t *testing.T) {+ ctx := t.Context()+ p := NewTextParser()++ // Single newlines now split too (previously only "\n\n" did).+ src := []byte("First line.\nSecond line.\nThird line.")+ res := p.ParseWithResult(ctx, "doc.txt", src)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if len(res.JSON) != 3 {+ t.Fatalf("JSON len = %d, want 3 (single-newline split)", len(res.JSON))+ }++ // Sentence delimiters split and keep the delimiter attached. The period+ // "." is NOT in the default set, so "Foo. Bar" stays joined until the ";".+ // TrimSpace drops the incidental leading space before each delimiter (the+ // package's established convention, also used by markdown leafText).+ src = []byte("Hello! World? Foo. Bar; Baz。 Qux!")+ res = p.ParseWithResult(ctx, "doc.txt", src)+ want := []string{"Hello!", "World?", "Foo. Bar;", "Baz。", "Qux!"}+ if len(res.JSON) != len(want) {+ t.Fatalf("JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)+ }+ for i, w := range want {+ if got := res.JSON[i]["text"]; got != w {+ t.Errorf("JSON[%d].text = %v, want %v", i, got, w)+ }+ }++ // Chinese sentence delimiters split the same way.+ src = []byte("这是第一句。这是第二句!第三句?结尾。")+ res = p.ParseWithResult(ctx, "doc.txt", src)+ if len(res.JSON) != 4 {+ t.Fatalf("JSON len = %d, want 4 (CJK delimiter split)", len(res.JSON))+ }+}++// TestTextParser_ParseWithResult_NewlineNormalization pins the+// normalizeTextNewlines contract: CRLF ("\r\n") and lone-CR ("\r") line+// endings fold to LF before splitting, so every variant of the same logical+// content yields identical items. This mirrors rag/nlp/delim.+// normalize_text_newlines, which is what Python splits on, so Windows-line+// documents parse identically to Unix ones.+func TestTextParser_ParseWithResult_NewlineNormalization(t *testing.T) {+ ctx := t.Context()+ p := NewTextParser()++ // Same logical content expressed with LF, CRLF, and lone-CR line endings.+ lf := "First line.\nSecond line! Third? Fourth."+ crlf := strings.ReplaceAll(lf, "\n", "\r\n")+ cr := strings.ReplaceAll(lf, "\n", "\r")++ extract := func(src string) []string {+ res := p.ParseWithResult(ctx, "doc.txt", []byte(src))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ out := make([]string, 0, len(res.JSON))+ for _, it := range res.JSON {+ if txt, _ := it["text"].(string); txt != "" {+ out = append(out, txt)+ }+ }+ return out+ }++ want := extract(lf)+ if len(want) == 0 {+ t.Fatal("LF baseline produced no items")+ }+ for _, variant := range []struct {+ name string+ src string+ }{+ {"crlf", crlf},+ {"cr", cr},+ } {+ got := extract(variant.src)+ if len(got) != len(want) {+ t.Fatalf("%s: JSON len = %d, want %d: %#v", variant.name, len(got), len(want), got)+ }+ for i := range want {+ if got[i] != want[i] {+ t.Errorf("%s: JSON[%d].text = %q, want %q", variant.name, i, got[i], want[i])+ }+ }+ }+}++// TestTextParser_AlignmentGolden verifies Go's ParseWithResult output is+// content-equivalent to Python's _code on the shared sample, using the shared+// concatenation-normalization alignment tool (align_test.go). Python applies+// the OVER_CAP token merge (chunking ownership retained by the Go Chunker per+// contract #17799), so item counts differ; the+// comparison normalizes both (delimiters stripped, whitespace collapsed) and+// joins on whitespace, so only CONTENT equivalence — not byte-exact layout — is+// checked. The golden files are a NORMALIZED content baseline, not a verbatim+// Python transcript: a fresh _code run at chunk_token_num=128 may merge into a+// different item count/structure (e.g. the en sample collapses to one chunk while+// the golden keeps prose and code as two items) and may collapse inter-sentence+// newlines, so do not treat them as byte-exact.+//+// No generator script is committed. The baseline meta records generator, sample,+// delimiter, keep_delimiters and chunk_token_num (see textcode.python.en/zh.golden.json);+// sample, delimiter, keep_delimiters, chunk_token_num): call the python flow+// _code on the sample with keep_delimiters=True and the default delimiter set,+// then project each merged section to {"text": section[0], "doc_type_kwd": "text"}.+func TestTextParser_AlignmentGolden(t *testing.T) {+ ctx := t.Context()+ p := NewTextParser()++ cases := []struct {+ name string+ sample string+ golden string+ }{+ {"en", "testdata/textcode.sample.en.txt", "testdata/textcode.python.en.golden.json"},+ {"zh", "testdata/textcode.sample.zh.txt", "testdata/textcode.python.zh.golden.json"},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ sample, err := os.ReadFile(tc.sample)+ if err != nil {+ t.Fatalf("read sample: %v", err)+ }+ res := p.ParseWithResult(ctx, tc.sample, sample)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }++ gd := LoadGoldenDoc(t, tc.golden)+ ignore := AcceptedDivergences(gd.Meta)++ goText := FilterOutDocTypes(FilterByDocType(res.JSON, "text"), ignore)+ pyText := FilterOutDocTypes(FilterByDocType(gd.Items, "text"), ignore)++ if ok, diff := CompareAlignment(goText, pyText, TextCodeAlignOptions(DefaultTextCodeDelimiter)); !ok {+ t.Fatalf("text&code parser not aligned with Python golden:%s", diff)+ }+ })+ }+}++// TestTextParser_AdjacentDelimiters pins Go's behavior on adjacent+// delimiters, confirming it matches Python's deepdoc TxtParser.parser_txt+// delimiter-loop exactly (not a divergence). Both ports run the same+// re.split(r"(%s)" % dels, txt) loop with keep_delimiters=True and merge a+// run of adjacent delimiters into the preceding segment, so the standalone+// second delimiter is dropped on both sides: "a!!b" → ["a!", "b"] (verified+// against deepdoc/parser/txt_parser.py). The alignment test's delimiter-strip+// normalization also reconciles this, but this test guards splitCapturingDelims+// directly so a future silent change there is caught independently.+func TestTextParser_AdjacentDelimiters(t *testing.T) {+ ctx := t.Context()+ p := NewTextParser()++ // Two adjacent sentence delimiters: Go (and Python's parser_txt) merge+ // them into the preceding segment and drop the standalone second delimiter.+ src := []byte("a!!b")+ res := p.ParseWithResult(ctx, "doc.txt", src)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ want := []string{"a!", "b"}+ if len(res.JSON) != len(want) {+ t.Fatalf("adjacent delimiters: JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)+ }+ for i, w := range want {+ if got := res.JSON[i]["text"]; got != w {+ t.Errorf("adjacent delimiters: JSON[%d].text = %v, want %v", i, got, w)+ }+ }++ // Delimiters separated by text each attach to their own segment (no merge+ // across the gap).+ src = []byte("x?y!z")+ res = p.ParseWithResult(ctx, "doc.txt", src)+ want = []string{"x?", "y!", "z"}+ if len(res.JSON) != len(want) {+ t.Fatalf("mixed delimiters: JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)+ }+ for i, w := range want {+ if got := res.JSON[i]["text"]; got != w {+ t.Errorf("mixed delimiters: JSON[%d].text = %v, want %v", i, got, w)+ }+ }+}diff --git a/internal/parser/parser/testdata/textcode.python.en.golden.json b/internal/parser/parser/testdata/textcode.python.en.golden.jsonnew file mode 100644index 0000000000..6cfda25b04--- /dev/null+++ b/internal/parser/parser/testdata/textcode.python.en.golden.json@@ -0,0 +1,23 @@+{+ "meta": {+ "generator": "rag/flow/parser/parser.py:_code",+ "sample": "internal/parser/parser/testdata/textcode.sample.en.txt",+ "delimiter": "\n!?;。;!?",+ "chunk_token_num": 128,+ "keep_delimiters": true,+ "separate_tables": false,+ "accepted_divergences": [],+ "python_engine": "deepdoc.parser.txt_parser.RAGFlowTxtParser",+ "note": "No generator script is committed. To regenerate: call _code on the sample with keep_delimiters=True (chunk_token_num=128, default delimiter set), project each merged section to {\"text\": section[0], \"doc_type_kwd\": \"text\"}, then dump {meta, items}. The golden is a NORMALIZED content baseline, not a verbatim Python transcript (see TestTextParser_AlignmentGolden): a fresh _code run at chunk_token_num=128 may merge into a different item count/structure and may collapse inter-sentence newlines, so this file is not byte-exact."+ },+ "items": [+ {+ "text": "RAGFlow parses plain text and source code through the text&code family. This is the first English sentence. Here is a second sentence with more detail!\n Is this a question that the parser should handle?\nA blank line separates paragraphs. Semicolons also act as delimiters;\n this clause stays attached to the previous one. The parser keeps the trailing punctuation so sentence boundaries survive the split.",+ "doc_type_kwd": "text"+ },+ {+ "text": "def greet(name):\n\n return f\"hello, {name}\"\n\ndef main():\n\n print(greet(\"world\"))\n\nLong code lines are kept as their own segments. The parser does not perform the token merge;\n the downstream chunker owns chunking per PARSER_ALIGNMENT_HANDOFF.md section 2.3.\n",+ "doc_type_kwd": "text"+ }+ ]+}diff --git a/internal/parser/parser/testdata/textcode.python.zh.golden.json b/internal/parser/parser/testdata/textcode.python.zh.golden.jsonnew file mode 100644index 0000000000..fc7b6e836b--- /dev/null+++ b/internal/parser/parser/testdata/textcode.python.zh.golden.json@@ -0,0 +1,19 @@+{+ "meta": {+ "generator": "rag/flow/parser/parser.py:_code",+ "sample": "internal/parser/parser/testdata/textcode.sample.zh.txt",+ "delimiter": "\n!?;。;!?",+ "chunk_token_num": 128,+ "keep_delimiters": true,+ "separate_tables": false,+ "accepted_divergences": [],+ "python_engine": "deepdoc.parser.txt_parser.RAGFlowTxtParser",+ "note": "No generator script is committed. To regenerate: call _code on the sample with keep_delimiters=True (chunk_token_num=128, default delimiter set), project each merged section to {\"text\": section[0], \"doc_type_kwd\": \"text\"}, then dump {meta, items}. The golden is a NORMALIZED content baseline, not a verbatim Python transcript (see TestTextParser_AlignmentGolden): a fresh _code run at chunk_token_num=128 may merge into a different item count/structure and may collapse inter-sentence newlines, so this file is not byte-exact."+ },+ "items": [+ {+ "text": "这是第一段中文。逗号不是分隔符,但句号是!第二句以感叹号结尾?第三句以问号结尾。中文段落同样按默认分隔符切分。",+ "doc_type_kwd": "text"+ }+ ]+}diff --git a/internal/parser/parser/testdata/textcode.sample.en.txt b/internal/parser/parser/testdata/textcode.sample.en.txtnew file mode 100644index 0000000000..8ed59c03ee--- /dev/null+++ b/internal/parser/parser/testdata/textcode.sample.en.txt@@ -0,0 +1,11 @@+RAGFlow parses plain text and source code through the text&code family. This is the first English sentence. Here is a second sentence with more detail! Is this a question that the parser should handle?++A blank line separates paragraphs. Semicolons also act as delimiters; this clause stays attached to the previous one. The parser keeps the trailing punctuation so sentence boundaries survive the split.++def greet(name):+ return f"hello, {name}"++def main():+ print(greet("world"))++Long code lines are kept as their own segments. The parser does not perform the token merge; the downstream chunker owns chunking per PARSER_ALIGNMENT_HANDOFF.md section 2.3.diff --git a/internal/parser/parser/testdata/textcode.sample.zh.txt b/internal/parser/parser/testdata/textcode.sample.zh.txtnew file mode 100644index 0000000000..59c5857e1b--- /dev/null+++ b/internal/parser/parser/testdata/textcode.sample.zh.txt@@ -0,0 +1 @@+这是第一段中文。逗号不是分隔符,但句号是!第二句以感叹号结尾?第三句以问号结尾。中文段落同样按默认分隔符切分。diff --git a/internal/parser/parser/text_parser.go b/internal/parser/parser/text_parser.goindex 6f2f03769f..188f4a5bfb 100644--- a/internal/parser/parser/text_parser.go+++ b/internal/parser/parser/text_parser.go@@ -22,36 +22,31 @@// side needs a parser for these families so `text&code` resolves to a// real ParseResultProducer.//-// TextParser fills that gap with a minimal but real implementation:-// it splits the input into paragraph-sized items and emits the-// python-compatible `{text, doc_type_kwd:"text"}` shape. The-// python TxtParser additionally does layout-aware section-// detection; the Go version is intentionally simpler because (a)-// no production template currently relies on text&code for richer-// structure than paragraph items.+// TextParser fills that gap with a real implementation: it splits the+// input into fine segments on the flow parser's default delimiter set and+// emits the python-compatible `{text, doc_type_kwd:"text"}` shape. Block+// boundaries converge to the Python flow TxtParser (which uses the same+// delimiter set); the OVER_CAP token merge that Python applies afterwards is+// intentionally NOT performed here — chunking ownership stays with the+// downstream Chunker (contract #17799), so+// item counts differ from Python's merged chunks and are reconciled by the+// stitch-compare alignment test (align_test.go).package parserimport (- "bytes""context"+ "regexp""strings")// TextParser is the text&code family parser. It implements the// structured ParseResultProducer contract directly.-type TextParser struct {- // maxItemBytes caps each emitted item's text length. The- // python TxtParser uses similar paragraph-style chunking;- // 8192 bytes is a conservative ceiling that prevents the- // downstream chunker from receiving oversized inputs.- maxItemBytes int-}+type TextParser struct{}-// NewTextParser constructs a TextParser with the default-// paragraph-sized chunking ceiling.+// NewTextParser constructs a TextParser.func NewTextParser() *TextParser {- return &TextParser{maxItemBytes: 8192}+ return &TextParser{}}// ParseWithResult emits one item per non-empty paragraph. The@@ -65,7 +60,7 @@ func (p *TextParser) ParseWithResult(ctx context.Context, filename string, dataif !utf8Valid(data) {return ParseResult{Err: errInvalidUTF8}}- items := textParserItems(data, p.maxItemBytes)+ items := textParserItems(data)if items == nil {items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}}@@ -140,32 +135,109 @@ func decodeRune(p []byte) (rune, int) {return 0xFFFD, 1}-// textParserItems splits `data` into paragraph-sized chunks. The-// split rule mirrors the python TxtParser: blank lines separate-// paragraphs; long paragraphs are sliced at maxItemBytes boundaries.-func textParserItems(data []byte, maxItemBytes int) []map[string]any {+// defaultTextDelimiterPattern is the regexp alternation of the flow parser's+// default delimiter set DefaultTextCodeDelimiter (rag/flow/parser/parser.py:_code+// → deepdoc TxtParser default "\n!?;。;!?"), each rune re.escape'd to mirror+// rag/nlp/delim.compile_delimiter_pattern. The Parser component has no user-facing+// delimiter config entry, so this default+// is exactly what the python flow always splits on. The shared DefaultTextCodeDelimiter+// const lives in delimiter.go so production and the alignment tests use one source+// of truth. Go's regexp.Split drops captured delimiters, so splitCapturingDelims+// walks the match indexes manually to reproduce python's re.split(r"(%s)" % pattern, txt)+// interleaving.+var (+ defaultTextDelimiterPattern = buildDelimiterPattern(DefaultTextCodeDelimiter)+ textDelimiterSplitRe = regexp.MustCompile(defaultTextDelimiterPattern)+ textDelimiterExactRe = regexp.MustCompile("^(?:" + defaultTextDelimiterPattern + ")$")+)++// buildDelimiterPattern builds an alternation of re.escape'd delimiter runes+// (longest-first is a no-op here: every delimiter in the default set is a+// single rune, so insertion order is preserved like python's stable sort).+func buildDelimiterPattern(delims string) string {+ parts := make([]string, 0, len(delims))+ for _, r := range delims {+ parts = append(parts, regexp.QuoteMeta(string(r)))+ }+ return strings.Join(parts, "|")+}++// normalizeTextNewlines folds CRLF and standalone CR to LF, mirroring+// rag/nlp/delim.normalize_text_newlines so Windows-line-ending documents split+// identically to Unix ones.+func normalizeTextNewlines(s string) string {+ if s == "" {+ return s+ }+ s = strings.ReplaceAll(s, "\r\n", "\n")+ return strings.ReplaceAll(s, "\r", "\n")+}++// splitCapturingDelims reproduces python re.split(r"(%s)" % pattern, s): it+// splits on the regexp and includes each matched delimiter as its own element+// (with empty strings between adjacent delimiters) so callers can keep or drop+// them. Go's regexp.Split discards captured groups, hence the manual walk.+func splitCapturingDelims(s string, re *regexp.Regexp) []string {+ locs := re.FindAllStringIndex(s, -1)+ if len(locs) == 0 {+ return []string{s}+ }+ out := make([]string, 0, 2*len(locs)+1)+ prev := 0+ for _, loc := range locs {+ out = append(out, s[prev:loc[0]])+ out = append(out, s[loc[0]:loc[1]])+ prev = loc[1]+ }+ out = append(out, s[prev:])+ return out+}++// textParserItems splits data into fine segments on the flow parser's default+// delimiter set, mirroring deepdoc.parser.txt_parser.TxtParser.parser_txt up to+// (but not including) the OVER_CAP token merge. The token merge is intentionally+// NOT performed here — chunking ownership stays with the downstream Chunker per+// contract #17799 — so item counts differ from the+// python flow's merged chunks and are reconciled by the stitch-compare alignment+// test (align_test.go).+//+// Unlike the python signature default keep_delimiters=False, the flow _code+// path calls TxtParser with keep_delimiters=True, so each segment keeps its+// trailing delimiter attached (sentence-ending punctuation preserved for code+// and prose).+//+// No per-item byte cap is applied: the parser is a pure delimiter splitter, just+// like python's parser_txt (which also does no size slicing). Sizing belongs to+// the chunker and the embedding truncation step, so a continuous run longer than+// the embedding token budget (e.g. a minified / no-newline file) is kept as one+// item here and collapsed to one chunk downstream — matching python's behaviour+// rather than diverging from it.+func textParserItems(data []byte) []map[string]any {+ txt := normalizeTextNewlines(string(data))+ secs := splitCapturingDelims(txt, textDelimiterSplitRe)++ var paras []string+ for i, sec := range secs {+ if textDelimiterExactRe.MatchString(sec) {+ continue+ }+ if sec == "" {+ continue+ }+ // keep_delimiters=True: append the delimiter to the segment it+ // follows, mirroring python's parser_txt.+ if i+1 < len(secs) && textDelimiterExactRe.MatchString(secs[i+1]) {+ sec += secs[i+1]+ }+ paras = append(paras, sec)+ }+var items []map[string]any- for _, raw := range bytes.Split(data, []byte("\n\n")) {- text := strings.TrimSpace(string(raw))+ for _, para := range paras {+ text := strings.TrimSpace(para)if text == "" {continue}- if maxItemBytes > 0 && len(text) > maxItemBytes {- // Slice at the nearest newline below maxItemBytes;- // falls back to a hard slice when no newline exists.- cut := strings.LastIndex(text[:maxItemBytes], "\n")- if cut <= 0 {- cut = maxItemBytes- }- items = append(items, map[string]any{- "text": strings.TrimSpace(text[:cut]),- "doc_type_kwd": "text",- })- text = strings.TrimSpace(text[cut:])- if text == "" {- continue- }- }items = append(items, map[string]any{"text": text,"doc_type_kwd": "text",