Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): derive eng from content for DOCX/HTML TOC removal (#17585)

Jackde17dda5f817
diff --git a/internal/parser/parser/docx_parser.go b/internal/parser/parser/docx_parser.go
index 6e4177ced5..4405b17096 100644
--- a/internal/parser/parser/docx_parser.go
+++ b/internal/parser/parser/docx_parser.go
@@ -108,7 +108,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data
// (mirrors Python parser.py:892-893 remove_toc_word).
if p.RemoveTOC {
outlines := extractDOCXOutlines(irJSON)
- sections = removeTOCWord(sections, outlines, false)
+ sections = removeTOCWord(sections, outlines, isEnglishItems(sections))
}
if len(sections) == 0 {
sections = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
@@ -150,7 +150,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data
for _, ln := range lines {
lineItems = append(lineItems, map[string]any{"text": ln})
}
- filtered := removeTOCWord(lineItems, outlines, false)
+ filtered := removeTOCWord(lineItems, outlines, isEnglishItems(lineItems))
rebuilt := make([]string, 0, len(filtered))
for _, item := range filtered {
rebuilt = append(rebuilt, itemText(item))
diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.go
index f613528a70..13d9ab28f3 100644
--- a/internal/parser/parser/html_parser.go
+++ b/internal/parser/parser/html_parser.go
@@ -86,7 +86,7 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data
// remove_toc: post-parse text heuristic (mirrors Python
// parser.py:1087-1088 remove_toc → remove_contents_table).
if p.RemoveTOC {
- items = removeContentsTable(items, false)
+ items = removeContentsTable(items, isEnglishItems(items))
}
if items == nil {
items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
diff --git a/internal/parser/parser/text_toc.go b/internal/parser/parser/text_toc.go
index bfdedd8db3..e31e42d4d6 100644
--- a/internal/parser/parser/text_toc.go
+++ b/internal/parser/parser/text_toc.go
@@ -105,3 +105,44 @@ func tocPrefix(text string, eng bool) string {
}
return string(runes[:3])
}
+
+// isEnglishTexts reports whether the given texts read as English. It returns
+// true when more than 80% of the (up to 200) sampled non-empty segments consist
+// solely of ASCII letters/digits/punctuation. The result selects the TOC-entry
+// prefix strategy: a 2-word prefix for English content, otherwise a 3-character
+// prefix (used by removeContentsTable / tocPrefix).
+func isEnglishTexts(texts []string) bool {
+ var sampled []string
+ for _, t := range texts {
+ if s := strings.TrimSpace(t); s != "" {
+ sampled = append(sampled, s)
+ }
+ if len(sampled) >= 200 {
+ break
+ }
+ }
+ if len(sampled) == 0 {
+ return false
+ }
+ pat := regexp.MustCompile("^[" + "`" + "a-zA-Z0-9\\s.,':;/\"?<>!()-]+$")
+ eng := 0
+ for _, t := range sampled {
+ if pat.MatchString(t) {
+ eng++
+ }
+ }
+ return float64(eng)/float64(len(sampled)) > 0.8
+}
+
+// isEnglishItems reports whether the parser items read as English, classifying
+// each item's text with isEnglishTexts. It is passed to removeTOCWord /
+// removeContentsTable so English documents use the 2-word TOC prefix.
+func isEnglishItems(items []map[string]any) bool {
+ texts := make([]string, 0, len(items))
+ for _, it := range items {
+ if t := itemText(it); t != "" {
+ texts = append(texts, t)
+ }
+ }
+ return isEnglishTexts(texts)
+}
diff --git a/internal/parser/parser/text_toc_test.go b/internal/parser/parser/text_toc_test.go
index a4c3a66a63..01ceb8137b 100644
--- a/internal/parser/parser/text_toc_test.go
+++ b/internal/parser/parser/text_toc_test.go
@@ -106,3 +106,60 @@ func TestRemoveContentsTable(t *testing.T) {
})
}
}
+
+// TestIsEnglishTexts classifies text as English when >80% of sampled segments
+// are ASCII.
+func TestIsEnglishTexts(t *testing.T) {
+ cases := []struct {
+ name string
+ texts []string
+ want bool
+ }{
+ {name: "empty", texts: nil, want: false},
+ {name: "all english", texts: []string{"Chapter One", "Section Two", "Body text here"}, want: true},
+ {name: "all chinese", texts: []string{"第一章 概述", "第二章 方法", "正文内容"}, want: false},
+ {name: "mostly english (>80%)", texts: []string{
+ "Introduction to the system", "Background and related work",
+ "Methodology details", "Evaluation results", "Conclusion",
+ "第一章", // single CJK line among 6 (>80% english)
+ }, want: true},
+ {name: "exactly 80% english", texts: []string{
+ "Introduction", "Background", "Method", "Conclusion", "第一章",
+ }, want: false}, // 4/5 == 80% is NOT > 80%
+ {name: "mostly chinese (<80% english)", texts: []string{
+ "第一章 概述", "第二章 方法", "第三章 结果", "English abstract only",
+ }, want: false},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := isEnglishTexts(tc.texts); got != tc.want {
+ t.Errorf("isEnglishTexts(%v) = %v, want %v", tc.texts, got, tc.want)
+ }
+ })
+ }
+}
+
+// TestRemoveTOCWordEnglishDetection verifies English content uses the 2-word
+// TOC prefix, so "Chapter 2 Method" is NOT over-deleted (a 3-character prefix
+// would have dropped it).
+func TestRemoveTOCWordEnglishDetection(t *testing.T) {
+ items := []map[string]any{
+ {"text": "Intro"},
+ {"text": "Contents"},
+ {"text": "Chapter 1 Overview"},
+ {"text": "Chapter 2 Method"},
+ {"text": "Body starts"},
+ }
+ if !isEnglishItems(items) {
+ t.Fatalf("expected isEnglishItems=true for English content")
+ }
+ got := removeContentsTable(items, isEnglishItems(items))
+ want := []map[string]any{
+ {"text": "Intro"},
+ {"text": "Chapter 2 Method"},
+ {"text": "Body starts"},
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("English TOC detection: got %v, want %v", got, want)
+ }
+}