RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): derive eng from content for DOCX/HTML TOC removal (#17585)
Jackde17dda5f817
diff --git a/internal/parser/parser/docx_parser.go b/internal/parser/parser/docx_parser.goindex 6e4177ced5..4405b17096 100644--- a/internal/parser/parser/docx_parser.go+++ b/internal/parser/parser/docx_parser.go@@ -108,7 +108,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data// (mirrors Python parser.py:892-893 remove_toc_word).if p.RemoveTOC {outlines := extractDOCXOutlines(irJSON)- sections = removeTOCWord(sections, outlines, false)+ sections = removeTOCWord(sections, outlines, isEnglishItems(sections))}if len(sections) == 0 {sections = []map[string]any{{"text": "", "doc_type_kwd": "text"}}@@ -150,7 +150,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, datafor _, ln := range lines {lineItems = append(lineItems, map[string]any{"text": ln})}- filtered := removeTOCWord(lineItems, outlines, false)+ filtered := removeTOCWord(lineItems, outlines, isEnglishItems(lineItems))rebuilt := make([]string, 0, len(filtered))for _, item := range filtered {rebuilt = append(rebuilt, itemText(item))diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.goindex f613528a70..13d9ab28f3 100644--- a/internal/parser/parser/html_parser.go+++ b/internal/parser/parser/html_parser.go@@ -86,7 +86,7 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data// remove_toc: post-parse text heuristic (mirrors Python// parser.py:1087-1088 remove_toc → remove_contents_table).if p.RemoveTOC {- items = removeContentsTable(items, false)+ items = removeContentsTable(items, isEnglishItems(items))}if items == nil {items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}diff --git a/internal/parser/parser/text_toc.go b/internal/parser/parser/text_toc.goindex bfdedd8db3..e31e42d4d6 100644--- a/internal/parser/parser/text_toc.go+++ b/internal/parser/parser/text_toc.go@@ -105,3 +105,44 @@ func tocPrefix(text string, eng bool) string {}return string(runes[:3])}++// isEnglishTexts reports whether the given texts read as English. It returns+// true when more than 80% of the (up to 200) sampled non-empty segments consist+// solely of ASCII letters/digits/punctuation. The result selects the TOC-entry+// prefix strategy: a 2-word prefix for English content, otherwise a 3-character+// prefix (used by removeContentsTable / tocPrefix).+func isEnglishTexts(texts []string) bool {+ var sampled []string+ for _, t := range texts {+ if s := strings.TrimSpace(t); s != "" {+ sampled = append(sampled, s)+ }+ if len(sampled) >= 200 {+ break+ }+ }+ if len(sampled) == 0 {+ return false+ }+ pat := regexp.MustCompile("^[" + "`" + "a-zA-Z0-9\\s.,':;/\"?<>!()-]+$")+ eng := 0+ for _, t := range sampled {+ if pat.MatchString(t) {+ eng+++ }+ }+ return float64(eng)/float64(len(sampled)) > 0.8+}++// isEnglishItems reports whether the parser items read as English, classifying+// each item's text with isEnglishTexts. It is passed to removeTOCWord /+// removeContentsTable so English documents use the 2-word TOC prefix.+func isEnglishItems(items []map[string]any) bool {+ texts := make([]string, 0, len(items))+ for _, it := range items {+ if t := itemText(it); t != "" {+ texts = append(texts, t)+ }+ }+ return isEnglishTexts(texts)+}diff --git a/internal/parser/parser/text_toc_test.go b/internal/parser/parser/text_toc_test.goindex a4c3a66a63..01ceb8137b 100644--- a/internal/parser/parser/text_toc_test.go+++ b/internal/parser/parser/text_toc_test.go@@ -106,3 +106,60 @@ func TestRemoveContentsTable(t *testing.T) {})}}++// TestIsEnglishTexts classifies text as English when >80% of sampled segments+// are ASCII.+func TestIsEnglishTexts(t *testing.T) {+ cases := []struct {+ name string+ texts []string+ want bool+ }{+ {name: "empty", texts: nil, want: false},+ {name: "all english", texts: []string{"Chapter One", "Section Two", "Body text here"}, want: true},+ {name: "all chinese", texts: []string{"第一章 概述", "第二章 方法", "正文内容"}, want: false},+ {name: "mostly english (>80%)", texts: []string{+ "Introduction to the system", "Background and related work",+ "Methodology details", "Evaluation results", "Conclusion",+ "第一章", // single CJK line among 6 (>80% english)+ }, want: true},+ {name: "exactly 80% english", texts: []string{+ "Introduction", "Background", "Method", "Conclusion", "第一章",+ }, want: false}, // 4/5 == 80% is NOT > 80%+ {name: "mostly chinese (<80% english)", texts: []string{+ "第一章 概述", "第二章 方法", "第三章 结果", "English abstract only",+ }, want: false},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ if got := isEnglishTexts(tc.texts); got != tc.want {+ t.Errorf("isEnglishTexts(%v) = %v, want %v", tc.texts, got, tc.want)+ }+ })+ }+}++// TestRemoveTOCWordEnglishDetection verifies English content uses the 2-word+// TOC prefix, so "Chapter 2 Method" is NOT over-deleted (a 3-character prefix+// would have dropped it).+func TestRemoveTOCWordEnglishDetection(t *testing.T) {+ items := []map[string]any{+ {"text": "Intro"},+ {"text": "Contents"},+ {"text": "Chapter 1 Overview"},+ {"text": "Chapter 2 Method"},+ {"text": "Body starts"},+ }+ if !isEnglishItems(items) {+ t.Fatalf("expected isEnglishItems=true for English content")+ }+ got := removeContentsTable(items, isEnglishItems(items))+ want := []map[string]any{+ {"text": "Intro"},+ {"text": "Chapter 2 Method"},+ {"text": "Body starts"},+ }+ if !reflect.DeepEqual(got, want) {+ t.Errorf("English TOC detection: got %v, want %v", got, want)+ }+}