RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): break EPUB text at the end of every block element (#21104)
Lukas7ee5174fc335
diff --git a/internal/parser/parser/epub_parser.go b/internal/parser/parser/epub_parser.goindex 7999fb68fa..acf6d1d0b9 100644--- a/internal/parser/parser/epub_parser.go+++ b/internal/parser/parser/epub_parser.go@@ -197,6 +197,8 @@ var (epubTagRe = regexp.MustCompile(`<[^>]+>`)epubEntityRe = regexp.MustCompile(`&(?:[a-zA-Z][a-zA-Z0-9]*|#[0-9]+|#[xX][0-9a-fA-F]+);`)epubWSRe = regexp.MustCompile(`\s+`)+ // Closing tags of block elements, or a void <hr>, that end a run of text.+ epubBlockEndRe = regexp.MustCompile(`^(?:</(?:title|main|section|article|header|footer|aside|nav|figure|figcaption|caption|table|pre|address|dl|dt|dd|ul|ol)[\s>]|<hr[\s/>])`))// extractEPUBTextItems reads every spine item and returns one text item per@@ -292,7 +294,11 @@ func stripHTMLTags(s string) string {strings.HasPrefix(lower, "</h6"),strings.HasPrefix(lower, "</tr"),strings.HasPrefix(lower, "</li"),- strings.HasPrefix(lower, "</blockquote"):+ strings.HasPrefix(lower, "</blockquote"),+ // Other elements a reader shows as blocks. Without a break, the+ // head's <title> ran into the first word of the body, and a term+ // into its definition ("TermDefinition").+ epubBlockEndRe.MatchString(lower):return "\n"case strings.HasPrefix(lower, "</td"),strings.HasPrefix(lower, "</th"):diff --git a/internal/parser/parser/epub_parser_test.go b/internal/parser/parser/epub_parser_test.goindex f80a2d8630..8fba6626ca 100644--- a/internal/parser/parser/epub_parser_test.go+++ b/internal/parser/parser/epub_parser_test.go@@ -207,6 +207,23 @@ func TestEPUBParser_ResolvesManifestHrefsAsURLs(t *testing.T) {}}+// The head's <title> and block elements outside the old list ended without a+// break, so their text ran into the next word: "Testchapter text" and+// "TermDefinition".+func TestEPUBParser_SeparatesBlockElements(t *testing.T) {+ res := parseTestEPUB(t, []epubTestChapter{+ {name: "ch1.xhtml", content: epubTestHTML(+ "<section>chapter text</section><dl><dt>Term</dt><dd>Definition</dd></dl>" ++ "<figure><figcaption>Figure 1</figcaption></figure><main>Afterword</main>Epilogue")},+ })+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if got, want := epubTestText(res), "Test chapter text Term Definition Figure 1 Afterword Epilogue"; got != want {+ t.Errorf("text = %q, want %q", got, want)+ }+}+// XHTML has no and EPUB tools often write numeric character// references; only a fixed list of named entities was decoded, so// "don’t" reached the index as written.