Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): break EPUB text at the end of every block element (#21104)

Lukas7ee5174fc335
diff --git a/internal/parser/parser/epub_parser.go b/internal/parser/parser/epub_parser.go
index 7999fb68fa..acf6d1d0b9 100644
--- a/internal/parser/parser/epub_parser.go
+++ b/internal/parser/parser/epub_parser.go
@@ -197,6 +197,8 @@ var (
epubTagRe = regexp.MustCompile(`<[^>]+>`)
epubEntityRe = regexp.MustCompile(`&(?:[a-zA-Z][a-zA-Z0-9]*|#[0-9]+|#[xX][0-9a-fA-F]+);`)
epubWSRe = regexp.MustCompile(`\s+`)
+ // Closing tags of block elements, or a void <hr>, that end a run of text.
+ epubBlockEndRe = regexp.MustCompile(`^(?:</(?:title|main|section|article|header|footer|aside|nav|figure|figcaption|caption|table|pre|address|dl|dt|dd|ul|ol)[\s>]|<hr[\s/>])`)
)
// extractEPUBTextItems reads every spine item and returns one text item per
@@ -292,7 +294,11 @@ func stripHTMLTags(s string) string {
strings.HasPrefix(lower, "</h6"),
strings.HasPrefix(lower, "</tr"),
strings.HasPrefix(lower, "</li"),
- strings.HasPrefix(lower, "</blockquote"):
+ strings.HasPrefix(lower, "</blockquote"),
+ // Other elements a reader shows as blocks. Without a break, the
+ // head's <title> ran into the first word of the body, and a term
+ // into its definition ("TermDefinition").
+ epubBlockEndRe.MatchString(lower):
return "\n"
case strings.HasPrefix(lower, "</td"),
strings.HasPrefix(lower, "</th"):
diff --git a/internal/parser/parser/epub_parser_test.go b/internal/parser/parser/epub_parser_test.go
index f80a2d8630..8fba6626ca 100644
--- a/internal/parser/parser/epub_parser_test.go
+++ b/internal/parser/parser/epub_parser_test.go
@@ -207,6 +207,23 @@ func TestEPUBParser_ResolvesManifestHrefsAsURLs(t *testing.T) {
}
}
+// The head's <title> and block elements outside the old list ended without a
+// break, so their text ran into the next word: "Testchapter text" and
+// "TermDefinition".
+func TestEPUBParser_SeparatesBlockElements(t *testing.T) {
+ res := parseTestEPUB(t, []epubTestChapter{
+ {name: "ch1.xhtml", content: epubTestHTML(
+ "<section>chapter text</section><dl><dt>Term</dt><dd>Definition</dd></dl>" +
+ "<figure><figcaption>Figure 1</figcaption></figure><main>Afterword</main>Epilogue")},
+ })
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if got, want := epubTestText(res), "Test chapter text Term Definition Figure 1 Afterword Epilogue"; got != want {
+ t.Errorf("text = %q, want %q", got, want)
+ }
+}
+
// XHTML has no &nbsp; and EPUB tools often write numeric character
// references; only a fixed list of named entities was decoded, so
// "don&#8217;t" reached the index as written.