Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): decode numeric and all named character references in EPUB text (#21103)

Lukas6488930b7047
diff --git a/internal/parser/parser/epub_parser.go b/internal/parser/parser/epub_parser.go
index 52b496cc89..7999fb68fa 100644
--- a/internal/parser/parser/epub_parser.go
+++ b/internal/parser/parser/epub_parser.go
@@ -34,6 +34,7 @@ import (
"context"
"encoding/xml"
"fmt"
+ "html"
"io"
"net/url"
"path"
@@ -194,7 +195,7 @@ var (
epubScriptRe = regexp.MustCompile(`(?is)<script[^>]*>.*?</script>`)
epubStyleRe = regexp.MustCompile(`(?is)<style[^>]*>.*?</style>`)
epubTagRe = regexp.MustCompile(`<[^>]+>`)
- epubEntityRe = regexp.MustCompile(`&[a-zA-Z]+;`)
+ epubEntityRe = regexp.MustCompile(`&(?:[a-zA-Z][a-zA-Z0-9]*|#[0-9]+|#[xX][0-9a-fA-F]+);`)
epubWSRe = regexp.MustCompile(`\s+`)
)
@@ -337,6 +338,13 @@ func xmlEntityValue(entity string) string {
case "&hellip;":
return "…"
default:
- return entity
+ // Numeric references (&#8217;, &#x2014;, and &#160; for a
+ // non-breaking space, which XHTML has no named entity for) and the
+ // other HTML named entities. One html cannot resolve stays as written.
+ decoded := html.UnescapeString(entity)
+ if decoded == "\u00a0" {
+ return " "
+ }
+ return decoded
}
}
diff --git a/internal/parser/parser/epub_parser_test.go b/internal/parser/parser/epub_parser_test.go
index 2d2170cf43..f80a2d8630 100644
--- a/internal/parser/parser/epub_parser_test.go
+++ b/internal/parser/parser/epub_parser_test.go
@@ -206,3 +206,19 @@ func TestEPUBParser_ResolvesManifestHrefsAsURLs(t *testing.T) {
}
}
}
+
+// XHTML has no &nbsp; and EPUB tools often write numeric character
+// references; only a fixed list of named entities was decoded, so
+// "don&#8217;t" reached the index as written.
+func TestEPUBParser_DecodesCharacterReferences(t *testing.T) {
+ res := parseTestEPUB(t, []epubTestChapter{
+ {name: "ch1.xhtml", content: epubTestHTML(
+ "<p>don&#8217;t&#160;stop &#x2014; caf&eacute; &frac12; cup &hellip; &amp; &#38; &bogus;</p>")},
+ })
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if text, want := epubTestText(res), "don’t stop — café ½ cup … & & &bogus;"; !strings.Contains(text, want) {
+ t.Errorf("text = %q, want it to contain %q", text, want)
+ }
+}