RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): decode numeric and all named character references in EPUB text (#21103)
Lukas6488930b7047
diff --git a/internal/parser/parser/epub_parser.go b/internal/parser/parser/epub_parser.goindex 52b496cc89..7999fb68fa 100644--- a/internal/parser/parser/epub_parser.go+++ b/internal/parser/parser/epub_parser.go@@ -34,6 +34,7 @@ import ("context""encoding/xml""fmt"+ "html""io""net/url""path"@@ -194,7 +195,7 @@ var (epubScriptRe = regexp.MustCompile(`(?is)<script[^>]*>.*?</script>`)epubStyleRe = regexp.MustCompile(`(?is)<style[^>]*>.*?</style>`)epubTagRe = regexp.MustCompile(`<[^>]+>`)- epubEntityRe = regexp.MustCompile(`&[a-zA-Z]+;`)+ epubEntityRe = regexp.MustCompile(`&(?:[a-zA-Z][a-zA-Z0-9]*|#[0-9]+|#[xX][0-9a-fA-F]+);`)epubWSRe = regexp.MustCompile(`\s+`))@@ -337,6 +338,13 @@ func xmlEntityValue(entity string) string {case "…":return "…"default:- return entity+ // Numeric references (’, —, and   for a+ // non-breaking space, which XHTML has no named entity for) and the+ // other HTML named entities. One html cannot resolve stays as written.+ decoded := html.UnescapeString(entity)+ if decoded == "\u00a0" {+ return " "+ }+ return decoded}}diff --git a/internal/parser/parser/epub_parser_test.go b/internal/parser/parser/epub_parser_test.goindex 2d2170cf43..f80a2d8630 100644--- a/internal/parser/parser/epub_parser_test.go+++ b/internal/parser/parser/epub_parser_test.go@@ -206,3 +206,19 @@ func TestEPUBParser_ResolvesManifestHrefsAsURLs(t *testing.T) {}}}++// XHTML has no and EPUB tools often write numeric character+// references; only a fixed list of named entities was decoded, so+// "don’t" reached the index as written.+func TestEPUBParser_DecodesCharacterReferences(t *testing.T) {+ res := parseTestEPUB(t, []epubTestChapter{+ {name: "ch1.xhtml", content: epubTestHTML(+ "<p>don’t stop — café ½ cup … & & &bogus;</p>")},+ })+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if text, want := epubTestText(res), "don’t stop — café ½ cup … & & &bogus;"; !strings.Contains(text, want) {+ t.Errorf("text = %q, want it to contain %q", text, want)+ }+}