Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): leave out the content of <template> elements (#21284)

Lukas8e31338dd219
diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.go
index c6eec770b3..3d5286bd19 100644
--- a/internal/parser/parser/email_parser.go
+++ b/internal/parser/parser/email_parser.go
@@ -544,7 +544,7 @@ func walkHTMLBodyText(n *html.Node, w *leafWriter) {
}
case html.ElementNode:
switch n.Data {
- case "head", "script", "style", "noscript":
+ case "head", "script", "style", "noscript", "template":
return
case "br":
w.hardBreak()
diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.go
index b9bc028149..41c68a360b 100644
--- a/internal/parser/parser/email_parser_test.go
+++ b/internal/parser/parser/email_parser_test.go
@@ -1506,6 +1506,7 @@ func TestHTMLBodyToText(t *testing.T) {
{"definition list and rule", `<div>A<dl><dt>Term</dt><dd>Def</dd></dl>B<hr>C</div>`, "A\nTerm\nDef\nB\nC"},
{"br forces a break", `one<br>two`, "one\ntwo"},
{"head script style skipped", `<html><head><title>t</title><style>.x{color:red}</style></head><body><p>visible</p><script>var x=1;</script></body></html>`, "visible"},
+ {"template content skipped", `<p>visible</p><template><p>Hello {{name}}</p></template>`, "visible"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.go
index 466fd5cf52..686a3b46b4 100644
--- a/internal/parser/parser/html_parser.go
+++ b/internal/parser/parser/html_parser.go
@@ -139,7 +139,7 @@ func walkHTMLBlocksWithState(root *html.Node, out *[]map[string]any, state *html
for child := root.FirstChild; child != nil; child = child.NextSibling {
// Invisible nodes neither show up nor end the run around them.
if child.Type == html.CommentNode || (child.Type == html.ElementNode &&
- (child.Data == "script" || child.Data == "style" || child.Data == "noscript")) {
+ (child.Data == "script" || child.Data == "style" || child.Data == "noscript" || child.Data == "template")) {
continue
}
if emitsLooseHTMLText(root) && (child.Type == html.TextNode || isHTMLInlineElement(child)) {
@@ -360,7 +360,8 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, layout str
case html.TextNode:
w.writeText(n.Data)
case html.ElementNode:
- if n.Data == "script" || n.Data == "style" || n.Data == "noscript" {
+ // A <template> holds markup a script may clone; the page never shows it.
+ if n.Data == "script" || n.Data == "style" || n.Data == "noscript" || n.Data == "template" {
return
}
if n.Data == "br" {
diff --git a/internal/parser/parser/html_parser_test.go b/internal/parser/parser/html_parser_test.go
index 22abc64812..216443c3c2 100644
--- a/internal/parser/parser/html_parser_test.go
+++ b/internal/parser/parser/html_parser_test.go
@@ -173,6 +173,22 @@ func TestHTMLParser_RemoveTOC(t *testing.T) {
}
}
+// TestHTMLParser_SkipsTemplateContent checks that markup inside a <template>
+// is not indexed. It is a pattern a script may clone (often with placeholders
+// such as {{name}}), never something the page shows, as the table parser's
+// inert-element list already treats it.
+func TestHTMLParser_SkipsTemplateContent(t *testing.T) {
+ const doc = `<html><body><p>Shown</p>Run <template>inline {{x}}</template>on` +
+ `<template><p>Hello {{name}}</p></template><p>Also shown</p></body></html>`
+ res := NewHTMLParser().ParseWithResult(context.Background(), "page.html", []byte(doc))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if got := itemsText(res); strings.Contains(got, "{{") {
+ t.Errorf("template content was indexed: %q", got)
+ }
+}
+
// TestHTMLParser_BodyTextAndInlineElementsFormOneItem checks that a run of
// text and inline elements directly under <body> is read as one item, the way
// a browser lays it out. Each text node and each inline element was its own