RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): leave out the content of <template> elements (#21284)
Lukas8e31338dd219
diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.goindex c6eec770b3..3d5286bd19 100644--- a/internal/parser/parser/email_parser.go+++ b/internal/parser/parser/email_parser.go@@ -544,7 +544,7 @@ func walkHTMLBodyText(n *html.Node, w *leafWriter) {}case html.ElementNode:switch n.Data {- case "head", "script", "style", "noscript":+ case "head", "script", "style", "noscript", "template":returncase "br":w.hardBreak()diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.goindex b9bc028149..41c68a360b 100644--- a/internal/parser/parser/email_parser_test.go+++ b/internal/parser/parser/email_parser_test.go@@ -1506,6 +1506,7 @@ func TestHTMLBodyToText(t *testing.T) {{"definition list and rule", `<div>A<dl><dt>Term</dt><dd>Def</dd></dl>B<hr>C</div>`, "A\nTerm\nDef\nB\nC"},{"br forces a break", `one<br>two`, "one\ntwo"},{"head script style skipped", `<html><head><title>t</title><style>.x{color:red}</style></head><body><p>visible</p><script>var x=1;</script></body></html>`, "visible"},+ {"template content skipped", `<p>visible</p><template><p>Hello {{name}}</p></template>`, "visible"},}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.goindex 466fd5cf52..686a3b46b4 100644--- a/internal/parser/parser/html_parser.go+++ b/internal/parser/parser/html_parser.go@@ -139,7 +139,7 @@ func walkHTMLBlocksWithState(root *html.Node, out *[]map[string]any, state *htmlfor child := root.FirstChild; child != nil; child = child.NextSibling {// Invisible nodes neither show up nor end the run around them.if child.Type == html.CommentNode || (child.Type == html.ElementNode &&- (child.Data == "script" || child.Data == "style" || child.Data == "noscript")) {+ (child.Data == "script" || child.Data == "style" || child.Data == "noscript" || child.Data == "template")) {continue}if emitsLooseHTMLText(root) && (child.Type == html.TextNode || isHTMLInlineElement(child)) {@@ -360,7 +360,8 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, layout strcase html.TextNode:w.writeText(n.Data)case html.ElementNode:- if n.Data == "script" || n.Data == "style" || n.Data == "noscript" {+ // A <template> holds markup a script may clone; the page never shows it.+ if n.Data == "script" || n.Data == "style" || n.Data == "noscript" || n.Data == "template" {return}if n.Data == "br" {diff --git a/internal/parser/parser/html_parser_test.go b/internal/parser/parser/html_parser_test.goindex 22abc64812..216443c3c2 100644--- a/internal/parser/parser/html_parser_test.go+++ b/internal/parser/parser/html_parser_test.go@@ -173,6 +173,22 @@ func TestHTMLParser_RemoveTOC(t *testing.T) {}}+// TestHTMLParser_SkipsTemplateContent checks that markup inside a <template>+// is not indexed. It is a pattern a script may clone (often with placeholders+// such as {{name}}), never something the page shows, as the table parser's+// inert-element list already treats it.+func TestHTMLParser_SkipsTemplateContent(t *testing.T) {+ const doc = `<html><body><p>Shown</p>Run <template>inline {{x}}</template>on` ++ `<template><p>Hello {{name}}</p></template><p>Also shown</p></body></html>`+ res := NewHTMLParser().ParseWithResult(context.Background(), "page.html", []byte(doc))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if got := itemsText(res); strings.Contains(got, "{{") {+ t.Errorf("template content was indexed: %q", got)+ }+}+// TestHTMLParser_BodyTextAndInlineElementsFormOneItem checks that a run of// text and inline elements directly under <body> is read as one item, the way// a browser lays it out. Each text node and each inline element was its own