High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix: TrimHTML keeps the base64 payload in the CSS quoting the serialiser actually emits and deletes an Italian/Portuguese 'Data:<date>' label, so anchor the rule on the RFC grammar behind one fixture table (PIN-226)
Luigi Agosti4bd375afa4eb
diff --git a/internal/htmltrim/trim.go b/internal/htmltrim/trim.goindex cf8b2551..3bedd7e0 100644--- a/internal/htmltrim/trim.go+++ b/internal/htmltrim/trim.go@@ -37,28 +37,10 @@ func TrimHTML(html string) string {}var (- reScript = regexp.MustCompile(`(?is)<script[^>]*>.*?</script>`)- reStyle = regexp.MustCompile(`(?is)<style[^>]*>.*?</style>`)- reComment = regexp.MustCompile(`(?s)<!--.*?-->`)- // reDataURI matches the URI itself, not the attribute value holding it. Anchored on- // the whole value ("data:[^"]*") it saw only src="data:…" and missed the shape that- // actually costs the budget, style="background:url(data:…)", where the URI is in the- // middle of a declaration. Replacing just the URI is also what keeps that declaration- // readable — url() rather than a blanked style attribute.- //- // The terminator set is what ends a URI in each host syntax: the quote of the- // attribute, the quote or paren of a CSS url(), and whitespace for a srcset- // candidate. A comma is NOT a terminator — it separates a data URI's own metadata- // from its payload, so excluding it would leave every base64 blob behind.- //- // Two guards, because each alone corrupts something. The leading delimiter (or start- // of input), captured in group 1 and re-emitted on replace, stops "data:" matching- // inside a word — without it "Metadata:" became "Meta". It has to include '>', or a- // URI opening an element's text content is missed and the whole blob survives. But '>'- // alone would then eat the prose label "<td>Data:</td>", so the payload must also look- // like a URI: a MIME-shaped prefix and one of '/', ';' or ','. A bare "data:" carries- // none of those and is left alone.- reDataURI = regexp.MustCompile(`(?i)(\A|["'(,=\s>])data:[a-z0-9.+-]*[/;,][^"'()<>\s]*`)+ reScript = regexp.MustCompile(`(?is)<script[^>]*>.*?</script>`)+ reStyle = regexp.MustCompile(`(?is)<style[^>]*>.*?</style>`)+ reComment = regexp.MustCompile(`(?s)<!--.*?-->`)+ reDataURI = regexp.MustCompile(`(?i)(\A|["'(,=\s>;])data:(?:[a-z][a-z0-9.+-]*/[a-z][a-z0-9.+-]*)?(?:;[a-z0-9.+-]+(?:=[a-z0-9.+-]*)?)*,[^"'()<>\s&]*`)reWhitespace = regexp.MustCompile(`[ \t]+`)reNewlines = regexp.MustCompile(`\n{3,}`))diff --git a/internal/htmltrim/trim_test.go b/internal/htmltrim/trim_test.goindex 83d7c364..86abeafb 100644--- a/internal/htmltrim/trim_test.go+++ b/internal/htmltrim/trim_test.go@@ -4,6 +4,8 @@ import ("strings""testing""unicode/utf8"++ "golang.org/x/net/html")func TestTrimHTMLStripsScriptsAndStyles(t *testing.T) {@@ -83,121 +85,248 @@ func TestTrimHTMLSpendsTheBudgetOnMarkupNotOnStrippedContent(t *testing.T) {}}-// A base64 image is usually the single largest token contributor on a page, and the cap-// is applied AFTER stripping — so one payload that survives spends the whole prompt-// budget on a picture and pushes the interactive markup off the end. The old pattern was-// anchored on the whole attribute value, so it only ever saw src="data:…" and missed the-// commonest carrier of all: a URL inside an inline style declaration.-func TestTrimHTMLStripsADataURIEmbeddedInAValue(t *testing.T) {- const payload = "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo="+// dataURIGuards names each guard in reDataURI against the corruption it alone prevents.+// Three rounds of tuning this one pattern each traded one corruption for another, because+// a guard added for one round's cases reads as redundant to the next round's editor. Every+// guard must be named by at least one row of dataURICases, so deleting the last row for a+// guard reds here rather than going quiet.+var dataURIGuards = map[string]string{+ "leading delimiter": `without it "Metadata:text/html,x" is cut to "Meta"`,+ "delimiter includes >": `without it a payload opening an element's text content survives whole`,+ "delimiter includes ;": `without it the ';' ending " is unlicensed and the escaped CSS payload survives whole`,+ "mediatype letter-initial": `without it a comma-separated pair of dates reads as a mediatype and "Data:30/07,31/07" is deleted`,+ "mediatype comma required": `without it every other compact "Data:<value>" is deleted, and "Data:" is the word for date in five languages`,+ "entity-safe payload end": `without it url("data:…") loses its closing entity and the declaration stops parsing`,+}- for _, tc := range []struct {- name string- html string- want string- }{- {- name: "css url, unquoted",- html: `<div style="background:url(data:image/png;base64,` + payload + `)">x</div>`,- want: `<div style="background:url()">x</div>`,- },- {- name: "css url, single-quoted",- html: `<div style="background-image:url('data:image/png;base64,` + payload + `')">x</div>`,- want: `<div style="background-image:url('')">x</div>`,- },- {- name: "css url among other declarations",- html: `<div style="color:red;background:url(data:image/gif;base64,` + payload + `);border:1px">x</div>`,- want: `<div style="color:red;background:url();border:1px">x</div>`,- },- {- name: "whole value, the already-covered form",- html: `<img src="data:image/png;base64,` + payload + `">`,- want: `<img src="">`,- },- } {+const dataURIPayload = "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo="++// dataURICases is the whole discovered class list for reDataURI in one table: every carrier+// a payload arrives in, and every prose shape that must survive. One table rather than a+// test per round, so a change to the pattern has to satisfy all of them at once.+var dataURICases = []struct {+ name string+ html string+ want string+ pins []string+}{+ {+ name: "attribute value, whole",+ html: `<img src="data:image/png;base64,` + dataURIPayload + `">`,+ want: `<img src="">`,+ },+ {+ name: "css url, unquoted",+ html: `<div style="background:url(data:image/png;base64,` + dataURIPayload + `)">x</div>`,+ want: `<div style="background:url()">x</div>`,+ },+ {+ name: "css url, single-quoted",+ html: `<div style="background-image:url('data:image/png;base64,` + dataURIPayload + `')">x</div>`,+ want: `<div style="background-image:url('')">x</div>`,+ },+ {+ name: "css url, escaped double-quoted, the only shape serialisation emits for it",+ html: `<div style="background:url("data:image/png;base64,` + dataURIPayload + `")">x</div>`,+ want: `<div style="background:url("")">x</div>`,+ pins: []string{"delimiter includes ;", "entity-safe payload end"},+ },+ {+ name: "css url among other declarations",+ html: `<div style="color:red;background:url(data:image/gif;base64,` + dataURIPayload + `);border:1px">x</div>`,+ want: `<div style="color:red;background:url();border:1px">x</div>`,+ },+ {+ name: "element text content",+ html: `<td>data:image/png;base64,` + dataURIPayload + `</td>`,+ want: `<td></td>`,+ pins: []string{"delimiter includes >"},+ },+ {+ name: "element text content, no base64",+ html: `<code>data:text/html,hello</code>`,+ want: `<code></code>`,+ pins: []string{"delimiter includes >"},+ },+ {+ name: "srcset candidate list",+ html: `<img srcset="data:image/png;base64,AAAABBBBCCCC 1x, data:image/png;base64,DDDDEEEEFFFF 2x" alt="pic">`,+ want: `<img srcset=" 1x, 2x" alt="pic">`,+ },+ {+ name: "empty mediatype, rfc-valid",+ html: `<td>data:,hello</td>`,+ want: `<td></td>`,+ pins: []string{"delimiter includes >"},+ },+ {+ name: "parameters only, rfc-valid",+ html: `<td>data:;base64,QUJD</td>`,+ want: `<td></td>`,+ pins: []string{"delimiter includes >"},+ },+ {+ name: "parameter with a value",+ html: `<td>data:text/plain;charset=utf-8,hello</td>`,+ want: `<td></td>`,+ pins: []string{"delimiter includes >"},+ },+ {+ name: "uppercase",+ html: `<td>DATA:IMAGE/PNG;BASE64,QUJD</td>`,+ want: `<td></td>`,+ pins: []string{"delimiter includes >"},+ },+ {+ name: "percent-encoded payload",+ html: `<td>data:text/plain,%20hi%21</td>`,+ want: `<td></td>`,+ pins: []string{"delimiter includes >"},+ },+ {+ name: "bare label",+ html: `<td>Data:</td>`,+ want: `<td>Data:</td>`,+ pins: []string{"mediatype comma required"},+ },+ {+ name: "spaced label",+ html: `<td>Data: 42</td>`,+ want: `<td>Data: 42</td>`,+ pins: []string{"mediatype comma required"},+ },+ {+ name: "compact label with a date",+ html: `<td>Data:30/07/2026</td>`,+ want: `<td>Data:30/07/2026</td>`,+ pins: []string{"mediatype comma required"},+ },+ {+ name: "compact label with a date, mid-sentence",+ html: `<p>Fattura Data:30/07/2026 totale</p>`,+ want: `<p>Fattura Data:30/07/2026 totale</p>`,+ pins: []string{"mediatype comma required"},+ },+ {+ name: "compact label with two dates",+ html: `<td>Data:30/07,31/07</td>`,+ want: `<td>Data:30/07,31/07</td>`,+ pins: []string{"mediatype letter-initial"},+ },+ {+ name: "compact label with a comma list",+ html: `<td>Data:1,2,3</td>`,+ want: `<td>Data:1,2,3</td>`,+ pins: []string{"mediatype comma required"},+ },+ {+ name: "compact label with a semicolon pair",+ html: `<td>Data:a;b</td>`,+ want: `<td>Data:a;b</td>`,+ pins: []string{"mediatype comma required"},+ },+ {+ name: "compact label with a single-letter path",+ html: `<td>Data:a/b</td>`,+ want: `<td>Data:a/b</td>`,+ pins: []string{"mediatype comma required"},+ },+ {+ name: "data: inside a word",+ html: `<p>Metadata: 2024</p>`,+ want: `<p>Metadata: 2024</p>`,+ },+ {+ name: "data: inside a word with a mime tail",+ html: `<td>metadata:image/png here</td>`,+ want: `<td>metadata:image/png here</td>`,+ },+ {+ name: "data: inside a word with a whole uri after it",+ html: `<td>Metadata:text/html,x</td>`,+ want: `<td>Metadata:text/html,x</td>`,+ pins: []string{"leading delimiter"},+ },+ {+ name: "data- attribute",+ html: `<div data-src="keep">x</div>`,+ want: `<div data-src="keep">x</div>`,+ },+}++func TestTrimHTMLHandlesEveryDataURICarrierAndProseShape(t *testing.T) {+ for _, tc := range dataURICases {t.Run(tc.name, func(t *testing.T) {got := TrimHTML(tc.html)- if strings.Contains(got, payload) {- t.Errorf("the base64 payload survived:\n%s", got)- }- if strings.Contains(got, "data:") {- t.Errorf("the data URI survived:\n%s", got)- }if got != tc.want {- t.Errorf("TrimHTML =\n %s\nwant\n %s", got, tc.want)+ t.Errorf("TrimHTML(%q) =\n %s\nwant\n %s", tc.html, got, tc.want)+ }+ if strings.Contains(got, dataURIPayload) {+ t.Errorf("the base64 payload survived and spends the whole prompt budget:\n%s", got)}})}}-// The widened pattern must not regress the forms that already stripped. srcset carries-// several URIs in one value separated by commas, and a comma also separates a data URI's-// own metadata from its payload — so the pattern cannot treat commas as a terminator,-// and this is the case that proves it does not.-func TestTrimHTMLStillStripsWholeValueDataURIs(t *testing.T) {- html := `<img srcset="data:image/png;base64,AAAABBBBCCCC 1x, data:image/png;base64,DDDDEEEEFFFF 2x" alt="pic">`+// The escaped row above is hand-written with the entity Chrome's serialiser emits. This+// derives the same shape by running a double-quoted CSS url() through a serialiser instead,+// so the row is pinned against what serialisation actually produces rather than against an+// approximation of it — and against either entity spelling a serialiser may choose.+func TestTrimHTMLStripsThePayloadASerialiserEscapedItself(t *testing.T) {+ div := &html.Node{+ Type: html.ElementNode,+ Data: "div",+ Attr: []html.Attribute{+ {Key: "id", Val: "d"},+ {Key: "style", Val: `background:url("data:image/png;base64,` + dataURIPayload + `")`},+ },+ }+ div.AppendChild(&html.Node{Type: html.TextNode, Data: "styled"})- got := TrimHTML(html)+ var rendered strings.Builder+ if err := html.Render(&rendered, div); err != nil {+ t.Fatalf("render: %v", err)+ }- for _, unwanted := range []string{"data:", "base64", "AAAABBBBCCCC", "DDDDEEEEFFFF"} {- if strings.Contains(got, unwanted) {- t.Errorf("srcset still carries %q:\n%s", unwanted, got)- }+ serialised := rendered.String()+ if strings.Contains(serialised, `url("data:`) {+ t.Fatalf("the serialiser left a bare quote, so this fixture is not the escaped shape:\n%s", serialised)}- // The descriptors and the rest of the tag are markup, not payload, and stay.- for _, want := range []string{"1x", "2x", `alt="pic"`, "<img"} {- if !strings.Contains(got, want) {- t.Errorf("srcset stripping removed %q, which is markup rather than payload:\n%s", want, got)- }+ if !strings.Contains(serialised, dataURIPayload) {+ t.Fatalf("the serialiser dropped the payload, so nothing is left to strip:\n%s", serialised)}-}-// A URI opening an element's text content is preceded by '>', not by a quote, paren, comma,-// '=' or whitespace. Licensing only those delimiters missed it and the whole blob survived-// — the budget cost this helper exists to prevent, reintroduced by the guard against-// over-matching. Text position is where a URI is most expensive, since nothing else in the-// pipeline strips it.-func TestTrimHTMLStripsADataURIOpeningTextContent(t *testing.T) {- blob := strings.Repeat("A", 64)+ open := strings.Index(serialised, "url(") + len("url(")+ entity := serialised[open:strings.Index(serialised, "data:")]- for _, tc := range []struct{ name, html string }{- {"table cell", `<td>data:image/png;base64,` + blob + `</td>`},- {"code block", `<code>data:text/html,hello</code>`},- } {- t.Run(tc.name, func(t *testing.T) {- got := TrimHTML(tc.html)- for _, unwanted := range []string{"data:", blob, "text/html"} {- if strings.Contains(got, unwanted) {- t.Errorf("still carries %q, so a URI in text position costs the whole budget:\n%s", unwanted, got)- }- }- })+ got := TrimHTML(serialised)+ if strings.Contains(got, dataURIPayload) {+ t.Errorf("the payload survived the shape serialisation actually emits:\n%s", got)+ }+ if want, have := strings.Count(serialised, entity), strings.Count(got, entity); have != want {+ t.Errorf("the strip took %d of the %d %s that quote the url — the declaration no longer parses:\n%s",+ want-have, want, entity, got)+ }+ if !strings.Contains(got, `)">styled`) {+ t.Errorf("the declaration stopped being readable:\n%s", got)}}-// The data-URI pattern strips a URI, not the letters "data:" wherever they fall. Widening-// it to match anywhere inside a value also let it match inside a word — "Metadata:" in page-// text became "Meta", eating the interactive markup this helper exists to carry. A real URI-// is preceded by a quote, paren, comma, '=', whitespace or '>', so those are what license-// the strip. Reverting either guard corrupts one of these.-func TestTrimHTMLLeavesDataColonInsideAWordAlone(t *testing.T) {- for _, tc := range []struct{ name, html, want string }{- {"prose label", `<p>Metadata: 2024</p>`, `<p>Metadata: 2024</p>`},- {"word with a mime tail", `<td>metadata:image/png here</td>`, `<td>metadata:image/png here</td>`},- {"data- attribute is unaffected", `<div data-src="keep">x</div>`, `<div data-src="keep">x</div>`},- // A field label opening a cell has a delimiter before it — the '>' that closes the- // tag — so the delimiter guard cannot save it. What does is the payload shape: a- // label carries no MIME prefix and no '/', ';' or ','.- {"bare label opening a cell", `<td>Data:</td>`, `<td>Data:</td>`},- {"labelled value opening a cell", `<td>Data: 42</td>`, `<td>Data: 42</td>`},- } {- t.Run(tc.name, func(t *testing.T) {- if got := TrimHTML(tc.html); got != tc.want {- t.Errorf("TrimHTML(%q) = %q, want it unchanged — 'data:' inside a word is not a URI", tc.html, got)+func TestEveryDataURIGuardIsPinnedByACase(t *testing.T) {+ pinned := map[string]int{}+ for _, tc := range dataURICases {+ for _, guard := range tc.pins {+ if _, ok := dataURIGuards[guard]; !ok {+ t.Errorf("case %q pins %q, which is not a declared guard", tc.name, guard)+ continue}- })+ pinned[guard]+++ }+ }+ for guard, corruption := range dataURIGuards {+ if pinned[guard] == 0 {+ t.Errorf("no case pins the %q guard, so removing it corrupts silently: %s", guard, corruption)+ }}}