Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(ai): match inline style values instead of substrings when sanitizing (#460)

Yiğit ERDOĞAN11e0935f6b10
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py
index 85c26ee..1ee8189 100644
--- a/scrapling/core/shell.py
+++ b/scrapling/core/shell.py
@@ -2,7 +2,7 @@
from sys import stderr
from copy import deepcopy
from functools import wraps
-from re import sub as re_sub, compile as re_compile
+from re import sub as re_sub, compile as re_compile, DOTALL, IGNORECASE
from collections import namedtuple
from shlex import split as shlex_split
from inspect import signature, Parameter
@@ -35,6 +35,7 @@ from scrapling.core._types import (
Callable,
Dict,
Any,
+ Match,
cast,
Optional,
Generator,
@@ -69,21 +70,60 @@ Request = namedtuple(
)
# Precompiled for the prompt injection sanitizer
-_HIDDEN_XPATH = XPath(
- './/*[contains(@style,"display:none") or contains(@style,"display: none")'
- ' or contains(@style,"visibility:hidden") or contains(@style,"visibility: hidden")'
- ' or contains(@style,"opacity:0") or contains(@style,"opacity: 0")'
- ' or contains(@style,"font-size:0") or contains(@style,"font-size: 0")'
- ' or contains(@style,"height:0") or contains(@style,"height: 0")'
- ' or contains(@style,"width:0") or contains(@style,"width: 0")]'
- " | .//*[@aria-hidden='true']"
- " | .//slot[@hidden]"
- " | .//template"
+_HIDDEN_XPATH = XPath(".//*[@style] | .//*[@aria-hidden='true'] | .//slot[@hidden] | .//template")
+_HIDING_DECLARATIONS = frozenset({("display", "none"), ("visibility", "hidden")})
+_ZERO_HIDING_PROPERTIES = frozenset({"opacity", "font-size", "height", "width", "max-height", "max-width"})
+# CSS reads CRLF, CR and FF as a single LF, which ends an unfinished string
+_CSS_NEWLINE_PATTERN = re_compile(r"\r\n?|\f")
+# Escapes, strings and unquoted `url()` tokens are matched first so a `/*`, a quote or a `;` inside them is never
+# read as CSS. An unquoted `url(` runs to the first unescaped `)` even when malformed, as browsers skip a bad url
+_STYLE_TOKEN_PATTERN = re_compile(
+ r"""(\\.)|"(?:[^"\\\n]|\\.)*"?|'(?:[^'\\\n]|\\.)*'?"""
+ r"""|(?<![\w-])url\((?![ \t\n]*["'])(?:[^)\\]|\\.?)*\)?|(/\*.*?(?:\*/|\Z))""",
+ DOTALL | IGNORECASE,
)
+_ZERO_VALUE_PATTERN = re_compile(r"[+-]?(?:0+(?:\.0+)?|\.0+)(?:e[+-]?[0-9]+)?[a-z%]*")
+_IMPORTANT_SUFFIX_PATTERN = re_compile(r"![ \t\n]*important$")
_ZWC_PATTERN = re_compile(r"[\u200b\u200c\u200d\ufeff\u2060\u180e]")
_CONTROL_CHARS_PATTERN = re_compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f]")
+def _blank_style_token(match: Match[str]) -> str:
+ """Keep an escape, replace a comment with a space and a string or `url()` with an empty string.
+
+ A comment separates tokens instead of joining them, so `no/**/ne` never reads as `none`,
+ and no hiding value contains a string or a `url()`, so their content can be dropped.
+ """
+ if match.group(1):
+ return match.group(1)
+ return " " if match.group(2) else '""'
+
+
+def _is_hidden_element(element: Any) -> bool:
+ """Check if an element selected by `_HIDDEN_XPATH` is hidden from the page's readers.
+
+ Inline styles are compared declaration by declaration instead of by substring, so a value that
+ only starts with a zero (`opacity:0.95`) or a longer property name (`line-height:0.9`) is kept.
+ """
+ if element.tag == "template" or element.get("aria-hidden") == "true":
+ return True
+ if element.tag == "slot" and element.get("hidden") is not None:
+ return True
+
+ style = _CSS_NEWLINE_PATTERN.sub("\n", element.get("style") or "")
+ style = _STYLE_TOKEN_PATTERN.sub(_blank_style_token, style)
+ for declaration in style.split(";"):
+ prop, _, value = declaration.partition(":")
+ prop = prop.strip(" \t\n").lower()
+ value = _IMPORTANT_SUFFIX_PATTERN.sub("", value.strip(" \t\n").lower()).rstrip(" \t\n")
+ if (prop, value) in _HIDING_DECLARATIONS:
+ return True
+ if prop in _ZERO_HIDING_PROPERTIES and _ZERO_VALUE_PATTERN.fullmatch(value):
+ return True
+
+ return False
+
+
# Suppress exit on error to handle parsing errors gracefully
class NoExitArgumentParser(ArgumentParser): # pragma: no cover
def error(self, message):
@@ -611,7 +651,8 @@ class Convertor:
"""
clean_root = deepcopy(page._root)
for element in cast(list, _HIDDEN_XPATH(clean_root)):
- element.drop_tree()
+ if _is_hidden_element(element):
+ element.drop_tree()
for element in clean_root.iter():
if element.text:
element.text = _CONTROL_CHARS_PATTERN.sub("", _ZWC_PATTERN.sub("", element.text))
diff --git a/tests/fetchers/test_response_markdown.py b/tests/fetchers/test_response_markdown.py
index 83f6dff..30adf27 100644
--- a/tests/fetchers/test_response_markdown.py
+++ b/tests/fetchers/test_response_markdown.py
@@ -78,3 +78,136 @@ class TestResponseMarkdown:
monkeypatch.setitem(sys.modules, "markdownify", None)
with pytest.raises(ModuleNotFoundError, match=r"scrapling\[rag\]"):
_make_response().markdown()
+
+
+STYLED_BUT_VISIBLE_HTML = """
+<html>
+ <body>
+ <div style="line-height:0.9"><h2>Compact heading</h2><p>Body of the first section.</p></div>
+ <p style="opacity:0.95">Almost opaque note.</p>
+ <p style="font-size:0.9rem">Slightly smaller note.</p>
+ <p style="width:0.5em">Narrow but visible.</p>
+ <p style="min-height:0">Flexbox child.</p>
+ <p style="opacity:0.95/*hidden*/">Commented but almost opaque.</p>
+ <p style="opacity:0.95 ! important">Important but almost opaque.</p>
+ <p style="opacity:5e-1">Half opaque note.</p>
+ <p style='--note:"a;display:none;b"'>Quoted separator note.</p>
+ <p style="background:url(a;display:none;b)">Unquoted url note.</p>
+ <p style='background:url( "a;display:none" )'>Quoted url note.</p>
+ <p style='--note:"a\\&#13;&#10;;display:none"'>Escaped line break note.</p>
+ <p style="display:&nbsp;none">Non-breaking space note.</p>
+ <p style="display:no/**/ne">Split keyword note.</p>
+ <p style="opacity:0/**/5">Split number note.</p>
+ <slot style="color:red">Styled slot note.</slot>
+ </body>
+</html>
+"""
+
+HIDDEN_HTML = """
+<html>
+ <body>
+ <p>Visible content.</p>
+ <p style="display:none">Hidden by display</p>
+ <p style="display: none">Hidden by spaced display</p>
+ <p style="visibility:hidden">Hidden by visibility</p>
+ <p style="opacity:0">Hidden by opacity</p>
+ <p style="opacity: 0">Hidden by spaced opacity</p>
+ <p style="height:0px">Hidden by height</p>
+ <p style="max-height:0">Hidden by max height</p>
+ <p style="width:0">Hidden by width</p>
+ <p style="font-size:0">Hidden by font size</p>
+ <p style="display:none !important">Hidden by important</p>
+ <p style="display:none ! important">Hidden by spaced important</p>
+ <p style="opacity:0 ! important">Hidden by spaced important opacity</p>
+ <p style="visibility:hidden!important">Hidden by unspaced important</p>
+ <p style="display:none/*hidden*/">Hidden by commented display</p>
+ <p style="opacity:0/*hidden*/">Hidden by commented opacity</p>
+ <p style="display:/* keep */none;">Hidden by comment inside declaration</p>
+ <p style="color:red/*;*/;visibility:hidden">Hidden after commented separator</p>
+ <p style="opacity:00">Hidden by double zero</p>
+ <p style="opacity:0e0">Hidden by exponent zero</p>
+ <p style="opacity:.0">Hidden by leading dot zero</p>
+ <p style="display:none/*unfinished">Hidden by unterminated comment</p>
+ <p style='--a:"/*";display:none;--b:"*/"'>Hidden between quoted comment markers</p>
+ <p style="background:url(/*);display:none;color:url(*/)">Hidden between url comment markers</p>
+ <p style="opacity:0.0">Hidden by decimal zero</p>
+ <p style="opacity:0/**/!/**/important">Hidden by commented important</p>
+ <p style="background:url(a\\)/*);display:none">Hidden after escaped url parenthesis</p>
+ <p style='background:url(a"/*);display:none;--b:"*/"'>Hidden after malformed url</p>
+ <p style='--x:\\";display:none'>Hidden after escaped quote</p>
+ <p style='--note:"unfinished&#13;;display:none'>Hidden after carriage return in string</p>
+ <p style='--note:"unfinished\f;display:none'>Hidden after form feed in string</p>
+ <p style='background:url(&nbsp;"/*);display:none;--b:"*/"'>Hidden after non-breaking space in url</p>
+ <p aria-hidden="true">Hidden by aria</p>
+ <slot hidden>Hidden by slot</slot>
+ <template><p>Hidden by template</p></template>
+ </body>
+</html>
+"""
+
+
+class TestResponseMarkdownSanitizerPrecision:
+ def test_partial_zero_values_are_not_treated_as_hidden(self):
+ """A style value that merely starts with a zero does not hide the element"""
+ md = _make_response(content=STYLED_BUT_VISIBLE_HTML).markdown()
+ for expected in (
+ "Compact heading",
+ "Body of the first section.",
+ "Almost opaque note.",
+ "Slightly smaller note.",
+ "Narrow but visible.",
+ "Flexbox child.",
+ "Commented but almost opaque.",
+ "Important but almost opaque.",
+ "Half opaque note.",
+ "Quoted separator note.",
+ "Unquoted url note.",
+ "Quoted url note.",
+ "Escaped line break note.",
+ "Non-breaking space note.",
+ "Split keyword note.",
+ "Split number note.",
+ "Styled slot note.",
+ ):
+ assert expected in md
+
+ def test_truly_hidden_content_is_still_removed(self):
+ md = _make_response(content=HIDDEN_HTML).markdown()
+ assert "Visible content." in md
+ for hidden in (
+ "Hidden by display",
+ "Hidden by spaced display",
+ "Hidden by visibility",
+ "Hidden by opacity",
+ "Hidden by spaced opacity",
+ "Hidden by height",
+ "Hidden by max height",
+ "Hidden by width",
+ "Hidden by font size",
+ "Hidden by important",
+ "Hidden by spaced important",
+ "Hidden by spaced important opacity",
+ "Hidden by unspaced important",
+ "Hidden by commented display",
+ "Hidden by commented opacity",
+ "Hidden by comment inside declaration",
+ "Hidden after commented separator",
+ "Hidden by double zero",
+ "Hidden by exponent zero",
+ "Hidden by leading dot zero",
+ "Hidden by unterminated comment",
+ "Hidden between quoted comment markers",
+ "Hidden between url comment markers",
+ "Hidden by decimal zero",
+ "Hidden by commented important",
+ "Hidden after escaped url parenthesis",
+ "Hidden after malformed url",
+ "Hidden after escaped quote",
+ "Hidden after carriage return in string",
+ "Hidden after form feed in string",
+ "Hidden after non-breaking space in url",
+ "Hidden by aria",
+ "Hidden by slot",
+ "Hidden by template",
+ ):
+ assert hidden not in md