🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(ai): match inline style values instead of substrings when sanitizing (#460)
Yiğit ERDOĞAN11e0935f6b10
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.pyindex 85c26ee..1ee8189 100644--- a/scrapling/core/shell.py+++ b/scrapling/core/shell.py@@ -2,7 +2,7 @@from sys import stderrfrom copy import deepcopyfrom functools import wraps-from re import sub as re_sub, compile as re_compile+from re import sub as re_sub, compile as re_compile, DOTALL, IGNORECASEfrom collections import namedtuplefrom shlex import split as shlex_splitfrom inspect import signature, Parameter@@ -35,6 +35,7 @@ from scrapling.core._types import (Callable,Dict,Any,+ Match,cast,Optional,Generator,@@ -69,21 +70,60 @@ Request = namedtuple()# Precompiled for the prompt injection sanitizer-_HIDDEN_XPATH = XPath(- './/*[contains(@style,"display:none") or contains(@style,"display: none")'- ' or contains(@style,"visibility:hidden") or contains(@style,"visibility: hidden")'- ' or contains(@style,"opacity:0") or contains(@style,"opacity: 0")'- ' or contains(@style,"font-size:0") or contains(@style,"font-size: 0")'- ' or contains(@style,"height:0") or contains(@style,"height: 0")'- ' or contains(@style,"width:0") or contains(@style,"width: 0")]'- " | .//*[@aria-hidden='true']"- " | .//slot[@hidden]"- " | .//template"+_HIDDEN_XPATH = XPath(".//*[@style] | .//*[@aria-hidden='true'] | .//slot[@hidden] | .//template")+_HIDING_DECLARATIONS = frozenset({("display", "none"), ("visibility", "hidden")})+_ZERO_HIDING_PROPERTIES = frozenset({"opacity", "font-size", "height", "width", "max-height", "max-width"})+# CSS reads CRLF, CR and FF as a single LF, which ends an unfinished string+_CSS_NEWLINE_PATTERN = re_compile(r"\r\n?|\f")+# Escapes, strings and unquoted `url()` tokens are matched first so a `/*`, a quote or a `;` inside them is never+# read as CSS. An unquoted `url(` runs to the first unescaped `)` even when malformed, as browsers skip a bad url+_STYLE_TOKEN_PATTERN = re_compile(+ r"""(\\.)|"(?:[^"\\\n]|\\.)*"?|'(?:[^'\\\n]|\\.)*'?"""+ r"""|(?<![\w-])url\((?![ \t\n]*["'])(?:[^)\\]|\\.?)*\)?|(/\*.*?(?:\*/|\Z))""",+ DOTALL | IGNORECASE,)+_ZERO_VALUE_PATTERN = re_compile(r"[+-]?(?:0+(?:\.0+)?|\.0+)(?:e[+-]?[0-9]+)?[a-z%]*")+_IMPORTANT_SUFFIX_PATTERN = re_compile(r"![ \t\n]*important$")_ZWC_PATTERN = re_compile(r"[\u200b\u200c\u200d\ufeff\u2060\u180e]")_CONTROL_CHARS_PATTERN = re_compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f]")+def _blank_style_token(match: Match[str]) -> str:+ """Keep an escape, replace a comment with a space and a string or `url()` with an empty string.++ A comment separates tokens instead of joining them, so `no/**/ne` never reads as `none`,+ and no hiding value contains a string or a `url()`, so their content can be dropped.+ """+ if match.group(1):+ return match.group(1)+ return " " if match.group(2) else '""'+++def _is_hidden_element(element: Any) -> bool:+ """Check if an element selected by `_HIDDEN_XPATH` is hidden from the page's readers.++ Inline styles are compared declaration by declaration instead of by substring, so a value that+ only starts with a zero (`opacity:0.95`) or a longer property name (`line-height:0.9`) is kept.+ """+ if element.tag == "template" or element.get("aria-hidden") == "true":+ return True+ if element.tag == "slot" and element.get("hidden") is not None:+ return True++ style = _CSS_NEWLINE_PATTERN.sub("\n", element.get("style") or "")+ style = _STYLE_TOKEN_PATTERN.sub(_blank_style_token, style)+ for declaration in style.split(";"):+ prop, _, value = declaration.partition(":")+ prop = prop.strip(" \t\n").lower()+ value = _IMPORTANT_SUFFIX_PATTERN.sub("", value.strip(" \t\n").lower()).rstrip(" \t\n")+ if (prop, value) in _HIDING_DECLARATIONS:+ return True+ if prop in _ZERO_HIDING_PROPERTIES and _ZERO_VALUE_PATTERN.fullmatch(value):+ return True++ return False++# Suppress exit on error to handle parsing errors gracefullyclass NoExitArgumentParser(ArgumentParser): # pragma: no coverdef error(self, message):@@ -611,7 +651,8 @@ class Convertor:"""clean_root = deepcopy(page._root)for element in cast(list, _HIDDEN_XPATH(clean_root)):- element.drop_tree()+ if _is_hidden_element(element):+ element.drop_tree()for element in clean_root.iter():if element.text:element.text = _CONTROL_CHARS_PATTERN.sub("", _ZWC_PATTERN.sub("", element.text))diff --git a/tests/fetchers/test_response_markdown.py b/tests/fetchers/test_response_markdown.pyindex 83f6dff..30adf27 100644--- a/tests/fetchers/test_response_markdown.py+++ b/tests/fetchers/test_response_markdown.py@@ -78,3 +78,136 @@ class TestResponseMarkdown:monkeypatch.setitem(sys.modules, "markdownify", None)with pytest.raises(ModuleNotFoundError, match=r"scrapling\[rag\]"):_make_response().markdown()+++STYLED_BUT_VISIBLE_HTML = """+<html>+ <body>+ <div style="line-height:0.9"><h2>Compact heading</h2><p>Body of the first section.</p></div>+ <p style="opacity:0.95">Almost opaque note.</p>+ <p style="font-size:0.9rem">Slightly smaller note.</p>+ <p style="width:0.5em">Narrow but visible.</p>+ <p style="min-height:0">Flexbox child.</p>+ <p style="opacity:0.95/*hidden*/">Commented but almost opaque.</p>+ <p style="opacity:0.95 ! important">Important but almost opaque.</p>+ <p style="opacity:5e-1">Half opaque note.</p>+ <p style='--note:"a;display:none;b"'>Quoted separator note.</p>+ <p style="background:url(a;display:none;b)">Unquoted url note.</p>+ <p style='background:url( "a;display:none" )'>Quoted url note.</p>+ <p style='--note:"a\\ ;display:none"'>Escaped line break note.</p>+ <p style="display: none">Non-breaking space note.</p>+ <p style="display:no/**/ne">Split keyword note.</p>+ <p style="opacity:0/**/5">Split number note.</p>+ <slot style="color:red">Styled slot note.</slot>+ </body>+</html>+"""++HIDDEN_HTML = """+<html>+ <body>+ <p>Visible content.</p>+ <p style="display:none">Hidden by display</p>+ <p style="display: none">Hidden by spaced display</p>+ <p style="visibility:hidden">Hidden by visibility</p>+ <p style="opacity:0">Hidden by opacity</p>+ <p style="opacity: 0">Hidden by spaced opacity</p>+ <p style="height:0px">Hidden by height</p>+ <p style="max-height:0">Hidden by max height</p>+ <p style="width:0">Hidden by width</p>+ <p style="font-size:0">Hidden by font size</p>+ <p style="display:none !important">Hidden by important</p>+ <p style="display:none ! important">Hidden by spaced important</p>+ <p style="opacity:0 ! important">Hidden by spaced important opacity</p>+ <p style="visibility:hidden!important">Hidden by unspaced important</p>+ <p style="display:none/*hidden*/">Hidden by commented display</p>+ <p style="opacity:0/*hidden*/">Hidden by commented opacity</p>+ <p style="display:/* keep */none;">Hidden by comment inside declaration</p>+ <p style="color:red/*;*/;visibility:hidden">Hidden after commented separator</p>+ <p style="opacity:00">Hidden by double zero</p>+ <p style="opacity:0e0">Hidden by exponent zero</p>+ <p style="opacity:.0">Hidden by leading dot zero</p>+ <p style="display:none/*unfinished">Hidden by unterminated comment</p>+ <p style='--a:"/*";display:none;--b:"*/"'>Hidden between quoted comment markers</p>+ <p style="background:url(/*);display:none;color:url(*/)">Hidden between url comment markers</p>+ <p style="opacity:0.0">Hidden by decimal zero</p>+ <p style="opacity:0/**/!/**/important">Hidden by commented important</p>+ <p style="background:url(a\\)/*);display:none">Hidden after escaped url parenthesis</p>+ <p style='background:url(a"/*);display:none;--b:"*/"'>Hidden after malformed url</p>+ <p style='--x:\\";display:none'>Hidden after escaped quote</p>+ <p style='--note:"unfinished ;display:none'>Hidden after carriage return in string</p>+ <p style='--note:"unfinished\f;display:none'>Hidden after form feed in string</p>+ <p style='background:url( "/*);display:none;--b:"*/"'>Hidden after non-breaking space in url</p>+ <p aria-hidden="true">Hidden by aria</p>+ <slot hidden>Hidden by slot</slot>+ <template><p>Hidden by template</p></template>+ </body>+</html>+"""+++class TestResponseMarkdownSanitizerPrecision:+ def test_partial_zero_values_are_not_treated_as_hidden(self):+ """A style value that merely starts with a zero does not hide the element"""+ md = _make_response(content=STYLED_BUT_VISIBLE_HTML).markdown()+ for expected in (+ "Compact heading",+ "Body of the first section.",+ "Almost opaque note.",+ "Slightly smaller note.",+ "Narrow but visible.",+ "Flexbox child.",+ "Commented but almost opaque.",+ "Important but almost opaque.",+ "Half opaque note.",+ "Quoted separator note.",+ "Unquoted url note.",+ "Quoted url note.",+ "Escaped line break note.",+ "Non-breaking space note.",+ "Split keyword note.",+ "Split number note.",+ "Styled slot note.",+ ):+ assert expected in md++ def test_truly_hidden_content_is_still_removed(self):+ md = _make_response(content=HIDDEN_HTML).markdown()+ assert "Visible content." in md+ for hidden in (+ "Hidden by display",+ "Hidden by spaced display",+ "Hidden by visibility",+ "Hidden by opacity",+ "Hidden by spaced opacity",+ "Hidden by height",+ "Hidden by max height",+ "Hidden by width",+ "Hidden by font size",+ "Hidden by important",+ "Hidden by spaced important",+ "Hidden by spaced important opacity",+ "Hidden by unspaced important",+ "Hidden by commented display",+ "Hidden by commented opacity",+ "Hidden by comment inside declaration",+ "Hidden after commented separator",+ "Hidden by double zero",+ "Hidden by exponent zero",+ "Hidden by leading dot zero",+ "Hidden by unterminated comment",+ "Hidden between quoted comment markers",+ "Hidden between url comment markers",+ "Hidden by decimal zero",+ "Hidden by commented important",+ "Hidden after escaped url parenthesis",+ "Hidden after malformed url",+ "Hidden after escaped quote",+ "Hidden after carriage return in string",+ "Hidden after form feed in string",+ "Hidden after non-breaking space in url",+ "Hidden by aria",+ "Hidden by slot",+ "Hidden by template",+ ):+ assert hidden not in md