Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(parser): keep filtering on blank class_ and escape CSS string values (#417)

Mrityunjay SIngh0003f6444b2b
diff --git a/scrapling/parser.py b/scrapling/parser.py
index 9366e74..abe77e2 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -53,6 +53,19 @@ _whitelisted = {
"for_": "for",
}
_T = TypeVar("_T")
+
+
+def _escape_css_string(value: str) -> str:
+ """Escape the characters that can't appear literally inside a CSS double-quoted string.
+
+ Line breaks have to be written as hexadecimal escapes, the trailing space ends the escape.
+ Backslashes are left alone on purpose: `cssselect` drops characters while unescaping them
+ (both `\\\\` and `\\5C ` lose the backslash and the character after it), so no form we could
+ emit here would match them anyway.
+ """
+ return value.replace('"', r"\"").replace("\n", r"\A ").replace("\r", r"\D ").replace("\f", r"\C ")
+
+
# Pre-compiled selectors for efficiency
_find_all_elements = XPath(".//*")
_find_all_elements_with_spaces = XPath(
@@ -760,13 +773,13 @@ class Selector(SelectorsGeneration):
for tag in tags:
selector = tag
for key, value in attributes.items():
- value = value.replace('"', r"\"") # Escape double quotes in user input
# Not escaping anything with the key so the user can pass patterns like {'href*': '/p/'} or get errors :)
- if key == "class":
+ if key == "class" and (class_names := value.split()):
# `class` is a space-separated list, so exact-match [class="x"] misses class="x y"; match each name with ~=
- selector += "".join('[class~="{}"]'.format(t) for t in value.split())
+ # An empty/blank value has no names to match, so it falls through to the exact match below
+ selector += "".join('[class~="{}"]'.format(_escape_css_string(name)) for name in class_names)
else:
- selector += '[{}="{}"]'.format(key, value)
+ selector += '[{}="{}"]'.format(key, _escape_css_string(value))
if selector != "*":
selectors.append(selector)
diff --git a/tests/parser/test_general.py b/tests/parser/test_general.py
index 7311ba2..cc57d33 100644
--- a/tests/parser/test_general.py
+++ b/tests/parser/test_general.py
@@ -175,6 +175,22 @@ class TestFindByClass:
assert len(page.find_all("div", class_="stock hidden")) == 3
assert page.find_all("div", class_="hidden nonexistent") == []
+ def test_find_all_keeps_filtering_on_a_blank_class(self):
+ """A blank class has no names to match, so it must match `class=""` instead of being dropped"""
+ blank_class = Selector('<div class="">empty</div><div class="a">a</div><div>none</div>')
+
+ assert [element.get_all_text() for element in blank_class.find_all("div", class_="")] == ["empty"]
+ assert blank_class.find_all("div", class_=" ") == []
+
+
+class TestFindByEscapedAttributes:
+ def test_find_all_matches_values_needing_css_escapes(self):
+ """Quotes and line breaks can't appear literally inside a CSS string, so they have to be escaped"""
+ page = Selector('<a title=\'say "hi"\'>quoted</a><a title="one\ntwo">broken</a>')
+
+ assert page.find("a", title='say "hi"').get_all_text() == "quoted"
+ assert page.find("a", title="one\ntwo").get_all_text() == "broken"
+
# Error Handling Tests
class TestErrorHandling: