Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(parser/find_all): Logic issues made conditions used sometimes in a (and) fashion and other times (OR)

Karim shoair1962c11f64fd
diff --git a/scrapling/parser.py b/scrapling/parser.py
index 4e96501..e01c680 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -19,7 +19,7 @@ from scrapling.core.storage_adaptors import (SQLiteStorageSystem,
StorageSystemMixin, _StorageTools)
from scrapling.core.translator import HTMLTranslator
from scrapling.core.utils import (clean_spaces, flatten, html_forbidden,
- is_jsonable, log, lru_cache)
+ is_jsonable, log)
class Adaptor(SelectorsGeneration):
@@ -285,7 +285,6 @@ class Adaptor(SelectorsGeneration):
return self.__handle_element(self._root.getparent())
@property
- @lru_cache(None, True)
def below_elements(self) -> 'Adaptors[Adaptor]':
"""Return all elements under the current element in the DOM tree"""
below = self._root.xpath('.//*')
@@ -603,12 +602,12 @@ class Adaptor(SelectorsGeneration):
raise TypeError('Nested Iterables are not accepted, only iterables of tag names are accepted')
tags.update(set(arg))
- elif type(arg) is dict:
+ elif isinstance(arg, dict):
if not all([(type(k) is str and type(v) is str) for k, v in arg.items()]):
raise TypeError('Nested dictionaries are not accepted, only string keys and string values are accepted')
attributes.update(arg)
- elif type(arg) is re.Pattern:
+ elif isinstance(arg, re.Pattern):
patterns.add(arg)
elif callable(arg):
@@ -629,14 +628,14 @@ class Adaptor(SelectorsGeneration):
attributes[attribute_name] = value
# It's easier and faster to build a selector than traversing the tree
- tags = tags or ['']
+ tags = tags or ['*']
for tag in tags:
selector = tag
for key, value in attributes.items():
value = value.replace('"', r'\"') # Escape double quotes in user input
# Not escaping anything with the key so the user can pass patterns like {'href*': '/p/'} or get errors :)
selector += '[{}="{}"]'.format(key, value)
- if selector:
+ if selector != '*':
selectors.append(selector)
if selectors:
@@ -650,12 +649,13 @@ class Adaptor(SelectorsGeneration):
for function in functions:
results = results.filter(function)
else:
+ results = results or self.below_elements
for pattern in patterns:
- results.extend(self.find_by_regex(pattern, first_match=False))
+ results = results.filter(lambda e: e.text.re(pattern, check_match=True))
# Collect element if it fulfills passed function otherwise
for function in functions:
- results.extend((results or self.below_elements).filter(function))
+ results = results.filter(function)
return results