🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Making `find_all`/`find` methods on steroids
Karim shoair6b62579a7e22
diff --git a/scrapling/parser.py b/scrapling/parser.pyindex 61d4796..6530c68 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -1,13 +1,14 @@import os+import re+import inspectfrom difflib import SequenceMatcherfrom scrapling.core.translator import HTMLTranslatorfrom scrapling.core.mixins import SelectorsGenerationfrom scrapling.core.custom_types import TextHandler, TextHandlers, AttributesHandlerfrom scrapling.core.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools-from scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, _is_iterable, html_forbidden+from scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbiddenfrom scrapling.core._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex, Iterable-from lxml import etree, htmlfrom cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors@@ -542,10 +543,10 @@ class Adaptor(SelectorsGeneration):except (SelectorError, SelectorSyntaxError, etree.XPathError, etree.XPathEvalError):raise SelectorSyntaxError(f"Invalid XPath selector: {selector}")- def find_all(self, *args, **kwargs) -> Union['Adaptors[Adaptor]', List]:- """Find elements by their tag name and filter them based on attributes for ease..+ def find_all(self, *args: Union[str, Iterable[str], Pattern, Callable, Dict[str, str]], **kwargs: str) -> Union['Adaptors[Adaptor]', List]:+ """Find elements by filters of your creations for ease..- :param args: Tag name(s), an iterable of tag names, or a dictionary of elements' attributes. Leave empty for selecting all.+ :param args: Tag name(s), an iterable of tag names, regex patterns, function, or a dictionary of elements' attributes. Leave empty for selecting all.:param kwargs: The attributes you want to filter elements based on it.:return: The `Adaptors` object of the elements or empty list"""@@ -560,9 +561,18 @@ class Adaptor(SelectorsGeneration):if not args and not kwargs:raise TypeError('You have to pass something to search with, like tag name(s), tag attributes, or both.')- tags = set()- selectors = []attributes = dict()+ tags, patterns = set(), set()+ results, functions, selectors = [], [], []++ def _search_tree(element: Adaptor, filter_function: Callable) -> None:+ """Collect element if it fulfills passed function otherwise, traverse the children tree and iterate"""+ if filter_function(element):+ results.append(element)++ for branch in element.children:+ _search_tree(branch, filter_function)+# Brace yourself for a wonderful journey!for arg in args:if type(arg) is str:@@ -578,6 +588,15 @@ class Adaptor(SelectorsGeneration):raise TypeError('Nested dictionaries are not accepted, only string keys and string values are accepted')attributes.update(arg)+ elif type(arg) is re.Pattern:+ patterns.add(arg)++ elif callable(arg):+ if len(inspect.signature(arg).parameters) > 0:+ functions.append(arg)+ else:+ raise TypeError("Callable filter function must have at least one argument to take `Adaptor` objects.")+else:raise TypeError(f'Argument with type "{type(arg)}" is not accepted, please read the docs.')@@ -597,14 +616,32 @@ class Adaptor(SelectorsGeneration):value = value.replace('"', r'\"') # Escape double quotes in user input# Not escaping anything with the key so the user can pass patterns like {'href*': '/p/'} or get errors :)selector += '[{}="{}"]'.format(key, value)- selectors.append(selector)+ if selector:+ selectors.append(selector)- return self.css(', '.join(selectors))+ if selectors:+ results = self.css(', '.join(selectors))+ if results:+ # From the results, get the ones that fulfill passed regex patterns+ for pattern in patterns:+ results = results.filter(lambda e: e.text.re(pattern, check_match=True))++ # From the results, get the ones that fulfill passed functions+ for function in functions:+ results = results.filter(function)+ else:+ for pattern in patterns:+ results.extend(self.find_by_regex(pattern, first_match=False))++ for function in functions:+ _search_tree(self, function)++ return self.__convert_results(results)- def find(self, *args, **kwargs) -> Union['Adaptor', None]:- """Find elements by their tag name and filter them based on attributes for ease then return the first result. Otherwise return `None`.+ def find(self, *args: Union[str, Iterable[str], Pattern, Callable, Dict[str, str]], **kwargs: str) -> Union['Adaptor', None]:+ """Find elements by filters of your creations for ease then return the first result. Otherwise return `None`.- :param args: Tag name(s), an iterable of tag names, or a dictionary of elements' attributes. Leave empty for selecting all.+ :param args: Tag name(s), an iterable of tag names, regex patterns, function, or a dictionary of elements' attributes. Leave empty for selecting all.:param kwargs: The attributes you want to filter elements based on it.:return: The `Adaptor` object of the element or `None` if the result didn't match"""@@ -882,10 +919,10 @@ class Adaptor(SelectorsGeneration):return self.__convert_results(results)def find_by_regex(- self, query: str, first_match: bool = True, case_sensitive: bool = False, clean_match: bool = True+ self, query: Union[str, Pattern[str]], first_match: bool = True, case_sensitive: bool = False, clean_match: bool = True) -> Union['Adaptors[Adaptor]', 'Adaptor', List]:"""Find elements that its text content matches the input regex pattern.- :param query: Regex query to match+ :param query: Regex query/pattern to match:param first_match: Return first element that matches conditions, enabled by default:param case_sensitive: if enabled, letters case will be taken into consideration in the regex:param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching