🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor(api)!: Unifying log under 1 logger and removing debug parameter
Karim shoair193827e27b2b
diff --git a/.github/ISSUE_TEMPLATE/01-bug_report.yml b/.github/ISSUE_TEMPLATE/01-bug_report.ymlindex 6a34895..0340da1 100644--- a/.github/ISSUE_TEMPLATE/01-bug_report.yml+++ b/.github/ISSUE_TEMPLATE/01-bug_report.yml@@ -65,7 +65,7 @@ body:- type: textareaattributes:- label: "Actual behavior (Remember to use `debug` parameter)"+ label: "Actual behavior"validations:required: truediff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 2e3b3b2..8adf4d1 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -19,7 +19,11 @@ tests/test_parser_functions.py ................ [100%]=============================== 16 passed in 0.22s ================================```-Also, consider setting `debug` to `True` while initializing the Adaptor object so it's easier to know what's happening in the background.+Also, consider setting the scrapling logging level to `debug` so it's easier to know what's happening in the background.+```python+>>> import logging+>>> logging.getLogger("scrapling").setLevel(logging.DEBUG)+```### The process is straight-forward.diff --git a/README.md b/README.mdindex 2f8455c..a717b7d 100644--- a/README.md+++ b/README.md@@ -219,7 +219,7 @@ You might be slightly confused by now so let me clear things up. All fetcher-typ```pythonfrom scrapling import Fetcher, StealthyFetcher, PlayWrightFetcher```-All of them can take these initialization arguments: `auto_match`, `huge_tree`, `keep_comments`, `keep_cdata`, `storage`, `storage_args`, and `debug`, which are the same ones you give to the `Adaptor` class.+All of them can take these initialization arguments: `auto_match`, `huge_tree`, `keep_comments`, `keep_cdata`, `storage`, and `storage_args`, which are the same ones you give to the `Adaptor` class.If you don't want to pass arguments to the generated `Adaptor` object and want to use the default values, you can use this import instead for cleaner code:```pythondiff --git a/benchmarks.py b/benchmarks.pyindex de647d6..28af680 100644--- a/benchmarks.py+++ b/benchmarks.py@@ -64,9 +64,9 @@ def test_pyquery():@benchmarkdef test_scrapling():# No need to do `.extract()` like parsel to extract text- # Also, this is faster than `[t.text for t in Adaptor(large_html, auto_match=False, debug=False).css('.item')]`+ # Also, this is faster than `[t.text for t in Adaptor(large_html, auto_match=False).css('.item')]`# for obvious reasons, of course.- return Adaptor(large_html, auto_match=False, debug=False).css('.item::text')+ return Adaptor(large_html, auto_match=False).css('.item::text')@benchmark@@ -103,7 +103,7 @@ def test_scrapling_text(request_html):# Will loop over resulted elements to get text too to make comparison even more fair otherwise Scrapling will be even fasterreturn [element.text for element in Adaptor(- request_html, auto_match=False, debug=False+ request_html, auto_match=False).find_by_text('Tipping the Velvet', first_match=True).find_similar(ignore_attributes=['title'])]diff --git a/scrapling/core/storage_adaptors.py b/scrapling/core/storage_adaptors.pyindex 983e863..c614643 100644--- a/scrapling/core/storage_adaptors.py+++ b/scrapling/core/storage_adaptors.py@@ -1,4 +1,3 @@-import loggingimport sqlite3import threadingfrom abc import ABC, abstractmethod@@ -9,7 +8,7 @@ from lxml import htmlfrom tldextract import extract as tldfrom scrapling.core._types import Dict, Optional, Union-from scrapling.core.utils import _StorageTools, cache+from scrapling.core.utils import _StorageTools, log, lru_cacheclass StorageSystemMixin(ABC):@@ -20,7 +19,7 @@ class StorageSystemMixin(ABC):"""self.url = url- @cache(None, typed=True)+ @lru_cache(None, typed=True)def _get_base_url(self, default_value: str = 'default') -> str:if not self.url or type(self.url) is not str:return default_value@@ -52,7 +51,7 @@ class StorageSystemMixin(ABC):raise NotImplementedError('Storage system must implement `save` method')@staticmethod- @cache(None, typed=True)+ @lru_cache(None, typed=True)def _get_hash(identifier: str) -> str:"""If you want to hash identifier in your storage system, use this safer"""identifier = identifier.lower().strip()@@ -64,7 +63,7 @@ class StorageSystemMixin(ABC):return f"{hash_value}_{len(identifier)}" # Length to reduce collision chance-@cache(None, typed=True)+@lru_cache(None, typed=True)class SQLiteStorageSystem(StorageSystemMixin):"""The recommended system to use, it's race condition safe and thread safe.Mainly built so the library can run in threaded frameworks like scrapy or threaded tools@@ -86,7 +85,7 @@ class SQLiteStorageSystem(StorageSystemMixin):self.connection.execute("PRAGMA journal_mode=WAL")self.cursor = self.connection.cursor()self._setup_database()- logging.debug(+ log.debug(f'Storage system loaded with arguments (storage_file="{storage_file}", url="{url}")')diff --git a/scrapling/core/translator.py b/scrapling/core/translator.pyindex aa6211e..263a24a 100644--- a/scrapling/core/translator.py+++ b/scrapling/core/translator.py@@ -17,7 +17,7 @@ from cssselect.xpath import XPathExpr as OriginalXPathExprfrom w3lib.html import HTML5_WHITESPACEfrom scrapling.core._types import Any, Optional, Protocol, Self-from scrapling.core.utils import cache+from scrapling.core.utils import lru_cacheregex = f"[{HTML5_WHITESPACE}]+"replace_html5_whitespaces = re.compile(regex).sub@@ -139,6 +139,6 @@ class TranslatorMixin:class HTMLTranslator(TranslatorMixin, OriginalHTMLTranslator):- @cache(maxsize=256)+ @lru_cache(maxsize=256)def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str:return super().css_to_xpath(css, prefix)diff --git a/scrapling/core/utils.py b/scrapling/core/utils.pyindex a911757..27cb884 100644--- a/scrapling/core/utils.py+++ b/scrapling/core/utils.py@@ -9,18 +9,36 @@ from scrapling.core._types import Any, Dict, Iterable, Union# Using cache on top of a class is brilliant way to achieve Singleton design pattern without much code# functools.cache is available on Python 3.9+ only so let's keep lru_cache-from functools import lru_cache as cache # isort:skip-+from functools import lru_cache # isort:skiphtml_forbidden = {html.HtmlComment, }-logging.basicConfig(- level=logging.INFO,- format="[%(asctime)s] %(levelname)s: %(message)s",- datefmt="%Y-%m-%d %H:%M:%S",- handlers=[- logging.StreamHandler()- ]-)+++@lru_cache(1, typed=True)+def setup_logger():+ """Create and configure a logger with a standard format.++ :returns: logging.Logger: Configured logger instance+ """+ logger = logging.getLogger('scrapling')+ logger.setLevel(logging.INFO)++ formatter = logging.Formatter(+ fmt="[%(asctime)s] %(levelname)s: %(message)s",+ datefmt="%Y-%m-%d %H:%M:%S"+ )++ console_handler = logging.StreamHandler()+ console_handler.setFormatter(formatter)++ # Add handler to logger (if not already added)+ if not logger.handlers:+ logger.addHandler(console_handler)++ return logger+++log = setup_logger()def is_jsonable(content: Union[bytes, str]) -> bool:@@ -34,23 +52,6 @@ def is_jsonable(content: Union[bytes, str]) -> bool:return False-@cache(None, typed=True)-def setup_basic_logging(level: str = 'debug'):- levels = {- 'debug': logging.DEBUG,- 'info': logging.INFO,- 'warning': logging.WARNING,- 'error': logging.ERROR,- 'critical': logging.CRITICAL- }- formatter = logging.Formatter("[%(asctime)s] %(levelname)s: %(message)s", "%Y-%m-%d %H:%M:%S")- lvl = levels[level.lower()]- handler = logging.StreamHandler()- handler.setFormatter(formatter)- # Configure the root logger- logging.basicConfig(level=lvl, handlers=[handler])--def flatten(lst: Iterable):return list(chain.from_iterable(lst))@@ -114,7 +115,7 @@ class _StorageTools:# return _impl-@cache(None, typed=True)+@lru_cache(None, typed=True)def clean_spaces(string):string = string.replace('\t', ' ')string = re.sub('[\n|\r]', '', string)diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.pyindex e7a6575..d755275 100644--- a/scrapling/engines/camo.py+++ b/scrapling/engines/camo.py@@ -1,10 +1,9 @@-import logging-from camoufox import DefaultAddonsfrom camoufox.sync_api import Camoufoxfrom scrapling.core._types import (Callable, Dict, List, Literal, Optional,Union)+from scrapling.core.utils import logfrom scrapling.engines.toolbelt import (Response, StatusText,check_type_validity,construct_proxy_dict, do_nothing,@@ -63,7 +62,7 @@ class CamoufoxEngine:self.page_action = page_actionelse:self.page_action = do_nothing- logging.error('[Ignored] Argument "page_action" must be callable')+ log.error('[Ignored] Argument "page_action" must be callable')self.wait_selector = wait_selectorself.wait_selector_state = wait_selector_statediff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.pyindex 7d15174..8b2895c 100644--- a/scrapling/engines/pw.py+++ b/scrapling/engines/pw.py@@ -1,7 +1,7 @@import json-import loggingfrom scrapling.core._types import Callable, Dict, List, Optional, Union+from scrapling.core.utils import logfrom scrapling.engines.constants import (DEFAULT_STEALTH_FLAGS,NSTBROWSER_DEFAULT_QUERY)from scrapling.engines.toolbelt import (Response, StatusText,@@ -78,7 +78,7 @@ class PlaywrightEngine:self.page_action = page_actionelse:self.page_action = do_nothing- logging.error('[Ignored] Argument "page_action" must be callable')+ log.error('[Ignored] Argument "page_action" must be callable')self.wait_selector = wait_selectorself.wait_selector_state = wait_selector_statediff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex a091c4f..9bad034 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -1,5 +1,3 @@-import logging-import httpxfrom httpx._models import Response as httpxResponse@@ -36,7 +34,6 @@ class StaticEngine:# Validate headersif not headers.get('user-agent') and not headers.get('User-Agent'):headers['User-Agent'] = generate_headers(browser_mode=False).get('User-Agent')- logging.info(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.")if stealth:extra_headers = generate_headers(browser_mode=False)diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex 7c8acd6..6632b6b 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -2,13 +2,12 @@Functions related to custom types or type checking"""import inspect-import loggingfrom email.message import Messagefrom scrapling.core._types import (Any, Callable, Dict, List, Optional, Tuple,Type, Union)from scrapling.core.custom_types import MappingProxyType-from scrapling.core.utils import cache, setup_basic_logging+from scrapling.core.utils import log, lru_cachefrom scrapling.parser import Adaptor, SQLiteStorageSystem@@ -17,7 +16,7 @@ class ResponseEncoding:__ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"}@classmethod- @cache(maxsize=None)+ @lru_cache(maxsize=None)def __parse_content_type(cls, header_value: str) -> Tuple[str, Dict[str, str]]:"""Parse content type and parameters from a content-type header value.@@ -39,7 +38,7 @@ class ResponseEncoding:return content_type, params@classmethod- @cache(maxsize=None)+ @lru_cache(maxsize=None)def get_value(cls, content_type: Optional[str], text: Optional[str] = 'test') -> str:"""Determine the appropriate character encoding from a content-type header.@@ -98,7 +97,7 @@ class Response(Adaptor):# For back-ward compatibilityself.adaptor = self# For easier debugging while working from a Python shell- logging.info(f'Fetched ({status}) <{method} {url}> (referer: {request_headers.get("referer")})')+ log.info(f'Fetched ({status}) <{method} {url}> (referer: {request_headers.get("referer")})')# def __repr__(self):# return f'<{self.__class__.__name__} [{self.status} {self.reason}]>'@@ -107,7 +106,7 @@ class Response(Adaptor):class BaseFetcher:def __init__(self, huge_tree: bool = True, keep_comments: Optional[bool] = False, auto_match: Optional[bool] = True,- storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None, debug: Optional[bool] = False,+ storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None,automatch_domain: Optional[str] = None, keep_cdata: Optional[bool] = False,):"""Arguments below are the same from the Adaptor class so you can pass them directly, the rest of Adaptor's arguments@@ -124,7 +123,6 @@ class BaseFetcher:If empty, default values will be used.:param automatch_domain: For cases where you want to automatch selectors across different websites as if they were on the same website, use this argument to unify them.Otherwise, the domain of the request is used by default.- :param debug: Enable debug mode"""# Adaptor class parameters# I won't validate Adaptor's class parameters here again, I will leave it to be validated later@@ -134,14 +132,11 @@ class BaseFetcher:keep_cdata=keep_cdata,auto_match=auto_match,storage=storage,- storage_args=storage_args,- debug=debug,+ storage_args=storage_args)- # If the user used fetchers first, then configure the logger from here instead of the `Adaptor` class- setup_basic_logging(level='debug' if debug else 'info')if automatch_domain:if type(automatch_domain) is not str:- logging.warning('[Ignored] The argument "automatch_domain" must be of string type')+ log.warning('[Ignored] The argument "automatch_domain" must be of string type')else:self.adaptor_arguments.update({'automatch_domain': automatch_domain})@@ -217,7 +212,7 @@ class StatusText:})@classmethod- @cache(maxsize=128)+ @lru_cache(maxsize=128)def get(cls, status_code: int) -> str:"""Get the phrase for a given HTTP status code."""return cls._phrases.get(status_code, "Unknown Status Code")@@ -284,7 +279,7 @@ def check_type_validity(variable: Any, valid_types: Union[List[Type], None], deferror_msg = f'Argument "{var_name}" cannot be None'if critical:raise TypeError(error_msg)- logging.error(f'[Ignored] {error_msg}')+ log.error(f'[Ignored] {error_msg}')return default_value# If no valid_types specified and variable has a value, return it@@ -297,7 +292,7 @@ def check_type_validity(variable: Any, valid_types: Union[List[Type], None], deferror_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}'if critical:raise TypeError(error_msg)- logging.error(f'[Ignored] {error_msg}')+ log.error(f'[Ignored] {error_msg}')return default_valuereturn variablediff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.pyindex 5600003..a7bf633 100644--- a/scrapling/engines/toolbelt/fingerprints.py+++ b/scrapling/engines/toolbelt/fingerprints.py@@ -9,10 +9,10 @@ from browserforge.headers import Browser, HeaderGeneratorfrom tldextract import extractfrom scrapling.core._types import Dict, Union-from scrapling.core.utils import cache+from scrapling.core.utils import lru_cache-@cache(None, typed=True)+@lru_cache(None, typed=True)def generate_convincing_referer(url: str) -> str:"""Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website@@ -26,7 +26,7 @@ def generate_convincing_referer(url: str) -> str:return f'https://www.google.com/search?q={website_name}'-@cache(None, typed=True)+@lru_cache(None, typed=True)def get_os_name() -> Union[str, None]:"""Get the current OS name in the same format needed for browserforgediff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.pyindex 2d24cac..5811a76 100644--- a/scrapling/engines/toolbelt/navigation.py+++ b/scrapling/engines/toolbelt/navigation.py@@ -1,15 +1,13 @@"""Functions related to files and URLs"""--import loggingimport osfrom urllib.parse import urlencode, urlparsefrom playwright.sync_api import Routefrom scrapling.core._types import Dict, Optional, Union-from scrapling.core.utils import cache+from scrapling.core.utils import log, lru_cachefrom scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES@@ -20,7 +18,7 @@ def intercept_route(route: Route) -> Union[Route, None]::return: PlayWright `Route` object"""if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:- logging.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')+ log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')return route.abort()return route.continue_()@@ -97,7 +95,7 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:raise ValueError(f"Invalid CDP URL: {str(e)}")-@cache(None, typed=True)+@lru_cache(None, typed=True)def js_bypass_path(filename: str) -> str:"""Takes the base filename of JS file inside the `bypasses` folder then return the full path of itdiff --git a/scrapling/parser.py b/scrapling/parser.pyindex 123fc6b..47d2b32 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -18,12 +18,12 @@ from scrapling.core.storage_adaptors import (SQLiteStorageSystem,StorageSystemMixin, _StorageTools)from scrapling.core.translator import HTMLTranslatorfrom scrapling.core.utils import (clean_spaces, flatten, html_forbidden,- is_jsonable, logging, setup_basic_logging)+ is_jsonable, log)class Adaptor(SelectorsGeneration):__slots__ = (- 'url', 'encoding', '__auto_match_enabled', '_root', '_storage', '__debug',+ 'url', 'encoding', '__auto_match_enabled', '_root', '_storage','__keep_comments', '__huge_tree_enabled', '__attributes', '__text', '__tag','__keep_cdata', '__raw_body')@@ -41,7 +41,6 @@ class Adaptor(SelectorsGeneration):auto_match: Optional[bool] = True,storage: Any = SQLiteStorageSystem,storage_args: Optional[Dict] = None,- debug: Optional[bool] = True,**kwargs):"""The main class that works as a wrapper for the HTML input data. Using this class, you can search for elements@@ -67,7 +66,6 @@ class Adaptor(SelectorsGeneration)::param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info.:param storage_args: A dictionary of ``argument->value`` pairs to be passed for the storage class.If empty, default values will be used.- :param debug: Enable debug mode"""if root is None and not body and text is None:raise ValueError("Adaptor class needs text, body, or root arguments to work")@@ -106,7 +104,6 @@ class Adaptor(SelectorsGeneration):self._root = root- setup_basic_logging(level='debug' if debug else 'info')self.__auto_match_enabled = auto_matchif self.__auto_match_enabled:@@ -117,7 +114,7 @@ class Adaptor(SelectorsGeneration):}if not hasattr(storage, '__wrapped__'):- raise ValueError("Storage class must be wrapped with cache decorator, see docs for info")+ raise ValueError("Storage class must be wrapped with lru_cache decorator, see docs for info")if not issubclass(storage.__wrapped__, StorageSystemMixin):raise ValueError("Storage system must be inherited from class `StorageSystemMixin`")@@ -132,7 +129,6 @@ class Adaptor(SelectorsGeneration):# For selector stuffself.__attributes = Noneself.__tag = None- self.__debug = debug# No need to check if all response attributes exist or not because if `status` exist, then the rest exist (Save some CPU cycles for speed)self.__response_data = {key: getattr(self, key) for key in ('status', 'reason', 'cookies', 'headers', 'request_headers',)@@ -164,7 +160,7 @@ class Adaptor(SelectorsGeneration):text='', body=b'', # Since root argument is provided, both `text` and `body` will be ignored so this is just a fillerurl=self.url, encoding=self.encoding, auto_match=self.__auto_match_enabled,keep_comments=self.__keep_comments, keep_cdata=self.__keep_cdata,- huge_tree=self.__huge_tree_enabled, debug=self.__debug,+ huge_tree=self.__huge_tree_enabled,**self.__response_data)return element@@ -417,10 +413,10 @@ class Adaptor(SelectorsGeneration):if score_table:highest_probability = max(score_table.keys())if score_table[highest_probability] and highest_probability >= percentage:- logging.debug(f'Highest probability was {highest_probability}%')- logging.debug('Top 5 best matching elements are: ')+ log.debug(f'Highest probability was {highest_probability}%')+ log.debug('Top 5 best matching elements are: ')for percent in tuple(sorted(score_table.keys(), reverse=True))[:5]:- logging.debug(f'{percent} -> {self.__convert_results(score_table[percent])}')+ log.debug(f'{percent} -> {self.__convert_results(score_table[percent])}')if not adaptor_type:return score_table[highest_probability]return self.__convert_results(score_table[highest_probability])@@ -546,7 +542,7 @@ class Adaptor(SelectorsGeneration):if selected_elements:if not self.__auto_match_enabled and auto_save:- logging.warning("Argument `auto_save` will be ignored because `auto_match` wasn't enabled on initialization. Check docs for more info.")+ log.warning("Argument `auto_save` will be ignored because `auto_match` wasn't enabled on initialization. Check docs for more info.")elif self.__auto_match_enabled and auto_save:self.save(selected_elements[0], identifier or selector)@@ -565,7 +561,7 @@ class Adaptor(SelectorsGeneration):return self.__convert_results(selected_elements)elif not self.__auto_match_enabled and auto_match:- logging.warning("Argument `auto_match` will be ignored because `auto_match` wasn't enabled on initialization. Check docs for more info.")+ log.warning("Argument `auto_match` will be ignored because `auto_match` wasn't enabled on initialization. Check docs for more info.")return self.__convert_results(selected_elements)@@ -769,7 +765,7 @@ class Adaptor(SelectorsGeneration):self._storage.save(element, identifier)else:- logging.critical(+ log.critical("Can't use Auto-match features with disabled globally, you have to start a new class instance.")@@ -783,7 +779,7 @@ class Adaptor(SelectorsGeneration):if self.__auto_match_enabled:return self._storage.retrieve(identifier)- logging.critical(+ log.critical("Can't use Auto-match features with disabled globally, you have to start a new class instance.")diff --git a/tests/parser/test_automatch.py b/tests/parser/test_automatch.pyindex 1e78e87..79fe6b7 100644--- a/tests/parser/test_automatch.py+++ b/tests/parser/test_automatch.py@@ -42,8 +42,8 @@ class TestParserAutoMatch(unittest.TestCase):</div>'''- old_page = Adaptor(original_html, url='example.com', auto_match=True, debug=True)- new_page = Adaptor(changed_html, url='example.com', auto_match=True, debug=True)+ old_page = Adaptor(original_html, url='example.com', auto_match=True)+ new_page = Adaptor(changed_html, url='example.com', auto_match=True)# 'p1' was used as ID and now it's not and all the path elements have changes# Also at the same time testing auto-match vs combined selectorsdiff --git a/tests/parser/test_general.py b/tests/parser/test_general.pyindex ea1fb78..8ce369a 100644--- a/tests/parser/test_general.py+++ b/tests/parser/test_general.py@@ -74,7 +74,7 @@ class TestParser(unittest.TestCase):</body></html>'''- self.page = Adaptor(self.html, auto_match=False, debug=False)+ self.page = Adaptor(self.html, auto_match=False)def test_css_selector(self):"""Test Selecting elements with complex CSS selectors"""@@ -273,7 +273,7 @@ class TestParser(unittest.TestCase):large_html = '<html><body>' + '<div class="item">' * 5000 + '</div>' * 5000 + '</body></html>'start_time = time.time()- parsed = Adaptor(large_html, auto_match=False, debug=False)+ parsed = Adaptor(large_html, auto_match=False)elements = parsed.css('.item')end_time = time.time()