🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Adding documentation to the base fetcher...
Karim shoairbe350f21dd8e
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex 3d71b4f..c06d68d 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -6,6 +6,7 @@ import loggingfrom dataclasses import dataclass, fieldfrom scrapling.parser import Adaptor, SQLiteStorageSystem+from scrapling.core.utils import setup_basic_loggingfrom scrapling.core._types import Any, List, Type, Union, Optional, Dict@@ -36,15 +37,23 @@ class Response:class BaseFetcher:def __init__(- self,- # Adaptor class parameters- huge_tree: bool = True,- keep_comments: Optional[bool] = False,- auto_match: Optional[bool] = False,- storage: Any = SQLiteStorageSystem,- storage_args: Optional[Dict] = None,- debug: Optional[bool] = True,+ self, huge_tree: bool = True, keep_comments: Optional[bool] = False, auto_match: Optional[bool] = True,+ storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None, debug: Optional[bool] = True,):+ """Arguments below are the same from the Adaptor class so you can pass them directly, the rest of Adaptor's arguments+ are detected and passed automatically from the Fetcher based on the response for accessibility.++ :param huge_tree: Enabled by default, should always be enabled when parsing large HTML documents. This controls+ libxml2 feature that forbids parsing certain large documents to protect from possible memory exhaustion.+ :param keep_comments: While parsing the HTML body, drop comments or not. Disabled by default for obvious reasons+ :param auto_match: Globally turn-off the auto-match feature in all functions, this argument takes higher+ priority over all auto-match related arguments/functions in the class.+ :param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info.+ :param storage_args: A dictionary of ``argument->value`` pairs to be passed for the storage class.+ If empty, default values will be used.+ :param debug: Enable debug mode+ """+ # Adaptor class parameters# I won't validate Adaptor's class parameters here again, I will leave it to be validated laterself.adaptor_arguments = dict(huge_tree=huge_tree,@@ -54,6 +63,8 @@ class BaseFetcher:storage_args=storage_args,debug=debug,)+ # If the user used fetchers first, then configure the logger from here instead of the `Adaptor` class+ setup_basic_logging(level='debug' if debug else 'info')def check_if_engine_usable(engine):