Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

Adding documentation to the base fetcher...

Karim shoairbe350f21dd8e
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index 3d71b4f..c06d68d 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -6,6 +6,7 @@ import logging
from dataclasses import dataclass, field
from scrapling.parser import Adaptor, SQLiteStorageSystem
+from scrapling.core.utils import setup_basic_logging
from scrapling.core._types import Any, List, Type, Union, Optional, Dict
@@ -36,15 +37,23 @@ class Response:
class BaseFetcher:
def __init__(
- self,
- # Adaptor class parameters
- huge_tree: bool = True,
- keep_comments: Optional[bool] = False,
- auto_match: Optional[bool] = False,
- storage: Any = SQLiteStorageSystem,
- storage_args: Optional[Dict] = None,
- debug: Optional[bool] = True,
+ self, huge_tree: bool = True, keep_comments: Optional[bool] = False, auto_match: Optional[bool] = True,
+ storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None, debug: Optional[bool] = True,
):
+ """Arguments below are the same from the Adaptor class so you can pass them directly, the rest of Adaptor's arguments
+ are detected and passed automatically from the Fetcher based on the response for accessibility.
+
+ :param huge_tree: Enabled by default, should always be enabled when parsing large HTML documents. This controls
+ libxml2 feature that forbids parsing certain large documents to protect from possible memory exhaustion.
+ :param keep_comments: While parsing the HTML body, drop comments or not. Disabled by default for obvious reasons
+ :param auto_match: Globally turn-off the auto-match feature in all functions, this argument takes higher
+ priority over all auto-match related arguments/functions in the class.
+ :param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info.
+ :param storage_args: A dictionary of ``argument->value`` pairs to be passed for the storage class.
+ If empty, default values will be used.
+ :param debug: Enable debug mode
+ """
+ # Adaptor class parameters
# I won't validate Adaptor's class parameters here again, I will leave it to be validated later
self.adaptor_arguments = dict(
huge_tree=huge_tree,
@@ -54,6 +63,8 @@ class BaseFetcher:
storage_args=storage_args,
debug=debug,
)
+ # If the user used fetchers first, then configure the logger from here instead of the `Adaptor` class
+ setup_basic_logging(level='debug' if debug else 'info')
def check_if_engine_usable(engine):