Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

Toolbelt subpackage custom functions general adjustments

Karim shoaire270b0081126
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index c06d68d..9dee07f 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -5,13 +5,14 @@ import inspect
import logging
from dataclasses import dataclass, field
-from scrapling.parser import Adaptor, SQLiteStorageSystem
from scrapling.core.utils import setup_basic_logging
-from scrapling.core._types import Any, List, Type, Union, Optional, Dict
+from scrapling.parser import Adaptor, SQLiteStorageSystem
+from scrapling.core._types import Any, List, Type, Union, Optional, Dict, Callable
@dataclass(frozen=True)
class Response:
+ """This class is returned by all engines as a way to unify response type between different libraries."""
url: str
text: str
content: bytes
@@ -24,7 +25,8 @@ class Response:
adaptor_arguments: Dict = field(default_factory=dict)
@property
- def adaptor(self):
+ def adaptor(self) -> Union[Adaptor, None]:
+ """Generate Adaptor instance from this response if possible, otherwise return None"""
if self.content:
return Adaptor(body=self.content, url=self.url, encoding=self.encoding, **self.adaptor_arguments)
elif self.text:
@@ -67,7 +69,13 @@ class BaseFetcher:
setup_basic_logging(level='debug' if debug else 'info')
-def check_if_engine_usable(engine):
+def check_if_engine_usable(engine: Callable) -> Union[Callable, None]:
+ """This function check if the passed engine can be used by a Fetcher-type class or not.
+
+ :param engine: The engine class itself
+ :return: The engine class again if all checks out, otherwise raises error
+ :raise TypeError: If engine class don't have fetch method, If engine class have fetch attribute not method, or If engine class have fetch function but it doesn't take arguments
+ """
# if isinstance(engine, type):
# raise TypeError("Expected an engine instance, not a class definition of the engine")
diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py
index 111b17c..2af7b86 100644
--- a/scrapling/engines/toolbelt/navigation.py
+++ b/scrapling/engines/toolbelt/navigation.py
@@ -5,10 +5,11 @@ Functions related to files and URLs
import os
import logging
from urllib.parse import urlparse, urlencode
-from playwright.sync_api import Route
-from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
from scrapling.core._types import Union, Dict
+from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
+
+from playwright.sync_api import Route
def intercept_route(route: Route) -> Union[Route, None]: