Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

refactor: removing unused code

Karim shoair3754547e58d3
diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.py
index 796b89f..59b41fa 100644
--- a/scrapling/engines/toolbelt/__init__.py
+++ b/scrapling/engines/toolbelt/__init__.py
@@ -2,7 +2,6 @@ from .custom import (
BaseFetcher,
Response,
StatusText,
- check_if_engine_usable,
check_type_validity,
get_variable_name,
)
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index 63ea1fc..adbd4c2 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -2,12 +2,10 @@
Functions related to custom types or type checking
"""
-import inspect
from email.message import Message
from scrapling.core._types import (
Any,
- Callable,
Dict,
List,
Optional,
@@ -315,38 +313,6 @@ class StatusText:
return cls._phrases.get(status_code, "Unknown Status Code")
-def check_if_engine_usable(engine: Callable) -> Union[Callable, None]:
- """This function check if the passed engine can be used by a Fetcher-type class or not.
-
- :param engine: The engine class itself
- :return: The engine class again if all checks out, otherwise raises error
- :raise TypeError: If engine class don't have fetch method, If engine class have fetch attribute not method, or If engine class have fetch function but it doesn't take arguments
- """
- # if isinstance(engine, type):
- # raise TypeError("Expected an engine instance, not a class definition of the engine")
-
- if hasattr(engine, "fetch"):
- fetch_function = getattr(engine, "fetch")
- if callable(fetch_function):
- if len(inspect.signature(fetch_function).parameters) > 0:
- return engine
- else:
- # raise TypeError("Engine class instance must have a callable method 'fetch' with the first argument used for the url.")
- raise TypeError(
- "Engine class must have a callable method 'fetch' with the first argument used for the url."
- )
- else:
- # raise TypeError("Invalid engine instance! Engine class must have a callable method 'fetch'")
- raise TypeError(
- "Invalid engine class! Engine class must have a callable method 'fetch'"
- )
- else:
- # raise TypeError("Invalid engine instance! Engine class must have the method 'fetch'")
- raise TypeError(
- "Invalid engine class! Engine class must have the method 'fetch'"
- )
-
-
def get_variable_name(var: Any) -> Optional[str]:
"""Get the name of a variable using global and local scopes.
:param var: The variable to find the name for