Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: Fixes for the type checking in the main `init` file

Karim shoaireedfa855ab47
diff --git a/scrapling/__init__.py b/scrapling/__init__.py
index c4bd97d..d30a8db 100644
--- a/scrapling/__init__.py
+++ b/scrapling/__init__.py
@@ -2,27 +2,37 @@ __author__ = "Karim Shoair ([email protected])"
__version__ = "0.3.6"
__copyright__ = "Copyright (c) 2024 Karim Shoair"
+from typing import Any, TYPE_CHECKING
-# A lightweight approach to create a lazy loader for each import for backward compatibility
-# This will reduces initial memory footprint significantly (only loads what's used)
-def __getattr__(name):
- lazy_imports = {
- "Fetcher": ("scrapling.fetchers", "Fetcher"),
- "Selector": ("scrapling.parser", "Selector"),
- "Selectors": ("scrapling.parser", "Selectors"),
- "AttributesHandler": ("scrapling.core.custom_types", "AttributesHandler"),
- "TextHandler": ("scrapling.core.custom_types", "TextHandler"),
- "AsyncFetcher": ("scrapling.fetchers", "AsyncFetcher"),
- "StealthyFetcher": ("scrapling.fetchers", "StealthyFetcher"),
- "DynamicFetcher": ("scrapling.fetchers", "DynamicFetcher"),
- }
-
- if name in lazy_imports:
- module_path, class_name = lazy_imports[name]
+if TYPE_CHECKING:
+ from scrapling.parser import Selector, Selectors
+ from scrapling.core.custom_types import AttributesHandler, TextHandler
+ from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher
+
+
+# Lazy import mapping
+_LAZY_IMPORTS = {
+ "Fetcher": ("scrapling.fetchers", "Fetcher"),
+ "Selector": ("scrapling.parser", "Selector"),
+ "Selectors": ("scrapling.parser", "Selectors"),
+ "AttributesHandler": ("scrapling.core.custom_types", "AttributesHandler"),
+ "TextHandler": ("scrapling.core.custom_types", "TextHandler"),
+ "AsyncFetcher": ("scrapling.fetchers", "AsyncFetcher"),
+ "StealthyFetcher": ("scrapling.fetchers", "StealthyFetcher"),
+ "DynamicFetcher": ("scrapling.fetchers", "DynamicFetcher"),
+}
+__all__ = ["Selector", "Fetcher", "AsyncFetcher", "StealthyFetcher", "DynamicFetcher"]
+
+
+def __getattr__(name: str) -> Any:
+ if name in _LAZY_IMPORTS:
+ module_path, class_name = _LAZY_IMPORTS[name]
module = __import__(module_path, fromlist=[class_name])
return getattr(module, class_name)
else:
- raise AttributeError(f"module 'scrapling' has no attribute '{name}'")
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
-__all__ = ["Selector", "Fetcher", "AsyncFetcher", "StealthyFetcher", "DynamicFetcher"]
+def __dir__() -> list[str]:
+ """Support for dir() and autocomplete."""
+ return sorted(__all__ + ["fetchers", "parser", "cli", "core", "__author__", "__version__", "__copyright__"])
diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py
index cd8c9c0..59422f2 100644
--- a/scrapling/core/_types.py
+++ b/scrapling/core/_types.py
@@ -39,6 +39,4 @@ except ImportError: # pragma: no cover
try:
from typing_extensions import Self # Backport
except ImportError:
- from typing import TypeVar
-
Self = object
diff --git a/scrapling/engines/_browsers/__init__.py b/scrapling/engines/_browsers/__init__.py
index 2cc5947..e69de29 100644
--- a/scrapling/engines/_browsers/__init__.py
+++ b/scrapling/engines/_browsers/__init__.py
@@ -1,2 +0,0 @@
-from ._controllers import DynamicSession, AsyncDynamicSession
-from ._camoufox import StealthySession, AsyncStealthySession