Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

refactor: cleaner code for the fetcher's lazy loader

Karim shoair3d07a1533e0a
diff --git a/scrapling/__init__.py b/scrapling/__init__.py
index 40fd294..323be33 100644
--- a/scrapling/__init__.py
+++ b/scrapling/__init__.py
@@ -3,45 +3,24 @@ __version__ = "0.3-beta"
__copyright__ = "Copyright (c) 2024 Karim Shoair"
-# A lightweight approach to create lazy loader for each import for backward compatibility
+# A lightweight approach to create a lazy loader for each import for backward compatibility
# This will reduces initial memory footprint significantly (only loads what's used)
def __getattr__(name):
- if name == "Fetcher":
- from scrapling.fetchers import Fetcher as cls
-
- return cls
- elif name == "Selector":
- from scrapling.parser import Selector as cls
-
- return cls
- elif name == "Selectors":
- from scrapling.parser import Selectors as cls
-
- return cls
- elif name == "AttributesHandler":
- from scrapling.core.custom_types import AttributesHandler as cls
-
- return cls
- elif name == "TextHandler":
- from scrapling.core.custom_types import TextHandler as cls
-
- return cls
- elif name == "AsyncFetcher":
- from scrapling.fetchers import AsyncFetcher as cls
-
- return cls
- elif name == "StealthyFetcher":
- from scrapling.fetchers import StealthyFetcher as cls
-
- return cls
- elif name == "DynamicFetcher":
- from scrapling.fetchers import DynamicFetcher as cls
-
- return cls
- elif name == "CustomFetcher":
- from scrapling.fetchers import CustomFetcher as cls
-
- return cls
+ lazy_imports = {
+ "Fetcher": ("scrapling.fetchers", "Fetcher"),
+ "Selector": ("scrapling.parser", "Selector"),
+ "Selectors": ("scrapling.parser", "Selectors"),
+ "AttributesHandler": ("scrapling.core.custom_types", "AttributesHandler"),
+ "TextHandler": ("scrapling.core.custom_types", "TextHandler"),
+ "AsyncFetcher": ("scrapling.fetchers", "AsyncFetcher"),
+ "StealthyFetcher": ("scrapling.fetchers", "StealthyFetcher"),
+ "DynamicFetcher": ("scrapling.fetchers", "DynamicFetcher"),
+ }
+
+ if name in lazy_imports:
+ module_path, class_name = lazy_imports[name]
+ module = __import__(module_path, fromlist=[class_name])
+ return getattr(module, class_name)
else:
raise AttributeError(f"module 'scrapling' has no attribute '{name}'")