Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(defaults): Lazy Loading fetchers to lower memory usage on importing

Karim shoair9b0515b1cd2e
diff --git a/scrapling/defaults.py b/scrapling/defaults.py
index 20f0c44..4098f76 100644
--- a/scrapling/defaults.py
+++ b/scrapling/defaults.py
@@ -1,10 +1,19 @@
-from .fetchers import AsyncFetcher as _AsyncFetcher
-from .fetchers import Fetcher as _Fetcher
-from .fetchers import PlayWrightFetcher as _PlayWrightFetcher
-from .fetchers import StealthyFetcher as _StealthyFetcher
-
# If you are going to use Fetchers with the default settings, import them from this file instead for a cleaner looking code
-Fetcher = _Fetcher()
-AsyncFetcher = _AsyncFetcher()
-StealthyFetcher = _StealthyFetcher()
-PlayWrightFetcher = _PlayWrightFetcher()
+
+# A lightweight approach to create lazy loader for each import for backward compatibility
+# This will reduces initial memory footprint significantly (only loads what's used)
+def __getattr__(name):
+ if name == 'Fetcher':
+ from scrapling.fetchers import Fetcher as cls
+ return cls()
+ elif name == 'AsyncFetcher':
+ from scrapling.fetchers import AsyncFetcher as cls
+ return cls()
+ elif name == 'StealthyFetcher':
+ from scrapling.fetchers import StealthyFetcher as cls
+ return cls()
+ elif name == 'PlayWrightFetcher':
+ from scrapling.fetchers import PlayWrightFetcher as cls
+ return cls()
+ else:
+ raise AttributeError(f"module 'scrapling' has no attribute '{name}'")