Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(importing): Lazy Loading submodules to lower memory usage on importing

Karim shoairbbc05ca1645d
diff --git a/scrapling/__init__.py b/scrapling/__init__.py
index 0877054..e8461b5 100644
--- a/scrapling/__init__.py
+++ b/scrapling/__init__.py
@@ -1,12 +1,41 @@
-# Declare top-level shortcuts
-from scrapling.core.custom_types import AttributesHandler, TextHandler
-from scrapling.fetchers import (AsyncFetcher, CustomFetcher, Fetcher,
- PlayWrightFetcher, StealthyFetcher)
-from scrapling.parser import Adaptor, Adaptors
__author__ = "Karim Shoair ([email protected])"
__version__ = "0.2.97"
__copyright__ = "Copyright (c) 2024 Karim Shoair"
+# A lightweight approach to create lazy loader for each import for backward compatibility
+# This will reduces initial memory footprint significantly (only loads what's used)
+def __getattr__(name):
+ if name == 'Fetcher':
+ from scrapling.fetchers import Fetcher as cls
+ return cls
+ elif name == 'Adaptor':
+ from scrapling.parser import Adaptor as cls
+ return cls
+ elif name == 'Adaptors':
+ from scrapling.parser import Adaptors as cls
+ return cls
+ elif name == 'AttributesHandler':
+ from scrapling.core.custom_types import AttributesHandler as cls
+ return cls
+ elif name == 'TextHandler':
+ from scrapling.core.custom_types import TextHandler as cls
+ return cls
+ elif name == 'AsyncFetcher':
+ from scrapling.fetchers import AsyncFetcher as cls
+ return cls
+ elif name == 'StealthyFetcher':
+ from scrapling.fetchers import StealthyFetcher as cls
+ return cls
+ elif name == 'PlayWrightFetcher':
+ from scrapling.fetchers import PlayWrightFetcher as cls
+ return cls
+ elif name == 'CustomFetcher':
+ from scrapling.fetchers import CustomFetcher as cls
+ return cls
+ else:
+ raise AttributeError(f"module 'scrapling' has no attribute '{name}'")
+
+
__all__ = ['Adaptor', 'Fetcher', 'AsyncFetcher', 'StealthyFetcher', 'PlayWrightFetcher']