🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(defaults): Lazy Loading fetchers to lower memory usage on importing
Karim shoair9b0515b1cd2e
diff --git a/scrapling/defaults.py b/scrapling/defaults.pyindex 20f0c44..4098f76 100644--- a/scrapling/defaults.py+++ b/scrapling/defaults.py@@ -1,10 +1,19 @@-from .fetchers import AsyncFetcher as _AsyncFetcher-from .fetchers import Fetcher as _Fetcher-from .fetchers import PlayWrightFetcher as _PlayWrightFetcher-from .fetchers import StealthyFetcher as _StealthyFetcher-# If you are going to use Fetchers with the default settings, import them from this file instead for a cleaner looking code-Fetcher = _Fetcher()-AsyncFetcher = _AsyncFetcher()-StealthyFetcher = _StealthyFetcher()-PlayWrightFetcher = _PlayWrightFetcher()++# A lightweight approach to create lazy loader for each import for backward compatibility+# This will reduces initial memory footprint significantly (only loads what's used)+def __getattr__(name):+ if name == 'Fetcher':+ from scrapling.fetchers import Fetcher as cls+ return cls()+ elif name == 'AsyncFetcher':+ from scrapling.fetchers import AsyncFetcher as cls+ return cls()+ elif name == 'StealthyFetcher':+ from scrapling.fetchers import StealthyFetcher as cls+ return cls()+ elif name == 'PlayWrightFetcher':+ from scrapling.fetchers import PlayWrightFetcher as cls+ return cls()+ else:+ raise AttributeError(f"module 'scrapling' has no attribute '{name}'")