🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(importing): Lazy Loading submodules to lower memory usage on importing
Karim shoairbbc05ca1645d
diff --git a/scrapling/__init__.py b/scrapling/__init__.pyindex 0877054..e8461b5 100644--- a/scrapling/__init__.py+++ b/scrapling/__init__.py@@ -1,12 +1,41 @@-# Declare top-level shortcuts-from scrapling.core.custom_types import AttributesHandler, TextHandler-from scrapling.fetchers import (AsyncFetcher, CustomFetcher, Fetcher,- PlayWrightFetcher, StealthyFetcher)-from scrapling.parser import Adaptor, Adaptors__author__ = "Karim Shoair ([email protected])"__version__ = "0.2.97"__copyright__ = "Copyright (c) 2024 Karim Shoair"+# A lightweight approach to create lazy loader for each import for backward compatibility+# This will reduces initial memory footprint significantly (only loads what's used)+def __getattr__(name):+ if name == 'Fetcher':+ from scrapling.fetchers import Fetcher as cls+ return cls+ elif name == 'Adaptor':+ from scrapling.parser import Adaptor as cls+ return cls+ elif name == 'Adaptors':+ from scrapling.parser import Adaptors as cls+ return cls+ elif name == 'AttributesHandler':+ from scrapling.core.custom_types import AttributesHandler as cls+ return cls+ elif name == 'TextHandler':+ from scrapling.core.custom_types import TextHandler as cls+ return cls+ elif name == 'AsyncFetcher':+ from scrapling.fetchers import AsyncFetcher as cls+ return cls+ elif name == 'StealthyFetcher':+ from scrapling.fetchers import StealthyFetcher as cls+ return cls+ elif name == 'PlayWrightFetcher':+ from scrapling.fetchers import PlayWrightFetcher as cls+ return cls+ elif name == 'CustomFetcher':+ from scrapling.fetchers import CustomFetcher as cls+ return cls+ else:+ raise AttributeError(f"module 'scrapling' has no attribute '{name}'")++__all__ = ['Adaptor', 'Fetcher', 'AsyncFetcher', 'StealthyFetcher', 'PlayWrightFetcher']