🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor: cleaner code for the fetcher's lazy loader
Karim shoair3d07a1533e0a
diff --git a/scrapling/__init__.py b/scrapling/__init__.pyindex 40fd294..323be33 100644--- a/scrapling/__init__.py+++ b/scrapling/__init__.py@@ -3,45 +3,24 @@ __version__ = "0.3-beta"__copyright__ = "Copyright (c) 2024 Karim Shoair"-# A lightweight approach to create lazy loader for each import for backward compatibility+# A lightweight approach to create a lazy loader for each import for backward compatibility# This will reduces initial memory footprint significantly (only loads what's used)def __getattr__(name):- if name == "Fetcher":- from scrapling.fetchers import Fetcher as cls-- return cls- elif name == "Selector":- from scrapling.parser import Selector as cls-- return cls- elif name == "Selectors":- from scrapling.parser import Selectors as cls-- return cls- elif name == "AttributesHandler":- from scrapling.core.custom_types import AttributesHandler as cls-- return cls- elif name == "TextHandler":- from scrapling.core.custom_types import TextHandler as cls-- return cls- elif name == "AsyncFetcher":- from scrapling.fetchers import AsyncFetcher as cls-- return cls- elif name == "StealthyFetcher":- from scrapling.fetchers import StealthyFetcher as cls-- return cls- elif name == "DynamicFetcher":- from scrapling.fetchers import DynamicFetcher as cls-- return cls- elif name == "CustomFetcher":- from scrapling.fetchers import CustomFetcher as cls-- return cls+ lazy_imports = {+ "Fetcher": ("scrapling.fetchers", "Fetcher"),+ "Selector": ("scrapling.parser", "Selector"),+ "Selectors": ("scrapling.parser", "Selectors"),+ "AttributesHandler": ("scrapling.core.custom_types", "AttributesHandler"),+ "TextHandler": ("scrapling.core.custom_types", "TextHandler"),+ "AsyncFetcher": ("scrapling.fetchers", "AsyncFetcher"),+ "StealthyFetcher": ("scrapling.fetchers", "StealthyFetcher"),+ "DynamicFetcher": ("scrapling.fetchers", "DynamicFetcher"),+ }++ if name in lazy_imports:+ module_path, class_name = lazy_imports[name]+ module = __import__(module_path, fromlist=[class_name])+ return getattr(module, class_name)else:raise AttributeError(f"module 'scrapling' has no attribute '{name}'")