Scrapy, a fast high-level web crawling & scraping framework for Python.
Lazy creation of Downloader._slot_gc_loop. (#7210)
Andrey Rakhmatullin9bae1ee21f51
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 85fa8f3c2..c862ecdf9 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -108,6 +108,7 @@ def _get_concurrency_delay(class Downloader:DOWNLOAD_SLOT = "download_slot"+ _SLOT_GC_INTERVAL: float = 60.0 # secondsdef __init__(self, crawler: Crawler):self.crawler: Crawler = crawler@@ -125,10 +126,7 @@ class Downloader:self.middleware: DownloaderMiddlewareManager = (DownloaderMiddlewareManager.from_crawler(crawler))- self._slot_gc_loop: AsyncioLoopingCall | LoopingCall = create_looping_call(- self._slot_gc- )- self._slot_gc_loop.start(60)+ self._slot_gc_loop: AsyncioLoopingCall | LoopingCall | None = Noneself.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict("DOWNLOAD_SLOTS")@@ -172,6 +170,7 @@ class Downloader:randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)new_slot = Slot(conc, delay, randomize_delay)self.slots[key] = new_slot+ self._start_slot_gc()return key, self.slots[key]@@ -273,7 +272,7 @@ class Downloader:queue_dfd.callback(response) # awaited in _enqueue_request()def close(self) -> None:- self._slot_gc_loop.stop()+ self._stop_slot_gc()for slot in self.slots.values():slot.close()@@ -282,3 +281,14 @@ class Downloader:for key, slot in list(self.slots.items()):if not slot.active and slot.lastseen + slot.delay < mintime:self.slots.pop(key).close()++ def _start_slot_gc(self) -> None:+ if self._slot_gc_loop:+ return+ self._slot_gc_loop = create_looping_call(self._slot_gc)+ self._slot_gc_loop.start(self._SLOT_GC_INTERVAL, now=False)++ def _stop_slot_gc(self) -> None:+ if self._slot_gc_loop:+ self._slot_gc_loop.stop()+ self._slot_gc_loop = Nonediff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.pyindex 22127a4da..dffe95406 100644--- a/tests/test_downloaderslotssettings.py+++ b/tests/test_downloaderslotssettings.py@@ -99,9 +99,9 @@ def test_params():crawler = get_crawler(DefaultSpider, settings_dict=settings)crawler.spider = crawler._create_spider()downloader = Downloader(crawler)- downloader._slot_gc_loop.stop() # Prevent an unclean reactor.request = Request("https://example.com")_, actual = downloader._get_slot(request)+ downloader.close()expected = Slot(**params)for param in params:assert getattr(expected, param) == getattr(actual, param), (@@ -113,7 +113,6 @@ def test_get_slot_deprecated_spider_arg():crawler = get_crawler(DefaultSpider)crawler.spider = crawler._create_spider()downloader = Downloader(crawler)- downloader._slot_gc_loop.stop() # Prevent an unclean reactor.request = Request("https://example.com")with pytest.warns(@@ -122,6 +121,7 @@ def test_get_slot_deprecated_spider_arg():):key1, slot1 = downloader._get_slot(request, spider=crawler.spider)key2, slot2 = downloader._get_slot(request)+ downloader.close()assert key1 == key2assert slot1 == slot2diff --git a/tests/test_engine.py b/tests/test_engine.pyindex dc98364fc..d3485e344 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -614,7 +614,6 @@ def test_request_scheduled_signal(caplog):crawler = get_crawler(MySpider)engine = ExecutionEngine(crawler, lambda _: None)- engine.downloader._slot_gc_loop.stop()scheduler = TestScheduler()async def start():