Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Lazy creation of Downloader._slot_gc_loop. (#7210)

Andrey Rakhmatullin9bae1ee21f51
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 85fa8f3c2..c862ecdf9 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -108,6 +108,7 @@ def _get_concurrency_delay(
class Downloader:
DOWNLOAD_SLOT = "download_slot"
+ _SLOT_GC_INTERVAL: float = 60.0 # seconds
def __init__(self, crawler: Crawler):
self.crawler: Crawler = crawler
@@ -125,10 +126,7 @@ class Downloader:
self.middleware: DownloaderMiddlewareManager = (
DownloaderMiddlewareManager.from_crawler(crawler)
)
- self._slot_gc_loop: AsyncioLoopingCall | LoopingCall = create_looping_call(
- self._slot_gc
- )
- self._slot_gc_loop.start(60)
+ self._slot_gc_loop: AsyncioLoopingCall | LoopingCall | None = None
self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict(
"DOWNLOAD_SLOTS"
)
@@ -172,6 +170,7 @@ class Downloader:
randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)
new_slot = Slot(conc, delay, randomize_delay)
self.slots[key] = new_slot
+ self._start_slot_gc()
return key, self.slots[key]
@@ -273,7 +272,7 @@ class Downloader:
queue_dfd.callback(response) # awaited in _enqueue_request()
def close(self) -> None:
- self._slot_gc_loop.stop()
+ self._stop_slot_gc()
for slot in self.slots.values():
slot.close()
@@ -282,3 +281,14 @@ class Downloader:
for key, slot in list(self.slots.items()):
if not slot.active and slot.lastseen + slot.delay < mintime:
self.slots.pop(key).close()
+
+ def _start_slot_gc(self) -> None:
+ if self._slot_gc_loop:
+ return
+ self._slot_gc_loop = create_looping_call(self._slot_gc)
+ self._slot_gc_loop.start(self._SLOT_GC_INTERVAL, now=False)
+
+ def _stop_slot_gc(self) -> None:
+ if self._slot_gc_loop:
+ self._slot_gc_loop.stop()
+ self._slot_gc_loop = None
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
index 22127a4da..dffe95406 100644
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@@ -99,9 +99,9 @@ def test_params():
crawler = get_crawler(DefaultSpider, settings_dict=settings)
crawler.spider = crawler._create_spider()
downloader = Downloader(crawler)
- downloader._slot_gc_loop.stop() # Prevent an unclean reactor.
request = Request("https://example.com")
_, actual = downloader._get_slot(request)
+ downloader.close()
expected = Slot(**params)
for param in params:
assert getattr(expected, param) == getattr(actual, param), (
@@ -113,7 +113,6 @@ def test_get_slot_deprecated_spider_arg():
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
downloader = Downloader(crawler)
- downloader._slot_gc_loop.stop() # Prevent an unclean reactor.
request = Request("https://example.com")
with pytest.warns(
@@ -122,6 +121,7 @@ def test_get_slot_deprecated_spider_arg():
):
key1, slot1 = downloader._get_slot(request, spider=crawler.spider)
key2, slot2 = downloader._get_slot(request)
+ downloader.close()
assert key1 == key2
assert slot1 == slot2
diff --git a/tests/test_engine.py b/tests/test_engine.py
index dc98364fc..d3485e344 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -614,7 +614,6 @@ def test_request_scheduled_signal(caplog):
crawler = get_crawler(MySpider)
engine = ExecutionEngine(crawler, lambda _: None)
- engine.downloader._slot_gc_loop.stop()
scheduler = TestScheduler()
async def start():