Scrapy, a fast high-level web crawling & scraping framework for Python.
Allow Crawler.crawl_async() without asyncio. (#7202)
Andrey Rakhmatullina2463325db2f
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rstindex 9c324a031..c82e6c2f3 100644--- a/docs/topics/coroutines.rst+++ b/docs/topics/coroutines.rst@@ -81,8 +81,8 @@ These APIs have a coroutine-based implementation and a Deferred-based one:- :meth:`~scrapy.crawler.Crawler.crawl_async` (coroutine-based) and:meth:`~scrapy.crawler.Crawler.crawl` (Deferred-based): the former- doesn't support non-default reactors and so the latter should be used- with those.+ may be inconvenient to use in Deferred-based code so both are available,+ this may change in a future Scrapy version.- :class:`scrapy.crawler.AsyncCrawlerRunner` and its subclass:class:`scrapy.crawler.AsyncCrawlerProcess` (coroutine-based) anddiff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex ffbebe152..e21207222 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -19,7 +19,6 @@ from scrapy.extension import ExtensionManagerfrom scrapy.settings import Settings, overridden_settingsfrom scrapy.signalmanager import SignalManagerfrom scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader-from scrapy.utils.asyncio import is_asyncio_availablefrom scrapy.utils.defer import deferred_from_corofrom scrapy.utils.log import (configure_logging,@@ -169,13 +168,7 @@ class Crawler:.. versionadded:: VERSIONComplete when the crawl is finished.-- This function requires- :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be- installed."""- if not is_asyncio_available():- raise RuntimeError("Crawler.crawl_async() requires AsyncioSelectorReactor.")if self.crawling:raise RuntimeError("Crawling already taking place")if self._started:diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.pyindex a84bb9d8d..a35bfd590 100644--- a/tests/test_core_downloader.py+++ b/tests/test_core_downloader.py@@ -187,4 +187,4 @@ async def test_fetch_deprecated_spider_arg():ScrapyDeprecationWarning,match=r"The fetch\(\) method of .+\.CustomDownloader requires a spider argument",):- await maybe_deferred_to_future(crawler.crawl())+ await crawler.crawl_async()diff --git a/tests/test_core_scraper.py b/tests/test_core_scraper.pyindex c819e246e..47dee1271 100644--- a/tests/test_core_scraper.py+++ b/tests/test_core_scraper.py@@ -2,7 +2,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future+from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.test import get_crawlerfrom tests.spiders import SimpleSpider@@ -23,5 +23,5 @@ async def test_scraper_exception("scrapy.core.engine.Scraper.handle_spider_output_async",lambda *args, **kwargs: 1 / 0,)- await maybe_deferred_to_future(crawler.crawl(url=mockserver.url("/")))+ await crawler.crawl_async(url=mockserver.url("/"))assert "Scraper bug processing" in caplog.textdiff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex b0060a76a..699ac3911 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -98,7 +98,7 @@ class TestCrawl:settings = {"DOWNLOAD_DELAY": delay, "RANDOMIZE_DOWNLOAD_DELAY": randomize}crawler = get_crawler(FollowAllSpider, settings)- await maybe_deferred_to_future(crawler.crawl(**crawl_kwargs))+ await crawler.crawl_async(**crawl_kwargs)assert crawler.spiderassert isinstance(crawler.spider, FollowAllSpider)times = crawler.spider.times@@ -112,7 +112,7 @@ class TestCrawl:# code above to have any meaning.settings["DOWNLOAD_DELAY"] = 0crawler = get_crawler(FollowAllSpider, settings)- await maybe_deferred_to_future(crawler.crawl(**crawl_kwargs))+ await crawler.crawl_async(**crawl_kwargs)assert crawler.spiderassert isinstance(crawler.spider, FollowAllSpider)times = crawler.spider.timesdiff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 4f1045786..5204c9a73 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -111,7 +111,6 @@ class TestCrawler(TestBaseCrawler):with pytest.raises(RuntimeError, match="more than once on the same instance"):yield crawler.crawl()- @pytest.mark.only_asyncio@deferred_f_from_coro_fasync def test_crawler_crawl_async_twice_seq_unsupported(self):crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)@@ -552,7 +551,7 @@ class TestCrawlerLogging:assert get_scrapy_root_handler().level == logging.DEBUGcrawler = get_crawler(MySpider)assert get_scrapy_root_handler().level == logging.INFO- await maybe_deferred_to_future(crawler.crawl())+ await crawler.crawl_async()finally:_uninstall_scrapy_root_handler()diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.pyindex 5d88b59c1..22127a4da 100644--- a/tests/test_downloaderslotssettings.py+++ b/tests/test_downloaderslotssettings.py@@ -8,7 +8,7 @@ from scrapy import Requestfrom scrapy.core.downloader import Downloader, Slotfrom scrapy.crawler import CrawlerRunnerfrom scrapy.exceptions import ScrapyDeprecationWarning-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future+from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServer@@ -143,7 +143,7 @@ async def test_none_slot_with_priority_queue(DownloaderSlotsSettingsTestSpider,settings_dict={"SCHEDULER_PRIORITY_QUEUE": priority_queue_class},)- await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))+ await crawler.crawl_async(mockserver=mockserver)assert isinstance(crawler.spider, DownloaderSlotsSettingsTestSpider)assert hasattr(crawler.spider, "times")diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.pyindex ddf1f1fe0..c219a0f6f 100644--- a/tests/test_engine_loop.py+++ b/tests/test_engine_loop.py@@ -81,7 +81,7 @@ class TestMain:settings = {"SCHEDULER": MemoryScheduler}crawler = get_crawler(TestSpider, settings_dict=settings)crawler.signals.connect(track_url, signals.request_reached_downloader)- await maybe_deferred_to_future(crawler.crawl())+ await crawler.crawl_async()assert crawler.stats.get_value("finish_reason") == "finished"expected_urls = ["data:,a", "data:,b", "data:,c", "data:,d"]assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}"@@ -112,7 +112,7 @@ class TestMain:caplog.clear()with caplog.at_level(ERROR):- await maybe_deferred_to_future(crawler.crawl())+ await crawler.crawl_async()assert not caplog.recordsassert crawler.stats@@ -182,7 +182,7 @@ class TestRequestSendOrder:crawler = get_crawler(TestSpider, settings_dict=settings)crawler.signals.connect(track_num, signals.request_reached_downloader)- await maybe_deferred_to_future(crawler.crawl())+ await crawler.crawl_async()assert crawler.stats.get_value("finish_reason") == "finished"expected_nums = sorted(start_nums + cb_nums)assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}"diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex a423f5328..a9d608e52 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -862,7 +862,7 @@ class TestFeedExport(TestFeedExportBase):try:spider_cls.start_urls = [self.mockserver.url("/")]crawler = get_crawler(spider_cls, settings)- await maybe_deferred_to_future(crawler.crawl())+ await crawler.crawl_async()for file_path, feed_options in FEEDS.items():content[feed_options["format"]] = (@@ -1866,7 +1866,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):try:spider_cls.start_urls = [self.mockserver.url("/")]crawler = get_crawler(spider_cls, settings)- await maybe_deferred_to_future(crawler.crawl())+ await crawler.crawl_async()for file_path in FEEDS:content[str(file_path)] = (@@ -2363,7 +2363,7 @@ class TestBatchDeliveries(TestFeedExportBase):content: defaultdict[str, list[bytes]] = defaultdict(list)spider_cls.start_urls = [self.mockserver.url("/")]crawler = get_crawler(spider_cls, settings)- await maybe_deferred_to_future(crawler.crawl())+ await crawler.crawl_async()for path, feed in FEEDS.items():dir_name = Path(path).parentdiff --git a/tests/test_pipelines.py b/tests/test_pipelines.pyindex fc61d61ec..f315388cd 100644--- a/tests/test_pipelines.py+++ b/tests/test_pipelines.py@@ -159,7 +159,7 @@ class TestPipeline:@deferred_f_from_coro_fasync def test_pipeline(self, mockserver: MockServer, pipeline_class: type) -> None:crawler = self._create_crawler(pipeline_class)- await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))+ await crawler.crawl_async(mockserver=mockserver)assert len(self.items) == 1@deferred_f_from_coro_f@@ -179,7 +179,7 @@ class TestPipeline:match="DeferredPipeline.process_item returned a Deferred",),):- await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))+ await crawler.crawl_async(mockserver=mockserver)assert len(self.items) == 1@deferred_f_from_coro_f@@ -199,7 +199,7 @@ class TestPipeline:match=r"DeprecatedSpiderArgPipeline.process_item\(\) requires a spider argument",),):- await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))+ await crawler.crawl_async(mockserver=mockserver)assert len(self.items) == 1@@ -224,7 +224,7 @@ class TestPipeline:pipeline_class: type,) -> None:crawler = self._create_crawler(pipeline_class)- await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))+ await crawler.crawl_async(mockserver=mockserver)assert "Error processing {'field': 42}" in caplog.textassert "process_item error" in caplog.text@@ -247,7 +247,7 @@ class TestPipeline:) -> None:crawler = self._create_crawler(pipeline_class)with pytest.raises(ValueError, match="open_spider error"):- await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))+ await crawler.crawl_async(mockserver=mockserver)class TestCustomPipelineManager:@@ -287,7 +287,7 @@ class TestCustomPipelineManager:)crawler.spider = crawler._create_spider()crawler.signals.connect(_on_item_scraped, signals.item_scraped)- await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))+ await crawler.crawl_async(mockserver=mockserver)assert len(items) == 1@@ -345,7 +345,7 @@ class TestCustomPipelineManager:match=r"CustomPipelineManager overrides process_item\(\) but doesn't override process_item_async\(\)",),):- await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))+ await crawler.crawl_async(mockserver=mockserver)assert len(items) == 1@@ -407,7 +407,7 @@ class TestCustomPipelineManager:match=r"CustomPipelineManager doesn't define a process_item_async\(\) method",),):- await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))+ await crawler.crawl_async(mockserver=mockserver)assert len(items) == 1diff --git a/tests/test_signals.py b/tests/test_signals.pyindex 2e4f9ffb1..89b7a74dc 100644--- a/tests/test_signals.py+++ b/tests/test_signals.py@@ -2,7 +2,7 @@ import pytestfrom twisted.internet.defer import inlineCallbacksfrom scrapy import Request, Spider, signals-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future+from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.test import get_crawler, get_from_asyncio_queuefrom tests.mockserver.http import MockServer@@ -30,7 +30,7 @@ class TestMain:calls.append(object())crawler.signals.connect(track_call, signals.scheduler_empty)- await maybe_deferred_to_future(crawler.crawl())+ await crawler.crawl_async()assert len(calls) >= 1diff --git a/tests/test_spider.py b/tests/test_spider.pyindex dcc9d1e82..a1649aa6f 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -29,7 +29,7 @@ from scrapy.spiders import (XMLFeedSpider,)from scrapy.spiders.init import InitSpider-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future+from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.test import get_crawler, get_reactor_settingsfrom tests import get_testdata, tests_datadir@@ -155,7 +155,7 @@ class TestInitSpider(TestSpider):responses.append(response)crawler = get_crawler(TestSpider)- await maybe_deferred_to_future(crawler.crawl())+ await crawler.crawl_async()assert len(responses) == 1assert responses[0].url == "data:,"diff --git a/tests/test_spider_start.py b/tests/test_spider_start.pyindex d4eca85b8..e608d8975 100644--- a/tests/test_spider_start.py+++ b/tests/test_spider_start.py@@ -32,7 +32,7 @@ class TestMain:crawler = get_crawler(spider)crawler.signals.connect(track_item, signals.item_scraped)- await maybe_deferred_to_future(crawler.crawl())+ await crawler.crawl_async()assert crawler.statsassert crawler.stats.get_value("finish_reason") == "finished"assert actual_items == expected_itemsdiff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.pyindex 1808e087c..97d9d6470 100644--- a/tests/test_spidermiddleware_output_chain.py+++ b/tests/test_spidermiddleware_output_chain.py@@ -1,7 +1,7 @@from testfixtures import LogCapturefrom scrapy import Request, Spider-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future+from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServer@@ -333,7 +333,7 @@ class TestSpiderMiddleware:async def crawl_log(self, spider: type[Spider]) -> LogCapture:crawler = get_crawler(spider)with LogCapture() as log:- await maybe_deferred_to_future(crawler.crawl(mockserver=self.mockserver))+ await crawler.crawl_async(mockserver=self.mockserver)return log@deferred_f_from_coro_fdiff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.pyindex a525f991d..18868bf8c 100644--- a/tests/test_spidermiddleware_process_start.py+++ b/tests/test_spidermiddleware_process_start.py@@ -117,7 +117,7 @@ class TestMain:}crawler = get_crawler(spider_cls, settings_dict=settings)crawler.signals.connect(track_item, signals.item_scraped)- await maybe_deferred_to_future(crawler.crawl())+ await crawler.crawl_async()assert crawler.stats.get_value("finish_reason") == "finished"assert actual_items == expected_items, f"{actual_items=} != {expected_items=}"