Scrapy, a fast high-level web crawling & scraping framework for Python.
Enable in-process HTTP tests without a reactor. (#7254)
Andrey Rakhmatullinccfa052fa19f
diff --git a/conftest.py b/conftest.pyindex 26e0434ba..d49901a7c 100644--- a/conftest.py+++ b/conftest.py@@ -93,10 +93,7 @@ def pytest_runtest_setup(item):# Skip tests based on reactor markersreactor = item.config.getoption("--reactor")- if (- item.get_closest_marker("requires_reactor")- or item.get_closest_marker("requires_http_handler")- ) and reactor == "none":+ if item.get_closest_marker("requires_reactor") and reactor == "none":pytest.skip('This test is only run when the --reactor value is not "none"')if item.get_closest_marker("only_asyncio") and reactor not in {"asyncio", "none"}:diff --git a/pyproject.toml b/pyproject.tomlindex 22e3f7b22..7cb83c483 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -230,7 +230,6 @@ markers = ["only_asyncio: marks tests that require the asyncio loop to be used","only_not_asyncio: marks tests that require the asyncio loop to not be used","requires_reactor: marks tests that require a reactor",- "requires_http_handler: marks tests that require a HTTP handler","requires_uvloop: marks tests as only enabled when uvloop is known to be working","requires_botocore: marks tests that need botocore (but not boto3)","requires_boto3: marks tests that need botocore and boto3",diff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex e60ae18db..75f357d50 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -131,8 +131,8 @@ def get_reactor_settings() -> dict[str, Any]:settings["TWISTED_ENABLED"] = Falsesettings["DOWNLOAD_HANDLERS"] = {"ftp": None,- "http": None,- "https": None,+ "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",+ "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",}return settings@@ -208,7 +208,14 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]: # pragma: no coverreturn (client_mock, bucket_mock, blob_mock)-def get_web_client_agent_req(url: str) -> Deferred[TxResponse]:+def get_web_client_agent_req(url: str) -> Deferred[TxResponse]: # pragma: no cover+ warnings.warn(+ "The get_web_client_agent_req() function is deprecated"+ " and will be removed in a future version of Scrapy.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+from twisted.internet import reactoragent = Agent(reactor)diff --git a/tests/spiders.py b/tests/spiders.pyindex de2e64242..79565738d 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -18,7 +18,7 @@ from scrapy.linkextractors import LinkExtractorfrom scrapy.spiders import Spiderfrom scrapy.spiders.crawl import CrawlSpider, Rulefrom scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future-from scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_req+from scrapy.utils.test import get_from_asyncio_queueclass MockServerSpider(Spider):@@ -199,28 +199,24 @@ class AsyncDefDeferredDirectSpider(SimpleSpider):name = "asyncdef_deferred_direct"async def parse(self, response):- resp = await get_web_client_agent_req(self.mockserver.url("/status?n=200"))- yield {"code": resp.code}+ await defer.succeed(None)+ yield {"code": 200}class AsyncDefDeferredWrappedSpider(SimpleSpider):name = "asyncdef_deferred_wrapped"async def parse(self, response):- resp = await deferred_to_future(- get_web_client_agent_req(self.mockserver.url("/status?n=200"))- )- yield {"code": resp.code}+ await deferred_to_future(defer.succeed(None))+ yield {"code": 200}class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):name = "asyncdef_deferred_wrapped"async def parse(self, response):- resp = await maybe_deferred_to_future(- get_web_client_agent_req(self.mockserver.url("/status?n=200"))- )- yield {"code": resp.code}+ await maybe_deferred_to_future(defer.succeed(None))+ yield {"code": 200}class AsyncDefAsyncioGenSpider(SimpleSpider):diff --git a/tests/test_closespider.py b/tests/test_closespider.pyindex a6fd6dd19..9a7447e16 100644--- a/tests/test_closespider.py+++ b/tests/test_closespider.py@@ -1,5 +1,3 @@-import pytest-from scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServerfrom tests.spiders import (@@ -12,7 +10,6 @@ from tests.spiders import (from tests.utils.decorators import inline_callbacks_test[email protected]_http_handlerclass TestCloseSpider:@classmethoddef setup_class(cls):diff --git a/tests/test_contracts.py b/tests/test_contracts.pyindex f1678f912..a35ef7010 100644--- a/tests/test_contracts.py+++ b/tests/test_contracts.py@@ -501,7 +501,6 @@ class TestContractsManager:assert not self.results.failuresassert self.results.errors- @pytest.mark.requires_http_handler@inline_callbacks_testdef test_same_url(self):class TestSameUrlSpider(Spider):diff --git a/tests/test_core_scraper.py b/tests/test_core_scraper.pyindex f4ef5ec5d..1b8ba08ab 100644--- a/tests/test_core_scraper.py+++ b/tests/test_core_scraper.py@@ -2,17 +2,16 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING-import pytest-from scrapy.utils.test import get_crawlerfrom tests.spiders import SimpleSpiderfrom tests.utils.decorators import coroutine_testif TYPE_CHECKING:+ import pytest+from tests.mockserver.http import MockServer[email protected]_http_handler@coroutine_testasync def test_scraper_exception(mockserver: MockServer,diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex b206cfffa..7e9e029a1 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -14,11 +14,11 @@ from twisted.internet.ssl import Certificatefrom twisted.python.failure import Failurefrom scrapy import Spider, signals-from scrapy.crawler import CrawlerRunner+from scrapy.crawler import AsyncCrawlerRunner, CrawlerRunnerfrom scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownloadfrom scrapy.http import Requestfrom scrapy.http.response import Response-from scrapy.utils.defer import maybe_deferred_to_future+from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_futurefrom scrapy.utils.engine import format_engine_status, get_engine_statusfrom scrapy.utils.python import to_unicodefrom scrapy.utils.test import get_crawler, get_reactor_settings@@ -61,7 +61,6 @@ if TYPE_CHECKING:from scrapy.statscollectors import StatsCollector[email protected]_http_handler # easier than marking many individual testsclass TestCrawl:mockserver: MockServer@@ -402,9 +401,15 @@ with multiples lines)assert "Got response 200" in str(log)- @inline_callbacks_test- def test_crawl_multiple(self, caplog: pytest.LogCaptureFixture):- runner = CrawlerRunner(get_reactor_settings())+ @coroutine_test+ async def test_crawl_multiple(self, caplog: pytest.LogCaptureFixture) -> None:+ settings_dict = get_reactor_settings()+ runner_cls = (+ CrawlerRunner+ if settings_dict.get("TWISTED_ENABLED", True)+ else AsyncCrawlerRunner+ )+ runner = runner_cls(settings_dict)runner.crawl(SimpleSpider,self.mockserver.url("/status?n=200"),@@ -417,7 +422,7 @@ with multiples lines)with caplog.at_level(logging.DEBUG):- yield runner.join()+ await ensure_awaitable(runner.join())self._assert_retried(caplog.text)assert "Got response 200" in caplog.text@@ -429,7 +434,6 @@ with multiples linesassert "NotSupported: Unsupported URL scheme 'foo'" in caplog.text[email protected]_http_handlerclass TestCrawlSpider:mockserver: MockServer@@ -636,6 +640,11 @@ class TestCrawlSpider:yield crawler.crawl(seed=url, mockserver=self.mockserver)assert crawler.spider.meta["responses"][0].certificate is None+ @pytest.mark.xfail(+ 'config.getoption("--reactor") == "none"',+ reason="Not implemented in HttpxDownloadHandler",+ strict=True,+ )@pytest.mark.parametrize("url",[@@ -643,6 +652,7 @@ class TestCrawlSpider:pytest.param("/status?n=200",marks=pytest.mark.xfail(+ 'config.getoption("--reactor") != "none"',reason="With HTTP11DownloadHandler, responses with no body are returned early and contain no certificate",strict=True,),@@ -669,6 +679,7 @@ class TestCrawlSpider:pytest.param("/status?n=200",marks=pytest.mark.xfail(+ 'config.getoption("--reactor") != "none"',reason="With HTTP11DownloadHandler, responses with no body are returned early and contain no ip_address",strict=True,),diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 4589e667f..a8e635707 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -93,10 +93,11 @@ class TestLoad:crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers})dh = DownloadHandlers(crawler)assert "scheme" not in dh._schemes- for scheme in handlers: # force load handlers- dh._get_handler(scheme)+ assert dh._get_handler("scheme") is Noneassert "scheme" not in dh._handlersassert "scheme" in dh._notconfigured+ # get the handler again to cover the code that gets it from dh._notconfigured+ assert dh._get_handler("scheme") is Nonedef test_lazy_handlers(self):handlers = {"scheme": DummyLazyDH}@@ -108,8 +109,8 @@ class TestLoad:dh = DownloadHandlers(crawler)assert "scheme" in dh._schemesassert "scheme" not in dh._handlers- for scheme in handlers: # force load lazy handler- dh._get_handler(scheme)+ handler = dh._get_handler("scheme") # force load lazy handler+ assert handlerassert "scheme" in dh._handlersassert "scheme" not in dh._notconfigureddiff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.pyindex 5ddb1cdfa..5c31f68bc 100644--- a/tests/test_downloaderslotssettings.py+++ b/tests/test_downloaderslotssettings.py@@ -67,7 +67,6 @@ class TestCrawl:def setup_method(self):self.runner = CrawlerRunner()- @pytest.mark.requires_http_handler@inline_callbacks_testdef test_delay(self):crawler = get_crawler(DownloaderSlotsSettingsTestSpider)@@ -129,7 +128,6 @@ def test_get_slot_deprecated_spider_arg():assert slot1 == slot2[email protected]_http_handler@pytest.mark.parametrize("priority_queue_class",[diff --git a/tests/test_engine.py b/tests/test_engine.pyindex c77e3c5c9..4573c5001 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -373,7 +373,6 @@ class TestEngineBase:class TestEngine(TestEngineBase):- @pytest.mark.requires_http_handler@coroutine_testasync def test_crawler(self, mockserver: MockServer) -> None:for spider in (@@ -391,7 +390,6 @@ class TestEngine(TestEngineBase):self._assert_signals_caught(run)self._assert_bytes_received(run)- @pytest.mark.requires_http_handler@coroutine_testasync def test_crawler_dupefilter(self, mockserver: MockServer) -> None:run = CrawlerRun(DupeFilterSpider)@@ -399,14 +397,12 @@ class TestEngine(TestEngineBase):self._assert_scheduled_requests(run, count=8)self._assert_dropped_requests(run)- @pytest.mark.requires_http_handler@coroutine_testasync def test_crawler_itemerror(self, mockserver: MockServer) -> None:run = CrawlerRun(ItemZeroDivisionErrorSpider)await run.run(mockserver)self._assert_items_error(run)- @pytest.mark.requires_http_handler@coroutine_testasync def test_crawler_change_close_reason_on_idle(self, mockserver: MockServerdiff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.pyindex 9bdbfcc56..115deb821 100644--- a/tests/test_engine_loop.py+++ b/tests/test_engine_loop.py@@ -332,7 +332,6 @@ class TestRequestSendOrder:# Examples from the “Start requests” section of the documentation about# spiders.- @pytest.mark.requires_http_handler@coroutine_testasync def test_lazy(self):start_nums = [1, 2, 4]diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.pyindex 970fffbc9..091485781 100644--- a/tests/test_engine_stop_download_bytes.py+++ b/tests/test_engine_stop_download_bytes.py@@ -2,8 +2,6 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING-import pytest-from scrapy.exceptions import StopDownloadfrom tests.test_engine import (AttrsItemsSpider,@@ -16,6 +14,8 @@ from tests.test_engine import (from tests.utils.decorators import coroutine_testif TYPE_CHECKING:+ import pytest+from tests.mockserver.http import MockServer@@ -26,7 +26,6 @@ class BytesReceivedCrawlerRun(CrawlerRun):class TestBytesReceivedEngine(TestEngineBase):- @pytest.mark.requires_http_handler@coroutine_testasync def test_crawler(self, mockserver: MockServer, caplog: pytest.LogCaptureFixturediff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.pyindex e0008a53a..8ebc948e2 100644--- a/tests/test_engine_stop_download_headers.py+++ b/tests/test_engine_stop_download_headers.py@@ -2,8 +2,6 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING-import pytest-from scrapy.exceptions import StopDownloadfrom tests.test_engine import (AttrsItemsSpider,@@ -16,6 +14,8 @@ from tests.test_engine import (from tests.utils.decorators import coroutine_testif TYPE_CHECKING:+ import pytest+from tests.mockserver.http import MockServer@@ -26,7 +26,6 @@ class HeadersReceivedCrawlerRun(CrawlerRun):class TestHeadersReceivedEngine(TestEngineBase):- @pytest.mark.requires_http_handler@coroutine_testasync def test_crawler(self, mockserver: MockServer, caplog: pytest.LogCaptureFixturediff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 3a9c37042..1ca5f21f0 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -849,7 +849,6 @@ class ExceptionJsonItemExporter(JsonItemExporter):raise RuntimeError("foo")[email protected]_http_handlerclass TestFeedExport(TestFeedExportBase):async def run_and_export(self, spider_cls: type[Spider], settings: dict[str, Any]@@ -1703,6 +1702,7 @@ class TestFeedExport(TestFeedExportBase):data = await self.exported_no_data(settings)assert data["csv"] == b""+ @pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage@coroutine_testasync def test_multiple_feeds_success_logs_blocking_feed_storage(self):settings = {@@ -1834,7 +1834,6 @@ class TestFeedExport(TestFeedExportBase):assert not Storage.file_was_closed[email protected]_http_handlerclass TestFeedPostProcessedExports(TestFeedExportBase):items = [{"foo": "bar"}]expected = b"foo\r\nbar\r\n"@@ -2353,7 +2352,6 @@ class TestFeedPostProcessedExports(TestFeedExportBase):assert result == expected[email protected]_http_handlerclass TestBatchDeliveries(TestFeedExportBase):_file_mark = "_%(batch_time)s_#%(batch_id)02d_"diff --git a/tests/test_logformatter.py b/tests/test_logformatter.pyindex 66544a508..9806315b4 100644--- a/tests/test_logformatter.py+++ b/tests/test_logformatter.py@@ -253,7 +253,6 @@ class DropSomeItemsPipeline:self.drop = True[email protected]_http_handlerclass TestShowOrSkipMessages:@classmethoddef setup_class(cls):diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.pyindex acc07f48b..0376d27c4 100644--- a/tests/test_pipeline_crawl.py+++ b/tests/test_pipeline_crawl.py@@ -57,7 +57,6 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider):)[email protected]_http_handlerclass TestFileDownloadCrawl:pipeline_class = "scrapy.pipelines.files.FilesPipeline"store_setting_key = "FILES_STORE"diff --git a/tests/test_pipelines.py b/tests/test_pipelines.pyindex fb1fccfc7..753fcd42e 100644--- a/tests/test_pipelines.py+++ b/tests/test_pipelines.py@@ -127,7 +127,6 @@ class ItemSpider(Spider):return {"field": 42}[email protected]_http_handlerclass TestPipeline:def _on_item_scraped(self, item):assert isinstance(item, dict)@@ -266,7 +265,6 @@ class TestCustomPipelineManager:):itemproc.process_item({}, crawler.spider)- @pytest.mark.requires_http_handler@coroutine_testasync def test_integration_recommended(self, mockserver: MockServer) -> None:class CustomPipelineManager(ItemPipelineManager):@@ -293,7 +291,6 @@ class TestCustomPipelineManager:assert len(items) == 1- @pytest.mark.requires_http_handler@coroutine_testasync def test_integration_no_async_subclass(self, mockserver: MockServer) -> None:class CustomPipelineManager(ItemPipelineManager):@@ -352,7 +349,6 @@ class TestCustomPipelineManager:assert len(items) == 1- @pytest.mark.requires_http_handler@coroutine_testasync def test_integration_no_async_not_subclass(self, mockserver: MockServerdiff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.pyindex c78c882cf..912075401 100644--- a/tests/test_proxy_connect.py+++ b/tests/test_proxy_connect.py@@ -61,7 +61,6 @@ def _wrong_credentials(proxy_url):return urlunsplit(bad_auth_proxy)[email protected]_http_handler@pytest.mark.requires_mitmproxyclass TestProxyConnect:@classmethoddiff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.pyindex 66dd48737..38d56e9bd 100644--- a/tests/test_request_attribute_binding.py+++ b/tests/test_request_attribute_binding.py@@ -1,4 +1,3 @@-import pytestfrom testfixtures import LogCapturefrom scrapy import Request, signals@@ -63,7 +62,6 @@ class AlternativeCallbacksMiddleware:return response.replace(request=new_request)[email protected]_http_handlerclass TestCrawl:@classmethoddef setup_class(cls):diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.pyindex c8629df00..8d3977452 100644--- a/tests/test_request_cb_kwargs.py+++ b/tests/test_request_cb_kwargs.py@@ -1,4 +1,3 @@-import pytestfrom testfixtures import LogCapturefrom scrapy.http import Request@@ -149,7 +148,6 @@ class KeywordArgumentsSpider(MockServerSpider):self.crawler.stats.inc_value("boolean_checks", 1)[email protected]_http_handlerclass TestCallbackKeywordArguments:@classmethoddef setup_class(cls):diff --git a/tests/test_scheduler.py b/tests/test_scheduler.pyindex 873be0b8f..0d2d8be93 100644--- a/tests/test_scheduler.py+++ b/tests/test_scheduler.py@@ -369,7 +369,6 @@ class TestIntegrationWithDownloaderAwareInMemory:},)- @pytest.mark.requires_http_handler@inline_callbacks_testdef test_integration_downloader_aware_priority_queue(self):with MockServer() as mockserver:diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.pyindex 6234bef86..176fefbbf 100644--- a/tests/test_scheduler_base.py+++ b/tests/test_scheduler_base.py@@ -144,7 +144,6 @@ class TestSimpleScheduler(InterfaceCheckMixin):assert close_result == "close"[email protected]_http_handlerclass TestMinimalSchedulerCrawl:scheduler_cls = MinimalSchedulerdiff --git a/tests/test_signals.py b/tests/test_signals.pyindex faf24a129..f5f13b992 100644--- a/tests/test_signals.py+++ b/tests/test_signals.py@@ -50,7 +50,6 @@ class TestMockServer:item = await get_from_asyncio_queue(item)self.items.append(item)- @pytest.mark.requires_http_handler@pytest.mark.only_asyncio@inline_callbacks_testdef test_simple_pipeline(self):diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.pyindex 8dc6ff300..b15f68dd0 100644--- a/tests/test_spidermiddleware_httperror.py+++ b/tests/test_spidermiddleware_httperror.py@@ -191,7 +191,6 @@ class TestHttpErrorMiddlewareHandleAll:mw.process_spider_input(res402)[email protected]_http_handlerclass TestHttpErrorMiddlewareIntegrational:@classmethoddef setup_class(cls):diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.pyindex 80f5828c9..8cf08dd94 100644--- a/tests/test_spidermiddleware_output_chain.py+++ b/tests/test_spidermiddleware_output_chain.py@@ -1,4 +1,3 @@-import pytestfrom testfixtures import LogCapturefrom scrapy import Request, Spider@@ -319,7 +318,6 @@ class NotGeneratorOutputChainSpider(Spider):# ================================================================================[email protected]_http_handlerclass TestSpiderMiddleware:mockserver: MockServerdiff --git a/tox.ini b/tox.iniindex 213ecbef6..7656cb017 100644--- a/tox.ini+++ b/tox.ini@@ -168,6 +168,7 @@ commands =[testenv:no-reactor]deps ={[testenv]deps}+ httpxpytest-asynciocommands ={[testenv]commands} -p no:twisted --reactor=none@@ -183,6 +184,7 @@ setenv =basepython = {[pinned]basepython}deps ={[testenv:pinned]deps}+ httpx==0.26.0pytest-asynciocommands = {[pinned]commands} -p no:twisted --reactor=nonesetenv =