Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Enable in-process HTTP tests without a reactor. (#7254)

Andrey Rakhmatullinccfa052fa19f
diff --git a/conftest.py b/conftest.py
index 26e0434ba..d49901a7c 100644
--- a/conftest.py
+++ b/conftest.py
@@ -93,10 +93,7 @@ def pytest_runtest_setup(item):
# Skip tests based on reactor markers
reactor = item.config.getoption("--reactor")
- if (
- item.get_closest_marker("requires_reactor")
- or item.get_closest_marker("requires_http_handler")
- ) and reactor == "none":
+ if item.get_closest_marker("requires_reactor") and reactor == "none":
pytest.skip('This test is only run when the --reactor value is not "none"')
if item.get_closest_marker("only_asyncio") and reactor not in {"asyncio", "none"}:
diff --git a/pyproject.toml b/pyproject.toml
index 22e3f7b22..7cb83c483 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -230,7 +230,6 @@ markers = [
"only_asyncio: marks tests that require the asyncio loop to be used",
"only_not_asyncio: marks tests that require the asyncio loop to not be used",
"requires_reactor: marks tests that require a reactor",
- "requires_http_handler: marks tests that require a HTTP handler",
"requires_uvloop: marks tests as only enabled when uvloop is known to be working",
"requires_botocore: marks tests that need botocore (but not boto3)",
"requires_boto3: marks tests that need botocore and boto3",
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index e60ae18db..75f357d50 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -131,8 +131,8 @@ def get_reactor_settings() -> dict[str, Any]:
settings["TWISTED_ENABLED"] = False
settings["DOWNLOAD_HANDLERS"] = {
"ftp": None,
- "http": None,
- "https": None,
+ "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
+ "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
}
return settings
@@ -208,7 +208,14 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]: # pragma: no cover
return (client_mock, bucket_mock, blob_mock)
-def get_web_client_agent_req(url: str) -> Deferred[TxResponse]:
+def get_web_client_agent_req(url: str) -> Deferred[TxResponse]: # pragma: no cover
+ warnings.warn(
+ "The get_web_client_agent_req() function is deprecated"
+ " and will be removed in a future version of Scrapy.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+
from twisted.internet import reactor
agent = Agent(reactor)
diff --git a/tests/spiders.py b/tests/spiders.py
index de2e64242..79565738d 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -18,7 +18,7 @@ from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import Spider
from scrapy.spiders.crawl import CrawlSpider, Rule
from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
-from scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_req
+from scrapy.utils.test import get_from_asyncio_queue
class MockServerSpider(Spider):
@@ -199,28 +199,24 @@ class AsyncDefDeferredDirectSpider(SimpleSpider):
name = "asyncdef_deferred_direct"
async def parse(self, response):
- resp = await get_web_client_agent_req(self.mockserver.url("/status?n=200"))
- yield {"code": resp.code}
+ await defer.succeed(None)
+ yield {"code": 200}
class AsyncDefDeferredWrappedSpider(SimpleSpider):
name = "asyncdef_deferred_wrapped"
async def parse(self, response):
- resp = await deferred_to_future(
- get_web_client_agent_req(self.mockserver.url("/status?n=200"))
- )
- yield {"code": resp.code}
+ await deferred_to_future(defer.succeed(None))
+ yield {"code": 200}
class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):
name = "asyncdef_deferred_wrapped"
async def parse(self, response):
- resp = await maybe_deferred_to_future(
- get_web_client_agent_req(self.mockserver.url("/status?n=200"))
- )
- yield {"code": resp.code}
+ await maybe_deferred_to_future(defer.succeed(None))
+ yield {"code": 200}
class AsyncDefAsyncioGenSpider(SimpleSpider):
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index a6fd6dd19..9a7447e16 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -1,5 +1,3 @@
-import pytest
-
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import (
@@ -12,7 +10,6 @@ from tests.spiders import (
from tests.utils.decorators import inline_callbacks_test
[email protected]_http_handler
class TestCloseSpider:
@classmethod
def setup_class(cls):
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index f1678f912..a35ef7010 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -501,7 +501,6 @@ class TestContractsManager:
assert not self.results.failures
assert self.results.errors
- @pytest.mark.requires_http_handler
@inline_callbacks_test
def test_same_url(self):
class TestSameUrlSpider(Spider):
diff --git a/tests/test_core_scraper.py b/tests/test_core_scraper.py
index f4ef5ec5d..1b8ba08ab 100644
--- a/tests/test_core_scraper.py
+++ b/tests/test_core_scraper.py
@@ -2,17 +2,16 @@ from __future__ import annotations
from typing import TYPE_CHECKING
-import pytest
-
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
+ import pytest
+
from tests.mockserver.http import MockServer
[email protected]_http_handler
@coroutine_test
async def test_scraper_exception(
mockserver: MockServer,
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index b206cfffa..7e9e029a1 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -14,11 +14,11 @@ from twisted.internet.ssl import Certificate
from twisted.python.failure import Failure
from scrapy import Spider, signals
-from scrapy.crawler import CrawlerRunner
+from scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner
from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload
from scrapy.http import Request
from scrapy.http.response import Response
-from scrapy.utils.defer import maybe_deferred_to_future
+from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
from scrapy.utils.engine import format_engine_status, get_engine_status
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler, get_reactor_settings
@@ -61,7 +61,6 @@ if TYPE_CHECKING:
from scrapy.statscollectors import StatsCollector
[email protected]_http_handler # easier than marking many individual tests
class TestCrawl:
mockserver: MockServer
@@ -402,9 +401,15 @@ with multiples lines
)
assert "Got response 200" in str(log)
- @inline_callbacks_test
- def test_crawl_multiple(self, caplog: pytest.LogCaptureFixture):
- runner = CrawlerRunner(get_reactor_settings())
+ @coroutine_test
+ async def test_crawl_multiple(self, caplog: pytest.LogCaptureFixture) -> None:
+ settings_dict = get_reactor_settings()
+ runner_cls = (
+ CrawlerRunner
+ if settings_dict.get("TWISTED_ENABLED", True)
+ else AsyncCrawlerRunner
+ )
+ runner = runner_cls(settings_dict)
runner.crawl(
SimpleSpider,
self.mockserver.url("/status?n=200"),
@@ -417,7 +422,7 @@ with multiples lines
)
with caplog.at_level(logging.DEBUG):
- yield runner.join()
+ await ensure_awaitable(runner.join())
self._assert_retried(caplog.text)
assert "Got response 200" in caplog.text
@@ -429,7 +434,6 @@ with multiples lines
assert "NotSupported: Unsupported URL scheme 'foo'" in caplog.text
[email protected]_http_handler
class TestCrawlSpider:
mockserver: MockServer
@@ -636,6 +640,11 @@ class TestCrawlSpider:
yield crawler.crawl(seed=url, mockserver=self.mockserver)
assert crawler.spider.meta["responses"][0].certificate is None
+ @pytest.mark.xfail(
+ 'config.getoption("--reactor") == "none"',
+ reason="Not implemented in HttpxDownloadHandler",
+ strict=True,
+ )
@pytest.mark.parametrize(
"url",
[
@@ -643,6 +652,7 @@ class TestCrawlSpider:
pytest.param(
"/status?n=200",
marks=pytest.mark.xfail(
+ 'config.getoption("--reactor") != "none"',
reason="With HTTP11DownloadHandler, responses with no body are returned early and contain no certificate",
strict=True,
),
@@ -669,6 +679,7 @@ class TestCrawlSpider:
pytest.param(
"/status?n=200",
marks=pytest.mark.xfail(
+ 'config.getoption("--reactor") != "none"',
reason="With HTTP11DownloadHandler, responses with no body are returned early and contain no ip_address",
strict=True,
),
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 4589e667f..a8e635707 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -93,10 +93,11 @@ class TestLoad:
crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers})
dh = DownloadHandlers(crawler)
assert "scheme" not in dh._schemes
- for scheme in handlers: # force load handlers
- dh._get_handler(scheme)
+ assert dh._get_handler("scheme") is None
assert "scheme" not in dh._handlers
assert "scheme" in dh._notconfigured
+ # get the handler again to cover the code that gets it from dh._notconfigured
+ assert dh._get_handler("scheme") is None
def test_lazy_handlers(self):
handlers = {"scheme": DummyLazyDH}
@@ -108,8 +109,8 @@ class TestLoad:
dh = DownloadHandlers(crawler)
assert "scheme" in dh._schemes
assert "scheme" not in dh._handlers
- for scheme in handlers: # force load lazy handler
- dh._get_handler(scheme)
+ handler = dh._get_handler("scheme") # force load lazy handler
+ assert handler
assert "scheme" in dh._handlers
assert "scheme" not in dh._notconfigured
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
index 5ddb1cdfa..5c31f68bc 100644
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@@ -67,7 +67,6 @@ class TestCrawl:
def setup_method(self):
self.runner = CrawlerRunner()
- @pytest.mark.requires_http_handler
@inline_callbacks_test
def test_delay(self):
crawler = get_crawler(DownloaderSlotsSettingsTestSpider)
@@ -129,7 +128,6 @@ def test_get_slot_deprecated_spider_arg():
assert slot1 == slot2
[email protected]_http_handler
@pytest.mark.parametrize(
"priority_queue_class",
[
diff --git a/tests/test_engine.py b/tests/test_engine.py
index c77e3c5c9..4573c5001 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -373,7 +373,6 @@ class TestEngineBase:
class TestEngine(TestEngineBase):
- @pytest.mark.requires_http_handler
@coroutine_test
async def test_crawler(self, mockserver: MockServer) -> None:
for spider in (
@@ -391,7 +390,6 @@ class TestEngine(TestEngineBase):
self._assert_signals_caught(run)
self._assert_bytes_received(run)
- @pytest.mark.requires_http_handler
@coroutine_test
async def test_crawler_dupefilter(self, mockserver: MockServer) -> None:
run = CrawlerRun(DupeFilterSpider)
@@ -399,14 +397,12 @@ class TestEngine(TestEngineBase):
self._assert_scheduled_requests(run, count=8)
self._assert_dropped_requests(run)
- @pytest.mark.requires_http_handler
@coroutine_test
async def test_crawler_itemerror(self, mockserver: MockServer) -> None:
run = CrawlerRun(ItemZeroDivisionErrorSpider)
await run.run(mockserver)
self._assert_items_error(run)
- @pytest.mark.requires_http_handler
@coroutine_test
async def test_crawler_change_close_reason_on_idle(
self, mockserver: MockServer
diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py
index 9bdbfcc56..115deb821 100644
--- a/tests/test_engine_loop.py
+++ b/tests/test_engine_loop.py
@@ -332,7 +332,6 @@ class TestRequestSendOrder:
# Examples from the “Start requests” section of the documentation about
# spiders.
- @pytest.mark.requires_http_handler
@coroutine_test
async def test_lazy(self):
start_nums = [1, 2, 4]
diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py
index 970fffbc9..091485781 100644
--- a/tests/test_engine_stop_download_bytes.py
+++ b/tests/test_engine_stop_download_bytes.py
@@ -2,8 +2,6 @@ from __future__ import annotations
from typing import TYPE_CHECKING
-import pytest
-
from scrapy.exceptions import StopDownload
from tests.test_engine import (
AttrsItemsSpider,
@@ -16,6 +14,8 @@ from tests.test_engine import (
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
+ import pytest
+
from tests.mockserver.http import MockServer
@@ -26,7 +26,6 @@ class BytesReceivedCrawlerRun(CrawlerRun):
class TestBytesReceivedEngine(TestEngineBase):
- @pytest.mark.requires_http_handler
@coroutine_test
async def test_crawler(
self, mockserver: MockServer, caplog: pytest.LogCaptureFixture
diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py
index e0008a53a..8ebc948e2 100644
--- a/tests/test_engine_stop_download_headers.py
+++ b/tests/test_engine_stop_download_headers.py
@@ -2,8 +2,6 @@ from __future__ import annotations
from typing import TYPE_CHECKING
-import pytest
-
from scrapy.exceptions import StopDownload
from tests.test_engine import (
AttrsItemsSpider,
@@ -16,6 +14,8 @@ from tests.test_engine import (
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
+ import pytest
+
from tests.mockserver.http import MockServer
@@ -26,7 +26,6 @@ class HeadersReceivedCrawlerRun(CrawlerRun):
class TestHeadersReceivedEngine(TestEngineBase):
- @pytest.mark.requires_http_handler
@coroutine_test
async def test_crawler(
self, mockserver: MockServer, caplog: pytest.LogCaptureFixture
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 3a9c37042..1ca5f21f0 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -849,7 +849,6 @@ class ExceptionJsonItemExporter(JsonItemExporter):
raise RuntimeError("foo")
[email protected]_http_handler
class TestFeedExport(TestFeedExportBase):
async def run_and_export(
self, spider_cls: type[Spider], settings: dict[str, Any]
@@ -1703,6 +1702,7 @@ class TestFeedExport(TestFeedExportBase):
data = await self.exported_no_data(settings)
assert data["csv"] == b""
+ @pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage
@coroutine_test
async def test_multiple_feeds_success_logs_blocking_feed_storage(self):
settings = {
@@ -1834,7 +1834,6 @@ class TestFeedExport(TestFeedExportBase):
assert not Storage.file_was_closed
[email protected]_http_handler
class TestFeedPostProcessedExports(TestFeedExportBase):
items = [{"foo": "bar"}]
expected = b"foo\r\nbar\r\n"
@@ -2353,7 +2352,6 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert result == expected
[email protected]_http_handler
class TestBatchDeliveries(TestFeedExportBase):
_file_mark = "_%(batch_time)s_#%(batch_id)02d_"
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 66544a508..9806315b4 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -253,7 +253,6 @@ class DropSomeItemsPipeline:
self.drop = True
[email protected]_http_handler
class TestShowOrSkipMessages:
@classmethod
def setup_class(cls):
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index acc07f48b..0376d27c4 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -57,7 +57,6 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider):
)
[email protected]_http_handler
class TestFileDownloadCrawl:
pipeline_class = "scrapy.pipelines.files.FilesPipeline"
store_setting_key = "FILES_STORE"
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index fb1fccfc7..753fcd42e 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -127,7 +127,6 @@ class ItemSpider(Spider):
return {"field": 42}
[email protected]_http_handler
class TestPipeline:
def _on_item_scraped(self, item):
assert isinstance(item, dict)
@@ -266,7 +265,6 @@ class TestCustomPipelineManager:
):
itemproc.process_item({}, crawler.spider)
- @pytest.mark.requires_http_handler
@coroutine_test
async def test_integration_recommended(self, mockserver: MockServer) -> None:
class CustomPipelineManager(ItemPipelineManager):
@@ -293,7 +291,6 @@ class TestCustomPipelineManager:
assert len(items) == 1
- @pytest.mark.requires_http_handler
@coroutine_test
async def test_integration_no_async_subclass(self, mockserver: MockServer) -> None:
class CustomPipelineManager(ItemPipelineManager):
@@ -352,7 +349,6 @@ class TestCustomPipelineManager:
assert len(items) == 1
- @pytest.mark.requires_http_handler
@coroutine_test
async def test_integration_no_async_not_subclass(
self, mockserver: MockServer
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index c78c882cf..912075401 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -61,7 +61,6 @@ def _wrong_credentials(proxy_url):
return urlunsplit(bad_auth_proxy)
[email protected]_http_handler
@pytest.mark.requires_mitmproxy
class TestProxyConnect:
@classmethod
diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py
index 66dd48737..38d56e9bd 100644
--- a/tests/test_request_attribute_binding.py
+++ b/tests/test_request_attribute_binding.py
@@ -1,4 +1,3 @@
-import pytest
from testfixtures import LogCapture
from scrapy import Request, signals
@@ -63,7 +62,6 @@ class AlternativeCallbacksMiddleware:
return response.replace(request=new_request)
[email protected]_http_handler
class TestCrawl:
@classmethod
def setup_class(cls):
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index c8629df00..8d3977452 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -1,4 +1,3 @@
-import pytest
from testfixtures import LogCapture
from scrapy.http import Request
@@ -149,7 +148,6 @@ class KeywordArgumentsSpider(MockServerSpider):
self.crawler.stats.inc_value("boolean_checks", 1)
[email protected]_http_handler
class TestCallbackKeywordArguments:
@classmethod
def setup_class(cls):
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 873be0b8f..0d2d8be93 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -369,7 +369,6 @@ class TestIntegrationWithDownloaderAwareInMemory:
},
)
- @pytest.mark.requires_http_handler
@inline_callbacks_test
def test_integration_downloader_aware_priority_queue(self):
with MockServer() as mockserver:
diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py
index 6234bef86..176fefbbf 100644
--- a/tests/test_scheduler_base.py
+++ b/tests/test_scheduler_base.py
@@ -144,7 +144,6 @@ class TestSimpleScheduler(InterfaceCheckMixin):
assert close_result == "close"
[email protected]_http_handler
class TestMinimalSchedulerCrawl:
scheduler_cls = MinimalScheduler
diff --git a/tests/test_signals.py b/tests/test_signals.py
index faf24a129..f5f13b992 100644
--- a/tests/test_signals.py
+++ b/tests/test_signals.py
@@ -50,7 +50,6 @@ class TestMockServer:
item = await get_from_asyncio_queue(item)
self.items.append(item)
- @pytest.mark.requires_http_handler
@pytest.mark.only_asyncio
@inline_callbacks_test
def test_simple_pipeline(self):
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index 8dc6ff300..b15f68dd0 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -191,7 +191,6 @@ class TestHttpErrorMiddlewareHandleAll:
mw.process_spider_input(res402)
[email protected]_http_handler
class TestHttpErrorMiddlewareIntegrational:
@classmethod
def setup_class(cls):
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 80f5828c9..8cf08dd94 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -1,4 +1,3 @@
-import pytest
from testfixtures import LogCapture
from scrapy import Request, Spider
@@ -319,7 +318,6 @@ class NotGeneratorOutputChainSpider(Spider):
# ================================================================================
[email protected]_http_handler
class TestSpiderMiddleware:
mockserver: MockServer
diff --git a/tox.ini b/tox.ini
index 213ecbef6..7656cb017 100644
--- a/tox.ini
+++ b/tox.ini
@@ -168,6 +168,7 @@ commands =
[testenv:no-reactor]
deps =
{[testenv]deps}
+ httpx
pytest-asyncio
commands =
{[testenv]commands} -p no:twisted --reactor=none
@@ -183,6 +184,7 @@ setenv =
basepython = {[pinned]basepython}
deps =
{[testenv:pinned]deps}
+ httpx==0.26.0
pytest-asyncio
commands = {[pinned]commands} -p no:twisted --reactor=none
setenv =