Scrapy, a fast high-level web crawling & scraping framework for Python.
Improve test coverage for spider middlewares (#7664)
Adrian65e8954a0689
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex 2c9452174..264f685c1 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -308,6 +308,12 @@ class RefererMiddleware(BaseSpiderMiddleware):# Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-stringself.policies[""] = NoReferrerWhenDowngradePolicyif settings is None:+ warn(+ "Instantiating RefererMiddleware without a 'settings' argument is "+ "deprecated.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )returnsetting_policies = settings.getdict("REFERRER_POLICIES")for policy_name, policy_class_import_path in setting_policies.items():diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.pyindex 70326f3f1..4de57fda9 100644--- a/tests/test_spidermiddleware_base.py+++ b/tests/test_spidermiddleware_base.py@@ -56,6 +56,11 @@ async def test_processed_request(crawler: Crawler) -> None:spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3]for processed in [list(mw.process_spider_output(Response("data:,"), spider_output)),+ await collect_asyncgen(+ mw.process_spider_output_async(+ Response("data:,"), as_async_generator(spider_output)+ )+ ),await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),]:assert len(processed) == 3diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.pyindex 00e547305..307ed2440 100644--- a/tests/test_spidermiddleware_depth.py+++ b/tests/test_spidermiddleware_depth.py@@ -7,6 +7,7 @@ import pytestfrom scrapy.http import Request, Responsefrom scrapy.spidermiddlewares.depth import DepthMiddlewarefrom scrapy.spiders import Spider+from scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawlerif TYPE_CHECKING:@@ -55,3 +56,27 @@ def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> Nordm = stats.get_value("request_depth_max")assert rdm == 1+++def test_priority_and_non_verbose_stats() -> None:+ crawler = get_crawler(+ Spider,+ {"DEPTH_LIMIT": 0, "DEPTH_STATS_VERBOSE": False, "DEPTH_PRIORITY": 10},+ )+ assert crawler.stats is not None+ crawler.stats.open_spider()+ try:+ mw = build_from_crawler(DepthMiddleware, crawler)+ resp = Response("http://toscrape.com")+ resp.request = Request("http://toscrape.com")+ resp.request.meta["depth"] = 2+ out = list(mw.process_spider_output(resp, [Request("http://toscrape.com")]))+ assert len(out) == 1+ # priority is decremented by depth * DEPTH_PRIORITY+ assert out[0].priority == -30+ assert out[0].meta["depth"] == 3+ # non-verbose stats don't track per-depth counts but still track the max+ assert crawler.stats.get_value("request_depth_count/3") is None+ assert crawler.stats.get_value("request_depth_max") == 3+ finally:+ crawler.stats.close_spider()diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex b3a434c3f..32e83f990 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -6,7 +6,7 @@ from urllib.parse import urlparseimport pytest-from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.http import Request, Responsefrom scrapy.settings import Settingsfrom scrapy.spidermiddlewares.referer import (@@ -1017,6 +1017,14 @@ class TestPolicyMethodResponseParamRename:response=self.response, resp_or_url=self.response, request=self.request)+ def test_missing_response(self):+ with pytest.raises(TypeError, match="Missing required argument: 'response'"):+ self.mw.policy(request=self.request)++ def test_missing_request(self):+ with pytest.raises(TypeError, match="Missing required argument: 'request'"):+ self.mw.policy(response=self.response)+@coroutine_testasync def test_response_policy_only_supports_policy_names():@@ -1115,3 +1123,36 @@ async def test_referer_policies_setting():]assert len(output) == 1assert output[0].headers == {b"Referer": [b"https://python.org/"]}+++class TestReferrerPolicyHelpers:+ def test_origin_referrer_local_scheme(self):+ # A local scheme yields no referrer.+ assert UnsafeUrlPolicy().origin_referrer("data:,foo") is None++ def test_strip_url_empty(self):+ assert UnsafeUrlPolicy().strip_url("") is None++ def test_potentially_trustworthy_data_scheme(self):+ assert UnsafeUrlPolicy().potentially_trustworthy("data:,foo") is False+++def test_default_policy():+ crawler = get_crawler()+ mw = build_from_crawler(RefererMiddleware, crawler)+ assert mw.default_policy is DefaultReferrerPolicy+++def test_no_settings_constructor():+ with pytest.warns(+ ScrapyDeprecationWarning,+ match="Instantiating RefererMiddleware without a 'settings' argument",+ ):+ mw = RefererMiddleware()+ assert mw.default_policy is DefaultReferrerPolicy+++def test_not_configured_when_disabled():+ crawler = get_crawler(settings_dict={"REFERER_ENABLED": False})+ with pytest.raises(NotConfigured):+ build_from_crawler(RefererMiddleware, crawler)diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.pyindex 26397d37c..def3a3df3 100644--- a/tests/test_spidermiddleware_start.py+++ b/tests/test_spidermiddleware_start.py@@ -1,4 +1,4 @@-from scrapy.http import Request+from scrapy.http import Request, Responsefrom scrapy.spidermiddlewares.start import StartSpiderMiddlewarefrom scrapy.spiders import Spiderfrom scrapy.utils.misc import build_from_crawler@@ -23,3 +23,13 @@ class TestMiddleware:async for request in mw.process_start(start())]assert result == [True, True, False, "foo"]++ def test_spider_output_not_marked(self):+ # Requests from a non-None response (spider output) are not flagged.+ crawler = get_crawler(Spider)+ mw = build_from_crawler(StartSpiderMiddleware, crawler)+ response = Response("data:,")+ request = Request("data:,1")+ out = list(mw.process_spider_output(response, [request]))+ assert out == [request]+ assert "is_start_request" not in request.metadiff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.pyindex 750ae3b07..1ed3a5637 100644--- a/tests/test_spidermiddleware_urllength.py+++ b/tests/test_spidermiddleware_urllength.py@@ -5,9 +5,11 @@ from typing import TYPE_CHECKINGimport pytest+from scrapy.exceptions import NotConfiguredfrom scrapy.http import Request, Responsefrom scrapy.spidermiddlewares.urllength import UrlLengthMiddlewarefrom scrapy.spiders import Spider+from scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawlerif TYPE_CHECKING:@@ -46,6 +48,12 @@ def test_middleware_works(mw: UrlLengthMiddleware) -> None:assert process_spider_output(mw) == [short_url_req]+def test_not_configured_without_limit() -> None:+ crawler = get_crawler(Spider, {"URLLENGTH_LIMIT": 0})+ with pytest.raises(NotConfigured):+ build_from_crawler(UrlLengthMiddleware, crawler)++def test_logging(stats: StatsCollector, mw: UrlLengthMiddleware, caplog: pytest.LogCaptureFixture) -> None: