Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Improve test coverage for spider middlewares (#7664)

Adrian65e8954a0689
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 2c9452174..264f685c1 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -308,6 +308,12 @@ class RefererMiddleware(BaseSpiderMiddleware):
# Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string
self.policies[""] = NoReferrerWhenDowngradePolicy
if settings is None:
+ warn(
+ "Instantiating RefererMiddleware without a 'settings' argument is "
+ "deprecated.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
return
setting_policies = settings.getdict("REFERRER_POLICIES")
for policy_name, policy_class_import_path in setting_policies.items():
diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py
index 70326f3f1..4de57fda9 100644
--- a/tests/test_spidermiddleware_base.py
+++ b/tests/test_spidermiddleware_base.py
@@ -56,6 +56,11 @@ async def test_processed_request(crawler: Crawler) -> None:
spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3]
for processed in [
list(mw.process_spider_output(Response("data:,"), spider_output)),
+ await collect_asyncgen(
+ mw.process_spider_output_async(
+ Response("data:,"), as_async_generator(spider_output)
+ )
+ ),
await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),
]:
assert len(processed) == 3
diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py
index 00e547305..307ed2440 100644
--- a/tests/test_spidermiddleware_depth.py
+++ b/tests/test_spidermiddleware_depth.py
@@ -7,6 +7,7 @@ import pytest
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.depth import DepthMiddleware
from scrapy.spiders import Spider
+from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
@@ -55,3 +56,27 @@ def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> No
rdm = stats.get_value("request_depth_max")
assert rdm == 1
+
+
+def test_priority_and_non_verbose_stats() -> None:
+ crawler = get_crawler(
+ Spider,
+ {"DEPTH_LIMIT": 0, "DEPTH_STATS_VERBOSE": False, "DEPTH_PRIORITY": 10},
+ )
+ assert crawler.stats is not None
+ crawler.stats.open_spider()
+ try:
+ mw = build_from_crawler(DepthMiddleware, crawler)
+ resp = Response("http://toscrape.com")
+ resp.request = Request("http://toscrape.com")
+ resp.request.meta["depth"] = 2
+ out = list(mw.process_spider_output(resp, [Request("http://toscrape.com")]))
+ assert len(out) == 1
+ # priority is decremented by depth * DEPTH_PRIORITY
+ assert out[0].priority == -30
+ assert out[0].meta["depth"] == 3
+ # non-verbose stats don't track per-depth counts but still track the max
+ assert crawler.stats.get_value("request_depth_count/3") is None
+ assert crawler.stats.get_value("request_depth_max") == 3
+ finally:
+ crawler.stats.close_spider()
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index b3a434c3f..32e83f990 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -6,7 +6,7 @@ from urllib.parse import urlparse
import pytest
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.spidermiddlewares.referer import (
@@ -1017,6 +1017,14 @@ class TestPolicyMethodResponseParamRename:
response=self.response, resp_or_url=self.response, request=self.request
)
+ def test_missing_response(self):
+ with pytest.raises(TypeError, match="Missing required argument: 'response'"):
+ self.mw.policy(request=self.request)
+
+ def test_missing_request(self):
+ with pytest.raises(TypeError, match="Missing required argument: 'request'"):
+ self.mw.policy(response=self.response)
+
@coroutine_test
async def test_response_policy_only_supports_policy_names():
@@ -1115,3 +1123,36 @@ async def test_referer_policies_setting():
]
assert len(output) == 1
assert output[0].headers == {b"Referer": [b"https://python.org/"]}
+
+
+class TestReferrerPolicyHelpers:
+ def test_origin_referrer_local_scheme(self):
+ # A local scheme yields no referrer.
+ assert UnsafeUrlPolicy().origin_referrer("data:,foo") is None
+
+ def test_strip_url_empty(self):
+ assert UnsafeUrlPolicy().strip_url("") is None
+
+ def test_potentially_trustworthy_data_scheme(self):
+ assert UnsafeUrlPolicy().potentially_trustworthy("data:,foo") is False
+
+
+def test_default_policy():
+ crawler = get_crawler()
+ mw = build_from_crawler(RefererMiddleware, crawler)
+ assert mw.default_policy is DefaultReferrerPolicy
+
+
+def test_no_settings_constructor():
+ with pytest.warns(
+ ScrapyDeprecationWarning,
+ match="Instantiating RefererMiddleware without a 'settings' argument",
+ ):
+ mw = RefererMiddleware()
+ assert mw.default_policy is DefaultReferrerPolicy
+
+
+def test_not_configured_when_disabled():
+ crawler = get_crawler(settings_dict={"REFERER_ENABLED": False})
+ with pytest.raises(NotConfigured):
+ build_from_crawler(RefererMiddleware, crawler)
diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py
index 26397d37c..def3a3df3 100644
--- a/tests/test_spidermiddleware_start.py
+++ b/tests/test_spidermiddleware_start.py
@@ -1,4 +1,4 @@
-from scrapy.http import Request
+from scrapy.http import Request, Response
from scrapy.spidermiddlewares.start import StartSpiderMiddleware
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
@@ -23,3 +23,13 @@ class TestMiddleware:
async for request in mw.process_start(start())
]
assert result == [True, True, False, "foo"]
+
+ def test_spider_output_not_marked(self):
+ # Requests from a non-None response (spider output) are not flagged.
+ crawler = get_crawler(Spider)
+ mw = build_from_crawler(StartSpiderMiddleware, crawler)
+ response = Response("data:,")
+ request = Request("data:,1")
+ out = list(mw.process_spider_output(response, [request]))
+ assert out == [request]
+ assert "is_start_request" not in request.meta
diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py
index 750ae3b07..1ed3a5637 100644
--- a/tests/test_spidermiddleware_urllength.py
+++ b/tests/test_spidermiddleware_urllength.py
@@ -5,9 +5,11 @@ from typing import TYPE_CHECKING
import pytest
+from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware
from scrapy.spiders import Spider
+from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
@@ -46,6 +48,12 @@ def test_middleware_works(mw: UrlLengthMiddleware) -> None:
assert process_spider_output(mw) == [short_url_req]
+def test_not_configured_without_limit() -> None:
+ crawler = get_crawler(Spider, {"URLLENGTH_LIMIT": 0})
+ with pytest.raises(NotConfigured):
+ build_from_crawler(UrlLengthMiddleware, crawler)
+
+
def test_logging(
stats: StatsCollector, mw: UrlLengthMiddleware, caplog: pytest.LogCaptureFixture
) -> None: