Scrapy, a fast high-level web crawling & scraping framework for Python.
Type downloader middleware tests (#7858)
Adrian2b2e18199b0b
diff --git a/pyproject.toml b/pyproject.tomlindex 1cbd39946..11e971a35 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -125,17 +125,6 @@ module = ["tests.test_contracts","tests.test_core_downloader","tests.test_downloader_handler_twisted_ftp",- "tests.test_downloadermiddleware_cookies",- "tests.test_downloadermiddleware_httpauth",- "tests.test_downloadermiddleware_httpcache",- "tests.test_downloadermiddleware_httpcompression",- "tests.test_downloadermiddleware_httpproxy",- "tests.test_downloadermiddleware_offsite",- "tests.test_downloadermiddleware_redirect",- "tests.test_downloadermiddleware_redirect_base",- "tests.test_downloadermiddleware_redirect_metarefresh",- "tests.test_downloadermiddleware_retry",- "tests.test_downloadermiddleware_robotstxt","tests.test_downloaderslotssettings","tests.test_dupefilters","tests.test_engine_loop",@@ -167,7 +156,6 @@ module = ["tests.test_request_cb_kwargs","tests.test_request_dict","tests.test_request_left",- "tests.test_robotstxt_interface","tests.test_scheduler_base","tests.test_settings","tests.test_spider",diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.pyindex 7d67bb6d7..7c53b6b48 100644--- a/scrapy/http/request/__init__.py+++ b/scrapy/http/request/__init__.py@@ -50,7 +50,9 @@ class VerboseCookie(TypedDict):secure: NotRequired[bool]-CookiesT: TypeAlias = dict[str | bytes, str | bytes] | list[VerboseCookie]+CookiesT: TypeAlias = (+ dict[str | bytes, str | bytes | bool | float | int] | list[VerboseCookie]+)RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.pyindex 7ad103f27..8d999d952 100644--- a/tests/test_downloadermiddleware_cookies.py+++ b/tests/test_downloadermiddleware_cookies.py@@ -1,5 +1,6 @@import loggingfrom collections.abc import Iterable+from typing import Anyimport pytest@@ -219,7 +220,7 @@ class TestCookiesMiddleware:def test_complex_cookies(self):# merge some cookies into jar- cookies = [+ cookies: list[VerboseCookie] = [{"name": "C1","value": "value1",@@ -483,13 +484,13 @@ class TestCookiesMiddleware:def _test_cookie_redirect(self,- source,- target,+ source: str | dict[str, Any],+ target: str | dict[str, Any],*,- cookies1,- cookies2,- ):- input_cookies = {"a": "b"}+ cookies1: bool,+ cookies2: bool,+ ) -> None:+ input_cookies: CookiesT = {"a": "b"}if not isinstance(source, dict):source = {"url": source}@@ -551,11 +552,11 @@ class TestCookiesMiddleware:def _test_cookie_header_redirect(self,- source,- target,+ source: str | dict[str, Any],+ target: str | dict[str, Any],*,- cookies2,- ):+ cookies2: bool,+ ) -> None:"""Test the handling of a user-defined Cookie header when building aredirect follow-up request.@@ -623,14 +624,14 @@ class TestCookiesMiddleware:def _test_user_set_cookie_domain_followup(self,- url1,- url2,- domain,+ url1: str,+ url2: str,+ domain: str,*,- cookies1,- cookies2,- ):- input_cookies = [+ cookies1: bool,+ cookies2: bool,+ ) -> None:+ input_cookies: list[VerboseCookie] = [{"name": "a","value": "b",@@ -686,16 +687,16 @@ class TestCookiesMiddleware:def _test_server_set_cookie_domain_followup(self,- url1,- url2,- domain,+ url1: str,+ url2: str,+ domain: str,*,- cookies,- ):+ cookies: bool,+ ) -> None:request1 = Request(url1)self.mw.process_request(request1)- input_cookies = [+ input_cookies: list[VerboseCookie] = [{"name": "a","value": "b",@@ -747,8 +748,14 @@ class TestCookiesMiddleware:)def _test_cookie_redirect_scheme_change(- self, secure, from_scheme, to_scheme, cookies1, cookies2, cookies3- ):+ self,+ secure: bool | object,+ from_scheme: str,+ to_scheme: str,+ cookies1: bool,+ cookies2: bool,+ cookies3: bool,+ ) -> None:"""When a redirect causes the URL scheme to change from *from_scheme*to *to_scheme*, while domain and port remain the same, and given acookie on the initial request with its secure attribute set to@@ -756,10 +763,11 @@ class TestCookiesMiddleware:initial request (*cookies1*), if it should be kept by the redirectmiddleware (*cookies2*), and if it should be present on the Cookieheader in the redirected request (*cookie3*)."""- cookie_kwargs = {}+ cookie: VerboseCookie = {"name": "a", "value": "b"}if secure is not UNSET:- cookie_kwargs["secure"] = secure- input_cookies = [{"name": "a", "value": "b", **cookie_kwargs}]+ assert isinstance(secure, bool)+ cookie["secure"] = secure+ input_cookies = [cookie]request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies)self.mw.process_request(request1)diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.pyindex 827133d2e..dd5af3bc5 100644--- a/tests/test_downloadermiddleware_httpauth.py+++ b/tests/test_downloadermiddleware_httpauth.py@@ -1,3 +1,5 @@+from typing import Any+import pytestfrom w3lib.http import basic_auth_header@@ -10,8 +12,10 @@ from scrapy.utils.test import get_crawler_DOMAIN_NOT_SET = object()-def make_mw(user="", passwd="", domain=_DOMAIN_NOT_SET):- settings: dict = {+def make_mw(+ user: str = "", passwd: str = "", domain: str | object = _DOMAIN_NOT_SET+) -> HttpAuthMiddleware:+ settings: dict[str, Any] = {"HTTPAUTH_USER": user,"HTTPAUTH_PASS": passwd,}diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.pyindex 6e8486eb8..dc8228470 100644--- a/tests/test_downloadermiddleware_httpcache.py+++ b/tests/test_downloadermiddleware_httpcache.py@@ -94,14 +94,14 @@ class TestBase:finally:mw.spider_closed(crawler.spider)- def assertEqualResponse(self, response1, response2):+ def assertEqualResponse(self, response1: Response, response2: Response) -> None:assert response1.url == response2.urlassert response1.status == response2.statusassert response1.headers == response2.headersassert response1.body == response2.body-class StorageTestMixin:+class StorageTestMixin(TestBase):"""Mixin containing storage-specific test methods."""def _corrupt_cache_entry(@@ -135,6 +135,8 @@ class StorageTestMixin:def test_corrupted_cache_entry_is_a_miss(self, caplog):with self._middleware() as mw:spider = mw.crawler.spider+ assert spider+ assert mw.crawler.statsmw.storage.store_response(spider, self.request, self.response)self._corrupt_cache_entry(mw.storage, spider, self.request)@@ -155,6 +157,8 @@ class StorageTestMixin:def test_corrupted_cache_entry_ignore_missing(self):with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:spider = mw.crawler.spider+ assert spider+ assert mw.crawler.statsmw.storage.store_response(spider, self.request, self.response)self._corrupt_cache_entry(mw.storage, spider, self.request)@@ -180,7 +184,7 @@ class StorageTestMixin:self.assertEqualResponse(response, cached_response)-class PolicyTestMixin:+class PolicyTestMixin(TestBase):"""Mixin containing policy-specific test methods."""def test_dont_cache(self):@@ -302,6 +306,7 @@ class DummyPolicyTestMixin(PolicyTestMixin):assert mw.process_request(self.request) is Nonefresh_response = self.response.replace(body=b"new body")response = mw.process_response(self.request, fresh_response)+ assert isinstance(response, Response)self.assertEqualResponse(self.response, response)assert "cached" in response.flagsassert mw.stats.get_value("httpcache/revalidate") == 1@@ -313,12 +318,12 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):@staticmethoddef _process_requestresponse(mw: HttpCacheMiddleware, request: Request, response: Response | None- ) -> Response | Request:- result = None+ ) -> Response:+ result: Request | Response | None = Nonetry:result = mw.process_request(request)if result:- assert isinstance(result, (Request, Response))+ assert isinstance(result, Response)return resultassert response is not Noneresult = mw.process_response(request, response)@@ -346,6 +351,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):res2 = self._process_requestresponse(mw, req0, res0)assert "cached" not in res2.flagsres3 = mw.process_request(req0)+ assert isinstance(res3, Response)assert "cached" in res3.flagsself.assertEqualResponse(res2, res3)# request with no-cache directive must not return cached response@@ -634,6 +640,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):assert mw.process_request(req0) is Noneres1 = mw.process_exception(req0, e("foo"))# Use cached response as recovery+ assert isinstance(res1, Response)assert "cached" in res1.flagsself.assertEqualResponse(res0, res1)# Do not use cached response for unhandled exceptions@@ -684,26 +691,22 @@ class DbmStorageTestMixin(StorageTestMixin):class TestFilesystemStorageWithDummyPolicy(- TestBase, FilesystemStorageTestMixin, DummyPolicyTestMixin+ FilesystemStorageTestMixin, DummyPolicyTestMixin):policy_class = "scrapy.extensions.httpcache.DummyPolicy"class TestFilesystemStorageWithRFC2616Policy(- TestBase, FilesystemStorageTestMixin, RFC2616PolicyTestMixin+ FilesystemStorageTestMixin, RFC2616PolicyTestMixin):policy_class = "scrapy.extensions.httpcache.RFC2616Policy"-class TestDbmStorageWithDummyPolicy(- TestBase, DbmStorageTestMixin, DummyPolicyTestMixin-):+class TestDbmStorageWithDummyPolicy(DbmStorageTestMixin, DummyPolicyTestMixin):policy_class = "scrapy.extensions.httpcache.DummyPolicy"-class TestDbmStorageWithRFC2616Policy(- TestBase, DbmStorageTestMixin, RFC2616PolicyTestMixin-):+class TestDbmStorageWithRFC2616Policy(DbmStorageTestMixin, RFC2616PolicyTestMixin):policy_class = "scrapy.extensions.httpcache.RFC2616Policy"diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 55f06b396..fa0707491 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -3,6 +3,7 @@ from importlib.util import find_specfrom io import BytesIOfrom logging import WARNINGfrom pathlib import Path+from typing import Anyimport pytestfrom w3lib.encoding import resolve_encoding@@ -15,6 +16,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarfrom scrapy.http import HtmlResponse, Request, Responsefrom scrapy.responsetypes import responsetypesfrom scrapy.spiders import Spider+from scrapy.utils._compression import _DecompressionMaxSizeExceededfrom scrapy.utils.gz import gunzipfrom scrapy.utils.test import get_crawlerfrom tests import tests_datadir@@ -72,6 +74,7 @@ class TestHttpCompression:def setup_method(self):self.crawler = get_crawler(Spider)self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)+ assert self.crawler.statsself.crawler.stats.open_spider()def _getresponse(self, coding: str) -> Response:@@ -96,7 +99,8 @@ class TestHttpCompression:)return response- def assertStatsEqual(self, key, value):+ def assertStatsEqual(self, key: str, value: Any) -> None:+ assert self.crawler.statsassert self.crawler.stats.get_value(key) == value, str(self.crawler.stats.get_stats())@@ -145,6 +149,7 @@ class TestHttpCompression:def test_process_response_gzip(self):response = self._getresponse("gzip")+ assert response.requestrequest = response.requestassert response.headers["Content-Encoding"] == b"gzip"@@ -159,6 +164,7 @@ class TestHttpCompression:_skip_if_no_br()response = self._getresponse("br")+ assert response.requestrequest = response.requestassert response.headers["Content-Encoding"] == b"br"newresponse = self.mw.process_response(request, response)@@ -172,6 +178,7 @@ class TestHttpCompression:if find_spec("brotli") is not None or find_spec("brotlicffi") is not None:pytest.skip("Requires not having brotli support")response = self._getresponse("br")+ assert response.requestrequest = response.requestassert response.headers["Content-Encoding"] == b"br"caplog.clear()@@ -201,6 +208,7 @@ class TestHttpCompression:if not check_key.startswith("zstd-"):continueresponse = self._getresponse(check_key)+ assert response.requestrequest = response.requestassert response.headers["Content-Encoding"] == b"zstd"newresponse = self.mw.process_response(request, response)@@ -216,6 +224,7 @@ class TestHttpCompression:if find_spec("zstandard") is not None:pytest.skip("Requires not having zstandard support")response = self._getresponse("zstd-static-content-size")+ assert response.requestrequest = response.requestassert response.headers["Content-Encoding"] == b"zstd"caplog.clear()@@ -239,6 +248,7 @@ class TestHttpCompression:def test_process_response_rawdeflate(self):response = self._getresponse("rawdeflate")+ assert response.requestrequest = response.requestassert response.headers["Content-Encoding"] == b"deflate"@@ -251,6 +261,7 @@ class TestHttpCompression:def test_process_response_zlibdelate(self):response = self._getresponse("zlibdeflate")+ assert response.requestrequest = response.requestassert response.headers["Content-Encoding"] == b"deflate"@@ -275,6 +286,7 @@ class TestHttpCompression:def test_multipleencodings(self):response = self._getresponse("gzip")response.headers["Content-Encoding"] = ["uuencode", "gzip"]+ assert response.requestrequest = response.requestnewresponse = self.mw.process_response(request, response)assert newresponse is not response@@ -282,6 +294,7 @@ class TestHttpCompression:def test_multi_compression_single_header(self):response = self._getresponse("gzip-deflate")+ assert response.requestrequest = response.requestnewresponse = self.mw.process_response(request, response)assert newresponse is not response@@ -293,6 +306,7 @@ class TestHttpCompression:) -> None:response = self._getresponse("gzip-deflate")response.headers["Content-Encoding"] = [b"gzip, foo, deflate"]+ assert response.requestrequest = response.requestcaplog.clear()with caplog.at_level(@@ -315,6 +329,7 @@ class TestHttpCompression:def test_multi_compression_multiple_header(self):response = self._getresponse("gzip-deflate")response.headers["Content-Encoding"] = ["gzip", "deflate"]+ assert response.requestrequest = response.requestnewresponse = self.mw.process_response(request, response)assert newresponse is not response@@ -324,6 +339,7 @@ class TestHttpCompression:def test_multi_compression_multiple_header_invalid_compression(self):response = self._getresponse("gzip-deflate")response.headers["Content-Encoding"] = ["gzip", "foo", "deflate"]+ assert response.requestrequest = response.requestnewresponse = self.mw.process_response(request, response)assert newresponse is not response@@ -332,6 +348,7 @@ class TestHttpCompression:def test_multi_compression_single_and_multiple_header(self):response = self._getresponse("gzip-deflate-gzip")response.headers["Content-Encoding"] = ["gzip", "deflate, gzip"]+ assert response.requestrequest = response.requestnewresponse = self.mw.process_response(request, response)assert newresponse is not response@@ -341,6 +358,7 @@ class TestHttpCompression:def test_multi_compression_single_and_multiple_header_invalid_compression(self):response = self._getresponse("gzip-deflate")response.headers["Content-Encoding"] = ["gzip", "foo,deflate"]+ assert response.requestrequest = response.requestnewresponse = self.mw.process_response(request, response)assert newresponse is not response@@ -397,9 +415,7 @@ class TestHttpCompression:self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))def test_process_response_no_content_type_header(self):- headers = {- "Content-Encoding": "identity",- }+ headers = {b"Content-Encoding": b"identity"}plainbody = (b"<html><head><title>Some page</title>"b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">'@@ -414,6 +430,7 @@ class TestHttpCompression:newresponse = self.mw.process_response(request, response)assert isinstance(newresponse, respcls)+ assert isinstance(newresponse, HtmlResponse)assert newresponse.body == plainbodyassert newresponse.encoding == resolve_encoding("gb2312")self.assertStatsEqual("httpcompression/response_count", 1)@@ -422,6 +439,7 @@ class TestHttpCompression:def test_process_response_gzipped_contenttype(self):response = self._getresponse("gzip")response.headers["Content-Type"] = "application/gzip"+ assert response.requestrequest = response.requestnewresponse = self.mw.process_response(request, response)@@ -434,6 +452,7 @@ class TestHttpCompression:def test_process_response_gzip_app_octetstream_contenttype(self):response = self._getresponse("gzip")response.headers["Content-Type"] = "application/octet-stream"+ assert response.requestrequest = response.requestnewresponse = self.mw.process_response(request, response)@@ -446,6 +465,7 @@ class TestHttpCompression:def test_process_response_gzip_binary_octetstream_contenttype(self):response = self._getresponse("x-gzip")response.headers["Content-Type"] = "binary/octet-stream"+ assert response.requestrequest = response.requestnewresponse = self.mw.process_response(request, response)@@ -504,6 +524,7 @@ class TestHttpCompression:def test_process_response_head_request_no_decode_required(self):response = self._getresponse("gzip")response.headers["Content-Type"] = "application/gzip"+ assert response.requestrequest = response.requestrequest.method = "HEAD"response = response.replace(body=None)@@ -513,7 +534,7 @@ class TestHttpCompression:self.assertStatsEqual("httpcompression/response_count", None)self.assertStatsEqual("httpcompression/response_bytes", None)- def _test_compression_bomb_setting(self, compression_id):+ def _test_compression_bomb_setting(self, compression_id: str) -> None:settings = {"DOWNLOAD_MAXSIZE": 1_000_000}crawler = get_crawler(Spider, settings_dict=settings)spider = crawler._create_spider("scrapytest.org")@@ -521,9 +542,12 @@ class TestHttpCompression:mw.open_spider(spider)response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B+ assert response.requestwith pytest.raises(IgnoreRequest) as exc_info:mw.process_response(response.request, response)- assert exc_info.value.__cause__.decompressed_size < 1_100_000+ cause = exc_info.value.__cause__+ assert isinstance(cause, _DecompressionMaxSizeExceeded)+ assert cause.decompressed_size < 1_100_000def test_compression_bomb_setting_br(self):_skip_if_no_br()@@ -549,6 +573,7 @@ class TestHttpCompression:mw.open_spider(spider)response = self._getresponse("bomb-gzip") # 11_511_612 B+ assert response.requestcaplog.clear()with (caplog.at_level(@@ -565,7 +590,7 @@ class TestHttpCompression:)]- def _test_compression_bomb_spider_attr(self, compression_id):+ def _test_compression_bomb_spider_attr(self, compression_id: str) -> None:class DownloadMaxSizeSpider(Spider):download_maxsize = 1_000_000@@ -575,9 +600,12 @@ class TestHttpCompression:mw.open_spider(spider)response = self._getresponse(f"bomb-{compression_id}")+ assert response.requestwith pytest.raises(IgnoreRequest) as exc_info:mw.process_response(response.request, response)- assert exc_info.value.__cause__.decompressed_size < 1_100_000+ cause = exc_info.value.__cause__+ assert isinstance(cause, _DecompressionMaxSizeExceeded)+ assert cause.decompressed_size < 1_100_000@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_compression_bomb_spider_attr_br(self):@@ -599,7 +627,7 @@ class TestHttpCompression:self._test_compression_bomb_spider_attr("zstd")- def _test_compression_bomb_request_meta(self, compression_id):+ def _test_compression_bomb_request_meta(self, compression_id: str) -> None:crawler = get_crawler(Spider)spider = crawler._create_spider("scrapytest.org")mw = HttpCompressionMiddleware.from_crawler(crawler)@@ -607,9 +635,12 @@ class TestHttpCompression:response = self._getresponse(f"bomb-{compression_id}")response.meta["download_maxsize"] = 1_000_000+ assert response.requestwith pytest.raises(IgnoreRequest) as exc_info:mw.process_response(response.request, response)- assert exc_info.value.__cause__.decompressed_size < 1_100_000+ cause = exc_info.value.__cause__+ assert isinstance(cause, _DecompressionMaxSizeExceeded)+ assert cause.decompressed_size < 1_100_000def test_compression_bomb_request_meta_br(self):_skip_if_no_br()@@ -789,7 +820,7 @@ class TestHttpCompression:self._test_download_warnsize_request_meta(caplog, "zstd")- def _get_truncated_response(self, compression_id):+ def _get_truncated_response(self, compression_id: str) -> Response:crawler = get_crawler(Spider)spider = crawler._create_spider("scrapytest.org")mw = HttpCompressionMiddleware.from_crawler(crawler)@@ -797,7 +828,10 @@ class TestHttpCompression:response = self._getresponse(compression_id)truncated_body = response.body[: len(response.body) // 2]response = response.replace(body=truncated_body)- return mw.process_response(response.request, response)+ assert response.request+ new_response = mw.process_response(response.request, response)+ assert isinstance(new_response, Response)+ return new_responsedef test_process_truncated_response_br(self):_skip_if_no_br()diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.pyindex 7ed848764..54d4601a7 100644--- a/tests/test_downloadermiddleware_httpproxy.py+++ b/tests/test_downloadermiddleware_httpproxy.py@@ -14,7 +14,8 @@ class TestHttpProxyMiddleware:self._oldenv = os.environ.copy()def teardown_method(self):- os.environ = self._oldenv+ os.environ.clear()+ os.environ.update(self._oldenv)def test_not_enabled(self):crawler = get_crawler(Spider, {"HTTPPROXY_ENABLED": False})@@ -22,7 +23,8 @@ class TestHttpProxyMiddleware:HttpProxyMiddleware.from_crawler(crawler)def test_no_environment_proxies(self):- os.environ = {"dummy_proxy": "reset_env_and_do_not_raise"}+ os.environ.clear()+ os.environ["dummy_proxy"] = "reset_env_and_do_not_raise"mw = HttpProxyMiddleware()for url in ("http://e.com", "https://e.com", "file:///tmp/a"):diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.pyindex 78efb0191..cb17c2553 100644--- a/tests/test_downloadermiddleware_offsite.py+++ b/tests/test_downloadermiddleware_offsite.py@@ -1,4 +1,5 @@import re+from typing import Anyimport pytest@@ -53,7 +54,7 @@ def test_process_request_dont_filter(value, filtered):crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])mw = OffsiteMiddleware.from_crawler(crawler)mw.spider_opened(crawler.spider)- kwargs = {}+ kwargs: dict[str, Any] = {}if value is not UNSET:kwargs["dont_filter"] = valuerequest = Request("https://b.example", **kwargs)@@ -82,7 +83,7 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])mw = OffsiteMiddleware.from_crawler(crawler)mw.spider_opened(crawler.spider)- kwargs = {"meta": {}}+ kwargs: dict[str, Any] = {"meta": {}}if allow_offsite is not UNSET:kwargs["meta"]["allow_offsite"] = allow_offsiteif dont_filter is not UNSET:@@ -105,7 +106,7 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):)def test_process_request_no_allowed_domains(value):crawler = get_crawler(Spider)- kwargs = {}+ kwargs: dict[str, Any] = {}if value is not UNSET:kwargs["allowed_domains"] = valuecrawler.spider = crawler._create_spider(name="a", **kwargs)@@ -152,7 +153,7 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):mw.spider_opened(crawler.spider)request = Request(url)if allowed:- assert mw.request_scheduled(request, crawler.spider) is None+ mw.request_scheduled(request, crawler.spider)else:with pytest.raises(IgnoreRequest):mw.request_scheduled(request, crawler.spider)@@ -172,7 +173,7 @@ def test_request_scheduled_dont_filter(value, filtered):crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])mw = OffsiteMiddleware.from_crawler(crawler)mw.spider_opened(crawler.spider)- kwargs = {}+ kwargs: dict[str, Any] = {}if value is not UNSET:kwargs["dont_filter"] = valuerequest = Request("https://b.example", **kwargs)@@ -180,7 +181,7 @@ def test_request_scheduled_dont_filter(value, filtered):with pytest.raises(IgnoreRequest):mw.request_scheduled(request, crawler.spider)else:- assert mw.request_scheduled(request, crawler.spider) is None+ mw.request_scheduled(request, crawler.spider)@pytest.mark.parametrize(@@ -193,14 +194,14 @@ def test_request_scheduled_dont_filter(value, filtered):)def test_request_scheduled_no_allowed_domains(value):crawler = get_crawler(Spider)- kwargs = {}+ kwargs: dict[str, Any] = {}if value is not UNSET:kwargs["allowed_domains"] = valuecrawler.spider = crawler._create_spider(name="a", **kwargs)mw = OffsiteMiddleware.from_crawler(crawler)mw.spider_opened(crawler.spider)request = Request("https://example.com")- assert mw.request_scheduled(request, crawler.spider) is None+ mw.request_scheduled(request, crawler.spider)def test_request_scheduled_invalid_domains():@@ -210,7 +211,7 @@ def test_request_scheduled_invalid_domains():mw = OffsiteMiddleware.from_crawler(crawler)mw.spider_opened(crawler.spider)request = Request("https://a.example")- assert mw.request_scheduled(request, crawler.spider) is None+ mw.request_scheduled(request, crawler.spider)for letter in ("b", "c"):request = Request(f"https://{letter}.example")with pytest.raises(IgnoreRequest):@@ -227,6 +228,7 @@ def test_repeated_offsite_domain():with pytest.raises(IgnoreRequest):mw.process_request(req1)assert "other.org" in mw.domains_seen+ assert crawler.statsassert crawler.stats.get_value("offsite/domains") == 1assert crawler.stats.get_value("offsite/filtered") == 1with pytest.raises(IgnoreRequest):diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.pyindex ef2774a93..de97aaadb 100644--- a/tests/test_downloadermiddleware_redirect.py+++ b/tests/test_downloadermiddleware_redirect.py@@ -309,7 +309,7 @@ class TestRedirectMiddleware(TestRedirectBase):url = "http://www.example.com/301"url2 = "http://www.example.com/redirected"- def _test_passthrough(req):+ def _test_passthrough(req: Request) -> None:rsp = Response(url, headers={"Location": url2}, status=301, request=req)r = self.mw.process_response(req, rsp)assert r is rsp@@ -404,15 +404,17 @@ def test_response_referrer_policy(policy, source_url, target_url, expected_referstatus=301,headers={"Location": target_url, **extra_headers},)- source_request = redirect_mw.process_response(source_request, response_redirect)- assert isinstance(source_request, Request)+ target_request = redirect_mw.process_response(source_request, response_redirect)+ assert isinstance(target_request, Request)- assert source_request.headers.get("Referer") == expected_referrer+ assert target_request.headers.get("Referer") == expected_referrerdef test_no_warning_when_referer_middleware_present(caplog):crawler = get_crawler()- crawler.get_spider_middleware = MagicMock(return_value=MagicMock())+ crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]+ return_value=MagicMock()+ )mw = build_from_crawler(RedirectMiddleware, crawler)caplog.clear()with caplog.at_level(logging.WARNING):@@ -426,7 +428,9 @@ def test_no_warning_when_referer_middleware_present(caplog):def test_warning_redirect_middleware(caplog):crawler = get_crawler()- crawler.get_spider_middleware = MagicMock(return_value=None)+ crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]+ return_value=None+ )mw = build_from_crawler(RedirectMiddleware, crawler)with caplog.at_level(logging.WARNING):mw._engine_started()@@ -449,7 +453,9 @@ def test_warning_subclass(caplog):passcrawler = get_crawler()- crawler.get_spider_middleware = MagicMock(return_value=None)+ crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]+ return_value=None+ )mw = build_from_crawler(MyRedirectMiddleware, crawler)with caplog.at_level(logging.WARNING):mw._engine_started()diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.pyindex aeae759a0..83dc6825f 100644--- a/tests/test_downloadermiddleware_redirect_metarefresh.py+++ b/tests/test_downloadermiddleware_redirect_metarefresh.py@@ -21,7 +21,7 @@ from tests.utils.redirect import ()-def meta_refresh_body(url, interval=5):+def meta_refresh_body(url: str, interval: int = 5) -> bytes:html = f"""<html><head><meta http-equiv="refresh" content="{interval};url={url}"/></head></html>"""return html.encode("utf-8")@@ -34,10 +34,14 @@ class TestMetaRefreshMiddleware(TestRedirectBase):crawler = get_crawler(Spider)self.mw = self.mwcls.from_crawler(crawler)- def _body(self, interval=5, url="http://example.org/newpage"):+ def _body(+ self, interval: int = 5, url: str = "http://example.org/newpage"+ ) -> bytes:return meta_refresh_body(url, interval)- def get_response(self, request, location):+ def get_response(+ self, request: Request, location: str, status: int = 302+ ) -> Response:return HtmlResponse(request.url, body=self._body(url=location))def test_meta_refresh(self):@@ -75,7 +79,7 @@ class TestMetaRefreshMiddleware(TestRedirectBase):assert "Content-Length" not in req2.headers, ("Content-Length header must not be present in redirected request")- assert not req2.body, f"Redirected body must be empty, not '{req2.body}'"+ assert not req2.body, f"Redirected body must be empty, not {req2.body!r}"def test_ignore_tags_default(self):req = Request(url="http://example.org")@@ -142,7 +146,9 @@ def test_meta_refresh_schemes(url, location, target):def test_warning_meta_refresh_middleware(caplog):crawler = get_crawler()- crawler.get_spider_middleware = MagicMock(return_value=None)+ crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]+ return_value=None+ )mw = build_from_crawler(MetaRefreshMiddleware, crawler)with caplog.at_level(logging.WARNING):mw._engine_started()diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.pyindex 410427b84..ab52590c7 100644--- a/tests/test_downloadermiddleware_retry.py+++ b/tests/test_downloadermiddleware_retry.py@@ -31,6 +31,7 @@ class TestRetry:req = Request("http://www.scrapytest.org/503")rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)req2 = self.mw.process_response(req, rsp)+ assert isinstance(req2, Request)assert req2.priority < req.prioritydef test_404(self):@@ -53,9 +54,9 @@ class TestRetry:rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)# first retry- req = self.mw.process_response(req, rsp)- assert isinstance(req, Request)- assert req.meta["retry_times"] == 1+ req2 = self.mw.process_response(req, rsp)+ assert isinstance(req2, Request)+ assert req2.meta["retry_times"] == 1def test_dont_retry_exc(self):req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})@@ -68,18 +69,19 @@ class TestRetry:rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)# first retry- req = self.mw.process_response(req, rsp)- assert isinstance(req, Request)- assert req.meta["retry_times"] == 1+ req2 = self.mw.process_response(req, rsp)+ assert isinstance(req2, Request)+ assert req2.meta["retry_times"] == 1# second retry- req = self.mw.process_response(req, rsp)- assert isinstance(req, Request)- assert req.meta["retry_times"] == 2+ req3 = self.mw.process_response(req2, rsp)+ assert isinstance(req3, Request)+ assert req3.meta["retry_times"] == 2# discard it- assert self.mw.process_response(req, rsp) is rsp+ assert self.mw.process_response(req3, rsp) is rsp+ assert self.crawler.statsassert self.crawler.stats.get_value("retry/max_reached") == 1assert (self.crawler.stats.get_value("retry/reason_count/503 Service Unavailable")@@ -131,6 +133,7 @@ class TestRetry:self._test_retry_exception(req, exc("foo"))stats = self.crawler.stats+ assert statsassert stats.get_value("retry/max_reached") == len(exceptions)assert stats.get_value("retry/count") == len(exceptions) * 2assert (@@ -149,29 +152,30 @@ class TestRetry:req = Request(f"http://www.scrapytest.org/{exc.__name__}")self._test_retry_exception(req, exc("foo"), mw)- def _test_retry_exception(self, req, exception, mw=None):+ def _test_retry_exception(+ self, req: Request, exception: Exception, mw: RetryMiddleware | None = None+ ) -> None:if mw is None:mw = self.mw# first retry- req = mw.process_exception(req, exception)- assert isinstance(req, Request)- assert req.meta["retry_times"] == 1+ req2 = mw.process_exception(req, exception)+ assert isinstance(req2, Request)+ assert req2.meta["retry_times"] == 1# second retry- req = mw.process_exception(req, exception)- assert isinstance(req, Request)- assert req.meta["retry_times"] == 2+ req3 = mw.process_exception(req2, exception)+ assert isinstance(req3, Request)+ assert req3.meta["retry_times"] == 2# discard it- req = mw.process_exception(req, exception)- assert req is None+ assert mw.process_exception(req3, exception) is Noneclass TestMaxRetryTimes:invalid_url = "http://www.scrapytest.org/invalid_url"- def get_middleware(self, settings=None):+ def get_middleware(self, settings: dict[str, Any] | None = None) -> RetryMiddleware:crawler = get_crawler(DefaultSpider, settings or {})crawler.spider = crawler._create_spider()return RetryMiddleware.from_crawler(crawler)@@ -275,20 +279,18 @@ class TestMaxRetryTimes:def _test_retry(self,- req,- exception,- max_retry_times,- middleware=None,- ):- middleware = middleware or self.mw-+ req: Request,+ exception: Exception,+ max_retry_times: int,+ middleware: RetryMiddleware,+ ) -> None:for _ in range(max_retry_times):- req = middleware.process_exception(req, exception)- assert isinstance(req, Request)+ result = middleware.process_exception(req, exception)+ assert isinstance(result, Request)+ req = result# discard it- req = middleware.process_exception(req, exception)- assert req is None+ assert middleware.process_exception(req, exception) is Noneclass TestGetRetryRequest:@@ -428,7 +430,7 @@ class TestGetRetryRequest:def test_no_spider(self):request = Request("https://example.com")with pytest.raises(TypeError):- get_retry_request(request) # pylint: disable=missing-kwoa+ get_retry_request(request) # type: ignore[call-arg] # pylint: disable=missing-kwoadef test_max_retry_times_setting(self):max_retry_times = 0@@ -471,6 +473,7 @@ class TestGetRetryRequest:request,spider=spider,)+ assert new_requestassert new_request.priority == priority_adjustdef test_priority_adjust_argument(self):@@ -482,6 +485,7 @@ class TestGetRetryRequest:spider=spider,priority_adjust=priority_adjust,)+ assert new_requestassert new_request.priority == priority_adjustdef test_log_extra_retry_success(self, caplog: pytest.LogCaptureFixture) -> None:@@ -732,6 +736,7 @@ class TestGetRetryRequest:reason=expected_reason,stats_base_key=stats_key,)+ assert spider.crawler.statsfor stat in (f"{stats_key}/count",f"{stats_key}/reason_count/{expected_reason}",diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex 1f2575f8f..793a2b5be 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -1,7 +1,6 @@from __future__ import annotationsimport asyncio-from typing import TYPE_CHECKINGfrom unittest import mockimport pytest@@ -19,9 +18,6 @@ from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_futurefrom tests.utils.decorators import coroutine_testfrom tests.utils.robotstxt import rerp_available-if TYPE_CHECKING:- from scrapy.crawler import Crawler-class TestRobotsTxtMiddleware:def setup_method(self) -> None:@@ -39,7 +35,7 @@ class TestRobotsTxtMiddleware:with pytest.raises(NotConfigured):RobotsTxtMiddleware(self.crawler)- def _get_successful_crawler(self) -> Crawler:+ def _get_successful_crawler(self) -> mock.MagicMock:crawler = self.crawlercrawler.settings.set("ROBOTSTXT_OBEY", True)ROBOTS = """@@ -54,8 +50,8 @@ Disallow: /some/randome/page.html""".encode()response = TextResponse("http://site.local/robots.txt", body=ROBOTS)- async def return_response(request):- deferred = Deferred()+ async def return_response(request: Request) -> Response:+ deferred: Deferred[Response] = Deferred()call_later(0, deferred.callback, response)return await maybe_deferred_to_future(deferred)@@ -130,15 +126,15 @@ Disallow: /some/randome/page.htmlRequest("http://site.local/static/", meta=meta), middleware)- def _get_garbage_crawler(self) -> Crawler:+ def _get_garbage_crawler(self) -> mock.MagicMock:crawler = self.crawlercrawler.settings.set("ROBOTSTXT_OBEY", True)response = Response("http://site.local/robots.txt", body=b"GIF89a\xd3\x00\xfe\x00\xa2")- async def return_response(request):- deferred = Deferred()+ async def return_response(request: Request) -> Response:+ deferred: Deferred[Response] = Deferred()call_later(0, deferred.callback, response)return await maybe_deferred_to_future(deferred)@@ -154,13 +150,13 @@ Disallow: /some/randome/page.htmlawait self.assertNotIgnored(Request("http://site.local/admin/main"), middleware)await self.assertNotIgnored(Request("http://site.local/static/"), middleware)- def _get_emptybody_crawler(self) -> Crawler:+ def _get_emptybody_crawler(self) -> mock.MagicMock:crawler = self.crawlercrawler.settings.set("ROBOTSTXT_OBEY", True)response = Response("http://site.local/robots.txt")- async def return_response(request):- deferred = Deferred()+ async def return_response(request: Request) -> Response:+ deferred: Deferred[Response] = Deferred()call_later(0, deferred.callback, response)return await maybe_deferred_to_future(deferred)@@ -180,8 +176,8 @@ Disallow: /some/randome/page.htmlself.crawler.settings.set("ROBOTSTXT_OBEY", True)err = CannotResolveHostError("Robotstxt address not found")- async def return_failure(request):- deferred = Deferred()+ async def return_failure(request: Request) -> Response:+ deferred: Deferred[Response] = Deferred()call_later(0, deferred.errback, failure.Failure(err))return await maybe_deferred_to_future(deferred)@@ -208,8 +204,8 @@ Disallow: /some/randome/page.htmlasync def test_ignore_robotstxt_request(self):self.crawler.settings.set("ROBOTSTXT_OBEY", True)- async def ignore_request(request):- deferred = Deferred()+ async def ignore_request(request: Request) -> Response:+ deferred: Deferred[Response] = Deferred()call_later(0, deferred.errback, failure.Failure(IgnoreRequest()))return await maybe_deferred_to_future(deferred)@@ -236,7 +232,7 @@ Disallow: /some/randome/page.html@coroutine_testasync def test_robotstxt_local_file(self):middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())- middleware.process_request_2 = mock.MagicMock()+ middleware.process_request_2 = mock.MagicMock() # type: ignore[method-assign]await middleware.process_request(Request("data:text/plain,Hello World data"))assert not middleware.process_request_2.calleddiff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.pyindex ea67877f8..755f29959 100644--- a/tests/test_robotstxt_interface.py+++ b/tests/test_robotstxt_interface.py@@ -1,3 +1,7 @@+from __future__ import annotations++from typing import TYPE_CHECKING+import pytestfrom scrapy.robotstxt import (@@ -10,22 +14,32 @@ from scrapy.robotstxt import (from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSERfrom tests.utils.robotstxt import rerp_available+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self++ from scrapy.crawler import Crawler+class BaseRobotParserTest:- def _setUp(self, parser_cls):+ parser_cls: type[RobotParser]++ def _setUp(self, parser_cls: type[RobotParser]) -> None:self.parser_cls = parser_cls+ def _parse(self, robotstxt_body: bytes) -> RobotParser:+ # The parser backends only use the crawler to get the spider to log with.+ return self.parser_cls.from_crawler(None, robotstxt_body) # type: ignore[arg-type]+def test_allowed(self):robotstxt_robotstxt_body = (b"User-agent: * \nDisallow: /disallowed \nAllow: /allowed \nCrawl-delay: 10")- rp = self.parser_cls.from_crawler(- crawler=None, robotstxt_body=robotstxt_robotstxt_body- )+ rp = self._parse(robotstxt_robotstxt_body)assert rp.allowed("https://www.site.local/allowed", "*")assert not rp.allowed("https://www.site.local/disallowed", "*")- def test_allowed_wildcards(self):+ def test_allowed_wildcards(self) -> None:robotstxt_robotstxt_body = b"""User-agent: firstDisallow: /disallowed/*/end$@@ -33,9 +47,7 @@ class BaseRobotParserTest:Allow: /*allowedDisallow: /"""- rp = self.parser_cls.from_crawler(- crawler=None, robotstxt_body=robotstxt_robotstxt_body- )+ rp = self._parse(robotstxt_robotstxt_body)assert rp.allowed("https://www.site.local/disallowed", "first")assert not rp.allowed("https://www.site.local/disallowed/xyz/end", "first")@@ -46,23 +58,19 @@ class BaseRobotParserTest:assert rp.allowed("https://www.site.local/is_still_allowed", "second")assert rp.allowed("https://www.site.local/is_allowed_too", "second")- def test_length_based_precedence(self):+ def test_length_based_precedence(self) -> None:robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page"- rp = self.parser_cls.from_crawler(- crawler=None, robotstxt_body=robotstxt_robotstxt_body- )+ rp = self._parse(robotstxt_robotstxt_body)assert rp.allowed("https://www.site.local/page", "*")- def test_order_based_precedence(self):+ def test_order_based_precedence(self) -> None:robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page"- rp = self.parser_cls.from_crawler(- crawler=None, robotstxt_body=robotstxt_robotstxt_body- )+ rp = self._parse(robotstxt_robotstxt_body)assert not rp.allowed("https://www.site.local/page", "*")def test_empty_response(self):"""empty response should equal 'allow all'"""- rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=b"")+ rp = self._parse(b"")assert rp.allowed("https://site.local/", "*")assert rp.allowed("https://site.local/", "chrome")assert rp.allowed("https://site.local/index.html", "*")@@ -71,9 +79,7 @@ class BaseRobotParserTest:def test_garbage_response(self):"""garbage response should be discarded, equal 'allow all'"""robotstxt_robotstxt_body = b"GIF89a\xd3\x00\xfe\x00\xa2"- rp = self.parser_cls.from_crawler(- crawler=None, robotstxt_body=robotstxt_robotstxt_body- )+ rp = self._parse(robotstxt_robotstxt_body)assert rp.allowed("https://site.local/", "*")assert rp.allowed("https://site.local/", "chrome")assert rp.allowed("https://site.local/index.html", "*")@@ -81,12 +87,12 @@ class BaseRobotParserTest:def test_crawl_delay(self):robotstxt_body = b"User-agent: *\nDisallow: /private\nCrawl-delay: 10\n"- rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body)+ rp = self._parse(robotstxt_body)assert rp.crawl_delay("*") == 10.0def test_crawl_delay_unset(self):robotstxt_body = b"User-agent: *\nDisallow: /private\n"- rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body)+ rp = self._parse(robotstxt_body)assert rp.crawl_delay("*") is Nonedef test_unicode_url_and_useragent(self):@@ -100,9 +106,7 @@ class BaseRobotParserTest:User-Agent: UnicödeBötDisallow: /some/randome/page.html""".encode()- rp = self.parser_cls.from_crawler(- crawler=None, robotstxt_body=robotstxt_robotstxt_body- )+ rp = self._parse(robotstxt_robotstxt_body)assert rp.allowed("https://site.local/", "*")assert not rp.allowed("https://site.local/admin/", "*")assert not rp.allowed("https://site.local/static/", "*")@@ -117,15 +121,13 @@ class TestRobotParser:def test_crawl_delay_unsupported(self):class AllowAllRobotParser(RobotParser):@classmethod- def from_crawler(cls, crawler, robotstxt_body):+ def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:return cls()- def allowed(self, url, user_agent):+ def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:return True- rp = AllowAllRobotParser.from_crawler(- crawler=None, robotstxt_body=b"User-agent: *\nCrawl-delay: 10\n"- )+ rp = AllowAllRobotParser()assert rp.crawl_delay("*") is None@@ -162,21 +164,21 @@ class TestPythonRobotParser(BaseRobotParserTest):not STDLIB_IMPROVED_ROBOTFILEPARSER,reason="RobotFileParser from this Python version does not support length based directives precedence.",)- def test_length_based_precedence(self):+ def test_length_based_precedence(self) -> None:super().test_length_based_precedence()@pytest.mark.skipif(STDLIB_IMPROVED_ROBOTFILEPARSER,reason="RobotFileParser from this Python version does not support order based directives precedence.",)- def test_order_based_precedence(self):+ def test_order_based_precedence(self) -> None:super().test_order_based_precedence()@pytest.mark.skipif(not STDLIB_IMPROVED_ROBOTFILEPARSER,reason="RobotFileParser from this Python version does not support wildcards.",)- def test_allowed_wildcards(self):+ def test_allowed_wildcards(self) -> None:super().test_allowed_wildcards()@@ -185,7 +187,7 @@ class TestRerpRobotParser(BaseRobotParserTest):def setup_method(self):super()._setUp(RerpRobotParser)- def test_length_based_precedence(self):+ def test_length_based_precedence(self) -> None:pytest.skip("Rerp does not support length based directives precedence.")@@ -193,5 +195,5 @@ class TestProtegoRobotParser(BaseRobotParserTest):def setup_method(self):super()._setUp(ProtegoRobotParser)- def test_order_based_precedence(self):+ def test_order_based_precedence(self) -> None:pytest.skip("Protego does not support order based directives precedence.")