Scrapy, a fast high-level web crawling & scraping framework for Python.
Add pyupgrade.
Andrey Rakhmatullinfeb0b8f7dcb7
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex a911d4cfe..f76a04ca1 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -22,3 +22,9 @@ repos:- id: blacken-docsadditional_dependencies:- black==24.2.0+- repo: https://github.com/asottile/pyupgrade+ rev: v3.15.2+ hooks:+ - id: pyupgrade+ args: [--py38-plus, --keep-runtime-typing]+ exclude: scrapy/__init__.pydiff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.pyindex 0e77cd2fe..9f6edb630 100644--- a/scrapy/core/downloader/contextfactory.py+++ b/scrapy/core/downloader/contextfactory.py@@ -107,7 +107,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECTreturn ctx- def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":+ def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:return ScrapyClientTLSOptions(hostname.decode("ascii"),self.getContext(),@@ -134,7 +134,7 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):``SSLv23_METHOD``) which allows TLS protocol negotiation."""- def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":+ def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:# trustRoot set to platformTrust() will use the platform's root CAs.## This means that a website like https://www.cacert.org will be rejected@@ -158,8 +158,8 @@ class AcceptableProtocolsContextFactory:self._wrapped_context_factory: Any = context_factoryself._acceptable_protocols: List[bytes] = acceptable_protocols- def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":- options: "ClientTLSOptions" = self._wrapped_context_factory.creatorForNetloc(+ def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:+ options: ClientTLSOptions = self._wrapped_context_factory.creatorForNetloc(hostname, port)_setAcceptableProtocols(options._ctx, self._acceptable_protocols)diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.pyindex 46587a898..0637f09d4 100644--- a/scrapy/downloadermiddlewares/retry.py+++ b/scrapy/downloadermiddlewares/retry.py@@ -147,9 +147,7 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):if not settings.getbool("RETRY_ENABLED"):raise NotConfiguredself.max_retry_times = settings.getint("RETRY_TIMES")- self.retry_http_codes = set(- int(x) for x in settings.getlist("RETRY_HTTP_CODES")- )+ self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")}self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")try:diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.pyindex 26726b662..a0fc7b99f 100644--- a/scrapy/extensions/debug.py+++ b/scrapy/extensions/debug.py@@ -55,7 +55,7 @@ class StackTraceDump:)def _thread_stacks(self) -> str:- id2name = dict((th.ident, th.name) for th in threading.enumerate())+ id2name = {th.ident: th.name for th in threading.enumerate()}dumps = ""for id_, frame in sys._current_frames().items():name = id2name.get(id_, "")diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.pyindex 77149333c..3da2e111d 100644--- a/scrapy/http/request/__init__.py+++ b/scrapy/http/request/__init__.py@@ -189,10 +189,10 @@ class Request(object_ref):def __repr__(self) -> str:return f"<{self.method} {self.url}>"- def copy(self) -> "Request":+ def copy(self) -> Request:return self.replace()- def replace(self, *args: Any, **kwargs: Any) -> "Request":+ def replace(self, *args: Any, **kwargs: Any) -> Request:"""Create a new Request with the same attributes except for those given new values"""for x in self.attributes:kwargs.setdefault(x, getattr(self, x))@@ -237,7 +237,7 @@ class Request(object_ref):request_kwargs.update(kwargs)return cls(**request_kwargs)- def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]:+ def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]:"""Return a dictionary containing the Request's data.Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.pyindex e20e7c438..096ecd370 100644--- a/scrapy/http/request/rpc.py+++ b/scrapy/http/request/rpc.py@@ -21,7 +21,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps)class XmlRpcRequest(Request):def __init__(self, *args: Any, encoding: Optional[str] = None, **kwargs: Any):if "body" not in kwargs and "params" in kwargs:- kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs)+ kw = {k: kwargs.pop(k) for k in DUMPS_ARGS if k in kwargs}kwargs["body"] = xmlrpclib.dumps(**kw)# spec defines that requests must use POST methoddiff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.pyindex 4448b6f4b..ea1db03f1 100644--- a/scrapy/settings/__init__.py+++ b/scrapy/settings/__init__.py@@ -275,7 +275,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):assert isinstance(value, (dict, list))return copy.deepcopy(value)- def getwithbase(self, name: _SettingsKeyT) -> "BaseSettings":+ def getwithbase(self, name: _SettingsKeyT) -> BaseSettings:"""Get a composition of a dictionary-like setting and its `_BASE`counterpart.@@ -438,7 +438,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):if self.frozen:raise TypeError("Trying to modify an immutable Settings object")- def copy(self) -> "Self":+ def copy(self) -> Self:"""Make a deep copy of current settings.@@ -460,7 +460,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):"""self.frozen = True- def frozencopy(self) -> "Self":+ def frozencopy(self) -> Self:"""Return an immutable copy of the current settings.diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.pyindex e9ea51aa1..550516141 100644--- a/scrapy/utils/benchserver.py+++ b/scrapy/utils/benchserver.py@@ -22,9 +22,7 @@ class Root(Resource):for nl in nlist:args["n"] = nlargstr = urlencode(args, doseq=True)- request.write(- f"<a href='/follow?{argstr}'>follow {nl}</a><br>".encode("utf8")- )+ request.write(f"<a href='/follow?{argstr}'>follow {nl}</a><br>".encode())request.write(b"</body></html>")return b""diff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex 42a6537a8..aa0b90ee8 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -49,9 +49,9 @@ def _serialize_headers(yield from request.headers.getlist(header)-_fingerprint_cache: (- "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"-)+_fingerprint_cache: WeakKeyDictionary[+ Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]+]_fingerprint_cache = WeakKeyDictionary()diff --git a/tests/mockserver.py b/tests/mockserver.pyindex 647b0682e..233f6b934 100644--- a/tests/mockserver.py+++ b/tests/mockserver.py@@ -189,10 +189,10 @@ class Raw(LeafResource):class Echo(LeafResource):def render_GET(self, request):output = {- "headers": dict(- (to_unicode(k), [to_unicode(v) for v in vs])+ "headers": {+ to_unicode(k): [to_unicode(v) for v in vs]for k, vs in request.requestHeaders.getAllRawHeaders()- ),+ },"body": to_unicode(request.content.read()),}return to_bytes(json.dumps(output))diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.pyindex 5eccd396a..6e343d035 100644--- a/tests/test_downloadermiddleware_cookies.py+++ b/tests/test_downloadermiddleware_cookies.py@@ -362,7 +362,7 @@ class CookiesMiddlewareTest(TestCase):def test_request_cookies_encoding(self):# 1) UTF8-encoded bytes- req1 = Request("http://example.org", cookies={"a": "á".encode("utf8")})+ req1 = Request("http://example.org", cookies={"a": "á".encode()})assert self.mw.process_request(req1, self.spider) is Noneself.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1")@@ -379,7 +379,7 @@ class CookiesMiddlewareTest(TestCase):@pytest.mark.xfail(reason="Cookie header is not currently being processed")def test_request_headers_cookie_encoding(self):# 1) UTF8-encoded bytes- req1 = Request("http://example.org", headers={"Cookie": "a=á".encode("utf8")})+ req1 = Request("http://example.org", headers={"Cookie": "a=á".encode()})assert self.mw.process_request(req1, self.spider) is Noneself.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1")diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.pyindex 4bfd34fe2..e37da9715 100644--- a/tests/test_downloadermiddleware_redirect.py+++ b/tests/test_downloadermiddleware_redirect.py@@ -1125,7 +1125,7 @@ class RedirectMiddlewareTest(Base.Test):def test_utf8_location(self):req = Request("http://scrapytest.org/first")- utf8_location = "/ação".encode("utf-8") # header using UTF-8 encoding+ utf8_location = "/ação".encode() # header using UTF-8 encodingresp = Response("http://scrapytest.org/first",headers={"Location": utf8_location},diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex 26898a6a1..e166cc000 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -40,9 +40,7 @@ Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:Disallow: /wiki/Käyttäjä:User-Agent: UnicödeBötDisallow: /some/randome/page.html-""".encode(- "utf-8"- )+""".encode()response = TextResponse("http://site.local/robots.txt", body=ROBOTS)def return_response(request):diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex d7560b5ff..3771df8f1 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -1359,13 +1359,13 @@ class FeedExportTest(FeedExportTestBase):items = [dict({"foo": "Test\xd6"})]formats = {- "json": '[{"foo": "Test\\u00d6"}]'.encode("utf-8"),- "jsonlines": '{"foo": "Test\\u00d6"}\n'.encode("utf-8"),+ "json": b'[{"foo": "Test\\u00d6"}]',+ "jsonlines": b'{"foo": "Test\\u00d6"}\n',"xml": ('<?xml version="1.0" encoding="utf-8"?>\n'"<items><item><foo>Test\xd6</foo></item></items>"- ).encode("utf-8"),- "csv": "foo\r\nTest\xd6\r\n".encode("utf-8"),+ ).encode(),+ "csv": "foo\r\nTest\xd6\r\n".encode(),}for fmt, expected in formats.items():@@ -1379,13 +1379,13 @@ class FeedExportTest(FeedExportTestBase):self.assertEqual(expected, data[fmt])formats = {- "json": '[{"foo": "Test\xd6"}]'.encode("latin-1"),- "jsonlines": '{"foo": "Test\xd6"}\n'.encode("latin-1"),+ "json": b'[{"foo": "Test\xd6"}]',+ "jsonlines": b'{"foo": "Test\xd6"}\n',"xml": (- '<?xml version="1.0" encoding="latin-1"?>\n'- "<items><item><foo>Test\xd6</foo></item></items>"- ).encode("latin-1"),- "csv": "foo\r\nTest\xd6\r\n".encode("latin-1"),+ b'<?xml version="1.0" encoding="latin-1"?>\n'+ b"<items><item><foo>Test\xd6</foo></item></items>"+ ),+ "csv": b"foo\r\nTest\xd6\r\n",}for fmt, expected in formats.items():@@ -1404,12 +1404,12 @@ class FeedExportTest(FeedExportTestBase):items = [dict({"foo": "FOO", "bar": "BAR"})]formats = {- "json": '[\n{"bar": "BAR"}\n]'.encode("utf-8"),+ "json": b'[\n{"bar": "BAR"}\n]',"xml": (- '<?xml version="1.0" encoding="latin-1"?>\n'- "<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>"- ).encode("latin-1"),- "csv": "bar,foo\r\nBAR,FOO\r\n".encode("utf-8"),+ b'<?xml version="1.0" encoding="latin-1"?>\n'+ b"<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>"+ ),+ "csv": b"bar,foo\r\nBAR,FOO\r\n",}settings = {@@ -1663,8 +1663,8 @@ class FeedExportTest(FeedExportTestBase):def test_extend_kwargs(self):items = [{"foo": "FOO", "bar": "BAR"}]- expected_with_title_csv = "foo,bar\r\nFOO,BAR\r\n".encode("utf-8")- expected_without_title_csv = "FOO,BAR\r\n".encode("utf-8")+ expected_with_title_csv = b"foo,bar\r\nFOO,BAR\r\n"+ expected_without_title_csv = b"FOO,BAR\r\n"test_cases = [# with title{@@ -2519,22 +2519,22 @@ class BatchDeliveriesTest(FeedExportTestBase):formats = {"json": [- '[\n{"bar": "BAR"}\n]'.encode("utf-8"),- '[\n{"bar": "BAR1"}\n]'.encode("utf-8"),+ b'[\n{"bar": "BAR"}\n]',+ b'[\n{"bar": "BAR1"}\n]',],"xml": [(- '<?xml version="1.0" encoding="latin-1"?>\n'- "<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>"- ).encode("latin-1"),+ b'<?xml version="1.0" encoding="latin-1"?>\n'+ b"<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>"+ ),(- '<?xml version="1.0" encoding="latin-1"?>\n'- "<items>\n <item>\n <foo>FOO1</foo>\n </item>\n</items>"- ).encode("latin-1"),+ b'<?xml version="1.0" encoding="latin-1"?>\n'+ b"<items>\n <item>\n <foo>FOO1</foo>\n </item>\n</items>"+ ),],"csv": [- "foo,bar\r\nFOO,BAR\r\n".encode("utf-8"),- "foo,bar\r\nFOO1,BAR1\r\n".encode("utf-8"),+ b"foo,bar\r\nFOO,BAR\r\n",+ b"foo,bar\r\nFOO1,BAR1\r\n",],}@@ -2577,8 +2577,8 @@ class BatchDeliveriesTest(FeedExportTestBase):items = [dict({"foo": "FOO"}), dict({"foo": "FOO1"})]formats = {"json": [- '[{"foo": "FOO"}]'.encode("utf-8"),- '[{"foo": "FOO1"}]'.encode("utf-8"),+ b'[{"foo": "FOO"}]',+ b'[{"foo": "FOO1"}]',],}settings = {diff --git a/tests/test_http_response.py b/tests/test_http_response.pyindex 80d46274b..b8a277295 100644--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@ -728,9 +728,7 @@ class TextResponseTest(BaseResponseTest):resp1 = self.response_class("http://example.com",encoding="utf8",- body='<html><body><a href="foo?привет">click me</a></body></html>'.encode(- "utf8"- ),+ body='<html><body><a href="foo?привет">click me</a></body></html>'.encode(),)req = self._assert_followed_url(resp1.css("a")[0],diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.pyindex 5a9a217ce..cd3442dd4 100644--- a/tests/test_pipeline_crawl.py+++ b/tests/test_pipeline_crawl.py@@ -107,9 +107,7 @@ class FileDownloadCrawlTestCase(TestCase):# check that the images/files checksums are what we know they should beif self.expected_checksums is not None:- checksums = set(- i["checksum"] for item in items for i in item[self.media_key]- )+ checksums = {i["checksum"] for item in items for i in item[self.media_key]}self.assertEqual(checksums, self.expected_checksums)# check that the image files where actually written to the media storediff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex 18a2454b3..7d7c78920 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -628,7 +628,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):class NoimagesDropTestCase(unittest.TestCase):def test_deprecation_warning(self):- arg = str()+ arg = ""with warnings.catch_warnings(record=True) as w:NoimagesDrop(arg)self.assertEqual(len(w), 1)diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.pyindex 2633cca5b..7be8150fc 100644--- a/tests/test_responsetypes.py+++ b/tests/test_responsetypes.py@@ -29,7 +29,7 @@ class ResponseTypesTest(unittest.TestCase):mappings = [(b'attachment; filename="data.xml"', XmlResponse),(b"attachment; filename=data.xml", XmlResponse),- ("attachment;filename=data£.tar.gz".encode("utf-8"), Response),+ ("attachment;filename=data£.tar.gz".encode(), Response),("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response),("attachment;filename=data高.doc".encode("gbk"), Response),("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse),diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.pyindex 6ad30deed..28ad910a8 100644--- a/tests/test_robotstxt_interface.py+++ b/tests/test_robotstxt_interface.py@@ -36,10 +36,10 @@ class BaseRobotParserTest:def test_allowed(self):robotstxt_robotstxt_body = (- "User-agent: * \n"- "Disallow: /disallowed \n"- "Allow: /allowed \n"- "Crawl-delay: 10".encode("utf-8")+ b"User-agent: * \n"+ b"Disallow: /disallowed \n"+ b"Allow: /allowed \n"+ b"Crawl-delay: 10")rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body@@ -48,15 +48,13 @@ class BaseRobotParserTest:self.assertFalse(rp.allowed("https://www.site.local/disallowed", "*"))def test_allowed_wildcards(self):- robotstxt_robotstxt_body = """User-agent: first+ robotstxt_robotstxt_body = b"""User-agent: firstDisallow: /disallowed/*/end$User-agent: secondAllow: /*allowedDisallow: /- """.encode(- "utf-8"- )+ """rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)@@ -77,18 +75,14 @@ class BaseRobotParserTest:self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second"))def test_length_based_precedence(self):- robotstxt_robotstxt_body = (- "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8")- )+ robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page"rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)self.assertTrue(rp.allowed("https://www.site.local/page", "*"))def test_order_based_precedence(self):- robotstxt_robotstxt_body = (- "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8")- )+ robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page"rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)@@ -123,9 +117,7 @@ class BaseRobotParserTest:Disallow: /wiki/Käyttäjä:User-Agent: UnicödeBöt- Disallow: /some/randome/page.html""".encode(- "utf-8"- )+ Disallow: /some/randome/page.html""".encode()rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)@@ -145,14 +137,14 @@ class BaseRobotParserTest:class DecodeRobotsTxtTest(unittest.TestCase):def test_native_string_conversion(self):- robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8")+ robotstxt_body = b"User-agent: *\nDisallow: /\n"decoded_content = decode_robotstxt(robotstxt_body, spider=None, to_native_str_type=True)self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")def test_decode_utf8(self):- robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8")+ robotstxt_body = b"User-agent: *\nDisallow: /\n"decoded_content = decode_robotstxt(robotstxt_body, spider=None)self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")