Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add pyupgrade.

Andrey Rakhmatullinfeb0b8f7dcb7
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index a911d4cfe..f76a04ca1 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -22,3 +22,9 @@ repos:
- id: blacken-docs
additional_dependencies:
- black==24.2.0
+- repo: https://github.com/asottile/pyupgrade
+ rev: v3.15.2
+ hooks:
+ - id: pyupgrade
+ args: [--py38-plus, --keep-runtime-typing]
+ exclude: scrapy/__init__.py
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 0e77cd2fe..9f6edb630 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -107,7 +107,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT
return ctx
- def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":
+ def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:
return ScrapyClientTLSOptions(
hostname.decode("ascii"),
self.getContext(),
@@ -134,7 +134,7 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
``SSLv23_METHOD``) which allows TLS protocol negotiation.
"""
- def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":
+ def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:
# trustRoot set to platformTrust() will use the platform's root CAs.
#
# This means that a website like https://www.cacert.org will be rejected
@@ -158,8 +158,8 @@ class AcceptableProtocolsContextFactory:
self._wrapped_context_factory: Any = context_factory
self._acceptable_protocols: List[bytes] = acceptable_protocols
- def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":
- options: "ClientTLSOptions" = self._wrapped_context_factory.creatorForNetloc(
+ def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:
+ options: ClientTLSOptions = self._wrapped_context_factory.creatorForNetloc(
hostname, port
)
_setAcceptableProtocols(options._ctx, self._acceptable_protocols)
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 46587a898..0637f09d4 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -147,9 +147,7 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
if not settings.getbool("RETRY_ENABLED"):
raise NotConfigured
self.max_retry_times = settings.getint("RETRY_TIMES")
- self.retry_http_codes = set(
- int(x) for x in settings.getlist("RETRY_HTTP_CODES")
- )
+ self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")}
self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
try:
diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py
index 26726b662..a0fc7b99f 100644
--- a/scrapy/extensions/debug.py
+++ b/scrapy/extensions/debug.py
@@ -55,7 +55,7 @@ class StackTraceDump:
)
def _thread_stacks(self) -> str:
- id2name = dict((th.ident, th.name) for th in threading.enumerate())
+ id2name = {th.ident: th.name for th in threading.enumerate()}
dumps = ""
for id_, frame in sys._current_frames().items():
name = id2name.get(id_, "")
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 77149333c..3da2e111d 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -189,10 +189,10 @@ class Request(object_ref):
def __repr__(self) -> str:
return f"<{self.method} {self.url}>"
- def copy(self) -> "Request":
+ def copy(self) -> Request:
return self.replace()
- def replace(self, *args: Any, **kwargs: Any) -> "Request":
+ def replace(self, *args: Any, **kwargs: Any) -> Request:
"""Create a new Request with the same attributes except for those given new values"""
for x in self.attributes:
kwargs.setdefault(x, getattr(self, x))
@@ -237,7 +237,7 @@ class Request(object_ref):
request_kwargs.update(kwargs)
return cls(**request_kwargs)
- def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]:
+ def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]:
"""Return a dictionary containing the Request's data.
Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.
diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py
index e20e7c438..096ecd370 100644
--- a/scrapy/http/request/rpc.py
+++ b/scrapy/http/request/rpc.py
@@ -21,7 +21,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps)
class XmlRpcRequest(Request):
def __init__(self, *args: Any, encoding: Optional[str] = None, **kwargs: Any):
if "body" not in kwargs and "params" in kwargs:
- kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs)
+ kw = {k: kwargs.pop(k) for k in DUMPS_ARGS if k in kwargs}
kwargs["body"] = xmlrpclib.dumps(**kw)
# spec defines that requests must use POST method
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 4448b6f4b..ea1db03f1 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -275,7 +275,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
assert isinstance(value, (dict, list))
return copy.deepcopy(value)
- def getwithbase(self, name: _SettingsKeyT) -> "BaseSettings":
+ def getwithbase(self, name: _SettingsKeyT) -> BaseSettings:
"""Get a composition of a dictionary-like setting and its `_BASE`
counterpart.
@@ -438,7 +438,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
if self.frozen:
raise TypeError("Trying to modify an immutable Settings object")
- def copy(self) -> "Self":
+ def copy(self) -> Self:
"""
Make a deep copy of current settings.
@@ -460,7 +460,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
"""
self.frozen = True
- def frozencopy(self) -> "Self":
+ def frozencopy(self) -> Self:
"""
Return an immutable copy of the current settings.
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index e9ea51aa1..550516141 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -22,9 +22,7 @@ class Root(Resource):
for nl in nlist:
args["n"] = nl
argstr = urlencode(args, doseq=True)
- request.write(
- f"<a href='/follow?{argstr}'>follow {nl}</a><br>".encode("utf8")
- )
+ request.write(f"<a href='/follow?{argstr}'>follow {nl}</a><br>".encode())
request.write(b"</body></html>")
return b""
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 42a6537a8..aa0b90ee8 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -49,9 +49,9 @@ def _serialize_headers(
yield from request.headers.getlist(header)
-_fingerprint_cache: (
- "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
-)
+_fingerprint_cache: WeakKeyDictionary[
+ Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]
+]
_fingerprint_cache = WeakKeyDictionary()
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 647b0682e..233f6b934 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -189,10 +189,10 @@ class Raw(LeafResource):
class Echo(LeafResource):
def render_GET(self, request):
output = {
- "headers": dict(
- (to_unicode(k), [to_unicode(v) for v in vs])
+ "headers": {
+ to_unicode(k): [to_unicode(v) for v in vs]
for k, vs in request.requestHeaders.getAllRawHeaders()
- ),
+ },
"body": to_unicode(request.content.read()),
}
return to_bytes(json.dumps(output))
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 5eccd396a..6e343d035 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -362,7 +362,7 @@ class CookiesMiddlewareTest(TestCase):
def test_request_cookies_encoding(self):
# 1) UTF8-encoded bytes
- req1 = Request("http://example.org", cookies={"a": "á".encode("utf8")})
+ req1 = Request("http://example.org", cookies={"a": "á".encode()})
assert self.mw.process_request(req1, self.spider) is None
self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1")
@@ -379,7 +379,7 @@ class CookiesMiddlewareTest(TestCase):
@pytest.mark.xfail(reason="Cookie header is not currently being processed")
def test_request_headers_cookie_encoding(self):
# 1) UTF8-encoded bytes
- req1 = Request("http://example.org", headers={"Cookie": "a=á".encode("utf8")})
+ req1 = Request("http://example.org", headers={"Cookie": "a=á".encode()})
assert self.mw.process_request(req1, self.spider) is None
self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1")
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 4bfd34fe2..e37da9715 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -1125,7 +1125,7 @@ class RedirectMiddlewareTest(Base.Test):
def test_utf8_location(self):
req = Request("http://scrapytest.org/first")
- utf8_location = "/ação".encode("utf-8") # header using UTF-8 encoding
+ utf8_location = "/ação".encode() # header using UTF-8 encoding
resp = Response(
"http://scrapytest.org/first",
headers={"Location": utf8_location},
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 26898a6a1..e166cc000 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -40,9 +40,7 @@ Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
Disallow: /wiki/Käyttäjä:
User-Agent: UnicödeBöt
Disallow: /some/randome/page.html
-""".encode(
- "utf-8"
- )
+""".encode()
response = TextResponse("http://site.local/robots.txt", body=ROBOTS)
def return_response(request):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index d7560b5ff..3771df8f1 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1359,13 +1359,13 @@ class FeedExportTest(FeedExportTestBase):
items = [dict({"foo": "Test\xd6"})]
formats = {
- "json": '[{"foo": "Test\\u00d6"}]'.encode("utf-8"),
- "jsonlines": '{"foo": "Test\\u00d6"}\n'.encode("utf-8"),
+ "json": b'[{"foo": "Test\\u00d6"}]',
+ "jsonlines": b'{"foo": "Test\\u00d6"}\n',
"xml": (
'<?xml version="1.0" encoding="utf-8"?>\n'
"<items><item><foo>Test\xd6</foo></item></items>"
- ).encode("utf-8"),
- "csv": "foo\r\nTest\xd6\r\n".encode("utf-8"),
+ ).encode(),
+ "csv": "foo\r\nTest\xd6\r\n".encode(),
}
for fmt, expected in formats.items():
@@ -1379,13 +1379,13 @@ class FeedExportTest(FeedExportTestBase):
self.assertEqual(expected, data[fmt])
formats = {
- "json": '[{"foo": "Test\xd6"}]'.encode("latin-1"),
- "jsonlines": '{"foo": "Test\xd6"}\n'.encode("latin-1"),
+ "json": b'[{"foo": "Test\xd6"}]',
+ "jsonlines": b'{"foo": "Test\xd6"}\n',
"xml": (
- '<?xml version="1.0" encoding="latin-1"?>\n'
- "<items><item><foo>Test\xd6</foo></item></items>"
- ).encode("latin-1"),
- "csv": "foo\r\nTest\xd6\r\n".encode("latin-1"),
+ b'<?xml version="1.0" encoding="latin-1"?>\n'
+ b"<items><item><foo>Test\xd6</foo></item></items>"
+ ),
+ "csv": b"foo\r\nTest\xd6\r\n",
}
for fmt, expected in formats.items():
@@ -1404,12 +1404,12 @@ class FeedExportTest(FeedExportTestBase):
items = [dict({"foo": "FOO", "bar": "BAR"})]
formats = {
- "json": '[\n{"bar": "BAR"}\n]'.encode("utf-8"),
+ "json": b'[\n{"bar": "BAR"}\n]',
"xml": (
- '<?xml version="1.0" encoding="latin-1"?>\n'
- "<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>"
- ).encode("latin-1"),
- "csv": "bar,foo\r\nBAR,FOO\r\n".encode("utf-8"),
+ b'<?xml version="1.0" encoding="latin-1"?>\n'
+ b"<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>"
+ ),
+ "csv": b"bar,foo\r\nBAR,FOO\r\n",
}
settings = {
@@ -1663,8 +1663,8 @@ class FeedExportTest(FeedExportTestBase):
def test_extend_kwargs(self):
items = [{"foo": "FOO", "bar": "BAR"}]
- expected_with_title_csv = "foo,bar\r\nFOO,BAR\r\n".encode("utf-8")
- expected_without_title_csv = "FOO,BAR\r\n".encode("utf-8")
+ expected_with_title_csv = b"foo,bar\r\nFOO,BAR\r\n"
+ expected_without_title_csv = b"FOO,BAR\r\n"
test_cases = [
# with title
{
@@ -2519,22 +2519,22 @@ class BatchDeliveriesTest(FeedExportTestBase):
formats = {
"json": [
- '[\n{"bar": "BAR"}\n]'.encode("utf-8"),
- '[\n{"bar": "BAR1"}\n]'.encode("utf-8"),
+ b'[\n{"bar": "BAR"}\n]',
+ b'[\n{"bar": "BAR1"}\n]',
],
"xml": [
(
- '<?xml version="1.0" encoding="latin-1"?>\n'
- "<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>"
- ).encode("latin-1"),
+ b'<?xml version="1.0" encoding="latin-1"?>\n'
+ b"<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>"
+ ),
(
- '<?xml version="1.0" encoding="latin-1"?>\n'
- "<items>\n <item>\n <foo>FOO1</foo>\n </item>\n</items>"
- ).encode("latin-1"),
+ b'<?xml version="1.0" encoding="latin-1"?>\n'
+ b"<items>\n <item>\n <foo>FOO1</foo>\n </item>\n</items>"
+ ),
],
"csv": [
- "foo,bar\r\nFOO,BAR\r\n".encode("utf-8"),
- "foo,bar\r\nFOO1,BAR1\r\n".encode("utf-8"),
+ b"foo,bar\r\nFOO,BAR\r\n",
+ b"foo,bar\r\nFOO1,BAR1\r\n",
],
}
@@ -2577,8 +2577,8 @@ class BatchDeliveriesTest(FeedExportTestBase):
items = [dict({"foo": "FOO"}), dict({"foo": "FOO1"})]
formats = {
"json": [
- '[{"foo": "FOO"}]'.encode("utf-8"),
- '[{"foo": "FOO1"}]'.encode("utf-8"),
+ b'[{"foo": "FOO"}]',
+ b'[{"foo": "FOO1"}]',
],
}
settings = {
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 80d46274b..b8a277295 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -728,9 +728,7 @@ class TextResponseTest(BaseResponseTest):
resp1 = self.response_class(
"http://example.com",
encoding="utf8",
- body='<html><body><a href="foo?привет">click me</a></body></html>'.encode(
- "utf8"
- ),
+ body='<html><body><a href="foo?привет">click me</a></body></html>'.encode(),
)
req = self._assert_followed_url(
resp1.css("a")[0],
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 5a9a217ce..cd3442dd4 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -107,9 +107,7 @@ class FileDownloadCrawlTestCase(TestCase):
# check that the images/files checksums are what we know they should be
if self.expected_checksums is not None:
- checksums = set(
- i["checksum"] for item in items for i in item[self.media_key]
- )
+ checksums = {i["checksum"] for item in items for i in item[self.media_key]}
self.assertEqual(checksums, self.expected_checksums)
# check that the image files where actually written to the media store
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 18a2454b3..7d7c78920 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -628,7 +628,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
class NoimagesDropTestCase(unittest.TestCase):
def test_deprecation_warning(self):
- arg = str()
+ arg = ""
with warnings.catch_warnings(record=True) as w:
NoimagesDrop(arg)
self.assertEqual(len(w), 1)
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 2633cca5b..7be8150fc 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -29,7 +29,7 @@ class ResponseTypesTest(unittest.TestCase):
mappings = [
(b'attachment; filename="data.xml"', XmlResponse),
(b"attachment; filename=data.xml", XmlResponse),
- ("attachment;filename=data£.tar.gz".encode("utf-8"), Response),
+ ("attachment;filename=data£.tar.gz".encode(), Response),
("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response),
("attachment;filename=data高.doc".encode("gbk"), Response),
("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse),
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 6ad30deed..28ad910a8 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -36,10 +36,10 @@ class BaseRobotParserTest:
def test_allowed(self):
robotstxt_robotstxt_body = (
- "User-agent: * \n"
- "Disallow: /disallowed \n"
- "Allow: /allowed \n"
- "Crawl-delay: 10".encode("utf-8")
+ b"User-agent: * \n"
+ b"Disallow: /disallowed \n"
+ b"Allow: /allowed \n"
+ b"Crawl-delay: 10"
)
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
@@ -48,15 +48,13 @@ class BaseRobotParserTest:
self.assertFalse(rp.allowed("https://www.site.local/disallowed", "*"))
def test_allowed_wildcards(self):
- robotstxt_robotstxt_body = """User-agent: first
+ robotstxt_robotstxt_body = b"""User-agent: first
Disallow: /disallowed/*/end$
User-agent: second
Allow: /*allowed
Disallow: /
- """.encode(
- "utf-8"
- )
+ """
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
@@ -77,18 +75,14 @@ class BaseRobotParserTest:
self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second"))
def test_length_based_precedence(self):
- robotstxt_robotstxt_body = (
- "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8")
- )
+ robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page"
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
self.assertTrue(rp.allowed("https://www.site.local/page", "*"))
def test_order_based_precedence(self):
- robotstxt_robotstxt_body = (
- "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8")
- )
+ robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page"
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
@@ -123,9 +117,7 @@ class BaseRobotParserTest:
Disallow: /wiki/Käyttäjä:
User-Agent: UnicödeBöt
- Disallow: /some/randome/page.html""".encode(
- "utf-8"
- )
+ Disallow: /some/randome/page.html""".encode()
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
@@ -145,14 +137,14 @@ class BaseRobotParserTest:
class DecodeRobotsTxtTest(unittest.TestCase):
def test_native_string_conversion(self):
- robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8")
+ robotstxt_body = b"User-agent: *\nDisallow: /\n"
decoded_content = decode_robotstxt(
robotstxt_body, spider=None, to_native_str_type=True
)
self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")
def test_decode_utf8(self):
- robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8")
+ robotstxt_body = b"User-agent: *\nDisallow: /\n"
decoded_content = decode_robotstxt(robotstxt_body, spider=None)
self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")