Scrapy, a fast high-level web crawling & scraping framework for Python.
Use f-strings where appropriate (#5246)
Georgiy Zatserklianyid3f1bf79e883
diff --git a/scrapy/__init__.py b/scrapy/__init__.pyindex 8a8065bf2..396f98219 100644--- a/scrapy/__init__.py+++ b/scrapy/__init__.py@@ -29,7 +29,7 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)# Check minimum required Python versionif sys.version_info < (3, 6):- print("Scrapy %s requires Python 3.6+" % __version__)+ print(f"Scrapy {__version__} requires Python 3.6+")sys.exit(1)diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.pyindex 073ef16bf..b5318c7bb 100644--- a/scrapy/core/downloader/contextfactory.py+++ b/scrapy/core/downloader/contextfactory.py@@ -135,11 +135,13 @@ def load_context_factory_from_settings(settings, crawler):settings=settings,crawler=crawler,)- msg = """- '%s' does not accept `method` argument (type OpenSSL.SSL method,\- e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\- Please upgrade your context factory class to handle them or ignore them.""" % (- settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)+ msg = (+ f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept "+ "a `method` argument (type OpenSSL.SSL method, e.g. "+ "OpenSSL.SSL.SSLv23_METHOD) and/or a `tls_verbose_logging` "+ "argument and/or a `tls_ciphers` argument. Please, upgrade your "+ "context factory class to handle them or ignore them."+ )warnings.warn(msg)return context_factorydiff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.pyindex 2b8990b75..19a56d9b6 100644--- a/scrapy/core/downloader/tls.py+++ b/scrapy/core/downloader/tls.py@@ -65,14 +65,14 @@ class ScrapyClientTLSOptions(ClientTLSOptions):verifyHostname(connection, self._hostnameASCII)except (CertificateError, VerificationError) as e:logger.warning(- 'Remote certificate is not valid for hostname "{}"; {}'.format(- self._hostnameASCII, e))+ 'Remote certificate is not valid for hostname "%s"; %s',+ self._hostnameASCII, e)except ValueError as e:logger.warning('Ignoring error while verifying certificate '- 'from host "{}" (exception: {})'.format(- self._hostnameASCII, repr(e)))+ 'from host "%s" (exception: %r)',+ self._hostnameASCII, e)DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.pyindex 62f1c3a29..80ed7ac75 100644--- a/scrapy/downloadermiddlewares/httpcache.py+++ b/scrapy/downloadermiddlewares/httpcache.py@@ -70,7 +70,7 @@ class HttpCacheMiddleware:self.stats.inc_value('httpcache/miss', spider=spider)if self.ignore_missing:self.stats.inc_value('httpcache/ignore', spider=spider)- raise IgnoreRequest("Ignored request not in cache: %s" % request)+ raise IgnoreRequest(f"Ignored request not in cache: {request}")return None # first time request# Return cached response only if not expireddiff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 0f5bf01d0..370723368 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -38,11 +38,10 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):kwargs['feed_options'] = feed_optionselse:warnings.warn(- "{} does not support the 'feed_options' keyword argument. Add a "+ f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a ""'feed_options' parameter to its signature to remove this ""warning. This parameter will become mandatory in a future "- "version of Scrapy."- .format(builder.__qualname__),+ "version of Scrapy.",category=ScrapyDeprecationWarning)return builder(*preargs, uri, *args, **kwargs)@@ -356,32 +355,28 @@ class FeedExporter:# properly closed.return defer.maybeDeferred(slot.storage.store, slot.file)slot.finish_exporting()- logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"- log_args = {'format': slot.format,- 'itemcount': slot.itemcount,- 'uri': slot.uri}+ logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"d = defer.maybeDeferred(slot.storage.store, slot.file)- # Use `largs=log_args` to copy log_args into function's scope- # instead of using `log_args` from the outer scoped.addCallback(- self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__+ self._handle_store_success, logmsg, spider, type(slot.storage).__name__)d.addErrback(- self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__+ self._handle_store_error, logmsg, spider, type(slot.storage).__name__)return d- def _handle_store_error(self, f, largs, logfmt, spider, slot_type):+ def _handle_store_error(self, f, logmsg, spider, slot_type):logger.error(- logfmt % "Error storing", largs,+ "Error storing %s", logmsg,exc_info=failure_to_exc_info(f), extra={'spider': spider})self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")- def _handle_store_success(self, f, largs, logfmt, spider, slot_type):+ def _handle_store_success(self, f, logmsg, spider, slot_type):logger.info(- logfmt % "Stored", largs, extra={'spider': spider}+ "Stored %s", logmsg,+ extra={'spider': spider})self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}")@@ -474,10 +469,10 @@ class FeedExporter:for uri_template, values in self.feeds.items():if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):logger.error(- '%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT '+ '%%(batch_time)s or %%(batch_id)d must be in the feed URI (%s) if FEED_EXPORT_BATCH_ITEM_COUNT ''setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: '- 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'- ''.format(uri_template)+ 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count',+ uri_template)return Falsereturn True@@ -526,7 +521,7 @@ class FeedExporter:instance = build_instance(feedcls)method_name = '__new__'if instance is None:- raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name))+ raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None")return instancedef _get_uri_params(self, spider, uri_params, slot=None):diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.pyindex e0c04b2de..d0ae29b90 100644--- a/scrapy/extensions/httpcache.py+++ b/scrapy/extensions/httpcache.py@@ -226,7 +226,7 @@ class DbmCacheStorage:dbpath = os.path.join(self.cachedir, f'{spider.name}.db')self.db = self.dbmodule.open(dbpath, 'c')- logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})+ logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})def close_spider(self, spider):self.db.close()@@ -280,7 +280,7 @@ class FilesystemCacheStorage:self._open = gzip.open if self.use_gzip else opendef open_spider(self, spider):- logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir},+ logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},extra={'spider': spider})def close_spider(self, spider):diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex 51cef1e91..11c4206c2 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -50,7 +50,7 @@ def load_object(path):return pathelse:raise TypeError("Unexpected argument type, expected string "- "or object, got: %s" % type(path))+ f"or object, got: {type(path)}")try:dot = path.rindex('.')diff --git a/tests/spiders.py b/tests/spiders.pyindex 5b45f897e..67dbbbe0f 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -86,7 +86,7 @@ class SimpleSpider(MetaSpider):self.start_urls = [url]def parse(self, response):- self.logger.info("Got response %d" % response.status)+ self.logger.info(f"Got response {response.status}")class AsyncDefSpider(SimpleSpider):@@ -95,7 +95,7 @@ class AsyncDefSpider(SimpleSpider):async def parse(self, response):await defer.succeed(42)- self.logger.info("Got response %d" % response.status)+ self.logger.info(f"Got response {response.status}")class AsyncDefAsyncioSpider(SimpleSpider):@@ -105,7 +105,7 @@ class AsyncDefAsyncioSpider(SimpleSpider):async def parse(self, response):await asyncio.sleep(0.2)status = await get_from_asyncio_queue(response.status)- self.logger.info("Got response %d" % status)+ self.logger.info(f"Got response {status}")class AsyncDefAsyncioReturnSpider(SimpleSpider):@@ -115,7 +115,7 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider):async def parse(self, response):await asyncio.sleep(0.2)status = await get_from_asyncio_queue(response.status)- self.logger.info("Got response %d" % status)+ self.logger.info(f"Got response {status}")return [{'id': 1}, {'id': 2}]@@ -126,7 +126,7 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):async def parse(self, response):await asyncio.sleep(0.1)status = await get_from_asyncio_queue(response.status)- self.logger.info("Got response %d" % status)+ self.logger.info(f"Got response {status}")return {"foo": 42}@@ -138,7 +138,7 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):await asyncio.sleep(0.2)req_id = response.meta.get('req_id', 0)status = await get_from_asyncio_queue(response.status)- self.logger.info("Got response %d, req_id %d" % (status, req_id))+ self.logger.info(f"Got response {status}, req_id {req_id}")if req_id > 0:returnreqs = []@@ -155,7 +155,7 @@ class AsyncDefAsyncioGenSpider(SimpleSpider):async def parse(self, response):await asyncio.sleep(0.2)yield {'foo': 42}- self.logger.info("Got response %d" % response.status)+ self.logger.info(f"Got response {response.status}")class AsyncDefAsyncioGenLoopSpider(SimpleSpider):@@ -166,7 +166,7 @@ class AsyncDefAsyncioGenLoopSpider(SimpleSpider):for i in range(10):await asyncio.sleep(0.1)yield {'foo': i}- self.logger.info("Got response %d" % response.status)+ self.logger.info(f"Got response {response.status}")class AsyncDefAsyncioGenComplexSpider(SimpleSpider):diff --git a/tests/test_closespider.py b/tests/test_closespider.pyindex 5ec5e2989..be8adadb3 100644--- a/tests/test_closespider.py+++ b/tests/test_closespider.py@@ -41,7 +41,7 @@ class TestCloseSpider(TestCase):yield crawler.crawl(total=1000000, mockserver=self.mockserver)reason = crawler.spider.meta['close_reason']self.assertEqual(reason, 'closespider_errorcount')- key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__)+ key = f'spider_exceptions/{crawler.spider.exception_cls.__name__}'errorcount = crawler.stats.get_value(key)self.assertTrue(errorcount >= close_on)diff --git a/tests/test_commands.py b/tests/test_commands.pyindex 086286b3a..75098a77a 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -498,7 +498,7 @@ class GenspiderCommandTest(CommandTest):self.find_in_file(join(self.proj_mod_path,'spiders', 'test_name.py'),r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1))- self.assertEqual('http://%s/' % domain,+ self.assertEqual(f'http://{domain}/',self.find_in_file(join(self.proj_mod_path,'spiders', 'test_name.py'),r'start_urls\s*=\s*\[\'(.+)\'\]').group(1))@@ -708,7 +708,7 @@ class MySpider(scrapy.Spider):])import asyncioloop = asyncio.new_event_loop()- self.assertIn("Using asyncio event loop: %s.%s" % (loop.__module__, loop.__class__.__name__), log)+ self.assertIn(f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", log)def test_output(self):spider_code = """diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.pyindex 8c8c30597..3a9db3ee5 100644--- a/tests/test_downloader_handlers_http2.py+++ b/tests/test_downloader_handlers_http2.py@@ -248,7 +248,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase):self.assertEqual(response.url, request.url)self.assertEqual(response.body, b'/')- http_proxy = '%s?noconnect' % self.getURL('')+ http_proxy = f"{self.getURL('')}?noconnect"request = Request('https://example.com', meta={'proxy': http_proxy})with self.assertWarnsRegex(Warning,diff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex b610087bd..579ef9fa2 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -1217,18 +1217,17 @@ class FormRequestTest(RequestTest):response, formcss="input[name='abc']")def test_from_response_valid_form_methods(self):- body = """<form action="post.php" method="%s">- <input type="hidden" name="one" value="1">- </form>"""-- for method in self.request_class.valid_form_methods:- response = _buildresponse(body % method)+ form_methods = [[method, method] for method in self.request_class.valid_form_methods]+ form_methods.append(['UNKNOWN', 'GET'])++ for method, expected in form_methods:+ response = _buildresponse(+ f'<form action="post.php" method="{method}">'+ '<input type="hidden" name="one" value="1">'+ '</form>'+ )r = self.request_class.from_response(response)- self.assertEqual(r.method, method)-- response = _buildresponse(body % 'UNKNOWN')- r = self.request_class.from_response(response)- self.assertEqual(r.method, 'GET')+ self.assertEqual(r.method, expected)def _buildresponse(body, **kwargs):