Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge remote-tracking branch 'upstream/master' into add-black-formatter
Emmanuel Rondanf2c22aaabbaf
diff --cc docs/contributing.rstindex f7f1218c8,9cfe10012..12f0e8d21--- a/docs/contributing.rst+++ b/docs/contributing.rst@@@ -179,27 -179,6 +179,28 @@@ ScrapySee https://help.github.com/en/github/using-git/setting-your-username-in-git forsetup instructions.+.. _pre-commit:++Pre-commit+==========++We use `pre-commit`_ to automatically address simple code issues before every+commit.++Before you start writing a patch:- #. `Install pre-commit <https://pre-commit.com/#installation>`_.- #. On the root of your local clone of the Scrapy repository, run the following command:++ #. `Install pre-commit <https://pre-commit.com/#installation>`_.++ #. On the root of your local clone of the Scrapy repository, run the following command:+++ .. code-block:: bash++ pre-commit install++Now our pre-commit hooks will run every time you create a Git commit. Upon+finding issues, pre-commit hooks aborts your commit, and they either fix+the corresponding issues automatically or only report them to you. If they fix+the issues automatically, creating your commit again should succeed. Otherwise,+you may need to address the corresponding issues manually first.+.. _documentation-policies:Documentation policiesdiff --cc scrapy/mail.pyindex fbde9c547,b8cc28335..fa1e55f1f--- a/scrapy/mail.py+++ b/scrapy/mail.py@@@ -151,44 -121,39 +153,57 @@@ class MailSenderdef _sent_failed(self, failure, to, cc, subject, nattachs):errstr = str(failure.value)- logger.error('Unable to send mail: To=%(mailto)s Cc=%(mailcc)s '- 'Subject="%(mailsubject)s" Attachs=%(mailattachs)d'- '- %(mailerr)s',- {'mailto': to, 'mailcc': cc, 'mailsubject': subject,- 'mailattachs': nattachs, 'mailerr': errstr})+ logger.error(+ "Unable to send mail: To=%(mailto)s Cc=%(mailcc)s "+ 'Subject="%(mailsubject)s" Attachs=%(mailattachs)d'+ "- %(mailerr)s",+ {+ "mailto": to,+ "mailcc": cc,+ "mailsubject": subject,+ "mailattachs": nattachs,+ "mailerr": errstr,+ },+ )def _sendmail(self, to_addrs, msg):- # Import twisted.mail here because it is not available in python3from twisted.internet import reactor- from twisted.mail.smtp import ESMTPSenderFactory+msg = BytesIO(msg)d = defer.Deferred()- factory = ESMTPSenderFactory(- self.smtpuser,- self.smtppass,- self.mailfrom,- to_addrs,- msg,- d,- heloFallback=True,- requireAuthentication=False,- requireTransportSecurity=self.smtptls,- )- factory.noisy = False++ factory = self._create_sender_factory(to_addrs, msg, d)if self.smtpssl:- reactor.connectSSL(self.smtphost, self.smtpport, factory, ssl.ClientContextFactory())+ reactor.connectSSL(+ self.smtphost, self.smtpport, factory, ssl.ClientContextFactory()+ )else:reactor.connectTCP(self.smtphost, self.smtpport, factory)return d++ def _create_sender_factory(self, to_addrs, msg, d):+ from twisted.mail.smtp import ESMTPSenderFactory++ factory_keywords = {- 'heloFallback': True,- 'requireAuthentication': False,- 'requireTransportSecurity': self.smtptls++ "heloFallback": True,++ "requireAuthentication": False,++ "requireTransportSecurity": self.smtptls,+ }++ # Newer versions of twisted require the hostname to use STARTTLS- if twisted_version >= Version('twisted', 21, 2, 0):- factory_keywords['hostname'] = self.smtphost-- factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d, **factory_keywords)++ if twisted_version >= Version("twisted", 21, 2, 0):++ factory_keywords["hostname"] = self.smtphost++++ factory = ESMTPSenderFactory(++ self.smtpuser,++ self.smtppass,++ self.mailfrom,++ to_addrs,++ msg,++ d,++ **factory_keywords++ )+ factory.noisy = False+ return factorydiff --cc tests/test_http_response.pyindex 8c422bb4f,3875bbcb1..beb0b712b--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@@ -519,8 -426,8 +519,9 @@@ class TextResponseTest(BaseResponseTest# Inferring encoding from body also cache decoded body as sideeffect,# this test tries to ensure that calling response.encoding and# response.text in indistinct order doesn't affect final++ # response.text in indistinct order doesn't affect final# values for encoding and decoded body.- url = 'http://example.com'+ url = "http://example.com"body = b"\xef\xbb\xbfWORD"headers = {"Content-type": ["text/html; charset=utf-8"]}diff --cc tests/test_mail.pyindex c78980d57,fd02020ee..0ee0400cd--- a/tests/test_mail.py+++ b/tests/test_mail.py@@@ -143,10 -122,21 +148,23 @@@ class MailSenderTest(unittest.TestCase)self.assertEqual(len(payload), 2)text, attach = payload- self.assertEqual(text.get_payload(decode=True).decode('utf-8'), body)- self.assertEqual(text.get_charset(), Charset('utf-8'))- self.assertEqual(attach.get_payload(decode=True).decode('utf-8'), body)+ self.assertEqual(text.get_payload(decode=True).decode("utf-8"), body)+ self.assertEqual(text.get_charset(), Charset("utf-8"))+ self.assertEqual(attach.get_payload(decode=True).decode("utf-8"), body)+ def test_create_sender_factory_with_host(self):- mailsender = MailSender(debug=False, smtphost='smtp.testhost.com')++ mailsender = MailSender(debug=False, smtphost="smtp.testhost.com")+- factory = mailsender._create_sender_factory(to_addrs=['[email protected]'], msg='test', d=defer.Deferred())++ factory = mailsender._create_sender_factory(++ to_addrs=["[email protected]"], msg="test", d=defer.Deferred()++ )+- context = factory.buildProtocol('[email protected]').context- if twisted_version >= Version('twisted', 21, 2, 0):++ context = factory.buildProtocol("[email protected]").context++ if twisted_version >= Version("twisted", 21, 2, 0):+ self.assertIsInstance(context, ClientTLSOptions)+ else:+ self.assertIsInstance(context, ClientContextFactory)+if __name__ == "__main__":unittest.main()