Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge remote-tracking branch 'upstream/master' into add-black-formatter

Emmanuel Rondanf2c22aaabbaf
diff --cc docs/contributing.rst
index f7f1218c8,9cfe10012..12f0e8d21
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@@ -179,27 -179,6 +179,28 @@@ Scrapy
See https://help.github.com/en/github/using-git/setting-your-username-in-git for
setup instructions.
+.. _pre-commit:
+
+Pre-commit
+==========
+
+We use `pre-commit`_ to automatically address simple code issues before every
+commit.
+
+Before you start writing a patch:
- #. `Install pre-commit <https://pre-commit.com/#installation>`_.
- #. On the root of your local clone of the Scrapy repository, run the following command:
++ #. `Install pre-commit <https://pre-commit.com/#installation>`_.
++ #. On the root of your local clone of the Scrapy repository, run the following command:
++
+ .. code-block:: bash
+
+ pre-commit install
+
+Now our pre-commit hooks will run every time you create a Git commit. Upon
+finding issues, pre-commit hooks aborts your commit, and they either fix
+the corresponding issues automatically or only report them to you. If they fix
+the issues automatically, creating your commit again should succeed. Otherwise,
+you may need to address the corresponding issues manually first.
+
.. _documentation-policies:
Documentation policies
diff --cc scrapy/mail.py
index fbde9c547,b8cc28335..fa1e55f1f
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@@ -151,44 -121,39 +153,57 @@@ class MailSender
def _sent_failed(self, failure, to, cc, subject, nattachs):
errstr = str(failure.value)
- logger.error('Unable to send mail: To=%(mailto)s Cc=%(mailcc)s '
- 'Subject="%(mailsubject)s" Attachs=%(mailattachs)d'
- '- %(mailerr)s',
- {'mailto': to, 'mailcc': cc, 'mailsubject': subject,
- 'mailattachs': nattachs, 'mailerr': errstr})
+ logger.error(
+ "Unable to send mail: To=%(mailto)s Cc=%(mailcc)s "
+ 'Subject="%(mailsubject)s" Attachs=%(mailattachs)d'
+ "- %(mailerr)s",
+ {
+ "mailto": to,
+ "mailcc": cc,
+ "mailsubject": subject,
+ "mailattachs": nattachs,
+ "mailerr": errstr,
+ },
+ )
def _sendmail(self, to_addrs, msg):
- # Import twisted.mail here because it is not available in python3
from twisted.internet import reactor
- from twisted.mail.smtp import ESMTPSenderFactory
+
msg = BytesIO(msg)
d = defer.Deferred()
- factory = ESMTPSenderFactory(
- self.smtpuser,
- self.smtppass,
- self.mailfrom,
- to_addrs,
- msg,
- d,
- heloFallback=True,
- requireAuthentication=False,
- requireTransportSecurity=self.smtptls,
- )
- factory.noisy = False
+
+ factory = self._create_sender_factory(to_addrs, msg, d)
if self.smtpssl:
- reactor.connectSSL(self.smtphost, self.smtpport, factory, ssl.ClientContextFactory())
+ reactor.connectSSL(
+ self.smtphost, self.smtpport, factory, ssl.ClientContextFactory()
+ )
else:
reactor.connectTCP(self.smtphost, self.smtpport, factory)
return d
+
+ def _create_sender_factory(self, to_addrs, msg, d):
+ from twisted.mail.smtp import ESMTPSenderFactory
+
+ factory_keywords = {
- 'heloFallback': True,
- 'requireAuthentication': False,
- 'requireTransportSecurity': self.smtptls
++ "heloFallback": True,
++ "requireAuthentication": False,
++ "requireTransportSecurity": self.smtptls,
+ }
+
+ # Newer versions of twisted require the hostname to use STARTTLS
- if twisted_version >= Version('twisted', 21, 2, 0):
- factory_keywords['hostname'] = self.smtphost
-
- factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d, **factory_keywords)
++ if twisted_version >= Version("twisted", 21, 2, 0):
++ factory_keywords["hostname"] = self.smtphost
++
++ factory = ESMTPSenderFactory(
++ self.smtpuser,
++ self.smtppass,
++ self.mailfrom,
++ to_addrs,
++ msg,
++ d,
++ **factory_keywords
++ )
+ factory.noisy = False
+ return factory
diff --cc tests/test_http_response.py
index 8c422bb4f,3875bbcb1..beb0b712b
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@@ -519,8 -426,8 +519,9 @@@ class TextResponseTest(BaseResponseTest
# Inferring encoding from body also cache decoded body as sideeffect,
# this test tries to ensure that calling response.encoding and
# response.text in indistinct order doesn't affect final
++ # response.text in indistinct order doesn't affect final
# values for encoding and decoded body.
- url = 'http://example.com'
+ url = "http://example.com"
body = b"\xef\xbb\xbfWORD"
headers = {"Content-type": ["text/html; charset=utf-8"]}
diff --cc tests/test_mail.py
index c78980d57,fd02020ee..0ee0400cd
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@@ -143,10 -122,21 +148,23 @@@ class MailSenderTest(unittest.TestCase)
self.assertEqual(len(payload), 2)
text, attach = payload
- self.assertEqual(text.get_payload(decode=True).decode('utf-8'), body)
- self.assertEqual(text.get_charset(), Charset('utf-8'))
- self.assertEqual(attach.get_payload(decode=True).decode('utf-8'), body)
+ self.assertEqual(text.get_payload(decode=True).decode("utf-8"), body)
+ self.assertEqual(text.get_charset(), Charset("utf-8"))
+ self.assertEqual(attach.get_payload(decode=True).decode("utf-8"), body)
+ def test_create_sender_factory_with_host(self):
- mailsender = MailSender(debug=False, smtphost='smtp.testhost.com')
++ mailsender = MailSender(debug=False, smtphost="smtp.testhost.com")
+
- factory = mailsender._create_sender_factory(to_addrs=['[email protected]'], msg='test', d=defer.Deferred())
++ factory = mailsender._create_sender_factory(
++ to_addrs=["[email protected]"], msg="test", d=defer.Deferred()
++ )
+
- context = factory.buildProtocol('[email protected]').context
- if twisted_version >= Version('twisted', 21, 2, 0):
++ context = factory.buildProtocol("[email protected]").context
++ if twisted_version >= Version("twisted", 21, 2, 0):
+ self.assertIsInstance(context, ClientTLSOptions)
+ else:
+ self.assertIsInstance(context, ClientContextFactory)
+
if __name__ == "__main__":
unittest.main()