Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix SMTP STARTTLS for Twisted >= 21.2.0 (#5406)

Tobias Mayrf449ee53778b
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 2a25ccd44..b8cc28335 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -12,7 +12,9 @@ from email.mime.text import MIMEText
from email.utils import formatdate
from io import BytesIO
+from twisted.python.versions import Version
from twisted.internet import defer, ssl
+from twisted import version as twisted_version
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import to_bytes
@@ -126,16 +128,11 @@ class MailSender:
'mailattachs': nattachs, 'mailerr': errstr})
def _sendmail(self, to_addrs, msg):
- # Import twisted.mail here because it is not available in python3
from twisted.internet import reactor
- from twisted.mail.smtp import ESMTPSenderFactory
msg = BytesIO(msg)
d = defer.Deferred()
- factory = ESMTPSenderFactory(
- self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d,
- heloFallback=True, requireAuthentication=False, requireTransportSecurity=self.smtptls,
- )
- factory.noisy = False
+
+ factory = self._create_sender_factory(to_addrs, msg, d)
if self.smtpssl:
reactor.connectSSL(self.smtphost, self.smtpport, factory, ssl.ClientContextFactory())
@@ -143,3 +140,20 @@ class MailSender:
reactor.connectTCP(self.smtphost, self.smtpport, factory)
return d
+
+ def _create_sender_factory(self, to_addrs, msg, d):
+ from twisted.mail.smtp import ESMTPSenderFactory
+
+ factory_keywords = {
+ 'heloFallback': True,
+ 'requireAuthentication': False,
+ 'requireTransportSecurity': self.smtptls
+ }
+
+ # Newer versions of twisted require the hostname to use STARTTLS
+ if twisted_version >= Version('twisted', 21, 2, 0):
+ factory_keywords['hostname'] = self.smtphost
+
+ factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d, **factory_keywords)
+ factory.noisy = False
+ return factory
diff --git a/tests/test_mail.py b/tests/test_mail.py
index 9b248fbfa..fd02020ee 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -4,6 +4,11 @@ import unittest
from io import BytesIO
from email.charset import Charset
+from twisted.internet._sslverify import ClientTLSOptions
+from twisted.internet.ssl import ClientContextFactory
+from twisted.python.versions import Version
+from twisted.internet import defer
+from twisted import version as twisted_version
from scrapy.mail import MailSender
@@ -121,6 +126,17 @@ class MailSenderTest(unittest.TestCase):
self.assertEqual(text.get_charset(), Charset('utf-8'))
self.assertEqual(attach.get_payload(decode=True).decode('utf-8'), body)
+ def test_create_sender_factory_with_host(self):
+ mailsender = MailSender(debug=False, smtphost='smtp.testhost.com')
+
+ factory = mailsender._create_sender_factory(to_addrs=['[email protected]'], msg='test', d=defer.Deferred())
+
+ context = factory.buildProtocol('[email protected]').context
+ if twisted_version >= Version('twisted', 21, 2, 0):
+ self.assertIsInstance(context, ClientTLSOptions)
+ else:
+ self.assertIsInstance(context, ClientContextFactory)
+
if __name__ == "__main__":
unittest.main()