Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove code required to support ancient twisted versions. See GH-1887.
Mikhail Korobov04b2f79e7a13
diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.pyindex 81da2615a..e4a7d8564 100644--- a/scrapy/core/downloader/handlers/http.py+++ b/scrapy/core/downloader/handlers/http.py@@ -1,10 +1,6 @@-from scrapy import twisted_version+from __future__ import absolute_importfrom .http10 import HTTP10DownloadHandler--if twisted_version >= (11, 1, 0):- from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler-else:- HTTPDownloadHandler = HTTP10DownloadHandler+from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler# backwards compatibilitydiff --git a/tests/mockserver.py b/tests/mockserver.pyindex a40e2e501..e611cc3ec 100644--- a/tests/mockserver.py+++ b/tests/mockserver.py@@ -2,32 +2,16 @@ from __future__ import print_functionimport sys, time, random, os, jsonfrom six.moves.urllib.parse import urlencodefrom subprocess import Popen, PIPE+from twisted.web.server import Site, NOT_DONE_YETfrom twisted.web.resource import Resource-from twisted.internet import reactor, defer, ssl-from scrapy import twisted_version-from scrapy.utils.python import to_bytes, to_unicode---if twisted_version < (11, 0, 0):- def deferLater(clock, delay, func, *args, **kw):- def _cancel_method():- _cancel_cb(None)- d.errback(Exception())+from twisted.web.test.test_webclient import PayloadResource+from twisted.web.server import GzipEncoderFactory+from twisted.web.resource import EncodingResourceWrapper+from twisted.internet import reactor, ssl+from twisted.internet.task import deferLater- def _cancel_cb(result):- if cl.active():- cl.cancel()- return result-- d = defer.Deferred()- d.cancel = _cancel_method- d.addCallback(lambda ignored: func(*args, **kw))- d.addBoth(_cancel_cb)- cl = clock.callLater(delay, d.callback, None)- return d-else:- from twisted.internet.task import deferLater+from scrapy.utils.python import to_bytes, to_unicodedef getarg(request, name, default=None, type=None):@@ -174,13 +158,8 @@ class Root(Resource):self.putChild(b"drop", Drop())self.putChild(b"raw", Raw())self.putChild(b"echo", Echo())-- if twisted_version > (12, 3, 0):- from twisted.web.test.test_webclient import PayloadResource- from twisted.web.server import GzipEncoderFactory- from twisted.web.resource import EncodingResourceWrapper- self.putChild(b"payload", PayloadResource())- self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))+ self.putChild(b"payload", PayloadResource())+ self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))def getChild(self, name, request):return selfdiff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 8d3b49d6a..6333efceb 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -17,7 +17,6 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \from twisted.cred import portal, checkers, credentialsfrom w3lib.url import path_to_file_uri-from scrapy import twisted_versionfrom scrapy.core.downloader.handlers import DownloadHandlersfrom scrapy.core.downloader.handlers.file import FileDownloadHandlerfrom scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler@@ -281,8 +280,6 @@ class Https10TestCase(Http10TestCase):class Http11TestCase(HttpTestCase):"""HTTP 1.1 test case"""download_handler_cls = HTTP11DownloadHandler- if twisted_version < (11, 1, 0):- skip = 'HTTP1.1 not supported in twisted < 11.1.0'def test_download_without_maxsize_limit(self):request = Request(self.getURL('file'))@@ -366,8 +363,6 @@ class Https11InvalidDNSId(Https11TestCase):class Http11MockServerTestCase(unittest.TestCase):"""HTTP 1.1 test case with MockServer"""- if twisted_version < (11, 1, 0):- skip = 'HTTP1.1 not supported in twisted < 11.1.0'def setUp(self):self.mockserver = MockServer()@@ -396,31 +391,27 @@ class Http11MockServerTestCase(unittest.TestCase):@defer.inlineCallbacksdef test_download_gzip_response(self):+ crawler = get_crawler(SingleRequestSpider)+ body = b'1' * 100 # PayloadResource requires body length to be 100+ request = Request('http://localhost:8998/payload', method='POST',+ body=body, meta={'download_maxsize': 50})+ yield crawler.crawl(seed=request)+ failure = crawler.spider.meta['failure']+ # download_maxsize < 100, hence the CancelledError+ self.assertIsInstance(failure.value, defer.CancelledError)- if twisted_version > (12, 3, 0):-- crawler = get_crawler(SingleRequestSpider)- body = b'1'*100 # PayloadResource requires body length to be 100- request = Request('http://localhost:8998/payload', method='POST', body=body, meta={'download_maxsize': 50})+ if six.PY2:+ request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')+ request = request.replace(url='http://localhost:8998/xpayload')yield crawler.crawl(seed=request)- failure = crawler.spider.meta['failure']- # download_maxsize < 100, hence the CancelledError- self.assertIsInstance(failure.value, defer.CancelledError)-- if six.PY2:- request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')- request = request.replace(url='http://localhost:8998/xpayload')- yield crawler.crawl(seed=request)- # download_maxsize = 50 is enough for the gzipped response- failure = crawler.spider.meta.get('failure')- self.assertTrue(failure == None)- reason = crawler.spider.meta['close_reason']- self.assertTrue(reason, 'finished')- else:- # See issue https://twistedmatrix.com/trac/ticket/8175- raise unittest.SkipTest("xpayload only enabled for PY2")+ # download_maxsize = 50 is enough for the gzipped response+ failure = crawler.spider.meta.get('failure')+ self.assertTrue(failure == None)+ reason = crawler.spider.meta['close_reason']+ self.assertTrue(reason, 'finished')else:- raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0")+ # See issue https://twistedmatrix.com/trac/ticket/8175+ raise unittest.SkipTest("xpayload only enabled for PY2")class UriResource(resource.Resource):@@ -500,8 +491,6 @@ class Http10ProxyTestCase(HttpProxyTestCase):class Http11ProxyTestCase(HttpProxyTestCase):download_handler_cls = HTTP11DownloadHandler- if twisted_version < (11, 1, 0):- skip = 'HTTP1.1 not supported in twisted < 11.1.0'@defer.inlineCallbacksdef test_download_with_proxy_https_timeout(self):@@ -692,8 +681,6 @@ class FTPTestCase(unittest.TestCase):username = "scrapy"password = "passwd"- if twisted_version < (10, 2, 0):- skip = "Twisted pre 10.2.0 doesn't allow to set home path other than /home"if six.PY3:skip = "Twisted missing ftp support for PY3"diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.pyindex eb17974bf..e129b71f8 100644--- a/tests/test_downloadermiddleware_retry.py+++ b/tests/test_downloadermiddleware_retry.py@@ -5,7 +5,6 @@ from twisted.internet.error import TimeoutError, DNSLookupError, \ConnectionLost, TCPTimedOutErrorfrom twisted.web.client import ResponseFailed-from scrapy import twisted_versionfrom scrapy.downloadermiddlewares.retry import RetryMiddlewarefrom scrapy.spiders import Spiderfrom scrapy.http import Request, Response@@ -74,9 +73,7 @@ class RetryTest(unittest.TestCase):def test_twistederrors(self):exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError,DNSLookupError, ConnectionRefusedError, ConnectionDone,- ConnectError, ConnectionLost]- if twisted_version >= (11, 1, 0): # http11 available- exceptions.append(ResponseFailed)+ ConnectError, ConnectionLost, ResponseFailed]for exc in exceptions:req = Request('http://www.scrapytest.org/%s' % exc.__name__)