Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove code required to support ancient twisted versions. See GH-1887.

Mikhail Korobov04b2f79e7a13
diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py
index 81da2615a..e4a7d8564 100644
--- a/scrapy/core/downloader/handlers/http.py
+++ b/scrapy/core/downloader/handlers/http.py
@@ -1,10 +1,6 @@
-from scrapy import twisted_version
+from __future__ import absolute_import
from .http10 import HTTP10DownloadHandler
-
-if twisted_version >= (11, 1, 0):
- from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
-else:
- HTTPDownloadHandler = HTTP10DownloadHandler
+from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
# backwards compatibility
diff --git a/tests/mockserver.py b/tests/mockserver.py
index a40e2e501..e611cc3ec 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -2,32 +2,16 @@ from __future__ import print_function
import sys, time, random, os, json
from six.moves.urllib.parse import urlencode
from subprocess import Popen, PIPE
+
from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.resource import Resource
-from twisted.internet import reactor, defer, ssl
-from scrapy import twisted_version
-from scrapy.utils.python import to_bytes, to_unicode
-
-
-if twisted_version < (11, 0, 0):
- def deferLater(clock, delay, func, *args, **kw):
- def _cancel_method():
- _cancel_cb(None)
- d.errback(Exception())
+from twisted.web.test.test_webclient import PayloadResource
+from twisted.web.server import GzipEncoderFactory
+from twisted.web.resource import EncodingResourceWrapper
+from twisted.internet import reactor, ssl
+from twisted.internet.task import deferLater
- def _cancel_cb(result):
- if cl.active():
- cl.cancel()
- return result
-
- d = defer.Deferred()
- d.cancel = _cancel_method
- d.addCallback(lambda ignored: func(*args, **kw))
- d.addBoth(_cancel_cb)
- cl = clock.callLater(delay, d.callback, None)
- return d
-else:
- from twisted.internet.task import deferLater
+from scrapy.utils.python import to_bytes, to_unicode
def getarg(request, name, default=None, type=None):
@@ -174,13 +158,8 @@ class Root(Resource):
self.putChild(b"drop", Drop())
self.putChild(b"raw", Raw())
self.putChild(b"echo", Echo())
-
- if twisted_version > (12, 3, 0):
- from twisted.web.test.test_webclient import PayloadResource
- from twisted.web.server import GzipEncoderFactory
- from twisted.web.resource import EncodingResourceWrapper
- self.putChild(b"payload", PayloadResource())
- self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
+ self.putChild(b"payload", PayloadResource())
+ self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
def getChild(self, name, request):
return self
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 8d3b49d6a..6333efceb 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -17,7 +17,6 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \
from twisted.cred import portal, checkers, credentials
from w3lib.url import path_to_file_uri
-from scrapy import twisted_version
from scrapy.core.downloader.handlers import DownloadHandlers
from scrapy.core.downloader.handlers.file import FileDownloadHandler
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler
@@ -281,8 +280,6 @@ class Https10TestCase(Http10TestCase):
class Http11TestCase(HttpTestCase):
"""HTTP 1.1 test case"""
download_handler_cls = HTTP11DownloadHandler
- if twisted_version < (11, 1, 0):
- skip = 'HTTP1.1 not supported in twisted < 11.1.0'
def test_download_without_maxsize_limit(self):
request = Request(self.getURL('file'))
@@ -366,8 +363,6 @@ class Https11InvalidDNSId(Https11TestCase):
class Http11MockServerTestCase(unittest.TestCase):
"""HTTP 1.1 test case with MockServer"""
- if twisted_version < (11, 1, 0):
- skip = 'HTTP1.1 not supported in twisted < 11.1.0'
def setUp(self):
self.mockserver = MockServer()
@@ -396,31 +391,27 @@ class Http11MockServerTestCase(unittest.TestCase):
@defer.inlineCallbacks
def test_download_gzip_response(self):
+ crawler = get_crawler(SingleRequestSpider)
+ body = b'1' * 100 # PayloadResource requires body length to be 100
+ request = Request('http://localhost:8998/payload', method='POST',
+ body=body, meta={'download_maxsize': 50})
+ yield crawler.crawl(seed=request)
+ failure = crawler.spider.meta['failure']
+ # download_maxsize < 100, hence the CancelledError
+ self.assertIsInstance(failure.value, defer.CancelledError)
- if twisted_version > (12, 3, 0):
-
- crawler = get_crawler(SingleRequestSpider)
- body = b'1'*100 # PayloadResource requires body length to be 100
- request = Request('http://localhost:8998/payload', method='POST', body=body, meta={'download_maxsize': 50})
+ if six.PY2:
+ request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
+ request = request.replace(url='http://localhost:8998/xpayload')
yield crawler.crawl(seed=request)
- failure = crawler.spider.meta['failure']
- # download_maxsize < 100, hence the CancelledError
- self.assertIsInstance(failure.value, defer.CancelledError)
-
- if six.PY2:
- request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
- request = request.replace(url='http://localhost:8998/xpayload')
- yield crawler.crawl(seed=request)
- # download_maxsize = 50 is enough for the gzipped response
- failure = crawler.spider.meta.get('failure')
- self.assertTrue(failure == None)
- reason = crawler.spider.meta['close_reason']
- self.assertTrue(reason, 'finished')
- else:
- # See issue https://twistedmatrix.com/trac/ticket/8175
- raise unittest.SkipTest("xpayload only enabled for PY2")
+ # download_maxsize = 50 is enough for the gzipped response
+ failure = crawler.spider.meta.get('failure')
+ self.assertTrue(failure == None)
+ reason = crawler.spider.meta['close_reason']
+ self.assertTrue(reason, 'finished')
else:
- raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0")
+ # See issue https://twistedmatrix.com/trac/ticket/8175
+ raise unittest.SkipTest("xpayload only enabled for PY2")
class UriResource(resource.Resource):
@@ -500,8 +491,6 @@ class Http10ProxyTestCase(HttpProxyTestCase):
class Http11ProxyTestCase(HttpProxyTestCase):
download_handler_cls = HTTP11DownloadHandler
- if twisted_version < (11, 1, 0):
- skip = 'HTTP1.1 not supported in twisted < 11.1.0'
@defer.inlineCallbacks
def test_download_with_proxy_https_timeout(self):
@@ -692,8 +681,6 @@ class FTPTestCase(unittest.TestCase):
username = "scrapy"
password = "passwd"
- if twisted_version < (10, 2, 0):
- skip = "Twisted pre 10.2.0 doesn't allow to set home path other than /home"
if six.PY3:
skip = "Twisted missing ftp support for PY3"
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index eb17974bf..e129b71f8 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -5,7 +5,6 @@ from twisted.internet.error import TimeoutError, DNSLookupError, \
ConnectionLost, TCPTimedOutError
from twisted.web.client import ResponseFailed
-from scrapy import twisted_version
from scrapy.downloadermiddlewares.retry import RetryMiddleware
from scrapy.spiders import Spider
from scrapy.http import Request, Response
@@ -74,9 +73,7 @@ class RetryTest(unittest.TestCase):
def test_twistederrors(self):
exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError,
DNSLookupError, ConnectionRefusedError, ConnectionDone,
- ConnectError, ConnectionLost]
- if twisted_version >= (11, 1, 0): # http11 available
- exceptions.append(ResponseFailed)
+ ConnectError, ConnectionLost, ResponseFailed]
for exc in exceptions:
req = Request('http://www.scrapytest.org/%s' % exc.__name__)