Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Bump Twisted dependency to 13.1.0 (released June 2013)

nyovc8cf1a303d22
diff --git a/requirements.txt b/requirements.txt
index cfa907050..64b6e771c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-Twisted>=10.0.0
+Twisted>=13.1.0
lxml
pyOpenSSL
cssselect>=0.9
diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py
index e902d6683..d20eda34f 100644
--- a/scrapy/xlib/tx/_newclient.py
+++ b/scrapy/xlib/tx/_newclient.py
@@ -38,26 +38,21 @@ from twisted.internet.error import ConnectionDone
from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred
from twisted.internet.defer import CancelledError
from twisted.internet.protocol import Protocol
-from twisted.protocols.basic import LineReceiver
-from twisted.web.iweb import UNKNOWN_LENGTH
-from twisted.web.http_headers import Headers
+#from twisted.protocols.basic import LineReceiver
+from twisted.web.iweb import UNKNOWN_LENGTH, IResponse
+#from twisted.web.http_headers import Headers
from twisted.web.http import NO_CONTENT, NOT_MODIFIED
from twisted.web.http import _DataLoss, PotentialDataLoss
from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder
from twisted.web._newclient import (
BadHeaders, ExcessWrite, ParseError, BadResponseVersion, _WrapperException,
- RequestGenerationFailed, RequestTransmissionFailed,
- WrongBodyLength, ResponseDone, RequestNotSent,
- LengthEnforcingConsumer, makeStatefulDispatcher, ChunkedEncoder,
- TransportProxyProducer,
+ RequestGenerationFailed, RequestTransmissionFailed, ConnectionAborted,
+ WrongBodyLength, ResponseDone, ResponseFailed, RequestNotSent,
+ ResponseNeverReceived, HTTPParser, HTTPClientParser, Request,
+ LengthEnforcingConsumer, makeStatefulDispatcher, Response, ChunkedEncoder,
+ TransportProxyProducer, HTTP11ClientProtocol
)
-# newer than 10.0.0
-#from twisted.web._newclient import (
-# ConnectionAborted, ResponseFailed, ResponseNeverReceived, HTTPParser,
-# HTTPClientParser, Request, Response, HTTP11ClientProtocol,
-#)
-from .iweb import IResponse
# States HTTPParser can be in
STATUS = 'STATUS'
@@ -130,7 +125,7 @@ class RequestTransmissionFailed(_WrapperException):
@ivar reasons: A C{list} of one or more L{Failure} instances giving the
reasons the request transmission was considered to have failed.
"""
-}}} '''
+
class ConnectionAborted(Exception):
@@ -139,7 +134,7 @@ class ConnectionAborted(Exception):
"""
-''' {{{
+
class WrongBodyLength(Exception):
"""
An L{IBodyProducer} declared the number of bytes it was going to
@@ -155,7 +150,7 @@ class ResponseDone(Exception):
protocol passed to L{Response.deliverBody} and indicates that the entire
response has been delivered.
"""
-}}} '''
+
class ResponseFailed(_WrapperException):
@@ -182,7 +177,7 @@ class ResponseNeverReceived(ResponseFailed):
"""
-''' {{{
+
class RequestNotSent(Exception):
"""
L{RequestNotSent} indicates that an attempt was made to issue a request but
@@ -191,7 +186,7 @@ class RequestNotSent(Exception):
to send a request using a protocol which is no longer connected to a
server.
"""
-}}} '''
+
def _callAppFunction(function):
@@ -777,7 +772,7 @@ class Request:
_callAppFunction(self.bodyProducer.stopProducing)
-''' {{{
+
class LengthEnforcingConsumer:
"""
An L{IConsumer} proxy which enforces an exact length requirement on the
@@ -1201,7 +1196,7 @@ class TransportProxyProducer:
"""
if self._producer is not None:
self._producer.pauseProducing()
-}}} '''
+
class HTTP11ClientProtocol(Protocol):
@@ -1527,3 +1522,4 @@ class HTTP11ClientProtocol(Protocol):
d = Deferred()
self._abortDeferreds.append(d)
return d
+}}} '''
diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.py
index 396115985..8e0b1df8b 100644
--- a/scrapy/xlib/tx/client.py
+++ b/scrapy/xlib/tx/client.py
@@ -29,23 +29,18 @@ from twisted.python.failure import Failure
from twisted.web import http
from twisted.internet import defer, protocol, task, reactor
from twisted.internet.interfaces import IProtocol
+from twisted.internet.endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint
from twisted.python import failure
from twisted.python.components import proxyForInterface
from twisted.web import error
-from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer
+from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IResponse
from twisted.web.http_headers import Headers
from twisted.web.client import (
- PartialDownloadError,
+ PartialDownloadError, FileBodyProducer,
+ CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent,
+ Agent, ProxyAgent, HTTPConnectionPool, readBody,
)
-# newer than 10.0.0
-#from twisted.web.client import (
-# CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent, FileBodyProducer,
-# HTTPConnectionPool, Agent, ProxyAgent,
-#)
-
-from .endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint
-from .iweb import IResponse
''' {{{
class PartialDownloadError(error.Error):
@@ -54,7 +49,7 @@ class PartialDownloadError(error.Error):
@ivar response: All of the response body which was downloaded.
"""
-}}} '''
+
class _URL(tuple):
"""
@@ -138,22 +133,21 @@ def _makeGetterFactory(url, factoryFactory, contextFactory=None,
else:
reactor.connectTCP(host, port, factory)
return factory
-
+}}} '''
# The code which follows is based on the new HTTP client implementation. It
# should be significantly better than anything above, though it is not yet
# feature equivalent.
-from twisted.web.error import SchemeNotSupported
-from ._newclient import Request, Response, HTTP11ClientProtocol
-from twisted.web._newclient import ResponseDone
-from ._newclient import ResponseFailed
-from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed
-from twisted.web._newclient import (
- PotentialDataLoss, _WrapperException)
-from ._newclient import (
- ResponseNeverReceived)
+#from twisted.web.error import SchemeNotSupported
+from twisted.web._newclient import Response
+#from twisted.web._newclient import Request, HTTP11ClientProtocol
+from twisted.web._newclient import ResponseDone, ResponseFailed
+#from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed
+#from twisted.web._newclient import (
+# ResponseNeverReceived, PotentialDataLoss, _WrapperException)
+''' {{{
try:
from twisted.internet.ssl import ClientContextFactory
except ImportError:
@@ -1170,7 +1164,7 @@ def readBody(response):
d = defer.Deferred()
response.deliverBody(_ReadBodyProtocol(response.code, response.phrase, d))
return d
-
+}}} '''
__all__ = [
diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.py
index 21a467433..3f4704064 100644
--- a/scrapy/xlib/tx/endpoints.py
+++ b/scrapy/xlib/tx/endpoints.py
@@ -21,38 +21,33 @@ from zope.interface import implementer, directlyProvides
import warnings
from twisted.internet import interfaces, defer, error, fdesc
-from twisted.internet.protocol import (
- ClientFactory, Protocol, Factory)
+#from twisted.internet.protocol import (
+# ClientFactory, Protocol)
+from twisted.internet.protocol import Factory
#from twisted.internet import threads, ProcessProtocol
from twisted.internet.interfaces import IStreamServerEndpointStringParser
-from twisted.internet.interfaces import IStreamClientEndpointStringParser
+#from twisted.internet.interfaces import IStreamClientEndpointStringParser
from twisted.python.filepath import FilePath
#from twisted.python.failure import Failure
#from twisted.python import log
-from twisted.python.components import proxyForInterface
+#from twisted.python.components import proxyForInterface
from twisted.plugin import IPlugin, getPlugins
#from twisted.internet import stdio
-# newer than 10.0.0
-#from twisted.internet.endpoints import (
-# TCP4ServerEndpoint, TCP6ServerEndpoint, TCP4ClientEndpoint, SSL4ServerEndpoint, SSL4ClientEndpoint,
-# UNIXServerEndpoint, UNIXClientEndpoint, AdoptedStreamServerEndpoint, connectProtocol,
-# quoteStringArgument,
-# serverFromString, #> using newer _parseSSL, _tokenize in _serverParsers
-# clientFromString, #> using newer _clientParsers
-# _WrappingProtocol, _WrappingFactory, _TCPServerEndpoint,
-# _parseTCP, _parseUNIX, _loadCAsFromDir,
-# _parseSSL, _tokenize,
-# _parseClientTCP, _parseClientSSL, _parseClientUNIX,
-#)
-
-from .interfaces import IFileDescriptorReceiver
+from twisted.internet.endpoints import (
+ clientFromString, serverFromString, quoteStringArgument,
+ TCP4ServerEndpoint, TCP6ServerEndpoint,
+ TCP4ClientEndpoint, TCP6ClientEndpoint,
+ UNIXServerEndpoint, UNIXClientEndpoint,
+ SSL4ServerEndpoint, SSL4ClientEndpoint,
+ AdoptedStreamServerEndpoint, connectProtocol,
+)
__all__ = ["TCP4ClientEndpoint", "SSL4ServerEndpoint"]
-
+''' {{{
class _WrappingProtocol(Protocol):
"""
Wrap another protocol in order to notify my user when a connection has
@@ -71,7 +66,7 @@ class _WrappingProtocol(Protocol):
self._wrappedProtocol = wrappedProtocol
for iface in [interfaces.IHalfCloseableProtocol,
- IFileDescriptorReceiver]:
+ interfaces.IFileDescriptorReceiver]:
if iface.providedBy(self._wrappedProtocol):
directlyProvides(self, iface)
@@ -609,6 +604,7 @@ class AdoptedStreamServerEndpoint(object):
+
def _parseTCP(factory, port, interface="", backlog=50):
"""
Internal parser function for L{_parseServer} to convert the string
@@ -1280,4 +1276,4 @@ def connectProtocol(endpoint, protocol):
def buildProtocol(self, addr):
return protocol
return endpoint.connect(OneShotFactory())
-
+}}} '''
diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py
index a715d4a05..7b2a78632 100644
--- a/scrapy/xlib/tx/interfaces.py
+++ b/scrapy/xlib/tx/interfaces.py
@@ -13,24 +13,21 @@ from zope.interface import Interface, Attribute
from twisted.internet.interfaces import (
IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL,
- IReactorUDP, IReactorMulticast, IReactorProcess,
+ IReactorWin32Events, IReactorUDP, IReactorMulticast, IReactorProcess,
IReactorTime, IDelayedCall, IReactorThreads, IReactorCore,
- IReactorPluggableResolver, IReactorFDSet,
+ IReactorPluggableResolver, IReactorDaemonize, IReactorFDSet,
IListeningPort, ILoggingContext, IFileDescriptor, IReadDescriptor,
IWriteDescriptor, IReadWriteDescriptor, IHalfCloseableDescriptor,
ISystemHandle, IConsumer, IProducer, IPushProducer, IPullProducer,
IProtocol, IProcessProtocol, IHalfCloseableProtocol,
- IProtocolFactory, ITransport, IProcessTransport, IServiceCollection,
+ IFileDescriptorReceiver, IProtocolFactory, ITransport, ITCPTransport,
+ IUNIXTransport,
+ ITLSTransport, ISSLTransport, IProcessTransport, IServiceCollection,
IUDPTransport, IUNIXDatagramTransport, IUNIXDatagramConnectedTransport,
- IMulticastTransport,
+ IMulticastTransport, IStreamClientEndpoint, IStreamServerEndpoint,
+ IStreamServerEndpointStringParser, IStreamClientEndpointStringParser,
+ IReactorUNIX, IReactorUNIXDatagram, IReactorSocket, IResolver
)
-# newer than 10.0.0
-#from twisted.internet.interfaces import (
-# IResolver, IReactorUNIX, IReactorUNIXDatagram, IReactorWin32Events, IReactorSocket,
-# IReactorDaemonize, IFileDescriptorReceiver, ITCPTransport, IUNIXTransport,
-# ITLSTransport, ISSLTransport, IStreamClientEndpoint, IStreamServerEndpoint,
-# IStreamServerEndpointStringParser, IStreamClientEndpointStringParser,
-#)
''' {{{
class IAddress(Interface):
@@ -95,7 +92,7 @@ class IResolverSimple(Interface):
@raise twisted.internet.defer.TimeoutError: Raised (asynchronously)
if the name cannot be resolved within the specified timeout period.
"""
-}}} '''
+
class IResolver(IResolverSimple):
@@ -635,7 +632,7 @@ class IResolver(IResolverSimple):
"""
-''' {{{
+
class IReactorTCP(Interface):
def listenTCP(port, factory, backlog=50, interface=''):
@@ -722,7 +719,7 @@ class IReactorSSL(Interface):
@param interface: the hostname to bind to, defaults to '' (all)
"""
-}}} '''
+
class IReactorUNIX(Interface):
@@ -850,7 +847,7 @@ class IReactorWin32Events(Interface):
"""
-''' {{{
+
class IReactorUDP(Interface):
"""
UDP socket methods.
@@ -889,7 +886,7 @@ class IReactorMulticast(Interface):
@see: L{twisted.internet.interfaces.IMulticastTransport}
@see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html}
"""
-}}} '''
+
class IReactorSocket(Interface):
@@ -991,7 +988,7 @@ class IReactorSocket(Interface):
"""
-''' {{{
+
class IReactorProcess(Interface):
def spawnProcess(processProtocol, executable, args=(), env={}, path=None,
@@ -1368,7 +1365,7 @@ class IReactorPluggableResolver(Interface):
@return: The previously installed resolver.
"""
-}}} '''
+
class IReactorDaemonize(Interface):
"""
@@ -1400,7 +1397,7 @@ class IReactorDaemonize(Interface):
"""
-''' {{{
+
class IReactorFDSet(Interface):
"""
Implement me to be able to use L{IFileDescriptor} type resources.
@@ -1884,7 +1881,7 @@ class IHalfCloseableProtocol(Interface):
This will never be called for TCP connections as TCP does not
support notification of this type of half-close.
"""
-}}} '''
+
class IFileDescriptorReceiver(Interface):
@@ -1905,7 +1902,7 @@ class IFileDescriptorReceiver(Interface):
"""
-''' {{{
+
class IProtocolFactory(Interface):
"""
Interface for protocol factories.
@@ -1995,7 +1992,7 @@ class ITransport(Interface):
@return: An L{IAddress} provider.
"""
-}}} '''
+
class ITCPTransport(ITransport):
"""
@@ -2116,7 +2113,7 @@ class ISSLTransport(ITCPTransport):
Return an object with the peer's certificate info.
"""
-''' {{{
+
class IProcessTransport(ITransport):
"""
A process transport.
@@ -2345,7 +2342,7 @@ class IMulticastTransport(Interface):
"""
Leave multicast group, return L{Deferred} of success.
"""
-}}} '''
+
class IStreamClientEndpoint(Interface):
"""
@@ -2461,3 +2458,4 @@ class IStreamClientEndpointStringParser(Interface):
@return: a client endpoint
@rtype: L{IStreamClientEndpoint}
"""
+}}} '''
diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.py
index 57a111411..32c88ff2d 100644
--- a/scrapy/xlib/tx/iweb.py
+++ b/scrapy/xlib/tx/iweb.py
@@ -15,14 +15,11 @@ from zope.interface import Interface, Attribute
#from twisted.internet.interfaces import IPushProducer
from twisted.web.iweb import (
- ICredentialFactory, IBodyProducer,
- UNKNOWN_LENGTH,
+ IRequest, ICredentialFactory, IBodyProducer, IRenderable, ITemplateLoader,
+ IResponse, _IRequestEncoder, _IRequestEncoderFactory, UNKNOWN_LENGTH,
)
-# newer than 10.0.0
-#from twisted.web.iweb import (
-# IRequest, IRenderable, ITemplateLoader, IResponse, _IRequestEncoder, _IRequestEncoderFactory,
-#)
+''' {{{
class IRequest(Interface):
"""
An HTTP request.
@@ -328,7 +325,7 @@ class IRequest(Interface):
"""
-''' {{{
+
class ICredentialFactory(Interface):
"""
A credential factory defines a way to generate a particular kind of
@@ -432,7 +429,7 @@ class IBodyProducer(IPushProducer):
L{Deferred<twisted.internet.defer.Deferred>} returned by
C{startProducing} is never fired.
"""
-}}} '''
+
class IRenderable(Interface):
@@ -584,7 +581,7 @@ class _IRequestEncoderFactory(Interface):
"""
-''' {{{
+
UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH"
}}} '''
__all__ = [
diff --git a/setup.py b/setup.py
index 92c114a7a..5e32d4240 100644
--- a/setup.py
+++ b/setup.py
@@ -41,7 +41,7 @@ setup(
'Topic :: Software Development :: Libraries :: Python Modules',
],
install_requires=[
- 'Twisted>=10.0.0',
+ 'Twisted>=13.1.0',
'w3lib>=1.15.0',
'queuelib',
'lxml',