Scrapy, a fast high-level web crawling & scraping framework for Python.
Bump Twisted dependency to 13.1.0 (released June 2013)
nyovc8cf1a303d22
diff --git a/requirements.txt b/requirements.txtindex cfa907050..64b6e771c 100644--- a/requirements.txt+++ b/requirements.txt@@ -1,4 +1,4 @@-Twisted>=10.0.0+Twisted>=13.1.0lxmlpyOpenSSLcssselect>=0.9diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.pyindex e902d6683..d20eda34f 100644--- a/scrapy/xlib/tx/_newclient.py+++ b/scrapy/xlib/tx/_newclient.py@@ -38,26 +38,21 @@ from twisted.internet.error import ConnectionDonefrom twisted.internet.defer import Deferred, succeed, fail, maybeDeferredfrom twisted.internet.defer import CancelledErrorfrom twisted.internet.protocol import Protocol-from twisted.protocols.basic import LineReceiver-from twisted.web.iweb import UNKNOWN_LENGTH-from twisted.web.http_headers import Headers+#from twisted.protocols.basic import LineReceiver+from twisted.web.iweb import UNKNOWN_LENGTH, IResponse+#from twisted.web.http_headers import Headersfrom twisted.web.http import NO_CONTENT, NOT_MODIFIEDfrom twisted.web.http import _DataLoss, PotentialDataLossfrom twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoderfrom twisted.web._newclient import (BadHeaders, ExcessWrite, ParseError, BadResponseVersion, _WrapperException,- RequestGenerationFailed, RequestTransmissionFailed,- WrongBodyLength, ResponseDone, RequestNotSent,- LengthEnforcingConsumer, makeStatefulDispatcher, ChunkedEncoder,- TransportProxyProducer,+ RequestGenerationFailed, RequestTransmissionFailed, ConnectionAborted,+ WrongBodyLength, ResponseDone, ResponseFailed, RequestNotSent,+ ResponseNeverReceived, HTTPParser, HTTPClientParser, Request,+ LengthEnforcingConsumer, makeStatefulDispatcher, Response, ChunkedEncoder,+ TransportProxyProducer, HTTP11ClientProtocol)-# newer than 10.0.0-#from twisted.web._newclient import (-# ConnectionAborted, ResponseFailed, ResponseNeverReceived, HTTPParser,-# HTTPClientParser, Request, Response, HTTP11ClientProtocol,-#)-from .iweb import IResponse# States HTTPParser can be inSTATUS = 'STATUS'@@ -130,7 +125,7 @@ class RequestTransmissionFailed(_WrapperException):@ivar reasons: A C{list} of one or more L{Failure} instances giving thereasons the request transmission was considered to have failed."""-}}} '''+class ConnectionAborted(Exception):@@ -139,7 +134,7 @@ class ConnectionAborted(Exception):"""-''' {{{+class WrongBodyLength(Exception):"""An L{IBodyProducer} declared the number of bytes it was going to@@ -155,7 +150,7 @@ class ResponseDone(Exception):protocol passed to L{Response.deliverBody} and indicates that the entireresponse has been delivered."""-}}} '''+class ResponseFailed(_WrapperException):@@ -182,7 +177,7 @@ class ResponseNeverReceived(ResponseFailed):"""-''' {{{+class RequestNotSent(Exception):"""L{RequestNotSent} indicates that an attempt was made to issue a request but@@ -191,7 +186,7 @@ class RequestNotSent(Exception):to send a request using a protocol which is no longer connected to aserver."""-}}} '''+def _callAppFunction(function):@@ -777,7 +772,7 @@ class Request:_callAppFunction(self.bodyProducer.stopProducing)-''' {{{+class LengthEnforcingConsumer:"""An L{IConsumer} proxy which enforces an exact length requirement on the@@ -1201,7 +1196,7 @@ class TransportProxyProducer:"""if self._producer is not None:self._producer.pauseProducing()-}}} '''+class HTTP11ClientProtocol(Protocol):@@ -1527,3 +1522,4 @@ class HTTP11ClientProtocol(Protocol):d = Deferred()self._abortDeferreds.append(d)return d+}}} '''diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.pyindex 396115985..8e0b1df8b 100644--- a/scrapy/xlib/tx/client.py+++ b/scrapy/xlib/tx/client.py@@ -29,23 +29,18 @@ from twisted.python.failure import Failurefrom twisted.web import httpfrom twisted.internet import defer, protocol, task, reactorfrom twisted.internet.interfaces import IProtocol+from twisted.internet.endpoints import TCP4ClientEndpoint, SSL4ClientEndpointfrom twisted.python import failurefrom twisted.python.components import proxyForInterfacefrom twisted.web import error-from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer+from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IResponsefrom twisted.web.http_headers import Headersfrom twisted.web.client import (- PartialDownloadError,+ PartialDownloadError, FileBodyProducer,+ CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent,+ Agent, ProxyAgent, HTTPConnectionPool, readBody,)-# newer than 10.0.0-#from twisted.web.client import (-# CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent, FileBodyProducer,-# HTTPConnectionPool, Agent, ProxyAgent,-#)--from .endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint-from .iweb import IResponse''' {{{class PartialDownloadError(error.Error):@@ -54,7 +49,7 @@ class PartialDownloadError(error.Error):@ivar response: All of the response body which was downloaded."""-}}} '''+class _URL(tuple):"""@@ -138,22 +133,21 @@ def _makeGetterFactory(url, factoryFactory, contextFactory=None,else:reactor.connectTCP(host, port, factory)return factory-+}}} '''# The code which follows is based on the new HTTP client implementation. It# should be significantly better than anything above, though it is not yet# feature equivalent.-from twisted.web.error import SchemeNotSupported-from ._newclient import Request, Response, HTTP11ClientProtocol-from twisted.web._newclient import ResponseDone-from ._newclient import ResponseFailed-from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed-from twisted.web._newclient import (- PotentialDataLoss, _WrapperException)-from ._newclient import (- ResponseNeverReceived)+#from twisted.web.error import SchemeNotSupported+from twisted.web._newclient import Response+#from twisted.web._newclient import Request, HTTP11ClientProtocol+from twisted.web._newclient import ResponseDone, ResponseFailed+#from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed+#from twisted.web._newclient import (+# ResponseNeverReceived, PotentialDataLoss, _WrapperException)+''' {{{try:from twisted.internet.ssl import ClientContextFactoryexcept ImportError:@@ -1170,7 +1164,7 @@ def readBody(response):d = defer.Deferred()response.deliverBody(_ReadBodyProtocol(response.code, response.phrase, d))return d-+}}} '''__all__ = [diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.pyindex 21a467433..3f4704064 100644--- a/scrapy/xlib/tx/endpoints.py+++ b/scrapy/xlib/tx/endpoints.py@@ -21,38 +21,33 @@ from zope.interface import implementer, directlyProvidesimport warningsfrom twisted.internet import interfaces, defer, error, fdesc-from twisted.internet.protocol import (- ClientFactory, Protocol, Factory)+#from twisted.internet.protocol import (+# ClientFactory, Protocol)+from twisted.internet.protocol import Factory#from twisted.internet import threads, ProcessProtocolfrom twisted.internet.interfaces import IStreamServerEndpointStringParser-from twisted.internet.interfaces import IStreamClientEndpointStringParser+#from twisted.internet.interfaces import IStreamClientEndpointStringParserfrom twisted.python.filepath import FilePath#from twisted.python.failure import Failure#from twisted.python import log-from twisted.python.components import proxyForInterface+#from twisted.python.components import proxyForInterfacefrom twisted.plugin import IPlugin, getPlugins#from twisted.internet import stdio-# newer than 10.0.0-#from twisted.internet.endpoints import (-# TCP4ServerEndpoint, TCP6ServerEndpoint, TCP4ClientEndpoint, SSL4ServerEndpoint, SSL4ClientEndpoint,-# UNIXServerEndpoint, UNIXClientEndpoint, AdoptedStreamServerEndpoint, connectProtocol,-# quoteStringArgument,-# serverFromString, #> using newer _parseSSL, _tokenize in _serverParsers-# clientFromString, #> using newer _clientParsers-# _WrappingProtocol, _WrappingFactory, _TCPServerEndpoint,-# _parseTCP, _parseUNIX, _loadCAsFromDir,-# _parseSSL, _tokenize,-# _parseClientTCP, _parseClientSSL, _parseClientUNIX,-#)--from .interfaces import IFileDescriptorReceiver+from twisted.internet.endpoints import (+ clientFromString, serverFromString, quoteStringArgument,+ TCP4ServerEndpoint, TCP6ServerEndpoint,+ TCP4ClientEndpoint, TCP6ClientEndpoint,+ UNIXServerEndpoint, UNIXClientEndpoint,+ SSL4ServerEndpoint, SSL4ClientEndpoint,+ AdoptedStreamServerEndpoint, connectProtocol,+)__all__ = ["TCP4ClientEndpoint", "SSL4ServerEndpoint"]-+''' {{{class _WrappingProtocol(Protocol):"""Wrap another protocol in order to notify my user when a connection has@@ -71,7 +66,7 @@ class _WrappingProtocol(Protocol):self._wrappedProtocol = wrappedProtocolfor iface in [interfaces.IHalfCloseableProtocol,- IFileDescriptorReceiver]:+ interfaces.IFileDescriptorReceiver]:if iface.providedBy(self._wrappedProtocol):directlyProvides(self, iface)@@ -609,6 +604,7 @@ class AdoptedStreamServerEndpoint(object):+def _parseTCP(factory, port, interface="", backlog=50):"""Internal parser function for L{_parseServer} to convert the string@@ -1280,4 +1276,4 @@ def connectProtocol(endpoint, protocol):def buildProtocol(self, addr):return protocolreturn endpoint.connect(OneShotFactory())-+}}} '''diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.pyindex a715d4a05..7b2a78632 100644--- a/scrapy/xlib/tx/interfaces.py+++ b/scrapy/xlib/tx/interfaces.py@@ -13,24 +13,21 @@ from zope.interface import Interface, Attributefrom twisted.internet.interfaces import (IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL,- IReactorUDP, IReactorMulticast, IReactorProcess,+ IReactorWin32Events, IReactorUDP, IReactorMulticast, IReactorProcess,IReactorTime, IDelayedCall, IReactorThreads, IReactorCore,- IReactorPluggableResolver, IReactorFDSet,+ IReactorPluggableResolver, IReactorDaemonize, IReactorFDSet,IListeningPort, ILoggingContext, IFileDescriptor, IReadDescriptor,IWriteDescriptor, IReadWriteDescriptor, IHalfCloseableDescriptor,ISystemHandle, IConsumer, IProducer, IPushProducer, IPullProducer,IProtocol, IProcessProtocol, IHalfCloseableProtocol,- IProtocolFactory, ITransport, IProcessTransport, IServiceCollection,+ IFileDescriptorReceiver, IProtocolFactory, ITransport, ITCPTransport,+ IUNIXTransport,+ ITLSTransport, ISSLTransport, IProcessTransport, IServiceCollection,IUDPTransport, IUNIXDatagramTransport, IUNIXDatagramConnectedTransport,- IMulticastTransport,+ IMulticastTransport, IStreamClientEndpoint, IStreamServerEndpoint,+ IStreamServerEndpointStringParser, IStreamClientEndpointStringParser,+ IReactorUNIX, IReactorUNIXDatagram, IReactorSocket, IResolver)-# newer than 10.0.0-#from twisted.internet.interfaces import (-# IResolver, IReactorUNIX, IReactorUNIXDatagram, IReactorWin32Events, IReactorSocket,-# IReactorDaemonize, IFileDescriptorReceiver, ITCPTransport, IUNIXTransport,-# ITLSTransport, ISSLTransport, IStreamClientEndpoint, IStreamServerEndpoint,-# IStreamServerEndpointStringParser, IStreamClientEndpointStringParser,-#)''' {{{class IAddress(Interface):@@ -95,7 +92,7 @@ class IResolverSimple(Interface):@raise twisted.internet.defer.TimeoutError: Raised (asynchronously)if the name cannot be resolved within the specified timeout period."""-}}} '''+class IResolver(IResolverSimple):@@ -635,7 +632,7 @@ class IResolver(IResolverSimple):"""-''' {{{+class IReactorTCP(Interface):def listenTCP(port, factory, backlog=50, interface=''):@@ -722,7 +719,7 @@ class IReactorSSL(Interface):@param interface: the hostname to bind to, defaults to '' (all)"""-}}} '''+class IReactorUNIX(Interface):@@ -850,7 +847,7 @@ class IReactorWin32Events(Interface):"""-''' {{{+class IReactorUDP(Interface):"""UDP socket methods.@@ -889,7 +886,7 @@ class IReactorMulticast(Interface):@see: L{twisted.internet.interfaces.IMulticastTransport}@see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html}"""-}}} '''+class IReactorSocket(Interface):@@ -991,7 +988,7 @@ class IReactorSocket(Interface):"""-''' {{{+class IReactorProcess(Interface):def spawnProcess(processProtocol, executable, args=(), env={}, path=None,@@ -1368,7 +1365,7 @@ class IReactorPluggableResolver(Interface):@return: The previously installed resolver."""-}}} '''+class IReactorDaemonize(Interface):"""@@ -1400,7 +1397,7 @@ class IReactorDaemonize(Interface):"""-''' {{{+class IReactorFDSet(Interface):"""Implement me to be able to use L{IFileDescriptor} type resources.@@ -1884,7 +1881,7 @@ class IHalfCloseableProtocol(Interface):This will never be called for TCP connections as TCP does notsupport notification of this type of half-close."""-}}} '''+class IFileDescriptorReceiver(Interface):@@ -1905,7 +1902,7 @@ class IFileDescriptorReceiver(Interface):"""-''' {{{+class IProtocolFactory(Interface):"""Interface for protocol factories.@@ -1995,7 +1992,7 @@ class ITransport(Interface):@return: An L{IAddress} provider."""-}}} '''+class ITCPTransport(ITransport):"""@@ -2116,7 +2113,7 @@ class ISSLTransport(ITCPTransport):Return an object with the peer's certificate info."""-''' {{{+class IProcessTransport(ITransport):"""A process transport.@@ -2345,7 +2342,7 @@ class IMulticastTransport(Interface):"""Leave multicast group, return L{Deferred} of success."""-}}} '''+class IStreamClientEndpoint(Interface):"""@@ -2461,3 +2458,4 @@ class IStreamClientEndpointStringParser(Interface):@return: a client endpoint@rtype: L{IStreamClientEndpoint}"""+}}} '''diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.pyindex 57a111411..32c88ff2d 100644--- a/scrapy/xlib/tx/iweb.py+++ b/scrapy/xlib/tx/iweb.py@@ -15,14 +15,11 @@ from zope.interface import Interface, Attribute#from twisted.internet.interfaces import IPushProducerfrom twisted.web.iweb import (- ICredentialFactory, IBodyProducer,- UNKNOWN_LENGTH,+ IRequest, ICredentialFactory, IBodyProducer, IRenderable, ITemplateLoader,+ IResponse, _IRequestEncoder, _IRequestEncoderFactory, UNKNOWN_LENGTH,)-# newer than 10.0.0-#from twisted.web.iweb import (-# IRequest, IRenderable, ITemplateLoader, IResponse, _IRequestEncoder, _IRequestEncoderFactory,-#)+''' {{{class IRequest(Interface):"""An HTTP request.@@ -328,7 +325,7 @@ class IRequest(Interface):"""-''' {{{+class ICredentialFactory(Interface):"""A credential factory defines a way to generate a particular kind of@@ -432,7 +429,7 @@ class IBodyProducer(IPushProducer):L{Deferred<twisted.internet.defer.Deferred>} returned byC{startProducing} is never fired."""-}}} '''+class IRenderable(Interface):@@ -584,7 +581,7 @@ class _IRequestEncoderFactory(Interface):"""-''' {{{+UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH"}}} '''__all__ = [diff --git a/setup.py b/setup.pyindex 92c114a7a..5e32d4240 100644--- a/setup.py+++ b/setup.py@@ -41,7 +41,7 @@ setup('Topic :: Software Development :: Libraries :: Python Modules',],install_requires=[- 'Twisted>=10.0.0',+ 'Twisted>=13.1.0','w3lib>=1.15.0','queuelib','lxml',