Scrapy, a fast high-level web crawling & scraping framework for Python.
Strip xlib.tx code of Twisted 10
nyove1ea0c433a96
diff --git a/scrapy/xlib/tx/__init__.py b/scrapy/xlib/tx/__init__.pyindex 1ac4e0108..1c9bf09e5 100644--- a/scrapy/xlib/tx/__init__.py+++ b/scrapy/xlib/tx/__init__.py@@ -15,9 +15,9 @@ else:client = endpoints = _Mock()-Agent = client.Agent-ProxyAgent = client.ProxyAgent-ResponseDone = client.ResponseDone-ResponseFailed = client.ResponseFailed-HTTPConnectionPool = client.HTTPConnectionPool-TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint+Agent = client.Agent # since < 11.1+ProxyAgent = client.ProxyAgent # since 11.1+ResponseDone = client.ResponseDone # since 11.1+ResponseFailed = client.ResponseFailed # since 11.1+HTTPConnectionPool = client.HTTPConnectionPool # since 12.1+TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint # since 10.1diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.pyindex 16d0ca6b4..e902d6683 100644--- a/scrapy/xlib/tx/_newclient.py+++ b/scrapy/xlib/tx/_newclient.py@@ -39,12 +39,25 @@ from twisted.internet.defer import Deferred, succeed, fail, maybeDeferredfrom twisted.internet.defer import CancelledErrorfrom twisted.internet.protocol import Protocolfrom twisted.protocols.basic import LineReceiver+from twisted.web.iweb import UNKNOWN_LENGTHfrom twisted.web.http_headers import Headersfrom twisted.web.http import NO_CONTENT, NOT_MODIFIEDfrom twisted.web.http import _DataLoss, PotentialDataLossfrom twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder-from .iweb import IResponse, UNKNOWN_LENGTH+from twisted.web._newclient import (+ BadHeaders, ExcessWrite, ParseError, BadResponseVersion, _WrapperException,+ RequestGenerationFailed, RequestTransmissionFailed,+ WrongBodyLength, ResponseDone, RequestNotSent,+ LengthEnforcingConsumer, makeStatefulDispatcher, ChunkedEncoder,+ TransportProxyProducer,+)+# newer than 10.0.0+#from twisted.web._newclient import (+# ConnectionAborted, ResponseFailed, ResponseNeverReceived, HTTPParser,+# HTTPClientParser, Request, Response, HTTP11ClientProtocol,+#)+from .iweb import IResponse# States HTTPParser can be inSTATUS = 'STATUS'@@ -52,7 +65,7 @@ HEADER = 'HEADER'BODY = 'BODY'DONE = 'DONE'-+''' {{{class BadHeaders(Exception):"""Headers passed to L{Request} were in some way invalid.@@ -117,7 +130,7 @@ class RequestTransmissionFailed(_WrapperException):@ivar reasons: A C{list} of one or more L{Failure} instances giving thereasons the request transmission was considered to have failed."""-+}}} '''class ConnectionAborted(Exception):@@ -126,7 +139,7 @@ class ConnectionAborted(Exception):"""-+''' {{{class WrongBodyLength(Exception):"""An L{IBodyProducer} declared the number of bytes it was going to@@ -142,7 +155,7 @@ class ResponseDone(Exception):protocol passed to L{Response.deliverBody} and indicates that the entireresponse has been delivered."""-+}}} '''class ResponseFailed(_WrapperException):@@ -169,7 +182,7 @@ class ResponseNeverReceived(ResponseFailed):"""-+''' {{{class RequestNotSent(Exception):"""L{RequestNotSent} indicates that an attempt was made to issue a request but@@ -178,7 +191,7 @@ class RequestNotSent(Exception):to send a request using a protocol which is no longer connected to aserver."""-+}}} '''def _callAppFunction(function):@@ -764,7 +777,7 @@ class Request:_callAppFunction(self.bodyProducer.stopProducing)-+''' {{{class LengthEnforcingConsumer:"""An L{IConsumer} proxy which enforces an exact length requirement on the@@ -1188,7 +1201,7 @@ class TransportProxyProducer:"""if self._producer is not None:self._producer.pauseProducing()-+}}} '''class HTTP11ClientProtocol(Protocol):diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.pyindex c3830dc47..396115985 100644--- a/scrapy/xlib/tx/client.py+++ b/scrapy/xlib/tx/client.py@@ -32,19 +32,29 @@ from twisted.internet.interfaces import IProtocolfrom twisted.python import failurefrom twisted.python.components import proxyForInterfacefrom twisted.web import error+from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducerfrom twisted.web.http_headers import Headers-from .endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint-from .iweb import IResponse, UNKNOWN_LENGTH, IBodyProducer+from twisted.web.client import (+ PartialDownloadError,+)+# newer than 10.0.0+#from twisted.web.client import (+# CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent, FileBodyProducer,+# HTTPConnectionPool, Agent, ProxyAgent,+#)+from .endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint+from .iweb import IResponse+''' {{{class PartialDownloadError(error.Error):"""Page was only partially downloaded, we got disconnected in middle.@ivar response: All of the response body which was downloaded."""-+}}} '''class _URL(tuple):"""@@ -136,10 +146,13 @@ def _makeGetterFactory(url, factoryFactory, contextFactory=None,from twisted.web.error import SchemeNotSupportedfrom ._newclient import Request, Response, HTTP11ClientProtocol-from ._newclient import ResponseDone, ResponseFailed-from ._newclient import RequestNotSent, RequestTransmissionFailed+from twisted.web._newclient import ResponseDone+from ._newclient import ResponseFailed+from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed+from twisted.web._newclient import (+ PotentialDataLoss, _WrapperException)from ._newclient import (- ResponseNeverReceived, PotentialDataLoss, _WrapperException)+ ResponseNeverReceived)try:from twisted.internet.ssl import ClientContextFactory@@ -1161,8 +1174,7 @@ def readBody(response):__all__ = [- 'PartialDownloadError', 'HTTPPageGetter', 'HTTPPageDownloader',- 'HTTPClientFactory', 'HTTPDownloader', 'getPage', 'downloadPage',+ 'PartialDownloadError','ResponseDone', 'Response', 'ResponseFailed', 'Agent', 'CookieAgent','ProxyAgent', 'ContentDecoderAgent', 'GzipDecoder', 'RedirectAgent','HTTPConnectionPool', 'readBody']diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.pyindex d8a92ccd0..21a467433 100644--- a/scrapy/xlib/tx/endpoints.py+++ b/scrapy/xlib/tx/endpoints.py@@ -15,23 +15,37 @@ parsed by the L{clientFromString} and L{serverFromString} functions.from __future__ import division, absolute_importimport os-import socket+#import socketfrom zope.interface import implementer, directlyProvidesimport warnings-from twisted.internet import interfaces, defer, error, fdesc, threads+from twisted.internet import interfaces, defer, error, fdescfrom twisted.internet.protocol import (- ClientFactory, Protocol, ProcessProtocol, Factory)+ ClientFactory, Protocol, Factory)+#from twisted.internet import threads, ProcessProtocolfrom twisted.internet.interfaces import IStreamServerEndpointStringParserfrom twisted.internet.interfaces import IStreamClientEndpointStringParserfrom twisted.python.filepath import FilePath-from twisted.python.failure import Failure-from twisted.python import log+#from twisted.python.failure import Failure+#from twisted.python import logfrom twisted.python.components import proxyForInterfacefrom twisted.plugin import IPlugin, getPlugins-from twisted.internet import stdio+#from twisted.internet import stdio++# newer than 10.0.0+#from twisted.internet.endpoints import (+# TCP4ServerEndpoint, TCP6ServerEndpoint, TCP4ClientEndpoint, SSL4ServerEndpoint, SSL4ClientEndpoint,+# UNIXServerEndpoint, UNIXClientEndpoint, AdoptedStreamServerEndpoint, connectProtocol,+# quoteStringArgument,+# serverFromString, #> using newer _parseSSL, _tokenize in _serverParsers+# clientFromString, #> using newer _clientParsers+# _WrappingProtocol, _WrappingFactory, _TCPServerEndpoint,+# _parseTCP, _parseUNIX, _loadCAsFromDir,+# _parseSSL, _tokenize,+# _parseClientTCP, _parseClientSSL, _parseClientUNIX,+#)from .interfaces import IFileDescriptorReceiverdiff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.pyindex f3e4ed5d8..a715d4a05 100644--- a/scrapy/xlib/tx/interfaces.py+++ b/scrapy/xlib/tx/interfaces.py@@ -11,7 +11,28 @@ from __future__ import division, absolute_importfrom zope.interface import Interface, Attribute-+from twisted.internet.interfaces import (+ IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL,+ IReactorUDP, IReactorMulticast, IReactorProcess,+ IReactorTime, IDelayedCall, IReactorThreads, IReactorCore,+ IReactorPluggableResolver, IReactorFDSet,+ IListeningPort, ILoggingContext, IFileDescriptor, IReadDescriptor,+ IWriteDescriptor, IReadWriteDescriptor, IHalfCloseableDescriptor,+ ISystemHandle, IConsumer, IProducer, IPushProducer, IPullProducer,+ IProtocol, IProcessProtocol, IHalfCloseableProtocol,+ IProtocolFactory, ITransport, IProcessTransport, IServiceCollection,+ IUDPTransport, IUNIXDatagramTransport, IUNIXDatagramConnectedTransport,+ IMulticastTransport,+)+# newer than 10.0.0+#from twisted.internet.interfaces import (+# IResolver, IReactorUNIX, IReactorUNIXDatagram, IReactorWin32Events, IReactorSocket,+# IReactorDaemonize, IFileDescriptorReceiver, ITCPTransport, IUNIXTransport,+# ITLSTransport, ISSLTransport, IStreamClientEndpoint, IStreamServerEndpoint,+# IStreamServerEndpointStringParser, IStreamClientEndpointStringParser,+#)++''' {{{class IAddress(Interface):"""An address, e.g. a TCP C{(host, port)}.@@ -74,7 +95,7 @@ class IResolverSimple(Interface):@raise twisted.internet.defer.TimeoutError: Raised (asynchronously)if the name cannot be resolved within the specified timeout period."""-+}}} '''class IResolver(IResolverSimple):@@ -614,7 +635,7 @@ class IResolver(IResolverSimple):"""-+''' {{{class IReactorTCP(Interface):def listenTCP(port, factory, backlog=50, interface=''):@@ -701,7 +722,7 @@ class IReactorSSL(Interface):@param interface: the hostname to bind to, defaults to '' (all)"""-+}}} '''class IReactorUNIX(Interface):@@ -829,7 +850,7 @@ class IReactorWin32Events(Interface):"""-+''' {{{class IReactorUDP(Interface):"""UDP socket methods.@@ -868,7 +889,7 @@ class IReactorMulticast(Interface):@see: L{twisted.internet.interfaces.IMulticastTransport}@see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html}"""-+}}} '''class IReactorSocket(Interface):@@ -970,7 +991,7 @@ class IReactorSocket(Interface):"""-+''' {{{class IReactorProcess(Interface):def spawnProcess(processProtocol, executable, args=(), env={}, path=None,@@ -1347,7 +1368,7 @@ class IReactorPluggableResolver(Interface):@return: The previously installed resolver."""-+}}} '''class IReactorDaemonize(Interface):"""@@ -1379,7 +1400,7 @@ class IReactorDaemonize(Interface):"""-+''' {{{class IReactorFDSet(Interface):"""Implement me to be able to use L{IFileDescriptor} type resources.@@ -1863,7 +1884,7 @@ class IHalfCloseableProtocol(Interface):This will never be called for TCP connections as TCP does notsupport notification of this type of half-close."""-+}}} '''class IFileDescriptorReceiver(Interface):@@ -1884,7 +1905,7 @@ class IFileDescriptorReceiver(Interface):"""-+''' {{{class IProtocolFactory(Interface):"""Interface for protocol factories.@@ -1974,7 +1995,7 @@ class ITransport(Interface):@return: An L{IAddress} provider."""-+}}} '''class ITCPTransport(ITransport):"""@@ -2095,7 +2116,7 @@ class ISSLTransport(ITCPTransport):Return an object with the peer's certificate info."""-+''' {{{class IProcessTransport(ITransport):"""A process transport.@@ -2324,7 +2345,7 @@ class IMulticastTransport(Interface):"""Leave multicast group, return L{Deferred} of success."""-+}}} '''class IStreamClientEndpoint(Interface):"""diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.pyindex ddcb6ed7a..57a111411 100644--- a/scrapy/xlib/tx/iweb.py+++ b/scrapy/xlib/tx/iweb.py@@ -12,8 +12,16 @@ Interface definitions for L{twisted.web}.from zope.interface import Interface, Attribute-from twisted.internet.interfaces import IPushProducer+#from twisted.internet.interfaces import IPushProducer+from twisted.web.iweb import (+ ICredentialFactory, IBodyProducer,+ UNKNOWN_LENGTH,+)+# newer than 10.0.0+#from twisted.web.iweb import (+# IRequest, IRenderable, ITemplateLoader, IResponse, _IRequestEncoder, _IRequestEncoderFactory,+#)class IRequest(Interface):"""@@ -320,7 +328,7 @@ class IRequest(Interface):"""-+''' {{{class ICredentialFactory(Interface):"""A credential factory defines a way to generate a particular kind of@@ -424,7 +432,7 @@ class IBodyProducer(IPushProducer):L{Deferred<twisted.internet.defer.Deferred>} returned byC{startProducing} is never fired."""-+}}} '''class IRenderable(Interface):@@ -576,9 +584,9 @@ class _IRequestEncoderFactory(Interface):"""-+''' {{{UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH"-+}}} '''__all__ = ["ICredentialFactory", "IRequest","IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder",