Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove obsolete xlib code for Twisted 13.1.0
nyov985755d1fe0a
diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.pyindex d20eda34f..39cd20f95 100644--- a/scrapy/xlib/tx/_newclient.py+++ b/scrapy/xlib/tx/_newclient.py@@ -38,9 +38,7 @@ from twisted.internet.error import ConnectionDonefrom twisted.internet.defer import Deferred, succeed, fail, maybeDeferredfrom twisted.internet.defer import CancelledErrorfrom twisted.internet.protocol import Protocol-#from twisted.protocols.basic import LineReceiverfrom twisted.web.iweb import UNKNOWN_LENGTH, IResponse-#from twisted.web.http_headers import Headersfrom twisted.web.http import NO_CONTENT, NOT_MODIFIEDfrom twisted.web.http import _DataLoss, PotentialDataLossfrom twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder@@ -59,1467 +57,3 @@ STATUS = 'STATUS'HEADER = 'HEADER'BODY = 'BODY'DONE = 'DONE'--''' {{{-class BadHeaders(Exception):- """- Headers passed to L{Request} were in some way invalid.- """----class ExcessWrite(Exception):- """- The body L{IBodyProducer} for a request tried to write data after- indicating it had finished writing data.- """---class ParseError(Exception):- """- Some received data could not be parsed.-- @ivar data: The string which could not be parsed.- """- def __init__(self, reason, data):- Exception.__init__(self, reason, data)- self.data = data----class BadResponseVersion(ParseError):- """- The version string in a status line was unparsable.- """----class _WrapperException(Exception):- """- L{_WrapperException} is the base exception type for exceptions which- include one or more other exceptions as the low-level causes.-- @ivar reasons: A list of exceptions. See subclass documentation for more- details.- """- def __init__(self, reasons):- Exception.__init__(self, reasons)- self.reasons = reasons----class RequestGenerationFailed(_WrapperException):- """- There was an error while creating the bytes which make up a request.-- @ivar reasons: A C{list} of one or more L{Failure} instances giving the- reasons the request generation was considered to have failed.- """----class RequestTransmissionFailed(_WrapperException):- """- There was an error while sending the bytes which make up a request.-- @ivar reasons: A C{list} of one or more L{Failure} instances giving the- reasons the request transmission was considered to have failed.- """----class ConnectionAborted(Exception):- """- The connection was explicitly aborted by application code.- """----class WrongBodyLength(Exception):- """- An L{IBodyProducer} declared the number of bytes it was going to- produce (via its C{length} attribute) and then produced a different number- of bytes.- """----class ResponseDone(Exception):- """- L{ResponseDone} may be passed to L{IProtocol.connectionLost} on the- protocol passed to L{Response.deliverBody} and indicates that the entire- response has been delivered.- """----class ResponseFailed(_WrapperException):- """- L{ResponseFailed} indicates that all of the response to a request was not- received for some reason.-- @ivar reasons: A C{list} of one or more L{Failure} instances giving the- reasons the response was considered to have failed.-- @ivar response: If specified, the L{Response} received from the server (and- in particular the status code and the headers).- """-- def __init__(self, reasons, response=None):- _WrapperException.__init__(self, reasons)- self.response = response----class ResponseNeverReceived(ResponseFailed):- """- A L{ResponseFailed} that knows no response bytes at all have been received.- """----class RequestNotSent(Exception):- """- L{RequestNotSent} indicates that an attempt was made to issue a request but- for reasons unrelated to the details of the request itself, the request- could not be sent. For example, this may indicate that an attempt was made- to send a request using a protocol which is no longer connected to a- server.- """----def _callAppFunction(function):- """- Call C{function}. If it raises an exception, log it with a minimal- description of the source.-- @return: C{None}- """- try:- function()- except:- log.err(None, "Unexpected exception from %s" % (- fullyQualifiedName(function),))----class HTTPParser(LineReceiver):- """- L{HTTPParser} handles the parsing side of HTTP processing. With a suitable- subclass, it can parse either the client side or the server side of the- connection.-- @ivar headers: All of the non-connection control message headers yet- received.-- @ivar state: State indicator for the response parsing state machine. One- of C{STATUS}, C{HEADER}, C{BODY}, C{DONE}.-- @ivar _partialHeader: C{None} or a C{list} of the lines of a multiline- header while that header is being received.- """-- # NOTE: According to HTTP spec, we're supposed to eat the- # 'Proxy-Authenticate' and 'Proxy-Authorization' headers also, but that- # doesn't sound like a good idea to me, because it makes it impossible to- # have a non-authenticating transparent proxy in front of an authenticating- # proxy. An authenticating proxy can eat them itself. -jknight- #- # Further, quoting- # http://homepages.tesco.net/J.deBoynePollard/FGA/web-proxy-connection-header.html- # regarding the 'Proxy-Connection' header:- #- # The Proxy-Connection: header is a mistake in how some web browsers- # use HTTP. Its name is the result of a false analogy. It is not a- # standard part of the protocol. There is a different standard- # protocol mechanism for doing what it does. And its existence- # imposes a requirement upon HTTP servers such that no proxy HTTP- # server can be standards-conforming in practice.- #- # -exarkun-- # Some servers (like http://news.ycombinator.com/) return status lines and- # HTTP headers delimited by \n instead of \r\n.- delimiter = '\n'-- CONNECTION_CONTROL_HEADERS = set([- 'content-length', 'connection', 'keep-alive', 'te', 'trailers',- 'transfer-encoding', 'upgrade', 'proxy-connection'])-- def connectionMade(self):- self.headers = Headers()- self.connHeaders = Headers()- self.state = STATUS- self._partialHeader = None--- def switchToBodyMode(self, decoder):- """- Switch to body parsing mode - interpret any more bytes delivered as- part of the message body and deliver them to the given decoder.- """- if self.state == BODY:- raise RuntimeError("already in body mode")-- self.bodyDecoder = decoder- self.state = BODY- self.setRawMode()--- def lineReceived(self, line):- """- Handle one line from a response.- """- # Handle the normal CR LF case.- if line[-1:] == '\r':- line = line[:-1]-- if self.state == STATUS:- self.statusReceived(line)- self.state = HEADER- elif self.state == HEADER:- if not line or line[0] not in ' \t':- if self._partialHeader is not None:- header = ''.join(self._partialHeader)- name, value = header.split(':', 1)- value = value.strip()- self.headerReceived(name, value)- if not line:- # Empty line means the header section is over.- self.allHeadersReceived()- else:- # Line not beginning with LWS is another header.- self._partialHeader = [line]- else:- # A line beginning with LWS is a continuation of a header- # begun on a previous line.- self._partialHeader.append(line)--- def rawDataReceived(self, data):- """- Pass data from the message body to the body decoder object.- """- self.bodyDecoder.dataReceived(data)--- def isConnectionControlHeader(self, name):- """- Return C{True} if the given lower-cased name is the name of a- connection control header (rather than an entity header).-- According to RFC 2616, section 14.10, the tokens in the Connection- header are probably relevant here. However, I am not sure what the- practical consequences of either implementing or ignoring that are.- So I leave it unimplemented for the time being.- """- return name in self.CONNECTION_CONTROL_HEADERS--- def statusReceived(self, status):- """- Callback invoked whenever the first line of a new message is received.- Override this.-- @param status: The first line of an HTTP request or response message- without trailing I{CR LF}.- @type status: C{str}- """--- def headerReceived(self, name, value):- """- Store the given header in C{self.headers}.- """- name = name.lower()- if self.isConnectionControlHeader(name):- headers = self.connHeaders- else:- headers = self.headers- headers.addRawHeader(name, value)--- def allHeadersReceived(self):- """- Callback invoked after the last header is passed to C{headerReceived}.- Override this to change to the C{BODY} or C{DONE} state.- """- self.switchToBodyMode(None)----class HTTPClientParser(HTTPParser):- """- An HTTP parser which only handles HTTP responses.-- @ivar request: The request with which the expected response is associated.- @type request: L{Request}-- @ivar NO_BODY_CODES: A C{set} of response codes which B{MUST NOT} have a- body.-- @ivar finisher: A callable to invoke when this response is fully parsed.-- @ivar _responseDeferred: A L{Deferred} which will be called back with the- response when all headers in the response have been received.- Thereafter, C{None}.-- @ivar _everReceivedData: C{True} if any bytes have been received.- """- NO_BODY_CODES = set([NO_CONTENT, NOT_MODIFIED])-- _transferDecoders = {- 'chunked': _ChunkedTransferDecoder,- }-- bodyDecoder = None-- def __init__(self, request, finisher):- self.request = request- self.finisher = finisher- self._responseDeferred = Deferred()- self._everReceivedData = False--- def dataReceived(self, data):- """- Override so that we know if any response has been received.- """- self._everReceivedData = True- HTTPParser.dataReceived(self, data)--- def parseVersion(self, strversion):- """- Parse version strings of the form Protocol '/' Major '.' Minor. E.g.- 'HTTP/1.1'. Returns (protocol, major, minor). Will raise ValueError- on bad syntax.- """- try:- proto, strnumber = strversion.split('/')- major, minor = strnumber.split('.')- major, minor = int(major), int(minor)- except ValueError as e:- raise BadResponseVersion(str(e), strversion)- if major < 0 or minor < 0:- raise BadResponseVersion("version may not be negative", strversion)- return (proto, major, minor)--- def statusReceived(self, status):- """- Parse the status line into its components and create a response object- to keep track of this response's state.- """- parts = status.split(' ', 2)- if len(parts) != 3:- raise ParseError("wrong number of parts", status)-- try:- statusCode = int(parts[1])- except ValueError:- raise ParseError("non-integer status code", status)-- self.response = Response(- self.parseVersion(parts[0]),- statusCode,- parts[2],- self.headers,- self.transport)--- def _finished(self, rest):- """- Called to indicate that an entire response has been received. No more- bytes will be interpreted by this L{HTTPClientParser}. Extra bytes are- passed up and the state of this L{HTTPClientParser} is set to I{DONE}.-- @param rest: A C{str} giving any extra bytes delivered to this- L{HTTPClientParser} which are not part of the response being- parsed.- """- self.state = DONE- self.finisher(rest)--- def isConnectionControlHeader(self, name):- """- Content-Length in the response to a HEAD request is an entity header,- not a connection control header.- """- if self.request.method == 'HEAD' and name == 'content-length':- return False- return HTTPParser.isConnectionControlHeader(self, name)--- def allHeadersReceived(self):- """- Figure out how long the response body is going to be by examining- headers and stuff.- """- if (self.response.code in self.NO_BODY_CODES- or self.request.method == 'HEAD'):- self.response.length = 0- self._finished(self.clearLineBuffer())- else:- transferEncodingHeaders = self.connHeaders.getRawHeaders(- 'transfer-encoding')- if transferEncodingHeaders:-- # This could be a KeyError. However, that would mean we do not- # know how to decode the response body, so failing the request- # is as good a behavior as any. Perhaps someday we will want- # to normalize/document/test this specifically, but failing- # seems fine to me for now.- transferDecoder = self._transferDecoders[transferEncodingHeaders[0].lower()]-- # If anyone ever invents a transfer encoding other than- # chunked (yea right), and that transfer encoding can predict- # the length of the response body, it might be sensible to- # allow the transfer decoder to set the response object's- # length attribute.- else:- contentLengthHeaders = self.connHeaders.getRawHeaders('content-length')- if contentLengthHeaders is None:- contentLength = None- elif len(contentLengthHeaders) == 1:- contentLength = int(contentLengthHeaders[0])- self.response.length = contentLength- else:- # "HTTP Message Splitting" or "HTTP Response Smuggling"- # potentially happening. Or it's just a buggy server.- raise ValueError(- "Too many Content-Length headers; response is invalid")-- if contentLength == 0:- self._finished(self.clearLineBuffer())- transferDecoder = None- else:- transferDecoder = lambda x, y: _IdentityTransferDecoder(- contentLength, x, y)-- if transferDecoder is None:- self.response._bodyDataFinished()- else:- # Make sure as little data as possible from the response body- # gets delivered to the response object until the response- # object actually indicates it is ready to handle bytes- # (probably because an application gave it a way to interpret- # them).- self.transport.pauseProducing()- self.switchToBodyMode(transferDecoder(- self.response._bodyDataReceived,- self._finished))-- # This must be last. If it were first, then application code might- # change some state (for example, registering a protocol to receive the- # response body). Then the pauseProducing above would be wrong since- # the response is ready for bytes and nothing else would ever resume- # the transport.- self._responseDeferred.callback(self.response)- del self._responseDeferred--- def connectionLost(self, reason):- if self.bodyDecoder is not None:- try:- try:- self.bodyDecoder.noMoreData()- except PotentialDataLoss:- self.response._bodyDataFinished(Failure())- except _DataLoss:- self.response._bodyDataFinished(- Failure(ResponseFailed([reason, Failure()],- self.response)))- else:- self.response._bodyDataFinished()- except:- # Handle exceptions from both the except suites and the else- # suite. Those functions really shouldn't raise exceptions,- # but maybe there's some buggy application code somewhere- # making things difficult.- log.err()- elif self.state != DONE:- if self._everReceivedData:- exceptionClass = ResponseFailed- else:- exceptionClass = ResponseNeverReceived- self._responseDeferred.errback(Failure(exceptionClass([reason])))- del self._responseDeferred----class Request:- """- A L{Request} instance describes an HTTP request to be sent to an HTTP- server.-- @ivar method: The HTTP method to for this request, ex: 'GET', 'HEAD',- 'POST', etc.- @type method: C{str}-- @ivar uri: The relative URI of the resource to request. For example,- C{'/foo/bar?baz=quux'}.- @type uri: C{str}-- @ivar headers: Headers to be sent to the server. It is important to- note that this object does not create any implicit headers. So it- is up to the HTTP Client to add required headers such as 'Host'.- @type headers: L{twisted.web.http_headers.Headers}-- @ivar bodyProducer: C{None} or an L{IBodyProducer} provider which- produces the content body to send to the remote HTTP server.-- @ivar persistent: Set to C{True} when you use HTTP persistent connection.- @type persistent: C{bool}- """- def __init__(self, method, uri, headers, bodyProducer, persistent=False):- self.method = method- self.uri = uri- self.headers = headers- self.bodyProducer = bodyProducer- self.persistent = persistent--- def _writeHeaders(self, transport, TEorCL):- hosts = self.headers.getRawHeaders('host', ())- if len(hosts) != 1:- raise BadHeaders("Exactly one Host header required")-- # In the future, having the protocol version be a parameter to this- # method would probably be good. It would be nice if this method- # weren't limited to issuing HTTP/1.1 requests.- requestLines = []- requestLines.append(- '%s %s HTTP/1.1\r\n' % (self.method, self.uri))- if not self.persistent:- requestLines.append('Connection: close\r\n')- if TEorCL is not None:- requestLines.append(TEorCL)- for name, values in self.headers.getAllRawHeaders():- requestLines.extend(['%s: %s\r\n' % (name, v) for v in values])- requestLines.append('\r\n')- transport.writeSequence(requestLines)--- def _writeToChunked(self, transport):- """- Write this request to the given transport using chunked- transfer-encoding to frame the body.- """- self._writeHeaders(transport, 'Transfer-Encoding: chunked\r\n')- encoder = ChunkedEncoder(transport)- encoder.registerProducer(self.bodyProducer, True)- d = self.bodyProducer.startProducing(encoder)-- def cbProduced(ignored):- encoder.unregisterProducer()- def ebProduced(err):- encoder._allowNoMoreWrites()- # Don't call the encoder's unregisterProducer because it will write- # a zero-length chunk. This would indicate to the server that the- # request body is complete. There was an error, though, so we- # don't want to do that.- transport.unregisterProducer()- return err- d.addCallbacks(cbProduced, ebProduced)- return d--- def _writeToContentLength(self, transport):- """- Write this request to the given transport using content-length to frame- the body.- """- self._writeHeaders(- transport,- 'Content-Length: %d\r\n' % (self.bodyProducer.length,))-- # This Deferred is used to signal an error in the data written to the- # encoder below. It can only errback and it will only do so before too- # many bytes have been written to the encoder and before the producer- # Deferred fires.- finishedConsuming = Deferred()-- # This makes sure the producer writes the correct number of bytes for- # the request body.- encoder = LengthEnforcingConsumer(- self.bodyProducer, transport, finishedConsuming)-- transport.registerProducer(self.bodyProducer, True)-- finishedProducing = self.bodyProducer.startProducing(encoder)-- def combine(consuming, producing):- # This Deferred is returned and will be fired when the first of- # consuming or producing fires. If it's cancelled, forward that- # cancellation to the producer.- def cancelConsuming(ign):- finishedProducing.cancel()- ultimate = Deferred(cancelConsuming)-- # Keep track of what has happened so far. This initially- # contains None, then an integer uniquely identifying what- # sequence of events happened. See the callbacks and errbacks- # defined below for the meaning of each value.- state = [None]-- def ebConsuming(err):- if state == [None]:- # The consuming Deferred failed first. This means the- # overall writeTo Deferred is going to errback now. The- # producing Deferred should not fire later (because the- # consumer should have called stopProducing on the- # producer), but if it does, a callback will be ignored- # and an errback will be logged.- state[0] = 1- ultimate.errback(err)- else:- # The consuming Deferred errbacked after the producing- # Deferred fired. This really shouldn't ever happen.- # If it does, I goofed. Log the error anyway, just so- # there's a chance someone might notice and complain.- log.err(- err,- "Buggy state machine in %r/[%d]: "- "ebConsuming called" % (self, state[0]))-- def cbProducing(result):- if state == [None]:- # The producing Deferred succeeded first. Nothing will- # ever happen to the consuming Deferred. Tell the- # encoder we're done so it can check what the producer- # wrote and make sure it was right.- state[0] = 2- try:- encoder._noMoreWritesExpected()- except:- # Fail the overall writeTo Deferred - something the- # producer did was wrong.- ultimate.errback()- else:- # Success - succeed the overall writeTo Deferred.- ultimate.callback(None)- # Otherwise, the consuming Deferred already errbacked. The- # producing Deferred wasn't supposed to fire, but it did- # anyway. It's buggy, but there's not really anything to be- # done about it. Just ignore this result.-- def ebProducing(err):- if state == [None]:- # The producing Deferred failed first. This means the- # overall writeTo Deferred is going to errback now.- # Tell the encoder that we're done so it knows to reject- # further writes from the producer (which should not- # happen, but the producer may be buggy).- state[0] = 3- encoder._allowNoMoreWrites()- ultimate.errback(err)- else:- # The producing Deferred failed after the consuming- # Deferred failed. It shouldn't have, so it's buggy.- # Log the exception in case anyone who can fix the code- # is watching.- log.err(err, "Producer is buggy")-- consuming.addErrback(ebConsuming)- producing.addCallbacks(cbProducing, ebProducing)-- return ultimate-- d = combine(finishedConsuming, finishedProducing)- def f(passthrough):- # Regardless of what happens with the overall Deferred, once it- # fires, the producer registered way up above the definition of- # combine should be unregistered.- transport.unregisterProducer()- return passthrough- d.addBoth(f)- return d--- def writeTo(self, transport):- """- Format this L{Request} as an HTTP/1.1 request and write it to the given- transport. If bodyProducer is not None, it will be associated with an- L{IConsumer}.-- @return: A L{Deferred} which fires with C{None} when the request has- been completely written to the transport or with a L{Failure} if- there is any problem generating the request bytes.- """- if self.bodyProducer is not None:- if self.bodyProducer.length is UNKNOWN_LENGTH:- return self._writeToChunked(transport)- else:- return self._writeToContentLength(transport)- else:- self._writeHeaders(transport, None)- return succeed(None)--- def stopWriting(self):- """- Stop writing this request to the transport. This can only be called- after C{writeTo} and before the L{Deferred} returned by C{writeTo}- fires. It should cancel any asynchronous task started by C{writeTo}.- The L{Deferred} returned by C{writeTo} need not be fired if this method- is called.- """- # If bodyProducer is None, then the Deferred returned by writeTo has- # fired already and this method cannot be called.- _callAppFunction(self.bodyProducer.stopProducing)----class LengthEnforcingConsumer:- """- An L{IConsumer} proxy which enforces an exact length requirement on the- total data written to it.-- @ivar _length: The number of bytes remaining to be written.-- @ivar _producer: The L{IBodyProducer} which is writing to this- consumer.-- @ivar _consumer: The consumer to which at most C{_length} bytes will be- forwarded.-- @ivar _finished: A L{Deferred} which will be fired with a L{Failure} if too- many bytes are written to this consumer.- """- def __init__(self, producer, consumer, finished):- self._length = producer.length- self._producer = producer- self._consumer = consumer- self._finished = finished--- def _allowNoMoreWrites(self):- """- Indicate that no additional writes are allowed. Attempts to write- after calling this method will be met with an exception.- """- self._finished = None--- def write(self, bytes):- """- Write C{bytes} to the underlying consumer unless- C{_noMoreWritesExpected} has been called or there are/have been too- many bytes.- """- if self._finished is None:- # No writes are supposed to happen any more. Try to convince the- # calling code to stop calling this method by calling its- # stopProducing method and then throwing an exception at it. This- # exception isn't documented as part of the API because you're- # never supposed to expect it: only buggy code will ever receive- # it.- self._producer.stopProducing()- raise ExcessWrite()-- if len(bytes) <= self._length:- self._length -= len(bytes)- self._consumer.write(bytes)- else:- # No synchronous exception is raised in *this* error path because- # we still have _finished which we can use to report the error to a- # better place than the direct caller of this method (some- # arbitrary application code).- _callAppFunction(self._producer.stopProducing)- self._finished.errback(WrongBodyLength("too many bytes written"))- self._allowNoMoreWrites()--- def _noMoreWritesExpected(self):- """- Called to indicate no more bytes will be written to this consumer.- Check to see that the correct number have been written.-- @raise WrongBodyLength: If not enough bytes have been written.- """- if self._finished is not None:- self._allowNoMoreWrites()- if self._length:- raise WrongBodyLength("too few bytes written")----def makeStatefulDispatcher(name, template):- """- Given a I{dispatch} name and a function, return a function which can be- used as a method and which, when called, will call another method defined- on the instance and return the result. The other method which is called is- determined by the value of the C{_state} attribute of the instance.-- @param name: A string which is used to construct the name of the subsidiary- method to invoke. The subsidiary method is named like C{'_%s_%s' %- (name, _state)}.-- @param template: A function object which is used to give the returned- function a docstring.-- @return: The dispatcher function.- """- def dispatcher(self, *args, **kwargs):- func = getattr(self, '_' + name + '_' + self._state, None)- if func is None:- raise RuntimeError(- "%r has no %s method in state %s" % (self, name, self._state))- return func(*args, **kwargs)- dispatcher.__doc__ = template.__doc__- return dispatcher----class Response:- """- A L{Response} instance describes an HTTP response received from an HTTP- server.-- L{Response} should not be subclassed or instantiated.-- @ivar _transport: The transport which is delivering this response.-- @ivar _bodyProtocol: The L{IProtocol} provider to which the body is- delivered. C{None} before one has been registered with- C{deliverBody}.-- @ivar _bodyBuffer: A C{list} of the strings passed to C{bodyDataReceived}- before C{deliverBody} is called. C{None} afterwards.-- @ivar _state: Indicates what state this L{Response} instance is in,- particularly with respect to delivering bytes from the response body- to an application-suppled protocol object. This may be one of- C{'INITIAL'}, C{'CONNECTED'}, C{'DEFERRED_CLOSE'}, or C{'FINISHED'},- with the following meanings:-- - INITIAL: This is the state L{Response} objects start in. No- protocol has yet been provided and the underlying transport may- still have bytes to deliver to it.-- - DEFERRED_CLOSE: If the underlying transport indicates all bytes- have been delivered but no application-provided protocol is yet- available, the L{Response} moves to this state. Data is- buffered and waiting for a protocol to be delivered to.-- - CONNECTED: If a protocol is provided when the state is INITIAL,- the L{Response} moves to this state. Any buffered data is- delivered and any data which arrives from the transport- subsequently is given directly to the protocol.-- - FINISHED: If a protocol is provided in the DEFERRED_CLOSE state,- the L{Response} moves to this state after delivering all- buffered data to the protocol. Otherwise, if the L{Response} is- in the CONNECTED state, if the transport indicates there is no- more data, the L{Response} moves to this state. Nothing else- can happen once the L{Response} is in this state.- """- implements(IResponse)-- length = UNKNOWN_LENGTH-- _bodyProtocol = None- _bodyFinished = False-- def __init__(self, version, code, phrase, headers, _transport):- self.version = version- self.code = code- self.phrase = phrase- self.headers = headers- self._transport = _transport- self._bodyBuffer = []- self._state = 'INITIAL'--- def deliverBody(self, protocol):- """- Dispatch the given L{IProtocol} depending of the current state of the- response.- """- deliverBody = makeStatefulDispatcher('deliverBody', deliverBody)--- def _deliverBody_INITIAL(self, protocol):- """- Deliver any buffered data to C{protocol} and prepare to deliver any- future data to it. Move to the C{'CONNECTED'} state.- """- # Now that there's a protocol to consume the body, resume the- # transport. It was previously paused by HTTPClientParser to avoid- # reading too much data before it could be handled.- self._transport.resumeProducing()-- protocol.makeConnection(self._transport)- self._bodyProtocol = protocol- for data in self._bodyBuffer:- self._bodyProtocol.dataReceived(data)- self._bodyBuffer = None- self._state = 'CONNECTED'--- def _deliverBody_CONNECTED(self, protocol):- """- It is invalid to attempt to deliver data to a protocol when it is- already being delivered to another protocol.- """- raise RuntimeError(- "Response already has protocol %r, cannot deliverBody "- "again" % (self._bodyProtocol,))--- def _deliverBody_DEFERRED_CLOSE(self, protocol):- """- Deliver any buffered data to C{protocol} and then disconnect the- protocol. Move to the C{'FINISHED'} state.- """- # Unlike _deliverBody_INITIAL, there is no need to resume the- # transport here because all of the response data has been received- # already. Some higher level code may want to resume the transport if- # that code expects further data to be received over it.-- protocol.makeConnection(self._transport)-- for data in self._bodyBuffer:- protocol.dataReceived(data)- self._bodyBuffer = None- protocol.connectionLost(self._reason)- self._state = 'FINISHED'--- def _deliverBody_FINISHED(self, protocol):- """- It is invalid to attempt to deliver data to a protocol after the- response body has been delivered to another protocol.- """- raise RuntimeError(- "Response already finished, cannot deliverBody now.")--- def _bodyDataReceived(self, data):- """- Called by HTTPClientParser with chunks of data from the response body.- They will be buffered or delivered to the protocol passed to- deliverBody.- """- _bodyDataReceived = makeStatefulDispatcher('bodyDataReceived',- _bodyDataReceived)--- def _bodyDataReceived_INITIAL(self, data):- """- Buffer any data received for later delivery to a protocol passed to- C{deliverBody}.-- Little or no data should be buffered by this method, since the- transport has been paused and will not be resumed until a protocol- is supplied.- """- self._bodyBuffer.append(data)--- def _bodyDataReceived_CONNECTED(self, data):- """- Deliver any data received to the protocol to which this L{Response}- is connected.- """- self._bodyProtocol.dataReceived(data)--- def _bodyDataReceived_DEFERRED_CLOSE(self, data):- """- It is invalid for data to be delivered after it has been indicated- that the response body has been completely delivered.- """- raise RuntimeError("Cannot receive body data after _bodyDataFinished")--- def _bodyDataReceived_FINISHED(self, data):- """- It is invalid for data to be delivered after the response body has- been delivered to a protocol.- """- raise RuntimeError("Cannot receive body data after protocol disconnected")--- def _bodyDataFinished(self, reason=None):- """- Called by HTTPClientParser when no more body data is available. If the- optional reason is supplied, this indicates a problem or potential- problem receiving all of the response body.- """- _bodyDataFinished = makeStatefulDispatcher('bodyDataFinished',- _bodyDataFinished)--- def _bodyDataFinished_INITIAL(self, reason=None):- """- Move to the C{'DEFERRED_CLOSE'} state to wait for a protocol to- which to deliver the response body.- """- self._state = 'DEFERRED_CLOSE'- if reason is None:- reason = Failure(ResponseDone("Response body fully received"))- self._reason = reason--- def _bodyDataFinished_CONNECTED(self, reason=None):- """- Disconnect the protocol and move to the C{'FINISHED'} state.- """- if reason is None:- reason = Failure(ResponseDone("Response body fully received"))- self._bodyProtocol.connectionLost(reason)- self._bodyProtocol = None- self._state = 'FINISHED'--- def _bodyDataFinished_DEFERRED_CLOSE(self):- """- It is invalid to attempt to notify the L{Response} of the end of the- response body data more than once.- """- raise RuntimeError("Cannot finish body data more than once")--- def _bodyDataFinished_FINISHED(self):- """- It is invalid to attempt to notify the L{Response} of the end of the- response body data more than once.- """- raise RuntimeError("Cannot finish body data after protocol disconnected")----class ChunkedEncoder:- """- Helper object which exposes L{IConsumer} on top of L{HTTP11ClientProtocol}- for streaming request bodies to the server.- """- implements(IConsumer)-- def __init__(self, transport):- self.transport = transport--- def _allowNoMoreWrites(self):- """- Indicate that no additional writes are allowed. Attempts to write- after calling this method will be met with an exception.- """- self.transport = None--- def registerProducer(self, producer, streaming):- """- Register the given producer with C{self.transport}.- """- self.transport.registerProducer(producer, streaming)--- def write(self, data):- """- Write the given request body bytes to the transport using chunked- encoding.-- @type data: C{str}- """- if self.transport is None:- raise ExcessWrite()- self.transport.writeSequence(("%x\r\n" % len(data), data, "\r\n"))--- def unregisterProducer(self):- """- Indicate that the request body is complete and finish the request.- """- self.write('')- self.transport.unregisterProducer()- self._allowNoMoreWrites()----class TransportProxyProducer:- """- An L{IPushProducer} implementation which wraps another such thing and- proxies calls to it until it is told to stop.-- @ivar _producer: The wrapped L{IPushProducer} provider or C{None} after- this proxy has been stopped.- """- implements(IPushProducer)-- # LineReceiver uses this undocumented attribute of transports to decide- # when to stop calling lineReceived or rawDataReceived (if it finds it to- # be true, it doesn't bother to deliver any more data). Set disconnecting- # to False here and never change it to true so that all data is always- # delivered to us and so that LineReceiver doesn't fail with an- # AttributeError.- disconnecting = False-- def __init__(self, producer):- self._producer = producer--- def _stopProxying(self):- """- Stop forwarding calls of L{IPushProducer} methods to the underlying- L{IPushProvider} provider.- """- self._producer = None--- def stopProducing(self):- """- Proxy the stoppage to the underlying producer, unless this proxy has- been stopped.- """- if self._producer is not None:- self._producer.stopProducing()--- def resumeProducing(self):- """- Proxy the resumption to the underlying producer, unless this proxy has- been stopped.- """- if self._producer is not None:- self._producer.resumeProducing()--- def pauseProducing(self):- """- Proxy the pause to the underlying producer, unless this proxy has been- stopped.- """- if self._producer is not None:- self._producer.pauseProducing()----class HTTP11ClientProtocol(Protocol):- """- L{HTTP11ClientProtocol} is an implementation of the HTTP 1.1 client- protocol. It supports as few features as possible.-- @ivar _parser: After a request is issued, the L{HTTPClientParser} to- which received data making up the response to that request is- delivered.-- @ivar _finishedRequest: After a request is issued, the L{Deferred} which- will fire when a L{Response} object corresponding to that request is- available. This allows L{HTTP11ClientProtocol} to fail the request- if there is a connection or parsing problem.-- @ivar _currentRequest: After a request is issued, the L{Request}- instance used to make that request. This allows- L{HTTP11ClientProtocol} to stop request generation if necessary (for- example, if the connection is lost).-- @ivar _transportProxy: After a request is issued, the- L{TransportProxyProducer} to which C{_parser} is connected. This- allows C{_parser} to pause and resume the transport in a way which- L{HTTP11ClientProtocol} can exert some control over.-- @ivar _responseDeferred: After a request is issued, the L{Deferred} from- C{_parser} which will fire with a L{Response} when one has been- received. This is eventually chained with C{_finishedRequest}, but- only in certain cases to avoid double firing that Deferred.-- @ivar _state: Indicates what state this L{HTTP11ClientProtocol} instance- is in with respect to transmission of a request and reception of a- response. This may be one of the following strings:-- - QUIESCENT: This is the state L{HTTP11ClientProtocol} instances- start in. Nothing is happening: no request is being sent and no- response is being received or expected.-- - TRANSMITTING: When a request is made (via L{request}), the- instance moves to this state. L{Request.writeTo} has been used- to start to send a request but it has not yet finished.-- - TRANSMITTING_AFTER_RECEIVING_RESPONSE: The server has returned a- complete response but the request has not yet been fully sent- yet. The instance will remain in this state until the request- is fully sent.-- - GENERATION_FAILED: There was an error while the request. The- request was not fully sent to the network.-- - WAITING: The request was fully sent to the network. The- instance is now waiting for the response to be fully received.-- - ABORTING: Application code has requested that the HTTP connection- be aborted.-- - CONNECTION_LOST: The connection has been lost.-- @ivar _abortDeferreds: A list of C{Deferred} instances that will fire when- the connection is lost.- """- _state = 'QUIESCENT'- _parser = None- _finishedRequest = None- _currentRequest = None- _transportProxy = None- _responseDeferred = None--- def __init__(self, quiescentCallback=lambda c: None):- self._quiescentCallback = quiescentCallback- self._abortDeferreds = []--- @property- def state(self):- return self._state--- def request(self, request):- """- Issue C{request} over C{self.transport} and return a L{Deferred} which- will fire with a L{Response} instance or an error.-- @param request: The object defining the parameters of the request to- issue.- @type request: L{Request}-- @rtype: L{Deferred}- @return: The deferred may errback with L{RequestGenerationFailed} if- the request was not fully written to the transport due to a local- error. It may errback with L{RequestTransmissionFailed} if it was- not fully written to the transport due to a network error. It may- errback with L{ResponseFailed} if the request was sent (not- necessarily received) but some or all of the response was lost. It- may errback with L{RequestNotSent} if it is not possible to send- any more requests using this L{HTTP11ClientProtocol}.- """- if self._state != 'QUIESCENT':- return fail(RequestNotSent())-- self._state = 'TRANSMITTING'- _requestDeferred = maybeDeferred(request.writeTo, self.transport)-- def cancelRequest(ign):- # Explicitly cancel the request's deferred if it's still trying to- # write when this request is cancelled.- if self._state in (- 'TRANSMITTING', 'TRANSMITTING_AFTER_RECEIVING_RESPONSE'):- _requestDeferred.cancel()- else:- self.transport.abortConnection()- self._disconnectParser(Failure(CancelledError()))- self._finishedRequest = Deferred(cancelRequest)-- # Keep track of the Request object in case we need to call stopWriting- # on it.- self._currentRequest = request-- self._transportProxy = TransportProxyProducer(self.transport)- self._parser = HTTPClientParser(request, self._finishResponse)- self._parser.makeConnection(self._transportProxy)- self._responseDeferred = self._parser._responseDeferred-- def cbRequestWrotten(ignored):- if self._state == 'TRANSMITTING':- self._state = 'WAITING'- self._responseDeferred.chainDeferred(self._finishedRequest)-- def ebRequestWriting(err):- if self._state == 'TRANSMITTING':- self._state = 'GENERATION_FAILED'- self.transport.abortConnection()- self._finishedRequest.errback(- Failure(RequestGenerationFailed([err])))- else:- log.err(err, 'Error writing request, but not in valid state '- 'to finalize request: %s' % self._state)-- _requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting)-- return self._finishedRequest--- def _finishResponse(self, rest):- """- Called by an L{HTTPClientParser} to indicate that it has parsed a- complete response.-- @param rest: A C{str} giving any trailing bytes which were given to- the L{HTTPClientParser} which were not part of the response it- was parsing.- """- _finishResponse = makeStatefulDispatcher('finishResponse', _finishResponse)--- def _finishResponse_WAITING(self, rest):- # Currently the rest parameter is ignored. Don't forget to use it if- # we ever add support for pipelining. And maybe check what trailers- # mean.- if self._state == 'WAITING':- self._state = 'QUIESCENT'- else:- # The server sent the entire response before we could send the- # whole request. That sucks. Oh well. Fire the request()- # Deferred with the response. But first, make sure that if the- # request does ever finish being written that it won't try to fire- # that Deferred.- self._state = 'TRANSMITTING_AFTER_RECEIVING_RESPONSE'- self._responseDeferred.chainDeferred(self._finishedRequest)-- # This will happen if we're being called due to connection being lost;- # if so, no need to disconnect parser again, or to call- # _quiescentCallback.- if self._parser is None:- return-- reason = ConnectionDone("synthetic!")- connHeaders = self._parser.connHeaders.getRawHeaders('connection', ())- if (('close' in connHeaders) or self._state != "QUIESCENT" or- not self._currentRequest.persistent):- self._giveUp(Failure(reason))- else:- # We call the quiescent callback first, to ensure connection gets- # added back to connection pool before we finish the request.- try:- self._quiescentCallback(self)- except:- # If callback throws exception, just log it and disconnect;- # keeping persistent connections around is an optimisation:- log.err()- self.transport.loseConnection()- self._disconnectParser(reason)--- _finishResponse_TRANSMITTING = _finishResponse_WAITING--- def _disconnectParser(self, reason):- """- If there is still a parser, call its C{connectionLost} method with the- given reason. If there is not, do nothing.-- @type reason: L{Failure}- """- if self._parser is not None:- parser = self._parser- self._parser = None- self._currentRequest = None- self._finishedRequest = None- self._responseDeferred = None-- # The parser is no longer allowed to do anything to the real- # transport. Stop proxying from the parser's transport to the real- # transport before telling the parser it's done so that it can't do- # anything.- self._transportProxy._stopProxying()- self._transportProxy = None- parser.connectionLost(reason)--- def _giveUp(self, reason):- """- Lose the underlying connection and disconnect the parser with the given- L{Failure}.-- Use this method instead of calling the transport's loseConnection- method directly otherwise random things will break.- """- self.transport.loseConnection()- self._disconnectParser(reason)--- def dataReceived(self, bytes):- """- Handle some stuff from some place.- """- try:- self._parser.dataReceived(bytes)- except:- self._giveUp(Failure())--- def connectionLost(self, reason):- """- The underlying transport went away. If appropriate, notify the parser- object.- """- connectionLost = makeStatefulDispatcher('connectionLost', connectionLost)--- def _connectionLost_QUIESCENT(self, reason):- """- Nothing is currently happening. Move to the C{'CONNECTION_LOST'}- state but otherwise do nothing.- """- self._state = 'CONNECTION_LOST'--- def _connectionLost_GENERATION_FAILED(self, reason):- """- The connection was in an inconsistent state. Move to the- C{'CONNECTION_LOST'} state but otherwise do nothing.- """- self._state = 'CONNECTION_LOST'--- def _connectionLost_TRANSMITTING(self, reason):- """- Fail the L{Deferred} for the current request, notify the request- object that it does not need to continue transmitting itself, and- move to the C{'CONNECTION_LOST'} state.- """- self._state = 'CONNECTION_LOST'- self._finishedRequest.errback(- Failure(RequestTransmissionFailed([reason])))- del self._finishedRequest-- # Tell the request that it should stop bothering now.- self._currentRequest.stopWriting()--- def _connectionLost_TRANSMITTING_AFTER_RECEIVING_RESPONSE(self, reason):- """- Move to the C{'CONNECTION_LOST'} state.- """- self._state = 'CONNECTION_LOST'--- def _connectionLost_WAITING(self, reason):- """- Disconnect the response parser so that it can propagate the event as- necessary (for example, to call an application protocol's- C{connectionLost} method, or to fail a request L{Deferred}) and move- to the C{'CONNECTION_LOST'} state.- """- self._disconnectParser(reason)- self._state = 'CONNECTION_LOST'--- def _connectionLost_ABORTING(self, reason):- """- Disconnect the response parser with a L{ConnectionAborted} failure, and- move to the C{'CONNECTION_LOST'} state.- """- self._disconnectParser(Failure(ConnectionAborted()))- self._state = 'CONNECTION_LOST'- for d in self._abortDeferreds:- d.callback(None)- self._abortDeferreds = []--- def abort(self):- """- Close the connection and cause all outstanding L{request} L{Deferred}s- to fire with an error.- """- if self._state == "CONNECTION_LOST":- return succeed(None)- self.transport.loseConnection()- self._state = 'ABORTING'- d = Deferred()- self._abortDeferreds.append(d)- return d-}}} '''diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.pyindex 8e0b1df8b..c2d50648a 100644--- a/scrapy/xlib/tx/client.py+++ b/scrapy/xlib/tx/client.py@@ -42,1129 +42,13 @@ from twisted.web.client import (Agent, ProxyAgent, HTTPConnectionPool, readBody,)-''' {{{-class PartialDownloadError(error.Error):- """- Page was only partially downloaded, we got disconnected in middle.-- @ivar response: All of the response body which was downloaded.- """---class _URL(tuple):- """- A parsed URL.-- At some point this should be replaced with a better URL implementation.- """- def __new__(self, scheme, host, port, path):- return tuple.__new__(_URL, (scheme, host, port, path))--- def __init__(self, scheme, host, port, path):- self.scheme = scheme- self.host = host- self.port = port- self.path = path---def _parse(url, defaultPort=None):- """- Split the given URL into the scheme, host, port, and path.-- @type url: C{bytes}- @param url: An URL to parse.-- @type defaultPort: C{int} or C{None}- @param defaultPort: An alternate value to use as the port if the URL does- not include one.-- @return: A four-tuple of the scheme, host, port, and path of the URL. All- of these are C{bytes} instances except for port, which is an C{int}.- """- url = url.strip()- parsed = http.urlparse(url)- scheme = parsed[0]- path = urlunparse((b'', b'') + parsed[2:])-- if defaultPort is None:- if scheme == b'https':- defaultPort = 443- else:- defaultPort = 80-- host, port = parsed[1], defaultPort- if b':' in host:- host, port = host.split(b':')- try:- port = int(port)- except ValueError:- port = defaultPort-- if path == b'':- path = b'/'-- return _URL(scheme, host, port, path)---def _makeGetterFactory(url, factoryFactory, contextFactory=None,- *args, **kwargs):- """- Create and connect an HTTP page getting factory.-- Any additional positional or keyword arguments are used when calling- C{factoryFactory}.-- @param factoryFactory: Factory factory that is called with C{url}, C{args}- and C{kwargs} to produce the getter-- @param contextFactory: Context factory to use when creating a secure- connection, defaulting to C{None}-- @return: The factory created by C{factoryFactory}- """- scheme, host, port, path = _parse(url)- factory = factoryFactory(url, *args, **kwargs)- if scheme == b'https':- from twisted.internet import ssl- if contextFactory is None:- contextFactory = ssl.ClientContextFactory()- reactor.connectSSL(host, port, factory, contextFactory)- else:- reactor.connectTCP(host, port, factory)- return factory-}}} '''# The code which follows is based on the new HTTP client implementation. It# should be significantly better than anything above, though it is not yet# feature equivalent.-#from twisted.web.error import SchemeNotSupportedfrom twisted.web._newclient import Response-#from twisted.web._newclient import Request, HTTP11ClientProtocolfrom twisted.web._newclient import ResponseDone, ResponseFailed-#from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed-#from twisted.web._newclient import (-# ResponseNeverReceived, PotentialDataLoss, _WrapperException)--''' {{{-try:- from twisted.internet.ssl import ClientContextFactory-except ImportError:- class WebClientContextFactory(object):- """- A web context factory which doesn't work because the necessary SSL- support is missing.- """- def getContext(self, hostname, port):- raise NotImplementedError("SSL support unavailable")-else:- class WebClientContextFactory(ClientContextFactory):- """- A web context factory which ignores the hostname and port and does no- certificate verification.- """- def getContext(self, hostname, port):- return ClientContextFactory.getContext(self)----class _WebToNormalContextFactory(object):- """- Adapt a web context factory to a normal context factory.-- @ivar _webContext: A web context factory which accepts a hostname and port- number to its C{getContext} method.-- @ivar _hostname: The hostname which will be passed to- C{_webContext.getContext}.-- @ivar _port: The port number which will be passed to- C{_webContext.getContext}.- """- def __init__(self, webContext, hostname, port):- self._webContext = webContext- self._hostname = hostname- self._port = port--- def getContext(self):- """- Called the wrapped web context factory's C{getContext} method with a- hostname and port number and return the resulting context object.- """- return self._webContext.getContext(self._hostname, self._port)----@implementer(IBodyProducer)-class FileBodyProducer(object):- """- L{FileBodyProducer} produces bytes from an input file object incrementally- and writes them to a consumer.-- Since file-like objects cannot be read from in an event-driven manner,- L{FileBodyProducer} uses a L{Cooperator} instance to schedule reads from- the file. This process is also paused and resumed based on notifications- from the L{IConsumer} provider being written to.-- The file is closed after it has been read, or if the producer is stopped- early.-- @ivar _inputFile: Any file-like object, bytes read from which will be- written to a consumer.-- @ivar _cooperate: A method like L{Cooperator.cooperate} which is used to- schedule all reads.-- @ivar _readSize: The number of bytes to read from C{_inputFile} at a time.- """-- # Python 2.4 doesn't have these symbolic constants- _SEEK_SET = getattr(os, 'SEEK_SET', 0)- _SEEK_END = getattr(os, 'SEEK_END', 2)-- def __init__(self, inputFile, cooperator=task, readSize=2 ** 16):- self._inputFile = inputFile- self._cooperate = cooperator.cooperate- self._readSize = readSize- self.length = self._determineLength(inputFile)--- def _determineLength(self, fObj):- """- Determine how many bytes can be read out of C{fObj} (assuming it is not- modified from this point on). If the determination cannot be made,- return C{UNKNOWN_LENGTH}.- """- try:- seek = fObj.seek- tell = fObj.tell- except AttributeError:- return UNKNOWN_LENGTH- originalPosition = tell()- seek(0, self._SEEK_END)- end = tell()- seek(originalPosition, self._SEEK_SET)- return end - originalPosition--- def stopProducing(self):- """- Permanently stop writing bytes from the file to the consumer by- stopping the underlying L{CooperativeTask}.- """- self._inputFile.close()- self._task.stop()--- def startProducing(self, consumer):- """- Start a cooperative task which will read bytes from the input file and- write them to C{consumer}. Return a L{Deferred} which fires after all- bytes have been written.-- @param consumer: Any L{IConsumer} provider- """- self._task = self._cooperate(self._writeloop(consumer))- d = self._task.whenDone()- def maybeStopped(reason):- # IBodyProducer.startProducing's Deferred isn't support to fire if- # stopProducing is called.- reason.trap(task.TaskStopped)- return defer.Deferred()- d.addCallbacks(lambda ignored: None, maybeStopped)- return d--- def _writeloop(self, consumer):- """- Return an iterator which reads one chunk of bytes from the input file- and writes them to the consumer for each time it is iterated.- """- while True:- bytes = self._inputFile.read(self._readSize)- if not bytes:- self._inputFile.close()- break- consumer.write(bytes)- yield None--- def pauseProducing(self):- """- Temporarily suspend copying bytes from the input file to the consumer- by pausing the L{CooperativeTask} which drives that activity.- """- self._task.pause()--- def resumeProducing(self):- """- Undo the effects of a previous C{pauseProducing} and resume copying- bytes to the consumer by resuming the L{CooperativeTask} which drives- the write activity.- """- self._task.resume()----class _HTTP11ClientFactory(protocol.Factory):- """- A factory for L{HTTP11ClientProtocol}, used by L{HTTPConnectionPool}.-- @ivar _quiescentCallback: The quiescent callback to be passed to protocol- instances, used to return them to the connection pool.-- @since: 11.1- """- def __init__(self, quiescentCallback):- self._quiescentCallback = quiescentCallback--- def buildProtocol(self, addr):- return HTTP11ClientProtocol(self._quiescentCallback)----class _RetryingHTTP11ClientProtocol(object):- """- A wrapper for L{HTTP11ClientProtocol} that automatically retries requests.-- @ivar _clientProtocol: The underlying L{HTTP11ClientProtocol}.-- @ivar _newConnection: A callable that creates a new connection for a- retry.- """-- def __init__(self, clientProtocol, newConnection):- self._clientProtocol = clientProtocol- self._newConnection = newConnection--- def _shouldRetry(self, method, exception, bodyProducer):- """- Indicate whether request should be retried.-- Only returns C{True} if method is idempotent, no response was- received, the reason for the failed request was not due to- user-requested cancellation, and no body was sent. The latter- requirement may be relaxed in the future, and PUT added to approved- method list.- """- if method not in ("GET", "HEAD", "OPTIONS", "DELETE", "TRACE"):- return False- if not isinstance(exception, (RequestNotSent, RequestTransmissionFailed,- ResponseNeverReceived)):- return False- if isinstance(exception, _WrapperException):- for failure in exception.reasons:- if failure.check(defer.CancelledError):- return False- if bodyProducer is not None:- return False- return True--- def request(self, request):- """- Do a request, and retry once (with a new connection) it it fails in- a retryable manner.-- @param request: A L{Request} instance that will be requested using the- wrapped protocol.- """- d = self._clientProtocol.request(request)-- def failed(reason):- if self._shouldRetry(request.method, reason.value,- request.bodyProducer):- return self._newConnection().addCallback(- lambda connection: connection.request(request))- else:- return reason- d.addErrback(failed)- return d----class HTTPConnectionPool(object):- """- A pool of persistent HTTP connections.-- Features:- - Cached connections will eventually time out.- - Limits on maximum number of persistent connections.-- Connections are stored using keys, which should be chosen such that any- connections stored under a given key can be used interchangeably.-- Failed requests done using previously cached connections will be retried- once if they use an idempotent method (e.g. GET), in case the HTTP server- timed them out.-- @ivar persistent: Boolean indicating whether connections should be- persistent. Connections are persistent by default.-- @ivar maxPersistentPerHost: The maximum number of cached persistent- connections for a C{host:port} destination.- @type maxPersistentPerHost: C{int}-- @ivar cachedConnectionTimeout: Number of seconds a cached persistent- connection will stay open before disconnecting.-- @ivar retryAutomatically: C{boolean} indicating whether idempotent- requests should be retried once if no response was received.-- @ivar _factory: The factory used to connect to the proxy.-- @ivar _connections: Map (scheme, host, port) to lists of- L{HTTP11ClientProtocol} instances.-- @ivar _timeouts: Map L{HTTP11ClientProtocol} instances to a- C{IDelayedCall} instance of their timeout.-- @since: 12.1- """-- _factory = _HTTP11ClientFactory- maxPersistentPerHost = 2- cachedConnectionTimeout = 240- retryAutomatically = True-- def __init__(self, reactor, persistent=True):- self._reactor = reactor- self.persistent = persistent- self._connections = {}- self._timeouts = {}--- def getConnection(self, key, endpoint):- """- Supply a connection, newly created or retrieved from the pool, to be- used for one HTTP request.-- The connection will remain out of the pool (not available to be- returned from future calls to this method) until one HTTP request has- been completed over it.-- Afterwards, if the connection is still open, it will automatically be- added to the pool.-- @param key: A unique key identifying connections that can be used- interchangeably.-- @param endpoint: An endpoint that can be used to open a new connection- if no cached connection is available.-- @return: A C{Deferred} that will fire with a L{HTTP11ClientProtocol}- (or a wrapper) that can be used to send a single HTTP request.- """- # Try to get cached version:- connections = self._connections.get(key)- while connections:- connection = connections.pop(0)- # Cancel timeout:- self._timeouts[connection].cancel()- del self._timeouts[connection]- if connection.state == "QUIESCENT":- if self.retryAutomatically:- newConnection = lambda: self._newConnection(key, endpoint)- connection = _RetryingHTTP11ClientProtocol(- connection, newConnection)- return defer.succeed(connection)-- return self._newConnection(key, endpoint)--- def _newConnection(self, key, endpoint):- """- Create a new connection.-- This implements the new connection code path for L{getConnection}.- """- def quiescentCallback(protocol):- self._putConnection(key, protocol)- factory = self._factory(quiescentCallback)- return endpoint.connect(factory)--- def _removeConnection(self, key, connection):- """- Remove a connection from the cache and disconnect it.- """- connection.transport.loseConnection()- self._connections[key].remove(connection)- del self._timeouts[connection]--- def _putConnection(self, key, connection):- """- Return a persistent connection to the pool. This will be called by- L{HTTP11ClientProtocol} when the connection becomes quiescent.- """- if connection.state != "QUIESCENT":- # Log with traceback for debugging purposes:- try:- raise RuntimeError(- "BUG: Non-quiescent protocol added to connection pool.")- except:- log.err()- return- connections = self._connections.setdefault(key, [])- if len(connections) == self.maxPersistentPerHost:- dropped = connections.pop(0)- dropped.transport.loseConnection()- self._timeouts[dropped].cancel()- del self._timeouts[dropped]- connections.append(connection)- cid = self._reactor.callLater(self.cachedConnectionTimeout,- self._removeConnection,- key, connection)- self._timeouts[connection] = cid--- def closeCachedConnections(self):- """- Close all persistent connections and remove them from the pool.-- @return: L{defer.Deferred} that fires when all connections have been- closed.- """- results = []- for protocols in self._connections.itervalues():- for p in protocols:- results.append(p.abort())- self._connections = {}- for dc in self._timeouts.values():- dc.cancel()- self._timeouts = {}- return defer.gatherResults(results).addCallback(lambda ign: None)----class _AgentBase(object):- """- Base class offering common facilities for L{Agent}-type classes.-- @ivar _reactor: The C{IReactorTime} implementation which will be used by- the pool, and perhaps by subclasses as well.-- @ivar _pool: The L{HTTPConnectionPool} used to manage HTTP connections.- """-- def __init__(self, reactor, pool):- if pool is None:- pool = HTTPConnectionPool(reactor, False)- self._reactor = reactor- self._pool = pool--- def _computeHostValue(self, scheme, host, port):- """- Compute the string to use for the value of the I{Host} header, based on- the given scheme, host name, and port number.- """- if (scheme, port) in (('http', 80), ('https', 443)):- return host- return '%s:%d' % (host, port)--- def _requestWithEndpoint(self, key, endpoint, method, parsedURI,- headers, bodyProducer, requestPath):- """- Issue a new request, given the endpoint and the path sent as part of- the request.- """- # Create minimal headers, if necessary:- if headers is None:- headers = Headers()- if not headers.hasHeader('host'):- #headers = headers.copy() # not supported in twisted <= 11.1, and it doesn't affects us- headers.addRawHeader(- 'host', self._computeHostValue(parsedURI.scheme, parsedURI.host,- parsedURI.port))-- d = self._pool.getConnection(key, endpoint)- def cbConnected(proto):- return proto.request(- Request(method, requestPath, headers, bodyProducer,- persistent=self._pool.persistent))- d.addCallback(cbConnected)- return d----class Agent(_AgentBase):- """- L{Agent} is a very basic HTTP client. It supports I{HTTP} and I{HTTPS}- scheme URIs (but performs no certificate checking by default).-- @param pool: A L{HTTPConnectionPool} instance, or C{None}, in which case a- non-persistent L{HTTPConnectionPool} instance will be created.-- @ivar _contextFactory: A web context factory which will be used to create- SSL context objects for any SSL connections the agent needs to make.-- @ivar _connectTimeout: If not C{None}, the timeout passed to C{connectTCP}- or C{connectSSL} for specifying the connection timeout.-- @ivar _bindAddress: If not C{None}, the address passed to C{connectTCP} or- C{connectSSL} for specifying the local address to bind to.-- @since: 9.0- """-- def __init__(self, reactor, contextFactory=WebClientContextFactory(),- connectTimeout=None, bindAddress=None,- pool=None):- _AgentBase.__init__(self, reactor, pool)- self._contextFactory = contextFactory- self._connectTimeout = connectTimeout- self._bindAddress = bindAddress--- def _wrapContextFactory(self, host, port):- """- Create and return a normal context factory wrapped around- C{self._contextFactory} in such a way that C{self._contextFactory} will- have the host and port information passed to it.-- @param host: A C{str} giving the hostname which will be connected to in- order to issue a request.-- @param port: An C{int} giving the port number the connection will be- on.-- @return: A context factory suitable to be passed to- C{reactor.connectSSL}.- """- return _WebToNormalContextFactory(self._contextFactory, host, port)--- def _getEndpoint(self, scheme, host, port):- """- Get an endpoint for the given host and port, using a transport- selected based on scheme.-- @param scheme: A string like C{'http'} or C{'https'} (the only two- supported values) to use to determine how to establish the- connection.-- @param host: A C{str} giving the hostname which will be connected to in- order to issue a request.-- @param port: An C{int} giving the port number the connection will be- on.-- @return: An endpoint which can be used to connect to given address.- """- kwargs = {}- if self._connectTimeout is not None:- kwargs['timeout'] = self._connectTimeout- kwargs['bindAddress'] = self._bindAddress- if scheme == 'http':- return TCP4ClientEndpoint(self._reactor, host, port, **kwargs)- elif scheme == 'https':- return SSL4ClientEndpoint(self._reactor, host, port,- self._wrapContextFactory(host, port),- **kwargs)- else:- raise SchemeNotSupported("Unsupported scheme: %r" % (scheme,))--- def request(self, method, uri, headers=None, bodyProducer=None):- """- Issue a new request.-- @param method: The request method to send.- @type method: C{str}-- @param uri: The request URI send.- @type uri: C{str}-- @param headers: The request headers to send. If no I{Host} header is- included, one will be added based on the request URI.- @type headers: L{Headers}-- @param bodyProducer: An object which will produce the request body or,- if the request body is to be empty, L{None}.- @type bodyProducer: L{IBodyProducer} provider-- @return: A L{Deferred} which fires with the result of the request (a- L{twisted.web.iweb.IResponse} provider), or fails if there is a- problem setting up a connection over which to issue the request.- It may also fail with L{SchemeNotSupported} if the scheme of the- given URI is not supported.- @rtype: L{Deferred}- """- parsedURI = _parse(uri)- try:- endpoint = self._getEndpoint(parsedURI.scheme, parsedURI.host,- parsedURI.port)- except SchemeNotSupported:- return defer.fail(Failure())- key = (parsedURI.scheme, parsedURI.host, parsedURI.port)- return self._requestWithEndpoint(key, endpoint, method, parsedURI,- headers, bodyProducer, parsedURI.path)----class ProxyAgent(_AgentBase):- """- An HTTP agent able to cross HTTP proxies.-- @ivar _proxyEndpoint: The endpoint used to connect to the proxy.-- @since: 11.1- """-- def __init__(self, endpoint, reactor=None, pool=None):- if reactor is None:- from twisted.internet import reactor- _AgentBase.__init__(self, reactor, pool)- self._proxyEndpoint = endpoint--- def request(self, method, uri, headers=None, bodyProducer=None):- """- Issue a new request via the configured proxy.- """- # Cache *all* connections under the same key, since we are only- # connecting to a single destination, the proxy:- key = ("http-proxy", self._proxyEndpoint)-- # To support proxying HTTPS via CONNECT, we will use key- # ("http-proxy-CONNECT", scheme, host, port), and an endpoint that- # wraps _proxyEndpoint with an additional callback to do the CONNECT.- return self._requestWithEndpoint(key, self._proxyEndpoint, method,- _parse(uri), headers, bodyProducer,- uri)----class _FakeUrllib2Request(object):- """- A fake C{urllib2.Request} object for C{cookielib} to work with.-- @see: U{http://docs.python.org/library/urllib2.html#request-objects}-- @type uri: C{str}- @ivar uri: Request URI.-- @type headers: L{twisted.web.http_headers.Headers}- @ivar headers: Request headers.-- @type type: C{str}- @ivar type: The scheme of the URI.-- @type host: C{str}- @ivar host: The host[:port] of the URI.-- @since: 11.1- """- def __init__(self, uri):- self.uri = uri- self.headers = Headers()- self.type, rest = splittype(self.uri)- self.host, rest = splithost(rest)--- def has_header(self, header):- return self.headers.hasHeader(header)--- def add_unredirected_header(self, name, value):- self.headers.addRawHeader(name, value)--- def get_full_url(self):- return self.uri--- def get_header(self, name, default=None):- headers = self.headers.getRawHeaders(name, default)- if headers is not None:- return headers[0]- return None--- def get_host(self):- return self.host--- def get_type(self):- return self.type--- def is_unverifiable(self):- # In theory this shouldn't be hardcoded.- return False----class _FakeUrllib2Response(object):- """- A fake C{urllib2.Response} object for C{cookielib} to work with.-- @type response: C{twisted.web.iweb.IResponse}- @ivar response: Underlying Twisted Web response.-- @since: 11.1- """- def __init__(self, response):- self.response = response--- def info(self):- class _Meta(object):- def getheaders(zelf, name):- return self.response.headers.getRawHeaders(name, [])- return _Meta()----class CookieAgent(object):- """- L{CookieAgent} extends the basic L{Agent} to add RFC-compliant- handling of HTTP cookies. Cookies are written to and extracted- from a C{cookielib.CookieJar} instance.-- The same cookie jar instance will be used for any requests through this- agent, mutating it whenever a I{Set-Cookie} header appears in a response.-- @type _agent: L{twisted.web.client.Agent}- @ivar _agent: Underlying Twisted Web agent to issue requests through.-- @type cookieJar: C{cookielib.CookieJar}- @ivar cookieJar: Initialized cookie jar to read cookies from and store- cookies to.-- @since: 11.1- """- def __init__(self, agent, cookieJar):- self._agent = agent- self.cookieJar = cookieJar--- def request(self, method, uri, headers=None, bodyProducer=None):- """- Issue a new request to the wrapped L{Agent}.-- Send a I{Cookie} header if a cookie for C{uri} is stored in- L{CookieAgent.cookieJar}. Cookies are automatically extracted and- stored from requests.-- If a C{'cookie'} header appears in C{headers} it will override the- automatic cookie header obtained from the cookie jar.-- @see: L{Agent.request}- """- if headers is None:- headers = Headers()- lastRequest = _FakeUrllib2Request(uri)- # Setting a cookie header explicitly will disable automatic request- # cookies.- if not headers.hasHeader('cookie'):- self.cookieJar.add_cookie_header(lastRequest)- cookieHeader = lastRequest.get_header('Cookie', None)- if cookieHeader is not None:- headers = headers.copy()- headers.addRawHeader('cookie', cookieHeader)-- d = self._agent.request(method, uri, headers, bodyProducer)- d.addCallback(self._extractCookies, lastRequest)- return d--- def _extractCookies(self, response, request):- """- Extract response cookies and store them in the cookie jar.-- @type response: L{twisted.web.iweb.IResponse}- @param response: Twisted Web response.-- @param request: A urllib2 compatible request object.- """- resp = _FakeUrllib2Response(response)- self.cookieJar.extract_cookies(resp, request)- return response----class GzipDecoder(proxyForInterface(IResponse)):- """- A wrapper for a L{Response} instance which handles gzip'ed body.-- @ivar original: The original L{Response} object.-- @since: 11.1- """-- def __init__(self, response):- self.original = response- self.length = UNKNOWN_LENGTH--- def deliverBody(self, protocol):- """- Override C{deliverBody} to wrap the given C{protocol} with- L{_GzipProtocol}.- """- self.original.deliverBody(_GzipProtocol(protocol, self.original))----class _GzipProtocol(proxyForInterface(IProtocol)):- """- A L{Protocol} implementation which wraps another one, transparently- decompressing received data.-- @ivar _zlibDecompress: A zlib decompress object used to decompress the data- stream.-- @ivar _response: A reference to the original response, in case of errors.-- @since: 11.1- """-- def __init__(self, protocol, response):- self.original = protocol- self._response = response- self._zlibDecompress = zlib.decompressobj(16 + zlib.MAX_WBITS)--- def dataReceived(self, data):- """- Decompress C{data} with the zlib decompressor, forwarding the raw data- to the original protocol.- """- try:- rawData = self._zlibDecompress.decompress(data)- except zlib.error:- raise ResponseFailed([failure.Failure()], self._response)- if rawData:- self.original.dataReceived(rawData)--- def connectionLost(self, reason):- """- Forward the connection lost event, flushing remaining data from the- decompressor if any.- """- try:- rawData = self._zlibDecompress.flush()- except zlib.error:- raise ResponseFailed([reason, failure.Failure()], self._response)- if rawData:- self.original.dataReceived(rawData)- self.original.connectionLost(reason)----class ContentDecoderAgent(object):- """- An L{Agent} wrapper to handle encoded content.-- It takes care of declaring the support for content in the- I{Accept-Encoding} header, and automatically decompresses the received data- if it's effectively using compression.-- @param decoders: A list or tuple of (name, decoder) objects. The name- declares which decoding the decoder supports, and the decoder must- return a response object when called/instantiated. For example,- C{(('gzip', GzipDecoder))}. The order determines how the decoders are- going to be advertized to the server.-- @since: 11.1- """-- def __init__(self, agent, decoders):- self._agent = agent- self._decoders = dict(decoders)- self._supported = ','.join([decoder[0] for decoder in decoders])--- def request(self, method, uri, headers=None, bodyProducer=None):- """- Send a client request which declares supporting compressed content.-- @see: L{Agent.request}.- """- if headers is None:- headers = Headers()- else:- headers = headers.copy()- headers.addRawHeader('accept-encoding', self._supported)- deferred = self._agent.request(method, uri, headers, bodyProducer)- return deferred.addCallback(self._handleResponse)--- def _handleResponse(self, response):- """- Check if the response is encoded, and wrap it to handle decompression.- """- contentEncodingHeaders = response.headers.getRawHeaders(- 'content-encoding', [])- contentEncodingHeaders = ','.join(contentEncodingHeaders).split(',')- while contentEncodingHeaders:- name = contentEncodingHeaders.pop().strip()- decoder = self._decoders.get(name)- if decoder is not None:- response = decoder(response)- else:- # Add it back- contentEncodingHeaders.append(name)- break- if contentEncodingHeaders:- response.headers.setRawHeaders(- 'content-encoding', [','.join(contentEncodingHeaders)])- else:- response.headers.removeHeader('content-encoding')- return response----class RedirectAgent(object):- """- An L{Agent} wrapper which handles HTTP redirects.-- The implementation is rather strict: 301 and 302 behaves like 307, not- redirecting automatically on methods different from C{GET} and C{HEAD}.-- @param redirectLimit: The maximum number of times the agent is allowed to- follow redirects before failing with a L{error.InfiniteRedirection}.-- @since: 11.1- """-- def __init__(self, agent, redirectLimit=20):- self._agent = agent- self._redirectLimit = redirectLimit--- def request(self, method, uri, headers=None, bodyProducer=None):- """- Send a client request following HTTP redirects.-- @see: L{Agent.request}.- """- deferred = self._agent.request(method, uri, headers, bodyProducer)- return deferred.addCallback(- self._handleResponse, method, uri, headers, 0)--- def _handleRedirect(self, response, method, uri, headers, redirectCount):- """- Handle a redirect response, checking the number of redirects already- followed, and extracting the location header fields.- """- if redirectCount >= self._redirectLimit:- err = error.InfiniteRedirection(- response.code,- 'Infinite redirection detected',- location=uri)- raise ResponseFailed([failure.Failure(err)], response)- locationHeaders = response.headers.getRawHeaders('location', [])- if not locationHeaders:- err = error.RedirectWithNoLocation(- response.code, 'No location header field', uri)- raise ResponseFailed([failure.Failure(err)], response)- location = locationHeaders[0]- deferred = self._agent.request(method, location, headers)- return deferred.addCallback(- self._handleResponse, method, uri, headers, redirectCount + 1)--- def _handleResponse(self, response, method, uri, headers, redirectCount):- """- Handle the response, making another request if it indicates a redirect.- """- if response.code in (http.MOVED_PERMANENTLY, http.FOUND,- http.TEMPORARY_REDIRECT):- if method not in ('GET', 'HEAD'):- err = error.PageRedirect(response.code, location=uri)- raise ResponseFailed([failure.Failure(err)], response)- return self._handleRedirect(response, method, uri, headers,- redirectCount)- elif response.code == http.SEE_OTHER:- return self._handleRedirect(response, 'GET', uri, headers,- redirectCount)- return response----class _ReadBodyProtocol(protocol.Protocol):- """- Protocol that collects data sent to it.-- This is a helper for L{IResponse.deliverBody}, which collects the body and- fires a deferred with it.-- @ivar deferred: See L{__init__}.- @ivar status: See L{__init__}.- @ivar message: See L{__init__}.-- @ivar dataBuffer: list of byte-strings received- @type dataBuffer: L{list} of L{bytes}- """-- def __init__(self, status, message, deferred):- """- @param status: Status of L{IResponse}- @ivar status: L{int}-- @param message: Message of L{IResponse}- @type message: L{bytes}-- @param deferred: deferred to fire when response is complete- @type deferred: L{Deferred} firing with L{bytes}- """- self.deferred = deferred- self.status = status- self.message = message- self.dataBuffer = []--- def dataReceived(self, data):- """- Accumulate some more bytes from the response.- """- self.dataBuffer.append(data)--- def connectionLost(self, reason):- """- Deliver the accumulated response bytes to the waiting L{Deferred}, if- the response body has been completely received without error.- """- if reason.check(ResponseDone):- self.deferred.callback(b''.join(self.dataBuffer))- elif reason.check(PotentialDataLoss):- self.deferred.errback(- PartialDownloadError(self.status, self.message,- b''.join(self.dataBuffer)))- else:- self.deferred.errback(reason)----def readBody(response):- """- Get the body of an L{IResponse} and return it as a byte string.-- This is a helper function for clients that don't want to incrementally- receive the body of an HTTP response.-- @param response: The HTTP response for which the body will be read.- @type response: L{IResponse} provider-- @return: A L{Deferred} which will fire with the body of the response.- """- d = defer.Deferred()- response.deliverBody(_ReadBodyProtocol(response.code, response.phrase, d))- return d-}}} '''__all__ = [diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.pyindex 3f4704064..197e43ed3 100644--- a/scrapy/xlib/tx/endpoints.py+++ b/scrapy/xlib/tx/endpoints.py@@ -14,27 +14,6 @@ parsed by the L{clientFromString} and L{serverFromString} functions.from __future__ import division, absolute_import-import os-#import socket--from zope.interface import implementer, directlyProvides-import warnings--from twisted.internet import interfaces, defer, error, fdesc-#from twisted.internet.protocol import (-# ClientFactory, Protocol)-from twisted.internet.protocol import Factory-#from twisted.internet import threads, ProcessProtocol-from twisted.internet.interfaces import IStreamServerEndpointStringParser-#from twisted.internet.interfaces import IStreamClientEndpointStringParser-from twisted.python.filepath import FilePath-#from twisted.python.failure import Failure-#from twisted.python import log-#from twisted.python.components import proxyForInterface--from twisted.plugin import IPlugin, getPlugins-#from twisted.internet import stdio-from twisted.internet.endpoints import (clientFromString, serverFromString, quoteStringArgument,TCP4ServerEndpoint, TCP6ServerEndpoint,@@ -44,1236 +23,4 @@ from twisted.internet.endpoints import (AdoptedStreamServerEndpoint, connectProtocol,)-__all__ = ["TCP4ClientEndpoint", "SSL4ServerEndpoint"]--''' {{{-class _WrappingProtocol(Protocol):- """- Wrap another protocol in order to notify my user when a connection has- been made.- """-- def __init__(self, connectedDeferred, wrappedProtocol):- """- @param connectedDeferred: The L{Deferred} that will callback- with the C{wrappedProtocol} when it is connected.-- @param wrappedProtocol: An L{IProtocol} provider that will be- connected.- """- self._connectedDeferred = connectedDeferred- self._wrappedProtocol = wrappedProtocol-- for iface in [interfaces.IHalfCloseableProtocol,- interfaces.IFileDescriptorReceiver]:- if iface.providedBy(self._wrappedProtocol):- directlyProvides(self, iface)--- def logPrefix(self):- """- Transparently pass through the wrapped protocol's log prefix.- """- if interfaces.ILoggingContext.providedBy(self._wrappedProtocol):- return self._wrappedProtocol.logPrefix()- return self._wrappedProtocol.__class__.__name__--- def connectionMade(self):- """- Connect the C{self._wrappedProtocol} to our C{self.transport} and- callback C{self._connectedDeferred} with the C{self._wrappedProtocol}- """- self._wrappedProtocol.makeConnection(self.transport)- self._connectedDeferred.callback(self._wrappedProtocol)--- def dataReceived(self, data):- """- Proxy C{dataReceived} calls to our C{self._wrappedProtocol}- """- return self._wrappedProtocol.dataReceived(data)--- def fileDescriptorReceived(self, descriptor):- """- Proxy C{fileDescriptorReceived} calls to our C{self._wrappedProtocol}- """- return self._wrappedProtocol.fileDescriptorReceived(descriptor)--- def connectionLost(self, reason):- """- Proxy C{connectionLost} calls to our C{self._wrappedProtocol}- """- return self._wrappedProtocol.connectionLost(reason)--- def readConnectionLost(self):- """- Proxy L{IHalfCloseableProtocol.readConnectionLost} to our- C{self._wrappedProtocol}- """- self._wrappedProtocol.readConnectionLost()--- def writeConnectionLost(self):- """- Proxy L{IHalfCloseableProtocol.writeConnectionLost} to our- C{self._wrappedProtocol}- """- self._wrappedProtocol.writeConnectionLost()----class _WrappingFactory(ClientFactory):- """- Wrap a factory in order to wrap the protocols it builds.-- @ivar _wrappedFactory: A provider of I{IProtocolFactory} whose buildProtocol- method will be called and whose resulting protocol will be wrapped.-- @ivar _onConnection: A L{Deferred} that fires when the protocol is- connected-- @ivar _connector: A L{connector <twisted.internet.interfaces.IConnector>}- that is managing the current or previous connection attempt.- """- protocol = _WrappingProtocol-- def __init__(self, wrappedFactory):- """- @param wrappedFactory: A provider of I{IProtocolFactory} whose- buildProtocol method will be called and whose resulting protocol- will be wrapped.- """- self._wrappedFactory = wrappedFactory- self._onConnection = defer.Deferred(canceller=self._canceller)--- def startedConnecting(self, connector):- """- A connection attempt was started. Remember the connector which started- said attempt, for use later.- """- self._connector = connector--- def _canceller(self, deferred):- """- The outgoing connection attempt was cancelled. Fail that L{Deferred}- with an L{error.ConnectingCancelledError}.-- @param deferred: The L{Deferred <defer.Deferred>} that was cancelled;- should be the same as C{self._onConnection}.- @type deferred: L{Deferred <defer.Deferred>}-- @note: This relies on startedConnecting having been called, so it may- seem as though there's a race condition where C{_connector} may not- have been set. However, using public APIs, this condition is- impossible to catch, because a connection API- (C{connectTCP}/C{SSL}/C{UNIX}) is always invoked before a- L{_WrappingFactory}'s L{Deferred <defer.Deferred>} is returned to- C{connect()}'s caller.-- @return: C{None}- """- deferred.errback(- error.ConnectingCancelledError(- self._connector.getDestination()))- self._connector.stopConnecting()--- def doStart(self):- """- Start notifications are passed straight through to the wrapped factory.- """- self._wrappedFactory.doStart()--- def doStop(self):- """- Stop notifications are passed straight through to the wrapped factory.- """- self._wrappedFactory.doStop()--- def buildProtocol(self, addr):- """- Proxy C{buildProtocol} to our C{self._wrappedFactory} or errback- the C{self._onConnection} L{Deferred}.-- @return: An instance of L{_WrappingProtocol} or C{None}- """- try:- proto = self._wrappedFactory.buildProtocol(addr)- except:- self._onConnection.errback()- else:- return self.protocol(self._onConnection, proto)--- def clientConnectionFailed(self, connector, reason):- """- Errback the C{self._onConnection} L{Deferred} when the- client connection fails.- """- if not self._onConnection.called:- self._onConnection.errback(reason)------@implementer(interfaces.ITransport)-class _ProcessEndpointTransport(proxyForInterface(- interfaces.IProcessTransport, '_process')):- """- An L{ITransport} provider for the L{IProtocol} instance passed to the- process endpoint.-- @ivar _process: An active process transport which will be used by write- methods on this object to write data to a child process.- @type _process: L{interfaces.IProcessTransport} provider- """-- def write(self, data):- """- Write to the child process's standard input.-- @param data: The data to write on stdin.- """- self._process.writeToChild(0, data)--- def writeSequence(self, data):- """- Write a list of strings to child process's stdin.-- @param data: The list of chunks to write on stdin.- """- for chunk in data:- self._process.writeToChild(0, chunk)---@implementer(interfaces.IStreamServerEndpoint)-class _TCPServerEndpoint(object):- """- A TCP server endpoint interface- """-- def __init__(self, reactor, port, backlog, interface):- """- @param reactor: An L{IReactorTCP} provider.-- @param port: The port number used for listening- @type port: int-- @param backlog: Size of the listen queue- @type backlog: int-- @param interface: The hostname to bind to- @type interface: str- """- self._reactor = reactor- self._port = port- self._backlog = backlog- self._interface = interface--- def listen(self, protocolFactory):- """- Implement L{IStreamServerEndpoint.listen} to listen on a TCP- socket- """- return defer.execute(self._reactor.listenTCP,- self._port,- protocolFactory,- backlog=self._backlog,- interface=self._interface)----class TCP4ServerEndpoint(_TCPServerEndpoint):- """- Implements TCP server endpoint with an IPv4 configuration- """- def __init__(self, reactor, port, backlog=50, interface=''):- """- @param reactor: An L{IReactorTCP} provider.-- @param port: The port number used for listening- @type port: int-- @param backlog: Size of the listen queue- @type backlog: int-- @param interface: The hostname to bind to, defaults to '' (all)- @type interface: str- """- _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface)----class TCP6ServerEndpoint(_TCPServerEndpoint):- """- Implements TCP server endpoint with an IPv6 configuration- """- def __init__(self, reactor, port, backlog=50, interface='::'):- """- @param reactor: An L{IReactorTCP} provider.-- @param port: The port number used for listening- @type port: int-- @param backlog: Size of the listen queue- @type backlog: int-- @param interface: The hostname to bind to, defaults to '' (all)- @type interface: str- """- _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface)----@implementer(interfaces.IStreamClientEndpoint)-class TCP4ClientEndpoint(object):- """- TCP client endpoint with an IPv4 configuration.- """-- def __init__(self, reactor, host, port, timeout=30, bindAddress=None):- """- @param reactor: An L{IReactorTCP} provider-- @param host: A hostname, used when connecting- @type host: str-- @param port: The port number, used when connecting- @type port: int-- @param timeout: The number of seconds to wait before assuming the- connection has failed.- @type timeout: int-- @param bindAddress: A (host, port) tuple of local address to bind to,- or None.- @type bindAddress: tuple- """- self._reactor = reactor- self._host = host- self._port = port- self._timeout = timeout- self._bindAddress = bindAddress--- def connect(self, protocolFactory):- """- Implement L{IStreamClientEndpoint.connect} to connect via TCP.- """- try:- wf = _WrappingFactory(protocolFactory)- self._reactor.connectTCP(- self._host, self._port, wf,- timeout=self._timeout, bindAddress=self._bindAddress)- return wf._onConnection- except:- return defer.fail()-----@implementer(interfaces.IStreamServerEndpoint)-class SSL4ServerEndpoint(object):- """- SSL secured TCP server endpoint with an IPv4 configuration.- """-- def __init__(self, reactor, port, sslContextFactory,- backlog=50, interface=''):- """- @param reactor: An L{IReactorSSL} provider.-- @param port: The port number used for listening- @type port: int-- @param sslContextFactory: An instance of- L{twisted.internet.ssl.ContextFactory}.-- @param backlog: Size of the listen queue- @type backlog: int-- @param interface: The hostname to bind to, defaults to '' (all)- @type interface: str- """- self._reactor = reactor- self._port = port- self._sslContextFactory = sslContextFactory- self._backlog = backlog- self._interface = interface--- def listen(self, protocolFactory):- """- Implement L{IStreamServerEndpoint.listen} to listen for SSL on a- TCP socket.- """- return defer.execute(self._reactor.listenSSL, self._port,- protocolFactory,- contextFactory=self._sslContextFactory,- backlog=self._backlog,- interface=self._interface)----@implementer(interfaces.IStreamClientEndpoint)-class SSL4ClientEndpoint(object):- """- SSL secured TCP client endpoint with an IPv4 configuration- """-- def __init__(self, reactor, host, port, sslContextFactory,- timeout=30, bindAddress=None):- """- @param reactor: An L{IReactorSSL} provider.-- @param host: A hostname, used when connecting- @type host: str-- @param port: The port number, used when connecting- @type port: int-- @param sslContextFactory: SSL Configuration information as an instance- of L{twisted.internet.ssl.ContextFactory}.-- @param timeout: Number of seconds to wait before assuming the- connection has failed.- @type timeout: int-- @param bindAddress: A (host, port) tuple of local address to bind to,- or None.- @type bindAddress: tuple- """- self._reactor = reactor- self._host = host- self._port = port- self._sslContextFactory = sslContextFactory- self._timeout = timeout- self._bindAddress = bindAddress--- def connect(self, protocolFactory):- """- Implement L{IStreamClientEndpoint.connect} to connect with SSL over- TCP.- """- try:- wf = _WrappingFactory(protocolFactory)- self._reactor.connectSSL(- self._host, self._port, wf, self._sslContextFactory,- timeout=self._timeout, bindAddress=self._bindAddress)- return wf._onConnection- except:- return defer.fail()----@implementer(interfaces.IStreamServerEndpoint)-class UNIXServerEndpoint(object):- """- UnixSocket server endpoint.- """- def __init__(self, reactor, address, backlog=50, mode=0o666, wantPID=0):- """- @param reactor: An L{IReactorUNIX} provider.- @param address: The path to the Unix socket file, used when listening- @param backlog: number of connections to allow in backlog.- @param mode: mode to set on the unix socket. This parameter is- deprecated. Permissions should be set on the directory which- contains the UNIX socket.- @param wantPID: If True, create a pidfile for the socket.- """- self._reactor = reactor- self._address = address- self._backlog = backlog- self._mode = mode- self._wantPID = wantPID--- def listen(self, protocolFactory):- """- Implement L{IStreamServerEndpoint.listen} to listen on a UNIX socket.- """- return defer.execute(self._reactor.listenUNIX, self._address,- protocolFactory,- backlog=self._backlog,- mode=self._mode,- wantPID=self._wantPID)----@implementer(interfaces.IStreamClientEndpoint)-class UNIXClientEndpoint(object):- """- UnixSocket client endpoint.- """- def __init__(self, reactor, path, timeout=30, checkPID=0):- """- @param reactor: An L{IReactorUNIX} provider.-- @param path: The path to the Unix socket file, used when connecting- @type path: str-- @param timeout: Number of seconds to wait before assuming the- connection has failed.- @type timeout: int-- @param checkPID: If True, check for a pid file to verify that a server- is listening.- @type checkPID: bool- """- self._reactor = reactor- self._path = path- self._timeout = timeout- self._checkPID = checkPID--- def connect(self, protocolFactory):- """- Implement L{IStreamClientEndpoint.connect} to connect via a- UNIX Socket- """- try:- wf = _WrappingFactory(protocolFactory)- self._reactor.connectUNIX(- self._path, wf,- timeout=self._timeout,- checkPID=self._checkPID)- return wf._onConnection- except:- return defer.fail()----@implementer(interfaces.IStreamServerEndpoint)-class AdoptedStreamServerEndpoint(object):- """- An endpoint for listening on a file descriptor initialized outside of- Twisted.-- @ivar _used: A C{bool} indicating whether this endpoint has been used to- listen with a factory yet. C{True} if so.- """- _close = os.close- _setNonBlocking = staticmethod(fdesc.setNonBlocking)-- def __init__(self, reactor, fileno, addressFamily):- """- @param reactor: An L{IReactorSocket} provider.-- @param fileno: An integer file descriptor corresponding to a listening- I{SOCK_STREAM} socket.-- @param addressFamily: The address family of the socket given by- C{fileno}.- """- self.reactor = reactor- self.fileno = fileno- self.addressFamily = addressFamily- self._used = False--- def listen(self, factory):- """- Implement L{IStreamServerEndpoint.listen} to start listening on, and- then close, C{self._fileno}.- """- if self._used:- return defer.fail(error.AlreadyListened())- self._used = True-- try:- self._setNonBlocking(self.fileno)- port = self.reactor.adoptStreamPort(- self.fileno, self.addressFamily, factory)- self._close(self.fileno)- except:- return defer.fail()- return defer.succeed(port)-----def _parseTCP(factory, port, interface="", backlog=50):- """- Internal parser function for L{_parseServer} to convert the string- arguments for a TCP(IPv4) stream endpoint into the structured arguments.-- @param factory: the protocol factory being parsed, or C{None}. (This was a- leftover argument from when this code was in C{strports}, and is now- mostly None and unused.)-- @type factory: L{IProtocolFactory} or C{NoneType}-- @param port: the integer port number to bind- @type port: C{str}-- @param interface: the interface IP to listen on- @param backlog: the length of the listen queue- @type backlog: C{str}-- @return: a 2-tuple of (args, kwargs), describing the parameters to- L{IReactorTCP.listenTCP} (or, modulo argument 2, the factory, arguments- to L{TCP4ServerEndpoint}.- """- return (int(port), factory), {'interface': interface,- 'backlog': int(backlog)}----def _parseUNIX(factory, address, mode='666', backlog=50, lockfile=True):- """- Internal parser function for L{_parseServer} to convert the string- arguments for a UNIX (AF_UNIX/SOCK_STREAM) stream endpoint into the- structured arguments.-- @param factory: the protocol factory being parsed, or C{None}. (This was a- leftover argument from when this code was in C{strports}, and is now- mostly None and unused.)-- @type factory: L{IProtocolFactory} or C{NoneType}-- @param address: the pathname of the unix socket- @type address: C{str}-- @param backlog: the length of the listen queue- @type backlog: C{str}-- @param lockfile: A string '0' or '1', mapping to True and False- respectively. See the C{wantPID} argument to C{listenUNIX}-- @return: a 2-tuple of (args, kwargs), describing the parameters to- L{IReactorTCP.listenUNIX} (or, modulo argument 2, the factory,- arguments to L{UNIXServerEndpoint}.- """- return (- (address, factory),- {'mode': int(mode, 8), 'backlog': int(backlog),- 'wantPID': bool(int(lockfile))})----def _parseSSL(factory, port, privateKey="server.pem", certKey=None,- sslmethod=None, interface='', backlog=50):- """- Internal parser function for L{_parseServer} to convert the string- arguments for an SSL (over TCP/IPv4) stream endpoint into the structured- arguments.-- @param factory: the protocol factory being parsed, or C{None}. (This was a- leftover argument from when this code was in C{strports}, and is now- mostly None and unused.)- @type factory: L{IProtocolFactory} or C{NoneType}-- @param port: the integer port number to bind- @type port: C{str}-- @param interface: the interface IP to listen on- @param backlog: the length of the listen queue- @type backlog: C{str}-- @param privateKey: The file name of a PEM format private key file.- @type privateKey: C{str}-- @param certKey: The file name of a PEM format certificate file.- @type certKey: C{str}-- @param sslmethod: The string name of an SSL method, based on the name of a- constant in C{OpenSSL.SSL}. Must be one of: "SSLv23_METHOD",- "SSLv2_METHOD", "SSLv3_METHOD", "TLSv1_METHOD".- @type sslmethod: C{str}-- @return: a 2-tuple of (args, kwargs), describing the parameters to- L{IReactorSSL.listenSSL} (or, modulo argument 2, the factory, arguments- to L{SSL4ServerEndpoint}.- """- from twisted.internet import ssl- if certKey is None:- certKey = privateKey- kw = {}- if sslmethod is not None:- kw['method'] = getattr(ssl.SSL, sslmethod)- else:- kw['method'] = ssl.SSL.SSLv23_METHOD- certPEM = FilePath(certKey).getContent()- keyPEM = FilePath(privateKey).getContent()- privateCertificate = ssl.PrivateCertificate.loadPEM(certPEM + keyPEM)- cf = ssl.CertificateOptions(- privateKey=privateCertificate.privateKey.original,- certificate=privateCertificate.original,- **kw- )- return ((int(port), factory, cf),- {'interface': interface, 'backlog': int(backlog)})----@implementer(IPlugin, IStreamServerEndpointStringParser)-class _StandardIOParser(object):- """- Stream server endpoint string parser for the Standard I/O type.-- @ivar prefix: See L{IStreamClientEndpointStringParser.prefix}.- """- prefix = "stdio"-- def _parseServer(self, reactor):- """- Internal parser function for L{_parseServer} to convert the string- arguments into structured arguments for the L{StandardIOEndpoint}-- @param reactor: Reactor for the endpoint- """- return StandardIOEndpoint(reactor)--- def parseStreamServer(self, reactor, *args, **kwargs):- # Redirects to another function (self._parseServer), tricks zope.interface- # into believing the interface is correctly implemented.- return self._parseServer(reactor)-----@implementer(IPlugin, IStreamServerEndpointStringParser)-class _TCP6ServerParser(object):- """- Stream server endpoint string parser for the TCP6ServerEndpoint type.-- @ivar prefix: See L{IStreamClientEndpointStringParser.prefix}.- """- prefix = "tcp6" # Used in _parseServer to identify the plugin with the endpoint type-- def _parseServer(self, reactor, port, backlog=50, interface='::'):- """- Internal parser function for L{_parseServer} to convert the string- arguments into structured arguments for the L{TCP6ServerEndpoint}-- @param reactor: An L{IReactorTCP} provider.-- @param port: The port number used for listening- @type port: int-- @param backlog: Size of the listen queue- @type backlog: int-- @param interface: The hostname to bind to- @type interface: str- """- port = int(port)- backlog = int(backlog)- return TCP6ServerEndpoint(reactor, port, backlog, interface)--- def parseStreamServer(self, reactor, *args, **kwargs):- # Redirects to another function (self._parseServer), tricks zope.interface- # into believing the interface is correctly implemented.- return self._parseServer(reactor, *args, **kwargs)----_serverParsers = {"tcp": _parseTCP,- "unix": _parseUNIX,- "ssl": _parseSSL,- }--_OP, _STRING = range(2)--def _tokenize(description):- """- Tokenize a strports string and yield each token.-- @param description: a string as described by L{serverFromString} or- L{clientFromString}.-- @return: an iterable of 2-tuples of (L{_OP} or L{_STRING}, string). Tuples- starting with L{_OP} will contain a second element of either ':' (i.e.- 'next parameter') or '=' (i.e. 'assign parameter value'). For example,- the string 'hello:greet\=ing=world' would result in a generator- yielding these values::-- _STRING, 'hello'- _OP, ':'- _STRING, 'greet=ing'- _OP, '='- _STRING, 'world'- """- current = ''- ops = ':='- nextOps = {':': ':=', '=': ':'}- description = iter(description)- for n in description:- if n in ops:- yield _STRING, current- yield _OP, n- current = ''- ops = nextOps[n]- elif n == '\\':- current += next(description)- else:- current += n- yield _STRING, current----def _parse(description):- """- Convert a description string into a list of positional and keyword- parameters, using logic vaguely like what Python does.-- @param description: a string as described by L{serverFromString} or- L{clientFromString}.-- @return: a 2-tuple of C{(args, kwargs)}, where 'args' is a list of all- ':'-separated C{str}s not containing an '=' and 'kwargs' is a map of- all C{str}s which do contain an '='. For example, the result of- C{_parse('a:b:d=1:c')} would be C{(['a', 'b', 'c'], {'d': '1'})}.- """- args, kw = [], {}- def add(sofar):- if len(sofar) == 1:- args.append(sofar[0])- else:- kw[sofar[0]] = sofar[1]- sofar = ()- for (type, value) in _tokenize(description):- if type is _STRING:- sofar += (value,)- elif value == ':':- add(sofar)- sofar = ()- add(sofar)- return args, kw---# Mappings from description "names" to endpoint constructors.-_endpointServerFactories = {- 'TCP': TCP4ServerEndpoint,- 'SSL': SSL4ServerEndpoint,- 'UNIX': UNIXServerEndpoint,- }--_endpointClientFactories = {- 'TCP': TCP4ClientEndpoint,- 'SSL': SSL4ClientEndpoint,- 'UNIX': UNIXClientEndpoint,- }---_NO_DEFAULT = object()--def _parseServer(description, factory, default=None):- """- Parse a stports description into a 2-tuple of arguments and keyword values.-- @param description: A description in the format explained by- L{serverFromString}.- @type description: C{str}-- @param factory: A 'factory' argument; this is left-over from- twisted.application.strports, it's not really used.- @type factory: L{IProtocolFactory} or L{None}-- @param default: Deprecated argument, specifying the default parser mode to- use for unqualified description strings (those which do not have a ':'- and prefix).- @type default: C{str} or C{NoneType}-- @return: a 3-tuple of (plugin or name, arguments, keyword arguments)- """- args, kw = _parse(description)- if not args or (len(args) == 1 and not kw):- deprecationMessage = (- "Unqualified strport description passed to 'service'."- "Use qualified endpoint descriptions; for example, 'tcp:%s'."- % (description,))- if default is None:- default = 'tcp'- warnings.warn(- deprecationMessage, category=DeprecationWarning, stacklevel=4)- elif default is _NO_DEFAULT:- raise ValueError(deprecationMessage)- # If the default has been otherwise specified, the user has already- # been warned.- args[0:0] = [default]- endpointType = args[0]- parser = _serverParsers.get(endpointType)- if parser is None:- # If the required parser is not found in _server, check if- # a plugin exists for the endpointType- for plugin in getPlugins(IStreamServerEndpointStringParser):- if plugin.prefix == endpointType:- return (plugin, args[1:], kw)- raise ValueError("Unknown endpoint type: '%s'" % (endpointType,))- return (endpointType.upper(),) + parser(factory, *args[1:], **kw)----def _serverFromStringLegacy(reactor, description, default):- """- Underlying implementation of L{serverFromString} which avoids exposing the- deprecated 'default' argument to anything but L{strports.service}.- """- nameOrPlugin, args, kw = _parseServer(description, None, default)- if type(nameOrPlugin) is not str:- plugin = nameOrPlugin- return plugin.parseStreamServer(reactor, *args, **kw)- else:- name = nameOrPlugin- # Chop out the factory.- args = args[:1] + args[2:]- return _endpointServerFactories[name](reactor, *args, **kw)----def serverFromString(reactor, description):- """- Construct a stream server endpoint from an endpoint description string.-- The format for server endpoint descriptions is a simple string. It is a- prefix naming the type of endpoint, then a colon, then the arguments for- that endpoint.-- For example, you can call it like this to create an endpoint that will- listen on TCP port 80::-- serverFromString(reactor, "tcp:80")-- Additional arguments may be specified as keywords, separated with colons.- For example, you can specify the interface for a TCP server endpoint to- bind to like this::-- serverFromString(reactor, "tcp:80:interface=127.0.0.1")-- SSL server endpoints may be specified with the 'ssl' prefix, and the- private key and certificate files may be specified by the C{privateKey} and- C{certKey} arguments::-- serverFromString(reactor, "ssl:443:privateKey=key.pem:certKey=crt.pem")-- If a private key file name (C{privateKey}) isn't provided, a "server.pem"- file is assumed to exist which contains the private key. If the certificate- file name (C{certKey}) isn't provided, the private key file is assumed to- contain the certificate as well.-- You may escape colons in arguments with a backslash, which you will need to- use if you want to specify a full pathname argument on Windows::-- serverFromString(reactor,- "ssl:443:privateKey=C\\:/key.pem:certKey=C\\:/cert.pem")-- finally, the 'unix' prefix may be used to specify a filesystem UNIX socket,- optionally with a 'mode' argument to specify the mode of the socket file- created by C{listen}::-- serverFromString(reactor, "unix:/var/run/finger")- serverFromString(reactor, "unix:/var/run/finger:mode=660")-- This function is also extensible; new endpoint types may be registered as- L{IStreamServerEndpointStringParser} plugins. See that interface for more- information.-- @param reactor: The server endpoint will be constructed with this reactor.-- @param description: The strports description to parse.-- @return: A new endpoint which can be used to listen with the parameters- given by by C{description}.-- @rtype: L{IStreamServerEndpoint<twisted.internet.interfaces.IStreamServerEndpoint>}-- @raise ValueError: when the 'description' string cannot be parsed.-- @since: 10.2- """- return _serverFromStringLegacy(reactor, description, _NO_DEFAULT)----def quoteStringArgument(argument):- """- Quote an argument to L{serverFromString} and L{clientFromString}. Since- arguments are separated with colons and colons are escaped with- backslashes, some care is necessary if, for example, you have a pathname,- you may be tempted to interpolate into a string like this::-- serverFromString("ssl:443:privateKey=%s" % (myPathName,))-- This may appear to work, but will have portability issues (Windows- pathnames, for example). Usually you should just construct the appropriate- endpoint type rather than interpolating strings, which in this case would- be L{SSL4ServerEndpoint}. There are some use-cases where you may need to- generate such a string, though; for example, a tool to manipulate a- configuration file which has strports descriptions in it. To be correct in- those cases, do this instead::-- serverFromString("ssl:443:privateKey=%s" %- (quoteStringArgument(myPathName),))-- @param argument: The part of the endpoint description string you want to- pass through.-- @type argument: C{str}-- @return: The quoted argument.-- @rtype: C{str}- """- return argument.replace('\\', '\\\\').replace(':', '\\:')----def _parseClientTCP(*args, **kwargs):- """- Perform any argument value coercion necessary for TCP client parameters.-- Valid positional arguments to this function are host and port.-- Valid keyword arguments to this function are all L{IReactorTCP.connectTCP}- arguments.-- @return: The coerced values as a C{dict}.- """-- if len(args) == 2:- kwargs['port'] = int(args[1])- kwargs['host'] = args[0]- elif len(args) == 1:- if 'host' in kwargs:- kwargs['port'] = int(args[0])- else:- kwargs['host'] = args[0]-- try:- kwargs['port'] = int(kwargs['port'])- except KeyError:- pass-- try:- kwargs['timeout'] = int(kwargs['timeout'])- except KeyError:- pass- return kwargs----def _loadCAsFromDir(directoryPath):- """- Load certificate-authority certificate objects in a given directory.-- @param directoryPath: a L{FilePath} pointing at a directory to load .pem- files from.-- @return: a C{list} of L{OpenSSL.crypto.X509} objects.- """- from twisted.internet import ssl-- caCerts = {}- for child in directoryPath.children():- if not child.basename().split('.')[-1].lower() == 'pem':- continue- try:- data = child.getContent()- except IOError:- # Permission denied, corrupt disk, we don't care.- continue- try:- theCert = ssl.Certificate.loadPEM(data)- except ssl.SSL.Error:- # Duplicate certificate, invalid certificate, etc. We don't care.- pass- else:- caCerts[theCert.digest()] = theCert.original- return caCerts.values()----def _parseClientSSL(*args, **kwargs):- """- Perform any argument value coercion necessary for SSL client parameters.-- Valid keyword arguments to this function are all L{IReactorSSL.connectSSL}- arguments except for C{contextFactory}. Instead, C{certKey} (the path name- of the certificate file) C{privateKey} (the path name of the private key- associated with the certificate) are accepted and used to construct a- context factory.-- Valid positional arguments to this function are host and port.-- @param caCertsDir: The one parameter which is not part of- L{IReactorSSL.connectSSL}'s signature, this is a path name used to- construct a list of certificate authority certificates. The directory- will be scanned for files ending in C{.pem}, all of which will be- considered valid certificate authorities for this connection.-- @type caCertsDir: C{str}-- @return: The coerced values as a C{dict}.- """- from twisted.internet import ssl- kwargs = _parseClientTCP(*args, **kwargs)- certKey = kwargs.pop('certKey', None)- privateKey = kwargs.pop('privateKey', None)- caCertsDir = kwargs.pop('caCertsDir', None)- if certKey is not None:- certx509 = ssl.Certificate.loadPEM(- FilePath(certKey).getContent()).original- else:- certx509 = None- if privateKey is not None:- privateKey = ssl.PrivateCertificate.loadPEM(- FilePath(privateKey).getContent()).privateKey.original- else:- privateKey = None- if caCertsDir is not None:- verify = True- caCerts = _loadCAsFromDir(FilePath(caCertsDir))- else:- verify = False- caCerts = None- kwargs['sslContextFactory'] = ssl.CertificateOptions(- method=ssl.SSL.SSLv23_METHOD,- certificate=certx509,- privateKey=privateKey,- verify=verify,- caCerts=caCerts- )- return kwargs----def _parseClientUNIX(*args, **kwargs):- """- Perform any argument value coercion necessary for UNIX client parameters.-- Valid keyword arguments to this function are all L{IReactorUNIX.connectUNIX}- keyword arguments except for C{checkPID}. Instead, C{lockfile} is accepted- and has the same meaning. Also C{path} is used instead of C{address}.-- Valid positional arguments to this function are C{path}.-- @return: The coerced values as a C{dict}.- """- if len(args) == 1:- kwargs['path'] = args[0]-- try:- kwargs['checkPID'] = bool(int(kwargs.pop('lockfile')))- except KeyError:- pass- try:- kwargs['timeout'] = int(kwargs['timeout'])- except KeyError:- pass- return kwargs--_clientParsers = {- 'TCP': _parseClientTCP,- 'SSL': _parseClientSSL,- 'UNIX': _parseClientUNIX,- }----def clientFromString(reactor, description):- """- Construct a client endpoint from a description string.-- Client description strings are much like server description strings,- although they take all of their arguments as keywords, aside from host and- port.-- You can create a TCP client endpoint with the 'host' and 'port' arguments,- like so::-- clientFromString(reactor, "tcp:host=www.example.com:port=80")-- or, without specifying host and port keywords::-- clientFromString(reactor, "tcp:www.example.com:80")-- Or you can specify only one or the other, as in the following 2 examples::-- clientFromString(reactor, "tcp:host=www.example.com:80")- clientFromString(reactor, "tcp:www.example.com:port=80")-- or an SSL client endpoint with those arguments, plus the arguments used by- the server SSL, for a client certificate::-- clientFromString(reactor, "ssl:web.example.com:443:"- "privateKey=foo.pem:certKey=foo.pem")-- to specify your certificate trust roots, you can identify a directory with- PEM files in it with the C{caCertsDir} argument::-- clientFromString(reactor, "ssl:host=web.example.com:port=443:"- "caCertsDir=/etc/ssl/certs")-- You can create a UNIX client endpoint with the 'path' argument and optional- 'lockfile' and 'timeout' arguments::-- clientFromString(reactor, "unix:path=/var/foo/bar:lockfile=1:timeout=9")-- or, with the path as a positional argument with or without optional- arguments as in the following 2 examples::-- clientFromString(reactor, "unix:/var/foo/bar")- clientFromString(reactor, "unix:/var/foo/bar:lockfile=1:timeout=9")-- This function is also extensible; new endpoint types may be registered as- L{IStreamClientEndpointStringParser} plugins. See that interface for more- information.-- @param reactor: The client endpoint will be constructed with this reactor.-- @param description: The strports description to parse.-- @return: A new endpoint which can be used to connect with the parameters- given by by C{description}.- @rtype: L{IStreamClientEndpoint<twisted.internet.interfaces.IStreamClientEndpoint>}-- @since: 10.2- """- args, kwargs = _parse(description)- aname = args.pop(0)- name = aname.upper()- for plugin in getPlugins(IStreamClientEndpointStringParser):- if plugin.prefix.upper() == name:- return plugin.parseStreamClient(*args, **kwargs)- if name not in _clientParsers:- raise ValueError("Unknown endpoint type: %r" % (aname,))- kwargs = _clientParsers[name](*args, **kwargs)- return _endpointClientFactories[name](reactor, **kwargs)----def connectProtocol(endpoint, protocol):- """- Connect a protocol instance to an endpoint.-- This allows using a client endpoint without having to create a factory.-- @param endpoint: A client endpoint to connect to.-- @param protocol: A protocol instance.-- @return: The result of calling C{connect} on the endpoint, i.e. a- L{Deferred} that will fire with the protocol when connected, or an- appropriate error.- """- class OneShotFactory(Factory):- def buildProtocol(self, addr):- return protocol- return endpoint.connect(OneShotFactory())-}}} '''diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.pyindex 7b2a78632..fdcbf3977 100644--- a/scrapy/xlib/tx/interfaces.py+++ b/scrapy/xlib/tx/interfaces.py@@ -9,8 +9,6 @@ Maintainer: Itamar Shtull-Trauringfrom __future__ import division, absolute_import-from zope.interface import Interface, Attribute-from twisted.internet.interfaces import (IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL,IReactorWin32Events, IReactorUDP, IReactorMulticast, IReactorProcess,@@ -28,2434 +26,3 @@ from twisted.internet.interfaces import (IStreamServerEndpointStringParser, IStreamClientEndpointStringParser,IReactorUNIX, IReactorUNIXDatagram, IReactorSocket, IResolver)--''' {{{-class IAddress(Interface):- """- An address, e.g. a TCP C{(host, port)}.-- Default implementations are in L{twisted.internet.address}.- """--### Reactor Interfaces--class IConnector(Interface):- """- Object used to interface between connections and protocols.-- Each L{IConnector} manages one connection.- """-- def stopConnecting():- """- Stop attempting to connect.- """-- def disconnect():- """- Disconnect regardless of the connection state.-- If we are connected, disconnect, if we are trying to connect,- stop trying.- """-- def connect():- """- Try to connect to remote address.- """-- def getDestination():- """- Return destination this will try to connect to.-- @return: An object which provides L{IAddress}.- """----class IResolverSimple(Interface):- def getHostByName(name, timeout = (1, 3, 11, 45)):- """- Resolve the domain name C{name} into an IP address.-- @type name: C{str}- @type timeout: C{tuple}- @rtype: L{twisted.internet.defer.Deferred}- @return: The callback of the Deferred that is returned will be- passed a string that represents the IP address of the specified- name, or the errback will be called if the lookup times out. If- multiple types of address records are associated with the name,- A6 records will be returned in preference to AAAA records, which- will be returned in preference to A records. If there are multiple- records of the type to be returned, one will be selected at random.-- @raise twisted.internet.defer.TimeoutError: Raised (asynchronously)- if the name cannot be resolved within the specified timeout period.- """----class IResolver(IResolverSimple):- def query(query, timeout=None):- """- Dispatch C{query} to the method which can handle its type.-- @type query: L{twisted.names.dns.Query}- @param query: The DNS query being issued, to which a response is to be- generated.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupAddress(name, timeout=None):- """- Perform an A record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupAddress6(name, timeout=None):- """- Perform an A6 record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupIPV6Address(name, timeout=None):- """- Perform an AAAA record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupMailExchange(name, timeout=None):- """- Perform an MX record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupNameservers(name, timeout=None):- """- Perform an NS record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupCanonicalName(name, timeout=None):- """- Perform a CNAME record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupMailBox(name, timeout=None):- """- Perform an MB record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupMailGroup(name, timeout=None):- """- Perform an MG record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupMailRename(name, timeout=None):- """- Perform an MR record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupPointer(name, timeout=None):- """- Perform a PTR record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupAuthority(name, timeout=None):- """- Perform an SOA record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupNull(name, timeout=None):- """- Perform a NULL record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupWellKnownServices(name, timeout=None):- """- Perform a WKS record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupHostInfo(name, timeout=None):- """- Perform a HINFO record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupMailboxInfo(name, timeout=None):- """- Perform an MINFO record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupText(name, timeout=None):- """- Perform a TXT record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupResponsibility(name, timeout=None):- """- Perform an RP record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupAFSDatabase(name, timeout=None):- """- Perform an AFSDB record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupService(name, timeout=None):- """- Perform an SRV record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupAllRecords(name, timeout=None):- """- Perform an ALL_RECORD lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupSenderPolicy(name, timeout= 10):- """- Perform a SPF record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupNamingAuthorityPointer(name, timeout=None):- """- Perform a NAPTR record lookup.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: Sequence of C{int}- @param timeout: Number of seconds after which to reissue the query.- When the last timeout expires, the query is considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances. The first element of the- tuple gives answers. The second element of the tuple gives- authorities. The third element of the tuple gives additional- information. The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """--- def lookupZone(name, timeout=None):- """- Perform an AXFR record lookup.-- NB This is quite different from other DNS requests. See- U{http://cr.yp.to/djbdns/axfr-notes.html} for more- information.-- NB Unlike other C{lookup*} methods, the timeout here is not a- list of ints, it is a single int.-- @type name: C{str}- @param name: DNS name to resolve.-- @type timeout: C{int}- @param timeout: When this timeout expires, the query is- considered failed.-- @rtype: L{Deferred}- @return: A L{Deferred} which fires with a three-tuple of lists of- L{twisted.names.dns.RRHeader} instances.- The first element of the tuple gives answers.- The second and third elements are always empty.- The L{Deferred} may instead fail with one of the- exceptions defined in L{twisted.names.error} or with- C{NotImplementedError}.- """----class IReactorTCP(Interface):-- def listenTCP(port, factory, backlog=50, interface=''):- """- Connects a given protocol factory to the given numeric TCP/IP port.-- @param port: a port number on which to listen-- @param factory: a L{twisted.internet.protocol.ServerFactory} instance-- @param backlog: size of the listen queue-- @param interface: The local IPv4 or IPv6 address to which to bind;- defaults to '', ie all IPv4 addresses. To bind to all IPv4 and IPv6- addresses, you must call this method twice.-- @return: an object that provides L{IListeningPort}.-- @raise CannotListenError: as defined here- L{twisted.internet.error.CannotListenError},- if it cannot listen on this port (e.g., it- cannot bind to the required port number)- """-- def connectTCP(host, port, factory, timeout=30, bindAddress=None):- """- Connect a TCP client.-- @param host: a host name-- @param port: a port number-- @param factory: a L{twisted.internet.protocol.ClientFactory} instance-- @param timeout: number of seconds to wait before assuming the- connection has failed.-- @param bindAddress: a (host, port) tuple of local address to bind- to, or None.-- @return: An object which provides L{IConnector}. This connector will- call various callbacks on the factory when a connection is- made, failed, or lost - see- L{ClientFactory<twisted.internet.protocol.ClientFactory>}- docs for details.- """--class IReactorSSL(Interface):-- def connectSSL(host, port, factory, contextFactory, timeout=30, bindAddress=None):- """- Connect a client Protocol to a remote SSL socket.-- @param host: a host name-- @param port: a port number-- @param factory: a L{twisted.internet.protocol.ClientFactory} instance-- @param contextFactory: a L{twisted.internet.ssl.ClientContextFactory} object.-- @param timeout: number of seconds to wait before assuming the- connection has failed.-- @param bindAddress: a (host, port) tuple of local address to bind to,- or C{None}.-- @return: An object which provides L{IConnector}.- """-- def listenSSL(port, factory, contextFactory, backlog=50, interface=''):- """- Connects a given protocol factory to the given numeric TCP/IP port.- The connection is a SSL one, using contexts created by the context- factory.-- @param port: a port number on which to listen-- @param factory: a L{twisted.internet.protocol.ServerFactory} instance-- @param contextFactory: a L{twisted.internet.ssl.ContextFactory} instance-- @param backlog: size of the listen queue-- @param interface: the hostname to bind to, defaults to '' (all)- """----class IReactorUNIX(Interface):- """- UNIX socket methods.- """-- def connectUNIX(address, factory, timeout=30, checkPID=0):- """- Connect a client protocol to a UNIX socket.-- @param address: a path to a unix socket on the filesystem.-- @param factory: a L{twisted.internet.protocol.ClientFactory} instance-- @param timeout: number of seconds to wait before assuming the connection- has failed.-- @param checkPID: if True, check for a pid file to verify that a server- is listening. If C{address} is a Linux abstract namespace path,- this must be C{False}.-- @return: An object which provides L{IConnector}.- """--- def listenUNIX(address, factory, backlog=50, mode=0o666, wantPID=0):- """- Listen on a UNIX socket.-- @param address: a path to a unix socket on the filesystem.-- @param factory: a L{twisted.internet.protocol.Factory} instance.-- @param backlog: number of connections to allow in backlog.-- @param mode: The mode (B{not} umask) to set on the unix socket. See- platform specific documentation for information about how this- might affect connection attempts.- @type mode: C{int}-- @param wantPID: if True, create a pidfile for the socket. If C{address}- is a Linux abstract namespace path, this must be C{False}.-- @return: An object which provides L{IListeningPort}.- """----class IReactorUNIXDatagram(Interface):- """- Datagram UNIX socket methods.- """-- def connectUNIXDatagram(address, protocol, maxPacketSize=8192, mode=0o666, bindAddress=None):- """- Connect a client protocol to a datagram UNIX socket.-- @param address: a path to a unix socket on the filesystem.-- @param protocol: a L{twisted.internet.protocol.ConnectedDatagramProtocol} instance-- @param maxPacketSize: maximum packet size to accept-- @param mode: The mode (B{not} umask) to set on the unix socket. See- platform specific documentation for information about how this- might affect connection attempts.- @type mode: C{int}-- @param bindAddress: address to bind to-- @return: An object which provides L{IConnector}.- """--- def listenUNIXDatagram(address, protocol, maxPacketSize=8192, mode=0o666):- """- Listen on a datagram UNIX socket.-- @param address: a path to a unix socket on the filesystem.-- @param protocol: a L{twisted.internet.protocol.DatagramProtocol} instance.-- @param maxPacketSize: maximum packet size to accept-- @param mode: The mode (B{not} umask) to set on the unix socket. See- platform specific documentation for information about how this- might affect connection attempts.- @type mode: C{int}-- @return: An object which provides L{IListeningPort}.- """----class IReactorWin32Events(Interface):- """- Win32 Event API methods-- @since: 10.2- """-- def addEvent(event, fd, action):- """- Add a new win32 event to the event loop.-- @param event: a Win32 event object created using win32event.CreateEvent()-- @param fd: an instance of L{twisted.internet.abstract.FileDescriptor}-- @param action: a string that is a method name of the fd instance.- This method is called in response to the event.-- @return: None- """--- def removeEvent(event):- """- Remove an event.-- @param event: a Win32 event object added using L{IReactorWin32Events.addEvent}-- @return: None- """----class IReactorUDP(Interface):- """- UDP socket methods.- """-- def listenUDP(port, protocol, interface='', maxPacketSize=8192):- """- Connects a given DatagramProtocol to the given numeric UDP port.-- @return: object which provides L{IListeningPort}.- """----class IReactorMulticast(Interface):- """- UDP socket methods that support multicast.-- IMPORTANT: This is an experimental new interface. It may change- without backwards compatibility. Suggestions are welcome.- """-- def listenMulticast(port, protocol, interface='', maxPacketSize=8192,- listenMultiple=False):- """- Connects a given- L{DatagramProtocol<twisted.internet.protocol.DatagramProtocol>} to the- given numeric UDP port.-- @param listenMultiple: If set to True, allows multiple sockets to- bind to the same address and port number at the same time.- @type listenMultiple: C{bool}-- @returns: An object which provides L{IListeningPort}.-- @see: L{twisted.internet.interfaces.IMulticastTransport}- @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html}- """----class IReactorSocket(Interface):- """- Methods which allow a reactor to use externally created sockets.-- For example, to use C{adoptStreamPort} to implement behavior equivalent- to that of L{IReactorTCP.listenTCP}, you might write code like this::-- from socket import SOMAXCONN, AF_INET, SOCK_STREAM, socket- portSocket = socket(AF_INET, SOCK_STREAM)- # Set FD_CLOEXEC on port, left as an exercise. Then make it into a- # non-blocking listening port:- portSocket.setblocking(False)- portSocket.bind(('192.168.1.2', 12345))- portSocket.listen(SOMAXCONN)-- # Now have the reactor use it as a TCP port- port = reactor.adoptStreamPort(- portSocket.fileno(), AF_INET, YourFactory())-- # portSocket itself is no longer necessary, and needs to be cleaned- # up by us.- portSocket.close()-- # Whenever the server is no longer needed, stop it as usual.- stoppedDeferred = port.stopListening()-- Another potential use is to inherit a listening descriptor from a parent- process (for example, systemd or launchd), or to receive one over a UNIX- domain socket.-- Some plans for extending this interface exist. See:-- - U{http://twistedmatrix.com/trac/ticket/5570}: established connections- - U{http://twistedmatrix.com/trac/ticket/5573}: AF_UNIX ports- - U{http://twistedmatrix.com/trac/ticket/5574}: SOCK_DGRAM sockets- """-- def adoptStreamPort(fileDescriptor, addressFamily, factory):- """- Add an existing listening I{SOCK_STREAM} socket to the reactor to- monitor for new connections to accept and handle.-- @param fileDescriptor: A file descriptor associated with a socket which- is already bound to an address and marked as listening. The socket- must be set non-blocking. Any additional flags (for example,- close-on-exec) must also be set by application code. Application- code is responsible for closing the file descriptor, which may be- done as soon as C{adoptStreamPort} returns.- @type fileDescriptor: C{int}-- @param addressFamily: The address family (or I{domain}) of the socket.- For example, L{socket.AF_INET6}.-- @param factory: A L{ServerFactory} instance to use to create new- protocols to handle connections accepted via this socket.-- @return: An object providing L{IListeningPort}.-- @raise UnsupportedAddressFamily: If the given address family is not- supported by this reactor, or not supported with the given socket- type.-- @raise UnsupportedSocketType: If the given socket type is not supported- by this reactor, or not supported with the given socket type.- """--- def adoptStreamConnection(fileDescriptor, addressFamily, factory):- """- Add an existing connected I{SOCK_STREAM} socket to the reactor to- monitor for data.-- Note that the given factory won't have its C{startFactory} and- C{stopFactory} methods called, as there is no sensible time to call- them in this situation.-- @param fileDescriptor: A file descriptor associated with a socket which- is already connected. The socket must be set non-blocking. Any- additional flags (for example, close-on-exec) must also be set by- application code. Application code is responsible for closing the- file descriptor, which may be done as soon as- C{adoptStreamConnection} returns.- @type fileDescriptor: C{int}-- @param addressFamily: The address family (or I{domain}) of the socket.- For example, L{socket.AF_INET6}.-- @param factory: A L{ServerFactory} instance to use to create a new- protocol to handle the connection via this socket.-- @raise UnsupportedAddressFamily: If the given address family is not- supported by this reactor, or not supported with the given socket- type.-- @raise UnsupportedSocketType: If the given socket type is not supported- by this reactor, or not supported with the given socket type.- """----class IReactorProcess(Interface):-- def spawnProcess(processProtocol, executable, args=(), env={}, path=None,- uid=None, gid=None, usePTY=0, childFDs=None):- """- Spawn a process, with a process protocol.-- @type processProtocol: L{IProcessProtocol} provider- @param processProtocol: An object which will be notified of all- events related to the created process.-- @param executable: the file name to spawn - the full path should be- used.-- @param args: the command line arguments to pass to the process; a- sequence of strings. The first string should be the- executable's name.-- @type env: a C{dict} mapping C{str} to C{str}, or C{None}.- @param env: the environment variables to pass to the child process. The- resulting behavior varies between platforms. If- - C{env} is not set:- - On POSIX: pass an empty environment.- - On Windows: pass C{os.environ}.- - C{env} is C{None}:- - On POSIX: pass C{os.environ}.- - On Windows: pass C{os.environ}.- - C{env} is a C{dict}:- - On POSIX: pass the key/value pairs in C{env} as the- complete environment.- - On Windows: update C{os.environ} with the key/value- pairs in the C{dict} before passing it. As a- consequence of U{bug #1640- <http://twistedmatrix.com/trac/ticket/1640>}, passing- keys with empty values in an effort to unset- environment variables I{won't} unset them.-- @param path: the path to run the subprocess in - defaults to the- current directory.-- @param uid: user ID to run the subprocess as. (Only available on- POSIX systems.)-- @param gid: group ID to run the subprocess as. (Only available on- POSIX systems.)-- @param usePTY: if true, run this process in a pseudo-terminal.- optionally a tuple of C{(masterfd, slavefd, ttyname)},- in which case use those file descriptors.- (Not available on all systems.)-- @param childFDs: A dictionary mapping file descriptors in the new child- process to an integer or to the string 'r' or 'w'.-- If the value is an integer, it specifies a file- descriptor in the parent process which will be mapped- to a file descriptor (specified by the key) in the- child process. This is useful for things like inetd- and shell-like file redirection.-- If it is the string 'r', a pipe will be created and- attached to the child at that file descriptor: the- child will be able to write to that file descriptor- and the parent will receive read notification via the- L{IProcessProtocol.childDataReceived} callback. This- is useful for the child's stdout and stderr.-- If it is the string 'w', similar setup to the previous- case will occur, with the pipe being readable by the- child instead of writeable. The parent process can- write to that file descriptor using- L{IProcessTransport.writeToChild}. This is useful for- the child's stdin.-- If childFDs is not passed, the default behaviour is to- use a mapping that opens the usual stdin/stdout/stderr- pipes.-- @see: L{twisted.internet.protocol.ProcessProtocol}-- @return: An object which provides L{IProcessTransport}.-- @raise OSError: Raised with errno C{EAGAIN} or C{ENOMEM} if there are- insufficient system resources to create a new process.- """--class IReactorTime(Interface):- """- Time methods that a Reactor should implement.- """-- def seconds():- """- Get the current time in seconds.-- @return: A number-like object of some sort.- """--- def callLater(delay, callable, *args, **kw):- """- Call a function later.-- @type delay: C{float}- @param delay: the number of seconds to wait.-- @param callable: the callable object to call later.-- @param args: the arguments to call it with.-- @param kw: the keyword arguments to call it with.-- @return: An object which provides L{IDelayedCall} and can be used to- cancel the scheduled call, by calling its C{cancel()} method.- It also may be rescheduled by calling its C{delay()} or- C{reset()} methods.- """--- def getDelayedCalls():- """- Retrieve all currently scheduled delayed calls.-- @return: A tuple of all L{IDelayedCall} providers representing all- currently scheduled calls. This is everything that has been- returned by C{callLater} but not yet called or canceled.- """---class IDelayedCall(Interface):- """- A scheduled call.-- There are probably other useful methods we can add to this interface;- suggestions are welcome.- """-- def getTime():- """- Get time when delayed call will happen.-- @return: time in seconds since epoch (a float).- """-- def cancel():- """- Cancel the scheduled call.-- @raises twisted.internet.error.AlreadyCalled: if the call has already- happened.- @raises twisted.internet.error.AlreadyCancelled: if the call has already- been cancelled.- """-- def delay(secondsLater):- """- Delay the scheduled call.-- @param secondsLater: how many seconds from its current firing time to delay-- @raises twisted.internet.error.AlreadyCalled: if the call has already- happened.- @raises twisted.internet.error.AlreadyCancelled: if the call has already- been cancelled.- """-- def reset(secondsFromNow):- """- Reset the scheduled call's timer.-- @param secondsFromNow: how many seconds from now it should fire,- equivalent to C{.cancel()} and then doing another- C{reactor.callLater(secondsLater, ...)}-- @raises twisted.internet.error.AlreadyCalled: if the call has already- happened.- @raises twisted.internet.error.AlreadyCancelled: if the call has already- been cancelled.- """-- def active():- """- @return: True if this call is still active, False if it has been- called or cancelled.- """--class IReactorThreads(Interface):- """- Dispatch methods to be run in threads.-- Internally, this should use a thread pool and dispatch methods to them.- """-- def getThreadPool():- """- Return the threadpool used by L{callInThread}. Create it first if- necessary.-- @rtype: L{twisted.python.threadpool.ThreadPool}- """--- def callInThread(callable, *args, **kwargs):- """- Run the callable object in a separate thread.- """--- def callFromThread(callable, *args, **kw):- """- Cause a function to be executed by the reactor thread.-- Use this method when you want to run a function in the reactor's thread- from another thread. Calling L{callFromThread} should wake up the main- thread (where L{reactor.run()<reactor.run>} is executing) and run the- given callable in that thread.-- If you're writing a multi-threaded application the C{callable} may need- to be thread safe, but this method doesn't require it as such. If you- want to call a function in the next mainloop iteration, but you're in- the same thread, use L{callLater} with a delay of 0.- """--- def suggestThreadPoolSize(size):- """- Suggest the size of the internal threadpool used to dispatch functions- passed to L{callInThread}.- """---class IReactorCore(Interface):- """- Core methods that a Reactor must implement.- """-- running = Attribute(- "A C{bool} which is C{True} from I{during startup} to "- "I{during shutdown} and C{False} the rest of the time.")--- def resolve(name, timeout=10):- """- Return a L{twisted.internet.defer.Deferred} that will resolve a hostname.- """-- def run():- """- Fire 'startup' System Events, move the reactor to the 'running'- state, then run the main loop until it is stopped with C{stop()} or- C{crash()}.- """-- def stop():- """- Fire 'shutdown' System Events, which will move the reactor to the- 'stopped' state and cause C{reactor.run()} to exit.- """-- def crash():- """- Stop the main loop *immediately*, without firing any system events.-- This is named as it is because this is an extremely "rude" thing to do;- it is possible to lose data and put your system in an inconsistent- state by calling this. However, it is necessary, as sometimes a system- can become wedged in a pre-shutdown call.- """-- def iterate(delay=0):- """- Run the main loop's I/O polling function for a period of time.-- This is most useful in applications where the UI is being drawn "as- fast as possible", such as games. All pending L{IDelayedCall}s will- be called.-- The reactor must have been started (via the C{run()} method) prior to- any invocations of this method. It must also be stopped manually- after the last call to this method (via the C{stop()} method). This- method is not re-entrant: you must not call it recursively; in- particular, you must not call it while the reactor is running.- """-- def fireSystemEvent(eventType):- """- Fire a system-wide event.-- System-wide events are things like 'startup', 'shutdown', and- 'persist'.- """-- def addSystemEventTrigger(phase, eventType, callable, *args, **kw):- """- Add a function to be called when a system event occurs.-- Each "system event" in Twisted, such as 'startup', 'shutdown', and- 'persist', has 3 phases: 'before', 'during', and 'after' (in that- order, of course). These events will be fired internally by the- Reactor.-- An implementor of this interface must only implement those events- described here.-- Callbacks registered for the "before" phase may return either None or a- Deferred. The "during" phase will not execute until all of the- Deferreds from the "before" phase have fired.-- Once the "during" phase is running, all of the remaining triggers must- execute; their return values must be ignored.-- @param phase: a time to call the event -- either the string 'before',- 'after', or 'during', describing when to call it- relative to the event's execution.-- @param eventType: this is a string describing the type of event.-- @param callable: the object to call before shutdown.-- @param args: the arguments to call it with.-- @param kw: the keyword arguments to call it with.-- @return: an ID that can be used to remove this call with- removeSystemEventTrigger.- """-- def removeSystemEventTrigger(triggerID):- """- Removes a trigger added with addSystemEventTrigger.-- @param triggerID: a value returned from addSystemEventTrigger.-- @raise KeyError: If there is no system event trigger for the given- C{triggerID}.-- @raise ValueError: If there is no system event trigger for the given- C{triggerID}.-- @raise TypeError: If there is no system event trigger for the given- C{triggerID}.- """-- def callWhenRunning(callable, *args, **kw):- """- Call a function when the reactor is running.-- If the reactor has not started, the callable will be scheduled- to run when it does start. Otherwise, the callable will be invoked- immediately.-- @param callable: the callable object to call later.-- @param args: the arguments to call it with.-- @param kw: the keyword arguments to call it with.-- @return: None if the callable was invoked, otherwise a system- event id for the scheduled call.- """---class IReactorPluggableResolver(Interface):- """- A reactor with a pluggable name resolver interface.- """-- def installResolver(resolver):- """- Set the internal resolver to use to for name lookups.-- @type resolver: An object implementing the L{IResolverSimple} interface- @param resolver: The new resolver to use.-- @return: The previously installed resolver.- """---class IReactorDaemonize(Interface):- """- A reactor which provides hooks that need to be called before and after- daemonization.-- Notes:- - This interface SHOULD NOT be called by applications.- - This interface should only be implemented by reactors as a workaround- (in particular, it's implemented currently only by kqueue()).- For details please see the comments on ticket #1918.- """-- def beforeDaemonize():- """- Hook to be called immediately before daemonization. No reactor methods- may be called until L{afterDaemonize} is called.-- @return: C{None}.- """--- def afterDaemonize():- """- Hook to be called immediately after daemonization. This may only be- called after L{beforeDaemonize} had been called previously.-- @return: C{None}.- """----class IReactorFDSet(Interface):- """- Implement me to be able to use L{IFileDescriptor} type resources.-- This assumes that your main-loop uses UNIX-style numeric file descriptors- (or at least similarly opaque IDs returned from a .fileno() method)- """-- def addReader(reader):- """- I add reader to the set of file descriptors to get read events for.-- @param reader: An L{IReadDescriptor} provider that will be checked for- read events until it is removed from the reactor with- L{removeReader}.-- @return: C{None}.- """-- def addWriter(writer):- """- I add writer to the set of file descriptors to get write events for.-- @param writer: An L{IWriteDescriptor} provider that will be checked for- write events until it is removed from the reactor with- L{removeWriter}.-- @return: C{None}.- """-- def removeReader(reader):- """- Removes an object previously added with L{addReader}.-- @return: C{None}.- """-- def removeWriter(writer):- """- Removes an object previously added with L{addWriter}.-- @return: C{None}.- """-- def removeAll():- """- Remove all readers and writers.-- Should not remove reactor internal reactor connections (like a waker).-- @return: A list of L{IReadDescriptor} and L{IWriteDescriptor} providers- which were removed.- """-- def getReaders():- """- Return the list of file descriptors currently monitored for input- events by the reactor.-- @return: the list of file descriptors monitored for input events.- @rtype: C{list} of C{IReadDescriptor}- """-- def getWriters():- """- Return the list file descriptors currently monitored for output events- by the reactor.-- @return: the list of file descriptors monitored for output events.- @rtype: C{list} of C{IWriteDescriptor}- """---class IListeningPort(Interface):- """- A listening port.- """-- def startListening():- """- Start listening on this port.-- @raise CannotListenError: If it cannot listen on this port (e.g., it is- a TCP port and it cannot bind to the required- port number).- """-- def stopListening():- """- Stop listening on this port.-- If it does not complete immediately, will return Deferred that fires- upon completion.- """-- def getHost():- """- Get the host that this port is listening for.-- @return: An L{IAddress} provider.- """---class ILoggingContext(Interface):- """- Give context information that will be used to log events generated by- this item.- """-- def logPrefix():- """- @return: Prefix used during log formatting to indicate context.- @rtype: C{str}- """----class IFileDescriptor(ILoggingContext):- """- An interface representing a UNIX-style numeric file descriptor.- """-- def fileno():- """- @raise: If the descriptor no longer has a valid file descriptor- number associated with it.-- @return: The platform-specified representation of a file descriptor- number. Or C{-1} if the descriptor no longer has a valid file- descriptor number associated with it. As long as the descriptor- is valid, calls to this method on a particular instance must- return the same value.- """--- def connectionLost(reason):- """- Called when the connection was lost.-- This is called when the connection on a selectable object has been- lost. It will be called whether the connection was closed explicitly,- an exception occurred in an event handler, or the other end of the- connection closed it first.-- See also L{IHalfCloseableDescriptor} if your descriptor wants to be- notified separately of the two halves of the connection being closed.-- @param reason: A failure instance indicating the reason why the- connection was lost. L{error.ConnectionLost} and- L{error.ConnectionDone} are of special note, but the- failure may be of other classes as well.- """----class IReadDescriptor(IFileDescriptor):- """- An L{IFileDescriptor} that can read.-- This interface is generally used in conjunction with L{IReactorFDSet}.- """-- def doRead():- """- Some data is available for reading on your descriptor.-- @return: If an error is encountered which causes the descriptor to- no longer be valid, a L{Failure} should be returned. Otherwise,- C{None}.- """---class IWriteDescriptor(IFileDescriptor):- """- An L{IFileDescriptor} that can write.-- This interface is generally used in conjunction with L{IReactorFDSet}.- """-- def doWrite():- """- Some data can be written to your descriptor.-- @return: If an error is encountered which causes the descriptor to- no longer be valid, a L{Failure} should be returned. Otherwise,- C{None}.- """---class IReadWriteDescriptor(IReadDescriptor, IWriteDescriptor):- """- An L{IFileDescriptor} that can both read and write.- """---class IHalfCloseableDescriptor(Interface):- """- A descriptor that can be half-closed.- """-- def writeConnectionLost(reason):- """- Indicates write connection was lost.- """-- def readConnectionLost(reason):- """- Indicates read connection was lost.- """---class ISystemHandle(Interface):- """- An object that wraps a networking OS-specific handle.- """-- def getHandle():- """- Return a system- and reactor-specific handle.-- This might be a socket.socket() object, or some other type of- object, depending on which reactor is being used. Use and- manipulate at your own risk.-- This might be used in cases where you want to set specific- options not exposed by the Twisted APIs.- """---class IConsumer(Interface):- """- A consumer consumes data from a producer.- """-- def registerProducer(producer, streaming):- """- Register to receive data from a producer.-- This sets self to be a consumer for a producer. When this object runs- out of data (as when a send(2) call on a socket succeeds in moving the- last data from a userspace buffer into a kernelspace buffer), it will- ask the producer to resumeProducing().-- For L{IPullProducer} providers, C{resumeProducing} will be called once- each time data is required.-- For L{IPushProducer} providers, C{pauseProducing} will be called- whenever the write buffer fills up and C{resumeProducing} will only be- called when it empties.-- @type producer: L{IProducer} provider-- @type streaming: C{bool}- @param streaming: C{True} if C{producer} provides L{IPushProducer},- C{False} if C{producer} provides L{IPullProducer}.-- @raise RuntimeError: If a producer is already registered.-- @return: C{None}- """--- def unregisterProducer():- """- Stop consuming data from a producer, without disconnecting.- """--- def write(data):- """- The producer will write data by calling this method.-- The implementation must be non-blocking and perform whatever- buffering is necessary. If the producer has provided enough data- for now and it is a L{IPushProducer}, the consumer may call its- C{pauseProducing} method.- """----class IProducer(Interface):- """- A producer produces data for a consumer.-- Typically producing is done by calling the write method of an class- implementing L{IConsumer}.- """-- def stopProducing():- """- Stop producing data.-- This tells a producer that its consumer has died, so it must stop- producing data for good.- """---class IPushProducer(IProducer):- """- A push producer, also known as a streaming producer is expected to- produce (write to this consumer) data on a continuous basis, unless- it has been paused. A paused push producer will resume producing- after its resumeProducing() method is called. For a push producer- which is not pauseable, these functions may be noops.- """-- def pauseProducing():- """- Pause producing data.-- Tells a producer that it has produced too much data to process for- the time being, and to stop until resumeProducing() is called.- """- def resumeProducing():- """- Resume producing data.-- This tells a producer to re-add itself to the main loop and produce- more data for its consumer.- """--class IPullProducer(IProducer):- """- A pull producer, also known as a non-streaming producer, is- expected to produce data each time resumeProducing() is called.- """-- def resumeProducing():- """- Produce data for the consumer a single time.-- This tells a producer to produce data for the consumer once- (not repeatedly, once only). Typically this will be done- by calling the consumer's write() method a single time with- produced data.- """--class IProtocol(Interface):-- def dataReceived(data):- """- Called whenever data is received.-- Use this method to translate to a higher-level message. Usually, some- callback will be made upon the receipt of each complete protocol- message.-- @param data: a string of indeterminate length. Please keep in mind- that you will probably need to buffer some data, as partial- (or multiple) protocol messages may be received! I recommend- that unit tests for protocols call through to this method with- differing chunk sizes, down to one byte at a time.- """-- def connectionLost(reason):- """- Called when the connection is shut down.-- Clear any circular references here, and any external references- to this Protocol. The connection has been closed. The C{reason}- Failure wraps a L{twisted.internet.error.ConnectionDone} or- L{twisted.internet.error.ConnectionLost} instance (or a subclass- of one of those).-- @type reason: L{twisted.python.failure.Failure}- """-- def makeConnection(transport):- """- Make a connection to a transport and a server.- """-- def connectionMade():- """- Called when a connection is made.-- This may be considered the initializer of the protocol, because- it is called when the connection is completed. For clients,- this is called once the connection to the server has been- established; for servers, this is called after an accept() call- stops blocking and a socket has been received. If you need to- send any greeting or initial message, do it here.- """---class IProcessProtocol(Interface):- """- Interface for process-related event handlers.- """-- def makeConnection(process):- """- Called when the process has been created.-- @type process: L{IProcessTransport} provider- @param process: An object representing the process which has been- created and associated with this protocol.- """--- def childDataReceived(childFD, data):- """- Called when data arrives from the child process.-- @type childFD: C{int}- @param childFD: The file descriptor from which the data was- received.-- @type data: C{str}- @param data: The data read from the child's file descriptor.- """--- def childConnectionLost(childFD):- """- Called when a file descriptor associated with the child process is- closed.-- @type childFD: C{int}- @param childFD: The file descriptor which was closed.- """--- def processExited(reason):- """- Called when the child process exits.-- @type reason: L{twisted.python.failure.Failure}- @param reason: A failure giving the reason the child process- terminated. The type of exception for this failure is either- L{twisted.internet.error.ProcessDone} or- L{twisted.internet.error.ProcessTerminated}.-- @since: 8.2- """--- def processEnded(reason):- """- Called when the child process exits and all file descriptors associated- with it have been closed.-- @type reason: L{twisted.python.failure.Failure}- @param reason: A failure giving the reason the child process- terminated. The type of exception for this failure is either- L{twisted.internet.error.ProcessDone} or- L{twisted.internet.error.ProcessTerminated}.- """----class IHalfCloseableProtocol(Interface):- """- Implemented to indicate they want notification of half-closes.-- TCP supports the notion of half-closing the connection, e.g.- closing the write side but still not stopping reading. A protocol- that implements this interface will be notified of such events,- instead of having connectionLost called.- """-- def readConnectionLost():- """- Notification of the read connection being closed.-- This indicates peer did half-close of write side. It is now- the responsibility of the this protocol to call- loseConnection(). In addition, the protocol MUST make sure a- reference to it still exists (i.e. by doing a callLater with- one of its methods, etc.) as the reactor will only have a- reference to it if it is writing.-- If the protocol does not do so, it might get garbage collected- without the connectionLost method ever being called.- """-- def writeConnectionLost():- """- Notification of the write connection being closed.-- This will never be called for TCP connections as TCP does not- support notification of this type of half-close.- """----class IFileDescriptorReceiver(Interface):- """- Protocols may implement L{IFileDescriptorReceiver} to receive file- descriptors sent to them. This is useful in conjunction with- L{IUNIXTransport}, which allows file descriptors to be sent between- processes on a single host.- """- def fileDescriptorReceived(descriptor):- """- Called when a file descriptor is received over the connection.-- @param descriptor: The descriptor which was received.- @type descriptor: C{int}-- @return: C{None}- """----class IProtocolFactory(Interface):- """- Interface for protocol factories.- """-- def buildProtocol(addr):- """- Called when a connection has been established to addr.-- If None is returned, the connection is assumed to have been refused,- and the Port will close the connection.-- @type addr: (host, port)- @param addr: The address of the newly-established connection-- @return: None if the connection was refused, otherwise an object- providing L{IProtocol}.- """-- def doStart():- """- Called every time this is connected to a Port or Connector.- """-- def doStop():- """- Called every time this is unconnected from a Port or Connector.- """---class ITransport(Interface):- """- I am a transport for bytes.-- I represent (and wrap) the physical connection and synchronicity- of the framework which is talking to the network. I make no- representations about whether calls to me will happen immediately- or require returning to a control loop, or whether they will happen- in the same or another thread. Consider methods of this class- (aside from getPeer) to be 'thrown over the wall', to happen at some- indeterminate time.- """-- def write(data):- """- Write some data to the physical connection, in sequence, in a- non-blocking fashion.-- If possible, make sure that it is all written. No data will- ever be lost, although (obviously) the connection may be closed- before it all gets through.- """-- def writeSequence(data):- """- Write a list of strings to the physical connection.-- If possible, make sure that all of the data is written to- the socket at once, without first copying it all into a- single string.- """-- def loseConnection():- """- Close my connection, after writing all pending data.-- Note that if there is a registered producer on a transport it- will not be closed until the producer has been unregistered.- """-- def getPeer():- """- Get the remote address of this connection.-- Treat this method with caution. It is the unfortunate result of the- CGI and Jabber standards, but should not be considered reliable for- the usual host of reasons; port forwarding, proxying, firewalls, IP- masquerading, etc.-- @return: An L{IAddress} provider.- """-- def getHost():- """- Similar to getPeer, but returns an address describing this side of the- connection.-- @return: An L{IAddress} provider.- """---class ITCPTransport(ITransport):- """- A TCP based transport.- """-- def loseWriteConnection():- """- Half-close the write side of a TCP connection.-- If the protocol instance this is attached to provides- IHalfCloseableProtocol, it will get notified when the operation is- done. When closing write connection, as with loseConnection this will- only happen when buffer has emptied and there is no registered- producer.- """--- def abortConnection():- """- Close the connection abruptly.-- Discards any buffered data, stops any registered producer,- and, if possible, notifies the other end of the unclean- closure.-- @since: 11.1- """--- def getTcpNoDelay():- """- Return if C{TCP_NODELAY} is enabled.- """-- def setTcpNoDelay(enabled):- """- Enable/disable C{TCP_NODELAY}.-- Enabling C{TCP_NODELAY} turns off Nagle's algorithm. Small packets are- sent sooner, possibly at the expense of overall throughput.- """-- def getTcpKeepAlive():- """- Return if C{SO_KEEPALIVE} is enabled.- """-- def setTcpKeepAlive(enabled):- """- Enable/disable C{SO_KEEPALIVE}.-- Enabling C{SO_KEEPALIVE} sends packets periodically when the connection- is otherwise idle, usually once every two hours. They are intended- to allow detection of lost peers in a non-infinite amount of time.- """-- def getHost():- """- Returns L{IPv4Address} or L{IPv6Address}.- """-- def getPeer():- """- Returns L{IPv4Address} or L{IPv6Address}.- """----class IUNIXTransport(ITransport):- """- Transport for stream-oriented unix domain connections.- """- def sendFileDescriptor(descriptor):- """- Send a duplicate of this (file, socket, pipe, etc) descriptor to the- other end of this connection.-- The send is non-blocking and will be queued if it cannot be performed- immediately. The send will be processed in order with respect to other- C{sendFileDescriptor} calls on this transport, but not necessarily with- respect to C{write} calls on this transport. The send can only be- processed if there are also bytes in the normal connection-oriented send- buffer (ie, you must call C{write} at least as many times as you call- C{sendFileDescriptor}).-- @param descriptor: An C{int} giving a valid file descriptor in this- process. Note that a I{file descriptor} may actually refer to a- socket, a pipe, or anything else POSIX tries to treat in the same- way as a file.-- @return: C{None}- """----class ITLSTransport(ITCPTransport):- """- A TCP transport that supports switching to TLS midstream.-- Once TLS mode is started the transport will implement L{ISSLTransport}.- """-- def startTLS(contextFactory):- """- Initiate TLS negotiation.-- @param contextFactory: A context factory (see L{ssl.py<twisted.internet.ssl>})- """--class ISSLTransport(ITCPTransport):- """- A SSL/TLS based transport.- """-- def getPeerCertificate():- """- Return an object with the peer's certificate info.- """---class IProcessTransport(ITransport):- """- A process transport.- """-- pid = Attribute(- "From before L{IProcessProtocol.makeConnection} is called to before "- "L{IProcessProtocol.processEnded} is called, C{pid} is an L{int} "- "giving the platform process ID of this process. C{pid} is L{None} "- "at all other times.")-- def closeStdin():- """- Close stdin after all data has been written out.- """-- def closeStdout():- """- Close stdout.- """-- def closeStderr():- """- Close stderr.- """-- def closeChildFD(descriptor):- """- Close a file descriptor which is connected to the child process, identified- by its FD in the child process.- """-- def writeToChild(childFD, data):- """- Similar to L{ITransport.write} but also allows the file descriptor in- the child process which will receive the bytes to be specified.-- @type childFD: C{int}- @param childFD: The file descriptor to which to write.-- @type data: C{str}- @param data: The bytes to write.-- @return: C{None}-- @raise KeyError: If C{childFD} is not a file descriptor that was mapped- in the child when L{IReactorProcess.spawnProcess} was used to create- it.- """-- def loseConnection():- """- Close stdin, stderr and stdout.- """-- def signalProcess(signalID):- """- Send a signal to the process.-- @param signalID: can be- - one of C{"KILL"}, C{"TERM"}, or C{"INT"}.- These will be implemented in a- cross-platform manner, and so should be used- if possible.- - an integer, where it represents a POSIX- signal ID.-- @raise twisted.internet.error.ProcessExitedAlready: If the process has- already exited.- @raise OSError: If the C{os.kill} call fails with an errno different- from C{ESRCH}.- """---class IServiceCollection(Interface):- """- An object which provides access to a collection of services.- """-- def getServiceNamed(serviceName):- """- Retrieve the named service from this application.-- Raise a C{KeyError} if there is no such service name.- """-- def addService(service):- """- Add a service to this collection.- """-- def removeService(service):- """- Remove a service from this collection.- """---class IUDPTransport(Interface):- """- Transport for UDP DatagramProtocols.- """-- def write(packet, addr=None):- """- Write packet to given address.-- @param addr: a tuple of (ip, port). For connected transports must- be the address the transport is connected to, or None.- In non-connected mode this is mandatory.-- @raise twisted.internet.error.MessageLengthError: C{packet} was too- long.- """-- def connect(host, port):- """- Connect the transport to an address.-- This changes it to connected mode. Datagrams can only be sent to- this address, and will only be received from this address. In addition- the protocol's connectionRefused method might get called if destination- is not receiving datagrams.-- @param host: an IP address, not a domain name ('127.0.0.1', not 'localhost')- @param port: port to connect to.- """-- def getHost():- """- Returns L{IPv4Address}.- """-- def stopListening():- """- Stop listening on this port.-- If it does not complete immediately, will return L{Deferred} that fires- upon completion.- """----class IUNIXDatagramTransport(Interface):- """- Transport for UDP PacketProtocols.- """-- def write(packet, address):- """- Write packet to given address.- """-- def getHost():- """- Returns L{UNIXAddress}.- """---class IUNIXDatagramConnectedTransport(Interface):- """- Transport for UDP ConnectedPacketProtocols.- """-- def write(packet):- """- Write packet to address we are connected to.- """-- def getHost():- """- Returns L{UNIXAddress}.- """-- def getPeer():- """- Returns L{UNIXAddress}.- """---class IMulticastTransport(Interface):- """- Additional functionality for multicast UDP.- """-- def getOutgoingInterface():- """- Return interface of outgoing multicast packets.- """-- def setOutgoingInterface(addr):- """- Set interface for outgoing multicast packets.-- Returns Deferred of success.- """-- def getLoopbackMode():- """- Return if loopback mode is enabled.- """-- def setLoopbackMode(mode):- """- Set if loopback mode is enabled.- """-- def getTTL():- """- Get time to live for multicast packets.- """-- def setTTL(ttl):- """- Set time to live on multicast packets.- """-- def joinGroup(addr, interface=""):- """- Join a multicast group. Returns L{Deferred} of success or failure.-- If an error occurs, the returned L{Deferred} will fail with- L{error.MulticastJoinError}.- """-- def leaveGroup(addr, interface=""):- """- Leave multicast group, return L{Deferred} of success.- """---class IStreamClientEndpoint(Interface):- """- A stream client endpoint is a place that L{ClientFactory} can connect to.- For example, a remote TCP host/port pair would be a TCP client endpoint.-- @since: 10.1- """-- def connect(protocolFactory):- """- Connect the C{protocolFactory} to the location specified by this- L{IStreamClientEndpoint} provider.-- @param protocolFactory: A provider of L{IProtocolFactory}- @return: A L{Deferred} that results in an L{IProtocol} upon successful- connection otherwise a L{ConnectError}- """----class IStreamServerEndpoint(Interface):- """- A stream server endpoint is a place that a L{Factory} can listen for- incoming connections.-- @since: 10.1- """-- def listen(protocolFactory):- """- Listen with C{protocolFactory} at the location specified by this- L{IStreamServerEndpoint} provider.-- @param protocolFactory: A provider of L{IProtocolFactory}- @return: A L{Deferred} that results in an L{IListeningPort} or an- L{CannotListenError}- """----class IStreamServerEndpointStringParser(Interface):- """- An L{IStreamServerEndpointStringParser} is like an- L{IStreamClientEndpointStringParser}, except for L{IStreamServerEndpoint}s- instead of clients. It integrates with L{endpoints.serverFromString} in- much the same way.- """-- prefix = Attribute(- """- @see: L{IStreamClientEndpointStringParser.prefix}- """- )--- def parseStreamServer(reactor, *args, **kwargs):- """- Parse a stream server endpoint from a reactor and string-only arguments- and keyword arguments.-- @see: L{IStreamClientEndpointStringParser.parseStreamClient}-- @return: a stream server endpoint- @rtype: L{IStreamServerEndpoint}- """----class IStreamClientEndpointStringParser(Interface):- """- An L{IStreamClientEndpointStringParser} is a parser which can convert- a set of string C{*args} and C{**kwargs} into an L{IStreamClientEndpoint}- provider.-- This interface is really only useful in the context of the plugin system- for L{endpoints.clientFromString}. See the document entitled "I{The- Twisted Plugin System}" for more details on how to write a plugin.-- If you place an L{IStreamClientEndpointStringParser} plugin in the- C{twisted.plugins} package, that plugin's C{parseStreamClient} method will- be used to produce endpoints for any description string that begins with- the result of that L{IStreamClientEndpointStringParser}'s prefix attribute.- """-- prefix = Attribute(- """- A C{str}, the description prefix to respond to. For example, an- L{IStreamClientEndpointStringParser} plugin which had C{"foo"} for its- C{prefix} attribute would be called for endpoint descriptions like- C{"foo:bar:baz"} or C{"foo:"}.- """- )--- def parseStreamClient(*args, **kwargs):- """- This method is invoked by L{endpoints.clientFromString}, if the type of- endpoint matches the return value from this- L{IStreamClientEndpointStringParser}'s C{prefix} method.-- @param args: The string arguments, minus the endpoint type, in the- endpoint description string, parsed according to the rules- described in L{endpoints.quoteStringArgument}. For example, if the- description were C{"my-type:foo:bar:baz=qux"}, C{args} would be- C{('foo','bar')}-- @param kwargs: The string arguments from the endpoint description- passed as keyword arguments. For example, if the description were- C{"my-type:foo:bar:baz=qux"}, C{kwargs} would be- C{dict(baz='qux')}.-- @return: a client endpoint- @rtype: L{IStreamClientEndpoint}- """-}}} '''diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.pyindex 32c88ff2d..fd814dc22 100644--- a/scrapy/xlib/tx/iweb.py+++ b/scrapy/xlib/tx/iweb.py@@ -10,580 +10,11 @@ Interface definitions for L{twisted.web}.body is not known in advance."""-from zope.interface import Interface, Attribute--#from twisted.internet.interfaces import IPushProducer-from twisted.web.iweb import (IRequest, ICredentialFactory, IBodyProducer, IRenderable, ITemplateLoader,IResponse, _IRequestEncoder, _IRequestEncoderFactory, UNKNOWN_LENGTH,)-''' {{{-class IRequest(Interface):- """- An HTTP request.-- @since: 9.0- """-- method = Attribute("A C{str} giving the HTTP method that was used.")- uri = Attribute(- "A C{str} giving the full encoded URI which was requested (including "- "query arguments).")- path = Attribute(- "A C{str} giving the encoded query path of the request URI.")- args = Attribute(- "A mapping of decoded query argument names as C{str} to "- "corresponding query argument values as C{list}s of C{str}. "- "For example, for a URI with C{'foo=bar&foo=baz&quux=spam'} "- "for its query part, C{args} will be C{{'foo': ['bar', 'baz'], "- "'quux': ['spam']}}.")-- received_headers = Attribute(- "Backwards-compatibility access to C{requestHeaders}. Use "- "C{requestHeaders} instead. C{received_headers} behaves mostly "- "like a C{dict} and does not provide access to all header values.")-- requestHeaders = Attribute(- "A L{http_headers.Headers} instance giving all received HTTP request "- "headers.")-- content = Attribute(- "A file-like object giving the request body. This may be a file on "- "disk, a C{StringIO}, or some other type. The implementation is free "- "to decide on a per-request basis.")-- headers = Attribute(- "Backwards-compatibility access to C{responseHeaders}. Use"- "C{responseHeaders} instead. C{headers} behaves mostly like a "- "C{dict} and does not provide access to all header values nor "- "does it allow multiple values for one header to be set.")-- responseHeaders = Attribute(- "A L{http_headers.Headers} instance holding all HTTP response "- "headers to be sent.")-- def getHeader(key):- """- Get an HTTP request header.-- @type key: C{str}- @param key: The name of the header to get the value of.-- @rtype: C{str} or C{NoneType}- @return: The value of the specified header, or C{None} if that header- was not present in the request.- """--- def getCookie(key):- """- Get a cookie that was sent from the network.- """--- def getAllHeaders():- """- Return dictionary mapping the names of all received headers to the last- value received for each.-- Since this method does not return all header information,- C{requestHeaders.getAllRawHeaders()} may be preferred.- """--- def getRequestHostname():- """- Get the hostname that the user passed in to the request.-- This will either use the Host: header (if it is available) or the- host we are listening on if the header is unavailable.-- @returns: the requested hostname- @rtype: C{str}- """--- def getHost():- """- Get my originally requesting transport's host.-- @return: An L{IAddress<twisted.internet.interfaces.IAddress>}.- """--- def getClientIP():- """- Return the IP address of the client who submitted this request.-- @returns: the client IP address or C{None} if the request was submitted- over a transport where IP addresses do not make sense.- @rtype: L{str} or C{NoneType}- """--- def getClient():- """- Return the hostname of the IP address of the client who submitted this- request, if possible.-- This method is B{deprecated}. See L{getClientIP} instead.-- @rtype: C{NoneType} or L{str}- @return: The canonical hostname of the client, as determined by- performing a name lookup on the IP address of the client.- """--- def getUser():- """- Return the HTTP user sent with this request, if any.-- If no user was supplied, return the empty string.-- @returns: the HTTP user, if any- @rtype: C{str}- """--- def getPassword():- """- Return the HTTP password sent with this request, if any.-- If no password was supplied, return the empty string.-- @returns: the HTTP password, if any- @rtype: C{str}- """--- def isSecure():- """- Return True if this request is using a secure transport.-- Normally this method returns True if this request's HTTPChannel- instance is using a transport that implements ISSLTransport.-- This will also return True if setHost() has been called- with ssl=True.-- @returns: True if this request is secure- @rtype: C{bool}- """--- def getSession(sessionInterface=None):- """- Look up the session associated with this request or create a new one if- there is not one.-- @return: The L{Session} instance identified by the session cookie in- the request, or the C{sessionInterface} component of that session- if C{sessionInterface} is specified.- """--- def URLPath():- """- @return: A L{URLPath} instance which identifies the URL for which this- request is.- """--- def prePathURL():- """- @return: At any time during resource traversal, a L{str} giving an- absolute URL to the most nested resource which has yet been- reached.- """--- def rememberRootURL():- """- Remember the currently-processed part of the URL for later- recalling.- """--- def getRootURL():- """- Get a previously-remembered URL.- """--- # Methods for outgoing response- def finish():- """- Indicate that the response to this request is complete.- """--- def write(data):- """- Write some data to the body of the response to this request. Response- headers are written the first time this method is called, after which- new response headers may not be added.- """--- def addCookie(k, v, expires=None, domain=None, path=None, max_age=None, comment=None, secure=None):- """- Set an outgoing HTTP cookie.-- In general, you should consider using sessions instead of cookies, see- L{twisted.web.server.Request.getSession} and the- L{twisted.web.server.Session} class for details.- """--- def setResponseCode(code, message=None):- """- Set the HTTP response code.- """--- def setHeader(k, v):- """- Set an HTTP response header. Overrides any previously set values for- this header.-- @type name: C{str}- @param name: The name of the header for which to set the value.-- @type value: C{str}- @param value: The value to set for the named header.- """--- def redirect(url):- """- Utility function that does a redirect.-- The request should have finish() called after this.- """--- def setLastModified(when):- """- Set the C{Last-Modified} time for the response to this request.-- If I am called more than once, I ignore attempts to set Last-Modified- earlier, only replacing the Last-Modified time if it is to a later- value.-- If I am a conditional request, I may modify my response code to- L{NOT_MODIFIED<http.NOT_MODIFIED>} if appropriate for the time given.-- @param when: The last time the resource being returned was modified, in- seconds since the epoch.- @type when: L{int}, L{long} or L{float}-- @return: If I am a C{If-Modified-Since} conditional request and the time- given is not newer than the condition, I return- L{CACHED<http.CACHED>} to indicate that you should write no body.- Otherwise, I return a false value.- """--- def setETag(etag):- """- Set an C{entity tag} for the outgoing response.-- That's "entity tag" as in the HTTP/1.1 I{ETag} header, "used for- comparing two or more entities from the same requested resource."-- If I am a conditional request, I may modify my response code to- L{NOT_MODIFIED<http.NOT_MODIFIED>} or- L{PRECONDITION_FAILED<http.PRECONDITION_FAILED>}, if appropriate for the- tag given.-- @param etag: The entity tag for the resource being returned.- @type etag: C{str}-- @return: If I am a C{If-None-Match} conditional request and the tag- matches one in the request, I return L{CACHED<http.CACHED>} to- indicate that you should write no body. Otherwise, I return a- false value.- """--- def setHost(host, port, ssl=0):- """- Change the host and port the request thinks it's using.-- This method is useful for working with reverse HTTP proxies (e.g. both- Squid and Apache's mod_proxy can do this), when the address the HTTP- client is using is different than the one we're listening on.-- For example, Apache may be listening on https://www.example.com, and- then forwarding requests to http://localhost:8080, but we don't want- HTML produced by Twisted to say 'http://localhost:8080', they should- say 'https://www.example.com', so we do::-- request.setHost('www.example.com', 443, ssl=1)- """----class ICredentialFactory(Interface):- """- A credential factory defines a way to generate a particular kind of- authentication challenge and a way to interpret the responses to these- challenges. It creates- L{ICredentials<twisted.cred.credentials.ICredentials>} providers from- responses. These objects will be used with L{twisted.cred} to authenticate- an authorize requests.- """- scheme = Attribute(- "A C{str} giving the name of the authentication scheme with which "- "this factory is associated. For example, C{'basic'} or C{'digest'}.")--- def getChallenge(request):- """- Generate a new challenge to be sent to a client.-- @type peer: L{twisted.web.http.Request}- @param peer: The request the response to which this challenge will be- included.-- @rtype: C{dict}- @return: A mapping from C{str} challenge fields to associated C{str}- values.- """--- def decode(response, request):- """- Create a credentials object from the given response.-- @type response: C{str}- @param response: scheme specific response string-- @type request: L{twisted.web.http.Request}- @param request: The request being processed (from which the response- was taken).-- @raise twisted.cred.error.LoginFailed: If the response is invalid.-- @rtype: L{twisted.cred.credentials.ICredentials} provider- @return: The credentials represented by the given response.- """----class IBodyProducer(IPushProducer):- """- Objects which provide L{IBodyProducer} write bytes to an object which- provides L{IConsumer<twisted.internet.interfaces.IConsumer>} by calling its- C{write} method repeatedly.-- L{IBodyProducer} providers may start producing as soon as they have an- L{IConsumer<twisted.internet.interfaces.IConsumer>} provider. That is, they- should not wait for a C{resumeProducing} call to begin writing data.-- L{IConsumer.unregisterProducer<twisted.internet.interfaces.IConsumer.unregisterProducer>}- must not be called. Instead, the- L{Deferred<twisted.internet.defer.Deferred>} returned from C{startProducing}- must be fired when all bytes have been written.-- L{IConsumer.write<twisted.internet.interfaces.IConsumer.write>} may- synchronously invoke any of C{pauseProducing}, C{resumeProducing}, or- C{stopProducing}. These methods must be implemented with this in mind.-- @since: 9.0- """-- # Despite the restrictions above and the additional requirements of- # stopProducing documented below, this interface still needs to be an- # IPushProducer subclass. Providers of it will be passed to IConsumer- # providers which only know about IPushProducer and IPullProducer, not- # about this interface. This interface needs to remain close enough to one- # of those interfaces for consumers to work with it.-- length = Attribute(- """- C{length} is a C{int} indicating how many bytes in total this- L{IBodyProducer} will write to the consumer or L{UNKNOWN_LENGTH}- if this is not known in advance.- """)-- def startProducing(consumer):- """- Start producing to the given- L{IConsumer<twisted.internet.interfaces.IConsumer>} provider.-- @return: A L{Deferred<twisted.internet.defer.Deferred>} which fires with- C{None} when all bytes have been produced or with a- L{Failure<twisted.python.failure.Failure>} if there is any problem- before all bytes have been produced.- """--- def stopProducing():- """- In addition to the standard behavior of- L{IProducer.stopProducing<twisted.internet.interfaces.IProducer.stopProducing>}- (stop producing data), make sure the- L{Deferred<twisted.internet.defer.Deferred>} returned by- C{startProducing} is never fired.- """----class IRenderable(Interface):- """- An L{IRenderable} is an object that may be rendered by the- L{twisted.web.template} templating system.- """-- def lookupRenderMethod(name):- """- Look up and return the render method associated with the given name.-- @type name: C{str}- @param name: The value of a render directive encountered in the- document returned by a call to L{IRenderable.render}.-- @return: A two-argument callable which will be invoked with the request- being responded to and the tag object on which the render directive- was encountered.- """--- def render(request):- """- Get the document for this L{IRenderable}.-- @type request: L{IRequest} provider or C{NoneType}- @param request: The request in response to which this method is being- invoked.-- @return: An object which can be flattened.- """----class ITemplateLoader(Interface):- """- A loader for templates; something usable as a value for- L{twisted.web.template.Element}'s C{loader} attribute.- """-- def load():- """- Load a template suitable for rendering.-- @return: a C{list} of C{list}s, C{unicode} objects, C{Element}s and- other L{IRenderable} providers.- """----class IResponse(Interface):- """- An object representing an HTTP response received from an HTTP server.-- @since: 11.1- """-- version = Attribute(- "A three-tuple describing the protocol and protocol version "- "of the response. The first element is of type C{str}, the second "- "and third are of type C{int}. For example, C{('HTTP', 1, 1)}.")--- code = Attribute("The HTTP status code of this response, as a C{int}.")--- phrase = Attribute(- "The HTTP reason phrase of this response, as a C{str}.")--- headers = Attribute("The HTTP response L{Headers} of this response.")--- length = Attribute(- "The C{int} number of bytes expected to be in the body of this "- "response or L{UNKNOWN_LENGTH} if the server did not indicate how "- "many bytes to expect. For I{HEAD} responses, this will be 0; if "- "the response includes a I{Content-Length} header, it will be "- "available in C{headers}.")--- def deliverBody(protocol):- """- Register an L{IProtocol<twisted.internet.interfaces.IProtocol>} provider- to receive the response body.-- The protocol will be connected to a transport which provides- L{IPushProducer}. The protocol's C{connectionLost} method will be- called with:-- - ResponseDone, which indicates that all bytes from the response- have been successfully delivered.-- - PotentialDataLoss, which indicates that it cannot be determined- if the entire response body has been delivered. This only occurs- when making requests to HTTP servers which do not set- I{Content-Length} or a I{Transfer-Encoding} in the response.-- - ResponseFailed, which indicates that some bytes from the response- were lost. The C{reasons} attribute of the exception may provide- more specific indications as to why.- """----class _IRequestEncoder(Interface):- """- An object encoding data passed to L{IRequest.write}, for example for- compression purpose.-- @since: 12.3- """-- def encode(data):- """- Encode the data given and return the result.-- @param data: The content to encode.- @type data: C{str}-- @return: The encoded data.- @rtype: C{str}- """--- def finish():- """- Callback called when the request is closing.-- @return: If necessary, the pending data accumulated from previous- C{encode} calls.- @rtype: C{str}- """----class _IRequestEncoderFactory(Interface):- """- A factory for returing L{_IRequestEncoder} instances.-- @since: 12.3- """-- def encoderForRequest(request):- """- If applicable, returns a L{_IRequestEncoder} instance which will encode- the request.- """----UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH"-}}} '''__all__ = ["ICredentialFactory", "IRequest","IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder",