Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove obsolete xlib code for Twisted 13.1.0

nyov985755d1fe0a
diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py
index d20eda34f..39cd20f95 100644
--- a/scrapy/xlib/tx/_newclient.py
+++ b/scrapy/xlib/tx/_newclient.py
@@ -38,9 +38,7 @@ from twisted.internet.error import ConnectionDone
from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred
from twisted.internet.defer import CancelledError
from twisted.internet.protocol import Protocol
-#from twisted.protocols.basic import LineReceiver
from twisted.web.iweb import UNKNOWN_LENGTH, IResponse
-#from twisted.web.http_headers import Headers
from twisted.web.http import NO_CONTENT, NOT_MODIFIED
from twisted.web.http import _DataLoss, PotentialDataLoss
from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder
@@ -59,1467 +57,3 @@ STATUS = 'STATUS'
HEADER = 'HEADER'
BODY = 'BODY'
DONE = 'DONE'
-
-''' {{{
-class BadHeaders(Exception):
- """
- Headers passed to L{Request} were in some way invalid.
- """
-
-
-
-class ExcessWrite(Exception):
- """
- The body L{IBodyProducer} for a request tried to write data after
- indicating it had finished writing data.
- """
-
-
-class ParseError(Exception):
- """
- Some received data could not be parsed.
-
- @ivar data: The string which could not be parsed.
- """
- def __init__(self, reason, data):
- Exception.__init__(self, reason, data)
- self.data = data
-
-
-
-class BadResponseVersion(ParseError):
- """
- The version string in a status line was unparsable.
- """
-
-
-
-class _WrapperException(Exception):
- """
- L{_WrapperException} is the base exception type for exceptions which
- include one or more other exceptions as the low-level causes.
-
- @ivar reasons: A list of exceptions. See subclass documentation for more
- details.
- """
- def __init__(self, reasons):
- Exception.__init__(self, reasons)
- self.reasons = reasons
-
-
-
-class RequestGenerationFailed(_WrapperException):
- """
- There was an error while creating the bytes which make up a request.
-
- @ivar reasons: A C{list} of one or more L{Failure} instances giving the
- reasons the request generation was considered to have failed.
- """
-
-
-
-class RequestTransmissionFailed(_WrapperException):
- """
- There was an error while sending the bytes which make up a request.
-
- @ivar reasons: A C{list} of one or more L{Failure} instances giving the
- reasons the request transmission was considered to have failed.
- """
-
-
-
-class ConnectionAborted(Exception):
- """
- The connection was explicitly aborted by application code.
- """
-
-
-
-class WrongBodyLength(Exception):
- """
- An L{IBodyProducer} declared the number of bytes it was going to
- produce (via its C{length} attribute) and then produced a different number
- of bytes.
- """
-
-
-
-class ResponseDone(Exception):
- """
- L{ResponseDone} may be passed to L{IProtocol.connectionLost} on the
- protocol passed to L{Response.deliverBody} and indicates that the entire
- response has been delivered.
- """
-
-
-
-class ResponseFailed(_WrapperException):
- """
- L{ResponseFailed} indicates that all of the response to a request was not
- received for some reason.
-
- @ivar reasons: A C{list} of one or more L{Failure} instances giving the
- reasons the response was considered to have failed.
-
- @ivar response: If specified, the L{Response} received from the server (and
- in particular the status code and the headers).
- """
-
- def __init__(self, reasons, response=None):
- _WrapperException.__init__(self, reasons)
- self.response = response
-
-
-
-class ResponseNeverReceived(ResponseFailed):
- """
- A L{ResponseFailed} that knows no response bytes at all have been received.
- """
-
-
-
-class RequestNotSent(Exception):
- """
- L{RequestNotSent} indicates that an attempt was made to issue a request but
- for reasons unrelated to the details of the request itself, the request
- could not be sent. For example, this may indicate that an attempt was made
- to send a request using a protocol which is no longer connected to a
- server.
- """
-
-
-
-def _callAppFunction(function):
- """
- Call C{function}. If it raises an exception, log it with a minimal
- description of the source.
-
- @return: C{None}
- """
- try:
- function()
- except:
- log.err(None, "Unexpected exception from %s" % (
- fullyQualifiedName(function),))
-
-
-
-class HTTPParser(LineReceiver):
- """
- L{HTTPParser} handles the parsing side of HTTP processing. With a suitable
- subclass, it can parse either the client side or the server side of the
- connection.
-
- @ivar headers: All of the non-connection control message headers yet
- received.
-
- @ivar state: State indicator for the response parsing state machine. One
- of C{STATUS}, C{HEADER}, C{BODY}, C{DONE}.
-
- @ivar _partialHeader: C{None} or a C{list} of the lines of a multiline
- header while that header is being received.
- """
-
- # NOTE: According to HTTP spec, we're supposed to eat the
- # 'Proxy-Authenticate' and 'Proxy-Authorization' headers also, but that
- # doesn't sound like a good idea to me, because it makes it impossible to
- # have a non-authenticating transparent proxy in front of an authenticating
- # proxy. An authenticating proxy can eat them itself. -jknight
- #
- # Further, quoting
- # http://homepages.tesco.net/J.deBoynePollard/FGA/web-proxy-connection-header.html
- # regarding the 'Proxy-Connection' header:
- #
- # The Proxy-Connection: header is a mistake in how some web browsers
- # use HTTP. Its name is the result of a false analogy. It is not a
- # standard part of the protocol. There is a different standard
- # protocol mechanism for doing what it does. And its existence
- # imposes a requirement upon HTTP servers such that no proxy HTTP
- # server can be standards-conforming in practice.
- #
- # -exarkun
-
- # Some servers (like http://news.ycombinator.com/) return status lines and
- # HTTP headers delimited by \n instead of \r\n.
- delimiter = '\n'
-
- CONNECTION_CONTROL_HEADERS = set([
- 'content-length', 'connection', 'keep-alive', 'te', 'trailers',
- 'transfer-encoding', 'upgrade', 'proxy-connection'])
-
- def connectionMade(self):
- self.headers = Headers()
- self.connHeaders = Headers()
- self.state = STATUS
- self._partialHeader = None
-
-
- def switchToBodyMode(self, decoder):
- """
- Switch to body parsing mode - interpret any more bytes delivered as
- part of the message body and deliver them to the given decoder.
- """
- if self.state == BODY:
- raise RuntimeError("already in body mode")
-
- self.bodyDecoder = decoder
- self.state = BODY
- self.setRawMode()
-
-
- def lineReceived(self, line):
- """
- Handle one line from a response.
- """
- # Handle the normal CR LF case.
- if line[-1:] == '\r':
- line = line[:-1]
-
- if self.state == STATUS:
- self.statusReceived(line)
- self.state = HEADER
- elif self.state == HEADER:
- if not line or line[0] not in ' \t':
- if self._partialHeader is not None:
- header = ''.join(self._partialHeader)
- name, value = header.split(':', 1)
- value = value.strip()
- self.headerReceived(name, value)
- if not line:
- # Empty line means the header section is over.
- self.allHeadersReceived()
- else:
- # Line not beginning with LWS is another header.
- self._partialHeader = [line]
- else:
- # A line beginning with LWS is a continuation of a header
- # begun on a previous line.
- self._partialHeader.append(line)
-
-
- def rawDataReceived(self, data):
- """
- Pass data from the message body to the body decoder object.
- """
- self.bodyDecoder.dataReceived(data)
-
-
- def isConnectionControlHeader(self, name):
- """
- Return C{True} if the given lower-cased name is the name of a
- connection control header (rather than an entity header).
-
- According to RFC 2616, section 14.10, the tokens in the Connection
- header are probably relevant here. However, I am not sure what the
- practical consequences of either implementing or ignoring that are.
- So I leave it unimplemented for the time being.
- """
- return name in self.CONNECTION_CONTROL_HEADERS
-
-
- def statusReceived(self, status):
- """
- Callback invoked whenever the first line of a new message is received.
- Override this.
-
- @param status: The first line of an HTTP request or response message
- without trailing I{CR LF}.
- @type status: C{str}
- """
-
-
- def headerReceived(self, name, value):
- """
- Store the given header in C{self.headers}.
- """
- name = name.lower()
- if self.isConnectionControlHeader(name):
- headers = self.connHeaders
- else:
- headers = self.headers
- headers.addRawHeader(name, value)
-
-
- def allHeadersReceived(self):
- """
- Callback invoked after the last header is passed to C{headerReceived}.
- Override this to change to the C{BODY} or C{DONE} state.
- """
- self.switchToBodyMode(None)
-
-
-
-class HTTPClientParser(HTTPParser):
- """
- An HTTP parser which only handles HTTP responses.
-
- @ivar request: The request with which the expected response is associated.
- @type request: L{Request}
-
- @ivar NO_BODY_CODES: A C{set} of response codes which B{MUST NOT} have a
- body.
-
- @ivar finisher: A callable to invoke when this response is fully parsed.
-
- @ivar _responseDeferred: A L{Deferred} which will be called back with the
- response when all headers in the response have been received.
- Thereafter, C{None}.
-
- @ivar _everReceivedData: C{True} if any bytes have been received.
- """
- NO_BODY_CODES = set([NO_CONTENT, NOT_MODIFIED])
-
- _transferDecoders = {
- 'chunked': _ChunkedTransferDecoder,
- }
-
- bodyDecoder = None
-
- def __init__(self, request, finisher):
- self.request = request
- self.finisher = finisher
- self._responseDeferred = Deferred()
- self._everReceivedData = False
-
-
- def dataReceived(self, data):
- """
- Override so that we know if any response has been received.
- """
- self._everReceivedData = True
- HTTPParser.dataReceived(self, data)
-
-
- def parseVersion(self, strversion):
- """
- Parse version strings of the form Protocol '/' Major '.' Minor. E.g.
- 'HTTP/1.1'. Returns (protocol, major, minor). Will raise ValueError
- on bad syntax.
- """
- try:
- proto, strnumber = strversion.split('/')
- major, minor = strnumber.split('.')
- major, minor = int(major), int(minor)
- except ValueError as e:
- raise BadResponseVersion(str(e), strversion)
- if major < 0 or minor < 0:
- raise BadResponseVersion("version may not be negative", strversion)
- return (proto, major, minor)
-
-
- def statusReceived(self, status):
- """
- Parse the status line into its components and create a response object
- to keep track of this response's state.
- """
- parts = status.split(' ', 2)
- if len(parts) != 3:
- raise ParseError("wrong number of parts", status)
-
- try:
- statusCode = int(parts[1])
- except ValueError:
- raise ParseError("non-integer status code", status)
-
- self.response = Response(
- self.parseVersion(parts[0]),
- statusCode,
- parts[2],
- self.headers,
- self.transport)
-
-
- def _finished(self, rest):
- """
- Called to indicate that an entire response has been received. No more
- bytes will be interpreted by this L{HTTPClientParser}. Extra bytes are
- passed up and the state of this L{HTTPClientParser} is set to I{DONE}.
-
- @param rest: A C{str} giving any extra bytes delivered to this
- L{HTTPClientParser} which are not part of the response being
- parsed.
- """
- self.state = DONE
- self.finisher(rest)
-
-
- def isConnectionControlHeader(self, name):
- """
- Content-Length in the response to a HEAD request is an entity header,
- not a connection control header.
- """
- if self.request.method == 'HEAD' and name == 'content-length':
- return False
- return HTTPParser.isConnectionControlHeader(self, name)
-
-
- def allHeadersReceived(self):
- """
- Figure out how long the response body is going to be by examining
- headers and stuff.
- """
- if (self.response.code in self.NO_BODY_CODES
- or self.request.method == 'HEAD'):
- self.response.length = 0
- self._finished(self.clearLineBuffer())
- else:
- transferEncodingHeaders = self.connHeaders.getRawHeaders(
- 'transfer-encoding')
- if transferEncodingHeaders:
-
- # This could be a KeyError. However, that would mean we do not
- # know how to decode the response body, so failing the request
- # is as good a behavior as any. Perhaps someday we will want
- # to normalize/document/test this specifically, but failing
- # seems fine to me for now.
- transferDecoder = self._transferDecoders[transferEncodingHeaders[0].lower()]
-
- # If anyone ever invents a transfer encoding other than
- # chunked (yea right), and that transfer encoding can predict
- # the length of the response body, it might be sensible to
- # allow the transfer decoder to set the response object's
- # length attribute.
- else:
- contentLengthHeaders = self.connHeaders.getRawHeaders('content-length')
- if contentLengthHeaders is None:
- contentLength = None
- elif len(contentLengthHeaders) == 1:
- contentLength = int(contentLengthHeaders[0])
- self.response.length = contentLength
- else:
- # "HTTP Message Splitting" or "HTTP Response Smuggling"
- # potentially happening. Or it's just a buggy server.
- raise ValueError(
- "Too many Content-Length headers; response is invalid")
-
- if contentLength == 0:
- self._finished(self.clearLineBuffer())
- transferDecoder = None
- else:
- transferDecoder = lambda x, y: _IdentityTransferDecoder(
- contentLength, x, y)
-
- if transferDecoder is None:
- self.response._bodyDataFinished()
- else:
- # Make sure as little data as possible from the response body
- # gets delivered to the response object until the response
- # object actually indicates it is ready to handle bytes
- # (probably because an application gave it a way to interpret
- # them).
- self.transport.pauseProducing()
- self.switchToBodyMode(transferDecoder(
- self.response._bodyDataReceived,
- self._finished))
-
- # This must be last. If it were first, then application code might
- # change some state (for example, registering a protocol to receive the
- # response body). Then the pauseProducing above would be wrong since
- # the response is ready for bytes and nothing else would ever resume
- # the transport.
- self._responseDeferred.callback(self.response)
- del self._responseDeferred
-
-
- def connectionLost(self, reason):
- if self.bodyDecoder is not None:
- try:
- try:
- self.bodyDecoder.noMoreData()
- except PotentialDataLoss:
- self.response._bodyDataFinished(Failure())
- except _DataLoss:
- self.response._bodyDataFinished(
- Failure(ResponseFailed([reason, Failure()],
- self.response)))
- else:
- self.response._bodyDataFinished()
- except:
- # Handle exceptions from both the except suites and the else
- # suite. Those functions really shouldn't raise exceptions,
- # but maybe there's some buggy application code somewhere
- # making things difficult.
- log.err()
- elif self.state != DONE:
- if self._everReceivedData:
- exceptionClass = ResponseFailed
- else:
- exceptionClass = ResponseNeverReceived
- self._responseDeferred.errback(Failure(exceptionClass([reason])))
- del self._responseDeferred
-
-
-
-class Request:
- """
- A L{Request} instance describes an HTTP request to be sent to an HTTP
- server.
-
- @ivar method: The HTTP method to for this request, ex: 'GET', 'HEAD',
- 'POST', etc.
- @type method: C{str}
-
- @ivar uri: The relative URI of the resource to request. For example,
- C{'/foo/bar?baz=quux'}.
- @type uri: C{str}
-
- @ivar headers: Headers to be sent to the server. It is important to
- note that this object does not create any implicit headers. So it
- is up to the HTTP Client to add required headers such as 'Host'.
- @type headers: L{twisted.web.http_headers.Headers}
-
- @ivar bodyProducer: C{None} or an L{IBodyProducer} provider which
- produces the content body to send to the remote HTTP server.
-
- @ivar persistent: Set to C{True} when you use HTTP persistent connection.
- @type persistent: C{bool}
- """
- def __init__(self, method, uri, headers, bodyProducer, persistent=False):
- self.method = method
- self.uri = uri
- self.headers = headers
- self.bodyProducer = bodyProducer
- self.persistent = persistent
-
-
- def _writeHeaders(self, transport, TEorCL):
- hosts = self.headers.getRawHeaders('host', ())
- if len(hosts) != 1:
- raise BadHeaders("Exactly one Host header required")
-
- # In the future, having the protocol version be a parameter to this
- # method would probably be good. It would be nice if this method
- # weren't limited to issuing HTTP/1.1 requests.
- requestLines = []
- requestLines.append(
- '%s %s HTTP/1.1\r\n' % (self.method, self.uri))
- if not self.persistent:
- requestLines.append('Connection: close\r\n')
- if TEorCL is not None:
- requestLines.append(TEorCL)
- for name, values in self.headers.getAllRawHeaders():
- requestLines.extend(['%s: %s\r\n' % (name, v) for v in values])
- requestLines.append('\r\n')
- transport.writeSequence(requestLines)
-
-
- def _writeToChunked(self, transport):
- """
- Write this request to the given transport using chunked
- transfer-encoding to frame the body.
- """
- self._writeHeaders(transport, 'Transfer-Encoding: chunked\r\n')
- encoder = ChunkedEncoder(transport)
- encoder.registerProducer(self.bodyProducer, True)
- d = self.bodyProducer.startProducing(encoder)
-
- def cbProduced(ignored):
- encoder.unregisterProducer()
- def ebProduced(err):
- encoder._allowNoMoreWrites()
- # Don't call the encoder's unregisterProducer because it will write
- # a zero-length chunk. This would indicate to the server that the
- # request body is complete. There was an error, though, so we
- # don't want to do that.
- transport.unregisterProducer()
- return err
- d.addCallbacks(cbProduced, ebProduced)
- return d
-
-
- def _writeToContentLength(self, transport):
- """
- Write this request to the given transport using content-length to frame
- the body.
- """
- self._writeHeaders(
- transport,
- 'Content-Length: %d\r\n' % (self.bodyProducer.length,))
-
- # This Deferred is used to signal an error in the data written to the
- # encoder below. It can only errback and it will only do so before too
- # many bytes have been written to the encoder and before the producer
- # Deferred fires.
- finishedConsuming = Deferred()
-
- # This makes sure the producer writes the correct number of bytes for
- # the request body.
- encoder = LengthEnforcingConsumer(
- self.bodyProducer, transport, finishedConsuming)
-
- transport.registerProducer(self.bodyProducer, True)
-
- finishedProducing = self.bodyProducer.startProducing(encoder)
-
- def combine(consuming, producing):
- # This Deferred is returned and will be fired when the first of
- # consuming or producing fires. If it's cancelled, forward that
- # cancellation to the producer.
- def cancelConsuming(ign):
- finishedProducing.cancel()
- ultimate = Deferred(cancelConsuming)
-
- # Keep track of what has happened so far. This initially
- # contains None, then an integer uniquely identifying what
- # sequence of events happened. See the callbacks and errbacks
- # defined below for the meaning of each value.
- state = [None]
-
- def ebConsuming(err):
- if state == [None]:
- # The consuming Deferred failed first. This means the
- # overall writeTo Deferred is going to errback now. The
- # producing Deferred should not fire later (because the
- # consumer should have called stopProducing on the
- # producer), but if it does, a callback will be ignored
- # and an errback will be logged.
- state[0] = 1
- ultimate.errback(err)
- else:
- # The consuming Deferred errbacked after the producing
- # Deferred fired. This really shouldn't ever happen.
- # If it does, I goofed. Log the error anyway, just so
- # there's a chance someone might notice and complain.
- log.err(
- err,
- "Buggy state machine in %r/[%d]: "
- "ebConsuming called" % (self, state[0]))
-
- def cbProducing(result):
- if state == [None]:
- # The producing Deferred succeeded first. Nothing will
- # ever happen to the consuming Deferred. Tell the
- # encoder we're done so it can check what the producer
- # wrote and make sure it was right.
- state[0] = 2
- try:
- encoder._noMoreWritesExpected()
- except:
- # Fail the overall writeTo Deferred - something the
- # producer did was wrong.
- ultimate.errback()
- else:
- # Success - succeed the overall writeTo Deferred.
- ultimate.callback(None)
- # Otherwise, the consuming Deferred already errbacked. The
- # producing Deferred wasn't supposed to fire, but it did
- # anyway. It's buggy, but there's not really anything to be
- # done about it. Just ignore this result.
-
- def ebProducing(err):
- if state == [None]:
- # The producing Deferred failed first. This means the
- # overall writeTo Deferred is going to errback now.
- # Tell the encoder that we're done so it knows to reject
- # further writes from the producer (which should not
- # happen, but the producer may be buggy).
- state[0] = 3
- encoder._allowNoMoreWrites()
- ultimate.errback(err)
- else:
- # The producing Deferred failed after the consuming
- # Deferred failed. It shouldn't have, so it's buggy.
- # Log the exception in case anyone who can fix the code
- # is watching.
- log.err(err, "Producer is buggy")
-
- consuming.addErrback(ebConsuming)
- producing.addCallbacks(cbProducing, ebProducing)
-
- return ultimate
-
- d = combine(finishedConsuming, finishedProducing)
- def f(passthrough):
- # Regardless of what happens with the overall Deferred, once it
- # fires, the producer registered way up above the definition of
- # combine should be unregistered.
- transport.unregisterProducer()
- return passthrough
- d.addBoth(f)
- return d
-
-
- def writeTo(self, transport):
- """
- Format this L{Request} as an HTTP/1.1 request and write it to the given
- transport. If bodyProducer is not None, it will be associated with an
- L{IConsumer}.
-
- @return: A L{Deferred} which fires with C{None} when the request has
- been completely written to the transport or with a L{Failure} if
- there is any problem generating the request bytes.
- """
- if self.bodyProducer is not None:
- if self.bodyProducer.length is UNKNOWN_LENGTH:
- return self._writeToChunked(transport)
- else:
- return self._writeToContentLength(transport)
- else:
- self._writeHeaders(transport, None)
- return succeed(None)
-
-
- def stopWriting(self):
- """
- Stop writing this request to the transport. This can only be called
- after C{writeTo} and before the L{Deferred} returned by C{writeTo}
- fires. It should cancel any asynchronous task started by C{writeTo}.
- The L{Deferred} returned by C{writeTo} need not be fired if this method
- is called.
- """
- # If bodyProducer is None, then the Deferred returned by writeTo has
- # fired already and this method cannot be called.
- _callAppFunction(self.bodyProducer.stopProducing)
-
-
-
-class LengthEnforcingConsumer:
- """
- An L{IConsumer} proxy which enforces an exact length requirement on the
- total data written to it.
-
- @ivar _length: The number of bytes remaining to be written.
-
- @ivar _producer: The L{IBodyProducer} which is writing to this
- consumer.
-
- @ivar _consumer: The consumer to which at most C{_length} bytes will be
- forwarded.
-
- @ivar _finished: A L{Deferred} which will be fired with a L{Failure} if too
- many bytes are written to this consumer.
- """
- def __init__(self, producer, consumer, finished):
- self._length = producer.length
- self._producer = producer
- self._consumer = consumer
- self._finished = finished
-
-
- def _allowNoMoreWrites(self):
- """
- Indicate that no additional writes are allowed. Attempts to write
- after calling this method will be met with an exception.
- """
- self._finished = None
-
-
- def write(self, bytes):
- """
- Write C{bytes} to the underlying consumer unless
- C{_noMoreWritesExpected} has been called or there are/have been too
- many bytes.
- """
- if self._finished is None:
- # No writes are supposed to happen any more. Try to convince the
- # calling code to stop calling this method by calling its
- # stopProducing method and then throwing an exception at it. This
- # exception isn't documented as part of the API because you're
- # never supposed to expect it: only buggy code will ever receive
- # it.
- self._producer.stopProducing()
- raise ExcessWrite()
-
- if len(bytes) <= self._length:
- self._length -= len(bytes)
- self._consumer.write(bytes)
- else:
- # No synchronous exception is raised in *this* error path because
- # we still have _finished which we can use to report the error to a
- # better place than the direct caller of this method (some
- # arbitrary application code).
- _callAppFunction(self._producer.stopProducing)
- self._finished.errback(WrongBodyLength("too many bytes written"))
- self._allowNoMoreWrites()
-
-
- def _noMoreWritesExpected(self):
- """
- Called to indicate no more bytes will be written to this consumer.
- Check to see that the correct number have been written.
-
- @raise WrongBodyLength: If not enough bytes have been written.
- """
- if self._finished is not None:
- self._allowNoMoreWrites()
- if self._length:
- raise WrongBodyLength("too few bytes written")
-
-
-
-def makeStatefulDispatcher(name, template):
- """
- Given a I{dispatch} name and a function, return a function which can be
- used as a method and which, when called, will call another method defined
- on the instance and return the result. The other method which is called is
- determined by the value of the C{_state} attribute of the instance.
-
- @param name: A string which is used to construct the name of the subsidiary
- method to invoke. The subsidiary method is named like C{'_%s_%s' %
- (name, _state)}.
-
- @param template: A function object which is used to give the returned
- function a docstring.
-
- @return: The dispatcher function.
- """
- def dispatcher(self, *args, **kwargs):
- func = getattr(self, '_' + name + '_' + self._state, None)
- if func is None:
- raise RuntimeError(
- "%r has no %s method in state %s" % (self, name, self._state))
- return func(*args, **kwargs)
- dispatcher.__doc__ = template.__doc__
- return dispatcher
-
-
-
-class Response:
- """
- A L{Response} instance describes an HTTP response received from an HTTP
- server.
-
- L{Response} should not be subclassed or instantiated.
-
- @ivar _transport: The transport which is delivering this response.
-
- @ivar _bodyProtocol: The L{IProtocol} provider to which the body is
- delivered. C{None} before one has been registered with
- C{deliverBody}.
-
- @ivar _bodyBuffer: A C{list} of the strings passed to C{bodyDataReceived}
- before C{deliverBody} is called. C{None} afterwards.
-
- @ivar _state: Indicates what state this L{Response} instance is in,
- particularly with respect to delivering bytes from the response body
- to an application-suppled protocol object. This may be one of
- C{'INITIAL'}, C{'CONNECTED'}, C{'DEFERRED_CLOSE'}, or C{'FINISHED'},
- with the following meanings:
-
- - INITIAL: This is the state L{Response} objects start in. No
- protocol has yet been provided and the underlying transport may
- still have bytes to deliver to it.
-
- - DEFERRED_CLOSE: If the underlying transport indicates all bytes
- have been delivered but no application-provided protocol is yet
- available, the L{Response} moves to this state. Data is
- buffered and waiting for a protocol to be delivered to.
-
- - CONNECTED: If a protocol is provided when the state is INITIAL,
- the L{Response} moves to this state. Any buffered data is
- delivered and any data which arrives from the transport
- subsequently is given directly to the protocol.
-
- - FINISHED: If a protocol is provided in the DEFERRED_CLOSE state,
- the L{Response} moves to this state after delivering all
- buffered data to the protocol. Otherwise, if the L{Response} is
- in the CONNECTED state, if the transport indicates there is no
- more data, the L{Response} moves to this state. Nothing else
- can happen once the L{Response} is in this state.
- """
- implements(IResponse)
-
- length = UNKNOWN_LENGTH
-
- _bodyProtocol = None
- _bodyFinished = False
-
- def __init__(self, version, code, phrase, headers, _transport):
- self.version = version
- self.code = code
- self.phrase = phrase
- self.headers = headers
- self._transport = _transport
- self._bodyBuffer = []
- self._state = 'INITIAL'
-
-
- def deliverBody(self, protocol):
- """
- Dispatch the given L{IProtocol} depending of the current state of the
- response.
- """
- deliverBody = makeStatefulDispatcher('deliverBody', deliverBody)
-
-
- def _deliverBody_INITIAL(self, protocol):
- """
- Deliver any buffered data to C{protocol} and prepare to deliver any
- future data to it. Move to the C{'CONNECTED'} state.
- """
- # Now that there's a protocol to consume the body, resume the
- # transport. It was previously paused by HTTPClientParser to avoid
- # reading too much data before it could be handled.
- self._transport.resumeProducing()
-
- protocol.makeConnection(self._transport)
- self._bodyProtocol = protocol
- for data in self._bodyBuffer:
- self._bodyProtocol.dataReceived(data)
- self._bodyBuffer = None
- self._state = 'CONNECTED'
-
-
- def _deliverBody_CONNECTED(self, protocol):
- """
- It is invalid to attempt to deliver data to a protocol when it is
- already being delivered to another protocol.
- """
- raise RuntimeError(
- "Response already has protocol %r, cannot deliverBody "
- "again" % (self._bodyProtocol,))
-
-
- def _deliverBody_DEFERRED_CLOSE(self, protocol):
- """
- Deliver any buffered data to C{protocol} and then disconnect the
- protocol. Move to the C{'FINISHED'} state.
- """
- # Unlike _deliverBody_INITIAL, there is no need to resume the
- # transport here because all of the response data has been received
- # already. Some higher level code may want to resume the transport if
- # that code expects further data to be received over it.
-
- protocol.makeConnection(self._transport)
-
- for data in self._bodyBuffer:
- protocol.dataReceived(data)
- self._bodyBuffer = None
- protocol.connectionLost(self._reason)
- self._state = 'FINISHED'
-
-
- def _deliverBody_FINISHED(self, protocol):
- """
- It is invalid to attempt to deliver data to a protocol after the
- response body has been delivered to another protocol.
- """
- raise RuntimeError(
- "Response already finished, cannot deliverBody now.")
-
-
- def _bodyDataReceived(self, data):
- """
- Called by HTTPClientParser with chunks of data from the response body.
- They will be buffered or delivered to the protocol passed to
- deliverBody.
- """
- _bodyDataReceived = makeStatefulDispatcher('bodyDataReceived',
- _bodyDataReceived)
-
-
- def _bodyDataReceived_INITIAL(self, data):
- """
- Buffer any data received for later delivery to a protocol passed to
- C{deliverBody}.
-
- Little or no data should be buffered by this method, since the
- transport has been paused and will not be resumed until a protocol
- is supplied.
- """
- self._bodyBuffer.append(data)
-
-
- def _bodyDataReceived_CONNECTED(self, data):
- """
- Deliver any data received to the protocol to which this L{Response}
- is connected.
- """
- self._bodyProtocol.dataReceived(data)
-
-
- def _bodyDataReceived_DEFERRED_CLOSE(self, data):
- """
- It is invalid for data to be delivered after it has been indicated
- that the response body has been completely delivered.
- """
- raise RuntimeError("Cannot receive body data after _bodyDataFinished")
-
-
- def _bodyDataReceived_FINISHED(self, data):
- """
- It is invalid for data to be delivered after the response body has
- been delivered to a protocol.
- """
- raise RuntimeError("Cannot receive body data after protocol disconnected")
-
-
- def _bodyDataFinished(self, reason=None):
- """
- Called by HTTPClientParser when no more body data is available. If the
- optional reason is supplied, this indicates a problem or potential
- problem receiving all of the response body.
- """
- _bodyDataFinished = makeStatefulDispatcher('bodyDataFinished',
- _bodyDataFinished)
-
-
- def _bodyDataFinished_INITIAL(self, reason=None):
- """
- Move to the C{'DEFERRED_CLOSE'} state to wait for a protocol to
- which to deliver the response body.
- """
- self._state = 'DEFERRED_CLOSE'
- if reason is None:
- reason = Failure(ResponseDone("Response body fully received"))
- self._reason = reason
-
-
- def _bodyDataFinished_CONNECTED(self, reason=None):
- """
- Disconnect the protocol and move to the C{'FINISHED'} state.
- """
- if reason is None:
- reason = Failure(ResponseDone("Response body fully received"))
- self._bodyProtocol.connectionLost(reason)
- self._bodyProtocol = None
- self._state = 'FINISHED'
-
-
- def _bodyDataFinished_DEFERRED_CLOSE(self):
- """
- It is invalid to attempt to notify the L{Response} of the end of the
- response body data more than once.
- """
- raise RuntimeError("Cannot finish body data more than once")
-
-
- def _bodyDataFinished_FINISHED(self):
- """
- It is invalid to attempt to notify the L{Response} of the end of the
- response body data more than once.
- """
- raise RuntimeError("Cannot finish body data after protocol disconnected")
-
-
-
-class ChunkedEncoder:
- """
- Helper object which exposes L{IConsumer} on top of L{HTTP11ClientProtocol}
- for streaming request bodies to the server.
- """
- implements(IConsumer)
-
- def __init__(self, transport):
- self.transport = transport
-
-
- def _allowNoMoreWrites(self):
- """
- Indicate that no additional writes are allowed. Attempts to write
- after calling this method will be met with an exception.
- """
- self.transport = None
-
-
- def registerProducer(self, producer, streaming):
- """
- Register the given producer with C{self.transport}.
- """
- self.transport.registerProducer(producer, streaming)
-
-
- def write(self, data):
- """
- Write the given request body bytes to the transport using chunked
- encoding.
-
- @type data: C{str}
- """
- if self.transport is None:
- raise ExcessWrite()
- self.transport.writeSequence(("%x\r\n" % len(data), data, "\r\n"))
-
-
- def unregisterProducer(self):
- """
- Indicate that the request body is complete and finish the request.
- """
- self.write('')
- self.transport.unregisterProducer()
- self._allowNoMoreWrites()
-
-
-
-class TransportProxyProducer:
- """
- An L{IPushProducer} implementation which wraps another such thing and
- proxies calls to it until it is told to stop.
-
- @ivar _producer: The wrapped L{IPushProducer} provider or C{None} after
- this proxy has been stopped.
- """
- implements(IPushProducer)
-
- # LineReceiver uses this undocumented attribute of transports to decide
- # when to stop calling lineReceived or rawDataReceived (if it finds it to
- # be true, it doesn't bother to deliver any more data). Set disconnecting
- # to False here and never change it to true so that all data is always
- # delivered to us and so that LineReceiver doesn't fail with an
- # AttributeError.
- disconnecting = False
-
- def __init__(self, producer):
- self._producer = producer
-
-
- def _stopProxying(self):
- """
- Stop forwarding calls of L{IPushProducer} methods to the underlying
- L{IPushProvider} provider.
- """
- self._producer = None
-
-
- def stopProducing(self):
- """
- Proxy the stoppage to the underlying producer, unless this proxy has
- been stopped.
- """
- if self._producer is not None:
- self._producer.stopProducing()
-
-
- def resumeProducing(self):
- """
- Proxy the resumption to the underlying producer, unless this proxy has
- been stopped.
- """
- if self._producer is not None:
- self._producer.resumeProducing()
-
-
- def pauseProducing(self):
- """
- Proxy the pause to the underlying producer, unless this proxy has been
- stopped.
- """
- if self._producer is not None:
- self._producer.pauseProducing()
-
-
-
-class HTTP11ClientProtocol(Protocol):
- """
- L{HTTP11ClientProtocol} is an implementation of the HTTP 1.1 client
- protocol. It supports as few features as possible.
-
- @ivar _parser: After a request is issued, the L{HTTPClientParser} to
- which received data making up the response to that request is
- delivered.
-
- @ivar _finishedRequest: After a request is issued, the L{Deferred} which
- will fire when a L{Response} object corresponding to that request is
- available. This allows L{HTTP11ClientProtocol} to fail the request
- if there is a connection or parsing problem.
-
- @ivar _currentRequest: After a request is issued, the L{Request}
- instance used to make that request. This allows
- L{HTTP11ClientProtocol} to stop request generation if necessary (for
- example, if the connection is lost).
-
- @ivar _transportProxy: After a request is issued, the
- L{TransportProxyProducer} to which C{_parser} is connected. This
- allows C{_parser} to pause and resume the transport in a way which
- L{HTTP11ClientProtocol} can exert some control over.
-
- @ivar _responseDeferred: After a request is issued, the L{Deferred} from
- C{_parser} which will fire with a L{Response} when one has been
- received. This is eventually chained with C{_finishedRequest}, but
- only in certain cases to avoid double firing that Deferred.
-
- @ivar _state: Indicates what state this L{HTTP11ClientProtocol} instance
- is in with respect to transmission of a request and reception of a
- response. This may be one of the following strings:
-
- - QUIESCENT: This is the state L{HTTP11ClientProtocol} instances
- start in. Nothing is happening: no request is being sent and no
- response is being received or expected.
-
- - TRANSMITTING: When a request is made (via L{request}), the
- instance moves to this state. L{Request.writeTo} has been used
- to start to send a request but it has not yet finished.
-
- - TRANSMITTING_AFTER_RECEIVING_RESPONSE: The server has returned a
- complete response but the request has not yet been fully sent
- yet. The instance will remain in this state until the request
- is fully sent.
-
- - GENERATION_FAILED: There was an error while the request. The
- request was not fully sent to the network.
-
- - WAITING: The request was fully sent to the network. The
- instance is now waiting for the response to be fully received.
-
- - ABORTING: Application code has requested that the HTTP connection
- be aborted.
-
- - CONNECTION_LOST: The connection has been lost.
-
- @ivar _abortDeferreds: A list of C{Deferred} instances that will fire when
- the connection is lost.
- """
- _state = 'QUIESCENT'
- _parser = None
- _finishedRequest = None
- _currentRequest = None
- _transportProxy = None
- _responseDeferred = None
-
-
- def __init__(self, quiescentCallback=lambda c: None):
- self._quiescentCallback = quiescentCallback
- self._abortDeferreds = []
-
-
- @property
- def state(self):
- return self._state
-
-
- def request(self, request):
- """
- Issue C{request} over C{self.transport} and return a L{Deferred} which
- will fire with a L{Response} instance or an error.
-
- @param request: The object defining the parameters of the request to
- issue.
- @type request: L{Request}
-
- @rtype: L{Deferred}
- @return: The deferred may errback with L{RequestGenerationFailed} if
- the request was not fully written to the transport due to a local
- error. It may errback with L{RequestTransmissionFailed} if it was
- not fully written to the transport due to a network error. It may
- errback with L{ResponseFailed} if the request was sent (not
- necessarily received) but some or all of the response was lost. It
- may errback with L{RequestNotSent} if it is not possible to send
- any more requests using this L{HTTP11ClientProtocol}.
- """
- if self._state != 'QUIESCENT':
- return fail(RequestNotSent())
-
- self._state = 'TRANSMITTING'
- _requestDeferred = maybeDeferred(request.writeTo, self.transport)
-
- def cancelRequest(ign):
- # Explicitly cancel the request's deferred if it's still trying to
- # write when this request is cancelled.
- if self._state in (
- 'TRANSMITTING', 'TRANSMITTING_AFTER_RECEIVING_RESPONSE'):
- _requestDeferred.cancel()
- else:
- self.transport.abortConnection()
- self._disconnectParser(Failure(CancelledError()))
- self._finishedRequest = Deferred(cancelRequest)
-
- # Keep track of the Request object in case we need to call stopWriting
- # on it.
- self._currentRequest = request
-
- self._transportProxy = TransportProxyProducer(self.transport)
- self._parser = HTTPClientParser(request, self._finishResponse)
- self._parser.makeConnection(self._transportProxy)
- self._responseDeferred = self._parser._responseDeferred
-
- def cbRequestWrotten(ignored):
- if self._state == 'TRANSMITTING':
- self._state = 'WAITING'
- self._responseDeferred.chainDeferred(self._finishedRequest)
-
- def ebRequestWriting(err):
- if self._state == 'TRANSMITTING':
- self._state = 'GENERATION_FAILED'
- self.transport.abortConnection()
- self._finishedRequest.errback(
- Failure(RequestGenerationFailed([err])))
- else:
- log.err(err, 'Error writing request, but not in valid state '
- 'to finalize request: %s' % self._state)
-
- _requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting)
-
- return self._finishedRequest
-
-
- def _finishResponse(self, rest):
- """
- Called by an L{HTTPClientParser} to indicate that it has parsed a
- complete response.
-
- @param rest: A C{str} giving any trailing bytes which were given to
- the L{HTTPClientParser} which were not part of the response it
- was parsing.
- """
- _finishResponse = makeStatefulDispatcher('finishResponse', _finishResponse)
-
-
- def _finishResponse_WAITING(self, rest):
- # Currently the rest parameter is ignored. Don't forget to use it if
- # we ever add support for pipelining. And maybe check what trailers
- # mean.
- if self._state == 'WAITING':
- self._state = 'QUIESCENT'
- else:
- # The server sent the entire response before we could send the
- # whole request. That sucks. Oh well. Fire the request()
- # Deferred with the response. But first, make sure that if the
- # request does ever finish being written that it won't try to fire
- # that Deferred.
- self._state = 'TRANSMITTING_AFTER_RECEIVING_RESPONSE'
- self._responseDeferred.chainDeferred(self._finishedRequest)
-
- # This will happen if we're being called due to connection being lost;
- # if so, no need to disconnect parser again, or to call
- # _quiescentCallback.
- if self._parser is None:
- return
-
- reason = ConnectionDone("synthetic!")
- connHeaders = self._parser.connHeaders.getRawHeaders('connection', ())
- if (('close' in connHeaders) or self._state != "QUIESCENT" or
- not self._currentRequest.persistent):
- self._giveUp(Failure(reason))
- else:
- # We call the quiescent callback first, to ensure connection gets
- # added back to connection pool before we finish the request.
- try:
- self._quiescentCallback(self)
- except:
- # If callback throws exception, just log it and disconnect;
- # keeping persistent connections around is an optimisation:
- log.err()
- self.transport.loseConnection()
- self._disconnectParser(reason)
-
-
- _finishResponse_TRANSMITTING = _finishResponse_WAITING
-
-
- def _disconnectParser(self, reason):
- """
- If there is still a parser, call its C{connectionLost} method with the
- given reason. If there is not, do nothing.
-
- @type reason: L{Failure}
- """
- if self._parser is not None:
- parser = self._parser
- self._parser = None
- self._currentRequest = None
- self._finishedRequest = None
- self._responseDeferred = None
-
- # The parser is no longer allowed to do anything to the real
- # transport. Stop proxying from the parser's transport to the real
- # transport before telling the parser it's done so that it can't do
- # anything.
- self._transportProxy._stopProxying()
- self._transportProxy = None
- parser.connectionLost(reason)
-
-
- def _giveUp(self, reason):
- """
- Lose the underlying connection and disconnect the parser with the given
- L{Failure}.
-
- Use this method instead of calling the transport's loseConnection
- method directly otherwise random things will break.
- """
- self.transport.loseConnection()
- self._disconnectParser(reason)
-
-
- def dataReceived(self, bytes):
- """
- Handle some stuff from some place.
- """
- try:
- self._parser.dataReceived(bytes)
- except:
- self._giveUp(Failure())
-
-
- def connectionLost(self, reason):
- """
- The underlying transport went away. If appropriate, notify the parser
- object.
- """
- connectionLost = makeStatefulDispatcher('connectionLost', connectionLost)
-
-
- def _connectionLost_QUIESCENT(self, reason):
- """
- Nothing is currently happening. Move to the C{'CONNECTION_LOST'}
- state but otherwise do nothing.
- """
- self._state = 'CONNECTION_LOST'
-
-
- def _connectionLost_GENERATION_FAILED(self, reason):
- """
- The connection was in an inconsistent state. Move to the
- C{'CONNECTION_LOST'} state but otherwise do nothing.
- """
- self._state = 'CONNECTION_LOST'
-
-
- def _connectionLost_TRANSMITTING(self, reason):
- """
- Fail the L{Deferred} for the current request, notify the request
- object that it does not need to continue transmitting itself, and
- move to the C{'CONNECTION_LOST'} state.
- """
- self._state = 'CONNECTION_LOST'
- self._finishedRequest.errback(
- Failure(RequestTransmissionFailed([reason])))
- del self._finishedRequest
-
- # Tell the request that it should stop bothering now.
- self._currentRequest.stopWriting()
-
-
- def _connectionLost_TRANSMITTING_AFTER_RECEIVING_RESPONSE(self, reason):
- """
- Move to the C{'CONNECTION_LOST'} state.
- """
- self._state = 'CONNECTION_LOST'
-
-
- def _connectionLost_WAITING(self, reason):
- """
- Disconnect the response parser so that it can propagate the event as
- necessary (for example, to call an application protocol's
- C{connectionLost} method, or to fail a request L{Deferred}) and move
- to the C{'CONNECTION_LOST'} state.
- """
- self._disconnectParser(reason)
- self._state = 'CONNECTION_LOST'
-
-
- def _connectionLost_ABORTING(self, reason):
- """
- Disconnect the response parser with a L{ConnectionAborted} failure, and
- move to the C{'CONNECTION_LOST'} state.
- """
- self._disconnectParser(Failure(ConnectionAborted()))
- self._state = 'CONNECTION_LOST'
- for d in self._abortDeferreds:
- d.callback(None)
- self._abortDeferreds = []
-
-
- def abort(self):
- """
- Close the connection and cause all outstanding L{request} L{Deferred}s
- to fire with an error.
- """
- if self._state == "CONNECTION_LOST":
- return succeed(None)
- self.transport.loseConnection()
- self._state = 'ABORTING'
- d = Deferred()
- self._abortDeferreds.append(d)
- return d
-}}} '''
diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.py
index 8e0b1df8b..c2d50648a 100644
--- a/scrapy/xlib/tx/client.py
+++ b/scrapy/xlib/tx/client.py
@@ -42,1129 +42,13 @@ from twisted.web.client import (
Agent, ProxyAgent, HTTPConnectionPool, readBody,
)
-''' {{{
-class PartialDownloadError(error.Error):
- """
- Page was only partially downloaded, we got disconnected in middle.
-
- @ivar response: All of the response body which was downloaded.
- """
-
-
-class _URL(tuple):
- """
- A parsed URL.
-
- At some point this should be replaced with a better URL implementation.
- """
- def __new__(self, scheme, host, port, path):
- return tuple.__new__(_URL, (scheme, host, port, path))
-
-
- def __init__(self, scheme, host, port, path):
- self.scheme = scheme
- self.host = host
- self.port = port
- self.path = path
-
-
-def _parse(url, defaultPort=None):
- """
- Split the given URL into the scheme, host, port, and path.
-
- @type url: C{bytes}
- @param url: An URL to parse.
-
- @type defaultPort: C{int} or C{None}
- @param defaultPort: An alternate value to use as the port if the URL does
- not include one.
-
- @return: A four-tuple of the scheme, host, port, and path of the URL. All
- of these are C{bytes} instances except for port, which is an C{int}.
- """
- url = url.strip()
- parsed = http.urlparse(url)
- scheme = parsed[0]
- path = urlunparse((b'', b'') + parsed[2:])
-
- if defaultPort is None:
- if scheme == b'https':
- defaultPort = 443
- else:
- defaultPort = 80
-
- host, port = parsed[1], defaultPort
- if b':' in host:
- host, port = host.split(b':')
- try:
- port = int(port)
- except ValueError:
- port = defaultPort
-
- if path == b'':
- path = b'/'
-
- return _URL(scheme, host, port, path)
-
-
-def _makeGetterFactory(url, factoryFactory, contextFactory=None,
- *args, **kwargs):
- """
- Create and connect an HTTP page getting factory.
-
- Any additional positional or keyword arguments are used when calling
- C{factoryFactory}.
-
- @param factoryFactory: Factory factory that is called with C{url}, C{args}
- and C{kwargs} to produce the getter
-
- @param contextFactory: Context factory to use when creating a secure
- connection, defaulting to C{None}
-
- @return: The factory created by C{factoryFactory}
- """
- scheme, host, port, path = _parse(url)
- factory = factoryFactory(url, *args, **kwargs)
- if scheme == b'https':
- from twisted.internet import ssl
- if contextFactory is None:
- contextFactory = ssl.ClientContextFactory()
- reactor.connectSSL(host, port, factory, contextFactory)
- else:
- reactor.connectTCP(host, port, factory)
- return factory
-}}} '''
# The code which follows is based on the new HTTP client implementation. It
# should be significantly better than anything above, though it is not yet
# feature equivalent.
-#from twisted.web.error import SchemeNotSupported
from twisted.web._newclient import Response
-#from twisted.web._newclient import Request, HTTP11ClientProtocol
from twisted.web._newclient import ResponseDone, ResponseFailed
-#from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed
-#from twisted.web._newclient import (
-# ResponseNeverReceived, PotentialDataLoss, _WrapperException)
-
-''' {{{
-try:
- from twisted.internet.ssl import ClientContextFactory
-except ImportError:
- class WebClientContextFactory(object):
- """
- A web context factory which doesn't work because the necessary SSL
- support is missing.
- """
- def getContext(self, hostname, port):
- raise NotImplementedError("SSL support unavailable")
-else:
- class WebClientContextFactory(ClientContextFactory):
- """
- A web context factory which ignores the hostname and port and does no
- certificate verification.
- """
- def getContext(self, hostname, port):
- return ClientContextFactory.getContext(self)
-
-
-
-class _WebToNormalContextFactory(object):
- """
- Adapt a web context factory to a normal context factory.
-
- @ivar _webContext: A web context factory which accepts a hostname and port
- number to its C{getContext} method.
-
- @ivar _hostname: The hostname which will be passed to
- C{_webContext.getContext}.
-
- @ivar _port: The port number which will be passed to
- C{_webContext.getContext}.
- """
- def __init__(self, webContext, hostname, port):
- self._webContext = webContext
- self._hostname = hostname
- self._port = port
-
-
- def getContext(self):
- """
- Called the wrapped web context factory's C{getContext} method with a
- hostname and port number and return the resulting context object.
- """
- return self._webContext.getContext(self._hostname, self._port)
-
-
-
-@implementer(IBodyProducer)
-class FileBodyProducer(object):
- """
- L{FileBodyProducer} produces bytes from an input file object incrementally
- and writes them to a consumer.
-
- Since file-like objects cannot be read from in an event-driven manner,
- L{FileBodyProducer} uses a L{Cooperator} instance to schedule reads from
- the file. This process is also paused and resumed based on notifications
- from the L{IConsumer} provider being written to.
-
- The file is closed after it has been read, or if the producer is stopped
- early.
-
- @ivar _inputFile: Any file-like object, bytes read from which will be
- written to a consumer.
-
- @ivar _cooperate: A method like L{Cooperator.cooperate} which is used to
- schedule all reads.
-
- @ivar _readSize: The number of bytes to read from C{_inputFile} at a time.
- """
-
- # Python 2.4 doesn't have these symbolic constants
- _SEEK_SET = getattr(os, 'SEEK_SET', 0)
- _SEEK_END = getattr(os, 'SEEK_END', 2)
-
- def __init__(self, inputFile, cooperator=task, readSize=2 ** 16):
- self._inputFile = inputFile
- self._cooperate = cooperator.cooperate
- self._readSize = readSize
- self.length = self._determineLength(inputFile)
-
-
- def _determineLength(self, fObj):
- """
- Determine how many bytes can be read out of C{fObj} (assuming it is not
- modified from this point on). If the determination cannot be made,
- return C{UNKNOWN_LENGTH}.
- """
- try:
- seek = fObj.seek
- tell = fObj.tell
- except AttributeError:
- return UNKNOWN_LENGTH
- originalPosition = tell()
- seek(0, self._SEEK_END)
- end = tell()
- seek(originalPosition, self._SEEK_SET)
- return end - originalPosition
-
-
- def stopProducing(self):
- """
- Permanently stop writing bytes from the file to the consumer by
- stopping the underlying L{CooperativeTask}.
- """
- self._inputFile.close()
- self._task.stop()
-
-
- def startProducing(self, consumer):
- """
- Start a cooperative task which will read bytes from the input file and
- write them to C{consumer}. Return a L{Deferred} which fires after all
- bytes have been written.
-
- @param consumer: Any L{IConsumer} provider
- """
- self._task = self._cooperate(self._writeloop(consumer))
- d = self._task.whenDone()
- def maybeStopped(reason):
- # IBodyProducer.startProducing's Deferred isn't support to fire if
- # stopProducing is called.
- reason.trap(task.TaskStopped)
- return defer.Deferred()
- d.addCallbacks(lambda ignored: None, maybeStopped)
- return d
-
-
- def _writeloop(self, consumer):
- """
- Return an iterator which reads one chunk of bytes from the input file
- and writes them to the consumer for each time it is iterated.
- """
- while True:
- bytes = self._inputFile.read(self._readSize)
- if not bytes:
- self._inputFile.close()
- break
- consumer.write(bytes)
- yield None
-
-
- def pauseProducing(self):
- """
- Temporarily suspend copying bytes from the input file to the consumer
- by pausing the L{CooperativeTask} which drives that activity.
- """
- self._task.pause()
-
-
- def resumeProducing(self):
- """
- Undo the effects of a previous C{pauseProducing} and resume copying
- bytes to the consumer by resuming the L{CooperativeTask} which drives
- the write activity.
- """
- self._task.resume()
-
-
-
-class _HTTP11ClientFactory(protocol.Factory):
- """
- A factory for L{HTTP11ClientProtocol}, used by L{HTTPConnectionPool}.
-
- @ivar _quiescentCallback: The quiescent callback to be passed to protocol
- instances, used to return them to the connection pool.
-
- @since: 11.1
- """
- def __init__(self, quiescentCallback):
- self._quiescentCallback = quiescentCallback
-
-
- def buildProtocol(self, addr):
- return HTTP11ClientProtocol(self._quiescentCallback)
-
-
-
-class _RetryingHTTP11ClientProtocol(object):
- """
- A wrapper for L{HTTP11ClientProtocol} that automatically retries requests.
-
- @ivar _clientProtocol: The underlying L{HTTP11ClientProtocol}.
-
- @ivar _newConnection: A callable that creates a new connection for a
- retry.
- """
-
- def __init__(self, clientProtocol, newConnection):
- self._clientProtocol = clientProtocol
- self._newConnection = newConnection
-
-
- def _shouldRetry(self, method, exception, bodyProducer):
- """
- Indicate whether request should be retried.
-
- Only returns C{True} if method is idempotent, no response was
- received, the reason for the failed request was not due to
- user-requested cancellation, and no body was sent. The latter
- requirement may be relaxed in the future, and PUT added to approved
- method list.
- """
- if method not in ("GET", "HEAD", "OPTIONS", "DELETE", "TRACE"):
- return False
- if not isinstance(exception, (RequestNotSent, RequestTransmissionFailed,
- ResponseNeverReceived)):
- return False
- if isinstance(exception, _WrapperException):
- for failure in exception.reasons:
- if failure.check(defer.CancelledError):
- return False
- if bodyProducer is not None:
- return False
- return True
-
-
- def request(self, request):
- """
- Do a request, and retry once (with a new connection) it it fails in
- a retryable manner.
-
- @param request: A L{Request} instance that will be requested using the
- wrapped protocol.
- """
- d = self._clientProtocol.request(request)
-
- def failed(reason):
- if self._shouldRetry(request.method, reason.value,
- request.bodyProducer):
- return self._newConnection().addCallback(
- lambda connection: connection.request(request))
- else:
- return reason
- d.addErrback(failed)
- return d
-
-
-
-class HTTPConnectionPool(object):
- """
- A pool of persistent HTTP connections.
-
- Features:
- - Cached connections will eventually time out.
- - Limits on maximum number of persistent connections.
-
- Connections are stored using keys, which should be chosen such that any
- connections stored under a given key can be used interchangeably.
-
- Failed requests done using previously cached connections will be retried
- once if they use an idempotent method (e.g. GET), in case the HTTP server
- timed them out.
-
- @ivar persistent: Boolean indicating whether connections should be
- persistent. Connections are persistent by default.
-
- @ivar maxPersistentPerHost: The maximum number of cached persistent
- connections for a C{host:port} destination.
- @type maxPersistentPerHost: C{int}
-
- @ivar cachedConnectionTimeout: Number of seconds a cached persistent
- connection will stay open before disconnecting.
-
- @ivar retryAutomatically: C{boolean} indicating whether idempotent
- requests should be retried once if no response was received.
-
- @ivar _factory: The factory used to connect to the proxy.
-
- @ivar _connections: Map (scheme, host, port) to lists of
- L{HTTP11ClientProtocol} instances.
-
- @ivar _timeouts: Map L{HTTP11ClientProtocol} instances to a
- C{IDelayedCall} instance of their timeout.
-
- @since: 12.1
- """
-
- _factory = _HTTP11ClientFactory
- maxPersistentPerHost = 2
- cachedConnectionTimeout = 240
- retryAutomatically = True
-
- def __init__(self, reactor, persistent=True):
- self._reactor = reactor
- self.persistent = persistent
- self._connections = {}
- self._timeouts = {}
-
-
- def getConnection(self, key, endpoint):
- """
- Supply a connection, newly created or retrieved from the pool, to be
- used for one HTTP request.
-
- The connection will remain out of the pool (not available to be
- returned from future calls to this method) until one HTTP request has
- been completed over it.
-
- Afterwards, if the connection is still open, it will automatically be
- added to the pool.
-
- @param key: A unique key identifying connections that can be used
- interchangeably.
-
- @param endpoint: An endpoint that can be used to open a new connection
- if no cached connection is available.
-
- @return: A C{Deferred} that will fire with a L{HTTP11ClientProtocol}
- (or a wrapper) that can be used to send a single HTTP request.
- """
- # Try to get cached version:
- connections = self._connections.get(key)
- while connections:
- connection = connections.pop(0)
- # Cancel timeout:
- self._timeouts[connection].cancel()
- del self._timeouts[connection]
- if connection.state == "QUIESCENT":
- if self.retryAutomatically:
- newConnection = lambda: self._newConnection(key, endpoint)
- connection = _RetryingHTTP11ClientProtocol(
- connection, newConnection)
- return defer.succeed(connection)
-
- return self._newConnection(key, endpoint)
-
-
- def _newConnection(self, key, endpoint):
- """
- Create a new connection.
-
- This implements the new connection code path for L{getConnection}.
- """
- def quiescentCallback(protocol):
- self._putConnection(key, protocol)
- factory = self._factory(quiescentCallback)
- return endpoint.connect(factory)
-
-
- def _removeConnection(self, key, connection):
- """
- Remove a connection from the cache and disconnect it.
- """
- connection.transport.loseConnection()
- self._connections[key].remove(connection)
- del self._timeouts[connection]
-
-
- def _putConnection(self, key, connection):
- """
- Return a persistent connection to the pool. This will be called by
- L{HTTP11ClientProtocol} when the connection becomes quiescent.
- """
- if connection.state != "QUIESCENT":
- # Log with traceback for debugging purposes:
- try:
- raise RuntimeError(
- "BUG: Non-quiescent protocol added to connection pool.")
- except:
- log.err()
- return
- connections = self._connections.setdefault(key, [])
- if len(connections) == self.maxPersistentPerHost:
- dropped = connections.pop(0)
- dropped.transport.loseConnection()
- self._timeouts[dropped].cancel()
- del self._timeouts[dropped]
- connections.append(connection)
- cid = self._reactor.callLater(self.cachedConnectionTimeout,
- self._removeConnection,
- key, connection)
- self._timeouts[connection] = cid
-
-
- def closeCachedConnections(self):
- """
- Close all persistent connections and remove them from the pool.
-
- @return: L{defer.Deferred} that fires when all connections have been
- closed.
- """
- results = []
- for protocols in self._connections.itervalues():
- for p in protocols:
- results.append(p.abort())
- self._connections = {}
- for dc in self._timeouts.values():
- dc.cancel()
- self._timeouts = {}
- return defer.gatherResults(results).addCallback(lambda ign: None)
-
-
-
-class _AgentBase(object):
- """
- Base class offering common facilities for L{Agent}-type classes.
-
- @ivar _reactor: The C{IReactorTime} implementation which will be used by
- the pool, and perhaps by subclasses as well.
-
- @ivar _pool: The L{HTTPConnectionPool} used to manage HTTP connections.
- """
-
- def __init__(self, reactor, pool):
- if pool is None:
- pool = HTTPConnectionPool(reactor, False)
- self._reactor = reactor
- self._pool = pool
-
-
- def _computeHostValue(self, scheme, host, port):
- """
- Compute the string to use for the value of the I{Host} header, based on
- the given scheme, host name, and port number.
- """
- if (scheme, port) in (('http', 80), ('https', 443)):
- return host
- return '%s:%d' % (host, port)
-
-
- def _requestWithEndpoint(self, key, endpoint, method, parsedURI,
- headers, bodyProducer, requestPath):
- """
- Issue a new request, given the endpoint and the path sent as part of
- the request.
- """
- # Create minimal headers, if necessary:
- if headers is None:
- headers = Headers()
- if not headers.hasHeader('host'):
- #headers = headers.copy() # not supported in twisted <= 11.1, and it doesn't affects us
- headers.addRawHeader(
- 'host', self._computeHostValue(parsedURI.scheme, parsedURI.host,
- parsedURI.port))
-
- d = self._pool.getConnection(key, endpoint)
- def cbConnected(proto):
- return proto.request(
- Request(method, requestPath, headers, bodyProducer,
- persistent=self._pool.persistent))
- d.addCallback(cbConnected)
- return d
-
-
-
-class Agent(_AgentBase):
- """
- L{Agent} is a very basic HTTP client. It supports I{HTTP} and I{HTTPS}
- scheme URIs (but performs no certificate checking by default).
-
- @param pool: A L{HTTPConnectionPool} instance, or C{None}, in which case a
- non-persistent L{HTTPConnectionPool} instance will be created.
-
- @ivar _contextFactory: A web context factory which will be used to create
- SSL context objects for any SSL connections the agent needs to make.
-
- @ivar _connectTimeout: If not C{None}, the timeout passed to C{connectTCP}
- or C{connectSSL} for specifying the connection timeout.
-
- @ivar _bindAddress: If not C{None}, the address passed to C{connectTCP} or
- C{connectSSL} for specifying the local address to bind to.
-
- @since: 9.0
- """
-
- def __init__(self, reactor, contextFactory=WebClientContextFactory(),
- connectTimeout=None, bindAddress=None,
- pool=None):
- _AgentBase.__init__(self, reactor, pool)
- self._contextFactory = contextFactory
- self._connectTimeout = connectTimeout
- self._bindAddress = bindAddress
-
-
- def _wrapContextFactory(self, host, port):
- """
- Create and return a normal context factory wrapped around
- C{self._contextFactory} in such a way that C{self._contextFactory} will
- have the host and port information passed to it.
-
- @param host: A C{str} giving the hostname which will be connected to in
- order to issue a request.
-
- @param port: An C{int} giving the port number the connection will be
- on.
-
- @return: A context factory suitable to be passed to
- C{reactor.connectSSL}.
- """
- return _WebToNormalContextFactory(self._contextFactory, host, port)
-
-
- def _getEndpoint(self, scheme, host, port):
- """
- Get an endpoint for the given host and port, using a transport
- selected based on scheme.
-
- @param scheme: A string like C{'http'} or C{'https'} (the only two
- supported values) to use to determine how to establish the
- connection.
-
- @param host: A C{str} giving the hostname which will be connected to in
- order to issue a request.
-
- @param port: An C{int} giving the port number the connection will be
- on.
-
- @return: An endpoint which can be used to connect to given address.
- """
- kwargs = {}
- if self._connectTimeout is not None:
- kwargs['timeout'] = self._connectTimeout
- kwargs['bindAddress'] = self._bindAddress
- if scheme == 'http':
- return TCP4ClientEndpoint(self._reactor, host, port, **kwargs)
- elif scheme == 'https':
- return SSL4ClientEndpoint(self._reactor, host, port,
- self._wrapContextFactory(host, port),
- **kwargs)
- else:
- raise SchemeNotSupported("Unsupported scheme: %r" % (scheme,))
-
-
- def request(self, method, uri, headers=None, bodyProducer=None):
- """
- Issue a new request.
-
- @param method: The request method to send.
- @type method: C{str}
-
- @param uri: The request URI send.
- @type uri: C{str}
-
- @param headers: The request headers to send. If no I{Host} header is
- included, one will be added based on the request URI.
- @type headers: L{Headers}
-
- @param bodyProducer: An object which will produce the request body or,
- if the request body is to be empty, L{None}.
- @type bodyProducer: L{IBodyProducer} provider
-
- @return: A L{Deferred} which fires with the result of the request (a
- L{twisted.web.iweb.IResponse} provider), or fails if there is a
- problem setting up a connection over which to issue the request.
- It may also fail with L{SchemeNotSupported} if the scheme of the
- given URI is not supported.
- @rtype: L{Deferred}
- """
- parsedURI = _parse(uri)
- try:
- endpoint = self._getEndpoint(parsedURI.scheme, parsedURI.host,
- parsedURI.port)
- except SchemeNotSupported:
- return defer.fail(Failure())
- key = (parsedURI.scheme, parsedURI.host, parsedURI.port)
- return self._requestWithEndpoint(key, endpoint, method, parsedURI,
- headers, bodyProducer, parsedURI.path)
-
-
-
-class ProxyAgent(_AgentBase):
- """
- An HTTP agent able to cross HTTP proxies.
-
- @ivar _proxyEndpoint: The endpoint used to connect to the proxy.
-
- @since: 11.1
- """
-
- def __init__(self, endpoint, reactor=None, pool=None):
- if reactor is None:
- from twisted.internet import reactor
- _AgentBase.__init__(self, reactor, pool)
- self._proxyEndpoint = endpoint
-
-
- def request(self, method, uri, headers=None, bodyProducer=None):
- """
- Issue a new request via the configured proxy.
- """
- # Cache *all* connections under the same key, since we are only
- # connecting to a single destination, the proxy:
- key = ("http-proxy", self._proxyEndpoint)
-
- # To support proxying HTTPS via CONNECT, we will use key
- # ("http-proxy-CONNECT", scheme, host, port), and an endpoint that
- # wraps _proxyEndpoint with an additional callback to do the CONNECT.
- return self._requestWithEndpoint(key, self._proxyEndpoint, method,
- _parse(uri), headers, bodyProducer,
- uri)
-
-
-
-class _FakeUrllib2Request(object):
- """
- A fake C{urllib2.Request} object for C{cookielib} to work with.
-
- @see: U{http://docs.python.org/library/urllib2.html#request-objects}
-
- @type uri: C{str}
- @ivar uri: Request URI.
-
- @type headers: L{twisted.web.http_headers.Headers}
- @ivar headers: Request headers.
-
- @type type: C{str}
- @ivar type: The scheme of the URI.
-
- @type host: C{str}
- @ivar host: The host[:port] of the URI.
-
- @since: 11.1
- """
- def __init__(self, uri):
- self.uri = uri
- self.headers = Headers()
- self.type, rest = splittype(self.uri)
- self.host, rest = splithost(rest)
-
-
- def has_header(self, header):
- return self.headers.hasHeader(header)
-
-
- def add_unredirected_header(self, name, value):
- self.headers.addRawHeader(name, value)
-
-
- def get_full_url(self):
- return self.uri
-
-
- def get_header(self, name, default=None):
- headers = self.headers.getRawHeaders(name, default)
- if headers is not None:
- return headers[0]
- return None
-
-
- def get_host(self):
- return self.host
-
-
- def get_type(self):
- return self.type
-
-
- def is_unverifiable(self):
- # In theory this shouldn't be hardcoded.
- return False
-
-
-
-class _FakeUrllib2Response(object):
- """
- A fake C{urllib2.Response} object for C{cookielib} to work with.
-
- @type response: C{twisted.web.iweb.IResponse}
- @ivar response: Underlying Twisted Web response.
-
- @since: 11.1
- """
- def __init__(self, response):
- self.response = response
-
-
- def info(self):
- class _Meta(object):
- def getheaders(zelf, name):
- return self.response.headers.getRawHeaders(name, [])
- return _Meta()
-
-
-
-class CookieAgent(object):
- """
- L{CookieAgent} extends the basic L{Agent} to add RFC-compliant
- handling of HTTP cookies. Cookies are written to and extracted
- from a C{cookielib.CookieJar} instance.
-
- The same cookie jar instance will be used for any requests through this
- agent, mutating it whenever a I{Set-Cookie} header appears in a response.
-
- @type _agent: L{twisted.web.client.Agent}
- @ivar _agent: Underlying Twisted Web agent to issue requests through.
-
- @type cookieJar: C{cookielib.CookieJar}
- @ivar cookieJar: Initialized cookie jar to read cookies from and store
- cookies to.
-
- @since: 11.1
- """
- def __init__(self, agent, cookieJar):
- self._agent = agent
- self.cookieJar = cookieJar
-
-
- def request(self, method, uri, headers=None, bodyProducer=None):
- """
- Issue a new request to the wrapped L{Agent}.
-
- Send a I{Cookie} header if a cookie for C{uri} is stored in
- L{CookieAgent.cookieJar}. Cookies are automatically extracted and
- stored from requests.
-
- If a C{'cookie'} header appears in C{headers} it will override the
- automatic cookie header obtained from the cookie jar.
-
- @see: L{Agent.request}
- """
- if headers is None:
- headers = Headers()
- lastRequest = _FakeUrllib2Request(uri)
- # Setting a cookie header explicitly will disable automatic request
- # cookies.
- if not headers.hasHeader('cookie'):
- self.cookieJar.add_cookie_header(lastRequest)
- cookieHeader = lastRequest.get_header('Cookie', None)
- if cookieHeader is not None:
- headers = headers.copy()
- headers.addRawHeader('cookie', cookieHeader)
-
- d = self._agent.request(method, uri, headers, bodyProducer)
- d.addCallback(self._extractCookies, lastRequest)
- return d
-
-
- def _extractCookies(self, response, request):
- """
- Extract response cookies and store them in the cookie jar.
-
- @type response: L{twisted.web.iweb.IResponse}
- @param response: Twisted Web response.
-
- @param request: A urllib2 compatible request object.
- """
- resp = _FakeUrllib2Response(response)
- self.cookieJar.extract_cookies(resp, request)
- return response
-
-
-
-class GzipDecoder(proxyForInterface(IResponse)):
- """
- A wrapper for a L{Response} instance which handles gzip'ed body.
-
- @ivar original: The original L{Response} object.
-
- @since: 11.1
- """
-
- def __init__(self, response):
- self.original = response
- self.length = UNKNOWN_LENGTH
-
-
- def deliverBody(self, protocol):
- """
- Override C{deliverBody} to wrap the given C{protocol} with
- L{_GzipProtocol}.
- """
- self.original.deliverBody(_GzipProtocol(protocol, self.original))
-
-
-
-class _GzipProtocol(proxyForInterface(IProtocol)):
- """
- A L{Protocol} implementation which wraps another one, transparently
- decompressing received data.
-
- @ivar _zlibDecompress: A zlib decompress object used to decompress the data
- stream.
-
- @ivar _response: A reference to the original response, in case of errors.
-
- @since: 11.1
- """
-
- def __init__(self, protocol, response):
- self.original = protocol
- self._response = response
- self._zlibDecompress = zlib.decompressobj(16 + zlib.MAX_WBITS)
-
-
- def dataReceived(self, data):
- """
- Decompress C{data} with the zlib decompressor, forwarding the raw data
- to the original protocol.
- """
- try:
- rawData = self._zlibDecompress.decompress(data)
- except zlib.error:
- raise ResponseFailed([failure.Failure()], self._response)
- if rawData:
- self.original.dataReceived(rawData)
-
-
- def connectionLost(self, reason):
- """
- Forward the connection lost event, flushing remaining data from the
- decompressor if any.
- """
- try:
- rawData = self._zlibDecompress.flush()
- except zlib.error:
- raise ResponseFailed([reason, failure.Failure()], self._response)
- if rawData:
- self.original.dataReceived(rawData)
- self.original.connectionLost(reason)
-
-
-
-class ContentDecoderAgent(object):
- """
- An L{Agent} wrapper to handle encoded content.
-
- It takes care of declaring the support for content in the
- I{Accept-Encoding} header, and automatically decompresses the received data
- if it's effectively using compression.
-
- @param decoders: A list or tuple of (name, decoder) objects. The name
- declares which decoding the decoder supports, and the decoder must
- return a response object when called/instantiated. For example,
- C{(('gzip', GzipDecoder))}. The order determines how the decoders are
- going to be advertized to the server.
-
- @since: 11.1
- """
-
- def __init__(self, agent, decoders):
- self._agent = agent
- self._decoders = dict(decoders)
- self._supported = ','.join([decoder[0] for decoder in decoders])
-
-
- def request(self, method, uri, headers=None, bodyProducer=None):
- """
- Send a client request which declares supporting compressed content.
-
- @see: L{Agent.request}.
- """
- if headers is None:
- headers = Headers()
- else:
- headers = headers.copy()
- headers.addRawHeader('accept-encoding', self._supported)
- deferred = self._agent.request(method, uri, headers, bodyProducer)
- return deferred.addCallback(self._handleResponse)
-
-
- def _handleResponse(self, response):
- """
- Check if the response is encoded, and wrap it to handle decompression.
- """
- contentEncodingHeaders = response.headers.getRawHeaders(
- 'content-encoding', [])
- contentEncodingHeaders = ','.join(contentEncodingHeaders).split(',')
- while contentEncodingHeaders:
- name = contentEncodingHeaders.pop().strip()
- decoder = self._decoders.get(name)
- if decoder is not None:
- response = decoder(response)
- else:
- # Add it back
- contentEncodingHeaders.append(name)
- break
- if contentEncodingHeaders:
- response.headers.setRawHeaders(
- 'content-encoding', [','.join(contentEncodingHeaders)])
- else:
- response.headers.removeHeader('content-encoding')
- return response
-
-
-
-class RedirectAgent(object):
- """
- An L{Agent} wrapper which handles HTTP redirects.
-
- The implementation is rather strict: 301 and 302 behaves like 307, not
- redirecting automatically on methods different from C{GET} and C{HEAD}.
-
- @param redirectLimit: The maximum number of times the agent is allowed to
- follow redirects before failing with a L{error.InfiniteRedirection}.
-
- @since: 11.1
- """
-
- def __init__(self, agent, redirectLimit=20):
- self._agent = agent
- self._redirectLimit = redirectLimit
-
-
- def request(self, method, uri, headers=None, bodyProducer=None):
- """
- Send a client request following HTTP redirects.
-
- @see: L{Agent.request}.
- """
- deferred = self._agent.request(method, uri, headers, bodyProducer)
- return deferred.addCallback(
- self._handleResponse, method, uri, headers, 0)
-
-
- def _handleRedirect(self, response, method, uri, headers, redirectCount):
- """
- Handle a redirect response, checking the number of redirects already
- followed, and extracting the location header fields.
- """
- if redirectCount >= self._redirectLimit:
- err = error.InfiniteRedirection(
- response.code,
- 'Infinite redirection detected',
- location=uri)
- raise ResponseFailed([failure.Failure(err)], response)
- locationHeaders = response.headers.getRawHeaders('location', [])
- if not locationHeaders:
- err = error.RedirectWithNoLocation(
- response.code, 'No location header field', uri)
- raise ResponseFailed([failure.Failure(err)], response)
- location = locationHeaders[0]
- deferred = self._agent.request(method, location, headers)
- return deferred.addCallback(
- self._handleResponse, method, uri, headers, redirectCount + 1)
-
-
- def _handleResponse(self, response, method, uri, headers, redirectCount):
- """
- Handle the response, making another request if it indicates a redirect.
- """
- if response.code in (http.MOVED_PERMANENTLY, http.FOUND,
- http.TEMPORARY_REDIRECT):
- if method not in ('GET', 'HEAD'):
- err = error.PageRedirect(response.code, location=uri)
- raise ResponseFailed([failure.Failure(err)], response)
- return self._handleRedirect(response, method, uri, headers,
- redirectCount)
- elif response.code == http.SEE_OTHER:
- return self._handleRedirect(response, 'GET', uri, headers,
- redirectCount)
- return response
-
-
-
-class _ReadBodyProtocol(protocol.Protocol):
- """
- Protocol that collects data sent to it.
-
- This is a helper for L{IResponse.deliverBody}, which collects the body and
- fires a deferred with it.
-
- @ivar deferred: See L{__init__}.
- @ivar status: See L{__init__}.
- @ivar message: See L{__init__}.
-
- @ivar dataBuffer: list of byte-strings received
- @type dataBuffer: L{list} of L{bytes}
- """
-
- def __init__(self, status, message, deferred):
- """
- @param status: Status of L{IResponse}
- @ivar status: L{int}
-
- @param message: Message of L{IResponse}
- @type message: L{bytes}
-
- @param deferred: deferred to fire when response is complete
- @type deferred: L{Deferred} firing with L{bytes}
- """
- self.deferred = deferred
- self.status = status
- self.message = message
- self.dataBuffer = []
-
-
- def dataReceived(self, data):
- """
- Accumulate some more bytes from the response.
- """
- self.dataBuffer.append(data)
-
-
- def connectionLost(self, reason):
- """
- Deliver the accumulated response bytes to the waiting L{Deferred}, if
- the response body has been completely received without error.
- """
- if reason.check(ResponseDone):
- self.deferred.callback(b''.join(self.dataBuffer))
- elif reason.check(PotentialDataLoss):
- self.deferred.errback(
- PartialDownloadError(self.status, self.message,
- b''.join(self.dataBuffer)))
- else:
- self.deferred.errback(reason)
-
-
-
-def readBody(response):
- """
- Get the body of an L{IResponse} and return it as a byte string.
-
- This is a helper function for clients that don't want to incrementally
- receive the body of an HTTP response.
-
- @param response: The HTTP response for which the body will be read.
- @type response: L{IResponse} provider
-
- @return: A L{Deferred} which will fire with the body of the response.
- """
- d = defer.Deferred()
- response.deliverBody(_ReadBodyProtocol(response.code, response.phrase, d))
- return d
-}}} '''
__all__ = [
diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.py
index 3f4704064..197e43ed3 100644
--- a/scrapy/xlib/tx/endpoints.py
+++ b/scrapy/xlib/tx/endpoints.py
@@ -14,27 +14,6 @@ parsed by the L{clientFromString} and L{serverFromString} functions.
from __future__ import division, absolute_import
-import os
-#import socket
-
-from zope.interface import implementer, directlyProvides
-import warnings
-
-from twisted.internet import interfaces, defer, error, fdesc
-#from twisted.internet.protocol import (
-# ClientFactory, Protocol)
-from twisted.internet.protocol import Factory
-#from twisted.internet import threads, ProcessProtocol
-from twisted.internet.interfaces import IStreamServerEndpointStringParser
-#from twisted.internet.interfaces import IStreamClientEndpointStringParser
-from twisted.python.filepath import FilePath
-#from twisted.python.failure import Failure
-#from twisted.python import log
-#from twisted.python.components import proxyForInterface
-
-from twisted.plugin import IPlugin, getPlugins
-#from twisted.internet import stdio
-
from twisted.internet.endpoints import (
clientFromString, serverFromString, quoteStringArgument,
TCP4ServerEndpoint, TCP6ServerEndpoint,
@@ -44,1236 +23,4 @@ from twisted.internet.endpoints import (
AdoptedStreamServerEndpoint, connectProtocol,
)
-
__all__ = ["TCP4ClientEndpoint", "SSL4ServerEndpoint"]
-
-''' {{{
-class _WrappingProtocol(Protocol):
- """
- Wrap another protocol in order to notify my user when a connection has
- been made.
- """
-
- def __init__(self, connectedDeferred, wrappedProtocol):
- """
- @param connectedDeferred: The L{Deferred} that will callback
- with the C{wrappedProtocol} when it is connected.
-
- @param wrappedProtocol: An L{IProtocol} provider that will be
- connected.
- """
- self._connectedDeferred = connectedDeferred
- self._wrappedProtocol = wrappedProtocol
-
- for iface in [interfaces.IHalfCloseableProtocol,
- interfaces.IFileDescriptorReceiver]:
- if iface.providedBy(self._wrappedProtocol):
- directlyProvides(self, iface)
-
-
- def logPrefix(self):
- """
- Transparently pass through the wrapped protocol's log prefix.
- """
- if interfaces.ILoggingContext.providedBy(self._wrappedProtocol):
- return self._wrappedProtocol.logPrefix()
- return self._wrappedProtocol.__class__.__name__
-
-
- def connectionMade(self):
- """
- Connect the C{self._wrappedProtocol} to our C{self.transport} and
- callback C{self._connectedDeferred} with the C{self._wrappedProtocol}
- """
- self._wrappedProtocol.makeConnection(self.transport)
- self._connectedDeferred.callback(self._wrappedProtocol)
-
-
- def dataReceived(self, data):
- """
- Proxy C{dataReceived} calls to our C{self._wrappedProtocol}
- """
- return self._wrappedProtocol.dataReceived(data)
-
-
- def fileDescriptorReceived(self, descriptor):
- """
- Proxy C{fileDescriptorReceived} calls to our C{self._wrappedProtocol}
- """
- return self._wrappedProtocol.fileDescriptorReceived(descriptor)
-
-
- def connectionLost(self, reason):
- """
- Proxy C{connectionLost} calls to our C{self._wrappedProtocol}
- """
- return self._wrappedProtocol.connectionLost(reason)
-
-
- def readConnectionLost(self):
- """
- Proxy L{IHalfCloseableProtocol.readConnectionLost} to our
- C{self._wrappedProtocol}
- """
- self._wrappedProtocol.readConnectionLost()
-
-
- def writeConnectionLost(self):
- """
- Proxy L{IHalfCloseableProtocol.writeConnectionLost} to our
- C{self._wrappedProtocol}
- """
- self._wrappedProtocol.writeConnectionLost()
-
-
-
-class _WrappingFactory(ClientFactory):
- """
- Wrap a factory in order to wrap the protocols it builds.
-
- @ivar _wrappedFactory: A provider of I{IProtocolFactory} whose buildProtocol
- method will be called and whose resulting protocol will be wrapped.
-
- @ivar _onConnection: A L{Deferred} that fires when the protocol is
- connected
-
- @ivar _connector: A L{connector <twisted.internet.interfaces.IConnector>}
- that is managing the current or previous connection attempt.
- """
- protocol = _WrappingProtocol
-
- def __init__(self, wrappedFactory):
- """
- @param wrappedFactory: A provider of I{IProtocolFactory} whose
- buildProtocol method will be called and whose resulting protocol
- will be wrapped.
- """
- self._wrappedFactory = wrappedFactory
- self._onConnection = defer.Deferred(canceller=self._canceller)
-
-
- def startedConnecting(self, connector):
- """
- A connection attempt was started. Remember the connector which started
- said attempt, for use later.
- """
- self._connector = connector
-
-
- def _canceller(self, deferred):
- """
- The outgoing connection attempt was cancelled. Fail that L{Deferred}
- with an L{error.ConnectingCancelledError}.
-
- @param deferred: The L{Deferred <defer.Deferred>} that was cancelled;
- should be the same as C{self._onConnection}.
- @type deferred: L{Deferred <defer.Deferred>}
-
- @note: This relies on startedConnecting having been called, so it may
- seem as though there's a race condition where C{_connector} may not
- have been set. However, using public APIs, this condition is
- impossible to catch, because a connection API
- (C{connectTCP}/C{SSL}/C{UNIX}) is always invoked before a
- L{_WrappingFactory}'s L{Deferred <defer.Deferred>} is returned to
- C{connect()}'s caller.
-
- @return: C{None}
- """
- deferred.errback(
- error.ConnectingCancelledError(
- self._connector.getDestination()))
- self._connector.stopConnecting()
-
-
- def doStart(self):
- """
- Start notifications are passed straight through to the wrapped factory.
- """
- self._wrappedFactory.doStart()
-
-
- def doStop(self):
- """
- Stop notifications are passed straight through to the wrapped factory.
- """
- self._wrappedFactory.doStop()
-
-
- def buildProtocol(self, addr):
- """
- Proxy C{buildProtocol} to our C{self._wrappedFactory} or errback
- the C{self._onConnection} L{Deferred}.
-
- @return: An instance of L{_WrappingProtocol} or C{None}
- """
- try:
- proto = self._wrappedFactory.buildProtocol(addr)
- except:
- self._onConnection.errback()
- else:
- return self.protocol(self._onConnection, proto)
-
-
- def clientConnectionFailed(self, connector, reason):
- """
- Errback the C{self._onConnection} L{Deferred} when the
- client connection fails.
- """
- if not self._onConnection.called:
- self._onConnection.errback(reason)
-
-
-
-
-
-@implementer(interfaces.ITransport)
-class _ProcessEndpointTransport(proxyForInterface(
- interfaces.IProcessTransport, '_process')):
- """
- An L{ITransport} provider for the L{IProtocol} instance passed to the
- process endpoint.
-
- @ivar _process: An active process transport which will be used by write
- methods on this object to write data to a child process.
- @type _process: L{interfaces.IProcessTransport} provider
- """
-
- def write(self, data):
- """
- Write to the child process's standard input.
-
- @param data: The data to write on stdin.
- """
- self._process.writeToChild(0, data)
-
-
- def writeSequence(self, data):
- """
- Write a list of strings to child process's stdin.
-
- @param data: The list of chunks to write on stdin.
- """
- for chunk in data:
- self._process.writeToChild(0, chunk)
-
-
-@implementer(interfaces.IStreamServerEndpoint)
-class _TCPServerEndpoint(object):
- """
- A TCP server endpoint interface
- """
-
- def __init__(self, reactor, port, backlog, interface):
- """
- @param reactor: An L{IReactorTCP} provider.
-
- @param port: The port number used for listening
- @type port: int
-
- @param backlog: Size of the listen queue
- @type backlog: int
-
- @param interface: The hostname to bind to
- @type interface: str
- """
- self._reactor = reactor
- self._port = port
- self._backlog = backlog
- self._interface = interface
-
-
- def listen(self, protocolFactory):
- """
- Implement L{IStreamServerEndpoint.listen} to listen on a TCP
- socket
- """
- return defer.execute(self._reactor.listenTCP,
- self._port,
- protocolFactory,
- backlog=self._backlog,
- interface=self._interface)
-
-
-
-class TCP4ServerEndpoint(_TCPServerEndpoint):
- """
- Implements TCP server endpoint with an IPv4 configuration
- """
- def __init__(self, reactor, port, backlog=50, interface=''):
- """
- @param reactor: An L{IReactorTCP} provider.
-
- @param port: The port number used for listening
- @type port: int
-
- @param backlog: Size of the listen queue
- @type backlog: int
-
- @param interface: The hostname to bind to, defaults to '' (all)
- @type interface: str
- """
- _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface)
-
-
-
-class TCP6ServerEndpoint(_TCPServerEndpoint):
- """
- Implements TCP server endpoint with an IPv6 configuration
- """
- def __init__(self, reactor, port, backlog=50, interface='::'):
- """
- @param reactor: An L{IReactorTCP} provider.
-
- @param port: The port number used for listening
- @type port: int
-
- @param backlog: Size of the listen queue
- @type backlog: int
-
- @param interface: The hostname to bind to, defaults to '' (all)
- @type interface: str
- """
- _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface)
-
-
-
-@implementer(interfaces.IStreamClientEndpoint)
-class TCP4ClientEndpoint(object):
- """
- TCP client endpoint with an IPv4 configuration.
- """
-
- def __init__(self, reactor, host, port, timeout=30, bindAddress=None):
- """
- @param reactor: An L{IReactorTCP} provider
-
- @param host: A hostname, used when connecting
- @type host: str
-
- @param port: The port number, used when connecting
- @type port: int
-
- @param timeout: The number of seconds to wait before assuming the
- connection has failed.
- @type timeout: int
-
- @param bindAddress: A (host, port) tuple of local address to bind to,
- or None.
- @type bindAddress: tuple
- """
- self._reactor = reactor
- self._host = host
- self._port = port
- self._timeout = timeout
- self._bindAddress = bindAddress
-
-
- def connect(self, protocolFactory):
- """
- Implement L{IStreamClientEndpoint.connect} to connect via TCP.
- """
- try:
- wf = _WrappingFactory(protocolFactory)
- self._reactor.connectTCP(
- self._host, self._port, wf,
- timeout=self._timeout, bindAddress=self._bindAddress)
- return wf._onConnection
- except:
- return defer.fail()
-
-
-
-
-@implementer(interfaces.IStreamServerEndpoint)
-class SSL4ServerEndpoint(object):
- """
- SSL secured TCP server endpoint with an IPv4 configuration.
- """
-
- def __init__(self, reactor, port, sslContextFactory,
- backlog=50, interface=''):
- """
- @param reactor: An L{IReactorSSL} provider.
-
- @param port: The port number used for listening
- @type port: int
-
- @param sslContextFactory: An instance of
- L{twisted.internet.ssl.ContextFactory}.
-
- @param backlog: Size of the listen queue
- @type backlog: int
-
- @param interface: The hostname to bind to, defaults to '' (all)
- @type interface: str
- """
- self._reactor = reactor
- self._port = port
- self._sslContextFactory = sslContextFactory
- self._backlog = backlog
- self._interface = interface
-
-
- def listen(self, protocolFactory):
- """
- Implement L{IStreamServerEndpoint.listen} to listen for SSL on a
- TCP socket.
- """
- return defer.execute(self._reactor.listenSSL, self._port,
- protocolFactory,
- contextFactory=self._sslContextFactory,
- backlog=self._backlog,
- interface=self._interface)
-
-
-
-@implementer(interfaces.IStreamClientEndpoint)
-class SSL4ClientEndpoint(object):
- """
- SSL secured TCP client endpoint with an IPv4 configuration
- """
-
- def __init__(self, reactor, host, port, sslContextFactory,
- timeout=30, bindAddress=None):
- """
- @param reactor: An L{IReactorSSL} provider.
-
- @param host: A hostname, used when connecting
- @type host: str
-
- @param port: The port number, used when connecting
- @type port: int
-
- @param sslContextFactory: SSL Configuration information as an instance
- of L{twisted.internet.ssl.ContextFactory}.
-
- @param timeout: Number of seconds to wait before assuming the
- connection has failed.
- @type timeout: int
-
- @param bindAddress: A (host, port) tuple of local address to bind to,
- or None.
- @type bindAddress: tuple
- """
- self._reactor = reactor
- self._host = host
- self._port = port
- self._sslContextFactory = sslContextFactory
- self._timeout = timeout
- self._bindAddress = bindAddress
-
-
- def connect(self, protocolFactory):
- """
- Implement L{IStreamClientEndpoint.connect} to connect with SSL over
- TCP.
- """
- try:
- wf = _WrappingFactory(protocolFactory)
- self._reactor.connectSSL(
- self._host, self._port, wf, self._sslContextFactory,
- timeout=self._timeout, bindAddress=self._bindAddress)
- return wf._onConnection
- except:
- return defer.fail()
-
-
-
-@implementer(interfaces.IStreamServerEndpoint)
-class UNIXServerEndpoint(object):
- """
- UnixSocket server endpoint.
- """
- def __init__(self, reactor, address, backlog=50, mode=0o666, wantPID=0):
- """
- @param reactor: An L{IReactorUNIX} provider.
- @param address: The path to the Unix socket file, used when listening
- @param backlog: number of connections to allow in backlog.
- @param mode: mode to set on the unix socket. This parameter is
- deprecated. Permissions should be set on the directory which
- contains the UNIX socket.
- @param wantPID: If True, create a pidfile for the socket.
- """
- self._reactor = reactor
- self._address = address
- self._backlog = backlog
- self._mode = mode
- self._wantPID = wantPID
-
-
- def listen(self, protocolFactory):
- """
- Implement L{IStreamServerEndpoint.listen} to listen on a UNIX socket.
- """
- return defer.execute(self._reactor.listenUNIX, self._address,
- protocolFactory,
- backlog=self._backlog,
- mode=self._mode,
- wantPID=self._wantPID)
-
-
-
-@implementer(interfaces.IStreamClientEndpoint)
-class UNIXClientEndpoint(object):
- """
- UnixSocket client endpoint.
- """
- def __init__(self, reactor, path, timeout=30, checkPID=0):
- """
- @param reactor: An L{IReactorUNIX} provider.
-
- @param path: The path to the Unix socket file, used when connecting
- @type path: str
-
- @param timeout: Number of seconds to wait before assuming the
- connection has failed.
- @type timeout: int
-
- @param checkPID: If True, check for a pid file to verify that a server
- is listening.
- @type checkPID: bool
- """
- self._reactor = reactor
- self._path = path
- self._timeout = timeout
- self._checkPID = checkPID
-
-
- def connect(self, protocolFactory):
- """
- Implement L{IStreamClientEndpoint.connect} to connect via a
- UNIX Socket
- """
- try:
- wf = _WrappingFactory(protocolFactory)
- self._reactor.connectUNIX(
- self._path, wf,
- timeout=self._timeout,
- checkPID=self._checkPID)
- return wf._onConnection
- except:
- return defer.fail()
-
-
-
-@implementer(interfaces.IStreamServerEndpoint)
-class AdoptedStreamServerEndpoint(object):
- """
- An endpoint for listening on a file descriptor initialized outside of
- Twisted.
-
- @ivar _used: A C{bool} indicating whether this endpoint has been used to
- listen with a factory yet. C{True} if so.
- """
- _close = os.close
- _setNonBlocking = staticmethod(fdesc.setNonBlocking)
-
- def __init__(self, reactor, fileno, addressFamily):
- """
- @param reactor: An L{IReactorSocket} provider.
-
- @param fileno: An integer file descriptor corresponding to a listening
- I{SOCK_STREAM} socket.
-
- @param addressFamily: The address family of the socket given by
- C{fileno}.
- """
- self.reactor = reactor
- self.fileno = fileno
- self.addressFamily = addressFamily
- self._used = False
-
-
- def listen(self, factory):
- """
- Implement L{IStreamServerEndpoint.listen} to start listening on, and
- then close, C{self._fileno}.
- """
- if self._used:
- return defer.fail(error.AlreadyListened())
- self._used = True
-
- try:
- self._setNonBlocking(self.fileno)
- port = self.reactor.adoptStreamPort(
- self.fileno, self.addressFamily, factory)
- self._close(self.fileno)
- except:
- return defer.fail()
- return defer.succeed(port)
-
-
-
-
-def _parseTCP(factory, port, interface="", backlog=50):
- """
- Internal parser function for L{_parseServer} to convert the string
- arguments for a TCP(IPv4) stream endpoint into the structured arguments.
-
- @param factory: the protocol factory being parsed, or C{None}. (This was a
- leftover argument from when this code was in C{strports}, and is now
- mostly None and unused.)
-
- @type factory: L{IProtocolFactory} or C{NoneType}
-
- @param port: the integer port number to bind
- @type port: C{str}
-
- @param interface: the interface IP to listen on
- @param backlog: the length of the listen queue
- @type backlog: C{str}
-
- @return: a 2-tuple of (args, kwargs), describing the parameters to
- L{IReactorTCP.listenTCP} (or, modulo argument 2, the factory, arguments
- to L{TCP4ServerEndpoint}.
- """
- return (int(port), factory), {'interface': interface,
- 'backlog': int(backlog)}
-
-
-
-def _parseUNIX(factory, address, mode='666', backlog=50, lockfile=True):
- """
- Internal parser function for L{_parseServer} to convert the string
- arguments for a UNIX (AF_UNIX/SOCK_STREAM) stream endpoint into the
- structured arguments.
-
- @param factory: the protocol factory being parsed, or C{None}. (This was a
- leftover argument from when this code was in C{strports}, and is now
- mostly None and unused.)
-
- @type factory: L{IProtocolFactory} or C{NoneType}
-
- @param address: the pathname of the unix socket
- @type address: C{str}
-
- @param backlog: the length of the listen queue
- @type backlog: C{str}
-
- @param lockfile: A string '0' or '1', mapping to True and False
- respectively. See the C{wantPID} argument to C{listenUNIX}
-
- @return: a 2-tuple of (args, kwargs), describing the parameters to
- L{IReactorTCP.listenUNIX} (or, modulo argument 2, the factory,
- arguments to L{UNIXServerEndpoint}.
- """
- return (
- (address, factory),
- {'mode': int(mode, 8), 'backlog': int(backlog),
- 'wantPID': bool(int(lockfile))})
-
-
-
-def _parseSSL(factory, port, privateKey="server.pem", certKey=None,
- sslmethod=None, interface='', backlog=50):
- """
- Internal parser function for L{_parseServer} to convert the string
- arguments for an SSL (over TCP/IPv4) stream endpoint into the structured
- arguments.
-
- @param factory: the protocol factory being parsed, or C{None}. (This was a
- leftover argument from when this code was in C{strports}, and is now
- mostly None and unused.)
- @type factory: L{IProtocolFactory} or C{NoneType}
-
- @param port: the integer port number to bind
- @type port: C{str}
-
- @param interface: the interface IP to listen on
- @param backlog: the length of the listen queue
- @type backlog: C{str}
-
- @param privateKey: The file name of a PEM format private key file.
- @type privateKey: C{str}
-
- @param certKey: The file name of a PEM format certificate file.
- @type certKey: C{str}
-
- @param sslmethod: The string name of an SSL method, based on the name of a
- constant in C{OpenSSL.SSL}. Must be one of: "SSLv23_METHOD",
- "SSLv2_METHOD", "SSLv3_METHOD", "TLSv1_METHOD".
- @type sslmethod: C{str}
-
- @return: a 2-tuple of (args, kwargs), describing the parameters to
- L{IReactorSSL.listenSSL} (or, modulo argument 2, the factory, arguments
- to L{SSL4ServerEndpoint}.
- """
- from twisted.internet import ssl
- if certKey is None:
- certKey = privateKey
- kw = {}
- if sslmethod is not None:
- kw['method'] = getattr(ssl.SSL, sslmethod)
- else:
- kw['method'] = ssl.SSL.SSLv23_METHOD
- certPEM = FilePath(certKey).getContent()
- keyPEM = FilePath(privateKey).getContent()
- privateCertificate = ssl.PrivateCertificate.loadPEM(certPEM + keyPEM)
- cf = ssl.CertificateOptions(
- privateKey=privateCertificate.privateKey.original,
- certificate=privateCertificate.original,
- **kw
- )
- return ((int(port), factory, cf),
- {'interface': interface, 'backlog': int(backlog)})
-
-
-
-@implementer(IPlugin, IStreamServerEndpointStringParser)
-class _StandardIOParser(object):
- """
- Stream server endpoint string parser for the Standard I/O type.
-
- @ivar prefix: See L{IStreamClientEndpointStringParser.prefix}.
- """
- prefix = "stdio"
-
- def _parseServer(self, reactor):
- """
- Internal parser function for L{_parseServer} to convert the string
- arguments into structured arguments for the L{StandardIOEndpoint}
-
- @param reactor: Reactor for the endpoint
- """
- return StandardIOEndpoint(reactor)
-
-
- def parseStreamServer(self, reactor, *args, **kwargs):
- # Redirects to another function (self._parseServer), tricks zope.interface
- # into believing the interface is correctly implemented.
- return self._parseServer(reactor)
-
-
-
-
-@implementer(IPlugin, IStreamServerEndpointStringParser)
-class _TCP6ServerParser(object):
- """
- Stream server endpoint string parser for the TCP6ServerEndpoint type.
-
- @ivar prefix: See L{IStreamClientEndpointStringParser.prefix}.
- """
- prefix = "tcp6" # Used in _parseServer to identify the plugin with the endpoint type
-
- def _parseServer(self, reactor, port, backlog=50, interface='::'):
- """
- Internal parser function for L{_parseServer} to convert the string
- arguments into structured arguments for the L{TCP6ServerEndpoint}
-
- @param reactor: An L{IReactorTCP} provider.
-
- @param port: The port number used for listening
- @type port: int
-
- @param backlog: Size of the listen queue
- @type backlog: int
-
- @param interface: The hostname to bind to
- @type interface: str
- """
- port = int(port)
- backlog = int(backlog)
- return TCP6ServerEndpoint(reactor, port, backlog, interface)
-
-
- def parseStreamServer(self, reactor, *args, **kwargs):
- # Redirects to another function (self._parseServer), tricks zope.interface
- # into believing the interface is correctly implemented.
- return self._parseServer(reactor, *args, **kwargs)
-
-
-
-_serverParsers = {"tcp": _parseTCP,
- "unix": _parseUNIX,
- "ssl": _parseSSL,
- }
-
-_OP, _STRING = range(2)
-
-def _tokenize(description):
- """
- Tokenize a strports string and yield each token.
-
- @param description: a string as described by L{serverFromString} or
- L{clientFromString}.
-
- @return: an iterable of 2-tuples of (L{_OP} or L{_STRING}, string). Tuples
- starting with L{_OP} will contain a second element of either ':' (i.e.
- 'next parameter') or '=' (i.e. 'assign parameter value'). For example,
- the string 'hello:greet\=ing=world' would result in a generator
- yielding these values::
-
- _STRING, 'hello'
- _OP, ':'
- _STRING, 'greet=ing'
- _OP, '='
- _STRING, 'world'
- """
- current = ''
- ops = ':='
- nextOps = {':': ':=', '=': ':'}
- description = iter(description)
- for n in description:
- if n in ops:
- yield _STRING, current
- yield _OP, n
- current = ''
- ops = nextOps[n]
- elif n == '\\':
- current += next(description)
- else:
- current += n
- yield _STRING, current
-
-
-
-def _parse(description):
- """
- Convert a description string into a list of positional and keyword
- parameters, using logic vaguely like what Python does.
-
- @param description: a string as described by L{serverFromString} or
- L{clientFromString}.
-
- @return: a 2-tuple of C{(args, kwargs)}, where 'args' is a list of all
- ':'-separated C{str}s not containing an '=' and 'kwargs' is a map of
- all C{str}s which do contain an '='. For example, the result of
- C{_parse('a:b:d=1:c')} would be C{(['a', 'b', 'c'], {'d': '1'})}.
- """
- args, kw = [], {}
- def add(sofar):
- if len(sofar) == 1:
- args.append(sofar[0])
- else:
- kw[sofar[0]] = sofar[1]
- sofar = ()
- for (type, value) in _tokenize(description):
- if type is _STRING:
- sofar += (value,)
- elif value == ':':
- add(sofar)
- sofar = ()
- add(sofar)
- return args, kw
-
-
-# Mappings from description "names" to endpoint constructors.
-_endpointServerFactories = {
- 'TCP': TCP4ServerEndpoint,
- 'SSL': SSL4ServerEndpoint,
- 'UNIX': UNIXServerEndpoint,
- }
-
-_endpointClientFactories = {
- 'TCP': TCP4ClientEndpoint,
- 'SSL': SSL4ClientEndpoint,
- 'UNIX': UNIXClientEndpoint,
- }
-
-
-_NO_DEFAULT = object()
-
-def _parseServer(description, factory, default=None):
- """
- Parse a stports description into a 2-tuple of arguments and keyword values.
-
- @param description: A description in the format explained by
- L{serverFromString}.
- @type description: C{str}
-
- @param factory: A 'factory' argument; this is left-over from
- twisted.application.strports, it's not really used.
- @type factory: L{IProtocolFactory} or L{None}
-
- @param default: Deprecated argument, specifying the default parser mode to
- use for unqualified description strings (those which do not have a ':'
- and prefix).
- @type default: C{str} or C{NoneType}
-
- @return: a 3-tuple of (plugin or name, arguments, keyword arguments)
- """
- args, kw = _parse(description)
- if not args or (len(args) == 1 and not kw):
- deprecationMessage = (
- "Unqualified strport description passed to 'service'."
- "Use qualified endpoint descriptions; for example, 'tcp:%s'."
- % (description,))
- if default is None:
- default = 'tcp'
- warnings.warn(
- deprecationMessage, category=DeprecationWarning, stacklevel=4)
- elif default is _NO_DEFAULT:
- raise ValueError(deprecationMessage)
- # If the default has been otherwise specified, the user has already
- # been warned.
- args[0:0] = [default]
- endpointType = args[0]
- parser = _serverParsers.get(endpointType)
- if parser is None:
- # If the required parser is not found in _server, check if
- # a plugin exists for the endpointType
- for plugin in getPlugins(IStreamServerEndpointStringParser):
- if plugin.prefix == endpointType:
- return (plugin, args[1:], kw)
- raise ValueError("Unknown endpoint type: '%s'" % (endpointType,))
- return (endpointType.upper(),) + parser(factory, *args[1:], **kw)
-
-
-
-def _serverFromStringLegacy(reactor, description, default):
- """
- Underlying implementation of L{serverFromString} which avoids exposing the
- deprecated 'default' argument to anything but L{strports.service}.
- """
- nameOrPlugin, args, kw = _parseServer(description, None, default)
- if type(nameOrPlugin) is not str:
- plugin = nameOrPlugin
- return plugin.parseStreamServer(reactor, *args, **kw)
- else:
- name = nameOrPlugin
- # Chop out the factory.
- args = args[:1] + args[2:]
- return _endpointServerFactories[name](reactor, *args, **kw)
-
-
-
-def serverFromString(reactor, description):
- """
- Construct a stream server endpoint from an endpoint description string.
-
- The format for server endpoint descriptions is a simple string. It is a
- prefix naming the type of endpoint, then a colon, then the arguments for
- that endpoint.
-
- For example, you can call it like this to create an endpoint that will
- listen on TCP port 80::
-
- serverFromString(reactor, "tcp:80")
-
- Additional arguments may be specified as keywords, separated with colons.
- For example, you can specify the interface for a TCP server endpoint to
- bind to like this::
-
- serverFromString(reactor, "tcp:80:interface=127.0.0.1")
-
- SSL server endpoints may be specified with the 'ssl' prefix, and the
- private key and certificate files may be specified by the C{privateKey} and
- C{certKey} arguments::
-
- serverFromString(reactor, "ssl:443:privateKey=key.pem:certKey=crt.pem")
-
- If a private key file name (C{privateKey}) isn't provided, a "server.pem"
- file is assumed to exist which contains the private key. If the certificate
- file name (C{certKey}) isn't provided, the private key file is assumed to
- contain the certificate as well.
-
- You may escape colons in arguments with a backslash, which you will need to
- use if you want to specify a full pathname argument on Windows::
-
- serverFromString(reactor,
- "ssl:443:privateKey=C\\:/key.pem:certKey=C\\:/cert.pem")
-
- finally, the 'unix' prefix may be used to specify a filesystem UNIX socket,
- optionally with a 'mode' argument to specify the mode of the socket file
- created by C{listen}::
-
- serverFromString(reactor, "unix:/var/run/finger")
- serverFromString(reactor, "unix:/var/run/finger:mode=660")
-
- This function is also extensible; new endpoint types may be registered as
- L{IStreamServerEndpointStringParser} plugins. See that interface for more
- information.
-
- @param reactor: The server endpoint will be constructed with this reactor.
-
- @param description: The strports description to parse.
-
- @return: A new endpoint which can be used to listen with the parameters
- given by by C{description}.
-
- @rtype: L{IStreamServerEndpoint<twisted.internet.interfaces.IStreamServerEndpoint>}
-
- @raise ValueError: when the 'description' string cannot be parsed.
-
- @since: 10.2
- """
- return _serverFromStringLegacy(reactor, description, _NO_DEFAULT)
-
-
-
-def quoteStringArgument(argument):
- """
- Quote an argument to L{serverFromString} and L{clientFromString}. Since
- arguments are separated with colons and colons are escaped with
- backslashes, some care is necessary if, for example, you have a pathname,
- you may be tempted to interpolate into a string like this::
-
- serverFromString("ssl:443:privateKey=%s" % (myPathName,))
-
- This may appear to work, but will have portability issues (Windows
- pathnames, for example). Usually you should just construct the appropriate
- endpoint type rather than interpolating strings, which in this case would
- be L{SSL4ServerEndpoint}. There are some use-cases where you may need to
- generate such a string, though; for example, a tool to manipulate a
- configuration file which has strports descriptions in it. To be correct in
- those cases, do this instead::
-
- serverFromString("ssl:443:privateKey=%s" %
- (quoteStringArgument(myPathName),))
-
- @param argument: The part of the endpoint description string you want to
- pass through.
-
- @type argument: C{str}
-
- @return: The quoted argument.
-
- @rtype: C{str}
- """
- return argument.replace('\\', '\\\\').replace(':', '\\:')
-
-
-
-def _parseClientTCP(*args, **kwargs):
- """
- Perform any argument value coercion necessary for TCP client parameters.
-
- Valid positional arguments to this function are host and port.
-
- Valid keyword arguments to this function are all L{IReactorTCP.connectTCP}
- arguments.
-
- @return: The coerced values as a C{dict}.
- """
-
- if len(args) == 2:
- kwargs['port'] = int(args[1])
- kwargs['host'] = args[0]
- elif len(args) == 1:
- if 'host' in kwargs:
- kwargs['port'] = int(args[0])
- else:
- kwargs['host'] = args[0]
-
- try:
- kwargs['port'] = int(kwargs['port'])
- except KeyError:
- pass
-
- try:
- kwargs['timeout'] = int(kwargs['timeout'])
- except KeyError:
- pass
- return kwargs
-
-
-
-def _loadCAsFromDir(directoryPath):
- """
- Load certificate-authority certificate objects in a given directory.
-
- @param directoryPath: a L{FilePath} pointing at a directory to load .pem
- files from.
-
- @return: a C{list} of L{OpenSSL.crypto.X509} objects.
- """
- from twisted.internet import ssl
-
- caCerts = {}
- for child in directoryPath.children():
- if not child.basename().split('.')[-1].lower() == 'pem':
- continue
- try:
- data = child.getContent()
- except IOError:
- # Permission denied, corrupt disk, we don't care.
- continue
- try:
- theCert = ssl.Certificate.loadPEM(data)
- except ssl.SSL.Error:
- # Duplicate certificate, invalid certificate, etc. We don't care.
- pass
- else:
- caCerts[theCert.digest()] = theCert.original
- return caCerts.values()
-
-
-
-def _parseClientSSL(*args, **kwargs):
- """
- Perform any argument value coercion necessary for SSL client parameters.
-
- Valid keyword arguments to this function are all L{IReactorSSL.connectSSL}
- arguments except for C{contextFactory}. Instead, C{certKey} (the path name
- of the certificate file) C{privateKey} (the path name of the private key
- associated with the certificate) are accepted and used to construct a
- context factory.
-
- Valid positional arguments to this function are host and port.
-
- @param caCertsDir: The one parameter which is not part of
- L{IReactorSSL.connectSSL}'s signature, this is a path name used to
- construct a list of certificate authority certificates. The directory
- will be scanned for files ending in C{.pem}, all of which will be
- considered valid certificate authorities for this connection.
-
- @type caCertsDir: C{str}
-
- @return: The coerced values as a C{dict}.
- """
- from twisted.internet import ssl
- kwargs = _parseClientTCP(*args, **kwargs)
- certKey = kwargs.pop('certKey', None)
- privateKey = kwargs.pop('privateKey', None)
- caCertsDir = kwargs.pop('caCertsDir', None)
- if certKey is not None:
- certx509 = ssl.Certificate.loadPEM(
- FilePath(certKey).getContent()).original
- else:
- certx509 = None
- if privateKey is not None:
- privateKey = ssl.PrivateCertificate.loadPEM(
- FilePath(privateKey).getContent()).privateKey.original
- else:
- privateKey = None
- if caCertsDir is not None:
- verify = True
- caCerts = _loadCAsFromDir(FilePath(caCertsDir))
- else:
- verify = False
- caCerts = None
- kwargs['sslContextFactory'] = ssl.CertificateOptions(
- method=ssl.SSL.SSLv23_METHOD,
- certificate=certx509,
- privateKey=privateKey,
- verify=verify,
- caCerts=caCerts
- )
- return kwargs
-
-
-
-def _parseClientUNIX(*args, **kwargs):
- """
- Perform any argument value coercion necessary for UNIX client parameters.
-
- Valid keyword arguments to this function are all L{IReactorUNIX.connectUNIX}
- keyword arguments except for C{checkPID}. Instead, C{lockfile} is accepted
- and has the same meaning. Also C{path} is used instead of C{address}.
-
- Valid positional arguments to this function are C{path}.
-
- @return: The coerced values as a C{dict}.
- """
- if len(args) == 1:
- kwargs['path'] = args[0]
-
- try:
- kwargs['checkPID'] = bool(int(kwargs.pop('lockfile')))
- except KeyError:
- pass
- try:
- kwargs['timeout'] = int(kwargs['timeout'])
- except KeyError:
- pass
- return kwargs
-
-_clientParsers = {
- 'TCP': _parseClientTCP,
- 'SSL': _parseClientSSL,
- 'UNIX': _parseClientUNIX,
- }
-
-
-
-def clientFromString(reactor, description):
- """
- Construct a client endpoint from a description string.
-
- Client description strings are much like server description strings,
- although they take all of their arguments as keywords, aside from host and
- port.
-
- You can create a TCP client endpoint with the 'host' and 'port' arguments,
- like so::
-
- clientFromString(reactor, "tcp:host=www.example.com:port=80")
-
- or, without specifying host and port keywords::
-
- clientFromString(reactor, "tcp:www.example.com:80")
-
- Or you can specify only one or the other, as in the following 2 examples::
-
- clientFromString(reactor, "tcp:host=www.example.com:80")
- clientFromString(reactor, "tcp:www.example.com:port=80")
-
- or an SSL client endpoint with those arguments, plus the arguments used by
- the server SSL, for a client certificate::
-
- clientFromString(reactor, "ssl:web.example.com:443:"
- "privateKey=foo.pem:certKey=foo.pem")
-
- to specify your certificate trust roots, you can identify a directory with
- PEM files in it with the C{caCertsDir} argument::
-
- clientFromString(reactor, "ssl:host=web.example.com:port=443:"
- "caCertsDir=/etc/ssl/certs")
-
- You can create a UNIX client endpoint with the 'path' argument and optional
- 'lockfile' and 'timeout' arguments::
-
- clientFromString(reactor, "unix:path=/var/foo/bar:lockfile=1:timeout=9")
-
- or, with the path as a positional argument with or without optional
- arguments as in the following 2 examples::
-
- clientFromString(reactor, "unix:/var/foo/bar")
- clientFromString(reactor, "unix:/var/foo/bar:lockfile=1:timeout=9")
-
- This function is also extensible; new endpoint types may be registered as
- L{IStreamClientEndpointStringParser} plugins. See that interface for more
- information.
-
- @param reactor: The client endpoint will be constructed with this reactor.
-
- @param description: The strports description to parse.
-
- @return: A new endpoint which can be used to connect with the parameters
- given by by C{description}.
- @rtype: L{IStreamClientEndpoint<twisted.internet.interfaces.IStreamClientEndpoint>}
-
- @since: 10.2
- """
- args, kwargs = _parse(description)
- aname = args.pop(0)
- name = aname.upper()
- for plugin in getPlugins(IStreamClientEndpointStringParser):
- if plugin.prefix.upper() == name:
- return plugin.parseStreamClient(*args, **kwargs)
- if name not in _clientParsers:
- raise ValueError("Unknown endpoint type: %r" % (aname,))
- kwargs = _clientParsers[name](*args, **kwargs)
- return _endpointClientFactories[name](reactor, **kwargs)
-
-
-
-def connectProtocol(endpoint, protocol):
- """
- Connect a protocol instance to an endpoint.
-
- This allows using a client endpoint without having to create a factory.
-
- @param endpoint: A client endpoint to connect to.
-
- @param protocol: A protocol instance.
-
- @return: The result of calling C{connect} on the endpoint, i.e. a
- L{Deferred} that will fire with the protocol when connected, or an
- appropriate error.
- """
- class OneShotFactory(Factory):
- def buildProtocol(self, addr):
- return protocol
- return endpoint.connect(OneShotFactory())
-}}} '''
diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py
index 7b2a78632..fdcbf3977 100644
--- a/scrapy/xlib/tx/interfaces.py
+++ b/scrapy/xlib/tx/interfaces.py
@@ -9,8 +9,6 @@ Maintainer: Itamar Shtull-Trauring
from __future__ import division, absolute_import
-from zope.interface import Interface, Attribute
-
from twisted.internet.interfaces import (
IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL,
IReactorWin32Events, IReactorUDP, IReactorMulticast, IReactorProcess,
@@ -28,2434 +26,3 @@ from twisted.internet.interfaces import (
IStreamServerEndpointStringParser, IStreamClientEndpointStringParser,
IReactorUNIX, IReactorUNIXDatagram, IReactorSocket, IResolver
)
-
-''' {{{
-class IAddress(Interface):
- """
- An address, e.g. a TCP C{(host, port)}.
-
- Default implementations are in L{twisted.internet.address}.
- """
-
-### Reactor Interfaces
-
-class IConnector(Interface):
- """
- Object used to interface between connections and protocols.
-
- Each L{IConnector} manages one connection.
- """
-
- def stopConnecting():
- """
- Stop attempting to connect.
- """
-
- def disconnect():
- """
- Disconnect regardless of the connection state.
-
- If we are connected, disconnect, if we are trying to connect,
- stop trying.
- """
-
- def connect():
- """
- Try to connect to remote address.
- """
-
- def getDestination():
- """
- Return destination this will try to connect to.
-
- @return: An object which provides L{IAddress}.
- """
-
-
-
-class IResolverSimple(Interface):
- def getHostByName(name, timeout = (1, 3, 11, 45)):
- """
- Resolve the domain name C{name} into an IP address.
-
- @type name: C{str}
- @type timeout: C{tuple}
- @rtype: L{twisted.internet.defer.Deferred}
- @return: The callback of the Deferred that is returned will be
- passed a string that represents the IP address of the specified
- name, or the errback will be called if the lookup times out. If
- multiple types of address records are associated with the name,
- A6 records will be returned in preference to AAAA records, which
- will be returned in preference to A records. If there are multiple
- records of the type to be returned, one will be selected at random.
-
- @raise twisted.internet.defer.TimeoutError: Raised (asynchronously)
- if the name cannot be resolved within the specified timeout period.
- """
-
-
-
-class IResolver(IResolverSimple):
- def query(query, timeout=None):
- """
- Dispatch C{query} to the method which can handle its type.
-
- @type query: L{twisted.names.dns.Query}
- @param query: The DNS query being issued, to which a response is to be
- generated.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupAddress(name, timeout=None):
- """
- Perform an A record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupAddress6(name, timeout=None):
- """
- Perform an A6 record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupIPV6Address(name, timeout=None):
- """
- Perform an AAAA record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupMailExchange(name, timeout=None):
- """
- Perform an MX record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupNameservers(name, timeout=None):
- """
- Perform an NS record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupCanonicalName(name, timeout=None):
- """
- Perform a CNAME record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupMailBox(name, timeout=None):
- """
- Perform an MB record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupMailGroup(name, timeout=None):
- """
- Perform an MG record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupMailRename(name, timeout=None):
- """
- Perform an MR record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupPointer(name, timeout=None):
- """
- Perform a PTR record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupAuthority(name, timeout=None):
- """
- Perform an SOA record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupNull(name, timeout=None):
- """
- Perform a NULL record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupWellKnownServices(name, timeout=None):
- """
- Perform a WKS record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupHostInfo(name, timeout=None):
- """
- Perform a HINFO record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupMailboxInfo(name, timeout=None):
- """
- Perform an MINFO record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupText(name, timeout=None):
- """
- Perform a TXT record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupResponsibility(name, timeout=None):
- """
- Perform an RP record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupAFSDatabase(name, timeout=None):
- """
- Perform an AFSDB record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupService(name, timeout=None):
- """
- Perform an SRV record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupAllRecords(name, timeout=None):
- """
- Perform an ALL_RECORD lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupSenderPolicy(name, timeout= 10):
- """
- Perform a SPF record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupNamingAuthorityPointer(name, timeout=None):
- """
- Perform a NAPTR record lookup.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: Sequence of C{int}
- @param timeout: Number of seconds after which to reissue the query.
- When the last timeout expires, the query is considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances. The first element of the
- tuple gives answers. The second element of the tuple gives
- authorities. The third element of the tuple gives additional
- information. The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
- def lookupZone(name, timeout=None):
- """
- Perform an AXFR record lookup.
-
- NB This is quite different from other DNS requests. See
- U{http://cr.yp.to/djbdns/axfr-notes.html} for more
- information.
-
- NB Unlike other C{lookup*} methods, the timeout here is not a
- list of ints, it is a single int.
-
- @type name: C{str}
- @param name: DNS name to resolve.
-
- @type timeout: C{int}
- @param timeout: When this timeout expires, the query is
- considered failed.
-
- @rtype: L{Deferred}
- @return: A L{Deferred} which fires with a three-tuple of lists of
- L{twisted.names.dns.RRHeader} instances.
- The first element of the tuple gives answers.
- The second and third elements are always empty.
- The L{Deferred} may instead fail with one of the
- exceptions defined in L{twisted.names.error} or with
- C{NotImplementedError}.
- """
-
-
-
-class IReactorTCP(Interface):
-
- def listenTCP(port, factory, backlog=50, interface=''):
- """
- Connects a given protocol factory to the given numeric TCP/IP port.
-
- @param port: a port number on which to listen
-
- @param factory: a L{twisted.internet.protocol.ServerFactory} instance
-
- @param backlog: size of the listen queue
-
- @param interface: The local IPv4 or IPv6 address to which to bind;
- defaults to '', ie all IPv4 addresses. To bind to all IPv4 and IPv6
- addresses, you must call this method twice.
-
- @return: an object that provides L{IListeningPort}.
-
- @raise CannotListenError: as defined here
- L{twisted.internet.error.CannotListenError},
- if it cannot listen on this port (e.g., it
- cannot bind to the required port number)
- """
-
- def connectTCP(host, port, factory, timeout=30, bindAddress=None):
- """
- Connect a TCP client.
-
- @param host: a host name
-
- @param port: a port number
-
- @param factory: a L{twisted.internet.protocol.ClientFactory} instance
-
- @param timeout: number of seconds to wait before assuming the
- connection has failed.
-
- @param bindAddress: a (host, port) tuple of local address to bind
- to, or None.
-
- @return: An object which provides L{IConnector}. This connector will
- call various callbacks on the factory when a connection is
- made, failed, or lost - see
- L{ClientFactory<twisted.internet.protocol.ClientFactory>}
- docs for details.
- """
-
-class IReactorSSL(Interface):
-
- def connectSSL(host, port, factory, contextFactory, timeout=30, bindAddress=None):
- """
- Connect a client Protocol to a remote SSL socket.
-
- @param host: a host name
-
- @param port: a port number
-
- @param factory: a L{twisted.internet.protocol.ClientFactory} instance
-
- @param contextFactory: a L{twisted.internet.ssl.ClientContextFactory} object.
-
- @param timeout: number of seconds to wait before assuming the
- connection has failed.
-
- @param bindAddress: a (host, port) tuple of local address to bind to,
- or C{None}.
-
- @return: An object which provides L{IConnector}.
- """
-
- def listenSSL(port, factory, contextFactory, backlog=50, interface=''):
- """
- Connects a given protocol factory to the given numeric TCP/IP port.
- The connection is a SSL one, using contexts created by the context
- factory.
-
- @param port: a port number on which to listen
-
- @param factory: a L{twisted.internet.protocol.ServerFactory} instance
-
- @param contextFactory: a L{twisted.internet.ssl.ContextFactory} instance
-
- @param backlog: size of the listen queue
-
- @param interface: the hostname to bind to, defaults to '' (all)
- """
-
-
-
-class IReactorUNIX(Interface):
- """
- UNIX socket methods.
- """
-
- def connectUNIX(address, factory, timeout=30, checkPID=0):
- """
- Connect a client protocol to a UNIX socket.
-
- @param address: a path to a unix socket on the filesystem.
-
- @param factory: a L{twisted.internet.protocol.ClientFactory} instance
-
- @param timeout: number of seconds to wait before assuming the connection
- has failed.
-
- @param checkPID: if True, check for a pid file to verify that a server
- is listening. If C{address} is a Linux abstract namespace path,
- this must be C{False}.
-
- @return: An object which provides L{IConnector}.
- """
-
-
- def listenUNIX(address, factory, backlog=50, mode=0o666, wantPID=0):
- """
- Listen on a UNIX socket.
-
- @param address: a path to a unix socket on the filesystem.
-
- @param factory: a L{twisted.internet.protocol.Factory} instance.
-
- @param backlog: number of connections to allow in backlog.
-
- @param mode: The mode (B{not} umask) to set on the unix socket. See
- platform specific documentation for information about how this
- might affect connection attempts.
- @type mode: C{int}
-
- @param wantPID: if True, create a pidfile for the socket. If C{address}
- is a Linux abstract namespace path, this must be C{False}.
-
- @return: An object which provides L{IListeningPort}.
- """
-
-
-
-class IReactorUNIXDatagram(Interface):
- """
- Datagram UNIX socket methods.
- """
-
- def connectUNIXDatagram(address, protocol, maxPacketSize=8192, mode=0o666, bindAddress=None):
- """
- Connect a client protocol to a datagram UNIX socket.
-
- @param address: a path to a unix socket on the filesystem.
-
- @param protocol: a L{twisted.internet.protocol.ConnectedDatagramProtocol} instance
-
- @param maxPacketSize: maximum packet size to accept
-
- @param mode: The mode (B{not} umask) to set on the unix socket. See
- platform specific documentation for information about how this
- might affect connection attempts.
- @type mode: C{int}
-
- @param bindAddress: address to bind to
-
- @return: An object which provides L{IConnector}.
- """
-
-
- def listenUNIXDatagram(address, protocol, maxPacketSize=8192, mode=0o666):
- """
- Listen on a datagram UNIX socket.
-
- @param address: a path to a unix socket on the filesystem.
-
- @param protocol: a L{twisted.internet.protocol.DatagramProtocol} instance.
-
- @param maxPacketSize: maximum packet size to accept
-
- @param mode: The mode (B{not} umask) to set on the unix socket. See
- platform specific documentation for information about how this
- might affect connection attempts.
- @type mode: C{int}
-
- @return: An object which provides L{IListeningPort}.
- """
-
-
-
-class IReactorWin32Events(Interface):
- """
- Win32 Event API methods
-
- @since: 10.2
- """
-
- def addEvent(event, fd, action):
- """
- Add a new win32 event to the event loop.
-
- @param event: a Win32 event object created using win32event.CreateEvent()
-
- @param fd: an instance of L{twisted.internet.abstract.FileDescriptor}
-
- @param action: a string that is a method name of the fd instance.
- This method is called in response to the event.
-
- @return: None
- """
-
-
- def removeEvent(event):
- """
- Remove an event.
-
- @param event: a Win32 event object added using L{IReactorWin32Events.addEvent}
-
- @return: None
- """
-
-
-
-class IReactorUDP(Interface):
- """
- UDP socket methods.
- """
-
- def listenUDP(port, protocol, interface='', maxPacketSize=8192):
- """
- Connects a given DatagramProtocol to the given numeric UDP port.
-
- @return: object which provides L{IListeningPort}.
- """
-
-
-
-class IReactorMulticast(Interface):
- """
- UDP socket methods that support multicast.
-
- IMPORTANT: This is an experimental new interface. It may change
- without backwards compatibility. Suggestions are welcome.
- """
-
- def listenMulticast(port, protocol, interface='', maxPacketSize=8192,
- listenMultiple=False):
- """
- Connects a given
- L{DatagramProtocol<twisted.internet.protocol.DatagramProtocol>} to the
- given numeric UDP port.
-
- @param listenMultiple: If set to True, allows multiple sockets to
- bind to the same address and port number at the same time.
- @type listenMultiple: C{bool}
-
- @returns: An object which provides L{IListeningPort}.
-
- @see: L{twisted.internet.interfaces.IMulticastTransport}
- @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html}
- """
-
-
-
-class IReactorSocket(Interface):
- """
- Methods which allow a reactor to use externally created sockets.
-
- For example, to use C{adoptStreamPort} to implement behavior equivalent
- to that of L{IReactorTCP.listenTCP}, you might write code like this::
-
- from socket import SOMAXCONN, AF_INET, SOCK_STREAM, socket
- portSocket = socket(AF_INET, SOCK_STREAM)
- # Set FD_CLOEXEC on port, left as an exercise. Then make it into a
- # non-blocking listening port:
- portSocket.setblocking(False)
- portSocket.bind(('192.168.1.2', 12345))
- portSocket.listen(SOMAXCONN)
-
- # Now have the reactor use it as a TCP port
- port = reactor.adoptStreamPort(
- portSocket.fileno(), AF_INET, YourFactory())
-
- # portSocket itself is no longer necessary, and needs to be cleaned
- # up by us.
- portSocket.close()
-
- # Whenever the server is no longer needed, stop it as usual.
- stoppedDeferred = port.stopListening()
-
- Another potential use is to inherit a listening descriptor from a parent
- process (for example, systemd or launchd), or to receive one over a UNIX
- domain socket.
-
- Some plans for extending this interface exist. See:
-
- - U{http://twistedmatrix.com/trac/ticket/5570}: established connections
- - U{http://twistedmatrix.com/trac/ticket/5573}: AF_UNIX ports
- - U{http://twistedmatrix.com/trac/ticket/5574}: SOCK_DGRAM sockets
- """
-
- def adoptStreamPort(fileDescriptor, addressFamily, factory):
- """
- Add an existing listening I{SOCK_STREAM} socket to the reactor to
- monitor for new connections to accept and handle.
-
- @param fileDescriptor: A file descriptor associated with a socket which
- is already bound to an address and marked as listening. The socket
- must be set non-blocking. Any additional flags (for example,
- close-on-exec) must also be set by application code. Application
- code is responsible for closing the file descriptor, which may be
- done as soon as C{adoptStreamPort} returns.
- @type fileDescriptor: C{int}
-
- @param addressFamily: The address family (or I{domain}) of the socket.
- For example, L{socket.AF_INET6}.
-
- @param factory: A L{ServerFactory} instance to use to create new
- protocols to handle connections accepted via this socket.
-
- @return: An object providing L{IListeningPort}.
-
- @raise UnsupportedAddressFamily: If the given address family is not
- supported by this reactor, or not supported with the given socket
- type.
-
- @raise UnsupportedSocketType: If the given socket type is not supported
- by this reactor, or not supported with the given socket type.
- """
-
-
- def adoptStreamConnection(fileDescriptor, addressFamily, factory):
- """
- Add an existing connected I{SOCK_STREAM} socket to the reactor to
- monitor for data.
-
- Note that the given factory won't have its C{startFactory} and
- C{stopFactory} methods called, as there is no sensible time to call
- them in this situation.
-
- @param fileDescriptor: A file descriptor associated with a socket which
- is already connected. The socket must be set non-blocking. Any
- additional flags (for example, close-on-exec) must also be set by
- application code. Application code is responsible for closing the
- file descriptor, which may be done as soon as
- C{adoptStreamConnection} returns.
- @type fileDescriptor: C{int}
-
- @param addressFamily: The address family (or I{domain}) of the socket.
- For example, L{socket.AF_INET6}.
-
- @param factory: A L{ServerFactory} instance to use to create a new
- protocol to handle the connection via this socket.
-
- @raise UnsupportedAddressFamily: If the given address family is not
- supported by this reactor, or not supported with the given socket
- type.
-
- @raise UnsupportedSocketType: If the given socket type is not supported
- by this reactor, or not supported with the given socket type.
- """
-
-
-
-class IReactorProcess(Interface):
-
- def spawnProcess(processProtocol, executable, args=(), env={}, path=None,
- uid=None, gid=None, usePTY=0, childFDs=None):
- """
- Spawn a process, with a process protocol.
-
- @type processProtocol: L{IProcessProtocol} provider
- @param processProtocol: An object which will be notified of all
- events related to the created process.
-
- @param executable: the file name to spawn - the full path should be
- used.
-
- @param args: the command line arguments to pass to the process; a
- sequence of strings. The first string should be the
- executable's name.
-
- @type env: a C{dict} mapping C{str} to C{str}, or C{None}.
- @param env: the environment variables to pass to the child process. The
- resulting behavior varies between platforms. If
- - C{env} is not set:
- - On POSIX: pass an empty environment.
- - On Windows: pass C{os.environ}.
- - C{env} is C{None}:
- - On POSIX: pass C{os.environ}.
- - On Windows: pass C{os.environ}.
- - C{env} is a C{dict}:
- - On POSIX: pass the key/value pairs in C{env} as the
- complete environment.
- - On Windows: update C{os.environ} with the key/value
- pairs in the C{dict} before passing it. As a
- consequence of U{bug #1640
- <http://twistedmatrix.com/trac/ticket/1640>}, passing
- keys with empty values in an effort to unset
- environment variables I{won't} unset them.
-
- @param path: the path to run the subprocess in - defaults to the
- current directory.
-
- @param uid: user ID to run the subprocess as. (Only available on
- POSIX systems.)
-
- @param gid: group ID to run the subprocess as. (Only available on
- POSIX systems.)
-
- @param usePTY: if true, run this process in a pseudo-terminal.
- optionally a tuple of C{(masterfd, slavefd, ttyname)},
- in which case use those file descriptors.
- (Not available on all systems.)
-
- @param childFDs: A dictionary mapping file descriptors in the new child
- process to an integer or to the string 'r' or 'w'.
-
- If the value is an integer, it specifies a file
- descriptor in the parent process which will be mapped
- to a file descriptor (specified by the key) in the
- child process. This is useful for things like inetd
- and shell-like file redirection.
-
- If it is the string 'r', a pipe will be created and
- attached to the child at that file descriptor: the
- child will be able to write to that file descriptor
- and the parent will receive read notification via the
- L{IProcessProtocol.childDataReceived} callback. This
- is useful for the child's stdout and stderr.
-
- If it is the string 'w', similar setup to the previous
- case will occur, with the pipe being readable by the
- child instead of writeable. The parent process can
- write to that file descriptor using
- L{IProcessTransport.writeToChild}. This is useful for
- the child's stdin.
-
- If childFDs is not passed, the default behaviour is to
- use a mapping that opens the usual stdin/stdout/stderr
- pipes.
-
- @see: L{twisted.internet.protocol.ProcessProtocol}
-
- @return: An object which provides L{IProcessTransport}.
-
- @raise OSError: Raised with errno C{EAGAIN} or C{ENOMEM} if there are
- insufficient system resources to create a new process.
- """
-
-class IReactorTime(Interface):
- """
- Time methods that a Reactor should implement.
- """
-
- def seconds():
- """
- Get the current time in seconds.
-
- @return: A number-like object of some sort.
- """
-
-
- def callLater(delay, callable, *args, **kw):
- """
- Call a function later.
-
- @type delay: C{float}
- @param delay: the number of seconds to wait.
-
- @param callable: the callable object to call later.
-
- @param args: the arguments to call it with.
-
- @param kw: the keyword arguments to call it with.
-
- @return: An object which provides L{IDelayedCall} and can be used to
- cancel the scheduled call, by calling its C{cancel()} method.
- It also may be rescheduled by calling its C{delay()} or
- C{reset()} methods.
- """
-
-
- def getDelayedCalls():
- """
- Retrieve all currently scheduled delayed calls.
-
- @return: A tuple of all L{IDelayedCall} providers representing all
- currently scheduled calls. This is everything that has been
- returned by C{callLater} but not yet called or canceled.
- """
-
-
-class IDelayedCall(Interface):
- """
- A scheduled call.
-
- There are probably other useful methods we can add to this interface;
- suggestions are welcome.
- """
-
- def getTime():
- """
- Get time when delayed call will happen.
-
- @return: time in seconds since epoch (a float).
- """
-
- def cancel():
- """
- Cancel the scheduled call.
-
- @raises twisted.internet.error.AlreadyCalled: if the call has already
- happened.
- @raises twisted.internet.error.AlreadyCancelled: if the call has already
- been cancelled.
- """
-
- def delay(secondsLater):
- """
- Delay the scheduled call.
-
- @param secondsLater: how many seconds from its current firing time to delay
-
- @raises twisted.internet.error.AlreadyCalled: if the call has already
- happened.
- @raises twisted.internet.error.AlreadyCancelled: if the call has already
- been cancelled.
- """
-
- def reset(secondsFromNow):
- """
- Reset the scheduled call's timer.
-
- @param secondsFromNow: how many seconds from now it should fire,
- equivalent to C{.cancel()} and then doing another
- C{reactor.callLater(secondsLater, ...)}
-
- @raises twisted.internet.error.AlreadyCalled: if the call has already
- happened.
- @raises twisted.internet.error.AlreadyCancelled: if the call has already
- been cancelled.
- """
-
- def active():
- """
- @return: True if this call is still active, False if it has been
- called or cancelled.
- """
-
-class IReactorThreads(Interface):
- """
- Dispatch methods to be run in threads.
-
- Internally, this should use a thread pool and dispatch methods to them.
- """
-
- def getThreadPool():
- """
- Return the threadpool used by L{callInThread}. Create it first if
- necessary.
-
- @rtype: L{twisted.python.threadpool.ThreadPool}
- """
-
-
- def callInThread(callable, *args, **kwargs):
- """
- Run the callable object in a separate thread.
- """
-
-
- def callFromThread(callable, *args, **kw):
- """
- Cause a function to be executed by the reactor thread.
-
- Use this method when you want to run a function in the reactor's thread
- from another thread. Calling L{callFromThread} should wake up the main
- thread (where L{reactor.run()<reactor.run>} is executing) and run the
- given callable in that thread.
-
- If you're writing a multi-threaded application the C{callable} may need
- to be thread safe, but this method doesn't require it as such. If you
- want to call a function in the next mainloop iteration, but you're in
- the same thread, use L{callLater} with a delay of 0.
- """
-
-
- def suggestThreadPoolSize(size):
- """
- Suggest the size of the internal threadpool used to dispatch functions
- passed to L{callInThread}.
- """
-
-
-class IReactorCore(Interface):
- """
- Core methods that a Reactor must implement.
- """
-
- running = Attribute(
- "A C{bool} which is C{True} from I{during startup} to "
- "I{during shutdown} and C{False} the rest of the time.")
-
-
- def resolve(name, timeout=10):
- """
- Return a L{twisted.internet.defer.Deferred} that will resolve a hostname.
- """
-
- def run():
- """
- Fire 'startup' System Events, move the reactor to the 'running'
- state, then run the main loop until it is stopped with C{stop()} or
- C{crash()}.
- """
-
- def stop():
- """
- Fire 'shutdown' System Events, which will move the reactor to the
- 'stopped' state and cause C{reactor.run()} to exit.
- """
-
- def crash():
- """
- Stop the main loop *immediately*, without firing any system events.
-
- This is named as it is because this is an extremely "rude" thing to do;
- it is possible to lose data and put your system in an inconsistent
- state by calling this. However, it is necessary, as sometimes a system
- can become wedged in a pre-shutdown call.
- """
-
- def iterate(delay=0):
- """
- Run the main loop's I/O polling function for a period of time.
-
- This is most useful in applications where the UI is being drawn "as
- fast as possible", such as games. All pending L{IDelayedCall}s will
- be called.
-
- The reactor must have been started (via the C{run()} method) prior to
- any invocations of this method. It must also be stopped manually
- after the last call to this method (via the C{stop()} method). This
- method is not re-entrant: you must not call it recursively; in
- particular, you must not call it while the reactor is running.
- """
-
- def fireSystemEvent(eventType):
- """
- Fire a system-wide event.
-
- System-wide events are things like 'startup', 'shutdown', and
- 'persist'.
- """
-
- def addSystemEventTrigger(phase, eventType, callable, *args, **kw):
- """
- Add a function to be called when a system event occurs.
-
- Each "system event" in Twisted, such as 'startup', 'shutdown', and
- 'persist', has 3 phases: 'before', 'during', and 'after' (in that
- order, of course). These events will be fired internally by the
- Reactor.
-
- An implementor of this interface must only implement those events
- described here.
-
- Callbacks registered for the "before" phase may return either None or a
- Deferred. The "during" phase will not execute until all of the
- Deferreds from the "before" phase have fired.
-
- Once the "during" phase is running, all of the remaining triggers must
- execute; their return values must be ignored.
-
- @param phase: a time to call the event -- either the string 'before',
- 'after', or 'during', describing when to call it
- relative to the event's execution.
-
- @param eventType: this is a string describing the type of event.
-
- @param callable: the object to call before shutdown.
-
- @param args: the arguments to call it with.
-
- @param kw: the keyword arguments to call it with.
-
- @return: an ID that can be used to remove this call with
- removeSystemEventTrigger.
- """
-
- def removeSystemEventTrigger(triggerID):
- """
- Removes a trigger added with addSystemEventTrigger.
-
- @param triggerID: a value returned from addSystemEventTrigger.
-
- @raise KeyError: If there is no system event trigger for the given
- C{triggerID}.
-
- @raise ValueError: If there is no system event trigger for the given
- C{triggerID}.
-
- @raise TypeError: If there is no system event trigger for the given
- C{triggerID}.
- """
-
- def callWhenRunning(callable, *args, **kw):
- """
- Call a function when the reactor is running.
-
- If the reactor has not started, the callable will be scheduled
- to run when it does start. Otherwise, the callable will be invoked
- immediately.
-
- @param callable: the callable object to call later.
-
- @param args: the arguments to call it with.
-
- @param kw: the keyword arguments to call it with.
-
- @return: None if the callable was invoked, otherwise a system
- event id for the scheduled call.
- """
-
-
-class IReactorPluggableResolver(Interface):
- """
- A reactor with a pluggable name resolver interface.
- """
-
- def installResolver(resolver):
- """
- Set the internal resolver to use to for name lookups.
-
- @type resolver: An object implementing the L{IResolverSimple} interface
- @param resolver: The new resolver to use.
-
- @return: The previously installed resolver.
- """
-
-
-class IReactorDaemonize(Interface):
- """
- A reactor which provides hooks that need to be called before and after
- daemonization.
-
- Notes:
- - This interface SHOULD NOT be called by applications.
- - This interface should only be implemented by reactors as a workaround
- (in particular, it's implemented currently only by kqueue()).
- For details please see the comments on ticket #1918.
- """
-
- def beforeDaemonize():
- """
- Hook to be called immediately before daemonization. No reactor methods
- may be called until L{afterDaemonize} is called.
-
- @return: C{None}.
- """
-
-
- def afterDaemonize():
- """
- Hook to be called immediately after daemonization. This may only be
- called after L{beforeDaemonize} had been called previously.
-
- @return: C{None}.
- """
-
-
-
-class IReactorFDSet(Interface):
- """
- Implement me to be able to use L{IFileDescriptor} type resources.
-
- This assumes that your main-loop uses UNIX-style numeric file descriptors
- (or at least similarly opaque IDs returned from a .fileno() method)
- """
-
- def addReader(reader):
- """
- I add reader to the set of file descriptors to get read events for.
-
- @param reader: An L{IReadDescriptor} provider that will be checked for
- read events until it is removed from the reactor with
- L{removeReader}.
-
- @return: C{None}.
- """
-
- def addWriter(writer):
- """
- I add writer to the set of file descriptors to get write events for.
-
- @param writer: An L{IWriteDescriptor} provider that will be checked for
- write events until it is removed from the reactor with
- L{removeWriter}.
-
- @return: C{None}.
- """
-
- def removeReader(reader):
- """
- Removes an object previously added with L{addReader}.
-
- @return: C{None}.
- """
-
- def removeWriter(writer):
- """
- Removes an object previously added with L{addWriter}.
-
- @return: C{None}.
- """
-
- def removeAll():
- """
- Remove all readers and writers.
-
- Should not remove reactor internal reactor connections (like a waker).
-
- @return: A list of L{IReadDescriptor} and L{IWriteDescriptor} providers
- which were removed.
- """
-
- def getReaders():
- """
- Return the list of file descriptors currently monitored for input
- events by the reactor.
-
- @return: the list of file descriptors monitored for input events.
- @rtype: C{list} of C{IReadDescriptor}
- """
-
- def getWriters():
- """
- Return the list file descriptors currently monitored for output events
- by the reactor.
-
- @return: the list of file descriptors monitored for output events.
- @rtype: C{list} of C{IWriteDescriptor}
- """
-
-
-class IListeningPort(Interface):
- """
- A listening port.
- """
-
- def startListening():
- """
- Start listening on this port.
-
- @raise CannotListenError: If it cannot listen on this port (e.g., it is
- a TCP port and it cannot bind to the required
- port number).
- """
-
- def stopListening():
- """
- Stop listening on this port.
-
- If it does not complete immediately, will return Deferred that fires
- upon completion.
- """
-
- def getHost():
- """
- Get the host that this port is listening for.
-
- @return: An L{IAddress} provider.
- """
-
-
-class ILoggingContext(Interface):
- """
- Give context information that will be used to log events generated by
- this item.
- """
-
- def logPrefix():
- """
- @return: Prefix used during log formatting to indicate context.
- @rtype: C{str}
- """
-
-
-
-class IFileDescriptor(ILoggingContext):
- """
- An interface representing a UNIX-style numeric file descriptor.
- """
-
- def fileno():
- """
- @raise: If the descriptor no longer has a valid file descriptor
- number associated with it.
-
- @return: The platform-specified representation of a file descriptor
- number. Or C{-1} if the descriptor no longer has a valid file
- descriptor number associated with it. As long as the descriptor
- is valid, calls to this method on a particular instance must
- return the same value.
- """
-
-
- def connectionLost(reason):
- """
- Called when the connection was lost.
-
- This is called when the connection on a selectable object has been
- lost. It will be called whether the connection was closed explicitly,
- an exception occurred in an event handler, or the other end of the
- connection closed it first.
-
- See also L{IHalfCloseableDescriptor} if your descriptor wants to be
- notified separately of the two halves of the connection being closed.
-
- @param reason: A failure instance indicating the reason why the
- connection was lost. L{error.ConnectionLost} and
- L{error.ConnectionDone} are of special note, but the
- failure may be of other classes as well.
- """
-
-
-
-class IReadDescriptor(IFileDescriptor):
- """
- An L{IFileDescriptor} that can read.
-
- This interface is generally used in conjunction with L{IReactorFDSet}.
- """
-
- def doRead():
- """
- Some data is available for reading on your descriptor.
-
- @return: If an error is encountered which causes the descriptor to
- no longer be valid, a L{Failure} should be returned. Otherwise,
- C{None}.
- """
-
-
-class IWriteDescriptor(IFileDescriptor):
- """
- An L{IFileDescriptor} that can write.
-
- This interface is generally used in conjunction with L{IReactorFDSet}.
- """
-
- def doWrite():
- """
- Some data can be written to your descriptor.
-
- @return: If an error is encountered which causes the descriptor to
- no longer be valid, a L{Failure} should be returned. Otherwise,
- C{None}.
- """
-
-
-class IReadWriteDescriptor(IReadDescriptor, IWriteDescriptor):
- """
- An L{IFileDescriptor} that can both read and write.
- """
-
-
-class IHalfCloseableDescriptor(Interface):
- """
- A descriptor that can be half-closed.
- """
-
- def writeConnectionLost(reason):
- """
- Indicates write connection was lost.
- """
-
- def readConnectionLost(reason):
- """
- Indicates read connection was lost.
- """
-
-
-class ISystemHandle(Interface):
- """
- An object that wraps a networking OS-specific handle.
- """
-
- def getHandle():
- """
- Return a system- and reactor-specific handle.
-
- This might be a socket.socket() object, or some other type of
- object, depending on which reactor is being used. Use and
- manipulate at your own risk.
-
- This might be used in cases where you want to set specific
- options not exposed by the Twisted APIs.
- """
-
-
-class IConsumer(Interface):
- """
- A consumer consumes data from a producer.
- """
-
- def registerProducer(producer, streaming):
- """
- Register to receive data from a producer.
-
- This sets self to be a consumer for a producer. When this object runs
- out of data (as when a send(2) call on a socket succeeds in moving the
- last data from a userspace buffer into a kernelspace buffer), it will
- ask the producer to resumeProducing().
-
- For L{IPullProducer} providers, C{resumeProducing} will be called once
- each time data is required.
-
- For L{IPushProducer} providers, C{pauseProducing} will be called
- whenever the write buffer fills up and C{resumeProducing} will only be
- called when it empties.
-
- @type producer: L{IProducer} provider
-
- @type streaming: C{bool}
- @param streaming: C{True} if C{producer} provides L{IPushProducer},
- C{False} if C{producer} provides L{IPullProducer}.
-
- @raise RuntimeError: If a producer is already registered.
-
- @return: C{None}
- """
-
-
- def unregisterProducer():
- """
- Stop consuming data from a producer, without disconnecting.
- """
-
-
- def write(data):
- """
- The producer will write data by calling this method.
-
- The implementation must be non-blocking and perform whatever
- buffering is necessary. If the producer has provided enough data
- for now and it is a L{IPushProducer}, the consumer may call its
- C{pauseProducing} method.
- """
-
-
-
-class IProducer(Interface):
- """
- A producer produces data for a consumer.
-
- Typically producing is done by calling the write method of an class
- implementing L{IConsumer}.
- """
-
- def stopProducing():
- """
- Stop producing data.
-
- This tells a producer that its consumer has died, so it must stop
- producing data for good.
- """
-
-
-class IPushProducer(IProducer):
- """
- A push producer, also known as a streaming producer is expected to
- produce (write to this consumer) data on a continuous basis, unless
- it has been paused. A paused push producer will resume producing
- after its resumeProducing() method is called. For a push producer
- which is not pauseable, these functions may be noops.
- """
-
- def pauseProducing():
- """
- Pause producing data.
-
- Tells a producer that it has produced too much data to process for
- the time being, and to stop until resumeProducing() is called.
- """
- def resumeProducing():
- """
- Resume producing data.
-
- This tells a producer to re-add itself to the main loop and produce
- more data for its consumer.
- """
-
-class IPullProducer(IProducer):
- """
- A pull producer, also known as a non-streaming producer, is
- expected to produce data each time resumeProducing() is called.
- """
-
- def resumeProducing():
- """
- Produce data for the consumer a single time.
-
- This tells a producer to produce data for the consumer once
- (not repeatedly, once only). Typically this will be done
- by calling the consumer's write() method a single time with
- produced data.
- """
-
-class IProtocol(Interface):
-
- def dataReceived(data):
- """
- Called whenever data is received.
-
- Use this method to translate to a higher-level message. Usually, some
- callback will be made upon the receipt of each complete protocol
- message.
-
- @param data: a string of indeterminate length. Please keep in mind
- that you will probably need to buffer some data, as partial
- (or multiple) protocol messages may be received! I recommend
- that unit tests for protocols call through to this method with
- differing chunk sizes, down to one byte at a time.
- """
-
- def connectionLost(reason):
- """
- Called when the connection is shut down.
-
- Clear any circular references here, and any external references
- to this Protocol. The connection has been closed. The C{reason}
- Failure wraps a L{twisted.internet.error.ConnectionDone} or
- L{twisted.internet.error.ConnectionLost} instance (or a subclass
- of one of those).
-
- @type reason: L{twisted.python.failure.Failure}
- """
-
- def makeConnection(transport):
- """
- Make a connection to a transport and a server.
- """
-
- def connectionMade():
- """
- Called when a connection is made.
-
- This may be considered the initializer of the protocol, because
- it is called when the connection is completed. For clients,
- this is called once the connection to the server has been
- established; for servers, this is called after an accept() call
- stops blocking and a socket has been received. If you need to
- send any greeting or initial message, do it here.
- """
-
-
-class IProcessProtocol(Interface):
- """
- Interface for process-related event handlers.
- """
-
- def makeConnection(process):
- """
- Called when the process has been created.
-
- @type process: L{IProcessTransport} provider
- @param process: An object representing the process which has been
- created and associated with this protocol.
- """
-
-
- def childDataReceived(childFD, data):
- """
- Called when data arrives from the child process.
-
- @type childFD: C{int}
- @param childFD: The file descriptor from which the data was
- received.
-
- @type data: C{str}
- @param data: The data read from the child's file descriptor.
- """
-
-
- def childConnectionLost(childFD):
- """
- Called when a file descriptor associated with the child process is
- closed.
-
- @type childFD: C{int}
- @param childFD: The file descriptor which was closed.
- """
-
-
- def processExited(reason):
- """
- Called when the child process exits.
-
- @type reason: L{twisted.python.failure.Failure}
- @param reason: A failure giving the reason the child process
- terminated. The type of exception for this failure is either
- L{twisted.internet.error.ProcessDone} or
- L{twisted.internet.error.ProcessTerminated}.
-
- @since: 8.2
- """
-
-
- def processEnded(reason):
- """
- Called when the child process exits and all file descriptors associated
- with it have been closed.
-
- @type reason: L{twisted.python.failure.Failure}
- @param reason: A failure giving the reason the child process
- terminated. The type of exception for this failure is either
- L{twisted.internet.error.ProcessDone} or
- L{twisted.internet.error.ProcessTerminated}.
- """
-
-
-
-class IHalfCloseableProtocol(Interface):
- """
- Implemented to indicate they want notification of half-closes.
-
- TCP supports the notion of half-closing the connection, e.g.
- closing the write side but still not stopping reading. A protocol
- that implements this interface will be notified of such events,
- instead of having connectionLost called.
- """
-
- def readConnectionLost():
- """
- Notification of the read connection being closed.
-
- This indicates peer did half-close of write side. It is now
- the responsibility of the this protocol to call
- loseConnection(). In addition, the protocol MUST make sure a
- reference to it still exists (i.e. by doing a callLater with
- one of its methods, etc.) as the reactor will only have a
- reference to it if it is writing.
-
- If the protocol does not do so, it might get garbage collected
- without the connectionLost method ever being called.
- """
-
- def writeConnectionLost():
- """
- Notification of the write connection being closed.
-
- This will never be called for TCP connections as TCP does not
- support notification of this type of half-close.
- """
-
-
-
-class IFileDescriptorReceiver(Interface):
- """
- Protocols may implement L{IFileDescriptorReceiver} to receive file
- descriptors sent to them. This is useful in conjunction with
- L{IUNIXTransport}, which allows file descriptors to be sent between
- processes on a single host.
- """
- def fileDescriptorReceived(descriptor):
- """
- Called when a file descriptor is received over the connection.
-
- @param descriptor: The descriptor which was received.
- @type descriptor: C{int}
-
- @return: C{None}
- """
-
-
-
-class IProtocolFactory(Interface):
- """
- Interface for protocol factories.
- """
-
- def buildProtocol(addr):
- """
- Called when a connection has been established to addr.
-
- If None is returned, the connection is assumed to have been refused,
- and the Port will close the connection.
-
- @type addr: (host, port)
- @param addr: The address of the newly-established connection
-
- @return: None if the connection was refused, otherwise an object
- providing L{IProtocol}.
- """
-
- def doStart():
- """
- Called every time this is connected to a Port or Connector.
- """
-
- def doStop():
- """
- Called every time this is unconnected from a Port or Connector.
- """
-
-
-class ITransport(Interface):
- """
- I am a transport for bytes.
-
- I represent (and wrap) the physical connection and synchronicity
- of the framework which is talking to the network. I make no
- representations about whether calls to me will happen immediately
- or require returning to a control loop, or whether they will happen
- in the same or another thread. Consider methods of this class
- (aside from getPeer) to be 'thrown over the wall', to happen at some
- indeterminate time.
- """
-
- def write(data):
- """
- Write some data to the physical connection, in sequence, in a
- non-blocking fashion.
-
- If possible, make sure that it is all written. No data will
- ever be lost, although (obviously) the connection may be closed
- before it all gets through.
- """
-
- def writeSequence(data):
- """
- Write a list of strings to the physical connection.
-
- If possible, make sure that all of the data is written to
- the socket at once, without first copying it all into a
- single string.
- """
-
- def loseConnection():
- """
- Close my connection, after writing all pending data.
-
- Note that if there is a registered producer on a transport it
- will not be closed until the producer has been unregistered.
- """
-
- def getPeer():
- """
- Get the remote address of this connection.
-
- Treat this method with caution. It is the unfortunate result of the
- CGI and Jabber standards, but should not be considered reliable for
- the usual host of reasons; port forwarding, proxying, firewalls, IP
- masquerading, etc.
-
- @return: An L{IAddress} provider.
- """
-
- def getHost():
- """
- Similar to getPeer, but returns an address describing this side of the
- connection.
-
- @return: An L{IAddress} provider.
- """
-
-
-class ITCPTransport(ITransport):
- """
- A TCP based transport.
- """
-
- def loseWriteConnection():
- """
- Half-close the write side of a TCP connection.
-
- If the protocol instance this is attached to provides
- IHalfCloseableProtocol, it will get notified when the operation is
- done. When closing write connection, as with loseConnection this will
- only happen when buffer has emptied and there is no registered
- producer.
- """
-
-
- def abortConnection():
- """
- Close the connection abruptly.
-
- Discards any buffered data, stops any registered producer,
- and, if possible, notifies the other end of the unclean
- closure.
-
- @since: 11.1
- """
-
-
- def getTcpNoDelay():
- """
- Return if C{TCP_NODELAY} is enabled.
- """
-
- def setTcpNoDelay(enabled):
- """
- Enable/disable C{TCP_NODELAY}.
-
- Enabling C{TCP_NODELAY} turns off Nagle's algorithm. Small packets are
- sent sooner, possibly at the expense of overall throughput.
- """
-
- def getTcpKeepAlive():
- """
- Return if C{SO_KEEPALIVE} is enabled.
- """
-
- def setTcpKeepAlive(enabled):
- """
- Enable/disable C{SO_KEEPALIVE}.
-
- Enabling C{SO_KEEPALIVE} sends packets periodically when the connection
- is otherwise idle, usually once every two hours. They are intended
- to allow detection of lost peers in a non-infinite amount of time.
- """
-
- def getHost():
- """
- Returns L{IPv4Address} or L{IPv6Address}.
- """
-
- def getPeer():
- """
- Returns L{IPv4Address} or L{IPv6Address}.
- """
-
-
-
-class IUNIXTransport(ITransport):
- """
- Transport for stream-oriented unix domain connections.
- """
- def sendFileDescriptor(descriptor):
- """
- Send a duplicate of this (file, socket, pipe, etc) descriptor to the
- other end of this connection.
-
- The send is non-blocking and will be queued if it cannot be performed
- immediately. The send will be processed in order with respect to other
- C{sendFileDescriptor} calls on this transport, but not necessarily with
- respect to C{write} calls on this transport. The send can only be
- processed if there are also bytes in the normal connection-oriented send
- buffer (ie, you must call C{write} at least as many times as you call
- C{sendFileDescriptor}).
-
- @param descriptor: An C{int} giving a valid file descriptor in this
- process. Note that a I{file descriptor} may actually refer to a
- socket, a pipe, or anything else POSIX tries to treat in the same
- way as a file.
-
- @return: C{None}
- """
-
-
-
-class ITLSTransport(ITCPTransport):
- """
- A TCP transport that supports switching to TLS midstream.
-
- Once TLS mode is started the transport will implement L{ISSLTransport}.
- """
-
- def startTLS(contextFactory):
- """
- Initiate TLS negotiation.
-
- @param contextFactory: A context factory (see L{ssl.py<twisted.internet.ssl>})
- """
-
-class ISSLTransport(ITCPTransport):
- """
- A SSL/TLS based transport.
- """
-
- def getPeerCertificate():
- """
- Return an object with the peer's certificate info.
- """
-
-
-class IProcessTransport(ITransport):
- """
- A process transport.
- """
-
- pid = Attribute(
- "From before L{IProcessProtocol.makeConnection} is called to before "
- "L{IProcessProtocol.processEnded} is called, C{pid} is an L{int} "
- "giving the platform process ID of this process. C{pid} is L{None} "
- "at all other times.")
-
- def closeStdin():
- """
- Close stdin after all data has been written out.
- """
-
- def closeStdout():
- """
- Close stdout.
- """
-
- def closeStderr():
- """
- Close stderr.
- """
-
- def closeChildFD(descriptor):
- """
- Close a file descriptor which is connected to the child process, identified
- by its FD in the child process.
- """
-
- def writeToChild(childFD, data):
- """
- Similar to L{ITransport.write} but also allows the file descriptor in
- the child process which will receive the bytes to be specified.
-
- @type childFD: C{int}
- @param childFD: The file descriptor to which to write.
-
- @type data: C{str}
- @param data: The bytes to write.
-
- @return: C{None}
-
- @raise KeyError: If C{childFD} is not a file descriptor that was mapped
- in the child when L{IReactorProcess.spawnProcess} was used to create
- it.
- """
-
- def loseConnection():
- """
- Close stdin, stderr and stdout.
- """
-
- def signalProcess(signalID):
- """
- Send a signal to the process.
-
- @param signalID: can be
- - one of C{"KILL"}, C{"TERM"}, or C{"INT"}.
- These will be implemented in a
- cross-platform manner, and so should be used
- if possible.
- - an integer, where it represents a POSIX
- signal ID.
-
- @raise twisted.internet.error.ProcessExitedAlready: If the process has
- already exited.
- @raise OSError: If the C{os.kill} call fails with an errno different
- from C{ESRCH}.
- """
-
-
-class IServiceCollection(Interface):
- """
- An object which provides access to a collection of services.
- """
-
- def getServiceNamed(serviceName):
- """
- Retrieve the named service from this application.
-
- Raise a C{KeyError} if there is no such service name.
- """
-
- def addService(service):
- """
- Add a service to this collection.
- """
-
- def removeService(service):
- """
- Remove a service from this collection.
- """
-
-
-class IUDPTransport(Interface):
- """
- Transport for UDP DatagramProtocols.
- """
-
- def write(packet, addr=None):
- """
- Write packet to given address.
-
- @param addr: a tuple of (ip, port). For connected transports must
- be the address the transport is connected to, or None.
- In non-connected mode this is mandatory.
-
- @raise twisted.internet.error.MessageLengthError: C{packet} was too
- long.
- """
-
- def connect(host, port):
- """
- Connect the transport to an address.
-
- This changes it to connected mode. Datagrams can only be sent to
- this address, and will only be received from this address. In addition
- the protocol's connectionRefused method might get called if destination
- is not receiving datagrams.
-
- @param host: an IP address, not a domain name ('127.0.0.1', not 'localhost')
- @param port: port to connect to.
- """
-
- def getHost():
- """
- Returns L{IPv4Address}.
- """
-
- def stopListening():
- """
- Stop listening on this port.
-
- If it does not complete immediately, will return L{Deferred} that fires
- upon completion.
- """
-
-
-
-class IUNIXDatagramTransport(Interface):
- """
- Transport for UDP PacketProtocols.
- """
-
- def write(packet, address):
- """
- Write packet to given address.
- """
-
- def getHost():
- """
- Returns L{UNIXAddress}.
- """
-
-
-class IUNIXDatagramConnectedTransport(Interface):
- """
- Transport for UDP ConnectedPacketProtocols.
- """
-
- def write(packet):
- """
- Write packet to address we are connected to.
- """
-
- def getHost():
- """
- Returns L{UNIXAddress}.
- """
-
- def getPeer():
- """
- Returns L{UNIXAddress}.
- """
-
-
-class IMulticastTransport(Interface):
- """
- Additional functionality for multicast UDP.
- """
-
- def getOutgoingInterface():
- """
- Return interface of outgoing multicast packets.
- """
-
- def setOutgoingInterface(addr):
- """
- Set interface for outgoing multicast packets.
-
- Returns Deferred of success.
- """
-
- def getLoopbackMode():
- """
- Return if loopback mode is enabled.
- """
-
- def setLoopbackMode(mode):
- """
- Set if loopback mode is enabled.
- """
-
- def getTTL():
- """
- Get time to live for multicast packets.
- """
-
- def setTTL(ttl):
- """
- Set time to live on multicast packets.
- """
-
- def joinGroup(addr, interface=""):
- """
- Join a multicast group. Returns L{Deferred} of success or failure.
-
- If an error occurs, the returned L{Deferred} will fail with
- L{error.MulticastJoinError}.
- """
-
- def leaveGroup(addr, interface=""):
- """
- Leave multicast group, return L{Deferred} of success.
- """
-
-
-class IStreamClientEndpoint(Interface):
- """
- A stream client endpoint is a place that L{ClientFactory} can connect to.
- For example, a remote TCP host/port pair would be a TCP client endpoint.
-
- @since: 10.1
- """
-
- def connect(protocolFactory):
- """
- Connect the C{protocolFactory} to the location specified by this
- L{IStreamClientEndpoint} provider.
-
- @param protocolFactory: A provider of L{IProtocolFactory}
- @return: A L{Deferred} that results in an L{IProtocol} upon successful
- connection otherwise a L{ConnectError}
- """
-
-
-
-class IStreamServerEndpoint(Interface):
- """
- A stream server endpoint is a place that a L{Factory} can listen for
- incoming connections.
-
- @since: 10.1
- """
-
- def listen(protocolFactory):
- """
- Listen with C{protocolFactory} at the location specified by this
- L{IStreamServerEndpoint} provider.
-
- @param protocolFactory: A provider of L{IProtocolFactory}
- @return: A L{Deferred} that results in an L{IListeningPort} or an
- L{CannotListenError}
- """
-
-
-
-class IStreamServerEndpointStringParser(Interface):
- """
- An L{IStreamServerEndpointStringParser} is like an
- L{IStreamClientEndpointStringParser}, except for L{IStreamServerEndpoint}s
- instead of clients. It integrates with L{endpoints.serverFromString} in
- much the same way.
- """
-
- prefix = Attribute(
- """
- @see: L{IStreamClientEndpointStringParser.prefix}
- """
- )
-
-
- def parseStreamServer(reactor, *args, **kwargs):
- """
- Parse a stream server endpoint from a reactor and string-only arguments
- and keyword arguments.
-
- @see: L{IStreamClientEndpointStringParser.parseStreamClient}
-
- @return: a stream server endpoint
- @rtype: L{IStreamServerEndpoint}
- """
-
-
-
-class IStreamClientEndpointStringParser(Interface):
- """
- An L{IStreamClientEndpointStringParser} is a parser which can convert
- a set of string C{*args} and C{**kwargs} into an L{IStreamClientEndpoint}
- provider.
-
- This interface is really only useful in the context of the plugin system
- for L{endpoints.clientFromString}. See the document entitled "I{The
- Twisted Plugin System}" for more details on how to write a plugin.
-
- If you place an L{IStreamClientEndpointStringParser} plugin in the
- C{twisted.plugins} package, that plugin's C{parseStreamClient} method will
- be used to produce endpoints for any description string that begins with
- the result of that L{IStreamClientEndpointStringParser}'s prefix attribute.
- """
-
- prefix = Attribute(
- """
- A C{str}, the description prefix to respond to. For example, an
- L{IStreamClientEndpointStringParser} plugin which had C{"foo"} for its
- C{prefix} attribute would be called for endpoint descriptions like
- C{"foo:bar:baz"} or C{"foo:"}.
- """
- )
-
-
- def parseStreamClient(*args, **kwargs):
- """
- This method is invoked by L{endpoints.clientFromString}, if the type of
- endpoint matches the return value from this
- L{IStreamClientEndpointStringParser}'s C{prefix} method.
-
- @param args: The string arguments, minus the endpoint type, in the
- endpoint description string, parsed according to the rules
- described in L{endpoints.quoteStringArgument}. For example, if the
- description were C{"my-type:foo:bar:baz=qux"}, C{args} would be
- C{('foo','bar')}
-
- @param kwargs: The string arguments from the endpoint description
- passed as keyword arguments. For example, if the description were
- C{"my-type:foo:bar:baz=qux"}, C{kwargs} would be
- C{dict(baz='qux')}.
-
- @return: a client endpoint
- @rtype: L{IStreamClientEndpoint}
- """
-}}} '''
diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.py
index 32c88ff2d..fd814dc22 100644
--- a/scrapy/xlib/tx/iweb.py
+++ b/scrapy/xlib/tx/iweb.py
@@ -10,580 +10,11 @@ Interface definitions for L{twisted.web}.
body is not known in advance.
"""
-from zope.interface import Interface, Attribute
-
-#from twisted.internet.interfaces import IPushProducer
-
from twisted.web.iweb import (
IRequest, ICredentialFactory, IBodyProducer, IRenderable, ITemplateLoader,
IResponse, _IRequestEncoder, _IRequestEncoderFactory, UNKNOWN_LENGTH,
)
-''' {{{
-class IRequest(Interface):
- """
- An HTTP request.
-
- @since: 9.0
- """
-
- method = Attribute("A C{str} giving the HTTP method that was used.")
- uri = Attribute(
- "A C{str} giving the full encoded URI which was requested (including "
- "query arguments).")
- path = Attribute(
- "A C{str} giving the encoded query path of the request URI.")
- args = Attribute(
- "A mapping of decoded query argument names as C{str} to "
- "corresponding query argument values as C{list}s of C{str}. "
- "For example, for a URI with C{'foo=bar&foo=baz&quux=spam'} "
- "for its query part, C{args} will be C{{'foo': ['bar', 'baz'], "
- "'quux': ['spam']}}.")
-
- received_headers = Attribute(
- "Backwards-compatibility access to C{requestHeaders}. Use "
- "C{requestHeaders} instead. C{received_headers} behaves mostly "
- "like a C{dict} and does not provide access to all header values.")
-
- requestHeaders = Attribute(
- "A L{http_headers.Headers} instance giving all received HTTP request "
- "headers.")
-
- content = Attribute(
- "A file-like object giving the request body. This may be a file on "
- "disk, a C{StringIO}, or some other type. The implementation is free "
- "to decide on a per-request basis.")
-
- headers = Attribute(
- "Backwards-compatibility access to C{responseHeaders}. Use"
- "C{responseHeaders} instead. C{headers} behaves mostly like a "
- "C{dict} and does not provide access to all header values nor "
- "does it allow multiple values for one header to be set.")
-
- responseHeaders = Attribute(
- "A L{http_headers.Headers} instance holding all HTTP response "
- "headers to be sent.")
-
- def getHeader(key):
- """
- Get an HTTP request header.
-
- @type key: C{str}
- @param key: The name of the header to get the value of.
-
- @rtype: C{str} or C{NoneType}
- @return: The value of the specified header, or C{None} if that header
- was not present in the request.
- """
-
-
- def getCookie(key):
- """
- Get a cookie that was sent from the network.
- """
-
-
- def getAllHeaders():
- """
- Return dictionary mapping the names of all received headers to the last
- value received for each.
-
- Since this method does not return all header information,
- C{requestHeaders.getAllRawHeaders()} may be preferred.
- """
-
-
- def getRequestHostname():
- """
- Get the hostname that the user passed in to the request.
-
- This will either use the Host: header (if it is available) or the
- host we are listening on if the header is unavailable.
-
- @returns: the requested hostname
- @rtype: C{str}
- """
-
-
- def getHost():
- """
- Get my originally requesting transport's host.
-
- @return: An L{IAddress<twisted.internet.interfaces.IAddress>}.
- """
-
-
- def getClientIP():
- """
- Return the IP address of the client who submitted this request.
-
- @returns: the client IP address or C{None} if the request was submitted
- over a transport where IP addresses do not make sense.
- @rtype: L{str} or C{NoneType}
- """
-
-
- def getClient():
- """
- Return the hostname of the IP address of the client who submitted this
- request, if possible.
-
- This method is B{deprecated}. See L{getClientIP} instead.
-
- @rtype: C{NoneType} or L{str}
- @return: The canonical hostname of the client, as determined by
- performing a name lookup on the IP address of the client.
- """
-
-
- def getUser():
- """
- Return the HTTP user sent with this request, if any.
-
- If no user was supplied, return the empty string.
-
- @returns: the HTTP user, if any
- @rtype: C{str}
- """
-
-
- def getPassword():
- """
- Return the HTTP password sent with this request, if any.
-
- If no password was supplied, return the empty string.
-
- @returns: the HTTP password, if any
- @rtype: C{str}
- """
-
-
- def isSecure():
- """
- Return True if this request is using a secure transport.
-
- Normally this method returns True if this request's HTTPChannel
- instance is using a transport that implements ISSLTransport.
-
- This will also return True if setHost() has been called
- with ssl=True.
-
- @returns: True if this request is secure
- @rtype: C{bool}
- """
-
-
- def getSession(sessionInterface=None):
- """
- Look up the session associated with this request or create a new one if
- there is not one.
-
- @return: The L{Session} instance identified by the session cookie in
- the request, or the C{sessionInterface} component of that session
- if C{sessionInterface} is specified.
- """
-
-
- def URLPath():
- """
- @return: A L{URLPath} instance which identifies the URL for which this
- request is.
- """
-
-
- def prePathURL():
- """
- @return: At any time during resource traversal, a L{str} giving an
- absolute URL to the most nested resource which has yet been
- reached.
- """
-
-
- def rememberRootURL():
- """
- Remember the currently-processed part of the URL for later
- recalling.
- """
-
-
- def getRootURL():
- """
- Get a previously-remembered URL.
- """
-
-
- # Methods for outgoing response
- def finish():
- """
- Indicate that the response to this request is complete.
- """
-
-
- def write(data):
- """
- Write some data to the body of the response to this request. Response
- headers are written the first time this method is called, after which
- new response headers may not be added.
- """
-
-
- def addCookie(k, v, expires=None, domain=None, path=None, max_age=None, comment=None, secure=None):
- """
- Set an outgoing HTTP cookie.
-
- In general, you should consider using sessions instead of cookies, see
- L{twisted.web.server.Request.getSession} and the
- L{twisted.web.server.Session} class for details.
- """
-
-
- def setResponseCode(code, message=None):
- """
- Set the HTTP response code.
- """
-
-
- def setHeader(k, v):
- """
- Set an HTTP response header. Overrides any previously set values for
- this header.
-
- @type name: C{str}
- @param name: The name of the header for which to set the value.
-
- @type value: C{str}
- @param value: The value to set for the named header.
- """
-
-
- def redirect(url):
- """
- Utility function that does a redirect.
-
- The request should have finish() called after this.
- """
-
-
- def setLastModified(when):
- """
- Set the C{Last-Modified} time for the response to this request.
-
- If I am called more than once, I ignore attempts to set Last-Modified
- earlier, only replacing the Last-Modified time if it is to a later
- value.
-
- If I am a conditional request, I may modify my response code to
- L{NOT_MODIFIED<http.NOT_MODIFIED>} if appropriate for the time given.
-
- @param when: The last time the resource being returned was modified, in
- seconds since the epoch.
- @type when: L{int}, L{long} or L{float}
-
- @return: If I am a C{If-Modified-Since} conditional request and the time
- given is not newer than the condition, I return
- L{CACHED<http.CACHED>} to indicate that you should write no body.
- Otherwise, I return a false value.
- """
-
-
- def setETag(etag):
- """
- Set an C{entity tag} for the outgoing response.
-
- That's "entity tag" as in the HTTP/1.1 I{ETag} header, "used for
- comparing two or more entities from the same requested resource."
-
- If I am a conditional request, I may modify my response code to
- L{NOT_MODIFIED<http.NOT_MODIFIED>} or
- L{PRECONDITION_FAILED<http.PRECONDITION_FAILED>}, if appropriate for the
- tag given.
-
- @param etag: The entity tag for the resource being returned.
- @type etag: C{str}
-
- @return: If I am a C{If-None-Match} conditional request and the tag
- matches one in the request, I return L{CACHED<http.CACHED>} to
- indicate that you should write no body. Otherwise, I return a
- false value.
- """
-
-
- def setHost(host, port, ssl=0):
- """
- Change the host and port the request thinks it's using.
-
- This method is useful for working with reverse HTTP proxies (e.g. both
- Squid and Apache's mod_proxy can do this), when the address the HTTP
- client is using is different than the one we're listening on.
-
- For example, Apache may be listening on https://www.example.com, and
- then forwarding requests to http://localhost:8080, but we don't want
- HTML produced by Twisted to say 'http://localhost:8080', they should
- say 'https://www.example.com', so we do::
-
- request.setHost('www.example.com', 443, ssl=1)
- """
-
-
-
-class ICredentialFactory(Interface):
- """
- A credential factory defines a way to generate a particular kind of
- authentication challenge and a way to interpret the responses to these
- challenges. It creates
- L{ICredentials<twisted.cred.credentials.ICredentials>} providers from
- responses. These objects will be used with L{twisted.cred} to authenticate
- an authorize requests.
- """
- scheme = Attribute(
- "A C{str} giving the name of the authentication scheme with which "
- "this factory is associated. For example, C{'basic'} or C{'digest'}.")
-
-
- def getChallenge(request):
- """
- Generate a new challenge to be sent to a client.
-
- @type peer: L{twisted.web.http.Request}
- @param peer: The request the response to which this challenge will be
- included.
-
- @rtype: C{dict}
- @return: A mapping from C{str} challenge fields to associated C{str}
- values.
- """
-
-
- def decode(response, request):
- """
- Create a credentials object from the given response.
-
- @type response: C{str}
- @param response: scheme specific response string
-
- @type request: L{twisted.web.http.Request}
- @param request: The request being processed (from which the response
- was taken).
-
- @raise twisted.cred.error.LoginFailed: If the response is invalid.
-
- @rtype: L{twisted.cred.credentials.ICredentials} provider
- @return: The credentials represented by the given response.
- """
-
-
-
-class IBodyProducer(IPushProducer):
- """
- Objects which provide L{IBodyProducer} write bytes to an object which
- provides L{IConsumer<twisted.internet.interfaces.IConsumer>} by calling its
- C{write} method repeatedly.
-
- L{IBodyProducer} providers may start producing as soon as they have an
- L{IConsumer<twisted.internet.interfaces.IConsumer>} provider. That is, they
- should not wait for a C{resumeProducing} call to begin writing data.
-
- L{IConsumer.unregisterProducer<twisted.internet.interfaces.IConsumer.unregisterProducer>}
- must not be called. Instead, the
- L{Deferred<twisted.internet.defer.Deferred>} returned from C{startProducing}
- must be fired when all bytes have been written.
-
- L{IConsumer.write<twisted.internet.interfaces.IConsumer.write>} may
- synchronously invoke any of C{pauseProducing}, C{resumeProducing}, or
- C{stopProducing}. These methods must be implemented with this in mind.
-
- @since: 9.0
- """
-
- # Despite the restrictions above and the additional requirements of
- # stopProducing documented below, this interface still needs to be an
- # IPushProducer subclass. Providers of it will be passed to IConsumer
- # providers which only know about IPushProducer and IPullProducer, not
- # about this interface. This interface needs to remain close enough to one
- # of those interfaces for consumers to work with it.
-
- length = Attribute(
- """
- C{length} is a C{int} indicating how many bytes in total this
- L{IBodyProducer} will write to the consumer or L{UNKNOWN_LENGTH}
- if this is not known in advance.
- """)
-
- def startProducing(consumer):
- """
- Start producing to the given
- L{IConsumer<twisted.internet.interfaces.IConsumer>} provider.
-
- @return: A L{Deferred<twisted.internet.defer.Deferred>} which fires with
- C{None} when all bytes have been produced or with a
- L{Failure<twisted.python.failure.Failure>} if there is any problem
- before all bytes have been produced.
- """
-
-
- def stopProducing():
- """
- In addition to the standard behavior of
- L{IProducer.stopProducing<twisted.internet.interfaces.IProducer.stopProducing>}
- (stop producing data), make sure the
- L{Deferred<twisted.internet.defer.Deferred>} returned by
- C{startProducing} is never fired.
- """
-
-
-
-class IRenderable(Interface):
- """
- An L{IRenderable} is an object that may be rendered by the
- L{twisted.web.template} templating system.
- """
-
- def lookupRenderMethod(name):
- """
- Look up and return the render method associated with the given name.
-
- @type name: C{str}
- @param name: The value of a render directive encountered in the
- document returned by a call to L{IRenderable.render}.
-
- @return: A two-argument callable which will be invoked with the request
- being responded to and the tag object on which the render directive
- was encountered.
- """
-
-
- def render(request):
- """
- Get the document for this L{IRenderable}.
-
- @type request: L{IRequest} provider or C{NoneType}
- @param request: The request in response to which this method is being
- invoked.
-
- @return: An object which can be flattened.
- """
-
-
-
-class ITemplateLoader(Interface):
- """
- A loader for templates; something usable as a value for
- L{twisted.web.template.Element}'s C{loader} attribute.
- """
-
- def load():
- """
- Load a template suitable for rendering.
-
- @return: a C{list} of C{list}s, C{unicode} objects, C{Element}s and
- other L{IRenderable} providers.
- """
-
-
-
-class IResponse(Interface):
- """
- An object representing an HTTP response received from an HTTP server.
-
- @since: 11.1
- """
-
- version = Attribute(
- "A three-tuple describing the protocol and protocol version "
- "of the response. The first element is of type C{str}, the second "
- "and third are of type C{int}. For example, C{('HTTP', 1, 1)}.")
-
-
- code = Attribute("The HTTP status code of this response, as a C{int}.")
-
-
- phrase = Attribute(
- "The HTTP reason phrase of this response, as a C{str}.")
-
-
- headers = Attribute("The HTTP response L{Headers} of this response.")
-
-
- length = Attribute(
- "The C{int} number of bytes expected to be in the body of this "
- "response or L{UNKNOWN_LENGTH} if the server did not indicate how "
- "many bytes to expect. For I{HEAD} responses, this will be 0; if "
- "the response includes a I{Content-Length} header, it will be "
- "available in C{headers}.")
-
-
- def deliverBody(protocol):
- """
- Register an L{IProtocol<twisted.internet.interfaces.IProtocol>} provider
- to receive the response body.
-
- The protocol will be connected to a transport which provides
- L{IPushProducer}. The protocol's C{connectionLost} method will be
- called with:
-
- - ResponseDone, which indicates that all bytes from the response
- have been successfully delivered.
-
- - PotentialDataLoss, which indicates that it cannot be determined
- if the entire response body has been delivered. This only occurs
- when making requests to HTTP servers which do not set
- I{Content-Length} or a I{Transfer-Encoding} in the response.
-
- - ResponseFailed, which indicates that some bytes from the response
- were lost. The C{reasons} attribute of the exception may provide
- more specific indications as to why.
- """
-
-
-
-class _IRequestEncoder(Interface):
- """
- An object encoding data passed to L{IRequest.write}, for example for
- compression purpose.
-
- @since: 12.3
- """
-
- def encode(data):
- """
- Encode the data given and return the result.
-
- @param data: The content to encode.
- @type data: C{str}
-
- @return: The encoded data.
- @rtype: C{str}
- """
-
-
- def finish():
- """
- Callback called when the request is closing.
-
- @return: If necessary, the pending data accumulated from previous
- C{encode} calls.
- @rtype: C{str}
- """
-
-
-
-class _IRequestEncoderFactory(Interface):
- """
- A factory for returing L{_IRequestEncoder} instances.
-
- @since: 12.3
- """
-
- def encoderForRequest(request):
- """
- If applicable, returns a L{_IRequestEncoder} instance which will encode
- the request.
- """
-
-
-
-UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH"
-}}} '''
__all__ = [
"ICredentialFactory", "IRequest",
"IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder",