Scrapy, a fast high-level web crawling & scraping framework for Python.
Use HTTP pool and proper endpoint key for ProxyAgent
Paul Tremberthc8dc15869745
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 9bfdd803c..a4b077b57 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -14,7 +14,7 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTHfrom twisted.internet.error import TimeoutErrorfrom twisted.web.http import _DataLoss, PotentialDataLossfrom twisted.web.client import Agent, ProxyAgent, ResponseDone, \- HTTPConnectionPool, ResponseFailed+ HTTPConnectionPool, ResponseFailed, URIfrom twisted.internet.endpoints import TCP4ClientEndpointfrom scrapy.http import Headers@@ -228,10 +228,33 @@ class TunnelingAgent(Agent):headers, bodyProducer, requestPath)+class ScrapyProxyAgent(Agent):++ def __init__(self, reactor, proxyURI,+ connectTimeout=None, bindAddress=None, pool=None):+ super(ScrapyProxyAgent, self).__init__(reactor,+ connectTimeout=connectTimeout,+ bindAddress=bindAddress,+ pool=pool)+ self._proxyURI = URI.fromBytes(proxyURI)++ def request(self, method, uri, headers=None, bodyProducer=None):+ """+ Issue a new request via the configured proxy.+ """+ # Cache *all* connections under the same key, since we are only+ # connecting to a single destination, the proxy:+ proxyEndpoint = self._getEndpoint(self._proxyURI)+ key = ("http-proxy", self._proxyURI.host, self._proxyURI.port)+ return self._requestWithEndpoint(key, proxyEndpoint, method,+ URI.fromBytes(uri), headers,+ bodyProducer, uri)++class ScrapyAgent(object):_Agent = Agent- _ProxyAgent = ProxyAgent+ _ProxyAgent = ScrapyProxyAgent_TunnelingAgent = TunnelingAgentdef __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,@@ -260,9 +283,8 @@ class ScrapyAgent(object):contextFactory=self._contextFactory, connectTimeout=timeout,bindAddress=bindaddress, pool=self._pool)else:- endpoint = TCP4ClientEndpoint(reactor, proxyHost, proxyPort,- timeout=timeout, bindAddress=bindaddress)- return self._ProxyAgent(endpoint)+ return self._ProxyAgent(reactor, proxyURI=proxy,+ connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool)return self._Agent(reactor, contextFactory=self._contextFactory,connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool)