Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Use HTTP pool and proper endpoint key for ProxyAgent

Paul Tremberthc8dc15869745
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 9bfdd803c..a4b077b57 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -14,7 +14,7 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
from twisted.internet.error import TimeoutError
from twisted.web.http import _DataLoss, PotentialDataLoss
from twisted.web.client import Agent, ProxyAgent, ResponseDone, \
- HTTPConnectionPool, ResponseFailed
+ HTTPConnectionPool, ResponseFailed, URI
from twisted.internet.endpoints import TCP4ClientEndpoint
from scrapy.http import Headers
@@ -228,10 +228,33 @@ class TunnelingAgent(Agent):
headers, bodyProducer, requestPath)
+class ScrapyProxyAgent(Agent):
+
+ def __init__(self, reactor, proxyURI,
+ connectTimeout=None, bindAddress=None, pool=None):
+ super(ScrapyProxyAgent, self).__init__(reactor,
+ connectTimeout=connectTimeout,
+ bindAddress=bindAddress,
+ pool=pool)
+ self._proxyURI = URI.fromBytes(proxyURI)
+
+ def request(self, method, uri, headers=None, bodyProducer=None):
+ """
+ Issue a new request via the configured proxy.
+ """
+ # Cache *all* connections under the same key, since we are only
+ # connecting to a single destination, the proxy:
+ proxyEndpoint = self._getEndpoint(self._proxyURI)
+ key = ("http-proxy", self._proxyURI.host, self._proxyURI.port)
+ return self._requestWithEndpoint(key, proxyEndpoint, method,
+ URI.fromBytes(uri), headers,
+ bodyProducer, uri)
+
+
class ScrapyAgent(object):
_Agent = Agent
- _ProxyAgent = ProxyAgent
+ _ProxyAgent = ScrapyProxyAgent
_TunnelingAgent = TunnelingAgent
def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,
@@ -260,9 +283,8 @@ class ScrapyAgent(object):
contextFactory=self._contextFactory, connectTimeout=timeout,
bindAddress=bindaddress, pool=self._pool)
else:
- endpoint = TCP4ClientEndpoint(reactor, proxyHost, proxyPort,
- timeout=timeout, bindAddress=bindaddress)
- return self._ProxyAgent(endpoint)
+ return self._ProxyAgent(reactor, proxyURI=proxy,
+ connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool)
return self._Agent(reactor, contextFactory=self._contextFactory,
connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool)