Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

check for unparseable no_proxy values

drs-11959222df7e80
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 04da11311..d2665b655 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -13,7 +13,12 @@ class HttpProxyMiddleware:
self.auth_encoding = auth_encoding
self.proxies = {}
for type_, url in getproxies().items():
- self.proxies[type_] = self._get_proxy(url, type_)
+ try:
+ self.proxies[type_] = self._get_proxy(url, type_)
+ # some values such as '/var/run/docker.sock' can't be parsed
+ # by _parse_proxy and as such should be skipped
+ except ValueError:
+ continue
@classmethod
def from_crawler(cls, crawler):
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 351631eb8..81d6cc335 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -123,7 +123,11 @@ class TestHttpProxyMiddleware(TestCase):
def test_no_proxy(self):
os.environ['http_proxy'] = 'https://proxy.for.http:3128'
+ os.environ['no_proxy'] = '/var/run/docker.sock'
mw = HttpProxyMiddleware()
+ # '/var/run/docker.sock' may be used by the user for
+ # no_proxy value but is not parseable and should be skipped
+ assert 'no' not in mw.proxies
os.environ['no_proxy'] = '*'
req = Request('http://noproxy.com')