Scrapy, a fast high-level web crawling & scraping framework for Python.
check for unparseable no_proxy values
drs-11959222df7e80
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.pyindex 04da11311..d2665b655 100644--- a/scrapy/downloadermiddlewares/httpproxy.py+++ b/scrapy/downloadermiddlewares/httpproxy.py@@ -13,7 +13,12 @@ class HttpProxyMiddleware:self.auth_encoding = auth_encodingself.proxies = {}for type_, url in getproxies().items():- self.proxies[type_] = self._get_proxy(url, type_)+ try:+ self.proxies[type_] = self._get_proxy(url, type_)+ # some values such as '/var/run/docker.sock' can't be parsed+ # by _parse_proxy and as such should be skipped+ except ValueError:+ continue@classmethoddef from_crawler(cls, crawler):diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.pyindex 351631eb8..81d6cc335 100644--- a/tests/test_downloadermiddleware_httpproxy.py+++ b/tests/test_downloadermiddleware_httpproxy.py@@ -123,7 +123,11 @@ class TestHttpProxyMiddleware(TestCase):def test_no_proxy(self):os.environ['http_proxy'] = 'https://proxy.for.http:3128'+ os.environ['no_proxy'] = '/var/run/docker.sock'mw = HttpProxyMiddleware()+ # '/var/run/docker.sock' may be used by the user for+ # no_proxy value but is not parseable and should be skipped+ assert 'no' not in mw.proxiesos.environ['no_proxy'] = '*'req = Request('http://noproxy.com')