Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Restore tests/test_proxy_connect.py and update it to modern mitmproxy.

Andrey Rakhmatullinb0d6f4917d78
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index dd5b23cc3..f27e45a54 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,5 +1,6 @@
# Tests requirements
jmespath
+mitmproxy
pytest
pytest-cov
pytest-twisted
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
new file mode 100644
index 000000000..8142d9a41
--- /dev/null
+++ b/tests/test_proxy_connect.py
@@ -0,0 +1,134 @@
+import json
+import os
+import re
+from subprocess import Popen, PIPE
+import sys
+import time
+
+from six.moves.urllib.parse import urlsplit, urlunsplit
+from testfixtures import LogCapture
+
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+
+from scrapy.utils.test import get_crawler
+from scrapy.http import Request
+from tests.spiders import SimpleSpider, SingleRequestSpider
+from tests.mockserver import MockServer
+
+
+class MitmProxy:
+ auth_user = 'scrapy'
+ auth_pass = 'scrapy'
+
+ def start(self):
+ from scrapy.utils.test import get_testenv
+ script = """
+import sys
+from mitmproxy.tools.main import mitmdump
+sys.argv[0] = "mitmdump"
+sys.exit(mitmdump())
+ """
+ cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
+ 'keys', 'mitmproxy-ca.pem')
+ self.proc = Popen([sys.executable,
+ '-c', script,
+ '--listen-host', '127.0.0.1',
+ '--listen-port', '0',
+ '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass),
+ '--certs', cert_path,
+ '--ssl-insecure',
+ ],
+ stdout=PIPE, env=get_testenv())
+ line = self.proc.stdout.readline().decode('utf-8')
+ host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1)
+ address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port)
+ return address
+
+ def stop(self):
+ self.proc.kill()
+ self.proc.wait()
+ time.sleep(0.2)
+
+
+def _wrong_credentials(proxy_url):
+ bad_auth_proxy = list(urlsplit(proxy_url))
+ bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@')
+ return urlunsplit(bad_auth_proxy)
+
+
+class ProxyConnectTestCase(TestCase):
+
+ def setUp(self):
+ self.mockserver = MockServer()
+ self.mockserver.__enter__()
+ self._oldenv = os.environ.copy()
+
+ self._proxy = MitmProxy()
+ proxy_url = self._proxy.start()
+ os.environ['https_proxy'] = proxy_url
+ os.environ['http_proxy'] = proxy_url
+
+ def tearDown(self):
+ self.mockserver.__exit__(None, None, None)
+ self._proxy.stop()
+ os.environ = self._oldenv
+
+ @defer.inlineCallbacks
+ def test_https_connect_tunnel(self):
+ crawler = get_crawler(SimpleSpider)
+ with LogCapture() as l:
+ yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+ self._assert_got_response_code(200, l)
+
+ @defer.inlineCallbacks
+ def test_https_noconnect(self):
+ proxy = os.environ['https_proxy']
+ os.environ['https_proxy'] = proxy + '?noconnect'
+ crawler = get_crawler(SimpleSpider)
+ with LogCapture() as l:
+ yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+ self._assert_got_response_code(200, l)
+
+ @defer.inlineCallbacks
+ def test_https_connect_tunnel_error(self):
+ crawler = get_crawler(SimpleSpider)
+ with LogCapture() as l:
+ yield crawler.crawl("https://localhost:99999/status?n=200")
+ self._assert_got_tunnel_error(l)
+
+ @defer.inlineCallbacks
+ def test_https_tunnel_auth_error(self):
+ os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
+ crawler = get_crawler(SimpleSpider)
+ with LogCapture() as l:
+ yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+ # The proxy returns a 407 error code but it does not reach the client;
+ # he just sees a TunnelError.
+ self._assert_got_tunnel_error(l)
+
+ @defer.inlineCallbacks
+ def test_https_tunnel_without_leak_proxy_authorization_header(self):
+ request = Request(self.mockserver.url("/echo", is_secure=True))
+ crawler = get_crawler(SingleRequestSpider)
+ with LogCapture() as l:
+ yield crawler.crawl(seed=request)
+ self._assert_got_response_code(200, l)
+ echo = json.loads(crawler.spider.meta['responses'][0].body)
+ self.assertTrue('Proxy-Authorization' not in echo['headers'])
+
+ @defer.inlineCallbacks
+ def test_https_noconnect_auth_error(self):
+ os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'
+ crawler = get_crawler(SimpleSpider)
+ with LogCapture() as l:
+ yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+ self._assert_got_response_code(407, l)
+
+ def _assert_got_response_code(self, code, log):
+ print(log)
+ self.assertEqual(str(log).count('Crawled (%d)' % code), 1)
+
+ def _assert_got_tunnel_error(self, log):
+ print(log)
+ self.assertIn('TunnelError', str(log))