Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add a comment about the noconnect tests.

Andrey Rakhmatullinbeb7d80d6a8a
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 2435999f9..277455751 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -80,16 +80,6 @@ class ProxyConnectTestCase(TestCase):
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
self._assert_got_response_code(200, l)
- @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
- @defer.inlineCallbacks
- def test_https_noconnect(self):
- proxy = os.environ['https_proxy']
- os.environ['https_proxy'] = proxy + '?noconnect'
- crawler = get_crawler(SimpleSpider)
- with LogCapture() as l:
- yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
- self._assert_got_response_code(200, l)
-
@pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)
@defer.inlineCallbacks
def test_https_connect_tunnel_error(self):
@@ -118,6 +108,23 @@ class ProxyConnectTestCase(TestCase):
echo = json.loads(crawler.spider.meta['responses'][0].text)
self.assertTrue('Proxy-Authorization' not in echo['headers'])
+ # The noconnect mode isn't supported by the current mitmproxy, it returns
+ # "Invalid request scheme: https" as it doesn't seem to support full URLs in GET at all,
+ # and it's not clear what behavior is intended by Scrapy and by mitmproxy here.
+ # https://github.com/mitmproxy/mitmproxy/issues/848 may be related.
+ # The Scrapy noconnect mode was required, at least in the past, to work with Crawlera,
+ # and https://github.com/scrapy-plugins/scrapy-crawlera/pull/44 seems to be related.
+
+ @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
+ @defer.inlineCallbacks
+ def test_https_noconnect(self):
+ proxy = os.environ['https_proxy']
+ os.environ['https_proxy'] = proxy + '?noconnect'
+ crawler = get_crawler(SimpleSpider)
+ with LogCapture() as l:
+ yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+ self._assert_got_response_code(200, l)
+
@pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
@defer.inlineCallbacks
def test_https_noconnect_auth_error(self):