Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix and re-enable newer mitmproxy usage in tests.

Andrey Rakhmatullin68fccb1d58f2
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index dc0a82086..46d42e9f6 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -31,6 +31,7 @@ sys.exit(mitmdump())
self.proc = Popen(
[
sys.executable,
+ "-u",
"-c",
script,
"--listen-host",
@@ -46,7 +47,7 @@ sys.exit(mitmdump())
stdout=PIPE,
)
line = self.proc.stdout.readline().decode("utf-8")
- host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1)
+ host_port = re.search(r"listening at (?:http://)?([^:]+:\d+)", line).group(1)
address = f"http://{self.auth_user}:{self.auth_pass}@{host_port}"
return address
diff --git a/tox.ini b/tox.ini
index 932c0b805..d9dcacc01 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,11 +11,7 @@ minversion = 1.7.0
deps =
-rtests/requirements.txt
# mitmproxy does not support PyPy
- # Python 3.9+ requires mitmproxy >= 5.3.0
- # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0
- #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
- # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454
- mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'
+ mitmproxy; implementation_name != 'pypy'
passenv =
S3_TEST_FILE_URI
AWS_ACCESS_KEY_ID
@@ -87,7 +83,7 @@ deps =
lxml==4.4.1
-rtests/requirements.txt
- # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
+ # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies
# above, hence we do not install it in pinned environments at the moment
setenv =
_SCRAPY_PINNED=true