Scrapy, a fast high-level web crawling & scraping framework for Python.
Use monkeypatch for os.environ manipulations. (#8236)
Andrey Rakhmatulline2c18d9dc2c6
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.pyindex a7a6f2079..d4f6c4796 100644--- a/tests/test_downloadermiddleware_httpproxy.py+++ b/tests/test_downloadermiddleware_httpproxy.py@@ -1,3 +1,5 @@+from __future__ import annotations+import osimport pytest@@ -11,21 +13,15 @@ from scrapy.utils.test import get_crawlerclass TestHttpProxyMiddleware:- def setup_method(self):- self._oldenv = os.environ.copy()-- def teardown_method(self):- os.environ.clear()- os.environ.update(self._oldenv)-def test_not_enabled(self):crawler = get_crawler(Spider, {"HTTPPROXY_ENABLED": False})with pytest.raises(NotConfigured):build_from_crawler(HttpProxyMiddleware, crawler)- def test_no_environment_proxies(self):- os.environ.clear()- os.environ["dummy_proxy"] = "reset_env_and_do_not_raise"+ def test_no_environment_proxies(self, monkeypatch: pytest.MonkeyPatch) -> None:+ for name in list(os.environ):+ monkeypatch.delenv(name)+ monkeypatch.setenv("dummy_proxy", "reset_env_and_do_not_raise")mw = HttpProxyMiddleware()for url in ("http://e.com", "https://e.com", "file:///tmp/a"):@@ -34,10 +30,12 @@ class TestHttpProxyMiddleware:assert req.url == urlassert req.meta == {}- def test_environment_proxies(self):- os.environ["http_proxy"] = http_proxy = "https://proxy.for.http:3128"- os.environ["https_proxy"] = https_proxy = "http://proxy.for.https:8080"- os.environ.pop("file_proxy", None)+ def test_environment_proxies(self, monkeypatch: pytest.MonkeyPatch) -> None:+ http_proxy = "https://proxy.for.http:3128"+ https_proxy = "http://proxy.for.https:8080"+ monkeypatch.setenv("http_proxy", http_proxy)+ monkeypatch.setenv("https_proxy", https_proxy)+ monkeypatch.delenv("file_proxy", raising=False)mw = HttpProxyMiddleware()for url, proxy in [@@ -50,15 +48,15 @@ class TestHttpProxyMiddleware:assert req.url == urlassert req.meta.get("proxy") == proxy- def test_proxy_precedence_meta(self):- os.environ["http_proxy"] = "https://proxy.com"+ def test_proxy_precedence_meta(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("http_proxy", "https://proxy.com")mw = HttpProxyMiddleware()req = Request("http://scrapytest.org", meta={"proxy": "https://new.proxy:3128"})assert mw.process_request(req) is Noneassert req.meta == {"proxy": "https://new.proxy:3128"}- def test_proxy_auth(self):- os.environ["http_proxy"] = "https://user:pass@proxy:3128"+ def test_proxy_auth(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("http_proxy", "https://user:pass@proxy:3128")mw = HttpProxyMiddleware()req = Request("http://scrapytest.org")assert mw.process_request(req) is None@@ -75,8 +73,8 @@ class TestHttpProxyMiddleware:req.headers.get("Proxy-Authorization") == b"Basic dXNlcm5hbWU6cGFzc3dvcmQ=")- def test_proxy_auth_empty_passwd(self):- os.environ["http_proxy"] = "https://user:@proxy:3128"+ def test_proxy_auth_empty_passwd(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("http_proxy", "https://user:@proxy:3128")mw = HttpProxyMiddleware()req = Request("http://scrapytest.org")assert mw.process_request(req) is None@@ -90,9 +88,9 @@ class TestHttpProxyMiddleware:assert req.meta["proxy"] == "https://proxy:3128"assert req.headers.get("Proxy-Authorization") == b"Basic dXNlcm5hbWU6"- def test_proxy_auth_encoding(self):+ def test_proxy_auth_encoding(self, monkeypatch: pytest.MonkeyPatch) -> None:# utf-8 encoding- os.environ["http_proxy"] = "https://m\u00e1n:pass@proxy:3128"+ monkeypatch.setenv("http_proxy", "https://m\u00e1n:pass@proxy:3128")mw = HttpProxyMiddleware(auth_encoding="utf-8")req = Request("http://scrapytest.org")assert mw.process_request(req) is None@@ -122,41 +120,41 @@ class TestHttpProxyMiddleware:assert req.meta["proxy"] == "https://proxy:3128"assert req.headers.get("Proxy-Authorization") == b"Basic /HNlcjpwYXNz"- def test_proxy_already_seted(self):- os.environ["http_proxy"] = "https://proxy.for.http:3128"+ def test_proxy_already_seted(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("http_proxy", "https://proxy.for.http:3128")mw = HttpProxyMiddleware()req = Request("http://noproxy.com", meta={"proxy": None})assert mw.process_request(req) is Noneassert "proxy" in req.metaassert req.meta["proxy"] is None- def test_no_proxy(self):- os.environ["http_proxy"] = "https://proxy.for.http:3128"+ def test_no_proxy(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("http_proxy", "https://proxy.for.http:3128")mw = HttpProxyMiddleware()- os.environ["no_proxy"] = "*"+ monkeypatch.setenv("no_proxy", "*")req = Request("http://noproxy.com")assert mw.process_request(req) is Noneassert "proxy" not in req.meta- os.environ["no_proxy"] = "other.com"+ monkeypatch.setenv("no_proxy", "other.com")req = Request("http://noproxy.com")assert mw.process_request(req) is Noneassert "proxy" in req.meta- os.environ["no_proxy"] = "other.com,noproxy.com"+ monkeypatch.setenv("no_proxy", "other.com,noproxy.com")req = Request("http://noproxy.com")assert mw.process_request(req) is Noneassert "proxy" not in req.meta# proxy from meta['proxy'] takes precedence- os.environ["no_proxy"] = "*"+ monkeypatch.setenv("no_proxy", "*")req = Request("http://noproxy.com", meta={"proxy": "http://proxy.com"})assert mw.process_request(req) is Noneassert req.meta == {"proxy": "http://proxy.com"}- def test_no_proxy_invalid_values(self):- os.environ["no_proxy"] = "/var/run/docker.sock"+ def test_no_proxy_invalid_values(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("no_proxy", "/var/run/docker.sock")mw = HttpProxyMiddleware()# '/var/run/docker.sock' may be used by the user for# no_proxy value but is not parseable and should be skippeddiff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.pyindex b12942ccc..a5c7cc920 100644--- a/tests/test_utils_misc/__init__.py+++ b/tests/test_utils_misc/__init__.py@@ -148,13 +148,13 @@ def test_build_from_crawler() -> None:build_from_crawler(m, crawler, *args, **kwargs)-def test_set_environ() -> None:+def test_set_environ(monkeypatch: pytest.MonkeyPatch) -> None:assert os.environ.get("some_test_environ") is Nonewith set_environ(some_test_environ="test_value"):assert os.environ.get("some_test_environ") == "test_value"assert os.environ.get("some_test_environ") is None- os.environ["some_test_environ"] = "test"+ monkeypatch.setenv("some_test_environ", "test")assert os.environ.get("some_test_environ") == "test"with set_environ(some_test_environ="test_value"):assert os.environ.get("some_test_environ") == "test_value"