Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deduplicate the proxy redirect tests. (#8232)

Andrey Rakhmatullinde41ace8573b
diff --git a/tests/utils/bases/redirect.py b/tests/utils/bases/redirect.py
index 63e271ec9..85a771dd2 100644
--- a/tests/utils/bases/redirect.py
+++ b/tests/utils/bases/redirect.py
@@ -11,6 +11,29 @@ from scrapy.http import Request, Response
from scrapy.utils.misc import build_from_crawler, set_environ
from scrapy.utils.test import get_crawler
+PROXY_A = "https://a:@a.example"
+PROXY_B = "https://b:@b.example"
+
+# Proxy ID → (expected Proxy-Authorization header, expected proxy URL).
+PROXIES = {
+ "a": (b"Basic YTo=", "https://a.example"),
+ "b": (b"Basic Yjo=", "https://b.example"),
+}
+
+
+def assert_proxy(request: Request, expected: str | None) -> None:
+ """Check that *request* uses the proxy with the *expected* ID, or no proxy
+ at all if *expected* is ``None``."""
+ if expected is None:
+ assert "Proxy-Authorization" not in request.headers
+ assert "_auth_proxy" not in request.meta
+ assert "proxy" not in request.meta
+ return
+ auth, url = PROXIES[expected]
+ assert request.headers["Proxy-Authorization"] == auth
+ assert request.meta["_auth_proxy"] == url
+ assert request.meta["proxy"] == url
+
class TestRedirectBase(ABC):
mwcls: type[Any]
@@ -219,770 +242,230 @@ class TestRedirectBase(ABC):
assert isinstance(downgrade_redirect_request, Request)
assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict()
- def test_meta_proxy_http_absolute(self):
+ def _check_proxy_scenario(
+ self,
+ *,
+ env: dict[str, str],
+ meta: dict[str, Any],
+ url: str,
+ location1: str,
+ location2: str,
+ expected: tuple[str | None, ...],
+ ) -> None:
+ """Send a request through 2 redirects and check its proxy state at 5
+ points: after the first HttpProxyMiddleware.process_request() call and,
+ for each redirect, after the redirect middleware builds the redirected
+ request and after HttpProxyMiddleware.process_request() runs on it.
+
+ Each item of *expected* is a key of PROXIES, or ``None`` for no proxy.
+ """
crawler = get_crawler()
redirect_mw = build_from_crawler(self.mwcls, crawler)
- proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
+ with set_environ(**env):
+ proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
- meta = {"proxy": "https://a:@a.example"}
- request1 = Request("http://example.com", meta=meta)
+ request1 = Request(url, meta=meta)
proxy_mw.process_request(request1)
+ assert_proxy(request1, expected[0])
- assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request1.meta["_auth_proxy"] == "https://a.example"
- assert request1.meta["proxy"] == "https://a.example"
-
- response1 = self.get_response(request1, "http://example.com")
+ response1 = self.get_response(request1, location1)
request2 = redirect_mw.process_response(request1, response1)
-
assert isinstance(request2, Request)
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
+ assert_proxy(request2, expected[1])
proxy_mw.process_request(request2)
+ assert_proxy(request2, expected[2])
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- response2 = self.get_response(request2, "http://example.com")
+ response2 = self.get_response(request2, location2)
request3 = redirect_mw.process_response(request2, response2)
-
assert isinstance(request3, Request)
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
-
+ assert_proxy(request3, expected[3])
proxy_mw.process_request(request3)
+ assert_proxy(request3, expected[4])
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
-
- def test_meta_proxy_http_relative(self):
- crawler = get_crawler()
- redirect_mw = build_from_crawler(self.mwcls, crawler)
- proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
-
- meta = {"proxy": "https://a:@a.example"}
- request1 = Request("http://example.com", meta=meta)
- proxy_mw.process_request(request1)
-
- assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request1.meta["_auth_proxy"] == "https://a.example"
- assert request1.meta["proxy"] == "https://a.example"
-
- response1 = self.get_response(request1, "/a")
- request2 = redirect_mw.process_response(request1, response1)
-
- assert isinstance(request2, Request)
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- proxy_mw.process_request(request2)
-
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
+ # A proxy set through Request.meta is kept on redirects, even on
+ # cross-scheme ones.
- response2 = self.get_response(request2, "/a")
- request3 = redirect_mw.process_response(request2, response2)
-
- assert isinstance(request3, Request)
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
-
- proxy_mw.process_request(request3)
+ def test_meta_proxy_http_absolute(self):
+ self._check_proxy_scenario(
+ env={},
+ meta={"proxy": PROXY_A},
+ url="http://example.com",
+ location1="http://example.com",
+ location2="http://example.com",
+ expected=("a", "a", "a", "a", "a"),
+ )
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
+ def test_meta_proxy_http_relative(self):
+ self._check_proxy_scenario(
+ env={},
+ meta={"proxy": PROXY_A},
+ url="http://example.com",
+ location1="/a",
+ location2="/a",
+ expected=("a", "a", "a", "a", "a"),
+ )
def test_meta_proxy_https_absolute(self):
- crawler = get_crawler()
- redirect_mw = build_from_crawler(self.mwcls, crawler)
- proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
-
- meta = {"proxy": "https://a:@a.example"}
- request1 = Request("https://example.com", meta=meta)
- proxy_mw.process_request(request1)
-
- assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request1.meta["_auth_proxy"] == "https://a.example"
- assert request1.meta["proxy"] == "https://a.example"
-
- response1 = self.get_response(request1, "https://example.com")
- request2 = redirect_mw.process_response(request1, response1)
-
- assert isinstance(request2, Request)
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- proxy_mw.process_request(request2)
-
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- response2 = self.get_response(request2, "https://example.com")
- request3 = redirect_mw.process_response(request2, response2)
-
- assert isinstance(request3, Request)
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
-
- proxy_mw.process_request(request3)
-
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
+ self._check_proxy_scenario(
+ env={},
+ meta={"proxy": PROXY_A},
+ url="https://example.com",
+ location1="https://example.com",
+ location2="https://example.com",
+ expected=("a", "a", "a", "a", "a"),
+ )
def test_meta_proxy_https_relative(self):
- crawler = get_crawler()
- redirect_mw = build_from_crawler(self.mwcls, crawler)
- proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
-
- meta = {"proxy": "https://a:@a.example"}
- request1 = Request("https://example.com", meta=meta)
- proxy_mw.process_request(request1)
-
- assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request1.meta["_auth_proxy"] == "https://a.example"
- assert request1.meta["proxy"] == "https://a.example"
-
- response1 = self.get_response(request1, "/a")
- request2 = redirect_mw.process_response(request1, response1)
-
- assert isinstance(request2, Request)
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- proxy_mw.process_request(request2)
-
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- response2 = self.get_response(request2, "/a")
- request3 = redirect_mw.process_response(request2, response2)
-
- assert isinstance(request3, Request)
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
-
- proxy_mw.process_request(request3)
-
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
+ self._check_proxy_scenario(
+ env={},
+ meta={"proxy": PROXY_A},
+ url="https://example.com",
+ location1="/a",
+ location2="/a",
+ expected=("a", "a", "a", "a", "a"),
+ )
def test_meta_proxy_http_to_https(self):
- crawler = get_crawler()
- redirect_mw = build_from_crawler(self.mwcls, crawler)
- proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
-
- meta = {"proxy": "https://a:@a.example"}
- request1 = Request("http://example.com", meta=meta)
- proxy_mw.process_request(request1)
-
- assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request1.meta["_auth_proxy"] == "https://a.example"
- assert request1.meta["proxy"] == "https://a.example"
-
- response1 = self.get_response(request1, "https://example.com")
- request2 = redirect_mw.process_response(request1, response1)
-
- assert isinstance(request2, Request)
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- proxy_mw.process_request(request2)
-
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- response2 = self.get_response(request2, "http://example.com")
- request3 = redirect_mw.process_response(request2, response2)
-
- assert isinstance(request3, Request)
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
-
- proxy_mw.process_request(request3)
-
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
+ self._check_proxy_scenario(
+ env={},
+ meta={"proxy": PROXY_A},
+ url="http://example.com",
+ location1="https://example.com",
+ location2="http://example.com",
+ expected=("a", "a", "a", "a", "a"),
+ )
def test_meta_proxy_https_to_http(self):
- crawler = get_crawler()
- redirect_mw = build_from_crawler(self.mwcls, crawler)
- proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
-
- meta = {"proxy": "https://a:@a.example"}
- request1 = Request("https://example.com", meta=meta)
- proxy_mw.process_request(request1)
-
- assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request1.meta["_auth_proxy"] == "https://a.example"
- assert request1.meta["proxy"] == "https://a.example"
-
- response1 = self.get_response(request1, "http://example.com")
- request2 = redirect_mw.process_response(request1, response1)
-
- assert isinstance(request2, Request)
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- proxy_mw.process_request(request2)
-
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- response2 = self.get_response(request2, "https://example.com")
- request3 = redirect_mw.process_response(request2, response2)
-
- assert isinstance(request3, Request)
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
-
- proxy_mw.process_request(request3)
+ self._check_proxy_scenario(
+ env={},
+ meta={"proxy": PROXY_A},
+ url="https://example.com",
+ location1="http://example.com",
+ location2="https://example.com",
+ expected=("a", "a", "a", "a", "a"),
+ )
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
+ # A proxy set from the environment is scheme-specific, so a cross-scheme
+ # redirect drops it and HttpProxyMiddleware then sets the proxy of the new
+ # scheme, if any. Cross-scheme scenarios use proxy a for http and proxy b
+ # for https.
def test_system_proxy_http_absolute(self):
- crawler = get_crawler()
- redirect_mw = build_from_crawler(self.mwcls, crawler)
- env = {
- "http_proxy": "https://a:@a.example",
- }
- with set_environ(**env):
- proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
-
- request1 = Request("http://example.com")
- proxy_mw.process_request(request1)
-
- assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request1.meta["_auth_proxy"] == "https://a.example"
- assert request1.meta["proxy"] == "https://a.example"
-
- response1 = self.get_response(request1, "http://example.com")
- request2 = redirect_mw.process_response(request1, response1)
-
- assert isinstance(request2, Request)
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- proxy_mw.process_request(request2)
-
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- response2 = self.get_response(request2, "http://example.com")
- request3 = redirect_mw.process_response(request2, response2)
-
- assert isinstance(request3, Request)
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
-
- proxy_mw.process_request(request3)
-
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
+ self._check_proxy_scenario(
+ env={"http_proxy": PROXY_A},
+ meta={},
+ url="http://example.com",
+ location1="http://example.com",
+ location2="http://example.com",
+ expected=("a", "a", "a", "a", "a"),
+ )
def test_system_proxy_http_relative(self):
- crawler = get_crawler()
- redirect_mw = build_from_crawler(self.mwcls, crawler)
- env = {
- "http_proxy": "https://a:@a.example",
- }
- with set_environ(**env):
- proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
-
- request1 = Request("http://example.com")
- proxy_mw.process_request(request1)
-
- assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request1.meta["_auth_proxy"] == "https://a.example"
- assert request1.meta["proxy"] == "https://a.example"
-
- response1 = self.get_response(request1, "/a")
- request2 = redirect_mw.process_response(request1, response1)
-
- assert isinstance(request2, Request)
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- proxy_mw.process_request(request2)
-
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- response2 = self.get_response(request2, "/a")
- request3 = redirect_mw.process_response(request2, response2)
-
- assert isinstance(request3, Request)
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
-
- proxy_mw.process_request(request3)
-
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
+ self._check_proxy_scenario(
+ env={"http_proxy": PROXY_A},
+ meta={},
+ url="http://example.com",
+ location1="/a",
+ location2="/a",
+ expected=("a", "a", "a", "a", "a"),
+ )
def test_system_proxy_https_absolute(self):
- crawler = get_crawler()
- redirect_mw = build_from_crawler(self.mwcls, crawler)
- env = {
- "https_proxy": "https://a:@a.example",
- }
- with set_environ(**env):
- proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
-
- request1 = Request("https://example.com")
- proxy_mw.process_request(request1)
-
- assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request1.meta["_auth_proxy"] == "https://a.example"
- assert request1.meta["proxy"] == "https://a.example"
-
- response1 = self.get_response(request1, "https://example.com")
- request2 = redirect_mw.process_response(request1, response1)
-
- assert isinstance(request2, Request)
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- proxy_mw.process_request(request2)
-
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- response2 = self.get_response(request2, "https://example.com")
- request3 = redirect_mw.process_response(request2, response2)
-
- assert isinstance(request3, Request)
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
-
- proxy_mw.process_request(request3)
-
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
+ self._check_proxy_scenario(
+ env={"https_proxy": PROXY_A},
+ meta={},
+ url="https://example.com",
+ location1="https://example.com",
+ location2="https://example.com",
+ expected=("a", "a", "a", "a", "a"),
+ )
def test_system_proxy_https_relative(self):
- crawler = get_crawler()
- redirect_mw = build_from_crawler(self.mwcls, crawler)
- env = {
- "https_proxy": "https://a:@a.example",
- }
- with set_environ(**env):
- proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
-
- request1 = Request("https://example.com")
- proxy_mw.process_request(request1)
-
- assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request1.meta["_auth_proxy"] == "https://a.example"
- assert request1.meta["proxy"] == "https://a.example"
-
- response1 = self.get_response(request1, "/a")
- request2 = redirect_mw.process_response(request1, response1)
-
- assert isinstance(request2, Request)
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- proxy_mw.process_request(request2)
-
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- response2 = self.get_response(request2, "/a")
- request3 = redirect_mw.process_response(request2, response2)
-
- assert isinstance(request3, Request)
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
-
- proxy_mw.process_request(request3)
-
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
+ self._check_proxy_scenario(
+ env={"https_proxy": PROXY_A},
+ meta={},
+ url="https://example.com",
+ location1="/a",
+ location2="/a",
+ expected=("a", "a", "a", "a", "a"),
+ )
def test_system_proxy_proxied_http_to_proxied_https(self):
- crawler = get_crawler()
- redirect_mw = build_from_crawler(self.mwcls, crawler)
- env = {
- "http_proxy": "https://a:@a.example",
- "https_proxy": "https://b:@b.example",
- }
- with set_environ(**env):
- proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
-
- request1 = Request("http://example.com")
- proxy_mw.process_request(request1)
-
- assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request1.meta["_auth_proxy"] == "https://a.example"
- assert request1.meta["proxy"] == "https://a.example"
-
- response1 = self.get_response(request1, "https://example.com")
- request2 = redirect_mw.process_response(request1, response1)
-
- assert isinstance(request2, Request)
- assert "Proxy-Authorization" not in request2.headers
- assert "_auth_proxy" not in request2.meta
- assert "proxy" not in request2.meta
-
- proxy_mw.process_request(request2)
-
- assert request2.headers["Proxy-Authorization"] == b"Basic Yjo="
- assert request2.meta["_auth_proxy"] == "https://b.example"
- assert request2.meta["proxy"] == "https://b.example"
-
- response2 = self.get_response(request2, "http://example.com")
- request3 = redirect_mw.process_response(request2, response2)
-
- assert isinstance(request3, Request)
- assert "Proxy-Authorization" not in request3.headers
- assert "_auth_proxy" not in request3.meta
- assert "proxy" not in request3.meta
-
- proxy_mw.process_request(request3)
-
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
+ self._check_proxy_scenario(
+ env={"http_proxy": PROXY_A, "https_proxy": PROXY_B},
+ meta={},
+ url="http://example.com",
+ location1="https://example.com",
+ location2="http://example.com",
+ expected=("a", None, "b", None, "a"),
+ )
def test_system_proxy_proxied_http_to_unproxied_https(self):
- crawler = get_crawler()
- redirect_mw = build_from_crawler(self.mwcls, crawler)
- env = {
- "http_proxy": "https://a:@a.example",
- }
- with set_environ(**env):
- proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
-
- request1 = Request("http://example.com")
- proxy_mw.process_request(request1)
-
- assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request1.meta["_auth_proxy"] == "https://a.example"
- assert request1.meta["proxy"] == "https://a.example"
-
- response1 = self.get_response(request1, "https://example.com")
- request2 = redirect_mw.process_response(request1, response1)
-
- assert isinstance(request2, Request)
- assert "Proxy-Authorization" not in request2.headers
- assert "_auth_proxy" not in request2.meta
- assert "proxy" not in request2.meta
-
- proxy_mw.process_request(request2)
-
- assert "Proxy-Authorization" not in request2.headers
- assert "_auth_proxy" not in request2.meta
- assert "proxy" not in request2.meta
-
- response2 = self.get_response(request2, "http://example.com")
- request3 = redirect_mw.process_response(request2, response2)
-
- assert isinstance(request3, Request)
- assert "Proxy-Authorization" not in request3.headers
- assert "_auth_proxy" not in request3.meta
- assert "proxy" not in request3.meta
-
- proxy_mw.process_request(request3)
-
- assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request3.meta["_auth_proxy"] == "https://a.example"
- assert request3.meta["proxy"] == "https://a.example"
+ self._check_proxy_scenario(
+ env={"http_proxy": PROXY_A},
+ meta={},
+ url="http://example.com",
+ location1="https://example.com",
+ location2="http://example.com",
+ expected=("a", None, None, None, "a"),
+ )
def test_system_proxy_unproxied_http_to_proxied_https(self):
- crawler = get_crawler()
- redirect_mw = build_from_crawler(self.mwcls, crawler)
- env = {
- "https_proxy": "https://b:@b.example",
- }
- with set_environ(**env):
- proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
-
- request1 = Request("http://example.com")
- proxy_mw.process_request(request1)
-
- assert "Proxy-Authorization" not in request1.headers
- assert "_auth_proxy" not in request1.meta
- assert "proxy" not in request1.meta
-
- response1 = self.get_response(request1, "https://example.com")
- request2 = redirect_mw.process_response(request1, response1)
-
- assert isinstance(request2, Request)
- assert "Proxy-Authorization" not in request2.headers
- assert "_auth_proxy" not in request2.meta
- assert "proxy" not in request2.meta
-
- proxy_mw.process_request(request2)
-
- assert request2.headers["Proxy-Authorization"] == b"Basic Yjo="
- assert request2.meta["_auth_proxy"] == "https://b.example"
- assert request2.meta["proxy"] == "https://b.example"
-
- response2 = self.get_response(request2, "http://example.com")
- request3 = redirect_mw.process_response(request2, response2)
-
- assert isinstance(request3, Request)
- assert "Proxy-Authorization" not in request3.headers
- assert "_auth_proxy" not in request3.meta
- assert "proxy" not in request3.meta
-
- proxy_mw.process_request(request3)
-
- assert "Proxy-Authorization" not in request3.headers
- assert "_auth_proxy" not in request3.meta
- assert "proxy" not in request3.meta
+ self._check_proxy_scenario(
+ env={"https_proxy": PROXY_B},
+ meta={},
+ url="http://example.com",
+ location1="https://example.com",
+ location2="http://example.com",
+ expected=(None, None, "b", None, None),
+ )
def test_system_proxy_unproxied_http_to_unproxied_https(self):
- crawler = get_crawler()
- redirect_mw = build_from_crawler(self.mwcls, crawler)
- proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
-
- request1 = Request("http://example.com")
- proxy_mw.process_request(request1)
-
- assert "Proxy-Authorization" not in request1.headers
- assert "_auth_proxy" not in request1.meta
- assert "proxy" not in request1.meta
-
- response1 = self.get_response(request1, "https://example.com")
- request2 = redirect_mw.process_response(request1, response1)
-
- assert isinstance(request2, Request)
- assert "Proxy-Authorization" not in request2.headers
- assert "_auth_proxy" not in request2.meta
- assert "proxy" not in request2.meta
-
- proxy_mw.process_request(request2)
-
- assert "Proxy-Authorization" not in request2.headers
- assert "_auth_proxy" not in request2.meta
- assert "proxy" not in request2.meta
-
- response2 = self.get_response(request2, "http://example.com")
- request3 = redirect_mw.process_response(request2, response2)
-
- assert isinstance(request3, Request)
- assert "Proxy-Authorization" not in request3.headers
- assert "_auth_proxy" not in request3.meta
- assert "proxy" not in request3.meta
-
- proxy_mw.process_request(request3)
-
- assert "Proxy-Authorization" not in request3.headers
- assert "_auth_proxy" not in request3.meta
- assert "proxy" not in request3.meta
+ self._check_proxy_scenario(
+ env={},
+ meta={},
+ url="http://example.com",
+ location1="https://example.com",
+ location2="http://example.com",
+ expected=(None, None, None, None, None),
+ )
def test_system_proxy_proxied_https_to_proxied_http(self):
- crawler = get_crawler()
- redirect_mw = build_from_crawler(self.mwcls, crawler)
- env = {
- "http_proxy": "https://a:@a.example",
- "https_proxy": "https://b:@b.example",
- }
- with set_environ(**env):
- proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
-
- request1 = Request("https://example.com")
- proxy_mw.process_request(request1)
-
- assert request1.headers["Proxy-Authorization"] == b"Basic Yjo="
- assert request1.meta["_auth_proxy"] == "https://b.example"
- assert request1.meta["proxy"] == "https://b.example"
-
- response1 = self.get_response(request1, "http://example.com")
- request2 = redirect_mw.process_response(request1, response1)
-
- assert isinstance(request2, Request)
- assert "Proxy-Authorization" not in request2.headers
- assert "_auth_proxy" not in request2.meta
- assert "proxy" not in request2.meta
-
- proxy_mw.process_request(request2)
-
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- response2 = self.get_response(request2, "https://example.com")
- request3 = redirect_mw.process_response(request2, response2)
-
- assert isinstance(request3, Request)
- assert "Proxy-Authorization" not in request3.headers
- assert "_auth_proxy" not in request3.meta
- assert "proxy" not in request3.meta
-
- proxy_mw.process_request(request3)
-
- assert request3.headers["Proxy-Authorization"] == b"Basic Yjo="
- assert request3.meta["_auth_proxy"] == "https://b.example"
- assert request3.meta["proxy"] == "https://b.example"
+ self._check_proxy_scenario(
+ env={"http_proxy": PROXY_A, "https_proxy": PROXY_B},
+ meta={},
+ url="https://example.com",
+ location1="http://example.com",
+ location2="https://example.com",
+ expected=("b", None, "a", None, "b"),
+ )
def test_system_proxy_proxied_https_to_unproxied_http(self):
- crawler = get_crawler()
- redirect_mw = build_from_crawler(self.mwcls, crawler)
- env = {
- "https_proxy": "https://b:@b.example",
- }
- with set_environ(**env):
- proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
-
- request1 = Request("https://example.com")
- proxy_mw.process_request(request1)
-
- assert request1.headers["Proxy-Authorization"] == b"Basic Yjo="
- assert request1.meta["_auth_proxy"] == "https://b.example"
- assert request1.meta["proxy"] == "https://b.example"
-
- response1 = self.get_response(request1, "http://example.com")
- request2 = redirect_mw.process_response(request1, response1)
-
- assert isinstance(request2, Request)
- assert "Proxy-Authorization" not in request2.headers
- assert "_auth_proxy" not in request2.meta
- assert "proxy" not in request2.meta
-
- proxy_mw.process_request(request2)
-
- assert "Proxy-Authorization" not in request2.headers
- assert "_auth_proxy" not in request2.meta
- assert "proxy" not in request2.meta
-
- response2 = self.get_response(request2, "https://example.com")
- request3 = redirect_mw.process_response(request2, response2)
-
- assert isinstance(request3, Request)
- assert "Proxy-Authorization" not in request3.headers
- assert "_auth_proxy" not in request3.meta
- assert "proxy" not in request3.meta
-
- proxy_mw.process_request(request3)
-
- assert request3.headers["Proxy-Authorization"] == b"Basic Yjo="
- assert request3.meta["_auth_proxy"] == "https://b.example"
- assert request3.meta["proxy"] == "https://b.example"
+ self._check_proxy_scenario(
+ env={"https_proxy": PROXY_B},
+ meta={},
+ url="https://example.com",
+ location1="http://example.com",
+ location2="https://example.com",
+ expected=("b", None, None, None, "b"),
+ )
def test_system_proxy_unproxied_https_to_proxied_http(self):
- crawler = get_crawler()
- redirect_mw = build_from_crawler(self.mwcls, crawler)
- env = {
- "http_proxy": "https://a:@a.example",
- }
- with set_environ(**env):
- proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
-
- request1 = Request("https://example.com")
- proxy_mw.process_request(request1)
-
- assert "Proxy-Authorization" not in request1.headers
- assert "_auth_proxy" not in request1.meta
- assert "proxy" not in request1.meta
-
- response1 = self.get_response(request1, "http://example.com")
- request2 = redirect_mw.process_response(request1, response1)
-
- assert isinstance(request2, Request)
- assert "Proxy-Authorization" not in request2.headers
- assert "_auth_proxy" not in request2.meta
- assert "proxy" not in request2.meta
-
- proxy_mw.process_request(request2)
-
- assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
- assert request2.meta["_auth_proxy"] == "https://a.example"
- assert request2.meta["proxy"] == "https://a.example"
-
- response2 = self.get_response(request2, "https://example.com")
- request3 = redirect_mw.process_response(request2, response2)
-
- assert isinstance(request3, Request)
- assert "Proxy-Authorization" not in request3.headers
- assert "_auth_proxy" not in request3.meta
- assert "proxy" not in request3.meta
-
- proxy_mw.process_request(request3)
-
- assert "Proxy-Authorization" not in request3.headers
- assert "_auth_proxy" not in request3.meta
- assert "proxy" not in request3.meta
+ self._check_proxy_scenario(
+ env={"http_proxy": PROXY_A},
+ meta={},
+ url="https://example.com",
+ location1="http://example.com",
+ location2="https://example.com",
+ expected=(None, None, "a", None, None),
+ )
def test_system_proxy_unproxied_https_to_unproxied_http(self):
- crawler = get_crawler()
- redirect_mw = build_from_crawler(self.mwcls, crawler)
- proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
-
- request1 = Request("https://example.com")
- proxy_mw.process_request(request1)
-
- assert "Proxy-Authorization" not in request1.headers
- assert "_auth_proxy" not in request1.meta
- assert "proxy" not in request1.meta
-
- response1 = self.get_response(request1, "http://example.com")
- request2 = redirect_mw.process_response(request1, response1)
-
- assert isinstance(request2, Request)
- assert "Proxy-Authorization" not in request2.headers
- assert "_auth_proxy" not in request2.meta
- assert "proxy" not in request2.meta
-
- proxy_mw.process_request(request2)
-
- assert "Proxy-Authorization" not in request2.headers
- assert "_auth_proxy" not in request2.meta
- assert "proxy" not in request2.meta
-
- response2 = self.get_response(request2, "https://example.com")
- request3 = redirect_mw.process_response(request2, response2)
-
- assert isinstance(request3, Request)
- assert "Proxy-Authorization" not in request3.headers
- assert "_auth_proxy" not in request3.meta
- assert "proxy" not in request3.meta
-
- proxy_mw.process_request(request3)
-
- assert "Proxy-Authorization" not in request3.headers
- assert "_auth_proxy" not in request3.meta
- assert "proxy" not in request3.meta
+ self._check_proxy_scenario(
+ env={},
+ meta={},
+ url="https://example.com",
+ location1="http://example.com",
+ location2="https://example.com",
+ expected=(None, None, None, None, None),
+ )