Scrapy, a fast high-level web crawling & scraping framework for Python.
Improve test coverage for http/ (#7672)
Adrian0007676e8d52
diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.pyindex 660b76d08..ce9296764 100644--- a/tests/test_http_cookies.py+++ b/tests/test_http_cookies.py@@ -1,8 +1,59 @@+from http.cookiejar import DefaultCookiePolicy+from scrapy.http import Request, Response-from scrapy.http.cookies import WrappedRequest, WrappedResponse+from scrapy.http.cookies import CookieJar, WrappedRequest, WrappedResponsefrom scrapy.utils.httpobj import urlparse_cached+class TestCookieJar:+ def setup_method(self):+ self.jar = CookieJar()+ self.request = Request("http://example.com/")+ self.response = Response(+ "http://example.com/",+ headers={"Set-Cookie": "name=value; Domain=example.com; Path=/"},+ )++ def test_extract_cookies(self):+ assert len(self.jar) == 0+ self.jar.extract_cookies(self.response, self.request)+ assert len(self.jar) == 1+ cookie = next(iter(self.jar))+ assert cookie.name == "name"+ assert cookie.value == "value"+ assert ".example.com" in self.jar._cookies++ def test_make_cookies_and_set_cookie(self):+ cookies = self.jar.make_cookies(self.response, self.request)+ assert len(cookies) == 1+ jar = CookieJar()+ for cookie in cookies:+ jar.set_cookie(cookie)+ assert len(jar) == 1++ def test_clear(self):+ self.jar.extract_cookies(self.response, self.request)+ assert len(self.jar) == 1+ self.jar.clear()+ assert len(self.jar) == 0++ def test_clear_session_cookies(self):+ self.jar.extract_cookies(self.response, self.request)+ assert len(self.jar) == 1+ self.jar.clear_session_cookies()+ assert len(self.jar) == 0++ def test_set_policy(self):+ policy = DefaultCookiePolicy()+ self.jar.set_policy(policy)+ assert self.jar.jar._policy is policy++ def test_check_expired_frequency(self):+ jar = CookieJar(check_expired_frequency=1)+ jar.add_cookie_header(self.request)+ assert jar.processed == 1++class TestWrappedRequest:def setup_method(self):self.request = Request(diff --git a/tests/test_http_headers.py b/tests/test_http_headers.pyindex 243aa6afe..aff3562e3 100644--- a/tests/test_http_headers.py+++ b/tests/test_http_headers.py@@ -140,6 +140,7 @@ class TestHeaders:h1["foo"] = "bar"h1["foo"] = Noneh1.setdefault("foo", "bar")+ assert h1["foo"] is Noneassert h1.get("foo") is Noneassert h1.getlist("foo") == []diff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex e22689d49..1941b826f 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -24,6 +24,10 @@ class TestRequest:with pytest.raises(TypeError):self.request_class(123)+ # priority argument must be an integer+ with pytest.raises(TypeError, match="Request priority not an integer"):+ self.request_class("http://www.example.com", priority="1")+r = self.request_class("http://www.example.com")assert isinstance(r.url, str)assert r.url == "http://www.example.com"@@ -274,6 +278,12 @@ class TestRequest:assert r4.meta == {}assert r4.dont_filter is False+ # the cls argument allows changing the resulting class+ custom_request_cls = type("CustomRequest", (self.request_class,), {})+ r5 = r1.replace(cls=custom_request_cls)+ assert isinstance(r5, custom_request_cls)+ assert r5.url == r1.url+def test_method_always_str(self):r = self.request_class("http://www.example.com", method="POST")assert isinstance(r.method, str)diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.pyindex a4f87d50d..c667e48a1 100644--- a/tests/test_http_request_form.py+++ b/tests/test_http_request_form.py@@ -294,6 +294,9 @@ class TestFormRequest(TestRequest):assert request.method == "GET"request = FormRequest.from_response(response, method="POST")assert request.method == "POST"+ # an explicit method=None skips form-method normalization+ request = FormRequest.from_response(response, method=None)+ assert request.method == "NONE"def test_from_response_override_url(self):response = _buildresponse(diff --git a/tests/test_http_response.py b/tests/test_http_response.pyindex 079c547c7..a8ea4920e 100644--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@ -254,6 +254,11 @@ class TestResponse:with pytest.raises(ValueError, match="url can't be None"):r.follow(None)+ def test_follow_None_encoding(self):+ r = self.response_class("http://example.com")+ with pytest.raises(ValueError, match="encoding can't be None"):+ r.follow("foo", encoding=None)+@pytest.mark.xfail(not W3LIB_STRIPS_URLS,reason="https://github.com/scrapy/w3lib/pull/207",diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.pyindex 4b3fa2302..507fd1864 100644--- a/tests/test_http_response_text.py+++ b/tests/test_http_response_text.py@@ -14,6 +14,13 @@ from tests.test_http_response import TestResponseclass TestTextResponse(TestResponse):response_class = TextResponse+ def test_follow_None_encoding(self):+ # unlike the base Response, TextResponse.follow() falls back to the+ # response encoding when encoding is None instead of raising+ r = self.response_class("http://example.com", body=b"hello", encoding="cp1252")+ req = r.follow("foo", encoding=None)+ assert req.encoding == "cp1252"+def test_replace(self):super().test_replace()r1 = self.response_class(