Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Improve test coverage for http/ (#7672)

Adrian0007676e8d52
diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py
index 660b76d08..ce9296764 100644
--- a/tests/test_http_cookies.py
+++ b/tests/test_http_cookies.py
@@ -1,8 +1,59 @@
+from http.cookiejar import DefaultCookiePolicy
+
from scrapy.http import Request, Response
-from scrapy.http.cookies import WrappedRequest, WrappedResponse
+from scrapy.http.cookies import CookieJar, WrappedRequest, WrappedResponse
from scrapy.utils.httpobj import urlparse_cached
+class TestCookieJar:
+ def setup_method(self):
+ self.jar = CookieJar()
+ self.request = Request("http://example.com/")
+ self.response = Response(
+ "http://example.com/",
+ headers={"Set-Cookie": "name=value; Domain=example.com; Path=/"},
+ )
+
+ def test_extract_cookies(self):
+ assert len(self.jar) == 0
+ self.jar.extract_cookies(self.response, self.request)
+ assert len(self.jar) == 1
+ cookie = next(iter(self.jar))
+ assert cookie.name == "name"
+ assert cookie.value == "value"
+ assert ".example.com" in self.jar._cookies
+
+ def test_make_cookies_and_set_cookie(self):
+ cookies = self.jar.make_cookies(self.response, self.request)
+ assert len(cookies) == 1
+ jar = CookieJar()
+ for cookie in cookies:
+ jar.set_cookie(cookie)
+ assert len(jar) == 1
+
+ def test_clear(self):
+ self.jar.extract_cookies(self.response, self.request)
+ assert len(self.jar) == 1
+ self.jar.clear()
+ assert len(self.jar) == 0
+
+ def test_clear_session_cookies(self):
+ self.jar.extract_cookies(self.response, self.request)
+ assert len(self.jar) == 1
+ self.jar.clear_session_cookies()
+ assert len(self.jar) == 0
+
+ def test_set_policy(self):
+ policy = DefaultCookiePolicy()
+ self.jar.set_policy(policy)
+ assert self.jar.jar._policy is policy
+
+ def test_check_expired_frequency(self):
+ jar = CookieJar(check_expired_frequency=1)
+ jar.add_cookie_header(self.request)
+ assert jar.processed == 1
+
+
class TestWrappedRequest:
def setup_method(self):
self.request = Request(
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index 243aa6afe..aff3562e3 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -140,6 +140,7 @@ class TestHeaders:
h1["foo"] = "bar"
h1["foo"] = None
h1.setdefault("foo", "bar")
+ assert h1["foo"] is None
assert h1.get("foo") is None
assert h1.getlist("foo") == []
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index e22689d49..1941b826f 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -24,6 +24,10 @@ class TestRequest:
with pytest.raises(TypeError):
self.request_class(123)
+ # priority argument must be an integer
+ with pytest.raises(TypeError, match="Request priority not an integer"):
+ self.request_class("http://www.example.com", priority="1")
+
r = self.request_class("http://www.example.com")
assert isinstance(r.url, str)
assert r.url == "http://www.example.com"
@@ -274,6 +278,12 @@ class TestRequest:
assert r4.meta == {}
assert r4.dont_filter is False
+ # the cls argument allows changing the resulting class
+ custom_request_cls = type("CustomRequest", (self.request_class,), {})
+ r5 = r1.replace(cls=custom_request_cls)
+ assert isinstance(r5, custom_request_cls)
+ assert r5.url == r1.url
+
def test_method_always_str(self):
r = self.request_class("http://www.example.com", method="POST")
assert isinstance(r.method, str)
diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py
index a4f87d50d..c667e48a1 100644
--- a/tests/test_http_request_form.py
+++ b/tests/test_http_request_form.py
@@ -294,6 +294,9 @@ class TestFormRequest(TestRequest):
assert request.method == "GET"
request = FormRequest.from_response(response, method="POST")
assert request.method == "POST"
+ # an explicit method=None skips form-method normalization
+ request = FormRequest.from_response(response, method=None)
+ assert request.method == "NONE"
def test_from_response_override_url(self):
response = _buildresponse(
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 079c547c7..a8ea4920e 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -254,6 +254,11 @@ class TestResponse:
with pytest.raises(ValueError, match="url can't be None"):
r.follow(None)
+ def test_follow_None_encoding(self):
+ r = self.response_class("http://example.com")
+ with pytest.raises(ValueError, match="encoding can't be None"):
+ r.follow("foo", encoding=None)
+
@pytest.mark.xfail(
not W3LIB_STRIPS_URLS,
reason="https://github.com/scrapy/w3lib/pull/207",
diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py
index 4b3fa2302..507fd1864 100644
--- a/tests/test_http_response_text.py
+++ b/tests/test_http_response_text.py
@@ -14,6 +14,13 @@ from tests.test_http_response import TestResponse
class TestTextResponse(TestResponse):
response_class = TextResponse
+ def test_follow_None_encoding(self):
+ # unlike the base Response, TextResponse.follow() falls back to the
+ # response encoding when encoding is None instead of raising
+ r = self.response_class("http://example.com", body=b"hello", encoding="cp1252")
+ req = r.follow("foo", encoding=None)
+ assert req.encoding == "cp1252"
+
def test_replace(self):
super().test_replace()
r1 = self.response_class(