Scrapy, a fast high-level web crawling & scraping framework for Python.
Return the default from Headers.get() for headers set to None. (#8268)
Andrey Rakhmatullin406e5ce4919b
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.pyindex b880454ee..e6f9e6289 100644--- a/scrapy/http/headers.py+++ b/scrapy/http/headers.py@@ -145,13 +145,12 @@ class Headers(dict): # type: ignore[type-arg]return Nonedef get(self, key: str | bytes, def_val: Any = None) -> bytes | None:- try:- return cast(- "list[bytes]",- dict.get(self, self._normkey(key), self._normvalue(def_val)),- )[-1]- except IndexError:- return None+ values = cast("list[bytes] | None", dict.get(self, self._normkey(key)))+ if not values:+ # The header is missing, or it was set to None so that it is not+ # sent; neither has a value to return.+ values = self._normvalue(def_val)+ return values[-1] if values else Nonedef getlist(self, key: str | bytes, def_val: _RawValue | Iterable[_RawValue] | None = Nonediff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.pyindex f9986a849..ec054426a 100644--- a/tests/test_downloadermiddleware_httpcache.py+++ b/tests/test_downloadermiddleware_httpcache.py@@ -428,6 +428,18 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):self.assertEqualResponse(res5, res0b)assert "cached" in res5.flags+ def test_request_cache_control_none(self):+ res0 = Response(+ self.request.url, status=200, headers={"Expires": self.tomorrow}+ )+ req0 = Request("http://example.com", headers={"Cache-Control": None})+ with self._middleware() as mw:+ res1 = self._process_requestresponse(mw, req0, res0)+ assert "cached" not in res1.flags+ res2 = mw.process_request(req0)+ assert isinstance(res2, Response)+ assert "cached" in res2.flags+def test_response_cacheability(self):responses = [# 304 is not cacheable no matter what servers sendsdiff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex 5c9a5af7b..f55e57b4f 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -244,6 +244,19 @@ Disallow: /some/randome/page.htmlmiddleware.process_request_2(rp, Request("http://site.local/allowed"))rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot")+ @coroutine_test+ async def test_robotstxt_user_agent_none(self):+ middleware = build_from_crawler(+ RobotsTxtMiddleware, self._get_successful_crawler()+ )+ headers = {"User-Agent": None}+ await self.assertNotIgnored(+ Request("http://site.local/allowed", headers=headers), middleware+ )+ await self.assertIgnored(+ Request("http://site.local/admin/main", headers=headers), middleware+ )+@coroutine_testasync def test_robotstxt_local_file(self):middleware = build_from_crawler(diff --git a/tests/test_http_headers.py b/tests/test_http_headers.pyindex 7e4932601..3fc800dc2 100644--- a/tests/test_http_headers.py+++ b/tests/test_http_headers.py@@ -191,6 +191,7 @@ def test_none_value() -> None:h1.setdefault("foo", "bar")assert h1["foo"] is Noneassert h1.get("foo") is None+ assert h1.get("foo", "baz") == b"baz"assert h1.getlist("foo") == []