Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Return the default from Headers.get() for headers set to None. (#8268)

Andrey Rakhmatullin406e5ce4919b
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index b880454ee..e6f9e6289 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -145,13 +145,12 @@ class Headers(dict): # type: ignore[type-arg]
return None
def get(self, key: str | bytes, def_val: Any = None) -> bytes | None:
- try:
- return cast(
- "list[bytes]",
- dict.get(self, self._normkey(key), self._normvalue(def_val)),
- )[-1]
- except IndexError:
- return None
+ values = cast("list[bytes] | None", dict.get(self, self._normkey(key)))
+ if not values:
+ # The header is missing, or it was set to None so that it is not
+ # sent; neither has a value to return.
+ values = self._normvalue(def_val)
+ return values[-1] if values else None
def getlist(
self, key: str | bytes, def_val: _RawValue | Iterable[_RawValue] | None = None
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index f9986a849..ec054426a 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -428,6 +428,18 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
self.assertEqualResponse(res5, res0b)
assert "cached" in res5.flags
+ def test_request_cache_control_none(self):
+ res0 = Response(
+ self.request.url, status=200, headers={"Expires": self.tomorrow}
+ )
+ req0 = Request("http://example.com", headers={"Cache-Control": None})
+ with self._middleware() as mw:
+ res1 = self._process_requestresponse(mw, req0, res0)
+ assert "cached" not in res1.flags
+ res2 = mw.process_request(req0)
+ assert isinstance(res2, Response)
+ assert "cached" in res2.flags
+
def test_response_cacheability(self):
responses = [
# 304 is not cacheable no matter what servers sends
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 5c9a5af7b..f55e57b4f 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -244,6 +244,19 @@ Disallow: /some/randome/page.html
middleware.process_request_2(rp, Request("http://site.local/allowed"))
rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot")
+ @coroutine_test
+ async def test_robotstxt_user_agent_none(self):
+ middleware = build_from_crawler(
+ RobotsTxtMiddleware, self._get_successful_crawler()
+ )
+ headers = {"User-Agent": None}
+ await self.assertNotIgnored(
+ Request("http://site.local/allowed", headers=headers), middleware
+ )
+ await self.assertIgnored(
+ Request("http://site.local/admin/main", headers=headers), middleware
+ )
+
@coroutine_test
async def test_robotstxt_local_file(self):
middleware = build_from_crawler(
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index 7e4932601..3fc800dc2 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -191,6 +191,7 @@ def test_none_value() -> None:
h1.setdefault("foo", "bar")
assert h1["foo"] is None
assert h1.get("foo") is None
+ assert h1.get("foo", "baz") == b"baz"
assert h1.getlist("foo") == []