Scrapy, a fast high-level web crawling & scraping framework for Python.
Use canonical HTTP status names from stdlib. (#8487)
Andrey Rakhmatullin758eb5bb46e9
diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.pyindex 1a7fc89da..98eb9c723 100644--- a/scrapy/downloadermiddlewares/stats.py+++ b/scrapy/downloadermiddlewares/stats.py@@ -1,9 +1,8 @@from __future__ import annotations+from http.client import responsesfrom typing import TYPE_CHECKING, Self-from twisted.web import http-from scrapy.exceptions import NotConfiguredfrom scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.python import global_object_name@@ -29,7 +28,7 @@ def get_header_size(def get_status_size(response_status: int) -> int:- return len(http.RESPONSES.get(response_status, b"")) + 15+ return len(responses.get(response_status, "")) + 15# resp.status + b"\r\n" + b"HTTP/1.1 <100-599> "diff --git a/scrapy/utils/response.py b/scrapy/utils/response.pyindex 2fff5b13e..d39c72045 100644--- a/scrapy/utils/response.py+++ b/scrapy/utils/response.py@@ -11,11 +11,11 @@ import reimport tempfileimport warningsimport webbrowser+from http.client import responsesfrom pathlib import Pathfrom typing import TYPE_CHECKING, Anyimport frostwork-from twisted.web import httpfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http.headers import Headers@@ -109,7 +109,7 @@ def response_from_dict(d: dict[str, Any]) -> Response:def response_status_message(status: bytes | float | str) -> str:"""Return status code plus status text descriptive message"""status_int = int(status)- message = http.RESPONSES.get(status_int, b"Unknown Status").decode()+ message = responses.get(status_int, "Unknown Status")return f"{status_int} {message}"diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.pyindex f0588e056..9e2726c0d 100644--- a/tests/test_downloadermiddleware_stats.py+++ b/tests/test_downloadermiddleware_stats.py@@ -66,6 +66,18 @@ class TestDownloaderStats:self.mw.process_response(self.req, res)self.assertStatsEqual("downloader/response_bytes", len(raw_response))+ @pytest.mark.parametrize(+ ("status", "raw_response"),+ [+ (429, b"HTTP/1.1 429 Too Many Requests\r\n\r\nbody"),+ (599, b"HTTP/1.1 599 \r\n\r\nbody"),+ ],+ )+ def test_response_bytes_status(self, status: int, raw_response: bytes) -> None:+ res = Response("http://scrapytest.org", status=status, body=b"body")+ self.mw.process_response(self.req, res)+ self.assertStatsEqual("downloader/response_bytes", len(raw_response))+def test_response_bytes_body_file(self, tmp_path: Path) -> None:body_file = tmp_path / "body"body_file.write_bytes(b"body")diff --git a/tests/test_utils_response.py b/tests/test_utils_response.pyindex 74eec5c58..6750963c8 100644--- a/tests/test_utils_response.py+++ b/tests/test_utils_response.py@@ -68,6 +68,7 @@ def test_get_base_url() -> None:def test_response_status_message():assert response_status_message(200) == "200 OK"assert response_status_message(404) == "404 Not Found"+ assert response_status_message(429) == "429 Too Many Requests"assert response_status_message(573) == "573 Unknown Status"