Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Use canonical HTTP status names from stdlib. (#8487)

Andrey Rakhmatullin758eb5bb46e9
diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py
index 1a7fc89da..98eb9c723 100644
--- a/scrapy/downloadermiddlewares/stats.py
+++ b/scrapy/downloadermiddlewares/stats.py
@@ -1,9 +1,8 @@
from __future__ import annotations
+from http.client import responses
from typing import TYPE_CHECKING, Self
-from twisted.web import http
-
from scrapy.exceptions import NotConfigured
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.python import global_object_name
@@ -29,7 +28,7 @@ def get_header_size(
def get_status_size(response_status: int) -> int:
- return len(http.RESPONSES.get(response_status, b"")) + 15
+ return len(responses.get(response_status, "")) + 15
# resp.status + b"\r\n" + b"HTTP/1.1 <100-599> "
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 2fff5b13e..d39c72045 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -11,11 +11,11 @@ import re
import tempfile
import warnings
import webbrowser
+from http.client import responses
from pathlib import Path
from typing import TYPE_CHECKING, Any
import frostwork
-from twisted.web import http
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http.headers import Headers
@@ -109,7 +109,7 @@ def response_from_dict(d: dict[str, Any]) -> Response:
def response_status_message(status: bytes | float | str) -> str:
"""Return status code plus status text descriptive message"""
status_int = int(status)
- message = http.RESPONSES.get(status_int, b"Unknown Status").decode()
+ message = responses.get(status_int, "Unknown Status")
return f"{status_int} {message}"
diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py
index f0588e056..9e2726c0d 100644
--- a/tests/test_downloadermiddleware_stats.py
+++ b/tests/test_downloadermiddleware_stats.py
@@ -66,6 +66,18 @@ class TestDownloaderStats:
self.mw.process_response(self.req, res)
self.assertStatsEqual("downloader/response_bytes", len(raw_response))
+ @pytest.mark.parametrize(
+ ("status", "raw_response"),
+ [
+ (429, b"HTTP/1.1 429 Too Many Requests\r\n\r\nbody"),
+ (599, b"HTTP/1.1 599 \r\n\r\nbody"),
+ ],
+ )
+ def test_response_bytes_status(self, status: int, raw_response: bytes) -> None:
+ res = Response("http://scrapytest.org", status=status, body=b"body")
+ self.mw.process_response(self.req, res)
+ self.assertStatsEqual("downloader/response_bytes", len(raw_response))
+
def test_response_bytes_body_file(self, tmp_path: Path) -> None:
body_file = tmp_path / "body"
body_file.write_bytes(b"body")
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 74eec5c58..6750963c8 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -68,6 +68,7 @@ def test_get_base_url() -> None:
def test_response_status_message():
assert response_status_message(200) == "200 OK"
assert response_status_message(404) == "404 Not Found"
+ assert response_status_message(429) == "429 Too Many Requests"
assert response_status_message(573) == "573 Unknown Status"