Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Response.attributes (#5218)

Eugenio Lacuestacc89f6be381d
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index a6a3daf31..d3e08efd4 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -670,9 +670,6 @@ Response objects
.. autoclass:: Response
- A :class:`Response` object represents an HTTP response, which is usually
- downloaded (by the Downloader) and fed to the Spiders for processing.
-
:param url: the URL of this response
:type url: str
@@ -829,6 +826,8 @@ Response objects
handlers, i.e. for ``http(s)`` responses. For other handlers,
:attr:`protocol` is always ``None``.
+ .. autoattribute:: Response.attributes
+
.. method:: Response.copy()
Returns a new Response which is a copy of this Response.
@@ -925,6 +924,8 @@ TextResponse objects
A :class:`~scrapy.Selector` instance using the response as
target. The selector is lazily instantiated on first access.
+ .. autoattribute:: TextResponse.attributes
+
:class:`TextResponse` objects support the following methods in addition to
the standard :class:`Response` ones:
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 185a9bb67..4de6c9b5b 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -4,7 +4,7 @@ responses in Scrapy.
See documentation in docs/topics/request-response.rst
"""
-from typing import Generator
+from typing import Generator, Tuple
from urllib.parse import urljoin
from scrapy.exceptions import NotSupported
@@ -16,6 +16,19 @@ from scrapy.utils.trackref import object_ref
class Response(object_ref):
+ """An object that represents an HTTP response, which is usually
+ downloaded (by the Downloader) and fed to the Spiders for processing.
+ """
+
+ attributes: Tuple[str, ...] = (
+ "url", "status", "headers", "body", "flags", "request", "certificate", "ip_address", "protocol",
+ )
+ """A tuple of :class:`str` objects containing the name of all public
+ attributes of the class that are also keyword parameters of the
+ ``__init__`` method.
+
+ Currently used by :meth:`Response.replace`.
+ """
def __init__(
self,
@@ -97,12 +110,8 @@ class Response(object_ref):
return self.replace()
def replace(self, *args, **kwargs):
- """Create a new Response with the same attributes except for those
- given new values.
- """
- for x in [
- "url", "status", "headers", "body", "request", "flags", "certificate", "ip_address", "protocol",
- ]:
+ """Create a new Response with the same attributes except for those given new values"""
+ for x in self.attributes:
kwargs.setdefault(x, getattr(self, x))
cls = kwargs.pop('cls', self.__class__)
return cls(*args, **kwargs)
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index e36e14880..27bd55c07 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst
import json
import warnings
from contextlib import suppress
-from typing import Generator
+from typing import Generator, Tuple
from urllib.parse import urljoin
import parsel
@@ -30,6 +30,8 @@ class TextResponse(Response):
_DEFAULT_ENCODING = 'ascii'
_cached_decoded_json = _NONE
+ attributes: Tuple[str, ...] = Response.attributes + ("encoding",)
+
def __init__(self, *args, **kwargs):
self._encoding = kwargs.pop('encoding', None)
self._cached_benc = None
@@ -53,10 +55,6 @@ class TextResponse(Response):
else:
super()._set_body(body)
- def replace(self, *args, **kwargs):
- kwargs.setdefault('encoding', self.encoding)
- return Response.replace(self, *args, **kwargs)
-
@property
def encoding(self):
return self._declared_encoding() or self._body_inferred_encoding()
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 04a594d03..cf34a9e5c 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -820,3 +820,62 @@ class XmlResponseTest(TextResponseTest):
response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(),
response.selector.xpath("//s2:elem/text()").getall(),
)
+
+
+class CustomResponse(TextResponse):
+ attributes = TextResponse.attributes + ("foo", "bar")
+
+ def __init__(self, *args, **kwargs) -> None:
+ self.foo = kwargs.pop("foo", None)
+ self.bar = kwargs.pop("bar", None)
+ self.lost = kwargs.pop("lost", None)
+ super().__init__(*args, **kwargs)
+
+
+class CustomResponseTest(TextResponseTest):
+ response_class = CustomResponse
+
+ def test_copy(self):
+ super().test_copy()
+ r1 = self.response_class(url="https://example.org", status=200, foo="foo", bar="bar", lost="lost")
+ r2 = r1.copy()
+ self.assertIsInstance(r2, self.response_class)
+ self.assertEqual(r1.foo, r2.foo)
+ self.assertEqual(r1.bar, r2.bar)
+ self.assertEqual(r1.lost, "lost")
+ self.assertIsNone(r2.lost)
+
+ def test_replace(self):
+ super().test_replace()
+ r1 = self.response_class(url="https://example.org", status=200, foo="foo", bar="bar", lost="lost")
+
+ r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost")
+ self.assertIsInstance(r2, self.response_class)
+ self.assertEqual(r1.foo, "foo")
+ self.assertEqual(r1.bar, "bar")
+ self.assertEqual(r1.lost, "lost")
+ self.assertEqual(r2.foo, "new-foo")
+ self.assertEqual(r2.bar, "new-bar")
+ self.assertEqual(r2.lost, "new-lost")
+
+ r3 = r1.replace(foo="new-foo", bar="new-bar")
+ self.assertIsInstance(r3, self.response_class)
+ self.assertEqual(r1.foo, "foo")
+ self.assertEqual(r1.bar, "bar")
+ self.assertEqual(r1.lost, "lost")
+ self.assertEqual(r3.foo, "new-foo")
+ self.assertEqual(r3.bar, "new-bar")
+ self.assertIsNone(r3.lost)
+
+ r4 = r1.replace(foo="new-foo")
+ self.assertIsInstance(r4, self.response_class)
+ self.assertEqual(r1.foo, "foo")
+ self.assertEqual(r1.bar, "bar")
+ self.assertEqual(r1.lost, "lost")
+ self.assertEqual(r4.foo, "new-foo")
+ self.assertEqual(r4.bar, "bar")
+ self.assertIsNone(r4.lost)
+
+ with self.assertRaises(TypeError) as ctx:
+ r1.replace(unknown="unknown")
+ self.assertEqual(str(ctx.exception), "__init__() got an unexpected keyword argument 'unknown'")