Scrapy, a fast high-level web crawling & scraping framework for Python.
Response.attributes (#5218)
Eugenio Lacuestacc89f6be381d
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex a6a3daf31..d3e08efd4 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -670,9 +670,6 @@ Response objects.. autoclass:: Response- A :class:`Response` object represents an HTTP response, which is usually- downloaded (by the Downloader) and fed to the Spiders for processing.-:param url: the URL of this response:type url: str@@ -829,6 +826,8 @@ Response objectshandlers, i.e. for ``http(s)`` responses. For other handlers,:attr:`protocol` is always ``None``.+ .. autoattribute:: Response.attributes+.. method:: Response.copy()Returns a new Response which is a copy of this Response.@@ -925,6 +924,8 @@ TextResponse objectsA :class:`~scrapy.Selector` instance using the response astarget. The selector is lazily instantiated on first access.+ .. autoattribute:: TextResponse.attributes+:class:`TextResponse` objects support the following methods in addition tothe standard :class:`Response` ones:diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.pyindex 185a9bb67..4de6c9b5b 100644--- a/scrapy/http/response/__init__.py+++ b/scrapy/http/response/__init__.py@@ -4,7 +4,7 @@ responses in Scrapy.See documentation in docs/topics/request-response.rst"""-from typing import Generator+from typing import Generator, Tuplefrom urllib.parse import urljoinfrom scrapy.exceptions import NotSupported@@ -16,6 +16,19 @@ from scrapy.utils.trackref import object_refclass Response(object_ref):+ """An object that represents an HTTP response, which is usually+ downloaded (by the Downloader) and fed to the Spiders for processing.+ """++ attributes: Tuple[str, ...] = (+ "url", "status", "headers", "body", "flags", "request", "certificate", "ip_address", "protocol",+ )+ """A tuple of :class:`str` objects containing the name of all public+ attributes of the class that are also keyword parameters of the+ ``__init__`` method.++ Currently used by :meth:`Response.replace`.+ """def __init__(self,@@ -97,12 +110,8 @@ class Response(object_ref):return self.replace()def replace(self, *args, **kwargs):- """Create a new Response with the same attributes except for those- given new values.- """- for x in [- "url", "status", "headers", "body", "request", "flags", "certificate", "ip_address", "protocol",- ]:+ """Create a new Response with the same attributes except for those given new values"""+ for x in self.attributes:kwargs.setdefault(x, getattr(self, x))cls = kwargs.pop('cls', self.__class__)return cls(*args, **kwargs)diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex e36e14880..27bd55c07 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rstimport jsonimport warningsfrom contextlib import suppress-from typing import Generator+from typing import Generator, Tuplefrom urllib.parse import urljoinimport parsel@@ -30,6 +30,8 @@ class TextResponse(Response):_DEFAULT_ENCODING = 'ascii'_cached_decoded_json = _NONE+ attributes: Tuple[str, ...] = Response.attributes + ("encoding",)+def __init__(self, *args, **kwargs):self._encoding = kwargs.pop('encoding', None)self._cached_benc = None@@ -53,10 +55,6 @@ class TextResponse(Response):else:super()._set_body(body)- def replace(self, *args, **kwargs):- kwargs.setdefault('encoding', self.encoding)- return Response.replace(self, *args, **kwargs)-@propertydef encoding(self):return self._declared_encoding() or self._body_inferred_encoding()diff --git a/tests/test_http_response.py b/tests/test_http_response.pyindex 04a594d03..cf34a9e5c 100644--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@ -820,3 +820,62 @@ class XmlResponseTest(TextResponseTest):response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(),response.selector.xpath("//s2:elem/text()").getall(),)+++class CustomResponse(TextResponse):+ attributes = TextResponse.attributes + ("foo", "bar")++ def __init__(self, *args, **kwargs) -> None:+ self.foo = kwargs.pop("foo", None)+ self.bar = kwargs.pop("bar", None)+ self.lost = kwargs.pop("lost", None)+ super().__init__(*args, **kwargs)+++class CustomResponseTest(TextResponseTest):+ response_class = CustomResponse++ def test_copy(self):+ super().test_copy()+ r1 = self.response_class(url="https://example.org", status=200, foo="foo", bar="bar", lost="lost")+ r2 = r1.copy()+ self.assertIsInstance(r2, self.response_class)+ self.assertEqual(r1.foo, r2.foo)+ self.assertEqual(r1.bar, r2.bar)+ self.assertEqual(r1.lost, "lost")+ self.assertIsNone(r2.lost)++ def test_replace(self):+ super().test_replace()+ r1 = self.response_class(url="https://example.org", status=200, foo="foo", bar="bar", lost="lost")++ r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost")+ self.assertIsInstance(r2, self.response_class)+ self.assertEqual(r1.foo, "foo")+ self.assertEqual(r1.bar, "bar")+ self.assertEqual(r1.lost, "lost")+ self.assertEqual(r2.foo, "new-foo")+ self.assertEqual(r2.bar, "new-bar")+ self.assertEqual(r2.lost, "new-lost")++ r3 = r1.replace(foo="new-foo", bar="new-bar")+ self.assertIsInstance(r3, self.response_class)+ self.assertEqual(r1.foo, "foo")+ self.assertEqual(r1.bar, "bar")+ self.assertEqual(r1.lost, "lost")+ self.assertEqual(r3.foo, "new-foo")+ self.assertEqual(r3.bar, "new-bar")+ self.assertIsNone(r3.lost)++ r4 = r1.replace(foo="new-foo")+ self.assertIsInstance(r4, self.response_class)+ self.assertEqual(r1.foo, "foo")+ self.assertEqual(r1.bar, "bar")+ self.assertEqual(r1.lost, "lost")+ self.assertEqual(r4.foo, "new-foo")+ self.assertEqual(r4.bar, "bar")+ self.assertIsNone(r4.lost)++ with self.assertRaises(TypeError) as ctx:+ r1.replace(unknown="unknown")+ self.assertEqual(str(ctx.exception), "__init__() got an unexpected keyword argument 'unknown'")