Scrapy, a fast high-level web crawling & scraping framework for Python.
Serialize cached responses with Response.to_dict() (#7908)
Adriane30c44432439
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex c45ec0487..55945fd96 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -467,6 +467,9 @@ Filesystem storage backend (default)* ``response_headers`` - the response headers (in raw HTTP format)+ * ``response_data`` - the remaining data of the response, as returned by+ :meth:`Response.to_dict() <scrapy.http.Response.to_dict>`, pickled+* ``meta`` - some metadata of this cache resource in Python ``repr()``format (grep-friendly format)@@ -500,6 +503,11 @@ Writing your own storage backendYou can implement a cache storage backend by creating a Python class thatdefines the methods described below.+To store a response, use :meth:`Response.to_dict()+<scrapy.http.Response.to_dict>`, and to read it back, use+:func:`~scrapy.utils.response.response_from_dict`. That way responses of any+class, including those of third-party plugins, are cached and restored intact.+.. module:: scrapy.extensions.httpcache.. class:: CacheStoragediff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex 4a6d19ce2..9a39212fb 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -1288,6 +1288,10 @@ Response objects.. automethod:: Response.follow_all+ .. automethod:: Response.to_dict++ .. automethod:: Response.from_dict+.. _topics-request-response-ref-response-subclasses:@@ -1478,3 +1482,9 @@ JsonResponse objectsthat is used when the response has a `JSON MIME type<https://mimesniff.spec.whatwg.org/#json-mime-type>`_ in its `Content-Type`header.+++Other functions related to responses+====================================++.. autofunction:: scrapy.utils.response.response_from_dictdiff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.pyindex d008d0f67..f7b1c73d4 100644--- a/scrapy/extensions/httpcache.py+++ b/scrapy/extensions/httpcache.py@@ -12,11 +12,11 @@ from weakref import WeakKeyDictionaryfrom w3lib.http import headers_dict_to_raw, headers_raw_to_dict-from scrapy.http import Headers, Response-from scrapy.responsetypes import responsetypes+from scrapy.http import Responsefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.project import data_pathfrom scrapy.utils.python import to_bytes, to_unicode+from scrapy.utils.response import response_from_dictif TYPE_CHECKING:import os@@ -272,24 +272,13 @@ class DbmCacheStorage:data = self._read_data(spider, request)if data is None:return None # not cached- url = data["url"]- status = data["status"]- headers = Headers(data["headers"])- body = data["body"]- respcls = responsetypes.from_args(headers=headers, url=url, body=body)- return respcls(url=url, headers=headers, status=status, body=body)+ return response_from_dict(data)def store_response(self, spider: Spider, request: Request, response: Response) -> None:key = self._fingerprinter.fingerprint(request).hex()- data = {- "status": response.status,- "url": response.url,- "headers": dict(response.headers),- "body": response.body,- }- self.db[f"{key}_data"] = pickle.dumps(data, protocol=4)+ self.db[f"{key}_data"] = pickle.dumps(response.to_dict(), protocol=4)self.db[f"{key}_time"] = str(time())def _read_data(self, spider: Spider, request: Request) -> dict[str, Any] | None:@@ -340,11 +329,17 @@ class FilesystemCacheStorage:body = f.read()with self._open(rpath / "response_headers", "rb") as f:rawheaders = f.read()- url = metadata["response_url"]- status = metadata["status"]- headers = Headers(headers_raw_to_dict(rawheaders))- respcls = responsetypes.from_args(headers=headers, url=url, body=body)- return respcls(url=url, headers=headers, status=status, body=body)+ data = {+ "url": metadata["response_url"],+ "status": metadata["status"],+ "headers": headers_raw_to_dict(rawheaders),+ "body": body,+ }+ datapath = rpath / "response_data"+ if datapath.exists():+ with self._open(datapath, "rb") as f:+ data.update(pickle.load(f)) # noqa: S301+ return response_from_dict(data)def store_response(self, spider: Spider, request: Request, response: Response@@ -368,6 +363,13 @@ class FilesystemCacheStorage:f.write(headers_dict_to_raw(response.headers))with self._open(rpath / "response_body", "wb") as f:f.write(response.body)+ data = {+ key: value+ for key, value in response.to_dict().items()+ if key not in {"url", "status", "headers", "body"}+ }+ with self._open(rpath / "response_data", "wb") as f:+ pickle.dump(data, f, protocol=4)with self._open(rpath / "request_headers", "wb") as f:f.write(headers_dict_to_raw(request.headers))with self._open(rpath / "request_body", "wb") as f:diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.pyindex f91cb2095..2f148ff84 100644--- a/scrapy/http/response/__init__.py+++ b/scrapy/http/response/__init__.py@@ -194,6 +194,37 @@ class Response(object_ref):cls = self.__class__return cls(*args, **kwargs)+ def to_dict(self) -> dict[str, Any]:+ """Return a dictionary containing the Response's data.++ .. versionadded:: VERSION++ Use :func:`~scrapy.utils.response.response_from_dict` to convert back+ into a :class:`~scrapy.http.Response` object.++ :attr:`request` and :attr:`certificate` are left out, as they are tied+ to a single crawl. Everything else in :attr:`attributes` is included,+ so subclasses only need to override this method, and :meth:`from_dict`,+ if some of their attributes cannot be stored as is.+ """+ d: dict[str, Any] = {"headers": dict(self.headers)}+ for attr in self.attributes:+ if attr in {"request", "certificate"}:+ continue+ d.setdefault(attr, getattr(self, attr))+ if type(self) is not Response: # pylint: disable=unidiomatic-typecheck+ d["_class"] = self.__module__ + "." + self.__class__.__name__+ return d++ @classmethod+ def from_dict(cls, d: dict[str, Any]) -> Self:+ """Return a response built from the *d* dict, as returned by+ :meth:`to_dict`.++ .. versionadded:: VERSION+ """+ return cls(**{key: value for key, value in d.items() if key != "_class"})+def urljoin(self, url: str) -> str:"""Join this Response's url with a possible relative url to form anabsolute interpretation of the latter."""diff --git a/scrapy/utils/response.py b/scrapy/utils/response.pyindex c9aa416f1..65e95253a 100644--- a/scrapy/utils/response.py+++ b/scrapy/utils/response.py@@ -16,6 +16,8 @@ from weakref import WeakKeyDictionaryfrom twisted.web import httpfrom w3lib import html+from scrapy.http.headers import Headers+from scrapy.utils.misc import load_objectfrom scrapy.utils.python import to_bytes, to_unicodeif TYPE_CHECKING:@@ -54,6 +56,30 @@ def get_meta_refresh(return _metaref_cache[response]+def response_from_dict(d: dict[str, Any]) -> Response:+ """Return a response built from the *d* dict, as returned by+ :meth:`Response.to_dict() <scrapy.http.Response.to_dict>`.++ .. versionadded:: VERSION++ If *d* does not indicate a response class, e.g. because it comes from a+ plain :class:`~scrapy.http.Response` object or predates :meth:`~scrapy.http.Response.to_dict`,+ the class is guessed with :attr:`~scrapy.responsetypes.responsetypes`.+ """+ # Imported here to avoid a circular import.+ from scrapy.responsetypes import responsetypes # noqa: PLC0415++ d = {**d, "headers": Headers(d.get("headers") or {})}+ response_cls: type[Response] = (+ load_object(d["_class"])+ if "_class" in d+ else responsetypes.from_args(+ headers=d["headers"], url=d["url"], body=d.get("body")+ )+ )+ return response_cls.from_dict(d)++def response_status_message(status: bytes | float | str) -> str:"""Return status code plus status text descriptive message"""status_int = int(status)diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.pyindex c48f77ea5..6baf97896 100644--- a/tests/test_downloadermiddleware_httpcache.py+++ b/tests/test_downloadermiddleware_httpcache.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport email.utilsimport logging+import pickleimport shutilimport tempfileimport time@@ -26,6 +27,14 @@ if TYPE_CHECKING:from scrapy.crawler import Crawler+class CustomResponse(Response):+ attributes: tuple[str, ...] = (*Response.attributes, "custom")++ def __init__(self, *args: Any, custom: str | None = None, **kwargs: Any):+ self.custom = custom+ super().__init__(*args, **kwargs)++class AlwaysStalePolicy(DummyPolicy):""":class:`~scrapy.extensions.httpcache.DummyPolicy` that alwaysrevalidates cached responses."""@@ -110,6 +119,12 @@ class StorageTestMixin(TestBase):"""Make the cache entry of *request* unreadable for *storage*."""raise NotImplementedError+ def _downgrade_cache_entry(+ self, storage: Any, spider: Spider, request: Request+ ) -> None:+ """Rewrite the cache entry of *request* as Scrapy 2.14 would have."""+ raise NotImplementedError+def test_storage(self):with self._storage(HTTPCACHE_EXPIRATION_SECS=100) as (storage, crawler):request2 = self.request.copy()@@ -181,6 +196,38 @@ class StorageTestMixin(TestBase):assert isinstance(cached_response, HtmlResponse)self.assertEqualResponse(response, cached_response)+ def test_storage_response_class(self):+ with self._storage() as (storage, crawler):+ response = CustomResponse(+ "http://www.example.com", body=b"test body", custom="value"+ )+ storage.store_response(crawler.spider, self.request, response)+ cached_response = storage.retrieve_response(crawler.spider, self.request)+ assert isinstance(cached_response, CustomResponse)+ assert cached_response.custom == "value"++ def test_storage_encoding(self):+ """The encoding of the stored response is kept even when it cannot be+ inferred from the response data."""+ with self._storage() as (storage, crawler):+ response = HtmlResponse(+ "http://www.example.com",+ body='<meta charset="iso-8859-1">€'.encode(),+ encoding="utf-8",+ )+ storage.store_response(crawler.spider, self.request, response)+ cached_response = storage.retrieve_response(crawler.spider, self.request)+ assert cached_response.encoding == "utf-8"+ assert cached_response.text == response.text++ def test_storage_old_cache_entry(self):+ with self._storage() as (storage, crawler):+ storage.store_response(crawler.spider, self.request, self.response)+ self._downgrade_cache_entry(storage, crawler.spider, self.request)+ cached_response = storage.retrieve_response(crawler.spider, self.request)+ assert isinstance(cached_response, HtmlResponse)+ self.assertEqualResponse(self.response, cached_response)+class PolicyTestMixin(TestBase):"""Mixin containing policy-specific test methods."""@@ -708,6 +755,10 @@ class FilesystemStorageTestMixin(StorageTestMixin):rpath = Path(storage._get_request_path(spider, request))(rpath / "response_body").unlink()+ def _downgrade_cache_entry(self, storage, spider, request) -> None:+ rpath = Path(storage._get_request_path(spider, request))+ (rpath / "response_data").unlink()+class DbmStorageTestMixin(StorageTestMixin):storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"@@ -716,6 +767,14 @@ class DbmStorageTestMixin(StorageTestMixin):key = storage._fingerprinter.fingerprint(request).hex()storage.db[f"{key}_data"] = b"not a pickle"+ def _downgrade_cache_entry(self, storage, spider, request) -> None:+ key = storage._fingerprinter.fingerprint(request).hex()+ data = pickle.loads(storage.db[f"{key}_data"])+ data = {+ k: v for k, v in data.items() if k in ("status", "url", "headers", "body")+ }+ storage.db[f"{key}_data"] = pickle.dumps(data, protocol=4)+class TestFilesystemStorageWithDummyPolicy(FilesystemStorageTestMixin, DummyPolicyTestMixindiff --git a/tests/test_response_dict.py b/tests/test_response_dict.pynew file mode 100644index 000000000..e258a2316--- /dev/null+++ b/tests/test_response_dict.py@@ -0,0 +1,98 @@+from __future__ import annotations++from ipaddress import ip_address++from scrapy import Request+from scrapy.http import HtmlResponse, Response, TextResponse+from scrapy.utils.response import response_from_dict+++class CustomResponse(TextResponse):+ attributes: tuple[str, ...] = (*TextResponse.attributes, "custom")++ def __init__(self, *args, custom: str | None = None, **kwargs):+ self.custom = custom+ super().__init__(*args, **kwargs)+++class DynamicResponse(Response):+ """Response subclass that extends+ :attr:`~scrapy.http.Response.attributes` on instances, as plugins that+ support several Scrapy versions do."""++ def __init__(self, *args, custom: str | None = None, **kwargs):+ self.custom = custom+ super().__init__(*args, **kwargs)+ self.attributes = (*self.attributes, "custom")+++def test_basic() -> None:+ """The class of plain responses is not stored, it is guessed back from the+ response data."""+ response = Response("https://example.com", body=b"\x00\x01")+ assert "_class" not in response.to_dict()+ response2 = response_from_dict(response.to_dict())+ assert response2.__class__ is Response+ assert response2.url == response.url+++def test_all_attributes() -> None:+ response = HtmlResponse(+ url="https://example.com",+ status=201,+ headers={"Content-Type": "text/html; charset=latin-1"},+ body=b"\xa3",+ flags=["testFlag"],+ encoding="latin-1",+ ip_address=ip_address("127.0.0.1"),+ protocol="h2",+ )+ response2 = response_from_dict(response.to_dict())+ assert response2.__class__ is HtmlResponse+ for attribute in HtmlResponse.attributes:+ if attribute in {"request", "certificate"}:+ continue+ assert getattr(response2, attribute) == getattr(response, attribute)+++def test_custom_attributes() -> None:+ response = CustomResponse("https://example.com", custom="value")+ response2 = response_from_dict(response.to_dict())+ assert isinstance(response2, CustomResponse)+ assert response2.custom == "value"+++def test_custom_instance_attributes() -> None:+ response = DynamicResponse("https://example.com", custom="value")+ response2 = response_from_dict(response.to_dict())+ assert isinstance(response2, DynamicResponse)+ assert response2.custom == "value"+++def test_crawl_attributes_left_out() -> None:+ response = Response(+ "https://example.com",+ request=Request("https://example.com"),+ certificate=object(),+ )+ d = response.to_dict()+ assert "request" not in d+ assert "certificate" not in d+ response2 = response_from_dict(d)+ assert response2.request is None+ assert response2.certificate is None+++def test_unknown_class() -> None:+ """Dicts that do not indicate a response class, e.g. cache entries written+ by older Scrapy versions, get a class based on their data."""+ response2 = response_from_dict(+ {+ "url": "https://example.com",+ "status": 200,+ "headers": {b"Content-Type": [b"text/plain"]},+ "body": b"foo",+ }+ )+ assert response2.__class__ is TextResponse+ assert response2.text == "foo"