Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Serialize cached responses with Response.to_dict() (#7908)

Adriane30c44432439
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index c45ec0487..55945fd96 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -467,6 +467,9 @@ Filesystem storage backend (default)
* ``response_headers`` - the response headers (in raw HTTP format)
+ * ``response_data`` - the remaining data of the response, as returned by
+ :meth:`Response.to_dict() <scrapy.http.Response.to_dict>`, pickled
+
* ``meta`` - some metadata of this cache resource in Python ``repr()``
format (grep-friendly format)
@@ -500,6 +503,11 @@ Writing your own storage backend
You can implement a cache storage backend by creating a Python class that
defines the methods described below.
+To store a response, use :meth:`Response.to_dict()
+<scrapy.http.Response.to_dict>`, and to read it back, use
+:func:`~scrapy.utils.response.response_from_dict`. That way responses of any
+class, including those of third-party plugins, are cached and restored intact.
+
.. module:: scrapy.extensions.httpcache
.. class:: CacheStorage
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 4a6d19ce2..9a39212fb 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -1288,6 +1288,10 @@ Response objects
.. automethod:: Response.follow_all
+ .. automethod:: Response.to_dict
+
+ .. automethod:: Response.from_dict
+
.. _topics-request-response-ref-response-subclasses:
@@ -1478,3 +1482,9 @@ JsonResponse objects
that is used when the response has a `JSON MIME type
<https://mimesniff.spec.whatwg.org/#json-mime-type>`_ in its `Content-Type`
header.
+
+
+Other functions related to responses
+====================================
+
+.. autofunction:: scrapy.utils.response.response_from_dict
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index d008d0f67..f7b1c73d4 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -12,11 +12,11 @@ from weakref import WeakKeyDictionary
from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
-from scrapy.http import Headers, Response
-from scrapy.responsetypes import responsetypes
+from scrapy.http import Response
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.project import data_path
from scrapy.utils.python import to_bytes, to_unicode
+from scrapy.utils.response import response_from_dict
if TYPE_CHECKING:
import os
@@ -272,24 +272,13 @@ class DbmCacheStorage:
data = self._read_data(spider, request)
if data is None:
return None # not cached
- url = data["url"]
- status = data["status"]
- headers = Headers(data["headers"])
- body = data["body"]
- respcls = responsetypes.from_args(headers=headers, url=url, body=body)
- return respcls(url=url, headers=headers, status=status, body=body)
+ return response_from_dict(data)
def store_response(
self, spider: Spider, request: Request, response: Response
) -> None:
key = self._fingerprinter.fingerprint(request).hex()
- data = {
- "status": response.status,
- "url": response.url,
- "headers": dict(response.headers),
- "body": response.body,
- }
- self.db[f"{key}_data"] = pickle.dumps(data, protocol=4)
+ self.db[f"{key}_data"] = pickle.dumps(response.to_dict(), protocol=4)
self.db[f"{key}_time"] = str(time())
def _read_data(self, spider: Spider, request: Request) -> dict[str, Any] | None:
@@ -340,11 +329,17 @@ class FilesystemCacheStorage:
body = f.read()
with self._open(rpath / "response_headers", "rb") as f:
rawheaders = f.read()
- url = metadata["response_url"]
- status = metadata["status"]
- headers = Headers(headers_raw_to_dict(rawheaders))
- respcls = responsetypes.from_args(headers=headers, url=url, body=body)
- return respcls(url=url, headers=headers, status=status, body=body)
+ data = {
+ "url": metadata["response_url"],
+ "status": metadata["status"],
+ "headers": headers_raw_to_dict(rawheaders),
+ "body": body,
+ }
+ datapath = rpath / "response_data"
+ if datapath.exists():
+ with self._open(datapath, "rb") as f:
+ data.update(pickle.load(f)) # noqa: S301
+ return response_from_dict(data)
def store_response(
self, spider: Spider, request: Request, response: Response
@@ -368,6 +363,13 @@ class FilesystemCacheStorage:
f.write(headers_dict_to_raw(response.headers))
with self._open(rpath / "response_body", "wb") as f:
f.write(response.body)
+ data = {
+ key: value
+ for key, value in response.to_dict().items()
+ if key not in {"url", "status", "headers", "body"}
+ }
+ with self._open(rpath / "response_data", "wb") as f:
+ pickle.dump(data, f, protocol=4)
with self._open(rpath / "request_headers", "wb") as f:
f.write(headers_dict_to_raw(request.headers))
with self._open(rpath / "request_body", "wb") as f:
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index f91cb2095..2f148ff84 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -194,6 +194,37 @@ class Response(object_ref):
cls = self.__class__
return cls(*args, **kwargs)
+ def to_dict(self) -> dict[str, Any]:
+ """Return a dictionary containing the Response's data.
+
+ .. versionadded:: VERSION
+
+ Use :func:`~scrapy.utils.response.response_from_dict` to convert back
+ into a :class:`~scrapy.http.Response` object.
+
+ :attr:`request` and :attr:`certificate` are left out, as they are tied
+ to a single crawl. Everything else in :attr:`attributes` is included,
+ so subclasses only need to override this method, and :meth:`from_dict`,
+ if some of their attributes cannot be stored as is.
+ """
+ d: dict[str, Any] = {"headers": dict(self.headers)}
+ for attr in self.attributes:
+ if attr in {"request", "certificate"}:
+ continue
+ d.setdefault(attr, getattr(self, attr))
+ if type(self) is not Response: # pylint: disable=unidiomatic-typecheck
+ d["_class"] = self.__module__ + "." + self.__class__.__name__
+ return d
+
+ @classmethod
+ def from_dict(cls, d: dict[str, Any]) -> Self:
+ """Return a response built from the *d* dict, as returned by
+ :meth:`to_dict`.
+
+ .. versionadded:: VERSION
+ """
+ return cls(**{key: value for key, value in d.items() if key != "_class"})
+
def urljoin(self, url: str) -> str:
"""Join this Response's url with a possible relative url to form an
absolute interpretation of the latter."""
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index c9aa416f1..65e95253a 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -16,6 +16,8 @@ from weakref import WeakKeyDictionary
from twisted.web import http
from w3lib import html
+from scrapy.http.headers import Headers
+from scrapy.utils.misc import load_object
from scrapy.utils.python import to_bytes, to_unicode
if TYPE_CHECKING:
@@ -54,6 +56,30 @@ def get_meta_refresh(
return _metaref_cache[response]
+def response_from_dict(d: dict[str, Any]) -> Response:
+ """Return a response built from the *d* dict, as returned by
+ :meth:`Response.to_dict() <scrapy.http.Response.to_dict>`.
+
+ .. versionadded:: VERSION
+
+ If *d* does not indicate a response class, e.g. because it comes from a
+ plain :class:`~scrapy.http.Response` object or predates :meth:`~scrapy.http.Response.to_dict`,
+ the class is guessed with :attr:`~scrapy.responsetypes.responsetypes`.
+ """
+ # Imported here to avoid a circular import.
+ from scrapy.responsetypes import responsetypes # noqa: PLC0415
+
+ d = {**d, "headers": Headers(d.get("headers") or {})}
+ response_cls: type[Response] = (
+ load_object(d["_class"])
+ if "_class" in d
+ else responsetypes.from_args(
+ headers=d["headers"], url=d["url"], body=d.get("body")
+ )
+ )
+ return response_cls.from_dict(d)
+
+
def response_status_message(status: bytes | float | str) -> str:
"""Return status code plus status text descriptive message"""
status_int = int(status)
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index c48f77ea5..6baf97896 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import email.utils
import logging
+import pickle
import shutil
import tempfile
import time
@@ -26,6 +27,14 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
+class CustomResponse(Response):
+ attributes: tuple[str, ...] = (*Response.attributes, "custom")
+
+ def __init__(self, *args: Any, custom: str | None = None, **kwargs: Any):
+ self.custom = custom
+ super().__init__(*args, **kwargs)
+
+
class AlwaysStalePolicy(DummyPolicy):
""":class:`~scrapy.extensions.httpcache.DummyPolicy` that always
revalidates cached responses."""
@@ -110,6 +119,12 @@ class StorageTestMixin(TestBase):
"""Make the cache entry of *request* unreadable for *storage*."""
raise NotImplementedError
+ def _downgrade_cache_entry(
+ self, storage: Any, spider: Spider, request: Request
+ ) -> None:
+ """Rewrite the cache entry of *request* as Scrapy 2.14 would have."""
+ raise NotImplementedError
+
def test_storage(self):
with self._storage(HTTPCACHE_EXPIRATION_SECS=100) as (storage, crawler):
request2 = self.request.copy()
@@ -181,6 +196,38 @@ class StorageTestMixin(TestBase):
assert isinstance(cached_response, HtmlResponse)
self.assertEqualResponse(response, cached_response)
+ def test_storage_response_class(self):
+ with self._storage() as (storage, crawler):
+ response = CustomResponse(
+ "http://www.example.com", body=b"test body", custom="value"
+ )
+ storage.store_response(crawler.spider, self.request, response)
+ cached_response = storage.retrieve_response(crawler.spider, self.request)
+ assert isinstance(cached_response, CustomResponse)
+ assert cached_response.custom == "value"
+
+ def test_storage_encoding(self):
+ """The encoding of the stored response is kept even when it cannot be
+ inferred from the response data."""
+ with self._storage() as (storage, crawler):
+ response = HtmlResponse(
+ "http://www.example.com",
+ body='<meta charset="iso-8859-1">€'.encode(),
+ encoding="utf-8",
+ )
+ storage.store_response(crawler.spider, self.request, response)
+ cached_response = storage.retrieve_response(crawler.spider, self.request)
+ assert cached_response.encoding == "utf-8"
+ assert cached_response.text == response.text
+
+ def test_storage_old_cache_entry(self):
+ with self._storage() as (storage, crawler):
+ storage.store_response(crawler.spider, self.request, self.response)
+ self._downgrade_cache_entry(storage, crawler.spider, self.request)
+ cached_response = storage.retrieve_response(crawler.spider, self.request)
+ assert isinstance(cached_response, HtmlResponse)
+ self.assertEqualResponse(self.response, cached_response)
+
class PolicyTestMixin(TestBase):
"""Mixin containing policy-specific test methods."""
@@ -708,6 +755,10 @@ class FilesystemStorageTestMixin(StorageTestMixin):
rpath = Path(storage._get_request_path(spider, request))
(rpath / "response_body").unlink()
+ def _downgrade_cache_entry(self, storage, spider, request) -> None:
+ rpath = Path(storage._get_request_path(spider, request))
+ (rpath / "response_data").unlink()
+
class DbmStorageTestMixin(StorageTestMixin):
storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
@@ -716,6 +767,14 @@ class DbmStorageTestMixin(StorageTestMixin):
key = storage._fingerprinter.fingerprint(request).hex()
storage.db[f"{key}_data"] = b"not a pickle"
+ def _downgrade_cache_entry(self, storage, spider, request) -> None:
+ key = storage._fingerprinter.fingerprint(request).hex()
+ data = pickle.loads(storage.db[f"{key}_data"])
+ data = {
+ k: v for k, v in data.items() if k in ("status", "url", "headers", "body")
+ }
+ storage.db[f"{key}_data"] = pickle.dumps(data, protocol=4)
+
class TestFilesystemStorageWithDummyPolicy(
FilesystemStorageTestMixin, DummyPolicyTestMixin
diff --git a/tests/test_response_dict.py b/tests/test_response_dict.py
new file mode 100644
index 000000000..e258a2316
--- /dev/null
+++ b/tests/test_response_dict.py
@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+from ipaddress import ip_address
+
+from scrapy import Request
+from scrapy.http import HtmlResponse, Response, TextResponse
+from scrapy.utils.response import response_from_dict
+
+
+class CustomResponse(TextResponse):
+ attributes: tuple[str, ...] = (*TextResponse.attributes, "custom")
+
+ def __init__(self, *args, custom: str | None = None, **kwargs):
+ self.custom = custom
+ super().__init__(*args, **kwargs)
+
+
+class DynamicResponse(Response):
+ """Response subclass that extends
+ :attr:`~scrapy.http.Response.attributes` on instances, as plugins that
+ support several Scrapy versions do."""
+
+ def __init__(self, *args, custom: str | None = None, **kwargs):
+ self.custom = custom
+ super().__init__(*args, **kwargs)
+ self.attributes = (*self.attributes, "custom")
+
+
+def test_basic() -> None:
+ """The class of plain responses is not stored, it is guessed back from the
+ response data."""
+ response = Response("https://example.com", body=b"\x00\x01")
+ assert "_class" not in response.to_dict()
+ response2 = response_from_dict(response.to_dict())
+ assert response2.__class__ is Response
+ assert response2.url == response.url
+
+
+def test_all_attributes() -> None:
+ response = HtmlResponse(
+ url="https://example.com",
+ status=201,
+ headers={"Content-Type": "text/html; charset=latin-1"},
+ body=b"\xa3",
+ flags=["testFlag"],
+ encoding="latin-1",
+ ip_address=ip_address("127.0.0.1"),
+ protocol="h2",
+ )
+ response2 = response_from_dict(response.to_dict())
+ assert response2.__class__ is HtmlResponse
+ for attribute in HtmlResponse.attributes:
+ if attribute in {"request", "certificate"}:
+ continue
+ assert getattr(response2, attribute) == getattr(response, attribute)
+
+
+def test_custom_attributes() -> None:
+ response = CustomResponse("https://example.com", custom="value")
+ response2 = response_from_dict(response.to_dict())
+ assert isinstance(response2, CustomResponse)
+ assert response2.custom == "value"
+
+
+def test_custom_instance_attributes() -> None:
+ response = DynamicResponse("https://example.com", custom="value")
+ response2 = response_from_dict(response.to_dict())
+ assert isinstance(response2, DynamicResponse)
+ assert response2.custom == "value"
+
+
+def test_crawl_attributes_left_out() -> None:
+ response = Response(
+ "https://example.com",
+ request=Request("https://example.com"),
+ certificate=object(),
+ )
+ d = response.to_dict()
+ assert "request" not in d
+ assert "certificate" not in d
+ response2 = response_from_dict(d)
+ assert response2.request is None
+ assert response2.certificate is None
+
+
+def test_unknown_class() -> None:
+ """Dicts that do not indicate a response class, e.g. cache entries written
+ by older Scrapy versions, get a class based on their data."""
+ response2 = response_from_dict(
+ {
+ "url": "https://example.com",
+ "status": 200,
+ "headers": {b"Content-Type": [b"text/plain"]},
+ "body": b"foo",
+ }
+ )
+ assert response2.__class__ is TextResponse
+ assert response2.text == "foo"