Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

refactor tests/test_downloadermiddleware_httpcache.py (#6769)

Felipe Benevolo128cb551eb49
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 5fac88ed7..02f4f488e 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -15,8 +15,7 @@ from scrapy.utils.test import get_crawler
class TestBase:
- storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"
- policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
+ """Base class with common setup and helper methods."""
def setup_method(self):
self.yesterday = email.utils.formatdate(time.time() - 86400)
@@ -90,23 +89,10 @@ class TestBase:
)
assert request1.body == request2.body
- def test_dont_cache(self):
- with self._middleware() as mw:
- self.request.meta["dont_cache"] = True
- mw.process_response(self.request, self.response, self.spider)
- assert mw.storage.retrieve_response(self.spider, self.request) is None
-
- with self._middleware() as mw:
- self.request.meta["dont_cache"] = False
- mw.process_response(self.request, self.response, self.spider)
- if mw.policy.should_cache_response(self.response, self.request):
- assert isinstance(
- mw.storage.retrieve_response(self.spider, self.request),
- self.response.__class__,
- )
+class StorageTestMixin:
+ """Mixin containing storage-specific test methods."""
-class TestDefaultStorage(TestBase):
def test_storage(self):
with self._storage() as storage:
request2 = self.request.copy()
@@ -143,31 +129,27 @@ class TestDefaultStorage(TestBase):
self.assertEqualResponse(response, cached_response)
-class TestDbmStorage(TestDefaultStorage):
- storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
-
-
-class TestDbmStorageWithCustomDbmModule(TestDbmStorage):
- dbm_module = "tests.mocks.dummydbm"
-
- def _get_settings(self, **new_settings):
- new_settings.setdefault("HTTPCACHE_DBM_MODULE", self.dbm_module)
- return super()._get_settings(**new_settings)
-
- def test_custom_dbm_module_loaded(self):
- # make sure our dbm module has been loaded
- with self._storage() as storage:
- assert storage.dbmodule.__name__ == self.dbm_module
+class PolicyTestMixin:
+ """Mixin containing policy-specific test methods."""
+ def test_dont_cache(self):
+ with self._middleware() as mw:
+ self.request.meta["dont_cache"] = True
+ mw.process_response(self.request, self.response, self.spider)
+ assert mw.storage.retrieve_response(self.spider, self.request) is None
-class TestFilesystemStorageGzip(TestDefaultStorage):
- def _get_settings(self, **new_settings):
- new_settings.setdefault("HTTPCACHE_GZIP", True)
- return super()._get_settings(**new_settings)
+ with self._middleware() as mw:
+ self.request.meta["dont_cache"] = False
+ mw.process_response(self.request, self.response, self.spider)
+ if mw.policy.should_cache_response(self.response, self.request):
+ assert isinstance(
+ mw.storage.retrieve_response(self.spider, self.request),
+ self.response.__class__,
+ )
-class TestDummyPolicy(TestBase):
- policy_class = "scrapy.extensions.httpcache.DummyPolicy"
+class DummyPolicyTestMixin(PolicyTestMixin):
+ """Mixin containing dummy policy specific test methods."""
def test_middleware(self):
with self._middleware() as mw:
@@ -258,8 +240,8 @@ class TestDummyPolicy(TestBase):
assert "cached" in response.flags
-class TestRFC2616Policy(TestDefaultStorage):
- policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
+class RFC2616PolicyTestMixin(PolicyTestMixin):
+ """Mixin containing RFC2616 policy specific test methods."""
def _process_requestresponse(self, mw, request, response):
result = None
@@ -562,3 +544,51 @@ class TestRFC2616Policy(TestDefaultStorage):
res2 = self._process_requestresponse(mw, req0, None)
self.assertEqualResponse(res1, res2)
assert "cached" in res2.flags
+
+
+# Concrete test classes that combine storage and policy mixins
+
+
+class TestFilesystemStorageWithDummyPolicy(
+ TestBase, StorageTestMixin, DummyPolicyTestMixin
+):
+ storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"
+ policy_class = "scrapy.extensions.httpcache.DummyPolicy"
+
+
+class TestFilesystemStorageWithRFC2616Policy(
+ TestBase, StorageTestMixin, RFC2616PolicyTestMixin
+):
+ storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"
+ policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
+
+
+class TestDbmStorageWithDummyPolicy(TestBase, StorageTestMixin, DummyPolicyTestMixin):
+ storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
+ policy_class = "scrapy.extensions.httpcache.DummyPolicy"
+
+
+class TestDbmStorageWithRFC2616Policy(
+ TestBase, StorageTestMixin, RFC2616PolicyTestMixin
+):
+ storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
+ policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
+
+
+class TestDbmStorageWithCustomDbmModule(TestDbmStorageWithDummyPolicy):
+ dbm_module = "tests.mocks.dummydbm"
+
+ def _get_settings(self, **new_settings):
+ new_settings.setdefault("HTTPCACHE_DBM_MODULE", self.dbm_module)
+ return super()._get_settings(**new_settings)
+
+ def test_custom_dbm_module_loaded(self):
+ # make sure our dbm module has been loaded
+ with self._storage() as storage:
+ assert storage.dbmodule.__name__ == self.dbm_module
+
+
+class TestFilesystemStorageGzipWithDummyPolicy(TestFilesystemStorageWithDummyPolicy):
+ def _get_settings(self, **new_settings):
+ new_settings.setdefault("HTTPCACHE_GZIP", True)
+ return super()._get_settings(**new_settings)