Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(spiders): keep the request meta on cached responses (#419)

Yiğit ERDOĞAN63cdc99cfdbc
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py
index 2d19016..5fa58f0 100644
--- a/scrapling/spiders/engine.py
+++ b/scrapling/spiders/engine.py
@@ -200,6 +200,8 @@ class CrawlerEngine:
cached = await self._cache_manager.get(request._fp)
if cached is not None:
cached.request = request
+ # Cached responses are rebuilt without meta, so merge the request's in as the live path does
+ cached.meta = {**request.meta, **cached.meta}
self.stats.cache_hits += 1
self.stats.increment_requests_count(request.sid or self.session_manager.default_session_id)
self.stats.increment_response_bytes(request.domain, len(cached.body))
diff --git a/tests/spiders/test_cache.py b/tests/spiders/test_cache.py
index b29114c..b47e4aa 100644
--- a/tests/spiders/test_cache.py
+++ b/tests/spiders/test_cache.py
@@ -34,7 +34,6 @@ def _make_response(
class TestResponseCacheManager:
-
@pytest.mark.anyio
async def test_put_get_roundtrip(self):
with tempfile.TemporaryDirectory() as tmpdir:
@@ -268,7 +267,6 @@ class MockSpider:
class TestDevelopmentModeIntegration:
-
@pytest.mark.anyio
async def test_first_run_fetches_and_caches(self):
with tempfile.TemporaryDirectory() as tmpdir:
@@ -317,3 +315,36 @@ class TestDevelopmentModeIntegration:
sm.add("default", MockSession())
engine = CrawlerEngine(spider, sm)
assert engine._cache_manager is None
+
+ @pytest.mark.anyio
+ async def test_cached_run_keeps_the_request_meta(self):
+ """A cache hit must hand the callback the same meta the live fetch did, not an empty one"""
+
+ class MetaSpider(MockSpider):
+ async def parse(self, response) -> AsyncGenerator[Dict[str, Any] | Request | None, None]:
+ yield {"meta": dict(response.meta)}
+
+ async def start_requests(self) -> AsyncGenerator[Request, None]:
+ yield Request("https://example.com/page1", sid="default", meta={"page": 7})
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ spider = MetaSpider(cache_dir=tmpdir)
+ sm = SessionManager()
+ sm.add("default", MockSession())
+ await CrawlerEngine(spider, sm).crawl()
+
+ cached_spider = MetaSpider(cache_dir=tmpdir)
+ cached_sm = SessionManager()
+ cached_sm.add("default", MockSession())
+ cached_engine = CrawlerEngine(cached_spider, cached_sm)
+ await cached_engine.crawl()
+
+ assert cached_engine.stats.cache_hits == 1, (
+ f"Expected the second run to be served from the cache, got {cached_engine.stats.cache_hits} hits"
+ )
+ assert spider.scraped_items == [{"meta": {"page": 7}}], (
+ f"Expected the live run to see the request meta, got {spider.scraped_items}"
+ )
+ assert cached_spider.scraped_items == [{"meta": {"page": 7}}], (
+ f"Expected the cached run to see the request meta too, got {cached_spider.scraped_items}"
+ )