🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(spiders): keep the request meta on cached responses (#419)
Yiğit ERDOĞAN63cdc99cfdbc
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.pyindex 2d19016..5fa58f0 100644--- a/scrapling/spiders/engine.py+++ b/scrapling/spiders/engine.py@@ -200,6 +200,8 @@ class CrawlerEngine:cached = await self._cache_manager.get(request._fp)if cached is not None:cached.request = request+ # Cached responses are rebuilt without meta, so merge the request's in as the live path does+ cached.meta = {**request.meta, **cached.meta}self.stats.cache_hits += 1self.stats.increment_requests_count(request.sid or self.session_manager.default_session_id)self.stats.increment_response_bytes(request.domain, len(cached.body))diff --git a/tests/spiders/test_cache.py b/tests/spiders/test_cache.pyindex b29114c..b47e4aa 100644--- a/tests/spiders/test_cache.py+++ b/tests/spiders/test_cache.py@@ -34,7 +34,6 @@ def _make_response(class TestResponseCacheManager:-@pytest.mark.anyioasync def test_put_get_roundtrip(self):with tempfile.TemporaryDirectory() as tmpdir:@@ -268,7 +267,6 @@ class MockSpider:class TestDevelopmentModeIntegration:-@pytest.mark.anyioasync def test_first_run_fetches_and_caches(self):with tempfile.TemporaryDirectory() as tmpdir:@@ -317,3 +315,36 @@ class TestDevelopmentModeIntegration:sm.add("default", MockSession())engine = CrawlerEngine(spider, sm)assert engine._cache_manager is None++ @pytest.mark.anyio+ async def test_cached_run_keeps_the_request_meta(self):+ """A cache hit must hand the callback the same meta the live fetch did, not an empty one"""++ class MetaSpider(MockSpider):+ async def parse(self, response) -> AsyncGenerator[Dict[str, Any] | Request | None, None]:+ yield {"meta": dict(response.meta)}++ async def start_requests(self) -> AsyncGenerator[Request, None]:+ yield Request("https://example.com/page1", sid="default", meta={"page": 7})++ with tempfile.TemporaryDirectory() as tmpdir:+ spider = MetaSpider(cache_dir=tmpdir)+ sm = SessionManager()+ sm.add("default", MockSession())+ await CrawlerEngine(spider, sm).crawl()++ cached_spider = MetaSpider(cache_dir=tmpdir)+ cached_sm = SessionManager()+ cached_sm.add("default", MockSession())+ cached_engine = CrawlerEngine(cached_spider, cached_sm)+ await cached_engine.crawl()++ assert cached_engine.stats.cache_hits == 1, (+ f"Expected the second run to be served from the cache, got {cached_engine.stats.cache_hits} hits"+ )+ assert spider.scraped_items == [{"meta": {"page": 7}}], (+ f"Expected the live run to see the request meta, got {spider.scraped_items}"+ )+ assert cached_spider.scraped_items == [{"meta": {"page": 7}}], (+ f"Expected the cached run to see the request meta too, got {cached_spider.scraped_items}"+ )