🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(spiders): cache responses only after blocked-response checks (#446)
Kuang-xianxince294063e964
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.pyindex 5fa58f0..1b38b39 100644--- a/scrapling/spiders/engine.py+++ b/scrapling/spiders/engine.py@@ -236,7 +236,6 @@ class CrawlerEngine:if self._cache_manager and request._fp is not None:self.stats.cache_misses += 1- await self._cache_manager.put(request._fp, response, request._session_kwargs.get("method", "GET"))blocked = await self.spider.is_blocked(response)if self._autothrottle:@@ -264,6 +263,9 @@ class CrawlerEngine:log.warning(f"Max retries exceeded for blocked request: {request.url}")return+ if self._cache_manager and request._fp is not None:+ await self._cache_manager.put(request._fp, response, request._session_kwargs.get("method", "GET"))+await self._run_callbacks(request, response)async def _task_wrapper(self, request: Request) -> None:diff --git a/tests/spiders/test_cache.py b/tests/spiders/test_cache.pyindex b47e4aa..b525224 100644--- a/tests/spiders/test_cache.py+++ b/tests/spiders/test_cache.py@@ -267,6 +267,83 @@ class MockSpider:class TestDevelopmentModeIntegration:+ @pytest.mark.anyio+ @pytest.mark.parametrize("blocked_status", [403, 200])+ async def test_blocked_response_is_not_replayed_for_retry(self, tmp_path: Path, blocked_status: int) -> None:+ """Retries fetch again, and only the accepted response is replayed later."""++ class RecoveringSession(MockSession):+ async def fetch(self, url: str, **kwargs: Any) -> Response:+ """Return a blocked response followed by an accepted response."""+ self.fetch_count += 1+ response = _make_response(+ url=url,+ body=b"blocked" if self.fetch_count == 1 else b"success",+ status=blocked_status if self.fetch_count == 1 else 200,+ )+ response.meta["live_only"] = True+ return response++ class RetryingSpider(MockSpider):+ async def is_blocked(self, response: Response) -> bool:+ """Recognize blocked content independently of the HTTP status."""+ # Custom hooks may rely on live metadata that the disk cache omits.+ assert response.meta["live_only"] is True+ return response.body == b"blocked"++ async def parse(self, response: Response) -> AsyncGenerator[Dict[str, Any] | Request | None, None]:+ """Expose the body actually delivered to the callback."""+ yield {"body": response.body}++ session = RecoveringSession()+ spider = RetryingSpider(cache_dir=str(tmp_path))+ sm = SessionManager()+ sm.add("default", session)+ engine = CrawlerEngine(spider, sm)++ await engine.crawl()++ assert session.fetch_count == 2+ assert spider.scraped_items == [{"body": b"success"}]+ assert engine.stats.blocked_requests_count == 1+ assert engine.stats.cache_hits == 0+ assert engine.stats.cache_misses == 2++ # A later run should still replay the successful response.+ cached_session = MockSession()+ cached_spider = RetryingSpider(cache_dir=str(tmp_path))+ cached_sm = SessionManager()+ cached_sm.add("default", cached_session)+ cached_engine = CrawlerEngine(cached_spider, cached_sm)+ await cached_engine.crawl()++ assert cached_session.fetch_count == 0+ assert cached_spider.scraped_items == [{"body": b"success"}]+ assert cached_engine.stats.cache_hits == 1++ @pytest.mark.anyio+ async def test_exhausted_blocked_response_is_not_cached(self, tmp_path: Path) -> None:+ """Exhausting retries must not leave a blocked response on disk."""++ class BlockedSpider(MockSpider):+ async def is_blocked(self, response: Response) -> bool:+ """Reject every response for this crawl."""+ return True++ session = MockSession()+ spider = BlockedSpider(cache_dir=str(tmp_path))+ spider.max_blocked_retries = 0+ sm = SessionManager()+ sm.add("default", session)+ engine = CrawlerEngine(spider, sm)+ await engine.crawl()++ assert session.fetch_count == 1+ assert engine.stats.blocked_requests_count == 1+ assert engine.stats.cache_misses == 1+ assert not spider.scraped_items+ assert not list(tmp_path.glob("*.json"))+@pytest.mark.anyioasync def test_first_run_fetches_and_caches(self):with tempfile.TemporaryDirectory() as tmpdir: