🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat: add hook to run after collecting an item
Karim shoair3466bcab2c02
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.pyindex bd50c41..2b12683 100644--- a/scrapling/spiders/engine.py+++ b/scrapling/spiders/engine.py@@ -102,18 +102,15 @@ class CrawlerEngine:self.stats.offsite_requests_count += 1log.debug(f"Filtered offsite request to: {result.url}")elif isinstance(result, dict):- await self._handle_item(result)+ self.stats.items_scraped += 1+ self._items.append(result)if self._item_stream:await self._item_stream.send(result)+ await self.spider.on_scraped_item(result)log.debug(f"Scraped from {str(response)}\n{result}")except Exception as e:await self.spider.on_error(request, e)- async def _handle_item(self, item: dict[str, Any]) -> None:- """Handle a scraped item. Override or extend for item pipelines."""- self.stats.items_scraped += 1- self._items.append(item)-async def _task_wrapper(self, request: Request) -> None:"""Wrapper to track active task count."""try:diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.pyindex eb5987f..bc7874c 100644--- a/scrapling/spiders/spider.py+++ b/scrapling/spiders/spider.py@@ -158,6 +158,10 @@ class Spider(ABC):"""self.logger.error(error, exc_info=error)+ async def on_scraped_item(self, item: dict[str, Any]) -> None:+ """Handle a scraped item. Override or extend for item pipelines."""+ pass+async def is_blocked(self, response: "Response") -> bool:"""Check if the response is blocked. Users should override this for custom detection logic."""if response.status in BLOCKED_CODES: