🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(spiders): Change items hook to make it for processing items + add a stat for this
Karim shoairafe6b2e97bbd
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.pyindex e869e7f..751d87b 100644--- a/scrapling/spiders/engine.py+++ b/scrapling/spiders/engine.py@@ -102,12 +102,17 @@ class CrawlerEngine:self.stats.offsite_requests_count += 1log.debug(f"Filtered offsite request to: {result.url}")elif isinstance(result, dict):- self.stats.items_scraped += 1- self._items.append(result)- if self._item_stream:- await self._item_stream.send(result)- await self.spider.on_scraped_item(result)- log.debug(f"Scraped from {str(response)}\n{result}")+ processed_result = await self.spider.on_scraped_item(result)+ if processed_result:+ self.stats.items_scraped += 1+ log.debug(f"Scraped from {str(response)}\n{processed_result}")+ if self._item_stream:+ await self._item_stream.send(processed_result)+ else:+ self._items.append(processed_result)+ else:+ self.stats.items_dropped += 1+ log.warning(f"Dropped from {str(response)}\n{processed_result}")elif result is not None:log.error(f"Spider must return Request, dict or None, got '{type(result)}' in {request}")except Exception as e:diff --git a/scrapling/spiders/result.py b/scrapling/spiders/result.pyindex fc10f1b..7e38a81 100644--- a/scrapling/spiders/result.py+++ b/scrapling/spiders/result.py@@ -50,6 +50,7 @@ class CrawlStats:offsite_requests_count: int = 0response_bytes: int = 0items_scraped: int = 0+ items_dropped: int = 0start_time: float = 0.0end_time: float = 0.0download_delay: float = 0.0@@ -85,6 +86,7 @@ class CrawlStats:def to_dict(self) -> dict[str, Any]:return {"items_scraped": self.items_scraped,+ "items_dropped": self.items_dropped,"elapsed_seconds": round(self.elapsed_seconds, 2),"download_delay": round(self.download_delay, 2),"concurrent_requests": self.concurrent_requests,diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.pyindex 59ab61c..ac7174f 100644--- a/scrapling/spiders/spider.py+++ b/scrapling/spiders/spider.py@@ -160,9 +160,9 @@ class Spider(ABC):"""self.logger.error(error, exc_info=error)- async def on_scraped_item(self, item: dict[str, Any]) -> None:- """Handle a scraped item. Override or extend for item pipelines."""- pass+ async def on_scraped_item(self, item: Dict[str, Any]) -> Dict[str, Any] | None:+ """A hook to be overridden by users to do some processing on scraped items, return `None` to drop the item silently."""+ return itemasync def is_blocked(self, response: "Response") -> bool:"""Check if the response is blocked. Users should override this for custom detection logic."""