Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat: add hook to run after collecting an item

Karim shoair3466bcab2c02
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py
index bd50c41..2b12683 100644
--- a/scrapling/spiders/engine.py
+++ b/scrapling/spiders/engine.py
@@ -102,18 +102,15 @@ class CrawlerEngine:
self.stats.offsite_requests_count += 1
log.debug(f"Filtered offsite request to: {result.url}")
elif isinstance(result, dict):
- await self._handle_item(result)
+ self.stats.items_scraped += 1
+ self._items.append(result)
if self._item_stream:
await self._item_stream.send(result)
+ await self.spider.on_scraped_item(result)
log.debug(f"Scraped from {str(response)}\n{result}")
except Exception as e:
await self.spider.on_error(request, e)
- async def _handle_item(self, item: dict[str, Any]) -> None:
- """Handle a scraped item. Override or extend for item pipelines."""
- self.stats.items_scraped += 1
- self._items.append(item)
-
async def _task_wrapper(self, request: Request) -> None:
"""Wrapper to track active task count."""
try:
diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.py
index eb5987f..bc7874c 100644
--- a/scrapling/spiders/spider.py
+++ b/scrapling/spiders/spider.py
@@ -158,6 +158,10 @@ class Spider(ABC):
"""
self.logger.error(error, exc_info=error)
+ async def on_scraped_item(self, item: dict[str, Any]) -> None:
+ """Handle a scraped item. Override or extend for item pipelines."""
+ pass
+
async def is_blocked(self, response: "Response") -> bool:
"""Check if the response is blocked. Users should override this for custom detection logic."""
if response.status in BLOCKED_CODES: