Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(spiders): Change items hook to make it for processing items + add a stat for this

Karim shoairafe6b2e97bbd
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py
index e869e7f..751d87b 100644
--- a/scrapling/spiders/engine.py
+++ b/scrapling/spiders/engine.py
@@ -102,12 +102,17 @@ class CrawlerEngine:
self.stats.offsite_requests_count += 1
log.debug(f"Filtered offsite request to: {result.url}")
elif isinstance(result, dict):
- self.stats.items_scraped += 1
- self._items.append(result)
- if self._item_stream:
- await self._item_stream.send(result)
- await self.spider.on_scraped_item(result)
- log.debug(f"Scraped from {str(response)}\n{result}")
+ processed_result = await self.spider.on_scraped_item(result)
+ if processed_result:
+ self.stats.items_scraped += 1
+ log.debug(f"Scraped from {str(response)}\n{processed_result}")
+ if self._item_stream:
+ await self._item_stream.send(processed_result)
+ else:
+ self._items.append(processed_result)
+ else:
+ self.stats.items_dropped += 1
+ log.warning(f"Dropped from {str(response)}\n{processed_result}")
elif result is not None:
log.error(f"Spider must return Request, dict or None, got '{type(result)}' in {request}")
except Exception as e:
diff --git a/scrapling/spiders/result.py b/scrapling/spiders/result.py
index fc10f1b..7e38a81 100644
--- a/scrapling/spiders/result.py
+++ b/scrapling/spiders/result.py
@@ -50,6 +50,7 @@ class CrawlStats:
offsite_requests_count: int = 0
response_bytes: int = 0
items_scraped: int = 0
+ items_dropped: int = 0
start_time: float = 0.0
end_time: float = 0.0
download_delay: float = 0.0
@@ -85,6 +86,7 @@ class CrawlStats:
def to_dict(self) -> dict[str, Any]:
return {
"items_scraped": self.items_scraped,
+ "items_dropped": self.items_dropped,
"elapsed_seconds": round(self.elapsed_seconds, 2),
"download_delay": round(self.download_delay, 2),
"concurrent_requests": self.concurrent_requests,
diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.py
index 59ab61c..ac7174f 100644
--- a/scrapling/spiders/spider.py
+++ b/scrapling/spiders/spider.py
@@ -160,9 +160,9 @@ class Spider(ABC):
"""
self.logger.error(error, exc_info=error)
- async def on_scraped_item(self, item: dict[str, Any]) -> None:
- """Handle a scraped item. Override or extend for item pipelines."""
- pass
+ async def on_scraped_item(self, item: Dict[str, Any]) -> Dict[str, Any] | None:
+ """A hook to be overridden by users to do some processing on scraped items, return `None` to drop the item silently."""
+ return item
async def is_blocked(self, response: "Response") -> bool:
"""Check if the response is blocked. Users should override this for custom detection logic."""