Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add item to the extra kwargs when logging item processing (#8085)

Adriand9d37a6adad2
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 2d31d31ac..5baa36907 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -517,7 +517,8 @@ class Scraper:
logkws = self.logformatter.dropped(item, ex, response, self.crawler.spider)
if logkws is not None:
logger.log(
- *logformatter_adapter(logkws), extra={"spider": self.crawler.spider}
+ *logformatter_adapter(logkws),
+ extra={"spider": self.crawler.spider, "item": item},
)
await self.signals.send_catch_log_async(
signal=signals.item_dropped,
@@ -532,7 +533,7 @@ class Scraper:
)
logger.log(
*logformatter_adapter(logkws),
- extra={"spider": self.crawler.spider},
+ extra={"spider": self.crawler.spider, "item": item},
exc_info=True,
)
await self.signals.send_catch_log_async(
@@ -546,7 +547,8 @@ class Scraper:
logkws = self.logformatter.scraped(output, response, self.crawler.spider)
if logkws is not None:
logger.log(
- *logformatter_adapter(logkws), extra={"spider": self.crawler.spider}
+ *logformatter_adapter(logkws),
+ extra={"spider": self.crawler.spider, "item": output},
)
await self.signals.send_catch_log_async(
signal=signals.item_scraped,
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index dc70e6413..8da842da5 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -277,6 +277,22 @@ class TestShowOrSkipMessages:
assert "Crawled (200) <GET http://127.0.0.1:" in caplog.text
assert "Dropped: Ignoring item" in caplog.text
+ @coroutine_test
+ async def test_item_in_extra(
+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+ ) -> None:
+ crawler = get_crawler(ItemSpider, self.base_settings)
+ with caplog.at_level(logging.DEBUG):
+ await crawler.crawl_async(mockserver=mockserver)
+ scraped_record = next(
+ r for r in caplog.records if r.message.startswith("Scraped from")
+ )
+ dropped_record = next(
+ r for r in caplog.records if r.message.startswith("Dropped: Ignoring item")
+ )
+ assert hasattr(scraped_record, "item")
+ assert isinstance(cast("Any", dropped_record).item, (Item, dict))
+
@coroutine_test
async def test_skip_messages(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer