Scrapy, a fast high-level web crawling & scraping framework for Python.
Add item to the extra kwargs when logging item processing (#8085)
Adriand9d37a6adad2
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 2d31d31ac..5baa36907 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -517,7 +517,8 @@ class Scraper:logkws = self.logformatter.dropped(item, ex, response, self.crawler.spider)if logkws is not None:logger.log(- *logformatter_adapter(logkws), extra={"spider": self.crawler.spider}+ *logformatter_adapter(logkws),+ extra={"spider": self.crawler.spider, "item": item},)await self.signals.send_catch_log_async(signal=signals.item_dropped,@@ -532,7 +533,7 @@ class Scraper:)logger.log(*logformatter_adapter(logkws),- extra={"spider": self.crawler.spider},+ extra={"spider": self.crawler.spider, "item": item},exc_info=True,)await self.signals.send_catch_log_async(@@ -546,7 +547,8 @@ class Scraper:logkws = self.logformatter.scraped(output, response, self.crawler.spider)if logkws is not None:logger.log(- *logformatter_adapter(logkws), extra={"spider": self.crawler.spider}+ *logformatter_adapter(logkws),+ extra={"spider": self.crawler.spider, "item": output},)await self.signals.send_catch_log_async(signal=signals.item_scraped,diff --git a/tests/test_logformatter.py b/tests/test_logformatter.pyindex dc70e6413..8da842da5 100644--- a/tests/test_logformatter.py+++ b/tests/test_logformatter.py@@ -277,6 +277,22 @@ class TestShowOrSkipMessages:assert "Crawled (200) <GET http://127.0.0.1:" in caplog.textassert "Dropped: Ignoring item" in caplog.text+ @coroutine_test+ async def test_item_in_extra(+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer+ ) -> None:+ crawler = get_crawler(ItemSpider, self.base_settings)+ with caplog.at_level(logging.DEBUG):+ await crawler.crawl_async(mockserver=mockserver)+ scraped_record = next(+ r for r in caplog.records if r.message.startswith("Scraped from")+ )+ dropped_record = next(+ r for r in caplog.records if r.message.startswith("Dropped: Ignoring item")+ )+ assert hasattr(scraped_record, "item")+ assert isinstance(cast("Any", dropped_record).item, (Item, dict))+@coroutine_testasync def test_skip_messages(self, caplog: pytest.LogCaptureFixture, mockserver: MockServer