Scrapy, a fast high-level web crawling & scraping framework for Python.
Move LogCounterHandler into a new extension (#7046)
Andrey Rakhmatullin1c6ba00fd07e
diff --git a/docs/news.rst b/docs/news.rstindex 6a019e2fb..2a769bc1c 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -15,6 +15,14 @@ Backward-incompatible changes``True`` when running Scrapy via :ref:`its command-line tool<topics-commands-crawlerprocess>` to avoid a reactor mismatch exception.+- The ``log_count/*`` stats no longer count some of the early messages that+ they counted before. While the earliest log messages, emitted before the+ counter is initialized, were never counted, the counter initialization now+ happens later than in previous Scrapy versions. You may need to adjust+ expected values if you retrieve and compare values of these stats in your+ code.+ (:issue:`7046`)+- The classes listed below are now :term:`abstract base classes <abstractbase class>`. They cannot be instantiated directly and their subclassesneed to override the abstract methods listed below to be able to bediff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rstindex e1e3dd6b4..d2ddde207 100644--- a/docs/topics/extensions.rst+++ b/docs/topics/extensions.rst@@ -138,6 +138,14 @@ enabled (see :ref:`topics-stats`)... _topics-extensions-ref-telnetconsole:+Log Count extension+~~~~~~~~~~~~~~~~~~~++.. module:: scrapy.extensions.logcount+ :synopsis: Basic stats logging++.. autoclass:: LogCount+Telnet console extension~~~~~~~~~~~~~~~~~~~~~~~~diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex ef658e9b5..ffbebe152 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Any, TypeVarfrom twisted.internet.defer import Deferred, DeferredList, inlineCallbacks-from scrapy import Spider, signals+from scrapy import Spiderfrom scrapy.addons import AddonManagerfrom scrapy.core.engine import ExecutionEnginefrom scrapy.exceptions import ScrapyDeprecationWarning@@ -22,7 +22,6 @@ from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loaderfrom scrapy.utils.asyncio import is_asyncio_availablefrom scrapy.utils.defer import deferred_from_corofrom scrapy.utils.log import (- LogCounterHandler,configure_logging,get_scrapy_root_handler,install_scrapy_root_handler,@@ -97,13 +96,6 @@ class Crawler:self.addons.load_settings(self.settings)self.stats = load_object(self.settings["STATS_CLASS"])(self)- handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL"))- logging.root.addHandler(handler)- # lambda is assigned to Crawler attribute because this way it is not- # garbage collected after leaving the scope- self.__remove_handler = lambda: logging.root.removeHandler(handler)- self.signals.connect(self.__remove_handler, signals.engine_stopped)-lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])self.logformatter = lf_cls.from_crawler(self)diff --git a/scrapy/extensions/logcount.py b/scrapy/extensions/logcount.pynew file mode 100644index 000000000..04e570bbf--- /dev/null+++ b/scrapy/extensions/logcount.py@@ -0,0 +1,48 @@+from __future__ import annotations++import logging+from typing import TYPE_CHECKING++from scrapy import Spider, signals+from scrapy.utils.log import LogCounterHandler++if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self++ from scrapy.crawler import Crawler+++logger = logging.getLogger(__name__)+++class LogCount:+ """Install a log handler that counts log messages by level.++ The handler installed is :class:`scrapy.utils.log.LogCounterHandler`.+ The counts are stored in stats as ``log_count/<level>``.++ .. versionadded:: VERSION+ """++ def __init__(self, crawler: Crawler):+ self.crawler: Crawler = crawler+ self.handler: LogCounterHandler | None = None++ @classmethod+ def from_crawler(cls, crawler: Crawler) -> Self:+ o = cls(crawler)+ crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)+ crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)+ return o++ def spider_opened(self, spider: Spider) -> None:+ self.handler = LogCounterHandler(+ self.crawler, level=self.crawler.settings.get("LOG_LEVEL")+ )+ logging.root.addHandler(self.handler)++ def spider_closed(self, spider: Spider, reason: str) -> None:+ if self.handler:+ logging.root.removeHandler(self.handler)+ self.handler = Nonediff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex f306569e4..624e71774 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -308,6 +308,7 @@ if sys.platform == "win32":EXTENSIONS = {}EXTENSIONS_BASE = {"scrapy.extensions.corestats.CoreStats": 0,+ "scrapy.extensions.logcount.LogCount": 0,"scrapy.extensions.telnet.TelnetConsole": 0,"scrapy.extensions.memusage.MemoryUsage": 0,"scrapy.extensions.memdebug.MemoryDebugger": 0,diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 255a03a2a..47bed925a 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -401,7 +401,7 @@ with multiples linesassert "Got response 200" in str(log)@inlineCallbacks- def test_crawl_multiple(self):+ def test_crawl_multiple(self, caplog: pytest.LogCaptureFixture):runner = CrawlerRunner(get_reactor_settings())runner.crawl(SimpleSpider,@@ -414,11 +414,11 @@ with multiples linesmockserver=self.mockserver,)- with LogCapture() as log:+ with caplog.at_level(logging.DEBUG):yield runner.join()- self._assert_retried(log)- assert "Got response 200" in str(log)+ self._assert_retried(caplog.text)+ assert "Got response 200" in caplog.textclass TestCrawlSpider:diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex de20d6b90..66383b92b 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -516,10 +516,13 @@ class TestCrawlerLogging:get_crawler(MySpider)assert get_scrapy_root_handler() is None- def test_spider_custom_settings_log_level(self, tmp_path):+ @deferred_f_from_coro_f+ async def test_spider_custom_settings_log_level(self, tmp_path):log_file = Path(tmp_path, "log.txt")log_file.write_text("previous message\n", encoding="utf-8")+ info_count = None+class MySpider(scrapy.Spider):name = "spider"custom_settings = {@@ -527,15 +530,24 @@ class TestCrawlerLogging:"LOG_FILE": str(log_file),}+ async def start(self):+ info_count_start = crawler.stats.get_value("log_count/INFO")+ logging.debug("debug message") # noqa: LOG015+ logging.info("info message") # noqa: LOG015+ logging.warning("warning message") # noqa: LOG015+ logging.error("error message") # noqa: LOG015+ nonlocal info_count+ info_count = (+ crawler.stats.get_value("log_count/INFO") - info_count_start+ )+ return+ yield+configure_logging()assert get_scrapy_root_handler().level == logging.DEBUGcrawler = get_crawler(MySpider)assert get_scrapy_root_handler().level == logging.INFO- info_count = crawler.stats.get_value("log_count/INFO")- logging.debug("debug message") # noqa: LOG015- logging.info("info message") # noqa: LOG015- logging.warning("warning message") # noqa: LOG015- logging.error("error message") # noqa: LOG015+ await maybe_deferred_to_future(crawler.crawl())logged = log_file.read_text(encoding="utf-8")@@ -546,7 +558,7 @@ class TestCrawlerLogging:assert "error message" in loggedassert crawler.stats.get_value("log_count/ERROR") == 1assert crawler.stats.get_value("log_count/WARNING") == 1- assert crawler.stats.get_value("log_count/INFO") - info_count == 1+ assert info_count == 1assert crawler.stats.get_value("log_count/DEBUG", 0) == 0def test_spider_custom_settings_log_append(self, tmp_path):diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.pyindex a4233d447..03b941e4f 100644--- a/tests/test_downloadermiddleware.py+++ b/tests/test_downloadermiddleware.py@@ -32,8 +32,10 @@ class TestManagerBase:mwman = DownloaderMiddlewareManager.from_crawler(crawler)crawler.engine = crawler._create_engine()await crawler.engine.open_spider_async()- yield mwman- await crawler.engine.close_spider_async()+ try:+ yield mwman+ finally:+ await crawler.engine.close_spider_async()@staticmethodasync def _download(diff --git a/tests/test_zz_resources.py b/tests/test_zz_resources.pynew file mode 100644index 000000000..84b2910db--- /dev/null+++ b/tests/test_zz_resources.py@@ -0,0 +1,16 @@+"""Test that certain resources are not leaked during earlier tests."""++from __future__ import annotations++import logging++from scrapy.utils.log import LogCounterHandler+++def test_counter_handler() -> None:+ """Test that LogCounterHandler is always properly removed.++ It's added in Crawler.crawl{,_async}() and removed on engine_stopped.+ """+ c = sum(1 for h in logging.root.handlers if isinstance(h, LogCounterHandler))+ assert c == 0