Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix extra stdout output in tests (#7070)
Andrey Rakhmatullin10850e7d2933
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex 533906003..e45d34ac5 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -134,14 +134,21 @@ _scrapy_root_handler: logging.Handler | None = Nonedef install_scrapy_root_handler(settings: Settings) -> None:global _scrapy_root_handler # noqa: PLW0603 # pylint: disable=global-statement+ _uninstall_scrapy_root_handler()+ logging.root.setLevel(logging.NOTSET)+ _scrapy_root_handler = _get_handler(settings)+ logging.root.addHandler(_scrapy_root_handler)+++def _uninstall_scrapy_root_handler() -> None:+ global _scrapy_root_handler # noqa: PLW0603 # pylint: disable=global-statement+if (_scrapy_root_handler is not Noneand _scrapy_root_handler in logging.root.handlers):logging.root.removeHandler(_scrapy_root_handler)- logging.root.setLevel(logging.NOTSET)- _scrapy_root_handler = _get_handler(settings)- logging.root.addHandler(_scrapy_root_handler)+ _scrapy_root_handler = Nonedef get_scrapy_root_handler() -> logging.Handler | None:diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.pyindex 1edef0b4a..246066485 100644--- a/tests/test_command_startproject.py+++ b/tests/test_command_startproject.py@@ -17,9 +17,7 @@ from tests.test_commands import TestProjectBaseclass TestStartprojectCommand(TestProjectBase):def test_startproject(self):- p, out, err = self.proc("startproject", self.project_name)- print(out)- print(err, file=sys.stderr)+ p, _, _ = self.proc("startproject", self.project_name)assert p.returncode == 0assert Path(self.proj_path, "scrapy.cfg").exists()@@ -64,9 +62,7 @@ class TestStartprojectCommand(TestProjectBase):project_path = Path(project_dir, project_name)project_path.mkdir()- p, out, err = self.proc("startproject", project_name, cwd=project_dir)- print(out)- print(err, file=sys.stderr)+ p, _, _ = self.proc("startproject", project_name, cwd=project_dir)assert p.returncode == 0assert Path(project_path, "scrapy.cfg").exists()@@ -151,6 +147,8 @@ class TestStartprojectTemplates(TestProjectBase):project_name,),cwd=destination,+ stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL,env=self.env,)process.wait()@@ -204,6 +202,8 @@ class TestStartprojectTemplates(TestProjectBase):f"TEMPLATES_DIR={read_only_templates_dir}",),cwd=destination,+ stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL,env=self.env,)process.wait()@@ -263,6 +263,8 @@ class TestStartprojectTemplates(TestProjectBase):".",),cwd=project_dir,+ stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL,env=self.env,)process.wait()@@ -306,6 +308,8 @@ class TestStartprojectTemplates(TestProjectBase):project_name,),cwd=destination,+ stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL,env=self.env,)process.wait()diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 66383b92b..2be3b49b3 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -36,7 +36,11 @@ from scrapy.utils.defer import (deferred_from_coro,maybe_deferred_to_future,)-from scrapy.utils.log import configure_logging, get_scrapy_root_handler+from scrapy.utils.log import (+ _uninstall_scrapy_root_handler,+ configure_logging,+ get_scrapy_root_handler,+)from scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler, get_reactor_settingsfrom tests.mockserver.http import MockServer@@ -543,11 +547,14 @@ class TestCrawlerLogging:returnyield- configure_logging()- assert get_scrapy_root_handler().level == logging.DEBUG- crawler = get_crawler(MySpider)- assert get_scrapy_root_handler().level == logging.INFO- await maybe_deferred_to_future(crawler.crawl())+ try:+ configure_logging()+ assert get_scrapy_root_handler().level == logging.DEBUG+ crawler = get_crawler(MySpider)+ assert get_scrapy_root_handler().level == logging.INFO+ await maybe_deferred_to_future(crawler.crawl())+ finally:+ _uninstall_scrapy_root_handler()logged = log_file.read_text(encoding="utf-8")@@ -572,9 +579,12 @@ class TestCrawlerLogging:"LOG_FILE_APPEND": False,}- configure_logging()- get_crawler(MySpider)- logging.debug("debug message") # noqa: LOG015+ try:+ configure_logging()+ get_crawler(MySpider)+ logging.debug("debug message") # noqa: LOG015+ finally:+ _uninstall_scrapy_root_handler()logged = log_file.read_text(encoding="utf-8")@@ -629,24 +639,24 @@ class TestAsyncCrawlerRunner(TestBaseCrawler):class TestCrawlerProcess(TestBaseCrawler):def test_crawler_process_accepts_dict(self):- runner = CrawlerProcess({"foo": "bar"})+ runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False)assert runner.settings["foo"] == "bar"self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")def test_crawler_process_accepts_None(self):- runner = CrawlerProcess()+ runner = CrawlerProcess(install_root_handler=False)self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")@pytest.mark.only_asyncioclass TestAsyncCrawlerProcess(TestBaseCrawler):def test_crawler_process_accepts_dict(self):- runner = AsyncCrawlerProcess({"foo": "bar"})+ runner = AsyncCrawlerProcess({"foo": "bar"}, install_root_handler=False)assert runner.settings["foo"] == "bar"self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")def test_crawler_process_accepts_None(self):- runner = AsyncCrawlerProcess()+ runner = AsyncCrawlerProcess(install_root_handler=False)self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")@@ -1177,7 +1187,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):)def test_log_scrapy_info(settings, items, caplog):with caplog.at_level("INFO"):- CrawlerProcess(settings)+ CrawlerProcess(settings, install_root_handler=False)assert (caplog.records[0].getMessage()== f"Scrapy {scrapy.__version__} started (bot: scrapybot)"diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 4bfccdb18..430d599c8 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -489,6 +489,7 @@ class TestEngine(TestEngineBase):)p = subprocess.Popen(args,+ stdout=subprocess.DEVNULL,stderr=subprocess.PIPE,)diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex c5bd1b172..aa4477210 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -1717,7 +1717,6 @@ class TestFeedExport(TestFeedExportBase):with LogCapture() as log:await self.exported_data(items, settings)- print(log)for fmt in ["json", "xml", "csv"]:assert f"Stored {fmt} feed (2 items)" in str(log)@@ -1738,7 +1737,6 @@ class TestFeedExport(TestFeedExportBase):with LogCapture() as log:await self.exported_data(items, settings)- print(log)for fmt in ["json", "xml", "csv"]:assert f"Error storing {fmt} feed (2 items)" in str(log)diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.pyindex f9874f821..61d79743d 100644--- a/tests/test_proxy_connect.py+++ b/tests/test_proxy_connect.py@@ -116,9 +116,7 @@ class TestProxyConnect:assert "Proxy-Authorization" not in echo["headers"]def _assert_got_response_code(self, code, log):- print(log)assert str(log).count(f"Crawled ({code})") == 1def _assert_got_tunnel_error(self, log):- print(log)assert "TunnelError" in str(log)diff --git a/tests/test_zz_resources.py b/tests/test_zz_resources.pyindex 84b2910db..2560f8d7f 100644--- a/tests/test_zz_resources.py+++ b/tests/test_zz_resources.py@@ -8,9 +8,20 @@ from scrapy.utils.log import LogCounterHandlerdef test_counter_handler() -> None:- """Test that LogCounterHandler is always properly removed.+ """Test that ``LogCounterHandler`` is always properly removed.- It's added in Crawler.crawl{,_async}() and removed on engine_stopped.+ It's added in ``Crawler.crawl{,_async}()`` and removed on engine_stopped."""c = sum(1 for h in logging.root.handlers if isinstance(h, LogCounterHandler))assert c == 0+++def test_stderr_log_handler() -> None:+ """Test that the Scrapy root handler is always properly removed.++ It's added in ``configure_logging()``, called by ``{Async,}CrawlerProcess``+ (without ``install_root_handler=False``). It can be removed with+ ``_uninstall_scrapy_root_handler()`` if installing it was really neeeded.+ """+ c = sum(1 for h in logging.root.handlers if type(h) is logging.StreamHandler) # pylint: disable=unidiomatic-typecheck+ assert c == 0