Scrapy, a fast high-level web crawling & scraping framework for Python.
Add integration tests for modifying settings via args.
Andrey Rakhmatullin77f39be4073c
diff --git a/tests/CrawlerProcess/args_settings.py b/tests/CrawlerProcess/args_settings.pynew file mode 100644index 000000000..a46a8806b--- /dev/null+++ b/tests/CrawlerProcess/args_settings.py@@ -0,0 +1,24 @@+from typing import Any++import scrapy+from scrapy.crawler import Crawler, CrawlerProcess+++class NoRequestsSpider(scrapy.Spider):+ name = "no_request"++ @classmethod+ def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any):+ spider = super().from_crawler(crawler, *args, **kwargs)+ spider.settings.set("FOO", kwargs.get("foo"))+ return spider++ def start_requests(self):+ self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}")+ return []+++process = CrawlerProcess(settings={})++process.crawl(NoRequestsSpider, foo=42)+process.start()diff --git a/tests/test_commands.py b/tests/test_commands.pyindex b1d7be628..36f800850 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -965,6 +965,28 @@ class MySpider(scrapy.Spider):log,)+ def test_args_change_settings(self):+ spider_code = """+import scrapy++class MySpider(scrapy.Spider):+ name = 'myspider'++ @classmethod+ def from_crawler(cls, crawler, *args, **kwargs):+ spider = super().from_crawler(crawler, *args, **kwargs)+ spider.settings.set("FOO", kwargs.get("foo"))+ return spider++ def start_requests(self):+ self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}")+ return []+"""+ args = ["-a", "foo=42"]+ log = self.get_log(spider_code, args=args)+ self.assertIn("Spider closed (finished)", log)+ self.assertIn("The value of FOO is 42", log)+@skipIf(platform.system() != "Windows", "Windows required for .pyw files")class WindowsRunSpiderCommandTest(RunSpiderCommandTest):diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 120991ae7..bfae6c690 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -481,6 +481,11 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):self.assertNotIn("Using asyncio event loop: uvloop.Loop", log)self.assertIn("async pipeline opened!", log)+ def test_args_change_settings(self):+ log = self.run_script("args_settings.py")+ self.assertIn("Spider closed (finished)", log)+ self.assertIn("The value of FOO is 42", log)+class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):script_dir = Path(__file__).parent.resolve() / "CrawlerRunner"