Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add integration tests for modifying settings via args.

Andrey Rakhmatullin77f39be4073c
diff --git a/tests/CrawlerProcess/args_settings.py b/tests/CrawlerProcess/args_settings.py
new file mode 100644
index 000000000..a46a8806b
--- /dev/null
+++ b/tests/CrawlerProcess/args_settings.py
@@ -0,0 +1,24 @@
+from typing import Any
+
+import scrapy
+from scrapy.crawler import Crawler, CrawlerProcess
+
+
+class NoRequestsSpider(scrapy.Spider):
+ name = "no_request"
+
+ @classmethod
+ def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any):
+ spider = super().from_crawler(crawler, *args, **kwargs)
+ spider.settings.set("FOO", kwargs.get("foo"))
+ return spider
+
+ def start_requests(self):
+ self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}")
+ return []
+
+
+process = CrawlerProcess(settings={})
+
+process.crawl(NoRequestsSpider, foo=42)
+process.start()
diff --git a/tests/test_commands.py b/tests/test_commands.py
index b1d7be628..36f800850 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -965,6 +965,28 @@ class MySpider(scrapy.Spider):
log,
)
+ def test_args_change_settings(self):
+ spider_code = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+ name = 'myspider'
+
+ @classmethod
+ def from_crawler(cls, crawler, *args, **kwargs):
+ spider = super().from_crawler(crawler, *args, **kwargs)
+ spider.settings.set("FOO", kwargs.get("foo"))
+ return spider
+
+ def start_requests(self):
+ self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}")
+ return []
+"""
+ args = ["-a", "foo=42"]
+ log = self.get_log(spider_code, args=args)
+ self.assertIn("Spider closed (finished)", log)
+ self.assertIn("The value of FOO is 42", log)
+
@skipIf(platform.system() != "Windows", "Windows required for .pyw files")
class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 120991ae7..bfae6c690 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -481,6 +481,11 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
self.assertNotIn("Using asyncio event loop: uvloop.Loop", log)
self.assertIn("async pipeline opened!", log)
+ def test_args_change_settings(self):
+ log = self.run_script("args_settings.py")
+ self.assertIn("Spider closed (finished)", log)
+ self.assertIn("The value of FOO is 42", log)
+
class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
script_dir = Path(__file__).parent.resolve() / "CrawlerRunner"