Scrapy, a fast high-level web crawling & scraping framework for Python.
Let scrapy settings --get support add-ons and spiders (#7828)
Adrianff98cb2308a9
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rstindex 2294f2d23..71a1d003d 100644--- a/docs/topics/commands.rst+++ b/docs/topics/commands.rst@@ -630,6 +630,17 @@ Example usage::$ scrapy settings --get DOWNLOAD_DELAY0+The settings of enabled :ref:`add-ons <topics-addons>` are taken into account.+:ref:`Spider settings <spider-settings>` are only taken into account if you name+a spider with ``--spider``::++ $ scrapy settings --get DOWNLOAD_DELAY --spider myspider+ 2.0++.. versionchanged:: VERSION+ Add-on settings are now taken into account, and the ``--spider`` option was+ added.+.. command:: runspiderrunspiderdiff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.pyindex 26a97ccbb..edd697965 100644--- a/scrapy/commands/settings.py+++ b/scrapy/commands/settings.py@@ -1,9 +1,20 @@-import argparse+from __future__ import annotations+import json-from typing import Any, ClassVar+from typing import TYPE_CHECKING, Any, ClassVar+from scrapy import Spider+from scrapy.addons import AddonManagerfrom scrapy.commands import ScrapyCommand+from scrapy.crawler import Crawler+from scrapy.exceptions import UsageErrorfrom scrapy.settings import BaseSettings+from scrapy.spiderloader import get_spider_loader++if TYPE_CHECKING:+ import argparse++ from scrapy.settings import Settingsclass Command(ScrapyCommand):@@ -45,10 +56,33 @@ class Command(ScrapyCommand):metavar="SETTING",help="print setting value, interpreted as a list",)+ parser.add_argument(+ "--spider",+ dest="spider",+ metavar="SPIDER",+ help="also apply the settings of this spider",+ )- def run(self, args: list[str], opts: argparse.Namespace) -> None:+ def _build_settings(self, spider_name: str | None) -> Settings:+ """Return the settings as they would be during a crawl."""assert self.settings is not None- settings = self.settings+ # Must run before get_spider_loader(), because add-ons may change+ # SPIDER_MODULES and other pre-crawler settings that it relies on.+ AddonManager.load_pre_crawler_settings(self.settings)+ spidercls: type[Spider] = Spider+ if spider_name is not None:+ try:+ spidercls = get_spider_loader(self.settings).load(spider_name)+ except KeyError:+ raise UsageError(f"Unable to find spider: {spider_name}") from None+ # Building a Crawler applies the spider settings and does not install a+ # reactor or build any component.+ crawler = Crawler(spidercls, self.settings)+ crawler.addons.load_settings(crawler.settings)+ return crawler.settings++ def run(self, args: list[str], opts: argparse.Namespace) -> None:+ settings = self._build_settings(opts.spider)if opts.get:s = settings.get(opts.get)if isinstance(s, BaseSettings):diff --git a/tests/test_commands.py b/tests/test_commands.pyindex bb5c20f18..4eefa058f 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -773,6 +773,80 @@ class TestExecute:assert capsys.readouterr().out.strip() == "scrapybot"+class TestSettingsCommand(TestProjectBase):+ @pytest.fixture(autouse=True)+ def create_files(self, proj_path: Path) -> None:+ proj_mod_path = proj_path / self.project_name+ (proj_mod_path / "addons.py").write_text("""+class MyAddon:+ def update_settings(self, settings):+ settings.set("FROM_ADDON", "addon", priority="addon")+ settings.set("SPIDER_WINS", "addon", priority="addon")+ settings.set("SEEN_BY_ADDON", settings.get("FROM_SPIDER"), priority="addon")++ @classmethod+ def update_pre_crawler_settings(cls, settings):+ settings.set("FROM_ADDON_PRE_CRAWLER", "addon", priority="addon")+""")+ (proj_mod_path / "spiders" / "sp.py").write_text("""+import scrapy++class MySpider(scrapy.Spider):+ name = "sp"+ custom_settings = {+ "FROM_SPIDER": "spider",+ "SPIDER_WINS": "spider",+ }+""")+ self._append_settings(+ proj_mod_path,+ f'ADDONS = {{"{self.project_name}.addons.MyAddon": 100}}\n',+ )++ @staticmethod+ def _get(proj_path: Path, setting: str, *args: str) -> str:+ returncode, out, err = proc("settings", "--get", setting, *args, cwd=proj_path)+ assert returncode == 0, err+ return out.strip()++ def test_project_setting(self, proj_path: Path) -> None:+ assert self._get(proj_path, "BOT_NAME") == self.project_name++ def test_addon(self, proj_path: Path) -> None:+ assert self._get(proj_path, "FROM_ADDON") == "addon"++ def test_addon_pre_crawler(self, proj_path: Path) -> None:+ assert self._get(proj_path, "FROM_ADDON_PRE_CRAWLER") == "addon"++ def test_cmdline_beats_addon(self, proj_path: Path) -> None:+ assert (+ self._get(proj_path, "FROM_ADDON", "-s", "FROM_ADDON=cmdline") == "cmdline"+ )++ def test_no_spider(self, proj_path: Path) -> None:+ assert self._get(proj_path, "FROM_SPIDER") == "None"+ assert self._get(proj_path, "SPIDER_WINS") == "addon"++ def test_spider(self, proj_path: Path) -> None:+ assert self._get(proj_path, "FROM_SPIDER", "--spider", "sp") == "spider"+ assert self._get(proj_path, "FROM_ADDON", "--spider", "sp") == "addon"++ def test_spider_beats_addon(self, proj_path: Path) -> None:+ assert self._get(proj_path, "SPIDER_WINS", "--spider", "sp") == "spider"++ def test_addons_see_spider_settings(self, proj_path: Path) -> None:+ """Add-ons are loaded after spider settings are applied, as in a crawl."""+ assert self._get(proj_path, "SEEN_BY_ADDON") == "None"+ assert self._get(proj_path, "SEEN_BY_ADDON", "--spider", "sp") == "spider"++ def test_unknown_spider(self, proj_path: Path) -> None:+ returncode, _, err = proc(+ "settings", "--get", "FROM_SPIDER", "--spider", "nope", cwd=proj_path+ )+ assert returncode == 2+ assert "Unable to find spider: nope" in err++class TestBenchCommand:@pytest.mark.parametrize("use_reactor", [True, False])def test_run(self, use_reactor: bool) -> None: