Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Let scrapy settings --get support add-ons and spiders (#7828)

Adrianff98cb2308a9
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 2294f2d23..71a1d003d 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -630,6 +630,17 @@ Example usage::
$ scrapy settings --get DOWNLOAD_DELAY
0
+The settings of enabled :ref:`add-ons <topics-addons>` are taken into account.
+:ref:`Spider settings <spider-settings>` are only taken into account if you name
+a spider with ``--spider``::
+
+ $ scrapy settings --get DOWNLOAD_DELAY --spider myspider
+ 2.0
+
+.. versionchanged:: VERSION
+ Add-on settings are now taken into account, and the ``--spider`` option was
+ added.
+
.. command:: runspider
runspider
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index 26a97ccbb..edd697965 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -1,9 +1,20 @@
-import argparse
+from __future__ import annotations
+
import json
-from typing import Any, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar
+from scrapy import Spider
+from scrapy.addons import AddonManager
from scrapy.commands import ScrapyCommand
+from scrapy.crawler import Crawler
+from scrapy.exceptions import UsageError
from scrapy.settings import BaseSettings
+from scrapy.spiderloader import get_spider_loader
+
+if TYPE_CHECKING:
+ import argparse
+
+ from scrapy.settings import Settings
class Command(ScrapyCommand):
@@ -45,10 +56,33 @@ class Command(ScrapyCommand):
metavar="SETTING",
help="print setting value, interpreted as a list",
)
+ parser.add_argument(
+ "--spider",
+ dest="spider",
+ metavar="SPIDER",
+ help="also apply the settings of this spider",
+ )
- def run(self, args: list[str], opts: argparse.Namespace) -> None:
+ def _build_settings(self, spider_name: str | None) -> Settings:
+ """Return the settings as they would be during a crawl."""
assert self.settings is not None
- settings = self.settings
+ # Must run before get_spider_loader(), because add-ons may change
+ # SPIDER_MODULES and other pre-crawler settings that it relies on.
+ AddonManager.load_pre_crawler_settings(self.settings)
+ spidercls: type[Spider] = Spider
+ if spider_name is not None:
+ try:
+ spidercls = get_spider_loader(self.settings).load(spider_name)
+ except KeyError:
+ raise UsageError(f"Unable to find spider: {spider_name}") from None
+ # Building a Crawler applies the spider settings and does not install a
+ # reactor or build any component.
+ crawler = Crawler(spidercls, self.settings)
+ crawler.addons.load_settings(crawler.settings)
+ return crawler.settings
+
+ def run(self, args: list[str], opts: argparse.Namespace) -> None:
+ settings = self._build_settings(opts.spider)
if opts.get:
s = settings.get(opts.get)
if isinstance(s, BaseSettings):
diff --git a/tests/test_commands.py b/tests/test_commands.py
index bb5c20f18..4eefa058f 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -773,6 +773,80 @@ class TestExecute:
assert capsys.readouterr().out.strip() == "scrapybot"
+class TestSettingsCommand(TestProjectBase):
+ @pytest.fixture(autouse=True)
+ def create_files(self, proj_path: Path) -> None:
+ proj_mod_path = proj_path / self.project_name
+ (proj_mod_path / "addons.py").write_text("""
+class MyAddon:
+ def update_settings(self, settings):
+ settings.set("FROM_ADDON", "addon", priority="addon")
+ settings.set("SPIDER_WINS", "addon", priority="addon")
+ settings.set("SEEN_BY_ADDON", settings.get("FROM_SPIDER"), priority="addon")
+
+ @classmethod
+ def update_pre_crawler_settings(cls, settings):
+ settings.set("FROM_ADDON_PRE_CRAWLER", "addon", priority="addon")
+""")
+ (proj_mod_path / "spiders" / "sp.py").write_text("""
+import scrapy
+
+class MySpider(scrapy.Spider):
+ name = "sp"
+ custom_settings = {
+ "FROM_SPIDER": "spider",
+ "SPIDER_WINS": "spider",
+ }
+""")
+ self._append_settings(
+ proj_mod_path,
+ f'ADDONS = {{"{self.project_name}.addons.MyAddon": 100}}\n',
+ )
+
+ @staticmethod
+ def _get(proj_path: Path, setting: str, *args: str) -> str:
+ returncode, out, err = proc("settings", "--get", setting, *args, cwd=proj_path)
+ assert returncode == 0, err
+ return out.strip()
+
+ def test_project_setting(self, proj_path: Path) -> None:
+ assert self._get(proj_path, "BOT_NAME") == self.project_name
+
+ def test_addon(self, proj_path: Path) -> None:
+ assert self._get(proj_path, "FROM_ADDON") == "addon"
+
+ def test_addon_pre_crawler(self, proj_path: Path) -> None:
+ assert self._get(proj_path, "FROM_ADDON_PRE_CRAWLER") == "addon"
+
+ def test_cmdline_beats_addon(self, proj_path: Path) -> None:
+ assert (
+ self._get(proj_path, "FROM_ADDON", "-s", "FROM_ADDON=cmdline") == "cmdline"
+ )
+
+ def test_no_spider(self, proj_path: Path) -> None:
+ assert self._get(proj_path, "FROM_SPIDER") == "None"
+ assert self._get(proj_path, "SPIDER_WINS") == "addon"
+
+ def test_spider(self, proj_path: Path) -> None:
+ assert self._get(proj_path, "FROM_SPIDER", "--spider", "sp") == "spider"
+ assert self._get(proj_path, "FROM_ADDON", "--spider", "sp") == "addon"
+
+ def test_spider_beats_addon(self, proj_path: Path) -> None:
+ assert self._get(proj_path, "SPIDER_WINS", "--spider", "sp") == "spider"
+
+ def test_addons_see_spider_settings(self, proj_path: Path) -> None:
+ """Add-ons are loaded after spider settings are applied, as in a crawl."""
+ assert self._get(proj_path, "SEEN_BY_ADDON") == "None"
+ assert self._get(proj_path, "SEEN_BY_ADDON", "--spider", "sp") == "spider"
+
+ def test_unknown_spider(self, proj_path: Path) -> None:
+ returncode, _, err = proc(
+ "settings", "--get", "FROM_SPIDER", "--spider", "nope", cwd=proj_path
+ )
+ assert returncode == 2
+ assert "Unable to find spider: nope" in err
+
+
class TestBenchCommand:
@pytest.mark.parametrize("use_reactor", [True, False])
def test_run(self, use_reactor: bool) -> None: