Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

46.7 KB
Raw Blame
# pylint: disable=unsubscriptable-object,unsupported-membership-test
# (too many false positives)

import logging
import re
import textwrap
from pathlib import Path
from typing import Any
from unittest import mock

import pytest

from scrapy.core.downloader.handlers.file import FileDownloadHandler
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import (
    SETTINGS_PRIORITIES,
    BaseSettings,
    Settings,
    SettingsAttribute,
    get_settings_priority,
)
from scrapy.settings import default_settings as scrapy_default_settings
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.test import get_crawler

from . import default_settings

NON_COMPONENT_PRIORITY_DICT_BASE_SETTING_NAMES = {
    "DOWNLOAD_HANDLERS_BASE",
    "FEED_EXPORTERS_BASE",
    "FEED_STORAGES_BASE",
    "MEDIA_STORAGES_BASE",
}


class TestSettingsGlobalFuncs:
    def test_get_settings_priority(self):
        for prio_str, prio_num in SETTINGS_PRIORITIES.items():
            assert get_settings_priority(prio_str) == prio_num
        assert get_settings_priority(99) == 99


class TestSettingsAttribute:
    def setup_method(self):
        self.attribute = SettingsAttribute("value", 10)

    def test_set_greater_priority(self):
        self.attribute.set("value2", 20)
        assert self.attribute.value == "value2"
        assert self.attribute.priority == 20

    def test_set_equal_priority(self):
        self.attribute.set("value2", 10)
        assert self.attribute.value == "value2"
        assert self.attribute.priority == 10

    def test_set_less_priority(self):
        self.attribute.set("value2", 0)
        assert self.attribute.value == "value"
        assert self.attribute.priority == 10

    def test_overwrite_basesettings(self):
        original_dict = {"one": 10, "two": 20}
        original_settings = BaseSettings(original_dict, 0)
        attribute = SettingsAttribute(original_settings, 0)

        new_dict = {"three": 11, "four": 21}
        attribute.set(new_dict, 10)
        assert isinstance(attribute.value, BaseSettings)
        assert set(attribute.value) == set(new_dict)
        assert set(original_settings) == set(original_dict)

        new_settings = BaseSettings({"five": 12}, 0)
        attribute.set(new_settings, 0)  # Insufficient priority
        assert set(attribute.value) == set(new_dict)
        attribute.set(new_settings, 10)
        assert set(attribute.value) == set(new_settings)

    def test_repr(self):
        assert repr(self.attribute) == "<SettingsAttribute value='value' priority=10>"


class TestBaseSettings:
    def setup_method(self):
        self.settings = BaseSettings()

    def test_setdefault_not_existing_value(self):
        settings = BaseSettings()
        value = settings.setdefault("TEST_OPTION", "value")
        assert settings["TEST_OPTION"] == "value"
        assert value == "value"
        assert value is not None

    def test_setdefault_existing_value(self):
        settings = BaseSettings({"TEST_OPTION": "value"})
        value = settings.setdefault("TEST_OPTION", None)
        assert settings["TEST_OPTION"] == "value"
        assert value == "value"

    def test_set_new_attribute(self):
        self.settings.set("TEST_OPTION", "value", 0)
        assert "TEST_OPTION" in self.settings.attributes

        attr = self.settings.attributes["TEST_OPTION"]
        assert isinstance(attr, SettingsAttribute)
        assert attr.value == "value"
        assert attr.priority == 0

    def test_set_settingsattribute(self):
        myattr = SettingsAttribute(0, 30)  # Note priority 30
        self.settings.set("TEST_ATTR", myattr, 10)
        assert self.settings.get("TEST_ATTR") == 0
        assert self.settings.getpriority("TEST_ATTR") == 30

    def test_set_instance_identity_on_update(self):
        attr = SettingsAttribute("value", 0)
        self.settings.attributes = {"TEST_OPTION": attr}
        self.settings.set("TEST_OPTION", "othervalue", 10)

        assert "TEST_OPTION" in self.settings.attributes
        assert attr is self.settings.attributes["TEST_OPTION"]

    @pytest.mark.parametrize("priority", [0, 10, 20])
    def test_set_calls_settings_attributes_methods_on_update(
        self, priority: int
    ) -> None:
        attr = SettingsAttribute("value", 10)
        with (
            mock.patch.object(attr, "__setattr__") as mock_setattr,
            mock.patch.object(attr, "set") as mock_set,
        ):
            self.settings.attributes = {"TEST_OPTION": attr}

            self.settings.set("TEST_OPTION", "othervalue", priority)
            mock_set.assert_called_once_with("othervalue", priority)
            assert not mock_setattr.called

    def test_setitem(self):
        settings = BaseSettings()
        settings.set("key", "a", "default")
        settings["key"] = "b"
        assert settings["key"] == "b"
        assert settings.getpriority("key") == 20
        settings["key"] = "c"
        assert settings["key"] == "c"
        settings["key2"] = "x"
        assert "key2" in settings
        assert settings["key2"] == "x"
        assert settings.getpriority("key2") == 20

    def test_setdict_alias(self):
        with mock.patch.object(self.settings, "set") as mock_set:
            self.settings.setdict({"TEST_1": "value1", "TEST_2": "value2"}, 10)
            assert mock_set.call_count == 2
            calls = [
                mock.call("TEST_1", "value1", 10),
                mock.call("TEST_2", "value2", 10),
            ]
            mock_set.assert_has_calls(calls, any_order=True)

    def test_setmodule_only_load_uppercase_vars(self):
        class ModuleMock:
            UPPERCASE_VAR = "value"
            MIXEDcase_VAR = "othervalue"
            lowercase_var = "anothervalue"

        self.settings.attributes = {}
        self.settings.setmodule(ModuleMock(), 10)  # type: ignore[arg-type]
        assert "UPPERCASE_VAR" in self.settings.attributes
        assert "MIXEDcase_VAR" not in self.settings.attributes
        assert "lowercase_var" not in self.settings.attributes
        assert len(self.settings.attributes) == 1

    def test_setmodule_alias(self):
        with mock.patch.object(self.settings, "set") as mock_set:
            self.settings.setmodule(default_settings, 10)
            mock_set.assert_any_call("TEST_DEFAULT", "defvalue", 10)
            mock_set.assert_any_call("TEST_DICT", {"key": "val"}, 10)

    def test_setmodule_by_path(self):
        self.settings.attributes = {}
        self.settings.setmodule(default_settings, 10)
        ctrl_attributes = self.settings.attributes.copy()

        self.settings.attributes = {}
        self.settings.setmodule("tests.test_settings.default_settings", 10)

        assert set(self.settings.attributes) == set(ctrl_attributes)

        for key in ctrl_attributes:
            attr = self.settings.attributes[key]
            ctrl_attr = ctrl_attributes[key]
            assert attr.value == ctrl_attr.value
            assert attr.priority == ctrl_attr.priority

    def test_update(self):
        settings = BaseSettings({"key_lowprio": 0}, priority=0)
        settings.set("key_highprio", 10, priority=50)
        custom_settings = BaseSettings(
            {"key_lowprio": 1, "key_highprio": 11}, priority=30
        )
        custom_settings.set("newkey_one", None, priority=50)
        custom_dict = {"key_lowprio": 2, "key_highprio": 12, "newkey_two": None}

        settings.update(custom_dict, priority=20)
        assert settings["key_lowprio"] == 2
        assert settings.getpriority("key_lowprio") == 20
        assert settings["key_highprio"] == 10
        assert "newkey_two" in settings
        assert settings.getpriority("newkey_two") == 20

        settings.update(custom_settings)
        assert settings["key_lowprio"] == 1
        assert settings.getpriority("key_lowprio") == 30
        assert settings["key_highprio"] == 10
        assert "newkey_one" in settings
        assert settings.getpriority("newkey_one") == 50

        settings.update({"key_lowprio": 3}, priority=20)
        assert settings["key_lowprio"] == 1

    @pytest.mark.xfail(
        raises=TypeError, reason="BaseSettings.update doesn't support kwargs input"
    )
    def test_update_kwargs(self):
        settings = BaseSettings({"key": 0})
        settings.update(key=1)  # type: ignore[call-arg]  # pylint: disable=unexpected-keyword-arg

    def test_update_iterable(self):
        settings = BaseSettings({"key": 0}, priority=0)
        settings.update([("key", 1)], priority=10)
        assert settings["key"] == 1
        assert settings.getpriority("key") == 10

    def test_update_jsonstring(self):
        settings = BaseSettings({"number": 0, "dict": BaseSettings({"key": "val"})})
        settings.update('{"number": 1, "newnumber": 2}')
        assert settings["number"] == 1
        assert settings["newnumber"] == 2
        settings.set("dict", '{"key": "newval", "newkey": "newval2"}')
        assert settings["dict"]["key"] == "newval"
        assert settings["dict"]["newkey"] == "newval2"

    def test_delete(self):
        settings = BaseSettings({"key": None})
        settings.set("key_highprio", None, priority=50)
        settings.delete("key")
        settings.delete("key_highprio")
        assert "key" not in settings
        assert "key_highprio" in settings
        del settings["key_highprio"]
        assert "key_highprio" not in settings
        with pytest.raises(KeyError):
            settings.delete("notkey")
        with pytest.raises(KeyError):
            del settings["notkey"]

    def test_get(self):
        test_configuration = {
            "TEST_ENABLED1": "1",
            "TEST_ENABLED2": True,
            "TEST_ENABLED3": 1,
            "TEST_ENABLED4": "True",
            "TEST_ENABLED5": "true",
            "TEST_ENABLED_WRONG": "on",
            "TEST_DISABLED1": "0",
            "TEST_DISABLED2": False,
            "TEST_DISABLED3": 0,
            "TEST_DISABLED4": "False",
            "TEST_DISABLED5": "false",
            "TEST_DISABLED_WRONG": "off",
            "TEST_INT1": 123,
            "TEST_INT2": "123",
            "TEST_FLOAT1": 123.45,
            "TEST_FLOAT2": "123.45",
            "TEST_LIST1": ["one", "two"],
            "TEST_LIST2": "one,two",
            "TEST_LIST3": "",
            "TEST_STR": "value",
            "TEST_DICT1": {"key1": "val1", "ke2": 3},
            "TEST_DICT2": '{"key1": "val1", "ke2": 3}',
        }
        settings = self.settings
        settings.attributes = {
            key: SettingsAttribute(value, 0)
            for key, value in test_configuration.items()
        }

        assert settings.getbool("TEST_ENABLED1")
        assert settings.getbool("TEST_ENABLED2")
        assert settings.getbool("TEST_ENABLED3")
        assert settings.getbool("TEST_ENABLED4")
        assert settings.getbool("TEST_ENABLED5")
        assert not settings.getbool("TEST_ENABLEDx")
        assert settings.getbool("TEST_ENABLEDx", True)
        assert not settings.getbool("TEST_DISABLED1")
        assert not settings.getbool("TEST_DISABLED2")
        assert not settings.getbool("TEST_DISABLED3")
        assert not settings.getbool("TEST_DISABLED4")
        assert not settings.getbool("TEST_DISABLED5")
        assert settings.getint("TEST_INT1") == 123
        assert settings.getint("TEST_INT2") == 123
        assert settings.getint("TEST_INTx") == 0
        assert settings.getint("TEST_INTx", 45) == 45
        assert settings.getfloat("TEST_FLOAT1") == 123.45
        assert settings.getfloat("TEST_FLOAT2") == 123.45
        assert settings.getfloat("TEST_FLOATx") == 0.0
        assert settings.getfloat("TEST_FLOATx", 55.0) == 55.0
        assert settings.getlist("TEST_LIST1") == ["one", "two"]
        assert settings.getlist("TEST_LIST2") == ["one", "two"]
        assert settings.getlist("TEST_LIST3") == []
        assert settings.getlist("TEST_LISTx") == []
        assert settings.getlist("TEST_LISTx", ["default"]) == ["default"]
        assert settings["TEST_STR"] == "value"
        assert settings.get("TEST_STR") == "value"
        assert settings["TEST_STRx"] is None
        assert settings.get("TEST_STRx") is None
        assert settings.get("TEST_STRx", "default") == "default"
        assert settings.getdict("TEST_DICT1") == {"key1": "val1", "ke2": 3}
        assert settings.getdict("TEST_DICT2") == {"key1": "val1", "ke2": 3}
        assert settings.getdict("TEST_DICT3") == {}
        assert settings.getdict("TEST_DICT3", {"key1": 5}) == {"key1": 5}
        with pytest.raises(
            ValueError,
            match=r"dictionary update sequence element #0 has length 3; 2 is required|sequence of pairs expected",
        ):
            settings.getdict("TEST_LIST1")
        with pytest.raises(
            ValueError, match="Supported values for boolean settings are"
        ):
            settings.getbool("TEST_ENABLED_WRONG")
        with pytest.raises(
            ValueError, match="Supported values for boolean settings are"
        ):
            settings.getbool("TEST_DISABLED_WRONG")

    def test_getpriority(self):
        settings = BaseSettings({"key": "value"}, priority=99)
        assert settings.getpriority("key") == 99
        assert settings.getpriority("nonexistentkey") is None

    def test_getwithbase(self):
        s = BaseSettings(
            {
                "TEST_BASE": BaseSettings({"1": 1, "2": 2}, "project"),
                "TEST": BaseSettings({"1": 10, "3": 30}, "default"),
                "HASNOBASE": BaseSettings({"3": 3000}, "default"),
            }
        )
        s["TEST"].set("2", 200, "cmdline")
        assert set(s.getwithbase("TEST")) == {"1", "2", "3"}
        assert set(s.getwithbase("HASNOBASE")) == set(s["HASNOBASE"])
        assert s.getwithbase("NONEXISTENT") == {}

    def test_maxpriority(self):
        # Empty settings should return 'default'
        assert self.settings.maxpriority() == 0
        self.settings.set("A", 0, 10)
        self.settings.set("B", 0, 30)
        assert self.settings.maxpriority() == 30

    def test_copy(self):
        values = {
            "TEST_BOOL": True,
            "TEST_LIST": ["one", "two"],
            "TEST_LIST_OF_LISTS": [
                ["first_one", "first_two"],
                ["second_one", "second_two"],
            ],
        }
        self.settings.setdict(values)
        copy = self.settings.copy()
        self.settings.set("TEST_BOOL", False)
        assert copy.get("TEST_BOOL")

        test_list = self.settings.get("TEST_LIST")
        test_list.append("three")
        assert copy.get("TEST_LIST") == ["one", "two"]

        test_list_of_lists = self.settings.get("TEST_LIST_OF_LISTS")
        test_list_of_lists[0].append("first_three")
        assert copy.get("TEST_LIST_OF_LISTS")[0] == ["first_one", "first_two"]

    def test_copy_to_dict(self):
        s = BaseSettings(
            {
                "TEST_STRING": "a string",
                "TEST_LIST": [1, 2],
                "TEST_BOOLEAN": False,
                "TEST_BASE": BaseSettings({"foo": 1, "bar": 2}, "project"),
                "TEST": BaseSettings({"foo": 10, "baz": 30}, "default"),
                "HASNOBASE": BaseSettings({"baz": 3000}, "default"),
            }
        )
        assert s.copy_to_dict() == {
            "HASNOBASE": {"baz": 3000},
            "TEST": {"foo": 10, "baz": 30},
            "TEST_BASE": {"foo": 1, "bar": 2},
            "TEST_LIST": [1, 2],
            "TEST_BOOLEAN": False,
            "TEST_STRING": "a string",
        }

    def test_freeze(self):
        self.settings.freeze()
        with pytest.raises(
            TypeError, match="Trying to modify an immutable Settings object"
        ):
            self.settings.set("TEST_BOOL", False)

    def test_frozencopy(self):
        frozencopy = self.settings.frozencopy()
        assert frozencopy.frozen
        assert frozencopy is not self.settings

    def test_getwithbase_for_dotted_keys(self):
        settings = BaseSettings(
            {
                "FEED_EXPORTERS_BASE": BaseSettings({"json": "foo"}),
                "FEED_EXPORTERS": BaseSettings({"csv.gz": "bar"}),
            }
        )
        value = settings.getwithbase("FEED_EXPORTERS")
        assert isinstance(value, BaseSettings)
        assert dict(value) == {
            "json": "foo",
            "csv.gz": "bar",
        }

    @pytest.mark.parametrize("priority", [1, None])
    @pytest.mark.parametrize(
        ("key", "exception"),
        [
            pytest.param(1, TypeError, id="type-error"),
            pytest.param("foo", ValueError, id="value-error"),
            pytest.param("csv.gz", NameError, id="name-error"),
            pytest.param(
                "nonexistent.module.Component", ImportError, id="import-error"
            ),
        ],
    )
    def test_get_component_priority_dict_with_base_unresolvable_key(
        self, key: Any, exception: type[Exception], priority: int | None
    ) -> None:
        with pytest.raises(exception):
            load_object(key)

        settings = BaseSettings(
            {
                "FOO_BASE": BaseSettings({"csv.excel": 1}),
                "FOO": BaseSettings({key: priority}),
            }
        )
        with pytest.raises(ValueError, match=re.escape(f"{key!r}, a key of the FOO")):
            settings.get_component_priority_dict_with_base("FOO")

    def test_get_component_priority_dict_with_base_override_none_by_type(self):
        settings = BaseSettings()
        setting_names = set()
        for k, v in scrapy_default_settings.__dict__.items():
            if (
                not k.endswith("_BASE")
                or k in NON_COMPONENT_PRIORITY_DICT_BASE_SETTING_NAMES
            ):
                continue
            settings[k] = v
            setting_name = k[: -len("_BASE")]
            setting_names.add(setting_name)
            settings[setting_name] = {
                load_object(import_path): None for import_path in v
            }
        for setting_name in setting_names:
            value = settings.get_component_priority_dict_with_base(setting_name)
            assert not dict(value)

    def test_get_component_priority_dict_with_base_override_value_by_type(self):
        settings = BaseSettings()
        setting_names = set()
        value = 0
        for k, v in scrapy_default_settings.__dict__.items():
            if (
                not k.endswith("_BASE")
                or k in NON_COMPONENT_PRIORITY_DICT_BASE_SETTING_NAMES
            ):
                continue
            settings[k] = v
            setting_name = k[: -len("_BASE")]
            setting_names.add(setting_name)
            settings[setting_name] = {
                load_object(import_path): value for import_path in v
            }
        for setting_name in setting_names:
            assert (
                settings.get_component_priority_dict_with_base(setting_name)
                == settings[setting_name]
            )

    @pytest.mark.parametrize(
        "base_name", sorted(NON_COMPONENT_PRIORITY_DICT_BASE_SETTING_NAMES)
    )
    def test_getwithbase_for_non_component_priority_dicts(self, base_name: str) -> None:
        settings = BaseSettings()
        base_value = getattr(scrapy_default_settings, base_name)
        settings[base_name] = BaseSettings(base_value)
        assert len(base_value) >= 2
        keys = list(base_value)
        values = list(base_value.values())
        override = {keys[0]: values[1]}
        expected = dict(base_value)
        expected[keys[0]] = values[1]
        name = base_name[: -len("_BASE")]
        settings[name] = BaseSettings(override)
        value = settings.getwithbase(name)
        assert isinstance(value, BaseSettings)
        assert dict(value) == expected

    def test_get_component_priority_dict_with_base_warns_on_duplicate_import_paths(
        self, caplog
    ):
        settings = BaseSettings()
        settings["FOO"] = BaseSettings(
            {
                "scrapy.Request": 1,
                "scrapy.http.Request": 2,
            }
        )
        with caplog.at_level(logging.WARNING):
            value = settings.get_component_priority_dict_with_base("FOO")
        assert isinstance(value, BaseSettings)
        assert dict(value) == {"scrapy.http.Request": 2}
        assert caplog.records, "Expected a warning to be logged"
        msg = caplog.records[0].message
        assert "scrapy.http.request.Request" in msg

    def test_get_component_priority_dict_with_base_warns_on_duplicate_mixed_type_and_path(
        self, caplog
    ):
        settings = BaseSettings()
        settings["FOO"] = BaseSettings(
            {Component1: 1, "tests.test_settings.Component1": 2}  # type: ignore[arg-type]
        )
        with caplog.at_level(logging.WARNING):
            value = settings.get_component_priority_dict_with_base("FOO")
        assert isinstance(value, BaseSettings)
        assert dict(value) == {"tests.test_settings.Component1": 2}
        assert caplog.records, "Expected a warning to be logged"
        msg = caplog.records[0].message
        assert "tests.test_settings.Component1" in msg

    @pytest.mark.parametrize(
        ("value", "expected"),
        [
            ('["one", "two"]', ["one", "two"]),
            ('["a,b", "c"]', ["a,b", "c"]),
            ("[500, 502]", [500, 502]),
            ("[]", []),
            ("one,two", ["one", "two"]),
            ('{"a": 1}', ['{"a": 1}']),
            ("123", ["123"]),
            ("[one,two", ["[one", "two"]),
        ],
    )
    def test_getlist_str(self, value, expected):
        settings = BaseSettings({"S": value})
        assert settings.getlist("S") == expected

    def test_getdictorlist(self):
        settings = BaseSettings()

        # No value and no default → {}
        assert settings.getdictorlist("MISSING") == {}

        # String: valid JSON dict
        settings.set("S_DICT_STR", '{"key": "val"}')
        assert settings.getdictorlist("S_DICT_STR") == {"key": "val"}

        # String: valid JSON list
        settings.set("S_LIST_STR", '["a", "b"]')
        assert settings.getdictorlist("S_LIST_STR") == ["a", "b"]

        # String: invalid JSON → comma-split fallback
        settings.set("S_CSV", "a,b,c")
        assert settings.getdictorlist("S_CSV") == ["a", "b", "c"]

        # String: valid JSON but not dict or list → ValueError caught → comma-split
        settings.set("S_JSON_NUMBER", "123")
        assert settings.getdictorlist("S_JSON_NUMBER") == ["123"]

        # Tuple → list
        settings.set("S_TUPLE", ("x", "y"))
        assert settings.getdictorlist("S_TUPLE") == ["x", "y"]

        # Unsupported type → raises ValueError
        settings.set("S_INT", 42)
        with pytest.raises(ValueError, match="must be a dict, list, tuple, or string"):
            settings.getdictorlist("S_INT")

        # Dict value → deepcopy returned
        settings.set("S_DICT", {"key": "val"})
        assert settings.getdictorlist("S_DICT") == {"key": "val"}

        # List value → deepcopy returned
        settings.set("S_LIST", ["a", "b"])
        assert settings.getdictorlist("S_LIST") == ["a", "b"]

    def test_repr_pretty_(self):
        settings = BaseSettings({"key": "value"})
        mock_p = mock.Mock()

        settings._repr_pretty_(mock_p, cycle=False)
        assert mock_p.text.call_count == 1

        mock_p.reset_mock()
        settings._repr_pretty_(mock_p, cycle=True)
        mock_p.text.assert_called_once_with(repr(settings))

    def test_getwithbase_invalid_setting_name(self):
        settings = BaseSettings()
        with pytest.raises(
            ValueError, match="Base setting key must be a string, got 123"
        ):
            settings.getwithbase(123)  # type: ignore[arg-type]

    def test_get_component_priority_dict_with_base_invalid_setting_name(self):
        settings = BaseSettings()
        with pytest.raises(
            ValueError, match="Base setting key must be a string, got 123"
        ):
            settings.get_component_priority_dict_with_base(123)  # type: ignore[arg-type]


class TestSettings:
    def setup_method(self):
        self.settings = Settings()

    @mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {"default": 10})
    @mock.patch("scrapy.settings.default_settings", default_settings)
    def test_initial_defaults(self):
        settings = Settings()
        assert len(settings.attributes) == 2
        assert "TEST_DEFAULT" in settings.attributes

        attr = settings.attributes["TEST_DEFAULT"]
        assert isinstance(attr, SettingsAttribute)
        assert attr.value == "defvalue"
        assert attr.priority == 10

    @mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {})
    @mock.patch("scrapy.settings.default_settings", {})
    def test_initial_values(self):
        settings = Settings({"TEST_OPTION": "value"}, 10)
        assert len(settings.attributes) == 1
        assert "TEST_OPTION" in settings.attributes

        attr = settings.attributes["TEST_OPTION"]
        assert isinstance(attr, SettingsAttribute)
        assert attr.value == "value"
        assert attr.priority == 10

    @mock.patch("scrapy.settings.default_settings", default_settings)
    def test_autopromote_dicts(self):
        settings = Settings()
        mydict = settings.get("TEST_DICT")
        assert isinstance(mydict, BaseSettings)
        assert "key" in mydict
        assert mydict["key"] == "val"
        assert mydict.getpriority("key") == 0

    @mock.patch("scrapy.settings.default_settings", default_settings)
    def test_getdict_autodegrade_basesettings(self):
        settings = Settings()
        mydict = settings.getdict("TEST_DICT")
        assert isinstance(mydict, dict)
        assert len(mydict) == 1
        assert "key" in mydict
        assert mydict["key"] == "val"

    def test_passing_objects_as_values(self):
        class TestPipeline:
            def process_item(self, i):
                return i

        settings = Settings(
            {
                "ITEM_PIPELINES": {
                    TestPipeline: 800,
                },
                "DOWNLOAD_HANDLERS": {
                    "ftp": FileDownloadHandler,
                },
            }
        )

        assert "ITEM_PIPELINES" in settings.attributes

        mypipeline, priority = settings.getdict("ITEM_PIPELINES").popitem()
        assert priority == 800
        assert mypipeline == TestPipeline
        assert isinstance(mypipeline(), TestPipeline)
        assert mypipeline().process_item("item") == "item"

        myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp")
        assert myhandler == FileDownloadHandler
        myhandler_instance = build_from_crawler(myhandler, get_crawler())
        assert isinstance(myhandler_instance, FileDownloadHandler)
        assert hasattr(myhandler_instance, "download_request")

    def test_pop_item_with_default_value(self):
        settings = Settings()

        with pytest.raises(KeyError):
            settings.pop("DUMMY_CONFIG")

        dummy_config_value = settings.pop("DUMMY_CONFIG", "dummy_value")
        assert dummy_config_value == "dummy_value"

    def test_pop_item_with_immutable_settings(self):
        settings = Settings(
            {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"}
        )

        assert settings.pop("DUMMY_CONFIG") == "dummy_value"

        settings.freeze()

        with pytest.raises(
            TypeError, match="Trying to modify an immutable Settings object"
        ):
            settings.pop("OTHER_DUMMY_CONFIG")


@pytest.mark.parametrize(
    ("before", "name", "item", "after"),
    [
        ({}, "FOO", "BAR", {"FOO": ["BAR"]}),
        ({"FOO": []}, "FOO", "BAR", {"FOO": ["BAR"]}),
        ({"FOO": ["BAR"]}, "FOO", "BAZ", {"FOO": ["BAR", "BAZ"]}),
        ({"FOO": ["BAR"]}, "FOO", "BAR", {"FOO": ["BAR"]}),
        ({"FOO": ""}, "FOO", "BAR", {"FOO": ["BAR"]}),
        ({"FOO": "BAR"}, "FOO", "BAR", {"FOO": "BAR"}),
        ({"FOO": "BAR"}, "FOO", "BAZ", {"FOO": ["BAR", "BAZ"]}),
        ({"FOO": "BAR,BAZ"}, "FOO", "BAZ", {"FOO": "BAR,BAZ"}),
        ({"FOO": "BAR,BAZ"}, "FOO", "QUX", {"FOO": ["BAR", "BAZ", "QUX"]}),
    ],
)
def test_add_to_list(before, name, item, after):
    settings = BaseSettings(before, priority=0)
    settings.add_to_list(name, item)
    expected_priority = settings.getpriority(name) or 0
    expected_settings = BaseSettings(after, priority=expected_priority)
    assert settings == expected_settings, (
        f"{settings[name]=} != {expected_settings[name]=}"
    )
    assert settings.getpriority(name) == expected_settings.getpriority(name)


@pytest.mark.parametrize(
    ("before", "name", "item", "after"),
    [
        ({}, "FOO", "BAR", ValueError),
        ({"FOO": ["BAR"]}, "FOO", "BAR", {"FOO": []}),
        ({"FOO": ["BAR"]}, "FOO", "BAZ", ValueError),
        ({"FOO": ["BAR", "BAZ"]}, "FOO", "BAR", {"FOO": ["BAZ"]}),
        ({"FOO": ""}, "FOO", "BAR", ValueError),
        ({"FOO": "[]"}, "FOO", "BAR", ValueError),
        ({"FOO": "BAR"}, "FOO", "BAR", {"FOO": []}),
        ({"FOO": "BAR"}, "FOO", "BAZ", ValueError),
        ({"FOO": "BAR,BAZ"}, "FOO", "BAR", {"FOO": ["BAZ"]}),
    ],
)
def test_remove_from_list(before, name, item, after):
    settings = BaseSettings(before, priority=0)

    if isinstance(after, type) and issubclass(after, Exception):
        with pytest.raises(after):
            settings.remove_from_list(name, item)
        return

    settings.remove_from_list(name, item)
    expected_priority = settings.getpriority(name) or 0
    expected_settings = BaseSettings(after, priority=expected_priority)
    assert settings == expected_settings, (
        f"{settings[name]=} != {expected_settings[name]=}"
    )
    assert settings.getpriority(name) == expected_settings.getpriority(name)


def test_default_settings_all():
    deprecated = {"DNS_RESOLVER"}
    assert scrapy_default_settings.__all__ == sorted(
        name
        for name in vars(scrapy_default_settings)
        if name.isupper() and name not in deprecated
    )


@pytest.mark.parametrize(
    "name",
    [
        "DEFAULT_REQUEST_HEADERS",
        "DOWNLOADER_MIDDLEWARES_BASE",
        "DOWNLOAD_HANDLERS_BASE",
        "EXTENSIONS_BASE",
        "FEED_EXPORTERS_BASE",
        "FEED_STORAGES_BASE",
        "LOG_VERSIONS",
        "RETRY_EXCEPTIONS",
        "RETRY_HTTP_CODES",
        "SPIDER_CONTRACTS_BASE",
        "SPIDER_MIDDLEWARES_BASE",
    ],
)
def test_documented_default(name: str) -> None:
    # In the section of the setting, captures either the inline literal of
    # "Default: ``...``" or the indented block of the code-block directive
    # that follows "Default:".
    pattern = re.compile(
        rf"^\.\. setting:: {name}\n"
        r"(?:(?!^\.\. setting::).)*?"
        r"^Default:(?: ``([^`\n]+)``|\n\n\.\. code-block:: python\n\n"
        r"((?: {4}[^\n]*\n|\n)+))",
        re.DOTALL | re.MULTILINE,
    )
    docs_dir = Path(__file__).parents[2] / "docs"
    matches = [
        match
        for path in docs_dir.glob("**/*.rst")
        if (match := pattern.search(path.read_text(encoding="utf-8")))
    ]
    assert len(matches) == 1
    inline, block = matches[0].groups()
    # plain eval() because of OSError in RETRY_EXCEPTIONS
    documented = eval(inline or textwrap.dedent(block), {})
    assert documented == getattr(scrapy_default_settings, name)


def test_deprecated_dns_resolver_setting():
    settings = Settings()
    with pytest.warns(
        ScrapyDeprecationWarning,
        match="The DNS_RESOLVER setting is deprecated",
    ):
        settings.get("DNS_RESOLVER")


def test_deprecated_concurrent_requests_per_ip_setting():
    settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1})
    with pytest.warns(
        ScrapyDeprecationWarning,
        match="The CONCURRENT_REQUESTS_PER_IP setting is deprecated",
    ):
        settings.get("CONCURRENT_REQUESTS_PER_IP")


class Component1:
    pass


Component1Alias = Component1


class Component1Subclass(Component1):
    pass


Component1SubclassAlias = Component1Subclass


class Component2:
    pass


class Component3:
    pass


class Component4:
    pass


@pytest.mark.parametrize(
    ("before", "name", "old_cls", "new_cls", "priority", "after"),
    [
        ({}, "FOO", Component1, Component2, None, KeyError),
        (
            {"FOO": {Component1: 1}},
            "FOO",
            Component1,
            Component2,
            None,
            {"FOO": {Component2: 1}},
        ),
        (
            {"FOO": {Component1: 1}},
            "FOO",
            Component1,
            Component2,
            2,
            {"FOO": {Component2: 2}},
        ),
        (
            {"FOO": {"tests.test_settings.Component1": 1}},
            "FOO",
            Component1,
            Component2,
            None,
            {"FOO": {Component2: 1}},
        ),
        (
            {"FOO": {Component1Alias: 1}},
            "FOO",
            Component1,
            Component2,
            None,
            {"FOO": {Component2: 1}},
        ),
        (
            {"FOO": {Component1Alias: 1}},
            "FOO",
            Component1,
            Component2,
            2,
            {"FOO": {Component2: 2}},
        ),
        (
            {"FOO": {"tests.test_settings.Component1Alias": 1}},
            "FOO",
            Component1,
            Component2,
            None,
            {"FOO": {Component2: 1}},
        ),
        (
            {"FOO": {"tests.test_settings.Component1Alias": 1}},
            "FOO",
            Component1,
            Component2,
            2,
            {"FOO": {Component2: 2}},
        ),
        (
            {
                "FOO": {
                    "tests.test_settings.Component1": 1,
                    "tests.test_settings.Component1Alias": 2,
                }
            },
            "FOO",
            Component1,
            Component2,
            None,
            {"FOO": {Component2: 2}},
        ),
        (
            {
                "FOO": {
                    "tests.test_settings.Component1": 1,
                    "tests.test_settings.Component1Alias": 2,
                }
            },
            "FOO",
            Component1,
            Component2,
            3,
            {"FOO": {Component2: 3}},
        ),
        (
            {"FOO": '{"tests.test_settings.Component1": 1}'},
            "FOO",
            Component1,
            Component2,
            None,
            {"FOO": {Component2: 1}},
        ),
        (
            {"FOO": '{"tests.test_settings.Component1": 1}'},
            "FOO",
            Component1,
            Component2,
            2,
            {"FOO": {Component2: 2}},
        ),
        (
            {"FOO": '{"tests.test_settings.Component1Alias": 1}'},
            "FOO",
            Component1,
            Component2,
            None,
            {"FOO": {Component2: 1}},
        ),
        (
            {"FOO": '{"tests.test_settings.Component1Alias": 1}'},
            "FOO",
            Component1,
            Component2,
            2,
            {"FOO": {Component2: 2}},
        ),
        (
            {
                "FOO": '{"tests.test_settings.Component1": 1, "tests.test_settings.Component1Alias": 2}'
            },
            "FOO",
            Component1,
            Component2,
            None,
            {"FOO": {Component2: 2}},
        ),
        (
            {
                "FOO": '{"tests.test_settings.Component1": 1, "tests.test_settings.Component1Alias": 2}'
            },
            "FOO",
            Component1,
            Component2,
            3,
            {"FOO": {Component2: 3}},
        ),
        # If old_cls has None as value, raise KeyError.
        (
            {"FOO": {Component1: None}},
            "FOO",
            Component1,
            Component2,
            None,
            KeyError,
        ),
        (
            {"FOO": '{"tests.test_settings.Component1": null}'},
            "FOO",
            Component1,
            Component2,
            None,
            KeyError,
        ),
        (
            {"FOO": {Component1: None, "tests.test_settings.Component1": None}},
            "FOO",
            Component1,
            Component2,
            None,
            KeyError,
        ),
        (
            {"FOO": {Component1: 1, "tests.test_settings.Component1": None}},
            "FOO",
            Component1,
            Component2,
            None,
            KeyError,
        ),
        (
            {"FOO": {Component1: None, "tests.test_settings.Component1": 1}},
            "FOO",
            Component1,
            Component2,
            None,
            KeyError,
        ),
        # Unrelated components are kept as is, as expected.
        (
            {
                "FOO": {
                    Component1: 1,
                    "tests.test_settings.Component2": 2,
                    Component3: 3,
                }
            },
            "FOO",
            Component3,
            Component4,
            None,
            {
                "FOO": {
                    Component1: 1,
                    "tests.test_settings.Component2": 2,
                    Component4: 3,
                }
            },
        ),
    ],
)
def test_replace_in_component_priority_dict(
    before, name, old_cls, new_cls, priority, after
):
    settings = BaseSettings(before, priority=0)

    if isinstance(after, type) and issubclass(after, Exception):
        with pytest.raises(after):
            settings.replace_in_component_priority_dict(
                name, old_cls, new_cls, priority
            )
        return

    expected_priority = settings.getpriority(name) or 0
    settings.replace_in_component_priority_dict(name, old_cls, new_cls, priority)
    expected_settings = BaseSettings(after, priority=expected_priority)
    assert settings == expected_settings
    assert settings.getpriority(name) == expected_settings.getpriority(name)


@pytest.mark.parametrize(
    ("before", "name", "cls", "priority", "after"),
    [
        # Set
        ({}, "FOO", Component1, None, {"FOO": {Component1: None}}),
        ({}, "FOO", Component1, 0, {"FOO": {Component1: 0}}),
        ({}, "FOO", Component1, 1, {"FOO": {Component1: 1}}),
        # Add
        (
            {"FOO": {Component1: 0}},
            "FOO",
            Component2,
            None,
            {"FOO": {Component1: 0, Component2: None}},
        ),
        (
            {"FOO": {Component1: 0}},
            "FOO",
            Component2,
            0,
            {"FOO": {Component1: 0, Component2: 0}},
        ),
        (
            {"FOO": {Component1: 0}},
            "FOO",
            Component2,
            1,
            {"FOO": {Component1: 0, Component2: 1}},
        ),
        # Replace
        (
            {
                "FOO": {
                    Component1: None,
                    "tests.test_settings.Component1": 0,
                    "tests.test_settings.Component1Alias": 1,
                    Component1Subclass: None,
                    "tests.test_settings.Component1Subclass": 0,
                    "tests.test_settings.Component1SubclassAlias": 1,
                }
            },
            "FOO",
            Component1,
            None,
            {
                "FOO": {
                    Component1: None,
                    Component1Subclass: None,
                    "tests.test_settings.Component1Subclass": 0,
                    "tests.test_settings.Component1SubclassAlias": 1,
                }
            },
        ),
        (
            {
                "FOO": {
                    Component1: 0,
                    "tests.test_settings.Component1": 1,
                    "tests.test_settings.Component1Alias": None,
                    Component1Subclass: 0,
                    "tests.test_settings.Component1Subclass": 1,
                    "tests.test_settings.Component1SubclassAlias": None,
                }
            },
            "FOO",
            Component1,
            0,
            {
                "FOO": {
                    Component1: 0,
                    Component1Subclass: 0,
                    "tests.test_settings.Component1Subclass": 1,
                    "tests.test_settings.Component1SubclassAlias": None,
                }
            },
        ),
        (
            {
                "FOO": {
                    Component1: 1,
                    "tests.test_settings.Component1": None,
                    "tests.test_settings.Component1Alias": 0,
                    Component1Subclass: 1,
                    "tests.test_settings.Component1Subclass": None,
                    "tests.test_settings.Component1SubclassAlias": 0,
                }
            },
            "FOO",
            Component1,
            1,
            {
                "FOO": {
                    Component1: 1,
                    Component1Subclass: 1,
                    "tests.test_settings.Component1Subclass": None,
                    "tests.test_settings.Component1SubclassAlias": 0,
                }
            },
        ),
        # String-based setting values
        (
            {"FOO": '{"tests.test_settings.Component1": 0}'},
            "FOO",
            Component2,
            None,
            {"FOO": {"tests.test_settings.Component1": 0, Component2: None}},
        ),
        (
            {
                "FOO": """{
                    "tests.test_settings.Component1": 0,
                    "tests.test_settings.Component1Alias": 1,
                    "tests.test_settings.Component1Subclass": 0,
                    "tests.test_settings.Component1SubclassAlias": 1
                }"""
            },
            "FOO",
            Component1,
            None,
            {
                "FOO": {
                    Component1: None,
                    "tests.test_settings.Component1Subclass": 0,
                    "tests.test_settings.Component1SubclassAlias": 1,
                }
            },
        ),
    ],
)
def test_set_in_component_priority_dict(before, name, cls, priority, after):
    settings = BaseSettings(before, priority=0)
    expected_priority = settings.getpriority(name) or 0
    settings.set_in_component_priority_dict(name, cls, priority)
    expected_settings = BaseSettings(after, priority=expected_priority)
    assert settings == expected_settings
    assert settings.getpriority(name) == expected_settings.getpriority(name), (
        f"{settings.getpriority(name)=} != {expected_settings.getpriority(name)=}"
    )


@pytest.mark.parametrize(
    ("before", "name", "cls", "priority", "after"),
    [
        # Set
        ({}, "FOO", Component1, None, {"FOO": {Component1: None}}),
        ({}, "FOO", Component1, 0, {"FOO": {Component1: 0}}),
        ({}, "FOO", Component1, 1, {"FOO": {Component1: 1}}),
        # Add
        (
            {"FOO": {Component1: 0}},
            "FOO",
            Component2,
            None,
            {"FOO": {Component1: 0, Component2: None}},
        ),
        (
            {"FOO": {Component1: 0}},
            "FOO",
            Component2,
            0,
            {"FOO": {Component1: 0, Component2: 0}},
        ),
        (
            {"FOO": {Component1: 0}},
            "FOO",
            Component2,
            1,
            {"FOO": {Component1: 0, Component2: 1}},
        ),
        # Keep
        (
            {
                "FOO": {
                    Component1: None,
                    "tests.test_settings.Component1": 0,
                    "tests.test_settings.Component1Alias": 1,
                    Component1Subclass: None,
                    "tests.test_settings.Component1Subclass": 0,
                    "tests.test_settings.Component1SubclassAlias": 1,
                }
            },
            "FOO",
            Component1,
            None,
            {
                "FOO": {
                    Component1: None,
                    "tests.test_settings.Component1": 0,
                    "tests.test_settings.Component1Alias": 1,
                    Component1Subclass: None,
                    "tests.test_settings.Component1Subclass": 0,
                    "tests.test_settings.Component1SubclassAlias": 1,
                }
            },
        ),
        (
            {
                "FOO": {
                    Component1: 0,
                    "tests.test_settings.Component1": 1,
                    "tests.test_settings.Component1Alias": None,
                    Component1Subclass: 0,
                    "tests.test_settings.Component1Subclass": 1,
                    "tests.test_settings.Component1SubclassAlias": None,
                }
            },
            "FOO",
            Component1,
            0,
            {
                "FOO": {
                    Component1: 0,
                    "tests.test_settings.Component1": 1,
                    "tests.test_settings.Component1Alias": None,
                    Component1Subclass: 0,
                    "tests.test_settings.Component1Subclass": 1,
                    "tests.test_settings.Component1SubclassAlias": None,
                }
            },
        ),
        (
            {
                "FOO": {
                    Component1: 1,
                    "tests.test_settings.Component1": None,
                    "tests.test_settings.Component1Alias": 0,
                    Component1Subclass: 1,
                    "tests.test_settings.Component1Subclass": None,
                    "tests.test_settings.Component1SubclassAlias": 0,
                }
            },
            "FOO",
            Component1,
            1,
            {
                "FOO": {
                    Component1: 1,
                    "tests.test_settings.Component1": None,
                    "tests.test_settings.Component1Alias": 0,
                    Component1Subclass: 1,
                    "tests.test_settings.Component1Subclass": None,
                    "tests.test_settings.Component1SubclassAlias": 0,
                }
            },
        ),
        # String-based setting values
        (
            {"FOO": '{"tests.test_settings.Component1": 0}'},
            "FOO",
            Component2,
            None,
            {"FOO": {"tests.test_settings.Component1": 0, Component2: None}},
        ),
        (
            {
                "FOO": """{
                    "tests.test_settings.Component1": 0,
                    "tests.test_settings.Component1Alias": 1,
                    "tests.test_settings.Component1Subclass": 0,
                    "tests.test_settings.Component1SubclassAlias": 1
                }"""
            },
            "FOO",
            Component1,
            None,
            {
                "FOO": """{
                    "tests.test_settings.Component1": 0,
                    "tests.test_settings.Component1Alias": 1,
                    "tests.test_settings.Component1Subclass": 0,
                    "tests.test_settings.Component1SubclassAlias": 1
                }"""
            },
        ),
    ],
)
def test_setdefault_in_component_priority_dict(before, name, cls, priority, after):
    settings = BaseSettings(before, priority=0)
    expected_priority = settings.getpriority(name) or 0
    settings.setdefault_in_component_priority_dict(name, cls, priority)
    expected_settings = BaseSettings(after, priority=expected_priority)
    assert settings == expected_settings
    assert settings.getpriority(name) == expected_settings.getpriority(name)