Scrapy, a fast high-level web crawling & scraping framework for Python.
Typing for scrapy/settings/__init__.py.
Andrey Rakhmatullin187e8f9a2d2f
diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 69ff07bb7..192541dd0 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -55,7 +55,7 @@ class Crawler:def __init__(self,spidercls: Type[Spider],- settings: Union[None, dict, Settings] = None,+ settings: Union[None, Dict[str, Any], Settings] = None,init_reactor: bool = False,):if isinstance(spidercls, Spider):diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.pyindex cc44d67e8..658c27f0a 100644--- a/scrapy/settings/__init__.py+++ b/scrapy/settings/__init__.py@@ -1,12 +1,41 @@+from __future__ import annotations+import copyimport json-from collections.abc import MutableMappingfrom importlib import import_modulefrom pprint import pformat+from types import ModuleType+from typing import (+ TYPE_CHECKING,+ Any,+ Dict,+ Iterable,+ Iterator,+ List,+ Mapping,+ MutableMapping,+ Optional,+ Tuple,+ Union,+ cast,+)from scrapy.settings import default_settings-SETTINGS_PRIORITIES = {+# The key types are restricted in BaseSettings._get_key() to ones supported by JSON,+# see https://github.com/scrapy/scrapy/issues/5383.+_SettingsKeyT = Union[bool, float, int, str, None]++if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ # https://github.com/python/typing/issues/445#issuecomment-1131458824+ from _typeshed import SupportsItems+ from typing_extensions import Self++ _SettingsInputT = Union[SupportsItems[_SettingsKeyT, Any], str, None]+++SETTINGS_PRIORITIES: Dict[str, int] = {"default": 0,"command": 10,"project": 20,@@ -15,7 +44,7 @@ SETTINGS_PRIORITIES = {}-def get_settings_priority(priority):+def get_settings_priority(priority: Union[int, str]) -> int:"""Small helper function that looks up a given string priority in the:attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its@@ -34,14 +63,15 @@ class SettingsAttribute:for settings configuration, not this one."""- def __init__(self, value, priority):- self.value = value+ def __init__(self, value: Any, priority: int):+ self.value: Any = value+ self.priority: intif isinstance(self.value, BaseSettings):self.priority = max(self.value.maxpriority(), priority)else:self.priority = priority- def set(self, value, priority):+ def set(self, value: Any, priority: int) -> None:"""Sets value if priority is higher or equal than current priority."""if priority >= self.priority:if isinstance(self.value, BaseSettings):@@ -49,11 +79,11 @@ class SettingsAttribute:self.value = valueself.priority = priority- def __repr__(self):+ def __repr__(self) -> str:return f"<SettingsAttribute value={self.value!r} priority={self.priority}>"-class BaseSettings(MutableMapping):+class BaseSettings(MutableMapping[_SettingsKeyT, Any]):"""Instances of this class behave like dictionaries, but store prioritiesalong with their ``(key, value)`` pairs, and can be frozen (i.e. marked@@ -77,21 +107,23 @@ class BaseSettings(MutableMapping):__default = object()- def __init__(self, values=None, priority="project"):- self.frozen = False- self.attributes = {}+ def __init__(+ self, values: _SettingsInputT = None, priority: Union[int, str] = "project"+ ):+ self.frozen: bool = False+ self.attributes: dict[_SettingsKeyT, SettingsAttribute] = {}if values:self.update(values, priority)- def __getitem__(self, opt_name):+ def __getitem__(self, opt_name: _SettingsKeyT) -> Any:if opt_name not in self:return Nonereturn self.attributes[opt_name].value- def __contains__(self, name):+ def __contains__(self, name: Any) -> bool:return name in self.attributes- def get(self, name, default=None):+ def get(self, name: _SettingsKeyT, default: Any = None) -> Any:"""Get a setting value without affecting its original type.@@ -103,7 +135,7 @@ class BaseSettings(MutableMapping):"""return self[name] if self[name] is not None else default- def getbool(self, name, default=False):+ def getbool(self, name: _SettingsKeyT, default: bool = False) -> bool:"""Get a setting value as a boolean.@@ -133,7 +165,7 @@ class BaseSettings(MutableMapping):"'True'/'False' and 'true'/'false'")- def getint(self, name, default=0):+ def getint(self, name: _SettingsKeyT, default: int = 0) -> int:"""Get a setting value as an int.@@ -145,7 +177,7 @@ class BaseSettings(MutableMapping):"""return int(self.get(name, default))- def getfloat(self, name, default=0.0):+ def getfloat(self, name: _SettingsKeyT, default: float = 0.0) -> float:"""Get a setting value as a float.@@ -157,7 +189,9 @@ class BaseSettings(MutableMapping):"""return float(self.get(name, default))- def getlist(self, name, default=None):+ def getlist(+ self, name: _SettingsKeyT, default: Optional[List[Any]] = None+ ) -> List[Any]:"""Get a setting value as a list. If the setting original type is a list, acopy of it will be returned. If it's a string it will be split by ",".@@ -176,7 +210,9 @@ class BaseSettings(MutableMapping):value = value.split(",")return list(value)- def getdict(self, name, default=None):+ def getdict(+ self, name: _SettingsKeyT, default: Optional[Dict[Any, Any]] = None+ ) -> Dict[Any, Any]:"""Get a setting value as a dictionary. If the setting original type is adictionary, a copy of it will be returned. If it is a string it will be@@ -197,7 +233,11 @@ class BaseSettings(MutableMapping):value = json.loads(value)return dict(value)- def getdictorlist(self, name, default=None):+ def getdictorlist(+ self,+ name: _SettingsKeyT,+ default: Union[Dict[Any, Any], List[Any], None] = None,+ ) -> Union[Dict[Any, Any], List[Any]]:"""Get a setting value as either a :class:`dict` or a :class:`list`.If the setting is already a dict or a list, a copy of it will be@@ -224,24 +264,29 @@ class BaseSettings(MutableMapping):return {}if isinstance(value, str):try:- return json.loads(value)+ value_loaded = json.loads(value)+ assert isinstance(value_loaded, (dict, list))+ return value_loadedexcept ValueError:return value.split(",")+ assert isinstance(value, (dict, list))return copy.deepcopy(value)- def getwithbase(self, name):+ def getwithbase(self, name: _SettingsKeyT) -> "BaseSettings":"""Get a composition of a dictionary-like setting and its `_BASE`counterpart.:param name: name of the dictionary-like setting:type name: str"""+ if not isinstance(name, str):+ raise ValueError(f"Base setting key must be a string, got {name}")compbs = BaseSettings()compbs.update(self[name + "_BASE"])compbs.update(self[name])return compbs- def getpriority(self, name):+ def getpriority(self, name: _SettingsKeyT) -> Optional[int]:"""Return the current numerical priority value of a setting, or ``None`` ifthe given ``name`` does not exist.@@ -253,7 +298,7 @@ class BaseSettings(MutableMapping):return Nonereturn self.attributes[name].priority- def maxpriority(self):+ def maxpriority(self) -> int:"""Return the numerical value of the highest priority present throughoutall settings, or the numerical value for ``default`` from@@ -261,13 +306,15 @@ class BaseSettings(MutableMapping):stored."""if len(self) > 0:- return max(self.getpriority(name) for name in self)+ return max(cast(int, self.getpriority(name)) for name in self)return get_settings_priority("default")- def __setitem__(self, name, value):+ def __setitem__(self, name: _SettingsKeyT, value: Any) -> None:self.set(name, value)- def set(self, name, value, priority="project"):+ def set(+ self, name: _SettingsKeyT, value: Any, priority: Union[int, str] = "project"+ ) -> None:"""Store a key/value attribute with a given priority.@@ -295,17 +342,26 @@ class BaseSettings(MutableMapping):else:self.attributes[name].set(value, priority)- def setdefault(self, name, default=None, priority="project"):+ def setdefault(+ self,+ name: _SettingsKeyT,+ default: Any = None,+ priority: Union[int, str] = "project",+ ) -> Any:if name not in self:self.set(name, default, priority)return defaultreturn self.attributes[name].value- def setdict(self, values, priority="project"):+ def setdict(+ self, values: _SettingsInputT, priority: Union[int, str] = "project"+ ) -> None:self.update(values, priority)- def setmodule(self, module, priority="project"):+ def setmodule(+ self, module: Union[ModuleType, str], priority: Union[int, str] = "project"+ ) -> None:"""Store settings from a module with a given priority.@@ -327,7 +383,8 @@ class BaseSettings(MutableMapping):if key.isupper():self.set(key, getattr(module, key), priority)- def update(self, values, priority="project"):+ # BaseSettings.update() doesn't support all inputs that MutableMapping.update() supports+ def update(self, values: _SettingsInputT, priority: Union[int, str] = "project") -> None: # type: ignore[override]"""Store key/value pairs with a given priority.@@ -351,30 +408,34 @@ class BaseSettings(MutableMapping):"""self._assert_mutability()if isinstance(values, str):- values = json.loads(values)+ values = cast(dict, json.loads(values))if values is not None:if isinstance(values, BaseSettings):for name, value in values.items():- self.set(name, value, values.getpriority(name))+ self.set(name, value, cast(int, values.getpriority(name)))else:for name, value in values.items():self.set(name, value, priority)- def delete(self, name, priority="project"):+ def delete(+ self, name: _SettingsKeyT, priority: Union[int, str] = "project"+ ) -> None:+ if name not in self:+ raise KeyError(name)self._assert_mutability()priority = get_settings_priority(priority)- if priority >= self.getpriority(name):+ if priority >= cast(int, self.getpriority(name)):del self.attributes[name]- def __delitem__(self, name):+ def __delitem__(self, name: _SettingsKeyT) -> None:self._assert_mutability()del self.attributes[name]- def _assert_mutability(self):+ def _assert_mutability(self) -> None:if self.frozen:raise TypeError("Trying to modify an immutable Settings object")- def copy(self):+ def copy(self) -> "Self":"""Make a deep copy of current settings.@@ -386,7 +447,7 @@ class BaseSettings(MutableMapping):"""return copy.deepcopy(self)- def freeze(self):+ def freeze(self) -> None:"""Disable further changes to the current settings.@@ -396,7 +457,7 @@ class BaseSettings(MutableMapping):"""self.frozen = True- def frozencopy(self):+ def frozencopy(self) -> "Self":"""Return an immutable copy of the current settings.@@ -406,26 +467,26 @@ class BaseSettings(MutableMapping):copy.freeze()return copy- def __iter__(self):+ def __iter__(self) -> Iterator[_SettingsKeyT]:return iter(self.attributes)- def __len__(self):+ def __len__(self) -> int:return len(self.attributes)- def _to_dict(self):+ def _to_dict(self) -> Dict[_SettingsKeyT, Any]:return {self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v)for k, v in self.items()}- def _get_key(self, key_value):+ def _get_key(self, key_value: Any) -> _SettingsKeyT:return (key_valueif isinstance(key_value, (bool, float, int, str, type(None)))else str(key_value))- def copy_to_dict(self):+ def copy_to_dict(self) -> Dict[_SettingsKeyT, Any]:"""Make a copy of current settings and convert to a dict.@@ -441,13 +502,14 @@ class BaseSettings(MutableMapping):settings = self.copy()return settings._to_dict()- def _repr_pretty_(self, p, cycle):+ # https://ipython.readthedocs.io/en/stable/config/integrating.html#pretty-printing+ def _repr_pretty_(self, p: Any, cycle: bool) -> None:if cycle:p.text(repr(self))else:p.text(pformat(self.copy_to_dict()))- def pop(self, name, default=__default):+ def pop(self, name: _SettingsKeyT, default: Any = __default) -> Any:try:value = self.attributes[name].valueexcept KeyError:@@ -471,7 +533,9 @@ class Settings(BaseSettings):described on :ref:`topics-settings-ref` already populated."""- def __init__(self, values=None, priority="project"):+ def __init__(+ self, values: _SettingsInputT = None, priority: Union[int, str] = "project"+ ):# Do not pass kwarg values here. We don't want to promote user-defined# dicts, and we want to update, not replace, default dicts with the# values given by the user@@ -485,14 +549,16 @@ class Settings(BaseSettings):self.update(values, priority)-def iter_default_settings():+def iter_default_settings() -> Iterable[Tuple[str, Any]]:"""Return the default settings as an iterator of (name, value) tuples"""for name in dir(default_settings):if name.isupper():yield name, getattr(default_settings, name)-def overridden_settings(settings):+def overridden_settings(+ settings: Mapping[_SettingsKeyT, Any]+) -> Iterable[Tuple[str, Any]]:"""Return a dict of the settings that have been overridden"""for name, defvalue in iter_default_settings():value = settings[name]diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.pyindex 3ade1d105..0608527ae 100644--- a/scrapy/utils/conf.py+++ b/scrapy/utils/conf.py@@ -28,6 +28,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):compbs = BaseSettings()for k, v in compdict.items():prio = compdict.getpriority(k)+ assert prio is not Noneif compbs.getpriority(convert(k)) == prio:raise ValueError(f"Some paths in {list(compdict.keys())!r} "diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.pyindex 5fc825393..db000233e 100644--- a/tests/test_settings/__init__.py+++ b/tests/test_settings/__init__.py@@ -1,6 +1,8 @@import unittestfrom unittest import mock+import pytest+from scrapy.settings import (SETTINGS_PRIORITIES,BaseSettings,@@ -199,6 +201,21 @@ class BaseSettingsTest(unittest.TestCase):settings.update({"key_lowprio": 3}, priority=20)self.assertEqual(settings["key_lowprio"], 1)+ @pytest.mark.xfail(+ raises=TypeError, reason="BaseSettings.update doesn't support kwargs input"+ )+ def test_update_kwargs(self):+ settings = BaseSettings({"key": 0})+ settings.update(key=1)++ @pytest.mark.xfail(+ raises=AttributeError,+ reason="BaseSettings.update doesn't support iterable input",+ )+ def test_update_iterable(self):+ settings = BaseSettings({"key": 0})+ settings.update([("key", 1)])+def test_update_jsonstring(self):settings = BaseSettings({"number": 0, "dict": BaseSettings({"key": "val"})})settings.update('{"number": 1, "newnumber": 2}')@@ -217,6 +234,10 @@ class BaseSettingsTest(unittest.TestCase):self.assertIn("key_highprio", settings)del settings["key_highprio"]self.assertNotIn("key_highprio", settings)+ with self.assertRaises(KeyError):+ settings.delete("notkey")+ with self.assertRaises(KeyError):+ del settings["notkey"]def test_get(self):test_configuration = {diff --git a/tox.ini b/tox.iniindex fe7df9782..1aeb94215 100644--- a/tox.ini+++ b/tox.ini@@ -34,6 +34,7 @@ install_command =basepython = python3deps =mypy==1.4.1+ typing-extensions==4.7.1types-attrs==19.1.0types-lxml==2023.3.28types-Pillow==10.0.0.1