Scrapy, a fast high-level web crawling & scraping framework for Python.
Replace _SettingsKeyT with str. (#7586)
Andrey Rakhmatullind59f9b644af2
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.pyindex d4671fd35..be298e9b1 100644--- a/scrapy/settings/__init__.py+++ b/scrapy/settings/__init__.py@@ -16,10 +16,6 @@ from scrapy.utils.python import global_object_namelogger = getLogger(__name__)-# The key types are restricted in BaseSettings._get_key() to ones supported by JSON,-# see https://github.com/scrapy/scrapy/issues/5383.-_SettingsKey: TypeAlias = bool | float | int | str | None-if TYPE_CHECKING:from types import ModuleType@@ -29,7 +25,7 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self- _SettingsInput: TypeAlias = SupportsItems[_SettingsKey, Any] | str | None+ _SettingsInput: TypeAlias = SupportsItems[str, Any] | str | NoneSETTINGS_PRIORITIES: dict[str, int] = {@@ -80,7 +76,7 @@ class SettingsAttribute:return f"<SettingsAttribute value={self.value!r} priority={self.priority}>"-class BaseSettings(MutableMapping[_SettingsKey, Any]):+class BaseSettings(MutableMapping[str, Any]):"""Instances of this class behave like dictionaries, but store prioritiesalong with their ``(key, value)`` pairs, and can be frozen (i.e. marked@@ -106,11 +102,11 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):def __init__(self, values: _SettingsInput = None, priority: int | str = "project"):self.frozen: bool = False- self.attributes: dict[_SettingsKey, SettingsAttribute] = {}+ self.attributes: dict[str, SettingsAttribute] = {}if values:self.update(values, priority)- def __getitem__(self, opt_name: _SettingsKey) -> Any:+ def __getitem__(self, opt_name: str) -> Any:if opt_name not in self:return Nonereturn self.attributes[opt_name].value@@ -118,7 +114,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):def __contains__(self, name: Any) -> bool:return name in self.attributes- def add_to_list(self, name: _SettingsKey, item: Any) -> None:+ def add_to_list(self, name: str, item: Any) -> None:"""Append *item* to the :class:`list` setting with the specified *name*if *item* is not already in that list.@@ -129,7 +125,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):if item not in value:self.set(name, [*value, item], self.getpriority(name) or 0)- def remove_from_list(self, name: _SettingsKey, item: Any) -> None:+ def remove_from_list(self, name: str, item: Any) -> None:"""Remove *item* from the :class:`list` setting with the specified*name*.@@ -143,7 +139,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):raise ValueError(f"{item!r} not found in the {name} setting ({value!r}).")self.set(name, [v for v in value if v != item], self.getpriority(name) or 0)- def get(self, name: _SettingsKey, default: Any = None) -> Any: # pylint: disable=arguments-renamed+ def get(self, name: str, default: Any = None) -> Any: # pylint: disable=arguments-renamed"""Get a setting value without affecting its original type.@@ -172,7 +168,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):return self[name] if self[name] is not None else default- def getbool(self, name: _SettingsKey, default: bool = False) -> bool:+ def getbool(self, name: str, default: bool = False) -> bool:"""Get a setting value as a boolean.@@ -202,7 +198,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):"'True'/'False' and 'true'/'false'") from None- def getint(self, name: _SettingsKey, default: int = 0) -> int:+ def getint(self, name: str, default: int = 0) -> int:"""Get a setting value as an int.@@ -214,7 +210,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):"""return int(self.get(name, default))- def getfloat(self, name: _SettingsKey, default: float = 0.0) -> float:+ def getfloat(self, name: str, default: float = 0.0) -> float:"""Get a setting value as a float.@@ -226,9 +222,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):"""return float(self.get(name, default))- def getlist(- self, name: _SettingsKey, default: list[Any] | None = None- ) -> list[Any]:+ def getlist(self, name: str, default: list[Any] | None = None) -> list[Any]:"""Get a setting value as a list. If the setting original type is a list,a copy of it will be returned. If it's a string it will be split by@@ -251,7 +245,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):return list(value)def getdict(- self, name: _SettingsKey, default: dict[Any, Any] | None = None+ self, name: str, default: dict[Any, Any] | None = None) -> dict[Any, Any]:"""Get a setting value as a dictionary. If the setting original type is a@@ -275,7 +269,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):def getdictorlist(self,- name: _SettingsKey,+ name: str,default: dict[Any, Any] | list[Any] | tuple[Any] | None = None,) -> dict[Any, Any] | list[Any]:"""Get a setting value as either a :class:`dict` or a :class:`list`.@@ -322,7 +316,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):)return copy.deepcopy(value)- def getwithbase(self, name: _SettingsKey) -> BaseSettings:+ def getwithbase(self, name: str) -> BaseSettings:"""Get a composition of a dictionary-like setting and its ``_BASE``counterpart.@@ -341,7 +335,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):compbs.update(self[name])return compbs- def get_component_priority_dict_with_base(self, name: _SettingsKey) -> BaseSettings:+ def get_component_priority_dict_with_base(self, name: str) -> BaseSettings:"""Get a composition of a component priority dictionary setting andits ``_BASE`` counterpart.@@ -389,7 +383,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):{restore_key(k): v for k, v in result.items() if v is not None})- def getpriority(self, name: _SettingsKey) -> int | None:+ def getpriority(self, name: str) -> int | None:"""Return the current numerical priority value of a setting, or ``None`` ifthe given ``name`` does not exist.@@ -414,7 +408,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):def replace_in_component_priority_dict(self,- name: _SettingsKey,+ name: str,old_cls: type,new_cls: type,priority: int | None = None,@@ -453,12 +447,10 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):)self.set(name, component_priority_dict, priority=self.getpriority(name) or 0)- def __setitem__(self, name: _SettingsKey, value: Any) -> None:+ def __setitem__(self, name: str, value: Any) -> None:self.set(name, value)- def set(- self, name: _SettingsKey, value: Any, priority: int | str = "project"- ) -> None:+ def set(self, name: str, value: Any, priority: int | str = "project") -> None:"""Store a key/value attribute with a given priority.@@ -487,7 +479,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):self.attributes[name].set(value, priority)def set_in_component_priority_dict(- self, name: _SettingsKey, cls: type, priority: int | None+ self, name: str, cls: type, priority: int | None) -> None:"""Set the *cls* component in the *name* :ref:`component prioritydictionary <component-priority-dictionaries>` setting with *priority*.@@ -512,7 +504,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):def setdefault( # pylint: disable=arguments-renamedself,- name: _SettingsKey,+ name: str,default: Any = None,priority: int | str = "project",) -> Any:@@ -523,7 +515,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):return self.attributes[name].valuedef setdefault_in_component_priority_dict(- self, name: _SettingsKey, cls: type, priority: int | None+ self, name: str, cls: type, priority: int | None) -> None:"""Set the *cls* component in the *name* :ref:`component prioritydictionary <component-priority-dictionaries>` setting with *priority*@@ -592,7 +584,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):"""self._assert_mutability()if isinstance(values, str):- values = cast("dict[_SettingsKey, Any]", json.loads(values))+ values = cast("dict[str, Any]", json.loads(values))if values is not None:if isinstance(values, BaseSettings):for name, value in values.items():@@ -601,7 +593,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):for name, value in values.items():self.set(name, value, priority)- def delete(self, name: _SettingsKey, priority: int | str = "project") -> None:+ def delete(self, name: str, priority: int | str = "project") -> None:if name not in self:raise KeyError(name)self._assert_mutability()@@ -609,7 +601,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):if priority >= cast("int", self.getpriority(name)):del self.attributes[name]- def __delitem__(self, name: _SettingsKey) -> None:+ def __delitem__(self, name: str) -> None:self._assert_mutability()del self.attributes[name]@@ -649,26 +641,19 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):copy.freeze()return copy- def __iter__(self) -> Iterator[_SettingsKey]:+ def __iter__(self) -> Iterator[str]:return iter(self.attributes)def __len__(self) -> int:return len(self.attributes)- def _to_dict(self) -> dict[_SettingsKey, Any]:+ def _to_dict(self) -> dict[str, Any]:return {- self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v)+ str(k): (v._to_dict() if isinstance(v, BaseSettings) else v)for k, v in self.items()}- def _get_key(self, key_value: Any) -> _SettingsKey:- return (- key_value- if isinstance(key_value, (bool, float, int, str, type(None)))- else str(key_value)- )-- def copy_to_dict(self) -> dict[_SettingsKey, Any]:+ def copy_to_dict(self) -> dict[str, Any]:"""Make a copy of current settings and convert to a dict.@@ -691,7 +676,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):else:p.text(pformat(self.copy_to_dict()))- def pop(self, name: _SettingsKey, default: Any = __default) -> Any: # pylint: disable=arguments-renamed+ def pop(self, name: str, default: Any = __default) -> Any: # pylint: disable=arguments-renamedtry:value = self.attributes[name].valueexcept KeyError:@@ -735,7 +720,7 @@ def iter_default_settings() -> Iterable[tuple[str, Any]]:def overridden_settings(- settings: Mapping[_SettingsKey, Any],+ settings: Mapping[str, Any],) -> Iterable[tuple[str, Any]]:"""Return an iterable of the settings that have been overridden"""for name, defvalue in iter_default_settings():diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.pyindex 299a5d43f..6e8c67adb 100644--- a/scrapy/spiders/__init__.py+++ b/scrapy/spiders/__init__.py@@ -24,7 +24,7 @@ if TYPE_CHECKING:from scrapy.crawler import Crawlerfrom scrapy.http.request import CallbackT- from scrapy.settings import BaseSettings, _SettingsKey+ from scrapy.settings import BaseSettingsfrom scrapy.utils.log import SpiderLoggerAdapter@@ -37,7 +37,7 @@ class Spider(object_ref):"""name: str- custom_settings: dict[_SettingsKey, Any] | None = None+ custom_settings: dict[str, Any] | None = None#: Start URLs. See :meth:`start`.start_urls: list[str]diff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex ee65d4155..09b67805d 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -12,7 +12,7 @@ from twisted.python import log as twisted_logfrom twisted.python.failure import Failureimport scrapy-from scrapy.settings import Settings, _SettingsKey+from scrapy.settings import Settingsfrom scrapy.utils.versions import get_versionsif TYPE_CHECKING:@@ -89,7 +89,7 @@ DEFAULT_LOGGING = {def configure_logging(- settings: Settings | dict[_SettingsKey, Any] | None = None,+ settings: Settings | dict[str, Any] | None = None,install_root_handler: bool = True,) -> None:"""diff --git a/tests/spiders.py b/tests/spiders.pyindex 065a0e6d7..55d1ea365 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -22,7 +22,6 @@ from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_futurefrom scrapy.utils.test import get_from_asyncio_queueif TYPE_CHECKING:- from scrapy.settings import _SettingsKeyfrom tests.mockserver.http import MockServer@@ -418,7 +417,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):"""name = "crawl_spider_with_parse_method"- custom_settings: dict[_SettingsKey, Any] = {+ custom_settings: dict[str, Any] = {"RETRY_HTTP_CODES": [], # no need to retry}rules = (Rule(LinkExtractor(), callback="parse", follow=True),)diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 3cde38a6f..82956735b 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -22,7 +22,7 @@ from scrapy.crawler import ()from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.extensions.throttle import AutoThrottle-from scrapy.settings import Settings, _SettingsKey, default_settings+from scrapy.settings import Settings, default_settingsfrom scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_futurefrom scrapy.utils.log import (_uninstall_scrapy_root_handler,@@ -56,7 +56,7 @@ class TestBaseCrawler:class TestCrawler(TestBaseCrawler):def test_populate_spidercls_settings(self) -> None:- spider_settings: dict[_SettingsKey, Any] = {+ spider_settings: dict[str, Any] = {"TEST1": "spider","TEST2": "spider",}diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.pyindex 1d4c3d487..7b3c52d65 100644--- a/tests/test_settings/__init__.py+++ b/tests/test_settings/__init__.py@@ -384,15 +384,15 @@ class TestBaseSettings:"TEST_STRING": "a string","TEST_LIST": [1, 2],"TEST_BOOLEAN": False,- "TEST_BASE": BaseSettings({1: 1, 2: 2}, "project"),- "TEST": BaseSettings({1: 10, 3: 30}, "default"),- "HASNOBASE": BaseSettings({3: 3000}, "default"),+ "TEST_BASE": BaseSettings({"foo": 1, "bar": 2}, "project"),+ "TEST": BaseSettings({"foo": 10, "baz": 30}, "default"),+ "HASNOBASE": BaseSettings({"baz": 3000}, "default"),})assert s.copy_to_dict() == {- "HASNOBASE": {3: 3000},- "TEST": {1: 10, 3: 30},- "TEST_BASE": {1: 1, 2: 2},+ "HASNOBASE": {"baz": 3000},+ "TEST": {"foo": 10, "baz": 30},+ "TEST_BASE": {"foo": 1, "bar": 2},"TEST_LIST": [1, 2],"TEST_BOOLEAN": False,"TEST_STRING": "a string",