Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Replace _SettingsKeyT with str. (#7586)

Andrey Rakhmatullind59f9b644af2
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index d4671fd35..be298e9b1 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -16,10 +16,6 @@ from scrapy.utils.python import global_object_name
logger = getLogger(__name__)
-# The key types are restricted in BaseSettings._get_key() to ones supported by JSON,
-# see https://github.com/scrapy/scrapy/issues/5383.
-_SettingsKey: TypeAlias = bool | float | int | str | None
-
if TYPE_CHECKING:
from types import ModuleType
@@ -29,7 +25,7 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
- _SettingsInput: TypeAlias = SupportsItems[_SettingsKey, Any] | str | None
+ _SettingsInput: TypeAlias = SupportsItems[str, Any] | str | None
SETTINGS_PRIORITIES: dict[str, int] = {
@@ -80,7 +76,7 @@ class SettingsAttribute:
return f"<SettingsAttribute value={self.value!r} priority={self.priority}>"
-class BaseSettings(MutableMapping[_SettingsKey, Any]):
+class BaseSettings(MutableMapping[str, Any]):
"""
Instances of this class behave like dictionaries, but store priorities
along with their ``(key, value)`` pairs, and can be frozen (i.e. marked
@@ -106,11 +102,11 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
def __init__(self, values: _SettingsInput = None, priority: int | str = "project"):
self.frozen: bool = False
- self.attributes: dict[_SettingsKey, SettingsAttribute] = {}
+ self.attributes: dict[str, SettingsAttribute] = {}
if values:
self.update(values, priority)
- def __getitem__(self, opt_name: _SettingsKey) -> Any:
+ def __getitem__(self, opt_name: str) -> Any:
if opt_name not in self:
return None
return self.attributes[opt_name].value
@@ -118,7 +114,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
def __contains__(self, name: Any) -> bool:
return name in self.attributes
- def add_to_list(self, name: _SettingsKey, item: Any) -> None:
+ def add_to_list(self, name: str, item: Any) -> None:
"""Append *item* to the :class:`list` setting with the specified *name*
if *item* is not already in that list.
@@ -129,7 +125,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
if item not in value:
self.set(name, [*value, item], self.getpriority(name) or 0)
- def remove_from_list(self, name: _SettingsKey, item: Any) -> None:
+ def remove_from_list(self, name: str, item: Any) -> None:
"""Remove *item* from the :class:`list` setting with the specified
*name*.
@@ -143,7 +139,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
raise ValueError(f"{item!r} not found in the {name} setting ({value!r}).")
self.set(name, [v for v in value if v != item], self.getpriority(name) or 0)
- def get(self, name: _SettingsKey, default: Any = None) -> Any: # pylint: disable=arguments-renamed
+ def get(self, name: str, default: Any = None) -> Any: # pylint: disable=arguments-renamed
"""
Get a setting value without affecting its original type.
@@ -172,7 +168,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
return self[name] if self[name] is not None else default
- def getbool(self, name: _SettingsKey, default: bool = False) -> bool:
+ def getbool(self, name: str, default: bool = False) -> bool:
"""
Get a setting value as a boolean.
@@ -202,7 +198,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
"'True'/'False' and 'true'/'false'"
) from None
- def getint(self, name: _SettingsKey, default: int = 0) -> int:
+ def getint(self, name: str, default: int = 0) -> int:
"""
Get a setting value as an int.
@@ -214,7 +210,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
"""
return int(self.get(name, default))
- def getfloat(self, name: _SettingsKey, default: float = 0.0) -> float:
+ def getfloat(self, name: str, default: float = 0.0) -> float:
"""
Get a setting value as a float.
@@ -226,9 +222,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
"""
return float(self.get(name, default))
- def getlist(
- self, name: _SettingsKey, default: list[Any] | None = None
- ) -> list[Any]:
+ def getlist(self, name: str, default: list[Any] | None = None) -> list[Any]:
"""
Get a setting value as a list. If the setting original type is a list,
a copy of it will be returned. If it's a string it will be split by
@@ -251,7 +245,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
return list(value)
def getdict(
- self, name: _SettingsKey, default: dict[Any, Any] | None = None
+ self, name: str, default: dict[Any, Any] | None = None
) -> dict[Any, Any]:
"""
Get a setting value as a dictionary. If the setting original type is a
@@ -275,7 +269,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
def getdictorlist(
self,
- name: _SettingsKey,
+ name: str,
default: dict[Any, Any] | list[Any] | tuple[Any] | None = None,
) -> dict[Any, Any] | list[Any]:
"""Get a setting value as either a :class:`dict` or a :class:`list`.
@@ -322,7 +316,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
)
return copy.deepcopy(value)
- def getwithbase(self, name: _SettingsKey) -> BaseSettings:
+ def getwithbase(self, name: str) -> BaseSettings:
"""Get a composition of a dictionary-like setting and its ``_BASE``
counterpart.
@@ -341,7 +335,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
compbs.update(self[name])
return compbs
- def get_component_priority_dict_with_base(self, name: _SettingsKey) -> BaseSettings:
+ def get_component_priority_dict_with_base(self, name: str) -> BaseSettings:
"""Get a composition of a component priority dictionary setting and
its ``_BASE`` counterpart.
@@ -389,7 +383,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
{restore_key(k): v for k, v in result.items() if v is not None}
)
- def getpriority(self, name: _SettingsKey) -> int | None:
+ def getpriority(self, name: str) -> int | None:
"""
Return the current numerical priority value of a setting, or ``None`` if
the given ``name`` does not exist.
@@ -414,7 +408,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
def replace_in_component_priority_dict(
self,
- name: _SettingsKey,
+ name: str,
old_cls: type,
new_cls: type,
priority: int | None = None,
@@ -453,12 +447,10 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
)
self.set(name, component_priority_dict, priority=self.getpriority(name) or 0)
- def __setitem__(self, name: _SettingsKey, value: Any) -> None:
+ def __setitem__(self, name: str, value: Any) -> None:
self.set(name, value)
- def set(
- self, name: _SettingsKey, value: Any, priority: int | str = "project"
- ) -> None:
+ def set(self, name: str, value: Any, priority: int | str = "project") -> None:
"""
Store a key/value attribute with a given priority.
@@ -487,7 +479,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
self.attributes[name].set(value, priority)
def set_in_component_priority_dict(
- self, name: _SettingsKey, cls: type, priority: int | None
+ self, name: str, cls: type, priority: int | None
) -> None:
"""Set the *cls* component in the *name* :ref:`component priority
dictionary <component-priority-dictionaries>` setting with *priority*.
@@ -512,7 +504,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
def setdefault( # pylint: disable=arguments-renamed
self,
- name: _SettingsKey,
+ name: str,
default: Any = None,
priority: int | str = "project",
) -> Any:
@@ -523,7 +515,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
return self.attributes[name].value
def setdefault_in_component_priority_dict(
- self, name: _SettingsKey, cls: type, priority: int | None
+ self, name: str, cls: type, priority: int | None
) -> None:
"""Set the *cls* component in the *name* :ref:`component priority
dictionary <component-priority-dictionaries>` setting with *priority*
@@ -592,7 +584,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
"""
self._assert_mutability()
if isinstance(values, str):
- values = cast("dict[_SettingsKey, Any]", json.loads(values))
+ values = cast("dict[str, Any]", json.loads(values))
if values is not None:
if isinstance(values, BaseSettings):
for name, value in values.items():
@@ -601,7 +593,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
for name, value in values.items():
self.set(name, value, priority)
- def delete(self, name: _SettingsKey, priority: int | str = "project") -> None:
+ def delete(self, name: str, priority: int | str = "project") -> None:
if name not in self:
raise KeyError(name)
self._assert_mutability()
@@ -609,7 +601,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
if priority >= cast("int", self.getpriority(name)):
del self.attributes[name]
- def __delitem__(self, name: _SettingsKey) -> None:
+ def __delitem__(self, name: str) -> None:
self._assert_mutability()
del self.attributes[name]
@@ -649,26 +641,19 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
copy.freeze()
return copy
- def __iter__(self) -> Iterator[_SettingsKey]:
+ def __iter__(self) -> Iterator[str]:
return iter(self.attributes)
def __len__(self) -> int:
return len(self.attributes)
- def _to_dict(self) -> dict[_SettingsKey, Any]:
+ def _to_dict(self) -> dict[str, Any]:
return {
- self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v)
+ str(k): (v._to_dict() if isinstance(v, BaseSettings) else v)
for k, v in self.items()
}
- def _get_key(self, key_value: Any) -> _SettingsKey:
- return (
- key_value
- if isinstance(key_value, (bool, float, int, str, type(None)))
- else str(key_value)
- )
-
- def copy_to_dict(self) -> dict[_SettingsKey, Any]:
+ def copy_to_dict(self) -> dict[str, Any]:
"""
Make a copy of current settings and convert to a dict.
@@ -691,7 +676,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
else:
p.text(pformat(self.copy_to_dict()))
- def pop(self, name: _SettingsKey, default: Any = __default) -> Any: # pylint: disable=arguments-renamed
+ def pop(self, name: str, default: Any = __default) -> Any: # pylint: disable=arguments-renamed
try:
value = self.attributes[name].value
except KeyError:
@@ -735,7 +720,7 @@ def iter_default_settings() -> Iterable[tuple[str, Any]]:
def overridden_settings(
- settings: Mapping[_SettingsKey, Any],
+ settings: Mapping[str, Any],
) -> Iterable[tuple[str, Any]]:
"""Return an iterable of the settings that have been overridden"""
for name, defvalue in iter_default_settings():
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 299a5d43f..6e8c67adb 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -24,7 +24,7 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
from scrapy.http.request import CallbackT
- from scrapy.settings import BaseSettings, _SettingsKey
+ from scrapy.settings import BaseSettings
from scrapy.utils.log import SpiderLoggerAdapter
@@ -37,7 +37,7 @@ class Spider(object_ref):
"""
name: str
- custom_settings: dict[_SettingsKey, Any] | None = None
+ custom_settings: dict[str, Any] | None = None
#: Start URLs. See :meth:`start`.
start_urls: list[str]
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index ee65d4155..09b67805d 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -12,7 +12,7 @@ from twisted.python import log as twisted_log
from twisted.python.failure import Failure
import scrapy
-from scrapy.settings import Settings, _SettingsKey
+from scrapy.settings import Settings
from scrapy.utils.versions import get_versions
if TYPE_CHECKING:
@@ -89,7 +89,7 @@ DEFAULT_LOGGING = {
def configure_logging(
- settings: Settings | dict[_SettingsKey, Any] | None = None,
+ settings: Settings | dict[str, Any] | None = None,
install_root_handler: bool = True,
) -> None:
"""
diff --git a/tests/spiders.py b/tests/spiders.py
index 065a0e6d7..55d1ea365 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -22,7 +22,6 @@ from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
from scrapy.utils.test import get_from_asyncio_queue
if TYPE_CHECKING:
- from scrapy.settings import _SettingsKey
from tests.mockserver.http import MockServer
@@ -418,7 +417,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
"""
name = "crawl_spider_with_parse_method"
- custom_settings: dict[_SettingsKey, Any] = {
+ custom_settings: dict[str, Any] = {
"RETRY_HTTP_CODES": [], # no need to retry
}
rules = (Rule(LinkExtractor(), callback="parse", follow=True),)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 3cde38a6f..82956735b 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -22,7 +22,7 @@ from scrapy.crawler import (
)
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions.throttle import AutoThrottle
-from scrapy.settings import Settings, _SettingsKey, default_settings
+from scrapy.settings import Settings, default_settings
from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
from scrapy.utils.log import (
_uninstall_scrapy_root_handler,
@@ -56,7 +56,7 @@ class TestBaseCrawler:
class TestCrawler(TestBaseCrawler):
def test_populate_spidercls_settings(self) -> None:
- spider_settings: dict[_SettingsKey, Any] = {
+ spider_settings: dict[str, Any] = {
"TEST1": "spider",
"TEST2": "spider",
}
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 1d4c3d487..7b3c52d65 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -384,15 +384,15 @@ class TestBaseSettings:
"TEST_STRING": "a string",
"TEST_LIST": [1, 2],
"TEST_BOOLEAN": False,
- "TEST_BASE": BaseSettings({1: 1, 2: 2}, "project"),
- "TEST": BaseSettings({1: 10, 3: 30}, "default"),
- "HASNOBASE": BaseSettings({3: 3000}, "default"),
+ "TEST_BASE": BaseSettings({"foo": 1, "bar": 2}, "project"),
+ "TEST": BaseSettings({"foo": 10, "baz": 30}, "default"),
+ "HASNOBASE": BaseSettings({"baz": 3000}, "default"),
}
)
assert s.copy_to_dict() == {
- "HASNOBASE": {3: 3000},
- "TEST": {1: 10, 3: 30},
- "TEST_BASE": {1: 1, 2: 2},
+ "HASNOBASE": {"baz": 3000},
+ "TEST": {"foo": 10, "baz": 30},
+ "TEST_BASE": {"foo": 1, "bar": 2},
"TEST_LIST": [1, 2],
"TEST_BOOLEAN": False,
"TEST_STRING": "a string",