Scrapy, a fast high-level web crawling & scraping framework for Python.
Add Settings.getsecret() (#7591)
Adrian0d4b674ed976
diff --git a/conftest.py b/conftest.pyindex d113322fe..df22d521e 100644--- a/conftest.py+++ b/conftest.py@@ -289,6 +289,7 @@ def pytest_runtest_setup(item):"botocore","boto3","aioboto3",+ "keyring","psutil",]diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 3fc091ad7..68a8a6cf7 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -239,6 +239,60 @@ The ``settings`` object can be used like a :class:`dict` (e.g.which may be passed from the command line as strings, it is recommended to useone of the methods provided by the :class:`~scrapy.settings.Settings` API.+.. _secret-settings:++Secret settings+===============++Settings that hold credentials, such as :setting:`AWS_SECRET_ACCESS_KEY`, can+reference a secret stored elsewhere instead:++.. code-block:: python++ AWS_ACCESS_KEY_ID = {"env": "AWS_ACCESS_KEY_ID"}+ AWS_SECRET_ACCESS_KEY = {"keyring": "aws"}+ FTP_PASSWORD = {+ "keyring": {+ "username": "ftp",+ "service": "myproject",+ "backend": "keyring.backends.SecretService.Keyring",+ }+ }++``{"env": "VAR"}`` reads the ``VAR`` environment variable.++``{"keyring": …}`` reads from a keyring_ (21.7.0 or later) entry, given either+its username or an object with its ``username``, ``service`` (default:+``"scrapy"``) and the import path of the keyring ``backend`` to use instead of+the default one.++.. _keyring: https://pypi.org/project/keyring/++A missing environment variable or keyring entry raises :exc:`KeyError`.++From the command line, write references as JSON:++.. code-block:: shell++ scrapy crawl myspider -s 'AWS_ACCESS_KEY_ID={"env": "AWS_ACCESS_KEY_ID"}'++To set a secret that is itself a JSON object with an ``env``, ``keyring`` or+``raw`` key, wrap it in a ``raw`` key: ``'{"raw": {"env": "production", …}}'``.++To load environment variables from a :file:`.env` file, run Scrapy through+`dotenv run`_ or through a secrets manager command like 1Password’s `op run`_,+which also resolves secret references in the file:++.. code-block:: shell++ dotenv run -- scrapy crawl myspider+ op run --env-file=.env -- scrapy crawl myspider++.. _dotenv run: https://pypi.org/project/python-dotenv/+.. _op run: https://developer.1password.com/docs/cli/reference/commands/run/++Components read secret settings with+:meth:`~scrapy.settings.BaseSettings.getsecret`... _component-priority-dictionaries:@@ -473,6 +527,8 @@ Default: ``None``The AWS access key used by code that requires access to `Amazon Web services`_,such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.+This is a :ref:`secret setting <secret-settings>`.+.. setting:: AWS_ENDPOINT_URLAWS_ENDPOINT_URL@@ -521,6 +577,8 @@ Default: ``None``The AWS secret key used by code that requires access to `Amazon Web services`_,such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.+This is a :ref:`secret setting <secret-settings>`.+.. setting:: AWS_SESSION_TOKENAWS_SESSION_TOKEN@@ -532,6 +590,8 @@ The AWS security token used by code that requires access to `Amazon Web servicessuch as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`, when using`temporary security credentials`_.+This is a :ref:`secret setting <secret-settings>`.+.. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html.. setting:: AWS_USE_SSL@@ -1586,6 +1646,8 @@ Default: ``"guest"``The password to use for FTP connections when there is no ``"ftp_password"``in ``Request`` meta.+This is a :ref:`secret setting <secret-settings>`.+It can be overridden in a request in any of the following ways:- Specifying ``ftp_password`` in :attr:`Request.meta <scrapy.http.Request.meta>`diff --git a/pyproject.toml b/pyproject.tomlindex a7fef239f..4ac6ae660 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -123,6 +123,7 @@ pattern = "^(?P<version>.+)$"[tool.mypy]strict = trueuntyped_calls_exclude = [+ "keyring","twisted",]@@ -309,6 +310,7 @@ markers = ["requires_aioboto3: marks tests that need aiobotocore and aioboto3","requires_psutil: marks tests that need psutil","requires_mitmproxy: marks tests that need a mitmdump executable",+ "requires_keyring: marks tests that need keyring","requires_internet: marks tests that need real Internet access","walltime: marks benchmarks that need CodSpeed's walltime mode to be meaningful",]diff --git a/scrapy/core/downloader/handlers/_aioftp.py b/scrapy/core/downloader/handlers/_aioftp.pyindex 3903efcb6..b4381505d 100644--- a/scrapy/core/downloader/handlers/_aioftp.py+++ b/scrapy/core/downloader/handlers/_aioftp.py@@ -107,7 +107,7 @@ class AioftpDownloadHandler(BaseStreamingDownloadHandler[_AioftpResponse]):f"{type(self).__name__} does not support disabling FTP_PASSIVE_MODE.")self._user: str = crawler.settings["FTP_USER"]- self._password: str = crawler.settings["FTP_PASSWORD"]+ self._password: str = crawler.settings.getsecret("FTP_PASSWORD") or ""self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)@asynccontextmanagerdiff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.pyindex 25a29ec30..7ad7f2808 100644--- a/scrapy/core/downloader/handlers/ftp.py+++ b/scrapy/core/downloader/handlers/ftp.py@@ -132,7 +132,7 @@ class FTPDownloadHandler(BaseDownloadHandler):raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.")super().__init__(crawler)self.default_user = crawler.settings["FTP_USER"]- self.default_password = crawler.settings["FTP_PASSWORD"]+ self.default_password = crawler.settings.getsecret("FTP_PASSWORD")self.passive_mode = crawler.settings["FTP_PASSIVE_MODE"]self._default_maxsize: int = crawler.settings.getint("DOWNLOAD_MAXSIZE")self._default_warnsize: int = crawler.settings.getint("DOWNLOAD_WARNSIZE")diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.pyindex 68ec4a931..1702f1455 100644--- a/scrapy/core/downloader/handlers/s3.py+++ b/scrapy/core/downloader/handlers/s3.py@@ -23,9 +23,9 @@ class S3DownloadHandler(BaseDownloadHandler):raise NotConfigured("missing botocore library")super().__init__(crawler)- aws_access_key_id = crawler.settings["AWS_ACCESS_KEY_ID"]- aws_secret_access_key = crawler.settings["AWS_SECRET_ACCESS_KEY"]- aws_session_token = crawler.settings["AWS_SESSION_TOKEN"]+ aws_access_key_id = crawler.settings.getsecret("AWS_ACCESS_KEY_ID")+ aws_secret_access_key = crawler.settings.getsecret("AWS_SECRET_ACCESS_KEY")+ aws_session_token = crawler.settings.getsecret("AWS_SESSION_TOKEN")self.anon = not aws_access_key_id and not aws_secret_access_keyself._signer = Noneif not self.anon:@@ -36,7 +36,9 @@ class S3DownloadHandler(BaseDownloadHandler):# botocore.auth.BaseSigner doesn't have an __init__() with args, only subclasses doself._signer = SignerCls(botocore.credentials.Credentials( # type: ignore[call-arg]- aws_access_key_id, aws_secret_access_key, aws_session_token+ aws_access_key_id or "",+ aws_secret_access_key or "",+ aws_session_token,))diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex a33cf30d8..0ef16ab1d 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -345,9 +345,9 @@ class S3FeedStorage(BlockingFeedStorage):) -> Self:return cls(uri,- access_key=crawler.settings["AWS_ACCESS_KEY_ID"],- secret_key=crawler.settings["AWS_SECRET_ACCESS_KEY"],- session_token=crawler.settings["AWS_SESSION_TOKEN"],+ access_key=crawler.settings.getsecret("AWS_ACCESS_KEY_ID"),+ secret_key=crawler.settings.getsecret("AWS_SECRET_ACCESS_KEY"),+ session_token=crawler.settings.getsecret("AWS_SESSION_TOKEN"),acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None,endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None,region_name=crawler.settings["AWS_REGION_NAME"] or None,diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 8f4a76432..a28958f29 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -173,9 +173,9 @@ class S3FilesStore:# checksums that were requested when the object was uploaded.checksum_algorithm: str = "md5"- AWS_ACCESS_KEY_ID = None- AWS_SECRET_ACCESS_KEY = None- AWS_SESSION_TOKEN = None+ AWS_ACCESS_KEY_ID: str | None = None+ AWS_SECRET_ACCESS_KEY: str | None = None+ AWS_SESSION_TOKEN: str | None = NoneAWS_ENDPOINT_URL = NoneAWS_REGION_NAME = NoneAWS_USE_SSL = None@@ -201,9 +201,9 @@ class S3FilesStore:) -> dict[str, Any]:settings = crawler.settingsreturn {- "AWS_ACCESS_KEY_ID": settings["AWS_ACCESS_KEY_ID"],- "AWS_SECRET_ACCESS_KEY": settings["AWS_SECRET_ACCESS_KEY"],- "AWS_SESSION_TOKEN": settings["AWS_SESSION_TOKEN"],+ "AWS_ACCESS_KEY_ID": settings.getsecret("AWS_ACCESS_KEY_ID"),+ "AWS_SECRET_ACCESS_KEY": settings.getsecret("AWS_SECRET_ACCESS_KEY"),+ "AWS_SESSION_TOKEN": settings.getsecret("AWS_SESSION_TOKEN"),"AWS_ENDPOINT_URL": settings["AWS_ENDPOINT_URL"],"AWS_REGION_NAME": settings["AWS_REGION_NAME"],"AWS_USE_SSL": settings["AWS_USE_SSL"],@@ -489,7 +489,7 @@ class FTPFilesStore:settings = crawler.settingsreturn {"FTP_USERNAME": settings["FTP_USER"],- "FTP_PASSWORD": settings["FTP_PASSWORD"],+ "FTP_PASSWORD": settings.getsecret("FTP_PASSWORD"),"USE_ACTIVE_MODE": settings.getbool("FEED_STORAGE_FTP_ACTIVE"),"UPLOAD_TIMEOUT": settings.getfloat("UPLOAD_TIMEOUT") or None,}diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.pyindex f1b2cdca3..372a7be59 100644--- a/scrapy/settings/__init__.py+++ b/scrapy/settings/__init__.py@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any, Self, TypeAlias, castfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.settings import default_settings+from scrapy.utils._secrets import _resolve_secretfrom scrapy.utils.misc import load_objectfrom scrapy.utils.python import global_object_name@@ -322,6 +323,19 @@ class BaseSettings(MutableMapping[str, Any]):)return copy.deepcopy(value)+ def getsecret(self, name: str, default: str | None = None) -> str | None:+ """Get a setting value as a string, resolving :ref:`secret references+ <secret-settings>`.++ If the setting is not defined or its value is ``None``, *default* is+ returned. If the referenced environment variable or keyring entry does+ not exist, :exc:`KeyError` is raised.+ """+ value = self.get(name)+ if value is None:+ return default+ return _resolve_secret(value)+def getwithbase(self, name: str) -> BaseSettings:"""Get a composition of a dictionary-like setting and its ``_BASE``counterpart.diff --git a/scrapy/utils/_secrets.py b/scrapy/utils/_secrets.pynew file mode 100644index 000000000..870e113ed--- /dev/null+++ b/scrapy/utils/_secrets.py@@ -0,0 +1,78 @@+from __future__ import annotations++import json+import os+from typing import Any++from scrapy.utils.misc import load_object++_SOURCES = {"env", "keyring", "raw"}+++def _resolve_secret(value: Any) -> str:+ """Return the secret that *value* holds or references.++ See :ref:`secret-settings` for the supported references.+ """+ if isinstance(value, dict):+ spec = value+ elif isinstance(value, str):+ try:+ spec = json.loads(value)+ except ValueError:+ return value+ # Any other JSON value, e.g. a service-account blob, is a literal+ # secret.+ if not isinstance(spec, dict) or not _SOURCES & spec.keys():+ return value+ else:+ return str(value)++ if "raw" in spec:+ raw = spec["raw"]+ return raw if isinstance(raw, str) else json.dumps(raw)+ if "env" in spec:+ try:+ return os.environ[spec["env"]]+ except KeyError:+ raise KeyError(f"Environment variable {spec['env']!r} is not set") from None+ if "keyring" in spec:+ return _resolve_keyring(spec["keyring"])+ raise ValueError(+ f"Secret references need one of these keys: {', '.join(sorted(_SOURCES))}"+ )+++def _resolve_keyring(kr: Any) -> str:+ try:+ import keyring # noqa: PLC0415+ except ImportError:+ raise ImportError(+ "Install the 'keyring' package to use keyring-based secrets: "+ "pip install keyring"+ ) from None++ if isinstance(kr, str):+ username: str = kr+ service: str = "scrapy"+ backend_path: str | None = None+ elif isinstance(kr, dict):+ if "username" not in kr:+ raise ValueError('"keyring" object must have a "username" key')+ username = kr["username"]+ service = kr.get("service", "scrapy")+ backend_path = kr.get("backend")+ else:+ raise ValueError('"keyring" must be a string or an object')++ if backend_path is not None:+ backend_cls = load_object(backend_path)+ value: str | None = backend_cls().get_password(service, username)+ else:+ value = keyring.get_password(service, username)++ if value is None:+ raise KeyError(+ f"No keyring entry found for service={service!r}, username={username!r}"+ )+ return valuediff --git a/tests/test_settings/test_secrets.py b/tests/test_settings/test_secrets.pynew file mode 100644index 000000000..02ba9be0e--- /dev/null+++ b/tests/test_settings/test_secrets.py@@ -0,0 +1,136 @@+from __future__ import annotations++import sys+from typing import Any++import pytest++from scrapy.settings import BaseSettings+++class _InMemoryKeyring:+ passwords = {("myservice", "myuser"): "backend-secret"}++ def get_password(self, service: str, username: str) -> str | None:+ return self.passwords.get((service, username))+++def _getsecret(value: Any, default: str | None = None) -> str | None:+ return BaseSettings({"SECRET": value}).getsecret("SECRET", default)+++ ("value", "expected"),+ [+ ("my-api-key", "my-api-key"),+ ("", ""),+ (42, "42"),+ ('"json-string"', '"json-string"'),+ ('["a", "b"]', '["a", "b"]'),+ ("not {json}", "not {json}"),+ ('{"type": "service_account"}', '{"type": "service_account"}'),+ ('{"raw": "literal"}', "literal"),+ ({"raw": "literal"}, "literal"),+ ('{"raw": {"env": "production"}}', '{"env": "production"}'),+ ({"raw": [1, 2]}, "[1, 2]"),+ ],+)+def test_literal(value: Any, expected: str) -> None:+ assert _getsecret(value) == expected++[email protected](("default", "expected"), [(None, None), ("foo", "foo")])+def test_unset(default: str | None, expected: str | None) -> None:+ assert BaseSettings().getsecret("SECRET", default) == expected+ assert _getsecret(None, default) == expected++[email protected]("value", ['{"env": "MY_SECRET"}', {"env": "MY_SECRET"}])+def test_env(value: Any, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("MY_SECRET", "s3cr3t")+ assert _getsecret(value, "default") == "s3cr3t"+++def test_env_missing(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.delenv("MY_SECRET", raising=False)+ with pytest.raises(KeyError, match="MY_SECRET"):+ _getsecret({"env": "MY_SECRET"}, "default")+++def test_unknown_reference() -> None:+ with pytest.raises(ValueError, match="env, keyring, raw"):+ _getsecret({"type": "service_account"})++[email protected]_keyring+ ("value", "service", "username"),+ [+ ('{"keyring": "myuser"}', "scrapy", "myuser"),+ ({"keyring": {"username": "myuser"}}, "scrapy", "myuser"),+ (+ {"keyring": {"username": "myuser", "service": "myservice"}},+ "myservice",+ "myuser",+ ),+ ],+)+def test_keyring(+ value: Any, service: str, username: str, monkeypatch: pytest.MonkeyPatch+) -> None:+ import keyring # noqa: PLC0415++ calls = []++ def get_password(service: str, username: str) -> str:+ calls.append((service, username))+ return "keyring-secret"++ monkeypatch.setattr(keyring, "get_password", get_password)+ assert _getsecret(value) == "keyring-secret"+ assert calls == [(service, username)]++[email protected]_keyring+def test_keyring_backend() -> None:+ value = {+ "keyring": {+ "username": "myuser",+ "service": "myservice",+ "backend": f"{__name__}._InMemoryKeyring",+ }+ }+ assert _getsecret(value) == "backend-secret"++[email protected]_keyring+def test_keyring_missing() -> None:+ import keyring # noqa: PLC0415+ from keyring.backends.null import Keyring # noqa: PLC0415++ original = keyring.get_keyring()+ keyring.set_keyring(Keyring())+ try:+ with pytest.raises(KeyError, match="myuser"):+ _getsecret({"keyring": "myuser"})+ finally:+ keyring.set_keyring(original)++[email protected]_keyring+ ("value", "match"),+ [+ ({"keyring": 42}, '"keyring" must be'),+ ({"keyring": {"service": "myservice"}}, '"username" key'),+ ],+)+def test_keyring_invalid(value: Any, match: str) -> None:+ with pytest.raises(ValueError, match=match):+ _getsecret(value)+++def test_keyring_not_installed(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setitem(sys.modules, "keyring", None)+ with pytest.raises(ImportError, match="keyring"):+ _getsecret({"keyring": "myuser"})diff --git a/tox.ini b/tox.iniindex 8d0299cae..99ddce6cf 100644--- a/tox.ini+++ b/tox.ini@@ -69,6 +69,7 @@ deps =mypy==2.4.1typing-extensions==4.16.0Pillow==13.0.0+ keyring==22.3.0Protego==0.7.0Twisted==26.10.0aiodns==4.0.4@@ -187,6 +188,7 @@ basepython = python3deps ={[testenv]deps}Pillow+ keyringTwisted[http2]python-socks[asyncio]siosocks # SOCKS server for tests@@ -208,6 +210,7 @@ basepython = {[min]basepython}deps ={[min]deps}Pillow==9.2.0+ keyring==21.7.0Twisted[http2]==23.10.0h2==3.2.0python-socks[asyncio]==2.6.0