Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add Settings.getsecret() (#7591)

Adrian0d4b674ed976
diff --git a/conftest.py b/conftest.py
index d113322fe..df22d521e 100644
--- a/conftest.py
+++ b/conftest.py
@@ -289,6 +289,7 @@ def pytest_runtest_setup(item):
"botocore",
"boto3",
"aioboto3",
+ "keyring",
"psutil",
]
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 3fc091ad7..68a8a6cf7 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -239,6 +239,60 @@ The ``settings`` object can be used like a :class:`dict` (e.g.
which may be passed from the command line as strings, it is recommended to use
one of the methods provided by the :class:`~scrapy.settings.Settings` API.
+.. _secret-settings:
+
+Secret settings
+===============
+
+Settings that hold credentials, such as :setting:`AWS_SECRET_ACCESS_KEY`, can
+reference a secret stored elsewhere instead:
+
+.. code-block:: python
+
+ AWS_ACCESS_KEY_ID = {"env": "AWS_ACCESS_KEY_ID"}
+ AWS_SECRET_ACCESS_KEY = {"keyring": "aws"}
+ FTP_PASSWORD = {
+ "keyring": {
+ "username": "ftp",
+ "service": "myproject",
+ "backend": "keyring.backends.SecretService.Keyring",
+ }
+ }
+
+``{"env": "VAR"}`` reads the ``VAR`` environment variable.
+
+``{"keyring": …}`` reads from a keyring_ (21.7.0 or later) entry, given either
+its username or an object with its ``username``, ``service`` (default:
+``"scrapy"``) and the import path of the keyring ``backend`` to use instead of
+the default one.
+
+.. _keyring: https://pypi.org/project/keyring/
+
+A missing environment variable or keyring entry raises :exc:`KeyError`.
+
+From the command line, write references as JSON:
+
+.. code-block:: shell
+
+ scrapy crawl myspider -s 'AWS_ACCESS_KEY_ID={"env": "AWS_ACCESS_KEY_ID"}'
+
+To set a secret that is itself a JSON object with an ``env``, ``keyring`` or
+``raw`` key, wrap it in a ``raw`` key: ``'{"raw": {"env": "production", …}}'``.
+
+To load environment variables from a :file:`.env` file, run Scrapy through
+`dotenv run`_ or through a secrets manager command like 1Password’s `op run`_,
+which also resolves secret references in the file:
+
+.. code-block:: shell
+
+ dotenv run -- scrapy crawl myspider
+ op run --env-file=.env -- scrapy crawl myspider
+
+.. _dotenv run: https://pypi.org/project/python-dotenv/
+.. _op run: https://developer.1password.com/docs/cli/reference/commands/run/
+
+Components read secret settings with
+:meth:`~scrapy.settings.BaseSettings.getsecret`.
.. _component-priority-dictionaries:
@@ -473,6 +527,8 @@ Default: ``None``
The AWS access key used by code that requires access to `Amazon Web services`_,
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
+This is a :ref:`secret setting <secret-settings>`.
+
.. setting:: AWS_ENDPOINT_URL
AWS_ENDPOINT_URL
@@ -521,6 +577,8 @@ Default: ``None``
The AWS secret key used by code that requires access to `Amazon Web services`_,
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
+This is a :ref:`secret setting <secret-settings>`.
+
.. setting:: AWS_SESSION_TOKEN
AWS_SESSION_TOKEN
@@ -532,6 +590,8 @@ The AWS security token used by code that requires access to `Amazon Web services
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`, when using
`temporary security credentials`_.
+This is a :ref:`secret setting <secret-settings>`.
+
.. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html
.. setting:: AWS_USE_SSL
@@ -1586,6 +1646,8 @@ Default: ``"guest"``
The password to use for FTP connections when there is no ``"ftp_password"``
in ``Request`` meta.
+This is a :ref:`secret setting <secret-settings>`.
+
It can be overridden in a request in any of the following ways:
- Specifying ``ftp_password`` in :attr:`Request.meta <scrapy.http.Request.meta>`
diff --git a/pyproject.toml b/pyproject.toml
index a7fef239f..4ac6ae660 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -123,6 +123,7 @@ pattern = "^(?P<version>.+)$"
[tool.mypy]
strict = true
untyped_calls_exclude = [
+ "keyring",
"twisted",
]
@@ -309,6 +310,7 @@ markers = [
"requires_aioboto3: marks tests that need aiobotocore and aioboto3",
"requires_psutil: marks tests that need psutil",
"requires_mitmproxy: marks tests that need a mitmdump executable",
+ "requires_keyring: marks tests that need keyring",
"requires_internet: marks tests that need real Internet access",
"walltime: marks benchmarks that need CodSpeed's walltime mode to be meaningful",
]
diff --git a/scrapy/core/downloader/handlers/_aioftp.py b/scrapy/core/downloader/handlers/_aioftp.py
index 3903efcb6..b4381505d 100644
--- a/scrapy/core/downloader/handlers/_aioftp.py
+++ b/scrapy/core/downloader/handlers/_aioftp.py
@@ -107,7 +107,7 @@ class AioftpDownloadHandler(BaseStreamingDownloadHandler[_AioftpResponse]):
f"{type(self).__name__} does not support disabling FTP_PASSIVE_MODE."
)
self._user: str = crawler.settings["FTP_USER"]
- self._password: str = crawler.settings["FTP_PASSWORD"]
+ self._password: str = crawler.settings.getsecret("FTP_PASSWORD") or ""
self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)
@asynccontextmanager
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 25a29ec30..7ad7f2808 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -132,7 +132,7 @@ class FTPDownloadHandler(BaseDownloadHandler):
raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.")
super().__init__(crawler)
self.default_user = crawler.settings["FTP_USER"]
- self.default_password = crawler.settings["FTP_PASSWORD"]
+ self.default_password = crawler.settings.getsecret("FTP_PASSWORD")
self.passive_mode = crawler.settings["FTP_PASSIVE_MODE"]
self._default_maxsize: int = crawler.settings.getint("DOWNLOAD_MAXSIZE")
self._default_warnsize: int = crawler.settings.getint("DOWNLOAD_WARNSIZE")
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 68ec4a931..1702f1455 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -23,9 +23,9 @@ class S3DownloadHandler(BaseDownloadHandler):
raise NotConfigured("missing botocore library")
super().__init__(crawler)
- aws_access_key_id = crawler.settings["AWS_ACCESS_KEY_ID"]
- aws_secret_access_key = crawler.settings["AWS_SECRET_ACCESS_KEY"]
- aws_session_token = crawler.settings["AWS_SESSION_TOKEN"]
+ aws_access_key_id = crawler.settings.getsecret("AWS_ACCESS_KEY_ID")
+ aws_secret_access_key = crawler.settings.getsecret("AWS_SECRET_ACCESS_KEY")
+ aws_session_token = crawler.settings.getsecret("AWS_SESSION_TOKEN")
self.anon = not aws_access_key_id and not aws_secret_access_key
self._signer = None
if not self.anon:
@@ -36,7 +36,9 @@ class S3DownloadHandler(BaseDownloadHandler):
# botocore.auth.BaseSigner doesn't have an __init__() with args, only subclasses do
self._signer = SignerCls(
botocore.credentials.Credentials( # type: ignore[call-arg]
- aws_access_key_id, aws_secret_access_key, aws_session_token
+ aws_access_key_id or "",
+ aws_secret_access_key or "",
+ aws_session_token,
)
)
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index a33cf30d8..0ef16ab1d 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -345,9 +345,9 @@ class S3FeedStorage(BlockingFeedStorage):
) -> Self:
return cls(
uri,
- access_key=crawler.settings["AWS_ACCESS_KEY_ID"],
- secret_key=crawler.settings["AWS_SECRET_ACCESS_KEY"],
- session_token=crawler.settings["AWS_SESSION_TOKEN"],
+ access_key=crawler.settings.getsecret("AWS_ACCESS_KEY_ID"),
+ secret_key=crawler.settings.getsecret("AWS_SECRET_ACCESS_KEY"),
+ session_token=crawler.settings.getsecret("AWS_SESSION_TOKEN"),
acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None,
endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None,
region_name=crawler.settings["AWS_REGION_NAME"] or None,
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 8f4a76432..a28958f29 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -173,9 +173,9 @@ class S3FilesStore:
# checksums that were requested when the object was uploaded.
checksum_algorithm: str = "md5"
- AWS_ACCESS_KEY_ID = None
- AWS_SECRET_ACCESS_KEY = None
- AWS_SESSION_TOKEN = None
+ AWS_ACCESS_KEY_ID: str | None = None
+ AWS_SECRET_ACCESS_KEY: str | None = None
+ AWS_SESSION_TOKEN: str | None = None
AWS_ENDPOINT_URL = None
AWS_REGION_NAME = None
AWS_USE_SSL = None
@@ -201,9 +201,9 @@ class S3FilesStore:
) -> dict[str, Any]:
settings = crawler.settings
return {
- "AWS_ACCESS_KEY_ID": settings["AWS_ACCESS_KEY_ID"],
- "AWS_SECRET_ACCESS_KEY": settings["AWS_SECRET_ACCESS_KEY"],
- "AWS_SESSION_TOKEN": settings["AWS_SESSION_TOKEN"],
+ "AWS_ACCESS_KEY_ID": settings.getsecret("AWS_ACCESS_KEY_ID"),
+ "AWS_SECRET_ACCESS_KEY": settings.getsecret("AWS_SECRET_ACCESS_KEY"),
+ "AWS_SESSION_TOKEN": settings.getsecret("AWS_SESSION_TOKEN"),
"AWS_ENDPOINT_URL": settings["AWS_ENDPOINT_URL"],
"AWS_REGION_NAME": settings["AWS_REGION_NAME"],
"AWS_USE_SSL": settings["AWS_USE_SSL"],
@@ -489,7 +489,7 @@ class FTPFilesStore:
settings = crawler.settings
return {
"FTP_USERNAME": settings["FTP_USER"],
- "FTP_PASSWORD": settings["FTP_PASSWORD"],
+ "FTP_PASSWORD": settings.getsecret("FTP_PASSWORD"),
"USE_ACTIVE_MODE": settings.getbool("FEED_STORAGE_FTP_ACTIVE"),
"UPLOAD_TIMEOUT": settings.getfloat("UPLOAD_TIMEOUT") or None,
}
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index f1b2cdca3..372a7be59 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any, Self, TypeAlias, cast
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import default_settings
+from scrapy.utils._secrets import _resolve_secret
from scrapy.utils.misc import load_object
from scrapy.utils.python import global_object_name
@@ -322,6 +323,19 @@ class BaseSettings(MutableMapping[str, Any]):
)
return copy.deepcopy(value)
+ def getsecret(self, name: str, default: str | None = None) -> str | None:
+ """Get a setting value as a string, resolving :ref:`secret references
+ <secret-settings>`.
+
+ If the setting is not defined or its value is ``None``, *default* is
+ returned. If the referenced environment variable or keyring entry does
+ not exist, :exc:`KeyError` is raised.
+ """
+ value = self.get(name)
+ if value is None:
+ return default
+ return _resolve_secret(value)
+
def getwithbase(self, name: str) -> BaseSettings:
"""Get a composition of a dictionary-like setting and its ``_BASE``
counterpart.
diff --git a/scrapy/utils/_secrets.py b/scrapy/utils/_secrets.py
new file mode 100644
index 000000000..870e113ed
--- /dev/null
+++ b/scrapy/utils/_secrets.py
@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import json
+import os
+from typing import Any
+
+from scrapy.utils.misc import load_object
+
+_SOURCES = {"env", "keyring", "raw"}
+
+
+def _resolve_secret(value: Any) -> str:
+ """Return the secret that *value* holds or references.
+
+ See :ref:`secret-settings` for the supported references.
+ """
+ if isinstance(value, dict):
+ spec = value
+ elif isinstance(value, str):
+ try:
+ spec = json.loads(value)
+ except ValueError:
+ return value
+ # Any other JSON value, e.g. a service-account blob, is a literal
+ # secret.
+ if not isinstance(spec, dict) or not _SOURCES & spec.keys():
+ return value
+ else:
+ return str(value)
+
+ if "raw" in spec:
+ raw = spec["raw"]
+ return raw if isinstance(raw, str) else json.dumps(raw)
+ if "env" in spec:
+ try:
+ return os.environ[spec["env"]]
+ except KeyError:
+ raise KeyError(f"Environment variable {spec['env']!r} is not set") from None
+ if "keyring" in spec:
+ return _resolve_keyring(spec["keyring"])
+ raise ValueError(
+ f"Secret references need one of these keys: {', '.join(sorted(_SOURCES))}"
+ )
+
+
+def _resolve_keyring(kr: Any) -> str:
+ try:
+ import keyring # noqa: PLC0415
+ except ImportError:
+ raise ImportError(
+ "Install the 'keyring' package to use keyring-based secrets: "
+ "pip install keyring"
+ ) from None
+
+ if isinstance(kr, str):
+ username: str = kr
+ service: str = "scrapy"
+ backend_path: str | None = None
+ elif isinstance(kr, dict):
+ if "username" not in kr:
+ raise ValueError('"keyring" object must have a "username" key')
+ username = kr["username"]
+ service = kr.get("service", "scrapy")
+ backend_path = kr.get("backend")
+ else:
+ raise ValueError('"keyring" must be a string or an object')
+
+ if backend_path is not None:
+ backend_cls = load_object(backend_path)
+ value: str | None = backend_cls().get_password(service, username)
+ else:
+ value = keyring.get_password(service, username)
+
+ if value is None:
+ raise KeyError(
+ f"No keyring entry found for service={service!r}, username={username!r}"
+ )
+ return value
diff --git a/tests/test_settings/test_secrets.py b/tests/test_settings/test_secrets.py
new file mode 100644
index 000000000..02ba9be0e
--- /dev/null
+++ b/tests/test_settings/test_secrets.py
@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+import sys
+from typing import Any
+
+import pytest
+
+from scrapy.settings import BaseSettings
+
+
+class _InMemoryKeyring:
+ passwords = {("myservice", "myuser"): "backend-secret"}
+
+ def get_password(self, service: str, username: str) -> str | None:
+ return self.passwords.get((service, username))
+
+
+def _getsecret(value: Any, default: str | None = None) -> str | None:
+ return BaseSettings({"SECRET": value}).getsecret("SECRET", default)
+
+
+ ("value", "expected"),
+ [
+ ("my-api-key", "my-api-key"),
+ ("", ""),
+ (42, "42"),
+ ('"json-string"', '"json-string"'),
+ ('["a", "b"]', '["a", "b"]'),
+ ("not {json}", "not {json}"),
+ ('{"type": "service_account"}', '{"type": "service_account"}'),
+ ('{"raw": "literal"}', "literal"),
+ ({"raw": "literal"}, "literal"),
+ ('{"raw": {"env": "production"}}', '{"env": "production"}'),
+ ({"raw": [1, 2]}, "[1, 2]"),
+ ],
+)
+def test_literal(value: Any, expected: str) -> None:
+ assert _getsecret(value) == expected
+
+
[email protected](("default", "expected"), [(None, None), ("foo", "foo")])
+def test_unset(default: str | None, expected: str | None) -> None:
+ assert BaseSettings().getsecret("SECRET", default) == expected
+ assert _getsecret(None, default) == expected
+
+
[email protected]("value", ['{"env": "MY_SECRET"}', {"env": "MY_SECRET"}])
+def test_env(value: Any, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("MY_SECRET", "s3cr3t")
+ assert _getsecret(value, "default") == "s3cr3t"
+
+
+def test_env_missing(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.delenv("MY_SECRET", raising=False)
+ with pytest.raises(KeyError, match="MY_SECRET"):
+ _getsecret({"env": "MY_SECRET"}, "default")
+
+
+def test_unknown_reference() -> None:
+ with pytest.raises(ValueError, match="env, keyring, raw"):
+ _getsecret({"type": "service_account"})
+
+
+ ("value", "service", "username"),
+ [
+ ('{"keyring": "myuser"}', "scrapy", "myuser"),
+ ({"keyring": {"username": "myuser"}}, "scrapy", "myuser"),
+ (
+ {"keyring": {"username": "myuser", "service": "myservice"}},
+ "myservice",
+ "myuser",
+ ),
+ ],
+)
+def test_keyring(
+ value: Any, service: str, username: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ import keyring # noqa: PLC0415
+
+ calls = []
+
+ def get_password(service: str, username: str) -> str:
+ calls.append((service, username))
+ return "keyring-secret"
+
+ monkeypatch.setattr(keyring, "get_password", get_password)
+ assert _getsecret(value) == "keyring-secret"
+ assert calls == [(service, username)]
+
+
+def test_keyring_backend() -> None:
+ value = {
+ "keyring": {
+ "username": "myuser",
+ "service": "myservice",
+ "backend": f"{__name__}._InMemoryKeyring",
+ }
+ }
+ assert _getsecret(value) == "backend-secret"
+
+
+def test_keyring_missing() -> None:
+ import keyring # noqa: PLC0415
+ from keyring.backends.null import Keyring # noqa: PLC0415
+
+ original = keyring.get_keyring()
+ keyring.set_keyring(Keyring())
+ try:
+ with pytest.raises(KeyError, match="myuser"):
+ _getsecret({"keyring": "myuser"})
+ finally:
+ keyring.set_keyring(original)
+
+
+ ("value", "match"),
+ [
+ ({"keyring": 42}, '"keyring" must be'),
+ ({"keyring": {"service": "myservice"}}, '"username" key'),
+ ],
+)
+def test_keyring_invalid(value: Any, match: str) -> None:
+ with pytest.raises(ValueError, match=match):
+ _getsecret(value)
+
+
+def test_keyring_not_installed(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setitem(sys.modules, "keyring", None)
+ with pytest.raises(ImportError, match="keyring"):
+ _getsecret({"keyring": "myuser"})
diff --git a/tox.ini b/tox.ini
index 8d0299cae..99ddce6cf 100644
--- a/tox.ini
+++ b/tox.ini
@@ -69,6 +69,7 @@ deps =
mypy==2.4.1
typing-extensions==4.16.0
Pillow==13.0.0
+ keyring==22.3.0
Protego==0.7.0
Twisted==26.10.0
aiodns==4.0.4
@@ -187,6 +188,7 @@ basepython = python3
deps =
{[testenv]deps}
Pillow
+ keyring
Twisted[http2]
python-socks[asyncio]
siosocks # SOCKS server for tests
@@ -208,6 +210,7 @@ basepython = {[min]basepython}
deps =
{[min]deps}
Pillow==9.2.0
+ keyring==21.7.0
Twisted[http2]==23.10.0
h2==3.2.0
python-socks[asyncio]==2.6.0