Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Address PR review: fail open on a decrypt failure, fix the key-rotation and settings-save races

Timo Bozsolik-Torres809f4c2e6dc9
diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/GuardrailsSection.tsx b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/GuardrailsSection.tsx
index ddbab7327..863024a51 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/GuardrailsSection.tsx
+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/GuardrailsSection.tsx
@@ -171,11 +171,16 @@ export const GuardrailsSection = observer(function GuardrailsSection({
<Field>
<FieldLabel>
Switch Trust API key
- {settings.hasApiKey && (
+ {settings.hasApiKey && settings.apiKeyLast4 && (
<span className="ml-1 font-normal text-foreground-muted">
(configured, ending •••{settings.apiKeyLast4})
</span>
)}
+ {settings.hasApiKey && !settings.apiKeyLast4 && (
+ <span className="ml-1 font-normal text-destructive">
+ (configured, but unreadable — save a new key to replace it)
+ </span>
+ )}
</FieldLabel>
<Input
type="password"
diff --git a/core/switch_core/db/key_rotation.py b/core/switch_core/db/key_rotation.py
index 7a89b29e0..1c1a34411 100644
--- a/core/switch_core/db/key_rotation.py
+++ b/core/switch_core/db/key_rotation.py
@@ -26,6 +26,7 @@ from switch_core.db.models import (
MessagingInstall,
ProviderConnection,
ServerConnector,
+ TrustSettings,
)
from switch_core.db.session_scope import tenant_session
from switch_core.keys import Keyring
@@ -42,6 +43,12 @@ _ENCRYPTED_TEXT_COLUMNS: tuple[tuple[type[Any], str], ...] = (
(ProviderConnection, "encrypted_credential"),
(MachineWorkspace, "enrollment_code_encrypted"),
)
+# `trust_settings` is a global table (`rls_ddl.GLOBAL_TABLES`), not a tenant's —
+# one row for the deployment — so it is swept once here rather than through
+# the per-tenant loop below, which would run it redundantly once per tenant.
+_GLOBAL_ENCRYPTED_TEXT_COLUMNS: tuple[tuple[type[Any], str], ...] = (
+ (TrustSettings, "api_key_encrypted"),
+)
async def reencrypt_stale_text(
@@ -88,7 +95,8 @@ async def reencrypt_stored_secrets(
keyring: Keyring,
tenant_ids: list[str],
) -> None:
- """Bring every tenant's encrypted values onto the current key."""
+ """Bring every tenant's encrypted values, plus the deployment's global
+ ones, onto the current key."""
total = 0
for tenant_id in tenant_ids:
async with tenant_session(session_factory, tenant_id) as session:
@@ -97,6 +105,10 @@ async def reencrypt_stored_secrets(
for model, column in _ENCRYPTED_TEXT_COLUMNS:
total += await reencrypt_stale_text(session, keyring, model, column)
await session.commit()
+ async with session_factory() as session:
+ for model, column in _GLOBAL_ENCRYPTED_TEXT_COLUMNS:
+ total += await reencrypt_stale_text(session, keyring, model, column)
+ await session.commit()
if total:
logger.warning(
"Re-encrypted %d stored secret(s) under key %r. Older keys and "
diff --git a/core/switch_core/db/stores/trust_settings_store.py b/core/switch_core/db/stores/trust_settings_store.py
index 934c18d99..f264da1a0 100644
--- a/core/switch_core/db/stores/trust_settings_store.py
+++ b/core/switch_core/db/stores/trust_settings_store.py
@@ -14,9 +14,13 @@ class TrustSettingsStore:
No row, or one missing ``policy_id``/``api_key_encrypted``, means the
check is off — see ``switch_core.trust.client.DynamicTrustClient``.
- Merge semantics (e.g. keeping the existing API key when a caller doesn't
- supply a new one) are the caller's job; this store only ever writes the
- exact values it's given.
+ ``endpoint``/``policy_id`` are written exactly as given; ``upsert``'s
+ ``api_key_encrypted=None`` means "leave whatever is already stored
+ untouched" (there is no request shape that means "clear the key but keep
+ the row" — ``clear()`` drops the whole row). That merge happens in the
+ ``ON CONFLICT`` clause itself via ``COALESCE``, not in a caller that reads
+ the row first: two saves racing each other can't have one silently
+ overwrite the other's key with a value it read before the write.
"""
async def get(self, session: AsyncSession) -> TrustSettings | None:
@@ -33,23 +37,23 @@ class TrustSettingsStore:
policy_id: str | None,
api_key_encrypted: str | None,
) -> TrustSettings:
- stmt = (
- insert(TrustSettings)
- .values(
- id=_ROW_ID,
- endpoint=endpoint,
- policy_id=policy_id,
- api_key_encrypted=api_key_encrypted,
- )
- .on_conflict_do_update(
- index_elements=[TrustSettings.id],
- set_={
- "endpoint": endpoint,
- "policy_id": policy_id,
- "api_key_encrypted": api_key_encrypted,
- "updated_at": func.now(),
- },
- )
+ insert_stmt = insert(TrustSettings).values(
+ id=_ROW_ID,
+ endpoint=endpoint,
+ policy_id=policy_id,
+ api_key_encrypted=api_key_encrypted,
+ )
+ stmt = insert_stmt.on_conflict_do_update(
+ index_elements=[TrustSettings.id],
+ set_={
+ "endpoint": endpoint,
+ "policy_id": policy_id,
+ "api_key_encrypted": func.coalesce(
+ insert_stmt.excluded.api_key_encrypted,
+ TrustSettings.api_key_encrypted,
+ ),
+ "updated_at": func.now(),
+ },
)
await session.execute(stmt)
await session.flush()
diff --git a/core/switch_core/gateway/trust_settings.py b/core/switch_core/gateway/trust_settings.py
index f6d5d5525..9ab01ab7c 100644
--- a/core/switch_core/gateway/trust_settings.py
+++ b/core/switch_core/gateway/trust_settings.py
@@ -7,6 +7,7 @@ covers the whole deployment rather than one tenant's traffic.
from __future__ import annotations
+import logging
from typing import Annotated
from fastapi import APIRouter, Depends
@@ -21,9 +22,12 @@ from switch_core.gateway.schemas import (
TrustSettingsResponse,
TrustSettingsUpdateRequest,
)
+from switch_core.keys import UndecryptableError
router = APIRouter()
+logger = logging.getLogger(__name__)
+
# Matches the previous env-var default (`SwitchConfig.switch_trust_endpoint`,
# since removed) — shown until an operator saves their own.
DEFAULT_TRUST_ENDPOINT = "https://api.switchagents.ai"
@@ -40,11 +44,19 @@ def _to_response(
api_key_last4=None,
enabled=False,
)
- api_key_last4 = (
- config.keyring.decrypt(settings.api_key_encrypted)[-4:]
- if settings.api_key_encrypted
- else None
- )
+ api_key_last4 = None
+ if settings.api_key_encrypted:
+ try:
+ api_key_last4 = config.keyring.decrypt(settings.api_key_encrypted)[-4:]
+ except UndecryptableError:
+ # A key encrypted under one since dropped from SECRET_KEYS. Report
+ # it as configured but unreadable rather than 500ing — an operator
+ # has to be able to load this page to save a replacement key, the
+ # only way out of this state.
+ logger.warning(
+ "Stored Switch Trust API key is not decryptable under any "
+ "current key; reporting it as configured but unreadable."
+ )
return TrustSettingsResponse(
endpoint=settings.endpoint,
policy_id=settings.policy_id,
@@ -72,13 +84,13 @@ async def update_trust_settings(
_admin: Annotated[User, Depends(require_admin)],
) -> TrustSettingsResponse:
"""`req.api_key` omitted leaves whatever key is already stored untouched,
- so changing the endpoint or policy id doesn't force re-entering it."""
- store = TrustSettingsStore()
- existing = await store.get(session)
- api_key_encrypted = existing.api_key_encrypted if existing else None
- if req.api_key is not None:
- api_key_encrypted = config.keyring.encrypt(req.api_key)
- settings = await store.upsert(
+ so changing the endpoint or policy id doesn't force re-entering it — enforced
+ by `TrustSettingsStore.upsert` itself, not a read here, so two admins saving
+ at once can't have one drop the other's key."""
+ api_key_encrypted = (
+ config.keyring.encrypt(req.api_key) if req.api_key is not None else None
+ )
+ settings = await TrustSettingsStore().upsert(
session,
endpoint=req.endpoint,
policy_id=req.policy_id,
diff --git a/core/switch_core/trust/client.py b/core/switch_core/trust/client.py
index aa9b6a76d..70ec0d344 100644
--- a/core/switch_core/trust/client.py
+++ b/core/switch_core/trust/client.py
@@ -59,7 +59,8 @@ _REDACTION_PLACEHOLDER = "[redacted]"
class GuardrailsCheckError(RuntimeError):
- """A check did not complete: network error, timeout, or a non-2xx answer."""
+ """A check did not complete: network error, timeout, non-2xx answer, or a
+ settings read/decrypt failure before the request was even made."""
class GuardrailBlockedError(Exception):
@@ -152,13 +153,28 @@ class DynamicTrustClient:
async def check(
self, *, role: Role, content: str, room_id: str
) -> TrustCheckResult:
- async with self._session_factory() as session:
- settings = await self._store.get(session)
- if settings is None or not settings.policy_id or not settings.api_key_encrypted:
- return TrustCheckResult(outcome="ok", policy_id=None, policy_name=None)
+ # The DB read and the decrypt below are not network calls to Switch
+ # Trust, so neither raises `GuardrailsCheckError` on its own — but a
+ # stuck pool or an `UndecryptableError` from a key rotated out too
+ # soon is exactly the kind of failure `check_message` exists to fail
+ # open on, so anything either raises is folded into the same error.
+ try:
+ async with self._session_factory() as session:
+ settings = await self._store.get(session)
+ if (
+ settings is None
+ or not settings.policy_id
+ or not settings.api_key_encrypted
+ ):
+ return TrustCheckResult(outcome="ok", policy_id=None, policy_name=None)
+ api_key = self._keyring.decrypt(settings.api_key_encrypted)
+ except Exception as error:
+ raise GuardrailsCheckError(
+ f"Could not read Switch Trust settings: {error}"
+ ) from error
http_client = HttpTrustClient(
base_url=settings.endpoint,
- api_key=self._keyring.decrypt(settings.api_key_encrypted),
+ api_key=api_key,
policy_id=settings.policy_id,
timeout_seconds=self._timeout_seconds,
client=self._client,
diff --git a/core/tests/switch_core/db/stores/test_trust_settings_store.py b/core/tests/switch_core/db/stores/test_trust_settings_store.py
index 067074038..08f952c71 100644
--- a/core/tests/switch_core/db/stores/test_trust_settings_store.py
+++ b/core/tests/switch_core/db/stores/test_trust_settings_store.py
@@ -81,6 +81,34 @@ class TestTrustSettingsStore:
assert settings.api_key_encrypted == "enc:k"
assert settings.endpoint == "https://trust2.example"
+ async def test_upsert_with_no_key_given_preserves_the_stored_one(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """`api_key_encrypted=None` means "leave it alone", not "clear it" —
+ the merge a caller resolves by passing `None` rather than reading the
+ row first, so a concurrent save of just the endpoint can't race one
+ that just changed the key."""
+ store = TrustSettingsStore()
+ async with session_factory() as session:
+ await store.upsert(
+ session,
+ endpoint="https://trust.example",
+ policy_id="pol_123",
+ api_key_encrypted="enc:k",
+ )
+ await store.upsert(
+ session,
+ endpoint="https://trust2.example",
+ policy_id="pol_123",
+ api_key_encrypted=None,
+ )
+ await session.commit()
+
+ settings = await store.get(session)
+ assert settings is not None
+ assert settings.api_key_encrypted == "enc:k"
+ assert settings.endpoint == "https://trust2.example"
+
async def test_clear_removes_the_row(
self, session_factory: async_sessionmaker[AsyncSession]
) -> None:
diff --git a/core/tests/switch_core/db/test_key_rotation.py b/core/tests/switch_core/db/test_key_rotation.py
index f3639f716..21b34ec39 100644
--- a/core/tests/switch_core/db/test_key_rotation.py
+++ b/core/tests/switch_core/db/test_key_rotation.py
@@ -26,6 +26,7 @@ from switch_core.db.models import (
CollaborationBridge,
MessagingInstall,
Tenant,
+ TrustSettings,
User,
)
from switch_core.db.session_scope import tenant_session
@@ -163,6 +164,35 @@ async def test_a_value_no_key_opens_stops_the_boot(
await reencrypt_stored_secrets(rls_harness.restricted, too_early, [tenant_id])
+async def test_the_global_trust_settings_key_is_reencrypted_with_no_tenants(
+ rls_harness: RLSHarness, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """`trust_settings` is a global table (`rls_ddl.GLOBAL_TABLES`), swept
+ once outside the per-tenant loop — it must be reached even when there are
+ no tenants to iterate over at all."""
+ monkeypatch.setattr(encrypted_json, "_keyring", _OLD)
+ async with rls_harness.owner() as session:
+ session.add(
+ TrustSettings(
+ id="global",
+ endpoint="https://trust.example",
+ policy_id="pol_123",
+ api_key_encrypted=_legacy_encrypt("trust-key"),
+ )
+ )
+ await session.commit()
+
+ monkeypatch.setattr(encrypted_json, "_keyring", _NEW)
+ await reencrypt_stored_secrets(rls_harness.restricted, _NEW, [])
+
+ current = _NEW.current_prefix()
+ stored = await _raw(
+ rls_harness.owner, "trust_settings", "api_key_encrypted", "global"
+ )
+ assert isinstance(stored, str) and stored.startswith(current)
+ assert _NEW.decrypt(stored) == "trust-key"
+
+
async def test_a_hash_only_key_is_left_alone(
rls_harness: RLSHarness, monkeypatch: pytest.MonkeyPatch
) -> None:
diff --git a/core/tests/switch_core/db/test_tenant_exemption_allowlist.py b/core/tests/switch_core/db/test_tenant_exemption_allowlist.py
index b98a0529b..cdd70fb6f 100644
--- a/core/tests/switch_core/db/test_tenant_exemption_allowlist.py
+++ b/core/tests/switch_core/db/test_tenant_exemption_allowlist.py
@@ -184,6 +184,12 @@ _RAW_SESSION_FACTORY_MODULES = {
# and no policy (`rls_ddl.GLOBAL_TABLES`): a process serves every tenant,
# so its lease belongs to none. It touches only that table.
"switch_core.management.process_lease",
+ # The boot-time re-encryption sweep's per-tenant pass binds a tenant
+ # (`tenant_session`); this raw call is its separate pass over
+ # `trust_settings`'s encrypted key, a table with no tenant and no policy
+ # (`rls_ddl.GLOBAL_TABLES`) that a per-tenant loop would otherwise sweep
+ # once per tenant for no reason.
+ "switch_core.db.key_rotation",
# `DynamicTrustClient` reads `trust_settings` on every check, including
# from agent- and bridge-originated sends that have no request to inherit
# a tenant from. A table with no tenant and no policy
diff --git a/core/tests/switch_core/trust/test_client.py b/core/tests/switch_core/trust/test_client.py
index 9cad8cab9..98c489321 100644
--- a/core/tests/switch_core/trust/test_client.py
+++ b/core/tests/switch_core/trust/test_client.py
@@ -10,6 +10,7 @@ from dataclasses import dataclass
import httpx
import pytest
+from switch_core.keys import UndecryptableError
from switch_core.trust.client import (
DynamicTrustClient,
GuardrailBlockedError,
@@ -56,11 +57,16 @@ class _FakeKeyring:
return value.removeprefix("enc:")
-def _dynamic_client(row: _FakeRow | None, handler) -> DynamicTrustClient:
+class _UndecryptableKeyring:
+ def decrypt(self, value: str) -> str:
+ raise UndecryptableError("no key in the keyring opens this value")
+
+
+def _dynamic_client(row: _FakeRow | None, handler, keyring=None) -> DynamicTrustClient:
return DynamicTrustClient(
session_factory=_FakeSessionFactory(), # type: ignore[arg-type]
store=_FakeStore(row), # type: ignore[arg-type]
- keyring=_FakeKeyring(), # type: ignore[arg-type]
+ keyring=keyring or _FakeKeyring(), # type: ignore[arg-type]
client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
timeout_seconds=1.0,
)
@@ -113,6 +119,25 @@ async def test_dynamic_client_checks_against_the_current_row() -> None:
assert result.outcome == "ok"
+async def test_dynamic_client_turns_an_undecryptable_key_into_a_check_error() -> None:
+ """A key encrypted under a key since dropped from SECRET_KEYS must not
+ crash message sending — `check_message` only fails open on
+ `GuardrailsCheckError`, so the decrypt failure has to become one."""
+ row = _FakeRow(
+ endpoint="https://trust.example",
+ policy_id="pol_123",
+ api_key_encrypted="enc:k",
+ )
+ client = _dynamic_client(row, _unreachable, keyring=_UndecryptableKeyring())
+
+ with pytest.raises(GuardrailsCheckError):
+ await client.check(role="user", content="hi", room_id="room-1")
+
+ result = await check_message(client, role="user", content="hi", room_id="room-1")
+ assert result.outcome == "errored"
+
+
def _client(handler) -> HttpTrustClient:
return HttpTrustClient(
base_url="https://trust.example",