Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(db): make the exemption the runtime role's, and stop overstating it (CHOO-2623)

Wojciech Wojtyniakb465c83326ea
diff --git a/core/switch_core/bridges/agent/auth.py b/core/switch_core/bridges/agent/auth.py
index 859bee731..9e1cc1373 100644
--- a/core/switch_core/bridges/agent/auth.py
+++ b/core/switch_core/bridges/agent/auth.py
@@ -101,7 +101,7 @@ class BearerAuthMiddleware:
``self.app(...)`` is called below. The lookups that resolve the credential
itself (``_resolve_api_key``, ``_try_oidc``) run ahead of that, on
sessions of their own, and each is in two steps: the credential's *tenant*
- comes from one of the eight ``SECURITY DEFINER`` lookups that are the
+ comes from one of the seven ``SECURITY DEFINER`` lookups that are the
whole exemption from row-level security (``db/tenant_lookup.py``), and the
row itself is then read with that tenant bound, subject to the same
policies as everything else.
diff --git a/core/switch_core/bridges/collaboration/lifecycle_service.py b/core/switch_core/bridges/collaboration/lifecycle_service.py
index 2a1bc3bd2..77f72b271 100644
--- a/core/switch_core/bridges/collaboration/lifecycle_service.py
+++ b/core/switch_core/bridges/collaboration/lifecycle_service.py
@@ -24,7 +24,7 @@ from switch_core.db.stores.external_user_store import ExternalUserStore
from switch_core.db.stores.room_store import RoomStore
from switch_core.db.tenant_lookup import all_tenant_ids, tenant_of_collaboration_bridge
from switch_core.provisioning import Provisioning
-from switch_core.tenant_context import no_tenant
+from switch_core.tenant_context import current_tenant_id, no_tenant
if TYPE_CHECKING:
from switch_core.clients.client_lifecycle_service import ClientLifecycleService
@@ -247,13 +247,22 @@ class CollaborationBridgeLifecycleService:
if wanted is None:
return
+ # Captured before the loop below rebinds per tenant. This method is
+ # only ever reached from an authenticated request, so what is bound
+ # going in *is* the caller's tenant — the loop then replaces it, once
+ # per tenant, for the scoped read of that tenant's bridges. A call
+ # with nothing bound (no request behind it, if one ever exists) is
+ # treated as belonging to no tenant at all rather than guessed at: it
+ # can never match `other.tenant_id`, so it falls straight into the
+ # cross-tenant branch below and gets the non-disclosing message. Fail
+ # closed, not open.
+ caller_tenant_id = current_tenant_id()
+
# Deliberately across every tenant. Two tenants binding the same Slack
# workspace, or the same Teams listen port, is precisely the collision
# this exists to refuse — the resource is a property of the host and
# the platform, not of a tenant — so narrowing to the caller's tenant
- # would make it miss the case it was written for. Reached from an
- # authenticated request, so what is bound going in *is* the caller's
- # tenant; the loop replaces it per tenant rather than inheriting it.
+ # would make it miss the case it was written for.
existing: list[CollaborationBridge] = []
for tenant_id in await all_tenant_ids(self._session_factory):
async with tenant_session(self._session_factory, tenant_id) as session:
@@ -290,7 +299,13 @@ class CollaborationBridgeLifecycleService:
exc_info=True,
)
continue
- if held == wanted:
+ if held != wanted:
+ continue
+ if caller_tenant_id is not None and other.tenant_id == caller_tenant_id:
+ # The incumbent is the caller's own bridge, so naming it tells
+ # the caller nothing they cannot already see on their own
+ # bridge list — and the name is what turns this from "refused"
+ # into "refused, and here is the one to delete or move".
raise ValueError(
f"'{other.display_name}' already uses {wanted} on this "
f"instance, and two {bridge_type} bridges cannot share it. "
@@ -299,6 +314,36 @@ class CollaborationBridgeLifecycleService:
"chart publishes only one Teams port, so a second one needs "
"its own Service port and route."
)
+ # The incumbent belongs to a different tenant (or the caller's
+ # tenant could not be determined at all — see the fail-closed note
+ # above). Refuse the same way, but without the incumbent's display
+ # name or tenant: naming either would tell tenant A that tenant B
+ # exists, is on this instance, and holds this specific workspace or
+ # port, none of which A has any business learning from an error
+ # message. `wanted` itself is not a new disclosure — it is an echo
+ # of the connection_config the caller just submitted, not
+ # information about the incumbent.
+ #
+ # "already claimed on this instance" is still a narrow leak: it
+ # tells the caller *someone* holds this resource, which they would
+ # not otherwise know. That is unavoidable if the collision is to be
+ # refused at all rather than silently misconfigured, and it is a
+ # great deal less than a bridge name and a tenant identity.
+ logger.warning(
+ "Refused to register a %s bridge for tenant %s: %s is already "
+ "held by bridge %s in tenant %s",
+ bridge_type,
+ caller_tenant_id,
+ wanted,
+ other.id,
+ other.tenant_id,
+ )
+ raise ValueError(
+ f"{wanted} is already claimed by a {bridge_type} bridge on "
+ "this instance and cannot be shared. This is a host- and "
+ "platform-level limit, not specific to your workspace; an "
+ "operator can see which bridge holds it."
+ )
async def register(
self,
@@ -474,6 +519,7 @@ class CollaborationBridgeLifecycleService:
bridge_client = BridgeClient(
bridge_core=bridge_core,
client_id=bridge_client_record.id,
+ tenant_id=bridge_client_record.tenant_id,
matrix_user_id=bridge_client_record.matrix_user_id,
display_name=bridge_client_record.display_name,
session_factory=self._session_factory,
diff --git a/core/switch_core/clients/agent_client.py b/core/switch_core/clients/agent_client.py
index 11c76fbf2..401018720 100644
--- a/core/switch_core/clients/agent_client.py
+++ b/core/switch_core/clients/agent_client.py
@@ -53,7 +53,6 @@ from switch_core.db.stores.external_user_store import ExternalUserStore
from switch_core.db.stores.reference_store import ReferenceStore
from switch_core.db.stores.room_role_store import RoomRoleStore
from switch_core.db.stores.room_store import RoomStore
-from switch_core.db.tenant_lookup import tenant_of_client
from switch_core.delivery.addressing import (
ADDRESSING_DENIED_MESSAGE,
ADDRESSING_UNCLAIMED_MESSAGE,
@@ -302,17 +301,13 @@ class AgentClient(ClientBase[ClientConfig]):
"""Resolve the agent this client is, then run.
This runs at the top of the client's own task, which binds nothing, so
- the tenant is derived rather than inherited: `clients.id` is a primary
- key, and the exemption (`db/tenant_lookup.py`) turns it into the
- tenant that row is in. A *guessed* tenant would turn "this client is
- an agent" into "no agent found for client" and fail the client at boot
- over a row that exists, which is why this is a lookup rather than a
- default.
+ the tenant is carried rather than inherited: it comes from the
+ `clients` row this client was built from. A *guessed* tenant would
+ turn "this client is an agent" into "no agent found for client" and
+ fail the client at boot over a row that exists, which is why nothing
+ here falls back to whatever is ambient.
"""
- tenant_id = await tenant_of_client(self.session_factory, self.client_id)
- if tenant_id is None:
- raise RuntimeError(f"No client row for: {self.client_id}")
- async with tenant_session(self.session_factory, tenant_id) as session:
+ async with tenant_session(self.session_factory, self.tenant_id) as session:
agent = await self._agent_store.get_by_client_id(session, self.client_id)
if agent is None:
raise RuntimeError(f"No agent found for client: {self.client_id}")
diff --git a/core/switch_core/clients/client_base.py b/core/switch_core/clients/client_base.py
index 995074cf9..4964432d7 100644
--- a/core/switch_core/clients/client_base.py
+++ b/core/switch_core/clients/client_base.py
@@ -63,6 +63,7 @@ class ClientBaseKwargs[ConfigT: ClientConfig](TypedDict):
"""
client_id: str
+ tenant_id: str
matrix_user_id: str
display_name: str
session_factory: async_sessionmaker[AsyncSession]
@@ -78,6 +79,7 @@ class ClientBase[ConfigT: ClientConfig]:
self,
*,
client_id: str,
+ tenant_id: str,
matrix_user_id: str,
display_name: str,
session_factory: async_sessionmaker[AsyncSession],
@@ -86,6 +88,15 @@ class ClientBase[ConfigT: ClientConfig]:
transport_factory: Callable[[ClientBase[ConfigT]], MessageTransport],
) -> None:
self.client_id = client_id
+ # The tenant of this client's own row, carried rather than looked up.
+ # A client's task binds nothing on purpose and its handlers bind some
+ # *room's* tenant, so neither context answers "which tenant is this
+ # client" — but every caller that builds a client is holding the row
+ # that says, so the answer travels with the client instead of being
+ # asked of the database again on the far side. Required, not
+ # defaulted: a client that does not know its tenant cannot write a
+ # scoped row, and there is no sensible value to fall back to.
+ self.tenant_id = tenant_id
self.matrix_user_id = matrix_user_id
self.display_name = display_name
self.session_factory = session_factory
diff --git a/core/switch_core/clients/client_factory.py b/core/switch_core/clients/client_factory.py
index 5ba5cd2f3..50b75b1e8 100644
--- a/core/switch_core/clients/client_factory.py
+++ b/core/switch_core/clients/client_factory.py
@@ -64,6 +64,12 @@ class ClientFactory:
config = cls.config_class.model_validate(record.config or {})
return cls(
client_id=record.id,
+ # Straight off the row this client *is*. `create_client` writes
+ # that row inside a session with the tenant bound and the factory
+ # does not expire on commit, so a record that has just been
+ # flushed carries its tenant here as surely as one read back at
+ # boot does.
+ tenant_id=record.tenant_id,
matrix_user_id=record.matrix_user_id,
display_name=record.display_name,
session_factory=self._session_factory,
@@ -84,6 +90,7 @@ class ClientFactory:
return PostgresTransport(
user_id=client.matrix_user_id,
client_id=client.client_id,
+ tenant_id=client.tenant_id,
display_name=client.display_name,
session_factory=self._session_factory,
room_store=self._room_store,
diff --git a/core/switch_core/clients/client_lifecycle_service.py b/core/switch_core/clients/client_lifecycle_service.py
index d9c3c58e4..221361046 100644
--- a/core/switch_core/clients/client_lifecycle_service.py
+++ b/core/switch_core/clients/client_lifecycle_service.py
@@ -40,11 +40,12 @@ class ClientLifecycleService:
self._config = config
self._clients: dict[str, ClientBase[ClientConfig]] = {}
self._client_types: dict[str, str] = {}
- # Which tenant each running client's row belongs to. A `ClientBase`
- # does not carry it — a client's own work resolves the tenant from the
- # room it is acting on — but this registry holds every tenant's
- # clients at once, so anything picking clients *out* of it has to be
- # able to say which tenant it wants. See `get_by_type`.
+ # Which tenant each running client's row belongs to. The client
+ # carries the same value (`ClientBase.tenant_id`); this is the index
+ # on it, keyed like `_clients` and `_client_types` so the three are
+ # populated and emptied together and a caller picking clients *out* of
+ # a registry that holds every tenant's at once can say which tenant it
+ # wants without reaching into the objects. See `get_by_type`.
self._client_tenants: dict[str, str] = {}
self._tasks: dict[str, asyncio.Task[None]] = {}
@@ -65,7 +66,7 @@ class ClientLifecycleService:
wrong question to ask: it answers yes for a tenant that has none.
Which tenants exist is exactly what a bound session cannot see, so it
- comes from `all_tenant_ids` — one of the eight `SECURITY DEFINER`
+ comes from `all_tenant_ids` — one of the seven `SECURITY DEFINER`
lookups that make up the whole exemption from row-level security
(`db/tenant_lookup.py`). Everything after that is an ordinary scoped
read, one tenant at a time, and each row is created with that tenant
@@ -327,13 +328,15 @@ class ClientLifecycleService:
A puppet in particular is reused for every room the person it stands
for speaks in, so the first room's tenant is exactly the value that
must not survive into the second. Everything the client does binds
- the tenant of the room it is acting on. The one lookup that still runs
- with nothing bound is `PostgresTransport._tenant`, the exemption call
- (`tenant_of_client`) that answers *which* tenant this client's own row
- belongs to — asking that question on a session already bound to a
- tenant would beg it. Its room list is not exempt any more: once the
- transport knows its own tenant, `joined_rooms` reads under it, through
- an ordinary `tenant_session`, rather than unscoped.
+ the tenant of the room it is acting on.
+
+ Nothing in here runs unbound, and nothing has to ask the database
+ which tenant this client is in. A `ClientBase` carries its own
+ `tenant_id`, taken from the `clients` row it was built from, and hands
+ it to its transport; `joined_rooms` and every media read then go
+ through an ordinary `tenant_session` under that tenant. An exemption
+ call here would have been a question whose answer the caller was
+ already holding.
"""
with no_tenant():
try:
diff --git a/core/switch_core/config.py b/core/switch_core/config.py
index f6afc3d22..65d6e6d39 100644
--- a/core/switch_core/config.py
+++ b/core/switch_core/config.py
@@ -31,8 +31,19 @@ class SwitchConfig(BaseSettings):
# working would hand back the ownership exemption the policies rely on it
# not having.
#
- # Unset means "this deployment migrates somewhere else": boot applies no
- # migrations and expects the schema to already be at head.
+ # Boot always runs `alembic upgrade head`, unconditionally, whether or not
+ # this is set — that is `main()`'s job, not this setting's. What this
+ # setting decides is which connection that migration (and the runtime
+ # role's grant re-issue right after it) runs on: `migrations/env.py` uses
+ # the owner connection where one is configured here, and falls back to the
+ # runtime connection where none is. That fallback is what keeps `alembic`
+ # on the command line working against a scratch database a developer
+ # points it at, where the two roles are the same one. For a deployment
+ # that has moved DB_USER to a genuinely restricted runtime role without
+ # also setting this, the same fallback is what makes the failure loud: the
+ # runtime role cannot issue DDL, so the migration fails immediately with a
+ # permission error naming the statement it could not run, rather than
+ # boot silently skipping the migration or half-applying it.
db_owner_user: str | None = None
db_owner_password: str | None = None
@@ -369,10 +380,6 @@ class SwitchConfig(BaseSettings):
f"@{self.db_host}:{self.db_port}/{self.db_name}"
)
- @property
- def runs_migrations_at_boot(self) -> bool:
- return self.owner_database_url is not None
-
@property
def db_connect_args(self) -> dict[str, object]:
"""asyncpg connect args derived from config.
diff --git a/core/switch_core/db/boot_lock.py b/core/switch_core/db/boot_lock.py
new file mode 100644
index 000000000..c10ceabaa
--- /dev/null
+++ b/core/switch_core/db/boot_lock.py
@@ -0,0 +1,141 @@
+"""A Postgres session-level advisory lock around boot-time schema work.
+
+Two replicas of switch-core starting at once is not a hypothetical: it is what
+a rolling deploy or a crash-restart racing a fresh pod produces routinely, and
+two things `main()` does at boot are only safe run by one replica at a time
+across the whole deployment:
+
+- `alembic upgrade head`, plus the runtime role's grant re-issue that runs
+ immediately after it (`main._prepare_database`). Two replicas applying the
+ same migration concurrently contend on the same catalogue rows Postgres
+ itself locks for DDL — the usual outcome is a deadlock or a "duplicate
+ object" error, not a race either side quietly wins — and the grant re-issue
+ right after it touches `pg_default_acl`, which two `ALTER DEFAULT
+ PRIVILEGES` statements running at once contend on the same way.
+- The deployment-wide seeding in `run()` (the admin user and the
+ agent-registration bootstrap key). Unlike the migration, this is not one
+ transaction: `_seed_agent_registration_bootstrap_key` reads its own state,
+ decides what to do, and writes it back across several separate sessions, so
+ a transaction-scoped lock cannot cover it — two replicas mid-seed can each
+ read "no bootstrap key yet" and both insert one, colliding on the unique
+ `key_hash` at best and duplicating a key that is supposed to be one per
+ deployment at worst.
+
+Postgres's advisory locks exist for exactly this shape of problem: a lock with
+no table or row behind it, held for as long as an application decides rather
+than for one transaction. `pg_advisory_lock` is the session-level form —
+acquired and released explicitly (or implicitly when the session ends) by
+whichever database session took it, independent of whatever transactions come
+and go on that session in between. That is also why `boot_lock` opens a
+connection of its own on `NullPool` rather than borrowing one from a pool: a
+pooled connection can be checked back in — and handed to a different caller —
+before the code between acquire and release has finished, at which point the
+lock is still held, but by a connection nothing here controls any more, and it
+would not be released until the pool happened to recycle that connection.
+
+`pg_advisory_lock` takes a single 64-bit signed key, and every advisory lock
+in a database shares one flat namespace — there is no per-caller or per-module
+subspace to register into, and the session-level and transaction-level forms
+share it too, so `pg_advisory_xact_lock` is not a separate space to hide in.
+Coordination is therefore entirely by convention: every caller has to agree
+out of band on which integers are taken.
+
+Switch has one other advisory lock, and it is worth naming rather than
+assuming away: `db/stores/message_store.py` serialises a room's message
+sequence with `pg_advisory_xact_lock(hashtext(room_id))`. It cannot collide
+with `BOOT_LOCK_KEY`, and the reason is arithmetic rather than luck —
+`hashtext` returns `integer`, so every key that lock can ever take lies in the
+signed 32-bit range, and `BOOT_LOCK_KEY` is deliberately chosen far outside
+it. Keeping the two apart by *domain* rather than by a list of taken values is
+what makes this survive a room id nobody has seen yet. A future third use
+should either pick another constant outside the 32-bit range and say so here,
+or accept that it is sharing a space with every room in the deployment.
+
+One more property matters for where this sits in the boot sequence:
+`pg_advisory_lock` lives in `pg_catalog`, not in `public`. A `REVOKE ALL ON
+SCHEMA public FROM PUBLIC` — tightening what the runtime role can do in the
+application schema — has no bearing on whether it may call a `pg_catalog`
+function, so the lock is available on the runtime connection precisely when
+`DB_OWNER_USER` is unset and that connection is boot's only option (see
+`SwitchConfig.owner_database_url`).
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import create_async_engine
+from sqlalchemy.pool import NullPool
+
+from switch_core.config import SwitchConfig
+
+# The one key this module uses. Advisory locks share a single namespace per
+# database, so the exact value carries no meaning of its own — it only has to
+# be fixed, and outside the signed 32-bit range every `hashtext`-derived room
+# lock lives in (see the module docstring). Generated once as a random 63-bit
+# value and hardcoded rather than derived at call time, so a reader can see at
+# a glance that it never changes between boots or between replicas, which is
+# the one property that matters for two replicas to contend on the same lock
+# at all.
+#
+# `test_main.py` pins the 32-bit separation, so changing this value to
+# something that could collide fails a test rather than going unnoticed.
+BOOT_LOCK_KEY = 4_127_659_302_481_907_553
+
+
+@asynccontextmanager
+async def boot_lock(config: SwitchConfig) -> AsyncIterator[None]:
+ """Hold Switch's one boot-time advisory lock for the duration of the block.
+
+ Connects as the schema owner where one is configured
+ (`config.owner_database_url`) and as the runtime role otherwise — the same
+ rule `migrations/env.py` applies to the migration connection itself, and
+ for the same reason: on a fresh deployment the runtime role may not exist
+ yet, so the owner is the only connection guaranteed to be there, and where
+ neither role is distinguished (a developer's scratch database) the runtime
+ connection is both.
+
+ Uses `config.db_connect_args` rather than the application engine's
+ `app_connect_args` — the same choice `main._prepare_database` and
+ `migrations/env.py` make — because this connection carries no request
+ traffic and the idle-in-transaction timeout that only applies to the
+ application engine would be actively wrong here: the lock is meant to be
+ held across slow, multi-statement boot work, not torn down because it sat
+ idle between two of its steps.
+
+ A dedicated `NullPool` engine, opened and disposed around the lock rather
+ than reused: an advisory lock belongs to the session that took it, and the
+ only way to guarantee this code — not a pool, not a future checkout of the
+ same connection — controls when it is released is to own that connection
+ outright for exactly as long as the lock is held.
+
+ `isolation_level="AUTOCOMMIT"` so `pg_advisory_lock`/`pg_advisory_unlock`
+ each take effect immediately as their own statement, rather than SQLAlchemy
+ auto-beginning a transaction on the first `execute` and leaving it open,
+ uncommitted, for however long the guarded block takes to run. The lock
+ itself does not need a transaction — it is a property of the session, not
+ of any transaction on it — so there is nothing to commit and no reason to
+ hold one open around boot work that can take a while.
+ """
+ url = config.owner_database_url or config.database_url
+ engine = create_async_engine(
+ url,
+ poolclass=NullPool,
+ connect_args=config.db_connect_args,
+ isolation_level="AUTOCOMMIT",
+ )
+ try:
+ async with engine.connect() as connection:
+ await connection.execute(
+ text("SELECT pg_advisory_lock(:key)"), {"key": BOOT_LOCK_KEY}
+ )
+ try:
+ yield
+ finally:
+ await connection.execute(
+ text("SELECT pg_advisory_unlock(:key)"), {"key": BOOT_LOCK_KEY}
+ )
+ finally:
+ await engine.dispose()
diff --git a/core/switch_core/db/engine.py b/core/switch_core/db/engine.py
index 83a88b371..17f552e42 100644
--- a/core/switch_core/db/engine.py
+++ b/core/switch_core/db/engine.py
@@ -13,11 +13,13 @@ from sqlalchemy.pool import NullPool
from switch_core.config import SwitchConfig
-# Imported for its side effect: it registers the `after_begin` hook that stamps
-# the bound tenant onto every session's transaction. Here rather than at a
-# factory call site, so a process that builds an `async_sessionmaker` by hand
-# is covered too — it still needs an engine, and an engine comes from here.
-from switch_core.db import tenant_session # noqa: F401
+# Imported both for its side effect and for what it holds. The side effect is
+# the `after_begin` hook that stamps the bound tenant onto every session's
+# transaction, registered here rather than at a factory call site so that a
+# process building an `async_sessionmaker` by hand is covered too — it still
+# needs an engine, and an engine comes from here. What it holds is the session
+# class `create_session_factory` binds below.
+from switch_core.db import tenant_session
logger = logging.getLogger(__name__)
@@ -167,4 +169,16 @@ def create_unpooled_engine(config: SwitchConfig) -> AsyncEngine:
def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]:
- return async_sessionmaker(bind=engine, expire_on_commit=False)
+ """The factory every session in the process comes from.
+
+ `sync_session_class` is what puts the tenant checks on `Session.get`:
+ `AsyncSession` does no ORM work of its own, it drives a synchronous
+ `Session` underneath, and `get` answered from that session's identity map
+ is the one read that reaches neither the row-level-security policy nor the
+ `do_orm_execute` hook. See `db/tenant_session.TenantCheckedSession`.
+ """
+ return async_sessionmaker(
+ bind=engine,
+ expire_on_commit=False,
+ sync_session_class=tenant_session.TenantCheckedSession,
+ )
diff --git a/core/switch_core/db/models.py b/core/switch_core/db/models.py
index d09ecfd0e..957e0b8c0 100644
--- a/core/switch_core/db/models.py
+++ b/core/switch_core/db/models.py
@@ -1589,7 +1589,7 @@ event.listen(
# notify trigger's migration does.
attach_row_level_security(Base.metadata)
-# And the eight functions that are exempt from those policies, on the same
+# And the seven functions that are exempt from those policies, on the same
# reasoning again: a restricted role cannot boot without them, so a schema
# `create_all` built without them is not the schema the server runs against.
# After the tables, not before — see `db/tenant_lookup.py`.
diff --git a/core/switch_core/db/runtime_role.py b/core/switch_core/db/runtime_role.py
index 59e7f3aab..3a68f555b 100644
--- a/core/switch_core/db/runtime_role.py
+++ b/core/switch_core/db/runtime_role.py
@@ -11,17 +11,26 @@ Two halves live here.
**Grants.** The runtime role owns nothing and can create nothing, so it needs
to be given access to every table the owner created. The actual mechanism is
-the four explicit `GRANT`s, re-issued fresh on every boot, immediately after
-the migration that may have added a table: naming everything the schema
-contains at that moment cannot fall behind it, the way a rule that has to fire
-once per object, at the moment of its creation, can. The three `ALTER DEFAULT
-PRIVILEGES` statements alongside them are belt to that grant's braces, not a
-second mechanism for the same job: they cover whatever the owner creates
-*between* one boot and the next, at the cost of being easy to get subtly
-wrong as a strategy on their own — the setting is per-granting-role, and
-silently grants nothing for an object some other role creates — a gap the
-next boot's fresh `GRANT` closes regardless of whether the default privilege
-fired. Both are idempotent and cost one round trip at boot.
+the explicit `GRANT`s, re-issued fresh on every boot, immediately after the
+migration that may have added a table: naming everything the schema contains
+at that moment cannot fall behind it, the way a rule that has to fire once per
+object, at the moment of its creation, can. The `ALTER DEFAULT PRIVILEGES`
+statements alongside them are belt to that grant's braces, not a second
+mechanism for the same job: they cover whatever the owner creates *between*
+one boot and the next, at the cost of being easy to get subtly wrong as a
+strategy on their own — the setting is per-granting-role, and silently grants
+nothing for an object some other role creates — a gap the next boot's fresh
+`GRANT` closes regardless of whether the default privilege fired. Both are
+idempotent and cost one round trip at boot.
+
+**Revokes, for the same reason in reverse.** `EXECUTE` on a new function is
+granted to `PUBLIC` by default, so the exempt lookups in `db/tenant_lookup.py`
+arrive callable by every role that can connect to this database. Nothing about
+that is deliberate — it is the shape a `CREATE FUNCTION` has, and the
+migration that installs them has no role name to grant to. So the same pass
+takes it away and grants `EXECUTE` to the runtime role by name, which makes
+the grant, rather than a default, the reason those functions are callable at
+all.
**The self-check.** Questions asked of the runtime connection before the
server listens, in increasing order of how much they prove, because this
@@ -42,7 +51,9 @@ like one that is.
pass every check above it.
5. Every tenant lookup exists and this role may execute it, since a role that
cannot would authenticate nobody.
-6. **A read of `tenants` with nothing bound raises.** This is the one that
+6. `PUBLIC` may *not* execute them, so the exemption is reachable through the
+ runtime role's credentials rather than by anything holding a connection.
+7. **A read of `tenants` with nothing bound raises.** This is the one that
actually proves it, because the others are inferences from the catalogue
and this is the behaviour. `tenants` is the table to ask it of: the check
is only meaningful against a populated table — Postgres does not evaluate a
@@ -95,20 +106,53 @@ async def grant_runtime_role(owner: AsyncConnection, role: str) -> None:
Run as the owner, after `alembic upgrade head`. `EXECUTE ON ALL FUNCTIONS`
covers `require_tenant_id()` and the tenant lookups without naming them,
- so a ninth lookup needs no change here.
+ so an eighth lookup needs no change here.
- The four plain `GRANT`s are what actually does the job, and doing it fresh
- on every boot is what keeps it from falling behind: it names every table,
+ The plain `GRANT`s are what actually does the job, and doing it fresh on
+ every boot is what keeps it from falling behind: it names every table,
sequence and function the schema holds at that moment, so it cannot miss
one the way a rule that has to fire once per object, at creation, can. The
- three `ALTER DEFAULT PRIVILEGES` statements after them are belt to that
- grant's braces, covering an object the owner creates between this boot and
- the next — at the cost of being unfit to carry the job alone, since the
+ `ALTER DEFAULT PRIVILEGES` statements after them are belt to that grant's
+ braces, covering an object the owner creates between this boot and the
+ next — at the cost of being unfit to carry the job alone, since the
setting is per-granting-role and silently grants nothing for an object
some other role creates. The next boot's fresh `GRANT` would close that
gap either way, which is why the default privileges are additional rather
than load-bearing on their own.
+ **The two revokes are what make the grant mean something.** Without them
+ `EXECUTE` on the tenant lookups stays at the `PUBLIC` default a new
+ function is created with, so every role with `CONNECT` on this database
+ can resolve which tenant any user, credential, client, room, bridge or
+ connector belongs to — see `db/tenant_lookup.py` on what that discloses.
+ Revoking from `PUBLIC` and granting to `role` by name replaces "callable
+ by anyone" with "callable by the process".
+
+ Two things must survive that revoke, and it is worth saying which and why
+ rather than trusting the blanket grant to have covered them.
+ `require_tenant_id()` is called by every policy and is evaluated as the
+ *querying* role, so a runtime role without `EXECUTE` on it cannot read or
+ write a single scoped table — measured on 16: the read fails with
+ `permission denied for function require_tenant_id`, not with a policy
+ refusal, which is a boot-time outage dressed up as an authorization error.
+ The delivery trigger in `db/notify_ddl.py` is the opposite case: a trigger
+ function's `EXECUTE` is checked when the trigger is created, not each time
+ it fires, so an insert succeeds for a role that may not call it — also
+ measured, also on 16. The blanket `GRANT EXECUTE ON ALL FUNCTIONS` covers
+ both regardless, and `verify_restricted_role` exercises the first on every
+ boot.
+
+ The default-privileges revoke carries no `IN SCHEMA`, unlike every other
+ statement here, and that is not an oversight. A per-schema default ACL is
+ *unioned* with the built-in one rather than replacing it, so
+ `ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE EXECUTE ON FUNCTIONS
+ FROM PUBLIC` records nothing and a function created after it is still
+ `PUBLIC`-executable — verified on Postgres 16, where it leaves
+ `pg_default_acl` empty and the new function's `proacl` null. The
+ schema-less form is the one that takes, and it is the right scope anyway:
+ it is a statement about what this owner creates, and this owner creates
+ only this schema.
+
No `CREATE` on the schema, and no ownership: the role is meant to be
unable to alter the tables it reads, which is what keeps it subject to
their policies.
@@ -118,9 +162,11 @@ async def grant_runtime_role(owner: AsyncConnection, role: str) -> None:
f"GRANT USAGE ON SCHEMA public TO {identifier}",
f"GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO {identifier}",
f"GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO {identifier}",
+ "REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC",
f"GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO {identifier}",
f"ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO {identifier}",
f"ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO {identifier}",
+ "ALTER DEFAULT PRIVILEGES REVOKE EXECUTE ON FUNCTIONS FROM PUBLIC",
f"ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT EXECUTE ON FUNCTIONS TO {identifier}",
):
await owner.execute(text(statement))
@@ -146,6 +192,7 @@ async def verify_restricted_role(engine: AsyncEngine) -> None:
await _refuse_forced_row_level_security(connection)
await _require_every_policy(connection)
await _require_the_lookups(connection, role)
+ await _refuse_public_execute(connection)
# The probe is the one check that provokes an error on purpose, and a
# statement that raises aborts the transaction it ran in — every query
# after it on that connection fails with "current transaction is aborted"
@@ -400,6 +447,57 @@ async def _require_the_lookups(connection: AsyncConnection, role: str) -> None:
)
+async def _refuse_public_execute(connection: AsyncConnection) -> None:
+ """The exemption must be the runtime role's, not everybody's.
+
+ `EXECUTE` on a new function is granted to `PUBLIC`, so the lookups arrive
+ callable by every role that can connect — including one created later for
+ reporting, for a migration tool, or for a person. None of them can read a
+ row of a scoped table, which is the boundary that matters, and all of them
+ could enumerate the tenants and resolve any identifier they hold to one
+ (`db/tenant_lookup.py` states exactly what that discloses). `boot`'s own
+ `grant_runtime_role` closes it; this is the check that it did, and that
+ nothing has re-granted it since — a `GRANT EXECUTE ON ALL FUNCTIONS IN
+ SCHEMA public TO PUBLIC` issued to fix some unrelated permission error
+ would reopen it in one statement and leave every other check here passing.
+
+ `has_function_privilege('public', …)` is the pseudo-role spelling, which
+ the access-privilege inquiry functions accept in place of a role name.
+ Asked of the function's oid rather than its signature so a lookup that is
+ not installed at all is absent from the result instead of raising —
+ `_require_the_lookups` above is the check that says so, and it has already
+ run by the time this one does.
+ """
+ expected = sorted(lookup.name for lookup in TENANT_LOOKUPS)
+ rows = await connection.execute(
+ text(
+ """
+ SELECT p.proname
+ FROM pg_proc p
+ JOIN pg_namespace n ON n.oid = p.pronamespace
+ WHERE n.nspname = 'public' AND p.proname = ANY(:names)
+ AND has_function_privilege('public', p.oid, 'EXECUTE')
+ ORDER BY p.proname
+ """
+ ),
+ {"names": expected},
+ )
+ public = [row[0] for row in rows]
+ if public:
+ raise RuntimeRoleError(
+ f"PUBLIC holds EXECUTE on {public}, so every role with CONNECT on "
+ "this database can call the tenant lookups (db/tenant_lookup.py) "
+ "— enumerate every tenant, and resolve any user, credential, "
+ "client, room, bridge or connector it can name to the tenant that "
+ "owns it. That is the default a CREATE FUNCTION leaves behind, so "
+ "either the boot grants have never run against this database "
+ "(configure DB_OWNER_USER / DB_OWNER_PASSWORD) or something has "
+ "granted EXECUTE back to PUBLIC since. Revoke it: REVOKE EXECUTE "
+ "ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC, then grant it to "
+ "the runtime role by name."
+ )
+
+
def _sqlstate(exc: DBAPIError) -> str | None:
"""The SQLSTATE behind a driver error, however this driver spells it.
diff --git a/core/switch_core/db/session_scope.py b/core/switch_core/db/session_scope.py
index eb93d289f..d23a04025 100644
--- a/core/switch_core/db/session_scope.py
+++ b/core/switch_core/db/session_scope.py
@@ -26,9 +26,12 @@ seventeen call sites — the ones that went through this helper by name, which
is not the same count as the exemption's full inventory; see
`db/tenant_lookup.py` for the other two — either died at boot or silently
read nothing. The whole model was inverted, and `db/tenant_lookup.py` is what
-replaced it: eight `SECURITY DEFINER` functions that answer *which tenant*
+replaced it: seven `SECURITY DEFINER` functions that answer *which tenant*
and never return a row, so a cross-tenant question is asked in one place with
a fixed shape, and the work it fans out into is scoped like everything else.
+That module states exactly what the exemption concedes and what it does not;
+it is a boundary on rows rather than on the shape of the deployment, and the
+difference is worth reading there rather than guessing at from here.
What is left is the one helper. Two things still open a session with nothing
bound, and neither is cross-tenant:
diff --git a/core/switch_core/db/stores/tenant_member_store.py b/core/switch_core/db/stores/tenant_member_store.py
deleted file mode 100644
index 935c1ece0..000000000
--- a/core/switch_core/db/stores/tenant_member_store.py
+++ /dev/null
@@ -1,64 +0,0 @@
-from __future__ import annotations
-
-from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
-
-from switch_core.db.tenant_lookup import tenants_of_user
-
-
-class TenantMembershipError(Exception):
- """A user has zero, or more than one, tenant memberships.
-
- Phase 1 has exactly one tenant, so exactly one membership row is the only
- correct state: zero means the user was never enrolled (a bug in whatever
- created the account), and more than one is Phase 2's tenant-switching
- shape arriving early. Either way this must not be resolved by picking
- one — see `docs/old/multi-tenancy-phase1-db.md`, "Setting the tenant".
-
- Raised, not returned, but not left to reach the client either: the gateway
- turns it into a 403 naming no user id (`gateway/auth.py`), so a broken
- account gets an answer an operator can act on instead of a bare 500.
- """
-
-
-class TenantMemberStore:
- """Reads memberships. Deliberately does not write them.
-
- There was a `create` here and nothing ever called it. Membership is not a
- thing code decides to write on its own: it is written by, and only by, the
- paths that produce or repair an account, so that "every account has exactly
- one" holds by construction rather than by everyone remembering. That write
- is `UserStore.ensure_membership`, which is idempotent and derives the role
- from the user. A second, unguarded way in — taking `tenant_id`, `user_id`
- and `role` from whatever the caller felt like — is how an account ends up
- with two, and `get_sole_tenant_id` below refuses to pick between two just
- as firmly as it refuses to invent one out of zero.
- """
-
- async def get_sole_tenant_id(
- self,
- session_factory: async_sessionmaker[AsyncSession],
- user_id: str,
- ) -> str:
- """The one tenant this user belongs to, raising if that isn't true.
-
- Takes the factory rather than a session, unlike every other store
- method here, and the difference is the point: `tenant_members` is a
- scoped table, so a session cannot read the row that would tell it what
- to be scoped to. This goes through `tenants_of_user`, one of the eight
- `SECURITY DEFINER` lookups that make up the whole exemption from
- row-level security (`db/tenant_lookup.py`), which opens a session of
- its own with nothing bound and answers with tenant ids and nothing
- else.
-
- It used to run a plain `select` on a session the caller opened and
- left unbound. That worked only because every environment connected as
- the tables' owner; under the restricted runtime role it is one of the
- reads the policy refuses, and every gateway login failed on it.
- """
- tenant_ids = await tenants_of_user(session_factory, user_id)
- if len(tenant_ids) != 1:
- raise TenantMembershipError(
- f"user {user_id} has {len(tenant_ids)} tenant memberships; "
- "expected exactly 1"
- )
- return tenant_ids[0]
diff --git a/core/switch_core/db/stores/tenant_store.py b/core/switch_core/db/stores/tenant_store.py
index 0cb753226..79e1bd2bd 100644
--- a/core/switch_core/db/stores/tenant_store.py
+++ b/core/switch_core/db/stores/tenant_store.py
@@ -13,7 +13,7 @@ class TenantStore:
exactly its own row and no store method can enumerate them — the previous
one only appeared to because every caller handed it a session connected as
the tables' owner. The enumeration is `all_tenant_ids` in
- `db/tenant_lookup.py`, one of the eight `SECURITY DEFINER` functions that
+ `db/tenant_lookup.py`, one of the seven `SECURITY DEFINER` functions that
are the whole exemption from row-level security, and it takes a session
factory rather than a session precisely because there is no session it
could correctly run on.
diff --git a/core/switch_core/db/tenant_lookup.py b/core/switch_core/db/tenant_lookup.py
index 090994a23..75b307712 100644
--- a/core/switch_core/db/tenant_lookup.py
+++ b/core/switch_core/db/tenant_lookup.py
@@ -1,4 +1,4 @@
-"""The whole exemption from row-level security, written out as eight functions.
+"""The whole exemption from row-level security, written out as seven functions.
Row-level security is enforced by `require_tenant_id()` (`db/rls_ddl.py`),
which raises when no tenant is bound. That is the property everything else
@@ -26,13 +26,47 @@ design deliberately never sets `force row level security` — see
worth having:
- **They answer *which tenant*, never *what row*.** Every one returns
- `setof text`: tenant ids. The most a caller can extract is the tenant an
- identifier it already holds belongs to. Nothing crosses a tenant boundary
- except the boundary's own name.
+ `setof text`: tenant ids. What exactly that does and does not concede is
+ worth stating precisely, and is stated below rather than summarised here,
+ because the obvious summary of it is false.
- **They are a closed list.** `TENANT_LOOKUPS` below is the list; a test
compares it against the functions actually installed, against the
migration's frozen copy, and against what each one answers when called as
- the restricted role. Adding a ninth is an edit a reviewer sees.
+ the restricted role. Adding an eighth is an edit a reviewer sees.
+
+**What the exemption gives, stated exactly.** Every lookup returns `setof
+text` — tenant ids, never a row of a scoped table. That much is the part
+worth having: no customer data crosses a tenant boundary through this module,
+on any function, for any argument.
+
+Three things the shorter statement of that property — "the most any caller can
+extract is the tenant of an identifier it already holds" — got wrong, all of
+them about *metadata* rather than rows:
+
+1. `all_tenant_ids()` takes no identifier. It enumerates the deployment: how
+ many tenants there are and what their ids are. Nothing about it is narrowed
+ to what the caller already holds, and nothing can be — the boot fan-outs
+ and the runtime-state sweep exist precisely to visit tenants the caller has
+ never heard of.
+2. `EXECUTE` used to be left at the `PUBLIC` default a new function gets, so
+ any role with `CONNECT` on this database could call every one of them. That
+ is closed: `grant_runtime_role` revokes `EXECUTE` on the schema's functions
+ from `PUBLIC` and grants it to the runtime role by name, and the boot
+ self-check refuses to serve if `PUBLIC` has it back.
+3. `users` and `oidc_identities` carry no tenant and so no policy
+ (`db/rls_ddl.py`'s `GLOBAL_TABLES`) — a person is global, and the
+ per-tenant object is the `tenant_members` row. A session with nothing bound
+ may therefore read every user id, and feeding those one at a time to
+ `tenants_of_user` reconstructs the whole user-to-tenant membership graph.
+ That follows from the schema rather than from this module, and it is not
+ closed here.
+
+So the property this design actually holds is: **the exemption discloses the
+shape of the deployment — which tenants exist, and which tenant a given user,
+credential, room, bridge or connector belongs to — and no row of any
+tenant-scoped table.** It is a boundary on data, not on metadata. Narrowing
+the second is a question about who may hold the runtime role's credentials at
+all, since everything above is reachable by anyone who has them.
Everything else in the process binds a tenant. The nineteen sites that could
not are now one of three shapes, and the shape is the interesting part:
@@ -46,16 +80,26 @@ not are now one of three shapes, and the shape is the interesting part:
then one scoped pass per tenant. These sites already fanned out per row
and bound that row's tenant; the loop simply moved one level up, and the
read inside it is now subject to the policies like any other.
-3. **Per-item work that can derive its tenant** — a client's transport, a
- bridge or connector being started by id, a room reached by id. One lookup
- by the globally unique identifier the caller already has, and everything
- after it is scoped.
+3. **Per-item work that can derive its tenant** — a bridge or connector being
+ started by id, a room reached by id. One lookup by the globally unique
+ identifier the caller already has, and everything after it is scoped.
+
+**A lookup whose caller already knows the answer does not belong here.** There
+was an eighth, `tenant_of_client`, and it was the one called most: every
+client's transport asked it, once per transport, and so did every agent
+client's `start`. Both were built from a `clients` row that names the tenant
+in a column, so the question was asked of the database with the answer already
+in hand. `ClientBase` and `PostgresTransport` now take a `tenant_id` the way
+they take a `client_id`, and revision `b1d7c4f0a92e` drops the function. The
+test that keeps the list honest is the one that would have let this stand: a
+function nobody needs is still a function every role could call, so the shorter
+list is the whole point of noticing.
Why not the obvious alternatives is argued in
`docs/old/multi-tenancy-phase1-db.md`, "The bootstrap problem"; the short
version is that returning rows instead of tenant ids would put a second copy
of a dozen store methods in SQL and widen the exemption from "a tenant id"
-to "any row of nine tables", and that a second engine connected as the owner
+to "any row of every table they read", and that a second engine connected as the owner
would make the exemption ambient again — reachable from anything holding the
factory, and no longer a list anybody can read.
@@ -140,7 +184,7 @@ class TenantLookup:
return f"{self.name}({'' if self.argument is None else 'text'})"
-# The eight of them. Ordered as the three shapes above: enumeration, then
+# The seven of them. Ordered as the three shapes above: enumeration, then
# credential resolution, then deriving a tenant from an identifier in hand.
TENANT_LOOKUPS: tuple[TenantLookup, ...] = (
TenantLookup(
@@ -189,17 +233,6 @@ TENANT_LOOKUPS: tuple[TenantLookup, ...] = (
"picking."
),
),
- TenantLookup(
- name="tenant_of_client",
- argument="client_id",
- query="SELECT tenant_id FROM clients WHERE id = p_client_id",
- purpose=(
- "Which tenant a room client belongs to. Its transport's tenant for "
- "everything it does: `client_rooms` carries composite foreign keys "
- "to both `clients` and `rooms`, so a client's rooms are all in the "
- "client's own tenant."
- ),
- ),
TenantLookup(
name="tenant_of_room",
argument="room_id",
@@ -346,13 +379,6 @@ async def tenant_of_agent_oauth_client(
return _at_most_one(lookup, await _call(session_factory, lookup, oauth_client_id))
-async def tenant_of_client(
- session_factory: async_sessionmaker[AsyncSession], client_id: str
-) -> str | None:
- lookup = TENANT_LOOKUPS_BY_NAME["tenant_of_client"]
- return _at_most_one(lookup, await _call(session_factory, lookup, client_id))
-
-
async def tenant_of_room(
session_factory: async_sessionmaker[AsyncSession], room_id: str
) -> str | None:
diff --git a/core/switch_core/db/tenant_session.py b/core/switch_core/db/tenant_session.py
index 3c4213c75..0d500092f 100644
--- a/core/switch_core/db/tenant_session.py
+++ b/core/switch_core/db/tenant_session.py
@@ -44,10 +44,28 @@ session with nothing bound may only read what carries no policy at all —
whatever is bound, including nothing. Whether nothing bound then reads
quietly or is refused loudly is the policies' call, not this hook's, and it
used to be the former.
+
+**One read never reaches the database, and so reaches neither the policy nor
+the drift check.** `Session.get` matches a primary key against the objects the
+session has already loaded, and on a hit returns one with no statement, no
+flush and no transaction. Every other guarantee here is downstream of a round
+trip: the policy is the server's, and `do_orm_execute`/`before_flush` fire on
+statements and units of work. An identity-map hit is none of those, so a
+session reused across two tenants can be handed the first tenant's row back
+while the second is bound. `TenantCheckedSession` below closes it, on the row
+rather than on the transaction — the object in hand carries its own
+`tenant_id`, which is a more direct answer than any stamp — and
+`db/engine.create_session_factory` is what puts it under every session in the
+process. It fires only when the `get` issued no statement at all; one that
+went to the database was filtered on the way, and second-guessing it would
+apply the policy's semantics to the owner connection, which nothing else here
+does.
"""
from __future__ import annotations
+from typing import Any
+
from sqlalchemy import event, text
from sqlalchemy.engine import Connection
from sqlalchemy.orm import ORMExecuteState, Session, SessionTransaction
@@ -60,6 +78,13 @@ _registered = False
# so the drift check below can compare it against what is bound now.
_STAMPED = "switch_stamped_tenant_id"
+# Set on the session whenever a statement goes out through the ORM, and read
+# by `TenantCheckedSession.get` to tell "this answer came from the database"
+# from "this answer came out of the identity map". Only the second needs the
+# row check: a read that reached Postgres was filtered by the policy on the
+# way, or would have been on a connection the policy applies to.
+_ISSUED_A_STATEMENT = "switch_issued_a_statement"
+
class TenantBindingDriftError(RuntimeError):
"""A tenant was bound after this session's transaction had already begun.
@@ -97,6 +122,7 @@ def register_tenant_session_hook() -> None:
event.listens_for(Session, "after_begin")(_set_tenant_on_begin)
event.listens_for(Session, "after_transaction_end")(_forget_the_stamp)
event.listens_for(Session, "do_orm_execute")(_refuse_a_binding_that_came_too_late)
+ event.listens_for(Session, "do_orm_execute")(_note_that_a_statement_was_issued)
# `do_orm_execute` does not cover a flush: the INSERTs and UPDATEs the unit
# of work emits go to the connection directly rather than through
# `Session.execute`, so a late binding followed only by `session.add(...)`
@@ -162,6 +188,11 @@ def _refuse_a_binding_that_came_too_late(state: ORMExecuteState) -> None:
_refuse_drift(state.session)
+def _note_that_a_statement_was_issued(state: ORMExecuteState) -> None:
+ """Record that this session went to the database, for `get` to read back."""
+ state.session.info[_ISSUED_A_STATEMENT] = True
+
+
def _refuse_a_flush_that_drifted(
session: Session, flush_context: object, instances: object
) -> None:
@@ -188,4 +219,127 @@ def _refuse_drift(session: Session) -> None:
)
+class CrossTenantIdentityMapError(RuntimeError):
+ """`Session.get` answered with a row belonging to a different tenant.
+
+ Not the same failure as `TenantBindingDriftError`, and not caught by it.
+ Every check above is reached by *going to the database* — `do_orm_execute`
+ fires on a statement, `before_flush` on a unit of work — and both of those
+ are also where the policy would refuse a cross-tenant read even if the
+ check did not. `Session.get` answered from the identity map does neither.
+ It matches the primary key against objects the session has already loaded
+ and returns one, with no statement, no flush, no transaction and so no
+ policy: the two mechanisms that make this design work are both downstream
+ of a round trip that never happens.
+
+ So this is the one read in the process that a rebinding can quietly walk
+ past, and it gets a check of its own — on the row rather than on the
+ transaction, since the object in hand carries the answer directly.
+ """
+
+
+def _tenant_of(instance: object) -> str | None:
+ """The tenant `instance` belongs to, or None if its table carries none.
+
+ The same two cases `db/rls_ddl._tenant_column` derives the policy from, and
+ deliberately so: a check that disagreed with the policy about which column
+ identifies the tenant would refuse rows the database allows, or allow rows
+ it refuses. `tenants` is compared on `id`, because a tenant *is* the
+ boundary rather than belonging to one; everything else on `tenant_id`.
+ """
+ if getattr(type(instance), "__tablename__", None) == "tenants":
+ identifier = getattr(instance, "id", None)
+ return None if identifier is None else str(identifier)
+ tenant_id = getattr(instance, "tenant_id", None)
+ return None if tenant_id is None else str(tenant_id)
+
+
+def _refuse_a_row_from_another_tenant(instance: object) -> None:
+ """Raise when `instance` belongs to a tenant other than the bound one.
+
+ Compares the row against what is bound now rather than against what the
+ transaction was stamped with, because the failure being caught here does
+ not need a transaction to exist: a session that has committed still holds
+ its objects (`expire_on_commit=False`), so a `get` after the commit finds
+ them with no stamp to compare against at all.
+
+ Deliberately silent when nothing is bound. A read with no tenant bound is
+ a read the policies refuse, so an identity-map hit in that state is
+ equally suspect — but there is no tenant to say the row is *not* in, and
+ the honest boundary here is a comparison rather than a guess. What is left
+ uncovered is written down in `docs/old/multi-tenancy-phase1-db.md` rather
+ than half-checked.
+ """
+ tenant_id = _tenant_of(instance)
+ if tenant_id is None:
+ # A global table (`users`, `oidc_identities`, `feature_flags`) carries
+ # no tenant, so there is nothing to compare and nothing to refuse.
+ return
+ bound = current_tenant_id()
+ if bound is None or bound == tenant_id:
+ return
+ raise CrossTenantIdentityMapError(
+ f"Session.get answered with a {type(instance).__name__} in tenant "
+ f"{tenant_id!r} while tenant {bound!r} is bound. The row was already "
+ "in this session's identity map, so it came back without a statement "
+ "and without the row-level-security policy that would have refused "
+ "it. The session is being reused across two tenants, which is what "
+ "`db/session_scope.tenant_session(factory, tenant_id)` exists to "
+ "prevent: open a session inside the binding and let it end with it, "
+ "rather than rebinding around a session that outlives the tenant it "
+ "was opened for."
+ )
+
+
+class TenantCheckedSession(Session):
+ """`Session`, with the tenant checks extended to the one path they missed.
+
+ Used through `db/engine.create_session_factory`'s `sync_session_class`, so
+ every session this process opens is one of these — including the async
+ ones, since `AsyncSession.get` delegates to the sync session underneath.
+
+ Subclassing rather than adding another event listener, because there is no
+ event to listen to: SQLAlchemy fires `do_orm_execute` when `get` has to
+ issue a statement and not when it answers from the identity map, which is
+ exactly the case that needs checking.
+
+ **The row check runs only when `get` issued no statement**, and that
+ restriction is the whole of its correctness. A `get` that reached the
+ database was filtered by the policy on the way — or would have been, on a
+ connection the policy applies to — so the row it returns is the bound
+ tenant's by construction and there is nothing to check. Applying the check
+ to those as well would impose the policy's semantics on the *owner*
+ connection, which this design deliberately does not do anywhere else: the
+ unit suite and every fan-out rely on an owner connection reading across
+ tenants (see the "filters what it reads back" note in
+ `db/tenant_lookup.py`), and a test arranging a second tenant's fixture rows
+ would start failing for a reason that has nothing to do with a leak.
+
+ Worth knowing about the hole this closes: SQLAlchemy's identity map holds
+ **weak** references, so an object is only reachable through it while
+ something else still holds a strong one. That makes the exposure narrower
+ than it first looks and, more importantly, makes it *intermittent* —
+ whether the leak happens depends on whether a local variable happens to
+ still be in scope. Intermittent is worse than reliable for something nobody
+ would think to test, which is the argument for checking rather than
+ reasoning about how reachable it is.
+ """
+
+ def get(self, *args: Any, **kwargs: Any) -> Any:
+ _refuse_drift(self)
+ self.info[_ISSUED_A_STATEMENT] = False
+ instance = super().get(*args, **kwargs)
+ if instance is not None and not self.info[_ISSUED_A_STATEMENT]:
+ _refuse_a_row_from_another_tenant(instance)
+ return instance
+
+ def get_one(self, *args: Any, **kwargs: Any) -> Any:
+ _refuse_drift(self)
+ self.info[_ISSUED_A_STATEMENT] = False
+ instance = super().get_one(*args, **kwargs)
+ if not self.info[_ISSUED_A_STATEMENT]:
+ _refuse_a_row_from_another_tenant(instance)
+ return instance
+
+
register_tenant_session_hook()
diff --git a/core/switch_core/gateway/app.py b/core/switch_core/gateway/app.py
index fb0e3fa96..ab24b4e3c 100644
--- a/core/switch_core/gateway/app.py
+++ b/core/switch_core/gateway/app.py
@@ -21,7 +21,6 @@ from switch_core.db.stores.external_user_store import ExternalUserStore
from switch_core.db.stores.room_group_store import RoomGroupStore
from switch_core.db.stores.room_store import RoomStore
from switch_core.db.stores.server_connector_store import ServerConnectorStore
-from switch_core.db.stores.tenant_member_store import TenantMemberStore
from switch_core.db.stores.user_store import UserStore
from switch_core.gateway.agents import router as agents_router
from switch_core.gateway.api_keys import router as api_keys_router
@@ -57,7 +56,6 @@ def create_gateway_app(
user_store: UserStore,
external_user_store: ExternalUserStore,
api_key_store: ApiKeyStore,
- tenant_member_store: TenantMemberStore,
resource_service: ResourceService,
protocol: ProtocolService,
config: SwitchConfig,
@@ -77,7 +75,6 @@ def create_gateway_app(
user_store=user_store,
external_user_store=external_user_store,
api_key_store=api_key_store,
- tenant_member_store=tenant_member_store,
resource_service=resource_service,
protocol=protocol,
config=config,
diff --git a/core/switch_core/gateway/auth.py b/core/switch_core/gateway/auth.py
index c4b329fce..4aa2dfa1c 100644
--- a/core/switch_core/gateway/auth.py
+++ b/core/switch_core/gateway/auth.py
@@ -14,16 +14,12 @@ from switch_core.authz import Action, Principal, require
from switch_core.config import SwitchConfig
from switch_core.db.models import Room, User
from switch_core.db.stores.room_store import RoomStore
-from switch_core.db.stores.tenant_member_store import (
- TenantMembershipError,
- TenantMemberStore,
-)
from switch_core.db.stores.user_store import UserStore
+from switch_core.db.tenant_lookup import tenants_of_user
from switch_core.gateway.dependencies import (
get_config,
get_session,
get_session_factory,
- get_tenant_member_store,
get_user_store,
)
from switch_core.logging_context import bind_log_context, unbind_log_context
@@ -93,10 +89,70 @@ def decode_jwt(token: str, secret_key: str) -> dict:
raise HTTPException(status_code=401, detail="Invalid token")
+class TenantMembershipError(Exception):
+ """A user has zero, or more than one, tenant memberships.
+
+ Phase 1 has exactly one tenant, so exactly one membership row is the only
+ correct state: zero means the user was never enrolled (a bug in whatever
+ created the account), and more than one is Phase 2's tenant-switching
+ shape arriving early. Either way this must not be resolved by picking
+ one — see `docs/old/multi-tenancy-phase1-db.md`, "Setting the tenant".
+
+ Raised, not returned, but not left to reach the client either:
+ `get_current_user` below turns it into a 403 naming no user id, so a
+ broken account gets an answer an operator can act on instead of a bare
+ 500.
+ """
+
+
+async def sole_tenant_id(
+ session_factory: async_sessionmaker[AsyncSession], user_id: str
+) -> str:
+ """The one tenant this user belongs to, raising if that isn't true.
+
+ Here rather than on a store, and that placement is the point. This is the
+ only caller there has ever been, and it is an authentication step: the
+ JWT names a person, and a person is global (`users` carries no tenant),
+ so something has to turn "who" into "which tenant" before a scoped session
+ can exist at all.
+
+ It goes through `tenants_of_user`, one of the seven `SECURITY DEFINER`
+ lookups that make up the whole exemption from row-level security
+ (`db/tenant_lookup.py`), which opens a session of its own with nothing
+ bound and answers with tenant ids and nothing else. `tenant_members` is a
+ scoped table, so a session cannot read the row that would tell it what to
+ be scoped to; a plain `select` here worked only because every environment
+ once connected as the tables' owner, and under the restricted runtime role
+ it is one of the reads the policy refuses.
+
+ That is also why this is a function and not an injected dependency. It
+ reaches the exemption with an arbitrary user id, and as a store on the
+ dependency graph any endpoint could declare it and do the same. The
+ exemption's audit surface is the list of modules that import a lookup
+ (`tests/switch_core/db/test_tenant_exemption_allowlist.py`), and a
+ reachable-from-anywhere wrapper made that list say less than it looked
+ like it said.
+
+ Refusing rather than picking, on both sides of "exactly one": zero
+ memberships is an account that was never enrolled, and two is a decision
+ about which tenant's data the caller is about to see. Neither is a thing
+ to guess at. Membership is written by exactly one function,
+ `UserStore.ensure_membership`, which is idempotent and derives the role
+ from the user — so "every account has exactly one" holds by construction
+ rather than by every writer remembering, and a second, unguarded way to
+ write one is how an account ends up with two.
+ """
+ tenant_ids = await tenants_of_user(session_factory, user_id)
+ if len(tenant_ids) != 1:
+ raise TenantMembershipError(
+ f"user {user_id} has {len(tenant_ids)} tenant memberships; expected exactly 1"
+ )
+ return tenant_ids[0]
+
+
async def _resolve_tenant_id(
session_factory: async_sessionmaker[AsyncSession],
user_store: UserStore,
- tenant_member_store: TenantMemberStore,
user_id: str,
) -> str:
"""Which tenant the JWT's subject belongs to, before one is bound.
@@ -122,7 +178,7 @@ async def _resolve_tenant_id(
if not await user_store.exists(system_session, user_id):
raise HTTPException(status_code=401, detail="User not found")
try:
- return await tenant_member_store.get_sole_tenant_id(session_factory, user_id)
+ return await sole_tenant_id(session_factory, user_id)
except TenantMembershipError as exc:
# Phase 1 has exactly one tenant, so anything but one membership is a
# provisioning bug, not a credential problem — but the caller still
@@ -146,7 +202,6 @@ async def get_current_user(
async_sessionmaker[AsyncSession], Depends(get_session_factory)
],
user_store: Annotated[UserStore, Depends(get_user_store)],
- tenant_member_store: Annotated[TenantMemberStore, Depends(get_tenant_member_store)],
config: Annotated[SwitchConfig, Depends(get_config)],
) -> AsyncIterator[User]:
"""Authenticate the caller, bind their tenant, and load them on the
@@ -166,9 +221,7 @@ async def get_current_user(
payload = decode_jwt(token, config.jwt_secret_key)
user_id = payload["sub"]
- tenant_id = await _resolve_tenant_id(
- session_factory, user_store, tenant_member_store, user_id
- )
+ tenant_id = await _resolve_tenant_id(session_factory, user_store, user_id)
with tenant_scope(tenant_id):
log_token = bind_log_context(user_id=user_id, tenant_id=tenant_id)
diff --git a/core/switch_core/gateway/dependencies.py b/core/switch_core/gateway/dependencies.py
index 4b1a61e05..3879e7384 100644
--- a/core/switch_core/gateway/dependencies.py
+++ b/core/switch_core/gateway/dependencies.py
@@ -23,7 +23,6 @@ from switch_core.db.stores.external_user_store import ExternalUserStore
from switch_core.db.stores.room_group_store import RoomGroupStore
from switch_core.db.stores.room_store import RoomStore
from switch_core.db.stores.server_connector_store import ServerConnectorStore
-from switch_core.db.stores.tenant_member_store import TenantMemberStore
from switch_core.db.stores.user_store import UserStore
from switch_core.room_service import RoomService
from switch_core.rooms_yaml import RoomYamlService
@@ -47,7 +46,6 @@ def init_dependencies(
user_store: UserStore,
external_user_store: ExternalUserStore,
api_key_store: ApiKeyStore,
- tenant_member_store: TenantMemberStore,
resource_service: ResourceService,
protocol: ProtocolService,
config: SwitchConfig,
@@ -66,7 +64,6 @@ def init_dependencies(
_state["user_store"] = user_store
_state["external_user_store"] = external_user_store
_state["api_key_store"] = api_key_store
- _state["tenant_member_store"] = tenant_member_store
_state["resource_service"] = resource_service
_state["protocol"] = protocol
_state["config"] = config
@@ -184,10 +181,6 @@ def get_api_key_store() -> ApiKeyStore:
return _state["api_key_store"] # type: ignore[no-any-return]
-def get_tenant_member_store() -> TenantMemberStore:
- return _state["tenant_member_store"] # type: ignore[no-any-return]
-
-
def get_connector_lifecycle() -> ServerSideConnectorLifecycleService:
return _state["connector_lifecycle"] # type: ignore[no-any-return]
diff --git a/core/switch_core/main.py b/core/switch_core/main.py
index c595e1a87..55c0c29ac 100644
--- a/core/switch_core/main.py
+++ b/core/switch_core/main.py
@@ -76,6 +76,7 @@ from switch_core.clients.client_factory import ClientFactory
from switch_core.clients.client_lifecycle_service import ClientLifecycleService
from switch_core.config import SwitchConfig
from switch_core.crypto import encrypt_token
+from switch_core.db.boot_lock import boot_lock
from switch_core.db.engine import (
create_engine_from_config,
create_session_factory,
@@ -107,7 +108,6 @@ from switch_core.db.stores.room_role_store import RoomRoleStore
from switch_core.db.stores.room_store import RoomStore
from switch_core.db.stores.server_connector_store import ServerConnectorStore
from switch_core.db.stores.task_store import TaskStore
-from switch_core.db.stores.tenant_member_store import TenantMemberStore
from switch_core.db.stores.tenant_store import TenantStore
from switch_core.db.stores.user_store import UserStore
from switch_core.db.tenant_lookup import all_tenant_ids
@@ -251,7 +251,11 @@ async def _check_tenant_isolation(config: SwitchConfig, engine: AsyncEngine) ->
async def run(config: SwitchConfig) -> None:
# ── Database ─────────────────────────────────────────────────────────────
- await _prepare_database(config)
+ # The migration and the runtime role's grant re-issue already ran, under
+ # Switch's boot-time advisory lock, before this coroutine was ever started
+ # — see `main._migrate_and_grant`, which `main()` awaits first. Both use
+ # the schema owner's connection where one is configured, and neither
+ # belongs on the pooled application engine built below.
engine = create_engine_from_config(config)
await _check_tenant_isolation(config, engine)
session_factory = create_session_factory(engine)
@@ -276,7 +280,6 @@ async def run(config: SwitchConfig) -> None:
bridge_message_map_store = BridgeMessageMapStore()
user_store = UserStore()
api_key_store = ApiKeyStore()
- tenant_member_store = TenantMemberStore()
tenant_store = TenantStore()
reference_store = ReferenceStore()
reference_type_store = ReferenceTypeStore()
@@ -289,10 +292,20 @@ async def run(config: SwitchConfig) -> None:
media_store = MediaStore()
# ── Seed admin user + agent-registration bootstrap key ──────────────────
- await _seed_admin_user(session_factory, user_store, config)
- await _seed_agent_registration_bootstrap_key(
- session_factory, user_store, api_key_store, agent_store, config
- )
+ # A second acquisition of the boot lock, distinct from the one around the
+ # migration in `main._migrate_and_grant`: nothing between the two needs
+ # another replica kept out, so there is nothing to gain from holding one
+ # lock across the whole of boot instead of two narrower ones. This one
+ # cannot be a transaction-scoped `pg_advisory_xact_lock` the way a single
+ # migration transaction could be, though — the seeding below reads its own
+ # state and writes it back across several separate sessions (see
+ # `_seed_agent_registration_bootstrap_key`'s docstring), so only a
+ # session-level lock, held for the whole span, actually serialises it.
+ async with boot_lock(config):
+ await _seed_admin_user(session_factory, user_store, config)
+ await _seed_agent_registration_bootstrap_key(
+ session_factory, user_store, api_key_store, agent_store, config
+ )
# ── Event queue + request trackers ───────────────────────────────────────
event_buffer = EventBuffer()
@@ -474,7 +487,6 @@ async def run(config: SwitchConfig) -> None:
user_store=user_store,
external_user_store=external_user_store,
api_key_store=api_key_store,
- tenant_member_store=tenant_member_store,
resource_service=resource_service,
protocol=protocol,
config=config,
@@ -1017,6 +1029,50 @@ async def _shutdown(
os._exit(0)
+async def _migrate_and_grant(config: SwitchConfig) -> None:
+ """Apply pending migrations and reissue the runtime role's grants.
+
+ Migrations still run at boot, on the same schedule as before — what
+ changed is the connection they run on, and the lock now held around both
+ this and the grant re-issue. `migrations/env.py` points Alembic at
+ DB_OWNER_USER where one is configured, because DDL is exactly what the
+ runtime role is not allowed to issue. Where none is, this is unchanged
+ from before and runs as DB_USER, which is right for a developer pointing
+ at a scratch database and fails loudly and immediately for a deployment
+ that has moved to a restricted role without saying who its owner is.
+
+ Held under `db/boot_lock.boot_lock` because a rolling deploy, or a crash
+ racing a restart, starts more than one replica of this process at once,
+ and neither step here tolerates two replicas doing it at the same time:
+ two concurrent `alembic upgrade head` runs contend on the same catalogue
+ locks Postgres itself takes for DDL — the usual result is a deadlock or a
+ "duplicate object" error, not one side quietly winning — and
+ `_prepare_database`'s grant re-issue right after it touches
+ `pg_default_acl`, which two concurrent `ALTER DEFAULT PRIVILEGES`
+ statements contend on the same way. One acquisition covers both rather
+ than two: a table a migration just added is usable only once the grant
+ after it has run, so nothing is served by letting a second replica in
+ between them, and holding the lock across both is simpler than justifying
+ why it would be safe to drop in the gap.
+
+ `alembic_command.upgrade` is synchronous, and `migrations/env.py` calls
+ `asyncio.run` internally to drive its own async engine when it isn't
+ offline — which raises if called from a thread that already has a running
+ event loop. This coroutine has one, so the upgrade runs via
+ `asyncio.to_thread`, on a worker thread that starts with no event loop of
+ its own, which is exactly what that inner `asyncio.run` needs.
+ """
+ alembic_ini = Path(__file__).resolve().parent.parent / "alembic.ini"
+ alembic_cfg = AlembicConfig(str(alembic_ini))
+ async with boot_lock(config):
+ await asyncio.to_thread(alembic_command.upgrade, alembic_cfg, "head")
+ await _prepare_database(config)
+ logger.info(
+ "Database migrations applied as %s",
+ config.db_owner_user or config.db_user,
+ )
+
+
def main() -> None:
config = SwitchConfig()
running_version = switch_core_version()
@@ -1024,20 +1080,7 @@ def main() -> None:
logger.info("Starting switch-core %s", running_version or "(version unknown)")
- # Migrations still run at boot, and on the same schedule as before — what
- # changed is the connection they run on. `migrations/env.py` points Alembic
- # at DB_OWNER_USER where one is configured, because DDL is exactly what the
- # runtime role is not allowed to issue. Where none is, this is unchanged
- # from before and runs as DB_USER, which is right for a developer pointing
- # at a scratch database and fails loudly and immediately for a deployment
- # that has moved to a restricted role without saying who its owner is.
- alembic_ini = Path(__file__).resolve().parent.parent / "alembic.ini"
- alembic_cfg = AlembicConfig(str(alembic_ini))
- alembic_command.upgrade(alembic_cfg, "head")
- logger.info(
- "Database migrations applied as %s",
- config.db_owner_user or config.db_user,
- )
+ asyncio.run(_migrate_and_grant(config))
asyncio.run(run(config))
diff --git a/core/switch_core/migrations/versions/9c41a7b0e5d8_tenant_lookup_functions.py b/core/switch_core/migrations/versions/9c41a7b0e5d8_tenant_lookup_functions.py
index eefd3b59f..7b7f21be5 100644
--- a/core/switch_core/migrations/versions/9c41a7b0e5d8_tenant_lookup_functions.py
+++ b/core/switch_core/migrations/versions/9c41a7b0e5d8_tenant_lookup_functions.py
@@ -39,13 +39,18 @@ than a thing to check per function.
Postgres note on writing `SECURITY DEFINER` functions safely: a role able to
create a temporary table could otherwise shadow one of the tables read here.
-No grants and no roles are created here either. `EXECUTE` defaults to `PUBLIC`
-for a new function, which is what makes these callable by the runtime role
-without this migration having to know its name — and the most a caller can
-learn from any of them is the tenant an identifier it already holds belongs
-to. The runtime role's table grants are re-issued at boot by the owner
-connection (`db/runtime_role.py`), after this migration has run, so a table a
-later revision adds is granted without a runbook step.
+No grants and no roles are created here either, and the `EXECUTE` these
+functions are created with is not a property to lean on. A new function is
+`PUBLIC`-executable by default, which makes them callable by the runtime role
+without this migration having to know its name — and callable by every other
+role with `CONNECT` on this database just as readily, which is not the same
+thing and is not wanted. Narrowing it needs a role name, which only boot has:
+`db/runtime_role.grant_runtime_role` runs on the owner connection immediately
+after this migration, revokes `EXECUTE` on the schema's functions from
+`PUBLIC` and grants it to the runtime role by name, and the boot self-check
+beside it refuses to serve if `PUBLIC` ever holds it again. The same pass
+re-issues the role's table grants, so a table a later revision adds is granted
+without a runbook step.
The DDL below is a verbatim copy of `switch_core/db/tenant_lookup.py` as it
stood when this migration was written, copied rather than imported for the
diff --git a/core/switch_core/migrations/versions/b1d7c4f0a92e_drop_tenant_of_client.py b/core/switch_core/migrations/versions/b1d7c4f0a92e_drop_tenant_of_client.py
new file mode 100644
index 000000000..0b749ff7a
--- /dev/null
+++ b/core/switch_core/migrations/versions/b1d7c4f0a92e_drop_tenant_of_client.py
@@ -0,0 +1,60 @@
+"""drop tenant_of_client, the one lookup its callers never needed
+
+`9c41a7b0e5d8` installed eight `SECURITY DEFINER` functions as the whole
+exemption from row-level security. This drops one of them.
+
+`tenant_of_client(client_id)` answered "which tenant is this room client in",
+and it was the most-called of the eight: once per transport, on every client
+in the process, plus once more in `AgentClient.start`. Both callers were
+handed a `clients` row to begin with — `ClientFactory.create(record)` builds
+the client, and `record.tenant_id` is a column on that row — so the question
+went to the database with the answer already in hand. `ClientBase` and
+`PostgresTransport` now take a `tenant_id` alongside the `client_id` they
+already took, and nothing calls the function.
+
+Dropping it rather than leaving it installed is the point of the change.
+Every function in this set is callable by whoever holds the runtime role's
+credentials, and it answers a question about a tenant boundary; one that no
+code needs is still one an operator would have to audit, and the argument for
+the exemption is that its inventory is short enough to read. A lookup called
+on every transport is the one worth not having.
+
+The downgrade recreates it, verbatim as `9c41a7b0e5d8` wrote it, so a rollback
+past this revision lands on a schema that revision would recognise.
+
+Revision ID: b1d7c4f0a92e
+Revises: 9c41a7b0e5d8
+Create Date: 2026-09-10 00:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "b1d7c4f0a92e"
+down_revision: str | None = "9c41a7b0e5d8"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+# Frozen, like everything else in this chain: the text `9c41a7b0e5d8` ran, so
+# a downgrade puts back exactly what it created rather than whatever
+# `db/tenant_lookup.py` would build today — which, this revision having
+# landed, would build nothing at all.
+SECURE_SEARCH_PATH = "pg_catalog, public, pg_temp"
+
+CREATE_TENANT_OF_CLIENT = f"""CREATE OR REPLACE FUNCTION tenant_of_client(p_client_id text)
+ RETURNS SETOF text
+ LANGUAGE sql STABLE SECURITY DEFINER
+ SET search_path = {SECURE_SEARCH_PATH}
+AS $$SELECT tenant_id FROM clients WHERE id = p_client_id$$"""
+
+DROP_TENANT_OF_CLIENT = "DROP FUNCTION IF EXISTS tenant_of_client(text)"
+
+
+def upgrade() -> None:
+ op.execute(DROP_TENANT_OF_CLIENT)
+
+
+def downgrade() -> None:
+ op.execute(CREATE_TENANT_OF_CLIENT)
diff --git a/core/switch_core/transport/postgres.py b/core/switch_core/transport/postgres.py
index 9e83b0a2f..fe4574647 100644
--- a/core/switch_core/transport/postgres.py
+++ b/core/switch_core/transport/postgres.py
@@ -43,7 +43,6 @@ from typing import TYPE_CHECKING, Any
from switch_core.attachments import ATTACHMENT_GROUP_KEY
from switch_core.db.models import ClientRoom, MediaBlob, Message, MessageAttachment
from switch_core.db.session_scope import tenant_session
-from switch_core.db.tenant_lookup import tenant_of_client
from switch_core.messages.recorded_types import EPHEMERAL
from switch_core.messages.row import attachments_in, text_field, thread_root_of
from switch_core.tenant_context import no_tenant, tenant_scope
@@ -106,6 +105,7 @@ class PostgresTransport:
*,
user_id: str,
client_id: str,
+ tenant_id: str,
display_name: str,
session_factory: async_sessionmaker[AsyncSession],
room_store: RoomStore,
@@ -117,6 +117,20 @@ class PostgresTransport:
) -> None:
self.user_id = user_id
self.client_id = client_id
+ # The tenant everything this transport reads and writes is scoped to,
+ # handed in with the client id rather than looked up from it. Neither
+ # this transport's context nor its caller's is the right answer — the
+ # client's task binds nothing and its handlers bind some room's tenant
+ # — but the client row that named this `client_id` also named its
+ # tenant, so whoever built the transport already had it.
+ #
+ # It is the whole answer, not a starting point: `client_rooms` carries
+ # composite foreign keys to both `clients` and `rooms`, so every room
+ # this client can be a member of is in this tenant, and a room lookup
+ # scoped to it cannot return a subset. That is what makes
+ # `joined_rooms` safe to scope — see its docstring for the failure
+ # this replaces.
+ self.tenant_id = tenant_id
self.display_name = display_name
self._session_factory = session_factory
self._room_store = room_store
@@ -142,13 +156,6 @@ class PostgresTransport:
# A room's transport id never changes, so this only grows and never
# goes stale.
self._room_ids: dict[str, str] = {}
- # This client's own tenant, resolved once from its row and then held.
- # Not a guess and not ambient: `client_rooms` carries composite foreign
- # keys to both `clients` and `rooms`, so every room this client can be
- # a member of is in this tenant, and a room lookup scoped to it cannot
- # return a subset. That is what makes `joined_rooms` safe to scope —
- # see its docstring for the failure this replaces.
- self._tenant_id: str | None = None
# ── Session ───────────────────────────────────────────────────────────────
@@ -354,7 +361,7 @@ class PostgresTransport:
# per room. This used to be a per-room map filled from each room's own
# row; the map only ever held this value, and two sources for one fact
# is how they come to disagree.
- tenant_id = await self._tenant()
+ tenant_id = self.tenant_id
# Bound for the read *and* the delivery below: a handler (posting to a
# bridge, gating a command) opens its own sessions rather than reusing
# this one, and those still need the room's tenant. The contextvar
@@ -536,9 +543,7 @@ class PostgresTransport:
# row has to name a tenant that means something. It is the same answer
# a room's tenant would give — a client is only in rooms of its own —
# and it does not depend on there being a room in hand.
- async with tenant_session(
- self._session_factory, await self._tenant()
- ) as session:
+ async with tenant_session(self._session_factory, self.tenant_id) as session:
await self._media_store.put(
session,
MediaBlob(
@@ -559,9 +564,7 @@ class PostgresTransport:
bytes: an attachment the sender was told had been stored and a reader
gets back as a zero-byte file is the worst of the available answers.
"""
- async with tenant_session(
- self._session_factory, await self._tenant()
- ) as session:
+ async with tenant_session(self._session_factory, self.tenant_id) as session:
blob = await self._media_store.get(session, uri)
if blob is None:
raise TransportError(f"No media stored under {uri}")
@@ -655,16 +658,15 @@ class PostgresTransport:
async def joined_rooms(self) -> list[str]:
"""The transport-side ids of this client's rooms.
- Scoped to this client's own tenant, which is *derived* rather than
+ Scoped to this client's own tenant, which is *carried* rather than
inherited — the distinction the earlier version of this method was
right to insist on. It runs once, at the top of `receive_forever`, in
a task that binds nothing, so a tenant taken from whatever created the
task would return a subset with no error to say so, and the client
- would be silently deaf in every room it did not see, forever. That is
- still true. What has changed is where the tenant comes from: not from
- the caller's context but from this client's own row, through the
- exemption (`db/tenant_lookup.py`), which is the same answer the read
- itself would have given.
+ would be silently deaf in every room it did not see, forever. The
+ tenant this reads under comes from the client row the transport was
+ built from, which is the same answer the read itself would have given
+ and one nobody has to be in the right context to have.
The subset it cannot be is guaranteed by the schema rather than
assumed: `client_rooms` carries composite foreign keys to both
@@ -672,9 +674,7 @@ class PostgresTransport:
its own tenant. `tests/switch_core/transport/test_postgres_transport.py`
pins that, because the shape of this list rests on it.
"""
- async with tenant_session(
- self._session_factory, await self._tenant()
- ) as session:
+ async with tenant_session(self._session_factory, self.tenant_id) as session:
rooms = await self._room_store.get_for_client(session, self.client_id)
return [room.matrix_room_id for room in rooms if room.matrix_room_id]
@@ -714,7 +714,7 @@ class PostgresTransport:
both narrower and exact. Once cached, resolving costs nothing: no
session, no query, no round trip.
"""
- tenant_id = await self._tenant()
+ tenant_id = self.tenant_id
cached = self._room_ids.get(transport_room_id)
if cached is not None:
return cached, tenant_id
@@ -722,25 +722,6 @@ class PostgresTransport:
room_id = await self._resolve_room(session, transport_room_id)
return room_id, tenant_id
- async def _tenant(self) -> str:
- """This client's tenant, resolved once from its own row.
-
- Its own row rather than its context: this runs inside the client's
- long-lived task, which unbinds on purpose, and from handlers that have
- some *room's* tenant bound. Neither is the question being asked. The
- client id is a primary key, so the exemption
- (`db/tenant_lookup.py`) answers it exactly.
- """
- if self._tenant_id is None:
- tenant_id = await tenant_of_client(self._session_factory, self.client_id)
- if tenant_id is None:
- raise TransportError(
- f"Client {self.client_id} has no row, so its transport has "
- "no tenant to act in."
- )
- self._tenant_id = tenant_id
- return self._tenant_id
-
def to_inbound(
row: Message,
diff --git a/core/tests/conftest.py b/core/tests/conftest.py
index f0c70cd84..d15cdcc55 100644
--- a/core/tests/conftest.py
+++ b/core/tests/conftest.py
@@ -209,4 +209,17 @@ async def rls_harness(postgres_url: str) -> AsyncIterator[RLSHarness]:
# ROLE` succeed right after.
await conn.execute(text(f'DROP OWNED BY "{role}"'))
await conn.execute(text(f'DROP ROLE "{role}"'))
+ # `grant_runtime_role` also revokes PUBLIC's default EXECUTE on
+ # functions, and that one is recorded against the *owner* rather
+ # than against the throwaway role — so `DROP OWNED BY` above does
+ # not touch it and it outlives this test, governing every schema
+ # every later test in the session builds. Left in place it makes
+ # the checks about it pass for the wrong reason: the second test
+ # to ask whether a new function is PUBLIC-executable would be
+ # answered by the first test's leftover rather than by the grants
+ # under test. Granting it back restores the built-in default,
+ # which Postgres records by deleting the row.
+ await conn.execute(
+ text("ALTER DEFAULT PRIVILEGES GRANT EXECUTE ON FUNCTIONS TO PUBLIC")
+ )
await owner_engine.dispose()
diff --git a/core/tests/switch_core/bridges/collaboration/test_lifecycle_tenant_binding.py b/core/tests/switch_core/bridges/collaboration/test_lifecycle_tenant_binding.py
index a72c6000e..1299bd153 100644
--- a/core/tests/switch_core/bridges/collaboration/test_lifecycle_tenant_binding.py
+++ b/core/tests/switch_core/bridges/collaboration/test_lifecycle_tenant_binding.py
@@ -365,7 +365,7 @@ async def test_a_host_resource_conflict_is_looked_for_across_every_tenant(
service.register_adapter("teams", _PortHoldingAdapter, _StubConfig)
with tenant_scope(newcomer_tenant):
- with pytest.raises(ValueError, match="already uses port:3979"):
+ with pytest.raises(ValueError, match="port:3979 is already claimed"):
await service._reject_resource_conflict("teams", {"listen_port": 3979})
assert sorted(seen) == sorted(
@@ -422,3 +422,149 @@ async def test_a_tenant_scoped_read_of_the_same_data_would_have_missed_the_confl
"bridge — row-level security should have hidden it, which is what "
"the application code avoids by calling get_all unscoped"
)
+
+
+class _PortHoldingAdapter(_StubAdapter):
+ @staticmethod
+ def exclusive_resource(config: dict[str, Any]) -> str | None:
+ port = config.get("listen_port")
+ return f"port:{port}" if port else None
+
+
+async def test_a_same_tenant_conflict_names_the_incumbent_bridge(
+ session_factory: async_sessionmaker[AsyncSession],
+) -> None:
+ """The exclusivity check still has to be *actionable* for the ordinary
+ case: an operator who registers a second Teams bridge on a port their own
+ first one already holds needs to be told which of their own bridges to
+ delete or move. They can already see its name on their own bridge list, so
+ naming it here discloses nothing they could not already read."""
+ tenant = f"tenant-{uuid.uuid4().hex[:8]}"
+ async with session_factory() as session:
+ await _make_tenant(session, tenant)
+ client = Client(
+ tenant_id=tenant,
+ matrix_user_id=f"@bridge-{uuid.uuid4().hex[:8]}:test",
+ display_name="bridge client",
+ type="bridge",
+ )
+ session.add(client)
+ await session.flush()
+ session.add(
+ CollaborationBridge(
+ tenant_id=tenant,
+ type="teams",
+ display_name="Our Existing Teams",
+ client_id=client.id,
+ status="active",
+ connection_config={"listen_port": 3979},
+ )
+ )
+ await session.commit()
+
+ service = _service(session_factory)
+ service.register_adapter("teams", _PortHoldingAdapter, _StubConfig)
+
+ with tenant_scope(tenant):
+ with pytest.raises(ValueError) as excinfo:
+ await service._reject_resource_conflict("teams", {"listen_port": 3979})
+
+ assert "Our Existing Teams" in str(excinfo.value)
+
+
+async def test_a_cross_tenant_conflict_does_not_name_the_incumbent_or_its_tenant(
+ session_factory: async_sessionmaker[AsyncSession],
+) -> None:
+ """The other half of finding CHOO-2623's cross-tenant disclosure: refusing
+ the conflict is still required (two tenants cannot share a Teams port or a
+ Slack workspace), but the message reaching an unrelated tenant must not
+ leak who holds the resource or that they exist on this instance at all.
+ `wanted` (`port:3979`) is not itself a disclosure — it is an echo of the
+ connection_config the caller just submitted — but the incumbent's display
+ name and tenant id must appear nowhere in the raised message.
+ """
+ incumbent_tenant = f"tenant-{uuid.uuid4().hex[:8]}"
+ newcomer_tenant = f"tenant-{uuid.uuid4().hex[:8]}"
+ async with session_factory() as session:
+ await _make_tenant(session, incumbent_tenant)
+ await _make_tenant(session, newcomer_tenant)
+ client = Client(
+ tenant_id=incumbent_tenant,
+ matrix_user_id=f"@bridge-{uuid.uuid4().hex[:8]}:test",
+ display_name="Their Secret Teams",
+ type="bridge",
+ )
+ session.add(client)
+ await session.flush()
+ session.add(
+ CollaborationBridge(
+ tenant_id=incumbent_tenant,
+ type="teams",
+ display_name="Their Secret Teams",
+ client_id=client.id,
+ status="active",
+ connection_config={"listen_port": 3979},
+ )
+ )
+ await session.commit()
+
+ service = _service(session_factory)
+ service.register_adapter("teams", _PortHoldingAdapter, _StubConfig)
+
+ with tenant_scope(newcomer_tenant):
+ with pytest.raises(ValueError) as excinfo:
+ await service._reject_resource_conflict("teams", {"listen_port": 3979})
+
+ message = str(excinfo.value)
+ assert "Their Secret Teams" not in message
+ assert incumbent_tenant not in message
+ assert "already claimed" in message
+
+
[email protected]_ambient_tenant
+async def test_a_conflict_with_nothing_bound_fails_closed_to_the_non_disclosing_message(
+ session_factory: async_sessionmaker[AsyncSession],
+) -> None:
+ """A call reached with no tenant bound at all (no request behind it, if
+ one ever exists) must not be treated as though it were the incumbent's own
+ tenant purely because `None == None` would otherwise compare equal to an
+ unset `tenant_id` — there is no such column, but the principle is the
+ same one `exclude_bridge_id`'s own None-guard states a few lines above:
+ an absent value must not accidentally satisfy a check meant to require a
+ real match. Nothing bound must get the same refusal an unrelated tenant
+ gets, not the same one the incumbent's own tenant gets.
+ """
+ incumbent_tenant = f"tenant-{uuid.uuid4().hex[:8]}"
+ async with session_factory() as session:
+ await _make_tenant(session, incumbent_tenant)
+ client = Client(
+ tenant_id=incumbent_tenant,
+ matrix_user_id=f"@bridge-{uuid.uuid4().hex[:8]}:test",
+ display_name="Their Secret Teams",
+ type="bridge",
+ )
+ session.add(client)
+ await session.flush()
+ session.add(
+ CollaborationBridge(
+ tenant_id=incumbent_tenant,
+ type="teams",
+ display_name="Their Secret Teams",
+ client_id=client.id,
+ status="active",
+ connection_config={"listen_port": 3979},
+ )
+ )
+ await session.commit()
+
+ service = _service(session_factory)
+ service.register_adapter("teams", _PortHoldingAdapter, _StubConfig)
+
+ assert current_tenant_id() is None
+ with pytest.raises(ValueError) as excinfo:
+ await service._reject_resource_conflict("teams", {"listen_port": 3979})
+
+ message = str(excinfo.value)
+ assert "Their Secret Teams" not in message
+ assert incumbent_tenant not in message
+ assert "already claimed" in message
diff --git a/core/tests/switch_core/bridges/collaboration/test_teams_zero_config.py b/core/tests/switch_core/bridges/collaboration/test_teams_zero_config.py
index 0e53d1d00..53192baac 100644
--- a/core/tests/switch_core/bridges/collaboration/test_teams_zero_config.py
+++ b/core/tests/switch_core/bridges/collaboration/test_teams_zero_config.py
@@ -28,6 +28,7 @@ from switch_core.bridges.collaboration.teams.crypto import (
generate_encryption_keypair,
load_certificate_der_b64,
)
+from switch_core.tenant_context import tenant_scope
def _raw_config(**overrides: Any) -> dict[str, Any]:
@@ -454,16 +455,26 @@ async def _stored_bridge(**overrides: Any) -> Any:
async def test_second_teams_bridge_on_the_same_port_is_refused(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- """The failure this replaces was a bind error in a background task."""
+ """The failure this replaces was a bind error in a background task.
+
+ Registering from within the incumbent's own tenant (`tenant_scope`,
+ matching what `_stub_tenant_lookups` says every stored row belongs to) is
+ the realistic case this message is written for: the operator can already
+ see "Contoso Teams" on their own bridge list, so naming it here is what
+ turns the refusal into something actionable rather than a disclosure —
+ see `test_lifecycle_tenant_binding.py` for the cross-tenant case, where
+ naming it would be exactly that.
+ """
service = _service_with_existing([await _stored_bridge()], monkeypatch)
- with pytest.raises(ValueError) as excinfo:
- await service.register(
- bridge_type="teams",
- display_name="Second Teams",
- connection_config=_raw_config(),
- channel_creation_enabled=True,
- )
+ with tenant_scope(_STUB_TENANT):
+ with pytest.raises(ValueError) as excinfo:
+ await service.register(
+ bridge_type="teams",
+ display_name="Second Teams",
+ connection_config=_raw_config(),
+ channel_creation_enabled=True,
+ )
message = str(excinfo.value)
assert "Contoso Teams" in message
diff --git a/core/tests/switch_core/clients/test_client_construction.py b/core/tests/switch_core/clients/test_client_construction.py
index e6db41f70..0046050f1 100644
--- a/core/tests/switch_core/clients/test_client_construction.py
+++ b/core/tests/switch_core/clients/test_client_construction.py
@@ -26,6 +26,7 @@ from switch_core.clients.client_base import ClientBase, ClientConfig
def _base_kwargs() -> dict[str, Any]:
return {
"client_id": "client-1",
+ "tenant_id": "tenant-1",
"matrix_user_id": "@switch-agent-1:switch.local",
"display_name": "agent-one",
"session_factory": object(),
diff --git a/core/tests/switch_core/clients/test_client_tenant_plumbing.py b/core/tests/switch_core/clients/test_client_tenant_plumbing.py
new file mode 100644
index 000000000..09d217977
--- /dev/null
+++ b/core/tests/switch_core/clients/test_client_tenant_plumbing.py
@@ -0,0 +1,215 @@
+"""A client's tenant travels with the row it was built from (CHOO-2623).
+
+There was an eighth exempt lookup, `tenant_of_client`, and it was the most
+called of them: every `PostgresTransport` asked it once, and so did every
+`AgentClient.start`. Both were built from a `clients` row that names the
+tenant in a column, so the question went to the database with the answer
+already in hand — and a `SECURITY DEFINER` function nobody needs is still a
+function to audit and still one more thing the runtime role's credentials
+reach.
+
+So `ClientBase` and `PostgresTransport` take a `tenant_id` the way they take a
+`client_id`. That moves the risk rather than removing it: a lookup that
+resolves the tenant from a primary key cannot be given the wrong one, and a
+parameter can. These tests are what stands in for it.
+
+Three things are pinned:
+
+- **the factory**, which is the one place the value is read off a row, for
+ both the client and the transport it is handed;
+- **the write path**, because the freshest caller — a puppet minted
+ mid-conversation — passes a record that was flushed a moment ago rather than
+ read back, and `record.tenant_id` has to be populated by then;
+- **every call site**, by scanning for one that names a `client_id` and not a
+ tenant, since a constructor test proves nothing about the places that build
+ a client wrongly.
+"""
+
+from __future__ import annotations
+
+import ast
+import pathlib
+import uuid
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import MagicMock
+
+import pytest
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+import switch_core
+from switch_core.clients.client_base import ClientBase, ClientConfig
+from switch_core.clients.client_factory import ClientFactory
+from switch_core.clients.client_lifecycle_service import ClientLifecycleService
+from switch_core.db.models import TENANT_ZERO_ID, Client, Tenant
+from switch_core.db.stores.client_store import ClientStore
+from switch_core.db.stores.media_store import MediaStore
+from switch_core.db.stores.message_store import MessageStore
+from switch_core.db.stores.room_store import RoomStore
+from switch_core.db.stores.tenant_store import TenantStore
+from switch_core.tenant_context import tenant_scope
+from switch_core.transport.ephemeral import EphemeralBus
+from switch_core.transport.invites import InviteBus
+
+
+def _factory(session_factory: async_sessionmaker[AsyncSession]) -> ClientFactory:
+ factory = ClientFactory(
+ client_store=ClientStore(),
+ session_factory=session_factory,
+ config=SimpleNamespace(matrix_server_name="test"), # type: ignore[arg-type]
+ room_store=RoomStore(),
+ message_store=MessageStore(),
+ media_store=MediaStore(),
+ listener=MagicMock(),
+ invites=InviteBus(),
+ ephemeral=EphemeralBus(),
+ )
+ factory.register("user", ClientBase)
+ return factory
+
+
+def _record(tenant_id: str) -> Client:
+ suffix = uuid.uuid4().hex[:8]
+ return Client(
+ id=f"client-{suffix}",
+ tenant_id=tenant_id,
+ matrix_user_id=f"@puppet-{suffix}:test",
+ display_name="a puppet",
+ type="user",
+ )
+
+
+class TestTheFactoryReadsItOffTheRow:
+ def test_a_client_gets_its_own_rows_tenant(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ tenant_id = f"tenant-{uuid.uuid4().hex[:8]}"
+
+ client = _factory(session_factory).create(_record(tenant_id))
+
+ assert client.tenant_id == tenant_id
+
+ def test_the_transport_gets_the_clients_tenant(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """`transport_for` is public because a collaboration bridge builds its
+ own client, so this is the seam every transport in the process comes
+ through — and the one place a transport could be handed the wrong
+ tenant for every read and write it will ever do."""
+ tenant_id = f"tenant-{uuid.uuid4().hex[:8]}"
+ factory = _factory(session_factory)
+
+ client = factory.create(_record(tenant_id))
+ transport = factory.transport_for(client)
+
+ assert transport.tenant_id == tenant_id # type: ignore[attr-defined]
+
+ def test_a_client_cannot_be_built_without_one(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """Required rather than defaulted. A client that fell back to whatever
+ was ambient would be silently correct in the common case — boot, one
+ tenant — and silently wrong for the puppet a bridge mints while
+ handling another tenant's message."""
+ with pytest.raises(TypeError):
+ ClientBase( # type: ignore[call-arg]
+ client_id="c",
+ matrix_user_id="@a:test",
+ display_name="a client",
+ session_factory=session_factory,
+ client_store=ClientStore(),
+ config=ClientConfig(),
+ transport_factory=lambda client: MagicMock(),
+ )
+
+
+class TestAFreshlyWrittenRowAlreadyCarriesIt:
+ """`create_client` writes the row and hands it straight to the factory.
+
+ A puppet is created and started inside one inbound message, so the record
+ the factory sees has been flushed rather than read back. `tenant_id` is a
+ Python-side default (`db/models.py`'s `TenantScoped`) applied at flush, and
+ the session factory does not expire on commit, so the value is there — but
+ "is there" is exactly the kind of thing that is true until a `SessionMaker`
+ argument changes, so it is asserted rather than assumed.
+ """
+
+ async def test_the_record_it_returns_names_the_bound_tenant(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ tenant_id = f"tenant-{uuid.uuid4().hex[:8]}"
+ async with session_factory() as session:
+ session.add(Tenant(id=tenant_id, slug=tenant_id, name=tenant_id))
+ await session.commit()
+
+ service = ClientLifecycleService(
+ matrix_admin=MagicMock(),
+ client_store=ClientStore(),
+ tenant_store=TenantStore(),
+ client_factory=MagicMock(),
+ session_factory=session_factory,
+ config=SimpleNamespace(matrix_server_name="test"), # type: ignore[arg-type]
+ )
+
+ with tenant_scope(tenant_id):
+ record = await service.create_client(
+ client_type="user", display_name="a puppet"
+ )
+
+ assert record.tenant_id == tenant_id
+
+ async def test_the_ambient_tenant_is_what_lands_on_the_row(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """The other direction, so the test above is not satisfied by a
+ constant: with tenant zero bound, that is what the row gets."""
+ service = ClientLifecycleService(
+ matrix_admin=MagicMock(),
+ client_store=ClientStore(),
+ tenant_store=TenantStore(),
+ client_factory=MagicMock(),
+ session_factory=session_factory,
+ config=SimpleNamespace(matrix_server_name="test"), # type: ignore[arg-type]
+ )
+
+ record = await service.create_client(
+ client_type="user", display_name="another puppet"
+ )
+
+ assert record.tenant_id == TENANT_ZERO_ID
+
+
+def test_no_call_site_names_a_client_without_naming_its_tenant() -> None:
+ """Catch the caller, which the constructor tests above cannot.
+
+ A client or a transport is built in five places — the factory, the
+ collaboration bridge's lifecycle service, and the tests that stand in for
+ them — and the failure of passing the wrong tenant is not an exception. It
+ is a client reading and writing in somebody else's tenant, or, once the
+ policies refuse it, a client silently in no rooms at all. Both are
+ invisible in review, so scan for the shape instead: anything constructing
+ a thing that takes a `client_id` has to say which tenant it is for.
+ """
+ package = pathlib.Path(switch_core.__file__).resolve().parent
+ offenders: list[str] = []
+
+ for path in package.rglob("*.py"):
+ tree = ast.parse(path.read_text(), filename=str(path))
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Call):
+ continue
+ func: Any = node.func
+ name = getattr(func, "id", None) or getattr(func, "attr", "")
+ if not (name.endswith("Client") or name.endswith("Transport")):
+ continue
+ passed = {keyword.arg for keyword in node.keywords if keyword.arg}
+ if "client_id" in passed and "tenant_id" not in passed:
+ offenders.append(f"{path.relative_to(package)}: {name}(client_id=…)")
+
+ assert offenders == [], (
+ "a client or transport is built without a tenant; these call sites "
+ f"name a client_id and no tenant_id: {offenders}. Take it off the "
+ "`clients` row the client_id came from — every caller is holding one "
+ "— rather than from whatever tenant happens to be bound, which for a "
+ "puppet minted mid-conversation is the wrong one."
+ )
diff --git a/core/tests/switch_core/db/test_frozen_ddl_matches_create_all.py b/core/tests/switch_core/db/test_frozen_ddl_matches_create_all.py
index 6d5e88332..e0e3913a9 100644
--- a/core/tests/switch_core/db/test_frozen_ddl_matches_create_all.py
+++ b/core/tests/switch_core/db/test_frozen_ddl_matches_create_all.py
@@ -158,9 +158,9 @@ def _tenant_lookups(connection: Connection) -> dict[str, tuple[Any, ...]]:
re-evaluating them per row, `proconfig` carries the `search_path` that
stops a temporary table shadowing what they read, and
`pg_get_function_arguments` catches an argument list that drifted. A
- migration whose frozen copy said `SECURITY INVOKER` would install eight
- functions unable to resolve a tenant, and every other test in the suite
- would stay green because none of them runs that SQL.
+ migration whose frozen copy said `SECURITY INVOKER` would install a set
+ of functions unable to resolve a tenant, and every other test in the
+ suite would stay green because none of them runs that SQL.
"""
rows = connection.execute(
text(
@@ -185,6 +185,27 @@ def _tenant_lookups(connection: Connection) -> dict[str, tuple[Any, ...]]:
}
+def _security_definer_functions(connection: Connection) -> set[str]:
+ """Every `SECURITY DEFINER` function in the schema, by name.
+
+ The comparison above asks about the lookups the live module still
+ names, so a function removed from the module and never dropped by a
+ revision is in neither snapshot's dictionary and passes unseen — while a
+ real deployment, whose schema Alembic built, still has it installed,
+ still `SECURITY DEFINER`, and still answering. Comparing the whole set
+ catches that: the migrated database must hold exactly what `create_all`
+ builds, no more.
+ """
+ rows = connection.execute(
+ text(
+ "SELECT p.proname FROM pg_proc p "
+ "JOIN pg_namespace n ON n.oid = p.pronamespace "
+ "WHERE n.nspname = 'public' AND p.prosecdef"
+ )
+ )
+ return {row.proname for row in rows}
+
+
def _trigger_def(connection: Connection, table: str, trigger: str) -> str:
row = connection.execute(
text(
@@ -203,6 +224,7 @@ class _DDLSnapshot:
rls_enabled: dict[str, bool]
require_tenant_id_body: str
tenant_lookups: dict[str, tuple[Any, ...]]
+ security_definer_functions: set[str]
notify_function_body: str
notify_trigger_def: str
@@ -213,6 +235,7 @@ def _snapshot(connection: Connection) -> _DDLSnapshot:
rls_enabled=_rls_enabled(connection),
require_tenant_id_body=_function_body(connection, REQUIRE_TENANT_FUNCTION_NAME),
tenant_lookups=_tenant_lookups(connection),
+ security_definer_functions=_security_definer_functions(connection),
notify_function_body=_function_body(connection, NOTIFY_FUNCTION_NAME),
notify_trigger_def=_trigger_def(connection, "messages", NOTIFY_TRIGGER_NAME),
)
@@ -298,6 +321,17 @@ async def test_migration_ddl_matches_the_live_copy(ddl_parity_urls: Any) -> None
f" migration: {migrated.require_tenant_id_body!r}\n"
f" create_all: {live.require_tenant_id_body!r}"
)
+ assert migrated.security_definer_functions == live.security_definer_functions, (
+ "the Alembic chain installs a different set of SECURITY DEFINER "
+ "functions than db/tenant_lookup.py builds. Every one of them is "
+ "exempt from row-level security, so an extra is an exemption nobody "
+ "is auditing and a missing one is a credential nothing can resolve. "
+ "A lookup taken out of the module needs a revision that drops it:\n"
+ f" only in the migrated database: "
+ f"{sorted(migrated.security_definer_functions - live.security_definer_functions)}\n"
+ f" only in create_all's: "
+ f"{sorted(live.security_definer_functions - migrated.security_definer_functions)}"
+ )
assert migrated.tenant_lookups == live.tenant_lookups, (
"the migration's frozen tenant lookups no longer match "
"db/tenant_lookup.py's live copy. These are the whole exemption from "
diff --git a/core/tests/switch_core/db/test_runtime_role.py b/core/tests/switch_core/db/test_runtime_role.py
index c854c0a7e..f1d348716 100644
--- a/core/tests/switch_core/db/test_runtime_role.py
+++ b/core/tests/switch_core/db/test_runtime_role.py
@@ -26,6 +26,7 @@ import uuid
import pytest
from sqlalchemy import text
from sqlalchemy.engine import make_url
+from sqlalchemy.exc import DBAPIError
from sqlalchemy.ext.asyncio import create_async_engine
from switch_core.db.runtime_role import (
@@ -133,7 +134,7 @@ class TestItRefusesEveryShapeThatWouldMakeThePoliciesInert:
"""`force row level security` sounds like more isolation and is less.
It takes the ownership exemption away from the schema owner, and the
- eight tenant lookups (`db/tenant_lookup.py`) are `SECURITY DEFINER`
+ seven tenant lookups (`db/tenant_lookup.py`) are `SECURITY DEFINER`
functions running as exactly that owner. Force it and nothing in the
process can answer "which tenant is this credential in", so no request
authenticates — a total outage arriving as a wall of 401s. Boot says
@@ -175,6 +176,15 @@ class TestItRefusesEveryShapeThatWouldMakeThePoliciesInert:
unrelated reason, and a check keyed on the code alone would read that
as "the policy refused it" and let boot proceed on a connection that
cannot serve a single request.
+
+ Two assertions, because two checks now stand between such a role and
+ the probe. `verify_restricted_role` refuses it earlier and more
+ precisely: the lookups are no longer `PUBLIC`-executable, so a role
+ that was granted nothing cannot call them, and boot says *that* rather
+ than reporting a suspicious read. The probe is then asked directly on
+ the same connection, because the trap it guards against is a property
+ of the probe rather than of the order the checks happen to run in —
+ and the order is exactly the sort of thing a later edit changes.
"""
role = f"switch_ungranted_{uuid.uuid4().hex[:12]}"
password = uuid.uuid4().hex
@@ -192,8 +202,13 @@ class TestItRefusesEveryShapeThatWouldMakeThePoliciesInert:
try:
with pytest.raises(RuntimeRoleError) as raised:
await verify_restricted_role(engine)
- assert "not with 42501" in str(raised.value)
- assert "missing its grants" in str(raised.value)
+ assert "cannot execute" in str(raised.value)
+
+ async with engine.connect() as connection:
+ with pytest.raises(RuntimeRoleError) as probed:
+ await _refuse_a_connection_no_policy_stops(connection, role)
+ assert "not with 42501" in str(probed.value)
+ assert "missing its grants" in str(probed.value)
finally:
await engine.dispose()
async with rls_harness.owner_engine.begin() as connection:
@@ -224,6 +239,135 @@ class TestItRefusesEveryShapeThatWouldMakeThePoliciesInert:
assert "all_tenant_ids" in str(raised.value)
+class TestTheExemptionBelongsToTheRuntimeRoleAndNotToEverybody:
+ """`EXECUTE` on a new function is granted to `PUBLIC`, and these are new
+ functions.
+
+ Left at that default the tenant lookups are callable by every role with
+ `CONNECT` on the database — a reporting role, a migration tool's role, a
+ person's. None of them can read a row of a scoped table, which is the
+ boundary that matters, but all of them could enumerate the deployment's
+ tenants and resolve any identifier they hold to one. `grant_runtime_role`
+ closes it and the self-check refuses to serve if it reopens.
+ """
+
+ async def test_a_second_role_with_connect_cannot_call_a_lookup(
+ self, rls_harness: RLSHarness, postgres_url: str
+ ) -> None:
+ """The property itself, measured rather than read off the catalogue.
+
+ A role created after `grant_runtime_role` has run, given nothing but
+ the `USAGE` any role needs to see the schema at all, tries the
+ enumeration and is refused. Asked of a *second* role because the
+ runtime role must still be able to call it, and a check that looked
+ only at one of them could pass by breaking the wrong thing.
+ """
+ role = f"switch_bystander_{uuid.uuid4().hex[:12]}"
+ password = uuid.uuid4().hex
+ async with rls_harness.owner_engine.begin() as connection:
+ await connection.execute(
+ text(
+ f"CREATE ROLE \"{role}\" LOGIN PASSWORD '{password}' "
+ "NOSUPERUSER NOCREATEDB NOCREATEROLE NOBYPASSRLS"
+ )
+ )
+ await connection.execute(text(f'GRANT USAGE ON SCHEMA public TO "{role}"'))
+ engine = create_async_engine(
+ make_url(postgres_url).set(username=role, password=password)
+ )
+ try:
+ async with engine.connect() as connection:
+ with pytest.raises(DBAPIError) as raised:
+ await connection.execute(
+ text("SELECT count(*) FROM all_tenant_ids()")
+ )
+ assert "permission denied for function all_tenant_ids" in str(raised.value)
+
+ # And the runtime role still can, which is the half that makes the
+ # refusal above a boundary rather than a broken schema.
+ async with rls_harness.restricted_engine.connect() as connection:
+ assert (
+ await connection.execute(
+ text("SELECT count(*) FROM all_tenant_ids()")
+ )
+ ).scalar_one() >= 1
+ finally:
+ await engine.dispose()
+ async with rls_harness.owner_engine.begin() as connection:
+ await connection.execute(text(f'DROP OWNED BY "{role}"'))
+ await connection.execute(text(f'DROP ROLE "{role}"'))
+
+ async def test_a_function_created_after_the_grants_is_not_public_executable(
+ self, rls_harness: RLSHarness
+ ) -> None:
+ """The gap between one boot and the next.
+
+ A revision adding an eighth lookup creates it `PUBLIC`-executable, and
+ the boot that would revoke that has not happened yet. The default
+ privileges cover the window — and they only do so because the revoke
+ is issued without `IN SCHEMA`: a per-schema default ACL is unioned
+ with the built-in one rather than replacing it, so the schema-scoped
+ spelling records nothing and the new function arrives
+ `PUBLIC`-executable regardless. Measured here, because that is not
+ something to take on trust from a statement that reports success.
+ """
+ async with rls_harness.owner_engine.begin() as connection:
+ await connection.execute(
+ text(
+ "CREATE FUNCTION tenant_of_something_new(p_id text) "
+ "RETURNS SETOF text LANGUAGE sql STABLE SECURITY DEFINER "
+ "SET search_path = pg_catalog, public, pg_temp "
+ "AS $$SELECT id FROM tenants WHERE id = p_id$$"
+ )
+ )
+ try:
+ async with rls_harness.restricted_engine.connect() as connection:
+ public_may, runtime_may = (
+ await connection.execute(
+ text(
+ "SELECT has_function_privilege("
+ " 'public', 'tenant_of_something_new(text)', 'EXECUTE'), "
+ "has_function_privilege("
+ " current_user, 'tenant_of_something_new(text)', 'EXECUTE')"
+ )
+ )
+ ).one()
+ assert not public_may, (
+ "a function created between two boots is PUBLIC-executable, so "
+ "an eighth lookup would be open to every role until the next "
+ "restart revoked it"
+ )
+ assert runtime_may, (
+ "the default privileges revoked PUBLIC's EXECUTE and did not "
+ "grant the runtime role's, so a new function would be callable "
+ "by nobody until the next boot"
+ )
+ finally:
+ async with rls_harness.owner_engine.begin() as connection:
+ await connection.execute(
+ text("DROP FUNCTION tenant_of_something_new(text)")
+ )
+
+ async def test_granting_execute_back_to_public_is_refused_at_boot(
+ self, rls_harness: RLSHarness
+ ) -> None:
+ """One statement reopens it, and every other check here still passes.
+
+ `GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO PUBLIC` is the sort
+ of thing issued to clear an unrelated permission error. The role stays
+ restricted, owns nothing, forces nothing, and the unbound read of
+ `tenants` still raises — so nothing but this check would notice.
+ """
+ async with rls_harness.owner_engine.begin() as connection:
+ await connection.execute(
+ text("GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO PUBLIC")
+ )
+ with pytest.raises(RuntimeRoleError) as raised:
+ await verify_restricted_role(rls_harness.restricted_engine)
+ assert "PUBLIC holds EXECUTE" in str(raised.value)
+ assert "all_tenant_ids" in str(raised.value)
+
+
class TestTheProbeIsOnlyMeaningfulAgainstARow:
async def test_an_empty_tenants_table_refuses_rather_than_passing(
self, rls_harness: RLSHarness
diff --git a/core/tests/switch_core/db/test_tenant_exemption_allowlist.py b/core/tests/switch_core/db/test_tenant_exemption_allowlist.py
index 0bdaf3733..44b599076 100644
--- a/core/tests/switch_core/db/test_tenant_exemption_allowlist.py
+++ b/core/tests/switch_core/db/test_tenant_exemption_allowlist.py
@@ -11,11 +11,18 @@ session reads nothing. So these lists are now what they always should have
been: an inventory a reviewer can read, backed by a database that refuses
regardless.
-**`db/tenant_lookup.py`** is the whole exemption: eight `SECURITY DEFINER`
+**`db/tenant_lookup.py`** is the whole exemption: seven `SECURITY DEFINER`
functions that answer *which tenant* and never return a row. Every module that
imports one is named below. Keep the list short — it is the complete set of
places in the process that ask a question no tenant can be scoped to.
+Short in both directions. A module reaches the exemption if it *can* call a
+lookup, not if it happens to; a helper on an injected store, reachable by
+anything that declares the store, put one module on this list and the
+exemption within reach of every endpoint behind it. That is why
+`get_sole_tenant_id` lives in `gateway/auth.py` as a function rather than on a
+`TenantMemberStore`: the one caller it ever had is the one module named here.
+
**A raw `session_factory()` call** is the other surface. It inherits whatever
is ambient, which in background code is nothing at all, since the long-lived
tasks deliberately unbind (see `tenant_context.no_tenant`). Such a session can
@@ -55,20 +62,23 @@ _ALLOWED_MODULES = {
# Bearer and OIDC authentication: the credential's tenant, before its row.
"switch_core.bridges.agent.auth",
# The gateway's JWT subject resolves to its membership, and
- # `tenant_members` is scoped, so nothing else can answer it.
- "switch_core.db.stores.tenant_member_store",
+ # `tenant_members` is scoped, so nothing else can answer it. In the
+ # authentication module itself rather than on an injected store: a store
+ # is reachable by any endpoint that declares it, which would make this
+ # list name one module while the exemption was open to every route.
+ "switch_core.gateway.auth",
# Enumerating tenants at boot, and starting one bridge or one connector by
# id from a context bound to somebody else's tenant.
"switch_core.clients.client_lifecycle_service",
"switch_core.bridges.collaboration.lifecycle_service",
"switch_core.bridges.agent.server_connectors.lifecycle",
- # A client's own task binds nothing, so its transport and its agent
- # resolution derive the tenant from the client row.
- "switch_core.clients.agent_client",
- "switch_core.transport.postgres",
# `_room_tenant`'s fallback: which tenant is this room in, asked when the
# answer is not already cached alongside the channel mapping.
"switch_core.bridges.collaboration.bridge_core",
+ # `switch_core.transport.postgres` and `switch_core.clients.agent_client`
+ # came off this list with `tenant_of_client`: both were built from a
+ # `clients` row that already named the tenant, so they carry it instead of
+ # asking for it.
}
# Every module allowed to open a session straight from the factory. Not short,
@@ -101,7 +111,7 @@ _RAW_SESSION_FACTORY_MODULES = {
"switch_core.provisioning.postgres",
"switch_core.room_service",
# ── The exemption's own plumbing. It opens a session with nothing bound
- # on purpose and touches only the eight functions above, which are the one
+ # on purpose and touches only the seven functions above, which are the one
# thing a session with nothing bound may read.
"switch_core.db.tenant_lookup",
# `switch_core.transport.postgres` and `switch_core.bridges.agent.auth`
@@ -118,6 +128,9 @@ _FACTORY_ACCESSORS = {"create_session_factory", "get_session_factory"}
_LOOKUP_NAMES = {lookup.name for lookup in TENANT_LOOKUPS}
+_LOOKUP_MODULE = "switch_core.db.tenant_lookup"
+_LOOKUP_PACKAGE, _, _LOOKUP_MODULE_LEAF = _LOOKUP_MODULE.rpartition(".")
+
_PACKAGE_ROOT = Path(switch_core.__file__).resolve().parent
# The modules that define the two things, rather than reaching for them.
@@ -137,22 +150,43 @@ def _imports_a_lookup(path: Path) -> bool:
"""Whether `path` reaches for one of the exempt lookups.
Matched on the import rather than on the call, unlike the raw-factory
- detector below, because there is a single module to import from: an
- `import ... as` renames the local binding but not the name in the `from`
- clause, so aliasing cannot walk past this. Importing without calling is
- flagged too, which is the safe direction — a module that reaches for the
- exemption at all is one this list wants to name. Importing the DDL helpers
- is not reaching for it: `TENANT_LOOKUPS` and `attach_tenant_lookups`
- describe the exemption rather than using it.
+ detector below, because there is a single module to import from and
+ Python has only three ways to name it. All three are checked, because
+ checking one and describing it as "the import" is how an audit surface
+ comes to be narrower than the thing it claims to cover:
+
+ - `from switch_core.db.tenant_lookup import tenants_of_user` — the shape
+ every current caller uses. An `as` alias renames the local binding but
+ not the name inside the `from` clause, so aliasing cannot walk past it.
+ - `import switch_core.db.tenant_lookup [as x]` — binds the module, and
+ every function on it, under a name of the writer's choosing.
+ - `from switch_core.db import tenant_lookup [as x]` — the same thing
+ spelled so that the module name never appears in a `from` clause at all.
+
+ The last two are flagged on reaching the *module*, without asking which
+ attribute is used, because there is no attribute on it worth having except
+ a lookup and the DDL helpers, and a module holding the module object can
+ reach any of them at any point later.
+
+ Importing the DDL helpers *by name* is not reaching for the exemption:
+ `TENANT_LOOKUPS` and `attach_tenant_lookups` describe it rather than using
+ it, which is why the first shape asks which names are imported and the
+ other two cannot. Importing without calling is flagged too, which is the
+ safe direction — a module that reaches for the exemption at all is one
+ this list wants to name.
"""
tree = ast.parse(path.read_text(), filename=str(path))
for node in ast.walk(tree):
- if not isinstance(node, ast.ImportFrom):
- continue
- if node.module != "switch_core.db.tenant_lookup":
- continue
- if any(alias.name in _LOOKUP_NAMES for alias in node.names):
- return True
+ if isinstance(node, ast.Import):
+ if any(alias.name == _LOOKUP_MODULE for alias in node.names):
+ return True
+ elif isinstance(node, ast.ImportFrom):
+ if node.module == _LOOKUP_MODULE:
+ if any(alias.name in _LOOKUP_NAMES for alias in node.names):
+ return True
+ elif node.module == _LOOKUP_PACKAGE:
+ if any(alias.name == _LOOKUP_MODULE_LEAF for alias in node.names):
+ return True
return False
@@ -308,6 +342,72 @@ class TestTheDetectorsCatchANewCaller:
)
assert _imports_a_lookup(new_caller)
+ def test_importing_the_module_itself_is_detected(self, tmp_path: Path) -> None:
+ """`import switch_core.db.tenant_lookup` names no function in the
+ statement, so a detector that only reads `from ... import` clauses
+ reports the module clean while it calls whatever it likes off the
+ module object."""
+ new_caller = tmp_path / "a_new_background_job.py"
+ new_caller.write_text(
+ "import switch_core.db.tenant_lookup\n"
+ "async def sweep(session_factory):\n"
+ " return await switch_core.db.tenant_lookup.all_tenant_ids(\n"
+ " session_factory\n"
+ " )\n"
+ )
+ assert _imports_a_lookup(new_caller)
+
+ def test_an_aliased_module_import_is_detected(self, tmp_path: Path) -> None:
+ new_caller = tmp_path / "a_new_background_job.py"
+ new_caller.write_text(
+ "import switch_core.db.tenant_lookup as lookups\n"
+ "async def sweep(session_factory):\n"
+ " return await lookups.all_tenant_ids(session_factory)\n"
+ )
+ assert _imports_a_lookup(new_caller)
+
+ def test_importing_the_module_from_its_package_is_detected(
+ self, tmp_path: Path
+ ) -> None:
+ """`from switch_core.db import tenant_lookup` reaches the same module
+ without its dotted name ever appearing in a `from` clause."""
+ new_caller = tmp_path / "a_new_background_job.py"
+ new_caller.write_text(
+ "from switch_core.db import tenant_lookup\n"
+ "async def sweep(session_factory):\n"
+ " return await tenant_lookup.all_tenant_ids(session_factory)\n"
+ )
+ assert _imports_a_lookup(new_caller)
+
+ def test_an_aliased_package_relative_module_import_is_detected(
+ self, tmp_path: Path
+ ) -> None:
+ new_caller = tmp_path / "a_new_background_job.py"
+ new_caller.write_text(
+ "from switch_core.db import tenant_lookup as lookups\n"
+ "async def sweep(session_factory):\n"
+ " return await lookups.all_tenant_ids(session_factory)\n"
+ )
+ assert _imports_a_lookup(new_caller)
+
+ def test_a_neighbouring_module_from_the_same_package_is_not_flagged(
+ self, tmp_path: Path
+ ) -> None:
+ """The package-relative shape must key on the module, not the package.
+ `from switch_core.db import ...` is how half the tree reaches a
+ session helper, and flagging all of it would make the list meaningless
+ by being unreadable."""
+ clean = tmp_path / "an_ordinary_service.py"
+ clean.write_text(
+ "from switch_core.db import session_scope\n"
+ "async def work(session_factory, tenant_id):\n"
+ " async with session_scope.tenant_session(\n"
+ " session_factory, tenant_id\n"
+ " ) as session:\n"
+ " ...\n"
+ )
+ assert not _imports_a_lookup(clean)
+
def test_importing_the_ddl_helpers_is_not_reaching_for_it(
self, tmp_path: Path
) -> None:
diff --git a/core/tests/switch_core/db/test_tenant_lookup.py b/core/tests/switch_core/db/test_tenant_lookup.py
index 03d7f3b02..5debb4bd3 100644
--- a/core/tests/switch_core/db/test_tenant_lookup.py
+++ b/core/tests/switch_core/db/test_tenant_lookup.py
@@ -13,7 +13,7 @@ for were invisible for exactly that reason — they returned nothing, silently,
until a row existed — so the rule is written down here rather than left to
whoever adds the next test.
-Four things are pinned:
+Five things are pinned:
- **the catalogue**: the functions installed are exactly `TENANT_LOOKUPS`, and
each one is `SECURITY DEFINER`, `STABLE`, and carries a search path that a
@@ -22,6 +22,11 @@ Four things are pinned:
two tenants, where the equivalent ordinary read answers nothing;
- **the boundary**: they return tenant ids and never rows, and the tables they
read are still refused to the same role in the same session;
+- **what that boundary does not cover**: `users` carries no tenant and so no
+ policy, which makes the whole user-to-tenant membership graph readable with
+ nothing bound. Written down as a deliberate property rather than left to be
+ found, because the shorter statement of the boundary — "the tenant of an
+ identifier you already hold" — reads as though it were covered;
- **the constraint they rest on**: no scoped table has `force row level
security`, which would take the ownership exemption away from them and leave
nothing in the process able to resolve a credential.
@@ -59,7 +64,6 @@ from switch_core.db.tenant_lookup import (
create_lookup_ddl,
tenant_of_agent_oauth_client,
tenant_of_api_key,
- tenant_of_client,
tenant_of_collaboration_bridge,
tenant_of_room,
tenant_of_server_connector,
@@ -73,9 +77,17 @@ pytestmark = pytest.mark.no_ambient_tenant
# The revision that installs the exemption for a database built by Alembic.
_LOOKUP_REVISION = "9c41a7b0e5d8"
+# What a later revision took back out, and which one did it. A migration is
+# a record of a change that already happened, so `9c41a7b0e5d8` still names
+# every function it created; the live module names what the schema holds
+# today. The two agree only once the drops between them are accounted for,
+# and naming the revision here is what keeps 'we removed it from the module'
+# from passing as 'we removed it from the database'.
+_DROPPED_SINCE = {"tenant_of_client": "b1d7c4f0a92e"}
-def _migration_module() -> ModuleType:
- """The `9c41a7b0e5d8` revision module, loaded through Alembic.
+
+def _revision_module(revision: str) -> ModuleType:
+ """One revision module, loaded through Alembic.
Alembic's own loader rather than an `importlib` call on a path, so this
finds the file the same way a deployment would and fails the same way if
@@ -84,7 +96,12 @@ def _migration_module() -> ModuleType:
core = Path(switch_core.__file__).resolve().parents[1]
config = Config(str(core / "alembic.ini"))
config.set_main_option("script_location", str(core / "switch_core" / "migrations"))
- return ScriptDirectory.from_config(config).get_revision(_LOOKUP_REVISION).module
+ return ScriptDirectory.from_config(config).get_revision(revision).module
+
+
+def _migration_module() -> ModuleType:
+ """The revision that installed the exemption."""
+ return _revision_module(_LOOKUP_REVISION)
class _Fixture:
@@ -366,12 +383,11 @@ class TestWhatTheyAnswer:
await _two_populated_tenants(rls_harness)
assert await tenant_of_api_key(rls_harness.restricted, "no-such-hash") is None
- async def test_a_client_room_bridge_and_connector_resolve_to_their_tenant(
+ async def test_a_room_bridge_and_connector_resolve_to_their_tenant(
self, rls_harness: RLSHarness
) -> None:
fixture = await _two_populated_tenants(rls_harness)
restricted = rls_harness.restricted
- assert await tenant_of_client(restricted, fixture.client_a) == fixture.tenant_a
assert await tenant_of_room(restricted, fixture.room_a) == fixture.tenant_a
assert (
await tenant_of_collaboration_bridge(restricted, fixture.bridge_a)
@@ -398,6 +414,74 @@ class TestWhatTheyAnswer:
assert "refusing to pick one" in str(raised.value)
+class TestWhatTheExemptionDiscloses:
+ """The boundary is on rows, not on the shape of the deployment.
+
+ `db/tenant_lookup.py` used to claim that the most a caller could extract
+ was the tenant of an identifier it already held. That is false three ways,
+ and the two that are closed are pinned elsewhere: `all_tenant_ids()` takes
+ no identifier at all (`TestWhatTheyAnswer` above), and `PUBLIC` no longer
+ holds `EXECUTE` (`test_runtime_role.py`). The third is not closed, and is
+ pinned here as a deliberate property rather than left to be rediscovered
+ as a surprise.
+ """
+
+ async def test_the_membership_graph_is_readable_with_nothing_bound(
+ self, rls_harness: RLSHarness
+ ) -> None:
+ """`users` carries no tenant, so it carries no policy.
+
+ A person is global — the per-tenant object is the `tenant_members`
+ row — and `db/rls_ddl.py`'s `GLOBAL_TABLES` says so on purpose. The
+ consequence is that a session with nothing bound reads every user id,
+ and feeding those to `tenants_of_user` one at a time reconstructs the
+ whole user-to-tenant membership graph. No row of a scoped table
+ crosses a boundary doing it; what is disclosed is who exists and which
+ tenants they belong to.
+
+ Closing it means giving `users` a tenant, which it cannot have while a
+ person may belong to more than one, or taking `tenants_of_user` out of
+ the exemption, which is the read every gateway login depends on. So it
+ stands, and it stands as something written down: anyone holding the
+ runtime role's credentials can learn this, and that is the boundary
+ this design draws rather than an oversight in it.
+ """
+ fixture = await _two_populated_tenants(rls_harness)
+
+ async with rls_harness.restricted() as session:
+ user_ids = [
+ row[0] for row in await session.execute(text("SELECT id FROM users"))
+ ]
+ assert {fixture.user_a, fixture.user_in_both} <= set(user_ids)
+
+ graph = {
+ user_id: sorted(await tenants_of_user(rls_harness.restricted, user_id))
+ for user_id in user_ids
+ }
+ assert graph[fixture.user_a] == [fixture.tenant_a]
+ assert graph[fixture.user_in_both] == sorted(
+ [fixture.tenant_a, fixture.tenant_b]
+ )
+
+ async def test_the_membership_rows_themselves_are_still_refused(
+ self, rls_harness: RLSHarness
+ ) -> None:
+ """The half that is closed, next to the half that is not.
+
+ `tenant_members` is scoped, so the row — its role, when it was
+ written, everything on it other than the pair of ids — is refused to
+ the same session that just reconstructed the pairs. That is the
+ difference between disclosing metadata and disclosing data, and it is
+ the line the exemption actually holds.
+ """
+ await _two_populated_tenants(rls_harness)
+
+ async with rls_harness.restricted() as session:
+ with pytest.raises(DBAPIError) as raised:
+ await session.execute(text("SELECT role FROM tenant_members"))
+ assert "app.tenant_id is not set" in str(raised.value)
+
+
class TestTheBoundary:
async def test_a_lookup_leaves_the_tables_it_read_still_refused(
self, rls_harness: RLSHarness
@@ -470,6 +554,7 @@ class TestTheMigrationInstallsTheSameThing:
frozen = {
(name, parameters, signature, body)
for name, parameters, signature, body in module.LOOKUPS
+ if name not in _DROPPED_SINCE
}
live = {
(
@@ -485,7 +570,46 @@ class TestTheMigrationInstallsTheSameThing:
"db/tenant_lookup.py. A deployment built by Alembic would get the "
"migration's functions and every test above would still pass "
"against create_all's. If the divergence is deliberate, express "
- "it as a new migration rather than by editing this one."
+ "it as a new migration rather than by editing this one — and, if "
+ "the new migration drops a lookup, name it in _DROPPED_SINCE "
+ "above so this comparison stays exact rather than being loosened."
+ )
+
+ def test_the_dropped_lookup_is_dropped_by_a_revision_and_not_only_here(
+ self,
+ ) -> None:
+ """Removing a function from `db/tenant_lookup.py` removes it from what
+ `create_all` builds and from nothing else.
+
+ A deployment's schema is built by Alembic, so a lookup deleted from the
+ module and left in the chain is still installed, still `SECURITY
+ DEFINER`, and still answers — while every test in this file, which
+ builds its schema from the models, agrees it is gone. That is the exact
+ shape of failure the frozen-copy tests exist for, arrived at from the
+ other direction.
+
+ The downgrade is checked against `9c41a7b0e5d8`'s frozen text rather
+ than against what the module would build, because the module no longer
+ builds it at all: a rollback past this revision has to land on the
+ schema the revision below it created.
+ """
+ installed = _migration_module()
+ dropped = _revision_module(_DROPPED_SINCE["tenant_of_client"])
+ assert (
+ dropped.DROP_TENANT_OF_CLIENT
+ == "DROP FUNCTION IF EXISTS tenant_of_client(text)"
+ )
+ frozen = {
+ name: (parameters, body)
+ for name, parameters, _signature, body in installed.LOOKUPS
+ }
+ parameters, body = frozen["tenant_of_client"]
+ assert dropped.CREATE_TENANT_OF_CLIENT == installed.create_lookup_ddl(
+ "tenant_of_client", parameters, body
+ ), (
+ "the downgrade would recreate tenant_of_client with different DDL "
+ "from the one 9c41a7b0e5d8 installed, so a rollback past it lands "
+ "on a schema that revision would not recognise."
)
def test_the_statement_it_would_run_is_the_statement_the_module_builds(
@@ -497,7 +621,7 @@ class TestTheMigrationInstallsTheSameThing:
still say `SECURITY INVOKER`, or drop the `search_path`, and the check
above would pass. Nothing else would catch it — this suite builds its
schema from the models, so the migration's own DDL is never executed
- here, and an Alembic-built database would install eight functions
+ here, and an Alembic-built database would install functions
unable to resolve a tenant with the whole suite green. So what is
compared is the rendered statement, not the inputs to it.
"""
diff --git a/core/tests/switch_core/db/test_tenant_session_hook.py b/core/tests/switch_core/db/test_tenant_session_hook.py
index 9d4115f47..ae0c7ffab 100644
--- a/core/tests/switch_core/db/test_tenant_session_hook.py
+++ b/core/tests/switch_core/db/test_tenant_session_hook.py
@@ -25,7 +25,7 @@ from sqlalchemy.orm import Session
from switch_core.db import tenant_session
from switch_core.db.base import Base
from switch_core.db.engine import create_session_factory
-from switch_core.db.models import Tenant
+from switch_core.db.models import Client, Tenant, User
from switch_core.tenant_context import tenant_scope
TENANT_A = "tenant-hook-a"
@@ -304,3 +304,158 @@ class TestABindingThatArrivesTooLateIsRefused:
with tenant_scope(TENANT_B):
assert (await _pid_and_setting(session))[1] == TENANT_B
await session.commit()
+
+
+class TestARowFromAnotherTenantIsRefusedFromTheIdentityMap:
+ """`Session.get` answered from the identity map is the one read that
+ reaches neither the policy nor the drift check.
+
+ Both of the guarantees this design rests on are downstream of a round
+ trip: the row-level-security policy is the server's, and the drift hooks
+ fire on `Session.execute` and on a flush. A `get` whose primary key is
+ already in the session's identity map issues no statement, opens no
+ transaction and flushes nothing, so it goes past all three and hands back
+ an object loaded under whatever tenant was bound at the time.
+
+ `db/engine.create_session_factory` puts `TenantCheckedSession` under every
+ session in the process for exactly this. The first test below is the
+ defect, asserted against a session built the plain way, so this file
+ records what the subclass is worth rather than only that it is wired in.
+ """
+
+ @staticmethod
+ async def _two_tenants_and_a_client(
+ factory: async_sessionmaker[AsyncSession],
+ ) -> str:
+ """Tenant A, tenant B, and one client row in A. Returns the client id."""
+ client_id = "client-in-tenant-a"
+ async with factory() as session:
+ for tenant_id in (TENANT_A, TENANT_B):
+ session.add(Tenant(id=tenant_id, slug=tenant_id, name=tenant_id))
+ # Flushed before the client is added: nothing declares a
+ # relationship between the two mappers, so the unit of work has no
+ # dependency to sort them by and is free to insert the child first.
+ await session.flush()
+ session.add(
+ Client(
+ id=client_id,
+ tenant_id=TENANT_A,
+ matrix_user_id=f"@{client_id}:localhost",
+ display_name="a client",
+ type="agent",
+ )
+ )
+ await session.commit()
+ return client_id
+
+ async def test_a_plain_session_hands_back_the_other_tenants_row(
+ self, single_connection_session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """The defect, stated as a passing test so the fix has something to be
+ a fix of. A session built without `TenantCheckedSession` answers the
+ second `get` from the identity map: no statement reaches Postgres, so
+ the policy never sees it, and tenant B is handed tenant A's row."""
+ client_id = await self._two_tenants_and_a_client(
+ single_connection_session_factory
+ )
+ engine = single_connection_session_factory.kw["bind"]
+ plain = async_sessionmaker(bind=engine, expire_on_commit=False)
+ async with plain() as session:
+ with tenant_scope(TENANT_A):
+ loaded = await session.get(Client, client_id)
+ assert loaded is not None
+ await session.commit()
+ with tenant_scope(TENANT_B):
+ leaked = await session.get(Client, client_id)
+ assert leaked is loaded
+ assert leaked is not None and leaked.tenant_id == TENANT_A
+
+ async def test_the_production_factory_refuses_it(
+ self, single_connection_session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ client_id = await self._two_tenants_and_a_client(
+ single_connection_session_factory
+ )
+ async with single_connection_session_factory() as session:
+ with tenant_scope(TENANT_A):
+ loaded = await session.get(Client, client_id)
+ assert loaded is not None
+ await session.commit()
+ with tenant_scope(TENANT_B):
+ with pytest.raises(
+ tenant_session.CrossTenantIdentityMapError
+ ) as raised:
+ await session.get(Client, client_id)
+ message = str(raised.value)
+ assert TENANT_A in message and TENANT_B in message
+
+ async def test_the_same_tenant_reading_its_own_row_is_untouched(
+ self, single_connection_session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """The check has to be silent on the shape every request actually
+ takes, or it is a false positive on the whole codebase."""
+ client_id = await self._two_tenants_and_a_client(
+ single_connection_session_factory
+ )
+ async with single_connection_session_factory() as session:
+ with tenant_scope(TENANT_A):
+ loaded = await session.get(Client, client_id)
+ assert loaded is not None
+ await session.commit()
+ again = await session.get(Client, client_id)
+ assert again is not None and again.tenant_id == TENANT_A
+
+ async def test_a_get_that_actually_queried_is_not_second_guessed(
+ self, single_connection_session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """The check must fire on the identity map and nowhere else.
+
+ This fixture connects as the container's superuser, whom Postgres
+ exempts from every policy — the same connection the whole unit suite
+ uses, and the same one every fan-out has to filter its own results on
+ (`db/tenant_lookup.py`). So a `get` that really goes to the database
+ can and does return another tenant's row here. Refusing that would
+ impose the policy's semantics on the owner connection, which this
+ design does not do anywhere else, and would break every test that
+ arranges a second tenant's fixture rows.
+
+ `expire_all` is what forces the round trip: the object is still in the
+ identity map, but its attributes are gone, so `get` has to reload it.
+ """
+ client_id = await self._two_tenants_and_a_client(
+ single_connection_session_factory
+ )
+ async with single_connection_session_factory() as session:
+ with tenant_scope(TENANT_A):
+ loaded = await session.get(Client, client_id)
+ assert loaded is not None
+ await session.commit()
+ session.expire_all()
+ with tenant_scope(TENANT_B):
+ fetched = await session.get(Client, client_id)
+ assert fetched is not None and fetched.tenant_id == TENANT_A
+
+ async def test_a_global_row_is_not_refused(
+ self, single_connection_session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """`users` carries no tenant and no policy — a person is global — so
+ there is nothing to compare and nothing to refuse. Reading one from
+ two tenants is the ordinary case, not a leak."""
+ async with single_connection_session_factory() as session:
+ for tenant_id in (TENANT_A, TENANT_B):
+ session.add(Tenant(id=tenant_id, slug=tenant_id, name=tenant_id))
+ session.add(
+ User(
+ id="a-person",
+ name="A Person",
+ role="admin",
+ )
+ )
+ await session.commit()
+ async with single_connection_session_factory() as session:
+ with tenant_scope(TENANT_A):
+ assert await session.get(User, "a-person") is not None
+ await session.commit()
+ with tenant_scope(TENANT_B):
+ assert await session.get(User, "a-person") is not None
diff --git a/core/tests/switch_core/gateway/test_change_password.py b/core/tests/switch_core/gateway/test_change_password.py
index 56e13d9f7..b4c928c17 100644
--- a/core/tests/switch_core/gateway/test_change_password.py
+++ b/core/tests/switch_core/gateway/test_change_password.py
@@ -24,7 +24,6 @@ from pydantic import ValidationError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.db.models import User
-from switch_core.db.stores.tenant_member_store import TenantMemberStore
from switch_core.db.stores.user_store import UserStore
from switch_core.gateway import dependencies as gw_deps
from switch_core.gateway.auth import create_jwt, hash_password, verify_password
@@ -57,9 +56,6 @@ def _app(session_factory: async_sessionmaker[AsyncSession]) -> FastAPI:
app.dependency_overrides[gw_deps.get_system_session] = _session_dep
app.dependency_overrides[gw_deps.get_session_factory] = lambda: session_factory
app.dependency_overrides[gw_deps.get_user_store] = lambda: _USER_STORE
- app.dependency_overrides[gw_deps.get_tenant_member_store] = lambda: (
- TenantMemberStore()
- )
app.dependency_overrides[gw_deps.get_config] = lambda: SimpleNamespace(
jwt_secret_key=_SECRET,
gateway_cookie_secure=False,
diff --git a/core/tests/switch_core/gateway/test_reference_types_routes.py b/core/tests/switch_core/gateway/test_reference_types_routes.py
index b3cb7359a..a792f7c60 100644
--- a/core/tests/switch_core/gateway/test_reference_types_routes.py
+++ b/core/tests/switch_core/gateway/test_reference_types_routes.py
@@ -26,7 +26,6 @@ from switch_core.gateway.dependencies import (
get_resource_service,
get_session,
get_session_factory,
- get_tenant_member_store,
get_user_store,
)
from switch_core.gateway.references import (
@@ -139,11 +138,10 @@ def _app() -> FastAPI:
app.dependency_overrides[get_session] = _stub_session
# get_current_user resolves the caller's tenant on a short-lived session it
# opens from the factory itself (see gateway/auth.py), so the
- # anonymous-client test below reaches that dependency too and needs it —
- # and the membership store — stubbed. Neither is ever called: a request
- # with no cookie is refused before either is touched.
+ # anonymous-client test below reaches that dependency too and needs it
+ # stubbed. It is never called: a request with no cookie is refused
+ # before it is touched.
app.dependency_overrides[get_session_factory] = lambda: None
- app.dependency_overrides[get_tenant_member_store] = lambda: None
app.dependency_overrides[get_user_store] = lambda: None
app.dependency_overrides[get_config] = lambda: None
app.dependency_overrides[get_resource_service] = lambda: None
diff --git a/core/tests/switch_core/gateway/test_tenant_resolution.py b/core/tests/switch_core/gateway/test_tenant_resolution.py
index f48ae0d96..b99fc4b2e 100644
--- a/core/tests/switch_core/gateway/test_tenant_resolution.py
+++ b/core/tests/switch_core/gateway/test_tenant_resolution.py
@@ -2,12 +2,12 @@
of the *user*, from their membership row — never a guess, never a request
parameter.
-`TestGetSoleTenantId` covers `TenantMemberStore.get_sole_tenant_id` raising
-rather than picking a tenant when a user's memberships aren't exactly one. It
-takes a session factory rather than a session: `tenant_members` is a scoped
-table, so no session can read the row that says what it should be scoped to,
-and the store reads through `tenants_of_user` (`db/tenant_lookup.py`)
-instead, on a session of its own with nothing bound.
+`TestSoleTenantId` covers `gateway.auth.sole_tenant_id` raising rather than
+picking a tenant when a user's memberships aren't exactly one. It takes a
+session factory rather than a session: `tenant_members` is a scoped table, so
+no session can read the row that says what it should be scoped to, and it
+reads through `tenants_of_user` (`db/tenant_lookup.py`) instead, on a session
+of its own with nothing bound.
`TestAGatewayRequestBindsTheCallersTenant` drives a real HTTP request through
`get_current_user` end to end — cookie in, `app.tenant_id` on the database
session out — the same way `test_reference_types_routes.py` builds a
@@ -41,23 +41,23 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_asyn
from switch_core.db.base import Base
from switch_core.db.engine import create_session_factory
from switch_core.db.models import TENANT_ZERO_ID, Tenant, TenantMember, User
-from switch_core.db.stores.tenant_member_store import (
- TenantMembershipError,
- TenantMemberStore,
-)
from switch_core.db.stores.user_store import UserStore
from switch_core.gateway import dependencies as gw_deps
-from switch_core.gateway.auth import create_jwt, get_current_user
+from switch_core.gateway.auth import (
+ TenantMembershipError,
+ create_jwt,
+ get_current_user,
+ sole_tenant_id,
+)
_SECRET = "unit-test-jwt-key-unit-test-jwt-key-unit-test" # gitleaks:allow
TENANT_B = "tenant-resolution-b"
-class TestGetSoleTenantId:
+class TestSoleTenantId:
async def test_raises_when_the_user_has_no_membership(
self, session_factory: async_sessionmaker[AsyncSession]
) -> None:
- store = TenantMemberStore()
async with session_factory() as session:
user = User(name="orphan", email="[email protected]", role="user")
session.add(user)
@@ -65,12 +65,11 @@ class TestGetSoleTenantId:
user_id = user.id
with pytest.raises(TenantMembershipError):
- await store.get_sole_tenant_id(session_factory, user_id)
+ await sole_tenant_id(session_factory, user_id)
async def test_raises_when_the_user_has_more_than_one_membership(
self, session_factory: async_sessionmaker[AsyncSession]
) -> None:
- store = TenantMemberStore()
async with session_factory() as session:
user = User(name="dual", email="[email protected]", role="user")
session.add(user)
@@ -89,12 +88,11 @@ class TestGetSoleTenantId:
user_id = user.id
with pytest.raises(TenantMembershipError):
- await store.get_sole_tenant_id(session_factory, user_id)
+ await sole_tenant_id(session_factory, user_id)
async def test_returns_the_one_tenant_a_user_belongs_to(
self, session_factory: async_sessionmaker[AsyncSession]
) -> None:
- store = TenantMemberStore()
async with session_factory() as session:
user = User(name="single", email="[email protected]", role="user")
session.add(user)
@@ -105,9 +103,7 @@ class TestGetSoleTenantId:
await session.commit()
user_id = user.id
- assert (
- await store.get_sole_tenant_id(session_factory, user_id) == TENANT_ZERO_ID
- )
+ assert await sole_tenant_id(session_factory, user_id) == TENANT_ZERO_ID
def _app(session_factory: async_sessionmaker[AsyncSession]) -> FastAPI:
@@ -123,9 +119,6 @@ def _app(session_factory: async_sessionmaker[AsyncSession]) -> FastAPI:
app.dependency_overrides[gw_deps.get_session] = _session_dep
app.dependency_overrides[gw_deps.get_session_factory] = lambda: session_factory
app.dependency_overrides[gw_deps.get_user_store] = lambda: UserStore()
- app.dependency_overrides[gw_deps.get_tenant_member_store] = lambda: (
- TenantMemberStore()
- )
app.dependency_overrides[gw_deps.get_config] = lambda: SimpleNamespace(
jwt_secret_key=_SECRET
)
diff --git a/core/tests/switch_core/test_main.py b/core/tests/switch_core/test_main.py
index 5a2721df8..58baf3e19 100644
--- a/core/tests/switch_core/test_main.py
+++ b/core/tests/switch_core/test_main.py
@@ -1,18 +1,35 @@
-"""Seeding the deployment-wide agent-registration bootstrap key.
+"""Seeding the deployment-wide agent-registration bootstrap key, and the
+boot-time advisory lock that serialises it (and the migration) across
+concurrently-booting replicas.
See `bridges/agent/test_registration_bootstrap.py` for what the key resolves
-to; this covers the seeding lifecycle itself: fresh install, in-place
-rotation, legacy migration, and that a deliberate revocation (deleting the
-key from the gateway's API Keys page) survives a restart instead of being
-silently recreated — including when the admin email changes underneath it.
+to; `TestSeedAgentRegistrationBootstrapKey` below covers the seeding lifecycle
+itself: fresh install, in-place rotation, legacy migration, and that a
+deliberate revocation (deleting the key from the gateway's API Keys page)
+survives a restart instead of being silently recreated — including when the
+admin email changes underneath it.
+
+`TestBootLock` covers `db/boot_lock.py` against the real Postgres instance
+`postgres_url` provides (not a mock — an advisory lock is a property of a real
+database session, not something a fake connection can stand in for): that the
+lock is actually held while its block runs and free again once it exits, that
+two overlapping acquisitions of it genuinely serialise rather than merely
+appearing to, that it is released even when the guarded body raises, and that
+`asyncio.to_thread` — which is how `main._migrate_and_grant` runs the
+synchronous `alembic_command.upgrade` from inside a coroutine — really does
+give that call a thread with no event loop of its own, since `env.py`'s own
+`asyncio.run` would raise otherwise.
"""
from __future__ import annotations
+import asyncio
import hashlib
import pytest
-from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+from sqlalchemy import text
+from sqlalchemy.engine import make_url
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from switch_core.bridges.agent.registration_bootstrap import (
BOOTSTRAP_KEY_LABEL,
@@ -26,6 +43,7 @@ from switch_core.bridges.agent.registration_bootstrap import (
RETIRED_KEY_TYPE,
)
from switch_core.config import SwitchConfig
+from switch_core.db.boot_lock import BOOT_LOCK_KEY, boot_lock
from switch_core.db.models import Agent, ApiKey, Client, User
from switch_core.db.stores.agent_store import AgentStore
from switch_core.db.stores.api_key_store import ApiKeyStore
@@ -50,6 +68,29 @@ def _config(token: str, *, admin_email: str = ADMIN_EMAIL) -> SwitchConfig:
)
+def _config_for(postgres_url: str) -> SwitchConfig:
+ """A config whose `database_url` actually reaches `postgres_url`.
+
+ Unlike `_config` above, `boot_lock` opens a real connection with this —
+ `db_owner_user` is left unset, so it falls back to `database_url`, the
+ same rule `migrations/env.py` applies for the same reason (a fresh
+ deployment's runtime role may not exist yet).
+ """
+ url = make_url(postgres_url)
+ return SwitchConfig(
+ db_host=url.host or "localhost",
+ db_port=str(url.port),
+ db_user=url.username,
+ db_password=url.password,
+ db_name=url.database,
+ matrix_server_name="test",
+ agent_registration_token="unused",
+ jwt_secret_key="test-jwt-secret",
+ gateway_admin_email=ADMIN_EMAIL,
+ gateway_admin_password="unused",
+ )
+
+
async def _make_user(
session_factory: async_sessionmaker[AsyncSession],
*,
@@ -617,3 +658,140 @@ class TestSeedAgentRegistrationBootstrapKey:
async with session_factory() as session:
found = await api_key_store.get_by_hash(session, _hash("dev-test-token"))
assert found is None
+
+
+async def _try_lock_from_a_fresh_session(postgres_url: str) -> bool:
+ """Whether `BOOT_LOCK_KEY` is free, asked from a connection of its own.
+
+ `pg_try_advisory_lock` both answers the question and, if the answer is
+ yes, acquires the lock for the session that asked — so a caller getting
+ `True` back now holds it and must release it, exactly as a caller of
+ `boot_lock` itself would.
+ """
+ probe_engine = create_async_engine(postgres_url)
+ try:
+ async with probe_engine.connect() as probe:
+ return bool(
+ (
+ await probe.execute(
+ text("SELECT pg_try_advisory_lock(:key)"),
+ {"key": BOOT_LOCK_KEY},
+ )
+ ).scalar_one()
+ )
+ finally:
+ await probe_engine.dispose()
+
+
+async def _unlock_from_a_fresh_session(postgres_url: str) -> None:
+ """Undo `_try_lock_from_a_fresh_session`'s acquisition, on its own
+ connection — a session-level advisory lock is released by the session
+ that holds it, not by whichever session next asks about it."""
+ probe_engine = create_async_engine(postgres_url)
+ try:
+ async with probe_engine.connect() as probe:
+ await probe.execute(
+ text("SELECT pg_advisory_unlock(:key)"), {"key": BOOT_LOCK_KEY}
+ )
+ finally:
+ await probe_engine.dispose()
+
+
+class TestBootLock:
+ def test_the_key_cannot_collide_with_a_rooms_message_lock(self) -> None:
+ """Session-level and transaction-level advisory locks share one flat
+ namespace per database, and Switch takes a second one:
+ `db/stores/message_store.py` serialises a room's message sequence with
+ `pg_advisory_xact_lock(hashtext(room_id))`. A boot that collided with
+ it would block on an ordinary message write, and a message write on a
+ boot.
+
+ `hashtext` returns `integer`, so every key that lock can take lies in
+ the signed 32-bit range. Keeping `BOOT_LOCK_KEY` outside that range
+ separates the two by domain rather than by a list of values anyone has
+ thought to check, which is what makes it hold for a room id nobody has
+ created yet."""
+ assert abs(BOOT_LOCK_KEY) > 2**31
+
+ async def test_lock_is_held_during_the_block_and_released_after(
+ self, postgres_url: str
+ ) -> None:
+ config = _config_for(postgres_url)
+
+ async with boot_lock(config):
+ still_free = await _try_lock_from_a_fresh_session(postgres_url)
+ if still_free:
+ await _unlock_from_a_fresh_session(postgres_url)
+ assert still_free is False, (
+ "a second session was able to take BOOT_LOCK_KEY while "
+ "boot_lock's own block was still running — the lock was "
+ "never actually held"
+ )
+
+ now_free = await _try_lock_from_a_fresh_session(postgres_url)
+ assert now_free is True, "the lock was not released when the block exited"
+ await _unlock_from_a_fresh_session(postgres_url)
+
+ async def test_lock_is_released_even_when_the_guarded_body_raises(
+ self, postgres_url: str
+ ) -> None:
+ config = _config_for(postgres_url)
+
+ with pytest.raises(RuntimeError, match="boom"):
+ async with boot_lock(config):
+ raise RuntimeError("boom")
+
+ freed = await _try_lock_from_a_fresh_session(postgres_url)
+ assert freed is True, (
+ "the lock leaked after the guarded body raised — a stuck "
+ "advisory lock would wedge every future boot on this database"
+ )
+ await _unlock_from_a_fresh_session(postgres_url)
+
+ async def test_two_concurrent_callers_of_the_guarded_section_serialise(
+ self, postgres_url: str
+ ) -> None:
+ """Real overlap, not a stand-in for it: both callers are started
+ together with `asyncio.gather`, and each holds the lock across an
+ `await asyncio.sleep`, which is exactly the shape boot's own
+ critical sections have (a migration and a database round trip
+ respectively) — long enough that two callers *would* interleave if
+ the lock were not actually excluding one while the other runs.
+ """
+ config = _config_for(postgres_url)
+ events: list[str] = []
+
+ async def _guarded(label: str) -> None:
+ async with boot_lock(config):
+ events.append(f"{label}-start")
+ await asyncio.sleep(0.2)
+ events.append(f"{label}-end")
+
+ await asyncio.gather(_guarded("a"), _guarded("b"))
+
+ assert events in (
+ ["a-start", "a-end", "b-start", "b-end"],
+ ["b-start", "b-end", "a-start", "a-end"],
+ ), f"the two callers' work interleaved instead of serialising: {events}"
+
+ async def test_asyncio_to_thread_gives_alembic_a_thread_with_no_running_loop(
+ self,
+ ) -> None:
+ """`main._migrate_and_grant` runs the synchronous
+ `alembic_command.upgrade` — whose own `migrations/env.py` calls
+ `asyncio.run` internally to drive its async engine — via
+ `asyncio.to_thread`, from a coroutine that already has a running event
+ loop of its own. `asyncio.run` raises `RuntimeError` if it is called
+ while a loop is already running on its thread, so this only works
+ because a `to_thread` worker is a fresh OS thread with none — pinned
+ down directly here, independent of Alembic, rather than trusted.
+ """
+
+ def _sync_work_with_its_own_event_loop() -> int:
+ async def _inner() -> int:
+ return 42
+
+ return asyncio.run(_inner())
+
+ result = await asyncio.to_thread(_sync_work_with_its_own_event_loop)
+ assert result == 42
diff --git a/core/tests/switch_core/test_seed_admin_membership.py b/core/tests/switch_core/test_seed_admin_membership.py
index a7371fed6..5bcbff295 100644
--- a/core/tests/switch_core/test_seed_admin_membership.py
+++ b/core/tests/switch_core/test_seed_admin_membership.py
@@ -24,20 +24,23 @@ database client.
from __future__ import annotations
+import ast
import logging
+import pathlib
import pytest
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+import switch_core
from switch_core.config import SwitchConfig
from switch_core.db.models import TENANT_ZERO_ID, TenantMember, User
-from switch_core.db.stores.tenant_member_store import (
+from switch_core.db.stores.user_store import UserStore
+from switch_core.gateway.auth import (
TenantMembershipError,
- TenantMemberStore,
+ hash_password,
+ sole_tenant_id,
)
-from switch_core.db.stores.user_store import UserStore
-from switch_core.gateway.auth import hash_password
from switch_core.main import _seed_admin_user
from switch_core.tenant_context import tenant_scope
@@ -180,7 +183,7 @@ class TestWhatBeingStrandedCosts:
await _strand_the_admin(session_factory, user_id)
with pytest.raises(TenantMembershipError):
- await TenantMemberStore().get_sole_tenant_id(session_factory, user_id)
+ await sole_tenant_id(session_factory, user_id)
async def test_creating_a_user_still_writes_one(
self, session_factory: async_sessionmaker[AsyncSession]
@@ -200,13 +203,32 @@ class TestWhatBeingStrandedCosts:
]
-def test_the_membership_store_offers_no_second_way_to_write_one() -> None:
- """`TenantMemberStore.create` existed and nothing called it.
+def test_there_is_one_way_to_write_a_membership() -> None:
+ """`UserStore.ensure_membership` is it, and nothing else writes one.
- Removed rather than wired up: "exactly one membership per account" holds
- because a single idempotent function writes them all, and an unguarded
- `create(tenant_id=…, user_id=…, role=…)` beside it is an invitation to
- write a second — which `get_sole_tenant_id` rejects just as firmly as it
- rejects none.
+ "Exactly one membership per account" holds because a single idempotent
+ function writes them all. There was a `TenantMemberStore.create` beside
+ it, taking `tenant_id`, `user_id` and `role` from whatever the caller
+ felt like; nothing ever called it, and an unguarded second way in is how
+ an account ends up with two — which `sole_tenant_id` rejects just as
+ firmly as it rejects none. The store is gone, so this asserts the
+ property rather than the absence: only `UserStore` constructs a
+ `TenantMember`.
"""
- assert not hasattr(TenantMemberStore, "create")
+ package = pathlib.Path(switch_core.__file__).resolve().parent
+ writers = set()
+ for path in package.rglob("*.py"):
+ tree = ast.parse(path.read_text(), filename=str(path))
+ for node in ast.walk(tree):
+ # The constructor call, not the class statement: `db/models.py`
+ # declares `class TenantMember(Base)`, which a text scan reads as
+ # a write and an AST walk does not.
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
+ if node.func.id == "TenantMember":
+ writers.add(path.relative_to(package).as_posix())
+ assert sorted(writers) == ["db/stores/user_store.py"], (
+ "a membership row is written outside UserStore.ensure_membership by "
+ f"{sorted(writers)}. Every account must end up with exactly one, "
+ "which holds because one idempotent function writes them all — route "
+ "the new write through it rather than adding a second way in."
+ )
diff --git a/core/tests/switch_core/transport/test_postgres_transport.py b/core/tests/switch_core/transport/test_postgres_transport.py
index 6c5e41510..96e6542d9 100644
--- a/core/tests/switch_core/transport/test_postgres_transport.py
+++ b/core/tests/switch_core/transport/test_postgres_transport.py
@@ -20,7 +20,7 @@ import pytest
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
-from switch_core.db.models import Client, ClientRoom, Room, Tenant
+from switch_core.db.models import TENANT_ZERO_ID, Client, ClientRoom, Room, Tenant
from switch_core.db.session_scope import tenant_session
from switch_core.db.stores.media_store import MediaStore
from switch_core.db.stores.message_store import MessageStore
@@ -134,42 +134,29 @@ class _Received:
self.events.append(event)
-async def _seed_client(
- session_factory: async_sessionmaker[AsyncSession], client_id: str
-) -> None:
- """Give a transport's client a row, so it has a tenant to act in.
-
- A `PostgresTransport` resolves its own tenant from `clients` (see
- `PostgresTransport._tenant`), because its task binds nothing and
- everything it writes is scoped. A transport built for a client id with no
- row behind it is a transport for a client that does not exist, and it says
- so rather than falling back to whatever tenant happens to be ambient —
- which for `upload_media` would mean putting a blob in a tenant nobody
- chose. Tests that need only *a* client take one from here.
- """
- async with session_factory() as session:
- session.add(
- Client(
- id=client_id,
- matrix_user_id=f"@{client_id}-{uuid.uuid4().hex[:8]}:test",
- display_name="a client",
- type="agent",
- )
- )
- await session.commit()
-
-
def _transport(
session_factory: async_sessionmaker[AsyncSession],
*,
client_id: str,
user_id: str,
+ tenant_id: str = TENANT_ZERO_ID,
listener: _FakeListener | None = None,
ephemeral: EphemeralBus | None = None,
) -> PostgresTransport:
+ """A transport for `client_id`, acting in `tenant_id`.
+
+ The tenant is handed in because production hands it in: a transport is
+ built by `ClientFactory.transport_for` from a client that took it off
+ its own `clients` row, so there is nothing here for the transport to
+ look up and nothing ambient for it to fall back to. The default is the
+ tenant the `session_factory` fixture binds, which is the one every row
+ these tests write lands in; the handful that arrange their own tenant
+ pass it, and those are the tests where the value is the point.
+ """
return PostgresTransport(
user_id=user_id,
client_id=client_id,
+ tenant_id=tenant_id,
display_name="agent one",
session_factory=session_factory,
room_store=RoomStore(),
@@ -381,14 +368,14 @@ class TestRooms:
class TestMedia:
"""`media_blobs` is scoped, so a blob is written into the uploading
- client's tenant — which means every test here needs a client row for the
- transport to read that tenant off. Without one the transport refuses,
- which is the right answer and not the one these tests are about."""
+ client's tenant — the one the transport was built with, which is what
+ keeps a blob out of a tenant nobody chose. That the tenant is the
+ client's own is `TestATransportActsInItsClientsTenant`'s business; here
+ it is simply the tenant these rows are expected in."""
async def test_bytes_go_in_and_come_back(
self, session_factory: async_sessionmaker[AsyncSession]
) -> None:
- await _seed_client(session_factory, "c")
transport = _transport(session_factory, client_id="c", user_id="@a:test")
uploaded = await transport.upload_media(b"file contents", "text/plain", "a.txt")
@@ -402,7 +389,6 @@ class TestMedia:
self, session_factory: async_sessionmaker[AsyncSession]
) -> None:
"""Deduplicating would make one sender's delete another's data loss."""
- await _seed_client(session_factory, "c")
transport = _transport(session_factory, client_id="c", user_id="@a:test")
first = await transport.upload_media(b"same", "text/plain", "a.txt")
@@ -420,24 +406,36 @@ class TestMedia:
transport raises `TransportError` for having no tenant, and this test
would pass for entirely the wrong reason.
"""
- await _seed_client(session_factory, "c")
transport = _transport(session_factory, client_id="c", user_id="@a:test")
with pytest.raises(TransportError, match="No media stored under"):
await transport.download_media("switch-media://nothing")
- async def test_a_transport_whose_client_has_no_row_refuses_to_upload(
+ def test_a_transport_cannot_be_built_without_a_tenant(
self, session_factory: async_sessionmaker[AsyncSession]
) -> None:
- """The fail-loud half of the same rule. A blob has to name a tenant,
- and the only honest source for one is the uploading client's own row;
- with no row there is no tenant, and the alternatives — whatever is
- ambient, or tenant zero — are both a write into a real customer's
- data that nobody chose."""
- transport = _transport(session_factory, client_id="ghost", user_id="@a:test")
-
- with pytest.raises(TransportError, match="no tenant to act in"):
- await transport.upload_media(b"bytes", "text/plain", "a.txt")
+ """The fail-loud half of the same rule, moved to construction.
+
+ A blob has to name a tenant, and the alternatives to being told which
+ one — whatever is ambient, or tenant zero — are both a write into a
+ real customer's data that nobody chose. So `tenant_id` is a required
+ argument rather than something derived on the first write: a caller
+ that has not got one cannot build the transport at all, which is a
+ failure at wiring time instead of a wrong row at run time.
+ """
+ with pytest.raises(TypeError):
+ PostgresTransport( # type: ignore[call-arg]
+ user_id="@a:test",
+ client_id="ghost",
+ display_name="agent one",
+ session_factory=session_factory,
+ room_store=RoomStore(),
+ message_store=MessageStore(),
+ media_store=MediaStore(),
+ listener=_FakeListener(),
+ invites=InviteBus(),
+ ephemeral=EphemeralBus(),
+ )
class TestWhatIsNotBuiltYet:
@@ -969,7 +967,11 @@ class TestDeliveryBindsTheRoomsTenant:
listener = _FakeListener()
received = _Received()
transport = _transport(
- session_factory, client_id=client_id, user_id=user_id, listener=listener
+ session_factory,
+ client_id=client_id,
+ user_id=user_id,
+ tenant_id=tenant_id,
+ listener=listener,
)
transport.register_handlers(received.handlers())
await transport.join_room(transport_room_id)
@@ -989,7 +991,7 @@ class TestDeliveryBindsTheRoomsTenant:
class TestATransportActsInItsClientsTenant:
- """A transport's tenant is its client's, derived and never inherited.
+ """A transport's tenant is its client's, carried and never inherited.
Its task binds nothing: `ClientLifecycleService` unbinds before running a
client, so anything the transport did under an *inherited* tenant would be
@@ -1004,12 +1006,18 @@ class TestATransportActsInItsClientsTenant:
only be a guess and a guess returns a silent subset. Under a role the
policies apply to, nothing bound returns a silent *empty* — so "unscoped"
stopped being the safe answer and became the worst one. The tenant now
- comes from the client's own row, through the exemption
- (`db/tenant_lookup.py`), which is exact rather than a guess: `client_rooms`
- and `messages` both carry composite foreign keys through `tenant_id`, so a
- client cannot be a member of, or a sender in, a room outside its own
- tenant. `test_the_schema_forbids_a_client_row_in_another_tenants_room`
- below is what that rests on.
+ comes from the client's own `clients` row, handed to the transport when it
+ is built, which is exact rather than a guess: `client_rooms` and `messages`
+ both carry composite foreign keys through `tenant_id`, so a client cannot
+ be a member of, or a sender in, a room outside its own tenant.
+ `test_the_schema_forbids_a_client_row_in_another_tenants_room` below is
+ what that rests on.
+
+ That the value really is the client row's, rather than something a caller
+ invented, is the factory's business and is pinned in
+ `tests/switch_core/clients/test_client_tenant_plumbing.py`. What these
+ tests pin is the other half: given it, the transport reads and writes
+ under that tenant and never under whatever a caller had bound.
"""
@pytest.fixture(autouse=True)
@@ -1074,7 +1082,12 @@ class TestATransportActsInItsClientsTenant:
monkeypatch.setattr(RoomStore, "get_for_client", _spy)
- transport = _transport(session_factory, client_id=client_id, user_id=user_id)
+ transport = _transport(
+ session_factory,
+ client_id=client_id,
+ user_id=user_id,
+ tenant_id=tenant_id,
+ )
# Stands in for a task that inherited a tenant it has no business
# acting under — a bridge restart, a puppet minted mid-conversation.
with tenant_scope("some-other-tenant"):
@@ -1195,7 +1208,11 @@ class TestATransportActsInItsClientsTenant:
listener = _FakeListener()
received = _Received()
transport = _transport(
- session_factory, client_id=client_id, user_id=user_id, listener=listener
+ session_factory,
+ client_id=client_id,
+ user_id=user_id,
+ tenant_id=tenant_id,
+ listener=listener,
)
transport.register_handlers(received.handlers())
await transport.join_room(mxid_a)
@@ -1255,7 +1272,10 @@ class TestATransportActsInItsClientsTenant:
elsewhere_mxid = elsewhere.matrix_room_id
transport = _transport(
- rls_harness.restricted, client_id=client_id, user_id=user_id
+ rls_harness.restricted,
+ client_id=client_id,
+ user_id=user_id,
+ tenant_id=tenant_id,
)
with pytest.raises(TransportError, match="is not a Switch room"):
await transport.send_message(elsewhere_mxid, "hello", sender_name="admin")