Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(db): refuse a tenant bound after its transaction opened (CHOO-2623)

Wojciech Wojtyniakb2b76ed6219a
diff --git a/core/switch_core/db/tenant_session.py b/core/switch_core/db/tenant_session.py
index 87deb3201..4a48e8726 100644
--- a/core/switch_core/db/tenant_session.py
+++ b/core/switch_core/db/tenant_session.py
@@ -44,25 +44,53 @@ from __future__ import annotations
from sqlalchemy import event, text
from sqlalchemy.engine import Connection
-from sqlalchemy.orm import Session, SessionTransaction
+from sqlalchemy.orm import ORMExecuteState, Session, SessionTransaction
from switch_core.tenant_context import current_tenant_id
_registered = False
+# Where the hook records what it actually stamped on the current transaction,
+# so the drift check below can compare it against what is bound now.
+_STAMPED = "switch_stamped_tenant_id"
+
+
+class TenantBindingDriftError(RuntimeError):
+ """A tenant was bound after this session's transaction had already begun.
+
+ The failure this exists to make loud: `set_config` rides `after_begin`, so
+ it is issued once, when the transaction opens. Entering a `tenant_scope`
+ *inside* an open transaction rebinds the contextvar and nothing else — the
+ connection keeps whatever it was told at begin, which for a session opened
+ with nothing bound is nothing at all. Every read and write on it then runs
+ under the wrong tenant, or under none.
+
+ That is invisible on a connection Postgres exempts from the policies,
+ which is what every environment used before the runtime role, and it is a
+ refused write on one it does not. Either way the call site looks correct
+ and is not, which is the exact shape this whole design exists to prevent —
+ so it raises rather than being left to a code review to notice.
+
+ The fix is never to move the `tenant_scope` earlier by a line or two: it
+ is to open the session inside the binding, which is what
+ `db/session_scope.tenant_session` does and why it exists.
+ """
+
def register_tenant_session_hook() -> None:
- """Attach the `after_begin` hook, once per process.
+ """Attach the hooks, once per process.
Called at the bottom of this module, so importing it is enough. Idempotent
so that an explicit call — a test asserting the registration, say — cannot
- end up with the hook attached twice and the `set_config` issued twice per
+ end up with them attached twice and the `set_config` issued twice per
transaction.
"""
global _registered
if _registered:
return
event.listens_for(Session, "after_begin")(_set_tenant_on_begin)
+ event.listens_for(Session, "after_transaction_end")(_forget_the_stamp)
+ event.listens_for(Session, "do_orm_execute")(_refuse_a_binding_that_came_too_late)
_registered = True
@@ -70,6 +98,10 @@ def _set_tenant_on_begin(
session: Session, transaction: SessionTransaction, connection: Connection
) -> None:
tenant_id = current_tenant_id()
+ # Recorded even when it is None, because "this transaction was stamped
+ # with nothing" is exactly what the drift check needs to be able to tell
+ # apart from "no transaction has begun yet".
+ session.info[_STAMPED] = tenant_id
if tenant_id is None:
return
connection.execute(
@@ -78,4 +110,46 @@ def _set_tenant_on_begin(
)
+def _forget_the_stamp(session: Session, transaction: SessionTransaction) -> None:
+ """Drop the record when the outermost transaction ends.
+
+ A session outlives its transactions — commit and the next statement opens
+ another — and the next one gets its own `after_begin` and so its own
+ stamp. Leaving the old one behind would make the first statement after a
+ commit compare against a transaction that no longer exists. Only the
+ outermost, because a savepoint ending does not end the transaction the
+ `set_config` was issued on.
+ """
+ if transaction.parent is None:
+ session.info.pop(_STAMPED, None)
+
+
+def _refuse_a_binding_that_came_too_late(state: ORMExecuteState) -> None:
+ """Raise if what is bound now is not what this transaction was stamped with.
+
+ Fires before every `Session.execute`, which is every statement this
+ codebase issues through a session. For the first statement of a
+ transaction there is no stamp yet — `after_begin` has not run, since the
+ transaction begins as part of executing it — so there is nothing to
+ compare and nothing to refuse. From the second onwards the two must agree.
+ """
+ session = state.session
+ if _STAMPED not in session.info:
+ return
+ stamped = session.info[_STAMPED]
+ bound = current_tenant_id()
+ if bound == stamped:
+ return
+ raise TenantBindingDriftError(
+ f"this session's transaction was opened with tenant {stamped!r} and "
+ f"the tenant bound now is {bound!r}. The `set_config` that makes the "
+ "database agree is issued once, when the transaction begins, so "
+ "rebinding after that changes nothing the database can see: every "
+ "statement from here on runs under the tenant in the first half of "
+ "this message. Open the session inside the binding instead — "
+ "`db/session_scope.tenant_session(factory, tenant_id)` does exactly "
+ "that — or commit this transaction before rebinding."
+ )
+
+
register_tenant_session_hook()
diff --git a/core/tests/switch_core/db/stores/test_collaboration_bridge_store.py b/core/tests/switch_core/db/stores/test_collaboration_bridge_store.py
index 5bb9cd2ea..19b4953e9 100644
--- a/core/tests/switch_core/db/stores/test_collaboration_bridge_store.py
+++ b/core/tests/switch_core/db/stores/test_collaboration_bridge_store.py
@@ -7,6 +7,7 @@ import pytest
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.db.models import Client, CollaborationBridge, Tenant
+from switch_core.db.session_scope import tenant_session
from switch_core.db.stores.collaboration_bridge_store import CollaborationBridgeStore
from switch_core.tenant_context import tenant_scope
@@ -30,12 +31,6 @@ async def _make_bridge(session: AsyncSession) -> str:
return bridge.id
-async def _make_bridge_for_tenant(session: AsyncSession, tenant_id: str) -> str:
- """A bridge filed under `tenant_id` rather than the ambient tenant."""
- with tenant_scope(tenant_id):
- return await _make_bridge(session)
-
-
@pytest.mark.asyncio
async def test_agent_greetings_enabled_defaults_true(
session_factory: async_sessionmaker[AsyncSession],
@@ -168,6 +163,16 @@ async def test_get_default_does_not_choke_on_a_second_tenants_default(
matched both rows the moment a second tenant had a default and raised
`MultipleResultsFound`, a 500 out of `create_room` for every tenant, not
just the one that just onboarded.
+
+ The other tenant's bridge is created and nominated on a session opened
+ with `tenant_session`, not by entering `tenant_scope(other_tenant)` inside
+ the session already open for the own-tenant arrangement above. That
+ session's transaction was already stamped with the ambient tenant by its
+ first statement — the `set_config` a transaction gets is issued once, at
+ `after_begin` — so rebinding the contextvar mid-transaction would not move
+ what Postgres was told; it would only make the two disagree. Opening a
+ fresh session inside `tenant_session(other_tenant)` stamps its
+ transaction with the other tenant from the first statement on.
"""
store = CollaborationBridgeStore()
other_tenant = f"tenant-{uuid.uuid4().hex[:8]}"
@@ -176,12 +181,14 @@ async def test_get_default_does_not_choke_on_a_second_tenants_default(
session.add(Tenant(id=other_tenant, slug=other_tenant, name=other_tenant))
await session.flush()
own_bridge = await _make_bridge(session)
- other_bridge = await _make_bridge_for_tenant(session, other_tenant)
await store.set_default(session, own_bridge)
- with tenant_scope(other_tenant):
- await store.set_default(session, other_bridge)
await session.commit()
+ async with tenant_session(session_factory, other_tenant) as other_session:
+ other_bridge = await _make_bridge(other_session)
+ await store.set_default(other_session, other_bridge)
+ await other_session.commit()
+
async with session_factory() as session:
default = await store.get_default(session)
assert default is not None
@@ -203,29 +210,43 @@ async def test_set_default_does_not_demote_another_tenants_default(
has none of its own yet, and `set_default` then demotes a bridge it was
never asked about — silently disabling another tenant's default the first
time this tenant nominates its own.
+
+ The other tenant's bridge is arranged and read back through its own
+ session, opened with `tenant_session`, rather than by wrapping a
+ `tenant_scope(other_tenant)` block around statements on a session shared
+ with the own-tenant arrangement. A session's transaction is stamped with
+ whatever tenant was bound at its first statement — `set_config` rides
+ `after_begin`, issued once — so rebinding the contextvar afterwards, or
+ reverting it on the way out of a `with` block, changes nothing the
+ database was told; it only makes the next statement disagree with the
+ one before it. Two sessions, each opened inside the binding it needs,
+ keep every statement honest about which tenant it ran under.
"""
store = CollaborationBridgeStore()
other_tenant = f"tenant-{uuid.uuid4().hex[:8]}"
async with session_factory() as session:
session.add(Tenant(id=other_tenant, slug=other_tenant, name=other_tenant))
- await session.flush()
- other_bridge = await _make_bridge_for_tenant(session, other_tenant)
- with tenant_scope(other_tenant):
- await store.set_default(session, other_bridge)
+ await session.commit()
+ async with tenant_session(session_factory, other_tenant) as other_session:
+ other_bridge = await _make_bridge(other_session)
+ await store.set_default(other_session, other_bridge)
+ await other_session.commit()
+
+ async with session_factory() as session:
own_bridge = await _make_bridge(session)
await store.set_default(session, own_bridge)
await session.commit()
- async with session_factory() as session:
- with tenant_scope(other_tenant):
- other_default = await store.get(session, other_bridge)
+ async with tenant_session(session_factory, other_tenant) as other_session:
+ other_default = await store.get(other_session, other_bridge)
assert other_default is not None
assert other_default.is_default is True, (
"the other tenant's default was demoted by a write it was never part of"
)
+ async with session_factory() as session:
own_default = await store.get(session, own_bridge)
assert own_default is not None
assert own_default.is_default is True
diff --git a/core/tests/switch_core/db/stores/test_reference_type_store.py b/core/tests/switch_core/db/stores/test_reference_type_store.py
index 748d3be08..f12cc98a6 100644
--- a/core/tests/switch_core/db/stores/test_reference_type_store.py
+++ b/core/tests/switch_core/db/stores/test_reference_type_store.py
@@ -7,6 +7,7 @@ from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.db.models import Reference, ReferenceType, Tenant, User
+from switch_core.db.session_scope import tenant_session
from switch_core.db.stores.reference_store import ReferenceStore
from switch_core.db.stores.reference_type_store import ReferenceTypeStore
from switch_core.tenant_context import tenant_scope
@@ -347,6 +348,19 @@ class TestReferenceTypesAreScopedToTheBoundTenant:
filter on the row's own tenant is what keeps it single-valued, and
this is the arrangement that tells the two apart: the same slug in
tenant zero *and* in the tenant doing the create.
+
+ The other tenant's rows are all created and read through one session
+ opened with `tenant_session(other)`, not by entering
+ `tenant_scope(other)` around statements on the session already open
+ for tenant zero's row above. `set_config` is issued once, when a
+ transaction begins — that session's transaction was already stamped
+ with tenant zero by its first statement, so rebinding the contextvar
+ afterwards would not move what Postgres was told; every following
+ statement would still run under tenant zero, and the write meant to
+ land under `other` would clash with itself instead of with the row
+ this test is actually arranging a clash against. Opening a fresh
+ session inside `tenant_session(other)` stamps its transaction with
+ `other` from the first statement on.
"""
other = f"tenant-{uuid.uuid4().hex[:8]}"
store = ReferenceTypeStore()
@@ -355,31 +369,45 @@ class TestReferenceTypesAreScopedToTheBoundTenant:
ada = await _make_user(session, "ada")
await session.flush()
await store.create(session, _type("notion", ada.id))
- with tenant_scope(other):
- await store.create(session, _type("notion", ada.id))
await session.commit()
- with tenant_scope(other):
- with pytest.raises(
- ValueError, match="Reference type 'notion' already exists"
- ):
- await store.create(session, _type("notion", ada.id))
-
- # The savepoint means the caller's transaction survives it,
- # and neither tenant's row was disturbed.
- await store.create(session, _type("linear", ada.id))
- await session.commit()
- assert {rt.type for rt in await store.list_all(session)} == {
- "notion",
- "linear",
- }
+ async with tenant_session(session_factory, other) as other_session:
+ await store.create(other_session, _type("notion", ada.id))
+ await other_session.commit()
+
+ with pytest.raises(
+ ValueError, match="Reference type 'notion' already exists"
+ ):
+ await store.create(other_session, _type("notion", ada.id))
+
+ # The savepoint means the caller's transaction survives it,
+ # and neither tenant's row was disturbed.
+ await store.create(other_session, _type("linear", ada.id))
+ await other_session.commit()
+ assert {rt.type for rt in await store.list_all(other_session)} == {
+ "notion",
+ "linear",
+ }
+
+ async with session_factory() as session:
assert {rt.type for rt in await store.list_all(session)} == {"notion"}
async def test_counting_references_ignores_another_tenants(
self, session_factory: async_sessionmaker[AsyncSession]
) -> None:
"""This count decides whether a type may be deleted; another
- customer's references must not veto it."""
+ customer's references must not veto it.
+
+ The other tenant's type and reference are created on a session opened
+ with `tenant_session(other)`, separate from the session tenant zero's
+ row is arranged on. Entering `tenant_scope(other)` around statements
+ on that same session would rebind the contextvar without moving what
+ the transaction was stamped with at `after_begin` — the two would
+ disagree from the second statement on. The two counts are read back
+ the same way: each through its own session, opened inside the
+ binding it is asserting about, rather than by toggling `tenant_scope`
+ on one shared session between the two assertions.
+ """
other = f"tenant-{uuid.uuid4().hex[:8]}"
store = ReferenceTypeStore()
references = ReferenceStore()
@@ -388,11 +416,15 @@ class TestReferenceTypesAreScopedToTheBoundTenant:
ada = await _make_user(session, "ada")
await session.flush()
await store.create(session, _type("notion", ada.id))
- with tenant_scope(other):
- await store.create(session, _type("notion", ada.id))
- await references.create(session, _reference("notion", ada.id))
await session.commit()
+ async with tenant_session(session_factory, other) as other_session:
+ await store.create(other_session, _type("notion", ada.id))
+ await references.create(other_session, _reference("notion", ada.id))
+ await other_session.commit()
+
+ async with session_factory() as session:
assert await store.count_references_of_type(session, "notion") == 0
- with tenant_scope(other):
- assert await store.count_references_of_type(session, "notion") == 1
+
+ async with tenant_session(session_factory, other) as other_session:
+ assert await store.count_references_of_type(other_session, "notion") == 1