Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(gateway): bound workspaces created, under a lock on the creator

Petr Bauch6c32d4cdfcc8
diff --git a/core/switch_core/db/models.py b/core/switch_core/db/models.py
index 978e6c092..c31d70019 100644
--- a/core/switch_core/db/models.py
+++ b/core/switch_core/db/models.py
@@ -184,6 +184,13 @@ class User(Base):
created_at: Mapped[str] = mapped_column(
DateTime(timezone=True), server_default=func.now(), nullable=False
)
+ # Workspaces this person has created through self-service, which is what
+ # `GATEWAY_MAX_WORKSPACES_PER_USER` bounds. A count of creations rather
+ # than of current ownership, so handing a workspace to someone else does
+ # not free an allowance to create another.
+ workspaces_created: Mapped[int] = mapped_column(
+ Integer, server_default=text("0"), nullable=False
+ )
# An OIDC login matches an account on email case-insensitively (CHOO-2624):
diff --git a/core/switch_core/gateway/dependencies.py b/core/switch_core/gateway/dependencies.py
index c891bcd88..4e5aecbd4 100644
--- a/core/switch_core/gateway/dependencies.py
+++ b/core/switch_core/gateway/dependencies.py
@@ -119,7 +119,9 @@ async def get_system_session() -> AsyncIterator[AsyncSession]:
the way `get_current_user` does. `POST /tenants/{id}/switch`
(`gateway/tenants.py`) is a third, for a different reason — the caller is
authenticated (`get_authenticated_user_id`) but has, by construction, not
- yet selected the tenant this session opens for. Named separately from
+ yet selected the tenant this session opens for. `POST /tenants` is a
+ fourth: it locks the caller's `users` row, which carries no tenant, for
+ the length of a creation that has no tenant yet. Named separately from
`get_session` so that reads as a deliberate, reviewable exception rather
than an accidentally-unscoped session, and so the guard test above can
hold for `get_session` without exemptions. Nothing about opening it
diff --git a/core/switch_core/gateway/tenants.py b/core/switch_core/gateway/tenants.py
index 57e6b1464..4b2065dac 100644
--- a/core/switch_core/gateway/tenants.py
+++ b/core/switch_core/gateway/tenants.py
@@ -7,7 +7,8 @@ from datetime import UTC, datetime, timedelta
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Response
-from sqlalchemy.exc import IntegrityError
+from sqlalchemy import select
+from sqlalchemy.exc import DBAPIError, IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.bridges.agent.protocol.service import ProtocolService
@@ -201,13 +202,14 @@ async def list_tenants(
return await list_tenant_memberships(session_factory, user_store, user_id)
-async def _require_workspace_allowance(
- session_factory: async_sessionmaker[AsyncSession],
- user_store: UserStore,
- caller: AuthenticatedCaller,
- limit: int,
-) -> None:
- """Raise 403 unless `caller` may own one more workspace.
+_LOCK_NOT_AVAILABLE = "55P03"
+
+
+async def _lock_workspace_allowance(
+ session: AsyncSession, caller: AuthenticatedCaller, limit: int
+) -> User:
+ """Lock the caller's row and raise 403 unless they may create one more
+ workspace. Returns the locked row, for the caller to count the creation on.
Creating a workspace is not just a row: `all_tenant_ids()` drives a fan-out
per tenant at boot and a sweep every few seconds, so unbounded creation
@@ -216,33 +218,83 @@ async def _require_workspace_allowance(
route entirely, which is how a deployment that is not ready to offer
self-service says so.
- Operators are exempt before anything is read, both because the bypass is
- what `is_operator` means everywhere else in this codebase (`authz.py`) and
- because the person provisioning workspaces for other people is the one
- caller a self-service bound must not stop.
-
- Counts `owner` memberships, not memberships: an invitation into someone
- else's workspace must not spend an allowance, since the invitee cannot get
- it back without being removed. The count reuses `list_tenant_memberships`
- rather than asking the database for a number — there is no exemption that
- returns a caller's *roles* across tenants, only `tenants_of_user`, and
- adding one would mean a tenth `SECURITY DEFINER` lookup and a migration for
- an answer this bounds to a handful of rows by construction.
+ The bound is on workspaces created (`users.workspaces_created`), not on
+ workspaces owned: ownership can be handed to another account, so a count of
+ ownership could be reset by creating, handing over and stepping down, over
+ and over. An invitation into someone else's workspace spends nothing.
+
+ The row lock is what makes the check hold under concurrent requests. It
+ is taken `NOWAIT`, so a second request from the same person is refused
+ with 409 at once rather than holding a pool connection while it waits —
+ the request holding the lock needs more connections to provision, and a
+ burst of waiters could starve it of them. `FOR NO KEY UPDATE` rather than
+ `FOR UPDATE`, because the new owner membership's foreign key takes a key
+ share on this same row from another session, and `FOR UPDATE` would make
+ the request wait on itself.
"""
- if caller.is_operator:
- return
if limit == 0:
raise HTTPException(
status_code=403,
detail="Workspace creation is disabled on this deployment",
)
- memberships = await list_tenant_memberships(session_factory, user_store, caller.id)
- owned = sum(1 for membership in memberships if membership.role == "owner")
- if owned >= limit:
+ try:
+ user = await session.scalar(
+ select(User)
+ .where(User.id == caller.id)
+ .with_for_update(key_share=True, nowait=True)
+ )
+ except DBAPIError as exc:
+ if getattr(exc.orig, "sqlstate", None) != _LOCK_NOT_AVAILABLE:
+ raise
+ raise HTTPException(
+ status_code=409,
+ detail="Another workspace is being created for you; try again shortly",
+ ) from exc
+ if user is None:
+ raise HTTPException(status_code=401, detail="Unknown user")
+ if user.workspaces_created >= limit:
raise HTTPException(
status_code=403,
- detail=(f"You own {owned} workspaces, and this deployment allows {limit}"),
+ detail=(
+ f"You have created {user.workspaces_created} workspaces, and "
+ f"this deployment allows {limit}"
+ ),
+ )
+ return user
+
+
+async def _provision_workspace(
+ session_factory: async_sessionmaker[AsyncSession],
+ user_store: UserStore,
+ client_lifecycle: ClientLifecycleService,
+ caller: AuthenticatedCaller,
+ name: str,
+) -> Tenant:
+ slug = _derive_slug(name)
+ try:
+ tenant = await client_lifecycle.create_tenant(name, slug)
+ except IntegrityError as exc:
+ raise HTTPException(
+ status_code=409, detail=f"Slug already taken: {slug}"
+ ) from exc
+
+ try:
+ async with tenant_session(session_factory, tenant.id) as session:
+ await user_store.add_membership(
+ session, tenant_id=tenant.id, user_id=caller.id, role="owner"
+ )
+ await session.commit()
+ except Exception:
+ logger.error(
+ "Workspace %s (slug %s) was created but its owner membership for "
+ "user %s was not written: it now has no members and its slug is "
+ "taken. Insert the membership to repair it.",
+ tenant.id,
+ slug,
+ caller.id,
)
+ raise
+ return tenant
@router.post("/tenants", status_code=201)
@@ -255,11 +307,16 @@ async def create_tenant(
user_store: Annotated[UserStore, Depends(get_user_store)],
client_lifecycle: Annotated[ClientLifecycleService, Depends(get_client_lifecycle)],
config: Annotated[SwitchConfig, Depends(get_config)],
+ allowance: Annotated[AsyncSession, Depends(get_system_session)],
) -> TenantMembershipResponse:
"""Create a workspace. The caller becomes its `owner`.
- How many workspaces one person may own is bounded, and the bound is checked
- before anything is provisioned — see `_require_workspace_allowance`. This
+ How many workspaces one person may create is bounded, and the bound is
+ checked before anything is provisioned — see `_lock_workspace_allowance`.
+ Deployment operators are exempt, both because the bypass is what
+ `is_operator` means everywhere else in this codebase (`authz.py`) and
+ because the person provisioning workspaces for other people is the one
+ caller a self-service bound must not stop; nothing is counted for them. This
is the only route in this file with no tenant to authorize against, so the
limit is what stands in for the role check its neighbours have.
@@ -280,33 +337,19 @@ async def create_tenant(
— a 500 with the orphan unrecorded — is the silent degradation this
codebase refuses. An operator repairs it by inserting the membership.
"""
- await _require_workspace_allowance(
- session_factory, user_store, caller, config.gateway_max_workspaces_per_user
- )
- slug = _derive_slug(req.name)
- try:
- tenant = await client_lifecycle.create_tenant(req.name, slug)
- except IntegrityError as exc:
- raise HTTPException(
- status_code=409, detail=f"Slug already taken: {slug}"
- ) from exc
-
- try:
- async with tenant_session(session_factory, tenant.id) as session:
- await user_store.add_membership(
- session, tenant_id=tenant.id, user_id=caller.id, role="owner"
- )
- await session.commit()
- except Exception:
- logger.error(
- "Workspace %s (slug %s) was created but its owner membership for "
- "user %s was not written: it now has no members and its slug is "
- "taken. Insert the membership to repair it.",
- tenant.id,
- slug,
- caller.id,
+ if caller.is_operator:
+ tenant = await _provision_workspace(
+ session_factory, user_store, client_lifecycle, caller, req.name
)
- raise
+ else:
+ user = await _lock_workspace_allowance(
+ allowance, caller, config.gateway_max_workspaces_per_user
+ )
+ tenant = await _provision_workspace(
+ session_factory, user_store, client_lifecycle, caller, req.name
+ )
+ user.workspaces_created += 1
+ await allowance.commit()
return TenantMembershipResponse(
id=tenant.id, slug=tenant.slug, name=tenant.name, role="owner"
diff --git a/core/switch_core/migrations/versions/c4e9a1f7b203_users_workspaces_created.py b/core/switch_core/migrations/versions/c4e9a1f7b203_users_workspaces_created.py
new file mode 100644
index 000000000..ef0a49094
--- /dev/null
+++ b/core/switch_core/migrations/versions/c4e9a1f7b203_users_workspaces_created.py
@@ -0,0 +1,52 @@
+"""users.workspaces_created: the count the workspace-creation bound reads
+
+Revision ID: c4e9a1f7b203
+Revises: c4f7e2a90b13
+
+`GATEWAY_MAX_WORKSPACES_PER_USER` bounds how many workspaces one person may
+create. Counting the workspaces they currently own let the bound be reset by
+handing a workspace to a second account and stepping down, so the bound now
+reads a count of creations that only ever goes up.
+
+Existing people start from the workspaces they own today, which is what the
+bound counted until now, so nobody gains or loses an allowance when this runs.
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "c4e9a1f7b203"
+down_revision: str | Sequence[str] | None = "c4f7e2a90b13"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+ op.add_column(
+ "users",
+ sa.Column(
+ "workspaces_created",
+ sa.Integer(),
+ server_default=sa.text("0"),
+ nullable=False,
+ ),
+ )
+ op.execute(
+ """
+ UPDATE users
+ SET workspaces_created = owned.n
+ FROM (
+ SELECT user_id, count(*) AS n
+ FROM tenant_members
+ WHERE role = 'owner'
+ GROUP BY user_id
+ ) AS owned
+ WHERE users.id = owned.user_id
+ """
+ )
+
+
+def downgrade() -> None:
+ op.drop_column("users", "workspaces_created")
diff --git a/core/tests/switch_core/gateway/test_session_requires_authentication.py b/core/tests/switch_core/gateway/test_session_requires_authentication.py
index d362550c4..2c8b36c3d 100644
--- a/core/tests/switch_core/gateway/test_session_requires_authentication.py
+++ b/core/tests/switch_core/gateway/test_session_requires_authentication.py
@@ -49,6 +49,9 @@ _ROUTES_WITH_NO_TENANT_BOUND = {
("POST", "/auth/login"),
("GET", "/auth/oidc/callback"),
("POST", "/tenants/{tenant_id}/switch"),
+ # Locks the caller's `users` row, which has no tenant, while it creates a
+ # workspace that has none yet either.
+ ("POST", "/tenants"),
}
# Every route that does not resolve a tenant, for any reason. `get_current_user`
@@ -155,9 +158,9 @@ def test_every_route_taking_the_session_also_authenticates() -> None:
)
-def test_the_routes_with_no_tenant_bound_are_exactly_these_three() -> None:
+def test_the_routes_with_no_tenant_bound_are_exactly_these() -> None:
"""The exemption above is not a list to grow casually: `get_system_session`
- exists so that adding a fourth is a visible act, not a signature nobody
+ exists so that adding another is a visible act, not a signature nobody
reads."""
assert {
route.key for route in ROUTES if get_system_session in route.calls
diff --git a/core/tests/switch_core/gateway/test_tenant_api_routes.py b/core/tests/switch_core/gateway/test_tenant_api_routes.py
index 0991851dc..8dcd19bc1 100644
--- a/core/tests/switch_core/gateway/test_tenant_api_routes.py
+++ b/core/tests/switch_core/gateway/test_tenant_api_routes.py
@@ -16,6 +16,7 @@ depending on. The unique-slug behaviour under test comes from the same
from __future__ import annotations
+import asyncio
import uuid
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
@@ -303,7 +304,7 @@ class TestWorkspaceCreationLimit:
third = await client.post("/tenants", json={"name": "Third"})
assert third.status_code == 403
- assert "own 2 workspaces" in third.json()["detail"]
+ assert "created 2 workspaces" in third.json()["detail"]
async def test_nothing_is_provisioned_for_a_refused_caller(
self, session_factory: async_sessionmaker[AsyncSession]
@@ -327,6 +328,81 @@ class TestWorkspaceCreationLimit:
)
assert found.first() is None
+ async def test_handing_a_workspace_over_does_not_free_the_allowance(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ # What is bounded is creation. Were it ownership, creating, promoting a
+ # second account and stepping down would reset the count every time.
+ await _make_tenant(session_factory, TENANT_A)
+ user_id = await _make_member(
+ session_factory, name="hand-off", tenant_id=TENANT_A, role="member"
+ )
+ token = _token(user_id, "[email protected]", TENANT_A)
+
+ app = _app(session_factory, max_workspaces_per_user=1)
+ async with _client(app, token) as client:
+ first = await client.post("/tenants", json={"name": "Handed Over"})
+ async with session_factory() as session:
+ member = await session.get(TenantMember, (first.json()["id"], user_id))
+ assert member is not None
+ member.role = "member"
+ await session.commit()
+ second = await client.post("/tenants", json={"name": "One Too Many"})
+
+ assert first.status_code == 201
+ assert second.status_code == 403
+
+ async def test_concurrent_requests_cannot_all_pass_the_check(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ await _make_tenant(session_factory, TENANT_A)
+ user_id = await _make_member(
+ session_factory, name="burst", tenant_id=TENANT_A, role="member"
+ )
+ token = _token(user_id, "[email protected]", TENANT_A)
+
+ app = _app(session_factory, max_workspaces_per_user=1)
+ async with _client(app, token) as client:
+ responses = await asyncio.gather(
+ *(
+ client.post("/tenants", json={"name": f"Burst {i}"})
+ for i in range(5)
+ )
+ )
+
+ codes = sorted(r.status_code for r in responses)
+ assert codes.count(201) == 1
+ assert set(codes) - {201} <= {403, 409}
+ async with session_factory() as session:
+ owned = await session.scalars(
+ select(TenantMember).where(
+ TenantMember.user_id == user_id, TenantMember.role == "owner"
+ )
+ )
+ assert len(owned.all()) == 1
+
+ async def test_a_request_during_another_creation_is_refused_not_queued(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ # Waiting would hold a pool connection for as long as the other
+ # creation takes; refusing at once holds none.
+ await _make_tenant(session_factory, TENANT_A)
+ user_id = await _make_member(
+ session_factory, name="in-flight", tenant_id=TENANT_A, role="member"
+ )
+ token = _token(user_id, "[email protected]", TENANT_A)
+
+ app = _app(session_factory, max_workspaces_per_user=3)
+ async with session_factory() as holder:
+ await holder.scalar(
+ select(User).where(User.id == user_id).with_for_update(key_share=True)
+ )
+ async with _client(app, token) as client:
+ response = await client.post("/tenants", json={"name": "Queued"})
+
+ assert response.status_code == 409
+ assert "being created" in response.json()["detail"]
+
async def test_a_limit_of_zero_says_creation_is_disabled(
self, session_factory: async_sessionmaker[AsyncSession]
) -> None:
diff --git a/docs/old/multi-tenancy-phase2-tenants.md b/docs/old/multi-tenancy-phase2-tenants.md
index 37efb6d9c..65f6ae5b2 100644
--- a/docs/old/multi-tenancy-phase2-tenants.md
+++ b/docs/old/multi-tenancy-phase2-tenants.md
@@ -202,8 +202,13 @@ a sweep every few seconds, across clients, collaboration bridges, server
connectors and rooms, so unbounded self-service creation buys steady-state work
in the deployment, not storage. `GATEWAY_MAX_WORKSPACES_PER_USER` is both the
cap and the gate: at the limit `POST /tenants` is a 403, and `0` closes the
-route outright. It counts `owner` memberships only, so being invited into
-someone else's workspace never spends an allowance the invitee cannot get back.
+route outright. It counts workspaces a person has *created*
+(`users.workspaces_created`), not ones they own: ownership can be handed to a
+second account, so a count of ownership could be reset by creating, handing
+over and stepping down. Being invited into someone else's workspace spends
+nothing. The check, the creation and the count run under a lock on the
+person's `users` row, so concurrent requests cannot all pass a check that only
+one of them should.
Deployment operators are exempt, on the same grounds as every other operator
bypass — this bounds self-service, and an operator provisioning workspaces for
other people is not that.