Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
fix(organizations): make the WorkOS mirror recover, fence it, keep conflicts
Petr Bauchae44f61e5c5b
diff --git a/core/switch_core/db/audit.py b/core/switch_core/db/audit.pyindex 7e194faa7..ddf7b6785 100644--- a/core/switch_core/db/audit.py+++ b/core/switch_core/db/audit.py@@ -101,6 +101,10 @@ class ServerAuditAction(StrEnum):CONNECTOR_APP_SAVED = "connector_app.saved"CONNECTOR_APP_IMPORTED = "connector_app.imported"CONNECTOR_APP_REMOVED = "connector_app.removed"+ # WorkOS refused where the directory mirror had got to in its events, and+ # the mirror started again from now, with a full check of every linked+ # organization.+ DIRECTORY_EVENTS_RESTARTED = "directory.events_restarted"async def record_audit_event(diff --git a/core/switch_core/gateway/organizations.py b/core/switch_core/gateway/organizations.pyindex 0cbd02fd9..0ea8b9e68 100644--- a/core/switch_core/gateway/organizations.py+++ b/core/switch_core/gateway/organizations.py@@ -479,7 +479,6 @@ async def _end_access_through_organization(reached=reached,actor_id=actor_id,details={"reason": reason},- keep_agent_keys=False,)diff --git a/core/switch_core/gateway/tenants.py b/core/switch_core/gateway/tenants.pyindex f463ca1ba..6612b689a 100644--- a/core/switch_core/gateway/tenants.py+++ b/core/switch_core/gateway/tenants.py@@ -124,15 +124,12 @@ from switch_core.gateway.schemas import (UsageTotalResponse,)from switch_core.gateway.service_connections import service_refusal-from switch_core.gateway.workspace_access import (- finish_revocation,- revoke_workspace_access,-)from switch_core.gateway.workspace_creation import (NOTHING_CHANGED,create_organization_for,directory_failed,)+from switch_core.organization_access import AccessRevocation, remove_membershipfrom switch_core.organization_directory import DirectoryError, OrganizationDirectoryfrom switch_core.telemetry import emit_safelyfrom switch_core.telemetry.ages import age_hours@@ -1463,35 +1460,25 @@ async def remove_member(),)+ revocation = AccessRevocation(+ agent_store=agent_store,+ api_key_store=api_key_store,+ broker=broker,+ protocol=protocol,+ sign_in_flows=request.app.state.sign_in_flows,+ )try:- revoked = await revoke_workspace_access(+ revoked = await remove_membership(session,- user_id,- api_key_store=api_key_store,- broker=broker,- sign_in_flows=request.app.state.sign_in_flows,+ revocation,+ membership,+ actor_id=admin.id,+ details={},keep_agent_keys=False,)except ServiceError as error:return service_refusal(error)-- await record_audit_event(- session,- tenant_id=tenant_id,- actor_user_id=admin.id,- action=AuditAction.MEMBER_REMOVED,- target_type="user",- target_id=user_id,- details={- "role": membership.role,- "api_keys_deleted": len(revoked.key_hashes),- "service_connections_deleted": len(revoked.removed_services),- },- )- await session.delete(membership)await session.commit()- warning = await finish_revocation(- session, user_id, revoked, protocol=protocol, broker=broker- )+ warning = await revocation.finish(session, user_id, revoked)return {"ok": True, "warning": warning}diff --git a/core/switch_core/gateway/workspace_access.py b/core/switch_core/gateway/workspace_access.pyindex 7093b8d9c..c12ed72b9 100644--- a/core/switch_core/gateway/workspace_access.py+++ b/core/switch_core/gateway/workspace_access.py@@ -43,18 +43,22 @@ async def revoke_workspace_access(when a connection is busy, before anything is removed. The callercommits, then calls `finish_revocation`.- `keep_agent_keys` keeps the keys agents they own run on, for a caller- that leaves those agents in place rather than refusing while they exist.+ `keep_agent_keys` keeps the agent keys (`type` "agent") agents run on,+ for a caller that leaves those agents in place rather than refusing while+ they exist. Such a key authenticates as its agent, never as the person+ (`bridges/agent/auth.py`), so keeping it leaves the agent working and the+ person nothing of their own; every other key of theirs goes."""removed_services = await broker.remove_connections(session, user_id)keys = await api_key_store.get_by_user(session, user_id)- if keep_agent_keys and keys:+ if keep_agent_keys:+ agent_keys = [key.id for key in keys if key.type == "agent"]in_use = set(await session.scalars(- select(Agent.api_key_id).where(- Agent.api_key_id.in_([key.id for key in keys])- )+ select(Agent.api_key_id).where(Agent.api_key_id.in_(agent_keys)))+ if agent_keys+ else ())keys = [key for key in keys if key.id not in in_use]for key in keys:diff --git a/core/switch_core/main.py b/core/switch_core/main.pyindex 950ec3735..4cf8dd370 100644--- a/core/switch_core/main.py+++ b/core/switch_core/main.py@@ -202,6 +202,7 @@ from switch_core.observability.pool import (from switch_core.observability.query import instrument_queriesfrom switch_core.organization_access import AccessRevocationfrom switch_core.organization_directory import LocalDirectory, OrganizationDirectory+from switch_core.organization_mirror import MirrorDependenciesfrom switch_core.outbound import guarded_async_clientfrom switch_core.ownership import Ownershipfrom switch_core.process_identity import new_process_id@@ -1419,13 +1420,18 @@ async def run(config: SwitchConfig, process_id: str) -> None:WorkOSMirror(session_factory,organization_directory,- AccessRevocation(- agent_store=agent_store,- api_key_store=api_key_store,- broker=service_broker,- protocol=protocol,- sign_in_flows=gateway_app.state.sign_in_flows,+ MirrorDependencies(+ revocation=AccessRevocation(+ agent_store=agent_store,+ api_key_store=api_key_store,+ broker=service_broker,+ protocol=protocol,+ sign_in_flows=gateway_app.state.sign_in_flows,+ ),+ organization_store=organization_store,+ user_store=user_store,),+ OwnershipLeaseStore(),config.gateway_oidc_issuer_url,),)diff --git a/core/switch_core/organization_access.py b/core/switch_core/organization_access.pyindex dd02dea80..526708104 100644--- a/core/switch_core/organization_access.py+++ b/core/switch_core/organization_access.py@@ -1,10 +1,15 @@"""Ending someone's access to an organization's workspaces: what being demoted-from `org_admin`, or leaving the organization, takes away.+from `org_admin`, or leaving a workspace or the organization, takes away.-Shared by the organization routes (`gateway/organizations.py`), which refuse-first while the person owns agents there, and the directory mirror-(`organization_mirror.py`), which applies a change made in the directory-whatever it breaks and reports what it broke.+Shared by the workspace and organization routes (`gateway/tenants.py`,+`gateway/organizations.py`), which refuse first while the person owns agents+there, and the directory mirror (`organization_mirror.py`), which applies a+change made in the directory whatever it breaks and reports what it broke.++Each step works on one workspace's session and leaves the commit to the+caller, so whatever else belongs with it (a refusal, a conflict recorded for+the organization) commits or rolls back with it. After the commit, the caller+calls `finish_revocation` with what the step returned."""from __future__ import annotations@@ -23,6 +28,7 @@ from switch_core.db.session_scope import tenant_sessionfrom switch_core.db.stores.agent_store import AgentStorefrom switch_core.db.stores.api_key_store import ApiKeyStorefrom switch_core.gateway.workspace_access import (+ RevokedAccess,finish_revocation,revoke_workspace_access,)@@ -38,6 +44,14 @@ class AccessRevocation:protocol: AgentCoresign_in_flows: SignInFlows+ async def finish(+ self, session: AsyncSession, user_id: str, revoked: RevokedAccess+ ) -> str | None:+ """`finish_revocation`, once the step's transaction has committed."""+ return await finish_revocation(+ session, user_id, revoked, protocol=self.protocol, broker=self.broker+ )+async def workspaces_where_they_own_agents(session_factory: async_sessionmaker[AsyncSession],@@ -55,109 +69,113 @@ async def workspaces_where_they_own_agents(return found+async def end_organization_admin_access(+ session: AsyncSession,+ revocation: AccessRevocation,+ *,+ tenant_id: str,+ user_id: str,+ actor_id: str | None,+ details: dict[str, Any],+ keep_agent_keys: bool,+) -> RevokedAccess:+ """In workspace `tenant_id`, bound on `session`, where `user_id` stands+ only as their organization's `org_admin`: remove their keys, service+ connections and sign-ins under way, recorded in its audit log with+ `details`; `keep_agent_keys` as `revoke_workspace_access` has it. Raises+ `ServiceError` when a connection is busy."""+ revoked = await revoke_workspace_access(+ session,+ user_id,+ api_key_store=revocation.api_key_store,+ broker=revocation.broker,+ sign_in_flows=revocation.sign_in_flows,+ keep_agent_keys=keep_agent_keys,+ )+ await record_audit_event(+ session,+ tenant_id=tenant_id,+ actor_user_id=actor_id,+ action=AuditAction.ORGANIZATION_ADMIN_ACCESS_ENDED,+ target_type="user",+ target_id=user_id,+ details={+ **details,+ "api_keys_deleted": len(revoked.key_hashes),+ "service_connections_deleted": len(revoked.removed_services),+ },+ )+ return revoked+++async def remove_membership(+ session: AsyncSession,+ revocation: AccessRevocation,+ membership: TenantMember,+ *,+ actor_id: str | None,+ details: dict[str, Any],+ keep_agent_keys: bool,+) -> RevokedAccess:+ """Remove `membership` from its workspace, bound on `session`, with the+ member's keys, service connections and sign-ins under way there, recorded+ in its audit log with `details`; `keep_agent_keys` as+ `revoke_workspace_access` has it. Agents they own are left as they are.+ Raises `ServiceError` when a connection is busy, before anything is+ removed."""+ revoked = await revoke_workspace_access(+ session,+ membership.user_id,+ api_key_store=revocation.api_key_store,+ broker=revocation.broker,+ sign_in_flows=revocation.sign_in_flows,+ keep_agent_keys=keep_agent_keys,+ )+ await record_audit_event(+ session,+ tenant_id=membership.tenant_id,+ actor_user_id=actor_id,+ action=AuditAction.MEMBER_REMOVED,+ target_type="user",+ target_id=membership.user_id,+ details={+ **details,+ "role": membership.role,+ "api_keys_deleted": len(revoked.key_hashes),+ "service_connections_deleted": len(revoked.removed_services),+ },+ )+ await session.delete(membership)+ return revoked++async def end_access_through_organization(session_factory: async_sessionmaker[AsyncSession],revocation: AccessRevocation,*,user_id: str,reached: list[Tenant],- actor_id: str | None,+ actor_id: str,details: dict[str, Any],- keep_agent_keys: bool,) -> str | None:- """Take away what `user_id` holds in `reached`, the workspaces they stand- in only as their organization's `org_admin`: their keys, service- connections and sign-ins under way, one workspace at a time, each- recorded in that workspace's audit log with `details`;- `keep_agent_keys` as `revoke_workspace_access` has it. Raises- `ServiceError` when a connection is busy. Returns a warning naming what- could not be revoked at a vendor, or None."""+ """`end_organization_admin_access` in each of `reached`, one workspace at+ a time, each committed on its own, for a route that has refused already+ while they own agents there. Returns a warning naming what could not be+ revoked at a vendor, or None."""warnings = []for tenant in reached:async with tenant_session(session_factory, tenant.id) as session:- revoked = await revoke_workspace_access(- session,- user_id,- api_key_store=revocation.api_key_store,- broker=revocation.broker,- sign_in_flows=revocation.sign_in_flows,- keep_agent_keys=keep_agent_keys,- )- await record_audit_event(+ revoked = await end_organization_admin_access(session,+ revocation,tenant_id=tenant.id,- actor_user_id=actor_id,- action=AuditAction.ORGANIZATION_ADMIN_ACCESS_ENDED,- target_type="user",- target_id=user_id,- details={- **details,- "api_keys_deleted": len(revoked.key_hashes),- "service_connections_deleted": len(revoked.removed_services),- },+ user_id=user_id,+ actor_id=actor_id,+ details=details,+ keep_agent_keys=False,)await session.commit()- warning = await finish_revocation(- session,- user_id,- revoked,- protocol=revocation.protocol,- broker=revocation.broker,- )+ warning = await revocation.finish(session, user_id, revoked)if warning:warnings.append(f"{tenant.name}: {warning}")return " ".join(warnings) or None---async def remove_from_workspace(- session_factory: async_sessionmaker[AsyncSession],- revocation: AccessRevocation,- *,- user_id: str,- tenant: Tenant,- actor_id: str | None,- details: dict[str, Any],- keep_agent_keys: bool,-) -> str | None:- """Remove `user_id` from `tenant`, with their keys, service connections- and sign-ins under way there, recorded in its audit log with `details`.- Agents they own are left as they are, and with `keep_agent_keys` so are- the keys they run on. Raises `ServiceError` when a- connection is busy. Returns a warning naming what could not be revoked- at a vendor, or None."""- async with tenant_session(session_factory, tenant.id) as session:- membership = await session.get(TenantMember, (tenant.id, user_id))- if membership is None:- return None- revoked = await revoke_workspace_access(- session,- user_id,- api_key_store=revocation.api_key_store,- broker=revocation.broker,- sign_in_flows=revocation.sign_in_flows,- keep_agent_keys=keep_agent_keys,- )- await record_audit_event(- session,- tenant_id=tenant.id,- actor_user_id=actor_id,- action=AuditAction.MEMBER_REMOVED,- target_type="user",- target_id=user_id,- details={- **details,- "role": membership.role,- "api_keys_deleted": len(revoked.key_hashes),- "service_connections_deleted": len(revoked.removed_services),- },- )- await session.delete(membership)- await session.commit()- warning = await finish_revocation(- session,- user_id,- revoked,- protocol=revocation.protocol,- broker=revocation.broker,- )- return f"{tenant.name}: {warning}" if warning else Nonediff --git a/core/switch_core/organization_mirror.py b/core/switch_core/organization_mirror.pyindex 7cdefe2ba..4ded90840 100644--- a/core/switch_core/organization_mirror.py+++ b/core/switch_core/organization_mirror.py@@ -30,9 +30,10 @@ events and its daily check (`apply_from_directory`).from __future__ import annotationsimport logging+from dataclasses import dataclassfrom typing import Any-from sqlalchemy import delete, func, select, update+from sqlalchemy import delete, select, updatefrom sqlalchemy.dialects.postgresql import insertfrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker@@ -48,6 +49,8 @@ from switch_core.db.models import (TenantMember,)from switch_core.db.session_scope import organization_session, tenant_session+from switch_core.db.stores.organization_store import OrganizationStore+from switch_core.db.stores.user_store import UserStorefrom switch_core.db.tenant_lookup import directory_organizations_of_userfrom switch_core.gateway.auth import (workspaces_in_organization,@@ -55,9 +58,8 @@ from switch_core.gateway.auth import ()from switch_core.organization_access import (AccessRevocation,- end_access_through_organization,- remove_from_workspace,- workspaces_where_they_own_agents,+ end_organization_admin_access,+ remove_membership,)from switch_core.organization_directory import DirectoryMembership@@ -103,80 +105,128 @@ async def bring_in_memberships()+@dataclass(frozen=True)+class MirrorDependencies:+ """What applying a change from the directory needs."""++ revocation: AccessRevocation+ organization_store: OrganizationStore+ user_store: UserStore++async def apply_from_directory(session_factory: async_sessionmaker[AsyncSession],- revocation: AccessRevocation,+ dependencies: MirrorDependencies,*,organization_id: str,+ workos_organization_id: str,user_id: str,wanted: str | None,source: str,) -> None:"""Make `user_id`'s membership of `organization_id` what the directory- says, `wanted` (a role, or None for no membership). Raises `ServiceError`- when a connection to revoke is busy, having changed nothing that a retry- would not finish."""+ says, `wanted` (a role, or None for no membership), as read from WorkOS+ organization `workos_organization_id`. Nothing is changed unless the+ organization still follows that one.++ Each workspace is worked on in a transaction of its own, with the+ conflicts it raises recorded in the same one, so a retry after a failure+ part-way (`ServiceError`, a busy connection) finds the workspaces already+ done done, and their conflicts recorded.+ """async with organization_session(session_factory, organization_id) as session:- _, current = await _link_and_role(session, organization_id, user_id)+ linked, current = await _link_and_role(session, organization_id, user_id)+ if linked != workos_organization_id:+ logger.warning(+ "Organization %s no longer follows WorkOS organization %s; "+ "user %s's membership there is not applied",+ organization_id,+ workos_organization_id,+ user_id,+ )+ returnif current == wanted:return- if _grants(current, wanted):- assert wanted is not None+ if wanted is not None and _grants(current, wanted):await _grant(session, organization_id, user_id, current, wanted, source)return- admins = await _count_admins(session, organization_id)assert current is not None- conflicts: list[dict[str, Any]] = []- warnings: list[str] = []details = {"source": source}+ revocation = dependencies.revocation+ warnings: list[str] = []if current == ORGANIZATION_ADMIN_ROLE:- if admins <= 1:- conflicts.append({"kind": "last_org_admin"})- reached = await workspaces_reached_through_organization(+ reason = "demoted" if wanted is not None else "removed"+ for tenant in await workspaces_reached_through_organization(session_factory, user_id, organization_id- )- conflicts += await _agents_left(session_factory, revocation, user_id, reached)- warning = await end_access_through_organization(- session_factory,- revocation,- user_id=user_id,- reached=reached,- actor_id=None,- details={- "reason": "demoted" if wanted is not None else "removed",- **details,- },- keep_agent_keys=True,- )- if warning:- warnings.append(warning)+ ):+ async with tenant_session(session_factory, tenant.id) as session:+ await _conflict_if_agents_left(+ session, dependencies, organization_id, user_id, tenant, details+ )+ revoked = await end_organization_admin_access(+ session,+ revocation,+ tenant_id=tenant.id,+ user_id=user_id,+ actor_id=None,+ details={"reason": reason, **details},+ keep_agent_keys=True,+ )+ await session.commit()+ warning = await revocation.finish(session, user_id, revoked)+ if warning:+ warnings.append(f"{tenant.name}: {warning}")if wanted is None:- workspaces = await workspaces_in_organization(+ for tenant in await workspaces_in_organization(session_factory, user_id, organization_id- )- conflicts += await _agents_left(- session_factory, revocation, user_id, workspaces- )- for tenant in workspaces:- if await _is_last_owner(session_factory, tenant.id, user_id):- conflicts.append(- {"kind": "workspace_left_without_owner", "workspace": tenant.name}+ ):+ async with tenant_session(session_factory, tenant.id) as session:+ membership = await session.get(TenantMember, (tenant.id, user_id))+ if membership is None:+ continue+ await _conflict_if_agents_left(+ session, dependencies, organization_id, user_id, tenant, details)- warning = await remove_from_workspace(- session_factory,- revocation,- user_id=user_id,- tenant=tenant,- actor_id=None,- details=details,- keep_agent_keys=True,- )+ if (+ membership.role == "owner"+ and await dependencies.user_store.count_owners(session) <= 1+ ):+ await _conflict(+ session,+ organization_id,+ user_id,+ {+ **details,+ "kind": "workspace_left_without_owner",+ "workspace": tenant.name,+ },+ )+ revoked = await remove_membership(+ session,+ revocation,+ membership,+ actor_id=None,+ details=details,+ keep_agent_keys=True,+ )+ await session.commit()+ warning = await revocation.finish(session, user_id, revoked)if warning:- warnings.append(warning)+ warnings.append(f"{tenant.name}: {warning}")async with organization_session(session_factory, organization_id) as session:+ admins = await dependencies.organization_store.lock_admins(+ session, organization_id+ )+ _, current = await _link_and_role(session, organization_id, user_id)+ if current is None or current == wanted:+ return+ if current == ORGANIZATION_ADMIN_ROLE and admins <= 1:+ await _conflict(+ session, organization_id, user_id, {**details, "kind": "last_org_admin"}+ )if wanted is None:await session.execute(delete(OrganizationMember).where(@@ -202,23 +252,6 @@ async def apply_from_directory(**({"warning": " ".join(warnings)} if warnings else {}),},)- for conflict in conflicts:- logger.error(- "Applying WorkOS's change to user %s in organization %s broke "- "a rule Switch keeps: %s",- user_id,- organization_id,- conflict,- )- await record_organization_audit_event(- session,- organization_id=organization_id,- actor_user_id=None,- action=OrganizationAuditAction.DIRECTORY_CONFLICT,- target_type="user",- target_id=user_id,- details={**details, **conflict},- )await session.commit()@@ -298,43 +331,48 @@ async def _set_role()-async def _count_admins(session: AsyncSession, organization_id: str) -> int:- return (- await session.scalar(- select(func.count()).where(- OrganizationMember.organization_id == organization_id,- OrganizationMember.role == ORGANIZATION_ADMIN_ROLE,- )- )- ) or 0---async def _agents_left(- session_factory: async_sessionmaker[AsyncSession],- revocation: AccessRevocation,+async def _conflict_if_agents_left(+ session: AsyncSession,+ dependencies: MirrorDependencies,+ organization_id: str,user_id: str,- workspaces: list[Tenant],-) -> list[dict[str, Any]]:- """A conflict for each of `workspaces` where `user_id` owns agents, which- stay, with the keys they run on, owned by someone who can no longer act- there, for an admin to deal with."""- return [- {"kind": "agents_left_with_owner", "workspace": tenant.name}- for tenant in await workspaces_where_they_own_agents(- session_factory, revocation.agent_store, user_id, workspaces+ tenant: Tenant,+ details: dict[str, Any],+) -> None:+ """Agents `user_id` owns in `tenant`, bound on `session`, stay, with the+ keys they run on, owned by someone who can no longer act there, for an+ admin to deal with."""+ if await dependencies.revocation.agent_store.get_by_owner(session, user_id):+ await _conflict(+ session,+ organization_id,+ user_id,+ {**details, "kind": "agents_left_with_owner", "workspace": tenant.name},)- ]-async def _is_last_owner(- session_factory: async_sessionmaker[AsyncSession], tenant_id: str, user_id: str-) -> bool:- async with tenant_session(session_factory, tenant_id) as session:- owners = list(- await session.scalars(- select(TenantMember.user_id).where(- TenantMember.tenant_id == tenant_id, TenantMember.role == "owner"- )- )- )- return owners == [user_id]+async def _conflict(+ session: AsyncSession,+ organization_id: str,+ user_id: str,+ details: dict[str, Any],+) -> None:+ """Record, in the caller's transaction, that a change from the directory+ broke a rule Switch keeps. `session` may be bound to the organization or+ to one of its workspaces."""+ logger.error(+ "Applying WorkOS's change to user %s in organization %s broke a rule "+ "Switch keeps: %s",+ user_id,+ organization_id,+ details,+ )+ await record_organization_audit_event(+ session,+ organization_id=organization_id,+ actor_user_id=None,+ action=OrganizationAuditAction.DIRECTORY_CONFLICT,+ target_type="user",+ target_id=user_id,+ details=details,+ )diff --git a/core/switch_core/workos_directory.py b/core/switch_core/workos_directory.pyindex 3dbb9b052..944d2202f 100644--- a/core/switch_core/workos_directory.py+++ b/core/switch_core/workos_directory.py@@ -74,6 +74,19 @@ def switch_role(workos_role: str) -> str:)+class WorkOSRefused(DirectoryUnavailable):+ """WorkOS answered, with something other than 200."""++ def __init__(self, message: str, status_code: int) -> None:+ super().__init__(message)+ self.status_code = status_code+++class EventCursorRejected(DirectoryUnavailable):+ """WorkOS refused where the events were asked from: an event id or a+ start time it no longer keeps."""++def _unreadable(what: str) -> DirectoryUnavailable:return DirectoryUnavailable(f"WorkOS answered with {what} Switch cannot read.")@@ -99,8 +112,9 @@ class WorkOSDirectory:f"WorkOS could not be reached ({type(error).__name__}).") from errorif response.status_code != 200:- raise DirectoryUnavailable(- f"WorkOS answered {response.status_code} for {path}."+ raise WorkOSRefused(+ f"WorkOS answered {response.status_code} for {path}.",+ response.status_code,)try:body = response.json()@@ -201,7 +215,10 @@ class WorkOSDirectory:) -> list[dict[str, Any]]:"""One page of events of `types`, oldest first: those after event`after`, or from `range_start` (ISO 8601) when there is none yet.- Empty once there are no more."""+ Empty once there are no more. Each event is returned as WorkOS sent+ it, for the caller to skip one it can't read rather than lose the+ page. Raises `EventCursorRejected` when WorkOS refuses `after` or+ `range_start`."""params: dict[str, str | list[str]] = {"events": list(types),"limit": str(_PAGE),@@ -211,21 +228,18 @@ class WorkOSDirectory:params["after"] = afterelse:params["range_start"] = range_start- page = await self._get("/events", params)+ try:+ page = await self._get("/events", params)+ except WorkOSRefused as error:+ if error.status_code in (400, 404, 422):+ raise EventCursorRejected(error.message) from error+ raisedata = page.get("data")- if not isinstance(data, list):+ if not isinstance(data, list) or not all(+ isinstance(event, dict) for event in data+ ):raise _unreadable("a list of events")- events = []- for event in data:- if (- not isinstance(event, dict)- or not isinstance(event.get("id"), str)- or not isinstance(event.get("event"), str)- or not isinstance(event.get("data"), dict)- ):- raise _unreadable("an event")- events.append(event)- return events+ return dataasync def organization(self, workos_organization_id: str) -> dict[str, Any]:"""The WorkOS organization `workos_organization_id`, for an operatordiff --git a/core/switch_core/workos_mirror.py b/core/switch_core/workos_mirror.pyindex e3adc4dff..af4010383 100644--- a/core/switch_core/workos_mirror.py+++ b/core/switch_core/workos_mirror.py@@ -7,20 +7,34 @@ in WorkOS to a linked organization's members, applied in Switchresumable, and with no public endpoint or signing secret. An event is onlya prompt: applying it re-reads the person's current membership fromWorkOS, so events read out of order, twice, or after a later change all- end in the same state. An event that fails to apply stops the pass, and the- next pass starts again from it.+ end in the same state.+- **Failures.** An event that fails to apply stops the pass, and the next+ pass tries it again, up to `MAX_EVENT_ATTEMPTS` times. Then it is passed+ over and its organization is re-checked in full on every pass until that+ succeeds, so a removal waits on the failure, not on the next daily check.+ An event Switch can't read is logged and passed over.- **The daily check.** Every `CHECK_INTERVAL_SECONDS`, and when the mirrorstarts, each linked organization's members in WorkOS are compared with- Switch's: the backstop for events missed (the Events API keeps 90 days) or- from before the mirror ran. A Switch member with no WorkOS identity can't- be compared, and is logged and left.+ Switch's and the differences applied: the backstop for events missed or+ from before the mirror ran. One member or organization failing doesn't+ stop the rest; an organization with a failure is re-checked on every pass.+ A Switch member with no WorkOS identity can't be compared, and is logged+ and left.+- **Where the events are read from.** Until the first event, from when the+ mirror started, moved forward on every empty pass so it stays inside what+ WorkOS keeps (90 days). If WorkOS refuses the cursor anyway (the mirror+ was stopped for longer), the mirror starts again from now, says so in the+ log and the server's audit log, and checks every organization in full.People are matched to WorkOS users by the sign-in identity linking them(`oidc_identities`, the gateway's issuer and their WorkOS user id). SomeoneWorkOS lists who has never signed in to Switch has no account to add, and isbrought in when they first sign in.-One process runs it (`ownership.py`, kind `workos_mirror`).+One process runs it (`ownership.py`, kind `workos_mirror`), under the fence+it was started with: each change and each move of the cursor is made only+while that hold is still this process's, so a holder taken over without+knowing it stops rather than writing beside the new one."""from __future__ import annotations@@ -28,37 +42,44 @@ from __future__ import annotationsimport asyncioimport contextlibimport logging-from datetime import UTC, datetime+from collections.abc import Awaitable+from datetime import UTC, datetime, timedeltafrom typing import Any-from sqlalchemy import select+from sqlalchemy import exists, select, updatefrom sqlalchemy.dialects.postgresql import insertfrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker-from switch_core.connections.broker import ServiceErrorfrom switch_core.db.audit import (OrganizationAuditAction,+ ServerAuditAction,record_organization_audit_event,+ record_server_audit_event,)from switch_core.db.models import (DirectoryEventCursor,OidcIdentity,Organization,OrganizationMember,+ OwnershipLease,)from switch_core.db.session_scope import organization_session+from switch_core.db.stores.ownership_lease_store import OwnershipLeaseStorefrom switch_core.db.tenant_lookup import directory_organizations-from switch_core.organization_access import AccessRevocation-from switch_core.organization_directory import DirectoryUnavailable-from switch_core.organization_mirror import apply_from_directory+from switch_core.organization_mirror import MirrorDependencies, apply_from_directoryfrom switch_core.ownership import Fence, OwnedKind, Ownershipfrom switch_core.tenant_context import no_tenant-from switch_core.workos_directory import WorkOSDirectory+from switch_core.workos_directory import EventCursorRejected, WorkOSDirectorylogger = logging.getLogger(__name__)POLL_INTERVAL_SECONDS = 30.0CHECK_INTERVAL_SECONDS = 24 * 3600.0+MAX_EVENT_ATTEMPTS = 5+# How far back an empty pass leaves where the events are read from, so an+# event WorkOS records a little after the time it carries is not missed.+# Reading one twice changes nothing.+_START_OVERLAP = timedelta(minutes=5)CURSOR_SOURCE = "workos"EVENT_TYPES = ("organization_membership.created",@@ -73,112 +94,216 @@ MIRROR_KIND = "workos_mirror"MIRROR_KEY = "all"+class TakenOver(Exception):+ """The fence the mirror runs under is no longer this process's."""++class WorkOSMirror:def __init__(self,session_factory: async_sessionmaker[AsyncSession],directory: WorkOSDirectory,- revocation: AccessRevocation,+ dependencies: MirrorDependencies,+ leases: OwnershipLeaseStore,issuer: str,) -> None:self._session_factory = session_factoryself._directory = directory- self._revocation = revocation+ self._dependencies = dependencies+ self._leases = leasesself._issuer = issuer+ self._attempts: dict[str, int] = {}+ # Organizations to check in full on every pass until that succeeds.+ self._recheck: set[str] = set()- async def poll_once(self) -> int:- """Apply the events since the cursor, page by page, moving the cursor- past each one applied. Returns how many were read."""+ async def poll_once(self, fence: Fence) -> int:+ """Re-check what is due, then apply the events since the cursor, page+ by page, moving the cursor past each one. Returns how many events+ were read."""cursor = await self._cursor()+ if self._recheck:+ await self._check_all(fence, await self._linked(), only=set(self._recheck))+ polled_at = datetime.now(UTC)+ try:+ events = await self._directory.events(+ EVENT_TYPES,+ after=cursor.after,+ range_start=cursor.started_at.isoformat(),+ )+ except EventCursorRejected as error:+ await self._restart(fence, cursor, error.message)+ return 0+ if not events:+ if cursor.after is None:+ await self._write_cursor(+ fence, started_at=polled_at - _START_OVERLAP, after=None+ )+ return 0linked = await self._linked()+ following = {workos: own for own, workos in linked.items()}read = 0after = cursor.after- while True:+ while events:+ for event in events:+ event_id = event.get("id")+ if not isinstance(event_id, str):+ logger.error(+ "WorkOS sent an event with no id; passed over: %r", event+ )+ continue+ await self._apply_event_or_pass_over(fence, event_id, event, following)+ await self._write_cursor(fence, started_at=None, after=event_id)+ after = event_id+ read += 1+ if after is None:+ return readevents = await self._directory.events(EVENT_TYPES, after=after, range_start=cursor.started_at.isoformat())- if not events:- return read- for event in events:- await self._apply_event(event, linked)- after = event["id"]- await self._advance(after)- read += 1+ return read- async def check_once(self) -> None:+ async def check_once(self, fence: Fence) -> None:"""Compare every linked organization's members with WorkOS's, and- apply the difference. One organization failing doesn't stop the- others."""- for organization_id, workos_organization_id in (await self._linked()).items():- try:- await self._check(organization_id, workos_organization_id)- except (DirectoryUnavailable, ServiceError) as error:- logger.error(- "Daily check of organization %s against WorkOS stopped: %s",- organization_id,- error,- )+ apply the differences."""+ await self._check_all(fence, await self._linked(), only=None)- async def run(self) -> None:+ async def run(self, fence: Fence) -> None:"""Poll the events every `POLL_INTERVAL_SECONDS`, and check every- organization when starting and every `CHECK_INTERVAL_SECONDS`."""+ organization when starting and every `CHECK_INTERVAL_SECONDS`, until+ the fence is lost."""with no_tenant():since_check = CHECK_INTERVAL_SECONDSwhile True:- if since_check >= CHECK_INTERVAL_SECONDS:- since_check = 0.0- await self._logged(self.check_once(), "daily check")- await self._logged(self.poll_once(), "events")+ try:+ if since_check >= CHECK_INTERVAL_SECONDS:+ since_check = 0.0+ await self._logged(self.check_once(fence), "daily check")+ await self._logged(self.poll_once(fence), "events")+ except TakenOver:+ logger.warning(+ "WorkOS mirror stopped: another process holds it now"+ )+ returnawait asyncio.sleep(POLL_INTERVAL_SECONDS)since_check += POLL_INTERVAL_SECONDS@staticmethod- async def _logged(work: Any, what: str) -> None:+ async def _logged(work: Awaitable[Any], what: str) -> None:try:await work- except (DirectoryUnavailable, ServiceError) as error:- logger.error("WorkOS mirror %s: %s; trying again later", what, error)+ except TakenOver:+ raiseexcept Exception:logger.exception("WorkOS mirror %s failed; trying again later", what)- async def _apply_event(self, event: dict[str, Any], linked: dict[str, str]) -> None:- data = event["data"]- following = {workos: own for own, workos in linked.items()}- if event["event"] == "organization.deleted":- organization_id = following.get(str(data.get("id")))+ async def _apply_event_or_pass_over(+ self,+ fence: Fence,+ event_id: str,+ event: dict[str, Any],+ following: dict[str, str],+ ) -> None:+ try:+ await self._apply_event(fence, event, following)+ except TakenOver:+ raise+ except Exception:+ attempts = self._attempts.get(event_id, 0) + 1+ if attempts < MAX_EVENT_ATTEMPTS:+ self._attempts[event_id] = attempts+ raise+ self._attempts.pop(event_id, None)+ organization_id = following.get(_organization_named(event) or "")+ logger.exception(+ "WorkOS event %s failed %d times; passed over, and organization "+ "%s is checked in full until that succeeds",+ event_id,+ attempts,+ organization_id,+ )if organization_id is not None:- await self._organization_deleted(organization_id, data)+ self._recheck.add(organization_id)+ else:+ self._attempts.pop(event_id, None)++ async def _apply_event(+ self, fence: Fence, event: dict[str, Any], following: dict[str, str]+ ) -> None:+ data = event.get("data")+ kind = event.get("event")+ workos_organization_id = _organization_named(event)+ if not isinstance(data, dict) or not isinstance(kind, str):+ logger.error(+ "WorkOS sent an event Switch can't read; passed over: %r", event+ )+ return+ organization_id = following.get(workos_organization_id or "")+ if organization_id is None or workos_organization_id is None:+ return+ if kind == "organization.deleted":+ await self._organization_deleted(fence, organization_id, data)return- organization_id = following.get(str(data.get("organization_id")))subject = data.get("user_id")- if organization_id is None or not isinstance(subject, str):+ if not isinstance(subject, str):+ logger.error("WorkOS sent a membership event with no user; passed over")return- users = await self._users_of([subject])- user_id = users.get(subject)+ user_id = (await self._users_of([subject])).get(subject)if user_id is None:return- wanted = await self._directory.membership_role(subject, linked[organization_id])+ wanted = await self._directory.membership_role(subject, workos_organization_id)+ await self._ensure_held(fence)await apply_from_directory(self._session_factory,- self._revocation,+ self._dependencies,organization_id=organization_id,+ workos_organization_id=workos_organization_id,user_id=user_id,wanted=wanted,source=FROM_EVENT,)- async def _check(self, organization_id: str, workos_organization_id: str) -> None:+ async def _check_all(+ self, fence: Fence, linked: dict[str, str], only: set[str] | None+ ) -> None:+ for organization_id, workos_organization_id in linked.items():+ if only is not None and organization_id not in only:+ continue+ try:+ complete = await self._check(+ fence, organization_id, workos_organization_id+ )+ except TakenOver:+ raise+ except Exception:+ logger.exception(+ "Checking organization %s against WorkOS failed; checked "+ "again on the next pass",+ organization_id,+ )+ complete = False+ if complete:+ self._recheck.discard(organization_id)+ else:+ self._recheck.add(organization_id)+ # One unlinked since has nothing left to check.+ self._recheck &= set(linked)++ async def _check(+ self, fence: Fence, organization_id: str, workos_organization_id: str+ ) -> bool:+ """Apply where Switch's members differ from WorkOS's. Returns whether+ every difference was applied; one that fails is logged and the rest+ go ahead."""in_workos = await self._directory.organization_members(workos_organization_id)async with organization_session(self._session_factory, organization_id) as session:- in_switch = set(- await session.scalars(- select(OrganizationMember.user_id).where(- OrganizationMember.organization_id == organization_id- )+ rows = await session.execute(+ select(OrganizationMember.user_id, OrganizationMember.role).where(+ OrganizationMember.organization_id == organization_id))+ in_switch = {user_id: role for user_id, role in rows}users = await self._users_of(list(in_workos))subjects = await self._subjects_of(list(in_switch))wanted: dict[str, str | None] = {@@ -195,18 +320,32 @@ class WorkOSMirror:)continuewanted.setdefault(user_id, in_workos.get(subject))+ complete = Truefor user_id, role in sorted(wanted.items()):- await apply_from_directory(- self._session_factory,- self._revocation,- organization_id=organization_id,- user_id=user_id,- wanted=role,- source=FROM_CHECK,- )+ if in_switch.get(user_id) == role:+ continue+ await self._ensure_held(fence)+ try:+ await apply_from_directory(+ self._session_factory,+ self._dependencies,+ organization_id=organization_id,+ workos_organization_id=workos_organization_id,+ user_id=user_id,+ wanted=role,+ source=FROM_CHECK,+ )+ except Exception:+ logger.exception(+ "Applying WorkOS's membership of user %s in organization %s failed",+ user_id,+ organization_id,+ )+ complete = False+ return completeasync def _organization_deleted(- self, organization_id: str, data: dict[str, Any]+ self, fence: Fence, organization_id: str, data: dict[str, Any]) -> None:"""Deleting a linked organization in WorkOS removes nobody here: thatis too much to take away on one event. It is reported for an operator@@ -217,6 +356,7 @@ class WorkOSMirror:data.get("id"),organization_id,)+ await self._ensure_held(fence)async with organization_session(self._session_factory, organization_id) as session:@@ -234,6 +374,36 @@ class WorkOSMirror:)await session.commit()+ async def _restart(+ self, fence: Fence, cursor: DirectoryEventCursor, reason: str+ ) -> None:+ """Start the events again from now, and check every organization in+ full for what fell between."""+ logger.warning(+ "WorkOS refused the mirror's place in its events (after %s, from %s): "+ "%s; starting again from now and checking every organization",+ cursor.after,+ cursor.started_at.isoformat(),+ reason,+ )+ await self._write_cursor(fence, started_at=datetime.now(UTC), after=None)+ with no_tenant():+ async with self._session_factory() as session:+ await record_server_audit_event(+ session,+ actor_user_id=None,+ action=ServerAuditAction.DIRECTORY_EVENTS_RESTARTED,+ target_type="directory",+ target_id=CURSOR_SOURCE,+ details={+ "after": cursor.after,+ "started_at": cursor.started_at.isoformat(),+ "reason": reason,+ },+ )+ await session.commit()+ self._recheck |= set(await self._linked())+async def _linked(self) -> dict[str, str]:"""Each linked organization's WorkOS organization, by its id."""linked: dict[str, str] = {}@@ -287,46 +457,95 @@ class WorkOSMirror:await session.execute(insert(DirectoryEventCursor).values(- source=CURSOR_SOURCE, after=None, started_at=datetime.now(UTC)+ source=CURSOR_SOURCE,+ after=None,+ started_at=datetime.now(UTC),).on_conflict_do_nothing(index_elements=["source"]))await session.commit()cursor = await session.get(DirectoryEventCursor, CURSOR_SOURCE)assert cursor is not None+ session.expunge(cursor)return cursor- async def _advance(self, after: str) -> None:+ async def _write_cursor(+ self, fence: Fence, *, started_at: datetime | None, after: str | None+ ) -> None:+ """Set the cursor to `after` (and `started_at`, when given), in one+ statement that writes only while `fence` holds."""+ values: dict[str, Any] = {"after": after, "updated_at": datetime.now(UTC)}+ if started_at is not None:+ values["started_at"] = started_atwith no_tenant():async with self._session_factory() as session:- cursor = await session.get(DirectoryEventCursor, CURSOR_SOURCE)- assert cursor is not None- cursor.after = after- cursor.updated_at = datetime.now(UTC)+ written = await session.execute(+ update(DirectoryEventCursor)+ .where(DirectoryEventCursor.source == CURSOR_SOURCE, _holds(fence))+ .values(**values)+ .returning(DirectoryEventCursor.source)+ )+ if written.first() is None:+ raise TakenOver()await session.commit()+ async def _ensure_held(self, fence: Fence) -> None:+ """Raise `TakenOver` unless `fence` is still this process's, before a+ change it must not make beside a new holder."""+ with no_tenant():+ async with self._session_factory() as session:+ held = await self._leases.held(+ session,+ kind=fence.kind,+ tokens={fence.key: fence.token},+ process_id=fence.process_id,+ )+ if fence.key not in held:+ raise TakenOver()+++def _holds(fence: Fence) -> Any:+ return exists().where(+ OwnershipLease.kind == fence.kind,+ OwnershipLease.key == fence.key,+ OwnershipLease.token == fence.token,+ OwnershipLease.process_id == fence.process_id,+ )+++def _organization_named(event: dict[str, Any]) -> str | None:+ """The WorkOS organization an event is about."""+ data = event.get("data")+ if not isinstance(data, dict):+ return None+ key = "id" if event.get("event") == "organization.deleted" else "organization_id"+ value = data.get(key)+ return value if isinstance(value, str) else None+def own_workos_mirror(ownership: Ownership, mirror: WorkOSMirror) -> OwnedKind:"""Register the mirror as owned work, wanted by this process; it runs hereonly while it holds it."""running: dict[str, asyncio.Task[None]] = {}- def launch(key: str) -> None:- task = asyncio.create_task(mirror.run(), name="workos-mirror")+ def launch(key: str, fence: Fence) -> None:+ task = asyncio.create_task(mirror.run(fence), name="workos-mirror")running[key] = task- task.add_done_callback(lambda done: ended(key, done))+ task.add_done_callback(lambda done: ended(key, fence, done))- def ended(key: str, task: asyncio.Task[None]) -> None:+ def ended(key: str, fence: Fence, task: asyncio.Task[None]) -> None:if task.cancelled() or running.get(key) is not task:return- logger.error(- "WorkOS mirror stopped unexpectedly; restarting",- exc_info=task.exception(),- )- launch(key)+ error = task.exception()+ if error is None:+ # Returned on its own: taken over, which the next balance settles.+ running.pop(key, None)+ return+ logger.error("WorkOS mirror stopped unexpectedly; restarting", exc_info=error)+ launch(key, fence)async def start(key: str, fence: Fence) -> None:- launch(key)+ launch(key, fence)async def stop(key: str) -> None:task = running.pop(key, None)diff --git a/core/tests/switch_core/test_organization_mirror_apply.py b/core/tests/switch_core/test_organization_mirror_apply.pyindex 1fc064186..1b2751ca4 100644--- a/core/tests/switch_core/test_organization_mirror_apply.py+++ b/core/tests/switch_core/test_organization_mirror_apply.py@@ -13,7 +13,8 @@ import pytestfrom sqlalchemy import selectfrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker-from switch_core.connections.broker import ServiceBroker+from switch_core import organization_mirror+from switch_core.connections.broker import ServiceBroker, ServiceErrorfrom switch_core.connections.loader import CATALOGfrom switch_core.connections.sign_in_flows import SignInFlowsfrom switch_core.db.audit import AuditAction, OrganizationAuditAction@@ -31,11 +32,13 @@ from switch_core.db.models import ()from switch_core.db.stores.agent_store import AgentStorefrom switch_core.db.stores.api_key_store import ApiKeyStore+from switch_core.db.stores.organization_store import OrganizationStorefrom switch_core.db.stores.service_connection_store import ServiceConnectionStore+from switch_core.db.stores.user_store import UserStorefrom switch_core.db.stores.workspace_service_store import WorkspaceServiceStorefrom switch_core.keys import Keyringfrom switch_core.organization_access import AccessRevocation-from switch_core.organization_mirror import apply_from_directory+from switch_core.organization_mirror import MirrorDependencies, apply_from_directoryfrom tests.conftest import RLSHarnesspytestmark = pytest.mark.no_ambient_tenant@@ -49,6 +52,16 @@ def _id(prefix: str) -> str:return f"{prefix}-{uuid.uuid4().hex[:8]}"+def _dependencies(+ session_factory: async_sessionmaker[AsyncSession],+) -> MirrorDependencies:+ return MirrorDependencies(+ revocation=_revocation(session_factory),+ organization_store=OrganizationStore(),+ user_store=UserStore(),+ )++def _revocation(session_factory: async_sessionmaker[AsyncSession]) -> AccessRevocation:return AccessRevocation(agent_store=AgentStore(),@@ -79,6 +92,7 @@ class _World:def __init__(self, organization_id: str, joined: str, reached: str) -> None:self.organization_id = organization_id+ self.workos_organization_id = _id("org_workos")self.joined = joinedself.reached = reachedself.ada = _id("ada")@@ -99,7 +113,7 @@ class _World:id=world.organization_id,slug=world.organization_id,name="Acme",- workos_organization_id=_id("org_workos"),+ workos_organization_id=world.workos_organization_id,))for user_id in (world.ada, world.bea):@@ -230,11 +244,18 @@ async def _workspace_actions(owner, tenant_id: str) -> list[str]:)-async def _apply(rls_harness: RLSHarness, world: _World, wanted: str | None) -> None:+async def _apply(+ rls_harness: RLSHarness,+ world: _World,+ wanted: str | None,+ *,+ workos_organization_id: str | None = None,+) -> None:await apply_from_directory(rls_harness.restricted,- _revocation(rls_harness.restricted),+ _dependencies(rls_harness.restricted),organization_id=world.organization_id,+ workos_organization_id=workos_organization_id or world.workos_organization_id,user_id=world.ada,wanted=wanted,source="directory_event",@@ -283,6 +304,19 @@ class TestTakingAway:[(action, _)] = await _organization_events(owner, world.organization_id)assert action == OrganizationAuditAction.MEMBER_DEMOTED_FROM_DIRECTORY+ async def test_nothing_is_applied_once_the_organization_follows_another(+ self, rls_harness: RLSHarness+ ) -> None:+ """A role read from a WorkOS organization the organization no longer+ follows (unlinked or relinked meanwhile) is not WorkOS's say here."""+ owner = rls_harness.owner+ world = await _World.make(owner)++ await _apply(rls_harness, world, None, workos_organization_id=_id("org_old"))++ assert await _role(owner, world.organization_id, world.ada) == "org_admin"+ assert await _organization_events(owner, world.organization_id) == []+async def test_the_same_role_changes_nothing(self, rls_harness: RLSHarness) -> None:owner = rls_harness.ownerworld = await _World.make(owner)@@ -373,8 +407,9 @@ class TestGranting:await apply_from_directory(rls_harness.restricted,- _revocation(rls_harness.restricted),+ _dependencies(rls_harness.restricted),organization_id=world.organization_id,+ workos_organization_id=world.workos_organization_id,user_id=cy,wanted="member",source="daily_check",@@ -384,3 +419,54 @@ class TestGranting:[(action, details)] = await _organization_events(owner, world.organization_id)assert action == OrganizationAuditAction.MEMBER_ADDED_FROM_DIRECTORYassert details == {"source": "daily_check", "role": "member"}+++class TestARetryAfterAFailure:+ async def test_finds_the_workspaces_left_and_their_conflicts_recorded(+ self, rls_harness: RLSHarness, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ """Ada is the only owner of `joined` and owns an agent there; a second+ workspace she is in has a busy connection the first time."""+ owner = rls_harness.owner+ world = await _World.make(owner, ada_owns_joined=True)+ await _agent(owner, world.joined, world.ada)+ busy = _id("busy")+ async with owner() as session:+ session.add(+ Tenant(+ id=busy,+ slug=busy,+ name="Busy",+ organization_id=world.organization_id,+ )+ )+ await session.flush()+ session.add(TenantMember(tenant_id=busy, user_id=world.ada, role="member"))+ await session.commit()+ real = organization_mirror.remove_membership++ async def busy_once(session, revocation, membership, **kwargs):+ if membership.tenant_id == busy:+ raise ServiceError(+ 409, "internal", "A connection is busy.", retryable=True+ )+ return await real(session, revocation, membership, **kwargs)++ monkeypatch.setattr(organization_mirror, "remove_membership", busy_once)+ with pytest.raises(ServiceError):+ await _apply(rls_harness, world, None)+ monkeypatch.setattr(organization_mirror, "remove_membership", real)+ await _apply(rls_harness, world, None)++ conflicts = [+ details["kind"]+ for action, details in await _organization_events(+ owner, world.organization_id+ )+ if action == OrganizationAuditAction.DIRECTORY_CONFLICT+ ]+ assert sorted(conflicts) == [+ "agents_left_with_owner",+ "workspace_left_without_owner",+ ]+ assert await _role(owner, world.organization_id, world.ada) is Nonediff --git a/core/tests/switch_core/test_workos_directory.py b/core/tests/switch_core/test_workos_directory.pyindex 17fd8978a..8451e67c5 100644--- a/core/tests/switch_core/test_workos_directory.py+++ b/core/tests/switch_core/test_workos_directory.py@@ -15,7 +15,11 @@ from switch_core.organization_directory import (DirectoryRefusal,DirectoryUnavailable,)-from switch_core.workos_directory import WorkOSDirectory, customer_id_of+from switch_core.workos_directory import (+ EventCursorRejected,+ WorkOSDirectory,+ customer_id_of,+)API = "https://workos.example.test"KEY = "sk_test_example"@@ -267,9 +271,9 @@ class TestWhatTheMirrorReads:assert seen[1].url.params["after"] == "event_1"assert "range_start" not in seen[1].url.params- async def test_an_event_switch_cannot_read_is_unavailable(self) -> None:+ async def test_a_page_switch_cannot_read_is_unavailable(self) -> None:directory = _directory(- lambda request: httpx.Response(200, json={"data": [{"id": "event_1"}]})+ lambda request: httpx.Response(200, json={"data": ["not an event"]}))with pytest.raises(DirectoryUnavailable):@@ -277,6 +281,35 @@ class TestWhatTheMirrorReads:("organization.deleted",), after=None, range_start="x")+ async def test_an_event_is_handed_back_as_sent_for_the_caller_to_judge(+ self,+ ) -> None:+ directory = _directory(+ lambda request: httpx.Response(200, json={"data": [{"id": "event_1"}]})+ )++ assert await directory.events(+ ("organization.deleted",), after=None, range_start="x"+ ) == [{"id": "event_1"}]++ @pytest.mark.parametrize("status", [400, 404, 422])+ async def test_a_refused_cursor_says_so(self, status: int) -> None:+ directory = _directory(lambda request: httpx.Response(status))++ with pytest.raises(EventCursorRejected):+ await directory.events(+ ("organization.deleted",), after="event_old", range_start="x"+ )++ async def test_other_refusals_are_not_the_cursor(self) -> None:+ directory = _directory(lambda request: httpx.Response(401))++ with pytest.raises(DirectoryUnavailable) as refused:+ await directory.events(+ ("organization.deleted",), after="event_1", range_start="x"+ )+ assert not isinstance(refused.value, EventCursorRejected)+async def test_one_membership_is_read_as_it_is_now(self) -> None:def handler(request: httpx.Request) -> httpx.Response:assert request.url.params["user_id"] == "user_A"diff --git a/core/tests/switch_core/test_workos_mirror.py b/core/tests/switch_core/test_workos_mirror.pyindex 23df2534e..7a54c1b9e 100644--- a/core/tests/switch_core/test_workos_mirror.py+++ b/core/tests/switch_core/test_workos_mirror.py@@ -5,11 +5,13 @@ WorkOS; `owner` only arranges fixtures and reads results."""from __future__ import annotationsimport uuid+from datetime import UTC, datetime, timedeltafrom typing import Anyimport pytestfrom sqlalchemy import select+from switch_core import workos_mirrorfrom switch_core.db.audit import OrganizationAuditActionfrom switch_core.db.models import (DirectoryEventCursor,@@ -17,12 +19,25 @@ from switch_core.db.models import (Organization,OrganizationAuditEvent,OrganizationMember,+ OwnershipLease,+ ServerAuditEvent,+ SwitchCoreProcess,User,)+from switch_core.db.stores.ownership_lease_store import OwnershipLeaseStorefrom switch_core.organization_directory import DirectoryUnavailable-from switch_core.workos_mirror import EVENT_TYPES, WorkOSMirror+from switch_core.ownership import Fence+from switch_core.workos_directory import EventCursorRejected+from switch_core.workos_mirror import (+ EVENT_TYPES,+ MAX_EVENT_ATTEMPTS,+ MIRROR_KEY,+ MIRROR_KIND,+ TakenOver,+ WorkOSMirror,+)from tests.conftest import RLSHarness-from tests.switch_core.test_organization_mirror_apply import _revocation+from tests.switch_core.test_organization_mirror_apply import _dependenciespytestmark = pytest.mark.no_ambient_tenant@@ -43,6 +58,7 @@ class _WorkOS:self.page = pageself.asked: list[dict[str, Any]] = []self.fail_role_for: str | None = None+ self.reject_cursor = Falsedef change(self, subject: str, organization: str, role: str | None) -> None:if role is None:@@ -68,9 +84,11 @@ class _WorkOS:) -> list[dict[str, Any]]:assert types == EVENT_TYPESself.asked.append({"after": after, "range_start": range_start})+ if self.reject_cursor:+ raise EventCursorRejected("WorkOS answered 400 for /events.")start = 0if after is not None:- start = next(i for i, e in enumerate(self.log) if e["id"] == after) + 1+ start = next(i for i, e in enumerate(self.log) if e.get("id") == after) + 1return self.log[start : start + self.page]async def membership_role(self, subject: str, organization: str) -> str | None:@@ -139,13 +157,41 @@ async def _roles(owner, organization_id: str) -> dict[str, str]:return {user_id: role for user_id, role in rows}-def _mirror(rls_harness: RLSHarness, workos: _WorkOS) -> WorkOSMirror:- return WorkOSMirror(+class _Running:+ """A mirror and the fence it holds, as `ownership.py` would start it."""++ def __init__(self, mirror: WorkOSMirror, fence: Fence) -> None:+ self.mirror = mirror+ self.fence = fence++ async def poll_once(self) -> int:+ return await self.mirror.poll_once(self.fence)++ async def check_once(self) -> None:+ await self.mirror.check_once(self.fence)+++async def _mirror(rls_harness: RLSHarness, workos: _WorkOS) -> _Running:+ process_id = _id("process")+ async with rls_harness.owner() as session:+ session.add(SwitchCoreProcess(id=process_id))+ await session.flush()+ session.add(+ OwnershipLease(+ kind=MIRROR_KIND, key=MIRROR_KEY, process_id=process_id, token=1+ )+ )+ await session.commit()+ mirror = WorkOSMirror(rls_harness.restricted,workos, # type: ignore[arg-type]- _revocation(rls_harness.restricted),+ _dependencies(rls_harness.restricted),+ OwnershipLeaseStore(),ISSUER,)+ return _Running(+ mirror, Fence(kind=MIRROR_KIND, key=MIRROR_KEY, process_id=process_id, token=1)+ )class TestEvents:@@ -156,7 +202,7 @@ class TestEvents:organization_id, workos_id = await _linked(owner)ada = await _person(owner, "user_ADA")workos = _WorkOS(page=1)- mirror = _mirror(rls_harness, workos)+ mirror = await _mirror(rls_harness, workos)assert await mirror.poll_once() == 0workos.change("user_ADA", workos_id, "member")@@ -180,7 +226,7 @@ class TestEvents:organization_id, workos_id = await _linked(owner)await _person(owner, "user_ADA")workos = _WorkOS()- mirror = _mirror(rls_harness, workos)+ mirror = await _mirror(rls_harness, workos)await mirror.poll_once()workos.change("user_ADA", workos_id, "member")workos.change("user_ADA", workos_id, None)@@ -198,7 +244,7 @@ class TestEvents:await _member(owner, organization_id, bea, "org_admin")workos = _WorkOS()workos.memberships[("user_ADA", workos_id)] = "member"- mirror = _mirror(rls_harness, workos)+ mirror = await _mirror(rls_harness, workos)await mirror.poll_once()workos.change("user_ADA", workos_id, None)@@ -213,7 +259,7 @@ class TestEvents:organization_id, workos_id = await _linked(owner)await _person(owner, "user_ADA")workos = _WorkOS()- mirror = _mirror(rls_harness, workos)+ mirror = await _mirror(rls_harness, workos)await mirror.poll_once()workos.change("user_ADA", _id("org_elsewhere"), "org_admin")workos.change("user_NOBODY", workos_id, "org_admin")@@ -229,7 +275,7 @@ class TestEvents:organization_id, workos_id = await _linked(owner)ada = await _person(owner, "user_ADA")workos = _WorkOS()- mirror = _mirror(rls_harness, workos)+ mirror = await _mirror(rls_harness, workos)await mirror.poll_once()workos.change("user_ADA", workos_id, "member")workos.fail_role_for = "user_ADA"@@ -249,7 +295,7 @@ class TestEvents:ada = await _person(owner, "user_ADA")await _member(owner, organization_id, ada, "org_admin")workos = _WorkOS()- mirror = _mirror(rls_harness, workos)+ mirror = await _mirror(rls_harness, workos)await mirror.poll_once()workos.log.append({@@ -298,7 +344,7 @@ class TestDailyCheck:("user_STRANGER", workos_id): "member",}- await _mirror(rls_harness, workos).check_once()+ await (await _mirror(rls_harness, workos)).check_once()assert await _roles(owner, organization_id) == {ada: "member",@@ -307,3 +353,147 @@ class TestDailyCheck:eve: "org_admin",}assert f"User {dee} is in organization" in caplog.text+++class TestRecovering:+ async def test_an_event_failing_every_time_is_passed_over_and_its_organization_rechecked(+ self, rls_harness: RLSHarness, caplog: pytest.LogCaptureFixture+ ) -> None:+ owner = rls_harness.owner+ organization_id, workos_id = await _linked(owner)+ ada = await _person(owner, "user_ADA")+ bea = await _person(owner, "user_BEA")+ workos = _WorkOS()+ mirror = await _mirror(rls_harness, workos)+ await mirror.poll_once()+ workos.change("user_ADA", workos_id, "member")+ workos.change("user_BEA", workos_id, "member")+ workos.fail_role_for = "user_ADA"++ for _ in range(MAX_EVENT_ATTEMPTS - 1):+ with pytest.raises(DirectoryUnavailable):+ await mirror.poll_once()+ assert await mirror.poll_once() == 2++ assert await _roles(owner, organization_id) == {bea: "member"}+ assert "failed 5 times; passed over" in caplog.text+ workos.fail_role_for = None+ await mirror.poll_once()+ assert await _roles(owner, organization_id) == {ada: "member", bea: "member"}++ async def test_an_event_switch_cannot_read_is_passed_over(+ self, rls_harness: RLSHarness+ ) -> None:+ owner = rls_harness.owner+ organization_id, workos_id = await _linked(owner)+ ada = await _person(owner, "user_ADA")+ workos = _WorkOS()+ mirror = await _mirror(rls_harness, workos)+ await mirror.poll_once()+ workos.log.append({"event": "organization_membership.created"})+ workos.log.append(+ {"id": "event_bad", "event": "organization_membership.updated"}+ )+ workos.change("user_ADA", workos_id, "member")++ await mirror.poll_once()++ assert await _roles(owner, organization_id) == {ada: "member"}++ async def test_an_empty_pass_moves_the_start_forward(+ self, rls_harness: RLSHarness+ ) -> None:+ owner = rls_harness.owner+ workos = _WorkOS()+ mirror = await _mirror(rls_harness, workos)+ await mirror.poll_once()+ async with owner() as session:+ cursor = await session.get(DirectoryEventCursor, "workos")+ assert cursor is not None+ cursor.started_at = datetime.now(UTC) - timedelta(days=60)+ await session.commit()++ await mirror.poll_once()++ async with owner() as session:+ cursor = await session.get(DirectoryEventCursor, "workos")+ assert cursor is not None+ assert cursor.started_at > datetime.now(UTC) - timedelta(minutes=10)++ async def test_a_refused_cursor_starts_again_from_now_and_checks_everything(+ self, rls_harness: RLSHarness, caplog: pytest.LogCaptureFixture+ ) -> None:+ owner = rls_harness.owner+ organization_id, workos_id = await _linked(owner)+ ada = await _person(owner, "user_ADA")+ workos = _WorkOS()+ mirror = await _mirror(rls_harness, workos)+ await mirror.poll_once()+ workos.memberships[("user_ADA", workos_id)] = "org_admin"+ workos.reject_cursor = True++ assert await mirror.poll_once() == 0+ workos.reject_cursor = False+ await mirror.poll_once()++ assert await _roles(owner, organization_id) == {ada: "org_admin"}+ assert "starting again from now" in caplog.text+ async with owner() as session:+ [event] = await session.scalars(select(ServerAuditEvent))+ assert event.action == "directory.events_restarted"+++class TestOneHolder:+ async def test_a_mirror_taken_over_writes_nothing_more(+ self, rls_harness: RLSHarness+ ) -> None:+ owner = rls_harness.owner+ organization_id, workos_id = await _linked(owner)+ await _person(owner, "user_ADA")+ workos = _WorkOS()+ mirror = await _mirror(rls_harness, workos)+ await mirror.poll_once()+ workos.change("user_ADA", workos_id, "member")+ async with owner() as session:+ lease = await session.get(OwnershipLease, (MIRROR_KIND, MIRROR_KEY))+ assert lease is not None+ lease.token = 2+ await session.commit()++ with pytest.raises(TakenOver):+ await mirror.poll_once()++ assert await _roles(owner, organization_id) == {}+ async with owner() as session:+ cursor = await session.get(DirectoryEventCursor, "workos")+ assert cursor is not None and cursor.after is None+++class TestDailyCheckFailures:+ async def test_one_member_failing_does_not_stop_the_rest(+ self, rls_harness: RLSHarness, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ owner = rls_harness.owner+ organization_id, workos_id = await _linked(owner)+ ada = await _person(owner, "user_ADA")+ bea = await _person(owner, "user_BEA")+ workos = _WorkOS()+ workos.memberships = {+ ("user_ADA", workos_id): "member",+ ("user_BEA", workos_id): "member",+ }+ real = workos_mirror.apply_from_directory++ async def failing_for_ada(*args, **kwargs):+ if kwargs["user_id"] == ada:+ raise RuntimeError("database went away")+ return await real(*args, **kwargs)++ monkeypatch.setattr(workos_mirror, "apply_from_directory", failing_for_ada)+ mirror = await _mirror(rls_harness, workos)+ await mirror.check_once()+ assert await _roles(owner, organization_id) == {bea: "member"}++ monkeypatch.setattr(workos_mirror, "apply_from_directory", real)+ await mirror.poll_once()+ assert await _roles(owner, organization_id) == {ada: "member", bea: "member"}diff --git a/docs/old/workos-directory.md b/docs/old/workos-directory.mdindex c00320fbf..626d1c503 100644--- a/docs/old/workos-directory.md+++ b/docs/old/workos-directory.md@@ -67,6 +67,13 @@ every linked organization's members with WorkOS and applies the difference,which catches up anything from before it ran or missed since. It keeps itsplace in the database, so a restart resumes where it stopped.+A change that fails to apply is tried again on the next pass, up to five+times. It is then passed over, and that organization is checked in full on+every pass until the check succeeds, so one stuck change doesn't hold up the+rest. If WorkOS no longer has the events from where the mirror stopped (it+keeps 90 days), the mirror starts again from now, records that in the+server's audit log, and checks every linked organization in full.+For each person it matches (by the WorkOS user their account signs in as):- **Added or made `org_admin` in WorkOS:** added, or made Org Admin, as at@@ -85,8 +92,10 @@ recorded in the organization's audit log as "WorkOS change broke a Switchrule", naming what:- the organization's last Org Admin was removed or demoted;-- agents the person owns were left in place, still running on their keys,- owned by someone who can no longer act there, for an admin to delete;+- agents the person owns were left in place, owned by someone who can no+ longer act there, for an admin to delete. Each agent keeps its own key,+ which acts as the agent, never as the person; every key of the person's+ own goes;- a workspace was left with no owner (its organization's Org Admins stilladminister it).