Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(organizations): make the WorkOS mirror recover, fence it, keep conflicts

Petr Bauchae44f61e5c5b
diff --git a/core/switch_core/db/audit.py b/core/switch_core/db/audit.py
index 7e194faa7..ddf7b6785 100644
--- a/core/switch_core/db/audit.py
+++ b/core/switch_core/db/audit.py
@@ -101,6 +101,10 @@ class ServerAuditAction(StrEnum):
CONNECTOR_APP_SAVED = "connector_app.saved"
CONNECTOR_APP_IMPORTED = "connector_app.imported"
CONNECTOR_APP_REMOVED = "connector_app.removed"
+ # WorkOS refused where the directory mirror had got to in its events, and
+ # the mirror started again from now, with a full check of every linked
+ # organization.
+ DIRECTORY_EVENTS_RESTARTED = "directory.events_restarted"
async def record_audit_event(
diff --git a/core/switch_core/gateway/organizations.py b/core/switch_core/gateway/organizations.py
index 0cbd02fd9..0ea8b9e68 100644
--- a/core/switch_core/gateway/organizations.py
+++ b/core/switch_core/gateway/organizations.py
@@ -479,7 +479,6 @@ async def _end_access_through_organization(
reached=reached,
actor_id=actor_id,
details={"reason": reason},
- keep_agent_keys=False,
)
diff --git a/core/switch_core/gateway/tenants.py b/core/switch_core/gateway/tenants.py
index f463ca1ba..6612b689a 100644
--- a/core/switch_core/gateway/tenants.py
+++ b/core/switch_core/gateway/tenants.py
@@ -124,15 +124,12 @@ from switch_core.gateway.schemas import (
UsageTotalResponse,
)
from switch_core.gateway.service_connections import service_refusal
-from switch_core.gateway.workspace_access import (
- finish_revocation,
- revoke_workspace_access,
-)
from switch_core.gateway.workspace_creation import (
NOTHING_CHANGED,
create_organization_for,
directory_failed,
)
+from switch_core.organization_access import AccessRevocation, remove_membership
from switch_core.organization_directory import DirectoryError, OrganizationDirectory
from switch_core.telemetry import emit_safely
from switch_core.telemetry.ages import age_hours
@@ -1463,35 +1460,25 @@ async def remove_member(
),
)
+ revocation = AccessRevocation(
+ agent_store=agent_store,
+ api_key_store=api_key_store,
+ broker=broker,
+ protocol=protocol,
+ sign_in_flows=request.app.state.sign_in_flows,
+ )
try:
- revoked = await revoke_workspace_access(
+ revoked = await remove_membership(
session,
- user_id,
- api_key_store=api_key_store,
- broker=broker,
- sign_in_flows=request.app.state.sign_in_flows,
+ revocation,
+ membership,
+ actor_id=admin.id,
+ details={},
keep_agent_keys=False,
)
except ServiceError as error:
return service_refusal(error)
-
- await record_audit_event(
- session,
- tenant_id=tenant_id,
- actor_user_id=admin.id,
- action=AuditAction.MEMBER_REMOVED,
- target_type="user",
- target_id=user_id,
- details={
- "role": membership.role,
- "api_keys_deleted": len(revoked.key_hashes),
- "service_connections_deleted": len(revoked.removed_services),
- },
- )
- await session.delete(membership)
await session.commit()
- warning = await finish_revocation(
- session, user_id, revoked, protocol=protocol, broker=broker
- )
+ warning = await revocation.finish(session, user_id, revoked)
return {"ok": True, "warning": warning}
diff --git a/core/switch_core/gateway/workspace_access.py b/core/switch_core/gateway/workspace_access.py
index 7093b8d9c..c12ed72b9 100644
--- a/core/switch_core/gateway/workspace_access.py
+++ b/core/switch_core/gateway/workspace_access.py
@@ -43,18 +43,22 @@ async def revoke_workspace_access(
when a connection is busy, before anything is removed. The caller
commits, then calls `finish_revocation`.
- `keep_agent_keys` keeps the keys agents they own run on, for a caller
- that leaves those agents in place rather than refusing while they exist.
+ `keep_agent_keys` keeps the agent keys (`type` "agent") agents run on,
+ for a caller that leaves those agents in place rather than refusing while
+ they exist. Such a key authenticates as its agent, never as the person
+ (`bridges/agent/auth.py`), so keeping it leaves the agent working and the
+ person nothing of their own; every other key of theirs goes.
"""
removed_services = await broker.remove_connections(session, user_id)
keys = await api_key_store.get_by_user(session, user_id)
- if keep_agent_keys and keys:
+ if keep_agent_keys:
+ agent_keys = [key.id for key in keys if key.type == "agent"]
in_use = set(
await session.scalars(
- select(Agent.api_key_id).where(
- Agent.api_key_id.in_([key.id for key in keys])
- )
+ select(Agent.api_key_id).where(Agent.api_key_id.in_(agent_keys))
)
+ if agent_keys
+ else ()
)
keys = [key for key in keys if key.id not in in_use]
for key in keys:
diff --git a/core/switch_core/main.py b/core/switch_core/main.py
index 950ec3735..4cf8dd370 100644
--- a/core/switch_core/main.py
+++ b/core/switch_core/main.py
@@ -202,6 +202,7 @@ from switch_core.observability.pool import (
from switch_core.observability.query import instrument_queries
from switch_core.organization_access import AccessRevocation
from switch_core.organization_directory import LocalDirectory, OrganizationDirectory
+from switch_core.organization_mirror import MirrorDependencies
from switch_core.outbound import guarded_async_client
from switch_core.ownership import Ownership
from switch_core.process_identity import new_process_id
@@ -1419,13 +1420,18 @@ async def run(config: SwitchConfig, process_id: str) -> None:
WorkOSMirror(
session_factory,
organization_directory,
- AccessRevocation(
- agent_store=agent_store,
- api_key_store=api_key_store,
- broker=service_broker,
- protocol=protocol,
- sign_in_flows=gateway_app.state.sign_in_flows,
+ MirrorDependencies(
+ revocation=AccessRevocation(
+ agent_store=agent_store,
+ api_key_store=api_key_store,
+ broker=service_broker,
+ protocol=protocol,
+ sign_in_flows=gateway_app.state.sign_in_flows,
+ ),
+ organization_store=organization_store,
+ user_store=user_store,
),
+ OwnershipLeaseStore(),
config.gateway_oidc_issuer_url,
),
)
diff --git a/core/switch_core/organization_access.py b/core/switch_core/organization_access.py
index dd02dea80..526708104 100644
--- a/core/switch_core/organization_access.py
+++ b/core/switch_core/organization_access.py
@@ -1,10 +1,15 @@
"""Ending someone's access to an organization's workspaces: what being demoted
-from `org_admin`, or leaving the organization, takes away.
+from `org_admin`, or leaving a workspace or the organization, takes away.
-Shared by the organization routes (`gateway/organizations.py`), which refuse
-first while the person owns agents there, and the directory mirror
-(`organization_mirror.py`), which applies a change made in the directory
-whatever it breaks and reports what it broke.
+Shared by the workspace and organization routes (`gateway/tenants.py`,
+`gateway/organizations.py`), which refuse first while the person owns agents
+there, and the directory mirror (`organization_mirror.py`), which applies a
+change made in the directory whatever it breaks and reports what it broke.
+
+Each step works on one workspace's session and leaves the commit to the
+caller, so whatever else belongs with it (a refusal, a conflict recorded for
+the organization) commits or rolls back with it. After the commit, the caller
+calls `finish_revocation` with what the step returned.
"""
from __future__ import annotations
@@ -23,6 +28,7 @@ from switch_core.db.session_scope import tenant_session
from switch_core.db.stores.agent_store import AgentStore
from switch_core.db.stores.api_key_store import ApiKeyStore
from switch_core.gateway.workspace_access import (
+ RevokedAccess,
finish_revocation,
revoke_workspace_access,
)
@@ -38,6 +44,14 @@ class AccessRevocation:
protocol: AgentCore
sign_in_flows: SignInFlows
+ async def finish(
+ self, session: AsyncSession, user_id: str, revoked: RevokedAccess
+ ) -> str | None:
+ """`finish_revocation`, once the step's transaction has committed."""
+ return await finish_revocation(
+ session, user_id, revoked, protocol=self.protocol, broker=self.broker
+ )
+
async def workspaces_where_they_own_agents(
session_factory: async_sessionmaker[AsyncSession],
@@ -55,109 +69,113 @@ async def workspaces_where_they_own_agents(
return found
+async def end_organization_admin_access(
+ session: AsyncSession,
+ revocation: AccessRevocation,
+ *,
+ tenant_id: str,
+ user_id: str,
+ actor_id: str | None,
+ details: dict[str, Any],
+ keep_agent_keys: bool,
+) -> RevokedAccess:
+ """In workspace `tenant_id`, bound on `session`, where `user_id` stands
+ only as their organization's `org_admin`: remove their keys, service
+ connections and sign-ins under way, recorded in its audit log with
+ `details`; `keep_agent_keys` as `revoke_workspace_access` has it. Raises
+ `ServiceError` when a connection is busy."""
+ revoked = await revoke_workspace_access(
+ session,
+ user_id,
+ api_key_store=revocation.api_key_store,
+ broker=revocation.broker,
+ sign_in_flows=revocation.sign_in_flows,
+ keep_agent_keys=keep_agent_keys,
+ )
+ await record_audit_event(
+ session,
+ tenant_id=tenant_id,
+ actor_user_id=actor_id,
+ action=AuditAction.ORGANIZATION_ADMIN_ACCESS_ENDED,
+ target_type="user",
+ target_id=user_id,
+ details={
+ **details,
+ "api_keys_deleted": len(revoked.key_hashes),
+ "service_connections_deleted": len(revoked.removed_services),
+ },
+ )
+ return revoked
+
+
+async def remove_membership(
+ session: AsyncSession,
+ revocation: AccessRevocation,
+ membership: TenantMember,
+ *,
+ actor_id: str | None,
+ details: dict[str, Any],
+ keep_agent_keys: bool,
+) -> RevokedAccess:
+ """Remove `membership` from its workspace, bound on `session`, with the
+ member's keys, service connections and sign-ins under way there, recorded
+ in its audit log with `details`; `keep_agent_keys` as
+ `revoke_workspace_access` has it. Agents they own are left as they are.
+ Raises `ServiceError` when a connection is busy, before anything is
+ removed."""
+ revoked = await revoke_workspace_access(
+ session,
+ membership.user_id,
+ api_key_store=revocation.api_key_store,
+ broker=revocation.broker,
+ sign_in_flows=revocation.sign_in_flows,
+ keep_agent_keys=keep_agent_keys,
+ )
+ await record_audit_event(
+ session,
+ tenant_id=membership.tenant_id,
+ actor_user_id=actor_id,
+ action=AuditAction.MEMBER_REMOVED,
+ target_type="user",
+ target_id=membership.user_id,
+ details={
+ **details,
+ "role": membership.role,
+ "api_keys_deleted": len(revoked.key_hashes),
+ "service_connections_deleted": len(revoked.removed_services),
+ },
+ )
+ await session.delete(membership)
+ return revoked
+
+
async def end_access_through_organization(
session_factory: async_sessionmaker[AsyncSession],
revocation: AccessRevocation,
*,
user_id: str,
reached: list[Tenant],
- actor_id: str | None,
+ actor_id: str,
details: dict[str, Any],
- keep_agent_keys: bool,
) -> str | None:
- """Take away what `user_id` holds in `reached`, the workspaces they stand
- in only as their organization's `org_admin`: their keys, service
- connections and sign-ins under way, one workspace at a time, each
- recorded in that workspace's audit log with `details`;
- `keep_agent_keys` as `revoke_workspace_access` has it. Raises
- `ServiceError` when a connection is busy. Returns a warning naming what
- could not be revoked at a vendor, or None."""
+ """`end_organization_admin_access` in each of `reached`, one workspace at
+ a time, each committed on its own, for a route that has refused already
+ while they own agents there. Returns a warning naming what could not be
+ revoked at a vendor, or None."""
warnings = []
for tenant in reached:
async with tenant_session(session_factory, tenant.id) as session:
- revoked = await revoke_workspace_access(
- session,
- user_id,
- api_key_store=revocation.api_key_store,
- broker=revocation.broker,
- sign_in_flows=revocation.sign_in_flows,
- keep_agent_keys=keep_agent_keys,
- )
- await record_audit_event(
+ revoked = await end_organization_admin_access(
session,
+ revocation,
tenant_id=tenant.id,
- actor_user_id=actor_id,
- action=AuditAction.ORGANIZATION_ADMIN_ACCESS_ENDED,
- target_type="user",
- target_id=user_id,
- details={
- **details,
- "api_keys_deleted": len(revoked.key_hashes),
- "service_connections_deleted": len(revoked.removed_services),
- },
+ user_id=user_id,
+ actor_id=actor_id,
+ details=details,
+ keep_agent_keys=False,
)
await session.commit()
- warning = await finish_revocation(
- session,
- user_id,
- revoked,
- protocol=revocation.protocol,
- broker=revocation.broker,
- )
+ warning = await revocation.finish(session, user_id, revoked)
if warning:
warnings.append(f"{tenant.name}: {warning}")
return " ".join(warnings) or None
-
-
-async def remove_from_workspace(
- session_factory: async_sessionmaker[AsyncSession],
- revocation: AccessRevocation,
- *,
- user_id: str,
- tenant: Tenant,
- actor_id: str | None,
- details: dict[str, Any],
- keep_agent_keys: bool,
-) -> str | None:
- """Remove `user_id` from `tenant`, with their keys, service connections
- and sign-ins under way there, recorded in its audit log with `details`.
- Agents they own are left as they are, and with `keep_agent_keys` so are
- the keys they run on. Raises `ServiceError` when a
- connection is busy. Returns a warning naming what could not be revoked
- at a vendor, or None."""
- async with tenant_session(session_factory, tenant.id) as session:
- membership = await session.get(TenantMember, (tenant.id, user_id))
- if membership is None:
- return None
- revoked = await revoke_workspace_access(
- session,
- user_id,
- api_key_store=revocation.api_key_store,
- broker=revocation.broker,
- sign_in_flows=revocation.sign_in_flows,
- keep_agent_keys=keep_agent_keys,
- )
- await record_audit_event(
- session,
- tenant_id=tenant.id,
- actor_user_id=actor_id,
- action=AuditAction.MEMBER_REMOVED,
- target_type="user",
- target_id=user_id,
- details={
- **details,
- "role": membership.role,
- "api_keys_deleted": len(revoked.key_hashes),
- "service_connections_deleted": len(revoked.removed_services),
- },
- )
- await session.delete(membership)
- await session.commit()
- warning = await finish_revocation(
- session,
- user_id,
- revoked,
- protocol=revocation.protocol,
- broker=revocation.broker,
- )
- return f"{tenant.name}: {warning}" if warning else None
diff --git a/core/switch_core/organization_mirror.py b/core/switch_core/organization_mirror.py
index 7cdefe2ba..4ded90840 100644
--- a/core/switch_core/organization_mirror.py
+++ b/core/switch_core/organization_mirror.py
@@ -30,9 +30,10 @@ events and its daily check (`apply_from_directory`).
from __future__ import annotations
import logging
+from dataclasses import dataclass
from typing import Any
-from sqlalchemy import delete, func, select, update
+from sqlalchemy import delete, select, update
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
@@ -48,6 +49,8 @@ from switch_core.db.models import (
TenantMember,
)
from switch_core.db.session_scope import organization_session, tenant_session
+from switch_core.db.stores.organization_store import OrganizationStore
+from switch_core.db.stores.user_store import UserStore
from switch_core.db.tenant_lookup import directory_organizations_of_user
from switch_core.gateway.auth import (
workspaces_in_organization,
@@ -55,9 +58,8 @@ from switch_core.gateway.auth import (
)
from switch_core.organization_access import (
AccessRevocation,
- end_access_through_organization,
- remove_from_workspace,
- workspaces_where_they_own_agents,
+ end_organization_admin_access,
+ remove_membership,
)
from switch_core.organization_directory import DirectoryMembership
@@ -103,80 +105,128 @@ async def bring_in_memberships(
)
+@dataclass(frozen=True)
+class MirrorDependencies:
+ """What applying a change from the directory needs."""
+
+ revocation: AccessRevocation
+ organization_store: OrganizationStore
+ user_store: UserStore
+
+
async def apply_from_directory(
session_factory: async_sessionmaker[AsyncSession],
- revocation: AccessRevocation,
+ dependencies: MirrorDependencies,
*,
organization_id: str,
+ workos_organization_id: str,
user_id: str,
wanted: str | None,
source: str,
) -> None:
"""Make `user_id`'s membership of `organization_id` what the directory
- says, `wanted` (a role, or None for no membership). Raises `ServiceError`
- when a connection to revoke is busy, having changed nothing that a retry
- would not finish."""
+ says, `wanted` (a role, or None for no membership), as read from WorkOS
+ organization `workos_organization_id`. Nothing is changed unless the
+ organization still follows that one.
+
+ Each workspace is worked on in a transaction of its own, with the
+ conflicts it raises recorded in the same one, so a retry after a failure
+ part-way (`ServiceError`, a busy connection) finds the workspaces already
+ done done, and their conflicts recorded.
+ """
async with organization_session(session_factory, organization_id) as session:
- _, current = await _link_and_role(session, organization_id, user_id)
+ linked, current = await _link_and_role(session, organization_id, user_id)
+ if linked != workos_organization_id:
+ logger.warning(
+ "Organization %s no longer follows WorkOS organization %s; "
+ "user %s's membership there is not applied",
+ organization_id,
+ workos_organization_id,
+ user_id,
+ )
+ return
if current == wanted:
return
- if _grants(current, wanted):
- assert wanted is not None
+ if wanted is not None and _grants(current, wanted):
await _grant(session, organization_id, user_id, current, wanted, source)
return
- admins = await _count_admins(session, organization_id)
assert current is not None
- conflicts: list[dict[str, Any]] = []
- warnings: list[str] = []
details = {"source": source}
+ revocation = dependencies.revocation
+ warnings: list[str] = []
if current == ORGANIZATION_ADMIN_ROLE:
- if admins <= 1:
- conflicts.append({"kind": "last_org_admin"})
- reached = await workspaces_reached_through_organization(
+ reason = "demoted" if wanted is not None else "removed"
+ for tenant in await workspaces_reached_through_organization(
session_factory, user_id, organization_id
- )
- conflicts += await _agents_left(session_factory, revocation, user_id, reached)
- warning = await end_access_through_organization(
- session_factory,
- revocation,
- user_id=user_id,
- reached=reached,
- actor_id=None,
- details={
- "reason": "demoted" if wanted is not None else "removed",
- **details,
- },
- keep_agent_keys=True,
- )
- if warning:
- warnings.append(warning)
+ ):
+ async with tenant_session(session_factory, tenant.id) as session:
+ await _conflict_if_agents_left(
+ session, dependencies, organization_id, user_id, tenant, details
+ )
+ revoked = await end_organization_admin_access(
+ session,
+ revocation,
+ tenant_id=tenant.id,
+ user_id=user_id,
+ actor_id=None,
+ details={"reason": reason, **details},
+ keep_agent_keys=True,
+ )
+ await session.commit()
+ warning = await revocation.finish(session, user_id, revoked)
+ if warning:
+ warnings.append(f"{tenant.name}: {warning}")
if wanted is None:
- workspaces = await workspaces_in_organization(
+ for tenant in await workspaces_in_organization(
session_factory, user_id, organization_id
- )
- conflicts += await _agents_left(
- session_factory, revocation, user_id, workspaces
- )
- for tenant in workspaces:
- if await _is_last_owner(session_factory, tenant.id, user_id):
- conflicts.append(
- {"kind": "workspace_left_without_owner", "workspace": tenant.name}
+ ):
+ async with tenant_session(session_factory, tenant.id) as session:
+ membership = await session.get(TenantMember, (tenant.id, user_id))
+ if membership is None:
+ continue
+ await _conflict_if_agents_left(
+ session, dependencies, organization_id, user_id, tenant, details
)
- warning = await remove_from_workspace(
- session_factory,
- revocation,
- user_id=user_id,
- tenant=tenant,
- actor_id=None,
- details=details,
- keep_agent_keys=True,
- )
+ if (
+ membership.role == "owner"
+ and await dependencies.user_store.count_owners(session) <= 1
+ ):
+ await _conflict(
+ session,
+ organization_id,
+ user_id,
+ {
+ **details,
+ "kind": "workspace_left_without_owner",
+ "workspace": tenant.name,
+ },
+ )
+ revoked = await remove_membership(
+ session,
+ revocation,
+ membership,
+ actor_id=None,
+ details=details,
+ keep_agent_keys=True,
+ )
+ await session.commit()
+ warning = await revocation.finish(session, user_id, revoked)
if warning:
- warnings.append(warning)
+ warnings.append(f"{tenant.name}: {warning}")
async with organization_session(session_factory, organization_id) as session:
+ admins = await dependencies.organization_store.lock_admins(
+ session, organization_id
+ )
+ _, current = await _link_and_role(session, organization_id, user_id)
+ if current is None or current == wanted:
+ return
+ if current == ORGANIZATION_ADMIN_ROLE and admins <= 1:
+ await _conflict(
+ session, organization_id, user_id, {**details, "kind": "last_org_admin"}
+ )
if wanted is None:
await session.execute(
delete(OrganizationMember).where(
@@ -202,23 +252,6 @@ async def apply_from_directory(
**({"warning": " ".join(warnings)} if warnings else {}),
},
)
- for conflict in conflicts:
- logger.error(
- "Applying WorkOS's change to user %s in organization %s broke "
- "a rule Switch keeps: %s",
- user_id,
- organization_id,
- conflict,
- )
- await record_organization_audit_event(
- session,
- organization_id=organization_id,
- actor_user_id=None,
- action=OrganizationAuditAction.DIRECTORY_CONFLICT,
- target_type="user",
- target_id=user_id,
- details={**details, **conflict},
- )
await session.commit()
@@ -298,43 +331,48 @@ async def _set_role(
)
-async def _count_admins(session: AsyncSession, organization_id: str) -> int:
- return (
- await session.scalar(
- select(func.count()).where(
- OrganizationMember.organization_id == organization_id,
- OrganizationMember.role == ORGANIZATION_ADMIN_ROLE,
- )
- )
- ) or 0
-
-
-async def _agents_left(
- session_factory: async_sessionmaker[AsyncSession],
- revocation: AccessRevocation,
+async def _conflict_if_agents_left(
+ session: AsyncSession,
+ dependencies: MirrorDependencies,
+ organization_id: str,
user_id: str,
- workspaces: list[Tenant],
-) -> list[dict[str, Any]]:
- """A conflict for each of `workspaces` where `user_id` owns agents, which
- stay, with the keys they run on, owned by someone who can no longer act
- there, for an admin to deal with."""
- return [
- {"kind": "agents_left_with_owner", "workspace": tenant.name}
- for tenant in await workspaces_where_they_own_agents(
- session_factory, revocation.agent_store, user_id, workspaces
+ tenant: Tenant,
+ details: dict[str, Any],
+) -> None:
+ """Agents `user_id` owns in `tenant`, bound on `session`, stay, with the
+ keys they run on, owned by someone who can no longer act there, for an
+ admin to deal with."""
+ if await dependencies.revocation.agent_store.get_by_owner(session, user_id):
+ await _conflict(
+ session,
+ organization_id,
+ user_id,
+ {**details, "kind": "agents_left_with_owner", "workspace": tenant.name},
)
- ]
-async def _is_last_owner(
- session_factory: async_sessionmaker[AsyncSession], tenant_id: str, user_id: str
-) -> bool:
- async with tenant_session(session_factory, tenant_id) as session:
- owners = list(
- await session.scalars(
- select(TenantMember.user_id).where(
- TenantMember.tenant_id == tenant_id, TenantMember.role == "owner"
- )
- )
- )
- return owners == [user_id]
+async def _conflict(
+ session: AsyncSession,
+ organization_id: str,
+ user_id: str,
+ details: dict[str, Any],
+) -> None:
+ """Record, in the caller's transaction, that a change from the directory
+ broke a rule Switch keeps. `session` may be bound to the organization or
+ to one of its workspaces."""
+ logger.error(
+ "Applying WorkOS's change to user %s in organization %s broke a rule "
+ "Switch keeps: %s",
+ user_id,
+ organization_id,
+ details,
+ )
+ await record_organization_audit_event(
+ session,
+ organization_id=organization_id,
+ actor_user_id=None,
+ action=OrganizationAuditAction.DIRECTORY_CONFLICT,
+ target_type="user",
+ target_id=user_id,
+ details=details,
+ )
diff --git a/core/switch_core/workos_directory.py b/core/switch_core/workos_directory.py
index 3dbb9b052..944d2202f 100644
--- a/core/switch_core/workos_directory.py
+++ b/core/switch_core/workos_directory.py
@@ -74,6 +74,19 @@ def switch_role(workos_role: str) -> str:
)
+class WorkOSRefused(DirectoryUnavailable):
+ """WorkOS answered, with something other than 200."""
+
+ def __init__(self, message: str, status_code: int) -> None:
+ super().__init__(message)
+ self.status_code = status_code
+
+
+class EventCursorRejected(DirectoryUnavailable):
+ """WorkOS refused where the events were asked from: an event id or a
+ start time it no longer keeps."""
+
+
def _unreadable(what: str) -> DirectoryUnavailable:
return DirectoryUnavailable(f"WorkOS answered with {what} Switch cannot read.")
@@ -99,8 +112,9 @@ class WorkOSDirectory:
f"WorkOS could not be reached ({type(error).__name__})."
) from error
if response.status_code != 200:
- raise DirectoryUnavailable(
- f"WorkOS answered {response.status_code} for {path}."
+ raise WorkOSRefused(
+ f"WorkOS answered {response.status_code} for {path}.",
+ response.status_code,
)
try:
body = response.json()
@@ -201,7 +215,10 @@ class WorkOSDirectory:
) -> list[dict[str, Any]]:
"""One page of events of `types`, oldest first: those after event
`after`, or from `range_start` (ISO 8601) when there is none yet.
- Empty once there are no more."""
+ Empty once there are no more. Each event is returned as WorkOS sent
+ it, for the caller to skip one it can't read rather than lose the
+ page. Raises `EventCursorRejected` when WorkOS refuses `after` or
+ `range_start`."""
params: dict[str, str | list[str]] = {
"events": list(types),
"limit": str(_PAGE),
@@ -211,21 +228,18 @@ class WorkOSDirectory:
params["after"] = after
else:
params["range_start"] = range_start
- page = await self._get("/events", params)
+ try:
+ page = await self._get("/events", params)
+ except WorkOSRefused as error:
+ if error.status_code in (400, 404, 422):
+ raise EventCursorRejected(error.message) from error
+ raise
data = page.get("data")
- if not isinstance(data, list):
+ if not isinstance(data, list) or not all(
+ isinstance(event, dict) for event in data
+ ):
raise _unreadable("a list of events")
- events = []
- for event in data:
- if (
- not isinstance(event, dict)
- or not isinstance(event.get("id"), str)
- or not isinstance(event.get("event"), str)
- or not isinstance(event.get("data"), dict)
- ):
- raise _unreadable("an event")
- events.append(event)
- return events
+ return data
async def organization(self, workos_organization_id: str) -> dict[str, Any]:
"""The WorkOS organization `workos_organization_id`, for an operator
diff --git a/core/switch_core/workos_mirror.py b/core/switch_core/workos_mirror.py
index e3adc4dff..af4010383 100644
--- a/core/switch_core/workos_mirror.py
+++ b/core/switch_core/workos_mirror.py
@@ -7,20 +7,34 @@ in WorkOS to a linked organization's members, applied in Switch
resumable, and with no public endpoint or signing secret. An event is only
a prompt: applying it re-reads the person's current membership from
WorkOS, so events read out of order, twice, or after a later change all
- end in the same state. An event that fails to apply stops the pass, and the
- next pass starts again from it.
+ end in the same state.
+- **Failures.** An event that fails to apply stops the pass, and the next
+ pass tries it again, up to `MAX_EVENT_ATTEMPTS` times. Then it is passed
+ over and its organization is re-checked in full on every pass until that
+ succeeds, so a removal waits on the failure, not on the next daily check.
+ An event Switch can't read is logged and passed over.
- **The daily check.** Every `CHECK_INTERVAL_SECONDS`, and when the mirror
starts, each linked organization's members in WorkOS are compared with
- Switch's: the backstop for events missed (the Events API keeps 90 days) or
- from before the mirror ran. A Switch member with no WorkOS identity can't
- be compared, and is logged and left.
+ Switch's and the differences applied: the backstop for events missed or
+ from before the mirror ran. One member or organization failing doesn't
+ stop the rest; an organization with a failure is re-checked on every pass.
+ A Switch member with no WorkOS identity can't be compared, and is logged
+ and left.
+- **Where the events are read from.** Until the first event, from when the
+ mirror started, moved forward on every empty pass so it stays inside what
+ WorkOS keeps (90 days). If WorkOS refuses the cursor anyway (the mirror
+ was stopped for longer), the mirror starts again from now, says so in the
+ log and the server's audit log, and checks every organization in full.
People are matched to WorkOS users by the sign-in identity linking them
(`oidc_identities`, the gateway's issuer and their WorkOS user id). Someone
WorkOS lists who has never signed in to Switch has no account to add, and is
brought in when they first sign in.
-One process runs it (`ownership.py`, kind `workos_mirror`).
+One process runs it (`ownership.py`, kind `workos_mirror`), under the fence
+it was started with: each change and each move of the cursor is made only
+while that hold is still this process's, so a holder taken over without
+knowing it stops rather than writing beside the new one.
"""
from __future__ import annotations
@@ -28,37 +42,44 @@ from __future__ import annotations
import asyncio
import contextlib
import logging
-from datetime import UTC, datetime
+from collections.abc import Awaitable
+from datetime import UTC, datetime, timedelta
from typing import Any
-from sqlalchemy import select
+from sqlalchemy import exists, select, update
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
-from switch_core.connections.broker import ServiceError
from switch_core.db.audit import (
OrganizationAuditAction,
+ ServerAuditAction,
record_organization_audit_event,
+ record_server_audit_event,
)
from switch_core.db.models import (
DirectoryEventCursor,
OidcIdentity,
Organization,
OrganizationMember,
+ OwnershipLease,
)
from switch_core.db.session_scope import organization_session
+from switch_core.db.stores.ownership_lease_store import OwnershipLeaseStore
from switch_core.db.tenant_lookup import directory_organizations
-from switch_core.organization_access import AccessRevocation
-from switch_core.organization_directory import DirectoryUnavailable
-from switch_core.organization_mirror import apply_from_directory
+from switch_core.organization_mirror import MirrorDependencies, apply_from_directory
from switch_core.ownership import Fence, OwnedKind, Ownership
from switch_core.tenant_context import no_tenant
-from switch_core.workos_directory import WorkOSDirectory
+from switch_core.workos_directory import EventCursorRejected, WorkOSDirectory
logger = logging.getLogger(__name__)
POLL_INTERVAL_SECONDS = 30.0
CHECK_INTERVAL_SECONDS = 24 * 3600.0
+MAX_EVENT_ATTEMPTS = 5
+# How far back an empty pass leaves where the events are read from, so an
+# event WorkOS records a little after the time it carries is not missed.
+# Reading one twice changes nothing.
+_START_OVERLAP = timedelta(minutes=5)
CURSOR_SOURCE = "workos"
EVENT_TYPES = (
"organization_membership.created",
@@ -73,112 +94,216 @@ MIRROR_KIND = "workos_mirror"
MIRROR_KEY = "all"
+class TakenOver(Exception):
+ """The fence the mirror runs under is no longer this process's."""
+
+
class WorkOSMirror:
def __init__(
self,
session_factory: async_sessionmaker[AsyncSession],
directory: WorkOSDirectory,
- revocation: AccessRevocation,
+ dependencies: MirrorDependencies,
+ leases: OwnershipLeaseStore,
issuer: str,
) -> None:
self._session_factory = session_factory
self._directory = directory
- self._revocation = revocation
+ self._dependencies = dependencies
+ self._leases = leases
self._issuer = issuer
+ self._attempts: dict[str, int] = {}
+ # Organizations to check in full on every pass until that succeeds.
+ self._recheck: set[str] = set()
- async def poll_once(self) -> int:
- """Apply the events since the cursor, page by page, moving the cursor
- past each one applied. Returns how many were read."""
+ async def poll_once(self, fence: Fence) -> int:
+ """Re-check what is due, then apply the events since the cursor, page
+ by page, moving the cursor past each one. Returns how many events
+ were read."""
cursor = await self._cursor()
+ if self._recheck:
+ await self._check_all(fence, await self._linked(), only=set(self._recheck))
+ polled_at = datetime.now(UTC)
+ try:
+ events = await self._directory.events(
+ EVENT_TYPES,
+ after=cursor.after,
+ range_start=cursor.started_at.isoformat(),
+ )
+ except EventCursorRejected as error:
+ await self._restart(fence, cursor, error.message)
+ return 0
+ if not events:
+ if cursor.after is None:
+ await self._write_cursor(
+ fence, started_at=polled_at - _START_OVERLAP, after=None
+ )
+ return 0
linked = await self._linked()
+ following = {workos: own for own, workos in linked.items()}
read = 0
after = cursor.after
- while True:
+ while events:
+ for event in events:
+ event_id = event.get("id")
+ if not isinstance(event_id, str):
+ logger.error(
+ "WorkOS sent an event with no id; passed over: %r", event
+ )
+ continue
+ await self._apply_event_or_pass_over(fence, event_id, event, following)
+ await self._write_cursor(fence, started_at=None, after=event_id)
+ after = event_id
+ read += 1
+ if after is None:
+ return read
events = await self._directory.events(
EVENT_TYPES, after=after, range_start=cursor.started_at.isoformat()
)
- if not events:
- return read
- for event in events:
- await self._apply_event(event, linked)
- after = event["id"]
- await self._advance(after)
- read += 1
+ return read
- async def check_once(self) -> None:
+ async def check_once(self, fence: Fence) -> None:
"""Compare every linked organization's members with WorkOS's, and
- apply the difference. One organization failing doesn't stop the
- others."""
- for organization_id, workos_organization_id in (await self._linked()).items():
- try:
- await self._check(organization_id, workos_organization_id)
- except (DirectoryUnavailable, ServiceError) as error:
- logger.error(
- "Daily check of organization %s against WorkOS stopped: %s",
- organization_id,
- error,
- )
+ apply the differences."""
+ await self._check_all(fence, await self._linked(), only=None)
- async def run(self) -> None:
+ async def run(self, fence: Fence) -> None:
"""Poll the events every `POLL_INTERVAL_SECONDS`, and check every
- organization when starting and every `CHECK_INTERVAL_SECONDS`."""
+ organization when starting and every `CHECK_INTERVAL_SECONDS`, until
+ the fence is lost."""
with no_tenant():
since_check = CHECK_INTERVAL_SECONDS
while True:
- if since_check >= CHECK_INTERVAL_SECONDS:
- since_check = 0.0
- await self._logged(self.check_once(), "daily check")
- await self._logged(self.poll_once(), "events")
+ try:
+ if since_check >= CHECK_INTERVAL_SECONDS:
+ since_check = 0.0
+ await self._logged(self.check_once(fence), "daily check")
+ await self._logged(self.poll_once(fence), "events")
+ except TakenOver:
+ logger.warning(
+ "WorkOS mirror stopped: another process holds it now"
+ )
+ return
await asyncio.sleep(POLL_INTERVAL_SECONDS)
since_check += POLL_INTERVAL_SECONDS
@staticmethod
- async def _logged(work: Any, what: str) -> None:
+ async def _logged(work: Awaitable[Any], what: str) -> None:
try:
await work
- except (DirectoryUnavailable, ServiceError) as error:
- logger.error("WorkOS mirror %s: %s; trying again later", what, error)
+ except TakenOver:
+ raise
except Exception:
logger.exception("WorkOS mirror %s failed; trying again later", what)
- async def _apply_event(self, event: dict[str, Any], linked: dict[str, str]) -> None:
- data = event["data"]
- following = {workos: own for own, workos in linked.items()}
- if event["event"] == "organization.deleted":
- organization_id = following.get(str(data.get("id")))
+ async def _apply_event_or_pass_over(
+ self,
+ fence: Fence,
+ event_id: str,
+ event: dict[str, Any],
+ following: dict[str, str],
+ ) -> None:
+ try:
+ await self._apply_event(fence, event, following)
+ except TakenOver:
+ raise
+ except Exception:
+ attempts = self._attempts.get(event_id, 0) + 1
+ if attempts < MAX_EVENT_ATTEMPTS:
+ self._attempts[event_id] = attempts
+ raise
+ self._attempts.pop(event_id, None)
+ organization_id = following.get(_organization_named(event) or "")
+ logger.exception(
+ "WorkOS event %s failed %d times; passed over, and organization "
+ "%s is checked in full until that succeeds",
+ event_id,
+ attempts,
+ organization_id,
+ )
if organization_id is not None:
- await self._organization_deleted(organization_id, data)
+ self._recheck.add(organization_id)
+ else:
+ self._attempts.pop(event_id, None)
+
+ async def _apply_event(
+ self, fence: Fence, event: dict[str, Any], following: dict[str, str]
+ ) -> None:
+ data = event.get("data")
+ kind = event.get("event")
+ workos_organization_id = _organization_named(event)
+ if not isinstance(data, dict) or not isinstance(kind, str):
+ logger.error(
+ "WorkOS sent an event Switch can't read; passed over: %r", event
+ )
+ return
+ organization_id = following.get(workos_organization_id or "")
+ if organization_id is None or workos_organization_id is None:
+ return
+ if kind == "organization.deleted":
+ await self._organization_deleted(fence, organization_id, data)
return
- organization_id = following.get(str(data.get("organization_id")))
subject = data.get("user_id")
- if organization_id is None or not isinstance(subject, str):
+ if not isinstance(subject, str):
+ logger.error("WorkOS sent a membership event with no user; passed over")
return
- users = await self._users_of([subject])
- user_id = users.get(subject)
+ user_id = (await self._users_of([subject])).get(subject)
if user_id is None:
return
- wanted = await self._directory.membership_role(subject, linked[organization_id])
+ wanted = await self._directory.membership_role(subject, workos_organization_id)
+ await self._ensure_held(fence)
await apply_from_directory(
self._session_factory,
- self._revocation,
+ self._dependencies,
organization_id=organization_id,
+ workos_organization_id=workos_organization_id,
user_id=user_id,
wanted=wanted,
source=FROM_EVENT,
)
- async def _check(self, organization_id: str, workos_organization_id: str) -> None:
+ async def _check_all(
+ self, fence: Fence, linked: dict[str, str], only: set[str] | None
+ ) -> None:
+ for organization_id, workos_organization_id in linked.items():
+ if only is not None and organization_id not in only:
+ continue
+ try:
+ complete = await self._check(
+ fence, organization_id, workos_organization_id
+ )
+ except TakenOver:
+ raise
+ except Exception:
+ logger.exception(
+ "Checking organization %s against WorkOS failed; checked "
+ "again on the next pass",
+ organization_id,
+ )
+ complete = False
+ if complete:
+ self._recheck.discard(organization_id)
+ else:
+ self._recheck.add(organization_id)
+ # One unlinked since has nothing left to check.
+ self._recheck &= set(linked)
+
+ async def _check(
+ self, fence: Fence, organization_id: str, workos_organization_id: str
+ ) -> bool:
+ """Apply where Switch's members differ from WorkOS's. Returns whether
+ every difference was applied; one that fails is logged and the rest
+ go ahead."""
in_workos = await self._directory.organization_members(workos_organization_id)
async with organization_session(
self._session_factory, organization_id
) as session:
- in_switch = set(
- await session.scalars(
- select(OrganizationMember.user_id).where(
- OrganizationMember.organization_id == organization_id
- )
+ rows = await session.execute(
+ select(OrganizationMember.user_id, OrganizationMember.role).where(
+ OrganizationMember.organization_id == organization_id
)
)
+ in_switch = {user_id: role for user_id, role in rows}
users = await self._users_of(list(in_workos))
subjects = await self._subjects_of(list(in_switch))
wanted: dict[str, str | None] = {
@@ -195,18 +320,32 @@ class WorkOSMirror:
)
continue
wanted.setdefault(user_id, in_workos.get(subject))
+ complete = True
for user_id, role in sorted(wanted.items()):
- await apply_from_directory(
- self._session_factory,
- self._revocation,
- organization_id=organization_id,
- user_id=user_id,
- wanted=role,
- source=FROM_CHECK,
- )
+ if in_switch.get(user_id) == role:
+ continue
+ await self._ensure_held(fence)
+ try:
+ await apply_from_directory(
+ self._session_factory,
+ self._dependencies,
+ organization_id=organization_id,
+ workos_organization_id=workos_organization_id,
+ user_id=user_id,
+ wanted=role,
+ source=FROM_CHECK,
+ )
+ except Exception:
+ logger.exception(
+ "Applying WorkOS's membership of user %s in organization %s failed",
+ user_id,
+ organization_id,
+ )
+ complete = False
+ return complete
async def _organization_deleted(
- self, organization_id: str, data: dict[str, Any]
+ self, fence: Fence, organization_id: str, data: dict[str, Any]
) -> None:
"""Deleting a linked organization in WorkOS removes nobody here: that
is too much to take away on one event. It is reported for an operator
@@ -217,6 +356,7 @@ class WorkOSMirror:
data.get("id"),
organization_id,
)
+ await self._ensure_held(fence)
async with organization_session(
self._session_factory, organization_id
) as session:
@@ -234,6 +374,36 @@ class WorkOSMirror:
)
await session.commit()
+ async def _restart(
+ self, fence: Fence, cursor: DirectoryEventCursor, reason: str
+ ) -> None:
+ """Start the events again from now, and check every organization in
+ full for what fell between."""
+ logger.warning(
+ "WorkOS refused the mirror's place in its events (after %s, from %s): "
+ "%s; starting again from now and checking every organization",
+ cursor.after,
+ cursor.started_at.isoformat(),
+ reason,
+ )
+ await self._write_cursor(fence, started_at=datetime.now(UTC), after=None)
+ with no_tenant():
+ async with self._session_factory() as session:
+ await record_server_audit_event(
+ session,
+ actor_user_id=None,
+ action=ServerAuditAction.DIRECTORY_EVENTS_RESTARTED,
+ target_type="directory",
+ target_id=CURSOR_SOURCE,
+ details={
+ "after": cursor.after,
+ "started_at": cursor.started_at.isoformat(),
+ "reason": reason,
+ },
+ )
+ await session.commit()
+ self._recheck |= set(await self._linked())
+
async def _linked(self) -> dict[str, str]:
"""Each linked organization's WorkOS organization, by its id."""
linked: dict[str, str] = {}
@@ -287,46 +457,95 @@ class WorkOSMirror:
await session.execute(
insert(DirectoryEventCursor)
.values(
- source=CURSOR_SOURCE, after=None, started_at=datetime.now(UTC)
+ source=CURSOR_SOURCE,
+ after=None,
+ started_at=datetime.now(UTC),
)
.on_conflict_do_nothing(index_elements=["source"])
)
await session.commit()
cursor = await session.get(DirectoryEventCursor, CURSOR_SOURCE)
assert cursor is not None
+ session.expunge(cursor)
return cursor
- async def _advance(self, after: str) -> None:
+ async def _write_cursor(
+ self, fence: Fence, *, started_at: datetime | None, after: str | None
+ ) -> None:
+ """Set the cursor to `after` (and `started_at`, when given), in one
+ statement that writes only while `fence` holds."""
+ values: dict[str, Any] = {"after": after, "updated_at": datetime.now(UTC)}
+ if started_at is not None:
+ values["started_at"] = started_at
with no_tenant():
async with self._session_factory() as session:
- cursor = await session.get(DirectoryEventCursor, CURSOR_SOURCE)
- assert cursor is not None
- cursor.after = after
- cursor.updated_at = datetime.now(UTC)
+ written = await session.execute(
+ update(DirectoryEventCursor)
+ .where(DirectoryEventCursor.source == CURSOR_SOURCE, _holds(fence))
+ .values(**values)
+ .returning(DirectoryEventCursor.source)
+ )
+ if written.first() is None:
+ raise TakenOver()
await session.commit()
+ async def _ensure_held(self, fence: Fence) -> None:
+ """Raise `TakenOver` unless `fence` is still this process's, before a
+ change it must not make beside a new holder."""
+ with no_tenant():
+ async with self._session_factory() as session:
+ held = await self._leases.held(
+ session,
+ kind=fence.kind,
+ tokens={fence.key: fence.token},
+ process_id=fence.process_id,
+ )
+ if fence.key not in held:
+ raise TakenOver()
+
+
+def _holds(fence: Fence) -> Any:
+ return exists().where(
+ OwnershipLease.kind == fence.kind,
+ OwnershipLease.key == fence.key,
+ OwnershipLease.token == fence.token,
+ OwnershipLease.process_id == fence.process_id,
+ )
+
+
+def _organization_named(event: dict[str, Any]) -> str | None:
+ """The WorkOS organization an event is about."""
+ data = event.get("data")
+ if not isinstance(data, dict):
+ return None
+ key = "id" if event.get("event") == "organization.deleted" else "organization_id"
+ value = data.get(key)
+ return value if isinstance(value, str) else None
+
def own_workos_mirror(ownership: Ownership, mirror: WorkOSMirror) -> OwnedKind:
"""Register the mirror as owned work, wanted by this process; it runs here
only while it holds it."""
running: dict[str, asyncio.Task[None]] = {}
- def launch(key: str) -> None:
- task = asyncio.create_task(mirror.run(), name="workos-mirror")
+ def launch(key: str, fence: Fence) -> None:
+ task = asyncio.create_task(mirror.run(fence), name="workos-mirror")
running[key] = task
- task.add_done_callback(lambda done: ended(key, done))
+ task.add_done_callback(lambda done: ended(key, fence, done))
- def ended(key: str, task: asyncio.Task[None]) -> None:
+ def ended(key: str, fence: Fence, task: asyncio.Task[None]) -> None:
if task.cancelled() or running.get(key) is not task:
return
- logger.error(
- "WorkOS mirror stopped unexpectedly; restarting",
- exc_info=task.exception(),
- )
- launch(key)
+ error = task.exception()
+ if error is None:
+ # Returned on its own: taken over, which the next balance settles.
+ running.pop(key, None)
+ return
+ logger.error("WorkOS mirror stopped unexpectedly; restarting", exc_info=error)
+ launch(key, fence)
async def start(key: str, fence: Fence) -> None:
- launch(key)
+ launch(key, fence)
async def stop(key: str) -> None:
task = running.pop(key, None)
diff --git a/core/tests/switch_core/test_organization_mirror_apply.py b/core/tests/switch_core/test_organization_mirror_apply.py
index 1fc064186..1b2751ca4 100644
--- a/core/tests/switch_core/test_organization_mirror_apply.py
+++ b/core/tests/switch_core/test_organization_mirror_apply.py
@@ -13,7 +13,8 @@ import pytest
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
-from switch_core.connections.broker import ServiceBroker
+from switch_core import organization_mirror
+from switch_core.connections.broker import ServiceBroker, ServiceError
from switch_core.connections.loader import CATALOG
from switch_core.connections.sign_in_flows import SignInFlows
from switch_core.db.audit import AuditAction, OrganizationAuditAction
@@ -31,11 +32,13 @@ from switch_core.db.models import (
)
from switch_core.db.stores.agent_store import AgentStore
from switch_core.db.stores.api_key_store import ApiKeyStore
+from switch_core.db.stores.organization_store import OrganizationStore
from switch_core.db.stores.service_connection_store import ServiceConnectionStore
+from switch_core.db.stores.user_store import UserStore
from switch_core.db.stores.workspace_service_store import WorkspaceServiceStore
from switch_core.keys import Keyring
from switch_core.organization_access import AccessRevocation
-from switch_core.organization_mirror import apply_from_directory
+from switch_core.organization_mirror import MirrorDependencies, apply_from_directory
from tests.conftest import RLSHarness
pytestmark = pytest.mark.no_ambient_tenant
@@ -49,6 +52,16 @@ def _id(prefix: str) -> str:
return f"{prefix}-{uuid.uuid4().hex[:8]}"
+def _dependencies(
+ session_factory: async_sessionmaker[AsyncSession],
+) -> MirrorDependencies:
+ return MirrorDependencies(
+ revocation=_revocation(session_factory),
+ organization_store=OrganizationStore(),
+ user_store=UserStore(),
+ )
+
+
def _revocation(session_factory: async_sessionmaker[AsyncSession]) -> AccessRevocation:
return AccessRevocation(
agent_store=AgentStore(),
@@ -79,6 +92,7 @@ class _World:
def __init__(self, organization_id: str, joined: str, reached: str) -> None:
self.organization_id = organization_id
+ self.workos_organization_id = _id("org_workos")
self.joined = joined
self.reached = reached
self.ada = _id("ada")
@@ -99,7 +113,7 @@ class _World:
id=world.organization_id,
slug=world.organization_id,
name="Acme",
- workos_organization_id=_id("org_workos"),
+ workos_organization_id=world.workos_organization_id,
)
)
for user_id in (world.ada, world.bea):
@@ -230,11 +244,18 @@ async def _workspace_actions(owner, tenant_id: str) -> list[str]:
)
-async def _apply(rls_harness: RLSHarness, world: _World, wanted: str | None) -> None:
+async def _apply(
+ rls_harness: RLSHarness,
+ world: _World,
+ wanted: str | None,
+ *,
+ workos_organization_id: str | None = None,
+) -> None:
await apply_from_directory(
rls_harness.restricted,
- _revocation(rls_harness.restricted),
+ _dependencies(rls_harness.restricted),
organization_id=world.organization_id,
+ workos_organization_id=workos_organization_id or world.workos_organization_id,
user_id=world.ada,
wanted=wanted,
source="directory_event",
@@ -283,6 +304,19 @@ class TestTakingAway:
[(action, _)] = await _organization_events(owner, world.organization_id)
assert action == OrganizationAuditAction.MEMBER_DEMOTED_FROM_DIRECTORY
+ async def test_nothing_is_applied_once_the_organization_follows_another(
+ self, rls_harness: RLSHarness
+ ) -> None:
+ """A role read from a WorkOS organization the organization no longer
+ follows (unlinked or relinked meanwhile) is not WorkOS's say here."""
+ owner = rls_harness.owner
+ world = await _World.make(owner)
+
+ await _apply(rls_harness, world, None, workos_organization_id=_id("org_old"))
+
+ assert await _role(owner, world.organization_id, world.ada) == "org_admin"
+ assert await _organization_events(owner, world.organization_id) == []
+
async def test_the_same_role_changes_nothing(self, rls_harness: RLSHarness) -> None:
owner = rls_harness.owner
world = await _World.make(owner)
@@ -373,8 +407,9 @@ class TestGranting:
await apply_from_directory(
rls_harness.restricted,
- _revocation(rls_harness.restricted),
+ _dependencies(rls_harness.restricted),
organization_id=world.organization_id,
+ workos_organization_id=world.workos_organization_id,
user_id=cy,
wanted="member",
source="daily_check",
@@ -384,3 +419,54 @@ class TestGranting:
[(action, details)] = await _organization_events(owner, world.organization_id)
assert action == OrganizationAuditAction.MEMBER_ADDED_FROM_DIRECTORY
assert details == {"source": "daily_check", "role": "member"}
+
+
+class TestARetryAfterAFailure:
+ async def test_finds_the_workspaces_left_and_their_conflicts_recorded(
+ self, rls_harness: RLSHarness, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ """Ada is the only owner of `joined` and owns an agent there; a second
+ workspace she is in has a busy connection the first time."""
+ owner = rls_harness.owner
+ world = await _World.make(owner, ada_owns_joined=True)
+ await _agent(owner, world.joined, world.ada)
+ busy = _id("busy")
+ async with owner() as session:
+ session.add(
+ Tenant(
+ id=busy,
+ slug=busy,
+ name="Busy",
+ organization_id=world.organization_id,
+ )
+ )
+ await session.flush()
+ session.add(TenantMember(tenant_id=busy, user_id=world.ada, role="member"))
+ await session.commit()
+ real = organization_mirror.remove_membership
+
+ async def busy_once(session, revocation, membership, **kwargs):
+ if membership.tenant_id == busy:
+ raise ServiceError(
+ 409, "internal", "A connection is busy.", retryable=True
+ )
+ return await real(session, revocation, membership, **kwargs)
+
+ monkeypatch.setattr(organization_mirror, "remove_membership", busy_once)
+ with pytest.raises(ServiceError):
+ await _apply(rls_harness, world, None)
+ monkeypatch.setattr(organization_mirror, "remove_membership", real)
+ await _apply(rls_harness, world, None)
+
+ conflicts = [
+ details["kind"]
+ for action, details in await _organization_events(
+ owner, world.organization_id
+ )
+ if action == OrganizationAuditAction.DIRECTORY_CONFLICT
+ ]
+ assert sorted(conflicts) == [
+ "agents_left_with_owner",
+ "workspace_left_without_owner",
+ ]
+ assert await _role(owner, world.organization_id, world.ada) is None
diff --git a/core/tests/switch_core/test_workos_directory.py b/core/tests/switch_core/test_workos_directory.py
index 17fd8978a..8451e67c5 100644
--- a/core/tests/switch_core/test_workos_directory.py
+++ b/core/tests/switch_core/test_workos_directory.py
@@ -15,7 +15,11 @@ from switch_core.organization_directory import (
DirectoryRefusal,
DirectoryUnavailable,
)
-from switch_core.workos_directory import WorkOSDirectory, customer_id_of
+from switch_core.workos_directory import (
+ EventCursorRejected,
+ WorkOSDirectory,
+ customer_id_of,
+)
API = "https://workos.example.test"
KEY = "sk_test_example"
@@ -267,9 +271,9 @@ class TestWhatTheMirrorReads:
assert seen[1].url.params["after"] == "event_1"
assert "range_start" not in seen[1].url.params
- async def test_an_event_switch_cannot_read_is_unavailable(self) -> None:
+ async def test_a_page_switch_cannot_read_is_unavailable(self) -> None:
directory = _directory(
- lambda request: httpx.Response(200, json={"data": [{"id": "event_1"}]})
+ lambda request: httpx.Response(200, json={"data": ["not an event"]})
)
with pytest.raises(DirectoryUnavailable):
@@ -277,6 +281,35 @@ class TestWhatTheMirrorReads:
("organization.deleted",), after=None, range_start="x"
)
+ async def test_an_event_is_handed_back_as_sent_for_the_caller_to_judge(
+ self,
+ ) -> None:
+ directory = _directory(
+ lambda request: httpx.Response(200, json={"data": [{"id": "event_1"}]})
+ )
+
+ assert await directory.events(
+ ("organization.deleted",), after=None, range_start="x"
+ ) == [{"id": "event_1"}]
+
+ @pytest.mark.parametrize("status", [400, 404, 422])
+ async def test_a_refused_cursor_says_so(self, status: int) -> None:
+ directory = _directory(lambda request: httpx.Response(status))
+
+ with pytest.raises(EventCursorRejected):
+ await directory.events(
+ ("organization.deleted",), after="event_old", range_start="x"
+ )
+
+ async def test_other_refusals_are_not_the_cursor(self) -> None:
+ directory = _directory(lambda request: httpx.Response(401))
+
+ with pytest.raises(DirectoryUnavailable) as refused:
+ await directory.events(
+ ("organization.deleted",), after="event_1", range_start="x"
+ )
+ assert not isinstance(refused.value, EventCursorRejected)
+
async def test_one_membership_is_read_as_it_is_now(self) -> None:
def handler(request: httpx.Request) -> httpx.Response:
assert request.url.params["user_id"] == "user_A"
diff --git a/core/tests/switch_core/test_workos_mirror.py b/core/tests/switch_core/test_workos_mirror.py
index 23df2534e..7a54c1b9e 100644
--- a/core/tests/switch_core/test_workos_mirror.py
+++ b/core/tests/switch_core/test_workos_mirror.py
@@ -5,11 +5,13 @@ WorkOS; `owner` only arranges fixtures and reads results."""
from __future__ import annotations
import uuid
+from datetime import UTC, datetime, timedelta
from typing import Any
import pytest
from sqlalchemy import select
+from switch_core import workos_mirror
from switch_core.db.audit import OrganizationAuditAction
from switch_core.db.models import (
DirectoryEventCursor,
@@ -17,12 +19,25 @@ from switch_core.db.models import (
Organization,
OrganizationAuditEvent,
OrganizationMember,
+ OwnershipLease,
+ ServerAuditEvent,
+ SwitchCoreProcess,
User,
)
+from switch_core.db.stores.ownership_lease_store import OwnershipLeaseStore
from switch_core.organization_directory import DirectoryUnavailable
-from switch_core.workos_mirror import EVENT_TYPES, WorkOSMirror
+from switch_core.ownership import Fence
+from switch_core.workos_directory import EventCursorRejected
+from switch_core.workos_mirror import (
+ EVENT_TYPES,
+ MAX_EVENT_ATTEMPTS,
+ MIRROR_KEY,
+ MIRROR_KIND,
+ TakenOver,
+ WorkOSMirror,
+)
from tests.conftest import RLSHarness
-from tests.switch_core.test_organization_mirror_apply import _revocation
+from tests.switch_core.test_organization_mirror_apply import _dependencies
pytestmark = pytest.mark.no_ambient_tenant
@@ -43,6 +58,7 @@ class _WorkOS:
self.page = page
self.asked: list[dict[str, Any]] = []
self.fail_role_for: str | None = None
+ self.reject_cursor = False
def change(self, subject: str, organization: str, role: str | None) -> None:
if role is None:
@@ -68,9 +84,11 @@ class _WorkOS:
) -> list[dict[str, Any]]:
assert types == EVENT_TYPES
self.asked.append({"after": after, "range_start": range_start})
+ if self.reject_cursor:
+ raise EventCursorRejected("WorkOS answered 400 for /events.")
start = 0
if after is not None:
- start = next(i for i, e in enumerate(self.log) if e["id"] == after) + 1
+ start = next(i for i, e in enumerate(self.log) if e.get("id") == after) + 1
return self.log[start : start + self.page]
async def membership_role(self, subject: str, organization: str) -> str | None:
@@ -139,13 +157,41 @@ async def _roles(owner, organization_id: str) -> dict[str, str]:
return {user_id: role for user_id, role in rows}
-def _mirror(rls_harness: RLSHarness, workos: _WorkOS) -> WorkOSMirror:
- return WorkOSMirror(
+class _Running:
+ """A mirror and the fence it holds, as `ownership.py` would start it."""
+
+ def __init__(self, mirror: WorkOSMirror, fence: Fence) -> None:
+ self.mirror = mirror
+ self.fence = fence
+
+ async def poll_once(self) -> int:
+ return await self.mirror.poll_once(self.fence)
+
+ async def check_once(self) -> None:
+ await self.mirror.check_once(self.fence)
+
+
+async def _mirror(rls_harness: RLSHarness, workos: _WorkOS) -> _Running:
+ process_id = _id("process")
+ async with rls_harness.owner() as session:
+ session.add(SwitchCoreProcess(id=process_id))
+ await session.flush()
+ session.add(
+ OwnershipLease(
+ kind=MIRROR_KIND, key=MIRROR_KEY, process_id=process_id, token=1
+ )
+ )
+ await session.commit()
+ mirror = WorkOSMirror(
rls_harness.restricted,
workos, # type: ignore[arg-type]
- _revocation(rls_harness.restricted),
+ _dependencies(rls_harness.restricted),
+ OwnershipLeaseStore(),
ISSUER,
)
+ return _Running(
+ mirror, Fence(kind=MIRROR_KIND, key=MIRROR_KEY, process_id=process_id, token=1)
+ )
class TestEvents:
@@ -156,7 +202,7 @@ class TestEvents:
organization_id, workos_id = await _linked(owner)
ada = await _person(owner, "user_ADA")
workos = _WorkOS(page=1)
- mirror = _mirror(rls_harness, workos)
+ mirror = await _mirror(rls_harness, workos)
assert await mirror.poll_once() == 0
workos.change("user_ADA", workos_id, "member")
@@ -180,7 +226,7 @@ class TestEvents:
organization_id, workos_id = await _linked(owner)
await _person(owner, "user_ADA")
workos = _WorkOS()
- mirror = _mirror(rls_harness, workos)
+ mirror = await _mirror(rls_harness, workos)
await mirror.poll_once()
workos.change("user_ADA", workos_id, "member")
workos.change("user_ADA", workos_id, None)
@@ -198,7 +244,7 @@ class TestEvents:
await _member(owner, organization_id, bea, "org_admin")
workos = _WorkOS()
workos.memberships[("user_ADA", workos_id)] = "member"
- mirror = _mirror(rls_harness, workos)
+ mirror = await _mirror(rls_harness, workos)
await mirror.poll_once()
workos.change("user_ADA", workos_id, None)
@@ -213,7 +259,7 @@ class TestEvents:
organization_id, workos_id = await _linked(owner)
await _person(owner, "user_ADA")
workos = _WorkOS()
- mirror = _mirror(rls_harness, workos)
+ mirror = await _mirror(rls_harness, workos)
await mirror.poll_once()
workos.change("user_ADA", _id("org_elsewhere"), "org_admin")
workos.change("user_NOBODY", workos_id, "org_admin")
@@ -229,7 +275,7 @@ class TestEvents:
organization_id, workos_id = await _linked(owner)
ada = await _person(owner, "user_ADA")
workos = _WorkOS()
- mirror = _mirror(rls_harness, workos)
+ mirror = await _mirror(rls_harness, workos)
await mirror.poll_once()
workos.change("user_ADA", workos_id, "member")
workos.fail_role_for = "user_ADA"
@@ -249,7 +295,7 @@ class TestEvents:
ada = await _person(owner, "user_ADA")
await _member(owner, organization_id, ada, "org_admin")
workos = _WorkOS()
- mirror = _mirror(rls_harness, workos)
+ mirror = await _mirror(rls_harness, workos)
await mirror.poll_once()
workos.log.append(
{
@@ -298,7 +344,7 @@ class TestDailyCheck:
("user_STRANGER", workos_id): "member",
}
- await _mirror(rls_harness, workos).check_once()
+ await (await _mirror(rls_harness, workos)).check_once()
assert await _roles(owner, organization_id) == {
ada: "member",
@@ -307,3 +353,147 @@ class TestDailyCheck:
eve: "org_admin",
}
assert f"User {dee} is in organization" in caplog.text
+
+
+class TestRecovering:
+ async def test_an_event_failing_every_time_is_passed_over_and_its_organization_rechecked(
+ self, rls_harness: RLSHarness, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ owner = rls_harness.owner
+ organization_id, workos_id = await _linked(owner)
+ ada = await _person(owner, "user_ADA")
+ bea = await _person(owner, "user_BEA")
+ workos = _WorkOS()
+ mirror = await _mirror(rls_harness, workos)
+ await mirror.poll_once()
+ workos.change("user_ADA", workos_id, "member")
+ workos.change("user_BEA", workos_id, "member")
+ workos.fail_role_for = "user_ADA"
+
+ for _ in range(MAX_EVENT_ATTEMPTS - 1):
+ with pytest.raises(DirectoryUnavailable):
+ await mirror.poll_once()
+ assert await mirror.poll_once() == 2
+
+ assert await _roles(owner, organization_id) == {bea: "member"}
+ assert "failed 5 times; passed over" in caplog.text
+ workos.fail_role_for = None
+ await mirror.poll_once()
+ assert await _roles(owner, organization_id) == {ada: "member", bea: "member"}
+
+ async def test_an_event_switch_cannot_read_is_passed_over(
+ self, rls_harness: RLSHarness
+ ) -> None:
+ owner = rls_harness.owner
+ organization_id, workos_id = await _linked(owner)
+ ada = await _person(owner, "user_ADA")
+ workos = _WorkOS()
+ mirror = await _mirror(rls_harness, workos)
+ await mirror.poll_once()
+ workos.log.append({"event": "organization_membership.created"})
+ workos.log.append(
+ {"id": "event_bad", "event": "organization_membership.updated"}
+ )
+ workos.change("user_ADA", workos_id, "member")
+
+ await mirror.poll_once()
+
+ assert await _roles(owner, organization_id) == {ada: "member"}
+
+ async def test_an_empty_pass_moves_the_start_forward(
+ self, rls_harness: RLSHarness
+ ) -> None:
+ owner = rls_harness.owner
+ workos = _WorkOS()
+ mirror = await _mirror(rls_harness, workos)
+ await mirror.poll_once()
+ async with owner() as session:
+ cursor = await session.get(DirectoryEventCursor, "workos")
+ assert cursor is not None
+ cursor.started_at = datetime.now(UTC) - timedelta(days=60)
+ await session.commit()
+
+ await mirror.poll_once()
+
+ async with owner() as session:
+ cursor = await session.get(DirectoryEventCursor, "workos")
+ assert cursor is not None
+ assert cursor.started_at > datetime.now(UTC) - timedelta(minutes=10)
+
+ async def test_a_refused_cursor_starts_again_from_now_and_checks_everything(
+ self, rls_harness: RLSHarness, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ owner = rls_harness.owner
+ organization_id, workos_id = await _linked(owner)
+ ada = await _person(owner, "user_ADA")
+ workos = _WorkOS()
+ mirror = await _mirror(rls_harness, workos)
+ await mirror.poll_once()
+ workos.memberships[("user_ADA", workos_id)] = "org_admin"
+ workos.reject_cursor = True
+
+ assert await mirror.poll_once() == 0
+ workos.reject_cursor = False
+ await mirror.poll_once()
+
+ assert await _roles(owner, organization_id) == {ada: "org_admin"}
+ assert "starting again from now" in caplog.text
+ async with owner() as session:
+ [event] = await session.scalars(select(ServerAuditEvent))
+ assert event.action == "directory.events_restarted"
+
+
+class TestOneHolder:
+ async def test_a_mirror_taken_over_writes_nothing_more(
+ self, rls_harness: RLSHarness
+ ) -> None:
+ owner = rls_harness.owner
+ organization_id, workos_id = await _linked(owner)
+ await _person(owner, "user_ADA")
+ workos = _WorkOS()
+ mirror = await _mirror(rls_harness, workos)
+ await mirror.poll_once()
+ workos.change("user_ADA", workos_id, "member")
+ async with owner() as session:
+ lease = await session.get(OwnershipLease, (MIRROR_KIND, MIRROR_KEY))
+ assert lease is not None
+ lease.token = 2
+ await session.commit()
+
+ with pytest.raises(TakenOver):
+ await mirror.poll_once()
+
+ assert await _roles(owner, organization_id) == {}
+ async with owner() as session:
+ cursor = await session.get(DirectoryEventCursor, "workos")
+ assert cursor is not None and cursor.after is None
+
+
+class TestDailyCheckFailures:
+ async def test_one_member_failing_does_not_stop_the_rest(
+ self, rls_harness: RLSHarness, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ owner = rls_harness.owner
+ organization_id, workos_id = await _linked(owner)
+ ada = await _person(owner, "user_ADA")
+ bea = await _person(owner, "user_BEA")
+ workos = _WorkOS()
+ workos.memberships = {
+ ("user_ADA", workos_id): "member",
+ ("user_BEA", workos_id): "member",
+ }
+ real = workos_mirror.apply_from_directory
+
+ async def failing_for_ada(*args, **kwargs):
+ if kwargs["user_id"] == ada:
+ raise RuntimeError("database went away")
+ return await real(*args, **kwargs)
+
+ monkeypatch.setattr(workos_mirror, "apply_from_directory", failing_for_ada)
+ mirror = await _mirror(rls_harness, workos)
+ await mirror.check_once()
+ assert await _roles(owner, organization_id) == {bea: "member"}
+
+ monkeypatch.setattr(workos_mirror, "apply_from_directory", real)
+ await mirror.poll_once()
+ assert await _roles(owner, organization_id) == {ada: "member", bea: "member"}
diff --git a/docs/old/workos-directory.md b/docs/old/workos-directory.md
index c00320fbf..626d1c503 100644
--- a/docs/old/workos-directory.md
+++ b/docs/old/workos-directory.md
@@ -67,6 +67,13 @@ every linked organization's members with WorkOS and applies the difference,
which catches up anything from before it ran or missed since. It keeps its
place in the database, so a restart resumes where it stopped.
+A change that fails to apply is tried again on the next pass, up to five
+times. It is then passed over, and that organization is checked in full on
+every pass until the check succeeds, so one stuck change doesn't hold up the
+rest. If WorkOS no longer has the events from where the mirror stopped (it
+keeps 90 days), the mirror starts again from now, records that in the
+server's audit log, and checks every linked organization in full.
+
For each person it matches (by the WorkOS user their account signs in as):
- **Added or made `org_admin` in WorkOS:** added, or made Org Admin, as at
@@ -85,8 +92,10 @@ recorded in the organization's audit log as "WorkOS change broke a Switch
rule", naming what:
- the organization's last Org Admin was removed or demoted;
-- agents the person owns were left in place, still running on their keys,
- owned by someone who can no longer act there, for an admin to delete;
+- agents the person owns were left in place, owned by someone who can no
+ longer act there, for an admin to delete. Each agent keeps its own key,
+ which acts as the agent, never as the person; every key of the person's
+ own goes;
- a workspace was left with no owner (its organization's Org Admins still
administer it).