Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
fix(organizations): withdraw services after the setting commits, and revoke through a usable app
Petr Bauch1e889256f9b2
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 1d2eac972..b264908fb 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -128,7 +128,8 @@ version of their own to them without also giving them a release of their own.workspaces. Unlocked, its setting applies where a workspace admin hasn'tset the service themselves; locked, it applies everywhere and workspaceadmins can't change it. Wherever a change leaves a service off, the- tokens issued for it there are revoked. A workspace admin can drop their+ tokens issued for it there are revoked; any a change could not reach,+ the periodic token upkeep revokes within five minutes. A workspace admin can drop theirown switch (`DELETE /gateway/workspace/services/{service}`) to take theorganization's setting.diff --git a/core/switch_core/connections/app_credentials.py b/core/switch_core/connections/app_credentials.pyindex c9f73c7aa..c218f0101 100644--- a/core/switch_core/connections/app_credentials.py+++ b/core/switch_core/connections/app_credentials.py@@ -279,7 +279,9 @@ def load_saved_app(decrypts the row's. Raises `AppLoadError` when it cannot be used."""try:opened = secrets()- except UndecryptableError:+ except (UndecryptableError, ValueError):+ # No key opens it, or what it opens is not a payload of secrets:+ # either way, only entering the secrets again mends it.raise UnreadableAppError(UNREADABLE) from Nonevalues = AppValues(client_id, dict(settings), opened)if spec.kind == "github_app":diff --git a/core/switch_core/connections/broker.py b/core/switch_core/connections/broker.pyindex c726e6158..021c5afd0 100644--- a/core/switch_core/connections/broker.py+++ b/core/switch_core/connections/broker.py@@ -1252,7 +1252,11 @@ class ServiceBroker:f"{name} is not set up on this server, so Switch could not "f"revoke the sign-in. Revoke it at {name}.")- for adapter in (here, server):+ # A sign-in from before clients were recorded predates organizations'+ # apps, so it was made through the server's.+ for adapter in (+ (here, server) if oauth_client_id is not None else (server, here)+ ):if adapter is not None and (oauth_client_id is Noneor adapter.current_client_id() in (None, oauth_client_id)@@ -2197,15 +2201,28 @@ class ServiceBroker:)await self._store.clear_expired(session, now)await self._store.queue_orphaned(session)+ # And every live token for a service now off in the workspace, by+ # whichever switch: a change that withdrew a service but could not+ # finish queueing its tokens (an organization's, across many+ # workspaces) is finished here.+ switched_off = await self.workspace_disabled(session)+ if switched_off:+ await self._store.queue_revocation(+ session,+ ServiceTokenIssuance.revoke_requested.is_(False),+ ServiceTokenIssuance.service.in_(switched_off),+ )until = now + REVOCATION_CLAIMrows = await self._store.claim_revocations(session, conditions, now=now, until=until, limit=REVOCATION_BATCH)claimed = [(row.id, row.service, row.encrypted_token) for row in rows]# Once per service, before the revocations run side by side. An- # issued token is revoked with itself, so any app's adapter will do.+ # issued token is revoked with itself, so any app's adapter will do:+ # the one that applies here, or the server's where that one can't be+ # used (an organization's app whose secrets no key opens).adapters = {- service: await self._here(session, service)+ service: await self._here(session, service) or self._adapters.get(service)for service in {service for _, service, _ in claimed}}await session.commit()diff --git a/core/switch_core/connections/flows.py b/core/switch_core/connections/flows.pyindex d98f36f2b..a8a5f15ab 100644--- a/core/switch_core/connections/flows.py+++ b/core/switch_core/connections/flows.py@@ -372,6 +372,13 @@ class ServiceFlows:changes={"account_label": exchange.account.label, "warning": warning},)+ async def refused(self, flow: ServiceFlow, error: str) -> None:+ """The browser could not be sent to the vendor: the flow ends with+ why, for Console to show, rather than waiting until it expires."""+ await self._flows.advance(+ flow.stored, expect=("pending",), to="failed", changes={"error": error}+ )+async def failed(self, flow: StoredFlow, error: str) -> None:await self._flows.advance(flow, expect=("checking",), to="failed", changes={"error": error}diff --git a/core/switch_core/db/stores/workspace_service_store.py b/core/switch_core/db/stores/workspace_service_store.pyindex e9ad829b0..cff991207 100644--- a/core/switch_core/db/stores/workspace_service_store.py+++ b/core/switch_core/db/stores/workspace_service_store.py@@ -178,12 +178,6 @@ class WorkspaceServiceStore:if not switch.enabled}- async def is_enabled(- self, session: AsyncSession, service: str, *, organizations: bool- ) -> bool:- switches = await self.switches(session, [service], organizations=organizations)- return switches[service].enabled-async def organization_settings(self, session: AsyncSession) -> dict[str, OrganizationServiceSetting]:diff --git a/core/switch_core/gateway/auth.py b/core/switch_core/gateway/auth.pyindex 6f4c6162d..0a14bde84 100644--- a/core/switch_core/gateway/auth.py+++ b/core/switch_core/gateway/auth.py@@ -422,15 +422,11 @@ async def organization_workspace_ids(admin_id: str,organization_id: str,) -> list[str]:- """Every workspace of `organization_id`, as its `org_admin` `admin_id`- reaches them: an `org_admin` may bind each one. Each read on its own- short session, as `workspaces_in_organization` does."""+ """The ids of `organization_workspaces`."""return [tenant.id- for tenant in await _in_organization(- session_factory,- await tenants_bindable_by_user(session_factory, admin_id),- organization_id,+ for tenant in await organization_workspaces(+ session_factory, admin_id, organization_id)]diff --git a/core/switch_core/gateway/organization_services.py b/core/switch_core/gateway/organization_services.pyindex f971b2c0c..f8bd1792f 100644--- a/core/switch_core/gateway/organization_services.py+++ b/core/switch_core/gateway/organization_services.py@@ -11,16 +11,16 @@ service out of all its workspaces. Clearing the setting gives up theorganization's say.A change that leaves the service off in a workspace withdraws what it was-issued there, as a workspace admin's switch does, and must not leave a token-live behind a lock. A workspace's tokens can only be queued from a session-bound to it, so one transaction cannot cover the setting and every-workspace. Instead, every workspace the change would switch off has its-tokens queued first, each in its own transaction, and if any cannot be, the-setting is not saved. Then the setting commits. Then each of those-workspaces is swept again, for a token issued in between, and the vendor-revocations run; a workspace that fails there is logged and named in the-warning, and the rest carry on. A queued token the vendor has not revoked-yet is retried by the periodic tick. Grants are kept.+issued there, as a workspace admin's switch does. A workspace's tokens can+only be queued from a session bound to it, so one transaction cannot cover+the setting and every workspace. The setting commits first, with its audit+row; then every workspace is visited, a few at a time, and its tokens for a+service now off there are queued and revoked. A workspace that fails is+logged and named in the warning, and the rest carry on. Nothing is lost if+one does, or the process stops part way: the periodic upkeep+(`connections/maintenance.py`) queues every live token for a service that is+off in its workspace on each pass, within five minutes, and retries what the+vendor has not revoked yet. Grants are kept.`org_admin` only, acting on the bound workspace's organization. Mounted onlywith both `service_connections` and `organization_connector_apps` on, the@@ -29,7 +29,9 @@ flag the rest of the organization's say in services ships under.from __future__ import annotations+import asyncioimport logging+from collections.abc import Awaitable, Callablefrom datetime import datetimefrom typing import Annotated, Any@@ -57,6 +59,7 @@ from switch_core.db.stores.organization_store import OrganizationStorefrom switch_core.db.stores.user_store import UserStorefrom switch_core.db.stores.workspace_service_store import (OrganizationSetting,+ ServiceSwitch,WorkspaceServiceStore,)from switch_core.gateway.auth import get_current_user, organization_workspaces@@ -71,6 +74,10 @@ from switch_core.gateway.organizations import require_organization_adminlogger = logging.getLogger(__name__)router = APIRouter(prefix="/organizations/current/services")+# Workspaces visited at once by one request: enough that a large+# organization's page or change does not walk them one by one, few enough to+# leave the connection pool to everyone else.+CONCURRENT_WORKSPACES = 4SETTINGS = OrganizationServiceStore()WORKSPACES = WorkspaceServiceStore()@@ -121,13 +128,13 @@ async def _reach("""For each service: in how many of the workspaces it is on, in how manytheir admin set it themselves, and how many people, each once, have itconnected across them. A workspace's rows are readable only bound to it,- so each is read on its own short session, in two queries, after the- request's has let its connection go."""+ so each is read on its own short session, in two queries, a few at a time,+ after the request's has let its connection go."""services = _services()- on: dict[str, int] = {}- own: dict[str, int] = {}- people: dict[str, set[str]] = {}- for workspace in workspaces:++ async def visit(+ workspace: Tenant,+ ) -> tuple[dict[str, tuple[ServiceSwitch, bool]], list[tuple[str, str]]]:async with tenant_session(session_factory, workspace.id) as session:switches = await WORKSPACES.switches_under(session, services, settings)connected = await session.execute(@@ -138,8 +145,14 @@ async def _reach().distinct())- for service, user_id in connected.tuples():- people.setdefault(service, set()).add(user_id)+ return switches, list(connected.tuples())++ on: dict[str, int] = {}+ own: dict[str, int] = {}+ people: dict[str, set[str]] = {}+ for switches, connected in await _each_workspace(workspaces, visit):+ for service, user_id in connected:+ people.setdefault(service, set()).add(user_id)for service, (switch, set_here) in switches.items():if switch.enabled:on[service] = on.get(service, 0) + 1@@ -148,48 +161,19 @@ async def _reach(return on, own, {service: len(users) for service, users in people.items()}-async def _queue_where_off(- session_factory: async_sessionmaker[AsyncSession],- broker: ServiceBroker,- workspaces: list[Tenant],- service: str,- setting: OrganizationSetting | None,-) -> list[Tenant]:- """Before the setting is saved: queue what each workspace `setting` would- switch `service` off in was issued for it, each in its own transaction.- The workspaces it would switch off. Raises if any cannot be queued, so- the caller saves nothing: no lock is put in force with tokens behind it- left live."""- issued_for_it = (ServiceTokenIssuance.service == service,)- off: list[Tenant] = []- for workspace in workspaces:- try:- async with tenant_session(session_factory, workspace.id) as session:- (switch, _) = (- await WORKSPACES.switches_under(- session,- [service],- {} if setting is None else {service: setting},- )- )[service]- if switch.enabled:- continue- await broker.queue_revocation(session, *issued_for_it)- await session.commit()- except Exception:- logger.exception(- "Could not queue %s's tokens in workspace %s before an "- "organization switched it off there; the change is not saved",- service,- workspace.id,- )- raise HTTPException(- status_code=503,- detail=f"Switch couldn't withdraw {_definition_name(service)} from "- f"{workspace.name}, so nothing was changed. Try again.",- ) from None- off.append(workspace)- return off+async def _each_workspace[T](+ workspaces: list[Tenant], visit: Callable[[Tenant], Awaitable[T]]+) -> list[T]:+ """`visit` every workspace, `CONCURRENT_WORKSPACES` at a time, each on+ sessions of its own: a large organization's are not walked one by one+ inside one request."""+ gate = asyncio.Semaphore(CONCURRENT_WORKSPACES)++ async def one(workspace: Tenant) -> T:+ async with gate:+ return await visit(workspace)++ return list(await asyncio.gather(*(one(workspace) for workspace in workspaces)))async def _withdraw(@@ -198,35 +182,48 @@ async def _withdraw(workspaces: list[Tenant],service: str,) -> str | None:- """After the setting has committed: in each workspace it switched- `service` off in, queue again what was issued in between, then revoke.- A workspace that fails is logged and named; the rest carry on. The- warnings, once each, of what could not be taken back yet."""+ """After the setting has committed: in each workspace where `service` is+ now off, queue what was issued for it and revoke. A workspace that fails+ is logged and named; the upkeep finishes it. The warnings, once each, of+ what could not be taken back yet."""issued_for_it = (ServiceTokenIssuance.service == service,)- warnings: list[str] = []- failed: list[str] = []- for workspace in workspaces:++ async def visit(workspace: Tenant) -> tuple[str | None, bool]:try:async with tenant_session(session_factory, workspace.id) as session:+ switch = (+ await WORKSPACES.switches(session, [service], organizations=True)+ )[service]+ if switch.enabled:+ await session.commit()+ return None, Trueawait broker.queue_revocation(session, *issued_for_it)await session.commit()- warning = await broker.revoke_withdrawn(session, issued_for_it)+ return await broker.revoke_withdrawn(session, issued_for_it), Trueexcept Exception:logger.exception(- "Could not finish withdrawing %s in workspace %s after its "- "organization switched it off there",+ "Could not withdraw %s in workspace %s after its organization "+ "switched it off there; the upkeep finishes it",service,workspace.id,)- failed.append(workspace.name)- continue+ return None, False++ outcomes = await _each_workspace(workspaces, visit)+ warnings: list[str] = []+ for warning, _ in outcomes:if warning is not None and warning not in warnings:warnings.append(warning)+ failed = [+ workspace.name+ for workspace, (_, done) in zip(workspaces, outcomes, strict=True)+ if not done+ ]if failed:warnings.append(- f"Switch couldn't finish withdrawing {_definition_name(service)} in "- f"{', '.join(failed)}. What it had queued there is revoked by its "- "periodic retry; tokens issued since may stay valid until they expire."+ f"Switch couldn't withdraw {_definition_name(service)} in "+ f"{', '.join(failed)} yet. It finishes within five minutes; until "+ "then, access already given to agents there may keep working.")return " ".join(warnings) or None@@ -243,19 +240,6 @@ async def _apply() -> str | None:"""Put `setting` (None: no say) in force for `service`, withdrawing itwherever that switches it off, as the module describes."""- # Only switching off, or giving up a setting of on, can leave a workspace- # without the service.- can_switch_off = (setting is not None and not setting.enabled) or (- setting is None and previous is not None and previous.enabled- )- # The request's connection goes back before the scans open one per workspace.- await session.commit()- workspaces = (- await organization_workspaces(session_factory, user.id, organization_id)- if can_switch_off- else []- )- off = await _queue_where_off(session_factory, broker, workspaces, service, setting)if setting is None:await SETTINGS.clear(session, organization_id, service)else:@@ -284,7 +268,15 @@ async def _apply(details=details,)await session.commit()- return await _withdraw(session_factory, broker, off, service)+ # Only on and locked switches nothing off anywhere: anything else, even+ # unlocking a setting of on, can leave a workspace whose own switch is+ # off without it.+ if setting is not None and setting.enabled and setting.locked:+ return None+ workspaces = await organization_workspaces(+ session_factory, user.id, organization_id+ )+ return await _withdraw(session_factory, broker, workspaces, service)@router.get("")diff --git a/core/switch_core/gateway/service_flows.py b/core/switch_core/gateway/service_flows.pyindex cbdf0a797..bba3e5c6f 100644--- a/core/switch_core/gateway/service_flows.py+++ b/core/switch_core/gateway/service_flows.py@@ -186,11 +186,17 @@ async def authorize(raise FlowError(INTERRUPTED)except FlowError:return _page("Connections", INTERRUPTED, 400)+ refusal: ServiceError | Noneasync with tenant_session(session_factory, flow.tenant_id) as session:try:definition, adapter = await broker.sign_in(session, service)- except ServiceError:- return _page("Connections", INTERRUPTED, 400)+ except ServiceError as error:+ refusal = error+ else:+ refusal = None+ if refusal is not None:+ await flows.refused(flow, refusal.message)+ return _page("Connections", escape(refusal.message), refusal.status_code)try:url, nonce = await flows.vendor_url(flow, adapter)except (FlowError, ServiceAdapterError):diff --git a/core/tests/switch_core/connections/test_app_credentials.py b/core/tests/switch_core/connections/test_app_credentials.pyindex 8aa1c12af..aa89c8af8 100644--- a/core/tests/switch_core/connections/test_app_credentials.py+++ b/core/tests/switch_core/connections/test_app_credentials.py@@ -19,8 +19,11 @@ from switch_core.connections.adapters.github import GitHubAdapterfrom switch_core.connections.adapters.oauth import OAuthAdapterfrom switch_core.connections.app_credentials import (SECRET_NAMES,+ UNREADABLE,ConnectorApps,+ UnreadableAppError,install_github_app,+ load_saved_app,)from switch_core.connections.broker import ServiceBrokerfrom switch_core.connections.loader import CATALOG@@ -148,6 +151,28 @@ class TestSpecs:assert specs["github"].secret_names == {"client_secret", "private_key"}+class TestSavedSecrets:+ def test_what_opens_to_no_payload_of_secrets_is_unreadable(+ self, session_factory+ ) -> None:+ """A key opens it, but what it opens is not JSON: only entering the+ secrets again mends it, as when no key opens it."""+ spec = _apps(session_factory).specs["google-workspace"]++ def not_json() -> dict[str, str]:+ return json.loads("not json")++ with pytest.raises(UnreadableAppError) as unreadable:+ load_saved_app(+ spec,+ client_id="example-client-id",+ settings={},+ secrets=not_json,+ public_url=PUBLIC_URL,+ )+ assert str(unreadable.value) == UNREADABLE++class TestPrecedence:async def test_nothing_set_is_not_set_up(self, session_factory) -> None:apps = _apps(session_factory)diff --git a/core/tests/switch_core/connections/test_organization_apps.py b/core/tests/switch_core/connections/test_organization_apps.pyindex 0968f0b45..a5beea59a 100644--- a/core/tests/switch_core/connections/test_organization_apps.py+++ b/core/tests/switch_core/connections/test_organization_apps.py@@ -51,6 +51,7 @@ from switch_core.signals import SignalBusfrom tests.conftest import TEST_KEYRING, RLSHarnessfrom tests.switch_core.connections.adapters.test_github import _configfrom tests.switch_core.connections.test_app_credentials import _rsa_pem+from tests.switch_core.gateway.test_workspace_services import _issuancepytestmark = pytest.mark.no_ambient_tenant@@ -330,12 +331,51 @@ class TestRevoking:# Made through the server's app, before the organization saved its own.before = await broker._revoker(session, GOOGLE, SERVER_CLIENT)since = await broker._revoker(session, GOOGLE, "organization-client-id")+ # From before clients were recorded, and so before organizations'+ # apps: the server's.+ legacy = await broker._revoker(session, GOOGLE, None)with pytest.raises(ServiceUnavailableError) as gone:await broker._revoker(session, GOOGLE, "replaced-client-id")assert before.current_client_id() == SERVER_CLIENT+ assert legacy.current_client_id() == SERVER_CLIENTassert since.current_client_id() == "organization-client-id"assert "no longer uses the Google Workspace app" in str(gone.value)+ async def test_an_issued_token_is_revoked_where_the_organizations_app_is_unusable(+ self, rls_harness: RLSHarness, tmp_path: Path+ ) -> None:+ """An issued token is revoked with itself, so the server's adapter+ will do when no key opens the organization's app: the token does not+ wait in the queue until someone saves the app again."""+ organization_id, tenant_id = await _workspace(+ rls_harness.owner, in_organization=True+ )+ assert organization_id is not None+ retired = Keyring.parse("retired:" + "r" * 40, legacy_secret=None)+ await _save(rls_harness.owner, organization_id, "lost-client", keyring=retired)+ async with rls_harness.owner() as session:+ user = User(name="ada", email=f"{_id('ada')}@example.invalid", role="user")+ session.add(user)+ await session.flush()+ issuance = _issuance(+ None, user.id, GOOGLE, "ya29.issued", tenant_id=tenant_id+ )+ issuance.revoke_requested = True+ session.add(issuance)+ await session.commit()+ broker = await _Server(rls_harness.restricted, tmp_path).start()+ revoked: list[str] = []++ async def revoke_issued(token: str) -> None:+ revoked.append(token)++ broker._adapters[GOOGLE].revoke_issued = revoke_issued # type: ignore[method-assign]+ async with tenant_session(rls_harness.restricted, tenant_id) as session:+ remaining = await broker.revoke_pending(session, ())++ assert revoked == ["ya29.issued"]+ assert remaining is False+class TestKept:async def test_what_a_workspace_belongs_to_is_read_once(diff --git a/core/tests/switch_core/db/test_workspace_service_store.py b/core/tests/switch_core/db/test_workspace_service_store.pyindex 403363cd7..2d0f41ff7 100644--- a/core/tests/switch_core/db/test_workspace_service_store.py+++ b/core/tests/switch_core/db/test_workspace_service_store.py@@ -56,8 +56,11 @@ class TestSwitch:async with session_factory() as session:assert await STORE.on_by_default(session) is Falseassert (- await STORE.is_enabled(session, "datadog", organizations=False) is False- )+ "datadog"+ not in await STORE.disabled_services(+ session, ["datadog"], organizations=False+ )+ ) is Falseassert await STORE.disabled_services(session, SERVICES, organizations=False) == set(SERVICES)@@ -69,8 +72,11 @@ class TestSwitch:await _on_by_default(session_factory)async with session_factory() as session:assert (- await STORE.is_enabled(session, "datadog", organizations=False) is True- )+ "datadog"+ not in await STORE.disabled_services(+ session, ["datadog"], organizations=False+ )+ ) is Trueassert (await STORE.disabled_services(session, SERVICES, organizations=False)== set()@@ -84,12 +90,17 @@ class TestSwitch:await STORE.set_enabled(session, "datadog", True, admin)await session.commit()assert (- await STORE.is_enabled(session, "datadog", organizations=False) is True- )+ "datadog"+ not in await STORE.disabled_services(+ session, ["datadog"], organizations=False+ )+ ) is Trueassert (- await STORE.is_enabled(session, "atlassian", organizations=False)- is False- )+ "atlassian"+ not in await STORE.disabled_services(+ session, ["atlassian"], organizations=False+ )+ ) is Falseassert await STORE.disabled_services(session, SERVICES, organizations=False) == {@@ -100,8 +111,11 @@ class TestSwitch:await STORE.set_enabled(session, "datadog", False, admin)await session.commit()assert (- await STORE.is_enabled(session, "datadog", organizations=False) is False- )+ "datadog"+ not in await STORE.disabled_services(+ session, ["datadog"], organizations=False+ )+ ) is Falseassert await STORE.disabled_services(session, SERVICES, organizations=False) == set(SERVICES)@@ -115,12 +129,17 @@ class TestSwitch:await STORE.set_enabled(session, "datadog", False, admin)await session.commit()assert (- await STORE.is_enabled(session, "datadog", organizations=False) is False- )+ "datadog"+ not in await STORE.disabled_services(+ session, ["datadog"], organizations=False+ )+ ) is Falseassert (- await STORE.is_enabled(session, "atlassian", organizations=False)- is True- )+ "atlassian"+ not in await STORE.disabled_services(+ session, ["atlassian"], organizations=False+ )+ ) is Trueassert await STORE.disabled_services(session, SERVICES, organizations=False) == {"datadog"}@@ -128,8 +147,11 @@ class TestSwitch:await STORE.set_enabled(session, "datadog", True, admin)await session.commit()assert (- await STORE.is_enabled(session, "datadog", organizations=False) is True- )+ "datadog"+ not in await STORE.disabled_services(+ session, ["datadog"], organizations=False+ )+ ) is Trueassert (await STORE.disabled_services(session, SERVICES, organizations=False)== set()@@ -184,8 +206,11 @@ class TestIsolation:)assert raw.all() == []assert (- await STORE.is_enabled(session, "datadog", organizations=False) is True- )+ "datadog"+ not in await STORE.disabled_services(+ session, ["datadog"], organizations=False+ )+ ) is Trueassert (await STORE.disabled_services(session, SERVICES, organizations=False)== set()diff --git a/core/tests/switch_core/gateway/test_organization_services.py b/core/tests/switch_core/gateway/test_organization_services.pyindex 17911a8b4..e8dd783f0 100644--- a/core/tests/switch_core/gateway/test_organization_services.py+++ b/core/tests/switch_core/gateway/test_organization_services.py@@ -22,6 +22,7 @@ from switch_core.connections.broker import (ServiceError,organization_locked,)+from switch_core.connections.maintenance import maintain_oncefrom switch_core.connections.organization_apps import OrganizationAppsfrom switch_core.db.audit import OrganizationAuditActionfrom switch_core.db.models import (@@ -278,6 +279,55 @@ class TestUnlocked:# On in the sibling, which never switched it, off here.assert organization["github"]["workspaces_on"] == 1+ async def test_unlocking_on_withdraws_where_the_workspace_had_it_off(+ self, harness: Harness, vendor: FakeVendor, session_factory+ ) -> None:+ """Locked on overrode a workspace's own switch of off; unlocked, that+ switch is in force again, and what was issued under the lock goes."""+ admin = await _org_admin(session_factory)+ workspace_admin = await _person(+ session_factory, "workspace-admin", member_role="admin"+ )+ assert (+ await _call(+ harness,+ workspace_admin,+ "PUT",+ f"{WORKSPACE}/github",+ {"enabled": False},+ )+ ).status_code == 200+ await _call(+ harness,+ admin,+ "PUT",+ f"{ORGANIZATION}/github",+ {"enabled": True, "locked": True},+ )+ ada = await add_member(session_factory, "ada")+ agent_id, _ = await agent_with_key(session_factory, ada, "builder")+ github = await _granted(session_factory, ada, agent_id, "github")+ async with session_factory() as session:+ session.add(_issuance(github, ada.id, "github", "ghs_here"))+ await session.commit()+ async with tenant_session(session_factory, SIBLING) as session:+ session.add(+ _issuance(None, ada.id, "github", "ghs_elsewhere", tenant_id=SIBLING)+ )+ await session.commit()++ unlocked = await _call(+ harness,+ admin,+ "PUT",+ f"{ORGANIZATION}/github",+ {"enabled": True, "locked": False},+ )++ assert unlocked.status_code == 200, unlocked.text+ # Off here by the workspace's own switch; on in the sibling.+ assert vendor.revoked == ["ghs_here"]+class TestClearing:async def test_clearing_gives_up_the_organizations_say(@@ -334,10 +384,18 @@ async def _connected_in(session_factory, user: User, tenant_id: str) -> None:class TestWithdrawingSafely:- async def test_nothing_is_saved_when_a_workspace_cannot_be_prepared(- self, harness: Harness, session_factory, monkeypatch+ async def test_a_workspace_that_cannot_be_reached_is_finished_by_the_upkeep(+ self, harness: Harness, vendor: FakeVendor, session_factory, monkeypatch) -> None:+ """The setting commits first: a workspace whose tokens could not be+ queued is named, and the next upkeep pass queues and revokes them."""admin = await _org_admin(session_factory)+ ada = await add_member(session_factory, "ada")+ async with tenant_session(session_factory, SIBLING) as session:+ session.add(+ _issuance(None, ada.id, "github", "ghs_elsewhere", tenant_id=SIBLING)+ )+ await session.commit()broker: ServiceBroker = harness.gateway_app.state.service_brokerqueue = broker.queue_revocation@@ -356,10 +414,16 @@ class TestWithdrawingSafely:)organization = _by_slug(await _call(harness, admin, "GET", ORGANIZATION))- assert answer.status_code == 503- assert "nothing was changed" in answer.json()["detail"]- assert organization["github"]["setting"] is None- assert await _events(session_factory) == []+ assert answer.status_code == 200, answer.text+ assert "Sibling" in answer.json()["warning"]+ assert organization["github"]["setting"] == {"enabled": False, "locked": True}+ assert len(await _events(session_factory)) == 1+ assert vendor.revoked == []++ monkeypatch.undo()+ await maintain_once(session_factory, broker, prune=False)++ assert vendor.revoked == ["ghs_elsewhere"]async def test_a_workspace_failing_afterwards_is_named_and_the_rest_carry_on(self, harness: Harness, vendor: FakeVendor, session_factory, monkeypatchdiff --git a/core/tests/switch_core/gateway/test_service_flows.py b/core/tests/switch_core/gateway/test_service_flows.pyindex f9ea00ed3..39c807add 100644--- a/core/tests/switch_core/gateway/test_service_flows.py+++ b/core/tests/switch_core/gateway/test_service_flows.py@@ -11,6 +11,7 @@ import jsonimport secretsimport shutilfrom datetime import UTC, datetime, timedelta+from html import escapefrom pathlib import Pathfrom urllib.parse import parse_qs, urlsplit@@ -19,6 +20,7 @@ from sqlalchemy import select, updatefrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmakerfrom switch_core.connections.adapters.oauth import OAuthAdapter+from switch_core.connections.app_credentials import UNREADABLEfrom switch_core.connections.broker import ServiceBrokerfrom switch_core.connections.flows import ServiceFlowsfrom switch_core.connections.loader import CATALOG_ROOT, load_catalog@@ -27,6 +29,7 @@ from switch_core.connections.oauth_clients import (RegisteredClient,redirect_uris,)+from switch_core.connections.organization_apps import OrganizationAppfrom switch_core.connections.sign_in_flows import MAX_FLOWS, SignInFlowsfrom switch_core.db.models import ServiceConnection, SignInFlow, Userfrom switch_core.db.stores.service_connection_store import ServiceConnectionStore@@ -115,7 +118,7 @@ def _world(token_retention=timedelta(days=30),workspace_services=WorkspaceServiceStore(),workspace_gate=workspace_gate,- public_url=None,+ public_url=public_url,)harness = build_harness(session_factory)harness.gateway_app.state.service_broker = broker@@ -494,6 +497,27 @@ class _AskedOrganizationApps:self.tenants.append(tenant_id)+class _UnreadableLater:+ """Organizations' apps on: none saved, until `now`, when the+ organization's app is one whose secrets Switch can't read."""++ def __init__(self) -> None:+ self.now = False++ async def for_workspace(+ self, session: AsyncSession, tenant_id: str, service: str+ ) -> OrganizationApp | None:+ if not self.now:+ return None+ return OrganizationApp(+ organization_id="organization",+ adapter=None,+ github_app=None,+ problem=UNREADABLE,+ changed_at=datetime.now(UTC),+ )++class TestCoreCallback:async def test_with_organizations_apps_the_browser_leaves_through_the_flows_workspace(self, session_factory, tmp_path@@ -516,6 +540,33 @@ class TestCoreCallback:flow_tenant = (await _flow_row(session_factory, started.json()["id"])).tenant_idassert asked.tenants == [flow_tenant, flow_tenant]+ async def test_an_app_unusable_by_then_ends_the_flow_with_why(+ self, session_factory, tmp_path+ ) -> None:+ """The organization's app was usable when the sign-in started and is+ not when the browser leaves: the page says why, and so does the flow+ Console is waiting on."""+ unreadable = _UnreadableLater()+ world = _world(+ session_factory,+ tmp_path / "catalog",+ redirect="[core]",+ public_url=PUBLIC,+ organization_apps=unreadable,+ )+ ada = await add_member(session_factory, "ada")+ async with world.harness.client() as client:+ started = await Console(client, ada).start()+ assert started.status_code == 200, started.text+ unreadable.now = True+ left = await client.get(started.json()["url"].removeprefix(PUBLIC))++ assert left.status_code == 503+ assert escape(UNREADABLE) in left.text+ row = await _flow_row(session_factory, started.json()["id"])+ assert row.status == "failed"+ assert row.error == UNREADABLE+async def test_returns_through_core_straight_to_console(self, session_factory, tmp_path) -> None:diff --git a/core/tests/switch_core/gateway/test_workspace_services.py b/core/tests/switch_core/gateway/test_workspace_services.pyindex 7535152aa..c9ebe02c2 100644--- a/core/tests/switch_core/gateway/test_workspace_services.py+++ b/core/tests/switch_core/gateway/test_workspace_services.py@@ -344,6 +344,10 @@ class TestSwitchingOff:_issuance(None, ada.id, "github", "ghs_elsewhere", tenant_id=TENANT_A))await session.commit()+ # On here, so its token is live: switching GitHub off must leave it.+ assert (+ await _put(harness, admin, "atlassian", {"enabled": True})+ ).status_code == 200response = await _put(harness, admin, "github", {"enabled": False})