Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

One cloud machine per user, with a controller per workspace (#752)

Louis Amaudruz8372d8824c9f
diff --git a/console/apps/switch-console-desktop/src/renderer/features/locations/components/add-agent-modal/new-agent-form.tsx b/console/apps/switch-console-desktop/src/renderer/features/locations/components/add-agent-modal/new-agent-form.tsx
index ad9f975de..589e7864c 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/locations/components/add-agent-modal/new-agent-form.tsx
+++ b/console/apps/switch-console-desktop/src/renderer/features/locations/components/add-agent-modal/new-agent-form.tsx
@@ -981,6 +981,6 @@ function cloudStartingText(machine: CloudMachine): string {
if (machine.desired_state === 'stopped')
return 'Your cloud machine is stopped. Start it from Your Agents, then add the agent.';
return machine.controller_id === null
- ? 'Starting your cloud machine. Its first start takes a few minutes…'
+ ? 'Starting your cloud machine for this workspace. If it already runs agents in another workspace, it restarts once to join this one. This takes a few minutes…'
: 'Waiting for your cloud machine to come online…';
}
diff --git a/core/switch_core/clients/agent_consumer.py b/core/switch_core/clients/agent_consumer.py
index 0389af290..d0a58d848 100644
--- a/core/switch_core/clients/agent_consumer.py
+++ b/core/switch_core/clients/agent_consumer.py
@@ -53,7 +53,7 @@ from switch_core.clients.mentions import (
strip_emphasis as _strip_emphasis,
)
from switch_core.clients.room_meta import RoomMeta
-from switch_core.db.models import Agent, HostedMachine
+from switch_core.db.models import Agent, CloudMachine
from switch_core.db.session_scope import tenant_session
from switch_core.db.stores.agent_session_store import AgentSessionStore
from switch_core.db.stores.agent_store import AgentStore
@@ -245,7 +245,7 @@ class CloudMachineState:
owner_stopped: bool
@classmethod
- def of(cls, machine: HostedMachine) -> CloudMachineState:
+ def of(cls, machine: CloudMachine) -> CloudMachineState:
return cls(
state=machine.state,
revision=machine.revision,
diff --git a/core/switch_core/db/key_rotation.py b/core/switch_core/db/key_rotation.py
index 148b6e36b..7a89b29e0 100644
--- a/core/switch_core/db/key_rotation.py
+++ b/core/switch_core/db/key_rotation.py
@@ -22,7 +22,7 @@ from switch_core.db.encrypted_json import reencrypt_stale_values
from switch_core.db.models import (
ApiKey,
CollaborationBridge,
- HostedMachine,
+ MachineWorkspace,
MessagingInstall,
ProviderConnection,
ServerConnector,
@@ -40,7 +40,7 @@ _ENCRYPTED_TEXT_COLUMNS: tuple[tuple[type[Any], str], ...] = (
(ApiKey, "encrypted_key"),
(MessagingInstall, "encrypted_bot_token"),
(ProviderConnection, "encrypted_credential"),
- (HostedMachine, "enrollment_code_encrypted"),
+ (MachineWorkspace, "enrollment_code_encrypted"),
)
diff --git a/core/switch_core/db/models.py b/core/switch_core/db/models.py
index b73879aaf..855de57bf 100644
--- a/core/switch_core/db/models.py
+++ b/core/switch_core/db/models.py
@@ -311,40 +311,37 @@ class ProviderConnection(TenantScoped, Base):
)
-class HostedMachine(TenantScoped, Base):
- """One user's cloud VM, whose agents controller runs that user's cloud agents."""
+class CloudMachine(Base):
+ """One user's Switch cloud VM, whichever workspaces it serves.
- __tablename__ = "hosted_machines"
+ Global, as the person who owns it is: one VM and one disk per user, with
+ one lifecycle. What it runs in each workspace is that workspace's
+ `MachineWorkspace` row, which is scoped. Nothing here names a workspace.
+ """
+
+ __tablename__ = "cloud_machines"
__table_args__ = (
- PrimaryKeyConstraint("tenant_id", "id"),
CheckConstraint(
"state IN ('queued', 'provisioning', 'ready', 'stopping', 'stopped', 'error', 'retained', 'deleting', 'deleted')",
- name="ck_hosted_machine_state",
+ name="ck_cloud_machine_state",
),
CheckConstraint(
"desired_state IN ('running', 'stopped', 'retained', 'deleted')",
- name="ck_hosted_machine_desired_state",
+ name="ck_cloud_machine_desired_state",
),
CheckConstraint(
"stop_reason IS NULL OR stop_reason IN ('idle', 'owner')",
- name="ck_hosted_machine_stop_reason",
- ),
- ForeignKeyConstraint(
- ["tenant_id", "controller_id"],
- ["agent_controllers.tenant_id", "agent_controllers.id"],
- name="fk_hosted_machines_controller",
- ondelete="SET NULL (controller_id)",
+ name="ck_cloud_machine_stop_reason",
),
Index(
- "uq_hosted_machine_owner",
- "tenant_id",
+ "uq_cloud_machine_owner",
"owner_id",
unique=True,
postgresql_where=text("state <> 'deleted'"),
),
)
- id: Mapped[str] = mapped_column(Text, nullable=False)
+ id: Mapped[str] = mapped_column(Text, primary_key=True)
owner_id: Mapped[str] = mapped_column(Text, ForeignKey("users.id"), nullable=False)
state: Mapped[str] = mapped_column(Text, nullable=False)
desired_state: Mapped[str] = mapped_column(
@@ -356,17 +353,13 @@ class HostedMachine(TenantScoped, Base):
data_volume_id: Mapped[str | None] = mapped_column(Text)
instance_id: Mapped[str | None] = mapped_column(Text)
retain_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
- # The agents controller the machine enrolled as, with a one-time code
- # minted for it; the owner's managed agents placed on it run there.
- controller_id: Mapped[str | None] = mapped_column(Text)
- # The code the machine enrolls with, kept for the revision it was issued
- # at, so every retry of that revision hands over the same one.
- enrollment_code_encrypted: Mapped[str | None] = mapped_column(Text)
- enrollment_code_revision: Mapped[int | None] = mapped_column(Integer)
active_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+ # The machine's disk and memory, and per workspace (keyed by its
+ # `MachineWorkspace` id) when its controller last reported and how many
+ # sessions it ran: `{disk, memory, controllers: {id: {at, sessions_running}}}`.
heartbeat: Mapped[dict | None] = mapped_column(JSONB)
heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
- #: When Core first saw the controller report `running` for this revision.
+ #: When Core first saw the machine reported `running` for this revision.
running_observed_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True)
)
@@ -380,6 +373,41 @@ class HostedMachine(TenantScoped, Base):
)
+class MachineWorkspace(TenantScoped, Base):
+ """A workspace on its owner's cloud machine: the agents controller the
+ machine runs for this workspace, enrolled with a one-time code minted here.
+ The owner's managed agents placed on that controller run on the machine."""
+
+ __tablename__ = "machine_workspaces"
+ __table_args__ = (
+ PrimaryKeyConstraint("tenant_id", "id"),
+ ForeignKeyConstraint(
+ ["tenant_id", "controller_id"],
+ ["agent_controllers.tenant_id", "agent_controllers.id"],
+ name="fk_machine_workspaces_controller",
+ ondelete="SET NULL (controller_id)",
+ ),
+ UniqueConstraint(
+ "tenant_id", "machine_id", name="uq_machine_workspaces_machine"
+ ),
+ Index("ix_machine_workspaces_machine_id", "machine_id"),
+ )
+
+ id: Mapped[str] = mapped_column(Text, nullable=False)
+ machine_id: Mapped[str] = mapped_column(
+ Text, ForeignKey("cloud_machines.id"), nullable=False
+ )
+ owner_id: Mapped[str] = mapped_column(Text, ForeignKey("users.id"), nullable=False)
+ controller_id: Mapped[str | None] = mapped_column(Text)
+ # The code the controller enrolls with, kept for the machine revision it
+ # was issued at, so every retry of that revision hands over the same one.
+ enrollment_code_encrypted: Mapped[str | None] = mapped_column(Text)
+ enrollment_code_revision: Mapped[int | None] = mapped_column(Integer)
+ created_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), server_default=func.now(), nullable=False
+ )
+
+
# ── Invitations ────────────────────────────────────────────────────────────────
@@ -2902,9 +2930,9 @@ class AgentControllerEnrollmentCode(TenantScoped, Base):
name="fk_agent_controller_enrollment_codes_controller",
),
ForeignKeyConstraint(
- ["tenant_id", "hosted_machine_id"],
- ["hosted_machines.tenant_id", "hosted_machines.id"],
- name="fk_agent_controller_enrollment_codes_hosted_machine",
+ ["tenant_id", "machine_workspace_id"],
+ ["machine_workspaces.tenant_id", "machine_workspaces.id"],
+ name="fk_agent_controller_enrollment_codes_machine_workspace",
),
Index("ix_agent_controller_enrollment_codes_tenant_id", "tenant_id"),
)
@@ -2921,7 +2949,7 @@ class AgentControllerEnrollmentCode(TenantScoped, Base):
controller_id: Mapped[str | None] = mapped_column(Text, nullable=True)
# Set for a code minted for a Switch cloud machine: what enrolls with it
# is that machine's controller.
- hosted_machine_id: Mapped[str | None] = mapped_column(Text, nullable=True)
+ machine_workspace_id: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), nullable=False
)
diff --git a/core/switch_core/db/rls_ddl.py b/core/switch_core/db/rls_ddl.py
index b45dd8bc4..cb2893d61 100644
--- a/core/switch_core/db/rls_ddl.py
+++ b/core/switch_core/db/rls_ddl.py
@@ -132,8 +132,16 @@ POLICY_NAME = "tenant_isolation"
# when it started, its last renewal and when it stopped. A process holds
# controller sockets for every tenant at once, so its lease belongs to none of
# them, and it holds no customer data, only that the process is alive.
+#
+# `cloud_machines` is a user's Switch cloud VM: one per person, whichever
+# workspaces it serves, as the person is global. It holds the VM's lifecycle,
+# disk and instance, and no workspace's data: what the machine runs for a
+# workspace is that workspace's `machine_workspaces` row, which is scoped, and
+# `tenants_of_cloud_machine` (`db/tenant_lookup.py`) is how the cloud
+# controller, which acts for no tenant, finds which workspaces those are.
GLOBAL_TABLES = frozenset(
{
+ "cloud_machines",
"users",
"oidc_identities",
"feature_flags",
diff --git a/core/switch_core/db/stores/agent_controller_store.py b/core/switch_core/db/stores/agent_controller_store.py
index 0868700be..b0fc90ab1 100644
--- a/core/switch_core/db/stores/agent_controller_store.py
+++ b/core/switch_core/db/stores/agent_controller_store.py
@@ -377,13 +377,13 @@ class AgentControllerStore:
owner_id: str,
api_key_id: str,
expires_at: datetime,
- hosted_machine_id: str | None,
+ machine_workspace_id: str | None,
) -> AgentControllerEnrollmentCode:
code = AgentControllerEnrollmentCode(
owner_id=owner_id,
api_key_id=api_key_id,
expires_at=expires_at,
- hosted_machine_id=hosted_machine_id,
+ machine_workspace_id=machine_workspace_id,
)
session.add(code)
await session.flush()
diff --git a/core/switch_core/db/stores/hosted_machine_store.py b/core/switch_core/db/stores/hosted_machine_store.py
index d2a780977..8796646f9 100644
--- a/core/switch_core/db/stores/hosted_machine_store.py
+++ b/core/switch_core/db/stores/hosted_machine_store.py
@@ -1,17 +1,21 @@
from __future__ import annotations
+from dataclasses import dataclass
from datetime import datetime, timedelta
from typing import Literal, cast
from uuid import uuid4
-from sqlalchemy import exists, func, select, text
-from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import func, select, text
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.db.models import (
AgentDefinition,
- HostedMachine,
+ CloudMachine,
+ MachineWorkspace,
require_tenant_id,
)
+from switch_core.db.tenant_lookup import tenants_of_cloud_machine
+from switch_core.tenant_context import tenant_scope
MACHINE_CONNECT_TIMEOUT = timedelta(minutes=10)
MACHINE_NEEDS_ATTENTION = (
@@ -24,9 +28,15 @@ MACHINES_FULL = "Every Switch cloud machine on this server is in use. Try again
MACHINE_BEING_REMOVED = (
"Your previous cloud machine is being removed. Try again in a minute."
)
+#: How many workspaces one machine serves: each runs its own agents controller.
+MAX_MACHINE_WORKSPACES = 8
+MACHINE_WORKSPACES_FULL = (
+ f"Your cloud machine already serves {MAX_MACHINE_WORKSPACES} workspaces, the most "
+ "it can. Contact your server administrator."
+)
-class HostedMachineConflict(Exception):
+class CloudMachineConflict(Exception):
"""A machine cannot take the requested change; the message is the 409 detail."""
@@ -38,24 +48,93 @@ async def _advisory_lock(session: AsyncSession, key: str) -> None:
async def lock_claims(session: AsyncSession) -> None:
- """The tenant-wide lock taken before claiming a machine."""
- await _advisory_lock(session, f"hosted-launches:{require_tenant_id()}")
+ """The server-wide lock taken before claiming a machine or joining one."""
+ await _advisory_lock(session, "cloud-machine-claims")
async def lock_machine(session: AsyncSession, machine_id: str) -> None:
- """The per-machine lock, taken after the tenant lock."""
- await _advisory_lock(session, f"hosted-machine:{require_tenant_id()}:{machine_id}")
+ """The per-machine lock, taken after the claims lock."""
+ await _advisory_lock(session, f"cloud-machine:{machine_id}")
+
+
+@dataclass(frozen=True)
+class WorkspaceOnMachine:
+ """What one workspace runs on a machine, as read with that workspace bound."""
+
+ tenant_id: str
+ id: str
+ controller_id: str | None
+ agent_count: int
+
+
+async def workspaces_on(
+ factory: async_sessionmaker[AsyncSession], machine_id: str
+) -> list[WorkspaceOnMachine]:
+ """Every workspace the machine serves, oldest first: each read on a
+ session of its own with that workspace bound, since the machine belongs
+ to none of them."""
+ found = []
+ for tenant_id in await tenants_of_cloud_machine(factory, machine_id):
+ with tenant_scope(tenant_id):
+ async with factory() as session:
+ workspace = await workspace_on(session, machine_id)
+ if workspace is None:
+ continue
+ found.append(
+ WorkspaceOnMachine(
+ tenant_id=tenant_id,
+ id=workspace.id,
+ controller_id=workspace.controller_id,
+ agent_count=len(
+ await managed_agent_ids(session, workspace.controller_id)
+ ),
+ )
+ )
+ return found
+
+
+async def workspace_on(
+ session: AsyncSession, machine_id: str
+) -> MachineWorkspace | None:
+ """The bound workspace's row on the machine, None when the machine does not serve it."""
+ return cast(
+ MachineWorkspace | None,
+ await session.scalar(
+ select(MachineWorkspace).where(
+ MachineWorkspace.tenant_id == require_tenant_id(),
+ MachineWorkspace.machine_id == machine_id,
+ )
+ ),
+ )
-def idle_sleeping(machine: HostedMachine) -> bool:
+async def managed_agent_ids(
+ session: AsyncSession, controller_id: str | None
+) -> list[str]:
+ """The bound workspace's managed agents placed on `controller_id`."""
+ if controller_id is None:
+ return []
+ return list(
+ await session.scalars(
+ select(AgentDefinition.agent_id)
+ .where(
+ AgentDefinition.tenant_id == require_tenant_id(),
+ AgentDefinition.controller_id == controller_id,
+ )
+ .order_by(AgentDefinition.agent_id)
+ )
+ )
+
+
+def idle_sleeping(machine: CloudMachine) -> bool:
return machine.desired_state == "stopped" and machine.stop_reason == "idle"
-def owner_stopped(machine: HostedMachine) -> bool:
+def owner_stopped(machine: CloudMachine) -> bool:
return machine.desired_state == "stopped" and machine.stop_reason == "owner"
-def machine_starting(machine: HostedMachine) -> bool:
+def machine_starting(machine: CloudMachine) -> bool:
return machine.desired_state == "running" and machine.state in {
"queued",
"provisioning",
@@ -65,7 +144,7 @@ def machine_starting(machine: HostedMachine) -> bool:
}
-def retention_expired(machine: HostedMachine, now: datetime) -> bool:
+def retention_expired(machine: CloudMachine, now: datetime) -> bool:
return (
machine.desired_state == "retained"
and machine.retain_until is not None
@@ -73,7 +152,7 @@ def retention_expired(machine: HostedMachine, now: datetime) -> bool:
)
-def claim_conflict(machine: HostedMachine, now: datetime) -> str | None:
+def claim_conflict(machine: CloudMachine, now: datetime) -> str | None:
"""The reason a live machine cannot be claimed, or None when it can."""
if machine.state == "error" and machine.error_code == "machine_needs_attention":
return MACHINE_NEEDS_ADMIN
@@ -88,13 +167,13 @@ def claim_conflict(machine: HostedMachine, now: datetime) -> str | None:
return None
-def bump_revision(machine: HostedMachine, now: datetime) -> None:
+def bump_revision(machine: CloudMachine, now: datetime) -> None:
machine.revision += 1
machine.running_observed_at = None
machine.updated_at = now
-def await_reconnect(machine: HostedMachine) -> None:
+def await_reconnect(machine: CloudMachine) -> None:
"""Make a still-ready machine prove itself again at its new revision.
A running observation leaves a ready machine ready, so without this a
@@ -104,42 +183,38 @@ def await_reconnect(machine: HostedMachine) -> None:
machine.state = "provisioning"
-class HostedMachineStore:
- async def get(self, session: AsyncSession, machine_id: str) -> HostedMachine | None:
- return await session.get(
- HostedMachine, (require_tenant_id(), machine_id), populate_existing=True
- )
+class CloudMachineStore:
+ async def get(self, session: AsyncSession, machine_id: str) -> CloudMachine | None:
+ return await session.get(CloudMachine, machine_id, populate_existing=True)
async def locked(
self, session: AsyncSession, machine_id: str
- ) -> HostedMachine | None:
+ ) -> CloudMachine | None:
await lock_machine(session, machine_id)
return await self.get(session, machine_id)
async def owned(
self, session: AsyncSession, machine_id: str, owner_id: str
- ) -> HostedMachine | None:
+ ) -> CloudMachine | None:
return cast(
- HostedMachine | None,
+ CloudMachine | None,
await session.scalar(
- select(HostedMachine).where(
- HostedMachine.tenant_id == require_tenant_id(),
- HostedMachine.id == machine_id,
- HostedMachine.owner_id == owner_id,
+ select(CloudMachine).where(
+ CloudMachine.id == machine_id,
+ CloudMachine.owner_id == owner_id,
)
),
)
async def live_for_owner(
self, session: AsyncSession, owner_id: str
- ) -> HostedMachine | None:
+ ) -> CloudMachine | None:
return cast(
- HostedMachine | None,
+ CloudMachine | None,
await session.scalar(
- select(HostedMachine).where(
- HostedMachine.tenant_id == require_tenant_id(),
- HostedMachine.owner_id == owner_id,
- HostedMachine.state != "deleted",
+ select(CloudMachine).where(
+ CloudMachine.owner_id == owner_id,
+ CloudMachine.state != "deleted",
)
),
)
@@ -147,23 +222,32 @@ class HostedMachineStore:
async def claim(
self,
session: AsyncSession,
+ factory: async_sessionmaker[AsyncSession],
*,
owner_id: str,
capacity: int,
now: datetime,
- ) -> HostedMachine:
- """The owner's machine, reused or newly claimed within `capacity`.
+ ) -> tuple[CloudMachine, MachineWorkspace]:
+ """The owner's machine, reused or newly claimed within `capacity`,
+ serving the bound workspace.
- The caller holds `lock_claims`, so no other claim in the tenant can
- count towards capacity meanwhile.
+ A machine joining a workspace takes a new revision, so it is prepared
+ again with a controller for it. The caller holds `lock_claims`, so no
+ other claim can count towards capacity or join the machine meanwhile.
"""
- tenant_id = require_tenant_id()
machine = await self.live_for_owner(session, owner_id)
if machine is not None:
machine = await self.locked(session, machine.id)
if machine is not None and machine.state != "deleted":
if (conflict := claim_conflict(machine, now)) is not None:
- raise HostedMachineConflict(conflict)
+ raise CloudMachineConflict(conflict)
+ workspace = await workspace_on(session, machine.id)
+ joined = workspace is None
+ if joined:
+ serving = await tenants_of_cloud_machine(factory, machine.id)
+ if len(serving) >= MAX_MACHINE_WORKSPACES:
+ raise CloudMachineConflict(MACHINE_WORKSPACES_FULL)
+ workspace = self._add_workspace(session, machine, now)
if machine.desired_state == "retained":
machine.desired_state = "running"
machine.retain_until = None
@@ -175,22 +259,23 @@ class HostedMachineStore:
self.start(machine, now)
else:
machine.active_at = now
+ if joined:
+ bump_revision(machine, now)
+ await_reconnect(machine)
+ assert workspace is not None
await session.flush()
- return machine
+ return machine, workspace
live = await session.scalar(
select(func.count())
- .select_from(HostedMachine)
- .where(
- HostedMachine.tenant_id == tenant_id,
- HostedMachine.state != "deleted",
- )
+ .select_from(CloudMachine)
+ .where(CloudMachine.state != "deleted")
)
if (live or 0) >= capacity:
- raise HostedMachineConflict(MACHINES_FULL)
+ raise CloudMachineConflict(MACHINES_FULL)
machine_id = str(uuid4())
await lock_machine(session, machine_id)
- machine = HostedMachine(
+ machine = CloudMachine(
id=machine_id,
owner_id=owner_id,
state="queued",
@@ -201,9 +286,23 @@ class HostedMachineStore:
)
session.add(machine)
await session.flush()
- return machine
+ workspace = self._add_workspace(session, machine, now)
+ await session.flush()
+ return machine, workspace
+
+ def _add_workspace(
+ self, session: AsyncSession, machine: CloudMachine, now: datetime
+ ) -> MachineWorkspace:
+ workspace = MachineWorkspace(
+ id=str(uuid4()),
+ machine_id=machine.id,
+ owner_id=machine.owner_id,
+ created_at=now,
+ )
+ session.add(workspace)
+ return workspace
- def start(self, machine: HostedMachine, now: datetime) -> None:
+ def start(self, machine: CloudMachine, now: datetime) -> None:
if machine.desired_state != "running":
machine.desired_state = "running"
machine.stop_reason = None
@@ -213,7 +312,7 @@ class HostedMachineStore:
def stop(
self,
- machine: HostedMachine,
+ machine: CloudMachine,
reason: Literal["idle", "owner"],
now: datetime,
) -> None:
@@ -221,7 +320,7 @@ class HostedMachineStore:
machine.stop_reason = reason
bump_revision(machine, now)
- def retry(self, machine: HostedMachine, now: datetime) -> None:
+ def retry(self, machine: CloudMachine, now: datetime) -> None:
machine.state = "queued"
machine.error = None
machine.error_code = None
@@ -229,19 +328,18 @@ class HostedMachineStore:
async def retain_if_empty(
self,
- session: AsyncSession,
- machine: HostedMachine,
+ machine: CloudMachine,
+ workspaces: list[WorkspaceOnMachine],
*,
retention_days: int,
now: datetime,
) -> bool:
- """Retain the machine's disk once no managed agent is placed on its
- controller.
+ """Retain the machine's disk once no workspace has a managed agent on it.
- Returns whether it did. The caller holds the machine lock and commits.
+ Returns whether it did. The caller holds the machine lock, read
+ `workspaces` under it (`workspaces_on`), and commits.
"""
- await session.flush()
- if await self.has_agents(session, machine):
+ if any(workspace.agent_count for workspace in workspaces):
return False
machine.desired_state = "retained"
machine.stop_reason = None
@@ -251,55 +349,26 @@ class HostedMachineStore:
async def release_if_empty(
self,
- session: AsyncSession,
- machine: HostedMachine,
+ machine: CloudMachine,
+ workspaces: list[WorkspaceOnMachine],
*,
retention_days: int,
now: datetime,
) -> bool:
- """Retain an agentless machine, expiring at once if its controller
- never enrolled: then its disk holds nothing worth keeping. The caller
+ """Retain an agentless machine, expiring at once if no controller of
+ it ever enrolled: then its disk holds nothing worth keeping. The caller
holds the machine lock and commits.
"""
if not await self.retain_if_empty(
- session, machine, retention_days=retention_days, now=now
+ machine, workspaces, retention_days=retention_days, now=now
):
return False
- if not await self.ever_hosted(session, machine):
+ if not ever_hosted(workspaces):
machine.retain_until = now
return True
- async def has_agents(self, session: AsyncSession, machine: HostedMachine) -> bool:
- """Whether a managed agent is still placed on the machine's controller."""
- return machine.controller_id is not None and bool(
- await session.scalar(
- select(
- exists().where(
- AgentDefinition.tenant_id == require_tenant_id(),
- AgentDefinition.controller_id == machine.controller_id,
- )
- )
- )
- )
-
- async def managed_agent_ids(
- self, session: AsyncSession, machine: HostedMachine
- ) -> list[str]:
- """The managed agents placed on the machine's controller."""
- if machine.controller_id is None:
- return []
- return list(
- await session.scalars(
- select(AgentDefinition.agent_id)
- .where(
- AgentDefinition.tenant_id == require_tenant_id(),
- AgentDefinition.controller_id == machine.controller_id,
- )
- .order_by(AgentDefinition.agent_id)
- )
- )
- async def ever_hosted(self, session: AsyncSession, machine: HostedMachine) -> bool:
- """Whether the machine's disk may hold anything worth keeping: its
- controller ever enrolled."""
- return machine.controller_id is not None
+def ever_hosted(workspaces: list[WorkspaceOnMachine]) -> bool:
+ """Whether the machine's disk may hold anything worth keeping: a
+ controller of it ever enrolled."""
+ return any(workspace.controller_id is not None for workspace in workspaces)
diff --git a/core/switch_core/db/tenant_lookup.py b/core/switch_core/db/tenant_lookup.py
index c2420893c..638dd2528 100644
--- a/core/switch_core/db/tenant_lookup.py
+++ b/core/switch_core/db/tenant_lookup.py
@@ -1,4 +1,4 @@
-"""The whole exemption from row-level security, written out as eleven functions.
+"""The whole exemption from row-level security, written out as twelve functions.
Row-level security is enforced by `require_tenant_id()` (`db/rls_ddl.py`),
which raises when no tenant is bound. That is the property everything else
@@ -151,6 +151,14 @@ tenant before any is bound. It discloses which workspaces are open to a
domain; the endpoint that calls it only ever passes the domain of the
signed-in caller's own address.
+**`tenants_of_cloud_machine` is the twelfth.** A user's cloud machine is
+global (`cloud_machines`, one VM per person), and the cloud controller that
+drives it acts for no tenant; what the machine runs in each workspace is that
+workspace's scoped `machine_workspaces` row. The lookup says which workspaces
+those are, so each row is read with its own tenant bound. It discloses which
+workspaces a machine, and so its owner, belongs to: what `tenants_of_user`
+already discloses about the same person.
+
Why not the obvious alternatives is argued in
`docs/old/multi-tenancy-phase1-db.md`, "The bootstrap problem"; the short
version is that returning rows instead of tenant ids would put a second copy
@@ -243,7 +251,7 @@ class TenantLookup:
return f"{self.name}({', '.join('text' for _ in self.arguments)})"
-# The eleven of them. Ordered as the three shapes above: enumeration, then
+# The twelve of them. Ordered as the three shapes above: enumeration, then
# credential resolution, then deriving a tenant from an identifier in hand.
TENANT_LOOKUPS: tuple[TenantLookup, ...] = (
TenantLookup(
@@ -358,6 +366,20 @@ TENANT_LOOKUPS: tuple[TenantLookup, ...] = (
"workspaces before any of them is bound."
),
),
+ TenantLookup(
+ name="tenants_of_cloud_machine",
+ arguments=("machine_id",),
+ query=(
+ "SELECT tenant_id FROM machine_workspaces "
+ "WHERE machine_id = p_machine_id ORDER BY created_at, tenant_id"
+ ),
+ purpose=(
+ "Which workspaces a user's cloud machine serves, oldest first. The "
+ "machine is global and acts for none of them; the cloud controller "
+ "reads each workspace's row on it, the controller it runs there "
+ "and its enrollment code, one bound tenant at a time."
+ ),
+ ),
TenantLookup(
name="tenant_of_messaging_install",
arguments=("platform", "external_workspace_id"),
@@ -431,12 +453,12 @@ def attach_tenant_lookups(metadata: MetaData) -> None:
# ── Calling them ──────────────────────────────────────────────────────────────
# One `text()` per lookup, written out rather than assembled from `lookup.name`
-# at call time: the eleven names are fixed and known here, so there is nothing
+# at call time: the twelve names are fixed and known here, so there is nothing
# for a call site to build. Each bind is named after the lookup's own argument,
# which is what lets `_call` zip them positionally against the dataclass and
# fail loudly on a mismatch rather than binding the workspace to the platform.
# The assertion below is what keeps this dict from quietly falling behind
-# `TENANT_LOOKUPS` — a twelfth lookup with no entry here fails at import, not
+# `TENANT_LOOKUPS` — a thirteenth lookup with no entry here fails at import, not
# with a `KeyError` on whatever request reaches it first.
_LOOKUP_STATEMENTS: dict[str, TextClause] = {
"all_tenant_ids": text("SELECT tenant_id FROM all_tenant_ids() AS tenant_id"),
@@ -468,6 +490,9 @@ _LOOKUP_STATEMENTS: dict[str, TextClause] = {
"tenants_open_to_domain": text(
"SELECT tenant_id FROM tenants_open_to_domain(:domain) AS tenant_id"
),
+ "tenants_of_cloud_machine": text(
+ "SELECT tenant_id FROM tenants_of_cloud_machine(:machine_id) AS tenant_id"
+ ),
"tenant_of_messaging_install": text(
"SELECT tenant_id FROM "
"tenant_of_messaging_install(:platform, :external_workspace_id) AS tenant_id"
@@ -598,3 +623,11 @@ async def tenant_of_messaging_install(
return _at_most_one(
lookup, await _call(session_factory, lookup, platform, external_workspace_id)
)
+
+
+async def tenants_of_cloud_machine(
+ session_factory: async_sessionmaker[AsyncSession], machine_id: str
+) -> list[str]:
+ return await _call(
+ session_factory, TENANT_LOOKUPS_BY_NAME["tenants_of_cloud_machine"], machine_id
+ )
diff --git a/core/switch_core/gateway/auth_routes.py b/core/switch_core/gateway/auth_routes.py
index 9cb5c401d..1566cd0fc 100644
--- a/core/switch_core/gateway/auth_routes.py
+++ b/core/switch_core/gateway/auth_routes.py
@@ -14,7 +14,7 @@ from switch_core.db.models import TENANT_ZERO_ID, User
from switch_core.db.stores.collaboration_bridge_store import CollaborationBridgeStore
from switch_core.db.stores.external_user_store import ExternalUserStore
from switch_core.db.stores.hosted_machine_store import (
- HostedMachineConflict,
+ CloudMachineConflict,
claim_conflict,
owner_stopped,
)
@@ -147,13 +147,16 @@ async def login(
async def _prewarm(
session: AsyncSession,
+ factory: async_sessionmaker[AsyncSession],
user_id: str,
config: SwitchConfig,
settings: HostedControllerSettings | None,
) -> SignupMachine:
try:
- machine = await ensure_machine(session, user_id, config, settings)
- except (MachineUnavailable, HostedMachineConflict) as error:
+ machine, _workspace = await ensure_machine(
+ session, factory, user_id, config, settings
+ )
+ except (MachineUnavailable, CloudMachineConflict) as error:
await session.rollback()
logger.warning(
"Signed-up user %s has no cloud machine warming: %s", user_id, error
@@ -196,6 +199,9 @@ async def signup(
req: SignupRequest,
response: Response,
session: Annotated[AsyncSession, Depends(get_system_session)],
+ session_factory: Annotated[
+ async_sessionmaker[AsyncSession], Depends(get_session_factory)
+ ],
user_store: Annotated[UserStore, Depends(get_user_store)],
config: Annotated[SwitchConfig, Depends(get_config)],
settings: Annotated[HostedControllerSettings | None, Depends(hosted_settings)],
@@ -249,7 +255,7 @@ async def signup(
TENANT_ZERO_ID,
)
signed_in = _session_response(user)
- machine = await _prewarm(session, user.id, config, settings)
+ machine = await _prewarm(session, session_factory, user.id, config, settings)
return SignupResponse(**signed_in.model_dump(), machine=machine)
diff --git a/core/switch_core/gateway/cloud_controllers.py b/core/switch_core/gateway/cloud_controllers.py
index a2013617f..c16b867d8 100644
--- a/core/switch_core/gateway/cloud_controllers.py
+++ b/core/switch_core/gateway/cloud_controllers.py
@@ -1,11 +1,12 @@
"""Switch cloud machines, which run the agents controller.
-A cloud machine boots the same `switch-agent-controller` any machine can
-run, and enrolls it with a one-time code Core mints for the machine and hands
-over when it prepares it. The controller it enrolls as is linked to the
-machine (`controller_id`); the owner's managed agents placed on it run there,
-each as a Linux user of its own, and its status reports are what make the
-machine ready.
+A cloud machine is its owner's, whichever workspaces it serves
+(`CloudMachine`). For each workspace it boots a `switch-agent-controller` of
+its own, and enrolls it with a one-time code Core mints in that workspace and
+hands over when it prepares the machine. The controller it enrolls as is
+linked to the workspace's row on the machine (`MachineWorkspace.controller_id`);
+the owner's managed agents placed on it run there, each as a Linux user of its
+own, and its status reports are what make the machine ready.
Enrollment codes and controllers belong to agent management, which Core does
not import (`management/wiring.py` installs it here with
@@ -22,10 +23,11 @@ from sqlalchemy.ext.asyncio import AsyncSession
from switch_core.db.models import (
AgentController,
- HostedMachine,
+ CloudMachine,
+ MachineWorkspace,
require_tenant_id,
)
-from switch_core.db.stores.hosted_machine_store import HostedMachineStore, idle_sleeping
+from switch_core.db.stores.hosted_machine_store import CloudMachineStore, idle_sleeping
from switch_core.keys import Keyring
# How recent the machine's last status report must be for its silence to count
@@ -37,9 +39,10 @@ class CloudEnrollment(Protocol):
"""What agent management does for a cloud machine's controller."""
async def machine_enrollment_code(
- self, session: AsyncSession, machine: HostedMachine, now: datetime
+ self, session: AsyncSession, workspace: MachineWorkspace, now: datetime
) -> str:
- """A new one-time code the machine's controller enrolls with. The caller commits."""
+ """A new one-time code the machine's controller for the bound workspace
+ enrolls with. The caller commits."""
...
@@ -64,13 +67,14 @@ def cloud_enrollment() -> CloudEnrollment:
async def live_controller(
- session: AsyncSession, machine: HostedMachine
+ session: AsyncSession, workspace: MachineWorkspace
) -> AgentController | None:
- """The controller the machine enrolled as, unless it was revoked since."""
- if machine.controller_id is None:
+ """The controller the machine enrolled as in the workspace, unless it was
+ revoked since."""
+ if workspace.controller_id is None:
return None
controller = await session.get(
- AgentController, machine.controller_id, populate_existing=True
+ AgentController, workspace.controller_id, populate_existing=True
)
if (
controller is None
@@ -82,36 +86,60 @@ async def live_controller(
async def enrollment_code(
- session: AsyncSession, machine: HostedMachine, keyring: Keyring, now: datetime
+ session: AsyncSession,
+ workspace: MachineWorkspace,
+ revision: int,
+ keyring: Keyring,
+ now: datetime,
) -> str:
- """The code the machine enrolls with at its current revision: the same one
- for every retry of a revision, a new one for a new revision. The caller
- commits."""
+ """The code the machine's controller for the workspace enrolls with at the
+ machine's `revision`: the same one for every retry of a revision, a new one
+ for a new revision. The caller commits."""
if (
- machine.enrollment_code_revision == machine.revision
- and machine.enrollment_code_encrypted is not None
+ workspace.enrollment_code_revision == revision
+ and workspace.enrollment_code_encrypted is not None
):
- return keyring.decrypt(machine.enrollment_code_encrypted)
- code = await cloud_enrollment().machine_enrollment_code(session, machine, now)
- machine.enrollment_code_encrypted = keyring.encrypt(code)
- machine.enrollment_code_revision = machine.revision
+ return keyring.decrypt(workspace.enrollment_code_encrypted)
+ code = await cloud_enrollment().machine_enrollment_code(session, workspace, now)
+ workspace.enrollment_code_encrypted = keyring.encrypt(code)
+ workspace.enrollment_code_revision = revision
return code
async def machine_of_controller(
session: AsyncSession, controller_id: str
-) -> HostedMachine | None:
- """The live machine running `controller_id`, locked; None for any other controller."""
- machine_id = await session.scalar(
- select(HostedMachine.id).where(
- HostedMachine.tenant_id == require_tenant_id(),
- HostedMachine.controller_id == controller_id,
- HostedMachine.state != "deleted",
+) -> tuple[CloudMachine, MachineWorkspace] | None:
+ """The live machine running `controller_id` for the bound workspace,
+ locked, with the workspace's row on it; None for any other controller."""
+ workspace = await session.scalar(
+ select(MachineWorkspace).where(
+ MachineWorkspace.tenant_id == require_tenant_id(),
+ MachineWorkspace.controller_id == controller_id,
)
)
- if machine_id is None:
+ if workspace is None:
return None
- return await HostedMachineStore().locked(session, machine_id)
+ machine = await CloudMachineStore().locked(session, workspace.machine_id)
+ if machine is None or machine.state == "deleted":
+ return None
+ return machine, workspace
+
+
+def controller_reports(machine: CloudMachine) -> dict[str, dict[str, Any]]:
+ """Per workspace on the machine (its `MachineWorkspace` id), what its
+ controller last reported: `{at, sessions_running}`."""
+ return dict((machine.heartbeat or {}).get("controllers") or {})
+
+
+def workspace_reported(machine: CloudMachine, workspace_id: str) -> bool:
+ """Whether the workspace's controller has reported since the machine was
+ last seen starting: its agents can run."""
+ report = controller_reports(machine).get(workspace_id)
+ return (
+ report is not None
+ and machine.running_observed_at is not None
+ and datetime.fromisoformat(report["at"]) > machine.running_observed_at
+ )
def _reading(free: Any, total: Any) -> dict[str, int] | None:
@@ -126,10 +154,17 @@ async def record_controller_status(
"""A status report from a cloud machine's controller is its heartbeat: it
records the machine's disk and memory, and makes a machine that started
at this revision ready. The caller commits."""
- machine = await machine_of_controller(session, controller_id)
- if machine is None:
+ found = await machine_of_controller(session, controller_id)
+ if found is None:
return
+ machine, workspace = found
now = datetime.now(UTC)
+ sessions = machine_reading.get("sessions_running")
+ reports = controller_reports(machine)
+ reports[workspace.id] = {
+ "at": now.isoformat(),
+ "sessions_running": sessions if isinstance(sessions, int) else None,
+ }
machine.heartbeat = {
"disk": _reading(
machine_reading.get("disk_free_bytes"),
@@ -139,13 +174,11 @@ async def record_controller_status(
machine_reading.get("mem_free_bytes"),
machine_reading.get("mem_total_bytes"),
),
+ "controllers": reports,
}
machine.heartbeat_at = now
- sessions = machine_reading.get("sessions_running")
- if isinstance(sessions, int):
- machine.heartbeat["sessions_running"] = sessions
- if sessions > 0:
- machine.active_at = now
+ if isinstance(sessions, int) and sessions > 0:
+ machine.active_at = now
if (
machine.state == "provisioning"
and machine.desired_state == "running"
@@ -159,15 +192,16 @@ async def record_controller_status(
async def wake_controller_machine(
session: AsyncSession, controller_id: str, now: datetime
-) -> HostedMachine | None:
+) -> CloudMachine | None:
"""The cloud machine running `controller_id`, started again if it was put
to sleep when idle, and kept awake otherwise. A machine its owner stopped
is left stopped. None for a controller that runs on no cloud machine. The
caller holds no other machine lock, and commits."""
- machine = await machine_of_controller(session, controller_id)
- if machine is None:
+ found = await machine_of_controller(session, controller_id)
+ if found is None:
return None
- store = HostedMachineStore()
+ machine, _workspace = found
+ store = CloudMachineStore()
if idle_sleeping(machine):
store.start(machine, now)
elif machine.desired_state == "running":
@@ -176,15 +210,19 @@ async def wake_controller_machine(
def controller_machine_idle(
- machine: HostedMachine, idle_after: timedelta, now: datetime
+ machine: CloudMachine, idle_after: timedelta, now: datetime
) -> bool:
- """Whether a controller machine has been idle for `idle_after`: its last
- status report is recent and says no session runs, and neither a session
- nor a message addressed to its agents has kept it active since."""
- heartbeat = machine.heartbeat or {}
+ """Whether a controller machine has been idle for `idle_after`: the last
+ status report of every controller it runs is recent and says no session
+ runs, and neither a session nor a message addressed to its agents has kept
+ it active since."""
+ reports = controller_reports(machine).values()
return (
- machine.heartbeat_at is not None
- and now - machine.heartbeat_at <= FRESH_HEARTBEAT
- and heartbeat.get("sessions_running") == 0
+ bool(reports)
+ and all(
+ report.get("sessions_running") == 0
+ and now - datetime.fromisoformat(report["at"]) <= FRESH_HEARTBEAT
+ for report in reports
+ )
and now - machine.active_at >= idle_after
)
diff --git a/core/switch_core/gateway/hosted_controller.py b/core/switch_core/gateway/hosted_controller.py
index a83a703f1..9c36aadb4 100644
--- a/core/switch_core/gateway/hosted_controller.py
+++ b/core/switch_core/gateway/hosted_controller.py
@@ -10,28 +10,29 @@ from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.config import SwitchConfig
-from switch_core.db.models import (
- HostedMachine,
- TenantMember,
- require_tenant_id,
-)
+from switch_core.db.models import CloudMachine, TenantMember
from switch_core.db.stores.hosted_machine_store import (
MACHINE_CONNECT_TIMEOUT,
- HostedMachineStore,
+ CloudMachineStore,
+ WorkspaceOnMachine,
bump_revision,
+ ever_hosted,
idle_sleeping,
retention_expired,
+ workspace_on,
+ workspaces_on,
)
from switch_core.gateway.cloud_controllers import (
CloudEnrollmentUnavailable,
controller_machine_idle,
+ controller_reports,
enrollment_code,
live_controller,
)
from switch_core.gateway.dependencies import get_config, get_session_factory
from switch_core.gateway.hosted_machines import controller_settings
from switch_core.providers.hosted import HostedControllerSettings
-from switch_core.tenant_context import tenant_scope
+from switch_core.tenant_context import no_tenant, tenant_scope
logger = logging.getLogger(__name__)
@@ -45,22 +46,24 @@ async def controller_session(
settings: Annotated[HostedControllerSettings, Depends(controller_settings)],
factory: Annotated[async_sessionmaker[AsyncSession], Depends(get_session_factory)],
) -> AsyncIterator[AsyncSession]:
- """Authenticate the operator controller and bind its configured tenant.
+ """Authenticate the operator controller, on a session with no tenant bound.
- The bearer credential can provision only this tenant's machines; it is
- never accepted as a user or agent credential on other routes.
+ A cloud machine belongs to its owner rather than to a workspace, so this
+ session reads machines and nothing scoped; what a machine runs in each
+ workspace is read with that workspace bound (`workspaces_on`). The bearer
+ credential is never accepted as a user or agent credential on other routes.
"""
supplied = request.headers.get("authorization", "")
if not secrets.compare_digest(
supplied, "Bearer " + settings.token.get_secret_value()
):
raise HTTPException(401, "Invalid cloud controller credential.")
- with tenant_scope(settings.tenant_id):
+ with no_tenant():
async with factory() as session:
yield session
-def machine_item(machine: HostedMachine) -> dict:
+def machine_item(machine: CloudMachine) -> dict:
return {
"machine_id": machine.id,
"state": machine.state,
@@ -70,17 +73,17 @@ def machine_item(machine: HostedMachine) -> dict:
"retain_until": machine.retain_until.isoformat()
if machine.retain_until
else None,
- "bundle_revision": machine.enrollment_code_revision,
+ "bundle_revision": machine.revision,
}
-def _idle_stops(machine: HostedMachine, idle_minutes: int) -> bool:
+def _idle_stops(machine: CloudMachine, idle_minutes: int) -> bool:
"""Whether an idle machine is put to sleep: a message addressed to one of
its agents wakes it again."""
return idle_minutes > 0
-def _connect_timed_out(machine: HostedMachine, now: datetime) -> bool:
+def _connect_timed_out(machine: CloudMachine, now: datetime) -> bool:
return (
machine.state == "provisioning"
and machine.running_observed_at is not None
@@ -92,7 +95,7 @@ def _connect_timed_out(machine: HostedMachine, now: datetime) -> bool:
)
-def _start_timed_out(machine: HostedMachine, now: datetime) -> bool:
+def _start_timed_out(machine: CloudMachine, now: datetime) -> bool:
return (
machine.state == "provisioning"
and machine.running_observed_at is None
@@ -100,9 +103,7 @@ def _start_timed_out(machine: HostedMachine, now: datetime) -> bool:
)
-def _errored_unclaimed(
- machine: HostedMachine, now: datetime, idle_minutes: int
-) -> bool:
+def _errored_unclaimed(machine: CloudMachine, now: datetime, idle_minutes: int) -> bool:
return (
idle_minutes > 0
and machine.state == "error"
@@ -111,7 +112,7 @@ def _errored_unclaimed(
)
-def _needs_sweep(machine: HostedMachine, now: datetime, idle_minutes: int) -> bool:
+def _needs_sweep(machine: CloudMachine, now: datetime, idle_minutes: int) -> bool:
return (
(machine.state == "queued" and now - machine.updated_at > QUEUED_TIMEOUT)
or _connect_timed_out(machine, now)
@@ -129,12 +130,13 @@ def _needs_sweep(machine: HostedMachine, now: datetime, idle_minutes: int) -> bo
async def _sweep(
session: AsyncSession,
- machine: HostedMachine,
+ machine: CloudMachine,
+ workspaces: list[WorkspaceOnMachine],
idle_minutes: int,
retention_days: int,
now: datetime,
) -> None:
- store = HostedMachineStore()
+ store = CloudMachineStore()
if machine.state == "queued" and now - machine.updated_at > QUEUED_TIMEOUT:
machine.state = "error"
machine.error_code = "machine_connect_timeout"
@@ -153,15 +155,13 @@ async def _sweep(
elif machine.state in {"retained", "error"} and retention_expired(machine, now):
machine.desired_state = "deleted"
bump_revision(machine, now)
- elif _errored_unclaimed(machine, now, idle_minutes) and not await store.ever_hosted(
- session, machine
- ):
+ elif _errored_unclaimed(machine, now, idle_minutes) and not ever_hosted(workspaces):
await store.release_if_empty(
- session, machine, retention_days=retention_days, now=now
+ machine, workspaces, retention_days=retention_days, now=now
)
elif idle_sleeping(machine):
await store.release_if_empty(
- session, machine, retention_days=retention_days, now=now
+ machine, workspaces, retention_days=retention_days, now=now
)
elif (
_idle_stops(machine, idle_minutes)
@@ -170,7 +170,7 @@ async def _sweep(
and controller_machine_idle(machine, timedelta(minutes=idle_minutes), now)
):
if not await store.release_if_empty(
- session, machine, retention_days=retention_days, now=now
+ machine, workspaces, retention_days=retention_days, now=now
):
store.stop(machine, "idle", now)
@@ -178,18 +178,16 @@ async def _sweep(
@router.get("/machines")
async def machines(
session: Annotated[AsyncSession, Depends(controller_session)],
+ factory: Annotated[async_sessionmaker[AsyncSession], Depends(get_session_factory)],
config: Annotated[SwitchConfig, Depends(get_config)],
) -> dict:
- store = HostedMachineStore()
+ store = CloudMachineStore()
now = datetime.now(UTC)
candidates = list(
await session.scalars(
- select(HostedMachine)
- .where(
- HostedMachine.tenant_id == require_tenant_id(),
- HostedMachine.state != "deleted",
- )
- .order_by(HostedMachine.created_at)
+ select(CloudMachine)
+ .where(CloudMachine.state != "deleted")
+ .order_by(CloudMachine.created_at)
)
)
for candidate in candidates:
@@ -200,18 +198,16 @@ async def machines(
await _sweep(
session,
machine,
+ await workspaces_on(factory, machine.id),
config.hosted_idle_stop_minutes,
config.hosted_disk_retention_days,
now,
)
await session.commit()
rows = await session.scalars(
- select(HostedMachine)
- .where(
- HostedMachine.tenant_id == require_tenant_id(),
- HostedMachine.state != "deleted",
- )
- .order_by(HostedMachine.created_at)
+ select(CloudMachine)
+ .where(CloudMachine.state != "deleted")
+ .order_by(CloudMachine.created_at)
.execution_options(populate_existing=True)
)
response = {"machines": [machine_item(machine) for machine in rows]}
@@ -219,8 +215,8 @@ async def machines(
return response
-async def _locked_machine(session: AsyncSession, machine_id: str) -> HostedMachine:
- machine = await HostedMachineStore().locked(session, machine_id)
+async def _locked_machine(session: AsyncSession, machine_id: str) -> CloudMachine:
+ machine = await CloudMachineStore().locked(session, machine_id)
if machine is None:
raise HTTPException(404, "Cloud machine not found.")
return machine
@@ -231,35 +227,73 @@ async def prepare(
machine_id: str,
response: Response,
session: Annotated[AsyncSession, Depends(controller_session)],
+ factory: Annotated[async_sessionmaker[AsyncSession], Depends(get_session_factory)],
settings: Annotated[HostedControllerSettings, Depends(controller_settings)],
config: Annotated[SwitchConfig, Depends(get_config)],
) -> dict:
+ """What the machine boots from at its current revision: per workspace it
+ serves, the controller it enrolled as there, or, until it has one that is
+ not revoked, a one-time code to enroll with. Never a long-lived credential:
+ the machine keeps the ones it enrolls with on its own disk."""
response.headers["Cache-Control"] = "no-store"
machine = await _locked_machine(session, machine_id)
if machine.state == "deleted" or machine.desired_state == "deleted":
raise HTTPException(409, "machine is deleted")
now = datetime.now(UTC)
- if await session.get(TenantMember, (require_tenant_id(), machine.owner_id)) is None:
+ controllers = []
+ for workspace in await workspaces_on(factory, machine.id):
+ with tenant_scope(workspace.tenant_id):
+ async with factory() as scoped:
+ if (
+ await scoped.get(
+ TenantMember, (workspace.tenant_id, machine.owner_id)
+ )
+ is None
+ ):
+ logger.warning(
+ "Cloud machine %s: its owner left workspace %s, so it runs no controller there.",
+ machine.id,
+ workspace.tenant_id,
+ )
+ continue
+ row = await workspace_on(scoped, machine.id)
+ assert row is not None
+ controller = await live_controller(scoped, row)
+ try:
+ code = (
+ None
+ if controller is not None
+ else await enrollment_code(
+ scoped, row, machine.revision, config.keyring, now
+ )
+ )
+ except CloudEnrollmentUnavailable as error:
+ raise HTTPException(409, str(error)) from None
+ await scoped.commit()
+ controllers.append(
+ {
+ "key": row.id,
+ "id": controller.id if controller is not None else None,
+ "enrollment_code": code,
+ }
+ )
+ served = {entry["key"] for entry in controllers}
+ reports = controller_reports(machine)
+ if set(reports) - served:
+ # A workspace the machine no longer runs a controller for must not keep
+ # it awake with a report that will never be fresh again.
+ machine.heartbeat = {
+ **(machine.heartbeat or {}),
+ "controllers": {
+ key: report for key, report in reports.items() if key in served
+ },
+ }
+ if not controllers:
machine.state = "error"
- machine.error = "The cloud machine's owner is no longer a workspace member."
+ machine.error = "The cloud machine's owner is no longer a member of any workspace it serves."
machine.updated_at = now
await session.commit()
- raise HTTPException(
- 409, "The cloud machine's owner is no longer a workspace member."
- )
- # What the machine boots from at its current revision: the controller it
- # enrolled as, or, until it has one that is not revoked, a one-time code to
- # enroll with. Never a long-lived credential: the machine keeps the one it
- # enrolls with on its own disk.
- controller = await live_controller(session, machine)
- try:
- code = (
- None
- if controller is not None
- else await enrollment_code(session, machine, config.keyring, now)
- )
- except CloudEnrollmentUnavailable as error:
- raise HTTPException(409, str(error)) from None
+ raise HTTPException(409, machine.error)
if machine.state == "queued":
machine.state = "provisioning"
machine.updated_at = now
@@ -268,10 +302,7 @@ async def prepare(
"revision": machine.revision,
"bundle_revision": machine.revision,
"api_endpoint": settings.agent_api_endpoint,
- "controller": {
- "id": controller.id if controller is not None else None,
- "enrollment_code": code,
- },
+ "controllers": controllers,
}
await session.commit()
return result
diff --git a/core/switch_core/gateway/hosted_machines.py b/core/switch_core/gateway/hosted_machines.py
index 0c97b37b4..c1ab9904b 100644
--- a/core/switch_core/gateway/hosted_machines.py
+++ b/core/switch_core/gateway/hosted_machines.py
@@ -3,21 +3,33 @@ from typing import Annotated, Literal, cast
from fastapi import APIRouter, Depends, HTTPException, Request
from pydantic import BaseModel, ConfigDict, Field
-from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.config import SwitchConfig
-from switch_core.db.models import HostedMachine, User, require_tenant_id
+from switch_core.db.models import (
+ CloudMachine,
+ MachineWorkspace,
+ User,
+ require_tenant_id,
+)
from switch_core.db.stores.hosted_machine_store import (
- HostedMachineConflict,
- HostedMachineStore,
+ MACHINE_CONNECT_TIMEOUT,
+ CloudMachineConflict,
+ CloudMachineStore,
claim_conflict,
idle_sleeping,
lock_claims,
+ managed_agent_ids,
owner_stopped,
+ workspace_on,
)
from switch_core.gateway.auth import get_current_user
-from switch_core.gateway.dependencies import get_config, get_session
+from switch_core.gateway.cloud_controllers import workspace_reported
+from switch_core.gateway.dependencies import (
+ get_config,
+ get_session,
+ get_session_factory,
+)
from switch_core.providers.hosted import HostedControllerSettings
router = APIRouter(prefix="/hosted-machines")
@@ -41,9 +53,10 @@ def controller_settings(request: Request) -> HostedControllerSettings:
def machines_enabled(config: SwitchConfig, settings: HostedControllerSettings) -> bool:
- """Whether the bound tenant may claim cloud machines on this server."""
- return (
- config.hosted_launch_capacity > 0 and settings.tenant_id == require_tenant_id()
+ """Whether the bound tenant may use cloud machines on this server."""
+ return config.hosted_launch_capacity > 0 and (
+ settings.allowed_tenant_ids is None
+ or require_tenant_id() in settings.allowed_tenant_ids
)
@@ -57,19 +70,50 @@ def _usage(heartbeat: dict | None, key: str) -> dict | None:
}
-async def machine_summary(session: AsyncSession, machine: HostedMachine) -> dict:
- """The machine as its owner sees it: `agents` are the managed agents
- placed on the controller it enrolled as, `controller_id` (null until it
- has)."""
+WORKSPACE_CONNECT_TIMEOUT = (
+ "The cloud machine started, but its controller for this workspace did not "
+ "connect to Switch within 10 minutes. Retry it in Switch Console, or ask your "
+ "administrator to check the machine's startup logs."
+)
+
+
+def workspace_state(
+ machine: CloudMachine, workspace: MachineWorkspace, now: datetime
+) -> tuple[str, str | None]:
+ """The machine's state as the bound workspace sees it, and why it is in
+ error: a ready machine is still starting here until this workspace's
+ controller has reported, and in error if it never does. A machine on its
+ way down is as it is everywhere."""
+ if (
+ machine.state != "ready"
+ or machine.desired_state != "running"
+ or workspace_reported(machine, workspace.id)
+ ):
+ return machine.state, machine.error
+ if (
+ machine.running_observed_at is not None
+ and now - machine.running_observed_at > MACHINE_CONNECT_TIMEOUT
+ ):
+ return "error", WORKSPACE_CONNECT_TIMEOUT
+ return "provisioning", None
+
+
+async def machine_summary(
+ session: AsyncSession, machine: CloudMachine, workspace: MachineWorkspace
+) -> dict:
+ """The machine as its owner sees it from the bound workspace: `agents` are
+ the workspace's managed agents placed on the controller the machine
+ enrolled as here, `controller_id` (null until it has)."""
+ state, error = workspace_state(machine, workspace, datetime.now(UTC))
return {
"machine_id": machine.id,
- "state": machine.state,
+ "state": state,
"desired_state": machine.desired_state,
"stop_reason": machine.stop_reason,
"sleeping": idle_sleeping(machine),
"revision": machine.revision,
"instance_type": machine.instance_type,
- "error": machine.error,
+ "error": error,
"error_code": machine.error_code,
"retain_until": None
if machine.retain_until is None
@@ -79,8 +123,8 @@ async def machine_summary(session: AsyncSession, machine: HostedMachine) -> dict
else machine.heartbeat_at.isoformat(),
"disk": _usage(machine.heartbeat, "disk"),
"memory": _usage(machine.heartbeat, "memory"),
- "agents": await HostedMachineStore().managed_agent_ids(session, machine),
- "controller_id": machine.controller_id,
+ "agents": await managed_agent_ids(session, workspace.controller_id),
+ "controller_id": workspace.controller_id,
}
@@ -90,32 +134,42 @@ class MachineUnavailable(Exception):
async def ensure_machine(
session: AsyncSession,
+ factory: async_sessionmaker[AsyncSession],
owner_id: str,
config: SwitchConfig,
settings: HostedControllerSettings | None,
-) -> HostedMachine:
- """The owner's machine, claimed and started so it warms before an agent needs it.
+) -> tuple[CloudMachine, MachineWorkspace]:
+ """The owner's machine, claimed or joined by the bound workspace and
+ started, so it warms before an agent needs it.
A machine its owner stopped, or one in error or being removed, is
- returned as it is.
+ returned as it is when it already serves the workspace.
Raises `MachineUnavailable` when the server offers the tenant no cloud
- machines and `HostedMachineConflict` when none can be had. The caller
+ machines and `CloudMachineConflict` when none can be had. The caller
commits.
"""
if settings is None or not machines_enabled(config, settings):
raise MachineUnavailable(MACHINES_DISABLED)
- machines = HostedMachineStore()
+ machines = CloudMachineStore()
await lock_claims(session)
existing = await machines.live_for_owner(session, owner_id)
if existing is not None:
existing = await machines.locked(session, existing.id)
- if existing is not None and (
- owner_stopped(existing)
- or claim_conflict(existing, datetime.now(UTC)) is not None
+ workspace = (
+ None if existing is None else await workspace_on(session, existing.id)
+ )
+ if (
+ existing is not None
+ and workspace is not None
+ and (
+ owner_stopped(existing)
+ or claim_conflict(existing, datetime.now(UTC)) is not None
+ )
):
- return existing
+ return existing, workspace
return await machines.claim(
session,
+ factory,
owner_id=owner_id,
capacity=config.hosted_launch_capacity,
now=datetime.now(UTC),
@@ -124,11 +178,13 @@ async def ensure_machine(
async def _owned(
session: AsyncSession, machine_id: str, owner_id: str
-) -> HostedMachine:
- machine = await HostedMachineStore().owned(session, machine_id, owner_id)
- if machine is None:
+) -> tuple[CloudMachine, MachineWorkspace]:
+ """The owner's machine, as it serves the bound workspace."""
+ machine = await CloudMachineStore().owned(session, machine_id, owner_id)
+ workspace = None if machine is None else await workspace_on(session, machine.id)
+ if machine is None or workspace is None:
raise HTTPException(404, "Cloud machine not found.")
- return machine
+ return machine, workspace
@router.get("")
@@ -136,33 +192,33 @@ async def owned_machines(
user: Annotated[User, Depends(get_current_user)],
session: Annotated[AsyncSession, Depends(get_session)],
) -> dict:
- machines = await session.scalars(
- select(HostedMachine)
- .where(
- HostedMachine.tenant_id == require_tenant_id(),
- HostedMachine.owner_id == user.id,
- HostedMachine.state != "deleted",
- )
- .order_by(HostedMachine.created_at)
- )
- return {"machines": [await machine_summary(session, row) for row in machines]}
+ """The owner's machine, when it serves the bound workspace: in another
+ workspace it is the same machine, joined from there."""
+ machine = await CloudMachineStore().live_for_owner(session, user.id)
+ workspace = None if machine is None else await workspace_on(session, machine.id)
+ if machine is None or workspace is None:
+ return {"machines": []}
+ return {"machines": [await machine_summary(session, machine, workspace)]}
@router.post("/ensure")
async def ensure(
user: Annotated[User, Depends(get_current_user)],
session: Annotated[AsyncSession, Depends(get_session)],
+ factory: Annotated[async_sessionmaker[AsyncSession], Depends(get_session_factory)],
config: Annotated[SwitchConfig, Depends(get_config)],
settings: Annotated[HostedControllerSettings | None, Depends(hosted_settings)],
) -> dict:
try:
- machine = await ensure_machine(session, user.id, config, settings)
+ machine, workspace = await ensure_machine(
+ session, factory, user.id, config, settings
+ )
except MachineUnavailable as error:
raise HTTPException(503, str(error)) from None
- except HostedMachineConflict as error:
+ except CloudMachineConflict as error:
raise HTTPException(409, str(error)) from None
await session.commit()
- return await machine_summary(session, machine)
+ return await machine_summary(session, machine, workspace)
@router.get("/{machine_id}")
@@ -171,7 +227,8 @@ async def status(
user: Annotated[User, Depends(get_current_user)],
session: Annotated[AsyncSession, Depends(get_session)],
) -> dict:
- return await machine_summary(session, await _owned(session, machine_id, user.id))
+ machine, workspace = await _owned(session, machine_id, user.id)
+ return await machine_summary(session, machine, workspace)
class MachineLifecycleRequest(BaseModel):
@@ -187,8 +244,8 @@ async def lifecycle(
user: Annotated[User, Depends(get_current_user)],
session: Annotated[AsyncSession, Depends(get_session)],
) -> dict:
- machines = HostedMachineStore()
- await _owned(session, machine_id, user.id)
+ machines = CloudMachineStore()
+ _machine, workspace = await _owned(session, machine_id, user.id)
machine = await machines.locked(session, machine_id)
assert machine is not None
if machine.revision != body.revision and not (
@@ -214,4 +271,4 @@ async def lifecycle(
raise HTTPException(409, "Only a machine in error can be retried.")
machines.retry(machine, now)
await session.commit()
- return {"machine": await machine_summary(session, machine)}
+ return {"machine": await machine_summary(session, machine, workspace)}
diff --git a/core/switch_core/management/service.py b/core/switch_core/management/service.py
index 677532f54..1248bd7f6 100644
--- a/core/switch_core/management/service.py
+++ b/core/switch_core/management/service.py
@@ -49,7 +49,7 @@ from switch_core.db.models import (
AgentController,
AgentControllerOperation,
ApiKey,
- HostedMachine,
+ MachineWorkspace,
)
from switch_core.db.models import AgentDefinition as AgentDefinitionRow
from switch_core.db.stores.agent_controller_operation_store import (
@@ -59,7 +59,7 @@ from switch_core.db.stores.agent_controller_store import AgentControllerStore
from switch_core.db.stores.agent_definition_store import AgentDefinitionStore
from switch_core.db.stores.agent_store import AgentStore
from switch_core.db.stores.api_key_store import ApiKeyStore
-from switch_core.db.stores.hosted_machine_store import HostedMachineStore
+from switch_core.db.stores.hosted_machine_store import CloudMachineStore
from switch_core.db.stores.switch_core_process_store import SwitchCoreProcessStore
from switch_core.gateway.cloud_controllers import (
record_controller_status,
@@ -231,29 +231,30 @@ class ManagementService:
owner_id=owner_id,
api_key_id=key.id,
expires_at=expires_at,
- hosted_machine_id=None,
+ machine_workspace_id=None,
)
await session.commit()
return code, expires_at
async def machine_enrollment_code(
- self, session: AsyncSession, machine: HostedMachine, now: datetime
+ self, session: AsyncSession, workspace: MachineWorkspace, now: datetime
) -> str:
- """A code for a Switch cloud machine's controller to enroll with: the
- owner's, valid long enough for the machine to boot, and binding what
- enrolls with it to the machine. The caller commits."""
+ """A code for a Switch cloud machine's controller in the bound workspace
+ to enroll with: the owner's, valid long enough for the machine to boot,
+ and binding what enrolls with it to the workspace's row on the machine.
+ The caller commits."""
key, code = await self._new_hash_only_key(
session,
- owner_id=machine.owner_id,
+ owner_id=workspace.owner_id,
key_type=CONTROLLER_ENROLLMENT_KEY_TYPE,
label="cloud machine enrollment code",
)
await self.controllers.create_enrollment_code(
session,
- owner_id=machine.owner_id,
+ owner_id=workspace.owner_id,
api_key_id=key.id,
expires_at=now + tokens.MACHINE_ENROLLMENT_CODE_LIFETIME,
- hosted_machine_id=machine.id,
+ machine_workspace_id=workspace.id,
)
return code
@@ -281,13 +282,20 @@ class ManagementService:
)
if consumed is None:
raise invalid
+ workspace = (
+ None
+ if consumed.machine_workspace_id is None
+ else await session.get(
+ MachineWorkspace, (tenant_id, consumed.machine_workspace_id)
+ )
+ )
machine = (
None
- if consumed.hosted_machine_id is None
- else await HostedMachineStore().locked(session, consumed.hosted_machine_id)
+ if workspace is None
+ else await CloudMachineStore().locked(session, workspace.machine_id)
)
- if consumed.hosted_machine_id is not None and (
- machine is None or machine.state == "deleted"
+ if consumed.machine_workspace_id is not None and (
+ workspace is None or machine is None or machine.state == "deleted"
):
raise invalid
controller_key, credential = await self._new_hash_only_key(
@@ -316,9 +324,9 @@ class ManagementService:
replaced: str | None = None
moved: list[AgentDefinitionRow] = []
revisions: dict[str, int] = {}
- if machine is not None:
- replaced = machine.controller_id
- machine.controller_id = controller.id
+ if workspace is not None:
+ replaced = workspace.controller_id
+ workspace.controller_id = controller.id
if replaced is not None and replaced != controller.id:
await self._revoke(session, tenant_id, replaced)
moved = await self._move_placed(
@@ -336,7 +344,7 @@ class ManagementService:
logger.info(
"Cloud machine %s enrolled again; revoked its previous controller %s "
"and moved its %d agent(s) to %s",
- consumed.hosted_machine_id,
+ workspace.machine_id if workspace is not None else None,
replaced,
len(moved),
controller.id,
diff --git a/core/switch_core/migrations/versions/b6e1c9d4a7f2_cloud_machine_per_user.py b/core/switch_core/migrations/versions/b6e1c9d4a7f2_cloud_machine_per_user.py
new file mode 100644
index 000000000..8d41ceafe
--- /dev/null
+++ b/core/switch_core/migrations/versions/b6e1c9d4a7f2_cloud_machine_per_user.py
@@ -0,0 +1,316 @@
+"""one cloud machine per user: `cloud_machines` and `machine_workspaces`
+
+A user's Switch cloud machine was a row of one workspace. It becomes the
+user's, whichever workspaces it serves: `cloud_machines` is global, as the
+person is, and holds the VM's lifecycle; `machine_workspaces` holds, per
+workspace and under row-level security, the agents controller the machine
+runs there and the code it enrolls with. `tenants_of_cloud_machine` says which
+workspaces a machine serves, for the cloud controller, which acts for none.
+
+Each existing machine keeps its id, and its workspace row takes the same id,
+so its controller and its enrollment carry over. Refuses to run when one user
+has live machines in two workspaces: nothing here could merge them.
+
+The lookup's DDL is a frozen copy of `switch_core/db/tenant_lookup.py`'s, as
+every revision in this chain copies rather than imports.
+
+Revision ID: b6e1c9d4a7f2
+Revises: a8c3e5f1d27b
+Create Date: 2026-10-09 15:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects.postgresql import JSONB
+
+revision: str = "b6e1c9d4a7f2"
+down_revision: str | None = "a8c3e5f1d27b"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+SECURE_SEARCH_PATH = "pg_catalog, public, pg_temp"
+
+CREATE_TENANTS_OF_CLOUD_MACHINE = f"""CREATE OR REPLACE FUNCTION tenants_of_cloud_machine(p_machine_id text)
+ RETURNS SETOF text
+ LANGUAGE sql STABLE SECURITY DEFINER
+ SET search_path = {SECURE_SEARCH_PATH}
+AS $$SELECT tenant_id FROM machine_workspaces WHERE machine_id = p_machine_id ORDER BY created_at, tenant_id$$"""
+
+DROP_TENANTS_OF_CLOUD_MACHINE = "DROP FUNCTION IF EXISTS tenants_of_cloud_machine(text)"
+
+STATES = "state IN ('queued', 'provisioning', 'ready', 'stopping', 'stopped', 'error', 'retained', 'deleting', 'deleted')"
+DESIRED_STATES = "desired_state IN ('running', 'stopped', 'retained', 'deleted')"
+STOP_REASONS = "stop_reason IS NULL OR stop_reason IN ('idle', 'owner')"
+
+
+def _timestamps() -> list[sa.Column]:
+ return [
+ sa.Column(
+ "created_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.func.now(),
+ nullable=False,
+ ),
+ sa.Column(
+ "updated_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.func.now(),
+ nullable=False,
+ ),
+ ]
+
+
+def upgrade() -> None:
+ connection = op.get_bind()
+ shared = connection.scalar(
+ sa.text(
+ "SELECT count(*) FROM (SELECT owner_id FROM hosted_machines "
+ "WHERE state <> 'deleted' GROUP BY owner_id HAVING count(*) > 1) AS owners"
+ )
+ )
+ if shared:
+ raise RuntimeError(
+ f"one cloud machine per user: {shared} user(s) have cloud machines that "
+ "are not deleted in more than one workspace. Remove all but one of each "
+ "user's cloud machines in Switch Console, then upgrade again."
+ )
+
+ op.create_table(
+ "cloud_machines",
+ sa.Column("id", sa.Text(), nullable=False),
+ sa.Column("owner_id", sa.Text(), sa.ForeignKey("users.id"), nullable=False),
+ sa.Column("state", sa.Text(), nullable=False),
+ sa.Column("desired_state", sa.Text(), nullable=False, server_default="running"),
+ sa.Column("stop_reason", sa.Text(), nullable=True),
+ sa.Column("revision", sa.Integer(), nullable=False, server_default="1"),
+ sa.Column("instance_type", sa.Text(), nullable=True),
+ sa.Column("data_volume_id", sa.Text(), nullable=True),
+ sa.Column("instance_id", sa.Text(), nullable=True),
+ sa.Column("retain_until", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("active_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("heartbeat", JSONB(), nullable=True),
+ sa.Column("heartbeat_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("running_observed_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("error", sa.Text(), nullable=True),
+ sa.Column("error_code", sa.Text(), nullable=True),
+ *_timestamps(),
+ sa.PrimaryKeyConstraint("id"),
+ sa.CheckConstraint(STATES, name="ck_cloud_machine_state"),
+ sa.CheckConstraint(DESIRED_STATES, name="ck_cloud_machine_desired_state"),
+ sa.CheckConstraint(STOP_REASONS, name="ck_cloud_machine_stop_reason"),
+ )
+ op.create_index(
+ "uq_cloud_machine_owner",
+ "cloud_machines",
+ ["owner_id"],
+ unique=True,
+ postgresql_where=sa.text("state <> 'deleted'"),
+ )
+ op.create_table(
+ "machine_workspaces",
+ sa.Column(
+ "tenant_id",
+ sa.Text(),
+ sa.ForeignKey("tenants.id", name="fk_machine_workspaces_tenant"),
+ nullable=False,
+ ),
+ sa.Column("id", sa.Text(), nullable=False),
+ sa.Column(
+ "machine_id", sa.Text(), sa.ForeignKey("cloud_machines.id"), nullable=False
+ ),
+ sa.Column("owner_id", sa.Text(), sa.ForeignKey("users.id"), nullable=False),
+ sa.Column("controller_id", sa.Text(), nullable=True),
+ sa.Column("enrollment_code_encrypted", sa.Text(), nullable=True),
+ sa.Column("enrollment_code_revision", sa.Integer(), nullable=True),
+ sa.Column(
+ "created_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.func.now(),
+ nullable=False,
+ ),
+ sa.PrimaryKeyConstraint("tenant_id", "id"),
+ sa.ForeignKeyConstraint(
+ ["tenant_id", "controller_id"],
+ ["agent_controllers.tenant_id", "agent_controllers.id"],
+ name="fk_machine_workspaces_controller",
+ ondelete="SET NULL (controller_id)",
+ ),
+ sa.UniqueConstraint(
+ "tenant_id", "machine_id", name="uq_machine_workspaces_machine"
+ ),
+ )
+ op.create_index(
+ "ix_machine_workspaces_machine_id", "machine_workspaces", ["machine_id"]
+ )
+ op.execute("ALTER TABLE machine_workspaces ENABLE ROW LEVEL SECURITY")
+ op.execute(
+ "CREATE POLICY tenant_isolation ON machine_workspaces FOR ALL USING (tenant_id = (SELECT require_tenant_id())) WITH CHECK (tenant_id = (SELECT require_tenant_id()))"
+ )
+
+ op.execute(
+ """
+ INSERT INTO cloud_machines (
+ id, owner_id, state, desired_state, stop_reason, revision, instance_type,
+ data_volume_id, instance_id, retain_until, active_at, heartbeat,
+ heartbeat_at, running_observed_at, error, error_code, created_at, updated_at
+ )
+ SELECT id, owner_id, state, desired_state, stop_reason, revision, instance_type,
+ data_volume_id, instance_id, retain_until, active_at,
+ CASE WHEN heartbeat IS NULL THEN NULL ELSE jsonb_build_object(
+ 'disk', heartbeat -> 'disk',
+ 'memory', heartbeat -> 'memory',
+ 'controllers', jsonb_build_object(id, jsonb_build_object(
+ 'at', to_jsonb(heartbeat_at),
+ 'sessions_running', heartbeat -> 'sessions_running'
+ ))
+ ) END,
+ heartbeat_at, running_observed_at, error, error_code, created_at, updated_at
+ FROM hosted_machines
+ """
+ )
+ op.execute(
+ """
+ INSERT INTO machine_workspaces (
+ tenant_id, id, machine_id, owner_id, controller_id,
+ enrollment_code_encrypted, enrollment_code_revision, created_at
+ )
+ SELECT tenant_id, id, id, owner_id, controller_id,
+ enrollment_code_encrypted, enrollment_code_revision, created_at
+ FROM hosted_machines
+ """
+ )
+
+ op.drop_constraint(
+ "fk_agent_controller_enrollment_codes_hosted_machine",
+ "agent_controller_enrollment_codes",
+ type_="foreignkey",
+ )
+ op.alter_column(
+ "agent_controller_enrollment_codes",
+ "hosted_machine_id",
+ new_column_name="machine_workspace_id",
+ )
+ op.create_foreign_key(
+ "fk_agent_controller_enrollment_codes_machine_workspace",
+ "agent_controller_enrollment_codes",
+ "machine_workspaces",
+ ["tenant_id", "machine_workspace_id"],
+ ["tenant_id", "id"],
+ )
+ op.drop_table("hosted_machines")
+ op.execute(CREATE_TENANTS_OF_CLOUD_MACHINE)
+
+
+def downgrade() -> None:
+ connection = op.get_bind()
+ shared = connection.scalar(
+ sa.text(
+ "SELECT count(*) FROM (SELECT machine_id FROM machine_workspaces "
+ "GROUP BY machine_id HAVING count(*) > 1) AS machines"
+ )
+ )
+ if shared:
+ raise RuntimeError(
+ f"one cloud machine per user cannot be undone: {shared} cloud machine(s) "
+ "serve more than one workspace."
+ )
+ op.execute(DROP_TENANTS_OF_CLOUD_MACHINE)
+ op.create_table(
+ "hosted_machines",
+ sa.Column(
+ "tenant_id",
+ sa.Text(),
+ sa.ForeignKey("tenants.id", name="fk_hosted_machines_tenant"),
+ nullable=False,
+ ),
+ sa.Column("id", sa.Text(), nullable=False),
+ sa.Column("owner_id", sa.Text(), sa.ForeignKey("users.id"), nullable=False),
+ sa.Column("state", sa.Text(), nullable=False),
+ sa.Column("desired_state", sa.Text(), nullable=False, server_default="running"),
+ sa.Column("stop_reason", sa.Text(), nullable=True),
+ sa.Column("revision", sa.Integer(), nullable=False, server_default="1"),
+ sa.Column("instance_type", sa.Text(), nullable=True),
+ sa.Column("data_volume_id", sa.Text(), nullable=True),
+ sa.Column("instance_id", sa.Text(), nullable=True),
+ sa.Column("retain_until", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("controller_id", sa.Text(), nullable=True),
+ sa.Column("enrollment_code_encrypted", sa.Text(), nullable=True),
+ sa.Column("enrollment_code_revision", sa.Integer(), nullable=True),
+ sa.Column("active_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("heartbeat", JSONB(), nullable=True),
+ sa.Column("heartbeat_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("running_observed_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("error", sa.Text(), nullable=True),
+ sa.Column("error_code", sa.Text(), nullable=True),
+ *_timestamps(),
+ sa.PrimaryKeyConstraint("tenant_id", "id"),
+ sa.CheckConstraint(STATES, name="ck_hosted_machine_state"),
+ sa.CheckConstraint(DESIRED_STATES, name="ck_hosted_machine_desired_state"),
+ sa.CheckConstraint(STOP_REASONS, name="ck_hosted_machine_stop_reason"),
+ sa.ForeignKeyConstraint(
+ ["tenant_id", "controller_id"],
+ ["agent_controllers.tenant_id", "agent_controllers.id"],
+ name="fk_hosted_machines_controller",
+ ondelete="SET NULL (controller_id)",
+ ),
+ )
+ op.create_index(
+ "uq_hosted_machine_owner",
+ "hosted_machines",
+ ["tenant_id", "owner_id"],
+ unique=True,
+ postgresql_where=sa.text("state <> 'deleted'"),
+ )
+ op.execute("ALTER TABLE hosted_machines ENABLE ROW LEVEL SECURITY")
+ op.execute(
+ "CREATE POLICY tenant_isolation ON hosted_machines FOR ALL USING (tenant_id = (SELECT require_tenant_id())) WITH CHECK (tenant_id = (SELECT require_tenant_id()))"
+ )
+ op.execute(
+ """
+ INSERT INTO hosted_machines (
+ tenant_id, id, owner_id, state, desired_state, stop_reason, revision,
+ instance_type, data_volume_id, instance_id, retain_until, controller_id,
+ enrollment_code_encrypted, enrollment_code_revision, active_at, heartbeat,
+ heartbeat_at, running_observed_at, error, error_code, created_at, updated_at
+ )
+ SELECT w.tenant_id, w.id, m.owner_id, m.state, m.desired_state, m.stop_reason,
+ m.revision, m.instance_type, m.data_volume_id, m.instance_id,
+ m.retain_until, w.controller_id, w.enrollment_code_encrypted,
+ w.enrollment_code_revision, m.active_at,
+ CASE WHEN m.heartbeat IS NULL THEN NULL ELSE jsonb_build_object(
+ 'disk', m.heartbeat -> 'disk',
+ 'memory', m.heartbeat -> 'memory',
+ 'sessions_running', m.heartbeat -> 'controllers' -> w.id -> 'sessions_running'
+ ) END,
+ m.heartbeat_at, m.running_observed_at, m.error, m.error_code,
+ m.created_at, m.updated_at
+ FROM machine_workspaces AS w JOIN cloud_machines AS m ON m.id = w.machine_id
+ """
+ )
+ op.drop_constraint(
+ "fk_agent_controller_enrollment_codes_machine_workspace",
+ "agent_controller_enrollment_codes",
+ type_="foreignkey",
+ )
+ op.alter_column(
+ "agent_controller_enrollment_codes",
+ "machine_workspace_id",
+ new_column_name="hosted_machine_id",
+ )
+ op.execute(
+ "UPDATE agent_controller_enrollment_codes AS c SET hosted_machine_id = NULL "
+ "WHERE hosted_machine_id IS NOT NULL AND NOT EXISTS (SELECT 1 FROM hosted_machines AS h "
+ "WHERE h.tenant_id = c.tenant_id AND h.id = c.hosted_machine_id)"
+ )
+ op.create_foreign_key(
+ "fk_agent_controller_enrollment_codes_hosted_machine",
+ "agent_controller_enrollment_codes",
+ "hosted_machines",
+ ["tenant_id", "hosted_machine_id"],
+ ["tenant_id", "id"],
+ )
+ op.drop_table("machine_workspaces")
+ op.drop_table("cloud_machines")
diff --git a/core/switch_core/providers/hosted.py b/core/switch_core/providers/hosted.py
index 7a291e7c5..ab37a0e12 100644
--- a/core/switch_core/providers/hosted.py
+++ b/core/switch_core/providers/hosted.py
@@ -1,3 +1,4 @@
+from typing import Annotated
from urllib.parse import urlsplit
from pydantic import (
@@ -12,7 +13,8 @@ from pydantic import (
class HostedControllerSettings(BaseModel):
model_config = ConfigDict(extra="forbid")
- tenant_id: str = Field(min_length=1)
+ # The workspaces whose members may use cloud machines; null for every one.
+ allowed_tenant_ids: list[Annotated[str, Field(min_length=1)]] | None
token: SecretStr = Field(min_length=32)
agent_api_endpoint: str
diff --git a/core/tests/switch_core/db/stores/test_agent_management_stores.py b/core/tests/switch_core/db/stores/test_agent_management_stores.py
index b039ff5e8..426bce638 100644
--- a/core/tests/switch_core/db/stores/test_agent_management_stores.py
+++ b/core/tests/switch_core/db/stores/test_agent_management_stores.py
@@ -235,14 +235,14 @@ class TestEnrollmentCodes:
owner_id=owner.id,
api_key_id=live.id,
expires_at=NOW + timedelta(minutes=10),
- hosted_machine_id=None,
+ machine_workspace_id=None,
)
await CONTROLLERS.create_enrollment_code(
session,
owner_id=owner.id,
api_key_id=stale.id,
expires_at=NOW - timedelta(seconds=1),
- hosted_machine_id=None,
+ machine_workspace_id=None,
)
first = await CONTROLLERS.consume_enrollment_code(
session, TENANT_ZERO_ID, live.id, NOW
diff --git a/core/tests/switch_core/db/stores/test_hosted_machine_store.py b/core/tests/switch_core/db/stores/test_hosted_machine_store.py
index af05c4dea..00ef061a5 100644
--- a/core/tests/switch_core/db/stores/test_hosted_machine_store.py
+++ b/core/tests/switch_core/db/stores/test_hosted_machine_store.py
@@ -4,29 +4,43 @@ from datetime import UTC, datetime, timedelta
import pytest
from switch_core.db.models import (
+ TENANT_ZERO_ID,
AgentDefinition,
- HostedMachine,
+ CloudMachine,
User,
- require_tenant_id,
)
from switch_core.db.stores.hosted_machine_store import (
MACHINE_BEING_REMOVED,
MACHINE_NEEDS_ADMIN,
MACHINE_NEEDS_ATTENTION,
+ MACHINE_WORKSPACES_FULL,
MACHINES_FULL,
- HostedMachineConflict,
- HostedMachineStore,
+ MAX_MACHINE_WORKSPACES,
+ CloudMachineConflict,
+ CloudMachineStore,
+ WorkspaceOnMachine,
+ ever_hosted,
idle_sleeping,
lock_claims,
machine_starting,
+ managed_agent_ids,
owner_stopped,
+ workspace_on,
+ workspaces_on,
)
+from switch_core.db.tenant_lookup import tenants_of_cloud_machine
+from switch_core.tenant_context import tenant_scope
from tests.switch_core.hosted_machine_helpers import (
+ add_tenant,
+ add_workspace,
link_controller,
place_managed_agent,
seed_machine,
)
+TENANT_B = "tenant-b"
+OTHER_WORKSPACES = [f"workspace-{index}" for index in range(MAX_MACHINE_WORKSPACES)]
+
@pytest.fixture
async def factory(session_factory):
@@ -43,21 +57,35 @@ async def factory(session_factory):
for owner in ("owner-a", "owner-b", "owner-c")
]
)
+ for tenant_id in (TENANT_B, *OTHER_WORKSPACES):
+ await add_tenant(session, tenant_id)
await session.commit()
return session_factory
-async def claim(factory, owner_id, capacity=2):
+async def claim_both(factory, owner_id, capacity=2):
+ """The owner's machine claimed with the bound workspace, and that workspace's row on it."""
async with factory() as session:
await lock_claims(session)
- machine = await HostedMachineStore().claim(
+ found = await CloudMachineStore().claim(
session,
+ factory,
owner_id=owner_id,
capacity=capacity,
now=datetime.now(UTC),
)
await session.commit()
- return machine
+ return found
+
+
+async def claim(factory, owner_id, capacity=2):
+ machine, _workspace = await claim_both(factory, owner_id, capacity)
+ return machine
+
+
+async def claim_in(factory, tenant_id, owner_id, capacity=2):
+ with tenant_scope(tenant_id):
+ return await claim_both(factory, owner_id, capacity)
async def seed(factory, **fields):
@@ -69,29 +97,35 @@ async def seed(factory, **fields):
async def reload(factory, machine_id):
async with factory() as session:
- machine = await HostedMachineStore().get(session, machine_id)
+ machine = await CloudMachineStore().get(session, machine_id)
assert machine is not None
return machine
async def test_claim_creates_a_queued_machine(factory):
- machine = await claim(factory, "owner-a")
+ machine, workspace = await claim_both(factory, "owner-a")
assert (
machine.owner_id,
machine.state,
machine.desired_state,
machine.revision,
- machine.controller_id,
- ) == ("owner-a", "queued", "running", 1, None)
+ ) == ("owner-a", "queued", "running", 1)
assert machine_starting(machine)
+ assert (
+ workspace.tenant_id,
+ workspace.machine_id,
+ workspace.owner_id,
+ workspace.controller_id,
+ ) == (TENANT_ZERO_ID, machine.id, "owner-a", None)
async def test_a_second_claim_reuses_the_owner_s_machine(factory):
- first = await claim(factory, "owner-a")
- second = await claim(factory, "owner-a")
- assert second.id == first.id
+ first, here = await claim_both(factory, "owner-a")
+ second, again = await claim_both(factory, "owner-a")
+ assert (second.id, again.id) == (first.id, here.id)
assert second.revision == 1
assert second.active_at >= first.active_at
+ assert await tenants_of_cloud_machine(factory, first.id) == [TENANT_ZERO_ID]
async def test_reusing_a_queued_machine_keeps_its_queued_clock(factory):
@@ -105,7 +139,7 @@ async def test_reusing_a_queued_machine_keeps_its_queued_clock(factory):
)
long_ago = datetime.now(UTC) - timedelta(minutes=11)
async with factory() as session:
- row = await HostedMachineStore().get(session, queued.id)
+ row = await CloudMachineStore().get(session, queued.id)
row.updated_at = long_ago
row.active_at = long_ago
await session.commit()
@@ -120,7 +154,7 @@ async def test_starting_a_running_machine_only_renews_activity(factory):
long_ago = datetime.now(UTC) - timedelta(minutes=11)
machine.updated_at = long_ago
now = datetime.now(UTC)
- HostedMachineStore().start(machine, now)
+ CloudMachineStore().start(machine, now)
assert (machine.updated_at, machine.active_at, machine.revision) == (
long_ago,
now,
@@ -138,7 +172,7 @@ async def test_a_retained_machine_is_reused_on_the_same_disk(factory):
revision=4,
)
async with factory() as session:
- row = await HostedMachineStore().get(session, retained.id)
+ row = await CloudMachineStore().get(session, retained.id)
row.retain_until = datetime.now(UTC) + timedelta(days=3)
await session.commit()
machine = await claim(factory, "owner-a")
@@ -214,13 +248,13 @@ async def test_a_machine_that_cannot_take_agents_refuses_the_claim(
revision=2,
)
async with factory() as session:
- row = await HostedMachineStore().get(session, seeded.id)
+ row = await CloudMachineStore().get(session, seeded.id)
assert row is not None
row.error_code = error_code
if retain_for is not None:
row.retain_until = datetime.now(UTC) + retain_for
await session.commit()
- with pytest.raises(HostedMachineConflict) as raised:
+ with pytest.raises(CloudMachineConflict) as raised:
await claim(factory, "owner-a")
assert str(raised.value) == detail
@@ -237,18 +271,18 @@ async def test_a_retained_machine_in_error_still_retaining_refuses_as_in_error(
revision=2,
)
async with factory() as session:
- row = await HostedMachineStore().get(session, seeded.id)
+ row = await CloudMachineStore().get(session, seeded.id)
assert row is not None
row.retain_until = datetime.now(UTC) + timedelta(days=3)
await session.commit()
- with pytest.raises(HostedMachineConflict) as raised:
+ with pytest.raises(CloudMachineConflict) as raised:
await claim(factory, "owner-a")
assert str(raised.value) == MACHINE_NEEDS_ATTENTION
async def test_capacity_caps_live_machines(factory):
await claim(factory, "owner-b", capacity=1)
- with pytest.raises(HostedMachineConflict) as raised:
+ with pytest.raises(CloudMachineConflict) as raised:
await claim(factory, "owner-a", capacity=1)
assert str(raised.value) == MACHINES_FULL
@@ -272,13 +306,13 @@ async def test_concurrent_claims_cannot_exceed_capacity(factory):
claim(factory, "owner-b", capacity=1),
return_exceptions=True,
)
- assert sum(isinstance(value, HostedMachine) for value in results) == 1
- assert sum(isinstance(value, HostedMachineConflict) for value in results) == 1
+ assert sum(isinstance(value, CloudMachine) for value in results) == 1
+ assert sum(isinstance(value, CloudMachineConflict) for value in results) == 1
async def test_start_stop_and_retry(factory):
machine = await claim(factory, "owner-a")
- store = HostedMachineStore()
+ store = CloudMachineStore()
now = datetime.now(UTC)
store.start(machine, now)
assert machine.revision == 1
@@ -318,7 +352,7 @@ async def test_starting_a_ready_machine_again_waits_for_it_to_reconnect(factory)
stop_reason=None,
revision=2,
)
- store = HostedMachineStore()
+ store = CloudMachineStore()
now = datetime.now(UTC)
machine.running_observed_at = now
store.start(machine, now)
@@ -369,33 +403,45 @@ async def ready_machine(session, owner_id="owner-a"):
)
+def counted(workspaces: list[WorkspaceOnMachine]) -> list[tuple[str, str | None, int]]:
+ return [
+ (workspace.tenant_id, workspace.controller_id, workspace.agent_count)
+ for workspace in workspaces
+ ]
+
+
async def test_retain_if_empty_retains_only_once_no_agent_is_placed(factory):
- store = HostedMachineStore()
+ store = CloudMachineStore()
async with factory() as session:
machine = await ready_machine(session)
controller = await link_controller(session, machine)
definition = await place_managed_agent(
session, owner_id="owner-a", controller_id=controller.id, name="helper"
)
- now = datetime.now(UTC)
- assert await store.has_agents(session, machine)
- assert not await store.retain_if_empty(
- session, machine, retention_days=7, now=now
- )
- assert (machine.desired_state, machine.revision) == ("running", 2)
- definition.controller_id = None
- assert await store.retain_if_empty(session, machine, retention_days=7, now=now)
- assert (
- machine.desired_state,
- machine.stop_reason,
- machine.revision,
- machine.retain_until,
- ) == ("retained", None, 3, now + timedelta(days=7))
await session.commit()
+ now = datetime.now(UTC)
+ workspaces = await workspaces_on(factory, machine.id)
+ assert counted(workspaces) == [(TENANT_ZERO_ID, controller.id, 1)]
+ assert not await store.retain_if_empty(
+ machine, workspaces, retention_days=7, now=now
+ )
+ assert (machine.desired_state, machine.revision) == ("running", 2)
+ async with factory() as session:
+ row = await session.get(AgentDefinition, definition.id)
+ row.controller_id = None
+ await session.commit()
+ workspaces = await workspaces_on(factory, machine.id)
+ assert counted(workspaces) == [(TENANT_ZERO_ID, controller.id, 0)]
+ assert await store.retain_if_empty(machine, workspaces, retention_days=7, now=now)
+ assert (
+ machine.desired_state,
+ machine.stop_reason,
+ machine.revision,
+ machine.retain_until,
+ ) == ("retained", None, 3, now + timedelta(days=7))
async def test_only_agents_on_the_machine_s_own_controller_count(factory):
- store = HostedMachineStore()
async with factory() as session:
machine = await ready_machine(session)
other = await seed_machine(
@@ -411,78 +457,94 @@ async def test_only_agents_on_the_machine_s_own_controller_count(factory):
await place_managed_agent(
session, owner_id="owner-b", controller_id=theirs.id, name="theirs"
)
- assert not await store.has_agents(session, machine)
- assert await store.managed_agent_ids(session, machine) == []
+ assert await managed_agent_ids(session, mine.id) == []
+ await session.commit()
+ assert counted(await workspaces_on(factory, machine.id)) == [
+ (TENANT_ZERO_ID, mine.id, 0)
+ ]
+ async with factory() as session:
placed = [
await place_managed_agent(
session, owner_id="owner-a", controller_id=mine.id, name=name
)
for name in ("reviewer", "builder")
]
- assert await store.has_agents(session, machine)
- assert await store.managed_agent_ids(session, machine) == sorted(
+ assert await managed_agent_ids(session, mine.id) == sorted(
definition.agent_id for definition in placed
)
+ await session.commit()
+ assert counted(await workspaces_on(factory, machine.id)) == [
+ (TENANT_ZERO_ID, mine.id, 2)
+ ]
async def test_a_machine_without_a_controller_has_no_agents(factory):
- store = HostedMachineStore()
async with factory() as session:
machine = await ready_machine(session)
- assert not await store.has_agents(session, machine)
- assert await store.managed_agent_ids(session, machine) == []
- assert not await store.ever_hosted(session, machine)
+ assert await managed_agent_ids(session, None) == []
+ await session.commit()
+ workspaces = await workspaces_on(factory, machine.id)
+ assert counted(workspaces) == [(TENANT_ZERO_ID, None, 0)]
+ assert not ever_hosted(workspaces)
async def test_release_if_empty_expires_a_machine_whose_controller_never_enrolled(
factory,
):
- store = HostedMachineStore()
+ store = CloudMachineStore()
async with factory() as session:
machine = await ready_machine(session)
- now = datetime.now(UTC)
- assert await store.release_if_empty(session, machine, retention_days=7, now=now)
- assert (machine.desired_state, machine.revision, machine.retain_until) == (
- "retained",
- 3,
- now,
- )
+ await session.commit()
+ now = datetime.now(UTC)
+ assert await store.release_if_empty(
+ machine, await workspaces_on(factory, machine.id), retention_days=7, now=now
+ )
+ assert (machine.desired_state, machine.revision, machine.retain_until) == (
+ "retained",
+ 3,
+ now,
+ )
async def test_release_if_empty_keeps_the_disk_of_a_machine_that_enrolled(factory):
- store = HostedMachineStore()
+ store = CloudMachineStore()
async with factory() as session:
machine = await ready_machine(session)
await link_controller(session, machine)
- assert await store.ever_hosted(session, machine)
- now = datetime.now(UTC)
- assert await store.release_if_empty(session, machine, retention_days=7, now=now)
- assert (machine.desired_state, machine.retain_until) == (
- "retained",
- now + timedelta(days=7),
- )
+ await session.commit()
+ workspaces = await workspaces_on(factory, machine.id)
+ assert ever_hosted(workspaces)
+ now = datetime.now(UTC)
+ assert await store.release_if_empty(machine, workspaces, retention_days=7, now=now)
+ assert (machine.desired_state, machine.retain_until) == (
+ "retained",
+ now + timedelta(days=7),
+ )
async def test_release_if_empty_leaves_a_machine_with_agents(factory):
- store = HostedMachineStore()
+ store = CloudMachineStore()
async with factory() as session:
machine = await ready_machine(session)
controller = await link_controller(session, machine)
await place_managed_agent(
session, owner_id="owner-a", controller_id=controller.id, name="helper"
)
- assert not await store.release_if_empty(
- session, machine, retention_days=7, now=datetime.now(UTC)
- )
- assert (machine.desired_state, machine.revision, machine.retain_until) == (
- "running",
- 2,
- None,
- )
+ await session.commit()
+ assert not await store.release_if_empty(
+ machine,
+ await workspaces_on(factory, machine.id),
+ retention_days=7,
+ now=datetime.now(UTC),
+ )
+ assert (machine.desired_state, machine.revision, machine.retain_until) == (
+ "running",
+ 2,
+ None,
+ )
async def test_removing_the_agent_takes_it_off_the_machine(factory):
- store = HostedMachineStore()
async with factory() as session:
machine = await ready_machine(session)
controller = await link_controller(session, machine)
@@ -494,15 +556,13 @@ async def test_removing_the_agent_takes_it_off_the_machine(factory):
row = await session.get(AgentDefinition, definition.id)
await session.delete(row)
await session.commit()
- async with factory() as session:
- machine = await store.get(session, machine.id)
- assert machine is not None
- assert not await store.has_agents(session, machine)
- assert await store.ever_hosted(session, machine)
+ workspaces = await workspaces_on(factory, machine.id)
+ assert counted(workspaces) == [(TENANT_ZERO_ID, controller.id, 0)]
+ assert ever_hosted(workspaces)
async def test_owned_and_live_for_owner(factory):
- store = HostedMachineStore()
+ store = CloudMachineStore()
async with factory() as session:
machine = await ready_machine(session)
await seed_machine(
@@ -521,4 +581,182 @@ async def test_owned_and_live_for_owner(factory):
assert await store.live_for_owner(session, "owner-b") is None
assert (await store.locked(session, machine.id)).id == machine.id
assert await store.locked(session, "missing") is None
- assert await session.get(HostedMachine, (require_tenant_id(), machine.id))
+ assert await session.get(CloudMachine, machine.id)
+
+
+async def test_a_machine_is_its_owner_s_in_every_workspace(factory):
+ """The machine is global: from a workspace it does not serve, it is still
+ the owner's, and that workspace has no row on it."""
+ store = CloudMachineStore()
+ async with factory() as session:
+ machine = await ready_machine(session)
+ await session.commit()
+ with tenant_scope(TENANT_B):
+ async with factory() as session:
+ assert (await store.live_for_owner(session, "owner-a")).id == machine.id
+ assert (await store.owned(session, machine.id, "owner-a")).id == machine.id
+ assert await workspace_on(session, machine.id) is None
+ assert counted(await workspaces_on(factory, machine.id)) == [
+ (TENANT_ZERO_ID, None, 0)
+ ]
+
+
+async def test_joining_from_another_workspace_reuses_the_machine(factory):
+ first, here = await claim_both(factory, "owner-a")
+ observed = datetime.now(UTC)
+ async with factory() as session:
+ row = await CloudMachineStore().get(session, first.id)
+ row.state = "ready"
+ row.running_observed_at = observed
+ await session.commit()
+
+ machine, there = await claim_in(factory, TENANT_B, "owner-a")
+ assert machine.id == first.id
+ assert there.id != here.id
+ assert (
+ there.tenant_id,
+ there.machine_id,
+ there.owner_id,
+ there.controller_id,
+ ) == (TENANT_B, machine.id, "owner-a", None)
+ # Joined, it is prepared again with a controller for the new workspace.
+ assert (
+ machine.state,
+ machine.desired_state,
+ machine.revision,
+ machine.running_observed_at,
+ ) == ("provisioning", "running", 2, None)
+ assert await tenants_of_cloud_machine(factory, machine.id) == [
+ TENANT_ZERO_ID,
+ TENANT_B,
+ ]
+ assert [workspace.id for workspace in await workspaces_on(factory, machine.id)] == [
+ here.id,
+ there.id,
+ ]
+
+ again, same = await claim_in(factory, TENANT_B, "owner-a")
+ assert (again.id, same.id, again.revision) == (machine.id, there.id, 2)
+ back, home = await claim_both(factory, "owner-a")
+ assert (back.id, home.id, back.revision) == (machine.id, here.id, 2)
+
+
+ ("state", "desired_state", "stop_reason"),
+ [
+ ("stopped", "stopped", "idle"),
+ ("stopped", "stopped", "owner"),
+ ("retained", "retained", None),
+ ],
+)
+async def test_joining_a_machine_at_rest_starts_it_once(
+ factory, state, desired_state, stop_reason
+):
+ seeded = await seed(
+ factory,
+ owner_id="owner-a",
+ state=state,
+ desired_state=desired_state,
+ stop_reason=stop_reason,
+ revision=3,
+ )
+ if desired_state == "retained":
+ async with factory() as session:
+ row = await CloudMachineStore().get(session, seeded.id)
+ row.retain_until = datetime.now(UTC) + timedelta(days=3)
+ await session.commit()
+ machine, workspace = await claim_in(factory, TENANT_B, "owner-a")
+ assert (
+ machine.id,
+ machine.desired_state,
+ machine.stop_reason,
+ machine.retain_until,
+ machine.revision,
+ ) == (seeded.id, "running", None, None, 4)
+ assert workspace.tenant_id == TENANT_B
+
+
+async def test_capacity_counts_machines_in_every_workspace(factory):
+ await claim_both(factory, "owner-a", capacity=1)
+ with pytest.raises(CloudMachineConflict) as raised:
+ await claim_in(factory, TENANT_B, "owner-b", capacity=1)
+ assert str(raised.value) == MACHINES_FULL
+ # Joining takes no machine of its own.
+ machine, _workspace = await claim_in(factory, TENANT_B, "owner-a", capacity=1)
+ assert machine.owner_id == "owner-a"
+
+
+async def test_a_machine_serves_at_most_eight_workspaces(factory):
+ machine = await claim(factory, "owner-a")
+ joined, refused = OTHER_WORKSPACES[:-1], OTHER_WORKSPACES[-1]
+ for tenant_id in joined:
+ await claim_in(factory, tenant_id, "owner-a")
+ assert len(await tenants_of_cloud_machine(factory, machine.id)) == (
+ MAX_MACHINE_WORKSPACES
+ )
+ with pytest.raises(CloudMachineConflict) as raised:
+ await claim_in(factory, refused, "owner-a")
+ assert str(raised.value) == MACHINE_WORKSPACES_FULL
+ assert refused not in await tenants_of_cloud_machine(factory, machine.id)
+ # A workspace it already serves still has it.
+ again, _workspace = await claim_in(factory, joined[0], "owner-a")
+ assert again.id == machine.id
+
+
+async def test_a_machine_is_empty_only_once_no_workspace_has_an_agent(factory):
+ store = CloudMachineStore()
+ async with factory() as session:
+ machine = await ready_machine(session)
+ mine = await link_controller(session, machine)
+ here = await place_managed_agent(
+ session, owner_id="owner-a", controller_id=mine.id, name="here"
+ )
+ await session.commit()
+ with tenant_scope(TENANT_B):
+ async with factory() as session:
+ await add_workspace(
+ session, machine, created_at=datetime.now(UTC) + timedelta(seconds=1)
+ )
+ theirs = await link_controller(session, machine)
+ there = [
+ await place_managed_agent(
+ session, owner_id="owner-a", controller_id=theirs.id, name=name
+ )
+ for name in ("there-1", "there-2")
+ ]
+ await session.commit()
+ workspaces = await workspaces_on(factory, machine.id)
+ assert counted(workspaces) == [
+ (TENANT_ZERO_ID, mine.id, 1),
+ (TENANT_B, theirs.id, 2),
+ ]
+ now = datetime.now(UTC)
+ assert not await store.retain_if_empty(
+ machine, workspaces, retention_days=7, now=now
+ )
+
+ async with factory() as session:
+ await session.delete(await session.get(AgentDefinition, here.id))
+ await session.commit()
+ workspaces = await workspaces_on(factory, machine.id)
+ assert counted(workspaces) == [
+ (TENANT_ZERO_ID, mine.id, 0),
+ (TENANT_B, theirs.id, 2),
+ ]
+ assert not await store.retain_if_empty(
+ machine, workspaces, retention_days=7, now=now
+ )
+ assert machine.desired_state == "running"
+
+ with tenant_scope(TENANT_B):
+ async with factory() as session:
+ for definition in there:
+ await session.delete(await session.get(AgentDefinition, definition.id))
+ await session.commit()
+ workspaces = await workspaces_on(factory, machine.id)
+ assert ever_hosted(workspaces)
+ assert await store.retain_if_empty(machine, workspaces, retention_days=7, now=now)
+ assert (machine.desired_state, machine.retain_until) == (
+ "retained",
+ now + timedelta(days=7),
+ )
diff --git a/core/tests/switch_core/db/test_tenant_exemption_allowlist.py b/core/tests/switch_core/db/test_tenant_exemption_allowlist.py
index baa13508f..5020bd8d1 100644
--- a/core/tests/switch_core/db/test_tenant_exemption_allowlist.py
+++ b/core/tests/switch_core/db/test_tenant_exemption_allowlist.py
@@ -84,6 +84,11 @@ _ALLOWED_MODULES = {
# is reachable by any endpoint that declares it, which would make this
# list name one module while the exemption was open to every route.
"switch_core.gateway.auth",
+ # A user's cloud machine is global and the cloud controller acts for no
+ # tenant: which workspaces a machine serves comes from
+ # `tenants_of_cloud_machine`, and each one's row on the machine is then
+ # read with that workspace bound.
+ "switch_core.db.stores.hosted_machine_store",
# Enumerating tenants at boot, and starting one bridge or one connector by
# id from a context bound to somebody else's tenant.
"switch_core.clients.client_lifecycle_service",
diff --git a/core/tests/switch_core/db/test_tenant_lookup.py b/core/tests/switch_core/db/test_tenant_lookup.py
index f1e180d7b..8d3934d4b 100644
--- a/core/tests/switch_core/db/test_tenant_lookup.py
+++ b/core/tests/switch_core/db/test_tenant_lookup.py
@@ -103,6 +103,7 @@ _DROPPED_SINCE = {"tenant_of_client": "b1d7c4f0a92e"}
_ADDED_SINCE = {
"tenant_of_invitation": "5daaea6b674d",
"tenant_of_messaging_install": "c8a4e21f6d30",
+ "tenants_of_cloud_machine": "b6e1c9d4a7f2",
"tenants_inviting_email": "4b8e2d61c9f7",
"tenants_open_to_domain": "7d3f5a19e2c8",
}
diff --git a/core/tests/switch_core/db/test_tenant_schema_catalogue.py b/core/tests/switch_core/db/test_tenant_schema_catalogue.py
index 1485b14e8..a74bf5557 100644
--- a/core/tests/switch_core/db/test_tenant_schema_catalogue.py
+++ b/core/tests/switch_core/db/test_tenant_schema_catalogue.py
@@ -93,10 +93,13 @@ class TestEveryTableIsScopedUnlessItIsNamedGlobal:
`users` is a person rather than a tenant member, `oidc_identities`
records how that person proves who they are, and `feature_flags` is a
deployment switch — a flag that has to vary per customer is a new
- scoped table, not a nullable column there. `alembic_version` is
+ scoped table, not a nullable column there. `cloud_machines` is a
+ person's VM, one per person whatever workspaces it serves; what it
+ runs in each is the scoped `machine_workspaces`. `alembic_version` is
global too but is not in this metadata: Alembic owns it.
"""
assert set(GLOBAL_TABLES) == {
+ "cloud_machines",
"users",
"oidc_identities",
"feature_flags",
diff --git a/core/tests/switch_core/fixtures/hosted_machines/prepare_controller_response.json b/core/tests/switch_core/fixtures/hosted_machines/prepare_controller_response.json
index e4034c590..1faac05b2 100644
--- a/core/tests/switch_core/fixtures/hosted_machines/prepare_controller_response.json
+++ b/core/tests/switch_core/fixtures/hosted_machines/prepare_controller_response.json
@@ -3,8 +3,11 @@
"revision": 1,
"bundle_revision": 1,
"api_endpoint": "https://switch.example.test/agent-api",
- "controller": {
- "id": null,
- "enrollment_code": "swce_SyntheticEnrollmentCode0000"
- }
+ "controllers": [
+ {
+ "key": "5e1c2b4a-0000-4000-8000-0000000000a1",
+ "id": null,
+ "enrollment_code": "swce_SyntheticEnrollmentCode0000"
+ }
+ ]
}
diff --git a/core/tests/switch_core/gateway/test_cloud_controller_machines.py b/core/tests/switch_core/gateway/test_cloud_controller_machines.py
index a30c82db9..f9c6c5a73 100644
--- a/core/tests/switch_core/gateway/test_cloud_controller_machines.py
+++ b/core/tests/switch_core/gateway/test_cloud_controller_machines.py
@@ -18,18 +18,26 @@ from types import SimpleNamespace
import httpx
import pytest
from fastapi import FastAPI
+from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.db.models import (
+ TENANT_ZERO_ID,
AgentController,
- HostedMachine,
+ AgentControllerEnrollmentCode,
+ CloudMachine,
+ MachineWorkspace,
+ TenantMember,
User,
require_tenant_id,
)
from switch_core.db.stores.hosted_machine_store import (
- HostedMachineStore,
+ CloudMachineStore,
bump_revision,
+ ever_hosted,
lock_claims,
+ workspace_on,
+ workspaces_on,
)
from switch_core.gateway.auth import get_current_user
from switch_core.gateway.cloud_controllers import (
@@ -46,6 +54,8 @@ from switch_core.gateway.hosted_machines import router as machines_router
from switch_core.management.placement import placement_refusal
from switch_core.management.process_lease import ProcessLeases
from switch_core.providers.hosted import HostedControllerSettings
+from switch_core.tenant_context import tenant_scope
+from tests.switch_core.hosted_machine_helpers import add_tenant
from tests.switch_core.management.harness import (
TEST_KEYRING,
Harness,
@@ -82,7 +92,7 @@ def _hosted_app(
) -> httpx.AsyncClient:
app = FastAPI()
app.state.hosted_controller_settings = HostedControllerSettings(
- tenant_id=require_tenant_id(),
+ allowed_tenant_ids=[require_tenant_id()],
token=TOKEN,
agent_api_endpoint=API_ENDPOINT,
)
@@ -113,11 +123,12 @@ def _hosted_app(
async def _claim(
session_factory: async_sessionmaker[AsyncSession], owner: User
-) -> HostedMachine:
+) -> CloudMachine:
async with session_factory() as session:
await lock_claims(session)
- machine = await HostedMachineStore().claim(
+ machine, _workspace = await CloudMachineStore().claim(
session,
+ session_factory,
owner_id=owner.id,
capacity=1,
now=datetime.now(UTC),
@@ -128,18 +139,38 @@ async def _claim(
async def _machine(
session_factory: async_sessionmaker[AsyncSession], machine_id: str
-) -> HostedMachine:
+) -> CloudMachine:
async with session_factory() as session:
- machine = await session.get(HostedMachine, (require_tenant_id(), machine_id))
+ machine = await session.get(CloudMachine, machine_id)
assert machine is not None
return machine
+async def _workspace(
+ session_factory: async_sessionmaker[AsyncSession], machine_id: str
+) -> MachineWorkspace:
+ """The bound workspace's row on the machine."""
+ async with session_factory() as session:
+ workspace = await workspace_on(session, machine_id)
+ assert workspace is not None
+ return workspace
+
+
+def _heartbeat(workspace_id: str, at: datetime, sessions: int) -> dict:
+ return {
+ "disk": None,
+ "memory": None,
+ "controllers": {
+ workspace_id: {"at": at.isoformat(), "sessions_running": sessions}
+ },
+ }
+
+
async def _update(
session_factory: async_sessionmaker[AsyncSession], machine_id: str, **values
) -> None:
async with session_factory() as session:
- machine = await HostedMachineStore().locked(session, machine_id)
+ machine = await CloudMachineStore().locked(session, machine_id)
assert machine is not None
for key, value in values.items():
setattr(machine, key, value)
@@ -150,7 +181,7 @@ async def _new_revision(
session_factory: async_sessionmaker[AsyncSession], machine_id: str
) -> None:
async with session_factory() as session:
- machine = await HostedMachineStore().locked(session, machine_id)
+ machine = await CloudMachineStore().locked(session, machine_id)
assert machine is not None
bump_revision(machine, datetime.now(UTC))
await session.commit()
@@ -213,18 +244,23 @@ class TestControllerMachines:
retried = await _prepare(hosted, machine.id)
assert first["bundle_revision"] == first["revision"]
assert first["api_endpoint"] == API_ENDPOINT
- code = first["controller"]["enrollment_code"]
+ [entry] = first["controllers"]
+ code = entry["enrollment_code"]
assert code.startswith("swce_")
- assert first["controller"]["id"] is None
+ assert entry["id"] is None
+ assert (
+ entry["key"]
+ == (await _workspace(harness.session_factory, machine.id)).id
+ )
# A retry of the same revision hands over the same code.
- assert retried["controller"] == first["controller"]
+ assert retried["controllers"] == first["controllers"]
assert "machine_capability" not in first
# The shape the hosted controller is tested against.
fixture = json.loads(
(FIXTURES / "prepare_controller_response.json").read_text()
)
assert set(first) == set(fixture)
- assert set(first["controller"]) == set(fixture["controller"])
+ assert set(entry) == set(fixture["controllers"][0])
async with harness.client() as client:
enrolled = await _enroll(client, code)
@@ -249,15 +285,87 @@ class TestControllerMachines:
assert controller.kind == "ec2"
assert controller.name == "Switch cloud"
assert (
- await _machine(harness.session_factory, machine.id)
+ await _workspace(harness.session_factory, machine.id)
).controller_id == (enrolled["controller_id"])
await _new_revision(harness.session_factory, machine.id)
enrolled_since = await _prepare(hosted, machine.id)
- assert enrolled_since["controller"] == {
- "id": enrolled["controller_id"],
- "enrollment_code": None,
- }
+ assert enrolled_since["controllers"] == [
+ {
+ "key": entry["key"],
+ "id": enrolled["controller_id"],
+ "enrollment_code": None,
+ }
+ ]
+
+ async def test_a_machine_serving_two_workspaces_enrolls_a_controller_in_each(
+ self, harness: Harness
+ ) -> None:
+ factory = harness.session_factory
+ owner = await add_member(factory, "ada")
+ async with factory() as session:
+ await add_tenant(session, "tenant-b")
+ await session.commit()
+ with tenant_scope("tenant-b"):
+ async with factory() as session:
+ session.add(
+ TenantMember(tenant_id="tenant-b", user_id=owner.id, role="member")
+ )
+ await session.commit()
+ machine = await _claim(factory, owner)
+ with tenant_scope("tenant-b"):
+ joined = await _claim(factory, owner)
+ there = await _workspace(factory, machine.id)
+ assert joined.id == machine.id
+ here = await _workspace(factory, machine.id)
+ async with _hosted_app(factory, owner) as hosted:
+ prepared = await _prepare(hosted, machine.id)
+ assert prepared["revision"] == 2
+ first, second = prepared["controllers"]
+ assert (first["key"], second["key"]) == (here.id, there.id)
+ assert first["id"] is None and second["id"] is None
+ assert first["enrollment_code"] != second["enrollment_code"]
+ # Each code is minted in the workspace its controller enrolls in.
+ for tenant_id, workspace in ((TENANT_ZERO_ID, here), ("tenant-b", there)):
+ with tenant_scope(tenant_id):
+ async with factory() as session:
+ codes = list(
+ await session.scalars(
+ select(AgentControllerEnrollmentCode).where(
+ AgentControllerEnrollmentCode.machine_workspace_id
+ == workspace.id
+ )
+ )
+ )
+ assert [(code.tenant_id, code.owner_id) for code in codes] == [
+ (tenant_id, owner.id)
+ ]
+
+ async with harness.client() as client:
+ enrolled = await _enroll(client, second["enrollment_code"])
+ assert (await _workspace(factory, machine.id)).controller_id is None
+ with tenant_scope("tenant-b"):
+ assert (await _workspace(factory, machine.id)).controller_id == (
+ enrolled["controller_id"]
+ )
+ async with factory() as session:
+ controller = await session.get(
+ AgentController, enrolled["controller_id"]
+ )
+ assert controller is not None
+ assert (controller.tenant_id, controller.kind) == (
+ "tenant-b",
+ "ec2",
+ )
+ again = await _prepare(hosted, machine.id)
+ assert again["controllers"] == [
+ first,
+ {
+ "key": there.id,
+ "id": enrolled["controller_id"],
+ "enrollment_code": None,
+ },
+ ]
async def test_its_status_reports_make_it_ready_and_its_agents_are_the_machines(
self, harness: Harness
@@ -265,7 +373,9 @@ class TestControllerMachines:
owner = await add_member(harness.session_factory, "ada")
machine = await _claim(harness.session_factory, owner)
async with _hosted_app(harness.session_factory, owner) as hosted:
- code = (await _prepare(hosted, machine.id))["controller"]["enrollment_code"]
+ code = (await _prepare(hosted, machine.id))["controllers"][0][
+ "enrollment_code"
+ ]
async with harness.client() as client:
enrolled = await _enroll(client, code)
token = await client.post(
@@ -315,7 +425,9 @@ class TestControllerMachines:
owner = await add_member(harness.session_factory, "ada")
machine = await _claim(harness.session_factory, owner)
async with _hosted_app(harness.session_factory, owner) as hosted:
- code = (await _prepare(hosted, machine.id))["controller"]["enrollment_code"]
+ code = (await _prepare(hosted, machine.id))["controllers"][0][
+ "enrollment_code"
+ ]
async with harness.client() as client:
first = await _enroll(client, code)
await _connected(client, harness, owner, first)
@@ -329,9 +441,9 @@ class TestControllerMachines:
)
assert revoked.status_code == 200, revoked.text
await _new_revision(harness.session_factory, machine.id)
- prepared = await _prepare(hosted, machine.id)
- assert prepared["controller"]["id"] is None
- second_code = prepared["controller"]["enrollment_code"]
+ [prepared] = (await _prepare(hosted, machine.id))["controllers"]
+ assert prepared["id"] is None
+ second_code = prepared["enrollment_code"]
assert second_code not in {None, code}
second = await _enroll(client, second_code)
agent = await client.get(
@@ -341,7 +453,7 @@ class TestControllerMachines:
assert agent.status_code == 200, agent.text
assert agent.json()["controller_id"] == second["controller_id"]
assert (
- await _machine(harness.session_factory, machine.id)
+ await _workspace(harness.session_factory, machine.id)
).controller_id == (second["controller_id"])
summary = await hosted.get(f"/hosted-machines/{machine.id}")
assert summary.json()["agents"] == [created.json()["agent_id"]]
@@ -352,7 +464,9 @@ class TestControllerMachines:
owner = await add_member(harness.session_factory, "ada")
machine = await _claim(harness.session_factory, owner)
async with _hosted_app(harness.session_factory, owner) as hosted:
- code = (await _prepare(hosted, machine.id))["controller"]["enrollment_code"]
+ code = (await _prepare(hosted, machine.id))["controllers"][0][
+ "enrollment_code"
+ ]
async with harness.client() as client:
enrolled = await _enroll(client, code)
await _connected(client, harness, owner, enrolled)
@@ -364,15 +478,16 @@ class TestControllerMachines:
desired_state="stopped",
)
assert created.status_code == 201, created.text
- store = HostedMachineStore()
+ store = CloudMachineStore()
async with harness.session_factory() as session:
locked = await store.locked(session, machine.id)
assert locked is not None
- assert await store.has_agents(session, locked)
+ workspaces = await workspaces_on(harness.session_factory, machine.id)
+ assert [workspace.agent_count for workspace in workspaces] == [1]
assert not await store.retain_if_empty(
- session, locked, retention_days=7, now=datetime.now(UTC)
+ locked, workspaces, retention_days=7, now=datetime.now(UTC)
)
- assert await store.ever_hosted(session, locked)
+ assert ever_hosted(workspaces)
await session.commit()
async with harness.client() as client:
deleted = await client.delete(
@@ -383,10 +498,11 @@ class TestControllerMachines:
async with harness.session_factory() as session:
locked = await store.locked(session, machine.id)
assert locked is not None
- assert not await store.has_agents(session, locked)
+ workspaces = await workspaces_on(harness.session_factory, machine.id)
+ assert [workspace.agent_count for workspace in workspaces] == [0]
now = datetime.now(UTC)
assert await store.retain_if_empty(
- session, locked, retention_days=7, now=now
+ locked, workspaces, retention_days=7, now=now
)
assert (locked.desired_state, locked.retain_until) == (
"retained",
@@ -407,13 +523,14 @@ class TestControllerMachines:
) -> None:
owner = await add_member(harness.session_factory, "ada")
machine = await _claim(harness.session_factory, owner)
+ workspace = await _workspace(harness.session_factory, machine.id)
now = datetime.now(UTC)
await _update(
harness.session_factory,
machine.id,
state="ready",
active_at=now - timedelta(hours=2),
- heartbeat={"disk": None, "memory": None, "sessions_running": sessions},
+ heartbeat=_heartbeat(workspace.id, now - heartbeat_age, sessions),
heartbeat_at=now - heartbeat_age,
)
async with _hosted_app(harness.session_factory, owner) as hosted:
@@ -432,7 +549,9 @@ class TestControllerMachines:
owner = await add_member(harness.session_factory, "ada")
machine = await _claim(harness.session_factory, owner)
async with _hosted_app(harness.session_factory, owner) as hosted:
- code = (await _prepare(hosted, machine.id))["controller"]["enrollment_code"]
+ code = (await _prepare(hosted, machine.id))["controllers"][0][
+ "enrollment_code"
+ ]
async with harness.client() as client:
enrolled = await _enroll(client, code)
await _connected(client, harness, owner, enrolled)
@@ -444,12 +563,13 @@ class TestControllerMachines:
)
assert created.status_code == 201, created.text
now = datetime.now(UTC)
+ workspace = await _workspace(harness.session_factory, machine.id)
await _update(
harness.session_factory,
machine.id,
state="ready",
active_at=now - timedelta(hours=2),
- heartbeat={"disk": None, "memory": None, "sessions_running": 0},
+ heartbeat=_heartbeat(workspace.id, now, 0),
heartbeat_at=now,
)
listed = await hosted.get("/hosted-controller/machines", headers=HEADERS)
@@ -463,7 +583,9 @@ class TestControllerMachines:
owner = await add_member(harness.session_factory, "ada")
machine = await _claim(harness.session_factory, owner)
async with _hosted_app(harness.session_factory, owner) as hosted:
- code = (await _prepare(hosted, machine.id))["controller"]["enrollment_code"]
+ code = (await _prepare(hosted, machine.id))["controllers"][0][
+ "enrollment_code"
+ ]
async with harness.client() as client:
enrolled = await _enroll(client, code)
await _update(
diff --git a/core/tests/switch_core/gateway/test_hosted_controller.py b/core/tests/switch_core/gateway/test_hosted_controller.py
index ffcf8d035..df466ac14 100644
--- a/core/tests/switch_core/gateway/test_hosted_controller.py
+++ b/core/tests/switch_core/gateway/test_hosted_controller.py
@@ -12,18 +12,28 @@ from uuid import uuid4
import httpx
import pytest
from fastapi import FastAPI
+from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.db.models import (
+ TENANT_ZERO_ID,
AgentController,
- HostedMachine,
+ AgentDefinition,
+ CloudMachine,
+ MachineWorkspace,
TenantMember,
User,
require_tenant_id,
)
-from switch_core.db.stores.hosted_machine_store import HostedMachineStore, lock_claims
+from switch_core.db.stores.hosted_machine_store import (
+ CloudMachineStore,
+ lock_claims,
+ workspace_on,
+)
from switch_core.gateway.auth import get_current_user
from switch_core.gateway.cloud_controllers import (
+ controller_machine_idle,
+ controller_reports,
record_controller_status,
set_cloud_enrollment,
)
@@ -36,7 +46,10 @@ from switch_core.gateway.hosted_controller import router
from switch_core.gateway.hosted_machines import router as machines_router
from switch_core.keys import Keyring
from switch_core.providers.hosted import HostedControllerSettings
+from switch_core.tenant_context import tenant_scope
from tests.switch_core.hosted_machine_helpers import (
+ add_tenant,
+ add_workspace,
link_controller,
place_managed_agent,
seed_machine,
@@ -63,15 +76,16 @@ def fixture(name: str) -> dict:
class FakeEnrollment:
- """Agent management's side of enrollment: mints a fresh code per call."""
+ """Agent management's side of enrollment: mints a fresh code per call,
+ noting the workspace it was minted in and the workspace row it is for."""
def __init__(self) -> None:
- self.minted: list[tuple[str, int]] = []
+ self.minted: list[tuple[str, str]] = []
async def machine_enrollment_code(
- self, session: AsyncSession, machine: HostedMachine, now: datetime
+ self, session: AsyncSession, workspace: MachineWorkspace, now: datetime
) -> str:
- self.minted.append((machine.id, machine.revision))
+ self.minted.append((require_tenant_id(), workspace.id))
return f"swce_synthetic-{len(self.minted)}"
@@ -80,6 +94,8 @@ class ControllerApp:
client: httpx.AsyncClient
factory: async_sessionmaker[AsyncSession]
machine_id: str
+ #: The machine's row in the bound workspace (tenant zero).
+ workspace_id: str
owner_id: str
config: SimpleNamespace
settings: HostedControllerSettings
@@ -107,9 +123,11 @@ async def controller_app(session_factory):
stop_reason=None,
revision=1,
)
+ workspace = await workspace_on(session, machine.id)
+ assert workspace is not None
await session.commit()
settings = HostedControllerSettings(
- tenant_id=require_tenant_id(),
+ allowed_tenant_ids=[require_tenant_id()],
token=TOKEN,
agent_api_endpoint="https://switch.example.com/api/agent",
)
@@ -146,6 +164,7 @@ async def controller_app(session_factory):
client=client,
factory=session_factory,
machine_id=machine.id,
+ workspace_id=workspace.id,
owner_id=owner.id,
config=config,
settings=settings,
@@ -155,16 +174,24 @@ async def controller_app(session_factory):
set_cloud_enrollment(None)
-async def machine_of(factory, machine_id: str) -> HostedMachine:
+async def machine_of(factory, machine_id: str) -> CloudMachine:
async with factory() as session:
- machine = await session.get(HostedMachine, (require_tenant_id(), machine_id))
+ machine = await session.get(CloudMachine, machine_id)
assert machine is not None
return machine
+async def workspace_of(factory, machine_id: str) -> MachineWorkspace:
+ """The bound workspace's row on the machine."""
+ async with factory() as session:
+ workspace = await workspace_on(session, machine_id)
+ assert workspace is not None
+ return workspace
+
+
async def update_machine(factory, machine_id: str, **values) -> None:
async with factory() as session:
- machine = await session.get(HostedMachine, (require_tenant_id(), machine_id))
+ machine = await session.get(CloudMachine, machine_id)
assert machine is not None
for key, value in values.items():
setattr(machine, key, value)
@@ -172,12 +199,11 @@ async def update_machine(factory, machine_id: str, **values) -> None:
async def enrolled(app: ControllerApp, *, agents: int) -> tuple[str, list[str]]:
- """Link the machine to the controller it enrolled as, with `agents`
- managed agents placed on it; the controller and agent ids."""
+ """Link the machine to the controller it enrolled as in the bound
+ workspace, with `agents` managed agents placed on it; the controller and
+ agent ids."""
async with app.factory() as session:
- machine = await session.get(
- HostedMachine, (require_tenant_id(), app.machine_id)
- )
+ machine = await session.get(CloudMachine, app.machine_id)
assert machine is not None
controller = await link_controller(session, machine)
placed = [
@@ -194,12 +220,65 @@ async def enrolled(app: ControllerApp, *, agents: int) -> tuple[str, list[str]]:
async def report_status(app: ControllerApp, controller_id: str, **reading) -> None:
- """A status report from the machine's controller."""
+ """A status report from the machine's controller in the bound workspace."""
async with app.factory() as session:
await record_controller_status(session, controller_id, reading)
await session.commit()
+@dataclass
+class Joined:
+ tenant_id: str
+ workspace_id: str
+ controller_id: str | None
+ agent_ids: list[str]
+
+
+async def join_workspace(
+ app: ControllerApp, tenant_id: str, *, agents: int | None, member: bool
+) -> Joined:
+ """Another workspace on the owner's machine, joined after the first: the
+ owner a member there when `member`, its controller enrolled with `agents`
+ placed on it unless `agents` is None."""
+ async with app.factory() as session:
+ await add_tenant(session, tenant_id)
+ await session.commit()
+ with tenant_scope(tenant_id):
+ async with app.factory() as session:
+ if member:
+ session.add(
+ TenantMember(
+ tenant_id=tenant_id, user_id=app.owner_id, role="member"
+ )
+ )
+ machine = await session.get(CloudMachine, app.machine_id)
+ assert machine is not None
+ workspace = await add_workspace(
+ session, machine, created_at=datetime.now(UTC) + timedelta(seconds=1)
+ )
+ controller_id = None
+ placed = []
+ if agents is not None:
+ controller = await link_controller(session, machine)
+ controller_id = controller.id
+ placed = [
+ await place_managed_agent(
+ session,
+ owner_id=app.owner_id,
+ controller_id=controller.id,
+ name=f"{tenant_id}-agent-{index}",
+ )
+ for index in range(agents)
+ ]
+ await session.commit()
+ return Joined(
+ tenant_id=tenant_id,
+ workspace_id=workspace.id,
+ controller_id=controller_id,
+ agent_ids=[definition.agent_id for definition in placed],
+ )
+
+
async def list_machines(client) -> list[dict]:
response = await client.get("/hosted-controller/machines", headers=HEADERS)
assert response.status_code == 200, response.text
@@ -248,7 +327,10 @@ async def test_controller_requires_its_own_credential(controller_app):
)
assert response.status_code == 401
saved = await machine_of(app.factory, app.machine_id)
- assert (saved.state, saved.enrollment_code_revision) == ("queued", None)
+ assert saved.state == "queued"
+ assert (
+ await workspace_of(app.factory, app.machine_id)
+ ).enrollment_code_revision is None
assert app.enrollment.minted == []
@@ -289,7 +371,7 @@ async def test_machines_lists_every_live_machine_with_the_contract_keys(
"revision": 1,
"data_volume_id": None,
"retain_until": None,
- "bundle_revision": None,
+ "bundle_revision": 1,
}
@@ -433,16 +515,23 @@ async def test_prepare_hands_over_one_enrollment_code_per_revision(controller_ap
body = first.json()
contract = fixture("prepare_controller_response.json")
assert set(body) == set(contract)
- assert set(body["controller"]) == set(contract["controller"])
+ [entry] = body["controllers"]
+ assert set(entry) == set(contract["controllers"][0])
assert body["machine_id"] == app.machine_id
assert body["revision"] == body["bundle_revision"] == 1
assert body["api_endpoint"] == app.settings.agent_api_endpoint
- assert body["controller"] == {"id": None, "enrollment_code": "swce_synthetic-1"}
- assert app.enrollment.minted == [(app.machine_id, 1)]
+ assert entry == {
+ "key": app.workspace_id,
+ "id": None,
+ "enrollment_code": "swce_synthetic-1",
+ }
+ minted = (require_tenant_id(), app.workspace_id)
+ assert app.enrollment.minted == [minted]
saved = await machine_of(app.factory, app.machine_id)
assert saved.state == "provisioning"
- assert saved.enrollment_code_revision == 1
- assert "swce_synthetic-1" not in (saved.enrollment_code_encrypted or "")
+ workspace = await workspace_of(app.factory, app.machine_id)
+ assert workspace.enrollment_code_revision == 1
+ assert "swce_synthetic-1" not in (workspace.enrollment_code_encrypted or "")
[item] = await list_machines(app.client)
assert item["bundle_revision"] == 1
assert "swce_" not in json.dumps(item)
@@ -450,9 +539,13 @@ async def test_prepare_hands_over_one_enrollment_code_per_revision(controller_ap
await update_machine(app.factory, app.machine_id, revision=2)
rotated = (await prepare(app.client, app.machine_id)).json()
assert rotated["revision"] == rotated["bundle_revision"] == 2
- assert rotated["controller"] == {"id": None, "enrollment_code": "swce_synthetic-2"}
- assert app.enrollment.minted == [(app.machine_id, 1), (app.machine_id, 2)]
- assert (await machine_of(app.factory, app.machine_id)).enrollment_code_revision == 2
+ assert rotated["controllers"] == [
+ {"key": app.workspace_id, "id": None, "enrollment_code": "swce_synthetic-2"}
+ ]
+ assert app.enrollment.minted == [minted, minted]
+ assert (
+ await workspace_of(app.factory, app.machine_id)
+ ).enrollment_code_revision == 2
async def test_prepare_names_the_controller_the_machine_enrolled_as(controller_app):
@@ -460,10 +553,9 @@ async def test_prepare_names_the_controller_the_machine_enrolled_as(controller_a
controller_id, _ = await enrolled(app, agents=0)
response = await prepare(app.client, app.machine_id)
assert response.status_code == 200, response.text
- assert response.json()["controller"] == {
- "id": controller_id,
- "enrollment_code": None,
- }
+ assert response.json()["controllers"] == [
+ {"key": app.workspace_id, "id": controller_id, "enrollment_code": None}
+ ]
assert app.enrollment.minted == []
@@ -478,10 +570,9 @@ async def test_prepare_mints_a_code_again_once_the_controller_is_revoked(
await session.commit()
response = await prepare(app.client, app.machine_id)
assert response.status_code == 200, response.text
- assert response.json()["controller"] == {
- "id": None,
- "enrollment_code": "swce_synthetic-1",
- }
+ assert response.json()["controllers"] == [
+ {"key": app.workspace_id, "id": None, "enrollment_code": "swce_synthetic-1"}
+ ]
async def test_prepare_leaves_a_machine_past_queued_in_its_state(controller_app):
@@ -504,7 +595,7 @@ async def test_prepare_refuses_unknown_and_deleted_machines(controller_app):
assert deleted.status_code == 409
assert deleted.json() == {"detail": "machine is deleted"}
assert (
- await machine_of(app.factory, app.machine_id)
+ await workspace_of(app.factory, app.machine_id)
).enrollment_code_revision is None
assert app.enrollment.minted == []
@@ -521,11 +612,146 @@ async def test_prepare_for_a_departed_owner_errors_the_machine(controller_app):
assert "enrollment_code" not in response.text
saved = await machine_of(app.factory, app.machine_id)
assert saved.state == "error"
- assert "workspace member" in saved.error
- assert saved.enrollment_code_revision is None
+ assert "no longer a member of any workspace it serves" in saved.error
+ assert (
+ await workspace_of(app.factory, app.machine_id)
+ ).enrollment_code_revision is None
assert app.enrollment.minted == []
+async def test_prepare_lists_a_controller_per_workspace_oldest_first(controller_app):
+ app = controller_app
+ second = await join_workspace(app, "tenant-b", agents=None, member=True)
+ third = await join_workspace(app, "tenant-c", agents=0, member=True)
+ first = await prepare(app.client, app.machine_id)
+ assert first.status_code == 200, first.text
+ assert first.json()["controllers"] == [
+ {"key": app.workspace_id, "id": None, "enrollment_code": "swce_synthetic-1"},
+ {
+ "key": second.workspace_id,
+ "id": None,
+ "enrollment_code": "swce_synthetic-2",
+ },
+ {"key": third.workspace_id, "id": third.controller_id, "enrollment_code": None},
+ ]
+ # Each code is minted in the workspace whose controller enrolls with it.
+ assert app.enrollment.minted == [
+ (TENANT_ZERO_ID, app.workspace_id),
+ ("tenant-b", second.workspace_id),
+ ]
+ with tenant_scope("tenant-b"):
+ assert (
+ await workspace_of(app.factory, app.machine_id)
+ ).enrollment_code_revision == 1
+ again = await prepare(app.client, app.machine_id)
+ assert again.json() == first.json()
+ assert len(app.enrollment.minted) == 2
+
+
+async def test_prepare_skips_a_workspace_its_owner_left(controller_app):
+ app = controller_app
+ left = await join_workspace(app, "tenant-b", agents=None, member=False)
+ stayed = await join_workspace(app, "tenant-c", agents=None, member=True)
+ async with app.factory() as session:
+ await session.delete(
+ await session.get(TenantMember, (TENANT_ZERO_ID, app.owner_id))
+ )
+ await session.commit()
+ response = await prepare(app.client, app.machine_id)
+ assert response.status_code == 200, response.text
+ assert response.json()["controllers"] == [
+ {"key": stayed.workspace_id, "id": None, "enrollment_code": "swce_synthetic-1"}
+ ]
+ assert app.enrollment.minted == [("tenant-c", stayed.workspace_id)]
+ saved = await machine_of(app.factory, app.machine_id)
+ assert (saved.state, saved.error) == ("provisioning", None)
+ with tenant_scope(left.tenant_id):
+ assert (
+ await workspace_of(app.factory, app.machine_id)
+ ).enrollment_code_revision is None
+
+
+async def test_each_workspace_s_controller_reports_on_its_own(controller_app):
+ app = controller_app
+ here, _ = await enrolled(app, agents=0)
+ there = await join_workspace(app, "tenant-b", agents=0, member=True)
+ await report_status(app, here, sessions_running=0, disk_free_bytes=1)
+ with tenant_scope(there.tenant_id):
+ await report_status(app, there.controller_id, sessions_running=3)
+ machine = await machine_of(app.factory, app.machine_id)
+ reports = controller_reports(machine)
+ assert set(reports) == {app.workspace_id, there.workspace_id}
+ assert (
+ reports[app.workspace_id]["sessions_running"],
+ reports[there.workspace_id]["sessions_running"],
+ ) == (0, 3)
+ now = datetime.now(UTC)
+ assert not controller_machine_idle(machine, timedelta(0), now)
+ with tenant_scope(there.tenant_id):
+ await report_status(app, there.controller_id, sessions_running=0)
+ machine = await machine_of(app.factory, app.machine_id)
+ assert controller_machine_idle(machine, timedelta(0), datetime.now(UTC))
+
+
+async def test_a_machine_sleeps_only_once_every_workspace_is_idle(controller_app):
+ app = controller_app
+ await _idle_ready(app, minutes=30)
+ there = await join_workspace(app, "tenant-b", agents=1, member=True)
+ reported = datetime.now(UTC) - timedelta(minutes=1)
+ await update_machine(
+ app.factory,
+ app.machine_id,
+ heartbeat=heartbeat_of(
+ {app.workspace_id: (reported, 0), there.workspace_id: (reported, 2)}
+ ),
+ )
+ [item] = await list_machines(app.client)
+ assert (item["desired_state"], item["revision"]) == ("running", 1)
+ await update_machine(
+ app.factory,
+ app.machine_id,
+ heartbeat=heartbeat_of(
+ {app.workspace_id: (reported, 0), there.workspace_id: (reported, 0)}
+ ),
+ )
+ [item] = await list_machines(app.client)
+ assert (item["desired_state"], item["revision"]) == ("stopped", 2)
+
+
+async def test_a_machine_is_retained_only_once_no_workspace_has_an_agent(
+ controller_app,
+):
+ app = controller_app
+ await _idle_ready(app, minutes=30, agents=0)
+ there = await join_workspace(app, "tenant-b", agents=1, member=True)
+ reported = datetime.now(UTC) - timedelta(minutes=1)
+ await update_machine(
+ app.factory,
+ app.machine_id,
+ heartbeat=heartbeat_of(
+ {app.workspace_id: (reported, 0), there.workspace_id: (reported, 0)}
+ ),
+ )
+ [item] = await list_machines(app.client)
+ # Workspace B still has an agent on it, so the machine sleeps on its disk.
+ assert (item["desired_state"], item["revision"]) == ("stopped", 2)
+ assert (await machine_of(app.factory, app.machine_id)).stop_reason == "idle"
+
+ with tenant_scope(there.tenant_id):
+ async with app.factory() as session:
+ for definition in await session.scalars(
+ select(AgentDefinition).where(
+ AgentDefinition.controller_id == there.controller_id
+ )
+ ):
+ await session.delete(definition)
+ await session.commit()
+ before = datetime.now(UTC)
+ [item] = await list_machines(app.client)
+ assert (item["desired_state"], item["revision"]) == ("retained", 3)
+ assert before + timedelta(days=7) <= datetime.fromisoformat(item["retain_until"])
+
+
async def test_controller_observation_fixture_is_accepted(controller_app):
app = controller_app
body = fixture("controller_observation.json")
@@ -799,6 +1025,19 @@ async def test_retention_sweep_keeps_an_errored_machine_inside_its_window(
assert item["retain_until"] == until.isoformat()
+def heartbeat_of(reports: dict[str, tuple[datetime, int | None]]) -> dict:
+ """A heartbeat with, per workspace row on the machine, when its controller
+ last reported and how many sessions it said ran."""
+ return {
+ "disk": None,
+ "memory": None,
+ "controllers": {
+ key: {"at": at.isoformat(), "sessions_running": sessions}
+ for key, (at, sessions) in reports.items()
+ },
+ }
+
+
async def _idle_ready(
app: ControllerApp,
*,
@@ -812,16 +1051,13 @@ async def _idle_ready(
minutes ago. The controller id."""
app.config.hosted_idle_stop_minutes = minutes
controller_id, _ = await enrolled(app, agents=agents)
- heartbeat: dict = {"disk": None, "memory": None}
- if sessions is not None:
- heartbeat["sessions_running"] = sessions
now = datetime.now(UTC)
await update_machine(
app.factory,
app.machine_id,
state="ready",
active_at=now - timedelta(minutes=31),
- heartbeat=heartbeat,
+ heartbeat=heartbeat_of({app.workspace_id: (now - reported, sessions)}),
heartbeat_at=now - reported,
running_observed_at=now - timedelta(hours=1),
)
@@ -942,8 +1178,10 @@ async def _empty_machine(app: ControllerApp, **values) -> tuple[str, str]:
stop_reason=None,
revision=1,
)
+ workspace = await workspace_on(session, machine.id)
+ assert workspace is not None
machine.active_at = datetime.now(UTC) - timedelta(minutes=31)
- machine.heartbeat = {"disk": None, "memory": None, "sessions_running": 0}
+ machine.heartbeat = heartbeat_of({workspace.id: (datetime.now(UTC), 0)})
machine.heartbeat_at = datetime.now(UTC)
for key, value in values.items():
setattr(machine, key, value)
@@ -1045,11 +1283,12 @@ async def test_recently_errored_empty_machine_is_left_alone(controller_app):
)
-async def _claim(factory, owner_id: str) -> HostedMachine:
+async def _claim(factory, owner_id: str) -> CloudMachine:
async with factory() as session:
await lock_claims(session)
- machine = await HostedMachineStore().claim(
+ machine, _workspace = await CloudMachineStore().claim(
session,
+ factory,
owner_id=owner_id,
capacity=2,
now=datetime.now(UTC),
@@ -1075,3 +1314,27 @@ async def test_released_machine_is_revived_or_replaced_by_a_claim(controller_app
replaced = await _claim(app.factory, owner_id)
assert replaced.id != machine_id
assert replaced.desired_state == "running"
+
+
+async def test_a_workspace_its_owner_left_no_longer_keeps_the_machine_awake(
+ controller_app,
+):
+ app = controller_app
+ here, _ = await enrolled(app, agents=0)
+ left = await join_workspace(app, "tenant-b", agents=0, member=True)
+ await report_status(app, here, sessions_running=0)
+ with tenant_scope(left.tenant_id):
+ await report_status(app, left.controller_id, sessions_running=0)
+ async with app.factory() as session:
+ await session.delete(
+ await session.get(TenantMember, (left.tenant_id, app.owner_id))
+ )
+ await session.commit()
+ assert set(controller_reports(await machine_of(app.factory, app.machine_id))) == {
+ app.workspace_id,
+ left.workspace_id,
+ }
+ response = await prepare(app.client, app.machine_id)
+ assert response.status_code == 200, response.text
+ machine = await machine_of(app.factory, app.machine_id)
+ assert set(controller_reports(machine)) == {app.workspace_id}
diff --git a/core/tests/switch_core/gateway/test_hosted_machines.py b/core/tests/switch_core/gateway/test_hosted_machines.py
index 0febe3098..01d1ca912 100644
--- a/core/tests/switch_core/gateway/test_hosted_machines.py
+++ b/core/tests/switch_core/gateway/test_hosted_machines.py
@@ -4,30 +4,36 @@ and retrying it."""
from __future__ import annotations
import json
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
from pathlib import Path
from types import SimpleNamespace
import pytest
-from switch_core.db.models import HostedMachine, require_tenant_id
+from switch_core.db.models import TENANT_ZERO_ID, CloudMachine
from switch_core.db.stores.hosted_machine_store import MACHINES_FULL
+from switch_core.db.tenant_lookup import tenants_of_cloud_machine
from switch_core.gateway.auth import get_current_user
+from switch_core.gateway.hosted_machines import WORKSPACE_CONNECT_TIMEOUT
+from switch_core.tenant_context import tenant_scope
from tests.switch_core.gateway.test_hosted_controller import ( # noqa: F401
ControllerApp,
controller_app,
enrolled,
+ join_workspace,
machine_of,
observe,
report_status,
update_machine,
+ workspace_of,
)
+from tests.switch_core.hosted_machine_helpers import add_tenant
FIXTURES = Path(__file__).parent.parent / "fixtures" / "hosted_machines"
HEARTBEAT = {
"disk": {"total_bytes": 214748364800, "available_bytes": 204010946560},
"memory": {"total_bytes": 17179869184, "available_bytes": 12884901888},
- "sessions_running": 0,
+ "controllers": {},
}
@@ -284,10 +290,11 @@ async def test_ensure_returns_the_owner_s_machine_and_claims_one_when_missing(
[],
)
async with app.factory() as session:
- machine = await session.get(
- HostedMachine, (require_tenant_id(), body["machine_id"])
- )
+ machine = await session.get(CloudMachine, body["machine_id"])
assert machine is not None
+ assert (await workspace_of(app.factory, body["machine_id"])).owner_id == (
+ app.owner_id
+ )
async def test_ensure_wakes_a_sleeping_machine_but_not_one_its_owner_stopped(
@@ -341,7 +348,7 @@ async def test_ensure_refuses_when_every_machine_is_in_use(controller_app): # n
assert refused.json() == {"detail": MACHINES_FULL}
[email protected]("disabled", ["capacity", "settings", "tenant"])
[email protected]("disabled", ["capacity", "settings", "tenant", "no-tenant"])
async def test_ensure_is_unavailable_when_machines_are_not_enabled(
controller_app, # noqa: F811
disabled,
@@ -352,12 +359,170 @@ async def test_ensure_is_unavailable_when_machines_are_not_enabled(
app.config.hosted_launch_capacity = 0
elif disabled == "settings":
fastapi_app.state.hosted_controller_settings = None
+ elif disabled == "tenant":
+ fastapi_app.state.hosted_controller_settings = app.settings.model_copy(
+ update={"allowed_tenant_ids": ["another-tenant"]}
+ )
else:
fastapi_app.state.hosted_controller_settings = app.settings.model_copy(
- update={"tenant_id": "another-tenant"}
+ update={"allowed_tenant_ids": []}
)
response = await _ensure(app)
assert response.status_code == 503
assert response.json() == {
"detail": "Switch cloud machines are not enabled on this server."
}
+
+
+async def test_every_workspace_may_use_machines_when_none_is_listed(
+ controller_app, # noqa: F811
+):
+ app = controller_app
+ fastapi_app = app.client._transport.app
+ fastapi_app.state.hosted_controller_settings = app.settings.model_copy(
+ update={"allowed_tenant_ids": None}
+ )
+ async with app.factory() as session:
+ await add_tenant(session, "tenant-b")
+ await session.commit()
+ with tenant_scope("tenant-b"):
+ joined = await _ensure(app)
+ assert joined.status_code == 200, joined.text
+ assert joined.json()["machine_id"] == app.machine_id
+ assert await tenants_of_cloud_machine(app.factory, app.machine_id) == [
+ TENANT_ZERO_ID,
+ "tenant-b",
+ ]
+
+
+async def test_a_workspace_off_the_list_is_refused_and_not_joined(
+ controller_app, # noqa: F811
+):
+ app = controller_app
+ async with app.factory() as session:
+ await add_tenant(session, "tenant-b")
+ await session.commit()
+ with tenant_scope("tenant-b"):
+ refused = await _ensure(app)
+ assert refused.status_code == 503
+ assert refused.json() == {
+ "detail": "Switch cloud machines are not enabled on this server."
+ }
+ assert await tenants_of_cloud_machine(app.factory, app.machine_id) == [
+ TENANT_ZERO_ID
+ ]
+ assert (await _ensure(app)).status_code == 200
+
+
+async def test_ensure_from_another_workspace_joins_the_owner_s_machine(
+ controller_app, # noqa: F811
+):
+ app = controller_app
+ app.settings.allowed_tenant_ids = None
+ await update_machine(
+ app.factory,
+ app.machine_id,
+ state="ready",
+ running_observed_at=datetime.now(UTC),
+ )
+ async with app.factory() as session:
+ await add_tenant(session, "tenant-b")
+ await session.commit()
+ with tenant_scope("tenant-b"):
+ joined = await _ensure(app)
+ assert joined.status_code == 200, joined.text
+ body = joined.json()
+ assert (body["machine_id"], body["state"], body["revision"]) == (
+ app.machine_id,
+ "provisioning",
+ 2,
+ )
+ assert (body["agents"], body["controller_id"]) == ([], None)
+ workspace = await workspace_of(app.factory, app.machine_id)
+ assert workspace.id != app.workspace_id
+ # Seen from the first workspace too, the machine is starting again.
+ here = (await app.client.get(f"/hosted-machines/{app.machine_id}")).json()
+ assert (here["state"], here["revision"]) == ("provisioning", 2)
+
+
+async def test_another_workspace_sees_the_machine_ready_once_its_controller_reports(
+ controller_app, # noqa: F811
+):
+ app = controller_app
+ here, _ = await enrolled(app, agents=1)
+ there = await join_workspace(app, "tenant-b", agents=2, member=True)
+ await observe(app.client, app.machine_id, state="running", revision=1)
+ await report_status(app, here, sessions_running=0)
+ assert (await machine_of(app.factory, app.machine_id)).state == "ready"
+ mine = (await app.client.get(f"/hosted-machines/{app.machine_id}")).json()
+ assert mine["state"] == "ready"
+ with tenant_scope(there.tenant_id):
+ theirs = (await app.client.get(f"/hosted-machines/{app.machine_id}")).json()
+ assert (
+ theirs["state"],
+ theirs["error"],
+ theirs["controller_id"],
+ sorted(theirs["agents"]),
+ ) == ("provisioning", None, there.controller_id, sorted(there.agent_ids))
+ assert (await app.client.get("/hosted-machines")).json()["machines"] == [theirs]
+ await report_status(app, there.controller_id, sessions_running=0)
+ theirs = (await app.client.get(f"/hosted-machines/{app.machine_id}")).json()
+ assert (theirs["state"], theirs["error"]) == ("ready", None)
+ assert mine["agents"] != theirs["agents"]
+
+
+async def test_a_workspace_whose_controller_never_connects_reads_as_in_error(
+ controller_app, # noqa: F811
+):
+ app = controller_app
+ here, _ = await enrolled(app, agents=0)
+ there = await join_workspace(app, "tenant-b", agents=0, member=True)
+ await update_machine(
+ app.factory,
+ app.machine_id,
+ state="provisioning",
+ running_observed_at=datetime.now(UTC) - timedelta(minutes=11),
+ )
+ await report_status(app, here, sessions_running=0)
+ assert (await machine_of(app.factory, app.machine_id)).state == "ready"
+ with tenant_scope(there.tenant_id):
+ theirs = (await app.client.get(f"/hosted-machines/{app.machine_id}")).json()
+ assert (theirs["state"], theirs["error"]) == ("error", WORKSPACE_CONNECT_TIMEOUT)
+ mine = (await app.client.get(f"/hosted-machines/{app.machine_id}")).json()
+ assert (mine["state"], mine["error"]) == ("ready", None)
+
+
+async def test_a_workspace_the_machine_does_not_serve_lists_no_machine(
+ controller_app, # noqa: F811
+):
+ app = controller_app
+ async with app.factory() as session:
+ await add_tenant(session, "tenant-b")
+ await session.commit()
+ with tenant_scope("tenant-b"):
+ assert (await app.client.get("/hosted-machines")).json() == {"machines": []}
+ assert (
+ await app.client.get(f"/hosted-machines/{app.machine_id}")
+ ).status_code == 404
+ assert (await _lifecycle(app, "stop", 1)).status_code == 404
+ assert (await machine_of(app.factory, app.machine_id)).desired_state == "running"
+
+
+async def test_stopping_a_ready_machine_reads_as_ready_until_it_stops(
+ controller_app, # noqa: F811
+):
+ app = controller_app
+ controller_id, _ = await enrolled(app, agents=0)
+ await observe(app.client, app.machine_id, state="running", revision=1)
+ await report_status(app, controller_id, sessions_running=0)
+ stopped = await _lifecycle(app, "stop", 1)
+ assert stopped.status_code == 200, stopped.text
+ machine = stopped.json()["machine"]
+ assert (machine["state"], machine["desired_state"], machine["revision"]) == (
+ "ready",
+ "stopped",
+ 2,
+ )
+ await observe(app.client, app.machine_id, state="stopping", revision=2)
+ summary = (await app.client.get(f"/hosted-machines/{app.machine_id}")).json()
+ assert summary["state"] == "stopping"
diff --git a/core/tests/switch_core/gateway/test_signup.py b/core/tests/switch_core/gateway/test_signup.py
index e0a5c35ad..847ba7c8b 100644
--- a/core/tests/switch_core/gateway/test_signup.py
+++ b/core/tests/switch_core/gateway/test_signup.py
@@ -12,12 +12,13 @@ from fastapi import FastAPI
from sqlalchemy import func, select, text
from switch_core.config import SwitchConfig
-from switch_core.db.models import TENANT_ZERO_ID, HostedMachine, TenantMember, User
+from switch_core.db.models import TENANT_ZERO_ID, CloudMachine, TenantMember, User
from switch_core.db.stores.hosted_machine_store import (
MACHINE_BEING_REMOVED,
MACHINE_NEEDS_ADMIN,
MACHINE_NEEDS_ATTENTION,
MACHINES_FULL,
+ workspace_on,
)
from switch_core.db.stores.user_store import UserStore
from switch_core.gateway.auth import decode_jwt, get_current_user, verify_password
@@ -47,7 +48,9 @@ async def signup_app(session_factory):
app = FastAPI()
app.include_router(auth_router)
app.include_router(machine_router)
- app.state.hosted_controller_settings = SimpleNamespace(tenant_id=TENANT_ZERO_ID)
+ app.state.hosted_controller_settings = SimpleNamespace(
+ allowed_tenant_ids=[TENANT_ZERO_ID]
+ )
config = SimpleNamespace(
gateway_signup_open=True,
gateway_signup_mode="default_tenant",
@@ -101,10 +104,10 @@ async def _user(app, email: str) -> User:
return user
-async def _machines(app) -> list[HostedMachine]:
+async def _machines(app) -> list[CloudMachine]:
with tenant_scope(TENANT_ZERO_ID):
async with app.factory() as session:
- return list(await session.scalars(select(HostedMachine)))
+ return list(await session.scalars(select(CloudMachine)))
async def _ensure(app, email: str):
@@ -208,6 +211,11 @@ async def test_signup_signs_in_a_tenant_zero_member_with_a_warming_machine(
"queued",
"running",
)
+ with tenant_scope(TENANT_ZERO_ID):
+ async with app.factory() as session:
+ workspace = await workspace_on(session, machine.id)
+ assert workspace is not None
+ assert (workspace.owner_id, workspace.controller_id) == (user.id, None)
login = await app.client.post(
"/auth/login", json={"email": "[email protected]", "password": PASSWORD}
@@ -260,6 +268,20 @@ async def test_signup_succeeds_without_cloud_capacity(signup_app, caplog):
await _user(app, "[email protected]")
+async def test_signup_warms_no_machine_outside_the_allowed_workspaces(signup_app):
+ app = signup_app
+ app.app.state.hosted_controller_settings = SimpleNamespace(
+ allowed_tenant_ids=["another-tenant"]
+ )
+ response = await _signup(app)
+ assert response.status_code == 201, response.text
+ assert response.json()["machine"] == {
+ "status": "unavailable",
+ "reason": MACHINES_DISABLED,
+ }
+ assert await _machines(app) == []
+
+
async def test_signup_succeeds_when_every_machine_is_in_use(signup_app):
app = signup_app
app.config.hosted_launch_capacity = 1
@@ -293,7 +315,7 @@ async def test_ensure_leaves_an_owner_stopped_machine_stopped(signup_app):
(machine,) = await _machines(app)
with tenant_scope(TENANT_ZERO_ID):
async with app.factory() as session:
- row = await session.get(HostedMachine, (TENANT_ZERO_ID, machine.id))
+ row = await session.get(CloudMachine, machine.id)
assert row is not None
row.state = "stopped"
row.desired_state = "stopped"
@@ -314,7 +336,7 @@ async def test_ensure_leaves_an_owner_stopped_machine_stopped(signup_app):
async def _set_machine(app, machine_id: str, **values) -> None:
with tenant_scope(TENANT_ZERO_ID):
async with app.factory() as session:
- row = await session.get(HostedMachine, (TENANT_ZERO_ID, machine_id))
+ row = await session.get(CloudMachine, machine_id)
assert row is not None
for key, value in values.items():
setattr(row, key, value)
@@ -401,7 +423,11 @@ async def test_ensure_returns_an_unclaimable_machine_as_it_is(
with tenant_scope(TENANT_ZERO_ID):
async with app.factory() as session:
prewarmed = await _prewarm(
- session, user.id, app.config, app.app.state.hosted_controller_settings
+ session,
+ app.factory,
+ user.id,
+ app.config,
+ app.app.state.hosted_controller_settings,
)
assert (prewarmed.status, prewarmed.reason) == ("unavailable", reason)
@@ -412,7 +438,7 @@ async def test_ensure_wakes_an_idle_sleeping_machine(signup_app):
(machine,) = await _machines(app)
with tenant_scope(TENANT_ZERO_ID):
async with app.factory() as session:
- row = await session.get(HostedMachine, (TENANT_ZERO_ID, machine.id))
+ row = await session.get(CloudMachine, machine.id)
assert row is not None
row.state = "stopped"
row.desired_state = "stopped"
diff --git a/core/tests/switch_core/hosted_machine_helpers.py b/core/tests/switch_core/hosted_machine_helpers.py
index 042dc8d12..3b4775779 100644
--- a/core/tests/switch_core/hosted_machine_helpers.py
+++ b/core/tests/switch_core/hosted_machine_helpers.py
@@ -1,4 +1,4 @@
-"""Rows for hosted-machine tests, written straight to the session (flushed, not committed)."""
+"""Rows for cloud-machine tests, written straight to the session (flushed, not committed)."""
from __future__ import annotations
@@ -13,8 +13,17 @@ from switch_core.db.models import (
AgentDefinition,
ApiKey,
Client,
- HostedMachine,
+ CloudMachine,
+ MachineWorkspace,
+ Tenant,
)
+from switch_core.db.stores.hosted_machine_store import workspace_on
+
+
+async def add_tenant(session: AsyncSession, tenant_id: str) -> None:
+ """Another workspace on the server."""
+ session.add(Tenant(id=tenant_id, slug=tenant_id, name=tenant_id))
+ await session.flush()
async def seed_machine(
@@ -25,9 +34,9 @@ async def seed_machine(
desired_state: str,
stop_reason: str | None,
revision: int,
-) -> HostedMachine:
+) -> CloudMachine:
now = datetime.now(UTC)
- machine = HostedMachine(
+ machine = CloudMachine(
id=str(uuid4()),
owner_id=owner_id,
state=state,
@@ -40,19 +49,38 @@ async def seed_machine(
)
session.add(machine)
await session.flush()
+ await add_workspace(session, machine, created_at=now)
return machine
+async def add_workspace(
+ session: AsyncSession, machine: CloudMachine, *, created_at: datetime
+) -> MachineWorkspace:
+ """The bound workspace's row on `machine`: the machine serves it."""
+ workspace = MachineWorkspace(
+ id=str(uuid4()),
+ machine_id=machine.id,
+ owner_id=machine.owner_id,
+ created_at=created_at,
+ )
+ session.add(workspace)
+ await session.flush()
+ return workspace
+
+
async def link_controller(
- session: AsyncSession, machine: HostedMachine
+ session: AsyncSession, machine: CloudMachine
) -> AgentController:
- """The cloud controller the machine enrolled as, linked to it."""
+ """The cloud controller the machine enrolled as in the bound workspace,
+ linked to the workspace's row on it."""
+ workspace = await workspace_on(session, machine.id)
+ assert workspace is not None
controller = AgentController(
owner_id=machine.owner_id, name="Switch cloud", kind="ec2"
)
session.add(controller)
await session.flush()
- machine.controller_id = controller.id
+ workspace.controller_id = controller.id
await session.flush()
return controller
diff --git a/core/tests/switch_core/test_migration_cloud_machine_per_user.py b/core/tests/switch_core/test_migration_cloud_machine_per_user.py
new file mode 100644
index 000000000..7a501daf2
--- /dev/null
+++ b/core/tests/switch_core/test_migration_cloud_machine_per_user.py
@@ -0,0 +1,226 @@
+"""`b6e1c9d4a7f2` makes a cloud machine its owner's rather than a workspace's.
+
+Each machine keeps its id, its workspace row takes the same id, and its
+controller, enrollment code and heartbeat carry over. It refuses a user with
+live machines in two workspaces, and its downgrade refuses a machine serving
+two.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import pytest
+from alembic.config import Config
+from alembic.runtime.environment import EnvironmentContext
+from alembic.script import ScriptDirectory
+from sqlalchemy import Connection, text
+from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
+
+_CORE = Path(__file__).resolve().parents[2]
+_DB = "migration_cloud_machine_per_user"
+
+_BEFORE = "a8c3e5f1d27b"
+_UNDER_TEST = "b6e1c9d4a7f2"
+
+
+def _script_directory(config: Config) -> ScriptDirectory:
+ config.set_main_option("script_location", str(_CORE / "switch_core" / "migrations"))
+ return ScriptDirectory.from_config(config)
+
+
+def _migrate(revision: str, downgrade: bool = False) -> Any:
+ def migrate(connection: Connection) -> None:
+ config = Config(str(_CORE / "alembic.ini"))
+ script = _script_directory(config)
+
+ def revisions(current_revision: str, context: Any) -> Any:
+ if downgrade:
+ return script._downgrade_revs(revision, current_revision)
+ return script._upgrade_revs(revision, current_revision)
+
+ with EnvironmentContext(config, script, fn=revisions) as environment:
+ environment.configure(connection=connection)
+ with environment.begin_transaction():
+ environment.run_migrations()
+
+ return migrate
+
+
+async def engine(postgres_url: str) -> Any:
+ admin = create_async_engine(postgres_url, isolation_level="AUTOCOMMIT")
+ async with admin.connect() as connection:
+ await connection.execute(text(f'DROP DATABASE IF EXISTS "{_DB}"'))
+ await connection.execute(text(f'CREATE DATABASE "{_DB}"'))
+ await admin.dispose()
+ base, _, _ = postgres_url.rpartition("/")
+ engine = create_async_engine(f"{base}/{_DB}")
+ try:
+ async with engine.begin() as connection:
+ await connection.run_sync(_migrate(_BEFORE))
+ yield engine
+ finally:
+ await engine.dispose()
+ admin = create_async_engine(postgres_url, isolation_level="AUTOCOMMIT")
+ async with admin.connect() as connection:
+ await connection.execute(text(f'DROP DATABASE IF EXISTS "{_DB}"'))
+ await admin.dispose()
+
+
+async def _run(engine: AsyncEngine, migrate: Any) -> None:
+ """Run a migration with foreign keys to tenants, users and controllers,
+ which the seeded rows do not have, left unchecked."""
+ async with engine.begin() as connection:
+ await connection.execute(text("SET LOCAL session_replication_role = replica"))
+ await connection.run_sync(migrate)
+
+
+async def _seed(engine: AsyncEngine, rows: str) -> None:
+ async with engine.begin() as connection:
+ # Foreign keys to tenants, users and controllers are beside the point here.
+ await connection.execute(text("SET LOCAL session_replication_role = replica"))
+ await connection.execute(
+ text(
+ f"""
+ INSERT INTO hosted_machines
+ (tenant_id, id, owner_id, state, desired_state, revision,
+ controller_id, enrollment_code_encrypted, enrollment_code_revision,
+ active_at, heartbeat, heartbeat_at)
+ VALUES {rows}
+ """
+ )
+ )
+
+
+HEARTBEAT = json.dumps(
+ {
+ "disk": {"total_bytes": 10, "available_bytes": 4},
+ "memory": None,
+ "sessions_running": 2,
+ }
+)
+
+
+async def test_each_machine_becomes_its_owners_with_its_workspace_row(
+ engine: AsyncEngine,
+) -> None:
+ await _seed(
+ engine,
+ f"""('t1', 'm1', 'ada', 'ready', 'running', 3, 'c1', 'sealed', 3, now(),
+ '{HEARTBEAT}', '2026-10-09T10:00:00+00:00'),
+ ('t1', 'old', 'ada', 'deleted', 'deleted', 9, NULL, NULL, NULL, now(),
+ NULL, NULL),
+ ('t2', 'm2', 'bob', 'queued', 'running', 1, NULL, NULL, NULL, now(),
+ NULL, NULL)""",
+ )
+ async with engine.begin() as connection:
+ await connection.execute(text("SET LOCAL session_replication_role = replica"))
+ await connection.execute(
+ text(
+ "INSERT INTO agent_controller_enrollment_codes "
+ "(tenant_id, id, owner_id, expires_at, hosted_machine_id) "
+ "VALUES ('t1', 'code-1', 'ada', now(), 'm1')"
+ )
+ )
+ await _run(engine, _migrate(_UNDER_TEST))
+
+ async with engine.connect() as connection:
+ machines = (
+ await connection.execute(
+ text(
+ "SELECT id, owner_id, state, revision, heartbeat "
+ "FROM cloud_machines ORDER BY id"
+ )
+ )
+ ).all()
+ workspaces = (
+ await connection.execute(
+ text(
+ "SELECT tenant_id, id, machine_id, owner_id, controller_id, "
+ "enrollment_code_encrypted, enrollment_code_revision "
+ "FROM machine_workspaces ORDER BY id"
+ )
+ )
+ ).all()
+ code = await connection.scalar(
+ text(
+ "SELECT machine_workspace_id FROM agent_controller_enrollment_codes "
+ "WHERE id = 'code-1'"
+ )
+ )
+ tenants = (
+ await connection.execute(text("SELECT tenants_of_cloud_machine('m1')"))
+ ).all()
+ gone = await connection.scalar(text("SELECT to_regclass('hosted_machines')"))
+
+ assert [(row[0], row[1], row[2], row[3]) for row in machines] == [
+ ("m1", "ada", "ready", 3),
+ ("m2", "bob", "queued", 1),
+ ("old", "ada", "deleted", 9),
+ ]
+ assert machines[0][4] == {
+ "disk": {"total_bytes": 10, "available_bytes": 4},
+ "memory": None,
+ "controllers": {
+ "m1": {"at": "2026-10-09T10:00:00+00:00", "sessions_running": 2}
+ },
+ }
+ assert machines[1][4] is None
+ assert workspaces == [
+ ("t1", "m1", "m1", "ada", "c1", "sealed", 3),
+ ("t2", "m2", "m2", "bob", None, None, None),
+ ("t1", "old", "old", "ada", None, None, None),
+ ]
+ assert code == "m1"
+ assert tenants == [("t1",)]
+ assert gone is None
+
+
+async def test_refuses_a_user_with_live_machines_in_two_workspaces(
+ engine: AsyncEngine,
+) -> None:
+ await _seed(
+ engine,
+ """('t1', 'm1', 'ada', 'ready', 'running', 1, NULL, NULL, NULL, now(), NULL, NULL),
+ ('t2', 'm2', 'ada', 'stopped', 'stopped', 1, NULL, NULL, NULL, now(), NULL, NULL)""",
+ )
+ with pytest.raises(RuntimeError, match="1 user\\(s\\) have cloud machines"):
+ await _run(engine, _migrate(_UNDER_TEST))
+
+
+async def test_downgrades_while_each_machine_serves_one_workspace(
+ engine: AsyncEngine,
+) -> None:
+ await _seed(
+ engine,
+ f"""('t1', 'm1', 'ada', 'ready', 'running', 3, 'c1', 'sealed', 3, now(),
+ '{HEARTBEAT}', '2026-10-09T10:00:00+00:00')""",
+ )
+ await _run(engine, _migrate(_UNDER_TEST))
+ async with engine.begin() as connection:
+ await connection.execute(text("SET LOCAL session_replication_role = replica"))
+ await connection.execute(
+ text(
+ "INSERT INTO machine_workspaces (tenant_id, id, machine_id, owner_id) "
+ "VALUES ('t2', 'w2', 'm1', 'ada')"
+ )
+ )
+ with pytest.raises(RuntimeError, match="serve more than one workspace"):
+ await _run(engine, _migrate(_BEFORE, downgrade=True))
+ async with engine.begin() as connection:
+ await connection.execute(text("DELETE FROM machine_workspaces WHERE id = 'w2'"))
+ await _run(engine, _migrate(_BEFORE, downgrade=True))
+ async with engine.connect() as connection:
+ row = (
+ await connection.execute(
+ text(
+ "SELECT tenant_id, id, owner_id, state, controller_id, heartbeat "
+ "FROM hosted_machines"
+ )
+ )
+ ).one()
+ assert row[:5] == ("t1", "m1", "ada", "ready", "c1")
+ assert row[5]["sessions_running"] == 2
diff --git a/deploy/hosted/README.md b/deploy/hosted/README.md
index b64fabae9..a74568212 100644
--- a/deploy/hosted/README.md
+++ b/deploy/hosted/README.md
@@ -1,9 +1,10 @@
# Switch cloud machines on EC2
-A Switch cloud machine is an EC2 VM, one per user, that runs the standard
-`switch-agent-controller`: the program a laptop or an SSH host runs agents
-with. The user's managed agents placed on it run there, each as a Linux user
-of its own. Each machine has a retained, encrypted EBS data disk.
+A Switch cloud machine is an EC2 VM, one per user whatever workspaces they
+use it in, that runs the standard `switch-agent-controller`, the program a
+laptop or an SSH host runs agents with, once per workspace. The user's managed
+agents placed on it run there, each as a Linux user of its own. Each machine
+has a retained, encrypted EBS data disk.
This directory holds what runs the machines:
@@ -27,15 +28,23 @@ authenticated HTTPS API, as any agents controller does.
## How a user gets a machine
Picking "Switch cloud" in Switch Console claims the user's machine
-(`POST /gateway/hosted-machines/ensure`) and creates a managed agent on its
-controller once the machine is ready. Self sign-up claims one too, so it warms
+(`POST /gateway/hosted-machines/ensure`), or joins it from this workspace when
+the user already has one, and creates a managed agent on the machine's
+controller for this workspace once it has reported. A machine serves up to 8
+workspaces. Self sign-up claims one too, so it warms
while the user signs in. The hosted controller polls Core for the machines that
-should exist, creates the data volume, and launches the VM with a bundle that
-carries a one-time enrollment code as its user data. The machine enrolls its
-controller with that code, and its status reports are its heartbeat from then
-on. An instance's user data is set when it launches: when the machine must
-enroll again, the hosted controller terminates the stopped instance and
-launches a new one with the new code, on the same data volume.
+should exist, creates the data volume, and launches the VM with a bundle as its
+user data. The machine runs one agents controller per workspace seated on it,
+and the bundle (version 5) lists them in Core's order, each under its seat key
+with either the controller it enrolled as or a one-time enrollment code. The
+machine enrolls each controller with its code, and their status reports are
+its heartbeat from then on. The machine sleeps only once every controller on it reports no session,
+and its disk is retained only once no workspace has an agent on it. An
+instance's user data is set when it launches:
+when a controller must enroll again or a workspace joins the machine, the hosted controller stops the instance if it is running, terminates
+it and launches a new one with the new bundle, on the same data volume. A
+bundle that only names the controllers enrolled with the codes the instance
+already holds does not replace the instance.
Provider logins are given to the machine on demand from Switch Console, sealed
to its controller's own key: Core relays only ciphertext.
@@ -47,9 +56,12 @@ Core needs agent management (`AGENT_MANAGEMENT_ENABLED`, with
above 0 without it.
Mount a private JSON file through `HOSTED_CONTROLLER_CONFIG_PATH` with
-`tenant_id`, a dedicated `token` of at least 32 characters and the HTTPS
-`agent_api_endpoint` the machines enroll against. Core refuses unknown keys.
-`HOSTED_LAUNCH_CAPACITY` is how many machines may exist at once, 0–100; 0
+`allowed_tenant_ids`, a dedicated `token` of at least 32 characters and the
+HTTPS `agent_api_endpoint` the machines enroll against. `allowed_tenant_ids`
+lists the workspaces whose members may use cloud machines, or is `null` for
+every workspace. Core refuses unknown keys, the old `tenant_id` among them.
+`HOSTED_LAUNCH_CAPACITY` is how many machines may exist at once across the
+server, one per user, 0–100; 0
disables cloud machines. Keep it no higher than the hosted controller's
`max_machines`. The backend chart exposes
`switchCore.hostedControllerSecret` (file `controller.json`),
@@ -61,10 +73,11 @@ the matching `token` and `instance_type`, and set the chart's
`gatewaySecretName` to it. `instance_type` is used for every new machine and
must be one of `allowed_instance_types` in both Terraform and the controller
configuration; `c7i.2xlarge` is recommended. The token authorizes only the
-hosted controller routes for its tenant. It is not a user or agent API key.
+hosted controller routes, for every machine on the server: their lifecycle
+and the controllers they run, never a workspace's agents, rooms or logins. It
+is not a user or agent API key.
-Only the configured tenant may claim machines. Keep every value and key out of
-this public repository.
+Keep every value and key out of this public repository.
## Prepare an environment
@@ -88,8 +101,8 @@ ingress and no Docker socket. VPC NACLs block RFC1918 egress but not IMDS or the
AWS resolver: agents cannot reach instance metadata (their units deny it), but
grant the instance role no authority anyway. The boot reads the machine's
bundle from instance metadata as root. Anyone in the AWS account who can read
-an instance's user data can read its enrollment code, which is spent at the
-first boot and expires after 30 minutes.
+an instance's user data can read its enrollment codes, each spent at the boot
+that enrolls its controller and expiring after 30 minutes.
## Controller configuration
@@ -127,8 +140,10 @@ switch-hosted-controller --config /etc/switch-hosted/controller.json delete <mac
switch-hosted-controller --config /etc/switch-hosted/controller.json upgrade <machine-id> --confirm-instance-id i-0123456789abcdef0
```
-`upgrade` moves a stopped machine whose instance is terminated onto the
-configured image, keeping its disk. A queued command is not confirmation that
+Every new instance of a machine launches from the configured image: a
+relaunch for new user data, a recovery, or a retained machine starting again
+moves it onto a new `image_id`, keeping its disk. `upgrade` does it for a
+stopped machine whose instance is terminated. A queued command is not confirmation that
AWS has done it. Deletion needs `--confirm-machine-id` and exactly one of
`--retain-volume` or `--delete-volume`. Deleting a machine does not remove
snapshots, roles, the NAT gateway or the controller's database.
@@ -154,6 +169,21 @@ owner stopped stays stopped.
A machine that does not start, or whose controller does not report, within 10
minutes goes to error. Use Retry in Switch Console.
+## Moving to one machine per user
+
+A machine used to belong to one workspace. It now belongs to its owner and
+runs a controller per workspace. To upgrade:
+
+1. Bake an image from this `machine/`, which reads bundle version 5, set it as
+ `image_id`, and deploy it before or with the new hosted controller: a
+ running machine on an older image is relaunched from the new one at its
+ next revision.
+2. Replace `tenant_id` in `controller.json` with `allowed_tenant_ids`: the
+ old workspace's id in a list to keep cloud machines to it, or `null`.
+3. Upgrade Core. Each machine keeps its id, its controller and its agents.
+ The migration refuses to run while one user has machines that are not
+ deleted in two workspaces.
+
## Moving off machine slots
Machines used to borrow one of a fixed pool of slots, each an IAM role and an
diff --git a/deploy/hosted/controller/src/switch_hosted_controller/cloud.py b/deploy/hosted/controller/src/switch_hosted_controller/cloud.py
index ffecf31db..8a7b2655e 100644
--- a/deploy/hosted/controller/src/switch_hosted_controller/cloud.py
+++ b/deploy/hosted/controller/src/switch_hosted_controller/cloud.py
@@ -29,6 +29,11 @@ class Ec2Cloud:
def availability_zone(self) -> str:
return self._config.availability_zone
+ @property
+ def image_id(self) -> str:
+ """The image a new instance launches with."""
+ return self._config.image_id
+
def validate_image(self, machine: Machine) -> None:
images = self._ec2.describe_images(ImageIds=[machine.image_id]).get("Images", [])
if len(images) != 1:
diff --git a/deploy/hosted/controller/src/switch_hosted_controller/gateway.py b/deploy/hosted/controller/src/switch_hosted_controller/gateway.py
index 85fe7db25..27b353215 100644
--- a/deploy/hosted/controller/src/switch_hosted_controller/gateway.py
+++ b/deploy/hosted/controller/src/switch_hosted_controller/gateway.py
@@ -20,6 +20,8 @@ from .store import MachineStore
logger = logging.getLogger(__name__)
ENROLLMENT_CODE_RE = re.compile(r"^swce_[A-Za-z0-9_-]{16,128}$")
+BUNDLE_VERSION = 5
+MAX_CONTROLLERS = 8
CORE_DESIRED_STATES = {"running", "stopped", "retained", "deleted"}
SETUP_FAILED_MESSAGE = (
"Cloud machine setup failed. Retry; if it still fails, contact your administrator."
@@ -383,35 +385,55 @@ class Gateway:
}
def bundle(self, prepared: dict, machine: Machine) -> dict:
- """The bundle a machine boots its agents controller from, its instance's
- user data: the controller it enrolled as, or the one-time code it
+ """The bundle a machine boots its agents controllers from, its
+ instance's user data: one controller per workspace seated on the
+ machine, each the controller it enrolled as or the one-time code it
enrolls with until it has one. Never a long-lived credential: the
- machine keeps the one it enrolls with on its own data volume."""
+ machine keeps the ones it enrolls with on its own data volume."""
endpoint = prepared.get("api_endpoint")
url = urlsplit(endpoint) if isinstance(endpoint, str) else None
if url is None or url.scheme != "https" or not url.hostname:
raise ConfigError("Cloud gateway returned no valid API endpoint.")
- controller = prepared.get("controller")
- if not isinstance(controller, dict):
- raise ConfigError("Cloud gateway returned no controller for the machine.")
- controller_id = controller.get("id")
- code = controller.get("enrollment_code")
- if controller_id is not None:
- if code is not None or not isinstance(controller_id, str):
- raise ConfigError("Cloud gateway returned an invalid controller.")
- controller_id = str(UUID(controller_id))
- elif not isinstance(code, str) or not ENROLLMENT_CODE_RE.fullmatch(code):
- raise ConfigError("Cloud gateway returned no valid enrollment code.")
+ controllers = prepared.get("controllers")
+ if not isinstance(controllers, list) or not 1 <= len(controllers) <= MAX_CONTROLLERS:
+ raise ConfigError("Cloud gateway returned no valid controllers for the machine.")
+ entries = [_controller_entry(controller) for controller in controllers]
+ if len({entry["key"] for entry in entries}) != len(entries):
+ raise ConfigError("Cloud gateway returned duplicate controller keys.")
return {
- "version": 4,
+ "version": BUNDLE_VERSION,
"installationId": self.config.installation_id,
"machineId": machine.machine_id,
"dataVolumeId": machine.data_volume_id,
"apiEndpoint": endpoint,
- "controller": {"id": controller_id, "enrollmentCode": code},
+ "controllers": entries,
}
+def _controller_entry(controller: Any) -> dict:
+ if not isinstance(controller, dict):
+ raise ConfigError("Cloud gateway returned an invalid controller.")
+ key = _uuid(controller.get("key"), "Cloud gateway returned an invalid controller key.")
+ controller_id = controller.get("id")
+ code = controller.get("enrollment_code")
+ if controller_id is not None:
+ if code is not None:
+ raise ConfigError("Cloud gateway returned an invalid controller.")
+ controller_id = _uuid(controller_id, "Cloud gateway returned an invalid controller.")
+ elif not isinstance(code, str) or not ENROLLMENT_CODE_RE.fullmatch(code):
+ raise ConfigError("Cloud gateway returned no valid enrollment code.")
+ return {"key": key, "id": controller_id, "enrollmentCode": code}
+
+
+def _uuid(value: Any, message: str) -> str:
+ if not isinstance(value, str):
+ raise ConfigError(message)
+ try:
+ return str(UUID(value))
+ except ValueError:
+ raise ConfigError(message) from None
+
+
def _at_rest(machine: Machine) -> bool:
return (
machine.desired_state in {DesiredState.STOPPED, DesiredState.RETAINED}
diff --git a/deploy/hosted/controller/src/switch_hosted_controller/reconciler.py b/deploy/hosted/controller/src/switch_hosted_controller/reconciler.py
index 5bf9a701d..9894c6be1 100644
--- a/deploy/hosted/controller/src/switch_hosted_controller/reconciler.py
+++ b/deploy/hosted/controller/src/switch_hosted_controller/reconciler.py
@@ -91,6 +91,12 @@ class Reconciler:
if not machine.instance_launch_issued:
if not self._unchanged(claim, DesiredState.RUNNING):
return self._store.cancel_queued_instance_launch(claim)
+ if machine.image_id != self._cloud.image_id:
+ # No instance is left, and the data volume is kept: the
+ # one moment the machine can move onto the configured image.
+ machine = self._store.use_image(claim, self._cloud.image_id)
+ if not self._same_claim(claim, machine):
+ return machine
self._cloud.validate_image(machine)
self._cloud.validate_capacity()
if not self._unchanged(claim, DesiredState.RUNNING):
@@ -133,7 +139,7 @@ class Reconciler:
):
# An instance's user data is set when it launches, so it
# makes way for one launched with the new bundle; the data
- # volume is kept.
+ # volume is kept. A running instance is stopped first.
machine = self._store.mark_instance_terminate_issued(claim)
if not self._same_claim(claim, machine):
return machine
@@ -145,6 +151,15 @@ class Reconciler:
return self._store.set_observed(claim, ObservedState.PROVISIONING, None)
if state != "running":
return self._attention(claim, f"instance is {state}; automatic replacement is disabled")
+ if (
+ machine.bundle is not None
+ and machine.instance_bundle is not None
+ and _bundle_ready(machine)
+ and needs_new_user_data(machine.instance_bundle, machine.bundle)
+ and self._unchanged(claim, DesiredState.RUNNING)
+ ):
+ self._cloud.stop_instance(machine)
+ return self._store.set_observed(claim, ObservedState.PROVISIONING, None)
attachments = volume.get("Attachments", [])
if volume["State"] == "available" and not attachments:
@@ -358,20 +373,36 @@ def needs_new_user_data(current: str | None, wanted: str) -> bool:
"""Whether an instance booting from the bundle `current` must make way for
one launched with `wanted`.
- Not when `wanted` only names the controller the machine enrolled as with
- the code in `current`: the boot keeps an enrollment made with that code.
+ Not when `wanted` only names, for a seat, the controller the machine
+ enrolled as with the code `current` holds for that seat: the boot keeps an
+ enrollment made with that code.
"""
if current is None:
return True
+ if current == wanted:
+ return False
have, want = json.loads(current), json.loads(wanted)
- have_controller, want_controller = have.pop("controller"), want.pop("controller")
- if have != want:
+ have_seats, want_seats = _seats(have), _seats(want)
+ if have_seats is None or want_seats is None or have != want:
return True
- if want_controller["enrollmentCode"] is not None:
- return want_controller["enrollmentCode"] != have_controller["enrollmentCode"]
- return have_controller["enrollmentCode"] is None and (
- have_controller["id"] != want_controller["id"]
- )
+ if have_seats.keys() != want_seats.keys():
+ return True
+ return any(_seat_needs_new_user_data(have_seats[key], want_seats[key]) for key in want_seats)
+
+
+def _seats(bundle: dict) -> dict[str | None, dict] | None:
+ """Pop the bundle's controllers off it, by seat key."""
+ if bundle.get("version") == 5:
+ return {entry["key"]: entry for entry in bundle.pop("controllers")}
+ if bundle.get("version") == 4:
+ return {None: bundle.pop("controller")}
+ return None
+
+
+def _seat_needs_new_user_data(have: dict, want: dict) -> bool:
+ if want["enrollmentCode"] is not None:
+ return want["enrollmentCode"] != have["enrollmentCode"]
+ return have["enrollmentCode"] is None and have["id"] != want["id"]
def _volume_mapping(instance: dict[str, Any], volume_id: str) -> dict[str, Any] | None:
diff --git a/deploy/hosted/controller/src/switch_hosted_controller/store.py b/deploy/hosted/controller/src/switch_hosted_controller/store.py
index fc711a314..4d7d9882e 100644
--- a/deploy/hosted/controller/src/switch_hosted_controller/store.py
+++ b/deploy/hosted/controller/src/switch_hosted_controller/store.py
@@ -296,6 +296,20 @@ class MachineStore:
def mark_volume_create_intent(self, claim: Machine) -> Machine:
return self._mark_intent(claim, "volume_create_intent")
+ def use_image(self, claim: Machine, image_id: str) -> Machine:
+ """Launch the machine's next instance from `image_id`: only while it has
+ no instance and no launch is issued, so nothing launched is told apart
+ from its launch request by the image."""
+ self._connection.execute(
+ """
+ UPDATE machines SET image_id = ?, updated_at = CURRENT_TIMESTAMP
+ WHERE machine_id = ? AND desired_revision = ? AND operation_id = ?
+ AND instance_id IS NULL AND instance_launch_issued = 0
+ """,
+ (image_id, claim.machine_id, claim.desired_revision, claim.operation_id),
+ )
+ return self.get(claim.machine_id)
+
def upgrade_terminated(self, claim: Machine, image_id: str) -> Machine:
if (
claim.desired_state is not DesiredState.STOPPED
diff --git a/deploy/hosted/controller/tests/test_bundle_revisions.py b/deploy/hosted/controller/tests/test_bundle_revisions.py
index 47129dd82..7d929661d 100644
--- a/deploy/hosted/controller/tests/test_bundle_revisions.py
+++ b/deploy/hosted/controller/tests/test_bundle_revisions.py
@@ -14,14 +14,20 @@ from switch_hosted_controller.store import MachineStore
VOLUME_ID = "vol-0123456789abcdef0"
INSTANCE_ID = "i-0123456789abcdef0"
+SEAT_A = "5e1c2b4a-0000-4000-8000-0000000000a1"
+SEAT_B = "5e1c2b4a-0000-4000-8000-0000000000b1"
+CONTROLLER_A = "9a1c2b4a-0000-4000-8000-0000000000aa"
-def code(revision: int) -> str:
- return f"swce_SyntheticCodeRevision{revision:04d}"
+def code(revision: int, seat: int = 0) -> str:
+ if seat == 0:
+ return f"swce_SyntheticCodeRevision{revision:04d}"
+ return f"swce_SyntheticCode{seat}Revision{revision:04d}"
class FakeCore:
- """Core's controller routes: one machine, a revision, one enrollment code per revision."""
+ """Core's controller routes: one machine, a revision, and one controller per
+ seated workspace, each enrolled or with one enrollment code per revision."""
def __init__(self, machine_id: str):
self.machine_id = machine_id
@@ -30,11 +36,12 @@ class FakeCore:
self.desired_state = "running"
self.retain_until: str | None = None
self.code_revision: int | None = None
- self.code = ""
+ self.seats = [SEAT_A]
+ self.codes: dict[str, tuple[int, str]] = {}
+ self.controller_ids: dict[str, str] = {}
self.prepared_revisions: list[int] = []
self.observations: list[dict] = []
self.prepare_failures = 0
- self.controller_id: str | None = None
def machine(self, revision: int | None = None) -> dict:
return {
@@ -60,22 +67,23 @@ class FakeCore:
self.prepare_failures -= 1
raise GatewayError(503)
self.prepared_revisions.append(self.revision)
- if self.controller_id is not None:
- controller = {"id": self.controller_id, "enrollment_code": None}
- else:
- if self.code_revision != self.revision:
- self.code_revision = self.revision
- self.code = code(self.revision)
- controller = {"id": None, "enrollment_code": self.code}
+ self.code_revision = self.revision
return {
"machine_id": self.machine_id,
"revision": self.revision,
"bundle_revision": self.revision,
"api_endpoint": "https://switch.example.test/agent-api",
- "controller": controller,
+ "controllers": [self.controller(index, key) for index, key in enumerate(self.seats)],
"extra": "ignored",
}
+ def controller(self, index: int, key: str) -> dict:
+ if key in self.controller_ids:
+ return {"key": key, "id": self.controller_ids[key], "enrollment_code": None}
+ if self.codes.get(key, (None, ""))[0] != self.revision:
+ self.codes[key] = (self.revision, code(self.revision, index))
+ return {"key": key, "id": None, "enrollment_code": self.codes[key][1]}
+
def sleep(self) -> None:
self.revision += 1
self.desired_state = "stopped"
@@ -87,6 +95,7 @@ class FakeCore:
class FakeCloud:
availability_zone = "us-east-1a"
+ image_id = "ami-0123456789abcdef0"
def __init__(self, store: MachineStore):
self.store = store
@@ -94,7 +103,7 @@ class FakeCloud:
self.instance: dict | None = None
self.user_data: str | None = None
self.calls: list[str] = []
- self.codes_at_boot: list[str] = []
+ self.codes_at_boot: list[list[str | None]] = []
def get_volume(self, machine):
return self.volume
@@ -127,7 +136,9 @@ class FakeCloud:
def _boot(self) -> None:
assert self.instance is not None and self.volume is not None
assert self.user_data is not None
- self.codes_at_boot.append(json.loads(self.user_data)["controller"]["enrollmentCode"])
+ self.codes_at_boot.append(
+ [entry["enrollmentCode"] for entry in json.loads(self.user_data)["controllers"]]
+ )
self.calls.append("boot")
self.instance["State"] = {"Name": "running"}
self.volume["State"] = "in-use"
@@ -217,7 +228,7 @@ def test_new_revision_prepares_once_and_records_its_bundle(tmp_path, instance_la
first_bundle = store.get(MACHINE_ID).bundle
assert core.prepared_revisions == [1]
assert store.get(MACHINE_ID).bundle_token == first
- assert stored_bundle(store)["controller"]["enrollmentCode"] == code(1)
+ assert stored_bundle(store)["controllers"][0]["enrollmentCode"] == code(1)
core.revision = 2
sync(gateway, core)
@@ -227,12 +238,12 @@ def test_new_revision_prepares_once_and_records_its_bundle(tmp_path, instance_la
assert core.prepared_revisions == [1, 2]
bundle = stored_bundle(store)
assert bundle == {
- "version": 4,
+ "version": 5,
"installationId": "test-installation",
"machineId": MACHINE_ID,
"dataVolumeId": VOLUME_ID,
"apiEndpoint": "https://switch.example.test/agent-api",
- "controller": {"id": None, "enrollmentCode": code(2)},
+ "controllers": [{"key": SEAT_A, "id": None, "enrollmentCode": code(2)}],
}
machine = store.get(MACHINE_ID)
assert machine.required_bundle_token == machine.bundle_token == second
@@ -252,7 +263,7 @@ def test_older_revision_is_a_no_op(tmp_path):
machine = store.get(MACHINE_ID)
assert machine.required_bundle_revision == 2
assert machine.bundle_token == token(core.machine_id, 2)
- assert stored_bundle(store)["controller"]["enrollmentCode"] == code(2)
+ assert stored_bundle(store)["controllers"][0]["enrollmentCode"] == code(2)
store.close()
@@ -263,12 +274,12 @@ def test_revision_moved_before_prepare_writes_nothing(tmp_path):
machine = store.get(MACHINE_ID)
assert machine.bundle_token == token(core.machine_id, 1)
assert machine.required_bundle_token == token(core.machine_id, 2)
- assert stored_bundle(store)["controller"]["enrollmentCode"] == code(1)
+ assert stored_bundle(store)["controllers"][0]["enrollmentCode"] == code(1)
sync(gateway, core)
assert core.prepared_revisions == [1, 3, 3]
assert store.get(MACHINE_ID).bundle_token == token(core.machine_id, 3)
- assert stored_bundle(store)["controller"]["enrollmentCode"] == code(3)
+ assert stored_bundle(store)["controllers"][0]["enrollmentCode"] == code(3)
store.close()
@@ -281,12 +292,12 @@ def test_failed_prepare_keeps_the_previous_bundle_until_a_retry_succeeds(tmp_pat
machine = store.get(MACHINE_ID)
assert machine.required_bundle_token == token(core.machine_id, 2)
assert machine.bundle_token == token(core.machine_id, 1)
- assert stored_bundle(store)["controller"]["enrollmentCode"] == code(1)
+ assert stored_bundle(store)["controllers"][0]["enrollmentCode"] == code(1)
sync(gateway, core)
assert core.prepared_revisions == [1, 2]
assert store.get(MACHINE_ID).bundle_token == token(core.machine_id, 2)
- assert stored_bundle(store)["controller"]["enrollmentCode"] == code(2)
+ assert stored_bundle(store)["controllers"][0]["enrollmentCode"] == code(2)
store.close()
@@ -296,7 +307,7 @@ def test_a_bundle_for_a_superseded_revision_is_not_recorded(tmp_path):
store.record_bundle(MACHINE_ID, token(core.machine_id, 1), '{"stale":true}')
machine = store.get(MACHINE_ID)
assert machine.bundle_token == token(core.machine_id, 1)
- assert stored_bundle(store)["controller"]["enrollmentCode"] == code(1)
+ assert stored_bundle(store)["controllers"][0]["enrollmentCode"] == code(1)
store.close()
@@ -316,7 +327,7 @@ def test_sleep_wake_refreshes_the_bundle_before_the_instance_starts(tmp_path):
machine = store.get(MACHINE_ID)
assert machine.observed_state is ObservedState.RUNNING
assert machine.instance_bundle == machine.bundle
- assert cloud.codes_at_boot == [code(1)]
+ assert cloud.codes_at_boot == [[code(1)]]
assert core.observations[-1] == {
"state": "running",
"revision": 1,
@@ -352,7 +363,7 @@ def test_sleep_wake_refreshes_the_bundle_before_the_instance_starts(tmp_path):
poll()
poll()
assert cloud.calls[-2:] == ["run_instance", "boot"]
- assert cloud.codes_at_boot == [code(1), code(3)]
+ assert cloud.codes_at_boot == [[code(1)], [code(3)]]
assert json.loads(cloud.user_data) == stored_bundle(store)
assert store.get(MACHINE_ID).bundle_token == woken
assert store.get(MACHINE_ID).instance_bundle == store.get(MACHINE_ID).bundle
@@ -388,21 +399,114 @@ def test_an_enrolled_machine_wakes_on_the_user_data_it_enrolled_with(tmp_path):
for _ in range(4):
poll()
enrolled_with = store.get(MACHINE_ID).bundle
- core.controller_id = "9a1c2b4a-0000-4000-8000-0000000000aa"
+ core.controller_ids[SEAT_A] = CONTROLLER_A
sleep_and_wake()
assert "terminate_instance" not in cloud.calls
assert cloud.calls[-2:] == ["start_instance", "boot"]
- assert json.loads(store.get(MACHINE_ID).bundle)["controller"] == {
- "id": core.controller_id,
- "enrollmentCode": None,
- }
+ assert json.loads(store.get(MACHINE_ID).bundle)["controllers"] == [
+ {"key": SEAT_A, "id": CONTROLLER_A, "enrollmentCode": None}
+ ]
assert store.get(MACHINE_ID).instance_bundle == cloud.user_data == enrolled_with
- core.controller_id = None
+ core.controller_ids.clear()
sleep_and_wake()
poll()
poll()
assert cloud.calls[-3:] == ["terminate_instance", "run_instance", "boot"]
- assert cloud.codes_at_boot == [code(1), code(1), code(5)]
+ assert cloud.codes_at_boot == [[code(1)], [code(1)], [code(5)]]
assert store.get(MACHINE_ID).instance_bundle == cloud.user_data == store.get(MACHINE_ID).bundle
+
+
+def running_machine(tmp_path):
+ store, core, gateway = harness(tmp_path)
+ cloud = FakeCloud(store)
+ reconciler = Reconciler(store, cloud)
+
+ def poll() -> None:
+ gateway.sync_machines(gateway.machines())
+ reconciler.reconcile_all()
+ gateway.report_observations(gateway.machines())
+
+ for _ in range(4):
+ poll()
+ machine = store.get(MACHINE_ID)
+ assert machine.observed_state is ObservedState.RUNNING
+ assert machine.instance_bundle == machine.bundle
+ return store, core, cloud, poll
+
+
+def test_a_workspace_joining_a_running_machine_relaunches_it_on_the_same_volume(tmp_path):
+ store, core, cloud, poll = running_machine(tmp_path)
+ core.controller_ids[SEAT_A] = CONTROLLER_A
+ core.seats.append(SEAT_B)
+ core.revision += 1
+
+ poll()
+ assert cloud.calls[-1] == "stop_instance"
+ assert core.observations[-1]["state"] == "provisioning"
+ assert json.loads(store.get(MACHINE_ID).bundle)["controllers"] == [
+ {"key": SEAT_A, "id": CONTROLLER_A, "enrollmentCode": None},
+ {"key": SEAT_B, "id": None, "enrollmentCode": code(2, 1)},
+ ]
+ poll()
+ assert cloud.calls[-1] == "terminate_instance"
+ poll()
+ poll()
+ assert cloud.calls[-2:] == ["run_instance", "boot"]
+ assert cloud.codes_at_boot == [[code(1)], [None, code(2, 1)]]
+ assert json.loads(cloud.user_data) == stored_bundle(store)
+ for _ in range(3):
+ poll()
+ machine = store.get(MACHINE_ID)
+ assert machine.observed_state is ObservedState.RUNNING
+ assert machine.instance_bundle == machine.bundle
+ assert machine.data_volume_id == VOLUME_ID
+ assert machine.previous_instance_id == INSTANCE_ID
+ assert machine.instance_id != INSTANCE_ID
+ assert machine.recovery_count == 0
+ assert cloud.calls.count("stop_instance") == 1
+ assert cloud.calls.count("terminate_instance") == 1
+ assert cloud.calls.count("run_instance") == 2
+ store.close()
+
+
+def test_a_revision_that_only_names_the_enrolled_controller_keeps_the_machine_running(
+ tmp_path,
+):
+ store, core, cloud, poll = running_machine(tmp_path)
+ enrolled_with = store.get(MACHINE_ID).bundle
+ core.controller_ids[SEAT_A] = CONTROLLER_A
+ core.revision += 1
+
+ for _ in range(3):
+ poll()
+ machine = store.get(MACHINE_ID)
+ assert core.prepared_revisions == [1, 2]
+ assert json.loads(machine.bundle)["controllers"] == [
+ {"key": SEAT_A, "id": CONTROLLER_A, "enrollmentCode": None}
+ ]
+ assert machine.instance_bundle == enrolled_with
+ assert machine.observed_state is ObservedState.RUNNING
+ assert machine.instance_id == INSTANCE_ID
+ assert cloud.calls.count("stop_instance") == 0
+ assert cloud.calls.count("run_instance") == 1
+ store.close()
+
+
+def test_a_running_instance_whose_user_data_is_unknown_is_left_running(tmp_path):
+ store, core, cloud, poll = running_machine(tmp_path)
+ store._connection.execute("UPDATE machines SET instance_bundle = NULL")
+ core.seats.append(SEAT_B)
+ core.revision += 1
+
+ for _ in range(3):
+ poll()
+ machine = store.get(MACHINE_ID)
+ assert len(json.loads(machine.bundle)["controllers"]) == 2
+ assert machine.instance_bundle is None
+ assert machine.observed_state is ObservedState.RUNNING
+ assert cloud.instance is not None and cloud.instance["State"]["Name"] == "running"
+ assert cloud.calls.count("stop_instance") == 0
+ assert cloud.calls.count("run_instance") == 1
+ store.close()
diff --git a/deploy/hosted/controller/tests/test_controller.py b/deploy/hosted/controller/tests/test_controller.py
index 7e52aad7e..9379df3b5 100644
--- a/deploy/hosted/controller/tests/test_controller.py
+++ b/deploy/hosted/controller/tests/test_controller.py
@@ -73,15 +73,19 @@ def insert_machine(store: MachineStore, cfg: ControllerConfig, machine_id: str)
)
+SEAT_A = "5e1c2b4a-0000-4000-8000-0000000000a1"
+SEAT_B = "5e1c2b4a-0000-4000-8000-0000000000b1"
+
+
def bundle_json(machine_id: str, enrollment_code: str) -> str:
return json.dumps(
{
- "version": 4,
+ "version": 5,
"installationId": "test-installation",
"machineId": machine_id,
"dataVolumeId": "vol-0123456789abcdef0",
"apiEndpoint": "https://switch.example.test/agent-api",
- "controller": {"id": None, "enrollmentCode": enrollment_code},
+ "controllers": [{"key": SEAT_A, "id": None, "enrollmentCode": enrollment_code}],
},
separators=(",", ":"),
)
@@ -441,9 +445,32 @@ def test_starting_an_instance_with_the_current_bundle_only_starts_it(tmp_path: P
store.close()
-def _with_controller(bundle: str, controller_id: str | None, enrollment_code: str | None) -> str:
+def _with_controller(
+ bundle: str, controller_id: str | None, enrollment_code: str | None, key: str = SEAT_A
+) -> str:
+ value = json.loads(bundle)
+ entry = {"key": key, "id": controller_id, "enrollmentCode": enrollment_code}
+ controllers = [item for item in value["controllers"] if item["key"] != key]
+ if len(controllers) == len(value["controllers"]):
+ value["controllers"] = [*controllers, entry]
+ else:
+ value["controllers"] = [
+ entry if item["key"] == key else item for item in value["controllers"]
+ ]
+ return json.dumps(value, separators=(",", ":"))
+
+
+def _without_seat(bundle: str, key: str) -> str:
value = json.loads(bundle)
- value["controller"] = {"id": controller_id, "enrollmentCode": enrollment_code}
+ value["controllers"] = [item for item in value["controllers"] if item["key"] != key]
+ return json.dumps(value, separators=(",", ":"))
+
+
+def _as_version_4(bundle: str) -> str:
+ value = json.loads(bundle)
+ [entry] = value.pop("controllers")
+ value["version"] = 4
+ value["controller"] = {"id": entry["id"], "enrollmentCode": entry["enrollmentCode"]}
return json.dumps(value, separators=(",", ":"))
@@ -489,6 +516,76 @@ def test_user_data_is_replaced_only_when_the_boot_would_need_it(current, wanted,
assert needs_new_user_data(current, wanted) is replaced
+TWO_SEATS = _with_controller(REVISION_1_BUNDLE, None, "swce_SyntheticSeatBRevision0001", SEAT_B)
+TWO_SEATS_ENROLLED_A = _with_controller(TWO_SEATS, CONTROLLER_A, None)
+
+
+ ("current", "wanted", "replaced"),
+ [
+ (REVISION_1_BUNDLE, TWO_SEATS, True),
+ (TWO_SEATS, REVISION_1_BUNDLE, True),
+ (TWO_SEATS, _without_seat(TWO_SEATS, SEAT_A), True),
+ (TWO_SEATS, TWO_SEATS, False),
+ (TWO_SEATS, TWO_SEATS_ENROLLED_A, False),
+ (
+ TWO_SEATS,
+ _with_controller(TWO_SEATS_ENROLLED_A, CONTROLLER_B, None, SEAT_B),
+ False,
+ ),
+ (
+ TWO_SEATS,
+ _with_controller(TWO_SEATS_ENROLLED_A, None, "swce_SyntheticSeatBRevision0002", SEAT_B),
+ True,
+ ),
+ (TWO_SEATS_ENROLLED_A, TWO_SEATS_ENROLLED_A, False),
+ (TWO_SEATS_ENROLLED_A, _with_controller(TWO_SEATS, CONTROLLER_B, None), True),
+ (TWO_SEATS_ENROLLED_A, TWO_SEATS, True),
+ (
+ TWO_SEATS_ENROLLED_A,
+ _with_controller(TWO_SEATS, None, "swce_SyntheticCodeRevision0002"),
+ True,
+ ),
+ (
+ _with_controller(REVISION_1_BUNDLE, CONTROLLER_A, None),
+ _with_controller(TWO_SEATS_ENROLLED_A, None, "swce_SyntheticSeatCRevision0001", SEAT_B),
+ True,
+ ),
+ (
+ TWO_SEATS,
+ json.dumps(
+ {
+ **json.loads(TWO_SEATS),
+ "controllers": json.loads(TWO_SEATS)["controllers"][::-1],
+ },
+ separators=(",", ":"),
+ ),
+ False,
+ ),
+ ],
+)
+def test_user_data_is_replaced_per_workspace_seat(current, wanted, replaced):
+ assert needs_new_user_data(current, wanted) is replaced
+
+
+ ("current", "wanted", "replaced"),
+ [
+ (_as_version_4(REVISION_1_BUNDLE), REVISION_1_BUNDLE, True),
+ (_as_version_4(REVISION_1_BUNDLE), _as_version_4(REVISION_1_BUNDLE), False),
+ (
+ _as_version_4(REVISION_1_BUNDLE),
+ _as_version_4(_with_controller(REVISION_1_BUNDLE, CONTROLLER_A, None)),
+ False,
+ ),
+ (_as_version_4(REVISION_1_BUNDLE), _as_version_4(REVISION_2_BUNDLE), True),
+ (json.dumps({"version": 3}), REVISION_1_BUNDLE, True),
+ ],
+)
+def test_user_data_of_another_bundle_version_is_replaced(current, wanted, replaced):
+ assert needs_new_user_data(current, wanted) is replaced
+
+
def test_starting_an_enrolled_instance_keeps_the_user_data_it_enrolled_with(tmp_path: Path):
cfg = config(tmp_path)
store, machine = stopped_instance_with_bundle(cfg, REVISION_1_BUNDLE)
diff --git a/deploy/hosted/controller/tests/test_gateway.py b/deploy/hosted/controller/tests/test_gateway.py
index 42da860ed..5cb514291 100644
--- a/deploy/hosted/controller/tests/test_gateway.py
+++ b/deploy/hosted/controller/tests/test_gateway.py
@@ -25,6 +25,7 @@ from switch_hosted_controller.store import CapacityError, MachineStore
VOLUME_ID = "vol-0123456789abcdef0"
CODE = "swce_SyntheticEnrollmentCode0000"
+SEAT = "5e1c2b4a-0000-4000-8000-0000000000a1"
OTHER_MACHINE_ID = "3f1c2b4a-0000-4000-8000-000000000002"
@@ -47,7 +48,7 @@ def prepared_machine(**overrides) -> dict:
"revision": 1,
"bundle_revision": 1,
"api_endpoint": "https://switch.example.test/agent-api",
- "controller": {"id": None, "enrollment_code": CODE},
+ "controllers": [{"key": SEAT, "id": None, "enrollment_code": CODE}],
**overrides,
}
@@ -105,12 +106,12 @@ def test_launch_retry_reuses_the_row_and_prepares_once(tmp_path):
assert machine.bundle_token == str(uuid5(NAMESPACE_URL, f"{MACHINE_ID}:1"))
assert machine.bundle == json.dumps(
{
- "version": 4,
+ "version": 5,
"installationId": cfg.installation_id,
"machineId": MACHINE_ID,
"dataVolumeId": VOLUME_ID,
"apiEndpoint": "https://switch.example.test/agent-api",
- "controller": {"id": None, "enrollmentCode": CODE},
+ "controllers": [{"key": SEAT, "id": None, "enrollmentCode": CODE}],
},
separators=(",", ":"),
)
@@ -132,7 +133,7 @@ def test_machine_without_a_controller_writes_no_bundle(tmp_path):
store = open_store(cfg)
gateway = make_gateway(cfg, store)
prepared = prepared_machine()
- del prepared["controller"]
+ del prepared["controllers"]
gateway.request = Mock(return_value=prepared)
core = CoreMachine.parse(core_machine())
gateway.sync_machine(core)
@@ -298,13 +299,17 @@ def test_core_machine_list_and_prepare_produce_the_bundle(tmp_path):
)
assert machine.bundle_token == bundle_token(listed["machine_id"], listed["revision"])
assert json.loads(machine.bundle) == {
- "version": 4,
+ "version": 5,
"installationId": "inst-test",
"machineId": prepared["machine_id"],
"dataVolumeId": listed["data_volume_id"],
"apiEndpoint": prepared["api_endpoint"],
- "controller": {"id": None, "enrollmentCode": prepared["controller"]["enrollment_code"]},
+ "controllers": [
+ {"key": entry["key"], "id": entry["id"], "enrollmentCode": entry["enrollment_code"]}
+ for entry in prepared["controllers"]
+ ],
}
+ assert prepared["controllers"]
store.close()
@@ -539,31 +544,105 @@ def test_invalid_core_machine_does_not_block_the_others(tmp_path):
store.close()
+SEAT_B = "5e1c2b4a-0000-4000-8000-0000000000b1"
+SEAT_C = "5e1c2b4a-0000-4000-8000-0000000000c1"
+CONTROLLER_ID = "9a1c2b4a-0000-4000-8000-0000000000aa"
+
+
+def seat(key: str, *, controller_id: str | None = None, code: str | None = CODE) -> dict:
+ return {"key": key, "id": controller_id, "enrollment_code": None if controller_id else code}
+
+
+def test_a_bundle_carries_one_controller_per_workspace_in_cores_order(tmp_path):
+ cfg = config(tmp_path)
+ store = open_store(cfg)
+ machine = store.record_volume(
+ insert_machine(store, cfg, MACHINE_ID).machine_id, VOLUME_ID, cfg.availability_zone
+ )
+ other_code = "swce_SyntheticEnrollmentCode0001"
+ prepared = prepared_machine(
+ controllers=[
+ seat(SEAT_C, code=other_code),
+ seat(SEAT, controller_id=CONTROLLER_ID),
+ {**seat(SEAT_B), "key": SEAT_B.upper()},
+ ]
+ )
+ assert make_gateway(cfg, store).bundle(prepared, machine) == {
+ "version": 5,
+ "installationId": cfg.installation_id,
+ "machineId": MACHINE_ID,
+ "dataVolumeId": VOLUME_ID,
+ "apiEndpoint": "https://switch.example.test/agent-api",
+ "controllers": [
+ {"key": SEAT_C, "id": None, "enrollmentCode": other_code},
+ {"key": SEAT, "id": CONTROLLER_ID, "enrollmentCode": None},
+ {"key": SEAT_B, "id": None, "enrollmentCode": CODE},
+ ],
+ }
+ store.close()
+
+
+def test_a_bundle_takes_up_to_eight_controllers(tmp_path):
+ cfg = config(tmp_path)
+ store = open_store(cfg)
+ machine = insert_machine(store, cfg, MACHINE_ID)
+ keys = [f"5e1c2b4a-0000-4000-8000-00000000000{index}" for index in range(8)]
+ prepared = prepared_machine(controllers=[seat(key) for key in keys])
+ bundle = make_gateway(cfg, store).bundle(prepared, machine)
+ assert [entry["key"] for entry in bundle["controllers"]] == keys
+ store.close()
+
+
@pytest.mark.parametrize(
- "controller",
+ "controllers",
[
None,
- {"id": None, "enrollment_code": None},
- {"id": None, "enrollment_code": "swcc_not-a-code-at-all-0000"},
- {"id": "not-a-uuid", "enrollment_code": None},
- {"id": MACHINE_ID, "enrollment_code": "swce_SyntheticEnrollmentCode0000"},
+ {"key": SEAT, "id": None, "enrollment_code": CODE},
+ [],
+ [seat(f"5e1c2b4a-0000-4000-8000-00000000000{index}") for index in range(9)],
+ [seat(SEAT), seat(SEAT, controller_id=CONTROLLER_ID)],
+ [seat(SEAT), seat(SEAT.upper())],
+ [None],
+ ["not-a-controller"],
+ [{"id": None, "enrollment_code": CODE}],
+ [{"key": None, "id": None, "enrollment_code": CODE}],
+ [{"key": "not-a-uuid", "id": None, "enrollment_code": CODE}],
+ [{"key": 7, "id": None, "enrollment_code": CODE}],
+ [{"key": SEAT, "id": None, "enrollment_code": None}],
+ [{"key": SEAT}],
+ [{"key": SEAT, "id": None, "enrollment_code": "swcc_not-a-code-at-all-0000"}],
+ [{"key": SEAT, "id": "not-a-uuid", "enrollment_code": None}],
+ [{"key": SEAT, "id": 7, "enrollment_code": None}],
+ [{"key": SEAT, "id": CONTROLLER_ID, "enrollment_code": CODE}],
+ [seat(SEAT_B), {"key": SEAT, "id": CONTROLLER_ID, "enrollment_code": CODE}],
],
)
-def test_a_controller_bundle_refuses_what_is_not_one(tmp_path, controller):
+def test_a_controller_bundle_refuses_what_is_not_one(tmp_path, controllers):
cfg = config(tmp_path)
store = open_store(cfg)
machine = insert_machine(store, cfg, MACHINE_ID)
- prepared = {**prepared_machine(), "controller": controller}
- with pytest.raises((ConfigError, ValueError)):
+ prepared = prepared_machine(controllers=controllers)
+ with pytest.raises(ConfigError):
+ make_gateway(cfg, store).bundle(prepared, machine)
+ store.close()
+
+
+def test_a_prepare_with_the_former_single_controller_is_refused(tmp_path):
+ cfg = config(tmp_path)
+ store = open_store(cfg)
+ machine = insert_machine(store, cfg, MACHINE_ID)
+ prepared = prepared_machine(controller={"id": None, "enrollment_code": CODE})
+ del prepared["controllers"]
+ with pytest.raises(ConfigError):
make_gateway(cfg, store).bundle(prepared, machine)
store.close()
-def test_an_enrolled_controller_machine_gets_its_controller_and_no_code(tmp_path):
+def test_an_enrolled_controller_gets_its_controller_and_no_code(tmp_path):
cfg = config(tmp_path)
store = open_store(cfg)
machine = insert_machine(store, cfg, MACHINE_ID)
- prepared = {**prepared_machine(), "controller": {"id": MACHINE_ID, "enrollment_code": None}}
+ prepared = prepared_machine(controllers=[seat(SEAT, controller_id=CONTROLLER_ID)])
bundle = make_gateway(cfg, store).bundle(prepared, machine)
- assert bundle["controller"] == {"id": MACHINE_ID, "enrollmentCode": None}
+ assert bundle["controllers"] == [{"key": SEAT, "id": CONTROLLER_ID, "enrollmentCode": None}]
store.close()
diff --git a/deploy/hosted/controller/tests/test_launch_request.py b/deploy/hosted/controller/tests/test_launch_request.py
index 2cec2188c..1c1b8104a 100644
--- a/deploy/hosted/controller/tests/test_launch_request.py
+++ b/deploy/hosted/controller/tests/test_launch_request.py
@@ -49,7 +49,7 @@ def test_run_request_boots_from_the_stored_bundle_with_the_shared_profile(tmp_pa
assert cloud.run_instance(machine) == "i-0123456789abcdef0"
bundle = json.loads(machine.bundle)
- assert bundle["version"] == 4
+ assert bundle["version"] == 5
assert bundle["machineId"] == MACHINE_ID
assert not {"slotId", "generation", "assignment", "assignmentSecretId"} & set(bundle)
store.close()
diff --git a/deploy/hosted/controller/tests/test_lifecycle_fencing.py b/deploy/hosted/controller/tests/test_lifecycle_fencing.py
index ad6de9e27..57ad661b6 100644
--- a/deploy/hosted/controller/tests/test_lifecycle_fencing.py
+++ b/deploy/hosted/controller/tests/test_lifecycle_fencing.py
@@ -276,7 +276,8 @@ def test_retained_with_a_missing_volume_needs_attention(tmp_path: Path):
class LaunchCloud:
- def __init__(self):
+ def __init__(self, image_id: str = "ami-0123456789abcdef0"):
+ self.image_id = image_id
self.launched = []
def get_volume(self, machine):
@@ -323,6 +324,35 @@ def test_running_after_retained_launches_the_next_instance_on_the_kept_volume(tm
store.close()
+def test_a_new_instance_moves_onto_the_configured_image(tmp_path: Path):
+ cfg = config(tmp_path)
+ store, machine = store_and_machine(cfg)
+ first = record_compute(store, machine, cfg.availability_zone)
+ store.set_desired(machine.machine_id, DesiredState.RETAINED, None)
+ Reconciler(store, DeletionCloud(["terminated"])).reconcile(machine.machine_id)
+ with_bundle(store, machine.machine_id, 2)
+ store.set_desired(machine.machine_id, DesiredState.RUNNING, None)
+ cloud = LaunchCloud(image_id="ami-22222222222222222")
+
+ relaunched = Reconciler(store, cloud).reconcile(machine.machine_id)
+
+ [launched] = cloud.launched
+ assert first.image_id == cfg.image_id
+ assert launched.image_id == relaunched.image_id == "ami-22222222222222222"
+ assert relaunched.data_volume_id == first.data_volume_id
+ store.close()
+
+
+def test_the_image_of_an_issued_launch_is_not_changed(tmp_path: Path):
+ cfg = config(tmp_path)
+ store, machine = store_and_machine(cfg)
+ machine = store.record_volume(machine.machine_id, "vol-0123456789abcdef0", "us-east-1a")
+ machine = store.mark_instance_launch_intent(machine)
+ machine = store.mark_instance_launch_issued(machine, datetime.now(UTC))
+ assert store.use_image(machine, "ami-22222222222222222").image_id == cfg.image_id
+ store.close()
+
+
def test_deleted_keeps_the_volume_until_retain_until(tmp_path: Path, monkeypatch):
cfg = config(tmp_path)
store, machine = store_and_machine(cfg)
diff --git a/deploy/hosted/controller/tests/test_retries.py b/deploy/hosted/controller/tests/test_retries.py
index f5a453b35..2963f6870 100644
--- a/deploy/hosted/controller/tests/test_retries.py
+++ b/deploy/hosted/controller/tests/test_retries.py
@@ -79,6 +79,7 @@ def ready_to_launch(tmp_path: Path):
)
machine = with_bundle(store, machine.machine_id, 1)
cloud = Mock(spec=Ec2Cloud)
+ cloud.image_id = cfg.image_id
cloud.get_volume.return_value = volume(cfg, machine)
cloud.get_instance.return_value = None
cloud.discover_instance.return_value = None
@@ -188,6 +189,7 @@ def test_definitely_rejected_volume_create_lets_retention_finish(tmp_path: Path)
cfg = config(tmp_path)
store, machine = store_and_machine(cfg)
cloud = Mock(spec=Ec2Cloud)
+ cloud.image_id = cfg.image_id
cloud.get_volume.return_value = None
cloud.create_volume.side_effect = client_error("VolumeLimitExceeded", 400, "CreateVolume")
Reconciler(store, cloud).reconcile_all()
diff --git a/deploy/hosted/machine/README.md b/deploy/hosted/machine/README.md
index 3f63a3083..a1ae0321f 100644
--- a/deploy/hosted/machine/README.md
+++ b/deploy/hosted/machine/README.md
@@ -1,34 +1,43 @@
# The Switch cloud machine image
A Switch cloud machine runs `switch-agent-controller`, the program any machine
-runs agents with. The owner's managed agents placed on it run there, each as a
-Linux user of its own, and they use the provider logins the owner gives the
-machine.
+runs agents with: one controller per workspace that uses the machine, up to 8.
+Each workspace's managed agents placed on it run under that workspace's
+controller, each as a Linux user of its own, and they use the provider logins
+the workspace gives the machine.
## How a machine comes up
1. Core claims the machine for its owner.
2. The hosted controller creates the data volume and asks Core to prepare the
- machine. Until a controller enrolls for the machine, Core answers with a
- one-time **enrollment code**. The code is bound to the machine and is valid
- for 30 minutes. A retry of the same revision gets the same code.
+ machine. Core lists one **seat** per workspace on the machine, the oldest
+ workspace first. Until a controller enrolls for a seat, Core answers with a
+ one-time **enrollment code** for it. The code is bound to the machine and
+ is valid for 30 minutes. A retry of the same revision gets the same code.
3. The hosted controller launches the instance from this image with the bundle
as its user data:
```json
- {"version": 4, "installationId": "…", "machineId": "…", "dataVolumeId": "vol-…",
+ {"version": 5, "installationId": "…", "machineId": "…", "dataVolumeId": "vol-…",
"apiEndpoint": "https://…",
- "controller": {"id": null, "enrollmentCode": "swce_…"}}
+ "controllers": [
+ {"key": "<uuid>", "id": "<uuid>", "enrollmentCode": null},
+ {"key": "<uuid>", "id": null, "enrollmentCode": "swce_…"}]}
```
- Once the machine has a controller, `controller.id` names it and
- `enrollmentCode` is null. The bundle never holds a long-lived credential.
+ `controllers` lists 1 to 8 seats, each with a `key` of its own (a UUID
+ naming the workspace's seat on this machine; the boot only uses it to tell
+ seats apart). Once a seat has a controller, its `id` names it and its
+ `enrollmentCode` is null; before, its `id` is null and it has a code. The
+ image reads version 5 only, strictly: a bundle with any other version, an
+ unknown key, a repeated seat key, no seat or more than 8 is refused. The
+ bundle never holds a long-lived credential.
An instance boots from the bundle it was launched with. When the machine
- needs a new enrollment code or a new endpoint, the hosted controller
- terminates the stopped instance and launches a new one with the new bundle,
- on the same data volume. A bundle that only names the controller the
- machine enrolled as is not one of those: the boot keeps an enrollment made
- with the code it already has.
+ needs a new enrollment code, a new endpoint or another set of seats, the
+ hosted controller terminates the stopped instance and launches a new one
+ with the new bundle, on the same data volume. A bundle that only names the
+ controller a seat enrolled as is not one of those: the boot keeps an
+ enrollment made with the code it already has.
4. `switch-machine-boot.service` runs as root at every boot
(`switch_machine_boot.py`):
- It reads the bundle from the instance's user data (IMDSv2) on every boot.
@@ -40,30 +49,77 @@ machine.
(`nodev,nosuid`).
- It checks the volume's marker (`/data/.switch-machine.json`), so the disk
of one machine is never used by another.
- - With a code, it enrolls the controller as `switch-controller` into
- `/data/.switch-controller`. The credential and the sealing key are kept
- there, on the machine's own volume. The SHA-256 of the code is kept in
- `/data/.switch-controller-code`. A directory that still holds an
- enrollment made with another code is set aside first: Switch handed over
- a new code, so it holds no live controller for this machine. One made
- with this same code is kept: an earlier attempt of this boot enrolled,
- then failed before Switch linked the controller.
+ - It gives each seat an **index** (see below), kept on the volume.
+ - For each seat with a code, it enrolls the seat's controller as the seat's
+ controller user into its data directory. The credential and the sealing
+ key are kept there, on the machine's own volume. The SHA-256 of the code
+ is kept in the seat's code file. A directory that still holds an
+ enrollment made with another code is set aside first, next to it, as
+ `<data directory>.replaced-<time>`: Switch handed over a new code, so it
+ holds no live controller for this seat. One made with this same code is
+ kept: an earlier attempt of this boot enrolled, then failed before Switch
+ linked the controller.
- Then, on every boot, it runs `switch-agent-controller install-service
- --separate-users --no-block` with the agents' directories in
- `/data/agents`. That command is idempotent, and the root volume can be
- new after an image change. It queues the controller's start: the
+ --separate-users --no-block` for each seat, with the seat's agents'
+ directory. That command is idempotent, and the root volume can be new
+ after an image change. It queues the controller's start: each
controller's unit is ordered after the boot service, so systemd starts it
once the boot completes.
-5. Core links the controller to the machine as a Switch cloud controller
- (kind `ec2`). The controller's status reports are the machine's heartbeat:
+ - A controller that fails to enroll or start does not keep the others from
+ starting. The boot tries it again 4 times, 30 s apart, then logs the
+ failure and completes anyway: failing the boot would stop the other
+ controllers, which are ordered after it. That workspace then shows the
+ machine in error in Switch Console, and a stop and start of the machine
+ tries again.
+5. Core links each controller to the machine as a Switch cloud controller
+ (kind `ec2`). The controllers' status reports are the machine's heartbeat:
the first one after the instance was seen running makes the machine
`ready`.
-If the owner removes the machine's controller in the Machines page, the next
-start of the machine gets a new code, and the old controller is replaced. A
-machine whose volume holds no enrollment and whose bundle has no code cannot
-start, and its boot log says so. Removing its controller in the Machines page
-gives it a new code.
+If a workspace removes the machine's controller in its Machines page, the next
+start of the machine gets a new code for that seat, and the old controller is
+replaced. A seat whose data directory holds no enrollment and whose bundle
+entry has no code cannot start, and the boot log says so. Removing its
+controller in the Machines page gives it a new code.
+
+## Controllers on a machine
+
+The first time the boot sees the volume, it numbers the bundle's seats 0, 1, …
+in bundle order. Core lists the oldest workspace first, so a volume that
+predates several controllers per machine keeps its enrollment at index 0. A
+seat added later gets the lowest index no other seat holds. The indexes are
+kept in `/data/.switch-controllers.json`
+(`{"version": 1, "controllers": {"<key>": <index>}}`, mode 0600), and an index
+is never given to another seat while its seat is in that file, which keeps
+seats that left. Since nothing is ever removed from it, a volume can have
+held at most 8 different seats; a new seat past that cannot start, and the
+boot log says so.
+
+Index `k`'s ids are fixed, because the data volume outlives the root volume:
+
+| | index 0 | index k ≥ 1 |
+|---|---|---|
+| controller user and its group (uid = gid) | `switch-controller`, 2000 | `switch-controller-<k>`, 2000 + 200·k |
+| agents' group | `switch-agents-2000`, 2001 | `switch-agents-<uid>`, uid + 1 |
+| agent users (NN = 01…agent users) | `sa2000-NN`, 2100 + NN | `sa<uid>-NN`, uid + 100 + NN |
+| controller unit | `switch-agent-controller-2000.service` | `switch-agent-controller-<uid>.service` |
+| data directory | `/data/.switch-controller` | `/data/controllers/<k>/data` |
+| code file | `/data/.switch-controller-code` | `/data/controllers/<k>/code` |
+| agents' directories | `/data/agents` | `/data/controllers/<k>/agents` |
+
+The image bakes index 0's users (`install.sh`). For an index k ≥ 1, the boot
+creates the users and groups that are missing with these ids, as `install.sh`
+does, before `install-service`, which then creates none; a user or group that
+exists with another id is refused. It also writes the drop-in that orders the
+seat's controller unit after the boot service
+(`/etc/systemd/system/switch-agent-controller-<uid>.service.d/after-boot.conf`)
+on every boot, since the root volume can be new.
+
+A seat that is in `/data/.switch-controllers.json` but not in the bundle (its
+workspace left the machine) has its controller stopped and disabled, and any
+of its agents still running stopped. Its data stays on the volume and its
+index stays reserved; the boot logs a warning. If the workspace comes back
+with the same seat key, its controller starts again from that data.
## Baking the image
@@ -77,10 +133,10 @@ directory. Then, as root:
- Checks the commands the boot needs, and that polkit is present.
- Installs the controller at `/opt/switch/controller`.
-- Creates the controller's user `switch-controller` (uid 2000), the agents'
- group `switch-agents-2000` (gid 2001) and one user per agent,
- `sa2000-01`… (uid 2101…).
-- Installs the boot service, and orders the controller's service after it.
+- Creates index 0's controller user `switch-controller` (uid 2000), the
+ agents' group `switch-agents-2000` (gid 2001) and one user per agent,
+ `sa2000-01`… (uid 2101…). The boot creates the other indexes' users.
+- Installs the boot service, and orders index 0's controller service after it.
- Writes `/etc/switch-hosted/machine.json` with the controller user, Node.js, the CLI,
the `PATH` the controller and its agents run with, and the number of agent
users.
@@ -88,7 +144,8 @@ directory. Then, as root:
The ids are fixed because the data volume outlives the root volume. Every
instance of the image must give the files on it the same owners.
-`agent-users` (16 by default) is how many agents a machine can run at once.
+`agent-users` (16 by default) is how many agents each controller on a machine
+can run at once.
## Running it
diff --git a/deploy/hosted/machine/install.sh b/deploy/hosted/machine/install.sh
index 310dd3dc4..81ca9d385 100644
--- a/deploy/hosted/machine/install.sh
+++ b/deploy/hosted/machine/install.sh
@@ -6,11 +6,12 @@
# Run as root while the AMI is built, after Node.js 24 is at /opt/switch/node
# and the provider CLIs are under /opt/switch (never under a home directory:
# agents run as users of their own and cannot reach one). It installs the
-# controller at /opt/switch/controller, makes the controller's user and the
-# agents' users and group with fixed ids (the data volume outlives the root
-# volume, so its files must keep meaning the same users on every instance),
-# and installs the boot service that mounts the data volume and starts the
-# controller (switch_machine_boot.py).
+# controller at /opt/switch/controller, makes the first controller's user and
+# its agents' users and group with fixed ids (the data volume outlives the root
+# volume, so its files must keep meaning the same users on every instance; the
+# boot makes the other controllers' with ids of their own), and installs the
+# boot service that mounts the data volume and starts the controllers
+# (switch_machine_boot.py).
set -eu
if [ "$(id -u)" -ne 0 ]; then
diff --git a/deploy/hosted/machine/switch-machine-boot.service b/deploy/hosted/machine/switch-machine-boot.service
index 912bf53dd..eaea986fe 100644
--- a/deploy/hosted/machine/switch-machine-boot.service
+++ b/deploy/hosted/machine/switch-machine-boot.service
@@ -1,5 +1,5 @@
[Unit]
-Description=Mount this Switch cloud machine's data volume and start its agents controller
+Description=Mount this Switch cloud machine's data volume and start its agents controllers
Wants=network-online.target
After=network-online.target
diff --git a/deploy/hosted/machine/switch_machine_boot.py b/deploy/hosted/machine/switch_machine_boot.py
index 59214a01f..40ce1b15c 100644
--- a/deploy/hosted/machine/switch_machine_boot.py
+++ b/deploy/hosted/machine/switch_machine_boot.py
@@ -1,16 +1,20 @@
#!/usr/bin/env python3
-"""Boot a Switch cloud machine that runs the agents controller.
+"""Boot a Switch cloud machine that runs one agents controller per workspace.
-Runs as root, once per boot, before the controller's service
+Runs as root, once per boot, before the controllers' services
(`switch-machine-boot.service`). It reads the bundle the hosted controller
-gave this machine as its instance's user data, mounts the machine's own data
-volume on /data,
-enrolls `switch-agent-controller` with the one-time code in the bundle when
-the data volume holds no live enrollment yet, and sets it up to run every
+gave this machine as its instance's user data and mounts the machine's own
+data volume on /data. Then, for each workspace seat the bundle lists, it
+enrolls a `switch-agent-controller` with the seat's one-time code when the
+data volume holds no live enrollment for it yet, and sets it up to run every
agent as a Linux user of its own (`install-service --separate-users`), its
state and its agents' directories on /data.
-The bundle never holds a long-lived credential: the controller keeps the one
+Each seat keeps an index on the data volume, from which its users, ids and
+paths are derived, so they mean the same on every instance. A seat that is no
+longer in the bundle has its controller stopped; its data stays on the volume.
+
+The bundle never holds a long-lived credential: each controller keeps the one
it enrolls with in its data directory, on the machine's own volume.
"""
@@ -41,10 +45,26 @@ DATA_MOUNT = Path("/data")
MACHINE_CONFIG_PATH = Path("/etc/switch-hosted/machine.json")
LOCK_PATH = Path("/run/lock/switch-machine-boot.lock")
MARKER_NAME = ".switch-machine.json"
+# Index 0's layout, which predates several controllers per machine.
CONTROLLER_DIR_NAME = ".switch-controller"
-# The SHA-256 of the code the enrollment in CONTROLLER_DIR_NAME was made with.
+# The SHA-256 of the code the enrollment in a data directory was made with.
CODE_RECORD_NAME = ".switch-controller-code"
AGENTS_DIR_NAME = "agents"
+# The other indexes' directories, each in CONTROLLERS_DIR_NAME/<index>.
+CONTROLLERS_DIR_NAME = "controllers"
+INDEXES_NAME = ".switch-controllers.json"
+MAX_CONTROLLERS = 8
+# The ids the image gives index 0 (install.sh); index k's are offset by k * UID_STRIDE.
+FIRST_CONTROLLER_UID = 2000
+UID_STRIDE = 200
+SYSTEMD_SYSTEM_DIR = Path("/etc/systemd/system")
+# What install.sh installs as controller-after-boot.conf for index 0.
+AFTER_BOOT_DROP_IN = """# The controller's state is on the data volume the boot service mounts.
+[Unit]
+Requires=switch-machine-boot.service
+After=switch-machine-boot.service
+RequiresMountsFor=/data
+"""
SYSTEMD_MOUNT = "/usr/bin/systemd-mount"
VOLUME_RE = re.compile(r"^vol-[0-9a-f]{8,17}$")
IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@/+\-]{0,199}$")
@@ -52,6 +72,10 @@ ENROLLMENT_CODE_RE = re.compile(r"^swce_[A-Za-z0-9_-]{16,128}$")
UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
MAX_JSON_BYTES = 128 * 1024
VOLUME_WAIT_SECONDS = 600
+# A controller that fails to enroll or start is tried again this many times,
+# this far apart, before the boot leaves it stopped and the others running.
+CONTROLLER_RETRIES = 4
+CONTROLLER_RETRY_SECONDS = 30
METADATA_WAIT_SECONDS = 300
# Instance metadata, IMDSv2: agents cannot reach it, root can.
METADATA_URL = "http://169.254.169.254/latest"
@@ -77,14 +101,82 @@ class MachineConfig:
agent_users: int
+@dataclass(frozen=True)
+class BundleController:
+ """A workspace's seat on this machine, and the controller it runs there."""
+
+ key: str
+ controller_id: str | None
+ enrollment_code: str | None
+
+
@dataclass(frozen=True)
class Bundle:
installation_id: str
machine_id: str
volume_id: str
api_endpoint: str
- controller_id: str | None
- enrollment_code: str | None
+ controllers: tuple[BundleController, ...]
+
+
+@dataclass(frozen=True)
+class Seat:
+ """The users, ids and paths of the controller with a given index."""
+
+ index: int
+ uid: int
+ user: str
+ data_dir: Path
+ code_record: Path
+ agents_dir: Path
+
+ @property
+ def gid(self) -> int:
+ return self.uid
+
+ @property
+ def agents_group(self) -> str:
+ return f"switch-agents-{self.uid}"
+
+ @property
+ def agents_gid(self) -> int:
+ return self.uid + 1
+
+ @property
+ def unit(self) -> str:
+ return f"switch-agent-controller-{self.uid}.service"
+
+ def agent_user(self, slot: int) -> str:
+ return f"sa{self.uid}-{slot:02d}"
+
+ def agent_uid(self, slot: int) -> int:
+ return self.uid + 100 + slot
+
+
+def seat_for(index: int, config: MachineConfig) -> Seat:
+ """Index 0 is the controller the image bakes, on the layout of a machine
+ with a single controller, so its disk keeps its enrollment."""
+ if not 0 <= index < MAX_CONTROLLERS:
+ raise BootError(f"There is no controller index {index}.")
+ uid = FIRST_CONTROLLER_UID + UID_STRIDE * index
+ if index == 0:
+ return Seat(
+ index=0,
+ uid=uid,
+ user=config.controller_user,
+ data_dir=DATA_MOUNT / CONTROLLER_DIR_NAME,
+ code_record=DATA_MOUNT / CODE_RECORD_NAME,
+ agents_dir=DATA_MOUNT / AGENTS_DIR_NAME,
+ )
+ home = DATA_MOUNT / CONTROLLERS_DIR_NAME / str(index)
+ return Seat(
+ index=index,
+ uid=uid,
+ user=f"{config.controller_user}-{index}",
+ data_dir=home / "data",
+ code_record=home / "code",
+ agents_dir=home / "agents",
+ )
def _load_json(path: Path, what: str) -> Any:
@@ -147,6 +239,8 @@ def parse_bundle(raw: str) -> Bundle:
"This instance's user data is not a machine bundle: only an instance the "
"hosted controller launched can boot as a Switch cloud machine."
) from None
+ if isinstance(value, dict) and value.get("version") != 5:
+ raise BootError("The machine bundle's version is not one this image reads.")
value = _strict(
value,
{
@@ -155,12 +249,10 @@ def parse_bundle(raw: str) -> Bundle:
"machineId",
"dataVolumeId",
"apiEndpoint",
- "controller",
+ "controllers",
},
"machine bundle",
)
- if value["version"] != 4:
- raise BootError("The machine bundle's version is not one this image reads.")
machine_id = value["machineId"]
if not isinstance(machine_id, str) or not UUID_RE.fullmatch(machine_id):
raise BootError("The machine bundle's machine id is invalid.")
@@ -171,9 +263,29 @@ def parse_bundle(raw: str) -> Bundle:
url = urlsplit(endpoint) if isinstance(endpoint, str) else None
if url is None or url.scheme != "https" or not url.hostname:
raise BootError("The machine bundle's server is not an HTTPS URL.")
- controller = _strict(
- value["controller"], {"id", "enrollmentCode"}, "machine bundle's controller"
+ entries = value["controllers"]
+ if not isinstance(entries, list) or not 1 <= len(entries) <= MAX_CONTROLLERS:
+ raise BootError(f"The machine bundle must list 1 to {MAX_CONTROLLERS} controllers.")
+ controllers = tuple(_bundle_controller(entry) for entry in entries)
+ if len({controller.key for controller in controllers}) != len(controllers):
+ raise BootError("The machine bundle lists a controller key twice.")
+ ids = [c.controller_id for c in controllers if c.controller_id is not None]
+ if len(set(ids)) != len(ids):
+ raise BootError("The machine bundle lists a controller twice.")
+ return Bundle(
+ installation_id=_identifier(value["installationId"], "installation id"),
+ machine_id=machine_id,
+ volume_id=volume_id,
+ api_endpoint=endpoint,
+ controllers=controllers,
)
+
+
+def _bundle_controller(entry: Any) -> BundleController:
+ controller = _strict(entry, {"key", "id", "enrollmentCode"}, "machine bundle's controller")
+ key = controller["key"]
+ if not isinstance(key, str) or not UUID_RE.fullmatch(key):
+ raise BootError("The machine bundle's controller key is invalid.")
controller_id, code = controller["id"], controller["enrollmentCode"]
if controller_id is not None:
if (
@@ -183,17 +295,8 @@ def parse_bundle(raw: str) -> Bundle:
):
raise BootError("The machine bundle's controller is invalid.")
elif not isinstance(code, str) or not ENROLLMENT_CODE_RE.fullmatch(code):
- raise BootError(
- "The machine bundle holds neither a controller nor an enrollment code."
- )
- return Bundle(
- installation_id=_identifier(value["installationId"], "installation id"),
- machine_id=machine_id,
- volume_id=volume_id,
- api_endpoint=endpoint,
- controller_id=controller_id,
- enrollment_code=code,
- )
+ raise BootError("The machine bundle holds neither a controller nor an enrollment code.")
+ return BundleController(key=key, controller_id=controller_id, enrollment_code=code)
def _user_data(opener: OpenerDirector) -> str:
@@ -204,9 +307,7 @@ def _user_data(opener: OpenerDirector) -> str:
)
with opener.open(token_request, timeout=5) as response:
token = response.read().decode()
- request = Request(
- f"{METADATA_URL}/user-data", headers={"X-aws-ec2-metadata-token": token}
- )
+ request = Request(f"{METADATA_URL}/user-data", headers={"X-aws-ec2-metadata-token": token})
try:
with opener.open(request, timeout=5) as response:
raw = response.read(MAX_JSON_BYTES + 1)
@@ -240,9 +341,7 @@ def read_bundle(
raise
except Exception as error:
if time.monotonic() > deadline:
- raise BootError(
- f"The instance's user data cannot be read: {error}"
- ) from None
+ raise BootError(f"The instance's user data cannot be read: {error}") from None
logger.warning("Waiting for instance metadata: %s", error)
sleep(5)
return parse_bundle(raw)
@@ -329,17 +428,13 @@ def prepare_storage(
if signatures or device.get("uuid"):
raise BootError("The data volume is not blank, but holds no filesystem.")
logger.info("Formatting the blank data volume %s", volume_id)
- commands.run(
- ["/usr/sbin/mkfs.ext4", "-q", "-m", "0", "-L", "switch-data", path]
- )
+ commands.run(["/usr/sbin/mkfs.ext4", "-q", "-m", "0", "-L", "switch-data", path])
commands.run(["/usr/bin/udevadm", "settle", "--timeout=30"])
device = _data_device(commands, volume_id)
if device is None:
raise BootError("The data volume went away while it was formatted.")
if device.get("fstype") != "ext4":
- raise BootError(
- "The data volume's filesystem is not ext4; refusing to mount it."
- )
+ raise BootError("The data volume's filesystem is not ext4; refusing to mount it.")
DATA_MOUNT.mkdir(mode=0o755, exist_ok=True)
mounted = commands.result(
[
@@ -355,9 +450,7 @@ def prepare_storage(
if os.path.realpath(mounted.stdout.strip()) != os.path.realpath(path):
raise BootError("/data is another device's mountpoint.")
return
- commands.run(
- [SYSTEMD_MOUNT, "--type=ext4", "--options=nodev,nosuid", path, str(DATA_MOUNT)]
- )
+ commands.run([SYSTEMD_MOUNT, "--type=ext4", "--options=nodev,nosuid", path, str(DATA_MOUNT)])
def reconcile_marker(data: Path, bundle: Bundle) -> None:
@@ -374,20 +467,175 @@ def reconcile_marker(data: Path, bundle: Bundle) -> None:
if marker.is_symlink():
raise BootError("The data volume's marker is a link.")
if json.loads(marker.read_text()) != wanted:
- raise BootError(
- "The data volume belongs to another machine; refusing to use it."
- )
+ raise BootError("The data volume belongs to another machine; refusing to use it.")
return
- temporary = data / f"{MARKER_NAME}.new"
- temporary.write_text(json.dumps(wanted))
- os.chmod(temporary, 0o600)
- os.replace(temporary, marker)
+ _write_atomically(marker, json.dumps(wanted), 0o600)
+
+
+def _write_atomically(path: Path, text: str, mode: int) -> None:
+ """Writes `path` through a fresh file renamed over it, never through a link."""
+ temporary = path.with_name(f"{path.name}.new")
+ temporary.unlink(missing_ok=True)
+ descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
+ with os.fdopen(descriptor, "w") as file:
+ os.fchmod(file.fileno(), mode)
+ file.write(text)
+ file.flush()
+ os.fsync(file.fileno())
+ os.replace(temporary, path)
+
+
+def load_indexes(data: Path) -> dict[str, int] | None:
+ """Each seat's index, as recorded on the data volume; None before the first."""
+ path = data / INDEXES_NAME
+ if not path.exists() and not path.is_symlink():
+ return None
+ value = _strict(
+ _load_json(path, "controllers' indexes"),
+ {"version", "controllers"},
+ "controllers' indexes",
+ )
+ indexes = value["controllers"]
+ if (
+ value["version"] != 1
+ or not isinstance(indexes, dict)
+ or not all(isinstance(key, str) and UUID_RE.fullmatch(key) for key in indexes)
+ or not all(
+ isinstance(index, int) and not isinstance(index, bool) and 0 <= index < MAX_CONTROLLERS
+ for index in indexes.values()
+ )
+ or len(set(indexes.values())) != len(indexes)
+ ):
+ raise BootError(f"The controllers' indexes at {path} are not ones this image reads.")
+ return dict(indexes)
+
+
+def assign_indexes(recorded: dict[str, int] | None, keys: list[str]) -> dict[str, int]:
+ """Every seat's index: the recorded ones, kept even for a seat that left the
+ machine, and the lowest free one for each new seat, in bundle order. A seat
+ there is no free index for is left out."""
+ indexes = dict(recorded or {})
+ for key in keys:
+ if key in indexes:
+ continue
+ free = [i for i in range(MAX_CONTROLLERS) if i not in indexes.values()]
+ if free:
+ indexes[key] = free[0]
+ return indexes
+
+
+def save_indexes(data: Path, indexes: dict[str, int]) -> None:
+ _write_atomically(
+ data / INDEXES_NAME,
+ json.dumps({"version": 1, "controllers": indexes}, indent=2, sort_keys=True),
+ 0o600,
+ )
+
+
+def _getent(commands: Commands, database: str, name: str) -> list[str] | None:
+ """The entry for `name`, split into its fields, or None when there is none."""
+ found = commands.result(["/usr/bin/getent", database, name])
+ if found.returncode == 2:
+ return None
+ if found.returncode != 0:
+ raise BootError(f"getent {database} {name} failed.")
+ return found.stdout.strip().split(":")
+
+
+def _ensure_group(commands: Commands, group: str, gid: int) -> None:
+ entry = _getent(commands, "group", group)
+ if entry is None:
+ commands.run(["/usr/sbin/groupadd", "--system", "--gid", str(gid), group])
+ elif len(entry) < 3 or entry[2] != str(gid):
+ raise BootError(
+ f"The group {group} exists with another gid than {gid}; refusing to use it."
+ )
+
+
+def _ensure_user(
+ commands: Commands, user: str, uid: int, group: str, gid: int, comment: str
+) -> None:
+ entry = _getent(commands, "passwd", user)
+ if entry is None:
+ commands.run(
+ [
+ "/usr/sbin/useradd",
+ "--system",
+ "--uid",
+ str(uid),
+ "--gid",
+ group,
+ "--no-create-home",
+ "--home-dir",
+ "/nonexistent",
+ "--shell",
+ "/usr/sbin/nologin",
+ "--comment",
+ comment,
+ user,
+ ]
+ )
+ elif len(entry) < 4 or entry[2] != str(uid) or entry[3] != str(gid):
+ raise BootError(
+ f"The user {user} exists with another uid or group than {uid}:{gid}; "
+ "refusing to use it."
+ )
-def enrolled(commands: Commands, config: MachineConfig, data_dir: Path) -> bool:
- """Whether the data directory holds an enrollment that was not revoked."""
+def ensure_identities(commands: Commands, config: MachineConfig, seat: Seat) -> None:
+ """Creates the controller's user and its agents' group and users with the
+ seat's fixed ids, as install.sh does for index 0, so that install-service
+ finds them all and creates none with ids of its own choosing."""
+ _ensure_group(commands, seat.user, seat.gid)
+ _ensure_user(commands, seat.user, seat.uid, str(seat.gid), seat.gid, "Switch agents controller")
+ _ensure_group(commands, seat.agents_group, seat.agents_gid)
+ for slot in range(1, config.agent_users + 1):
+ _ensure_user(
+ commands,
+ seat.agent_user(slot),
+ seat.agent_uid(slot),
+ seat.agents_group,
+ seat.agents_gid,
+ "Switch agent",
+ )
+
+
+def write_drop_in(seat: Seat) -> None:
+ """Orders the seat's controller unit after this service, as install.sh
+ does for index 0. The root volume can be new, so it is written every boot."""
+ directory = SYSTEMD_SYSTEM_DIR / f"{seat.unit}.d"
+ directory.mkdir(mode=0o755, parents=True, exist_ok=True)
+ path = directory / "after-boot.conf"
+ if path.is_symlink() or not path.exists() or path.read_text() != AFTER_BOOT_DROP_IN:
+ _write_atomically(path, AFTER_BOOT_DROP_IN, 0o644)
+
+
+def stop_controller(commands: Commands, seat: Seat) -> None:
+ """Stops and disables a controller whose seat left the machine, and any of
+ its agents still running. Its data stays on the volume."""
+ stopped = commands.result(["/usr/bin/systemctl", "disable", "--now", seat.unit])
+ output = f"{stopped.stdout}\n{stopped.stderr}"
+ if stopped.returncode != 0 and not ("not loaded" in output or "does not exist" in output):
+ raise BootError(f"{seat.unit} cannot be stopped: {output.strip()}")
+ commands.run(["/usr/bin/systemctl", "stop", f"switch-agent-{seat.uid}@*.service"])
+
+
+def _ensure_root_directories(path: Path) -> None:
+ """Creates `path` and its parents below /data, root's and 0755, refusing a link."""
+ current = DATA_MOUNT
+ for part in path.relative_to(DATA_MOUNT).parts:
+ current = current / part
+ if current.is_symlink():
+ raise BootError(f"{current} is a link; refusing to use it.")
+ if not current.exists():
+ current.mkdir(mode=0o755)
+ os.chmod(current, 0o755)
+
+
+def enrolled(commands: Commands, config: MachineConfig, seat: Seat) -> bool:
+ """Whether the seat's data directory holds an enrollment that was not revoked."""
status = commands.result(
- _as_controller(config, [*_cli(config), "status", "--data-dir", str(data_dir)]),
+ _as_controller(seat, [*_cli(config), "status", "--data-dir", str(seat.data_dir)]),
env=_environment(config),
)
if status.returncode != 0:
@@ -405,58 +653,60 @@ def _environment(config: MachineConfig) -> dict[str, str]:
return {"PATH": config.path, "LANG": "C.UTF-8"}
-def _as_controller(config: MachineConfig, arguments: list[str]) -> list[str]:
- return ["/usr/sbin/runuser", "-u", config.controller_user, "--", *arguments]
+def _as_controller(seat: Seat, arguments: list[str]) -> list[str]:
+ return ["/usr/sbin/runuser", "-u", seat.user, "--", *arguments]
def start_controller(
commands: Commands,
config: MachineConfig,
- bundle: Bundle,
+ api_endpoint: str,
+ seat: Seat,
+ controller: BundleController,
now: Callable[[], float] = time.time,
) -> None:
- """Enrolls the controller when it must, then sets it up and starts it."""
- data_dir = DATA_MOUNT / CONTROLLER_DIR_NAME
+ """Enrolls the seat's controller when it must, then sets it up and starts it."""
+ _ensure_root_directories(seat.data_dir.parent)
+ data_dir = seat.data_dir
if not data_dir.exists():
data_dir.mkdir(mode=0o700)
- commands.run(["/usr/bin/chown", f"{config.controller_user}:", str(data_dir)])
- record = DATA_MOUNT / CODE_RECORD_NAME
- code = bundle.enrollment_code
+ commands.run(["/usr/bin/chown", f"{seat.user}:", str(data_dir)])
+ record = seat.code_record
+ code = controller.enrollment_code
if code is not None:
code_hash = hashlib.sha256(code.encode()).hexdigest()
- if enrolled(commands, config, data_dir):
+ if enrolled(commands, config, seat):
if _recorded(record) == code_hash:
# An earlier attempt of this boot enrolled with this very code
# and failed later on. Switch has not linked the controller
# yet, so it still hands the code over, but the code is spent.
- logger.info("The controller is already enrolled with this code")
+ logger.info("%s is already enrolled with this code", seat.user)
code = None
else:
# A new code, so Switch holds no live controller for this
- # machine: whatever this directory holds was revoked there.
- aside = DATA_MOUNT / f"{CONTROLLER_DIR_NAME}.replaced-{int(now())}"
+ # seat: whatever this directory holds was revoked there.
+ aside = data_dir.with_name(f"{data_dir.name}.replaced-{int(now())}")
logger.warning(
- "Switch gave a new code; setting the old enrollment aside in %s",
+ "Switch gave %s a new code; setting the old enrollment aside in %s",
+ seat.user,
aside,
)
os.replace(data_dir, aside)
data_dir.mkdir(mode=0o700)
- commands.run(
- ["/usr/bin/chown", f"{config.controller_user}:", str(data_dir)]
- )
+ commands.run(["/usr/bin/chown", f"{seat.user}:", str(data_dir)])
if code is not None:
# Recorded first: a crash right after enrolling must not leave an
# enrollment the next attempt would take for a revoked one.
_record(record, code_hash)
- logger.info("Enrolling the controller")
+ logger.info("Enrolling %s", seat.user)
commands.run(
_as_controller(
- config,
+ seat,
[
*_cli(config),
"enroll",
"--server",
- bundle.api_endpoint,
+ api_endpoint,
"--code",
code,
"--name",
@@ -469,11 +719,11 @@ def start_controller(
),
env=_environment(config),
)
- elif bundle.enrollment_code is None and not enrolled(commands, config, data_dir):
+ elif controller.enrollment_code is None and not enrolled(commands, config, seat):
raise BootError(
"The data volume holds no live enrollment, and Switch gave no code to enroll "
- "with. Remove this machine's controller in the Machines page; the next start "
- "gets a new code."
+ "with. Remove this machine's controller in the workspace's Machines page; the "
+ "next start gets a new code."
)
commands.run(
[
@@ -481,11 +731,11 @@ def start_controller(
"install-service",
"--separate-users",
"--user",
- config.controller_user,
+ seat.user,
"--data-dir",
str(data_dir),
"--agents-dir",
- str(DATA_MOUNT / AGENTS_DIR_NAME),
+ str(seat.agents_dir),
"--agent-users",
str(config.agent_users),
# The controller's unit is ordered after this service: a restart
@@ -506,10 +756,77 @@ def _recorded(path: Path) -> str | None:
def _record(path: Path, code_hash: str) -> None:
- temporary = path.with_name(f"{path.name}.new")
- temporary.write_text(code_hash)
- os.chmod(temporary, 0o600)
- os.replace(temporary, path)
+ _write_atomically(path, code_hash, 0o600)
+
+
+def boot_controllers(
+ commands: Commands,
+ config: MachineConfig,
+ bundle: Bundle,
+ now: Callable[[], float] = time.time,
+) -> dict[str, str]:
+ """Starts every controller the bundle lists and stops those whose seat left
+ the machine. One that fails does not keep the others from starting: the
+ failures are returned by seat key, one message each."""
+ keys = [controller.key for controller in bundle.controllers]
+ recorded = load_indexes(DATA_MOUNT)
+ indexes = assign_indexes(recorded, keys)
+ if indexes != recorded:
+ save_indexes(DATA_MOUNT, indexes)
+ failures: dict[str, str] = {}
+ for key, index in sorted(indexes.items(), key=lambda item: item[1]):
+ if key in keys:
+ continue
+ seat = seat_for(index, config)
+ logger.warning(
+ "The workspace seat %s left this machine: stopping its controller %s. "
+ "Its data stays on the volume.",
+ key,
+ seat.unit,
+ )
+ try:
+ stop_controller(commands, seat)
+ except BootError as error:
+ failures[key] = f"The controller of the seat {key} that left: {error}"
+ seats = {key: seat_for(indexes[key], config) for key in keys if key in indexes}
+ for seat in seats.values():
+ if seat.index:
+ write_drop_in(seat)
+ commands.run(["/usr/bin/systemctl", "daemon-reload"])
+ failures.update(start_controllers(commands, config, bundle, indexes, set(keys), now))
+ return failures
+
+
+def start_controllers(
+ commands: Commands,
+ config: MachineConfig,
+ bundle: Bundle,
+ indexes: dict[str, int],
+ keys: set[str],
+ now: Callable[[], float] = time.time,
+) -> dict[str, str]:
+ """Enrolls when it must and starts the controllers of the seats `keys`;
+ the failures by seat key."""
+ failures: dict[str, str] = {}
+ for controller in bundle.controllers:
+ if controller.key not in keys:
+ continue
+ if controller.key not in indexes:
+ failures[controller.key] = (
+ f"The seat {controller.key} has no free controller index: all "
+ f"{MAX_CONTROLLERS} are held by this machine's other seats, present or past."
+ )
+ continue
+ seat = seat_for(indexes[controller.key], config)
+ try:
+ if seat.index:
+ ensure_identities(commands, config, seat)
+ start_controller(commands, config, bundle.api_endpoint, seat, controller, now)
+ except (BootError, OSError) as error:
+ failures[controller.key] = (
+ f"The controller {seat.user} (seat {controller.key}): {error}"
+ )
+ return failures
def acquire_lock(path: Path = LOCK_PATH) -> Any:
@@ -537,14 +854,30 @@ def main(argv: list[str] | None = None) -> int:
commands = Commands()
prepare_storage(commands, bundle.volume_id)
reconcile_marker(DATA_MOUNT, bundle)
- start_controller(commands, config, bundle)
+ failures = boot_controllers(commands, config, bundle)
+ indexes = load_indexes(DATA_MOUNT) or {}
+ for _attempt in range(CONTROLLER_RETRIES):
+ if not failures or set(failures) - set(indexes):
+ break
+ for failure in failures.values():
+ logger.warning("%s; trying again", failure)
+ time.sleep(CONTROLLER_RETRY_SECONDS)
+ failures = start_controllers(commands, config, bundle, indexes, set(failures))
except NoBundle as error:
logger.error("%s", error)
return NO_BUNDLE_EXIT
except BootError as error:
logger.error("%s", error)
return 1
- logger.info("The controller is running")
+ # Not a failed boot: the controllers this boot started are ordered after it
+ # (`controller-after-boot.conf`), so failing it would stop them too.
+ for failure in failures.values():
+ logger.error("%s", failure)
+ logger.info(
+ "%d of the machine's %d controllers are running",
+ len(bundle.controllers) - len(failures),
+ len(bundle.controllers),
+ )
return 0
diff --git a/deploy/hosted/machine/tests/test_boot.py b/deploy/hosted/machine/tests/test_boot.py
index c5570ab0c..4b0922e55 100644
--- a/deploy/hosted/machine/tests/test_boot.py
+++ b/deploy/hosted/machine/tests/test_boot.py
@@ -1,11 +1,15 @@
-"""The boot of a Switch cloud machine that runs the agents controller."""
+"""The boot of a Switch cloud machine that runs one agents controller per workspace."""
from __future__ import annotations
+import contextlib
+import dataclasses
import hashlib
import importlib.util
import io
import json
+import logging
+import stat
import subprocess
import sys
from pathlib import Path
@@ -22,6 +26,10 @@ spec.loader.exec_module(boot)
MACHINE_ID = "3f1c2b4a-0000-4000-8000-000000000001"
CONTROLLER_ID = "9a1c2b4a-0000-4000-8000-0000000000aa"
+OTHER_CONTROLLER_ID = "9a1c2b4a-0000-4000-8000-0000000000bb"
+KEY_A = "5e1c2b4a-0000-4000-8000-00000000000a"
+KEY_B = "5e1c2b4a-0000-4000-8000-00000000000b"
+KEY_C = "5e1c2b4a-0000-4000-8000-00000000000c"
CODE = "swce_SyntheticEnrollmentCode0000"
VOLUME_ID = "vol-0123456789abcdef0"
CONFIG = boot.MachineConfig(
@@ -33,19 +41,32 @@ CONFIG = boot.MachineConfig(
)
-def bundle(**controller) -> str:
+def entry(key: str = KEY_A, **fields) -> dict:
+ return {"key": key, "id": None, "enrollmentCode": CODE, **fields}
+
+
+def enrolled_entry(key: str, controller_id: str = CONTROLLER_ID) -> dict:
+ return entry(key, id=controller_id, enrollmentCode=None)
+
+
+def bundle(*controllers: dict, **fields) -> str:
return json.dumps(
{
- "version": 4,
+ "version": 5,
"installationId": "inst-test",
"machineId": MACHINE_ID,
"dataVolumeId": VOLUME_ID,
"apiEndpoint": "https://switch.example.test/agent-api",
- "controller": {"id": None, "enrollmentCode": CODE, **controller},
+ "controllers": list(controllers) if controllers else [entry()],
+ **fields,
}
)
+def uuid_key(n: int) -> str:
+ return f"5e1c2b4a-0000-4000-8000-{n:012d}"
+
+
class FakeCommands:
"""Answers host commands from a table, and records them."""
@@ -73,30 +94,80 @@ class FakeCommands:
return [call for call in self.calls if word in call]
+def machine(monkeypatch, tmp_path) -> Path:
+ """A data volume and a root volume's systemd directory, both empty."""
+ data = tmp_path / "data"
+ data.mkdir()
+ monkeypatch.setattr(boot, "DATA_MOUNT", data)
+ monkeypatch.setattr(boot, "SYSTEMD_SYSTEM_DIR", tmp_path / "systemd")
+ return data
+
+
+def start(commands, raw: str | None = None, index: int = 0, now=lambda: 7) -> None:
+ parsed = boot.parse_bundle(raw or bundle())
+ boot.start_controller(
+ commands,
+ CONFIG,
+ parsed.api_endpoint,
+ boot.seat_for(index, CONFIG),
+ parsed.controllers[0],
+ now,
+ )
+
+
+def setups(commands: FakeCommands) -> list[str]:
+ """The controller users install-service was run for, in order."""
+ return [call[call.index("--user") + 1] for call in commands.ran("install-service")]
+
+
def test_the_bundle_is_read_strictly():
- parsed = boot.parse_bundle(bundle())
+ parsed = boot.parse_bundle(bundle(entry(KEY_A), enrolled_entry(KEY_B)))
assert parsed == boot.Bundle(
installation_id="inst-test",
machine_id=MACHINE_ID,
volume_id=VOLUME_ID,
api_endpoint="https://switch.example.test/agent-api",
- controller_id=None,
- enrollment_code=CODE,
+ controllers=(
+ boot.BundleController(key=KEY_A, controller_id=None, enrollment_code=CODE),
+ boot.BundleController(key=KEY_B, controller_id=CONTROLLER_ID, enrollment_code=None),
+ ),
)
- enrolled = boot.parse_bundle(bundle(id=CONTROLLER_ID, enrollmentCode=None))
- assert (enrolled.controller_id, enrolled.enrollment_code) == (CONTROLLER_ID, None)
+
+
+V4_BUNDLE = json.dumps(
+ {
+ "version": 4,
+ "installationId": "inst-test",
+ "machineId": MACHINE_ID,
+ "dataVolumeId": VOLUME_ID,
+ "apiEndpoint": "https://switch.example.test/agent-api",
+ "controller": {"id": None, "enrollmentCode": CODE},
+ }
+)
@pytest.mark.parametrize(
"raw",
[
- bundle(enrollmentCode=None),
- bundle(enrollmentCode="swcc_a-long-lived-credential-00"),
- bundle(id=CONTROLLER_ID),
- bundle().replace('"version": 4', '"version": 3'),
+ V4_BUNDLE,
+ bundle(version=4),
+ bundle(entry(enrollmentCode=None)),
+ bundle(entry(enrollmentCode="swcc_a-long-lived-credential-00")),
+ bundle(entry(id=CONTROLLER_ID)),
+ bundle(entry(key="seat-a")),
+ bundle(entry(key=KEY_A.upper())),
+ bundle({"id": None, "enrollmentCode": CODE}),
+ bundle({**entry(), "slotId": "slot-a"}),
+ bundle(entry(KEY_A), entry(KEY_A)),
+ bundle(enrolled_entry(KEY_A), enrolled_entry(KEY_B)),
+ bundle(controllers=[]),
+ bundle(*[entry(uuid_key(n)) for n in range(9)]),
+ bundle(controllers=entry()),
bundle().replace(VOLUME_ID, "vol-nope"),
bundle().replace("https://", "http://"),
json.dumps({**json.loads(bundle()), "slotId": "slot-a"}),
+ json.dumps({**json.loads(bundle()), "controller": entry()}),
],
)
def test_a_bundle_this_image_does_not_read_is_refused(raw):
@@ -104,6 +175,11 @@ def test_a_bundle_this_image_does_not_read_is_refused(raw):
boot.parse_bundle(raw)
+def test_a_bundle_lists_up_to_eight_controllers():
+ parsed = boot.parse_bundle(bundle(*[entry(uuid_key(n)) for n in range(8)]))
+ assert len(parsed.controllers) == 8
+
+
def test_user_data_that_is_not_a_bundle_is_not_a_cloud_machine():
with pytest.raises(boot.NoBundle, match="not a machine bundle"):
boot.parse_bundle("#cloud-config\nruncmd: []\n")
@@ -195,9 +271,7 @@ def test_a_blank_volume_is_formatted_and_mounted(monkeypatch, tmp_path):
]
-def test_a_volume_attached_late_is_waited_for_and_a_formatted_one_kept(
- monkeypatch, tmp_path
-):
+def test_a_volume_attached_late_is_waited_for_and_a_formatted_one_kept(monkeypatch, tmp_path):
monkeypatch.setattr(boot, "DATA_MOUNT", tmp_path / "data")
commands = FakeCommands(
{
@@ -225,20 +299,282 @@ def test_a_disk_with_unknown_contents_is_never_formatted(monkeypatch, tmp_path):
def test_the_volume_marker_refuses_another_machines_disk(tmp_path):
boot.reconcile_marker(tmp_path, boot.parse_bundle(bundle()))
- boot.reconcile_marker(
- tmp_path, boot.parse_bundle(bundle(id=CONTROLLER_ID, enrollmentCode=None))
- )
+ boot.reconcile_marker(tmp_path, boot.parse_bundle(bundle(enrolled_entry(KEY_A))))
other = bundle().replace(MACHINE_ID, "3f1c2b4a-0000-4000-8000-000000000002")
with pytest.raises(boot.BootError, match="another machine"):
boot.reconcile_marker(tmp_path, boot.parse_bundle(other))
-def test_a_fresh_machine_enrolls_then_runs_its_agents_as_users_of_their_own(
- monkeypatch, tmp_path
-):
- monkeypatch.setattr(boot, "DATA_MOUNT", tmp_path)
+def test_index_zero_is_the_baked_controller_on_the_single_controller_layout(machine):
+ seat = boot.seat_for(0, CONFIG)
+ assert (seat.uid, seat.gid, seat.user) == (2000, 2000, "switch-controller")
+ assert (seat.agents_group, seat.agents_gid) == ("switch-agents-2000", 2001)
+ assert (seat.agent_user(1), seat.agent_uid(1)) == ("sa2000-01", 2101)
+ assert (seat.agent_user(16), seat.agent_uid(16)) == ("sa2000-16", 2116)
+ assert seat.unit == "switch-agent-controller-2000.service"
+ assert seat.data_dir == machine / ".switch-controller"
+ assert seat.code_record == machine / ".switch-controller-code"
+ assert seat.agents_dir == machine / "agents"
+
+
+def test_another_index_has_ids_and_paths_of_its_own(machine):
+ seat = boot.seat_for(3, CONFIG)
+ assert (seat.uid, seat.gid, seat.user) == (2600, 2600, "switch-controller-3")
+ assert (seat.agents_group, seat.agents_gid) == ("switch-agents-2600", 2601)
+ assert (seat.agent_user(1), seat.agent_uid(1)) == ("sa2600-01", 2701)
+ assert (seat.agent_user(99), seat.agent_uid(99)) == ("sa2600-99", 2799)
+ assert seat.unit == "switch-agent-controller-2600.service"
+ assert seat.data_dir == machine / "controllers" / "3" / "data"
+ assert seat.code_record == machine / "controllers" / "3" / "code"
+ assert seat.agents_dir == machine / "controllers" / "3" / "agents"
+ last = boot.seat_for(7, CONFIG)
+ assert last.agent_uid(99) < boot.seat_for(0, CONFIG).uid + 8 * boot.UID_STRIDE
+ with pytest.raises(boot.BootError):
+ boot.seat_for(8, CONFIG)
+
+
+def test_a_new_seat_gets_the_lowest_free_index_and_a_recorded_one_keeps_its_own():
+ assert boot.assign_indexes(None, [KEY_B, KEY_A]) == {KEY_B: 0, KEY_A: 1}
+ assert boot.assign_indexes({KEY_A: 0, KEY_C: 2}, [KEY_C, KEY_B]) == {
+ KEY_A: 0,
+ KEY_B: 1,
+ KEY_C: 2,
+ }
+ full = {uuid_key(n): n for n in range(8)}
+ assert boot.assign_indexes(full, [KEY_A]) == full
+
+
+def test_the_first_boot_numbers_the_seats_in_bundle_order(machine):
+ commands = FakeCommands({"getent": [(2, "")]})
+ raw = bundle(enrolled_entry(KEY_B), enrolled_entry(KEY_A, OTHER_CONTROLLER_ID))
+ assert boot.boot_controllers(commands, CONFIG, boot.parse_bundle(raw)) == {}
+ path = machine / ".switch-controllers.json"
+ assert json.loads(path.read_text()) == {"version": 1, "controllers": {KEY_B: 0, KEY_A: 1}}
+ assert stat.S_IMODE(path.lstat().st_mode) == 0o600
+ assert setups(commands) == ["switch-controller", "switch-controller-1"]
+
+
+def test_a_seat_that_left_keeps_its_index_and_its_data_and_its_controller_is_stopped(machine):
+ boot.save_indexes(machine, {KEY_A: 0, KEY_B: 1})
+ left = boot.seat_for(1, CONFIG)
+ left.data_dir.mkdir(parents=True)
+ (left.data_dir / "controller.db").write_text("kept")
+ commands = FakeCommands()
+ raw = bundle(enrolled_entry(KEY_A))
+ assert boot.boot_controllers(commands, CONFIG, boot.parse_bundle(raw)) == {}
+ assert ["/usr/bin/systemctl", "disable", "--now", "switch-agent-controller-2200.service"] in (
+ commands.calls
+ )
+ assert ["/usr/bin/systemctl", "stop", "switch-agent-2200@*.service"] in commands.calls
+ assert (left.data_dir / "controller.db").read_text() == "kept"
+ assert setups(commands) == ["switch-controller"]
+ assert boot.load_indexes(machine) == {KEY_A: 0, KEY_B: 1}
+
+ again = FakeCommands({"getent": [(2, "")]})
+ raw = bundle(enrolled_entry(KEY_A), enrolled_entry(KEY_C, OTHER_CONTROLLER_ID))
+ assert boot.boot_controllers(again, CONFIG, boot.parse_bundle(raw)) == {}
+ assert boot.load_indexes(machine) == {KEY_A: 0, KEY_B: 1, KEY_C: 2}
+ assert setups(again) == ["switch-controller", "switch-controller-2"]
+
+
+def test_a_controller_that_left_is_stopped_even_when_its_unit_is_not_on_this_root_volume(machine):
+ boot.save_indexes(machine, {KEY_A: 0, KEY_B: 1})
+ missing = (
+ "Failed to disable unit: Unit file switch-agent-controller-2200.service does not exist."
+ )
+ commands = FakeCommands({"disable --now": [(1, missing)]})
+ assert (
+ boot.boot_controllers(commands, CONFIG, boot.parse_bundle(bundle(enrolled_entry(KEY_A))))
+ == {}
+ )
+ failing = FakeCommands({"disable --now": [(1, "Access denied")]})
+ [(key, failure)] = boot.boot_controllers(
+ failing, CONFIG, boot.parse_bundle(bundle(enrolled_entry(KEY_A)))
+ ).items()
+ assert key == KEY_B and KEY_B in failure
+ assert setups(failing) == ["switch-controller"]
+
+
+def test_a_seat_without_a_free_index_fails_alone(machine):
+ boot.save_indexes(machine, {uuid_key(n): n for n in range(1, 8)} | {KEY_A: 0})
+ commands = FakeCommands()
+ raw = bundle(enrolled_entry(KEY_A), enrolled_entry(KEY_B, OTHER_CONTROLLER_ID))
+ [(key, failure)] = boot.boot_controllers(commands, CONFIG, boot.parse_bundle(raw)).items()
+ assert key == KEY_B and "no free controller index" in failure
+ assert setups(commands) == ["switch-controller"]
+
+
+def test_the_indexes_are_never_read_through_a_link(machine):
+ elsewhere = machine.parent / "elsewhere.json"
+ elsewhere.write_text(json.dumps({"version": 1, "controllers": {KEY_A: 0}}))
+ (machine / ".switch-controllers.json").symlink_to(elsewhere)
+ with pytest.raises(boot.BootError, match="cannot be read"):
+ boot.load_indexes(machine)
+ with pytest.raises(boot.BootError):
+ boot.boot_controllers(FakeCommands(), CONFIG, boot.parse_bundle(bundle()))
+
+
+def test_the_indexes_are_never_written_through_a_link(machine):
+ victim = machine.parent / "victim"
+ victim.write_text("untouched")
+ (machine / ".switch-controllers.json.new").symlink_to(victim)
+ boot.save_indexes(machine, {KEY_A: 0})
+ assert victim.read_text() == "untouched"
+ path = machine / ".switch-controllers.json"
+ assert stat.S_ISREG(path.lstat().st_mode)
+ assert boot.load_indexes(machine) == {KEY_A: 0}
+
+
+ "recorded",
+ [
+ {"version": 2, "controllers": {}},
+ {"version": 1, "controllers": {KEY_A: 0, KEY_B: 0}},
+ {"version": 1, "controllers": {KEY_A: 8}},
+ {"version": 1, "controllers": {KEY_A: True}},
+ {"version": 1, "controllers": {"seat-a": 0}},
+ {"version": 1, "controllers": [KEY_A]},
+ {"version": 1, "controllers": {}, "extra": 1},
+ ],
+)
+def test_indexes_this_image_does_not_read_are_refused(machine, recorded):
+ (machine / ".switch-controllers.json").write_text(json.dumps(recorded))
+ with pytest.raises(boot.BootError):
+ boot.load_indexes(machine)
+
+
+def test_another_index_gets_its_users_with_their_fixed_ids():
+ commands = FakeCommands({"getent": [(2, "")]})
+ seat = boot.seat_for(1, CONFIG)
+ boot.ensure_identities(commands, dataclasses.replace(CONFIG, agent_users=2), seat)
+ nologin = [
+ "--no-create-home",
+ "--home-dir",
+ "/nonexistent",
+ "--shell",
+ "/usr/sbin/nologin",
+ ]
+ assert [call for call in commands.calls if "getent" not in call[0]] == [
+ ["/usr/sbin/groupadd", "--system", "--gid", "2200", "switch-controller-1"],
+ [
+ "/usr/sbin/useradd",
+ "--system",
+ "--uid",
+ "2200",
+ "--gid",
+ "2200",
+ *nologin,
+ "--comment",
+ "Switch agents controller",
+ "switch-controller-1",
+ ],
+ ["/usr/sbin/groupadd", "--system", "--gid", "2201", "switch-agents-2200"],
+ [
+ "/usr/sbin/useradd",
+ "--system",
+ "--uid",
+ "2301",
+ "--gid",
+ "switch-agents-2200",
+ *nologin,
+ "--comment",
+ "Switch agent",
+ "sa2200-01",
+ ],
+ [
+ "/usr/sbin/useradd",
+ "--system",
+ "--uid",
+ "2302",
+ "--gid",
+ "switch-agents-2200",
+ *nologin,
+ "--comment",
+ "Switch agent",
+ "sa2200-02",
+ ],
+ ]
+
+
+def test_users_that_exist_with_their_fixed_ids_are_kept():
+ commands = FakeCommands(
+ {
+ "getent passwd switch-controller-1": [
+ (0, "switch-controller-1:x:2200:2200::/nonexistent:/usr/sbin/nologin\n")
+ ],
+ "getent group switch-controller-1": [(0, "switch-controller-1:x:2200:\n")],
+ "getent group switch-agents-2200": [(0, "switch-agents-2200:x:2201:\n")],
+ "getent passwd sa2200-01": [
+ (0, "sa2200-01:x:2301:2201::/nonexistent:/usr/sbin/nologin")
+ ],
+ }
+ )
+ seat = boot.seat_for(1, CONFIG)
+ boot.ensure_identities(commands, dataclasses.replace(CONFIG, agent_users=1), seat)
+ assert all(call[0] == "/usr/bin/getent" for call in commands.calls)
+
+
+ "answers",
+ [
+ {"getent passwd switch-controller-1": [(0, "switch-controller-1:x:4242:2200::/:/bin/sh")]},
+ {"getent passwd switch-controller-1": [(0, "switch-controller-1:x:2200:100::/:/bin/sh")]},
+ {"getent group switch-controller-1": [(0, "switch-controller-1:x:4242:")]},
+ {"getent group switch-agents-2200": [(0, "switch-agents-2200:x:4242:")]},
+ {"getent passwd sa2200-01": [(0, "sa2200-01:x:4242:2201::/:/bin/sh")]},
+ ],
+)
+def test_a_user_or_group_with_another_id_is_refused(answers):
+ commands = FakeCommands({**answers, "getent": [(2, "")]})
+ seat = boot.seat_for(1, CONFIG)
+ with pytest.raises(boot.BootError, match="another"):
+ boot.ensure_identities(commands, dataclasses.replace(CONFIG, agent_users=1), seat)
+ refused = next(name for name in answers)
+ assert commands.calls[-1] == ["/usr/bin/getent", *refused.split()[1:]]
+
+
+def test_another_indexs_unit_is_ordered_after_the_boot(machine):
+ commands = FakeCommands({"getent": [(2, "")]})
+ raw = bundle(enrolled_entry(KEY_A), enrolled_entry(KEY_B, OTHER_CONTROLLER_ID))
+ assert boot.boot_controllers(commands, CONFIG, boot.parse_bundle(raw)) == {}
+ systemd = boot.SYSTEMD_SYSTEM_DIR
+ drop_in = systemd / "switch-agent-controller-2200.service.d" / "after-boot.conf"
+ assert drop_in.read_text() == (SCRIPT.parent / "controller-after-boot.conf").read_text()
+ assert stat.S_IMODE(drop_in.stat().st_mode) == 0o644
+ assert not (systemd / "switch-agent-controller-2000.service.d").exists()
+ reloads = [i for i, call in enumerate(commands.calls) if "daemon-reload" in call]
+ first_setup = commands.calls.index(commands.ran("install-service")[0])
+ assert len(reloads) == 1 and reloads[0] < first_setup
+ drop_in.write_text("stale")
+ again = FakeCommands({"getent": [(2, "")]})
+ assert boot.boot_controllers(again, CONFIG, boot.parse_bundle(raw)) == {}
+ assert drop_in.read_text() == boot.AFTER_BOOT_DROP_IN
+
+
+def test_another_index_enrolls_and_runs_on_its_own_paths_as_its_own_user(machine):
+ commands = FakeCommands({" status ": [(1, "Not enrolled.")]})
+ start(commands, index=2)
+ seat = boot.seat_for(2, CONFIG)
+ [enroll] = commands.ran("enroll")
+ assert enroll[:4] == ["/usr/sbin/runuser", "-u", "switch-controller-2", "--"]
+ assert enroll[enroll.index("--data-dir") + 1] == str(seat.data_dir)
+ assert ["/usr/bin/chown", "switch-controller-2:", str(seat.data_dir)] in commands.calls
+ assert stat.S_IMODE(seat.data_dir.stat().st_mode) == 0o700
+ for directory in (machine / "controllers", machine / "controllers" / "2"):
+ assert stat.S_IMODE(directory.stat().st_mode) == 0o755
+ assert (machine / "controllers" / "2" / "code").read_text() == (
+ hashlib.sha256(CODE.encode()).hexdigest()
+ )
+ assert not (machine / ".switch-controller").exists()
+ [setup] = commands.ran("install-service")
+ assert setup[setup.index("--user") + 1] == "switch-controller-2"
+ assert setup[setup.index("--data-dir") + 1] == str(seat.data_dir)
+ assert setup[setup.index("--agents-dir") + 1] == str(seat.agents_dir)
+
+
+def test_a_fresh_machine_enrolls_then_runs_its_agents_as_users_of_their_own(machine):
commands = FakeCommands({" status ": [(1, "Not enrolled.")]})
- boot.start_controller(commands, CONFIG, boot.parse_bundle(bundle()))
+ start(commands)
[enroll] = commands.ran("enroll")
assert enroll[:6] == [
"/usr/sbin/runuser",
@@ -249,77 +585,147 @@ def test_a_fresh_machine_enrolls_then_runs_its_agents_as_users_of_their_own(
"/opt/switch/controller/bin/switch-agent-controller",
]
assert enroll[enroll.index("--code") + 1] == CODE
- assert enroll[enroll.index("--data-dir") + 1] == str(
- tmp_path / ".switch-controller"
- )
+ assert enroll[enroll.index("--name") + 1] == "Switch cloud"
+ assert enroll[enroll.index("--data-dir") + 1] == str(machine / ".switch-controller")
[setup] = commands.ran("install-service")
assert setup[:2] == [
"/opt/switch/node/bin/node",
"/opt/switch/controller/bin/switch-agent-controller",
]
- assert setup[setup.index("--agents-dir") + 1] == str(tmp_path / "agents")
+ assert setup[setup.index("--agents-dir") + 1] == str(machine / "agents")
assert setup[setup.index("--user") + 1] == "switch-controller"
+ assert setup[setup.index("--agent-users") + 1] == "16"
assert "--no-block" in setup
assert commands.calls.index(enroll) < commands.calls.index(setup)
-def test_an_enrolled_machine_only_starts_its_controller(monkeypatch, tmp_path):
- monkeypatch.setattr(boot, "DATA_MOUNT", tmp_path)
- (tmp_path / ".switch-controller").mkdir()
+def test_an_enrolled_machine_only_starts_its_controller(machine):
+ (machine / ".switch-controller").mkdir()
commands = FakeCommands({" status ": [(0, "Controller: 9a1c…")]})
- boot.start_controller(
- commands,
- CONFIG,
- boot.parse_bundle(bundle(id=CONTROLLER_ID, enrollmentCode=None)),
- )
+ start(commands, bundle(enrolled_entry(KEY_A)))
assert commands.ran("enroll") == []
assert len(commands.ran("install-service")) == 1
-def test_a_new_code_sets_a_revoked_enrollment_aside(monkeypatch, tmp_path):
- monkeypatch.setattr(boot, "DATA_MOUNT", tmp_path)
- (tmp_path / ".switch-controller").mkdir()
- (tmp_path / ".switch-controller" / "controller.db").write_text("old")
- (tmp_path / ".switch-controller-code").write_text(
- hashlib.sha256(b"swce_old").hexdigest()
- )
+def test_a_new_code_sets_a_revoked_enrollment_aside(machine):
+ (machine / ".switch-controller").mkdir()
+ (machine / ".switch-controller" / "controller.db").write_text("old")
+ (machine / ".switch-controller-code").write_text(hashlib.sha256(b"swce_old").hexdigest())
commands = FakeCommands({" status ": [(0, "Controller: old")]})
- boot.start_controller(commands, CONFIG, boot.parse_bundle(bundle()), now=lambda: 7)
- assert (
- tmp_path / ".switch-controller.replaced-7" / "controller.db"
- ).read_text() == "old"
- assert not (tmp_path / ".switch-controller" / "controller.db").exists()
+ start(commands)
+ assert (machine / ".switch-controller.replaced-7" / "controller.db").read_text() == "old"
+ assert not (machine / ".switch-controller" / "controller.db").exists()
assert len(commands.ran("enroll")) == 1
- assert (tmp_path / ".switch-controller-code").read_text() != hashlib.sha256(
+ assert (machine / ".switch-controller-code").read_text() != hashlib.sha256(
b"swce_old"
).hexdigest()
-def test_an_enrollment_made_with_this_code_is_kept_when_the_boot_runs_again(
- monkeypatch, tmp_path
-):
- monkeypatch.setattr(boot, "DATA_MOUNT", tmp_path)
+def test_another_index_sets_its_revoked_enrollment_aside_next_to_it(machine):
+ seat = boot.seat_for(1, CONFIG)
+ seat.data_dir.mkdir(parents=True)
+ (seat.data_dir / "controller.db").write_text("old")
+ seat.code_record.write_text(hashlib.sha256(b"swce_old").hexdigest())
+ commands = FakeCommands({" status ": [(0, "Controller: old")]})
+ start(commands, index=1)
+ aside = machine / "controllers" / "1" / "data.replaced-7"
+ assert (aside / "controller.db").read_text() == "old"
+ assert len(commands.ran("enroll")) == 1
+
+
+def test_an_enrollment_made_with_this_code_is_kept_when_the_boot_runs_again(machine):
first = FakeCommands({" status ": [(1, "Not enrolled.")]})
- boot.start_controller(first, CONFIG, boot.parse_bundle(bundle()))
+ start(first)
assert len(first.ran("enroll")) == 1
# It failed after enrolling: systemd starts the boot again, and Switch,
# which has not linked the controller yet, still hands over the code.
- (tmp_path / ".switch-controller" / "controller.db").write_text("enrolled")
+ (machine / ".switch-controller" / "controller.db").write_text("enrolled")
again = FakeCommands({" status ": [(0, "Controller: new")]})
- boot.start_controller(again, CONFIG, boot.parse_bundle(bundle()), now=lambda: 7)
+ start(again)
assert again.ran("enroll") == []
- assert not (tmp_path / ".switch-controller.replaced-7").exists()
- assert (tmp_path / ".switch-controller" / "controller.db").read_text() == "enrolled"
+ assert not (machine / ".switch-controller.replaced-7").exists()
+ assert (machine / ".switch-controller" / "controller.db").read_text() == "enrolled"
assert len(again.ran("install-service")) == 1
-def test_no_enrollment_and_no_code_says_what_to_do(monkeypatch, tmp_path):
- monkeypatch.setattr(boot, "DATA_MOUNT", tmp_path)
+def test_no_enrollment_and_no_code_says_what_to_do(machine):
commands = FakeCommands({" status ": [(1, "Not enrolled.")]})
with pytest.raises(boot.BootError, match="Machines page"):
- boot.start_controller(
- commands,
- CONFIG,
- boot.parse_bundle(bundle(id=CONTROLLER_ID, enrollmentCode=None)),
- )
+ start(commands, bundle(enrolled_entry(KEY_A)))
assert commands.ran("install-service") == []
+
+
+def test_one_failing_controller_does_not_keep_the_others_from_starting(
+ machine, monkeypatch, caplog
+):
+ commands = FakeCommands(
+ {
+ "controllers/1/data": [(0, "Controller: b")],
+ " status ": [(1, "Not enrolled.")],
+ " enroll ": [(1, "")],
+ "getent": [(2, "")],
+ }
+ )
+ parsed = boot.parse_bundle(bundle(entry(KEY_A), enrolled_entry(KEY_B)))
+ monkeypatch.setattr(boot.os, "geteuid", lambda: 0)
+ monkeypatch.setattr(boot, "acquire_lock", contextlib.nullcontext)
+ monkeypatch.setattr(boot, "read_bundle", lambda: parsed)
+ monkeypatch.setattr(boot, "load_machine_config", lambda: CONFIG)
+ monkeypatch.setattr(boot, "Commands", lambda: commands)
+ monkeypatch.setattr(boot, "prepare_storage", lambda *_: None)
+ waits = []
+ monkeypatch.setattr(boot.time, "sleep", waits.append)
+ with caplog.at_level(logging.ERROR, logger="switch-machine-boot"):
+ assert boot.main([]) == 0
+ assert waits == [boot.CONTROLLER_RETRY_SECONDS] * boot.CONTROLLER_RETRIES
+ assert len(commands.ran("enroll")) == 1 + boot.CONTROLLER_RETRIES
+ assert setups(commands) == ["switch-controller-1"]
+ [failure] = [
+ record.getMessage() for record in caplog.records if record.levelno == logging.ERROR
+ ]
+ assert "switch-controller" in failure and KEY_A in failure
+
+
+def test_a_controller_that_fails_once_is_started_on_the_next_try(machine, monkeypatch):
+ commands = FakeCommands(
+ {" status ": [(1, "Not enrolled.")], " enroll ": [(1, ""), (0, "")], "getent": [(2, "")]}
+ )
+ parsed = boot.parse_bundle(bundle(entry(KEY_A)))
+ monkeypatch.setattr(boot.os, "geteuid", lambda: 0)
+ monkeypatch.setattr(boot, "acquire_lock", contextlib.nullcontext)
+ monkeypatch.setattr(boot, "read_bundle", lambda: parsed)
+ monkeypatch.setattr(boot, "load_machine_config", lambda: CONFIG)
+ monkeypatch.setattr(boot, "Commands", lambda: commands)
+ monkeypatch.setattr(boot, "prepare_storage", lambda *_: None)
+ waits = []
+ monkeypatch.setattr(boot.time, "sleep", waits.append)
+ assert boot.main([]) == 0
+ assert waits == [boot.CONTROLLER_RETRY_SECONDS]
+ assert len(commands.ran("enroll")) == 2
+ assert setups(commands) == ["switch-controller"]
+
+
+def test_a_machine_whose_controllers_all_start_boots(machine, monkeypatch):
+ commands = FakeCommands({"getent": [(2, "")]})
+ parsed = boot.parse_bundle(
+ bundle(enrolled_entry(KEY_A), enrolled_entry(KEY_B, OTHER_CONTROLLER_ID))
+ )
+ monkeypatch.setattr(boot.os, "geteuid", lambda: 0)
+ monkeypatch.setattr(boot, "acquire_lock", contextlib.nullcontext)
+ monkeypatch.setattr(boot, "read_bundle", lambda: parsed)
+ monkeypatch.setattr(boot, "load_machine_config", lambda: CONFIG)
+ monkeypatch.setattr(boot, "Commands", lambda: commands)
+ monkeypatch.setattr(boot, "prepare_storage", lambda *_: None)
+ assert boot.main([]) == 0
+ assert json.loads((machine / ".switch-machine.json").read_text())["machineId"] == MACHINE_ID
+ assert setups(commands) == ["switch-controller", "switch-controller-1"]
+
+
+def test_user_data_that_is_no_bundle_is_not_retried(monkeypatch):
+ def no_bundle():
+ raise boot.NoBundle("no bundle")
+
+ monkeypatch.setattr(boot.os, "geteuid", lambda: 0)
+ monkeypatch.setattr(boot, "acquire_lock", contextlib.nullcontext)
+ monkeypatch.setattr(boot, "read_bundle", no_bundle)
+ assert boot.main([]) == boot.NO_BUNDLE_EXIT