Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Machines show offline within seconds: connection transitions and a process lease in the database, set-info (#696, via #678)
Abel Dantasa3d58a6bd18d
diff --git a/console/apps/switch-console-desktop/src/main/core/embedded-controller/controller.ts b/console/apps/switch-console-desktop/src/main/core/embedded-controller/controller.tsindex bf27cc7cd..c980545c4 100644--- a/console/apps/switch-console-desktop/src/main/core/embedded-controller/controller.ts+++ b/console/apps/switch-console-desktop/src/main/core/embedded-controller/controller.ts@@ -30,6 +30,9 @@ export const embeddedControllerController = createRPCController({dismissRemoved: (serverId: string): Promise<void> => embeddedControllerService.dismiss(serverId),+ /** Enrolls this computer again when the server no longer knows the machine it was. */+ enrollAgain: (serverId: string): Promise<void> => embeddedControllerService.enrollAgain(serverId),+/*** Where this computer's controller puts a managed agent that names no* directory, `~` for the home directory; null for a name that cannot be adiff --git a/console/apps/switch-console-desktop/src/main/core/embedded-controller/embedded-controller-service.test.ts b/console/apps/switch-console-desktop/src/main/core/embedded-controller/embedded-controller-service.test.tsindex 082765f52..65999fd29 100644--- a/console/apps/switch-console-desktop/src/main/core/embedded-controller/embedded-controller-service.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/embedded-controller/embedded-controller-service.test.ts@@ -219,6 +219,33 @@ describe('EmbeddedControllerService', () => {expect(events.map((event) => event.phase.kind)).toEqual(['enrolling', 'running']);});+ it('enrolls again when Switch no longer lists the machine, forgetting the old identity', async () => {+ const running = await enabled();+ management.enroll.mockResolvedValueOnce({+ serverId: SERVER,+ apiUrl: 'https://switch.example.com',+ controllerId: 'controller-2',+ credential: 'swcc_second',+ });+ await running.enrollAgain(SERVER);+ expect(management.revoke).not.toHaveBeenCalled();+ expect(management.enroll).toHaveBeenCalledTimes(2);+ await waitFor(() => calls.length === 2, 'the new controller started');+ expect(storedRecord()).toMatchObject({ kind: 'enrolled', controllerId: 'controller-2' });+ expect(secrets.get(credentialSecretKey(SERVER))).toBe('swcc_second');+ });++ it('refuses to enroll again while Switch still lists the machine', async () => {+ const running = await enabled();+ management.read.mockResolvedValue({+ kind: 'ok',+ controller: { name: 'build-box', description: null, state: 'offline', lastSeenAt: null },+ agents: [],+ });+ await expect(running.enrollAgain(SERVER)).rejects.toThrow(/still lists this computer/);+ expect(management.enroll).toHaveBeenCalledTimes(1);+ });+it('restarts the controller with backoff when it exits on its own', async () => {await enabled();calls[0]!.child.exit(1);diff --git a/console/apps/switch-console-desktop/src/main/core/embedded-controller/embedded-controller-service.ts b/console/apps/switch-console-desktop/src/main/core/embedded-controller/embedded-controller-service.tsindex f66f31465..ba2d7c141 100644--- a/console/apps/switch-console-desktop/src/main/core/embedded-controller/embedded-controller-service.ts+++ b/console/apps/switch-console-desktop/src/main/core/embedded-controller/embedded-controller-service.ts@@ -289,6 +289,42 @@ export class EmbeddedControllerService {});}+ /**+ * Enrolls this computer afresh when the server no longer knows the machine+ * it was enrolled as: its database was reset or restored, say. The old+ * controller is stopped and its identity forgotten (there is nothing on the+ * server to revoke), then this computer enrolls again in the same+ * workspace. Refused while the server still lists the machine, which is+ * turned off and on instead.+ */+ async enrollAgain(serverId: string): Promise<void> {+ const record = await this.deps.records.get(serverId);+ if (record?.kind !== 'enrolled')+ throw new Error('This computer is not enrolled to run managed agents for this server.');+ const remote = await this.deps.management.read(record.workspaceId, record.controllerId);+ if (remote.kind !== 'ok')+ throw new Error(+ remote.kind === 'error'+ ? `Switch could not be asked about this computer: ${remote.message}`+ : 'This server no longer has agent management turned on.'+ );+ if (remote.controller)+ throw new Error(+ 'Switch still lists this computer as a machine. Turn it off and on again instead.'+ );+ await this.exclusive(serverId, async () => {+ const runner = this.runners.get(serverId);+ this.runners.delete(serverId);+ if (runner) await runner.supervisor.stop(this.deps.stopTimeoutMs);+ await this.forget(serverId);+ this.deps.log.warn('The server no longer knew this computer as a machine; enrolling again', {+ serverId,+ controllerId: record.controllerId,+ });+ });+ await this.enable(serverId, record.workspaceId);+ }+/*** Renames this computer as a machine on the server and/or changes its* description: what its owner sees on the Machines page, and what agentsdiff --git a/console/apps/switch-console-desktop/src/main/core/host-controllers/controller.ts b/console/apps/switch-console-desktop/src/main/core/host-controllers/controller.tsindex 5a17758fa..567abd0d6 100644--- a/console/apps/switch-console-desktop/src/main/core/host-controllers/controller.ts+++ b/console/apps/switch-console-desktop/src/main/core/host-controllers/controller.ts@@ -19,6 +19,10 @@ export const hostControllersController = createRPCController({restart: (params: { sshHost: string; serverId: string }): Promise<void> =>hostControllerService.restart(params.sshHost, params.serverId),+ /** Enrolls the host again when the server no longer knows the machine it was. */+ enrollAgain: (params: { sshHost: string; serverId: string }): Promise<void> =>+ hostControllerService.enrollAgain(params.sshHost, params.serverId),+disable: (params: { sshHost: string; serverId: string }): Promise<void> =>hostControllerService.disable(params.sshHost, params.serverId, { force: false }),diff --git a/console/apps/switch-console-desktop/src/main/core/host-controllers/host-controller-service.test.ts b/console/apps/switch-console-desktop/src/main/core/host-controllers/host-controller-service.test.tsindex 9ef14cd85..f5358fd52 100644--- a/console/apps/switch-console-desktop/src/main/core/host-controllers/host-controller-service.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/host-controllers/host-controller-service.test.ts@@ -308,6 +308,35 @@ describe('the host as a machine afterwards', () => {});});+describe('enrolling a host again that Switch no longer knows', () => {+ it('wipes the old controller on the host, without a revoke, and enrolls it afresh', async () => {+ const base = deps();+ const service = new HostControllerService({+ ...base,+ management: {+ ...base.management,+ read: async () => ({ kind: 'ok', controller: null, agents: [] }),+ },+ });+ await service.enable(HOST, SERVER, WORKSPACE);+ calls = [];+ host.enroll = { ok: true, controllerId: 'controller-8' };+ await service.enrollAgain(HOST, SERVER);+ expect(calls.slice(0, 2)).toEqual(['stop', 'forget']);+ expect(calls).not.toContain('revoke');+ expect(calls).toContain('code');+ expect(records.get(`${HOST}|${SERVER}`)?.controllerId).toBe('controller-8');+ });++ it('refuses while Switch still lists the host', async () => {+ const service = new HostControllerService(deps());+ await service.enable(HOST, SERVER, WORKSPACE);+ calls = [];+ await expect(service.enrollAgain(HOST, SERVER)).rejects.toThrow(/still lists build/);+ expect(calls).toEqual([]);+ });+});+describe('removing the host as a machine', () => {it('revokes the controller first, then stops it and clears it from the host', async () => {const service = new HostControllerService(deps());diff --git a/console/apps/switch-console-desktop/src/main/core/host-controllers/host-controller-service.ts b/console/apps/switch-console-desktop/src/main/core/host-controllers/host-controller-service.tsindex 2f1484bd0..191294bfe 100644--- a/console/apps/switch-console-desktop/src/main/core/host-controllers/host-controller-service.ts+++ b/console/apps/switch-console-desktop/src/main/core/host-controllers/host-controller-service.ts@@ -348,6 +348,45 @@ export class HostControllerService {});}+ /**+ * Enrolls the host afresh when the server no longer knows the machine it+ * was enrolled as: its database was reset or restored, say. The old+ * controller is stopped and its identity wiped on the host (there is nothing+ * on the server to revoke), then the host enrolls again in the same+ * workspace. Refused while the server still lists the machine, which is+ * turned off and on instead.+ */+ async enrollAgain(sshHost: string, serverId: string): Promise<void> {+ const record = await this.deps.records.get(sshHost, serverId);+ if (!record) throw new Error(`${sshHost} is not a machine for this server.`);+ const remote = await this.deps.management.read(record.workspaceId, record.controllerId);+ if (remote.kind !== 'ok')+ throw new Error(+ remote.kind === 'error'+ ? `Switch could not be asked about ${sshHost}: ${remote.message}`+ : 'This server no longer has agent management turned on.'+ );+ if (remote.controller)+ throw new Error(+ `Switch still lists ${sshHost} as a machine. Turn it off and on again instead.`+ );+ await this.exclusive(sshHost, serverId, async () => {+ const shell = await this.deps.shell(sshHost);+ try {+ await this.stopOn(shell, record, { turnOff: true, wipe: true });+ } finally {+ shell.close();+ }+ await this.deps.records.delete(sshHost, serverId);+ this.deps.log.warn('Switch no longer knew an SSH host as a machine; enrolling it again', {+ sshHost,+ serverId,+ controllerId: record.controllerId,+ });+ });+ await this.enable(sshHost, serverId, record.workspaceId);+ }+/*** The host is being removed from Console: its controllers go first. Refused,* with nothing changed, while any of them runs agents moved from this Console.diff --git a/console/apps/switch-console-desktop/src/main/core/switch-servers/gateway-client.ts b/console/apps/switch-console-desktop/src/main/core/switch-servers/gateway-client.tsindex 70e440b24..a1afbc752 100644--- a/console/apps/switch-console-desktop/src/main/core/switch-servers/gateway-client.ts+++ b/console/apps/switch-console-desktop/src/main/core/switch-servers/gateway-client.ts@@ -2610,7 +2610,7 @@ export type ManagementController = {/** What its owner says the machine is for; null when none was given. */description: string | null;kind: string;- state: 'online' | 'unknown' | 'revoked';+ state: 'online' | 'offline' | 'unknown' | 'revoked';lastSeenAt: string | null;revokedAt: string | null;/** Each provider as the controller last reported it; empty before it has reported. */@@ -2667,7 +2667,7 @@ type ManagementControllerJson = {name: string;description: string | null;kind: string;- state: 'online' | 'unknown' | 'revoked';+ state: 'online' | 'offline' | 'unknown' | 'revoked';last_seen_at: string | null;revoked_at: string | null;status?: unknown;diff --git a/console/apps/switch-console-desktop/src/renderer/features/managed-agents/managed-machine-card.tsx b/console/apps/switch-console-desktop/src/renderer/features/managed-agents/managed-machine-card.tsxindex d1a9c5476..5243a1cac 100644--- a/console/apps/switch-console-desktop/src/renderer/features/managed-agents/managed-machine-card.tsx+++ b/console/apps/switch-console-desktop/src/renderer/features/managed-agents/managed-machine-card.tsx@@ -115,7 +115,7 @@ function MachineCard({);}-function MachineStatePill({ state }: { state: 'online' | 'unknown' | 'revoked' }) {+function MachineStatePill({ state }: { state: 'online' | 'offline' | 'unknown' | 'revoked' }) {if (state === 'revoked')return <span className="shrink-0 text-xs text-foreground-muted">Removed</span>;const online = state === 'online';diff --git a/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine-card.tsx b/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine-card.tsxindex fd2dc3ab7..5291f4d98 100644--- a/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine-card.tsx+++ b/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine-card.tsx@@ -29,6 +29,7 @@ import {hostMachineStatus,hostStateKey,hostToggleBlocker,+ hostUnknownToServer,supervisionNote,} from './host-machine';@@ -110,6 +111,10 @@ export function HostMachineCard({mutationFn: () => rpc.hostControllers.restart({ sshHost, serverId }),...handlers,});+ const enrollAgain = useMutation({+ mutationFn: () => rpc.hostControllers.enrollAgain({ sshHost, serverId }),+ ...handlers,+ });const moveAll = useMutation({mutationFn: () => rpc.hostControllers.moveAll(sshHost),...handlers,@@ -183,7 +188,17 @@ export function HostMachineCard({{failureText(failure, 'That did not work.')}</p>)}- {canRestartHost(overview) && (+ {hostUnknownToServer(overview) && (+ <Button+ variant="outline"+ size="sm"+ disabled={enrollAgain.isPending}+ onClick={() => enrollAgain.mutate()}+ >+ <RefreshCw className="size-4" /> {enrollAgain.isPending ? 'Enrolling…' : 'Enroll again'}+ </Button>+ )}+ {canRestartHost(overview) && !hostUnknownToServer(overview) && (<Buttonvariant="outline"size="sm"diff --git a/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine.test.ts b/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine.test.tsindex 8a8968dfa..ffe1d43bf 100644--- a/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine.test.ts+++ b/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine.test.ts@@ -8,6 +8,7 @@ import {hostMachineStatus,hostStateKey,hostToggleBlocker,+ hostUnknownToServer,supervisionNote,} from './host-machine';@@ -45,6 +46,24 @@ describe('the state of a host as a machine', () => {).toEqual({ label: 'Setting up…', tone: 'busy', detail: 'Enrolling the host…' });});+ it('says Switch no longer knows the host when it is enrolled but not listed, whatever its process does', () => {+ const unknown = { ...ON, remote: { kind: 'ok' as const, controller: null, agents: [] } };+ expect(hostUnknownToServer(unknown)).toBe(true);+ expect(hostMachineStatus(unknown)).toMatchObject({+ label: 'Unknown to Switch',+ tone: 'error',+ detail: expect.stringContaining('Switch no longer knows build-box as a machine'),+ });+ const stopped = {+ ...unknown,+ process: { kind: 'stopped' as const, state: 'exited', code: 1, log: '' },+ };+ expect(hostMachineStatus(stopped).label).toBe('Unknown to Switch');+ expect(hostMachineStatus({ ...unknown, movedAgents: ['jack'] }).detail).toContain('jack');+ expect(hostUnknownToServer(OFF)).toBe(false);+ expect(hostUnknownToServer(ON)).toBe(false);+ });+it('is running when the host runs it and Switch sees it', () => {expect(hostMachineStatus(ON)).toEqual({ label: 'Running', tone: 'ok', detail: null });});diff --git a/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine.ts b/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine.tsindex f12bc89c9..f788ef0f1 100644--- a/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine.ts+++ b/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine.ts@@ -59,6 +59,16 @@ export function hostMachineStatus(overview: HostControllerOverview): HostMachinereturn { label: 'Off', tone: 'neutral', detail: null };}const failed = phase.kind === 'error' ? phase.message : null;+ if (hostUnknownToServer(overview)) {+ const why = `Switch no longer knows ${overview.sshHost} as a machine (its data was reset or restored, or the machine was deleted), so its controller cannot sign in. Enroll it again.`;+ return {+ label: 'Unknown to Switch',+ tone: 'error',+ detail: overview.movedAgents.length+ ? `${why} ${overview.movedAgents.join(', ')}, moved here from this Console, then need Stop managing, and can be moved again.`+ : why,+ };+ }if (process?.kind === 'unknown')return {label: 'Unknown',@@ -97,9 +107,7 @@ export function hostMachineStatus(overview: HostControllerOverview): HostMachinedetail:state === 'revoked'? 'Switch has removed this host; its controller is stopping its agents.'- : state === null- ? 'Switch does not list this host’s controller.'- : 'The controller runs but has not reached Switch recently. It keeps trying.',+ : 'The controller runs but has not reached Switch recently. It keeps trying.',};return { label: 'Running', tone: failed ? 'warn' : 'ok', detail: failed };}@@ -193,3 +201,14 @@ export function hostStateKey(overview: HostControllerOverview): string {export function failureAlreadyShown(overview: HostControllerOverview, message: string): boolean {return overview.phase.kind === 'error' && overview.phase.message.trim() === message.trim();}++/** Switch answered, and does not list the machine this host is enrolled as. */+export function hostUnknownToServer(overview: HostControllerOverview): boolean {+ return (+ overview.enrollment !== null &&+ overview.phase.kind !== 'installing' &&+ overview.phase.kind !== 'removing' &&+ overview.remote?.kind === 'ok' &&+ overview.remote.controller === null+ );+}diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine-card.tsx b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine-card.tsxindex 227bd9289..68933ff9e 100644--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine-card.tsx+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine-card.tsx@@ -38,6 +38,7 @@ import {type MachineStatusTone,toggleBlocker,toggleChecked,+ unknownToServer,} from './this-computer-machine';/** How often the card re-reads the server while it is on screen. */@@ -135,6 +136,10 @@ export const ThisComputerMachineCard = observer(function ThisComputerMachineCardmutationFn: () => rpc.embeddedController.dismissRemoved(serverId),...handlers,});+ const enrollAgain = useMutation({+ mutationFn: () => rpc.embeddedController.enrollAgain(serverId),+ ...handlers,+ });const overview = overviewQuery.data;// Nothing to say on a server this computer has nothing to do with and is not@@ -210,6 +215,17 @@ export const ThisComputerMachineCard = observer(function ThisComputerMachineCardStart again</Button>)}+ {unknownToServer(overview) && (+ <Button+ variant="outline"+ size="sm"+ disabled={enrollAgain.isPending}+ onClick={() => enrollAgain.mutate()}+ >+ <RefreshCw className="size-4" />+ {enrollAgain.isPending ? 'Enrolling…' : 'Enroll again'}+ </Button>+ )}{overview.phase.kind === 'removed' && (<Buttonvariant="outline"diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine.test.ts b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine.test.tsindex fcb66dfee..a605c7cfe 100644--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine.test.ts+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine.test.ts@@ -11,6 +11,7 @@ import {machineStatus,toggleBlocker,toggleChecked,+ unknownToServer,} from './this-computer-machine';const ENROLLMENT = {@@ -35,7 +36,9 @@ function overview(};}-const ok = (state: 'online' | 'unknown' | 'revoked' | null): EmbeddedControllerRemote => ({+const ok = (+ state: 'online' | 'offline' | 'unknown' | 'revoked' | null+): EmbeddedControllerRemote => ({kind: 'ok',controller: state ? { name: 'box', description: null, state, lastSeenAt: null } : null,agents: [],@@ -63,6 +66,21 @@ describe('machineStatus', () => {});});+ it('says Switch no longer knows the computer when it is enrolled but not listed, and offers to enroll again', () => {+ const unknown = overview(running, ok(null));+ expect(unknownToServer(unknown)).toBe(true);+ expect(machineStatus(unknown, LATER)).toMatchObject({+ label: 'Unknown to Switch',+ tone: 'error',+ detail: expect.stringContaining('Enroll it again.'),+ });+ const withMoved = { ...unknown, movedAgents: ['jack'] };+ expect(machineStatus(withMoved, LATER).detail).toContain('jack, moved here from this Console');+ expect(unknownToServer(overview(running, ok('offline')))).toBe(false);+ expect(unknownToServer(overview(running, { kind: 'error', message: 'down' }))).toBe(false);+ expect(unknownToServer(overview({ kind: 'off' }, ok(null), false))).toBe(false);+ });+it('says connecting, not disconnected, in the first minute after the controller starts', () => {const justStarted = Date.parse(running.since) + 5_000;expect(machineStatus(overview(running, ok('unknown')), justStarted)).toMatchObject({diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine.ts b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine.tsindex 26c6dc14e..7b5d78766 100644--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine.ts+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine.ts@@ -26,7 +26,21 @@ function clock(iso: string): string {/** How long after starting a controller that has not reached Switch reads as connecting, not disconnected. */export const CONNECTING_GRACE_MS = 60_000;+/** What the card says when Switch does not list the machine this computer is enrolled as. */+function unknownStatus(overview: EmbeddedControllerOverview): MachineStatus {+ const why =+ 'Switch no longer knows this computer as a machine (its data was reset or restored, or the machine was deleted), so its controller cannot sign in. Enroll it again.';+ return {+ label: 'Unknown to Switch',+ tone: 'error',+ detail: overview.movedAgents.length+ ? `${why} ${overview.movedAgents.join(', ')}, moved here from this Console, then need Stop managing, and can be moved again.`+ : why,+ };+}+export function machineStatus(overview: EmbeddedControllerOverview, nowMs: number): MachineStatus {+ if (unknownToServer(overview)) return unknownStatus(overview);const { phase, remote } = overview;switch (phase.kind) {case 'off':@@ -93,10 +107,7 @@ export function machineStatus(overview: EmbeddedControllerOverview, nowMs: numbereturn {label: 'Disconnected',tone: 'warn',- detail:- state === null- ? 'Switch does not list this computer’s controller.'- : 'The controller is running but has not reached Switch recently. It keeps trying.',+ detail: 'The controller is running but has not reached Switch recently. It keeps trying.',};}}@@ -165,3 +176,13 @@ export function machineStateKey(overview: EmbeddedControllerOverview): string {overview.movedAgents,]);}++/** Switch answered, and does not list the machine this computer is enrolled as. */+export function unknownToServer(overview: EmbeddedControllerOverview): boolean {+ return (+ overview.enrollment !== null &&+ overview.remote?.kind === 'ok' &&+ overview.remote.controller === null &&+ (overview.phase.kind === 'running' || overview.phase.kind === 'restarting')+ );+}diff --git a/console/apps/switch-console-desktop/src/shared/core/embedded-controller/embedded-controller.ts b/console/apps/switch-console-desktop/src/shared/core/embedded-controller/embedded-controller.tsindex 9817435f8..57f6b4b96 100644--- a/console/apps/switch-console-desktop/src/shared/core/embedded-controller/embedded-controller.ts+++ b/console/apps/switch-console-desktop/src/shared/core/embedded-controller/embedded-controller.ts@@ -56,7 +56,7 @@ export type EmbeddedControllerRemote =controller: {name: string;description: string | null;- state: 'online' | 'unknown' | 'revoked';+ state: 'online' | 'offline' | 'unknown' | 'revoked';lastSeenAt: string | null;} | null;agents: PlacedManagedAgent[];diff --git a/console/apps/switch-console-desktop/src/shared/core/host-controllers/host-controllers.ts b/console/apps/switch-console-desktop/src/shared/core/host-controllers/host-controllers.tsindex 93580417b..873818eb1 100644--- a/console/apps/switch-console-desktop/src/shared/core/host-controllers/host-controllers.ts+++ b/console/apps/switch-console-desktop/src/shared/core/host-controllers/host-controllers.ts@@ -34,7 +34,10 @@ export type HostControllerProcess =export type HostControllerRemote =| {kind: 'ok';- controller: { state: 'online' | 'unknown' | 'revoked'; lastSeenAt: string | null } | null;+ controller: {+ state: 'online' | 'offline' | 'unknown' | 'revoked';+ lastSeenAt: string | null;+ } | null;agents: PlacedManagedAgent[];}| { kind: 'unavailable' }diff --git a/console/apps/switch-console-desktop/src/shared/core/managed-agents/managed-agents.ts b/console/apps/switch-console-desktop/src/shared/core/managed-agents/managed-agents.tsindex 9db9448ea..53ff4fc53 100644--- a/console/apps/switch-console-desktop/src/shared/core/managed-agents/managed-agents.ts+++ b/console/apps/switch-console-desktop/src/shared/core/managed-agents/managed-agents.ts@@ -50,7 +50,7 @@ export type ManagedMachine = {id: string;name: string;kind: string;- state: 'online' | 'unknown' | 'revoked';+ state: 'online' | 'offline' | 'unknown' | 'revoked';};/** A provider on a machine, as the machine last reported it. */diff --git a/console/packages/agent-controller/README.md b/console/packages/agent-controller/README.mdindex 35ad52455..a88afe577 100644--- a/console/packages/agent-controller/README.md+++ b/console/packages/agent-controller/README.md@@ -49,6 +49,8 @@ node packages/agent-controller/dist/cli.mjs enroll \node packages/agent-controller/dist/cli.mjs run [--data-dir <dir>]node packages/agent-controller/dist/cli.mjs status [--data-dir <dir>]+node packages/agent-controller/dist/cli.mjs set-info [--name <name>] \+ [--description <text>] [--data-dir <dir>]```- `--server` is the agent bridge URL, and it must be `https`. Plain `http` is@@ -59,6 +61,13 @@ node packages/agent-controller/dist/cli.mjs status [--data-dir <dir>]characters). Its owner sees it in the gateway's Machines page, where both thename and the description can be changed later, and so do the agents allowedto manage agents for that owner.+- `set-info` changes the machine's name and/or description on the server after+ enrollment, with this controller's own credential, and records the new name+ in the data directory. Give `--name`, `--description` or both; `--description ""`+ clears the description. The limits are enrollment's (a name of at most 200+ characters, not blank). It needs the credential in the data directory, so a+ controller whose credential is handed over with `--credential-stdin` is+ renamed in the gateway instead. It can run while `run` does.- `status` reads only local state. It makes no network call.- Logging goes to stderr. Set the level with `SWITCH_CONTROLLER_LOG_LEVEL`(`debug`, `info`, `warn`, `error`; the default is `info`).@@ -78,7 +87,8 @@ node packages/agent-controller/dist/cli.mjs status [--data-dir <dir>]a newer controller wrote; a shared-host bundle that is missing, not a fileor unreadable; an unsupported platform (Windows); a data directory thatholds no identity, or no credential (revoked earlier and wiped, or never- enrolled); and with `--credential-stdin`, a credential that does not arrive+ enrolled); `set-info` with nothing to change, or a name or description past+ the limits; and with `--credential-stdin`, a credential that does not arrive(stdin is a terminal, closes empty, holds more than one token, cannot beread, or stays open past 10 s). The reason is the last line on stderr,prefixed `switch-agent-controller: `. Once the controller is running,diff --git a/console/packages/agent-controller/src/api.test.ts b/console/packages/agent-controller/src/api.test.tsindex 6c7e56cc8..5971332e8 100644--- a/console/packages/agent-controller/src/api.test.ts+++ b/console/packages/agent-controller/src/api.test.ts@@ -322,6 +322,37 @@ describe('the controller connection', () => {});});+describe('the machine', () => {+ it("changes the machine's name and description", async () => {+ const { client: c } = client();+ expect(await c.updateInfo({ name: 'build-box', description: 'The box' })).toEqual({+ id: core.controllerId,+ name: 'build-box',+ description: 'The box',+ });+ expect(await c.updateInfo({ description: null })).toMatchObject({ description: null });+ const sent = core.requests.filter((r) => r.method === 'PATCH');+ expect(sent.map((r) => r.body)).toEqual([+ { name: 'build-box', description: 'The box' },+ { description: null },+ ]);+ });++ it('says so when the server cannot rename from the controller', async () => {+ core.scripted.push({+ method: 'PATCH',+ path: `/v1/management/controllers/${core.controllerId}`,+ status: 405,+ body: { detail: 'Method Not Allowed' },+ });+ const error = await client()+ .client.updateInfo({ name: 'x' })+ .catch((e: unknown) => e);+ expect(error).toBeInstanceOf(ControllerApiError);+ expect(error).toMatchObject({ status: 405, code: 'not_supported' });+ });+});+describe('operations', () => {it('lists, claims, renews and reports', async () => {const { client: api } = client();diff --git a/console/packages/agent-controller/src/api.ts b/console/packages/agent-controller/src/api.tsindex 154bdb2c4..f3852b09f 100644--- a/console/packages/agent-controller/src/api.ts+++ b/console/packages/agent-controller/src/api.ts@@ -7,7 +7,10 @@ import {type Assignment,assignmentSchema,type ControllerConnection,+ type ControllerInfo,+ type ControllerInfoChange,controllerConnectionResponseSchema,+ controllerInfoResponseSchema,credentialRotateResponseSchema,type EnrollRequest,type EnrollResponse,@@ -352,6 +355,31 @@ export class ControllerClient {}}+ /** Renames this machine and/or changes its description, as its owner can in the gateway. */+ async updateInfo(change: ControllerInfoChange): Promise<ControllerInfo> {+ let response: Response;+ try {+ response = await this.request(this.controllerPath, { method: 'PATCH', body: change });+ } catch (error) {+ // A server from before this route answers with the framework's own+ // refusal, which carries no Switch error envelope.+ if (+ error instanceof ControllerApiError &&+ (error.status === 404 || error.status === 405) &&+ error.code === 'unexpected_response'+ )+ throw new ControllerApiError(+ error.status,+ 'not_supported',+ "This Switch server cannot rename a machine from its controller. Change the name and description in the gateway's Machines page instead.",+ false,+ null+ );+ throw error;+ }+ return parsed(response, controllerInfoResponseSchema);+ }+async rotateCredential(): Promise<string> {const response = await this.request(`${this.controllerPath}/credential/rotate`, {method: 'POST',diff --git a/console/packages/agent-controller/src/main.test.ts b/console/packages/agent-controller/src/main.test.tsindex 18fad20e5..6e1ddcbb3 100644--- a/console/packages/agent-controller/src/main.test.ts+++ b/console/packages/agent-controller/src/main.test.ts@@ -2,8 +2,10 @@ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';import { tmpdir } from 'node:os';import { join } from 'node:path';import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';-import { EXIT_CONFIGURATION, EXIT_OK } from './exit-codes';-import { main } from './main';+import { EXIT_CONFIGURATION, EXIT_FAILURE, EXIT_OK } from './exit-codes';+import { infoChange, main } from './main';+import { dataLayout } from './paths';+import { CONTROLLER_CREDENTIAL, FileSecretStore } from './secrets';import { ControllerStore } from './store';let dir: string;@@ -152,6 +154,114 @@ describe('main', () => {expect(lastLine()).toMatch(/must use https/);});+ it("set-info changes the machine's name and description, and records the name", async () => {+ const store = ControllerStore.open(join(dir, 'controller.db'));+ store.saveIdentity({+ controllerId: 'controller-1',+ server: 'https://switch.example.com',+ name: 'box',+ enrolledAt: '2026-01-01T00:00:00.000Z',+ });+ store.close();+ await new FileSecretStore(dataLayout(dir).secrets).set(CONTROLLER_CREDENTIAL, 'swcc_test');+ const calls: { method: string; url: string; body: unknown; auth: string | null }[] = [];+ vi.stubGlobal(+ 'fetch',+ vi.fn(async (url: string, init: RequestInit) => {+ const headers = new Headers(init.headers);+ calls.push({+ method: String(init.method),+ url,+ body: init.body ? JSON.parse(String(init.body)) : null,+ auth: headers.get('Authorization'),+ });+ const body = url.endsWith('/token')+ ? {+ access_token: 'swct_test',+ expires_at: new Date(Date.now() + 3_600_000).toISOString(),+ }+ : { id: 'controller-1', name: 'build-box', description: null, state: 'online' };+ return new Response(JSON.stringify(body), {+ status: 200,+ headers: { 'Content-Type': 'application/json' },+ });+ })+ );+ const code = await main([+ 'set-info',+ '--name',+ ' build-box ',+ '--description',+ '',+ '--data-dir',+ dir,+ ]);+ vi.unstubAllGlobals();+ expect(code).toBe(EXIT_OK);+ expect(calls.map((c) => [c.method, c.url])).toEqual([+ ['POST', 'https://switch.example.com/v1/management/controllers/controller-1/token'],+ ['PATCH', 'https://switch.example.com/v1/management/controllers/controller-1'],+ ]);+ expect(calls[0]!.body).toEqual({ credential: 'swcc_test' });+ expect(calls[1]!.body).toEqual({ name: 'build-box', description: null });+ expect(calls[1]!.auth).toBe('Bearer swct_test');+ const reopened = ControllerStore.open(join(dir, 'controller.db'));+ expect(reopened.identity()?.name).toBe('build-box');+ reopened.close();+ });++ it('set-info refuses what it cannot send, before calling Switch', async () => {+ const fetchSpy = vi.fn();+ vi.stubGlobal('fetch', fetchSpy);+ expect(await main(['set-info', '--data-dir', dir])).toBe(EXIT_CONFIGURATION);+ expect(lastLine()).toBe(+ 'switch-agent-controller: set-info needs --name, --description, or both.'+ );+ expect(await main(['set-info', '--name', ' ', '--data-dir', dir])).toBe(EXIT_CONFIGURATION);+ expect(lastLine()).toBe('switch-agent-controller: --name must not be blank.');+ expect(await main(['set-info', '--name', 'box', '--data-dir', dir])).toBe(EXIT_CONFIGURATION);+ expect(lastLine()).toMatch(/holds no enrolled controller/);+ vi.unstubAllGlobals();+ expect(fetchSpy).not.toHaveBeenCalled();+ });++ it('set-info exits 1 when Switch refuses the change', async () => {+ const store = ControllerStore.open(join(dir, 'controller.db'));+ store.saveIdentity({+ controllerId: 'controller-1',+ server: 'https://switch.example.com',+ name: 'box',+ enrolledAt: '2026-01-01T00:00:00.000Z',+ });+ store.close();+ await new FileSecretStore(dataLayout(dir).secrets).set(CONTROLLER_CREDENTIAL, 'swcc_test');+ vi.stubGlobal(+ 'fetch',+ vi.fn(async (url: string) =>+ url.endsWith('/token')+ ? new Response(+ JSON.stringify({+ access_token: 'swct_test',+ expires_at: new Date(Date.now() + 3_600_000).toISOString(),+ }),+ { status: 200 }+ )+ : new Response(+ JSON.stringify({+ error: { code: 'validation_error', message: 'name too long', retryable: false },+ }),+ { status: 422 }+ )+ )+ );+ expect(await main(['set-info', '--name', 'box', '--data-dir', dir])).toBe(EXIT_FAILURE);+ vi.unstubAllGlobals();+ expect(lastLine()).toBe('switch-agent-controller: validation_error: name too long');+ const reopened = ControllerStore.open(join(dir, 'controller.db'));+ expect(reopened.identity()?.name).toBe('box');+ reopened.close();+ });+it('exits 2 when it is not enrolled', async () => {expect(await main(['run', '--data-dir', dir, '--shared-host-bundle', bundle])).toBe(EXIT_CONFIGURATION@@ -159,3 +269,16 @@ describe('main', () => {expect(lastLine()).toMatch(/not enrolled/);});});++describe('infoChange', () => {+ it('trims, clears a blank description, and keeps to the limits', () => {+ expect(infoChange({ name: ' box ' })).toEqual({ name: 'box' });+ expect(infoChange({ description: ' ' })).toEqual({ description: null });+ expect(infoChange({ name: 'x'.repeat(200), description: 'y'.repeat(500) })).toEqual({+ name: 'x'.repeat(200),+ description: 'y'.repeat(500),+ });+ expect(() => infoChange({ name: 'x'.repeat(201) })).toThrow(/at most 200/);+ expect(() => infoChange({ description: 'y'.repeat(501) })).toThrow(/at most 500/);+ });+});diff --git a/console/packages/agent-controller/src/main.ts b/console/packages/agent-controller/src/main.tsindex b41b0d9e0..c2fbe0db2 100644--- a/console/packages/agent-controller/src/main.ts+++ b/console/packages/agent-controller/src/main.ts@@ -1,7 +1,14 @@import { hostname } from 'node:os';import { parseArgs } from 'node:util';import packageJson from '../package.json' with { type: 'json' };-import { ControllerApiError, enroll, normalizeServerUrl, nodeWebSocket } from './api';+import {+ AccessTokens,+ ControllerApiError,+ ControllerClient,+ enroll,+ normalizeServerUrl,+ nodeWebSocket,+} from './api';import { DEFAULT_TIMING, runController } from './controller';import { DetachedRuntime } from './detached-runtime';import { ConfigurationError, UsageError } from './errors';@@ -30,6 +37,7 @@ import {observeOnDisk,} from './runtime';import { AgentRuntimes } from './runtimes';+import type { ControllerInfoChange } from './schemas';import { CONTROLLER_CREDENTIAL, FileSecretStore, MemorySecretStore } from './secrets';import { contractPlatform, mapAgentProcess, PathProviderLocator } from './status';import { ControllerStore } from './store';@@ -38,6 +46,9 @@ export const VERSION: string = packageJson.version;const SIGNALS = ['SIGINT', 'SIGTERM'] as const;+const MAX_NAME = 200;+const MAX_DESCRIPTION = 500;+const USAGE = `Usage: switch-agent-controller <command> [options]Commands:@@ -54,6 +65,10 @@ Commands:the data directory holds none, and move the same identity to a newserver URL when it holds that one. --credential-stdin reads thecontroller credential from stdin and keeps it in memory only.+ set-info [--name <name>] [--description <text>] [--data-dir <dir>]+ Rename this machine and/or change its description on the server, with+ this controller's own credential. Either or both; --description ""+ clears the description. The same limits as at enrollment.status [--data-dir <dir>] [--shared-host-bundle <path>]Show this controller's identity and its agents, from local state only.@@ -98,8 +113,8 @@ async function enrollCommand(args: string[]): Promise<number> {const server = normalizeServerUrl(values.server);const name = values.name ?? hostname();const description = values.description?.trim() || undefined;- if (description !== undefined && description.length > 500)- throw new UsageError('--description must be at most 500 characters.');+ if (description !== undefined && description.length > MAX_DESCRIPTION)+ throw new UsageError(`--description must be at most ${MAX_DESCRIPTION} characters.`);const { dataDir, store, secrets } = await openState(values['data-dir']);try {const existing = store.identity();@@ -134,6 +149,82 @@ async function enrollCommand(args: string[]): Promise<number> {}}+/** The change `set-info` asks for, checked as the server checks it. */+export function infoChange(values: { name?: string; description?: string }): ControllerInfoChange {+ if (values.name === undefined && values.description === undefined)+ throw new UsageError('set-info needs --name, --description, or both.');+ const change: ControllerInfoChange = {};+ if (values.name !== undefined) {+ const name = values.name.trim();+ if (!name) throw new UsageError('--name must not be blank.');+ if (name.length > MAX_NAME)+ throw new UsageError(`--name must be at most ${MAX_NAME} characters.`);+ change.name = name;+ }+ if (values.description !== undefined) {+ const description = values.description.trim();+ if (description.length > MAX_DESCRIPTION)+ throw new UsageError(`--description must be at most ${MAX_DESCRIPTION} characters.`);+ change.description = description || null;+ }+ return change;+}++async function setInfoCommand(args: string[]): Promise<number> {+ const { values } = parseArgs({+ args,+ options: {+ name: { type: 'string' },+ description: { type: 'string' },+ 'data-dir': { type: 'string' },+ },+ strict: true,+ });+ const change = infoChange(values);+ const { dataDir, store, secrets } = await openState(values['data-dir']);+ try {+ const identity = store.identity();+ if (!identity)+ throw new ConfigurationError(`${dataDir} holds no enrolled controller. Enroll it first.`);+ if (store.revokedAt())+ throw new ConfigurationError(+ `Controller ${identity.controllerId} was revoked; it has to be enrolled again.`+ );+ const credential = await secrets.get(CONTROLLER_CREDENTIAL);+ if (!credential)+ throw new ConfigurationError(+ `${dataDir} holds no controller credential (it is handed over at run time, or was removed). Change the name and description in the gateway's Machines page instead.`+ );+ const log = createLogger({+ level: process.env.SWITCH_CONTROLLER_LOG_LEVEL,+ write: (line) => process.stderr.write(line),+ });+ const client = new ControllerClient({+ fetch,+ server: identity.server,+ controllerId: identity.controllerId,+ version: VERSION,+ tokens: new AccessTokens({+ fetch,+ server: identity.server,+ controllerId: identity.controllerId,+ credential: async () => credential,+ now: Date.now,+ log,+ }),+ openWebSocket: nodeWebSocket,+ });+ const updated = await client.updateInfo(change);+ store.saveName(updated.name);+ process.stdout.write(+ `Controller ${updated.id} is now "${updated.name}"${updated.description ? `: ${updated.description}` : ', with no description'}.\n`+ );+ return EXIT_OK;+ } finally {+ store.close();+ }+}+async function runCommand(args: string[]): Promise<number> {const { values } = parseArgs({args,@@ -310,6 +401,8 @@ export async function main(argv: string[]): Promise<number> {return await enrollCommand(rest);case 'run':return await runCommand(rest);+ case 'set-info':+ return await setInfoCommand(rest);case 'status':return await statusCommand(rest);case undefined:diff --git a/console/packages/agent-controller/src/schemas.test.ts b/console/packages/agent-controller/src/schemas.test.tsindex 0b1b20b56..38d7b5716 100644--- a/console/packages/agent-controller/src/schemas.test.ts+++ b/console/packages/agent-controller/src/schemas.test.ts@@ -7,6 +7,8 @@ import {controllerConnectionRequestSchema,controllerPongSchema,controllerConnectionResponseSchema,+ controllerInfoRequestSchema,+ controllerInfoResponseSchema,credentialRotateResponseSchema,enrollRequestSchema,enrollResponseSchema,@@ -48,6 +50,8 @@ const SCHEMA_FOR: Record<string, z.ZodType> = {'controller_pong.json': controllerPongSchema,'controller_connection_request.json': controllerConnectionRequestSchema,'controller_connection_response.json': controllerConnectionResponseSchema,+ 'controller_info_request.json': controllerInfoRequestSchema,+ 'controller_info_response.json': controllerInfoResponseSchema,'credential_rotate_response.json': credentialRotateResponseSchema,'enroll_request.json': enrollRequestSchema,'enroll_response.json': enrollResponseSchema,diff --git a/console/packages/agent-controller/src/schemas.ts b/console/packages/agent-controller/src/schemas.tsindex a4f4571b4..4bd3a0b97 100644--- a/console/packages/agent-controller/src/schemas.ts+++ b/console/packages/agent-controller/src/schemas.ts@@ -110,6 +110,21 @@ export type TokenResponse = z.infer<typeof tokenResponseSchema>;export const credentialRotateResponseSchema = z.object({ credential: id });+/** Renaming this machine, or changing its description: either or both; `description: null` clears it. */+export const controllerInfoRequestSchema = z.object({+ name: z.string().min(1).max(200).optional(),+ description: z.string().max(500).nullable().optional(),+});+export type ControllerInfoChange = z.infer<typeof controllerInfoRequestSchema>;++/** The machine as its owner's list shows it, after the change; only what the controller reads. */+export const controllerInfoResponseSchema = z.object({+ id,+ name: z.string().min(1),+ description: z.string().nullable(),+});+export type ControllerInfo = z.infer<typeof controllerInfoResponseSchema>;+// §2 Assignment (v1 definition)export const agentDefinitionSchema = z.object({diff --git a/console/packages/agent-controller/src/store.test.ts b/console/packages/agent-controller/src/store.test.tsindex f711061b1..bce76aead 100644--- a/console/packages/agent-controller/src/store.test.ts+++ b/console/packages/agent-controller/src/store.test.ts@@ -96,6 +96,21 @@ describe('ControllerStore', () => {store.close();});+ it('records a new name for the identity and keeps the rest', () => {+ const store = ControllerStore.open(path);+ expect(() => store.saveName('renamed')).toThrow(/no identity/);+ const identity = {+ controllerId: 'controller-1',+ server: 'https://switch.example.com',+ name: 'build-box',+ enrolledAt: '2026-01-01T00:00:00.000Z',+ };+ store.saveIdentity(identity);+ store.saveName('renamed');+ expect(store.identity()).toEqual({ ...identity, name: 'renamed' });+ store.close();+ });+it('caches the assignment with its ETag', () => {const store = ControllerStore.open(path);expect(store.cachedAssignment()).toEqual({ kind: 'none' });diff --git a/console/packages/agent-controller/src/store.ts b/console/packages/agent-controller/src/store.tsindex 1228cc510..1359fc3fb 100644--- a/console/packages/agent-controller/src/store.ts+++ b/console/packages/agent-controller/src/store.ts@@ -135,6 +135,13 @@ export class ControllerStore {});}+ /** Records the name the server now has for this controller. */+ saveName(name: string): void {+ if (!this.meta('controller_id'))+ throw new Error('There is no identity whose name could be changed.');+ this.setMeta('name', name);+ }+/** Moves the identity to another server URL, keeping everything else. */saveServer(server: string): void {if (!this.meta('controller_id'))diff --git a/console/packages/agent-controller/src/testing/fake-core.ts b/console/packages/agent-controller/src/testing/fake-core.tsindex a10238e0e..d741a84a2 100644--- a/console/packages/agent-controller/src/testing/fake-core.ts+++ b/console/packages/agent-controller/src/testing/fake-core.ts@@ -102,6 +102,8 @@ export class FakeCore {readonly results = new Map<string, OperationResult>();readonly opens: Record<string, AgentCursor>[] = [];readonly beats: Record<string, number>[] = [];+ /** The machine's name and description, as `PATCH .../controllers/{id}` changes them. */+ info: { name: string; description: string | null } = { name: 'laptop', description: null };/** Answers the next request to a path with this, once. */readonly scripted: { method: string; path: string; status: number; body: unknown }[] = [];/** What `GET .../media` sends, in these chunks; `waitBetween` holds back all but the first. */@@ -398,6 +400,12 @@ export class FakeCore {agents: this.bound(),});}+ if (method === 'PATCH' && url.pathname === base) {+ const change = body as { name?: string; description?: string | null };+ if (change.name !== undefined) this.info.name = change.name;+ if (change.description !== undefined) this.info.description = change.description;+ return this.json(res, 200, { id: this.controllerId, ...this.info, state: 'online' });+ }if (method === 'GET' && url.pathname === `${base}/assignment`) {const etag = `"${this.assignment.revision}"`;if (req.headers['if-none-match'] === etag) {diff --git a/console/packages/plugins/src/switch-skill/SKILL.md b/console/packages/plugins/src/switch-skill/SKILL.mdindex e000d5668..bb5172af2 100644--- a/console/packages/plugins/src/switch-skill/SKILL.md+++ b/console/packages/plugins/src/switch-skill/SKILL.md@@ -666,8 +666,9 @@ tools exist, and only there — if they are not in your tool list, this serverdoes not manage agents, so say so rather than looking for another way.- **`list_machines`** — your owner's machines: `id`, `name`, `description`,- `state` (`online` or `unknown`), the providers each has (`installed`,- `auth`) and how many agents it is running. Removed machines are left out.+ `state` (`online`, `offline`, or `unknown` if it never connected), the+ providers each has (`installed`, `auth`) and how many agents it is running.+ Removed machines are left out.- **`get_advanced_config`** — the advanced settings a provider's agents cancarry (the "Advanced configuration" Switch Console shows): each field's`key`, `label`, `type` (`text`, `textarea`, `number`, `boolean`, `list` ofdiff --git a/core/switch_core/bridges/agent/operations/agent_management.py b/core/switch_core/bridges/agent/operations/agent_management.pyindex 407f35777..af6f36ca9 100644--- a/core/switch_core/bridges/agent/operations/agent_management.py+++ b/core/switch_core/bridges/agent/operations/agent_management.py@@ -123,8 +123,9 @@ async def list_machines() -> list[dict[str, Any]]:A list of machines, each {id, name, description, kind, state,last_seen_at, providers, agents_running}.`kind` is "console" (Switch Console's own machine), "daemon" (a- headless controller) or "ec2". `state` is "online" (reporting now) or- "unknown" (has not reported recently: asleep, off or disconnected).+ headless controller) or "ec2". `state` is "online" (connected to Switch+ now), "offline" (was connected, and is not now: stopped, asleep, off+ or disconnected) or "unknown" (has never connected).`providers` lists the agent CLIs the machine last reported, each{provider, installed, version, auth}, `auth` being "ok", "missing","expired" or "unknown". `agents_running` counts the managed agents@@ -260,7 +261,7 @@ async def list_managed_agents() -> list[dict[str, Any]]:`directory` is the working directory the definition names (null onlywhen its machine has not said where it keeps workspaces).`machine` is {id, name, state} (null when the agent is not placed on- a machine), `state` being "online", "unknown" or "revoked".+ a machine), `state` being "online", "offline", "unknown" or "revoked".`desired_state` is what your owner wants: "running" or "stopped".`actual` is what the machine last reported for the agent,{process, reason, detail, applied_revision, since, directory}, ordiff --git a/core/switch_core/bridges/agent/protocol/controller_presence.py b/core/switch_core/bridges/agent/protocol/controller_presence.pyindex c59cfea84..d5df3250a 100644--- a/core/switch_core/bridges/agent/protocol/controller_presence.py+++ b/core/switch_core/bridges/agent/protocol/controller_presence.py@@ -19,7 +19,12 @@ the bearer middleware's act-as check, and the controller stream.Like the connection registry it is memory only. A restart forgets everycontroller connection, and Management loads the bindings again before the-agent bridge serves.+agent bridge serves. Each controller connection's transitions, its socket+attaching and its socket going (and why), are also handed to a ledger, which+Management installs (`use_ledger`) and which persists them, so that whether+a machine is connected can be read from any process. Heartbeats are not:+they stay here. The ledger is told synchronously, as each transition happens,+once per transition."""from __future__ import annotations@@ -32,7 +37,7 @@ import uuidfrom collections import dequefrom collections.abc import Callable, Iterablefrom dataclasses import dataclass, field, replace-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, Protocolfrom switch_core.bridges.agent.protocol.liveness import (HEARTBEAT_LAPSED,@@ -54,6 +59,13 @@ REVOKED = Closure(code="closed", message="the controller has been revoked", room_id=None)+# Why a connection's socket went, as the ledger records it. A connection+# closed with a closure records the closure's code (`heartbeat_lapsed`,+# `taken_over`) instead.+DISCONNECT_REVOKED = "revoked"+DISCONNECT_SOCKET_CLOSED = "socket_closed"+DISCONNECT_SERVER_SHUTDOWN = "server_shutdown"+# How many superseded connection ids a controller is remembered by, so a late# beat from one is told `taken_over` rather than `unknown_connection`._SUPERSEDED_REMEMBERED = 16@@ -162,6 +174,20 @@ class ControllerConnection:)+class ControllerConnectionLedger(Protocol):+ """Where controller connections' transitions are persisted.++ Called as each happens, once each, and must not block. `connected`: a+ socket attached to a connection that had none. `disconnected`: the+ socket went, with the reason to record: `socket_closed`,+ `server_shutdown`, `heartbeat_lapsed`, `taken_over` or `revoked`.+ """++ def connected(self, conn: ControllerConnection) -> None: ...++ def disconnected(self, conn: ControllerConnection, reason: str) -> None: ...++class ControllerPresence:def __init__(self, *, on_bound: Callable[[str], None]) -> None:# Called with each agent that becomes controller-backed, so the@@ -177,12 +203,35 @@ class ControllerPresence:self._rooms: dict[str, set[str]] = {}self._next_generation = secrets.randbits(32)self._auth_cache: ControllerAuthCache | None = None+ self._ledger: ControllerConnectionLedger | None = None+ # Set as the process begins shutting down: a socket going after that+ # went because the server closed it.+ self._shutting_down = Falsedef use_auth_cache(self, cache: ControllerAuthCache) -> None:"""The cache of controller-token reads to keep in step with thebindings and revocations recorded here."""self._auth_cache = cache+ def use_ledger(self, ledger: ControllerConnectionLedger) -> None:+ """Where each connection's socket attaching and going are persisted."""+ self._ledger = ledger++ def begin_shutdown(self) -> None:+ """The process is shutting down: sockets that go from now are+ recorded as closed by the server."""+ self._shutting_down = True++ def _record_connected(self, conn: ControllerConnection) -> None:+ if self._ledger is not None:+ self._ledger.connected(conn)++ def _record_disconnected(self, conn: ControllerConnection, reason: str) -> None:+ """Record the socket going, if one was attached: a connection whose+ socket already went was recorded then."""+ if conn.stream_attached and self._ledger is not None:+ self._ledger.disconnected(conn, reason)+def _invalidate_agent_auth(self, agent_id: str) -> None:if self._auth_cache is not None:self._auth_cache.invalidate_agent(agent_id)@@ -264,6 +313,7 @@ class ControllerPresence:conn = self._connections.pop(controller_id, None)if conn is not None:conn.closure = REVOKED+ self._record_disconnected(conn, DISCONNECT_REVOKED)conn.stream_attached = Falseconn.wake.set()@@ -399,6 +449,7 @@ class ControllerPresence:previous = self._connections.get(controller_id)if previous is not None:previous.closure = TAKEN_OVER+ self._record_disconnected(previous, TAKEN_OVER.code)previous.stream_attached = Falseprevious.wake.set()remembered = self._superseded.setdefault(@@ -448,8 +499,12 @@ class ControllerPresence:"""Attach a stream to the connection, displacing any attached before.Attaching counts as a beat: the stream is what the beat proves alive,- and the client cannot beat before it has one.+ and the client cannot beat before it has one. A connection that had+ no socket attached is recorded as connected; displacing an attached+ one changes nothing the ledger holds."""+ if not conn.stream_attached:+ self._record_connected(conn)conn.stream_token += 1conn.stream_attached = Trueconn.last_beat = time.monotonic()@@ -457,7 +512,16 @@ class ControllerPresence:return conn.stream_tokendef detach_stream(self, conn: ControllerConnection, token: int) -> None:+ """The socket holding this stream went. The connection stays open for+ the socket to reattach within the heartbeat TTL, but the machine is+ recorded as disconnected now."""if conn.stream_token == token:+ self._record_disconnected(+ conn,+ DISCONNECT_SERVER_SHUTDOWN+ if self._shutting_down+ else DISCONNECT_SOCKET_CLOSED,+ )conn.stream_attached = Falsedef beat(@@ -477,6 +541,10 @@ class ControllerPresence:if current is None or cursor > current:conn.resume_cursors[agent_id] = cursor+ def current_connection(self, controller_id: str) -> ControllerConnection | None:+ """The controller's open connection, live or not yet swept."""+ return self._connections.get(controller_id)+def is_current(self, conn: ControllerConnection) -> bool:return self._connections.get(conn.controller_id) is conn@@ -528,6 +596,7 @@ class ControllerPresence:closure.code,conn.beats,)+ self._record_disconnected(conn, closure.code)conn.stream_attached = Falseconn.wake.set()diff --git a/core/switch_core/db/models.py b/core/switch_core/db/models.pyindex 063502810..483927269 100644--- a/core/switch_core/db/models.py+++ b/core/switch_core/db/models.py@@ -2392,6 +2392,37 @@ class TelemetrySnapshotWatermark(Base):)+# ── Switch-core processes ────────────────────────────────────────────────────+++class SwitchCoreProcess(Base):+ """A running switch-core process's lease (`management/process_lease.py`).++ Each process holding controller sockets claims a row at startup, renews+ `beat_at` every few seconds, and sets `stopped_at` as it shuts down. A+ controller connection whose holding process's lease is stale, stopped or+ gone reads as offline, which is how a process that died without writing+ its connections' closings is noticed. One write per process per renewal,+ however many machines it holds.++ Not tenant-scoped: a process serves every tenant. Rows long stale are+ pruned by the live processes.+ """++ __tablename__ = "switch_core_processes"++ id: Mapped[str] = mapped_column(Text, primary_key=True)+ started_at: Mapped[datetime] = mapped_column(+ DateTime(timezone=True), server_default=func.now(), nullable=False+ )+ beat_at: Mapped[datetime] = mapped_column(+ DateTime(timezone=True), server_default=func.now(), nullable=False+ )+ stopped_at: Mapped[datetime | None] = mapped_column(+ DateTime(timezone=True), nullable=True+ )++# ── Feature flags ────────────────────────────────────────────────────────────@@ -3084,6 +3115,17 @@ class AgentController(TenantScoped, Base):`status_seq` its sequence number: a report with a sequence at or below it isignored. `assignment_revision` bumps on every change to the set of agentsthe controller should run, and is what its ETag carries.++ The `connection_*`, `connected_at` and `disconnect*` columns are the+ controller's socket as the switch-core process holding it last recorded it+ (`management/connection_ledger.py`): which connection, which process holds+ it (`connection_process_id`, a `switch_core_processes` row), when its+ socket attached, and when and why it went. Only transitions are written,+ never heartbeats; a process that dies without writing the closing is+ caught by its lease in `switch_core_processes` going stale. They describe+ the current connection, or the last one once it has closed; a new one+ replaces them. Every replica reads them, so whether the machine is+ connected does not depend on which process holds its socket."""__tablename__ = "agent_controllers"@@ -3123,6 +3165,19 @@ class AgentController(TenantScoped, Base):revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)+ connection_id: Mapped[str | None] = mapped_column(Text, nullable=True)+ # Not a foreign key: a lease row is pruned once long stale, and a+ # connection naming a process with no lease reads as lost.+ connection_process_id: Mapped[str | None] = mapped_column(Text, nullable=True)+ connected_at: Mapped[datetime | None] = mapped_column(+ DateTime(timezone=True), nullable=True+ )+ disconnected_at: Mapped[datetime | None] = mapped_column(+ DateTime(timezone=True), nullable=True+ )+ # `socket_closed`, `heartbeat_lapsed`, `taken_over`, `revoked` or+ # `server_shutdown`.+ disconnect_reason: Mapped[str | None] = mapped_column(Text, nullable=True)created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), nullable=False)diff --git a/core/switch_core/db/rls_ddl.py b/core/switch_core/db/rls_ddl.pyindex d2667b52f..b45dd8bc4 100644--- a/core/switch_core/db/rls_ddl.py+++ b/core/switch_core/db/rls_ddl.py@@ -127,6 +127,11 @@ POLICY_NAME = "tenant_isolation"# which reports counts and never an identifier — and scoping them would be# incoherent: a deployment running three tenants has one identity, not three,# and a milestone reported once per tenant would not be once-ever at all.+#+# `switch_core_processes` is each running switch-core process's lease: its id,+# when it started, its last renewal and when it stopped. A process holds+# controller sockets for every tenant at once, so its lease belongs to none of+# them, and it holds no customer data, only that the process is alive.GLOBAL_TABLES = frozenset({"users",@@ -135,6 +140,7 @@ GLOBAL_TABLES = frozenset("deployment_identity","telemetry_milestones","telemetry_snapshot_watermark",+ "switch_core_processes",})diff --git a/core/switch_core/db/stores/agent_controller_store.py b/core/switch_core/db/stores/agent_controller_store.pyindex e142757cb..0eadbb757 100644--- a/core/switch_core/db/stores/agent_controller_store.py+++ b/core/switch_core/db/stores/agent_controller_store.py@@ -164,6 +164,72 @@ class AgentControllerStore:)return result.scalar_one_or_none() is not None+ async def record_connected(+ self,+ session: AsyncSession,+ tenant_id: str,+ controller_id: str,+ *,+ connection_id: str,+ process_id: str,+ connected_at: datetime,+ ) -> bool:+ """Record that the controller's socket attached to this connection,+ held by this process. It replaces whatever the row held: the process+ holding the live socket is the authority. Returns whether the+ controller's row exists. `updated_at` is left alone: connecting is+ not an edit."""+ result = await session.execute(+ update(AgentController)+ .where(+ AgentController.tenant_id == tenant_id,+ AgentController.id == controller_id,+ )+ .values(+ connection_id=connection_id,+ connection_process_id=process_id,+ connected_at=connected_at,+ disconnected_at=None,+ disconnect_reason=None,+ updated_at=AgentController.updated_at,+ )+ .returning(AgentController.id)+ .execution_options(synchronize_session=False)+ )+ return result.scalar_one_or_none() is not None++ async def record_disconnected(+ self,+ session: AsyncSession,+ tenant_id: str,+ controller_id: str,+ *,+ connection_id: str,+ disconnected_at: datetime,+ reason: str,+ ) -> bool:+ """Record that the controller's socket on this connection went, and+ why. Written only while the row still names this connection, so a+ process closing a connection the controller has since replaced+ through another process leaves the replacement standing. Returns+ whether the row was written."""+ result = await session.execute(+ update(AgentController)+ .where(+ AgentController.tenant_id == tenant_id,+ AgentController.id == controller_id,+ AgentController.connection_id == connection_id,+ )+ .values(+ disconnected_at=disconnected_at,+ disconnect_reason=reason,+ updated_at=AgentController.updated_at,+ )+ .returning(AgentController.id)+ .execution_options(synchronize_session=False)+ )+ return result.scalar_one_or_none() is not None+async def set_credential(self,session: AsyncSession,diff --git a/core/switch_core/db/stores/switch_core_process_store.py b/core/switch_core/db/stores/switch_core_process_store.pynew file mode 100644index 000000000..f2e799a5e--- /dev/null+++ b/core/switch_core/db/stores/switch_core_process_store.py@@ -0,0 +1,72 @@+from __future__ import annotations++from dataclasses import dataclass+from datetime import datetime, timedelta++from sqlalchemy import delete, func, select, update+from sqlalchemy.dialects.postgresql import insert+from sqlalchemy.ext.asyncio import AsyncSession++from switch_core.db.models import SwitchCoreProcess+++@dataclass(frozen=True)+class ProcessLeaseRow:+ beat_at: datetime+ stopped_at: datetime | None+++class SwitchCoreProcessStore:+ """Each switch-core process's lease (`switch_core_processes`), a global+ table.++ Every time here is the database's own clock, so leases written by one+ process and read by another are compared on one clock whatever the two+ hosts' clocks say.+ """++ async def renew(self, session: AsyncSession, process_id: str) -> None:+ """Claim the lease, or renew it: its beat is now. A row pruned while+ this process could not reach the database is claimed again."""+ await session.execute(+ insert(SwitchCoreProcess)+ .values(id=process_id, started_at=func.now(), beat_at=func.now())+ .on_conflict_do_update(+ index_elements=[SwitchCoreProcess.id],+ set_={"beat_at": func.now(), "stopped_at": None},+ )+ )++ async def stop(self, session: AsyncSession, process_id: str) -> None:+ await session.execute(+ update(SwitchCoreProcess)+ .where(SwitchCoreProcess.id == process_id)+ .values(stopped_at=func.now())+ )++ async def prune(self, session: AsyncSession, older_than: timedelta) -> int:+ """Delete leases whose last beat is older than `older_than`. Returns+ how many."""+ result = await session.execute(+ delete(SwitchCoreProcess)+ .where(SwitchCoreProcess.beat_at < func.now() - older_than)+ .returning(SwitchCoreProcess.id)+ )+ return len(result.all())++ async def read(+ self, session: AsyncSession+ ) -> tuple[datetime, dict[str, ProcessLeaseRow]]:+ """The database's now, and every lease as it stands then."""+ now = (await session.execute(select(func.now()))).scalar_one()+ rows = await session.execute(+ select(+ SwitchCoreProcess.id,+ SwitchCoreProcess.beat_at,+ SwitchCoreProcess.stopped_at,+ )+ )+ return now, {+ process_id: ProcessLeaseRow(beat_at=beat_at, stopped_at=stopped_at)+ for process_id, beat_at, stopped_at in rows.all()+ }diff --git a/core/switch_core/main.py b/core/switch_core/main.pyindex 8c59e09c0..7103bd319 100644--- a/core/switch_core/main.py+++ b/core/switch_core/main.py@@ -32,6 +32,7 @@ from switch_core.bridges.agent.protocol.agent_connections import (AgentConnectionRegistry,)from switch_core.bridges.agent.protocol.agent_core import AgentCore+from switch_core.bridges.agent.protocol.controller_presence import ControllerPresencefrom switch_core.bridges.agent.protocol.event_buffer import EventBufferfrom switch_core.bridges.agent.registration_bootstrap import (BOOTSTRAP_KEY_LABEL,@@ -144,7 +145,7 @@ from switch_core.gateway.app import create_gateway_appfrom switch_core.gateway.auth import hash_passwordfrom switch_core.gateway.invite_mail import SmtpInviteMailerfrom switch_core.logging_config import configure_logging-from switch_core.management.wiring import create_management+from switch_core.management.wiring import Management, create_managementfrom switch_core.messages.notify import MessageListenerfrom switch_core.observability.bootstrap import (Observability,@@ -258,6 +259,32 @@ async def _connection_sweep_loop(protocol: AgentCore, lag: EventLoopLag) -> Nonelogger.exception("AgentConnection sweep failed")+# Under `_FORCED_EXIT_GRACE_SECONDS`, and first in the teardown: what it+# writes is what lets every other process see this one's machines go offline+# now rather than when its lease lapses.+_MANAGEMENT_STOP_SECONDS = 1.0+++async def _stop_management(management: Management) -> None:+ """Record the controller sockets that went and mark this process's lease+ stopped. Never raises: if it fails, the lease lapses on its own."""+ try:+ async with asyncio.timeout(_MANAGEMENT_STOP_SECONDS):+ await management.stop()+ except TimeoutError:+ logger.warning(+ "Gave up recording controller disconnections after %.1fs; their "+ "machines read offline once this process's lease lapses.",+ _MANAGEMENT_STOP_SECONDS,+ )+ except Exception:+ logger.warning(+ "Recording controller disconnections at shutdown failed; their "+ "machines read offline once this process's lease lapses.",+ exc_info=True,+ )++# The innermost of three nested budgets: under# `observability.logs.SHUTDOWN_FLUSH_SECONDS`, itself under# `_FORCED_EXIT_GRACE_SECONDS`.@@ -823,6 +850,7 @@ async def run(config: SwitchConfig) -> None:# Before the bridge serves: until Core knows which agents a controller# runs, their own keys would be let in and their presence misread.await management.load_bindings()+ await management.start()agent_bridge_app.mount("/gateway", gateway_app)@@ -875,6 +903,11 @@ async def run(config: SwitchConfig) -> None:connection_sweep_task = asyncio.create_task(_connection_sweep_loop(protocol, observability.lag))+ management_task = (+ asyncio.create_task(management.run())+ if management is not None+ else None+ )# Only when telemetry is on: the chart tells a customer that off# means nothing is collected, and the fan-out is not free.snapshot_task = (@@ -892,6 +925,10 @@ async def run(config: SwitchConfig) -> None:connection_sweep_task.cancel()if snapshot_task is not None:snapshot_task.cancel()+ if management is not None:+ assert management_task is not None+ management_task.cancel()+ await _stop_management(management)await message_listener.stop()await session_activity_listener.stop()# Before the operational flush, and bounded: the whole@@ -982,6 +1019,7 @@ async def run(config: SwitchConfig) -> None:lambda: asyncio.create_task(_shutdown(server,+ connections.controllers,client_lifecycle,collab_lifecycle,connector_lifecycle,@@ -1429,6 +1467,7 @@ async def _bootstrap_key_tenant(async def _shutdown(server: uvicorn.Server,+ controllers: ControllerPresence,client_lifecycle: ClientLifecycleService,collab_lifecycle: CollaborationBridgeLifecycleService,connector_lifecycle: ServerSideConnectorLifecycleService,@@ -1437,6 +1476,9 @@ async def _shutdown(discord_gateway_task: asyncio.Task[None] | None,) -> None:logger.info("Shutting down...")+ # Before uvicorn closes the sockets, so the controllers' are recorded as+ # closed by the server.+ controllers.begin_shutdown()server.should_exit = Trueawait connector_lifecycle.stop_all()await collab_lifecycle.stop_all()diff --git a/core/switch_core/management/agent_operations.py b/core/switch_core/management/agent_operations.pyindex 40549b808..d6f6ff3df 100644--- a/core/switch_core/management/agent_operations.py+++ b/core/switch_core/management/agent_operations.py@@ -29,7 +29,8 @@ from switch_core.db.session_scope import tenant_sessionfrom switch_core.management import reason_codesfrom switch_core.management.advanced_config import provider_fieldsfrom switch_core.management.errors import ManagementError-from switch_core.management.placement import provider_auth+from switch_core.management.placement import is_revoked, provider_auth+from switch_core.management.process_lease import ProcessLeasesfrom switch_core.management.schemas import (CreateManagedAgentRequest,DefinitionV1,@@ -65,9 +66,9 @@ def _placement_sentence(code: str, machine: str, provider: str) -> str | None:f"machine '{machine}' has been removed from Switch; pick another machine"),reason_codes.CONTROLLER_OFFLINE: (- f"machine '{machine}' has not reported recently (it may be asleep, "- "off or disconnected); your owner needs to bring it back online, or "- "pick another machine"+ f"machine '{machine}' is not connected to Switch or has not reported "+ "recently (it may be asleep, off or disconnected); your owner needs "+ "to bring it back online, or pick another machine"),reason_codes.PROVIDER_NOT_INSTALLED: (f"{provider} is not installed on machine '{machine}'; your owner "@@ -136,9 +137,10 @@ class ManagementAgentOperations:controllers = await self._service.controllers.list_for_owner(session, tenant_id, owner_id)+ leases = await self._service.leases(session)machines = []for controller in controllers:- state = self._service.state_of(controller)+ state = self._service.state_of(controller, leases)if state == "revoked":continuemachines.append(@@ -180,11 +182,7 @@ class ManagementAgentOperations:by_id = next((c for c in controllers if c.id == machine), None)if by_id is not None:return by_id- named = [- c- for c in controllers- if c.name == machine and self._service.state_of(c) != "revoked"- ]+ named = [c for c in controllers if c.name == machine and not is_revoked(c)]if len(named) == 1:return named[0]if not named:@@ -193,8 +191,9 @@ class ManagementAgentOperations:f"{nothing_done}: your owner has no machine with the id or "f"name '{machine}'. list_machines shows the machines you can use.",)+ leases = await self._service.leases(session)candidates = "; ".join(- _machine_line(c, self._service.state_of(c)) for c in named+ _machine_line(c, self._service.state_of(c, leases)) for c in named)raise AgentManagementRefused(reason_codes.VALIDATION_ERROR,@@ -259,6 +258,7 @@ class ManagementAgentOperations:row: AgentDefinitionRow,agent: Agent,controller: AgentController | None,+ leases: ProcessLeases,) -> dict[str, Any]:status = agent_status_from(controller, agent.id)return {@@ -275,7 +275,7 @@ class ManagementAgentOperations:else {"id": controller.id,"name": controller.name,- "state": self._service.state_of(controller),+ "state": self._service.state_of(controller, leases),},"desired_state": row.desired_state,"actual": None@@ -304,11 +304,13 @@ class ManagementAgentOperations:rows = await self._service.definitions.list_for_owner(session, tenant_id, owner_id)+ leases = await self._service.leases(session)return [self._managed_entry(row,agent,controllers.get(row.controller_id) if row.controller_id else None,+ leases,)for row, agent in rows]@@ -335,7 +337,9 @@ class ManagementAgentOperations:if row.controller_id is not Noneelse None)- return self._managed_entry(row, agent, controller)+ return self._managed_entry(+ row, agent, controller, await self._service.leases(session)+ )async def update_managed_agent(self,diff --git a/core/switch_core/management/connection_ledger.py b/core/switch_core/management/connection_ledger.pynew file mode 100644index 000000000..a00b0ab9e--- /dev/null+++ b/core/switch_core/management/connection_ledger.py@@ -0,0 +1,177 @@+"""Persisting each controller connection's transitions, so any process can+tell whether the machine is connected.++Core's `ControllerPresence` decides when a controller's socket attaches to+its connection and when it goes, in the memory of the process holding it,+and tells this ledger as each happens. Heartbeats stay there: nothing here is+written per beat. The ledger writes each transition to the controller's row+once:++- connected: `connection_id`, `connection_process_id` (this process),+ `connected_at`, and the previous closing cleared;+- disconnected: `disconnected_at` and `disconnect_reason` (`socket_closed`,+ `server_shutdown`, `heartbeat_lapsed`, `taken_over`, `revoked`).++That row, and the lease of the process it names (`process_lease.py`), are+what a machine's state is read from (`placement.controller_state`).++Which write wins, across processes: a connecting comes from the process+holding the live socket, and replaces whatever the row holds. A closing is+only written while the row still names that connection, so a process closing+a connection the controller has since replaced through another process+leaves the replacement standing. (Clocks are not compared: two processes'+clocks need not agree.)++Writes are queued, latest per controller, and written by one task (`run`),+woken by each transition, so a socket attaching or going never waits on the+database. A write that fails is kept, unless something newer for the same+controller has arrived meanwhile, and retried.+"""++from __future__ import annotations++import asyncio+import logging+from collections.abc import Callable+from dataclasses import dataclass+from datetime import datetime++from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker++from switch_core.bridges.agent.protocol.controller_presence import (+ ControllerConnection,+)+from switch_core.db.session_scope import tenant_session+from switch_core.db.stores.agent_controller_store import AgentControllerStore++logger = logging.getLogger(__name__)++# How long a failed write waits before it is tried again.+RETRY_SECONDS = 2.0+++@dataclass(frozen=True)+class _Transition:+ tenant_id: str+ controller_id: str+ connection_id: str+ at: datetime+ # None for a connecting; why the socket went for a closing.+ disconnect_reason: str | None+++class ControllerConnectionLedger:+ """Core's `ControllerConnectionLedger`, written to `agent_controllers`."""++ def __init__(+ self,+ *,+ process_id: str,+ session_factory: async_sessionmaker[AsyncSession],+ controllers: AgentControllerStore,+ clock: Callable[[], datetime],+ ) -> None:+ self._process_id = process_id+ self._session_factory = session_factory+ self._controllers = controllers+ self._clock = clock+ self._pending: dict[str, _Transition] = {}+ self._wake = asyncio.Event()+ self._lock = asyncio.Lock()++ def connected(self, conn: ControllerConnection) -> None:+ self._queue(conn, None)++ def disconnected(self, conn: ControllerConnection, reason: str) -> None:+ self._queue(conn, reason)++ def _queue(self, conn: ControllerConnection, reason: str | None) -> None:+ self._pending[conn.controller_id] = _Transition(+ tenant_id=conn.tenant_id,+ controller_id=conn.controller_id,+ connection_id=conn.id,+ at=self._clock(),+ disconnect_reason=reason,+ )+ self._wake.set()++ @property+ def pending(self) -> int:+ return len(self._pending)++ async def flush_all(self) -> None:+ """Write every queued transition. Raises the first failure, keeping+ what was not written."""+ async with self._lock:+ transitions = list(self._pending.values())+ self._pending.clear()+ if transitions:+ await self._write(transitions)++ async def run(self) -> None:+ """Write transitions as they are queued, until cancelled."""+ while True:+ await self._wake.wait()+ self._wake.clear()+ try:+ await self.flush_all()+ except Exception:+ logger.exception(+ "Recording controller connection transitions failed; %d kept "+ "and retried in %.0fs",+ len(self._pending),+ RETRY_SECONDS,+ )+ await asyncio.sleep(RETRY_SECONDS)+ self._wake.set()++ async def _write(self, transitions: list[_Transition]) -> None:+ by_tenant: dict[str, list[_Transition]] = {}+ for transition in transitions:+ by_tenant.setdefault(transition.tenant_id, []).append(transition)+ unwritten = list(transitions)+ try:+ for tenant_id, batch in by_tenant.items():+ async with tenant_session(self._session_factory, tenant_id) as session:+ for transition in batch:+ await self._write_one(session, transition)+ await session.commit()+ for transition in batch:+ unwritten.remove(transition)+ except BaseException:+ for transition in unwritten:+ self._pending.setdefault(transition.controller_id, transition)+ raise++ async def _write_one(self, session: AsyncSession, transition: _Transition) -> None:+ if transition.disconnect_reason is None:+ found = await self._controllers.record_connected(+ session,+ transition.tenant_id,+ transition.controller_id,+ connection_id=transition.connection_id,+ process_id=self._process_id,+ connected_at=transition.at,+ )+ if not found:+ logger.warning(+ "Controller %s connected, but has no row to record it on",+ transition.controller_id,+ )+ return+ written = await self._controllers.record_disconnected(+ session,+ transition.tenant_id,+ transition.controller_id,+ connection_id=transition.connection_id,+ disconnected_at=transition.at,+ reason=transition.disconnect_reason,+ )+ if not written:+ logger.info(+ "Controller %s connection %s went (%s), but its row names a newer "+ "connection; left as it is",+ transition.controller_id,+ transition.connection_id,+ transition.disconnect_reason,+ )diff --git a/core/switch_core/management/controller_routes.py b/core/switch_core/management/controller_routes.pyindex 66dfb2094..57f1db18b 100644--- a/core/switch_core/management/controller_routes.py+++ b/core/switch_core/management/controller_routes.py@@ -65,6 +65,7 @@ from switch_core.management.errors import ManagementError, ManagementRoute, errofrom switch_core.management.schemas import (MAX_STATUS_BYTES,ControllerConnectionRequest,+ ControllerInfoRequest,DefinitionV1,EnrollRequest,OperationResultRequest,@@ -210,6 +211,22 @@ async def exchange_token(# ── Controller access token ───────────────────────────────────────────────────[email protected]("/v1/management/controllers/{controller_id}")+async def update_controller_info(+ body: ControllerInfoRequest,+ principal: PathController,+ management: Management,+ session: Session,+) -> dict[str, Any]:+ """Rename this machine and/or change its description, as its owner can+ in the gateway. Returns the machine as the owner's list shows it."""+ return await management.update_own_controller(+ session,+ principal,+ {key: getattr(body, key) for key in body.model_fields_set},+ )++@router.post("/v1/management/controllers/{controller_id}/credential/rotate")async def rotate_credential(principal: PathController, management: Management, session: Sessiondiff --git a/core/switch_core/management/placement.py b/core/switch_core/management/placement.pyindex a59447d16..49c564833 100644--- a/core/switch_core/management/placement.py+++ b/core/switch_core/management/placement.py@@ -1,15 +1,30 @@"""Controller state, and whether an agent may be placed on a controller.-A controller is `revoked` once its credential is gone, `online` while its-last accepted status report is fresh, and `unknown` otherwise — including-before it has ever reported. Fresh means within three status intervals, the-interval being what each controller is told as `report_within_s`.--Placement reads the controller's last status and refuses, with a contract-reason code, when the controller is revoked or not online, when the-definition's provider is not installed there, or when its login is missing-or expired. A provider whose login is `unknown` passes: the controller could-not tell, and refusing would block every provider whose CLI has no probe.+A controller's state is read from its connection's recorded transitions+(`connection_ledger.py`) and the lease of the switch-core process holding it+(`process_lease.py`), so every process answers alike and a restart changes+nothing:++- `revoked`: its credential is gone.+- `online`: its socket is attached, and the process holding it is alive+ (its lease renewed within `LEASE_TTL`, and not stopped).+- `offline`: it has connected before, and its socket went (closed, its+ heartbeat lapsed, taken over and not replaced, the server shut down), or+ the process holding it stopped or died without saying so.+- `unknown`: no connection has ever been recorded for it: enrolled but never+ run, or connected only to a server that did not record connections.++Its status report is not what makes it online; the report carries the+details (providers, disk, agents).++Placement needs the controller online and its last status report fresh,+within three status intervals (the interval being what each controller is+told as `report_within_s`), since that report is what the provider checks+read. It refuses, with a contract reason code, when either is not so, when+the definition's provider is not installed there, or when its login is+missing or expired. A provider whose login is `unknown` passes: the+controller could not tell, and refusing would block every provider whose CLI+has no probe."""from __future__ import annotations@@ -20,27 +35,43 @@ from typing import Any, Literalfrom switch_core.db.models import AgentControllerfrom switch_core.management import reason_codesfrom switch_core.management.errors import ManagementError+from switch_core.management.process_lease import ProcessLeases-ControllerState = Literal["online", "unknown", "revoked"]+ControllerState = Literal["online", "offline", "unknown", "revoked"]STALE_AFTER_INTERVALS = 3PROVIDER_AUTH_STATES = frozenset({"ok", "expired", "missing", "unknown"})+def is_revoked(controller: AgentController) -> bool:+ return controller.revoked_at is not None or controller.api_key_id is None++def controller_state(- controller: AgentController, *, now: datetime, interval_seconds: int+ controller: AgentController, leases: ProcessLeases) -> ControllerState:- if controller.revoked_at is not None or controller.api_key_id is None:+ if is_revoked(controller):return "revoked"- if controller.status is None or controller.last_seen_at is None:- return "unknown"- stale_after = timedelta(seconds=STALE_AFTER_INTERVALS * interval_seconds)- if now - controller.last_seen_at > stale_after:+ if controller.connected_at is None:return "unknown"+ if controller.disconnected_at is not None:+ return "offline"+ if not leases.holds(controller.connection_process_id):+ return "offline"return "online"+def status_is_fresh(+ controller: AgentController, *, now: datetime, interval_seconds: int+) -> bool:+ """Whether the controller has reported status within three intervals."""+ if controller.status is None or controller.last_seen_at is None:+ return False+ stale_after = timedelta(seconds=STALE_AFTER_INTERVALS * interval_seconds)+ return now - controller.last_seen_at <= stale_after++def provider_auth(entry: dict[str, Any]) -> str:"""The entry's `auth`, with any value this server does not know read as unknown."""auth = entry.get("auth")@@ -51,14 +82,25 @@ def placement_refusal(controller: AgentController,provider: str,*,+ leases: ProcessLeases,now: datetime,interval_seconds: int,) -> tuple[str, str] | None:"""The reason code and message refusing this placement, or None to allow it."""- state = controller_state(controller, now=now, interval_seconds=interval_seconds)+ state = controller_state(controller, leases)if state == "revoked":return reason_codes.CONTROLLER_REVOKED, "the controller has been revoked"- if state != "online":+ if state == "unknown":+ return (+ reason_codes.CONTROLLER_OFFLINE,+ "the controller has never connected to Switch",+ )+ if state == "offline":+ return (+ reason_codes.CONTROLLER_OFFLINE,+ "the controller is not connected to Switch",+ )+ if not status_is_fresh(controller, now=now, interval_seconds=interval_seconds):return (reason_codes.CONTROLLER_OFFLINE,"the controller has not reported status recently",@@ -96,11 +138,16 @@ def require_placement(controller: AgentController,provider: str,*,+ leases: ProcessLeases,now: datetime,interval_seconds: int,) -> None:refusal = placement_refusal(- controller, provider, now=now, interval_seconds=interval_seconds+ controller,+ provider,+ leases=leases,+ now=now,+ interval_seconds=interval_seconds,)if refusal is not None:code, message = refusaldiff --git a/core/switch_core/management/process_lease.py b/core/switch_core/management/process_lease.pynew file mode 100644index 000000000..9e3d8192f--- /dev/null+++ b/core/switch_core/management/process_lease.py@@ -0,0 +1,131 @@+"""This switch-core process's lease, and reading every process's.++A controller's socket lives in the memory of the one process holding it, and+that process writes the connection's transitions to the controller's row+(`connection_ledger.py`). A process that dies writes no closing, so each+process also holds a lease in `switch_core_processes`: it claims it at+startup, renews it every `RENEW_INTERVAL`, and marks it stopped as it shuts+down. A connection reads as online only while its holding process's lease is+held: renewed within `LEASE_TTL` and not stopped. One write per process per+renewal, however many machines it holds.++Leases are written and compared on the database's clock, never a host's.+Leases not renewed for `PRUNE_AFTER` are deleted by whichever process renews+next; a connection naming a process with no lease reads as lost.+"""++from __future__ import annotations++import asyncio+import logging+from dataclasses import dataclass+from datetime import datetime, timedelta++from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker++from switch_core.db.stores.switch_core_process_store import (+ ProcessLeaseRow,+ SwitchCoreProcessStore,+)++logger = logging.getLogger(__name__)++RENEW_INTERVAL_SECONDS = 5.0++# Three renewals: one missed write, or a slow one, does not put every machine+# the process holds offline.+LEASE_TTL = timedelta(seconds=3 * RENEW_INTERVAL_SECONDS)++PRUNE_AFTER = timedelta(hours=1)++# Why a connection the row still shows open is not, when its process is the+# reason: the process shut down without writing it, or stopped renewing.+SERVER_SHUTDOWN = "server_shutdown"+SERVER_LOST = "server_lost"+++@dataclass(frozen=True)+class ProcessLeases:+ """Every process's lease as the database held it at `read_at`, its own+ clock."""++ read_at: datetime+ leases: dict[str, ProcessLeaseRow]++ def holds(self, process_id: str | None) -> bool:+ """Whether the process is alive and holding its sockets."""+ lease = self.leases.get(process_id) if process_id is not None else None+ return (+ lease is not None+ and lease.stopped_at is None+ and self.read_at - lease.beat_at < LEASE_TTL+ )++ def ended(self, process_id: str | None) -> tuple[datetime | None, str]:+ """When and why a process not holding stopped: shut down at+ `stopped_at`, or lost after its last renewal (when not known once its+ lease is pruned)."""+ lease = self.leases.get(process_id) if process_id is not None else None+ if lease is None:+ return None, SERVER_LOST+ if lease.stopped_at is not None:+ return lease.stopped_at, SERVER_SHUTDOWN+ return lease.beat_at, SERVER_LOST+++async def read_leases(+ session: AsyncSession, processes: SwitchCoreProcessStore+) -> ProcessLeases:+ """Every lease, in the caller's session; the table has no tenant policy,+ so any session reads it."""+ read_at, leases = await processes.read(session)+ return ProcessLeases(read_at=read_at, leases=leases)+++class ProcessLease:+ """This process's lease. The sessions are opened with no tenant bound:+ the table carries none."""++ def __init__(+ self,+ *,+ process_id: str,+ session_factory: async_sessionmaker[AsyncSession],+ processes: SwitchCoreProcessStore,+ ) -> None:+ self.process_id = process_id+ self._session_factory = session_factory+ self._processes = processes++ async def renew(self) -> None:+ """Claim or renew the lease, and prune leases long dead."""+ async with self._session_factory() as session:+ await self._processes.renew(session, self.process_id)+ pruned = await self._processes.prune(session, PRUNE_AFTER)+ await session.commit()+ if pruned:+ logger.info("Pruned %d switch-core process lease(s) long stale", pruned)++ async def stop(self) -> None:+ """Mark the lease stopped: every connection this process still shows+ open reads as offline from now."""+ async with self._session_factory() as session:+ await self._processes.stop(session, self.process_id)+ await session.commit()+ logger.info("Released switch-core process lease %s", self.process_id)++ async def run(self) -> None:+ """Renew every `RENEW_INTERVAL_SECONDS` until cancelled. A failed+ renewal is logged and retried at the next; three in a row and this+ process's machines read offline elsewhere."""+ while True:+ await asyncio.sleep(RENEW_INTERVAL_SECONDS)+ try:+ await self.renew()+ except Exception:+ logger.exception(+ "Renewing switch-core process lease %s failed; the machines "+ "it holds read offline once it is %s old",+ self.process_id,+ LEASE_TTL,+ )diff --git a/core/switch_core/management/schemas.py b/core/switch_core/management/schemas.pyindex f815994a8..7a475dec1 100644--- a/core/switch_core/management/schemas.py+++ b/core/switch_core/management/schemas.py@@ -15,13 +15,14 @@ same files.from __future__ import annotationsfrom datetime import UTC, datetime-from typing import Any, Literal+from typing import Any, Literal, Selffrom pydantic import BaseModel, ConfigDict, Field, field_validator, model_validatorfrom switch_core.db.models import Agent, AgentController, AgentControllerOperationfrom switch_core.db.models import AgentDefinition as AgentDefinitionRowfrom switch_core.management.advanced_config import validate_advanced_config+from switch_core.management.process_lease import ProcessLeasesProvider = Literal["claude", "codex", "opencode", "antigravity", "cursor"]ControllerKind = Literal["console", "daemon", "ec2"]@@ -344,7 +345,7 @@ class ConsoleControllerRequest(_GatewayBody):return _controller_description(value)-class UpdateControllerRequest(_GatewayBody):+class _ControllerDetailsChange(BaseModel):"""Rename a machine or change its description. Either or both; a key leftout is left as it is, and `description: null` (or blank) clears it — read`model_fields_set`, not the value."""@@ -369,12 +370,24 @@ class UpdateControllerRequest(_GatewayBody):return _controller_description(value)@model_validator(mode="after")- def _changes_something(self) -> UpdateControllerRequest:+ def _changes_something(self) -> Self:if not self.model_fields_set:raise ValueError("give a name, a description, or both")return self+class UpdateControllerRequest(_ControllerDetailsChange):+ """The owner's change, through the gateway."""++ model_config = ConfigDict(extra="forbid")+++class ControllerInfoRequest(_ControllerDetailsChange):+ """The controller's own change (`switch-agent-controller set-info`)."""++ model_config = ConfigDict(extra="ignore")++class CreateManagedAgentRequest(_GatewayBody):name: strdescription: str@@ -481,7 +494,32 @@ def workspaces_dir_of(controller: AgentController) -> str | None:return directory if isinstance(directory, str) and directory else None-def controller_view(controller: AgentController, state: str) -> dict[str, Any]:+def connection_view(+ controller: AgentController, leases: ProcessLeases+) -> dict[str, Any] | None:+ """The controller's connection as last recorded: the current one, or the+ last one once its socket went. None when none was ever recorded.++ A connection the row shows open whose holding process is no longer+ alive is shown as ended when and why that process stopped holding it:+ `server_shutdown` at its lease's `stopped_at`, or `server_lost` after its+ last renewal (when unknown once the lease is pruned)."""+ if controller.connected_at is None:+ return None+ disconnected_at = controller.disconnected_at+ reason = controller.disconnect_reason+ if disconnected_at is None and not leases.holds(controller.connection_process_id):+ disconnected_at, reason = leases.ended(controller.connection_process_id)+ return {+ "connected_at": wire_time(controller.connected_at),+ "disconnected_at": wire_time_or_none(disconnected_at),+ "disconnect_reason": reason,+ }+++def controller_view(+ controller: AgentController, state: str, leases: ProcessLeases+) -> dict[str, Any]:return {"id": controller.id,"name": controller.name,@@ -491,6 +529,7 @@ def controller_view(controller: AgentController, state: str) -> dict[str, Any]:"version": controller.version,"state": state,"last_seen_at": wire_time_or_none(controller.last_seen_at),+ "connection": connection_view(controller, leases),"status": controller.status,"assignment_revision": controller.assignment_revision,"workspaces_dir": workspaces_dir_of(controller),diff --git a/core/switch_core/management/service.py b/core/switch_core/management/service.pyindex b377577cf..bef779ab2 100644--- a/core/switch_core/management/service.py+++ b/core/switch_core/management/service.py@@ -58,6 +58,7 @@ from switch_core.db.stores.agent_controller_store import AgentControllerStorefrom switch_core.db.stores.agent_definition_store import AgentDefinitionStorefrom switch_core.db.stores.agent_store import AgentStorefrom switch_core.db.stores.api_key_store import ApiKeyStore+from switch_core.db.stores.switch_core_process_store import SwitchCoreProcessStorefrom switch_core.gateway.known_agents import KNOWN_AGENTS, KnownAgentfrom switch_core.management import reason_codes, tokensfrom switch_core.management.errors import ManagementError, not_found@@ -65,8 +66,10 @@ from switch_core.management.notifier import ControllerNotifierfrom switch_core.management.placement import (ControllerState,controller_state,+ is_revoked,require_placement,)+from switch_core.management.process_lease import ProcessLeases, read_leasesfrom switch_core.management.schemas import (PROVIDER_KNOWN_AGENT_TYPES,ControllerDescription,@@ -121,6 +124,7 @@ class ManagementService:operations: AgentControllerOperationStore,api_keys: ApiKeyStore,agents: AgentStore,+ processes: SwitchCoreProcessStore,presence: ControllerPresence,clock: Callable[[], datetime],) -> None:@@ -132,17 +136,20 @@ class ManagementService:self.operations = operationsself.api_keys = api_keysself.agents = agents+ self.processes = processesself._clock = clockdef now(self) -> datetime:return self._clock()- def state_of(self, controller: AgentController) -> ControllerState:- return controller_state(- controller,- now=self.now(),- interval_seconds=self.settings.status_interval_seconds,- )+ async def leases(self, session: AsyncSession) -> ProcessLeases:+ """Every switch-core process's lease, which a controller's state+ needs: read once per request, before the states it decides."""+ return await read_leases(session, self.processes)++ @staticmethod+ def state_of(controller: AgentController, leases: ProcessLeases) -> ControllerState:+ return controller_state(controller, leases)# ── Credentials and enrollment ────────────────────────────────────────────@@ -286,7 +293,7 @@ class ManagementService:controller = await self.controllers.get_by_api_key(session, tenant_id, key.id)if controller is None or controller.id != controller_id:raise invalid- if self.state_of(controller) == "revoked":+ if is_revoked(controller):raise ManagementError(401, reason_codes.CONTROLLER_REVOKED, "The controller has been revoked.")@@ -534,8 +541,9 @@ class ManagementService:controllers = await self.controllers.list_for_owner(session, tenant_id, owner_id)+ leases = await self.leases(session)return [- controller_view(controller, self.state_of(controller))+ controller_view(controller, self.state_of(controller, leases), leases)for controller in controllers]@@ -554,10 +562,33 @@ class ManagementService:that is the name the room is told when the machine is offline."""await self.owned_controller(session, tenant_id, owner_id, controller_id)+ return await self._update_details(session, tenant_id, controller_id, changes)++ async def update_own_controller(+ self,+ session: AsyncSession,+ principal: ControllerPrincipal,+ changes: dict[str, str | None],+ ) -> dict[str, Any]:+ """The controller renames itself or changes its description, with the+ same limits and effects as its owner's change."""+ await self._principal_controller(session, principal)+ return await self._update_details(+ session, principal.tenant_id, principal.controller_id, changes+ )++ async def _update_details(+ self,+ session: AsyncSession,+ tenant_id: str,+ controller_id: str,+ changes: dict[str, str | None],+ ) -> dict[str, Any]:controller = await self.controllers.update_details(session, tenant_id, controller_id, changes)- view = controller_view(controller, self.state_of(controller))+ leases = await self.leases(session)+ view = controller_view(controller, self.state_of(controller, leases), leases)await session.commit()if "name" in changes:self.presence.rename_controller(controller_id, controller.name)@@ -604,6 +635,7 @@ class ManagementService:row: AgentDefinitionRow,agent: Agent,controllers: dict[str, AgentController],+ leases: ProcessLeases,) -> dict[str, Any]:controller = Noneif row.controller_id is not None:@@ -618,15 +650,16 @@ class ManagementService:row,agent,controller,- self.state_of(controller) if controller is not None else None,+ self.state_of(controller, leases) if controller is not None else None,)async def list_managed_agents(self, session: AsyncSession, tenant_id: str, owner_id: str) -> list[dict[str, Any]]:controllers: dict[str, AgentController] = {}+ leases = await self.leases(session)return [- await self._view(session, tenant_id, row, agent, controllers)+ await self._view(session, tenant_id, row, agent, controllers, leases)for row, agent in await self.definitions.list_for_owner(session, tenant_id, owner_id)@@ -655,7 +688,9 @@ class ManagementService:row, agent = await self._owned_definition(session, tenant_id, owner_id, agent_id)- return await self._view(session, tenant_id, row, agent, {})+ return await self._view(+ session, tenant_id, row, agent, {}, await self.leases(session)+ )async def _check_target(self,@@ -676,6 +711,7 @@ class ManagementService:require_placement(controller,provider,+ leases=await self.leases(session),now=self.now(),interval_seconds=self.settings.status_interval_seconds,)@@ -777,7 +813,9 @@ class ManagementService:request.controller_id,)agent = await self._owned_agent(session, owner_id, result.agent_id)- return await self._view(session, tenant_id, row, agent, {})+ return await self._view(+ session, tenant_id, row, agent, {}, await self.leases(session)+ )async def put_managed_agent(self,@@ -874,7 +912,9 @@ class ManagementService:and existing.desired_state == target.desired_stateand existing.definition == target.definition):- return await self._view(session, tenant_id, existing, agent, {})+ return await self._view(+ session, tenant_id, existing, agent, {}, await self.leases(session)+ )previous_controller_id = existing.controller_id if existing else Noneif existing is None or existing.definition != target.definition:@@ -920,7 +960,9 @@ class ManagementService:await self._bind(session, tenant_id, row)self._nudge(revisions)agent = await self._owned_agent(session, owner_id, agent.id)- return await self._view(session, tenant_id, row, agent, {})+ return await self._view(+ session, tenant_id, row, agent, {}, await self.leases(session)+ )async def delete_managed_agent(self, session: AsyncSession, tenant_id: str, owner_id: str, agent_id: str@@ -991,7 +1033,7 @@ class ManagementService:controller = await self.owned_controller(session, tenant_id, owner_id, controller_id)- if self.state_of(controller) == "revoked":+ if is_revoked(controller):raise ManagementError(409, reason_codes.CONTROLLER_REVOKED, "The controller has been revoked.")diff --git a/core/switch_core/management/wiring.py b/core/switch_core/management/wiring.pyindex 06cd4fda3..f6d24f669 100644--- a/core/switch_core/management/wiring.py+++ b/core/switch_core/management/wiring.py@@ -8,10 +8,18 @@ bridge app is built, `install` adds the routes once both apps exist, andserves. `install` also hands Core the agent-facing side(`ManagementAgentOperations`), which is what makes the agent operations onmachines and managed agents exist.++Each process also holds a lease (`process_lease.py`) and writes its+controllers' connection transitions (`connection_ledger.py`): `start` claims+the lease before the bridge serves, `run` renews it and writes transitions+until cancelled, and `stop` writes what is left and marks the lease stopped."""from __future__ import annotations+import asyncio+import logging+import uuidfrom collections.abc import Callablefrom dataclasses import dataclassfrom datetime import UTC, datetime@@ -34,15 +42,20 @@ from switch_core.db.stores.agent_controller_store import AgentControllerStorefrom switch_core.db.stores.agent_definition_store import AgentDefinitionStorefrom switch_core.db.stores.agent_store import AgentStorefrom switch_core.db.stores.api_key_store import ApiKeyStore+from switch_core.db.stores.switch_core_process_store import SwitchCoreProcessStorefrom switch_core.management.agent_operations import ManagementAgentOperationsfrom switch_core.management.auth import ManagementAuthenticatorfrom switch_core.management.bindings import load_bindings+from switch_core.management.connection_ledger import ControllerConnectionLedgerfrom switch_core.management.controller_routes import router as controller_routerfrom switch_core.management.dependencies import init_management_dependenciesfrom switch_core.management.gateway_routes import router as gateway_routerfrom switch_core.management.notifier import ControllerNotifier+from switch_core.management.process_lease import ProcessLeasefrom switch_core.management.service import ManagementService, ManagementSettings+logger = logging.getLogger(__name__)+GATEWAY_PREFIX = "/management"@@ -56,6 +69,8 @@ class Management:authenticator: ManagementAuthenticatoragent_operations: ManagementAgentOperationssession_factory: async_sessionmaker[AsyncSession]+ lease: ProcessLease+ ledger: ControllerConnectionLedgerdef install(self,@@ -78,6 +93,26 @@ class Management:async with tenant_session(self.session_factory, tenant_id) as session:await self.service.forget_deleted_agent(session, tenant_id, agent_id)+ async def start(self) -> None:+ """Claim this process's lease, so the connections it records read as+ held. Raises when the database refuses: a process that cannot hold a+ lease would show every machine it connects as offline."""+ await self.lease.renew()+ logger.info("Claimed switch-core process lease %s", self.lease.process_id)++ async def run(self) -> None:+ """Renew the lease and write connection transitions, until cancelled."""+ await asyncio.gather(self.lease.run(), self.ledger.run())++ async def stop(self) -> None:+ """Write the transitions still queued, then mark the lease stopped.+ The lease is marked stopped even when the writes fail, so the+ machines this process held read offline either way."""+ try:+ await self.ledger.flush_all()+ finally:+ await self.lease.stop()+async def load_bindings(self) -> int:return await load_bindings(session_factory=self.session_factory,@@ -96,9 +131,18 @@ def build_management(presence: ControllerPresence,auth_cache: ControllerAuthCache,clock: Callable[[], datetime],+ process_id: str,) -> Management:controllers = AgentControllerStore()+ processes = SwitchCoreProcessStore()presence.use_auth_cache(auth_cache)+ ledger = ControllerConnectionLedger(+ process_id=process_id,+ session_factory=session_factory,+ controllers=controllers,+ clock=clock,+ )+ presence.use_ledger(ledger)service = ManagementService(settings=ManagementSettings(token_secret=token_secret,@@ -111,6 +155,7 @@ def build_management(operations=AgentControllerOperationStore(),api_keys=ApiKeyStore(),agents=AgentStore(),+ processes=processes,presence=presence,clock=clock,)@@ -128,6 +173,12 @@ def build_management(service=service, session_factory=session_factory),session_factory=session_factory,+ lease=ProcessLease(+ process_id=process_id,+ session_factory=session_factory,+ processes=processes,+ ),+ ledger=ledger,)@@ -157,4 +208,5 @@ def create_management(max_entries=config.agent_auth_cache_max_entries,),clock=utc_now,+ process_id=str(uuid.uuid4()),)diff --git a/core/switch_core/migrations/versions/fabf9b9bff78_controller_connection_transitions.py b/core/switch_core/migrations/versions/fabf9b9bff78_controller_connection_transitions.pynew file mode 100644index 000000000..6e969cc9d--- /dev/null+++ b/core/switch_core/migrations/versions/fabf9b9bff78_controller_connection_transitions.py@@ -0,0 +1,76 @@+"""agent controllers: record connection transitions and process leases++Four nullable columns on `agent_controllers` recording the controller's+connection as the switch-core process holding its socket last wrote it:+which connection, which process holds it, when its socket attached, and when+and why it went. Existing rows start with all null and fill in when their+controller next connects.++`switch_core_processes` holds each switch-core process's lease, renewed every+few seconds, so a connection whose process died without writing its closing+reads as offline. A global table: no tenant column and no row-level security+policy (`db/rls_ddl.py`'s `GLOBAL_TABLES`).++Revision ID: fabf9b9bff78+Revises: eb24eafa59a0+Create Date: 2026-10-07 12:00:00.000000++"""++from collections.abc import Sequence++import sqlalchemy as sa+from alembic import op++revision: str = "fabf9b9bff78"+down_revision: str | None = "eb24eafa59a0"+branch_labels: str | Sequence[str] | None = None+depends_on: str | Sequence[str] | None = None+++def upgrade() -> None:+ op.create_table(+ "switch_core_processes",+ sa.Column("id", sa.Text(), nullable=False),+ sa.Column(+ "started_at",+ sa.DateTime(timezone=True),+ server_default=sa.func.now(),+ nullable=False,+ ),+ sa.Column(+ "beat_at",+ sa.DateTime(timezone=True),+ server_default=sa.func.now(),+ nullable=False,+ ),+ sa.Column("stopped_at", sa.DateTime(timezone=True), nullable=True),+ sa.PrimaryKeyConstraint("id"),+ )+ op.add_column(+ "agent_controllers", sa.Column("connection_id", sa.Text(), nullable=True)+ )+ op.add_column(+ "agent_controllers",+ sa.Column("connection_process_id", sa.Text(), nullable=True),+ )+ op.add_column(+ "agent_controllers",+ sa.Column("connected_at", sa.DateTime(timezone=True), nullable=True),+ )+ op.add_column(+ "agent_controllers",+ sa.Column("disconnected_at", sa.DateTime(timezone=True), nullable=True),+ )+ op.add_column(+ "agent_controllers", sa.Column("disconnect_reason", sa.Text(), nullable=True)+ )+++def downgrade() -> None:+ op.drop_column("agent_controllers", "disconnect_reason")+ op.drop_column("agent_controllers", "disconnected_at")+ op.drop_column("agent_controllers", "connected_at")+ op.drop_column("agent_controllers", "connection_process_id")+ op.drop_column("agent_controllers", "connection_id")+ op.drop_table("switch_core_processes")diff --git a/core/tests/switch_core/db/test_tenant_exemption_allowlist.py b/core/tests/switch_core/db/test_tenant_exemption_allowlist.pyindex c60035744..baa13508f 100644--- a/core/tests/switch_core/db/test_tenant_exemption_allowlist.py+++ b/core/tests/switch_core/db/test_tenant_exemption_allowlist.py@@ -175,6 +175,10 @@ _RAW_SESSION_FACTORY_MODULES = {# Its own reads are the per-tenant fan-out above; this is the one global# read beside them, the deployment's user count."switch_core.telemetry.snapshot",+ # This process's lease in `switch_core_processes`, a table with no tenant+ # and no policy (`rls_ddl.GLOBAL_TABLES`): a process serves every tenant,+ # so its lease belongs to none. It touches only that table.+ "switch_core.management.process_lease",}# Calls that end in `session_factory` but hand one back rather than open adiff --git a/core/tests/switch_core/db/test_tenant_schema_catalogue.py b/core/tests/switch_core/db/test_tenant_schema_catalogue.pyindex d06f2a7dc..1485b14e8 100644--- a/core/tests/switch_core/db/test_tenant_schema_catalogue.py+++ b/core/tests/switch_core/db/test_tenant_schema_catalogue.py@@ -108,6 +108,10 @@ class TestEveryTableIsScopedUnlessItIsNamedGlobal:"deployment_identity","telemetry_milestones","telemetry_snapshot_watermark",+ # Each switch-core process's lease. A process holds controller+ # sockets for every tenant at once, and the row says only that it+ # is alive.+ "switch_core_processes",}async def test_scoped_is_everything_else(self) -> None:diff --git a/core/tests/switch_core/fixtures/agent_controllers/controller_info_request.json b/core/tests/switch_core/fixtures/agent_controllers/controller_info_request.jsonnew file mode 100644index 000000000..773fb5901--- /dev/null+++ b/core/tests/switch_core/fixtures/agent_controllers/controller_info_request.json@@ -0,0 +1,4 @@+{+ "name": "build-box",+ "description": "The build box in the office"+}diff --git a/core/tests/switch_core/fixtures/agent_controllers/controller_info_response.json b/core/tests/switch_core/fixtures/agent_controllers/controller_info_response.jsonnew file mode 100644index 000000000..01d5cd4fd--- /dev/null+++ b/core/tests/switch_core/fixtures/agent_controllers/controller_info_response.json@@ -0,0 +1,67 @@+{+ "id": "3f0f8a1e-5b7c-4c1e-9a52-2d6f1c9e0b11",+ "name": "build-box",+ "description": "The build box in the office",+ "kind": "console",+ "platform": {+ "os": "linux",+ "arch": "x64",+ "os_version": "6.1.0"+ },+ "version": "0.1.0",+ "state": "online",+ "last_seen_at": "2026-10-07T09:00:05Z",+ "connection": {+ "connected_at": "2026-10-07T09:00:00Z",+ "disconnected_at": null,+ "disconnect_reason": null+ },+ "status": {+ "seq": 42,+ "observed_at": "2026-01-01T00:00:00Z",+ "controller": {+ "version": "0.1.0",+ "protocol": 1,+ "assignment_revision": 3+ },+ "machine": {+ "platform": {+ "os": "linux",+ "arch": "x64",+ "os_version": "6.1.0"+ },+ "disk_free_bytes": 53687091200,+ "disk_total_bytes": 107374182400,+ "mem_free_bytes": 8589934592,+ "mem_total_bytes": 17179869184,+ "sessions_running": 1,+ "sessions_max": 4,+ "workspaces_dir": "/home/user/.local/state/switch/agent-controller/workspaces"+ },+ "providers": [+ {+ "provider": "claude",+ "installed": true,+ "version": "1.0.0",+ "auth": "ok",+ "auth_source": "local",+ "checked_at": "2026-01-01T00:00:00Z"+ }+ ],+ "tools": [+ {+ "tool": "git",+ "state": "ok"+ },+ {+ "tool": "gh",+ "state": "missing"+ }+ ],+ "agents": []+ },+ "assignment_revision": 0,+ "workspaces_dir": "/home/user/.local/state/switch/agent-controller/workspaces",+ "created_at": "2026-10-01T12:00:00Z",+ "revoked_at": null+}diff --git a/core/tests/switch_core/management/harness.py b/core/tests/switch_core/management/harness.pyindex f39bc50c7..38cb9f0e2 100644--- a/core/tests/switch_core/management/harness.py+++ b/core/tests/switch_core/management/harness.py@@ -199,6 +199,7 @@ def build_harness(presence=protocol.connections.controllers,auth_cache=controller_auth_cache,clock=clock,+ process_id=str(uuid.uuid4()),)async def _session() -> AsyncIterator[AsyncSession]:@@ -312,6 +313,8 @@ class EnrolledController:credential: straccess_token: strowner: User+ # To connect the controller as a running one is connected.+ harness: Harness@propertydef headers(self) -> dict[str, str]:@@ -344,6 +347,7 @@ async def enroll_console(credential=body["credential"],access_token=token.json()["access_token"],owner=owner,+ harness=harness,)@@ -355,6 +359,23 @@ async def report_status(providers: list[dict[str, Any]] | None = None,agents: list[dict[str, Any]] | None = None,) -> httpx.Response:+ """Report status as a running controller does: one whose socket is+ attached, connected first unless it already is."""+ await ensure_connected(client, controller)+ return await report_status_only(+ client, controller, seq, providers=providers, agents=agents+ )+++async def report_status_only(+ client: httpx.AsyncClient,+ controller: EnrolledController,+ seq: int,+ *,+ providers: list[dict[str, Any]] | None = None,+ agents: list[dict[str, Any]] | None = None,+) -> httpx.Response:+ """Report status without opening a connection."""return await client.put(f"/v1/management/controllers/{controller.controller_id}/status",json=status_report(seq, providers=providers, agents=agents),@@ -496,6 +517,38 @@ async def open_connection(return dict(response.json())+async def connect(+ client: httpx.AsyncClient, controller: EnrolledController+) -> dict[str, Any]:+ """Connect the controller as its socket would, and record it: the+ connection opened, its socket attached, this process's lease claimed and+ the transition written. Returns the open's response."""+ harness = controller.harness+ opened = await open_connection(client, controller)+ presence = harness.protocol.connections.controllers+ conn = presence.current_connection(controller.controller_id)+ assert conn is not None+ presence.attach_stream(conn)+ await harness.management.start()+ await harness.management.ledger.flush_all()+ return opened+++async def ensure_connected(+ client: httpx.AsyncClient, controller: EnrolledController+) -> None:+ """Connect the controller unless its socket is already attached, and+ record it either way."""+ harness = controller.harness+ presence = harness.protocol.connections.controllers+ conn = presence.current_connection(controller.controller_id)+ if conn is None or not conn.stream_attached:+ await connect(client, controller)+ return+ await harness.management.start()+ await harness.management.ledger.flush_all()++async def open_stream(harness: Harness,controller: EnrolledController,diff --git a/core/tests/switch_core/management/test_connection_state.py b/core/tests/switch_core/management/test_connection_state.pynew file mode 100644index 000000000..90e29dfdb--- /dev/null+++ b/core/tests/switch_core/management/test_connection_state.py@@ -0,0 +1,547 @@+"""A machine's online state, read from its connection's recorded transitions+and the lease of the process holding it.++The process holding a controller's socket writes each transition once, the+socket attaching and the socket going (and why), to the controller's row, and+never a heartbeat. Each process also renews a lease. A machine is online+while its socket is attached and its holding process's lease is fresh: so it+is the same from every process, and turns offline within seconds of the+controller stopping, its socket dropping, or the process holding it dying.+Runs against the real database.+"""++from __future__ import annotations++import time+import uuid+from datetime import UTC, datetime, timedelta+from typing import Any++import httpx+import pytest+from sqlalchemy import select, text+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker++from switch_core.bridges.agent.protocol.controller_presence import (+ ControllerConnection,+)+from switch_core.bridges.agent.protocol.liveness import HEARTBEAT_TTL_SECONDS+from switch_core.db.models import AgentController, SwitchCoreProcess+from switch_core.db.stores.agent_controller_store import AgentControllerStore+from switch_core.db.stores.switch_core_process_store import ProcessLeaseRow+from switch_core.management.connection_ledger import ControllerConnectionLedger+from switch_core.management.process_lease import LEASE_TTL, ProcessLeases+from tests.switch_core.management.harness import (+ EnrolledController,+ Harness,+ add_member,+ beat,+ build_harness,+ connect,+ cookies_for,+ create_managed_agent,+ enroll_console,+ open_connection,+ open_stream,+ place_agent,+ provider,+ report_status,+ report_status_only,+ take,+)+++def harness(session_factory: async_sessionmaker[AsyncSession]) -> Harness:+ return build_harness(session_factory)+++async def _row(harness: Harness, controller_id: str) -> AgentController:+ async with harness.session_factory() as session:+ result = await session.execute(+ select(AgentController).where(AgentController.id == controller_id)+ )+ return result.scalar_one()+++async def _listed(+ client: httpx.AsyncClient, controller: EnrolledController+) -> dict[str, Any]:+ response = await client.get(+ "/gateway/management/controllers", cookies=cookies_for(controller.owner)+ )+ assert response.status_code == 200, response.text+ [entry] = [c for c in response.json() if c["id"] == controller.controller_id]+ return dict(entry)+++async def _flush(harness: Harness) -> None:+ await harness.management.ledger.flush_all()+++async def _age_lease(harness: Harness, seconds: float) -> None:+ """Set this harness's process's last renewal `seconds` in the past, as a+ process that stopped renewing would leave it."""+ async with harness.session_factory() as session:+ await session.execute(+ text(+ "UPDATE switch_core_processes "+ "SET beat_at = now() - make_interval(secs => :seconds) "+ "WHERE id = :id"+ ),+ {"seconds": seconds, "id": harness.management.lease.process_id},+ )+ await session.commit()+++def _current(harness: Harness, controller: EnrolledController) -> ControllerConnection:+ conn = harness.protocol.connections.controllers.current_connection(+ controller.controller_id+ )+ assert conn is not None+ return conn+++class _CountingStore(AgentControllerStore):+ """Counts the transitions written, and fails the first `failures`."""++ def __init__(self, failures: int = 0) -> None:+ self.failures = failures+ self.connected = 0+ self.disconnected = 0++ def _maybe_fail(self) -> None:+ if self.failures:+ self.failures -= 1+ raise ConnectionError("the database went away")++ async def record_connected(self, *args: Any, **kwargs: Any) -> bool:+ self._maybe_fail()+ self.connected += 1+ return await super().record_connected(*args, **kwargs)++ async def record_disconnected(self, *args: Any, **kwargs: Any) -> bool:+ self._maybe_fail()+ self.disconnected += 1+ return await super().record_disconnected(*args, **kwargs)+++class TestLeases:+ """`ProcessLeases` alone: whether a process holds its sockets."""++ now = datetime(2026, 1, 1, tzinfo=UTC)++ def _leases(self, **rows: ProcessLeaseRow) -> ProcessLeases:+ return ProcessLeases(read_at=self.now, leases=rows)++ def test_a_fresh_lease_holds(self) -> None:+ leases = self._leases(p=ProcessLeaseRow(beat_at=self.now, stopped_at=None))+ assert leases.holds("p")++ def test_a_lease_older_than_the_ttl_does_not(self) -> None:+ beat_at = self.now - LEASE_TTL+ leases = self._leases(p=ProcessLeaseRow(beat_at=beat_at, stopped_at=None))+ assert not leases.holds("p")+ assert leases.ended("p") == (beat_at, "server_lost")++ def test_a_stopped_lease_does_not(self) -> None:+ stopped_at = self.now - timedelta(seconds=1)+ leases = self._leases(+ p=ProcessLeaseRow(beat_at=self.now, stopped_at=stopped_at)+ )+ assert not leases.holds("p")+ assert leases.ended("p") == (stopped_at, "server_shutdown")++ def test_a_missing_lease_does_not(self) -> None:+ leases = self._leases()+ assert not leases.holds("p")+ assert not leases.holds(None)+ assert leases.ended("p") == (None, "server_lost")+++class TestTheMachineState:+ async def test_a_machine_that_never_connected_is_unknown(+ self, harness: Harness+ ) -> None:+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ await report_status_only(client, controller, 1)+ listed = await _listed(client, controller)+ assert listed["state"] == "unknown"+ assert listed["connection"] is None+ assert listed["last_seen_at"] is not None++ async def test_an_open_connection_with_no_socket_is_not_online(+ self, harness: Harness+ ) -> None:+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ await harness.management.start()+ await open_connection(client, controller)+ await _flush(harness)+ listed = await _listed(client, controller)+ assert listed["state"] == "unknown"++ async def test_the_socket_attaching_is_online_and_going_is_offline_at_once(+ self, harness: Harness+ ) -> None:+ owner = await add_member(harness.session_factory, "ada")+ presence = harness.protocol.connections.controllers+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ agent_id = await place_agent(client, controller, name="reviewer")+ conn = _current(harness, controller)+ # The socket `place_agent` connected goes; a new one attaches.+ presence.detach_stream(conn, conn.stream_token)+ opened = {"connection_id": conn.id, "generation": conn.generation}+ stream = await open_stream(harness, controller, opened)+ await take(stream, 2)+ await _flush(harness)+ live = presence.is_live(agent_id)+ online = await _listed(client, controller)+ await stream.aclose() # type: ignore[attr-defined]+ await _flush(harness)+ offline = await _listed(client, controller)+ row = await _row(harness, controller.controller_id)++ assert live+ assert online["state"] == "online"+ assert online["connection"]["disconnected_at"] is None+ assert online["connection"]["disconnect_reason"] is None+ assert not presence.is_live(agent_id)+ assert offline["state"] == "offline"+ assert offline["connection"]["disconnect_reason"] == "socket_closed"+ assert offline["connection"]["disconnected_at"] is not None+ assert row.connection_id == conn.id+ assert row.connection_process_id == harness.management.lease.process_id++ async def test_a_socket_reattaching_to_its_connection_is_online_again(+ self, harness: Harness+ ) -> None:+ owner = await add_member(harness.session_factory, "ada")+ presence = harness.protocol.connections.controllers+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ await connect(client, controller)+ conn = _current(harness, controller)+ presence.detach_stream(conn, conn.stream_token)+ await _flush(harness)+ dropped = await _listed(client, controller)+ presence.attach_stream(conn)+ await _flush(harness)+ back = await _listed(client, controller)+ assert dropped["state"] == "offline"+ assert back["state"] == "online"+ assert back["connection"]["disconnected_at"] is None++ async def test_each_transition_is_written_once_and_no_beat_is_written(+ self, harness: Harness+ ) -> None:+ store = _CountingStore()+ presence = harness.protocol.connections.controllers+ ledger = ControllerConnectionLedger(+ process_id=harness.management.lease.process_id,+ session_factory=harness.session_factory,+ controllers=store,+ clock=harness.clock,+ )+ presence.use_ledger(ledger)+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ opened = await open_connection(client, controller)+ conn = _current(harness, controller)+ presence.attach_stream(conn)+ await ledger.flush_all()+ for _ in range(20):+ answer = beat(+ harness,+ controller,+ connection_id=opened["connection_id"],+ generation=opened["generation"],+ cursors={},+ )+ assert answer.status_code == 200, answer.text+ queued_by_beats = ledger.pending+ await ledger.flush_all()+ after_beats = (store.connected, store.disconnected)+ presence.detach_stream(conn, conn.stream_token)+ await ledger.flush_all()+ conn.last_beat = time.monotonic() - HEARTBEAT_TTL_SECONDS - 1+ swept = presence.sweep()+ await ledger.flush_all()+ assert queued_by_beats == 0+ assert after_beats == (1, 0)+ assert swept == [conn]+ assert (store.connected, store.disconnected) == (1, 1)+ row = await _row(harness, controller.controller_id)+ assert row.disconnect_reason == "socket_closed"++ async def test_a_lapsed_heartbeat_is_recorded_by_the_sweep(+ self, harness: Harness+ ) -> None:+ owner = await add_member(harness.session_factory, "ada")+ presence = harness.protocol.connections.controllers+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ await connect(client, controller)+ online = await _listed(client, controller)+ conn = _current(harness, controller)+ conn.last_beat = time.monotonic() - HEARTBEAT_TTL_SECONDS - 1+ presence.sweep()+ await _flush(harness)+ offline = await _listed(client, controller)+ assert online["state"] == "online"+ assert offline["state"] == "offline"+ assert offline["connection"]["disconnect_reason"] == "heartbeat_lapsed"++ async def test_a_socket_the_server_closes_as_it_shuts_down_says_so(+ self, harness: Harness+ ) -> None:+ owner = await add_member(harness.session_factory, "ada")+ presence = harness.protocol.connections.controllers+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ await connect(client, controller)+ presence.begin_shutdown()+ conn = _current(harness, controller)+ presence.detach_stream(conn, conn.stream_token)+ await _flush(harness)+ listed = await _listed(client, controller)+ assert listed["state"] == "offline"+ assert listed["connection"]["disconnect_reason"] == "server_shutdown"++ async def test_a_takeover_records_the_old_connection_taken_over(+ self, harness: Harness+ ) -> None:+ owner = await add_member(harness.session_factory, "ada")+ presence = harness.protocol.connections.controllers+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ await connect(client, controller)+ first = _current(harness, controller)+ await open_connection(client, controller)+ await _flush(harness)+ between = await _listed(client, controller)+ second = _current(harness, controller)+ presence.attach_stream(second)+ await _flush(harness)+ after = await _listed(client, controller)+ row = await _row(harness, controller.controller_id)+ assert between["state"] == "offline"+ assert between["connection"]["disconnect_reason"] == "taken_over"+ assert after["state"] == "online"+ assert row.connection_id == second.id != first.id++ async def test_a_revoked_machine_is_revoked_and_its_connection_closed(+ self, harness: Harness+ ) -> None:+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ await connect(client, controller)+ revoked = await client.delete(+ f"/gateway/management/controllers/{controller.controller_id}",+ cookies=cookies_for(owner),+ )+ await _flush(harness)+ listed = await _listed(client, controller)+ assert revoked.status_code == 200, revoked.text+ assert listed["state"] == "revoked"+ assert listed["connection"]["disconnect_reason"] == "revoked"++ async def test_placement_needs_the_machine_connected(+ self, harness: Harness+ ) -> None:+ owner = await add_member(harness.session_factory, "ada")+ presence = harness.protocol.connections.controllers+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ await report_status(client, controller, 1, providers=[provider("claude")])+ conn = _current(harness, controller)+ presence.detach_stream(conn, conn.stream_token)+ await _flush(harness)+ refused = await create_managed_agent(+ client, owner, name="reviewer", controller_id=controller.controller_id+ )+ assert refused.status_code == 409, refused.text+ assert refused.json()["error"] == {+ "code": "controller_offline",+ "message": "Cannot place the agent: the controller is not connected "+ "to Switch.",+ "retryable": False,+ }+++class TestTheProcessLease:+ async def test_a_lease_renewed_within_the_ttl_keeps_the_machine_online(+ self, harness: Harness+ ) -> None:+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ await connect(client, controller)+ await _age_lease(harness, LEASE_TTL.total_seconds() - 3)+ still_online = await _listed(client, controller)+ await _age_lease(harness, LEASE_TTL.total_seconds() + 1)+ lapsed = await _listed(client, controller)+ await harness.management.lease.renew()+ renewed = await _listed(client, controller)+ assert still_online["state"] == "online"+ assert lapsed["state"] == "offline"+ assert lapsed["connection"]["disconnect_reason"] == "server_lost"+ assert lapsed["connection"]["disconnected_at"] is not None+ assert renewed["state"] == "online"++ async def test_a_dead_processs_machines_read_offline_from_another(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ """The process holding the socket died: it wrote no closing, its lease+ stops being renewed, and every other process reads the machine+ offline, with nothing written per machine."""+ holder = build_harness(session_factory)+ other = build_harness(session_factory)+ owner = await add_member(session_factory, "ada")+ async with holder.client() as client, other.client() as elsewhere:+ controller = await enroll_console(holder, client, owner)+ await connect(client, controller)+ await other.management.start()+ online = await _listed(elsewhere, controller)+ await _age_lease(holder, LEASE_TTL.total_seconds() + 1)+ await other.management.lease.renew()+ offline = await _listed(elsewhere, controller)+ async with session_factory() as session:+ await session.execute(+ SwitchCoreProcess.__table__.delete().where(+ SwitchCoreProcess.id == holder.management.lease.process_id+ )+ )+ await session.commit()+ pruned = await _listed(elsewhere, controller)+ row = await _row(holder, controller.controller_id)+ assert (+ other.protocol.connections.controllers.current_connection(+ controller.controller_id+ )+ is None+ )+ assert online["state"] == "online"+ assert offline["state"] == "offline"+ assert offline["connection"]["disconnect_reason"] == "server_lost"+ assert pruned["state"] == "offline"+ assert pruned["connection"] == {+ "connected_at": online["connection"]["connected_at"],+ "disconnected_at": None,+ "disconnect_reason": "server_lost",+ }+ assert row.disconnected_at is None++ async def test_a_process_that_stops_takes_its_machines_offline(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ holder = build_harness(session_factory)+ other = build_harness(session_factory)+ owner = await add_member(session_factory, "ada")+ async with holder.client() as client, other.client() as elsewhere:+ controller = await enroll_console(holder, client, owner)+ await connect(client, controller)+ await holder.management.stop()+ listed = await _listed(elsewhere, controller)+ assert listed["state"] == "offline"+ assert listed["connection"]["disconnect_reason"] == "server_shutdown"+ assert listed["connection"]["disconnected_at"] is not None++ async def test_long_dead_leases_are_pruned(self, harness: Harness) -> None:+ dead = str(uuid.uuid4())+ async with harness.session_factory() as session:+ session.add(+ SwitchCoreProcess(+ id=dead,+ started_at=datetime.now(UTC) - timedelta(hours=3),+ beat_at=datetime.now(UTC) - timedelta(hours=2),+ )+ )+ await session.commit()+ await harness.management.lease.renew()+ async with harness.session_factory() as session:+ ids = set(+ (await session.execute(select(SwitchCoreProcess.id))).scalars().all()+ )+ assert dead not in ids+ assert harness.management.lease.process_id in ids+++class TestTheLedger:+ async def test_a_late_closing_of_a_replaced_connection_leaves_the_new_one(+ self, harness: Harness+ ) -> None:+ """A process that held an earlier connection and only now records its+ socket going does not overwrite the connection recorded since."""+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ await connect(client, controller)+ earlier = _current(harness, controller)+ await connect(client, controller)+ replacement = _current(harness, controller)+ elsewhere = ControllerConnectionLedger(+ process_id=str(uuid.uuid4()),+ session_factory=harness.session_factory,+ controllers=AgentControllerStore(),+ clock=harness.clock,+ )+ elsewhere.disconnected(earlier, "heartbeat_lapsed")+ await elsewhere.flush_all()+ listed = await _listed(client, controller)+ row = await _row(harness, controller.controller_id)+ assert row.connection_id == replacement.id+ assert listed["state"] == "online"+ assert listed["connection"]["disconnect_reason"] is None++ async def test_only_the_latest_transition_of_a_controller_is_written(+ self, harness: Harness+ ) -> None:+ store = _CountingStore()+ ledger = ControllerConnectionLedger(+ process_id=harness.management.lease.process_id,+ session_factory=harness.session_factory,+ controllers=store,+ clock=harness.clock,+ )+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ await connect(client, controller)+ conn = _current(harness, controller)+ ledger.disconnected(conn, "socket_closed")+ ledger.connected(conn)+ await ledger.flush_all()+ row = await _row(harness, controller.controller_id)+ assert (store.connected, store.disconnected) == (1, 0)+ assert row.disconnected_at is None++ async def test_a_failed_write_is_kept_for_the_next_flush(+ self, harness: Harness+ ) -> None:+ store = _CountingStore(failures=1)+ ledger = ControllerConnectionLedger(+ process_id=harness.management.lease.process_id,+ session_factory=harness.session_factory,+ controllers=store,+ clock=harness.clock,+ )+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ await connect(client, controller)+ conn = _current(harness, controller)+ ledger.disconnected(conn, "heartbeat_lapsed")+ with pytest.raises(ConnectionError):+ await ledger.flush_all()+ assert (await _row(harness, controller.controller_id)).disconnected_at is None+ await ledger.flush_all()+ row = await _row(harness, controller.controller_id)+ assert row.disconnect_reason == "heartbeat_lapsed"+ assert store.failures == 0diff --git a/core/tests/switch_core/management/test_contract_fixtures.py b/core/tests/switch_core/management/test_contract_fixtures.pyindex fc78a651f..58ba3e0fa 100644--- a/core/tests/switch_core/management/test_contract_fixtures.py+++ b/core/tests/switch_core/management/test_contract_fixtures.py@@ -28,6 +28,7 @@ from switch_core.db.stores.agent_store import AgentStorefrom switch_core.management import controller_routesfrom switch_core.management.schemas import (ControllerConnectionRequest,+ ControllerInfoRequest,EnrollRequest,OperationResultRequest,StatusReport,@@ -97,6 +98,7 @@ class TestRequestFixturesParse:ControllerConnectionRequest.model_validate(_fixture("controller_connection_request.json"))+ ControllerInfoRequest.model_validate(_fixture("controller_info_request.json"))class TestEnrollmentAndTokens:@@ -243,6 +245,23 @@ class TestControllerMessages:assert refused.json() == _fixture("error_response.json")+class TestTheControllersOwnInfo:+ async def test_set_info(self, harness: Harness) -> None:+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ await report_status(+ client, controller, 1, providers=[provider("claude")], agents=[]+ )+ updated = await client.patch(+ f"/v1/management/controllers/{controller.controller_id}",+ json=_fixture("controller_info_request.json"),+ headers=controller.headers,+ )+ assert updated.status_code == 200, updated.text+ assert_same_shape(updated.json(), _fixture("controller_info_response.json"))++class TestTheControllerConnection:async def test_open_and_beat(self, harness: Harness) -> None:owner = await add_member(harness.session_factory, "ada")diff --git a/core/tests/switch_core/management/test_controller_routes.py b/core/tests/switch_core/management/test_controller_routes.pyindex 4248b302f..65d9a33f8 100644--- a/core/tests/switch_core/management/test_controller_routes.py+++ b/core/tests/switch_core/management/test_controller_routes.py@@ -361,7 +361,7 @@ class TestAssignment:class TestStatus:- async def test_a_report_makes_the_controller_online_until_it_goes_stale(+ async def test_a_connected_controller_is_online_however_old_its_report(self, harness: Harness) -> None:owner = await add_member(harness.session_factory, "ada")@@ -379,7 +379,7 @@ class TestStatus:assert response.json() == {"assignment_revision": 0, "report_within_s": 60}assert online.json()[0]["state"] == "online"assert online.json()[0]["status"]["seq"] == 5- assert stale.json()[0]["state"] == "unknown"+ assert stale.json()[0]["state"] == "online"async def test_an_older_seq_is_ignored_but_answered(self, harness: Harness) -> None:owner = await add_member(harness.session_factory, "ada")@@ -790,6 +790,75 @@ class TestPlacementBindsTheAgentInCore:assert restarted.is_revoked(controller.controller_id)+class TestTheControllersOwnInfo:+ async def test_the_controller_renames_and_describes_itself(+ self, harness: Harness+ ) -> None:+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner, name="laptop")+ await report_status(client, controller, 1, providers=[provider("claude")])+ created = await create_managed_agent(+ client, owner, name="reviewer", controller_id=controller.controller_id+ )+ path = f"/v1/management/controllers/{controller.controller_id}"+ renamed = await client.patch(+ path,+ json={"name": " build box ", "description": " Under the desk "},+ headers=controller.headers,+ )+ cleared = await client.patch(+ path, json={"description": " "}, headers=controller.headers+ )+ listed = await client.get(+ "/gateway/management/controllers", cookies=cookies_for(owner)+ )+ assert renamed.status_code == 200, renamed.text+ assert (renamed.json()["name"], renamed.json()["description"]) == (+ "build box",+ "Under the desk",+ )+ assert renamed.json()["state"] == "online"+ assert cleared.json()["name"] == "build box"+ assert cleared.json()["description"] is None+ [stored] = listed.json()+ assert (stored["name"], stored["description"]) == ("build box", None)+ binding = harness.protocol.connections.controllers.binding(+ created.json()["agent_id"]+ )+ assert binding is not None and binding.controller_name == "build box"++ async def test_the_owners_limits_hold_and_only_its_own_machine(+ self, harness: Harness+ ) -> None:+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ mine = await enroll_console(harness, client, owner, name="mine")+ theirs = await enroll_console(harness, client, owner, name="theirs")+ path = f"/v1/management/controllers/{mine.controller_id}"+ refused = [+ await client.patch(path, json=body, headers=mine.headers)+ for body in (+ {},+ {"name": " "},+ {"name": None},+ {"name": "x" * 201},+ {"description": "x" * 501},+ )+ ]+ other = await client.patch(+ f"/v1/management/controllers/{theirs.controller_id}",+ json={"name": "taken"},+ headers=mine.headers,+ )+ anonymous = await client.patch(path, json={"name": "nobody"})+ assert [(r.status_code, r.json()["error"]["code"]) for r in refused] == [+ (422, "validation_error")+ ] * 5+ assert (other.status_code, other.json()["error"]["code"]) == (403, "forbidden")+ assert anonymous.status_code == 401++def test_bearer_helper() -> None:assert bearer("x") == {"Authorization": "Bearer x"}diff --git a/core/tests/switch_core/management/test_gateway_routes.py b/core/tests/switch_core/management/test_gateway_routes.pyindex 3dcf56319..69c3650b0 100644--- a/core/tests/switch_core/management/test_gateway_routes.py+++ b/core/tests/switch_core/management/test_gateway_routes.py@@ -27,6 +27,7 @@ from tests.switch_core.management.harness import (create_managed_agent,definition,enroll_console,+ ensure_connected,platform,provider,report_status,@@ -579,6 +580,7 @@ async def report_machine(del report["machine"]["workspaces_dir"]else:report["machine"]["workspaces_dir"] = workspaces_dir+ await ensure_connected(client, controller)response = await client.put(f"/v1/management/controllers/{controller.controller_id}/status",json=report,diff --git a/docs/design/agent-controllers-v1.md b/docs/design/agent-controllers-v1.mdindex 88ba97923..3ca8e85d8 100644--- a/docs/design/agent-controllers-v1.md+++ b/docs/design/agent-controllers-v1.md@@ -38,7 +38,8 @@ where it deliberately stops short.when the flag is on**: a model validator raises if it is missing or shorter than 32 chars. It isseparate from `jwt_secret_key` on purpose.- `controller_status_interval_seconds: int = 60`. Returned to controllers as `report_within_s`.- A controller is `unknown` after 3 intervals without a status.+ Placement needs a status within 3 intervals. Whether a machine is online is read from its+ recorded connection instead (see "Liveness" below).With the flag off, none of the routes below are mounted, and the middleware branch isinactive.@@ -50,7 +51,13 @@ inactive.`version` null, `public_key` null.- `api_key_id`: the credential, an `api_keys` row of type `controller`, holding the hash only.- `assignment_revision` int default 0, `status_seq` bigint null, `status` JSONB null.- - `last_seen_at` null, `revoked_at` null, `created_at`, `updated_at`.+ - `last_seen_at` null (the last status), `revoked_at` null, `created_at`, `updated_at`.+ - The connection, as the switch-core process holding its socket last recorded it (all null+ until the first): `connection_id`, `connection_process_id`, `connected_at`,+ `disconnected_at`, `disconnect_reason` (`socket_closed`, `server_shutdown`,+ `heartbeat_lapsed`, `taken_over`, `revoked`).+- `switch_core_processes` (global, no tenant): each switch-core process's lease, `id`,+ `started_at`, `beat_at`, `stopped_at` null.- `agent_controller_enrollment_codes`: `id`, `owner_id`, `api_key_id` (an `api_keys` row oftype `controller_enrollment`, so the existing global hash → tenant lookup works),`expires_at`, `used_at` null, `controller_id` null, `created_at`.@@ -88,6 +95,11 @@ Public (they authenticate through the body):Controller access token (`{id}` must match the token's `cid`, otherwise `403 forbidden`):- `POST /v1/management/controllers/{id}/credential/rotate` returns `{credential}`.+- `PATCH /v1/management/controllers/{id}`+ - Body: `{name?, description?}`, at least one: the controller renames its own machine+ (`switch-agent-controller set-info`). The same validation and effects as the owner's+ `PATCH /gateway/management/controllers/{id}`, including the rename reaching Core's+ bindings. Returns the controller as the owner's list shows it.- `GET /v1/management/controllers/{id}/assignment`- Honours `If-None-Match`; returns `200` with an `ETag` header, or `304`.- `PUT /v1/management/controllers/{id}/status`@@ -124,8 +136,12 @@ Controller access token (`{id}` must match the token's `cid`, otherwise `403 for- Body: `{name, description?, kind:"console", platform, version, public_key?}`.- Returns `{controller_id, credential}`.- `GET /gateway/management/controllers`- - Returns the list, each with its `description`, derived `state` (`online|unknown|revoked`),- `last_seen_at`, its last `status`, and `workspaces_dir`: the directory the controller makes+ - Returns the list, each with its `description`, derived `state`+ (`online|offline|unknown|revoked`, see "Liveness" below), `last_seen_at` (its last status),+ `connection` (`{connected_at, disconnected_at, disconnect_reason}` or null when it never+ connected; `disconnect_reason` is also `server_shutdown` or `server_lost` when the row+ shows it open but the process holding it stopped or died), its last `status`, and+ `workspaces_dir`: the directory the controller makesagents' workspaces in, from that status (`machine.workspaces_dir`), null when it has notreported one.- `PATCH /gateway/management/controllers/{id}`@@ -157,7 +173,8 @@ Controller access token (`{id}` must match the token's `cid`, otherwise `403 for**Placement checks** run on create, adopt and move, and on a change to `running`. Each failure returns `409` with a reason:- `controller_revoked`-- `controller_offline`: no status, or the last status is stale+- `controller_offline`: not online (never connected, its socket went, or the process+ holding it stopped), or no status, or the last status is stale- `provider_not_installed`- `provider_login_missing` or `provider_login_expired`@@ -283,6 +300,9 @@ exactly the answer a missing one does.- CLI `switch-agent-controller`:- `enroll --server <agent-bridge-url> --code <code> [--name] [--description] [--data-dir]`- `run [--data-dir]`+ - `set-info [--name] [--description] [--data-dir]`: renames the machine and/or changes its+ description on the server, with the controller's own credential, and records the new name+ locally.- `status [--data-dir]`- **Data dir:** `SWITCH_CONTROLLER_DATA_DIR`, otherwise the OS default.- macOS: `~/Library/Application Support/Switch/agent-controller`@@ -435,8 +455,25 @@ stream. The flag and everything else above stay as they are.offered the terminal command or told to open Switch Console.In-room session commands (`!reset`, `!compact`) are still relayed on thecontroller's stream with the room, and the controller picks the session.-- **Liveness** is "stream attached and beat within 6 s"; the connection sweep- closes lapsed controller connections.+- **Liveness** is "socket attached and a pong within 6 s"; the connection sweep+ closes lapsed controller connections. Heartbeats stay in the memory of the+ process holding the socket.+- **Recorded transitions.** `ControllerPresence` stays the in-memory fast path for+ routing and agent presence, and tells a ledger (`ControllerConnectionLedger`, Core's+ port, implemented by `management/connection_ledger.py`) of each transition, once:+ the socket attaching to a connection that had none, and the socket going, with why+ (`socket_closed`, `server_shutdown`, `heartbeat_lapsed`, `taken_over`, `revoked`).+ The ledger writes them to the controller's row from one background task, never+ per beat. A connecting replaces the row; a closing is written only while the row+ still names that connection, so a process closing a connection the controller has+ since replaced through another process leaves the replacement standing.+- **Process lease.** Each switch-core process renews a lease in+ `switch_core_processes` every 5 s (`management/process_lease.py`), one write per+ process however many machines it holds, and marks it stopped as it shuts down.+ A machine is online while its socket is recorded attached and its holding+ process's lease was renewed within 15 s and is not stopped; so a process that+ dies without writing its closings takes its machines offline within 15 s.+ Leases are written and compared on the database's clock.- **Holder id.** A controller-backed agent holds things under`controller:{controller_id}:{agent_id}`: the operation caller's session keyand session id, the reader of its unread counts, and the holder of a rolediff --git a/docs/design/controller-contract-v1.md b/docs/design/controller-contract-v1.mdindex 45727b353..af33bd6d1 100644--- a/docs/design/controller-contract-v1.md+++ b/docs/design/controller-contract-v1.md@@ -43,9 +43,18 @@ POST /v1/management/controllers/{id}/token auth: none (credential in b→ 200 { access_token: string, expires_at: Time } // about 1h. Claims: controller_id, owner_id, tenant_idPOST /v1/management/controllers/{id}/credential/rotate → 200 { credential: string }+PATCH /v1/management/controllers/{id} auth: the controller's access token+ { name?: string, description?: string | null } // either or both; null or blank clears the description+ → 200 Controller // the machine as its owner's list shows it+ → 422 validation_error // name blank or > 200 chars, description > 500, nothing givenDELETE /v1/management/controllers/{id} auth: owner (user). Revokes it, and every agent on it stops```+The `PATCH` is the controller renaming its own machine+(`switch-agent-controller set-info`), with the limits and effects of the+owner's own change in the gateway. A server that predates it answers `404` or+`405` without an error envelope.+```tstype Platform = { os: "macos" | "linux" | "windows"; arch: "arm64" | "x64"; os_version: string }```@@ -238,6 +247,16 @@ WS /v1/controllers/{id}/connection/ws?connection_id=…&generation=…close 1012 // the server is restarting: reconnect within a second```+**Connection transitions are recorded.** Whichever server process holds the+socket records, once each, when it attached and when and why it went+(`socket_closed`, `server_shutdown`, `heartbeat_lapsed`, `taken_over`,+`revoked`) on the controller's row. Pongs are not recorded. A controller+shutting down needs to say nothing: its socket closing is recorded at once,+and so is a dropped one, which reattaching records as connected again. Every+process reads the machine's state from that record and the holding process's+lease (§8, Machine state), so it does not depend on which process holds the+socket.+**Attachment is automatic.** Core attaches the agents currently bound to this controller (§7), and attaches or detaches them as bindings change. There's no per-agent subscribe call.**Core knows only whether an agent is connected.** A controller-backed agent is connected while its controller's socket is attached and answering pings, its owner has it `running`, and the controller is not revoked. Core does not know where the agent's sessions are, or whether one is running: the open and the pong carry no session map, and a body that still sends `placements` is refused with `422 validation_error` (the one exception to ignoring unknown fields, so a controller still reporting sessions is noticed rather than silently dropped). When a connected agent is addressed, Core delivers the message and posts nothing; if a session has to start, the agent's host posts "Starting a session…" itself. When it is not connected, Core tells the room why: stopped by its owner, its machine removed, or its machine offline.@@ -289,13 +308,21 @@ After acting, send status.- To move it, Management first unbinds it from A at revision r+1, and waits for A to report it stopped, or for A to go stale.- Then it binds the agent to B at revision r+2. B refuses to start a revision older than one it has already applied.-**Staleness.** No status for 3 × `report_within_s` makes a controller `unknown`, and so are its agents. Management then:-- shows them as unknown,+**Machine state.** Read from the recorded connection (§6), not from the status report:+- `online`: its socket is attached, and the server process holding it is alive (its lease renewed within 15 s, and not stopped).+- `offline`: it has connected before, and its socket went (closed, a lapsed heartbeat, a takeover not replaced, the server shutting down), or the server process holding it stopped or died. A controller that stops shows offline at once when its socket closes, within about 8 s when its heartbeat lapses with the socket still open, and within 15 s when the process holding its socket died.+- `unknown`: no connection has ever been recorded for it.+- `revoked`: its credential is gone.++While it is not online Management:+- shows it, and its agents, as offline or unknown,- refuses new placements on that controller,- may reassign its agents only if their definition allows it. Cloud agents stay put.-**Placement checks.** Before binding, Management checks the target's last status. It refuses with a reason if:-- the controller is `unknown`,+The status report carries the details (providers, disk, sessions, agents) and is what placement reads them from.++**Placement checks.** Before binding, Management checks the target. It refuses with a reason if:+- the controller is not `online`, or its last status is older than 3 × `report_within_s`,- the provider is not installed, or its login is not `ok`,- there's no session capacity, or the disk is nearly full.@@ -334,7 +361,7 @@ The personal agent relays that reason to the user as is.| `crash_loop` | status | 5 restarts in 10 minutes, stopped retrying || `out_of_memory` | status | Killed by the memory limit || `disk_full` / `capacity_exceeded` | status, placement | No disk / no session slots |-| `controller_offline` | placement | Target is `unknown` |+| `controller_offline` | placement | Target is not `online`, or its last status is stale || `internal` | any | Server or controller bug, with `retryable` set honestly |---diff --git a/gateway/src/data/management.test.ts b/gateway/src/data/management.test.tsindex 24a3d96d4..7964dc128 100644--- a/gateway/src/data/management.test.ts+++ b/gateway/src/data/management.test.ts@@ -5,6 +5,7 @@ import {enrollCommand,fetchManagementAvailable,managementError,+ shellQuote,} from "./management";function respond(status: number, body: unknown) {@@ -106,8 +107,29 @@ describe("createManagedAgent", () => {describe("enrollCommand", () => {it("names the server and the code", () => {- expect(enrollCommand("https://switch.example.com", "swce_example")).toBe(- "switch-agent-controller enroll --server https://switch.example.com --code swce_example",+ expect(+ enrollCommand("https://switch.example.com", "swce_example", { name: "", description: " " }),+ ).toBe("switch-agent-controller enroll --server https://switch.example.com --code swce_example");+ });++ it("adds the name and description, quoted for the shell", () => {+ expect(+ enrollCommand("https://switch.example.com", "swce_example", {+ name: " build-box ",+ description: "Bob's box: $HOME `x` \"y\"",+ }),+ ).toBe(+ "switch-agent-controller enroll --server https://switch.example.com --code swce_example" ++ " --name build-box --description 'Bob'\\''s box: $HOME `x` \"y\"'",);});});++describe("shellQuote", () => {+ it("leaves plain words alone and single-quotes the rest", () => {+ expect(shellQuote("swce_a-b.c")).toBe("swce_a-b.c");+ expect(shellQuote("two words")).toBe("'two words'");+ expect(shellQuote("it's")).toBe("'it'\\''s'");+ expect(shellQuote("")).toBe("''");+ });+});diff --git a/gateway/src/data/management.ts b/gateway/src/data/management.tsindex d444cb72f..f162ff7c4 100644--- a/gateway/src/data/management.ts+++ b/gateway/src/data/management.ts@@ -16,7 +16,26 @@ export const PROVIDERS: { value: Provider; label: string }[] = [export type DesiredState = "running" | "stopped";export type Isolation = "shared" | "isolated";-export type ControllerState = "online" | "unknown" | "revoked";+/**+ * `online`: its controller's socket to Switch is attached. `offline`: it was+ * connected and is not now (stopped, asleep, cut off, or the Switch server+ * holding it went away). `unknown`: it has never connected. `revoked`: removed+ * from Switch.+ */+export type ControllerState = "online" | "offline" | "unknown" | "revoked";++/** The machine's connection as Switch last recorded it: the current one, or the last. */+export interface ControllerConnection {+ /** When its socket last attached. */+ connected_at: string;+ /** Null while it is connected, and when the Switch server holding it was lost long ago. */+ disconnected_at: string | null;+ /**+ * Null while it is connected; else `socket_closed`, `heartbeat_lapsed`, `taken_over`,+ * `revoked`, `server_shutdown` or `server_lost`.+ */+ disconnect_reason: string | null;+}export interface Platform {os: string;@@ -74,7 +93,10 @@ export interface Controller {platform: Platform | null;version: string | null;state: ControllerState;+ /** When it last reported its status. */last_seen_at: string | null;+ /** Null when it has never connected. */+ connection: ControllerConnection | null;status: StatusReport | null;assignment_revision: number;created_at: string;@@ -258,7 +280,26 @@ export function createOperation(body: {return request<Operation>("/operations", { method: "POST", body: JSON.stringify(body) });}-/** The command a user runs on the machine to enroll it with a fresh code. */-export function enrollCommand(server: string, code: string): string {- return `switch-agent-controller enroll --server ${server} --code ${code}`;+/** One word for a POSIX shell: as it is when nothing in it is special, else single-quoted. */+export function shellQuote(value: string): string {+ if (/^[A-Za-z0-9_@%+=:,./-]+$/.test(value)) return value;+ return `'${value.replace(/'/g, `'\\''`)}'`;+}++/**+ * The command a user runs on the machine to enroll it with a fresh code, with+ * the machine's name and description when given (blank is left out: the name+ * then defaults to the host name).+ */+export function enrollCommand(+ server: string,+ code: string,+ machine: { name: string; description: string },+): string {+ const words = ["switch-agent-controller", "enroll", "--server", server, "--code", code];+ const name = machine.name.trim();+ const description = machine.description.trim();+ if (name) words.push("--name", name);+ if (description) words.push("--description", description);+ return words.map(shellQuote).join(" ");}diff --git a/gateway/src/pages/machines/AddMachineDialog.test.tsx b/gateway/src/pages/machines/AddMachineDialog.test.tsxindex 2fc69a757..5987efeff 100644--- a/gateway/src/pages/machines/AddMachineDialog.test.tsx+++ b/gateway/src/pages/machines/AddMachineDialog.test.tsx@@ -1,4 +1,4 @@-import { cleanup, render, screen } from "@testing-library/react";+import { cleanup, fireEvent, render, screen } from "@testing-library/react";import { afterEach, describe, expect, it, vi } from "vitest";import AddMachineDialog from "./AddMachineDialog";@@ -36,6 +36,33 @@ describe("AddMachineDialog", () => {expect(screen.queryByText(new RegExp(window.location.origin))).toBeNull();});+ it("puts the optional name and description into the command, quoted", async () => {+ issue("https://switch-api.example.test");+ render(<AddMachineDialog open onClose={() => {}} />);+ const base =+ "switch-agent-controller enroll --server https://switch-api.example.test --code swce_example";+ await screen.findByText(base);+ fireEvent.change(screen.getByLabelText("Name (optional)"), { target: { value: " build-box " } });+ fireEvent.change(screen.getByLabelText("Description (optional)"), {+ target: { value: "Bob's box in the office" },+ });+ expect(+ screen.getByText(`${base} --name build-box --description 'Bob'\\''s box in the office'`),+ ).toBeTruthy();+ });++ it("shows no command for a name or description that is too long", async () => {+ issue("https://switch-api.example.test");+ render(<AddMachineDialog open onClose={() => {}} />);+ await screen.findByText(/--code swce_example/);+ fireEvent.change(screen.getByLabelText("Description (optional)"), {+ target: { value: "x".repeat(501) },+ });+ expect(screen.getByText("At most 500 characters.")).toBeTruthy();+ expect(screen.queryByText(/--code swce_example/)).toBeNull();+ expect(screen.getByText("swce_example")).toBeTruthy();+ });+it("says what to configure instead of showing a command that cannot work", async () => {issue(null);render(<AddMachineDialog open onClose={() => {}} />);diff --git a/gateway/src/pages/machines/AddMachineDialog.tsx b/gateway/src/pages/machines/AddMachineDialog.tsxindex 5972afaf9..7246ffd50 100644--- a/gateway/src/pages/machines/AddMachineDialog.tsx+++ b/gateway/src/pages/machines/AddMachineDialog.tsx@@ -12,11 +12,18 @@ import {DialogTitle,IconButton,Stack,+ TextField,Tooltip,Typography,} from "@mui/material";import { useEffect, useState } from "react";-import { createEnrollmentCode, type EnrollmentCode, enrollCommand } from "../../data/management";+import {+ createEnrollmentCode,+ type EnrollmentCode,+ enrollCommand,+ MAX_MACHINE_DESCRIPTION,+ MAX_MACHINE_NAME,+} from "../../data/management";import { MONO_SX, formatDate } from "../../theme/hootFormat";function CopyBlock({ text, label }: { text: string; label: string }) {@@ -48,7 +55,8 @@ function CopyBlock({ text, label }: { text: string; label: string }) {/*** Issues a one-time enrollment code and shows the command that enrolls a* machine's agents controller with it. The code is shown once: closing the- * dialog discards it, and a new open issues a new one.+ * dialog discards it, and a new open issues a new one. The optional name and+ * description go into the command, so the machine enrolls with them.*/export default function AddMachineDialog({open,@@ -59,12 +67,23 @@ export default function AddMachineDialog({}) {const [enrollment, setEnrollment] = useState<EnrollmentCode | null>(null);const [error, setError] = useState<string | null>(null);+ const [name, setName] = useState("");+ const [description, setDescription] = useState("");++ const nameError =+ name.trim().length > MAX_MACHINE_NAME ? `At most ${MAX_MACHINE_NAME} characters.` : null;+ const descriptionError =+ description.trim().length > MAX_MACHINE_DESCRIPTION+ ? `At most ${MAX_MACHINE_DESCRIPTION} characters.`+ : null;useEffect(() => {if (!open) return;let cancelled = false;setEnrollment(null);setError(null);+ setName("");+ setDescription("");createEnrollmentCode().then((issued) => {if (!cancelled) setEnrollment(issued);@@ -90,14 +109,36 @@ export default function AddMachineDialog({{!error && !enrollment && <CircularProgress size={24} />}{enrollment && (<Stack spacing={2}>+ <Stack spacing={2}>+ <TextField+ label="Name (optional)"+ value={name}+ onChange={(e) => setName(e.target.value)}+ error={nameError !== null}+ helperText={nameError ?? "Defaults to the machine's host name."}+ size="small"+ fullWidth+ />+ <TextField+ label="Description (optional)"+ value={description}+ onChange={(e) => setDescription(e.target.value)}+ error={descriptionError !== null}+ helperText={descriptionError ?? "What the machine is for, e.g. the build box in the office."}+ size="small"+ fullWidth+ />+ </Stack><Box><Typography variant="subtitle2" gutterBottom>Enrollment command</Typography>- {enrollment.server_url ? (+ {enrollment.server_url && (nameError || descriptionError) ? (+ <Alert severity="warning">Shorten the name or description to see the command.</Alert>+ ) : enrollment.server_url ? (<><CopyBlock- text={enrollCommand(enrollment.server_url, enrollment.code)}+ text={enrollCommand(enrollment.server_url, enrollment.code, { name, description })}label="Copy command"/><Typography variant="caption" color="text.secondary">diff --git a/gateway/src/pages/machines/MachinesPage.test.tsx b/gateway/src/pages/machines/MachinesPage.test.tsxindex b3b981d47..4bd07da74 100644--- a/gateway/src/pages/machines/MachinesPage.test.tsx+++ b/gateway/src/pages/machines/MachinesPage.test.tsx@@ -12,6 +12,11 @@ const laptop: Controller = {version: "0.1.0",state: "online",last_seen_at: new Date().toISOString(),+ connection: {+ connected_at: new Date().toISOString(),+ disconnected_at: null,+ disconnect_reason: null,+ },assignment_revision: 3,created_at: "2026-10-01T00:00:00Z",revoked_at: null,@@ -151,6 +156,39 @@ describe("MachinesPage", () => {expect(screen.getAllByText("running")).toHaveLength(1);});+ it("shows a machine that stopped as offline, and its agents as unreachable", async () => {+ const stopped: Controller = {+ ...laptop,+ state: "offline",+ connection: {+ ...laptop.connection!,+ disconnected_at: new Date().toISOString(),+ disconnect_reason: "socket_closed",+ },+ };+ mockManagement({+ "GET /controllers": [200, [stopped]],+ "GET /agents": [200, [{ ...pmAgent, controller_state: "offline" }]],+ "GET /operations": [200, []],+ });+ render(<MachinesPage />);+ expect(await screen.findByText("offline")).toBeTruthy();+ expect(screen.getByText("Not connected")).toBeTruthy();+ expect(screen.getByText("machine offline")).toBeTruthy();+ expect(screen.queryByText("failed: invalid_credential")).toBeNull();+ });++ it("shows a machine that never connected as unknown", async () => {+ mockManagement({+ "GET /controllers": [200, [{ ...laptop, state: "unknown", connection: null, status: null }]],+ "GET /agents": [200, []],+ "GET /operations": [200, []],+ });+ render(<MachinesPage />);+ expect(await screen.findByText("unknown")).toBeTruthy();+ expect(screen.getByText("Unknown")).toBeTruthy();+ });+it("describes revoking without per-agent keys", async () => {mockManagement({"GET /controllers": [200, [laptop]],diff --git a/gateway/src/pages/machines/MachinesPage.tsx b/gateway/src/pages/machines/MachinesPage.tsxindex 0c019c624..c53ecb887 100644--- a/gateway/src/pages/machines/MachinesPage.tsx+++ b/gateway/src/pages/machines/MachinesPage.tsx@@ -46,14 +46,51 @@ import AddMachineDialog from "./AddMachineDialog";import EditMachineDialog from "./EditMachineDialog";import ManagedAgentDialog from "./ManagedAgentDialog";-const REFRESH_MS = 10_000;+const REFRESH_MS = 5_000;const STATE_COLOR: Record<ControllerState, "success" | "warning" | "default"> = {online: "success",- unknown: "warning",+ offline: "warning",+ unknown: "default",revoked: "default",};+const DISCONNECT_REASON: Record<string, string> = {+ socket_closed: "its controller stopped or lost its connection",+ heartbeat_lapsed: "its connection stopped responding",+ server_shutdown: "the Switch server it was connected to restarted",+ server_lost: "the Switch server it was connected to stopped responding",+ taken_over: "another instance of its controller took over",+ revoked: "it was removed",+};++/** What the state chip says on hover: when the machine connected, or when and why it stopped. */+function stateTitle(row: Controller): string {+ const connection = row.connection;+ if (row.state === "revoked") return "Removed from Switch.";+ if (row.state === "unknown" || connection === null)+ return "Has never connected to Switch. Start its controller with switch-agent-controller run.";+ if (row.state === "online") return `Connected since ${absoluteTitle(connection.connected_at)}`;+ const why = connection.disconnect_reason+ ? (DISCONNECT_REASON[connection.disconnect_reason] ?? connection.disconnect_reason)+ : null;+ const when =+ connection.disconnected_at !== null ? ` ${formatRelative(connection.disconnected_at)}` : "";+ return `Disconnected${when}${why ? `: ${why}` : ""}.`;+}++/**+ * The last time Switch heard from the machine at all: now while it is connected, else the+ * latest of its status report, its socket attaching and its socket going.+ */+function lastContact(row: Controller): string | null {+ if (row.state === "online") return new Date().toISOString();+ const times = [row.last_seen_at, row.connection?.connected_at, row.connection?.disconnected_at]+ .filter((t): t is string => typeof t === "string")+ .sort((a, b) => Date.parse(b) - Date.parse(a));+ return times[0] ?? null;+}+const PROCESS_COLOR: Record<string, "success" | "warning" | "error" | "default"> = {running: "success",starting: "warning",@@ -199,7 +236,9 @@ export default function MachinesPage() {headerName: "State",width: 110,renderCell: ({ row }) => (- <Chip size="small" color={STATE_COLOR[row.state]} label={row.state} />+ <Tooltip title={stateTitle(row)}>+ <Chip size="small" color={STATE_COLOR[row.state] ?? "default"} label={row.state} />+ </Tooltip>),},{@@ -245,9 +284,11 @@ export default function MachinesPage() {valueGetter: (_value, row) =>row.state === "revoked"? "None (revoked)"- : row.state === "unknown"- ? "Unknown"- : row.status+ : row.state === "offline"+ ? "Not connected"+ : row.state === "unknown"+ ? "Unknown"+ : row.status? `${row.status.agents.filter((a) => a.process === "running").length} / ${row.status.agents.length}`: EM_DASH,},@@ -255,9 +296,10 @@ export default function MachinesPage() {field: "last_seen_at",headerName: "Last seen",width: 130,+ valueGetter: (_value, row) => lastContact(row),renderCell: ({ row }) => (- <Tooltip title={absoluteTitle(row.last_seen_at)}>- <span>{formatRelative(row.last_seen_at)}</span>+ <Tooltip title={absoluteTitle(lastContact(row))}>+ <span>{formatRelative(lastContact(row))}</span></Tooltip>),},@@ -372,10 +414,16 @@ export default function MachinesPage() {<Chip size="small" label="not running: machine revoked" /></Tooltip>);+ if (row.controller_state === "offline")+ return (+ <Tooltip title="Its machine is not connected to Switch, so this agent is not reachable. It runs again once the machine's controller is back.">+ <Chip size="small" color="warning" label="machine offline" />+ </Tooltip>+ );if (row.controller_state === "unknown")return (- <Tooltip title="The machine has stopped reporting; this agent's state is not known.">- <Chip size="small" color="warning" label="unknown" />+ <Tooltip title="Its machine has never connected to Switch; this agent's state is not known.">+ <Chip size="small" label="unknown" /></Tooltip>);if (!row.status)