Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Machines show offline within seconds: connection transitions and a process lease in the database, set-info (#696, via #678)

Abel Dantasa3d58a6bd18d
diff --git a/console/apps/switch-console-desktop/src/main/core/embedded-controller/controller.ts b/console/apps/switch-console-desktop/src/main/core/embedded-controller/controller.ts
index bf27cc7cd..c980545c4 100644
--- a/console/apps/switch-console-desktop/src/main/core/embedded-controller/controller.ts
+++ b/console/apps/switch-console-desktop/src/main/core/embedded-controller/controller.ts
@@ -30,6 +30,9 @@ export const embeddedControllerController = createRPCController({
dismissRemoved: (serverId: string): Promise<void> => embeddedControllerService.dismiss(serverId),
+ /** Enrolls this computer again when the server no longer knows the machine it was. */
+ enrollAgain: (serverId: string): Promise<void> => embeddedControllerService.enrollAgain(serverId),
+
/**
* Where this computer's controller puts a managed agent that names no
* directory, `~` for the home directory; null for a name that cannot be a
diff --git a/console/apps/switch-console-desktop/src/main/core/embedded-controller/embedded-controller-service.test.ts b/console/apps/switch-console-desktop/src/main/core/embedded-controller/embedded-controller-service.test.ts
index 082765f52..65999fd29 100644
--- a/console/apps/switch-console-desktop/src/main/core/embedded-controller/embedded-controller-service.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/embedded-controller/embedded-controller-service.test.ts
@@ -219,6 +219,33 @@ describe('EmbeddedControllerService', () => {
expect(events.map((event) => event.phase.kind)).toEqual(['enrolling', 'running']);
});
+ it('enrolls again when Switch no longer lists the machine, forgetting the old identity', async () => {
+ const running = await enabled();
+ management.enroll.mockResolvedValueOnce({
+ serverId: SERVER,
+ apiUrl: 'https://switch.example.com',
+ controllerId: 'controller-2',
+ credential: 'swcc_second',
+ });
+ await running.enrollAgain(SERVER);
+ expect(management.revoke).not.toHaveBeenCalled();
+ expect(management.enroll).toHaveBeenCalledTimes(2);
+ await waitFor(() => calls.length === 2, 'the new controller started');
+ expect(storedRecord()).toMatchObject({ kind: 'enrolled', controllerId: 'controller-2' });
+ expect(secrets.get(credentialSecretKey(SERVER))).toBe('swcc_second');
+ });
+
+ it('refuses to enroll again while Switch still lists the machine', async () => {
+ const running = await enabled();
+ management.read.mockResolvedValue({
+ kind: 'ok',
+ controller: { name: 'build-box', description: null, state: 'offline', lastSeenAt: null },
+ agents: [],
+ });
+ await expect(running.enrollAgain(SERVER)).rejects.toThrow(/still lists this computer/);
+ expect(management.enroll).toHaveBeenCalledTimes(1);
+ });
+
it('restarts the controller with backoff when it exits on its own', async () => {
await enabled();
calls[0]!.child.exit(1);
diff --git a/console/apps/switch-console-desktop/src/main/core/embedded-controller/embedded-controller-service.ts b/console/apps/switch-console-desktop/src/main/core/embedded-controller/embedded-controller-service.ts
index f66f31465..ba2d7c141 100644
--- a/console/apps/switch-console-desktop/src/main/core/embedded-controller/embedded-controller-service.ts
+++ b/console/apps/switch-console-desktop/src/main/core/embedded-controller/embedded-controller-service.ts
@@ -289,6 +289,42 @@ export class EmbeddedControllerService {
});
}
+ /**
+ * Enrolls this computer afresh when the server no longer knows the machine
+ * it was enrolled as: its database was reset or restored, say. The old
+ * controller is stopped and its identity forgotten (there is nothing on the
+ * server to revoke), then this computer enrolls again in the same
+ * workspace. Refused while the server still lists the machine, which is
+ * turned off and on instead.
+ */
+ async enrollAgain(serverId: string): Promise<void> {
+ const record = await this.deps.records.get(serverId);
+ if (record?.kind !== 'enrolled')
+ throw new Error('This computer is not enrolled to run managed agents for this server.');
+ const remote = await this.deps.management.read(record.workspaceId, record.controllerId);
+ if (remote.kind !== 'ok')
+ throw new Error(
+ remote.kind === 'error'
+ ? `Switch could not be asked about this computer: ${remote.message}`
+ : 'This server no longer has agent management turned on.'
+ );
+ if (remote.controller)
+ throw new Error(
+ 'Switch still lists this computer as a machine. Turn it off and on again instead.'
+ );
+ await this.exclusive(serverId, async () => {
+ const runner = this.runners.get(serverId);
+ this.runners.delete(serverId);
+ if (runner) await runner.supervisor.stop(this.deps.stopTimeoutMs);
+ await this.forget(serverId);
+ this.deps.log.warn('The server no longer knew this computer as a machine; enrolling again', {
+ serverId,
+ controllerId: record.controllerId,
+ });
+ });
+ await this.enable(serverId, record.workspaceId);
+ }
+
/**
* Renames this computer as a machine on the server and/or changes its
* description: what its owner sees on the Machines page, and what agents
diff --git a/console/apps/switch-console-desktop/src/main/core/host-controllers/controller.ts b/console/apps/switch-console-desktop/src/main/core/host-controllers/controller.ts
index 5a17758fa..567abd0d6 100644
--- a/console/apps/switch-console-desktop/src/main/core/host-controllers/controller.ts
+++ b/console/apps/switch-console-desktop/src/main/core/host-controllers/controller.ts
@@ -19,6 +19,10 @@ export const hostControllersController = createRPCController({
restart: (params: { sshHost: string; serverId: string }): Promise<void> =>
hostControllerService.restart(params.sshHost, params.serverId),
+ /** Enrolls the host again when the server no longer knows the machine it was. */
+ enrollAgain: (params: { sshHost: string; serverId: string }): Promise<void> =>
+ hostControllerService.enrollAgain(params.sshHost, params.serverId),
+
disable: (params: { sshHost: string; serverId: string }): Promise<void> =>
hostControllerService.disable(params.sshHost, params.serverId, { force: false }),
diff --git a/console/apps/switch-console-desktop/src/main/core/host-controllers/host-controller-service.test.ts b/console/apps/switch-console-desktop/src/main/core/host-controllers/host-controller-service.test.ts
index 9ef14cd85..f5358fd52 100644
--- a/console/apps/switch-console-desktop/src/main/core/host-controllers/host-controller-service.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/host-controllers/host-controller-service.test.ts
@@ -308,6 +308,35 @@ describe('the host as a machine afterwards', () => {
});
});
+describe('enrolling a host again that Switch no longer knows', () => {
+ it('wipes the old controller on the host, without a revoke, and enrolls it afresh', async () => {
+ const base = deps();
+ const service = new HostControllerService({
+ ...base,
+ management: {
+ ...base.management,
+ read: async () => ({ kind: 'ok', controller: null, agents: [] }),
+ },
+ });
+ await service.enable(HOST, SERVER, WORKSPACE);
+ calls = [];
+ host.enroll = { ok: true, controllerId: 'controller-8' };
+ await service.enrollAgain(HOST, SERVER);
+ expect(calls.slice(0, 2)).toEqual(['stop', 'forget']);
+ expect(calls).not.toContain('revoke');
+ expect(calls).toContain('code');
+ expect(records.get(`${HOST}|${SERVER}`)?.controllerId).toBe('controller-8');
+ });
+
+ it('refuses while Switch still lists the host', async () => {
+ const service = new HostControllerService(deps());
+ await service.enable(HOST, SERVER, WORKSPACE);
+ calls = [];
+ await expect(service.enrollAgain(HOST, SERVER)).rejects.toThrow(/still lists build/);
+ expect(calls).toEqual([]);
+ });
+});
+
describe('removing the host as a machine', () => {
it('revokes the controller first, then stops it and clears it from the host', async () => {
const service = new HostControllerService(deps());
diff --git a/console/apps/switch-console-desktop/src/main/core/host-controllers/host-controller-service.ts b/console/apps/switch-console-desktop/src/main/core/host-controllers/host-controller-service.ts
index 2f1484bd0..191294bfe 100644
--- a/console/apps/switch-console-desktop/src/main/core/host-controllers/host-controller-service.ts
+++ b/console/apps/switch-console-desktop/src/main/core/host-controllers/host-controller-service.ts
@@ -348,6 +348,45 @@ export class HostControllerService {
});
}
+ /**
+ * Enrolls the host afresh when the server no longer knows the machine it
+ * was enrolled as: its database was reset or restored, say. The old
+ * controller is stopped and its identity wiped on the host (there is nothing
+ * on the server to revoke), then the host enrolls again in the same
+ * workspace. Refused while the server still lists the machine, which is
+ * turned off and on instead.
+ */
+ async enrollAgain(sshHost: string, serverId: string): Promise<void> {
+ const record = await this.deps.records.get(sshHost, serverId);
+ if (!record) throw new Error(`${sshHost} is not a machine for this server.`);
+ const remote = await this.deps.management.read(record.workspaceId, record.controllerId);
+ if (remote.kind !== 'ok')
+ throw new Error(
+ remote.kind === 'error'
+ ? `Switch could not be asked about ${sshHost}: ${remote.message}`
+ : 'This server no longer has agent management turned on.'
+ );
+ if (remote.controller)
+ throw new Error(
+ `Switch still lists ${sshHost} as a machine. Turn it off and on again instead.`
+ );
+ await this.exclusive(sshHost, serverId, async () => {
+ const shell = await this.deps.shell(sshHost);
+ try {
+ await this.stopOn(shell, record, { turnOff: true, wipe: true });
+ } finally {
+ shell.close();
+ }
+ await this.deps.records.delete(sshHost, serverId);
+ this.deps.log.warn('Switch no longer knew an SSH host as a machine; enrolling it again', {
+ sshHost,
+ serverId,
+ controllerId: record.controllerId,
+ });
+ });
+ await this.enable(sshHost, serverId, record.workspaceId);
+ }
+
/**
* The host is being removed from Console: its controllers go first. Refused,
* with nothing changed, while any of them runs agents moved from this Console.
diff --git a/console/apps/switch-console-desktop/src/main/core/switch-servers/gateway-client.ts b/console/apps/switch-console-desktop/src/main/core/switch-servers/gateway-client.ts
index 70e440b24..a1afbc752 100644
--- a/console/apps/switch-console-desktop/src/main/core/switch-servers/gateway-client.ts
+++ b/console/apps/switch-console-desktop/src/main/core/switch-servers/gateway-client.ts
@@ -2610,7 +2610,7 @@ export type ManagementController = {
/** What its owner says the machine is for; null when none was given. */
description: string | null;
kind: string;
- state: 'online' | 'unknown' | 'revoked';
+ state: 'online' | 'offline' | 'unknown' | 'revoked';
lastSeenAt: string | null;
revokedAt: string | null;
/** Each provider as the controller last reported it; empty before it has reported. */
@@ -2667,7 +2667,7 @@ type ManagementControllerJson = {
name: string;
description: string | null;
kind: string;
- state: 'online' | 'unknown' | 'revoked';
+ state: 'online' | 'offline' | 'unknown' | 'revoked';
last_seen_at: string | null;
revoked_at: string | null;
status?: unknown;
diff --git a/console/apps/switch-console-desktop/src/renderer/features/managed-agents/managed-machine-card.tsx b/console/apps/switch-console-desktop/src/renderer/features/managed-agents/managed-machine-card.tsx
index d1a9c5476..5243a1cac 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/managed-agents/managed-machine-card.tsx
+++ b/console/apps/switch-console-desktop/src/renderer/features/managed-agents/managed-machine-card.tsx
@@ -115,7 +115,7 @@ function MachineCard({
);
}
-function MachineStatePill({ state }: { state: 'online' | 'unknown' | 'revoked' }) {
+function MachineStatePill({ state }: { state: 'online' | 'offline' | 'unknown' | 'revoked' }) {
if (state === 'revoked')
return <span className="shrink-0 text-xs text-foreground-muted">Removed</span>;
const online = state === 'online';
diff --git a/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine-card.tsx b/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine-card.tsx
index fd2dc3ab7..5291f4d98 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine-card.tsx
+++ b/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine-card.tsx
@@ -29,6 +29,7 @@ import {
hostMachineStatus,
hostStateKey,
hostToggleBlocker,
+ hostUnknownToServer,
supervisionNote,
} from './host-machine';
@@ -110,6 +111,10 @@ export function HostMachineCard({
mutationFn: () => rpc.hostControllers.restart({ sshHost, serverId }),
...handlers,
});
+ const enrollAgain = useMutation({
+ mutationFn: () => rpc.hostControllers.enrollAgain({ sshHost, serverId }),
+ ...handlers,
+ });
const moveAll = useMutation({
mutationFn: () => rpc.hostControllers.moveAll(sshHost),
...handlers,
@@ -183,7 +188,17 @@ export function HostMachineCard({
{failureText(failure, 'That did not work.')}
</p>
)}
- {canRestartHost(overview) && (
+ {hostUnknownToServer(overview) && (
+ <Button
+ variant="outline"
+ size="sm"
+ disabled={enrollAgain.isPending}
+ onClick={() => enrollAgain.mutate()}
+ >
+ <RefreshCw className="size-4" /> {enrollAgain.isPending ? 'Enrolling…' : 'Enroll again'}
+ </Button>
+ )}
+ {canRestartHost(overview) && !hostUnknownToServer(overview) && (
<Button
variant="outline"
size="sm"
diff --git a/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine.test.ts b/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine.test.ts
index 8a8968dfa..ffe1d43bf 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine.test.ts
+++ b/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine.test.ts
@@ -8,6 +8,7 @@ import {
hostMachineStatus,
hostStateKey,
hostToggleBlocker,
+ hostUnknownToServer,
supervisionNote,
} from './host-machine';
@@ -45,6 +46,24 @@ describe('the state of a host as a machine', () => {
).toEqual({ label: 'Setting up…', tone: 'busy', detail: 'Enrolling the host…' });
});
+ it('says Switch no longer knows the host when it is enrolled but not listed, whatever its process does', () => {
+ const unknown = { ...ON, remote: { kind: 'ok' as const, controller: null, agents: [] } };
+ expect(hostUnknownToServer(unknown)).toBe(true);
+ expect(hostMachineStatus(unknown)).toMatchObject({
+ label: 'Unknown to Switch',
+ tone: 'error',
+ detail: expect.stringContaining('Switch no longer knows build-box as a machine'),
+ });
+ const stopped = {
+ ...unknown,
+ process: { kind: 'stopped' as const, state: 'exited', code: 1, log: '' },
+ };
+ expect(hostMachineStatus(stopped).label).toBe('Unknown to Switch');
+ expect(hostMachineStatus({ ...unknown, movedAgents: ['jack'] }).detail).toContain('jack');
+ expect(hostUnknownToServer(OFF)).toBe(false);
+ expect(hostUnknownToServer(ON)).toBe(false);
+ });
+
it('is running when the host runs it and Switch sees it', () => {
expect(hostMachineStatus(ON)).toEqual({ label: 'Running', tone: 'ok', detail: null });
});
diff --git a/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine.ts b/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine.ts
index f12bc89c9..f788ef0f1 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine.ts
+++ b/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-machine.ts
@@ -59,6 +59,16 @@ export function hostMachineStatus(overview: HostControllerOverview): HostMachine
return { label: 'Off', tone: 'neutral', detail: null };
}
const failed = phase.kind === 'error' ? phase.message : null;
+ if (hostUnknownToServer(overview)) {
+ const why = `Switch no longer knows ${overview.sshHost} as a machine (its data was reset or restored, or the machine was deleted), so its controller cannot sign in. Enroll it again.`;
+ return {
+ label: 'Unknown to Switch',
+ tone: 'error',
+ detail: overview.movedAgents.length
+ ? `${why} ${overview.movedAgents.join(', ')}, moved here from this Console, then need Stop managing, and can be moved again.`
+ : why,
+ };
+ }
if (process?.kind === 'unknown')
return {
label: 'Unknown',
@@ -97,9 +107,7 @@ export function hostMachineStatus(overview: HostControllerOverview): HostMachine
detail:
state === 'revoked'
? 'Switch has removed this host; its controller is stopping its agents.'
- : state === null
- ? 'Switch does not list this host’s controller.'
- : 'The controller runs but has not reached Switch recently. It keeps trying.',
+ : 'The controller runs but has not reached Switch recently. It keeps trying.',
};
return { label: 'Running', tone: failed ? 'warn' : 'ok', detail: failed };
}
@@ -193,3 +201,14 @@ export function hostStateKey(overview: HostControllerOverview): string {
export function failureAlreadyShown(overview: HostControllerOverview, message: string): boolean {
return overview.phase.kind === 'error' && overview.phase.message.trim() === message.trim();
}
+
+/** Switch answered, and does not list the machine this host is enrolled as. */
+export function hostUnknownToServer(overview: HostControllerOverview): boolean {
+ return (
+ overview.enrollment !== null &&
+ overview.phase.kind !== 'installing' &&
+ overview.phase.kind !== 'removing' &&
+ overview.remote?.kind === 'ok' &&
+ overview.remote.controller === null
+ );
+}
diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine-card.tsx b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine-card.tsx
index 227bd9289..68933ff9e 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine-card.tsx
+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine-card.tsx
@@ -38,6 +38,7 @@ import {
type MachineStatusTone,
toggleBlocker,
toggleChecked,
+ unknownToServer,
} from './this-computer-machine';
/** How often the card re-reads the server while it is on screen. */
@@ -135,6 +136,10 @@ export const ThisComputerMachineCard = observer(function ThisComputerMachineCard
mutationFn: () => rpc.embeddedController.dismissRemoved(serverId),
...handlers,
});
+ const enrollAgain = useMutation({
+ mutationFn: () => rpc.embeddedController.enrollAgain(serverId),
+ ...handlers,
+ });
const overview = overviewQuery.data;
// Nothing to say on a server this computer has nothing to do with and is not
@@ -210,6 +215,17 @@ export const ThisComputerMachineCard = observer(function ThisComputerMachineCard
Start again
</Button>
)}
+ {unknownToServer(overview) && (
+ <Button
+ variant="outline"
+ size="sm"
+ disabled={enrollAgain.isPending}
+ onClick={() => enrollAgain.mutate()}
+ >
+ <RefreshCw className="size-4" />
+ {enrollAgain.isPending ? 'Enrolling…' : 'Enroll again'}
+ </Button>
+ )}
{overview.phase.kind === 'removed' && (
<Button
variant="outline"
diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine.test.ts b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine.test.ts
index fcb66dfee..a605c7cfe 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine.test.ts
+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine.test.ts
@@ -11,6 +11,7 @@ import {
machineStatus,
toggleBlocker,
toggleChecked,
+ unknownToServer,
} from './this-computer-machine';
const ENROLLMENT = {
@@ -35,7 +36,9 @@ function overview(
};
}
-const ok = (state: 'online' | 'unknown' | 'revoked' | null): EmbeddedControllerRemote => ({
+const ok = (
+ state: 'online' | 'offline' | 'unknown' | 'revoked' | null
+): EmbeddedControllerRemote => ({
kind: 'ok',
controller: state ? { name: 'box', description: null, state, lastSeenAt: null } : null,
agents: [],
@@ -63,6 +66,21 @@ describe('machineStatus', () => {
});
});
+ it('says Switch no longer knows the computer when it is enrolled but not listed, and offers to enroll again', () => {
+ const unknown = overview(running, ok(null));
+ expect(unknownToServer(unknown)).toBe(true);
+ expect(machineStatus(unknown, LATER)).toMatchObject({
+ label: 'Unknown to Switch',
+ tone: 'error',
+ detail: expect.stringContaining('Enroll it again.'),
+ });
+ const withMoved = { ...unknown, movedAgents: ['jack'] };
+ expect(machineStatus(withMoved, LATER).detail).toContain('jack, moved here from this Console');
+ expect(unknownToServer(overview(running, ok('offline')))).toBe(false);
+ expect(unknownToServer(overview(running, { kind: 'error', message: 'down' }))).toBe(false);
+ expect(unknownToServer(overview({ kind: 'off' }, ok(null), false))).toBe(false);
+ });
+
it('says connecting, not disconnected, in the first minute after the controller starts', () => {
const justStarted = Date.parse(running.since) + 5_000;
expect(machineStatus(overview(running, ok('unknown')), justStarted)).toMatchObject({
diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine.ts b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine.ts
index 26c6dc14e..7b5d78766 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine.ts
+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/this-computer-machine.ts
@@ -26,7 +26,21 @@ function clock(iso: string): string {
/** How long after starting a controller that has not reached Switch reads as connecting, not disconnected. */
export const CONNECTING_GRACE_MS = 60_000;
+/** What the card says when Switch does not list the machine this computer is enrolled as. */
+function unknownStatus(overview: EmbeddedControllerOverview): MachineStatus {
+ const why =
+ 'Switch no longer knows this computer as a machine (its data was reset or restored, or the machine was deleted), so its controller cannot sign in. Enroll it again.';
+ return {
+ label: 'Unknown to Switch',
+ tone: 'error',
+ detail: overview.movedAgents.length
+ ? `${why} ${overview.movedAgents.join(', ')}, moved here from this Console, then need Stop managing, and can be moved again.`
+ : why,
+ };
+}
+
export function machineStatus(overview: EmbeddedControllerOverview, nowMs: number): MachineStatus {
+ if (unknownToServer(overview)) return unknownStatus(overview);
const { phase, remote } = overview;
switch (phase.kind) {
case 'off':
@@ -93,10 +107,7 @@ export function machineStatus(overview: EmbeddedControllerOverview, nowMs: numbe
return {
label: 'Disconnected',
tone: 'warn',
- detail:
- state === null
- ? 'Switch does not list this computer’s controller.'
- : 'The controller is running but has not reached Switch recently. It keeps trying.',
+ detail: 'The controller is running but has not reached Switch recently. It keeps trying.',
};
}
}
@@ -165,3 +176,13 @@ export function machineStateKey(overview: EmbeddedControllerOverview): string {
overview.movedAgents,
]);
}
+
+/** Switch answered, and does not list the machine this computer is enrolled as. */
+export function unknownToServer(overview: EmbeddedControllerOverview): boolean {
+ return (
+ overview.enrollment !== null &&
+ overview.remote?.kind === 'ok' &&
+ overview.remote.controller === null &&
+ (overview.phase.kind === 'running' || overview.phase.kind === 'restarting')
+ );
+}
diff --git a/console/apps/switch-console-desktop/src/shared/core/embedded-controller/embedded-controller.ts b/console/apps/switch-console-desktop/src/shared/core/embedded-controller/embedded-controller.ts
index 9817435f8..57f6b4b96 100644
--- a/console/apps/switch-console-desktop/src/shared/core/embedded-controller/embedded-controller.ts
+++ b/console/apps/switch-console-desktop/src/shared/core/embedded-controller/embedded-controller.ts
@@ -56,7 +56,7 @@ export type EmbeddedControllerRemote =
controller: {
name: string;
description: string | null;
- state: 'online' | 'unknown' | 'revoked';
+ state: 'online' | 'offline' | 'unknown' | 'revoked';
lastSeenAt: string | null;
} | null;
agents: PlacedManagedAgent[];
diff --git a/console/apps/switch-console-desktop/src/shared/core/host-controllers/host-controllers.ts b/console/apps/switch-console-desktop/src/shared/core/host-controllers/host-controllers.ts
index 93580417b..873818eb1 100644
--- a/console/apps/switch-console-desktop/src/shared/core/host-controllers/host-controllers.ts
+++ b/console/apps/switch-console-desktop/src/shared/core/host-controllers/host-controllers.ts
@@ -34,7 +34,10 @@ export type HostControllerProcess =
export type HostControllerRemote =
| {
kind: 'ok';
- controller: { state: 'online' | 'unknown' | 'revoked'; lastSeenAt: string | null } | null;
+ controller: {
+ state: 'online' | 'offline' | 'unknown' | 'revoked';
+ lastSeenAt: string | null;
+ } | null;
agents: PlacedManagedAgent[];
}
| { kind: 'unavailable' }
diff --git a/console/apps/switch-console-desktop/src/shared/core/managed-agents/managed-agents.ts b/console/apps/switch-console-desktop/src/shared/core/managed-agents/managed-agents.ts
index 9db9448ea..53ff4fc53 100644
--- a/console/apps/switch-console-desktop/src/shared/core/managed-agents/managed-agents.ts
+++ b/console/apps/switch-console-desktop/src/shared/core/managed-agents/managed-agents.ts
@@ -50,7 +50,7 @@ export type ManagedMachine = {
id: string;
name: string;
kind: string;
- state: 'online' | 'unknown' | 'revoked';
+ state: 'online' | 'offline' | 'unknown' | 'revoked';
};
/** A provider on a machine, as the machine last reported it. */
diff --git a/console/packages/agent-controller/README.md b/console/packages/agent-controller/README.md
index 35ad52455..a88afe577 100644
--- a/console/packages/agent-controller/README.md
+++ b/console/packages/agent-controller/README.md
@@ -49,6 +49,8 @@ node packages/agent-controller/dist/cli.mjs enroll \
node packages/agent-controller/dist/cli.mjs run [--data-dir <dir>]
node packages/agent-controller/dist/cli.mjs status [--data-dir <dir>]
+node packages/agent-controller/dist/cli.mjs set-info [--name <name>] \
+ [--description <text>] [--data-dir <dir>]
```
- `--server` is the agent bridge URL, and it must be `https`. Plain `http` is
@@ -59,6 +61,13 @@ node packages/agent-controller/dist/cli.mjs status [--data-dir <dir>]
characters). Its owner sees it in the gateway's Machines page, where both the
name and the description can be changed later, and so do the agents allowed
to manage agents for that owner.
+- `set-info` changes the machine's name and/or description on the server after
+ enrollment, with this controller's own credential, and records the new name
+ in the data directory. Give `--name`, `--description` or both; `--description ""`
+ clears the description. The limits are enrollment's (a name of at most 200
+ characters, not blank). It needs the credential in the data directory, so a
+ controller whose credential is handed over with `--credential-stdin` is
+ renamed in the gateway instead. It can run while `run` does.
- `status` reads only local state. It makes no network call.
- Logging goes to stderr. Set the level with `SWITCH_CONTROLLER_LOG_LEVEL`
(`debug`, `info`, `warn`, `error`; the default is `info`).
@@ -78,7 +87,8 @@ node packages/agent-controller/dist/cli.mjs status [--data-dir <dir>]
a newer controller wrote; a shared-host bundle that is missing, not a file
or unreadable; an unsupported platform (Windows); a data directory that
holds no identity, or no credential (revoked earlier and wiped, or never
- enrolled); and with `--credential-stdin`, a credential that does not arrive
+ enrolled); `set-info` with nothing to change, or a name or description past
+ the limits; and with `--credential-stdin`, a credential that does not arrive
(stdin is a terminal, closes empty, holds more than one token, cannot be
read, or stays open past 10 s). The reason is the last line on stderr,
prefixed `switch-agent-controller: `. Once the controller is running,
diff --git a/console/packages/agent-controller/src/api.test.ts b/console/packages/agent-controller/src/api.test.ts
index 6c7e56cc8..5971332e8 100644
--- a/console/packages/agent-controller/src/api.test.ts
+++ b/console/packages/agent-controller/src/api.test.ts
@@ -322,6 +322,37 @@ describe('the controller connection', () => {
});
});
+describe('the machine', () => {
+ it("changes the machine's name and description", async () => {
+ const { client: c } = client();
+ expect(await c.updateInfo({ name: 'build-box', description: 'The box' })).toEqual({
+ id: core.controllerId,
+ name: 'build-box',
+ description: 'The box',
+ });
+ expect(await c.updateInfo({ description: null })).toMatchObject({ description: null });
+ const sent = core.requests.filter((r) => r.method === 'PATCH');
+ expect(sent.map((r) => r.body)).toEqual([
+ { name: 'build-box', description: 'The box' },
+ { description: null },
+ ]);
+ });
+
+ it('says so when the server cannot rename from the controller', async () => {
+ core.scripted.push({
+ method: 'PATCH',
+ path: `/v1/management/controllers/${core.controllerId}`,
+ status: 405,
+ body: { detail: 'Method Not Allowed' },
+ });
+ const error = await client()
+ .client.updateInfo({ name: 'x' })
+ .catch((e: unknown) => e);
+ expect(error).toBeInstanceOf(ControllerApiError);
+ expect(error).toMatchObject({ status: 405, code: 'not_supported' });
+ });
+});
+
describe('operations', () => {
it('lists, claims, renews and reports', async () => {
const { client: api } = client();
diff --git a/console/packages/agent-controller/src/api.ts b/console/packages/agent-controller/src/api.ts
index 154bdb2c4..f3852b09f 100644
--- a/console/packages/agent-controller/src/api.ts
+++ b/console/packages/agent-controller/src/api.ts
@@ -7,7 +7,10 @@ import {
type Assignment,
assignmentSchema,
type ControllerConnection,
+ type ControllerInfo,
+ type ControllerInfoChange,
controllerConnectionResponseSchema,
+ controllerInfoResponseSchema,
credentialRotateResponseSchema,
type EnrollRequest,
type EnrollResponse,
@@ -352,6 +355,31 @@ export class ControllerClient {
}
}
+ /** Renames this machine and/or changes its description, as its owner can in the gateway. */
+ async updateInfo(change: ControllerInfoChange): Promise<ControllerInfo> {
+ let response: Response;
+ try {
+ response = await this.request(this.controllerPath, { method: 'PATCH', body: change });
+ } catch (error) {
+ // A server from before this route answers with the framework's own
+ // refusal, which carries no Switch error envelope.
+ if (
+ error instanceof ControllerApiError &&
+ (error.status === 404 || error.status === 405) &&
+ error.code === 'unexpected_response'
+ )
+ throw new ControllerApiError(
+ error.status,
+ 'not_supported',
+ "This Switch server cannot rename a machine from its controller. Change the name and description in the gateway's Machines page instead.",
+ false,
+ null
+ );
+ throw error;
+ }
+ return parsed(response, controllerInfoResponseSchema);
+ }
+
async rotateCredential(): Promise<string> {
const response = await this.request(`${this.controllerPath}/credential/rotate`, {
method: 'POST',
diff --git a/console/packages/agent-controller/src/main.test.ts b/console/packages/agent-controller/src/main.test.ts
index 18fad20e5..6e1ddcbb3 100644
--- a/console/packages/agent-controller/src/main.test.ts
+++ b/console/packages/agent-controller/src/main.test.ts
@@ -2,8 +2,10 @@ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
-import { EXIT_CONFIGURATION, EXIT_OK } from './exit-codes';
-import { main } from './main';
+import { EXIT_CONFIGURATION, EXIT_FAILURE, EXIT_OK } from './exit-codes';
+import { infoChange, main } from './main';
+import { dataLayout } from './paths';
+import { CONTROLLER_CREDENTIAL, FileSecretStore } from './secrets';
import { ControllerStore } from './store';
let dir: string;
@@ -152,6 +154,114 @@ describe('main', () => {
expect(lastLine()).toMatch(/must use https/);
});
+ it("set-info changes the machine's name and description, and records the name", async () => {
+ const store = ControllerStore.open(join(dir, 'controller.db'));
+ store.saveIdentity({
+ controllerId: 'controller-1',
+ server: 'https://switch.example.com',
+ name: 'box',
+ enrolledAt: '2026-01-01T00:00:00.000Z',
+ });
+ store.close();
+ await new FileSecretStore(dataLayout(dir).secrets).set(CONTROLLER_CREDENTIAL, 'swcc_test');
+ const calls: { method: string; url: string; body: unknown; auth: string | null }[] = [];
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async (url: string, init: RequestInit) => {
+ const headers = new Headers(init.headers);
+ calls.push({
+ method: String(init.method),
+ url,
+ body: init.body ? JSON.parse(String(init.body)) : null,
+ auth: headers.get('Authorization'),
+ });
+ const body = url.endsWith('/token')
+ ? {
+ access_token: 'swct_test',
+ expires_at: new Date(Date.now() + 3_600_000).toISOString(),
+ }
+ : { id: 'controller-1', name: 'build-box', description: null, state: 'online' };
+ return new Response(JSON.stringify(body), {
+ status: 200,
+ headers: { 'Content-Type': 'application/json' },
+ });
+ })
+ );
+ const code = await main([
+ 'set-info',
+ '--name',
+ ' build-box ',
+ '--description',
+ '',
+ '--data-dir',
+ dir,
+ ]);
+ vi.unstubAllGlobals();
+ expect(code).toBe(EXIT_OK);
+ expect(calls.map((c) => [c.method, c.url])).toEqual([
+ ['POST', 'https://switch.example.com/v1/management/controllers/controller-1/token'],
+ ['PATCH', 'https://switch.example.com/v1/management/controllers/controller-1'],
+ ]);
+ expect(calls[0]!.body).toEqual({ credential: 'swcc_test' });
+ expect(calls[1]!.body).toEqual({ name: 'build-box', description: null });
+ expect(calls[1]!.auth).toBe('Bearer swct_test');
+ const reopened = ControllerStore.open(join(dir, 'controller.db'));
+ expect(reopened.identity()?.name).toBe('build-box');
+ reopened.close();
+ });
+
+ it('set-info refuses what it cannot send, before calling Switch', async () => {
+ const fetchSpy = vi.fn();
+ vi.stubGlobal('fetch', fetchSpy);
+ expect(await main(['set-info', '--data-dir', dir])).toBe(EXIT_CONFIGURATION);
+ expect(lastLine()).toBe(
+ 'switch-agent-controller: set-info needs --name, --description, or both.'
+ );
+ expect(await main(['set-info', '--name', ' ', '--data-dir', dir])).toBe(EXIT_CONFIGURATION);
+ expect(lastLine()).toBe('switch-agent-controller: --name must not be blank.');
+ expect(await main(['set-info', '--name', 'box', '--data-dir', dir])).toBe(EXIT_CONFIGURATION);
+ expect(lastLine()).toMatch(/holds no enrolled controller/);
+ vi.unstubAllGlobals();
+ expect(fetchSpy).not.toHaveBeenCalled();
+ });
+
+ it('set-info exits 1 when Switch refuses the change', async () => {
+ const store = ControllerStore.open(join(dir, 'controller.db'));
+ store.saveIdentity({
+ controllerId: 'controller-1',
+ server: 'https://switch.example.com',
+ name: 'box',
+ enrolledAt: '2026-01-01T00:00:00.000Z',
+ });
+ store.close();
+ await new FileSecretStore(dataLayout(dir).secrets).set(CONTROLLER_CREDENTIAL, 'swcc_test');
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async (url: string) =>
+ url.endsWith('/token')
+ ? new Response(
+ JSON.stringify({
+ access_token: 'swct_test',
+ expires_at: new Date(Date.now() + 3_600_000).toISOString(),
+ }),
+ { status: 200 }
+ )
+ : new Response(
+ JSON.stringify({
+ error: { code: 'validation_error', message: 'name too long', retryable: false },
+ }),
+ { status: 422 }
+ )
+ )
+ );
+ expect(await main(['set-info', '--name', 'box', '--data-dir', dir])).toBe(EXIT_FAILURE);
+ vi.unstubAllGlobals();
+ expect(lastLine()).toBe('switch-agent-controller: validation_error: name too long');
+ const reopened = ControllerStore.open(join(dir, 'controller.db'));
+ expect(reopened.identity()?.name).toBe('box');
+ reopened.close();
+ });
+
it('exits 2 when it is not enrolled', async () => {
expect(await main(['run', '--data-dir', dir, '--shared-host-bundle', bundle])).toBe(
EXIT_CONFIGURATION
@@ -159,3 +269,16 @@ describe('main', () => {
expect(lastLine()).toMatch(/not enrolled/);
});
});
+
+describe('infoChange', () => {
+ it('trims, clears a blank description, and keeps to the limits', () => {
+ expect(infoChange({ name: ' box ' })).toEqual({ name: 'box' });
+ expect(infoChange({ description: ' ' })).toEqual({ description: null });
+ expect(infoChange({ name: 'x'.repeat(200), description: 'y'.repeat(500) })).toEqual({
+ name: 'x'.repeat(200),
+ description: 'y'.repeat(500),
+ });
+ expect(() => infoChange({ name: 'x'.repeat(201) })).toThrow(/at most 200/);
+ expect(() => infoChange({ description: 'y'.repeat(501) })).toThrow(/at most 500/);
+ });
+});
diff --git a/console/packages/agent-controller/src/main.ts b/console/packages/agent-controller/src/main.ts
index b41b0d9e0..c2fbe0db2 100644
--- a/console/packages/agent-controller/src/main.ts
+++ b/console/packages/agent-controller/src/main.ts
@@ -1,7 +1,14 @@
import { hostname } from 'node:os';
import { parseArgs } from 'node:util';
import packageJson from '../package.json' with { type: 'json' };
-import { ControllerApiError, enroll, normalizeServerUrl, nodeWebSocket } from './api';
+import {
+ AccessTokens,
+ ControllerApiError,
+ ControllerClient,
+ enroll,
+ normalizeServerUrl,
+ nodeWebSocket,
+} from './api';
import { DEFAULT_TIMING, runController } from './controller';
import { DetachedRuntime } from './detached-runtime';
import { ConfigurationError, UsageError } from './errors';
@@ -30,6 +37,7 @@ import {
observeOnDisk,
} from './runtime';
import { AgentRuntimes } from './runtimes';
+import type { ControllerInfoChange } from './schemas';
import { CONTROLLER_CREDENTIAL, FileSecretStore, MemorySecretStore } from './secrets';
import { contractPlatform, mapAgentProcess, PathProviderLocator } from './status';
import { ControllerStore } from './store';
@@ -38,6 +46,9 @@ export const VERSION: string = packageJson.version;
const SIGNALS = ['SIGINT', 'SIGTERM'] as const;
+const MAX_NAME = 200;
+const MAX_DESCRIPTION = 500;
+
const USAGE = `Usage: switch-agent-controller <command> [options]
Commands:
@@ -54,6 +65,10 @@ Commands:
the data directory holds none, and move the same identity to a new
server URL when it holds that one. --credential-stdin reads the
controller credential from stdin and keeps it in memory only.
+ set-info [--name <name>] [--description <text>] [--data-dir <dir>]
+ Rename this machine and/or change its description on the server, with
+ this controller's own credential. Either or both; --description ""
+ clears the description. The same limits as at enrollment.
status [--data-dir <dir>] [--shared-host-bundle <path>]
Show this controller's identity and its agents, from local state only.
@@ -98,8 +113,8 @@ async function enrollCommand(args: string[]): Promise<number> {
const server = normalizeServerUrl(values.server);
const name = values.name ?? hostname();
const description = values.description?.trim() || undefined;
- if (description !== undefined && description.length > 500)
- throw new UsageError('--description must be at most 500 characters.');
+ if (description !== undefined && description.length > MAX_DESCRIPTION)
+ throw new UsageError(`--description must be at most ${MAX_DESCRIPTION} characters.`);
const { dataDir, store, secrets } = await openState(values['data-dir']);
try {
const existing = store.identity();
@@ -134,6 +149,82 @@ async function enrollCommand(args: string[]): Promise<number> {
}
}
+/** The change `set-info` asks for, checked as the server checks it. */
+export function infoChange(values: { name?: string; description?: string }): ControllerInfoChange {
+ if (values.name === undefined && values.description === undefined)
+ throw new UsageError('set-info needs --name, --description, or both.');
+ const change: ControllerInfoChange = {};
+ if (values.name !== undefined) {
+ const name = values.name.trim();
+ if (!name) throw new UsageError('--name must not be blank.');
+ if (name.length > MAX_NAME)
+ throw new UsageError(`--name must be at most ${MAX_NAME} characters.`);
+ change.name = name;
+ }
+ if (values.description !== undefined) {
+ const description = values.description.trim();
+ if (description.length > MAX_DESCRIPTION)
+ throw new UsageError(`--description must be at most ${MAX_DESCRIPTION} characters.`);
+ change.description = description || null;
+ }
+ return change;
+}
+
+async function setInfoCommand(args: string[]): Promise<number> {
+ const { values } = parseArgs({
+ args,
+ options: {
+ name: { type: 'string' },
+ description: { type: 'string' },
+ 'data-dir': { type: 'string' },
+ },
+ strict: true,
+ });
+ const change = infoChange(values);
+ const { dataDir, store, secrets } = await openState(values['data-dir']);
+ try {
+ const identity = store.identity();
+ if (!identity)
+ throw new ConfigurationError(`${dataDir} holds no enrolled controller. Enroll it first.`);
+ if (store.revokedAt())
+ throw new ConfigurationError(
+ `Controller ${identity.controllerId} was revoked; it has to be enrolled again.`
+ );
+ const credential = await secrets.get(CONTROLLER_CREDENTIAL);
+ if (!credential)
+ throw new ConfigurationError(
+ `${dataDir} holds no controller credential (it is handed over at run time, or was removed). Change the name and description in the gateway's Machines page instead.`
+ );
+ const log = createLogger({
+ level: process.env.SWITCH_CONTROLLER_LOG_LEVEL,
+ write: (line) => process.stderr.write(line),
+ });
+ const client = new ControllerClient({
+ fetch,
+ server: identity.server,
+ controllerId: identity.controllerId,
+ version: VERSION,
+ tokens: new AccessTokens({
+ fetch,
+ server: identity.server,
+ controllerId: identity.controllerId,
+ credential: async () => credential,
+ now: Date.now,
+ log,
+ }),
+ openWebSocket: nodeWebSocket,
+ });
+ const updated = await client.updateInfo(change);
+ store.saveName(updated.name);
+ process.stdout.write(
+ `Controller ${updated.id} is now "${updated.name}"${updated.description ? `: ${updated.description}` : ', with no description'}.\n`
+ );
+ return EXIT_OK;
+ } finally {
+ store.close();
+ }
+}
+
async function runCommand(args: string[]): Promise<number> {
const { values } = parseArgs({
args,
@@ -310,6 +401,8 @@ export async function main(argv: string[]): Promise<number> {
return await enrollCommand(rest);
case 'run':
return await runCommand(rest);
+ case 'set-info':
+ return await setInfoCommand(rest);
case 'status':
return await statusCommand(rest);
case undefined:
diff --git a/console/packages/agent-controller/src/schemas.test.ts b/console/packages/agent-controller/src/schemas.test.ts
index 0b1b20b56..38d7b5716 100644
--- a/console/packages/agent-controller/src/schemas.test.ts
+++ b/console/packages/agent-controller/src/schemas.test.ts
@@ -7,6 +7,8 @@ import {
controllerConnectionRequestSchema,
controllerPongSchema,
controllerConnectionResponseSchema,
+ controllerInfoRequestSchema,
+ controllerInfoResponseSchema,
credentialRotateResponseSchema,
enrollRequestSchema,
enrollResponseSchema,
@@ -48,6 +50,8 @@ const SCHEMA_FOR: Record<string, z.ZodType> = {
'controller_pong.json': controllerPongSchema,
'controller_connection_request.json': controllerConnectionRequestSchema,
'controller_connection_response.json': controllerConnectionResponseSchema,
+ 'controller_info_request.json': controllerInfoRequestSchema,
+ 'controller_info_response.json': controllerInfoResponseSchema,
'credential_rotate_response.json': credentialRotateResponseSchema,
'enroll_request.json': enrollRequestSchema,
'enroll_response.json': enrollResponseSchema,
diff --git a/console/packages/agent-controller/src/schemas.ts b/console/packages/agent-controller/src/schemas.ts
index a4f4571b4..4bd3a0b97 100644
--- a/console/packages/agent-controller/src/schemas.ts
+++ b/console/packages/agent-controller/src/schemas.ts
@@ -110,6 +110,21 @@ export type TokenResponse = z.infer<typeof tokenResponseSchema>;
export const credentialRotateResponseSchema = z.object({ credential: id });
+/** Renaming this machine, or changing its description: either or both; `description: null` clears it. */
+export const controllerInfoRequestSchema = z.object({
+ name: z.string().min(1).max(200).optional(),
+ description: z.string().max(500).nullable().optional(),
+});
+export type ControllerInfoChange = z.infer<typeof controllerInfoRequestSchema>;
+
+/** The machine as its owner's list shows it, after the change; only what the controller reads. */
+export const controllerInfoResponseSchema = z.object({
+ id,
+ name: z.string().min(1),
+ description: z.string().nullable(),
+});
+export type ControllerInfo = z.infer<typeof controllerInfoResponseSchema>;
+
// §2 Assignment (v1 definition)
export const agentDefinitionSchema = z.object({
diff --git a/console/packages/agent-controller/src/store.test.ts b/console/packages/agent-controller/src/store.test.ts
index f711061b1..bce76aead 100644
--- a/console/packages/agent-controller/src/store.test.ts
+++ b/console/packages/agent-controller/src/store.test.ts
@@ -96,6 +96,21 @@ describe('ControllerStore', () => {
store.close();
});
+ it('records a new name for the identity and keeps the rest', () => {
+ const store = ControllerStore.open(path);
+ expect(() => store.saveName('renamed')).toThrow(/no identity/);
+ const identity = {
+ controllerId: 'controller-1',
+ server: 'https://switch.example.com',
+ name: 'build-box',
+ enrolledAt: '2026-01-01T00:00:00.000Z',
+ };
+ store.saveIdentity(identity);
+ store.saveName('renamed');
+ expect(store.identity()).toEqual({ ...identity, name: 'renamed' });
+ store.close();
+ });
+
it('caches the assignment with its ETag', () => {
const store = ControllerStore.open(path);
expect(store.cachedAssignment()).toEqual({ kind: 'none' });
diff --git a/console/packages/agent-controller/src/store.ts b/console/packages/agent-controller/src/store.ts
index 1228cc510..1359fc3fb 100644
--- a/console/packages/agent-controller/src/store.ts
+++ b/console/packages/agent-controller/src/store.ts
@@ -135,6 +135,13 @@ export class ControllerStore {
});
}
+ /** Records the name the server now has for this controller. */
+ saveName(name: string): void {
+ if (!this.meta('controller_id'))
+ throw new Error('There is no identity whose name could be changed.');
+ this.setMeta('name', name);
+ }
+
/** Moves the identity to another server URL, keeping everything else. */
saveServer(server: string): void {
if (!this.meta('controller_id'))
diff --git a/console/packages/agent-controller/src/testing/fake-core.ts b/console/packages/agent-controller/src/testing/fake-core.ts
index a10238e0e..d741a84a2 100644
--- a/console/packages/agent-controller/src/testing/fake-core.ts
+++ b/console/packages/agent-controller/src/testing/fake-core.ts
@@ -102,6 +102,8 @@ export class FakeCore {
readonly results = new Map<string, OperationResult>();
readonly opens: Record<string, AgentCursor>[] = [];
readonly beats: Record<string, number>[] = [];
+ /** The machine's name and description, as `PATCH .../controllers/{id}` changes them. */
+ info: { name: string; description: string | null } = { name: 'laptop', description: null };
/** Answers the next request to a path with this, once. */
readonly scripted: { method: string; path: string; status: number; body: unknown }[] = [];
/** What `GET .../media` sends, in these chunks; `waitBetween` holds back all but the first. */
@@ -398,6 +400,12 @@ export class FakeCore {
agents: this.bound(),
});
}
+ if (method === 'PATCH' && url.pathname === base) {
+ const change = body as { name?: string; description?: string | null };
+ if (change.name !== undefined) this.info.name = change.name;
+ if (change.description !== undefined) this.info.description = change.description;
+ return this.json(res, 200, { id: this.controllerId, ...this.info, state: 'online' });
+ }
if (method === 'GET' && url.pathname === `${base}/assignment`) {
const etag = `"${this.assignment.revision}"`;
if (req.headers['if-none-match'] === etag) {
diff --git a/console/packages/plugins/src/switch-skill/SKILL.md b/console/packages/plugins/src/switch-skill/SKILL.md
index e000d5668..bb5172af2 100644
--- a/console/packages/plugins/src/switch-skill/SKILL.md
+++ b/console/packages/plugins/src/switch-skill/SKILL.md
@@ -666,8 +666,9 @@ tools exist, and only there — if they are not in your tool list, this server
does not manage agents, so say so rather than looking for another way.
- **`list_machines`** — your owner's machines: `id`, `name`, `description`,
- `state` (`online` or `unknown`), the providers each has (`installed`,
- `auth`) and how many agents it is running. Removed machines are left out.
+ `state` (`online`, `offline`, or `unknown` if it never connected), the
+ providers each has (`installed`, `auth`) and how many agents it is running.
+ Removed machines are left out.
- **`get_advanced_config`** — the advanced settings a provider's agents can
carry (the "Advanced configuration" Switch Console shows): each field's
`key`, `label`, `type` (`text`, `textarea`, `number`, `boolean`, `list` of
diff --git a/core/switch_core/bridges/agent/operations/agent_management.py b/core/switch_core/bridges/agent/operations/agent_management.py
index 407f35777..af6f36ca9 100644
--- a/core/switch_core/bridges/agent/operations/agent_management.py
+++ b/core/switch_core/bridges/agent/operations/agent_management.py
@@ -123,8 +123,9 @@ async def list_machines() -> list[dict[str, Any]]:
A list of machines, each {id, name, description, kind, state,
last_seen_at, providers, agents_running}.
`kind` is "console" (Switch Console's own machine), "daemon" (a
- headless controller) or "ec2". `state` is "online" (reporting now) or
- "unknown" (has not reported recently: asleep, off or disconnected).
+ headless controller) or "ec2". `state` is "online" (connected to Switch
+ now), "offline" (was connected, and is not now: stopped, asleep, off
+ or disconnected) or "unknown" (has never connected).
`providers` lists the agent CLIs the machine last reported, each
{provider, installed, version, auth}, `auth` being "ok", "missing",
"expired" or "unknown". `agents_running` counts the managed agents
@@ -260,7 +261,7 @@ async def list_managed_agents() -> list[dict[str, Any]]:
`directory` is the working directory the definition names (null only
when its machine has not said where it keeps workspaces).
`machine` is {id, name, state} (null when the agent is not placed on
- a machine), `state` being "online", "unknown" or "revoked".
+ a machine), `state` being "online", "offline", "unknown" or "revoked".
`desired_state` is what your owner wants: "running" or "stopped".
`actual` is what the machine last reported for the agent,
{process, reason, detail, applied_revision, since, directory}, or
diff --git a/core/switch_core/bridges/agent/protocol/controller_presence.py b/core/switch_core/bridges/agent/protocol/controller_presence.py
index c59cfea84..d5df3250a 100644
--- a/core/switch_core/bridges/agent/protocol/controller_presence.py
+++ b/core/switch_core/bridges/agent/protocol/controller_presence.py
@@ -19,7 +19,12 @@ the bearer middleware's act-as check, and the controller stream.
Like the connection registry it is memory only. A restart forgets every
controller connection, and Management loads the bindings again before the
-agent bridge serves.
+agent bridge serves. Each controller connection's transitions, its socket
+attaching and its socket going (and why), are also handed to a ledger, which
+Management installs (`use_ledger`) and which persists them, so that whether
+a machine is connected can be read from any process. Heartbeats are not:
+they stay here. The ledger is told synchronously, as each transition happens,
+once per transition.
"""
from __future__ import annotations
@@ -32,7 +37,7 @@ import uuid
from collections import deque
from collections.abc import Callable, Iterable
from dataclasses import dataclass, field, replace
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Protocol
from switch_core.bridges.agent.protocol.liveness import (
HEARTBEAT_LAPSED,
@@ -54,6 +59,13 @@ REVOKED = Closure(
code="closed", message="the controller has been revoked", room_id=None
)
+# Why a connection's socket went, as the ledger records it. A connection
+# closed with a closure records the closure's code (`heartbeat_lapsed`,
+# `taken_over`) instead.
+DISCONNECT_REVOKED = "revoked"
+DISCONNECT_SOCKET_CLOSED = "socket_closed"
+DISCONNECT_SERVER_SHUTDOWN = "server_shutdown"
+
# How many superseded connection ids a controller is remembered by, so a late
# beat from one is told `taken_over` rather than `unknown_connection`.
_SUPERSEDED_REMEMBERED = 16
@@ -162,6 +174,20 @@ class ControllerConnection:
)
+class ControllerConnectionLedger(Protocol):
+ """Where controller connections' transitions are persisted.
+
+ Called as each happens, once each, and must not block. `connected`: a
+ socket attached to a connection that had none. `disconnected`: the
+ socket went, with the reason to record: `socket_closed`,
+ `server_shutdown`, `heartbeat_lapsed`, `taken_over` or `revoked`.
+ """
+
+ def connected(self, conn: ControllerConnection) -> None: ...
+
+ def disconnected(self, conn: ControllerConnection, reason: str) -> None: ...
+
+
class ControllerPresence:
def __init__(self, *, on_bound: Callable[[str], None]) -> None:
# Called with each agent that becomes controller-backed, so the
@@ -177,12 +203,35 @@ class ControllerPresence:
self._rooms: dict[str, set[str]] = {}
self._next_generation = secrets.randbits(32)
self._auth_cache: ControllerAuthCache | None = None
+ self._ledger: ControllerConnectionLedger | None = None
+ # Set as the process begins shutting down: a socket going after that
+ # went because the server closed it.
+ self._shutting_down = False
def use_auth_cache(self, cache: ControllerAuthCache) -> None:
"""The cache of controller-token reads to keep in step with the
bindings and revocations recorded here."""
self._auth_cache = cache
+ def use_ledger(self, ledger: ControllerConnectionLedger) -> None:
+ """Where each connection's socket attaching and going are persisted."""
+ self._ledger = ledger
+
+ def begin_shutdown(self) -> None:
+ """The process is shutting down: sockets that go from now are
+ recorded as closed by the server."""
+ self._shutting_down = True
+
+ def _record_connected(self, conn: ControllerConnection) -> None:
+ if self._ledger is not None:
+ self._ledger.connected(conn)
+
+ def _record_disconnected(self, conn: ControllerConnection, reason: str) -> None:
+ """Record the socket going, if one was attached: a connection whose
+ socket already went was recorded then."""
+ if conn.stream_attached and self._ledger is not None:
+ self._ledger.disconnected(conn, reason)
+
def _invalidate_agent_auth(self, agent_id: str) -> None:
if self._auth_cache is not None:
self._auth_cache.invalidate_agent(agent_id)
@@ -264,6 +313,7 @@ class ControllerPresence:
conn = self._connections.pop(controller_id, None)
if conn is not None:
conn.closure = REVOKED
+ self._record_disconnected(conn, DISCONNECT_REVOKED)
conn.stream_attached = False
conn.wake.set()
@@ -399,6 +449,7 @@ class ControllerPresence:
previous = self._connections.get(controller_id)
if previous is not None:
previous.closure = TAKEN_OVER
+ self._record_disconnected(previous, TAKEN_OVER.code)
previous.stream_attached = False
previous.wake.set()
remembered = self._superseded.setdefault(
@@ -448,8 +499,12 @@ class ControllerPresence:
"""Attach a stream to the connection, displacing any attached before.
Attaching counts as a beat: the stream is what the beat proves alive,
- and the client cannot beat before it has one.
+ and the client cannot beat before it has one. A connection that had
+ no socket attached is recorded as connected; displacing an attached
+ one changes nothing the ledger holds.
"""
+ if not conn.stream_attached:
+ self._record_connected(conn)
conn.stream_token += 1
conn.stream_attached = True
conn.last_beat = time.monotonic()
@@ -457,7 +512,16 @@ class ControllerPresence:
return conn.stream_token
def detach_stream(self, conn: ControllerConnection, token: int) -> None:
+ """The socket holding this stream went. The connection stays open for
+ the socket to reattach within the heartbeat TTL, but the machine is
+ recorded as disconnected now."""
if conn.stream_token == token:
+ self._record_disconnected(
+ conn,
+ DISCONNECT_SERVER_SHUTDOWN
+ if self._shutting_down
+ else DISCONNECT_SOCKET_CLOSED,
+ )
conn.stream_attached = False
def beat(
@@ -477,6 +541,10 @@ class ControllerPresence:
if current is None or cursor > current:
conn.resume_cursors[agent_id] = cursor
+ def current_connection(self, controller_id: str) -> ControllerConnection | None:
+ """The controller's open connection, live or not yet swept."""
+ return self._connections.get(controller_id)
+
def is_current(self, conn: ControllerConnection) -> bool:
return self._connections.get(conn.controller_id) is conn
@@ -528,6 +596,7 @@ class ControllerPresence:
closure.code,
conn.beats,
)
+ self._record_disconnected(conn, closure.code)
conn.stream_attached = False
conn.wake.set()
diff --git a/core/switch_core/db/models.py b/core/switch_core/db/models.py
index 063502810..483927269 100644
--- a/core/switch_core/db/models.py
+++ b/core/switch_core/db/models.py
@@ -2392,6 +2392,37 @@ class TelemetrySnapshotWatermark(Base):
)
+# ── Switch-core processes ────────────────────────────────────────────────────
+
+
+class SwitchCoreProcess(Base):
+ """A running switch-core process's lease (`management/process_lease.py`).
+
+ Each process holding controller sockets claims a row at startup, renews
+ `beat_at` every few seconds, and sets `stopped_at` as it shuts down. A
+ controller connection whose holding process's lease is stale, stopped or
+ gone reads as offline, which is how a process that died without writing
+ its connections' closings is noticed. One write per process per renewal,
+ however many machines it holds.
+
+ Not tenant-scoped: a process serves every tenant. Rows long stale are
+ pruned by the live processes.
+ """
+
+ __tablename__ = "switch_core_processes"
+
+ id: Mapped[str] = mapped_column(Text, primary_key=True)
+ started_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), server_default=func.now(), nullable=False
+ )
+ beat_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), server_default=func.now(), nullable=False
+ )
+ stopped_at: Mapped[datetime | None] = mapped_column(
+ DateTime(timezone=True), nullable=True
+ )
+
+
# ── Feature flags ────────────────────────────────────────────────────────────
@@ -3084,6 +3115,17 @@ class AgentController(TenantScoped, Base):
`status_seq` its sequence number: a report with a sequence at or below it is
ignored. `assignment_revision` bumps on every change to the set of agents
the controller should run, and is what its ETag carries.
+
+ The `connection_*`, `connected_at` and `disconnect*` columns are the
+ controller's socket as the switch-core process holding it last recorded it
+ (`management/connection_ledger.py`): which connection, which process holds
+ it (`connection_process_id`, a `switch_core_processes` row), when its
+ socket attached, and when and why it went. Only transitions are written,
+ never heartbeats; a process that dies without writing the closing is
+ caught by its lease in `switch_core_processes` going stale. They describe
+ the current connection, or the last one once it has closed; a new one
+ replaces them. Every replica reads them, so whether the machine is
+ connected does not depend on which process holds its socket.
"""
__tablename__ = "agent_controllers"
@@ -3123,6 +3165,19 @@ class AgentController(TenantScoped, Base):
revoked_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
+ connection_id: Mapped[str | None] = mapped_column(Text, nullable=True)
+ # Not a foreign key: a lease row is pruned once long stale, and a
+ # connection naming a process with no lease reads as lost.
+ connection_process_id: Mapped[str | None] = mapped_column(Text, nullable=True)
+ connected_at: Mapped[datetime | None] = mapped_column(
+ DateTime(timezone=True), nullable=True
+ )
+ disconnected_at: Mapped[datetime | None] = mapped_column(
+ DateTime(timezone=True), nullable=True
+ )
+ # `socket_closed`, `heartbeat_lapsed`, `taken_over`, `revoked` or
+ # `server_shutdown`.
+ disconnect_reason: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), nullable=False
)
diff --git a/core/switch_core/db/rls_ddl.py b/core/switch_core/db/rls_ddl.py
index d2667b52f..b45dd8bc4 100644
--- a/core/switch_core/db/rls_ddl.py
+++ b/core/switch_core/db/rls_ddl.py
@@ -127,6 +127,11 @@ POLICY_NAME = "tenant_isolation"
# which reports counts and never an identifier — and scoping them would be
# incoherent: a deployment running three tenants has one identity, not three,
# and a milestone reported once per tenant would not be once-ever at all.
+#
+# `switch_core_processes` is each running switch-core process's lease: its id,
+# when it started, its last renewal and when it stopped. A process holds
+# controller sockets for every tenant at once, so its lease belongs to none of
+# them, and it holds no customer data, only that the process is alive.
GLOBAL_TABLES = frozenset(
{
"users",
@@ -135,6 +140,7 @@ GLOBAL_TABLES = frozenset(
"deployment_identity",
"telemetry_milestones",
"telemetry_snapshot_watermark",
+ "switch_core_processes",
}
)
diff --git a/core/switch_core/db/stores/agent_controller_store.py b/core/switch_core/db/stores/agent_controller_store.py
index e142757cb..0eadbb757 100644
--- a/core/switch_core/db/stores/agent_controller_store.py
+++ b/core/switch_core/db/stores/agent_controller_store.py
@@ -164,6 +164,72 @@ class AgentControllerStore:
)
return result.scalar_one_or_none() is not None
+ async def record_connected(
+ self,
+ session: AsyncSession,
+ tenant_id: str,
+ controller_id: str,
+ *,
+ connection_id: str,
+ process_id: str,
+ connected_at: datetime,
+ ) -> bool:
+ """Record that the controller's socket attached to this connection,
+ held by this process. It replaces whatever the row held: the process
+ holding the live socket is the authority. Returns whether the
+ controller's row exists. `updated_at` is left alone: connecting is
+ not an edit."""
+ result = await session.execute(
+ update(AgentController)
+ .where(
+ AgentController.tenant_id == tenant_id,
+ AgentController.id == controller_id,
+ )
+ .values(
+ connection_id=connection_id,
+ connection_process_id=process_id,
+ connected_at=connected_at,
+ disconnected_at=None,
+ disconnect_reason=None,
+ updated_at=AgentController.updated_at,
+ )
+ .returning(AgentController.id)
+ .execution_options(synchronize_session=False)
+ )
+ return result.scalar_one_or_none() is not None
+
+ async def record_disconnected(
+ self,
+ session: AsyncSession,
+ tenant_id: str,
+ controller_id: str,
+ *,
+ connection_id: str,
+ disconnected_at: datetime,
+ reason: str,
+ ) -> bool:
+ """Record that the controller's socket on this connection went, and
+ why. Written only while the row still names this connection, so a
+ process closing a connection the controller has since replaced
+ through another process leaves the replacement standing. Returns
+ whether the row was written."""
+ result = await session.execute(
+ update(AgentController)
+ .where(
+ AgentController.tenant_id == tenant_id,
+ AgentController.id == controller_id,
+ AgentController.connection_id == connection_id,
+ )
+ .values(
+ disconnected_at=disconnected_at,
+ disconnect_reason=reason,
+ updated_at=AgentController.updated_at,
+ )
+ .returning(AgentController.id)
+ .execution_options(synchronize_session=False)
+ )
+ return result.scalar_one_or_none() is not None
+
async def set_credential(
self,
session: AsyncSession,
diff --git a/core/switch_core/db/stores/switch_core_process_store.py b/core/switch_core/db/stores/switch_core_process_store.py
new file mode 100644
index 000000000..f2e799a5e
--- /dev/null
+++ b/core/switch_core/db/stores/switch_core_process_store.py
@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+
+from sqlalchemy import delete, func, select, update
+from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from switch_core.db.models import SwitchCoreProcess
+
+
+@dataclass(frozen=True)
+class ProcessLeaseRow:
+ beat_at: datetime
+ stopped_at: datetime | None
+
+
+class SwitchCoreProcessStore:
+ """Each switch-core process's lease (`switch_core_processes`), a global
+ table.
+
+ Every time here is the database's own clock, so leases written by one
+ process and read by another are compared on one clock whatever the two
+ hosts' clocks say.
+ """
+
+ async def renew(self, session: AsyncSession, process_id: str) -> None:
+ """Claim the lease, or renew it: its beat is now. A row pruned while
+ this process could not reach the database is claimed again."""
+ await session.execute(
+ insert(SwitchCoreProcess)
+ .values(id=process_id, started_at=func.now(), beat_at=func.now())
+ .on_conflict_do_update(
+ index_elements=[SwitchCoreProcess.id],
+ set_={"beat_at": func.now(), "stopped_at": None},
+ )
+ )
+
+ async def stop(self, session: AsyncSession, process_id: str) -> None:
+ await session.execute(
+ update(SwitchCoreProcess)
+ .where(SwitchCoreProcess.id == process_id)
+ .values(stopped_at=func.now())
+ )
+
+ async def prune(self, session: AsyncSession, older_than: timedelta) -> int:
+ """Delete leases whose last beat is older than `older_than`. Returns
+ how many."""
+ result = await session.execute(
+ delete(SwitchCoreProcess)
+ .where(SwitchCoreProcess.beat_at < func.now() - older_than)
+ .returning(SwitchCoreProcess.id)
+ )
+ return len(result.all())
+
+ async def read(
+ self, session: AsyncSession
+ ) -> tuple[datetime, dict[str, ProcessLeaseRow]]:
+ """The database's now, and every lease as it stands then."""
+ now = (await session.execute(select(func.now()))).scalar_one()
+ rows = await session.execute(
+ select(
+ SwitchCoreProcess.id,
+ SwitchCoreProcess.beat_at,
+ SwitchCoreProcess.stopped_at,
+ )
+ )
+ return now, {
+ process_id: ProcessLeaseRow(beat_at=beat_at, stopped_at=stopped_at)
+ for process_id, beat_at, stopped_at in rows.all()
+ }
diff --git a/core/switch_core/main.py b/core/switch_core/main.py
index 8c59e09c0..7103bd319 100644
--- a/core/switch_core/main.py
+++ b/core/switch_core/main.py
@@ -32,6 +32,7 @@ from switch_core.bridges.agent.protocol.agent_connections import (
AgentConnectionRegistry,
)
from switch_core.bridges.agent.protocol.agent_core import AgentCore
+from switch_core.bridges.agent.protocol.controller_presence import ControllerPresence
from switch_core.bridges.agent.protocol.event_buffer import EventBuffer
from switch_core.bridges.agent.registration_bootstrap import (
BOOTSTRAP_KEY_LABEL,
@@ -144,7 +145,7 @@ from switch_core.gateway.app import create_gateway_app
from switch_core.gateway.auth import hash_password
from switch_core.gateway.invite_mail import SmtpInviteMailer
from switch_core.logging_config import configure_logging
-from switch_core.management.wiring import create_management
+from switch_core.management.wiring import Management, create_management
from switch_core.messages.notify import MessageListener
from switch_core.observability.bootstrap import (
Observability,
@@ -258,6 +259,32 @@ async def _connection_sweep_loop(protocol: AgentCore, lag: EventLoopLag) -> None
logger.exception("AgentConnection sweep failed")
+# Under `_FORCED_EXIT_GRACE_SECONDS`, and first in the teardown: what it
+# writes is what lets every other process see this one's machines go offline
+# now rather than when its lease lapses.
+_MANAGEMENT_STOP_SECONDS = 1.0
+
+
+async def _stop_management(management: Management) -> None:
+ """Record the controller sockets that went and mark this process's lease
+ stopped. Never raises: if it fails, the lease lapses on its own."""
+ try:
+ async with asyncio.timeout(_MANAGEMENT_STOP_SECONDS):
+ await management.stop()
+ except TimeoutError:
+ logger.warning(
+ "Gave up recording controller disconnections after %.1fs; their "
+ "machines read offline once this process's lease lapses.",
+ _MANAGEMENT_STOP_SECONDS,
+ )
+ except Exception:
+ logger.warning(
+ "Recording controller disconnections at shutdown failed; their "
+ "machines read offline once this process's lease lapses.",
+ exc_info=True,
+ )
+
+
# The innermost of three nested budgets: under
# `observability.logs.SHUTDOWN_FLUSH_SECONDS`, itself under
# `_FORCED_EXIT_GRACE_SECONDS`.
@@ -823,6 +850,7 @@ async def run(config: SwitchConfig) -> None:
# Before the bridge serves: until Core knows which agents a controller
# runs, their own keys would be let in and their presence misread.
await management.load_bindings()
+ await management.start()
agent_bridge_app.mount("/gateway", gateway_app)
@@ -875,6 +903,11 @@ async def run(config: SwitchConfig) -> None:
connection_sweep_task = asyncio.create_task(
_connection_sweep_loop(protocol, observability.lag)
)
+ management_task = (
+ asyncio.create_task(management.run())
+ if management is not None
+ else None
+ )
# Only when telemetry is on: the chart tells a customer that off
# means nothing is collected, and the fan-out is not free.
snapshot_task = (
@@ -892,6 +925,10 @@ async def run(config: SwitchConfig) -> None:
connection_sweep_task.cancel()
if snapshot_task is not None:
snapshot_task.cancel()
+ if management is not None:
+ assert management_task is not None
+ management_task.cancel()
+ await _stop_management(management)
await message_listener.stop()
await session_activity_listener.stop()
# Before the operational flush, and bounded: the whole
@@ -982,6 +1019,7 @@ async def run(config: SwitchConfig) -> None:
lambda: asyncio.create_task(
_shutdown(
server,
+ connections.controllers,
client_lifecycle,
collab_lifecycle,
connector_lifecycle,
@@ -1429,6 +1467,7 @@ async def _bootstrap_key_tenant(
async def _shutdown(
server: uvicorn.Server,
+ controllers: ControllerPresence,
client_lifecycle: ClientLifecycleService,
collab_lifecycle: CollaborationBridgeLifecycleService,
connector_lifecycle: ServerSideConnectorLifecycleService,
@@ -1437,6 +1476,9 @@ async def _shutdown(
discord_gateway_task: asyncio.Task[None] | None,
) -> None:
logger.info("Shutting down...")
+ # Before uvicorn closes the sockets, so the controllers' are recorded as
+ # closed by the server.
+ controllers.begin_shutdown()
server.should_exit = True
await connector_lifecycle.stop_all()
await collab_lifecycle.stop_all()
diff --git a/core/switch_core/management/agent_operations.py b/core/switch_core/management/agent_operations.py
index 40549b808..d6f6ff3df 100644
--- a/core/switch_core/management/agent_operations.py
+++ b/core/switch_core/management/agent_operations.py
@@ -29,7 +29,8 @@ from switch_core.db.session_scope import tenant_session
from switch_core.management import reason_codes
from switch_core.management.advanced_config import provider_fields
from switch_core.management.errors import ManagementError
-from switch_core.management.placement import provider_auth
+from switch_core.management.placement import is_revoked, provider_auth
+from switch_core.management.process_lease import ProcessLeases
from switch_core.management.schemas import (
CreateManagedAgentRequest,
DefinitionV1,
@@ -65,9 +66,9 @@ def _placement_sentence(code: str, machine: str, provider: str) -> str | None:
f"machine '{machine}' has been removed from Switch; pick another machine"
),
reason_codes.CONTROLLER_OFFLINE: (
- f"machine '{machine}' has not reported recently (it may be asleep, "
- "off or disconnected); your owner needs to bring it back online, or "
- "pick another machine"
+ f"machine '{machine}' is not connected to Switch or has not reported "
+ "recently (it may be asleep, off or disconnected); your owner needs "
+ "to bring it back online, or pick another machine"
),
reason_codes.PROVIDER_NOT_INSTALLED: (
f"{provider} is not installed on machine '{machine}'; your owner "
@@ -136,9 +137,10 @@ class ManagementAgentOperations:
controllers = await self._service.controllers.list_for_owner(
session, tenant_id, owner_id
)
+ leases = await self._service.leases(session)
machines = []
for controller in controllers:
- state = self._service.state_of(controller)
+ state = self._service.state_of(controller, leases)
if state == "revoked":
continue
machines.append(
@@ -180,11 +182,7 @@ class ManagementAgentOperations:
by_id = next((c for c in controllers if c.id == machine), None)
if by_id is not None:
return by_id
- named = [
- c
- for c in controllers
- if c.name == machine and self._service.state_of(c) != "revoked"
- ]
+ named = [c for c in controllers if c.name == machine and not is_revoked(c)]
if len(named) == 1:
return named[0]
if not named:
@@ -193,8 +191,9 @@ class ManagementAgentOperations:
f"{nothing_done}: your owner has no machine with the id or "
f"name '{machine}'. list_machines shows the machines you can use.",
)
+ leases = await self._service.leases(session)
candidates = "; ".join(
- _machine_line(c, self._service.state_of(c)) for c in named
+ _machine_line(c, self._service.state_of(c, leases)) for c in named
)
raise AgentManagementRefused(
reason_codes.VALIDATION_ERROR,
@@ -259,6 +258,7 @@ class ManagementAgentOperations:
row: AgentDefinitionRow,
agent: Agent,
controller: AgentController | None,
+ leases: ProcessLeases,
) -> dict[str, Any]:
status = agent_status_from(controller, agent.id)
return {
@@ -275,7 +275,7 @@ class ManagementAgentOperations:
else {
"id": controller.id,
"name": controller.name,
- "state": self._service.state_of(controller),
+ "state": self._service.state_of(controller, leases),
},
"desired_state": row.desired_state,
"actual": None
@@ -304,11 +304,13 @@ class ManagementAgentOperations:
rows = await self._service.definitions.list_for_owner(
session, tenant_id, owner_id
)
+ leases = await self._service.leases(session)
return [
self._managed_entry(
row,
agent,
controllers.get(row.controller_id) if row.controller_id else None,
+ leases,
)
for row, agent in rows
]
@@ -335,7 +337,9 @@ class ManagementAgentOperations:
if row.controller_id is not None
else None
)
- return self._managed_entry(row, agent, controller)
+ return self._managed_entry(
+ row, agent, controller, await self._service.leases(session)
+ )
async def update_managed_agent(
self,
diff --git a/core/switch_core/management/connection_ledger.py b/core/switch_core/management/connection_ledger.py
new file mode 100644
index 000000000..a00b0ab9e
--- /dev/null
+++ b/core/switch_core/management/connection_ledger.py
@@ -0,0 +1,177 @@
+"""Persisting each controller connection's transitions, so any process can
+tell whether the machine is connected.
+
+Core's `ControllerPresence` decides when a controller's socket attaches to
+its connection and when it goes, in the memory of the process holding it,
+and tells this ledger as each happens. Heartbeats stay there: nothing here is
+written per beat. The ledger writes each transition to the controller's row
+once:
+
+- connected: `connection_id`, `connection_process_id` (this process),
+ `connected_at`, and the previous closing cleared;
+- disconnected: `disconnected_at` and `disconnect_reason` (`socket_closed`,
+ `server_shutdown`, `heartbeat_lapsed`, `taken_over`, `revoked`).
+
+That row, and the lease of the process it names (`process_lease.py`), are
+what a machine's state is read from (`placement.controller_state`).
+
+Which write wins, across processes: a connecting comes from the process
+holding the live socket, and replaces whatever the row holds. A closing is
+only written while the row still names that connection, so a process closing
+a connection the controller has since replaced through another process
+leaves the replacement standing. (Clocks are not compared: two processes'
+clocks need not agree.)
+
+Writes are queued, latest per controller, and written by one task (`run`),
+woken by each transition, so a socket attaching or going never waits on the
+database. A write that fails is kept, unless something newer for the same
+controller has arrived meanwhile, and retried.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import datetime
+
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from switch_core.bridges.agent.protocol.controller_presence import (
+ ControllerConnection,
+)
+from switch_core.db.session_scope import tenant_session
+from switch_core.db.stores.agent_controller_store import AgentControllerStore
+
+logger = logging.getLogger(__name__)
+
+# How long a failed write waits before it is tried again.
+RETRY_SECONDS = 2.0
+
+
+@dataclass(frozen=True)
+class _Transition:
+ tenant_id: str
+ controller_id: str
+ connection_id: str
+ at: datetime
+ # None for a connecting; why the socket went for a closing.
+ disconnect_reason: str | None
+
+
+class ControllerConnectionLedger:
+ """Core's `ControllerConnectionLedger`, written to `agent_controllers`."""
+
+ def __init__(
+ self,
+ *,
+ process_id: str,
+ session_factory: async_sessionmaker[AsyncSession],
+ controllers: AgentControllerStore,
+ clock: Callable[[], datetime],
+ ) -> None:
+ self._process_id = process_id
+ self._session_factory = session_factory
+ self._controllers = controllers
+ self._clock = clock
+ self._pending: dict[str, _Transition] = {}
+ self._wake = asyncio.Event()
+ self._lock = asyncio.Lock()
+
+ def connected(self, conn: ControllerConnection) -> None:
+ self._queue(conn, None)
+
+ def disconnected(self, conn: ControllerConnection, reason: str) -> None:
+ self._queue(conn, reason)
+
+ def _queue(self, conn: ControllerConnection, reason: str | None) -> None:
+ self._pending[conn.controller_id] = _Transition(
+ tenant_id=conn.tenant_id,
+ controller_id=conn.controller_id,
+ connection_id=conn.id,
+ at=self._clock(),
+ disconnect_reason=reason,
+ )
+ self._wake.set()
+
+ @property
+ def pending(self) -> int:
+ return len(self._pending)
+
+ async def flush_all(self) -> None:
+ """Write every queued transition. Raises the first failure, keeping
+ what was not written."""
+ async with self._lock:
+ transitions = list(self._pending.values())
+ self._pending.clear()
+ if transitions:
+ await self._write(transitions)
+
+ async def run(self) -> None:
+ """Write transitions as they are queued, until cancelled."""
+ while True:
+ await self._wake.wait()
+ self._wake.clear()
+ try:
+ await self.flush_all()
+ except Exception:
+ logger.exception(
+ "Recording controller connection transitions failed; %d kept "
+ "and retried in %.0fs",
+ len(self._pending),
+ RETRY_SECONDS,
+ )
+ await asyncio.sleep(RETRY_SECONDS)
+ self._wake.set()
+
+ async def _write(self, transitions: list[_Transition]) -> None:
+ by_tenant: dict[str, list[_Transition]] = {}
+ for transition in transitions:
+ by_tenant.setdefault(transition.tenant_id, []).append(transition)
+ unwritten = list(transitions)
+ try:
+ for tenant_id, batch in by_tenant.items():
+ async with tenant_session(self._session_factory, tenant_id) as session:
+ for transition in batch:
+ await self._write_one(session, transition)
+ await session.commit()
+ for transition in batch:
+ unwritten.remove(transition)
+ except BaseException:
+ for transition in unwritten:
+ self._pending.setdefault(transition.controller_id, transition)
+ raise
+
+ async def _write_one(self, session: AsyncSession, transition: _Transition) -> None:
+ if transition.disconnect_reason is None:
+ found = await self._controllers.record_connected(
+ session,
+ transition.tenant_id,
+ transition.controller_id,
+ connection_id=transition.connection_id,
+ process_id=self._process_id,
+ connected_at=transition.at,
+ )
+ if not found:
+ logger.warning(
+ "Controller %s connected, but has no row to record it on",
+ transition.controller_id,
+ )
+ return
+ written = await self._controllers.record_disconnected(
+ session,
+ transition.tenant_id,
+ transition.controller_id,
+ connection_id=transition.connection_id,
+ disconnected_at=transition.at,
+ reason=transition.disconnect_reason,
+ )
+ if not written:
+ logger.info(
+ "Controller %s connection %s went (%s), but its row names a newer "
+ "connection; left as it is",
+ transition.controller_id,
+ transition.connection_id,
+ transition.disconnect_reason,
+ )
diff --git a/core/switch_core/management/controller_routes.py b/core/switch_core/management/controller_routes.py
index 66dfb2094..57f1db18b 100644
--- a/core/switch_core/management/controller_routes.py
+++ b/core/switch_core/management/controller_routes.py
@@ -65,6 +65,7 @@ from switch_core.management.errors import ManagementError, ManagementRoute, erro
from switch_core.management.schemas import (
MAX_STATUS_BYTES,
ControllerConnectionRequest,
+ ControllerInfoRequest,
DefinitionV1,
EnrollRequest,
OperationResultRequest,
@@ -210,6 +211,22 @@ async def exchange_token(
# ── Controller access token ───────────────────────────────────────────────────
[email protected]("/v1/management/controllers/{controller_id}")
+async def update_controller_info(
+ body: ControllerInfoRequest,
+ principal: PathController,
+ management: Management,
+ session: Session,
+) -> dict[str, Any]:
+ """Rename this machine and/or change its description, as its owner can
+ in the gateway. Returns the machine as the owner's list shows it."""
+ return await management.update_own_controller(
+ session,
+ principal,
+ {key: getattr(body, key) for key in body.model_fields_set},
+ )
+
+
@router.post("/v1/management/controllers/{controller_id}/credential/rotate")
async def rotate_credential(
principal: PathController, management: Management, session: Session
diff --git a/core/switch_core/management/placement.py b/core/switch_core/management/placement.py
index a59447d16..49c564833 100644
--- a/core/switch_core/management/placement.py
+++ b/core/switch_core/management/placement.py
@@ -1,15 +1,30 @@
"""Controller state, and whether an agent may be placed on a controller.
-A controller is `revoked` once its credential is gone, `online` while its
-last accepted status report is fresh, and `unknown` otherwise — including
-before it has ever reported. Fresh means within three status intervals, the
-interval being what each controller is told as `report_within_s`.
-
-Placement reads the controller's last status and refuses, with a contract
-reason code, when the controller is revoked or not online, when the
-definition's provider is not installed there, or when its login is missing
-or expired. A provider whose login is `unknown` passes: the controller could
-not tell, and refusing would block every provider whose CLI has no probe.
+A controller's state is read from its connection's recorded transitions
+(`connection_ledger.py`) and the lease of the switch-core process holding it
+(`process_lease.py`), so every process answers alike and a restart changes
+nothing:
+
+- `revoked`: its credential is gone.
+- `online`: its socket is attached, and the process holding it is alive
+ (its lease renewed within `LEASE_TTL`, and not stopped).
+- `offline`: it has connected before, and its socket went (closed, its
+ heartbeat lapsed, taken over and not replaced, the server shut down), or
+ the process holding it stopped or died without saying so.
+- `unknown`: no connection has ever been recorded for it: enrolled but never
+ run, or connected only to a server that did not record connections.
+
+Its status report is not what makes it online; the report carries the
+details (providers, disk, agents).
+
+Placement needs the controller online and its last status report fresh,
+within three status intervals (the interval being what each controller is
+told as `report_within_s`), since that report is what the provider checks
+read. It refuses, with a contract reason code, when either is not so, when
+the definition's provider is not installed there, or when its login is
+missing or expired. A provider whose login is `unknown` passes: the
+controller could not tell, and refusing would block every provider whose CLI
+has no probe.
"""
from __future__ import annotations
@@ -20,27 +35,43 @@ from typing import Any, Literal
from switch_core.db.models import AgentController
from switch_core.management import reason_codes
from switch_core.management.errors import ManagementError
+from switch_core.management.process_lease import ProcessLeases
-ControllerState = Literal["online", "unknown", "revoked"]
+ControllerState = Literal["online", "offline", "unknown", "revoked"]
STALE_AFTER_INTERVALS = 3
PROVIDER_AUTH_STATES = frozenset({"ok", "expired", "missing", "unknown"})
+def is_revoked(controller: AgentController) -> bool:
+ return controller.revoked_at is not None or controller.api_key_id is None
+
+
def controller_state(
- controller: AgentController, *, now: datetime, interval_seconds: int
+ controller: AgentController, leases: ProcessLeases
) -> ControllerState:
- if controller.revoked_at is not None or controller.api_key_id is None:
+ if is_revoked(controller):
return "revoked"
- if controller.status is None or controller.last_seen_at is None:
- return "unknown"
- stale_after = timedelta(seconds=STALE_AFTER_INTERVALS * interval_seconds)
- if now - controller.last_seen_at > stale_after:
+ if controller.connected_at is None:
return "unknown"
+ if controller.disconnected_at is not None:
+ return "offline"
+ if not leases.holds(controller.connection_process_id):
+ return "offline"
return "online"
+def status_is_fresh(
+ controller: AgentController, *, now: datetime, interval_seconds: int
+) -> bool:
+ """Whether the controller has reported status within three intervals."""
+ if controller.status is None or controller.last_seen_at is None:
+ return False
+ stale_after = timedelta(seconds=STALE_AFTER_INTERVALS * interval_seconds)
+ return now - controller.last_seen_at <= stale_after
+
+
def provider_auth(entry: dict[str, Any]) -> str:
"""The entry's `auth`, with any value this server does not know read as unknown."""
auth = entry.get("auth")
@@ -51,14 +82,25 @@ def placement_refusal(
controller: AgentController,
provider: str,
*,
+ leases: ProcessLeases,
now: datetime,
interval_seconds: int,
) -> tuple[str, str] | None:
"""The reason code and message refusing this placement, or None to allow it."""
- state = controller_state(controller, now=now, interval_seconds=interval_seconds)
+ state = controller_state(controller, leases)
if state == "revoked":
return reason_codes.CONTROLLER_REVOKED, "the controller has been revoked"
- if state != "online":
+ if state == "unknown":
+ return (
+ reason_codes.CONTROLLER_OFFLINE,
+ "the controller has never connected to Switch",
+ )
+ if state == "offline":
+ return (
+ reason_codes.CONTROLLER_OFFLINE,
+ "the controller is not connected to Switch",
+ )
+ if not status_is_fresh(controller, now=now, interval_seconds=interval_seconds):
return (
reason_codes.CONTROLLER_OFFLINE,
"the controller has not reported status recently",
@@ -96,11 +138,16 @@ def require_placement(
controller: AgentController,
provider: str,
*,
+ leases: ProcessLeases,
now: datetime,
interval_seconds: int,
) -> None:
refusal = placement_refusal(
- controller, provider, now=now, interval_seconds=interval_seconds
+ controller,
+ provider,
+ leases=leases,
+ now=now,
+ interval_seconds=interval_seconds,
)
if refusal is not None:
code, message = refusal
diff --git a/core/switch_core/management/process_lease.py b/core/switch_core/management/process_lease.py
new file mode 100644
index 000000000..9e3d8192f
--- /dev/null
+++ b/core/switch_core/management/process_lease.py
@@ -0,0 +1,131 @@
+"""This switch-core process's lease, and reading every process's.
+
+A controller's socket lives in the memory of the one process holding it, and
+that process writes the connection's transitions to the controller's row
+(`connection_ledger.py`). A process that dies writes no closing, so each
+process also holds a lease in `switch_core_processes`: it claims it at
+startup, renews it every `RENEW_INTERVAL`, and marks it stopped as it shuts
+down. A connection reads as online only while its holding process's lease is
+held: renewed within `LEASE_TTL` and not stopped. One write per process per
+renewal, however many machines it holds.
+
+Leases are written and compared on the database's clock, never a host's.
+Leases not renewed for `PRUNE_AFTER` are deleted by whichever process renews
+next; a connection naming a process with no lease reads as lost.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from switch_core.db.stores.switch_core_process_store import (
+ ProcessLeaseRow,
+ SwitchCoreProcessStore,
+)
+
+logger = logging.getLogger(__name__)
+
+RENEW_INTERVAL_SECONDS = 5.0
+
+# Three renewals: one missed write, or a slow one, does not put every machine
+# the process holds offline.
+LEASE_TTL = timedelta(seconds=3 * RENEW_INTERVAL_SECONDS)
+
+PRUNE_AFTER = timedelta(hours=1)
+
+# Why a connection the row still shows open is not, when its process is the
+# reason: the process shut down without writing it, or stopped renewing.
+SERVER_SHUTDOWN = "server_shutdown"
+SERVER_LOST = "server_lost"
+
+
+@dataclass(frozen=True)
+class ProcessLeases:
+ """Every process's lease as the database held it at `read_at`, its own
+ clock."""
+
+ read_at: datetime
+ leases: dict[str, ProcessLeaseRow]
+
+ def holds(self, process_id: str | None) -> bool:
+ """Whether the process is alive and holding its sockets."""
+ lease = self.leases.get(process_id) if process_id is not None else None
+ return (
+ lease is not None
+ and lease.stopped_at is None
+ and self.read_at - lease.beat_at < LEASE_TTL
+ )
+
+ def ended(self, process_id: str | None) -> tuple[datetime | None, str]:
+ """When and why a process not holding stopped: shut down at
+ `stopped_at`, or lost after its last renewal (when not known once its
+ lease is pruned)."""
+ lease = self.leases.get(process_id) if process_id is not None else None
+ if lease is None:
+ return None, SERVER_LOST
+ if lease.stopped_at is not None:
+ return lease.stopped_at, SERVER_SHUTDOWN
+ return lease.beat_at, SERVER_LOST
+
+
+async def read_leases(
+ session: AsyncSession, processes: SwitchCoreProcessStore
+) -> ProcessLeases:
+ """Every lease, in the caller's session; the table has no tenant policy,
+ so any session reads it."""
+ read_at, leases = await processes.read(session)
+ return ProcessLeases(read_at=read_at, leases=leases)
+
+
+class ProcessLease:
+ """This process's lease. The sessions are opened with no tenant bound:
+ the table carries none."""
+
+ def __init__(
+ self,
+ *,
+ process_id: str,
+ session_factory: async_sessionmaker[AsyncSession],
+ processes: SwitchCoreProcessStore,
+ ) -> None:
+ self.process_id = process_id
+ self._session_factory = session_factory
+ self._processes = processes
+
+ async def renew(self) -> None:
+ """Claim or renew the lease, and prune leases long dead."""
+ async with self._session_factory() as session:
+ await self._processes.renew(session, self.process_id)
+ pruned = await self._processes.prune(session, PRUNE_AFTER)
+ await session.commit()
+ if pruned:
+ logger.info("Pruned %d switch-core process lease(s) long stale", pruned)
+
+ async def stop(self) -> None:
+ """Mark the lease stopped: every connection this process still shows
+ open reads as offline from now."""
+ async with self._session_factory() as session:
+ await self._processes.stop(session, self.process_id)
+ await session.commit()
+ logger.info("Released switch-core process lease %s", self.process_id)
+
+ async def run(self) -> None:
+ """Renew every `RENEW_INTERVAL_SECONDS` until cancelled. A failed
+ renewal is logged and retried at the next; three in a row and this
+ process's machines read offline elsewhere."""
+ while True:
+ await asyncio.sleep(RENEW_INTERVAL_SECONDS)
+ try:
+ await self.renew()
+ except Exception:
+ logger.exception(
+ "Renewing switch-core process lease %s failed; the machines "
+ "it holds read offline once it is %s old",
+ self.process_id,
+ LEASE_TTL,
+ )
diff --git a/core/switch_core/management/schemas.py b/core/switch_core/management/schemas.py
index f815994a8..7a475dec1 100644
--- a/core/switch_core/management/schemas.py
+++ b/core/switch_core/management/schemas.py
@@ -15,13 +15,14 @@ same files.
from __future__ import annotations
from datetime import UTC, datetime
-from typing import Any, Literal
+from typing import Any, Literal, Self
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from switch_core.db.models import Agent, AgentController, AgentControllerOperation
from switch_core.db.models import AgentDefinition as AgentDefinitionRow
from switch_core.management.advanced_config import validate_advanced_config
+from switch_core.management.process_lease import ProcessLeases
Provider = Literal["claude", "codex", "opencode", "antigravity", "cursor"]
ControllerKind = Literal["console", "daemon", "ec2"]
@@ -344,7 +345,7 @@ class ConsoleControllerRequest(_GatewayBody):
return _controller_description(value)
-class UpdateControllerRequest(_GatewayBody):
+class _ControllerDetailsChange(BaseModel):
"""Rename a machine or change its description. Either or both; a key left
out is left as it is, and `description: null` (or blank) clears it — read
`model_fields_set`, not the value."""
@@ -369,12 +370,24 @@ class UpdateControllerRequest(_GatewayBody):
return _controller_description(value)
@model_validator(mode="after")
- def _changes_something(self) -> UpdateControllerRequest:
+ def _changes_something(self) -> Self:
if not self.model_fields_set:
raise ValueError("give a name, a description, or both")
return self
+class UpdateControllerRequest(_ControllerDetailsChange):
+ """The owner's change, through the gateway."""
+
+ model_config = ConfigDict(extra="forbid")
+
+
+class ControllerInfoRequest(_ControllerDetailsChange):
+ """The controller's own change (`switch-agent-controller set-info`)."""
+
+ model_config = ConfigDict(extra="ignore")
+
+
class CreateManagedAgentRequest(_GatewayBody):
name: str
description: str
@@ -481,7 +494,32 @@ def workspaces_dir_of(controller: AgentController) -> str | None:
return directory if isinstance(directory, str) and directory else None
-def controller_view(controller: AgentController, state: str) -> dict[str, Any]:
+def connection_view(
+ controller: AgentController, leases: ProcessLeases
+) -> dict[str, Any] | None:
+ """The controller's connection as last recorded: the current one, or the
+ last one once its socket went. None when none was ever recorded.
+
+ A connection the row shows open whose holding process is no longer
+ alive is shown as ended when and why that process stopped holding it:
+ `server_shutdown` at its lease's `stopped_at`, or `server_lost` after its
+ last renewal (when unknown once the lease is pruned)."""
+ if controller.connected_at is None:
+ return None
+ disconnected_at = controller.disconnected_at
+ reason = controller.disconnect_reason
+ if disconnected_at is None and not leases.holds(controller.connection_process_id):
+ disconnected_at, reason = leases.ended(controller.connection_process_id)
+ return {
+ "connected_at": wire_time(controller.connected_at),
+ "disconnected_at": wire_time_or_none(disconnected_at),
+ "disconnect_reason": reason,
+ }
+
+
+def controller_view(
+ controller: AgentController, state: str, leases: ProcessLeases
+) -> dict[str, Any]:
return {
"id": controller.id,
"name": controller.name,
@@ -491,6 +529,7 @@ def controller_view(controller: AgentController, state: str) -> dict[str, Any]:
"version": controller.version,
"state": state,
"last_seen_at": wire_time_or_none(controller.last_seen_at),
+ "connection": connection_view(controller, leases),
"status": controller.status,
"assignment_revision": controller.assignment_revision,
"workspaces_dir": workspaces_dir_of(controller),
diff --git a/core/switch_core/management/service.py b/core/switch_core/management/service.py
index b377577cf..bef779ab2 100644
--- a/core/switch_core/management/service.py
+++ b/core/switch_core/management/service.py
@@ -58,6 +58,7 @@ from switch_core.db.stores.agent_controller_store import AgentControllerStore
from switch_core.db.stores.agent_definition_store import AgentDefinitionStore
from switch_core.db.stores.agent_store import AgentStore
from switch_core.db.stores.api_key_store import ApiKeyStore
+from switch_core.db.stores.switch_core_process_store import SwitchCoreProcessStore
from switch_core.gateway.known_agents import KNOWN_AGENTS, KnownAgent
from switch_core.management import reason_codes, tokens
from switch_core.management.errors import ManagementError, not_found
@@ -65,8 +66,10 @@ from switch_core.management.notifier import ControllerNotifier
from switch_core.management.placement import (
ControllerState,
controller_state,
+ is_revoked,
require_placement,
)
+from switch_core.management.process_lease import ProcessLeases, read_leases
from switch_core.management.schemas import (
PROVIDER_KNOWN_AGENT_TYPES,
ControllerDescription,
@@ -121,6 +124,7 @@ class ManagementService:
operations: AgentControllerOperationStore,
api_keys: ApiKeyStore,
agents: AgentStore,
+ processes: SwitchCoreProcessStore,
presence: ControllerPresence,
clock: Callable[[], datetime],
) -> None:
@@ -132,17 +136,20 @@ class ManagementService:
self.operations = operations
self.api_keys = api_keys
self.agents = agents
+ self.processes = processes
self._clock = clock
def now(self) -> datetime:
return self._clock()
- def state_of(self, controller: AgentController) -> ControllerState:
- return controller_state(
- controller,
- now=self.now(),
- interval_seconds=self.settings.status_interval_seconds,
- )
+ async def leases(self, session: AsyncSession) -> ProcessLeases:
+ """Every switch-core process's lease, which a controller's state
+ needs: read once per request, before the states it decides."""
+ return await read_leases(session, self.processes)
+
+ @staticmethod
+ def state_of(controller: AgentController, leases: ProcessLeases) -> ControllerState:
+ return controller_state(controller, leases)
# ── Credentials and enrollment ────────────────────────────────────────────
@@ -286,7 +293,7 @@ class ManagementService:
controller = await self.controllers.get_by_api_key(session, tenant_id, key.id)
if controller is None or controller.id != controller_id:
raise invalid
- if self.state_of(controller) == "revoked":
+ if is_revoked(controller):
raise ManagementError(
401, reason_codes.CONTROLLER_REVOKED, "The controller has been revoked."
)
@@ -534,8 +541,9 @@ class ManagementService:
controllers = await self.controllers.list_for_owner(
session, tenant_id, owner_id
)
+ leases = await self.leases(session)
return [
- controller_view(controller, self.state_of(controller))
+ controller_view(controller, self.state_of(controller, leases), leases)
for controller in controllers
]
@@ -554,10 +562,33 @@ class ManagementService:
that is the name the room is told when the machine is offline.
"""
await self.owned_controller(session, tenant_id, owner_id, controller_id)
+ return await self._update_details(session, tenant_id, controller_id, changes)
+
+ async def update_own_controller(
+ self,
+ session: AsyncSession,
+ principal: ControllerPrincipal,
+ changes: dict[str, str | None],
+ ) -> dict[str, Any]:
+ """The controller renames itself or changes its description, with the
+ same limits and effects as its owner's change."""
+ await self._principal_controller(session, principal)
+ return await self._update_details(
+ session, principal.tenant_id, principal.controller_id, changes
+ )
+
+ async def _update_details(
+ self,
+ session: AsyncSession,
+ tenant_id: str,
+ controller_id: str,
+ changes: dict[str, str | None],
+ ) -> dict[str, Any]:
controller = await self.controllers.update_details(
session, tenant_id, controller_id, changes
)
- view = controller_view(controller, self.state_of(controller))
+ leases = await self.leases(session)
+ view = controller_view(controller, self.state_of(controller, leases), leases)
await session.commit()
if "name" in changes:
self.presence.rename_controller(controller_id, controller.name)
@@ -604,6 +635,7 @@ class ManagementService:
row: AgentDefinitionRow,
agent: Agent,
controllers: dict[str, AgentController],
+ leases: ProcessLeases,
) -> dict[str, Any]:
controller = None
if row.controller_id is not None:
@@ -618,15 +650,16 @@ class ManagementService:
row,
agent,
controller,
- self.state_of(controller) if controller is not None else None,
+ self.state_of(controller, leases) if controller is not None else None,
)
async def list_managed_agents(
self, session: AsyncSession, tenant_id: str, owner_id: str
) -> list[dict[str, Any]]:
controllers: dict[str, AgentController] = {}
+ leases = await self.leases(session)
return [
- await self._view(session, tenant_id, row, agent, controllers)
+ await self._view(session, tenant_id, row, agent, controllers, leases)
for row, agent in await self.definitions.list_for_owner(
session, tenant_id, owner_id
)
@@ -655,7 +688,9 @@ class ManagementService:
row, agent = await self._owned_definition(
session, tenant_id, owner_id, agent_id
)
- return await self._view(session, tenant_id, row, agent, {})
+ return await self._view(
+ session, tenant_id, row, agent, {}, await self.leases(session)
+ )
async def _check_target(
self,
@@ -676,6 +711,7 @@ class ManagementService:
require_placement(
controller,
provider,
+ leases=await self.leases(session),
now=self.now(),
interval_seconds=self.settings.status_interval_seconds,
)
@@ -777,7 +813,9 @@ class ManagementService:
request.controller_id,
)
agent = await self._owned_agent(session, owner_id, result.agent_id)
- return await self._view(session, tenant_id, row, agent, {})
+ return await self._view(
+ session, tenant_id, row, agent, {}, await self.leases(session)
+ )
async def put_managed_agent(
self,
@@ -874,7 +912,9 @@ class ManagementService:
and existing.desired_state == target.desired_state
and existing.definition == target.definition
):
- return await self._view(session, tenant_id, existing, agent, {})
+ return await self._view(
+ session, tenant_id, existing, agent, {}, await self.leases(session)
+ )
previous_controller_id = existing.controller_id if existing else None
if existing is None or existing.definition != target.definition:
@@ -920,7 +960,9 @@ class ManagementService:
await self._bind(session, tenant_id, row)
self._nudge(revisions)
agent = await self._owned_agent(session, owner_id, agent.id)
- return await self._view(session, tenant_id, row, agent, {})
+ return await self._view(
+ session, tenant_id, row, agent, {}, await self.leases(session)
+ )
async def delete_managed_agent(
self, session: AsyncSession, tenant_id: str, owner_id: str, agent_id: str
@@ -991,7 +1033,7 @@ class ManagementService:
controller = await self.owned_controller(
session, tenant_id, owner_id, controller_id
)
- if self.state_of(controller) == "revoked":
+ if is_revoked(controller):
raise ManagementError(
409, reason_codes.CONTROLLER_REVOKED, "The controller has been revoked."
)
diff --git a/core/switch_core/management/wiring.py b/core/switch_core/management/wiring.py
index 06cd4fda3..f6d24f669 100644
--- a/core/switch_core/management/wiring.py
+++ b/core/switch_core/management/wiring.py
@@ -8,10 +8,18 @@ bridge app is built, `install` adds the routes once both apps exist, and
serves. `install` also hands Core the agent-facing side
(`ManagementAgentOperations`), which is what makes the agent operations on
machines and managed agents exist.
+
+Each process also holds a lease (`process_lease.py`) and writes its
+controllers' connection transitions (`connection_ledger.py`): `start` claims
+the lease before the bridge serves, `run` renews it and writes transitions
+until cancelled, and `stop` writes what is left and marks the lease stopped.
"""
from __future__ import annotations
+import asyncio
+import logging
+import uuid
from collections.abc import Callable
from dataclasses import dataclass
from datetime import UTC, datetime
@@ -34,15 +42,20 @@ from switch_core.db.stores.agent_controller_store import AgentControllerStore
from switch_core.db.stores.agent_definition_store import AgentDefinitionStore
from switch_core.db.stores.agent_store import AgentStore
from switch_core.db.stores.api_key_store import ApiKeyStore
+from switch_core.db.stores.switch_core_process_store import SwitchCoreProcessStore
from switch_core.management.agent_operations import ManagementAgentOperations
from switch_core.management.auth import ManagementAuthenticator
from switch_core.management.bindings import load_bindings
+from switch_core.management.connection_ledger import ControllerConnectionLedger
from switch_core.management.controller_routes import router as controller_router
from switch_core.management.dependencies import init_management_dependencies
from switch_core.management.gateway_routes import router as gateway_router
from switch_core.management.notifier import ControllerNotifier
+from switch_core.management.process_lease import ProcessLease
from switch_core.management.service import ManagementService, ManagementSettings
+logger = logging.getLogger(__name__)
+
GATEWAY_PREFIX = "/management"
@@ -56,6 +69,8 @@ class Management:
authenticator: ManagementAuthenticator
agent_operations: ManagementAgentOperations
session_factory: async_sessionmaker[AsyncSession]
+ lease: ProcessLease
+ ledger: ControllerConnectionLedger
def install(
self,
@@ -78,6 +93,26 @@ class Management:
async with tenant_session(self.session_factory, tenant_id) as session:
await self.service.forget_deleted_agent(session, tenant_id, agent_id)
+ async def start(self) -> None:
+ """Claim this process's lease, so the connections it records read as
+ held. Raises when the database refuses: a process that cannot hold a
+ lease would show every machine it connects as offline."""
+ await self.lease.renew()
+ logger.info("Claimed switch-core process lease %s", self.lease.process_id)
+
+ async def run(self) -> None:
+ """Renew the lease and write connection transitions, until cancelled."""
+ await asyncio.gather(self.lease.run(), self.ledger.run())
+
+ async def stop(self) -> None:
+ """Write the transitions still queued, then mark the lease stopped.
+ The lease is marked stopped even when the writes fail, so the
+ machines this process held read offline either way."""
+ try:
+ await self.ledger.flush_all()
+ finally:
+ await self.lease.stop()
+
async def load_bindings(self) -> int:
return await load_bindings(
session_factory=self.session_factory,
@@ -96,9 +131,18 @@ def build_management(
presence: ControllerPresence,
auth_cache: ControllerAuthCache,
clock: Callable[[], datetime],
+ process_id: str,
) -> Management:
controllers = AgentControllerStore()
+ processes = SwitchCoreProcessStore()
presence.use_auth_cache(auth_cache)
+ ledger = ControllerConnectionLedger(
+ process_id=process_id,
+ session_factory=session_factory,
+ controllers=controllers,
+ clock=clock,
+ )
+ presence.use_ledger(ledger)
service = ManagementService(
settings=ManagementSettings(
token_secret=token_secret,
@@ -111,6 +155,7 @@ def build_management(
operations=AgentControllerOperationStore(),
api_keys=ApiKeyStore(),
agents=AgentStore(),
+ processes=processes,
presence=presence,
clock=clock,
)
@@ -128,6 +173,12 @@ def build_management(
service=service, session_factory=session_factory
),
session_factory=session_factory,
+ lease=ProcessLease(
+ process_id=process_id,
+ session_factory=session_factory,
+ processes=processes,
+ ),
+ ledger=ledger,
)
@@ -157,4 +208,5 @@ def create_management(
max_entries=config.agent_auth_cache_max_entries,
),
clock=utc_now,
+ process_id=str(uuid.uuid4()),
)
diff --git a/core/switch_core/migrations/versions/fabf9b9bff78_controller_connection_transitions.py b/core/switch_core/migrations/versions/fabf9b9bff78_controller_connection_transitions.py
new file mode 100644
index 000000000..6e969cc9d
--- /dev/null
+++ b/core/switch_core/migrations/versions/fabf9b9bff78_controller_connection_transitions.py
@@ -0,0 +1,76 @@
+"""agent controllers: record connection transitions and process leases
+
+Four nullable columns on `agent_controllers` recording the controller's
+connection as the switch-core process holding its socket last wrote it:
+which connection, which process holds it, when its socket attached, and when
+and why it went. Existing rows start with all null and fill in when their
+controller next connects.
+
+`switch_core_processes` holds each switch-core process's lease, renewed every
+few seconds, so a connection whose process died without writing its closing
+reads as offline. A global table: no tenant column and no row-level security
+policy (`db/rls_ddl.py`'s `GLOBAL_TABLES`).
+
+Revision ID: fabf9b9bff78
+Revises: eb24eafa59a0
+Create Date: 2026-10-07 12:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "fabf9b9bff78"
+down_revision: str | None = "eb24eafa59a0"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+ op.create_table(
+ "switch_core_processes",
+ sa.Column("id", sa.Text(), nullable=False),
+ sa.Column(
+ "started_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.func.now(),
+ nullable=False,
+ ),
+ sa.Column(
+ "beat_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.func.now(),
+ nullable=False,
+ ),
+ sa.Column("stopped_at", sa.DateTime(timezone=True), nullable=True),
+ sa.PrimaryKeyConstraint("id"),
+ )
+ op.add_column(
+ "agent_controllers", sa.Column("connection_id", sa.Text(), nullable=True)
+ )
+ op.add_column(
+ "agent_controllers",
+ sa.Column("connection_process_id", sa.Text(), nullable=True),
+ )
+ op.add_column(
+ "agent_controllers",
+ sa.Column("connected_at", sa.DateTime(timezone=True), nullable=True),
+ )
+ op.add_column(
+ "agent_controllers",
+ sa.Column("disconnected_at", sa.DateTime(timezone=True), nullable=True),
+ )
+ op.add_column(
+ "agent_controllers", sa.Column("disconnect_reason", sa.Text(), nullable=True)
+ )
+
+
+def downgrade() -> None:
+ op.drop_column("agent_controllers", "disconnect_reason")
+ op.drop_column("agent_controllers", "disconnected_at")
+ op.drop_column("agent_controllers", "connected_at")
+ op.drop_column("agent_controllers", "connection_process_id")
+ op.drop_column("agent_controllers", "connection_id")
+ op.drop_table("switch_core_processes")
diff --git a/core/tests/switch_core/db/test_tenant_exemption_allowlist.py b/core/tests/switch_core/db/test_tenant_exemption_allowlist.py
index c60035744..baa13508f 100644
--- a/core/tests/switch_core/db/test_tenant_exemption_allowlist.py
+++ b/core/tests/switch_core/db/test_tenant_exemption_allowlist.py
@@ -175,6 +175,10 @@ _RAW_SESSION_FACTORY_MODULES = {
# Its own reads are the per-tenant fan-out above; this is the one global
# read beside them, the deployment's user count.
"switch_core.telemetry.snapshot",
+ # This process's lease in `switch_core_processes`, a table with no tenant
+ # and no policy (`rls_ddl.GLOBAL_TABLES`): a process serves every tenant,
+ # so its lease belongs to none. It touches only that table.
+ "switch_core.management.process_lease",
}
# Calls that end in `session_factory` but hand one back rather than open a
diff --git a/core/tests/switch_core/db/test_tenant_schema_catalogue.py b/core/tests/switch_core/db/test_tenant_schema_catalogue.py
index d06f2a7dc..1485b14e8 100644
--- a/core/tests/switch_core/db/test_tenant_schema_catalogue.py
+++ b/core/tests/switch_core/db/test_tenant_schema_catalogue.py
@@ -108,6 +108,10 @@ class TestEveryTableIsScopedUnlessItIsNamedGlobal:
"deployment_identity",
"telemetry_milestones",
"telemetry_snapshot_watermark",
+ # Each switch-core process's lease. A process holds controller
+ # sockets for every tenant at once, and the row says only that it
+ # is alive.
+ "switch_core_processes",
}
async def test_scoped_is_everything_else(self) -> None:
diff --git a/core/tests/switch_core/fixtures/agent_controllers/controller_info_request.json b/core/tests/switch_core/fixtures/agent_controllers/controller_info_request.json
new file mode 100644
index 000000000..773fb5901
--- /dev/null
+++ b/core/tests/switch_core/fixtures/agent_controllers/controller_info_request.json
@@ -0,0 +1,4 @@
+{
+ "name": "build-box",
+ "description": "The build box in the office"
+}
diff --git a/core/tests/switch_core/fixtures/agent_controllers/controller_info_response.json b/core/tests/switch_core/fixtures/agent_controllers/controller_info_response.json
new file mode 100644
index 000000000..01d5cd4fd
--- /dev/null
+++ b/core/tests/switch_core/fixtures/agent_controllers/controller_info_response.json
@@ -0,0 +1,67 @@
+{
+ "id": "3f0f8a1e-5b7c-4c1e-9a52-2d6f1c9e0b11",
+ "name": "build-box",
+ "description": "The build box in the office",
+ "kind": "console",
+ "platform": {
+ "os": "linux",
+ "arch": "x64",
+ "os_version": "6.1.0"
+ },
+ "version": "0.1.0",
+ "state": "online",
+ "last_seen_at": "2026-10-07T09:00:05Z",
+ "connection": {
+ "connected_at": "2026-10-07T09:00:00Z",
+ "disconnected_at": null,
+ "disconnect_reason": null
+ },
+ "status": {
+ "seq": 42,
+ "observed_at": "2026-01-01T00:00:00Z",
+ "controller": {
+ "version": "0.1.0",
+ "protocol": 1,
+ "assignment_revision": 3
+ },
+ "machine": {
+ "platform": {
+ "os": "linux",
+ "arch": "x64",
+ "os_version": "6.1.0"
+ },
+ "disk_free_bytes": 53687091200,
+ "disk_total_bytes": 107374182400,
+ "mem_free_bytes": 8589934592,
+ "mem_total_bytes": 17179869184,
+ "sessions_running": 1,
+ "sessions_max": 4,
+ "workspaces_dir": "/home/user/.local/state/switch/agent-controller/workspaces"
+ },
+ "providers": [
+ {
+ "provider": "claude",
+ "installed": true,
+ "version": "1.0.0",
+ "auth": "ok",
+ "auth_source": "local",
+ "checked_at": "2026-01-01T00:00:00Z"
+ }
+ ],
+ "tools": [
+ {
+ "tool": "git",
+ "state": "ok"
+ },
+ {
+ "tool": "gh",
+ "state": "missing"
+ }
+ ],
+ "agents": []
+ },
+ "assignment_revision": 0,
+ "workspaces_dir": "/home/user/.local/state/switch/agent-controller/workspaces",
+ "created_at": "2026-10-01T12:00:00Z",
+ "revoked_at": null
+}
diff --git a/core/tests/switch_core/management/harness.py b/core/tests/switch_core/management/harness.py
index f39bc50c7..38cb9f0e2 100644
--- a/core/tests/switch_core/management/harness.py
+++ b/core/tests/switch_core/management/harness.py
@@ -199,6 +199,7 @@ def build_harness(
presence=protocol.connections.controllers,
auth_cache=controller_auth_cache,
clock=clock,
+ process_id=str(uuid.uuid4()),
)
async def _session() -> AsyncIterator[AsyncSession]:
@@ -312,6 +313,8 @@ class EnrolledController:
credential: str
access_token: str
owner: User
+ # To connect the controller as a running one is connected.
+ harness: Harness
@property
def headers(self) -> dict[str, str]:
@@ -344,6 +347,7 @@ async def enroll_console(
credential=body["credential"],
access_token=token.json()["access_token"],
owner=owner,
+ harness=harness,
)
@@ -355,6 +359,23 @@ async def report_status(
providers: list[dict[str, Any]] | None = None,
agents: list[dict[str, Any]] | None = None,
) -> httpx.Response:
+ """Report status as a running controller does: one whose socket is
+ attached, connected first unless it already is."""
+ await ensure_connected(client, controller)
+ return await report_status_only(
+ client, controller, seq, providers=providers, agents=agents
+ )
+
+
+async def report_status_only(
+ client: httpx.AsyncClient,
+ controller: EnrolledController,
+ seq: int,
+ *,
+ providers: list[dict[str, Any]] | None = None,
+ agents: list[dict[str, Any]] | None = None,
+) -> httpx.Response:
+ """Report status without opening a connection."""
return await client.put(
f"/v1/management/controllers/{controller.controller_id}/status",
json=status_report(seq, providers=providers, agents=agents),
@@ -496,6 +517,38 @@ async def open_connection(
return dict(response.json())
+async def connect(
+ client: httpx.AsyncClient, controller: EnrolledController
+) -> dict[str, Any]:
+ """Connect the controller as its socket would, and record it: the
+ connection opened, its socket attached, this process's lease claimed and
+ the transition written. Returns the open's response."""
+ harness = controller.harness
+ opened = await open_connection(client, controller)
+ presence = harness.protocol.connections.controllers
+ conn = presence.current_connection(controller.controller_id)
+ assert conn is not None
+ presence.attach_stream(conn)
+ await harness.management.start()
+ await harness.management.ledger.flush_all()
+ return opened
+
+
+async def ensure_connected(
+ client: httpx.AsyncClient, controller: EnrolledController
+) -> None:
+ """Connect the controller unless its socket is already attached, and
+ record it either way."""
+ harness = controller.harness
+ presence = harness.protocol.connections.controllers
+ conn = presence.current_connection(controller.controller_id)
+ if conn is None or not conn.stream_attached:
+ await connect(client, controller)
+ return
+ await harness.management.start()
+ await harness.management.ledger.flush_all()
+
+
async def open_stream(
harness: Harness,
controller: EnrolledController,
diff --git a/core/tests/switch_core/management/test_connection_state.py b/core/tests/switch_core/management/test_connection_state.py
new file mode 100644
index 000000000..90e29dfdb
--- /dev/null
+++ b/core/tests/switch_core/management/test_connection_state.py
@@ -0,0 +1,547 @@
+"""A machine's online state, read from its connection's recorded transitions
+and the lease of the process holding it.
+
+The process holding a controller's socket writes each transition once, the
+socket attaching and the socket going (and why), to the controller's row, and
+never a heartbeat. Each process also renews a lease. A machine is online
+while its socket is attached and its holding process's lease is fresh: so it
+is the same from every process, and turns offline within seconds of the
+controller stopping, its socket dropping, or the process holding it dying.
+Runs against the real database.
+"""
+
+from __future__ import annotations
+
+import time
+import uuid
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+import httpx
+import pytest
+from sqlalchemy import select, text
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from switch_core.bridges.agent.protocol.controller_presence import (
+ ControllerConnection,
+)
+from switch_core.bridges.agent.protocol.liveness import HEARTBEAT_TTL_SECONDS
+from switch_core.db.models import AgentController, SwitchCoreProcess
+from switch_core.db.stores.agent_controller_store import AgentControllerStore
+from switch_core.db.stores.switch_core_process_store import ProcessLeaseRow
+from switch_core.management.connection_ledger import ControllerConnectionLedger
+from switch_core.management.process_lease import LEASE_TTL, ProcessLeases
+from tests.switch_core.management.harness import (
+ EnrolledController,
+ Harness,
+ add_member,
+ beat,
+ build_harness,
+ connect,
+ cookies_for,
+ create_managed_agent,
+ enroll_console,
+ open_connection,
+ open_stream,
+ place_agent,
+ provider,
+ report_status,
+ report_status_only,
+ take,
+)
+
+
+def harness(session_factory: async_sessionmaker[AsyncSession]) -> Harness:
+ return build_harness(session_factory)
+
+
+async def _row(harness: Harness, controller_id: str) -> AgentController:
+ async with harness.session_factory() as session:
+ result = await session.execute(
+ select(AgentController).where(AgentController.id == controller_id)
+ )
+ return result.scalar_one()
+
+
+async def _listed(
+ client: httpx.AsyncClient, controller: EnrolledController
+) -> dict[str, Any]:
+ response = await client.get(
+ "/gateway/management/controllers", cookies=cookies_for(controller.owner)
+ )
+ assert response.status_code == 200, response.text
+ [entry] = [c for c in response.json() if c["id"] == controller.controller_id]
+ return dict(entry)
+
+
+async def _flush(harness: Harness) -> None:
+ await harness.management.ledger.flush_all()
+
+
+async def _age_lease(harness: Harness, seconds: float) -> None:
+ """Set this harness's process's last renewal `seconds` in the past, as a
+ process that stopped renewing would leave it."""
+ async with harness.session_factory() as session:
+ await session.execute(
+ text(
+ "UPDATE switch_core_processes "
+ "SET beat_at = now() - make_interval(secs => :seconds) "
+ "WHERE id = :id"
+ ),
+ {"seconds": seconds, "id": harness.management.lease.process_id},
+ )
+ await session.commit()
+
+
+def _current(harness: Harness, controller: EnrolledController) -> ControllerConnection:
+ conn = harness.protocol.connections.controllers.current_connection(
+ controller.controller_id
+ )
+ assert conn is not None
+ return conn
+
+
+class _CountingStore(AgentControllerStore):
+ """Counts the transitions written, and fails the first `failures`."""
+
+ def __init__(self, failures: int = 0) -> None:
+ self.failures = failures
+ self.connected = 0
+ self.disconnected = 0
+
+ def _maybe_fail(self) -> None:
+ if self.failures:
+ self.failures -= 1
+ raise ConnectionError("the database went away")
+
+ async def record_connected(self, *args: Any, **kwargs: Any) -> bool:
+ self._maybe_fail()
+ self.connected += 1
+ return await super().record_connected(*args, **kwargs)
+
+ async def record_disconnected(self, *args: Any, **kwargs: Any) -> bool:
+ self._maybe_fail()
+ self.disconnected += 1
+ return await super().record_disconnected(*args, **kwargs)
+
+
+class TestLeases:
+ """`ProcessLeases` alone: whether a process holds its sockets."""
+
+ now = datetime(2026, 1, 1, tzinfo=UTC)
+
+ def _leases(self, **rows: ProcessLeaseRow) -> ProcessLeases:
+ return ProcessLeases(read_at=self.now, leases=rows)
+
+ def test_a_fresh_lease_holds(self) -> None:
+ leases = self._leases(p=ProcessLeaseRow(beat_at=self.now, stopped_at=None))
+ assert leases.holds("p")
+
+ def test_a_lease_older_than_the_ttl_does_not(self) -> None:
+ beat_at = self.now - LEASE_TTL
+ leases = self._leases(p=ProcessLeaseRow(beat_at=beat_at, stopped_at=None))
+ assert not leases.holds("p")
+ assert leases.ended("p") == (beat_at, "server_lost")
+
+ def test_a_stopped_lease_does_not(self) -> None:
+ stopped_at = self.now - timedelta(seconds=1)
+ leases = self._leases(
+ p=ProcessLeaseRow(beat_at=self.now, stopped_at=stopped_at)
+ )
+ assert not leases.holds("p")
+ assert leases.ended("p") == (stopped_at, "server_shutdown")
+
+ def test_a_missing_lease_does_not(self) -> None:
+ leases = self._leases()
+ assert not leases.holds("p")
+ assert not leases.holds(None)
+ assert leases.ended("p") == (None, "server_lost")
+
+
+class TestTheMachineState:
+ async def test_a_machine_that_never_connected_is_unknown(
+ self, harness: Harness
+ ) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ await report_status_only(client, controller, 1)
+ listed = await _listed(client, controller)
+ assert listed["state"] == "unknown"
+ assert listed["connection"] is None
+ assert listed["last_seen_at"] is not None
+
+ async def test_an_open_connection_with_no_socket_is_not_online(
+ self, harness: Harness
+ ) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ await harness.management.start()
+ await open_connection(client, controller)
+ await _flush(harness)
+ listed = await _listed(client, controller)
+ assert listed["state"] == "unknown"
+
+ async def test_the_socket_attaching_is_online_and_going_is_offline_at_once(
+ self, harness: Harness
+ ) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ presence = harness.protocol.connections.controllers
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ agent_id = await place_agent(client, controller, name="reviewer")
+ conn = _current(harness, controller)
+ # The socket `place_agent` connected goes; a new one attaches.
+ presence.detach_stream(conn, conn.stream_token)
+ opened = {"connection_id": conn.id, "generation": conn.generation}
+ stream = await open_stream(harness, controller, opened)
+ await take(stream, 2)
+ await _flush(harness)
+ live = presence.is_live(agent_id)
+ online = await _listed(client, controller)
+ await stream.aclose() # type: ignore[attr-defined]
+ await _flush(harness)
+ offline = await _listed(client, controller)
+ row = await _row(harness, controller.controller_id)
+
+ assert live
+ assert online["state"] == "online"
+ assert online["connection"]["disconnected_at"] is None
+ assert online["connection"]["disconnect_reason"] is None
+ assert not presence.is_live(agent_id)
+ assert offline["state"] == "offline"
+ assert offline["connection"]["disconnect_reason"] == "socket_closed"
+ assert offline["connection"]["disconnected_at"] is not None
+ assert row.connection_id == conn.id
+ assert row.connection_process_id == harness.management.lease.process_id
+
+ async def test_a_socket_reattaching_to_its_connection_is_online_again(
+ self, harness: Harness
+ ) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ presence = harness.protocol.connections.controllers
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ await connect(client, controller)
+ conn = _current(harness, controller)
+ presence.detach_stream(conn, conn.stream_token)
+ await _flush(harness)
+ dropped = await _listed(client, controller)
+ presence.attach_stream(conn)
+ await _flush(harness)
+ back = await _listed(client, controller)
+ assert dropped["state"] == "offline"
+ assert back["state"] == "online"
+ assert back["connection"]["disconnected_at"] is None
+
+ async def test_each_transition_is_written_once_and_no_beat_is_written(
+ self, harness: Harness
+ ) -> None:
+ store = _CountingStore()
+ presence = harness.protocol.connections.controllers
+ ledger = ControllerConnectionLedger(
+ process_id=harness.management.lease.process_id,
+ session_factory=harness.session_factory,
+ controllers=store,
+ clock=harness.clock,
+ )
+ presence.use_ledger(ledger)
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ opened = await open_connection(client, controller)
+ conn = _current(harness, controller)
+ presence.attach_stream(conn)
+ await ledger.flush_all()
+ for _ in range(20):
+ answer = beat(
+ harness,
+ controller,
+ connection_id=opened["connection_id"],
+ generation=opened["generation"],
+ cursors={},
+ )
+ assert answer.status_code == 200, answer.text
+ queued_by_beats = ledger.pending
+ await ledger.flush_all()
+ after_beats = (store.connected, store.disconnected)
+ presence.detach_stream(conn, conn.stream_token)
+ await ledger.flush_all()
+ conn.last_beat = time.monotonic() - HEARTBEAT_TTL_SECONDS - 1
+ swept = presence.sweep()
+ await ledger.flush_all()
+ assert queued_by_beats == 0
+ assert after_beats == (1, 0)
+ assert swept == [conn]
+ assert (store.connected, store.disconnected) == (1, 1)
+ row = await _row(harness, controller.controller_id)
+ assert row.disconnect_reason == "socket_closed"
+
+ async def test_a_lapsed_heartbeat_is_recorded_by_the_sweep(
+ self, harness: Harness
+ ) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ presence = harness.protocol.connections.controllers
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ await connect(client, controller)
+ online = await _listed(client, controller)
+ conn = _current(harness, controller)
+ conn.last_beat = time.monotonic() - HEARTBEAT_TTL_SECONDS - 1
+ presence.sweep()
+ await _flush(harness)
+ offline = await _listed(client, controller)
+ assert online["state"] == "online"
+ assert offline["state"] == "offline"
+ assert offline["connection"]["disconnect_reason"] == "heartbeat_lapsed"
+
+ async def test_a_socket_the_server_closes_as_it_shuts_down_says_so(
+ self, harness: Harness
+ ) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ presence = harness.protocol.connections.controllers
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ await connect(client, controller)
+ presence.begin_shutdown()
+ conn = _current(harness, controller)
+ presence.detach_stream(conn, conn.stream_token)
+ await _flush(harness)
+ listed = await _listed(client, controller)
+ assert listed["state"] == "offline"
+ assert listed["connection"]["disconnect_reason"] == "server_shutdown"
+
+ async def test_a_takeover_records_the_old_connection_taken_over(
+ self, harness: Harness
+ ) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ presence = harness.protocol.connections.controllers
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ await connect(client, controller)
+ first = _current(harness, controller)
+ await open_connection(client, controller)
+ await _flush(harness)
+ between = await _listed(client, controller)
+ second = _current(harness, controller)
+ presence.attach_stream(second)
+ await _flush(harness)
+ after = await _listed(client, controller)
+ row = await _row(harness, controller.controller_id)
+ assert between["state"] == "offline"
+ assert between["connection"]["disconnect_reason"] == "taken_over"
+ assert after["state"] == "online"
+ assert row.connection_id == second.id != first.id
+
+ async def test_a_revoked_machine_is_revoked_and_its_connection_closed(
+ self, harness: Harness
+ ) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ await connect(client, controller)
+ revoked = await client.delete(
+ f"/gateway/management/controllers/{controller.controller_id}",
+ cookies=cookies_for(owner),
+ )
+ await _flush(harness)
+ listed = await _listed(client, controller)
+ assert revoked.status_code == 200, revoked.text
+ assert listed["state"] == "revoked"
+ assert listed["connection"]["disconnect_reason"] == "revoked"
+
+ async def test_placement_needs_the_machine_connected(
+ self, harness: Harness
+ ) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ presence = harness.protocol.connections.controllers
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ await report_status(client, controller, 1, providers=[provider("claude")])
+ conn = _current(harness, controller)
+ presence.detach_stream(conn, conn.stream_token)
+ await _flush(harness)
+ refused = await create_managed_agent(
+ client, owner, name="reviewer", controller_id=controller.controller_id
+ )
+ assert refused.status_code == 409, refused.text
+ assert refused.json()["error"] == {
+ "code": "controller_offline",
+ "message": "Cannot place the agent: the controller is not connected "
+ "to Switch.",
+ "retryable": False,
+ }
+
+
+class TestTheProcessLease:
+ async def test_a_lease_renewed_within_the_ttl_keeps_the_machine_online(
+ self, harness: Harness
+ ) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ await connect(client, controller)
+ await _age_lease(harness, LEASE_TTL.total_seconds() - 3)
+ still_online = await _listed(client, controller)
+ await _age_lease(harness, LEASE_TTL.total_seconds() + 1)
+ lapsed = await _listed(client, controller)
+ await harness.management.lease.renew()
+ renewed = await _listed(client, controller)
+ assert still_online["state"] == "online"
+ assert lapsed["state"] == "offline"
+ assert lapsed["connection"]["disconnect_reason"] == "server_lost"
+ assert lapsed["connection"]["disconnected_at"] is not None
+ assert renewed["state"] == "online"
+
+ async def test_a_dead_processs_machines_read_offline_from_another(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """The process holding the socket died: it wrote no closing, its lease
+ stops being renewed, and every other process reads the machine
+ offline, with nothing written per machine."""
+ holder = build_harness(session_factory)
+ other = build_harness(session_factory)
+ owner = await add_member(session_factory, "ada")
+ async with holder.client() as client, other.client() as elsewhere:
+ controller = await enroll_console(holder, client, owner)
+ await connect(client, controller)
+ await other.management.start()
+ online = await _listed(elsewhere, controller)
+ await _age_lease(holder, LEASE_TTL.total_seconds() + 1)
+ await other.management.lease.renew()
+ offline = await _listed(elsewhere, controller)
+ async with session_factory() as session:
+ await session.execute(
+ SwitchCoreProcess.__table__.delete().where(
+ SwitchCoreProcess.id == holder.management.lease.process_id
+ )
+ )
+ await session.commit()
+ pruned = await _listed(elsewhere, controller)
+ row = await _row(holder, controller.controller_id)
+ assert (
+ other.protocol.connections.controllers.current_connection(
+ controller.controller_id
+ )
+ is None
+ )
+ assert online["state"] == "online"
+ assert offline["state"] == "offline"
+ assert offline["connection"]["disconnect_reason"] == "server_lost"
+ assert pruned["state"] == "offline"
+ assert pruned["connection"] == {
+ "connected_at": online["connection"]["connected_at"],
+ "disconnected_at": None,
+ "disconnect_reason": "server_lost",
+ }
+ assert row.disconnected_at is None
+
+ async def test_a_process_that_stops_takes_its_machines_offline(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ holder = build_harness(session_factory)
+ other = build_harness(session_factory)
+ owner = await add_member(session_factory, "ada")
+ async with holder.client() as client, other.client() as elsewhere:
+ controller = await enroll_console(holder, client, owner)
+ await connect(client, controller)
+ await holder.management.stop()
+ listed = await _listed(elsewhere, controller)
+ assert listed["state"] == "offline"
+ assert listed["connection"]["disconnect_reason"] == "server_shutdown"
+ assert listed["connection"]["disconnected_at"] is not None
+
+ async def test_long_dead_leases_are_pruned(self, harness: Harness) -> None:
+ dead = str(uuid.uuid4())
+ async with harness.session_factory() as session:
+ session.add(
+ SwitchCoreProcess(
+ id=dead,
+ started_at=datetime.now(UTC) - timedelta(hours=3),
+ beat_at=datetime.now(UTC) - timedelta(hours=2),
+ )
+ )
+ await session.commit()
+ await harness.management.lease.renew()
+ async with harness.session_factory() as session:
+ ids = set(
+ (await session.execute(select(SwitchCoreProcess.id))).scalars().all()
+ )
+ assert dead not in ids
+ assert harness.management.lease.process_id in ids
+
+
+class TestTheLedger:
+ async def test_a_late_closing_of_a_replaced_connection_leaves_the_new_one(
+ self, harness: Harness
+ ) -> None:
+ """A process that held an earlier connection and only now records its
+ socket going does not overwrite the connection recorded since."""
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ await connect(client, controller)
+ earlier = _current(harness, controller)
+ await connect(client, controller)
+ replacement = _current(harness, controller)
+ elsewhere = ControllerConnectionLedger(
+ process_id=str(uuid.uuid4()),
+ session_factory=harness.session_factory,
+ controllers=AgentControllerStore(),
+ clock=harness.clock,
+ )
+ elsewhere.disconnected(earlier, "heartbeat_lapsed")
+ await elsewhere.flush_all()
+ listed = await _listed(client, controller)
+ row = await _row(harness, controller.controller_id)
+ assert row.connection_id == replacement.id
+ assert listed["state"] == "online"
+ assert listed["connection"]["disconnect_reason"] is None
+
+ async def test_only_the_latest_transition_of_a_controller_is_written(
+ self, harness: Harness
+ ) -> None:
+ store = _CountingStore()
+ ledger = ControllerConnectionLedger(
+ process_id=harness.management.lease.process_id,
+ session_factory=harness.session_factory,
+ controllers=store,
+ clock=harness.clock,
+ )
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ await connect(client, controller)
+ conn = _current(harness, controller)
+ ledger.disconnected(conn, "socket_closed")
+ ledger.connected(conn)
+ await ledger.flush_all()
+ row = await _row(harness, controller.controller_id)
+ assert (store.connected, store.disconnected) == (1, 0)
+ assert row.disconnected_at is None
+
+ async def test_a_failed_write_is_kept_for_the_next_flush(
+ self, harness: Harness
+ ) -> None:
+ store = _CountingStore(failures=1)
+ ledger = ControllerConnectionLedger(
+ process_id=harness.management.lease.process_id,
+ session_factory=harness.session_factory,
+ controllers=store,
+ clock=harness.clock,
+ )
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ await connect(client, controller)
+ conn = _current(harness, controller)
+ ledger.disconnected(conn, "heartbeat_lapsed")
+ with pytest.raises(ConnectionError):
+ await ledger.flush_all()
+ assert (await _row(harness, controller.controller_id)).disconnected_at is None
+ await ledger.flush_all()
+ row = await _row(harness, controller.controller_id)
+ assert row.disconnect_reason == "heartbeat_lapsed"
+ assert store.failures == 0
diff --git a/core/tests/switch_core/management/test_contract_fixtures.py b/core/tests/switch_core/management/test_contract_fixtures.py
index fc78a651f..58ba3e0fa 100644
--- a/core/tests/switch_core/management/test_contract_fixtures.py
+++ b/core/tests/switch_core/management/test_contract_fixtures.py
@@ -28,6 +28,7 @@ from switch_core.db.stores.agent_store import AgentStore
from switch_core.management import controller_routes
from switch_core.management.schemas import (
ControllerConnectionRequest,
+ ControllerInfoRequest,
EnrollRequest,
OperationResultRequest,
StatusReport,
@@ -97,6 +98,7 @@ class TestRequestFixturesParse:
ControllerConnectionRequest.model_validate(
_fixture("controller_connection_request.json")
)
+ ControllerInfoRequest.model_validate(_fixture("controller_info_request.json"))
class TestEnrollmentAndTokens:
@@ -243,6 +245,23 @@ class TestControllerMessages:
assert refused.json() == _fixture("error_response.json")
+class TestTheControllersOwnInfo:
+ async def test_set_info(self, harness: Harness) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ await report_status(
+ client, controller, 1, providers=[provider("claude")], agents=[]
+ )
+ updated = await client.patch(
+ f"/v1/management/controllers/{controller.controller_id}",
+ json=_fixture("controller_info_request.json"),
+ headers=controller.headers,
+ )
+ assert updated.status_code == 200, updated.text
+ assert_same_shape(updated.json(), _fixture("controller_info_response.json"))
+
+
class TestTheControllerConnection:
async def test_open_and_beat(self, harness: Harness) -> None:
owner = await add_member(harness.session_factory, "ada")
diff --git a/core/tests/switch_core/management/test_controller_routes.py b/core/tests/switch_core/management/test_controller_routes.py
index 4248b302f..65d9a33f8 100644
--- a/core/tests/switch_core/management/test_controller_routes.py
+++ b/core/tests/switch_core/management/test_controller_routes.py
@@ -361,7 +361,7 @@ class TestAssignment:
class TestStatus:
- async def test_a_report_makes_the_controller_online_until_it_goes_stale(
+ async def test_a_connected_controller_is_online_however_old_its_report(
self, harness: Harness
) -> None:
owner = await add_member(harness.session_factory, "ada")
@@ -379,7 +379,7 @@ class TestStatus:
assert response.json() == {"assignment_revision": 0, "report_within_s": 60}
assert online.json()[0]["state"] == "online"
assert online.json()[0]["status"]["seq"] == 5
- assert stale.json()[0]["state"] == "unknown"
+ assert stale.json()[0]["state"] == "online"
async def test_an_older_seq_is_ignored_but_answered(self, harness: Harness) -> None:
owner = await add_member(harness.session_factory, "ada")
@@ -790,6 +790,75 @@ class TestPlacementBindsTheAgentInCore:
assert restarted.is_revoked(controller.controller_id)
+class TestTheControllersOwnInfo:
+ async def test_the_controller_renames_and_describes_itself(
+ self, harness: Harness
+ ) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner, name="laptop")
+ await report_status(client, controller, 1, providers=[provider("claude")])
+ created = await create_managed_agent(
+ client, owner, name="reviewer", controller_id=controller.controller_id
+ )
+ path = f"/v1/management/controllers/{controller.controller_id}"
+ renamed = await client.patch(
+ path,
+ json={"name": " build box ", "description": " Under the desk "},
+ headers=controller.headers,
+ )
+ cleared = await client.patch(
+ path, json={"description": " "}, headers=controller.headers
+ )
+ listed = await client.get(
+ "/gateway/management/controllers", cookies=cookies_for(owner)
+ )
+ assert renamed.status_code == 200, renamed.text
+ assert (renamed.json()["name"], renamed.json()["description"]) == (
+ "build box",
+ "Under the desk",
+ )
+ assert renamed.json()["state"] == "online"
+ assert cleared.json()["name"] == "build box"
+ assert cleared.json()["description"] is None
+ [stored] = listed.json()
+ assert (stored["name"], stored["description"]) == ("build box", None)
+ binding = harness.protocol.connections.controllers.binding(
+ created.json()["agent_id"]
+ )
+ assert binding is not None and binding.controller_name == "build box"
+
+ async def test_the_owners_limits_hold_and_only_its_own_machine(
+ self, harness: Harness
+ ) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ mine = await enroll_console(harness, client, owner, name="mine")
+ theirs = await enroll_console(harness, client, owner, name="theirs")
+ path = f"/v1/management/controllers/{mine.controller_id}"
+ refused = [
+ await client.patch(path, json=body, headers=mine.headers)
+ for body in (
+ {},
+ {"name": " "},
+ {"name": None},
+ {"name": "x" * 201},
+ {"description": "x" * 501},
+ )
+ ]
+ other = await client.patch(
+ f"/v1/management/controllers/{theirs.controller_id}",
+ json={"name": "taken"},
+ headers=mine.headers,
+ )
+ anonymous = await client.patch(path, json={"name": "nobody"})
+ assert [(r.status_code, r.json()["error"]["code"]) for r in refused] == [
+ (422, "validation_error")
+ ] * 5
+ assert (other.status_code, other.json()["error"]["code"]) == (403, "forbidden")
+ assert anonymous.status_code == 401
+
+
def test_bearer_helper() -> None:
assert bearer("x") == {"Authorization": "Bearer x"}
diff --git a/core/tests/switch_core/management/test_gateway_routes.py b/core/tests/switch_core/management/test_gateway_routes.py
index 3dcf56319..69c3650b0 100644
--- a/core/tests/switch_core/management/test_gateway_routes.py
+++ b/core/tests/switch_core/management/test_gateway_routes.py
@@ -27,6 +27,7 @@ from tests.switch_core.management.harness import (
create_managed_agent,
definition,
enroll_console,
+ ensure_connected,
platform,
provider,
report_status,
@@ -579,6 +580,7 @@ async def report_machine(
del report["machine"]["workspaces_dir"]
else:
report["machine"]["workspaces_dir"] = workspaces_dir
+ await ensure_connected(client, controller)
response = await client.put(
f"/v1/management/controllers/{controller.controller_id}/status",
json=report,
diff --git a/docs/design/agent-controllers-v1.md b/docs/design/agent-controllers-v1.md
index 88ba97923..3ca8e85d8 100644
--- a/docs/design/agent-controllers-v1.md
+++ b/docs/design/agent-controllers-v1.md
@@ -38,7 +38,8 @@ where it deliberately stops short.
when the flag is on**: a model validator raises if it is missing or shorter than 32 chars. It is
separate from `jwt_secret_key` on purpose.
- `controller_status_interval_seconds: int = 60`. Returned to controllers as `report_within_s`.
- A controller is `unknown` after 3 intervals without a status.
+ Placement needs a status within 3 intervals. Whether a machine is online is read from its
+ recorded connection instead (see "Liveness" below).
With the flag off, none of the routes below are mounted, and the middleware branch is
inactive.
@@ -50,7 +51,13 @@ inactive.
`version` null, `public_key` null.
- `api_key_id`: the credential, an `api_keys` row of type `controller`, holding the hash only.
- `assignment_revision` int default 0, `status_seq` bigint null, `status` JSONB null.
- - `last_seen_at` null, `revoked_at` null, `created_at`, `updated_at`.
+ - `last_seen_at` null (the last status), `revoked_at` null, `created_at`, `updated_at`.
+ - The connection, as the switch-core process holding its socket last recorded it (all null
+ until the first): `connection_id`, `connection_process_id`, `connected_at`,
+ `disconnected_at`, `disconnect_reason` (`socket_closed`, `server_shutdown`,
+ `heartbeat_lapsed`, `taken_over`, `revoked`).
+- `switch_core_processes` (global, no tenant): each switch-core process's lease, `id`,
+ `started_at`, `beat_at`, `stopped_at` null.
- `agent_controller_enrollment_codes`: `id`, `owner_id`, `api_key_id` (an `api_keys` row of
type `controller_enrollment`, so the existing global hash → tenant lookup works),
`expires_at`, `used_at` null, `controller_id` null, `created_at`.
@@ -88,6 +95,11 @@ Public (they authenticate through the body):
Controller access token (`{id}` must match the token's `cid`, otherwise `403 forbidden`):
- `POST /v1/management/controllers/{id}/credential/rotate` returns `{credential}`.
+- `PATCH /v1/management/controllers/{id}`
+ - Body: `{name?, description?}`, at least one: the controller renames its own machine
+ (`switch-agent-controller set-info`). The same validation and effects as the owner's
+ `PATCH /gateway/management/controllers/{id}`, including the rename reaching Core's
+ bindings. Returns the controller as the owner's list shows it.
- `GET /v1/management/controllers/{id}/assignment`
- Honours `If-None-Match`; returns `200` with an `ETag` header, or `304`.
- `PUT /v1/management/controllers/{id}/status`
@@ -124,8 +136,12 @@ Controller access token (`{id}` must match the token's `cid`, otherwise `403 for
- Body: `{name, description?, kind:"console", platform, version, public_key?}`.
- Returns `{controller_id, credential}`.
- `GET /gateway/management/controllers`
- - Returns the list, each with its `description`, derived `state` (`online|unknown|revoked`),
- `last_seen_at`, its last `status`, and `workspaces_dir`: the directory the controller makes
+ - Returns the list, each with its `description`, derived `state`
+ (`online|offline|unknown|revoked`, see "Liveness" below), `last_seen_at` (its last status),
+ `connection` (`{connected_at, disconnected_at, disconnect_reason}` or null when it never
+ connected; `disconnect_reason` is also `server_shutdown` or `server_lost` when the row
+ shows it open but the process holding it stopped or died), its last `status`, and
+ `workspaces_dir`: the directory the controller makes
agents' workspaces in, from that status (`machine.workspaces_dir`), null when it has not
reported one.
- `PATCH /gateway/management/controllers/{id}`
@@ -157,7 +173,8 @@ Controller access token (`{id}` must match the token's `cid`, otherwise `403 for
**Placement checks** run on create, adopt and move, and on a change to `running`. Each failure returns `409` with a reason:
- `controller_revoked`
-- `controller_offline`: no status, or the last status is stale
+- `controller_offline`: not online (never connected, its socket went, or the process
+ holding it stopped), or no status, or the last status is stale
- `provider_not_installed`
- `provider_login_missing` or `provider_login_expired`
@@ -283,6 +300,9 @@ exactly the answer a missing one does.
- CLI `switch-agent-controller`:
- `enroll --server <agent-bridge-url> --code <code> [--name] [--description] [--data-dir]`
- `run [--data-dir]`
+ - `set-info [--name] [--description] [--data-dir]`: renames the machine and/or changes its
+ description on the server, with the controller's own credential, and records the new name
+ locally.
- `status [--data-dir]`
- **Data dir:** `SWITCH_CONTROLLER_DATA_DIR`, otherwise the OS default.
- macOS: `~/Library/Application Support/Switch/agent-controller`
@@ -435,8 +455,25 @@ stream. The flag and everything else above stay as they are.
offered the terminal command or told to open Switch Console.
In-room session commands (`!reset`, `!compact`) are still relayed on the
controller's stream with the room, and the controller picks the session.
-- **Liveness** is "stream attached and beat within 6 s"; the connection sweep
- closes lapsed controller connections.
+- **Liveness** is "socket attached and a pong within 6 s"; the connection sweep
+ closes lapsed controller connections. Heartbeats stay in the memory of the
+ process holding the socket.
+- **Recorded transitions.** `ControllerPresence` stays the in-memory fast path for
+ routing and agent presence, and tells a ledger (`ControllerConnectionLedger`, Core's
+ port, implemented by `management/connection_ledger.py`) of each transition, once:
+ the socket attaching to a connection that had none, and the socket going, with why
+ (`socket_closed`, `server_shutdown`, `heartbeat_lapsed`, `taken_over`, `revoked`).
+ The ledger writes them to the controller's row from one background task, never
+ per beat. A connecting replaces the row; a closing is written only while the row
+ still names that connection, so a process closing a connection the controller has
+ since replaced through another process leaves the replacement standing.
+- **Process lease.** Each switch-core process renews a lease in
+ `switch_core_processes` every 5 s (`management/process_lease.py`), one write per
+ process however many machines it holds, and marks it stopped as it shuts down.
+ A machine is online while its socket is recorded attached and its holding
+ process's lease was renewed within 15 s and is not stopped; so a process that
+ dies without writing its closings takes its machines offline within 15 s.
+ Leases are written and compared on the database's clock.
- **Holder id.** A controller-backed agent holds things under
`controller:{controller_id}:{agent_id}`: the operation caller's session key
and session id, the reader of its unread counts, and the holder of a role
diff --git a/docs/design/controller-contract-v1.md b/docs/design/controller-contract-v1.md
index 45727b353..af33bd6d1 100644
--- a/docs/design/controller-contract-v1.md
+++ b/docs/design/controller-contract-v1.md
@@ -43,9 +43,18 @@ POST /v1/management/controllers/{id}/token auth: none (credential in b
→ 200 { access_token: string, expires_at: Time } // about 1h. Claims: controller_id, owner_id, tenant_id
POST /v1/management/controllers/{id}/credential/rotate → 200 { credential: string }
+PATCH /v1/management/controllers/{id} auth: the controller's access token
+ { name?: string, description?: string | null } // either or both; null or blank clears the description
+ → 200 Controller // the machine as its owner's list shows it
+ → 422 validation_error // name blank or > 200 chars, description > 500, nothing given
DELETE /v1/management/controllers/{id} auth: owner (user). Revokes it, and every agent on it stops
```
+The `PATCH` is the controller renaming its own machine
+(`switch-agent-controller set-info`), with the limits and effects of the
+owner's own change in the gateway. A server that predates it answers `404` or
+`405` without an error envelope.
+
```ts
type Platform = { os: "macos" | "linux" | "windows"; arch: "arm64" | "x64"; os_version: string }
```
@@ -238,6 +247,16 @@ WS /v1/controllers/{id}/connection/ws?connection_id=…&generation=…
close 1012 // the server is restarting: reconnect within a second
```
+**Connection transitions are recorded.** Whichever server process holds the
+socket records, once each, when it attached and when and why it went
+(`socket_closed`, `server_shutdown`, `heartbeat_lapsed`, `taken_over`,
+`revoked`) on the controller's row. Pongs are not recorded. A controller
+shutting down needs to say nothing: its socket closing is recorded at once,
+and so is a dropped one, which reattaching records as connected again. Every
+process reads the machine's state from that record and the holding process's
+lease (§8, Machine state), so it does not depend on which process holds the
+socket.
+
**Attachment is automatic.** Core attaches the agents currently bound to this controller (§7), and attaches or detaches them as bindings change. There's no per-agent subscribe call.
**Core knows only whether an agent is connected.** A controller-backed agent is connected while its controller's socket is attached and answering pings, its owner has it `running`, and the controller is not revoked. Core does not know where the agent's sessions are, or whether one is running: the open and the pong carry no session map, and a body that still sends `placements` is refused with `422 validation_error` (the one exception to ignoring unknown fields, so a controller still reporting sessions is noticed rather than silently dropped). When a connected agent is addressed, Core delivers the message and posts nothing; if a session has to start, the agent's host posts "Starting a session…" itself. When it is not connected, Core tells the room why: stopped by its owner, its machine removed, or its machine offline.
@@ -289,13 +308,21 @@ After acting, send status.
- To move it, Management first unbinds it from A at revision r+1, and waits for A to report it stopped, or for A to go stale.
- Then it binds the agent to B at revision r+2. B refuses to start a revision older than one it has already applied.
-**Staleness.** No status for 3 × `report_within_s` makes a controller `unknown`, and so are its agents. Management then:
-- shows them as unknown,
+**Machine state.** Read from the recorded connection (§6), not from the status report:
+- `online`: its socket is attached, and the server process holding it is alive (its lease renewed within 15 s, and not stopped).
+- `offline`: it has connected before, and its socket went (closed, a lapsed heartbeat, a takeover not replaced, the server shutting down), or the server process holding it stopped or died. A controller that stops shows offline at once when its socket closes, within about 8 s when its heartbeat lapses with the socket still open, and within 15 s when the process holding its socket died.
+- `unknown`: no connection has ever been recorded for it.
+- `revoked`: its credential is gone.
+
+While it is not online Management:
+- shows it, and its agents, as offline or unknown,
- refuses new placements on that controller,
- may reassign its agents only if their definition allows it. Cloud agents stay put.
-**Placement checks.** Before binding, Management checks the target's last status. It refuses with a reason if:
-- the controller is `unknown`,
+The status report carries the details (providers, disk, sessions, agents) and is what placement reads them from.
+
+**Placement checks.** Before binding, Management checks the target. It refuses with a reason if:
+- the controller is not `online`, or its last status is older than 3 × `report_within_s`,
- the provider is not installed, or its login is not `ok`,
- there's no session capacity, or the disk is nearly full.
@@ -334,7 +361,7 @@ The personal agent relays that reason to the user as is.
| `crash_loop` | status | 5 restarts in 10 minutes, stopped retrying |
| `out_of_memory` | status | Killed by the memory limit |
| `disk_full` / `capacity_exceeded` | status, placement | No disk / no session slots |
-| `controller_offline` | placement | Target is `unknown` |
+| `controller_offline` | placement | Target is not `online`, or its last status is stale |
| `internal` | any | Server or controller bug, with `retryable` set honestly |
---
diff --git a/gateway/src/data/management.test.ts b/gateway/src/data/management.test.ts
index 24a3d96d4..7964dc128 100644
--- a/gateway/src/data/management.test.ts
+++ b/gateway/src/data/management.test.ts
@@ -5,6 +5,7 @@ import {
enrollCommand,
fetchManagementAvailable,
managementError,
+ shellQuote,
} from "./management";
function respond(status: number, body: unknown) {
@@ -106,8 +107,29 @@ describe("createManagedAgent", () => {
describe("enrollCommand", () => {
it("names the server and the code", () => {
- expect(enrollCommand("https://switch.example.com", "swce_example")).toBe(
- "switch-agent-controller enroll --server https://switch.example.com --code swce_example",
+ expect(
+ enrollCommand("https://switch.example.com", "swce_example", { name: "", description: " " }),
+ ).toBe("switch-agent-controller enroll --server https://switch.example.com --code swce_example");
+ });
+
+ it("adds the name and description, quoted for the shell", () => {
+ expect(
+ enrollCommand("https://switch.example.com", "swce_example", {
+ name: " build-box ",
+ description: "Bob's box: $HOME `x` \"y\"",
+ }),
+ ).toBe(
+ "switch-agent-controller enroll --server https://switch.example.com --code swce_example" +
+ " --name build-box --description 'Bob'\\''s box: $HOME `x` \"y\"'",
);
});
});
+
+describe("shellQuote", () => {
+ it("leaves plain words alone and single-quotes the rest", () => {
+ expect(shellQuote("swce_a-b.c")).toBe("swce_a-b.c");
+ expect(shellQuote("two words")).toBe("'two words'");
+ expect(shellQuote("it's")).toBe("'it'\\''s'");
+ expect(shellQuote("")).toBe("''");
+ });
+});
diff --git a/gateway/src/data/management.ts b/gateway/src/data/management.ts
index d444cb72f..f162ff7c4 100644
--- a/gateway/src/data/management.ts
+++ b/gateway/src/data/management.ts
@@ -16,7 +16,26 @@ export const PROVIDERS: { value: Provider; label: string }[] = [
export type DesiredState = "running" | "stopped";
export type Isolation = "shared" | "isolated";
-export type ControllerState = "online" | "unknown" | "revoked";
+/**
+ * `online`: its controller's socket to Switch is attached. `offline`: it was
+ * connected and is not now (stopped, asleep, cut off, or the Switch server
+ * holding it went away). `unknown`: it has never connected. `revoked`: removed
+ * from Switch.
+ */
+export type ControllerState = "online" | "offline" | "unknown" | "revoked";
+
+/** The machine's connection as Switch last recorded it: the current one, or the last. */
+export interface ControllerConnection {
+ /** When its socket last attached. */
+ connected_at: string;
+ /** Null while it is connected, and when the Switch server holding it was lost long ago. */
+ disconnected_at: string | null;
+ /**
+ * Null while it is connected; else `socket_closed`, `heartbeat_lapsed`, `taken_over`,
+ * `revoked`, `server_shutdown` or `server_lost`.
+ */
+ disconnect_reason: string | null;
+}
export interface Platform {
os: string;
@@ -74,7 +93,10 @@ export interface Controller {
platform: Platform | null;
version: string | null;
state: ControllerState;
+ /** When it last reported its status. */
last_seen_at: string | null;
+ /** Null when it has never connected. */
+ connection: ControllerConnection | null;
status: StatusReport | null;
assignment_revision: number;
created_at: string;
@@ -258,7 +280,26 @@ export function createOperation(body: {
return request<Operation>("/operations", { method: "POST", body: JSON.stringify(body) });
}
-/** The command a user runs on the machine to enroll it with a fresh code. */
-export function enrollCommand(server: string, code: string): string {
- return `switch-agent-controller enroll --server ${server} --code ${code}`;
+/** One word for a POSIX shell: as it is when nothing in it is special, else single-quoted. */
+export function shellQuote(value: string): string {
+ if (/^[A-Za-z0-9_@%+=:,./-]+$/.test(value)) return value;
+ return `'${value.replace(/'/g, `'\\''`)}'`;
+}
+
+/**
+ * The command a user runs on the machine to enroll it with a fresh code, with
+ * the machine's name and description when given (blank is left out: the name
+ * then defaults to the host name).
+ */
+export function enrollCommand(
+ server: string,
+ code: string,
+ machine: { name: string; description: string },
+): string {
+ const words = ["switch-agent-controller", "enroll", "--server", server, "--code", code];
+ const name = machine.name.trim();
+ const description = machine.description.trim();
+ if (name) words.push("--name", name);
+ if (description) words.push("--description", description);
+ return words.map(shellQuote).join(" ");
}
diff --git a/gateway/src/pages/machines/AddMachineDialog.test.tsx b/gateway/src/pages/machines/AddMachineDialog.test.tsx
index 2fc69a757..5987efeff 100644
--- a/gateway/src/pages/machines/AddMachineDialog.test.tsx
+++ b/gateway/src/pages/machines/AddMachineDialog.test.tsx
@@ -1,4 +1,4 @@
-import { cleanup, render, screen } from "@testing-library/react";
+import { cleanup, fireEvent, render, screen } from "@testing-library/react";
import { afterEach, describe, expect, it, vi } from "vitest";
import AddMachineDialog from "./AddMachineDialog";
@@ -36,6 +36,33 @@ describe("AddMachineDialog", () => {
expect(screen.queryByText(new RegExp(window.location.origin))).toBeNull();
});
+ it("puts the optional name and description into the command, quoted", async () => {
+ issue("https://switch-api.example.test");
+ render(<AddMachineDialog open onClose={() => {}} />);
+ const base =
+ "switch-agent-controller enroll --server https://switch-api.example.test --code swce_example";
+ await screen.findByText(base);
+ fireEvent.change(screen.getByLabelText("Name (optional)"), { target: { value: " build-box " } });
+ fireEvent.change(screen.getByLabelText("Description (optional)"), {
+ target: { value: "Bob's box in the office" },
+ });
+ expect(
+ screen.getByText(`${base} --name build-box --description 'Bob'\\''s box in the office'`),
+ ).toBeTruthy();
+ });
+
+ it("shows no command for a name or description that is too long", async () => {
+ issue("https://switch-api.example.test");
+ render(<AddMachineDialog open onClose={() => {}} />);
+ await screen.findByText(/--code swce_example/);
+ fireEvent.change(screen.getByLabelText("Description (optional)"), {
+ target: { value: "x".repeat(501) },
+ });
+ expect(screen.getByText("At most 500 characters.")).toBeTruthy();
+ expect(screen.queryByText(/--code swce_example/)).toBeNull();
+ expect(screen.getByText("swce_example")).toBeTruthy();
+ });
+
it("says what to configure instead of showing a command that cannot work", async () => {
issue(null);
render(<AddMachineDialog open onClose={() => {}} />);
diff --git a/gateway/src/pages/machines/AddMachineDialog.tsx b/gateway/src/pages/machines/AddMachineDialog.tsx
index 5972afaf9..7246ffd50 100644
--- a/gateway/src/pages/machines/AddMachineDialog.tsx
+++ b/gateway/src/pages/machines/AddMachineDialog.tsx
@@ -12,11 +12,18 @@ import {
DialogTitle,
IconButton,
Stack,
+ TextField,
Tooltip,
Typography,
} from "@mui/material";
import { useEffect, useState } from "react";
-import { createEnrollmentCode, type EnrollmentCode, enrollCommand } from "../../data/management";
+import {
+ createEnrollmentCode,
+ type EnrollmentCode,
+ enrollCommand,
+ MAX_MACHINE_DESCRIPTION,
+ MAX_MACHINE_NAME,
+} from "../../data/management";
import { MONO_SX, formatDate } from "../../theme/hootFormat";
function CopyBlock({ text, label }: { text: string; label: string }) {
@@ -48,7 +55,8 @@ function CopyBlock({ text, label }: { text: string; label: string }) {
/**
* Issues a one-time enrollment code and shows the command that enrolls a
* machine's agents controller with it. The code is shown once: closing the
- * dialog discards it, and a new open issues a new one.
+ * dialog discards it, and a new open issues a new one. The optional name and
+ * description go into the command, so the machine enrolls with them.
*/
export default function AddMachineDialog({
open,
@@ -59,12 +67,23 @@ export default function AddMachineDialog({
}) {
const [enrollment, setEnrollment] = useState<EnrollmentCode | null>(null);
const [error, setError] = useState<string | null>(null);
+ const [name, setName] = useState("");
+ const [description, setDescription] = useState("");
+
+ const nameError =
+ name.trim().length > MAX_MACHINE_NAME ? `At most ${MAX_MACHINE_NAME} characters.` : null;
+ const descriptionError =
+ description.trim().length > MAX_MACHINE_DESCRIPTION
+ ? `At most ${MAX_MACHINE_DESCRIPTION} characters.`
+ : null;
useEffect(() => {
if (!open) return;
let cancelled = false;
setEnrollment(null);
setError(null);
+ setName("");
+ setDescription("");
createEnrollmentCode()
.then((issued) => {
if (!cancelled) setEnrollment(issued);
@@ -90,14 +109,36 @@ export default function AddMachineDialog({
{!error && !enrollment && <CircularProgress size={24} />}
{enrollment && (
<Stack spacing={2}>
+ <Stack spacing={2}>
+ <TextField
+ label="Name (optional)"
+ value={name}
+ onChange={(e) => setName(e.target.value)}
+ error={nameError !== null}
+ helperText={nameError ?? "Defaults to the machine's host name."}
+ size="small"
+ fullWidth
+ />
+ <TextField
+ label="Description (optional)"
+ value={description}
+ onChange={(e) => setDescription(e.target.value)}
+ error={descriptionError !== null}
+ helperText={descriptionError ?? "What the machine is for, e.g. the build box in the office."}
+ size="small"
+ fullWidth
+ />
+ </Stack>
<Box>
<Typography variant="subtitle2" gutterBottom>
Enrollment command
</Typography>
- {enrollment.server_url ? (
+ {enrollment.server_url && (nameError || descriptionError) ? (
+ <Alert severity="warning">Shorten the name or description to see the command.</Alert>
+ ) : enrollment.server_url ? (
<>
<CopyBlock
- text={enrollCommand(enrollment.server_url, enrollment.code)}
+ text={enrollCommand(enrollment.server_url, enrollment.code, { name, description })}
label="Copy command"
/>
<Typography variant="caption" color="text.secondary">
diff --git a/gateway/src/pages/machines/MachinesPage.test.tsx b/gateway/src/pages/machines/MachinesPage.test.tsx
index b3b981d47..4bd07da74 100644
--- a/gateway/src/pages/machines/MachinesPage.test.tsx
+++ b/gateway/src/pages/machines/MachinesPage.test.tsx
@@ -12,6 +12,11 @@ const laptop: Controller = {
version: "0.1.0",
state: "online",
last_seen_at: new Date().toISOString(),
+ connection: {
+ connected_at: new Date().toISOString(),
+ disconnected_at: null,
+ disconnect_reason: null,
+ },
assignment_revision: 3,
created_at: "2026-10-01T00:00:00Z",
revoked_at: null,
@@ -151,6 +156,39 @@ describe("MachinesPage", () => {
expect(screen.getAllByText("running")).toHaveLength(1);
});
+ it("shows a machine that stopped as offline, and its agents as unreachable", async () => {
+ const stopped: Controller = {
+ ...laptop,
+ state: "offline",
+ connection: {
+ ...laptop.connection!,
+ disconnected_at: new Date().toISOString(),
+ disconnect_reason: "socket_closed",
+ },
+ };
+ mockManagement({
+ "GET /controllers": [200, [stopped]],
+ "GET /agents": [200, [{ ...pmAgent, controller_state: "offline" }]],
+ "GET /operations": [200, []],
+ });
+ render(<MachinesPage />);
+ expect(await screen.findByText("offline")).toBeTruthy();
+ expect(screen.getByText("Not connected")).toBeTruthy();
+ expect(screen.getByText("machine offline")).toBeTruthy();
+ expect(screen.queryByText("failed: invalid_credential")).toBeNull();
+ });
+
+ it("shows a machine that never connected as unknown", async () => {
+ mockManagement({
+ "GET /controllers": [200, [{ ...laptop, state: "unknown", connection: null, status: null }]],
+ "GET /agents": [200, []],
+ "GET /operations": [200, []],
+ });
+ render(<MachinesPage />);
+ expect(await screen.findByText("unknown")).toBeTruthy();
+ expect(screen.getByText("Unknown")).toBeTruthy();
+ });
+
it("describes revoking without per-agent keys", async () => {
mockManagement({
"GET /controllers": [200, [laptop]],
diff --git a/gateway/src/pages/machines/MachinesPage.tsx b/gateway/src/pages/machines/MachinesPage.tsx
index 0c019c624..c53ecb887 100644
--- a/gateway/src/pages/machines/MachinesPage.tsx
+++ b/gateway/src/pages/machines/MachinesPage.tsx
@@ -46,14 +46,51 @@ import AddMachineDialog from "./AddMachineDialog";
import EditMachineDialog from "./EditMachineDialog";
import ManagedAgentDialog from "./ManagedAgentDialog";
-const REFRESH_MS = 10_000;
+const REFRESH_MS = 5_000;
const STATE_COLOR: Record<ControllerState, "success" | "warning" | "default"> = {
online: "success",
- unknown: "warning",
+ offline: "warning",
+ unknown: "default",
revoked: "default",
};
+const DISCONNECT_REASON: Record<string, string> = {
+ socket_closed: "its controller stopped or lost its connection",
+ heartbeat_lapsed: "its connection stopped responding",
+ server_shutdown: "the Switch server it was connected to restarted",
+ server_lost: "the Switch server it was connected to stopped responding",
+ taken_over: "another instance of its controller took over",
+ revoked: "it was removed",
+};
+
+/** What the state chip says on hover: when the machine connected, or when and why it stopped. */
+function stateTitle(row: Controller): string {
+ const connection = row.connection;
+ if (row.state === "revoked") return "Removed from Switch.";
+ if (row.state === "unknown" || connection === null)
+ return "Has never connected to Switch. Start its controller with switch-agent-controller run.";
+ if (row.state === "online") return `Connected since ${absoluteTitle(connection.connected_at)}`;
+ const why = connection.disconnect_reason
+ ? (DISCONNECT_REASON[connection.disconnect_reason] ?? connection.disconnect_reason)
+ : null;
+ const when =
+ connection.disconnected_at !== null ? ` ${formatRelative(connection.disconnected_at)}` : "";
+ return `Disconnected${when}${why ? `: ${why}` : ""}.`;
+}
+
+/**
+ * The last time Switch heard from the machine at all: now while it is connected, else the
+ * latest of its status report, its socket attaching and its socket going.
+ */
+function lastContact(row: Controller): string | null {
+ if (row.state === "online") return new Date().toISOString();
+ const times = [row.last_seen_at, row.connection?.connected_at, row.connection?.disconnected_at]
+ .filter((t): t is string => typeof t === "string")
+ .sort((a, b) => Date.parse(b) - Date.parse(a));
+ return times[0] ?? null;
+}
+
const PROCESS_COLOR: Record<string, "success" | "warning" | "error" | "default"> = {
running: "success",
starting: "warning",
@@ -199,7 +236,9 @@ export default function MachinesPage() {
headerName: "State",
width: 110,
renderCell: ({ row }) => (
- <Chip size="small" color={STATE_COLOR[row.state]} label={row.state} />
+ <Tooltip title={stateTitle(row)}>
+ <Chip size="small" color={STATE_COLOR[row.state] ?? "default"} label={row.state} />
+ </Tooltip>
),
},
{
@@ -245,9 +284,11 @@ export default function MachinesPage() {
valueGetter: (_value, row) =>
row.state === "revoked"
? "None (revoked)"
- : row.state === "unknown"
- ? "Unknown"
- : row.status
+ : row.state === "offline"
+ ? "Not connected"
+ : row.state === "unknown"
+ ? "Unknown"
+ : row.status
? `${row.status.agents.filter((a) => a.process === "running").length} / ${row.status.agents.length}`
: EM_DASH,
},
@@ -255,9 +296,10 @@ export default function MachinesPage() {
field: "last_seen_at",
headerName: "Last seen",
width: 130,
+ valueGetter: (_value, row) => lastContact(row),
renderCell: ({ row }) => (
- <Tooltip title={absoluteTitle(row.last_seen_at)}>
- <span>{formatRelative(row.last_seen_at)}</span>
+ <Tooltip title={absoluteTitle(lastContact(row))}>
+ <span>{formatRelative(lastContact(row))}</span>
</Tooltip>
),
},
@@ -372,10 +414,16 @@ export default function MachinesPage() {
<Chip size="small" label="not running: machine revoked" />
</Tooltip>
);
+ if (row.controller_state === "offline")
+ return (
+ <Tooltip title="Its machine is not connected to Switch, so this agent is not reachable. It runs again once the machine's controller is back.">
+ <Chip size="small" color="warning" label="machine offline" />
+ </Tooltip>
+ );
if (row.controller_state === "unknown")
return (
- <Tooltip title="The machine has stopped reporting; this agent's state is not known.">
- <Chip size="small" color="warning" label="unknown" />
+ <Tooltip title="Its machine has never connected to Switch; this agent's state is not known.">
+ <Chip size="small" label="unknown" />
</Tooltip>
);
if (!row.status)