Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

feat(console): move a machine's agents to managed in one batch (#701) (#704)

Louis Amaudruzead70e6ddd17
diff --git a/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration-service.test.ts b/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration-service.test.ts
index 775d2fe26..71099c46d 100644
--- a/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration-service.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration-service.test.ts
@@ -66,6 +66,8 @@ type World = {
controllerRunning: boolean[];
failAdoptOn: string | null;
failStopWatchers: boolean;
+ /** Agents whose watcher does not stop when a batch turns them off. */
+ stuckWatchers: string[];
failStartFresh: boolean;
failDesiredState: boolean;
releaseOutcome: 'released' | 'already_gone';
@@ -143,6 +145,18 @@ function deps(): AgentMigrationDeps {
startConsoleWatcher: async (agent) => {
world.calls.push(`start console ${agent.name}`);
},
+ stopConsoleWatchers: async (agents) => {
+ world.calls.push(`stop console batch ${agents.map((agent) => agent.name).join(',')}`);
+ if (world.failStopWatchers) throw new Error('watcher did not stop');
+ return new Map(
+ agents.map((agent) => [
+ agent.id,
+ world.stuckWatchers.includes(agent.name)
+ ? 'Its watcher on the host did not stop.'
+ : null,
+ ])
+ );
+ },
handoff: async (_agent, request: HandoffRequest) => {
const ids = request.identities.map((identity) => identity.switchAgentId).join(',');
world.calls.push(
@@ -177,6 +191,10 @@ function deps(): AgentMigrationDeps {
restore: async (_agent, identity) => {
world.calls.push(`restore ${identity.slug}`);
},
+ stashMany: async (items) => {
+ world.calls.push(`stash batch ${items.map((item) => item.identity.slug).join(',')}`);
+ return new Map(items.map((item) => [item.agent.id, true]));
+ },
},
store: {
list: async () => [...world.records.values()],
@@ -233,6 +251,7 @@ beforeEach(() => {
controllerRunning: [false],
failAdoptOn: null,
failStopWatchers: false,
+ stuckWatchers: [],
failStartFresh: false,
failDesiredState: false,
releaseOutcome: 'released',
@@ -898,6 +917,64 @@ describe('moving every agent automatically', () => {
expect(await migration.overview()).toMatchObject({ leftAlone: 1, unasked: 1, running: false });
});
+ it('does what happens on a machine once for all its agents, not once per agent', async () => {
+ const SECOND: MigrationAgent = {
+ ...REMOTE,
+ id: 'agent-5',
+ name: 'second',
+ switchAgentId: 'switch-5',
+ };
+ all = [REMOTE, SECOND];
+ await service().migrateEverything();
+ expect(world.calls.filter((call) => call.startsWith('stop console'))).toEqual([
+ 'stop console batch trainer,second',
+ ]);
+ expect(world.calls.filter((call) => call.startsWith('stash'))).toEqual([
+ 'stash batch trainer,second',
+ ]);
+ expect(world.calls.filter((call) => call.startsWith('start-fresh'))).toEqual([
+ 'start-fresh controller switch-2,switch-5',
+ ]);
+ expect(world.calls.filter((call) => call.startsWith('desired'))).toEqual([
+ 'desired switch-2 running',
+ 'desired switch-5 running',
+ ]);
+ });
+
+ it('undoes only the agent whose watcher would not stop, and moves the rest', async () => {
+ const SECOND: MigrationAgent = {
+ ...REMOTE,
+ id: 'agent-5',
+ name: 'second',
+ switchAgentId: 'switch-5',
+ };
+ all = [REMOTE, SECOND];
+ world.stuckWatchers = ['trainer'];
+ const migration = service();
+ await migration.migrateEverything();
+ expect(world.calls).toContain('release switch-2');
+ expect(world.calls).toContain('start console trainer');
+ expect(world.calls).toContain('start-fresh controller switch-5');
+ expect(world.records.has(REMOTE.id)).toBe(false);
+ expect(world.records.get(SECOND.id)?.identities[0]?.credentialsStashed).toBe(true);
+ expect(migration.migrationProblems()).toEqual([
+ expect.objectContaining({
+ name: 'trainer',
+ message: expect.stringMatching(/^Could not stop Console’s watcher for trainer/),
+ }),
+ ]);
+ });
+
+ it('undoes every agent of a machine when starting them on its controller fails', async () => {
+ all = [REMOTE];
+ world.failStartFresh = true;
+ const migration = service();
+ await migration.migrateEverything();
+ expect(world.records.size).toBe(0);
+ expect(world.calls).toContain('restore trainer');
+ expect(migration.migrationProblems()[0]?.message).toMatch(/^Could not move trainer/);
+ });
+
it('runs one pass at a time', async () => {
const migration = service();
await Promise.all([migration.migrateEverything(), migration.migrateEverything()]);
diff --git a/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration-service.ts b/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration-service.ts
index b8d7277f7..8567f41da 100644
--- a/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration-service.ts
+++ b/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration-service.ts
@@ -113,6 +113,11 @@ export interface MigrationMachinePort {
): Promise<{ roomId: string; reason: string }[]>;
/** Stops Console's watcher for the agent, and every session it runs. */
stopConsoleWatcher(agent: MigrationAgent): Promise<void>;
+ /**
+ * Stops Console's watchers for several agents on one machine at once. Per
+ * agent, null once its watcher is down, or why it is not.
+ */
+ stopConsoleWatchers(agents: MigrationAgent[]): Promise<Map<string, string | null>>;
/** Puts Console's watcher for the agent back as its settings say. */
startConsoleWatcher(agent: MigrationAgent): Promise<void>;
handoff(agent: MigrationAgent, request: HandoffRequest): Promise<HandoffResult>;
@@ -122,6 +127,10 @@ export interface MigrationMachinePort {
export interface MigrationCredentialsPort {
/** Keeps the file's token in the encrypted secrets store and removes the file. False when there was none. */
stash(agent: MigrationAgent, identity: { slug: string; switchAgentId: string }): Promise<boolean>;
+ /** `stash` for several agents on one machine at once: per agent, the outcome or the error. */
+ stashMany(
+ items: { agent: MigrationAgent; identity: { slug: string; switchAgentId: string } }[]
+ ): Promise<Map<string, boolean | Error>>;
/** Writes the file back, from the kept token or, failing that, from Switch; then forgets the kept one. */
restore(agent: MigrationAgent, identity: { slug: string; switchAgentId: string }): Promise<void>;
}
@@ -501,20 +510,10 @@ export class AgentMigrationService {
return;
}
if (waiting.length === 0) this.backOff(key, now, this.deps.healIntervalMs);
- for (const agent of group) {
- const record = managed.get(agent.id);
- if (!record && !this.due(agent.id, now)) continue;
+ for (const [agentId, record] of managed) {
+ const agent = group.find((candidate) => candidate.id === agentId)!;
try {
- if (record) {
- if (record.controllerId !== controllerId) await this.replace(agent, record, controllerId);
- } else {
- const state = await this.state(agent.id);
- if (state.runner === 'console') {
- if (state.blocker) throw new MigrationBlockedError(state.blocker);
- await this.moveToManaged(agent.id);
- this.deps.log.info('Moved an agent to managed', { agentId: agent.id });
- }
- }
+ if (record.controllerId !== controllerId) await this.replace(agent, record, controllerId);
this.problems.delete(agent.id);
this.retryAt.delete(agent.id);
} catch (error) {
@@ -522,6 +521,282 @@ export class AgentMigrationService {
this.problem(agent, message(error));
}
}
+ const due = waiting.filter((agent) => this.due(agent.id, now));
+ if (!due.length) return;
+ const failures = await this.moveBatch(due);
+ for (const agent of due) {
+ const failure = failures.get(agent.id);
+ if (failure === undefined) {
+ this.problems.delete(agent.id);
+ this.retryAt.delete(agent.id);
+ } else {
+ this.backOff(agent.id, now);
+ this.problem(agent, message(failure));
+ }
+ }
+ }
+
+ /**
+ * Moves several agents on one machine onto its controller together: what
+ * happens on the machine (turning Console's watchers off, keeping their
+ * credentials aside, starting them on the controller) is one command for all
+ * of them rather than one per agent. Each agent still goes through the
+ * steps of `move`, and one that fails at any step is undone on its own,
+ * leaving the rest to carry on. Returns why each agent that did not move
+ * did not.
+ */
+ private async moveBatch(agents: MigrationAgent[]): Promise<Map<string, unknown>> {
+ const failures = new Map<string, unknown>();
+ const claimed: MigrationAgent[] = [];
+ for (const agent of agents) {
+ if (this.operations.has(agent.id)) {
+ failures.set(agent.id, new MigrationBlockedError('This agent is already being moved.'));
+ continue;
+ }
+ this.operations.set(agent.id, { kind: 'moving', stage: 'checking' });
+ claimed.push(agent);
+ }
+ try {
+ await this.moveClaimed(claimed, failures);
+ } finally {
+ for (const agent of claimed) {
+ this.operations.delete(agent.id);
+ const record = await this.deps.store.get(agent.id).catch(() => null);
+ this.deps.emit({
+ agentId: agent.id,
+ runner: record ? 'managed' : 'console',
+ operation: null,
+ });
+ }
+ }
+ return failures;
+ }
+
+ private async moveClaimed(
+ agents: MigrationAgent[],
+ failures: Map<string, unknown>
+ ): Promise<void> {
+ const first = agents[0];
+ if (!first) return;
+ const lookup = await this.deps.targets.resolve(first);
+ const target = lookup.target;
+ if (!target) {
+ const blocked = new MigrationBlockedError(
+ lookup.blocker ?? 'There is no machine to move it to.'
+ );
+ for (const agent of agents) failures.set(agent.id, blocked);
+ return;
+ }
+ const fail = (moving: Moving, error: unknown) => failures.set(moving.agent.id, error);
+
+ let batch: Moving[] = [];
+ for (const agent of agents) {
+ try {
+ if (target.workspaceId !== agent.workspaceId)
+ throw new MigrationBlockedError(
+ 'The machine runs managed agents for another workspace on this server; an agent can only be placed on a machine of its own workspace.'
+ );
+ batch.push({
+ agent,
+ target,
+ definition: await this.deps.definitions.build(agent),
+ stoppedByHand: await this.deps.agents.stoppedByHand(agent.id),
+ identity: {
+ switchAgentId: agent.switchAgentId!,
+ slug: agent.name,
+ subagent: null,
+ credentialsStashed: false,
+ controllerRoot: target.watcherRoot(agent.switchAgentId!),
+ },
+ });
+ } catch (error) {
+ failures.set(agent.id, error);
+ }
+ }
+
+ await Promise.all(batch.map((moving) => this.tellTurnsCut(moving.agent)));
+
+ const adopted: Moving[] = [];
+ for (const moving of batch) {
+ this.stage(moving.agent.id, 'adopting');
+ try {
+ await this.deps.management.adopt(moving.agent.workspaceId!, moving.identity.switchAgentId, {
+ controller_id: target.controllerId,
+ desired_state: 'stopped',
+ definition: moving.definition.definition,
+ });
+ adopted.push(moving);
+ } catch (error) {
+ fail(moving, error);
+ }
+ }
+ batch = adopted;
+
+ const records = new Map<string, ManagedAgentRecord>();
+ for (const moving of batch) {
+ this.stage(moving.agent.id, 'stopping-console-watcher');
+ records.set(moving.agent.id, this.recordOf(moving));
+ }
+ const stopped = new Map<string, string | null>();
+ try {
+ for (const record of records.values()) await this.deps.store.set(record);
+ for (const [agentId, outcome] of await this.deps.machine.stopConsoleWatchers(
+ batch.map((moving) => moving.agent)
+ ))
+ stopped.set(agentId, outcome);
+ } catch (error) {
+ for (const moving of batch) stopped.set(moving.agent.id, message(error));
+ }
+ batch = await this.keep(batch, async (moving) => {
+ const why = stopped.has(moving.agent.id)
+ ? stopped.get(moving.agent.id)
+ : 'The machine did not report on its watcher.';
+ if (why === null) return;
+ this.deps.log.error('Could not stop Console’s watcher for an agent being moved; undoing', {
+ agentId: moving.agent.id,
+ error: why,
+ });
+ await this.releaseQuietly(moving.agent.workspaceId!, [moving.identity.switchAgentId]);
+ await this.quietly('forget the managed record', () =>
+ this.deps.store.delete(moving.agent.id)
+ );
+ await this.quietly('start Console’s watcher again', () =>
+ this.deps.machine.startConsoleWatcher(moving.agent)
+ );
+ fail(
+ moving,
+ new Error(
+ `Could not stop Console’s watcher for ${moving.agent.name}, so it stays with this Console: ${why}`
+ )
+ );
+ return 'undone';
+ });
+
+ for (const moving of batch) this.stage(moving.agent.id, 'releasing');
+ let stashed: Map<string, boolean | Error>;
+ try {
+ stashed = await this.deps.credentials.stashMany(
+ batch.map((moving) => ({ agent: moving.agent, identity: moving.identity }))
+ );
+ } catch (error) {
+ stashed = new Map(
+ batch.map((moving) => [
+ moving.agent.id,
+ error instanceof Error ? error : new Error(message(error)),
+ ])
+ );
+ }
+ batch = await this.keep(batch, async (moving) => {
+ const outcome =
+ stashed.get(moving.agent.id) ??
+ new Error('The machine did not report on its credentials file.');
+ if (outcome instanceof Error) {
+ await this.undoInBatch(moving, records.get(moving.agent.id)!, false, outcome, fail);
+ return 'undone';
+ }
+ moving.identity.credentialsStashed = outcome;
+ await this.deps.store.set({
+ ...records.get(moving.agent.id)!,
+ identities: [moving.identity],
+ });
+ });
+
+ if (batch.length) {
+ try {
+ await this.deps.machine.handoff(batch[0]!.agent, {
+ op: 'start-fresh',
+ side: 'controller',
+ identities: this.handoffIdentities(batch.map((moving) => moving.identity)),
+ });
+ } catch (error) {
+ for (const moving of batch)
+ await this.undoInBatch(moving, records.get(moving.agent.id)!, false, error, fail);
+ batch = [];
+ }
+ }
+
+ for (const moving of batch) {
+ if (moving.stoppedByHand) continue;
+ try {
+ await this.deps.management.setDesiredState(
+ moving.agent.workspaceId!,
+ moving.identity.switchAgentId,
+ 'running'
+ );
+ } catch (error) {
+ await this.undoInBatch(moving, records.get(moving.agent.id)!, false, error, fail);
+ continue;
+ }
+ this.deps.log.info('Moved an agent onto its controller', {
+ agentId: moving.agent.id,
+ controllerId: target.controllerId,
+ });
+ }
+ for (const moving of batch)
+ if (moving.stoppedByHand)
+ this.deps.log.info('Moved an agent onto its controller, stopped as it was', {
+ agentId: moving.agent.id,
+ controllerId: target.controllerId,
+ });
+ }
+
+ /** The moves for which `step` did not undo anything. */
+ private async keep(
+ batch: Moving[],
+ step: (moving: Moving) => Promise<'undone' | undefined>
+ ): Promise<Moving[]> {
+ const kept: Moving[] = [];
+ for (const moving of batch) if ((await step(moving)) !== 'undone') kept.push(moving);
+ return kept;
+ }
+
+ private async undoInBatch(
+ moving: Moving,
+ record: ManagedAgentRecord,
+ ranOnController: boolean,
+ error: unknown,
+ fail: (moving: Moving, error: unknown) => void
+ ): Promise<void> {
+ this.deps.log.error('Could not finish moving an agent to its controller; undoing', {
+ agentId: moving.agent.id,
+ error: message(error),
+ });
+ await this.undoMove(
+ moving.agent,
+ moving.target,
+ record,
+ this.handoffIdentities([moving.identity]),
+ moving.identity,
+ ranOnController
+ );
+ fail(
+ moving,
+ new Error(
+ `Could not move ${moving.agent.name}, so it stays with this Console: ${message(error)}`,
+ {
+ cause: error,
+ }
+ )
+ );
+ }
+
+ private recordOf(moving: Moving): ManagedAgentRecord {
+ const { agent, target, identity } = moving;
+ return {
+ agentId: agent.id,
+ workspaceId: agent.workspaceId!,
+ controllerId: target.controllerId,
+ placement:
+ target.display.kind === 'this-computer'
+ ? { kind: 'this-computer', serverId: target.display.serverId }
+ : {
+ kind: 'ssh-host',
+ sshHost: target.display.sshHost,
+ serverId: target.display.serverId,
+ },
+ identities: [identity],
+ movedAt: new Date(this.deps.now()).toISOString(),
+ };
}
/**
@@ -792,21 +1067,7 @@ export class AgentMigrationService {
definition: definition.definition,
});
- const record: ManagedAgentRecord = {
- agentId: agent.id,
- workspaceId,
- controllerId: target.controllerId,
- placement:
- target.display.kind === 'this-computer'
- ? { kind: 'this-computer', serverId: target.display.serverId }
- : {
- kind: 'ssh-host',
- sshHost: target.display.sshHost,
- serverId: target.display.serverId,
- },
- identities: [identity],
- movedAt: new Date(this.deps.now()).toISOString(),
- };
+ const record = this.recordOf(moving);
this.stage(agent.id, 'stopping-console-watcher');
try {
await this.deps.store.set(record);
diff --git a/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration.ts b/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration.ts
index eb5f90ef1..25f07e34e 100644
--- a/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration.ts
+++ b/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration.ts
@@ -57,6 +57,7 @@ import {
type MigrationManagementPort,
type TargetLookup,
} from './agent-migration-service';
+import { deleteFiles, readFiles, stopWatchers } from './host-batch';
import {
credentialsStashKey,
deleteManagedAgentRecord,
@@ -401,6 +402,35 @@ const machine: MigrationMachinePort = {
if (agent.sshHost) return;
await stopLocalSessionsOf(agent.switchAgentId!);
},
+ stopConsoleWatchers: async (agents) => {
+ const outcomes = new Map<string, string | null>();
+ const first = agents[0];
+ if (!first) return outcomes;
+ if (!first.sshHost) {
+ for (const agent of agents)
+ try {
+ await machine.stopConsoleWatcher(agent);
+ outcomes.set(agent.id, null);
+ } catch (error) {
+ outcomes.set(agent.id, error instanceof Error ? error.message : String(error));
+ }
+ return outcomes;
+ }
+ for (const agent of agents) autoSessionWatcher.forgetRetries(agent.id);
+ const stopped = await stopWatchers(
+ machineScript(first),
+ agents.map((agent) => agent.switchAgentId!),
+ { waitMs: 20_000, killWaitMs: 5_000 }
+ );
+ for (const agent of agents)
+ outcomes.set(
+ agent.id,
+ agent.switchAgentId! in stopped
+ ? (stopped[agent.switchAgentId!] ?? null)
+ : 'The host did not report on its watcher.'
+ );
+ return outcomes;
+ },
startConsoleWatcher: async (agent) => {
await autoSessionWatcher.bringUp(agent.id, 'explicit');
},
@@ -409,6 +439,32 @@ const machine: MigrationMachinePort = {
const credentialsLog = { warn: (...input: unknown[]) => log.warn('agent-migration:', ...input) };
+/**
+ * Keeps the token of an agent's credentials file in the encrypted secrets
+ * store, refusing a file that is incomplete or names another agent. The
+ * caller removes the file once this returns.
+ */
+async function keepCredentials(
+ agent: MigrationAgent,
+ identity: { slug: string; switchAgentId: string },
+ raw: string
+): Promise<void> {
+ const relPath = agentSettingsRelativePath(identity.slug);
+ const parsed = parseSwitchAgentCredentials(raw, credentialsLog);
+ if (!parsed)
+ throw new Error(
+ `${relPath} does not hold complete Switch credentials, so it is left as it is.`
+ );
+ if (parsed.agentId !== identity.switchAgentId)
+ throw new Error(
+ `${relPath} names Switch agent ${parsed.agentId}, not ${identity.switchAgentId}; it is left as it is.`
+ );
+ await encryptedAppSecretsStore.setSecret(
+ credentialsStashKey(agent.id, identity.switchAgentId),
+ JSON.stringify({ endpoint: parsed.apiEndpoint, token: parsed.token })
+ );
+}
+
const credentials: MigrationCredentialsPort = {
stash: async (agent, identity) => {
const relPath = agentSettingsRelativePath(identity.slug);
@@ -416,25 +472,48 @@ const credentials: MigrationCredentialsPort = {
try {
const raw = await workdir.fs.read(relPath);
if (raw === null) return false;
- const parsed = parseSwitchAgentCredentials(raw, credentialsLog);
- if (!parsed)
- throw new Error(
- `${relPath} does not hold complete Switch credentials, so it is left as it is.`
- );
- if (parsed.agentId !== identity.switchAgentId)
- throw new Error(
- `${relPath} names Switch agent ${parsed.agentId}, not ${identity.switchAgentId}; it is left as it is.`
- );
- await encryptedAppSecretsStore.setSecret(
- credentialsStashKey(agent.id, identity.switchAgentId),
- JSON.stringify({ endpoint: parsed.apiEndpoint, token: parsed.token })
- );
+ await keepCredentials(agent, identity, raw);
await workdir.fs.delete(relPath);
return true;
} finally {
workdir.close();
}
},
+ stashMany: async (items) => {
+ const outcomes = new Map<string, boolean | Error>();
+ const first = items[0];
+ if (!first) return outcomes;
+ if (!first.agent.sshHost) {
+ for (const { agent, identity } of items)
+ try {
+ outcomes.set(agent.id, await credentials.stash(agent, identity));
+ } catch (error) {
+ outcomes.set(agent.id, error instanceof Error ? error : new Error(String(error)));
+ }
+ return outcomes;
+ }
+ const run = machineScript(first.agent);
+ const pathOf = (item: (typeof items)[number]) =>
+ posix.join(item.agent.dir, agentSettingsRelativePath(item.identity.slug));
+ const files = await readFiles(run, items.map(pathOf));
+ const kept: string[] = [];
+ for (const item of items) {
+ const raw = files[pathOf(item)] ?? null;
+ if (raw === null) {
+ outcomes.set(item.agent.id, false);
+ continue;
+ }
+ try {
+ await keepCredentials(item.agent, item.identity, raw);
+ kept.push(pathOf(item));
+ outcomes.set(item.agent.id, true);
+ } catch (error) {
+ outcomes.set(item.agent.id, error instanceof Error ? error : new Error(String(error)));
+ }
+ }
+ if (kept.length) await deleteFiles(run, kept);
+ return outcomes;
+ },
restore: async (agent, identity) => {
const key = credentialsStashKey(agent.id, identity.switchAgentId);
if ((await identityInFile(agent, identity.slug)) === identity.switchAgentId) {
diff --git a/console/apps/switch-console-desktop/src/main/core/agent-migration/host-batch.test.ts b/console/apps/switch-console-desktop/src/main/core/agent-migration/host-batch.test.ts
new file mode 100644
index 000000000..5e7dee265
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/main/core/agent-migration/host-batch.test.ts
@@ -0,0 +1,106 @@
+import { type ChildProcess, execFile, spawn } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { promisify } from 'node:util';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { deleteFiles, readFiles, stopWatchers } from './host-batch';
+import type { MachineScript } from './session-handoff';
+
+const execute = promisify(execFile);
+
+let home: string;
+const children: ChildProcess[] = [];
+
+/** `node -e` with `home` as its home directory, as the scripts run on a host. */
+const run: MachineScript = async (script, args) =>
+ (
+ await execute(process.execPath, ['-e', script, ...args], {
+ env: { ...process.env, HOME: home },
+ })
+ ).stdout;
+
+const rootOf = (identity: string) =>
+ join(
+ home,
+ '.local',
+ 'state',
+ 'switch',
+ 'sdk-watchers',
+ createHash('sha256').update(identity).digest('hex')
+ );
+
+/** A stand-in watcher: a process whose command line names the shared host, recorded as the root's owner. */
+async function watcher(
+ identity: string,
+ behaviour: 'exits-when-off' | 'ignores-sigterm' | 'not-ours'
+) {
+ const root = rootOf(identity);
+ await mkdir(join(root, 'supervisor'), { recursive: true });
+ await writeFile(join(root, 'config.json'), JSON.stringify({ session: { agentId: identity } }));
+ await writeFile(join(root, 'watch.json'), JSON.stringify({ enabled: true }));
+ const body = {
+ 'exits-when-off': `const fs=require('fs');setInterval(()=>{if(!JSON.parse(fs.readFileSync(process.argv[1]+'/watch.json','utf8')).enabled)process.exit(0)},100)`,
+ 'ignores-sigterm': `process.on('SIGTERM',()=>{});setInterval(()=>{},1000)`,
+ 'not-ours': `setInterval(()=>{},1000)`,
+ }[behaviour];
+ const name = behaviour === 'not-ours' ? 'something-else' : 'shared-host-test.mjs';
+ const child = spawn(process.execPath, ['-e', body, root, name], { stdio: 'ignore' });
+ children.push(child);
+ await writeFile(join(root, 'shared-owner.lock'), JSON.stringify({ pid: child.pid }));
+ return child;
+}
+
+const exited = (child: ChildProcess) => child.exitCode !== null || child.signalCode !== null;
+
+beforeEach(async () => {
+ home = await mkdtemp(join(tmpdir(), 'host-batch-'));
+});
+
+afterEach(async () => {
+ for (const child of children.splice(0)) if (!exited(child)) child.kill('SIGKILL');
+ await rm(home, { recursive: true, force: true });
+});
+
+describe('stopping several watchers at once', () => {
+ it('turns each off and waits for them together', async () => {
+ const a = await watcher('agent-a', 'exits-when-off');
+ const b = await watcher('agent-b', 'exits-when-off');
+ const outcome = await stopWatchers(run, ['agent-a', 'agent-b', 'agent-none'], {
+ waitMs: 5_000,
+ killWaitMs: 1_000,
+ });
+ expect(outcome).toEqual({ 'agent-a': null, 'agent-b': null, 'agent-none': null });
+ expect(JSON.parse(await readFile(join(rootOf('agent-a'), 'watch.json'), 'utf8'))).toEqual({
+ enabled: false,
+ spawn: false,
+ });
+ await expect.poll(() => exited(a) && exited(b)).toBe(true);
+ });
+
+ it('kills a watcher that does not stop by itself', async () => {
+ const stuck = await watcher('agent-stuck', 'ignores-sigterm');
+ const outcome = await stopWatchers(run, ['agent-stuck'], { waitMs: 300, killWaitMs: 300 });
+ expect(outcome).toEqual({ 'agent-stuck': null });
+ await expect.poll(() => exited(stuck)).toBe(true);
+ });
+
+ it('never signals a process that is not a watcher, and says the watcher is still up', async () => {
+ const other = await watcher('agent-other', 'not-ours');
+ const outcome = await stopWatchers(run, ['agent-other'], { waitMs: 200, killWaitMs: 200 });
+ expect(outcome['agent-other']).toMatch(/did not stop/);
+ expect(exited(other)).toBe(false);
+ });
+});
+
+describe('credentials files on a host', () => {
+ it('reads several in one command, and removes the ones asked', async () => {
+ const one = join(home, 'one.json');
+ const two = join(home, 'two.json');
+ await writeFile(one, '{"a":1}');
+ expect(await readFiles(run, [one, two])).toEqual({ [one]: '{"a":1}', [two]: null });
+ await deleteFiles(run, [one, two]);
+ expect(await readFiles(run, [one])).toEqual({ [one]: null });
+ });
+});
diff --git a/console/apps/switch-console-desktop/src/main/core/agent-migration/host-batch.ts b/console/apps/switch-console-desktop/src/main/core/agent-migration/host-batch.ts
new file mode 100644
index 000000000..43572f4ac
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/main/core/agent-migration/host-batch.ts
@@ -0,0 +1,132 @@
+import { z } from 'zod';
+import { WATCHER_ROOT } from '@main/core/sdk-host/state-roots';
+import type { MachineScript } from './session-handoff';
+
+/**
+ * Turns off the Console watchers of several agents on one host, in one
+ * command, and waits for them all at once.
+ *
+ * Moving an agent makes Switch close its watcher's connection, which the
+ * watcher's supervisor takes for a crash and answers by sleeping before a
+ * restart; one asleep then is slow to notice it was turned off. So a watcher
+ * still up when the wait runs out is sent SIGTERM, then SIGKILL, rather than
+ * left running. Only a process whose command line names the shared host is
+ * signalled, so a PID file outliving its process cannot take another one down.
+ *
+ * Takes the JSON of `{ identities, waitMs, killWaitMs }` and prints, per
+ * identity, null when its watcher is down or the reason it is not.
+ */
+export const STOP_WATCHERS_SCRIPT = String.raw`${WATCHER_ROOT}
+const fs = require('node:fs'), path = require('node:path'), crypto = require('node:crypto');
+const cp = require('node:child_process');
+const o = JSON.parse(process.argv[1]);
+const base = path.join(require('node:os').homedir(), '.local', 'state', 'switch', 'sdk-watchers');
+const put = (file, data) => {
+ const tmp = file + '.' + crypto.randomUUID();
+ fs.writeFileSync(tmp, JSON.stringify(data), { mode: 0o600 });
+ fs.renameSync(tmp, file);
+};
+const pidIn = (root, file) => {
+ try {
+ const pid = JSON.parse(fs.readFileSync(path.join(root, file), 'utf8')).pid;
+ return Number.isSafeInteger(pid) && pid > 0 ? pid : null;
+ } catch (e) {
+ if (e.code === 'ENOENT') return null;
+ throw e;
+ }
+};
+const alive = (pid) => {
+ try { process.kill(pid, 0); return true; } catch (e) { return e.code === 'EPERM'; }
+};
+const commandOf = (pid) => {
+ try { return fs.readFileSync('/proc/' + pid + '/cmdline', 'utf8').split('\0').join(' '); }
+ catch (e) {
+ try { return cp.execFileSync('ps', ['-o', 'command=', '-p', String(pid)], { encoding: 'utf8' }); }
+ catch { return ''; }
+ }
+};
+const sleep = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+const out = {}, roots = {};
+for (const identity of o.identities) {
+ try {
+ const root = watcherRoot(base, identity);
+ if (!fs.existsSync(root)) { out[identity] = null; continue; }
+ try { fs.unlinkSync(path.join(root, 'taken-over.json')); } catch (e) { if (e.code !== 'ENOENT') throw e; }
+ put(path.join(root, 'watch.json'), { enabled: false, spawn: false });
+ roots[identity] = root;
+ } catch (e) {
+ out[identity] = e.message;
+ }
+}
+const live = (identity) =>
+ ['shared-owner.lock', 'supervisor/owner.json']
+ .map((file) => pidIn(roots[identity], file))
+ .filter((pid) => pid !== null && alive(pid));
+const signal = (identities, sig) => {
+ for (const identity of identities)
+ for (const pid of live(identity))
+ if (commandOf(pid).includes('shared-host')) try { process.kill(pid, sig); } catch {}
+};
+let deadline = Date.now() + o.waitMs, stage = 0;
+for (;;) {
+ const pending = Object.keys(roots).filter((identity) => live(identity).length > 0);
+ if (!pending.length) break;
+ if (Date.now() >= deadline) {
+ if (stage === 2) {
+ for (const identity of pending) out[identity] = 'Its watcher on the host did not stop, even when killed.';
+ break;
+ }
+ signal(pending, stage === 0 ? 'SIGTERM' : 'SIGKILL');
+ stage += 1;
+ deadline = Date.now() + o.killWaitMs;
+ }
+ sleep(200);
+}
+for (const identity of Object.keys(roots)) if (!(identity in out)) out[identity] = null;
+console.log(JSON.stringify(out));
+`;
+
+/**
+ * Reads several files on a host in one command. Takes a JSON array of absolute
+ * paths and prints an object from path to its text, or null where there is none.
+ */
+export const READ_FILES_SCRIPT = String.raw`
+const fs = require('node:fs');
+const out = {};
+for (const file of JSON.parse(process.argv[1])) {
+ try { out[file] = fs.readFileSync(file, 'utf8'); }
+ catch (e) { if (e.code !== 'ENOENT') throw e; out[file] = null; }
+}
+console.log(JSON.stringify(out));
+`;
+
+/** Removes several files on a host in one command. Takes a JSON array of absolute paths. */
+export const DELETE_FILES_SCRIPT = String.raw`
+const fs = require('node:fs');
+for (const file of JSON.parse(process.argv[1])) fs.rmSync(file, { force: true });
+console.log('{}');
+`;
+
+const lastLine = (stdout: string): unknown => JSON.parse(stdout.trim().split('\n').at(-1) ?? '');
+
+/** Per identity, null when its watcher is down, or why it is not. */
+export async function stopWatchers(
+ run: MachineScript,
+ identities: string[],
+ timing: { waitMs: number; killWaitMs: number }
+): Promise<Record<string, string | null>> {
+ const stdout = await run(STOP_WATCHERS_SCRIPT, [JSON.stringify({ identities, ...timing })]);
+ return z.record(z.string(), z.string().nullable()).parse(lastLine(stdout));
+}
+
+export async function readFiles(
+ run: MachineScript,
+ paths: string[]
+): Promise<Record<string, string | null>> {
+ const stdout = await run(READ_FILES_SCRIPT, [JSON.stringify(paths)]);
+ return z.record(z.string(), z.string().nullable()).parse(lastLine(stdout));
+}
+
+export async function deleteFiles(run: MachineScript, paths: string[]): Promise<void> {
+ await run(DELETE_FILES_SCRIPT, [JSON.stringify(paths)]);
+}
diff --git a/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.ts b/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.ts
index 629bc0fb4..4107ae1e8 100644
--- a/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.ts
+++ b/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.ts
@@ -257,6 +257,11 @@ class AutoSessionWatcher {
this.retries.delete(agentId);
}
+ /** Stops retrying the agent's watcher, for a caller that turns it off by other means. */
+ forgetRetries(agentId: string): void {
+ this.cancelRetry(agentId);
+ }
+
stopForAgent(agentId: string): Promise<void> {
this.cancelRetry(agentId);
return configureAgentHost(agentId, { connected: false, spawning: false }, 'restore');