Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
feat(console): move a machine's agents to managed in one batch (#701) (#704)
Louis Amaudruzead70e6ddd17
diff --git a/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration-service.test.ts b/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration-service.test.tsindex 775d2fe26..71099c46d 100644--- a/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration-service.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration-service.test.ts@@ -66,6 +66,8 @@ type World = {controllerRunning: boolean[];failAdoptOn: string | null;failStopWatchers: boolean;+ /** Agents whose watcher does not stop when a batch turns them off. */+ stuckWatchers: string[];failStartFresh: boolean;failDesiredState: boolean;releaseOutcome: 'released' | 'already_gone';@@ -143,6 +145,18 @@ function deps(): AgentMigrationDeps {startConsoleWatcher: async (agent) => {world.calls.push(`start console ${agent.name}`);},+ stopConsoleWatchers: async (agents) => {+ world.calls.push(`stop console batch ${agents.map((agent) => agent.name).join(',')}`);+ if (world.failStopWatchers) throw new Error('watcher did not stop');+ return new Map(+ agents.map((agent) => [+ agent.id,+ world.stuckWatchers.includes(agent.name)+ ? 'Its watcher on the host did not stop.'+ : null,+ ])+ );+ },handoff: async (_agent, request: HandoffRequest) => {const ids = request.identities.map((identity) => identity.switchAgentId).join(',');world.calls.push(@@ -177,6 +191,10 @@ function deps(): AgentMigrationDeps {restore: async (_agent, identity) => {world.calls.push(`restore ${identity.slug}`);},+ stashMany: async (items) => {+ world.calls.push(`stash batch ${items.map((item) => item.identity.slug).join(',')}`);+ return new Map(items.map((item) => [item.agent.id, true]));+ },},store: {list: async () => [...world.records.values()],@@ -233,6 +251,7 @@ beforeEach(() => {controllerRunning: [false],failAdoptOn: null,failStopWatchers: false,+ stuckWatchers: [],failStartFresh: false,failDesiredState: false,releaseOutcome: 'released',@@ -898,6 +917,64 @@ describe('moving every agent automatically', () => {expect(await migration.overview()).toMatchObject({ leftAlone: 1, unasked: 1, running: false });});+ it('does what happens on a machine once for all its agents, not once per agent', async () => {+ const SECOND: MigrationAgent = {+ ...REMOTE,+ id: 'agent-5',+ name: 'second',+ switchAgentId: 'switch-5',+ };+ all = [REMOTE, SECOND];+ await service().migrateEverything();+ expect(world.calls.filter((call) => call.startsWith('stop console'))).toEqual([+ 'stop console batch trainer,second',+ ]);+ expect(world.calls.filter((call) => call.startsWith('stash'))).toEqual([+ 'stash batch trainer,second',+ ]);+ expect(world.calls.filter((call) => call.startsWith('start-fresh'))).toEqual([+ 'start-fresh controller switch-2,switch-5',+ ]);+ expect(world.calls.filter((call) => call.startsWith('desired'))).toEqual([+ 'desired switch-2 running',+ 'desired switch-5 running',+ ]);+ });++ it('undoes only the agent whose watcher would not stop, and moves the rest', async () => {+ const SECOND: MigrationAgent = {+ ...REMOTE,+ id: 'agent-5',+ name: 'second',+ switchAgentId: 'switch-5',+ };+ all = [REMOTE, SECOND];+ world.stuckWatchers = ['trainer'];+ const migration = service();+ await migration.migrateEverything();+ expect(world.calls).toContain('release switch-2');+ expect(world.calls).toContain('start console trainer');+ expect(world.calls).toContain('start-fresh controller switch-5');+ expect(world.records.has(REMOTE.id)).toBe(false);+ expect(world.records.get(SECOND.id)?.identities[0]?.credentialsStashed).toBe(true);+ expect(migration.migrationProblems()).toEqual([+ expect.objectContaining({+ name: 'trainer',+ message: expect.stringMatching(/^Could not stop Console’s watcher for trainer/),+ }),+ ]);+ });++ it('undoes every agent of a machine when starting them on its controller fails', async () => {+ all = [REMOTE];+ world.failStartFresh = true;+ const migration = service();+ await migration.migrateEverything();+ expect(world.records.size).toBe(0);+ expect(world.calls).toContain('restore trainer');+ expect(migration.migrationProblems()[0]?.message).toMatch(/^Could not move trainer/);+ });+it('runs one pass at a time', async () => {const migration = service();await Promise.all([migration.migrateEverything(), migration.migrateEverything()]);diff --git a/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration-service.ts b/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration-service.tsindex b8d7277f7..8567f41da 100644--- a/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration-service.ts+++ b/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration-service.ts@@ -113,6 +113,11 @@ export interface MigrationMachinePort {): Promise<{ roomId: string; reason: string }[]>;/** Stops Console's watcher for the agent, and every session it runs. */stopConsoleWatcher(agent: MigrationAgent): Promise<void>;+ /**+ * Stops Console's watchers for several agents on one machine at once. Per+ * agent, null once its watcher is down, or why it is not.+ */+ stopConsoleWatchers(agents: MigrationAgent[]): Promise<Map<string, string | null>>;/** Puts Console's watcher for the agent back as its settings say. */startConsoleWatcher(agent: MigrationAgent): Promise<void>;handoff(agent: MigrationAgent, request: HandoffRequest): Promise<HandoffResult>;@@ -122,6 +127,10 @@ export interface MigrationMachinePort {export interface MigrationCredentialsPort {/** Keeps the file's token in the encrypted secrets store and removes the file. False when there was none. */stash(agent: MigrationAgent, identity: { slug: string; switchAgentId: string }): Promise<boolean>;+ /** `stash` for several agents on one machine at once: per agent, the outcome or the error. */+ stashMany(+ items: { agent: MigrationAgent; identity: { slug: string; switchAgentId: string } }[]+ ): Promise<Map<string, boolean | Error>>;/** Writes the file back, from the kept token or, failing that, from Switch; then forgets the kept one. */restore(agent: MigrationAgent, identity: { slug: string; switchAgentId: string }): Promise<void>;}@@ -501,20 +510,10 @@ export class AgentMigrationService {return;}if (waiting.length === 0) this.backOff(key, now, this.deps.healIntervalMs);- for (const agent of group) {- const record = managed.get(agent.id);- if (!record && !this.due(agent.id, now)) continue;+ for (const [agentId, record] of managed) {+ const agent = group.find((candidate) => candidate.id === agentId)!;try {- if (record) {- if (record.controllerId !== controllerId) await this.replace(agent, record, controllerId);- } else {- const state = await this.state(agent.id);- if (state.runner === 'console') {- if (state.blocker) throw new MigrationBlockedError(state.blocker);- await this.moveToManaged(agent.id);- this.deps.log.info('Moved an agent to managed', { agentId: agent.id });- }- }+ if (record.controllerId !== controllerId) await this.replace(agent, record, controllerId);this.problems.delete(agent.id);this.retryAt.delete(agent.id);} catch (error) {@@ -522,6 +521,282 @@ export class AgentMigrationService {this.problem(agent, message(error));}}+ const due = waiting.filter((agent) => this.due(agent.id, now));+ if (!due.length) return;+ const failures = await this.moveBatch(due);+ for (const agent of due) {+ const failure = failures.get(agent.id);+ if (failure === undefined) {+ this.problems.delete(agent.id);+ this.retryAt.delete(agent.id);+ } else {+ this.backOff(agent.id, now);+ this.problem(agent, message(failure));+ }+ }+ }++ /**+ * Moves several agents on one machine onto its controller together: what+ * happens on the machine (turning Console's watchers off, keeping their+ * credentials aside, starting them on the controller) is one command for all+ * of them rather than one per agent. Each agent still goes through the+ * steps of `move`, and one that fails at any step is undone on its own,+ * leaving the rest to carry on. Returns why each agent that did not move+ * did not.+ */+ private async moveBatch(agents: MigrationAgent[]): Promise<Map<string, unknown>> {+ const failures = new Map<string, unknown>();+ const claimed: MigrationAgent[] = [];+ for (const agent of agents) {+ if (this.operations.has(agent.id)) {+ failures.set(agent.id, new MigrationBlockedError('This agent is already being moved.'));+ continue;+ }+ this.operations.set(agent.id, { kind: 'moving', stage: 'checking' });+ claimed.push(agent);+ }+ try {+ await this.moveClaimed(claimed, failures);+ } finally {+ for (const agent of claimed) {+ this.operations.delete(agent.id);+ const record = await this.deps.store.get(agent.id).catch(() => null);+ this.deps.emit({+ agentId: agent.id,+ runner: record ? 'managed' : 'console',+ operation: null,+ });+ }+ }+ return failures;+ }++ private async moveClaimed(+ agents: MigrationAgent[],+ failures: Map<string, unknown>+ ): Promise<void> {+ const first = agents[0];+ if (!first) return;+ const lookup = await this.deps.targets.resolve(first);+ const target = lookup.target;+ if (!target) {+ const blocked = new MigrationBlockedError(+ lookup.blocker ?? 'There is no machine to move it to.'+ );+ for (const agent of agents) failures.set(agent.id, blocked);+ return;+ }+ const fail = (moving: Moving, error: unknown) => failures.set(moving.agent.id, error);++ let batch: Moving[] = [];+ for (const agent of agents) {+ try {+ if (target.workspaceId !== agent.workspaceId)+ throw new MigrationBlockedError(+ 'The machine runs managed agents for another workspace on this server; an agent can only be placed on a machine of its own workspace.'+ );+ batch.push({+ agent,+ target,+ definition: await this.deps.definitions.build(agent),+ stoppedByHand: await this.deps.agents.stoppedByHand(agent.id),+ identity: {+ switchAgentId: agent.switchAgentId!,+ slug: agent.name,+ subagent: null,+ credentialsStashed: false,+ controllerRoot: target.watcherRoot(agent.switchAgentId!),+ },+ });+ } catch (error) {+ failures.set(agent.id, error);+ }+ }++ await Promise.all(batch.map((moving) => this.tellTurnsCut(moving.agent)));++ const adopted: Moving[] = [];+ for (const moving of batch) {+ this.stage(moving.agent.id, 'adopting');+ try {+ await this.deps.management.adopt(moving.agent.workspaceId!, moving.identity.switchAgentId, {+ controller_id: target.controllerId,+ desired_state: 'stopped',+ definition: moving.definition.definition,+ });+ adopted.push(moving);+ } catch (error) {+ fail(moving, error);+ }+ }+ batch = adopted;++ const records = new Map<string, ManagedAgentRecord>();+ for (const moving of batch) {+ this.stage(moving.agent.id, 'stopping-console-watcher');+ records.set(moving.agent.id, this.recordOf(moving));+ }+ const stopped = new Map<string, string | null>();+ try {+ for (const record of records.values()) await this.deps.store.set(record);+ for (const [agentId, outcome] of await this.deps.machine.stopConsoleWatchers(+ batch.map((moving) => moving.agent)+ ))+ stopped.set(agentId, outcome);+ } catch (error) {+ for (const moving of batch) stopped.set(moving.agent.id, message(error));+ }+ batch = await this.keep(batch, async (moving) => {+ const why = stopped.has(moving.agent.id)+ ? stopped.get(moving.agent.id)+ : 'The machine did not report on its watcher.';+ if (why === null) return;+ this.deps.log.error('Could not stop Console’s watcher for an agent being moved; undoing', {+ agentId: moving.agent.id,+ error: why,+ });+ await this.releaseQuietly(moving.agent.workspaceId!, [moving.identity.switchAgentId]);+ await this.quietly('forget the managed record', () =>+ this.deps.store.delete(moving.agent.id)+ );+ await this.quietly('start Console’s watcher again', () =>+ this.deps.machine.startConsoleWatcher(moving.agent)+ );+ fail(+ moving,+ new Error(+ `Could not stop Console’s watcher for ${moving.agent.name}, so it stays with this Console: ${why}`+ )+ );+ return 'undone';+ });++ for (const moving of batch) this.stage(moving.agent.id, 'releasing');+ let stashed: Map<string, boolean | Error>;+ try {+ stashed = await this.deps.credentials.stashMany(+ batch.map((moving) => ({ agent: moving.agent, identity: moving.identity }))+ );+ } catch (error) {+ stashed = new Map(+ batch.map((moving) => [+ moving.agent.id,+ error instanceof Error ? error : new Error(message(error)),+ ])+ );+ }+ batch = await this.keep(batch, async (moving) => {+ const outcome =+ stashed.get(moving.agent.id) ??+ new Error('The machine did not report on its credentials file.');+ if (outcome instanceof Error) {+ await this.undoInBatch(moving, records.get(moving.agent.id)!, false, outcome, fail);+ return 'undone';+ }+ moving.identity.credentialsStashed = outcome;+ await this.deps.store.set({+ ...records.get(moving.agent.id)!,+ identities: [moving.identity],+ });+ });++ if (batch.length) {+ try {+ await this.deps.machine.handoff(batch[0]!.agent, {+ op: 'start-fresh',+ side: 'controller',+ identities: this.handoffIdentities(batch.map((moving) => moving.identity)),+ });+ } catch (error) {+ for (const moving of batch)+ await this.undoInBatch(moving, records.get(moving.agent.id)!, false, error, fail);+ batch = [];+ }+ }++ for (const moving of batch) {+ if (moving.stoppedByHand) continue;+ try {+ await this.deps.management.setDesiredState(+ moving.agent.workspaceId!,+ moving.identity.switchAgentId,+ 'running'+ );+ } catch (error) {+ await this.undoInBatch(moving, records.get(moving.agent.id)!, false, error, fail);+ continue;+ }+ this.deps.log.info('Moved an agent onto its controller', {+ agentId: moving.agent.id,+ controllerId: target.controllerId,+ });+ }+ for (const moving of batch)+ if (moving.stoppedByHand)+ this.deps.log.info('Moved an agent onto its controller, stopped as it was', {+ agentId: moving.agent.id,+ controllerId: target.controllerId,+ });+ }++ /** The moves for which `step` did not undo anything. */+ private async keep(+ batch: Moving[],+ step: (moving: Moving) => Promise<'undone' | undefined>+ ): Promise<Moving[]> {+ const kept: Moving[] = [];+ for (const moving of batch) if ((await step(moving)) !== 'undone') kept.push(moving);+ return kept;+ }++ private async undoInBatch(+ moving: Moving,+ record: ManagedAgentRecord,+ ranOnController: boolean,+ error: unknown,+ fail: (moving: Moving, error: unknown) => void+ ): Promise<void> {+ this.deps.log.error('Could not finish moving an agent to its controller; undoing', {+ agentId: moving.agent.id,+ error: message(error),+ });+ await this.undoMove(+ moving.agent,+ moving.target,+ record,+ this.handoffIdentities([moving.identity]),+ moving.identity,+ ranOnController+ );+ fail(+ moving,+ new Error(+ `Could not move ${moving.agent.name}, so it stays with this Console: ${message(error)}`,+ {+ cause: error,+ }+ )+ );+ }++ private recordOf(moving: Moving): ManagedAgentRecord {+ const { agent, target, identity } = moving;+ return {+ agentId: agent.id,+ workspaceId: agent.workspaceId!,+ controllerId: target.controllerId,+ placement:+ target.display.kind === 'this-computer'+ ? { kind: 'this-computer', serverId: target.display.serverId }+ : {+ kind: 'ssh-host',+ sshHost: target.display.sshHost,+ serverId: target.display.serverId,+ },+ identities: [identity],+ movedAt: new Date(this.deps.now()).toISOString(),+ };}/**@@ -792,21 +1067,7 @@ export class AgentMigrationService {definition: definition.definition,});- const record: ManagedAgentRecord = {- agentId: agent.id,- workspaceId,- controllerId: target.controllerId,- placement:- target.display.kind === 'this-computer'- ? { kind: 'this-computer', serverId: target.display.serverId }- : {- kind: 'ssh-host',- sshHost: target.display.sshHost,- serverId: target.display.serverId,- },- identities: [identity],- movedAt: new Date(this.deps.now()).toISOString(),- };+ const record = this.recordOf(moving);this.stage(agent.id, 'stopping-console-watcher');try {await this.deps.store.set(record);diff --git a/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration.ts b/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration.tsindex eb5f90ef1..25f07e34e 100644--- a/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration.ts+++ b/console/apps/switch-console-desktop/src/main/core/agent-migration/agent-migration.ts@@ -57,6 +57,7 @@ import {type MigrationManagementPort,type TargetLookup,} from './agent-migration-service';+import { deleteFiles, readFiles, stopWatchers } from './host-batch';import {credentialsStashKey,deleteManagedAgentRecord,@@ -401,6 +402,35 @@ const machine: MigrationMachinePort = {if (agent.sshHost) return;await stopLocalSessionsOf(agent.switchAgentId!);},+ stopConsoleWatchers: async (agents) => {+ const outcomes = new Map<string, string | null>();+ const first = agents[0];+ if (!first) return outcomes;+ if (!first.sshHost) {+ for (const agent of agents)+ try {+ await machine.stopConsoleWatcher(agent);+ outcomes.set(agent.id, null);+ } catch (error) {+ outcomes.set(agent.id, error instanceof Error ? error.message : String(error));+ }+ return outcomes;+ }+ for (const agent of agents) autoSessionWatcher.forgetRetries(agent.id);+ const stopped = await stopWatchers(+ machineScript(first),+ agents.map((agent) => agent.switchAgentId!),+ { waitMs: 20_000, killWaitMs: 5_000 }+ );+ for (const agent of agents)+ outcomes.set(+ agent.id,+ agent.switchAgentId! in stopped+ ? (stopped[agent.switchAgentId!] ?? null)+ : 'The host did not report on its watcher.'+ );+ return outcomes;+ },startConsoleWatcher: async (agent) => {await autoSessionWatcher.bringUp(agent.id, 'explicit');},@@ -409,6 +439,32 @@ const machine: MigrationMachinePort = {const credentialsLog = { warn: (...input: unknown[]) => log.warn('agent-migration:', ...input) };+/**+ * Keeps the token of an agent's credentials file in the encrypted secrets+ * store, refusing a file that is incomplete or names another agent. The+ * caller removes the file once this returns.+ */+async function keepCredentials(+ agent: MigrationAgent,+ identity: { slug: string; switchAgentId: string },+ raw: string+): Promise<void> {+ const relPath = agentSettingsRelativePath(identity.slug);+ const parsed = parseSwitchAgentCredentials(raw, credentialsLog);+ if (!parsed)+ throw new Error(+ `${relPath} does not hold complete Switch credentials, so it is left as it is.`+ );+ if (parsed.agentId !== identity.switchAgentId)+ throw new Error(+ `${relPath} names Switch agent ${parsed.agentId}, not ${identity.switchAgentId}; it is left as it is.`+ );+ await encryptedAppSecretsStore.setSecret(+ credentialsStashKey(agent.id, identity.switchAgentId),+ JSON.stringify({ endpoint: parsed.apiEndpoint, token: parsed.token })+ );+}+const credentials: MigrationCredentialsPort = {stash: async (agent, identity) => {const relPath = agentSettingsRelativePath(identity.slug);@@ -416,25 +472,48 @@ const credentials: MigrationCredentialsPort = {try {const raw = await workdir.fs.read(relPath);if (raw === null) return false;- const parsed = parseSwitchAgentCredentials(raw, credentialsLog);- if (!parsed)- throw new Error(- `${relPath} does not hold complete Switch credentials, so it is left as it is.`- );- if (parsed.agentId !== identity.switchAgentId)- throw new Error(- `${relPath} names Switch agent ${parsed.agentId}, not ${identity.switchAgentId}; it is left as it is.`- );- await encryptedAppSecretsStore.setSecret(- credentialsStashKey(agent.id, identity.switchAgentId),- JSON.stringify({ endpoint: parsed.apiEndpoint, token: parsed.token })- );+ await keepCredentials(agent, identity, raw);await workdir.fs.delete(relPath);return true;} finally {workdir.close();}},+ stashMany: async (items) => {+ const outcomes = new Map<string, boolean | Error>();+ const first = items[0];+ if (!first) return outcomes;+ if (!first.agent.sshHost) {+ for (const { agent, identity } of items)+ try {+ outcomes.set(agent.id, await credentials.stash(agent, identity));+ } catch (error) {+ outcomes.set(agent.id, error instanceof Error ? error : new Error(String(error)));+ }+ return outcomes;+ }+ const run = machineScript(first.agent);+ const pathOf = (item: (typeof items)[number]) =>+ posix.join(item.agent.dir, agentSettingsRelativePath(item.identity.slug));+ const files = await readFiles(run, items.map(pathOf));+ const kept: string[] = [];+ for (const item of items) {+ const raw = files[pathOf(item)] ?? null;+ if (raw === null) {+ outcomes.set(item.agent.id, false);+ continue;+ }+ try {+ await keepCredentials(item.agent, item.identity, raw);+ kept.push(pathOf(item));+ outcomes.set(item.agent.id, true);+ } catch (error) {+ outcomes.set(item.agent.id, error instanceof Error ? error : new Error(String(error)));+ }+ }+ if (kept.length) await deleteFiles(run, kept);+ return outcomes;+ },restore: async (agent, identity) => {const key = credentialsStashKey(agent.id, identity.switchAgentId);if ((await identityInFile(agent, identity.slug)) === identity.switchAgentId) {diff --git a/console/apps/switch-console-desktop/src/main/core/agent-migration/host-batch.test.ts b/console/apps/switch-console-desktop/src/main/core/agent-migration/host-batch.test.tsnew file mode 100644index 000000000..5e7dee265--- /dev/null+++ b/console/apps/switch-console-desktop/src/main/core/agent-migration/host-batch.test.ts@@ -0,0 +1,106 @@+import { type ChildProcess, execFile, spawn } from 'node:child_process';+import { createHash } from 'node:crypto';+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { promisify } from 'node:util';+import { afterEach, beforeEach, describe, expect, it } from 'vitest';+import { deleteFiles, readFiles, stopWatchers } from './host-batch';+import type { MachineScript } from './session-handoff';++const execute = promisify(execFile);++let home: string;+const children: ChildProcess[] = [];++/** `node -e` with `home` as its home directory, as the scripts run on a host. */+const run: MachineScript = async (script, args) =>+ (+ await execute(process.execPath, ['-e', script, ...args], {+ env: { ...process.env, HOME: home },+ })+ ).stdout;++const rootOf = (identity: string) =>+ join(+ home,+ '.local',+ 'state',+ 'switch',+ 'sdk-watchers',+ createHash('sha256').update(identity).digest('hex')+ );++/** A stand-in watcher: a process whose command line names the shared host, recorded as the root's owner. */+async function watcher(+ identity: string,+ behaviour: 'exits-when-off' | 'ignores-sigterm' | 'not-ours'+) {+ const root = rootOf(identity);+ await mkdir(join(root, 'supervisor'), { recursive: true });+ await writeFile(join(root, 'config.json'), JSON.stringify({ session: { agentId: identity } }));+ await writeFile(join(root, 'watch.json'), JSON.stringify({ enabled: true }));+ const body = {+ 'exits-when-off': `const fs=require('fs');setInterval(()=>{if(!JSON.parse(fs.readFileSync(process.argv[1]+'/watch.json','utf8')).enabled)process.exit(0)},100)`,+ 'ignores-sigterm': `process.on('SIGTERM',()=>{});setInterval(()=>{},1000)`,+ 'not-ours': `setInterval(()=>{},1000)`,+ }[behaviour];+ const name = behaviour === 'not-ours' ? 'something-else' : 'shared-host-test.mjs';+ const child = spawn(process.execPath, ['-e', body, root, name], { stdio: 'ignore' });+ children.push(child);+ await writeFile(join(root, 'shared-owner.lock'), JSON.stringify({ pid: child.pid }));+ return child;+}++const exited = (child: ChildProcess) => child.exitCode !== null || child.signalCode !== null;++beforeEach(async () => {+ home = await mkdtemp(join(tmpdir(), 'host-batch-'));+});++afterEach(async () => {+ for (const child of children.splice(0)) if (!exited(child)) child.kill('SIGKILL');+ await rm(home, { recursive: true, force: true });+});++describe('stopping several watchers at once', () => {+ it('turns each off and waits for them together', async () => {+ const a = await watcher('agent-a', 'exits-when-off');+ const b = await watcher('agent-b', 'exits-when-off');+ const outcome = await stopWatchers(run, ['agent-a', 'agent-b', 'agent-none'], {+ waitMs: 5_000,+ killWaitMs: 1_000,+ });+ expect(outcome).toEqual({ 'agent-a': null, 'agent-b': null, 'agent-none': null });+ expect(JSON.parse(await readFile(join(rootOf('agent-a'), 'watch.json'), 'utf8'))).toEqual({+ enabled: false,+ spawn: false,+ });+ await expect.poll(() => exited(a) && exited(b)).toBe(true);+ });++ it('kills a watcher that does not stop by itself', async () => {+ const stuck = await watcher('agent-stuck', 'ignores-sigterm');+ const outcome = await stopWatchers(run, ['agent-stuck'], { waitMs: 300, killWaitMs: 300 });+ expect(outcome).toEqual({ 'agent-stuck': null });+ await expect.poll(() => exited(stuck)).toBe(true);+ });++ it('never signals a process that is not a watcher, and says the watcher is still up', async () => {+ const other = await watcher('agent-other', 'not-ours');+ const outcome = await stopWatchers(run, ['agent-other'], { waitMs: 200, killWaitMs: 200 });+ expect(outcome['agent-other']).toMatch(/did not stop/);+ expect(exited(other)).toBe(false);+ });+});++describe('credentials files on a host', () => {+ it('reads several in one command, and removes the ones asked', async () => {+ const one = join(home, 'one.json');+ const two = join(home, 'two.json');+ await writeFile(one, '{"a":1}');+ expect(await readFiles(run, [one, two])).toEqual({ [one]: '{"a":1}', [two]: null });+ await deleteFiles(run, [one, two]);+ expect(await readFiles(run, [one])).toEqual({ [one]: null });+ });+});diff --git a/console/apps/switch-console-desktop/src/main/core/agent-migration/host-batch.ts b/console/apps/switch-console-desktop/src/main/core/agent-migration/host-batch.tsnew file mode 100644index 000000000..43572f4ac--- /dev/null+++ b/console/apps/switch-console-desktop/src/main/core/agent-migration/host-batch.ts@@ -0,0 +1,132 @@+import { z } from 'zod';+import { WATCHER_ROOT } from '@main/core/sdk-host/state-roots';+import type { MachineScript } from './session-handoff';++/**+ * Turns off the Console watchers of several agents on one host, in one+ * command, and waits for them all at once.+ *+ * Moving an agent makes Switch close its watcher's connection, which the+ * watcher's supervisor takes for a crash and answers by sleeping before a+ * restart; one asleep then is slow to notice it was turned off. So a watcher+ * still up when the wait runs out is sent SIGTERM, then SIGKILL, rather than+ * left running. Only a process whose command line names the shared host is+ * signalled, so a PID file outliving its process cannot take another one down.+ *+ * Takes the JSON of `{ identities, waitMs, killWaitMs }` and prints, per+ * identity, null when its watcher is down or the reason it is not.+ */+export const STOP_WATCHERS_SCRIPT = String.raw`${WATCHER_ROOT}+const fs = require('node:fs'), path = require('node:path'), crypto = require('node:crypto');+const cp = require('node:child_process');+const o = JSON.parse(process.argv[1]);+const base = path.join(require('node:os').homedir(), '.local', 'state', 'switch', 'sdk-watchers');+const put = (file, data) => {+ const tmp = file + '.' + crypto.randomUUID();+ fs.writeFileSync(tmp, JSON.stringify(data), { mode: 0o600 });+ fs.renameSync(tmp, file);+};+const pidIn = (root, file) => {+ try {+ const pid = JSON.parse(fs.readFileSync(path.join(root, file), 'utf8')).pid;+ return Number.isSafeInteger(pid) && pid > 0 ? pid : null;+ } catch (e) {+ if (e.code === 'ENOENT') return null;+ throw e;+ }+};+const alive = (pid) => {+ try { process.kill(pid, 0); return true; } catch (e) { return e.code === 'EPERM'; }+};+const commandOf = (pid) => {+ try { return fs.readFileSync('/proc/' + pid + '/cmdline', 'utf8').split('\0').join(' '); }+ catch (e) {+ try { return cp.execFileSync('ps', ['-o', 'command=', '-p', String(pid)], { encoding: 'utf8' }); }+ catch { return ''; }+ }+};+const sleep = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);+const out = {}, roots = {};+for (const identity of o.identities) {+ try {+ const root = watcherRoot(base, identity);+ if (!fs.existsSync(root)) { out[identity] = null; continue; }+ try { fs.unlinkSync(path.join(root, 'taken-over.json')); } catch (e) { if (e.code !== 'ENOENT') throw e; }+ put(path.join(root, 'watch.json'), { enabled: false, spawn: false });+ roots[identity] = root;+ } catch (e) {+ out[identity] = e.message;+ }+}+const live = (identity) =>+ ['shared-owner.lock', 'supervisor/owner.json']+ .map((file) => pidIn(roots[identity], file))+ .filter((pid) => pid !== null && alive(pid));+const signal = (identities, sig) => {+ for (const identity of identities)+ for (const pid of live(identity))+ if (commandOf(pid).includes('shared-host')) try { process.kill(pid, sig); } catch {}+};+let deadline = Date.now() + o.waitMs, stage = 0;+for (;;) {+ const pending = Object.keys(roots).filter((identity) => live(identity).length > 0);+ if (!pending.length) break;+ if (Date.now() >= deadline) {+ if (stage === 2) {+ for (const identity of pending) out[identity] = 'Its watcher on the host did not stop, even when killed.';+ break;+ }+ signal(pending, stage === 0 ? 'SIGTERM' : 'SIGKILL');+ stage += 1;+ deadline = Date.now() + o.killWaitMs;+ }+ sleep(200);+}+for (const identity of Object.keys(roots)) if (!(identity in out)) out[identity] = null;+console.log(JSON.stringify(out));+`;++/**+ * Reads several files on a host in one command. Takes a JSON array of absolute+ * paths and prints an object from path to its text, or null where there is none.+ */+export const READ_FILES_SCRIPT = String.raw`+const fs = require('node:fs');+const out = {};+for (const file of JSON.parse(process.argv[1])) {+ try { out[file] = fs.readFileSync(file, 'utf8'); }+ catch (e) { if (e.code !== 'ENOENT') throw e; out[file] = null; }+}+console.log(JSON.stringify(out));+`;++/** Removes several files on a host in one command. Takes a JSON array of absolute paths. */+export const DELETE_FILES_SCRIPT = String.raw`+const fs = require('node:fs');+for (const file of JSON.parse(process.argv[1])) fs.rmSync(file, { force: true });+console.log('{}');+`;++const lastLine = (stdout: string): unknown => JSON.parse(stdout.trim().split('\n').at(-1) ?? '');++/** Per identity, null when its watcher is down, or why it is not. */+export async function stopWatchers(+ run: MachineScript,+ identities: string[],+ timing: { waitMs: number; killWaitMs: number }+): Promise<Record<string, string | null>> {+ const stdout = await run(STOP_WATCHERS_SCRIPT, [JSON.stringify({ identities, ...timing })]);+ return z.record(z.string(), z.string().nullable()).parse(lastLine(stdout));+}++export async function readFiles(+ run: MachineScript,+ paths: string[]+): Promise<Record<string, string | null>> {+ const stdout = await run(READ_FILES_SCRIPT, [JSON.stringify(paths)]);+ return z.record(z.string(), z.string().nullable()).parse(lastLine(stdout));+}++export async function deleteFiles(run: MachineScript, paths: string[]): Promise<void> {+ await run(DELETE_FILES_SCRIPT, [JSON.stringify(paths)]);+}diff --git a/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.ts b/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.tsindex 629bc0fb4..4107ae1e8 100644--- a/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.ts+++ b/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.ts@@ -257,6 +257,11 @@ class AutoSessionWatcher {this.retries.delete(agentId);}+ /** Stops retrying the agent's watcher, for a caller that turns it off by other means. */+ forgetRetries(agentId: string): void {+ this.cancelRetry(agentId);+ }+stopForAgent(agentId: string): Promise<void> {this.cancelRetry(agentId);return configureAgentHost(agentId, { connected: false, spawning: false }, 'restore');