Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
import type {
OpenAgentStream,
ProviderReadiness,
SessionStartedNotice,
} from '@switch-console/agent-providers';
import { AgentHub } from './agent-hub';
import {
AccessTokens,
ControllerClient,
type Fetch,
isCredentialRefused,
isRevoked,
isUpgradeRequired,
type OpenWebSocket,
} from './api';
import { ConfigurationError } from './errors';
import { HubSocket } from './hub-socket';
import { errorMessage, type Logger } from './log';
import { processPendingOperations } from './operations';
import { isSafeSegment } from './paths';
import type { ProviderRechecks } from './provider-rechecks';
import { definitionProblem, reconcile, type ReconcileDeps, startAgent } from './reconcile';
import { LocalRelay, RELAY_TOKEN_PREFIX } from './relay';
import { UpstreamForwarder } from './relay-forward';
import type { AgentRuntime } from './runtime';
import {
type AgentCursor,
type Assignment,
isProvider,
type Provider,
type StatusReport,
statusReportSchema,
type ToolStatus,
type UpdatedBy,
} from './schemas';
import { type GivenLogin, registerSealingKey, SealedLogins, sealingKeys } from './sealed-logins';
import { CONTROLLER_CREDENTIAL, type SecretStore } from './secrets';
import type { SelfUpdater } from './self-update';
import { SocketCalls } from './socket-calls';
import {
noSessionMidTurn,
type ProviderLocator,
type ProviderSeed,
ProviderStatuses,
StatusCollector,
statusFingerprint,
} from './status';
import type { ControllerStore } from './store';
import { type ControllerFrame, runControllerStream } from './stream';
export type ControllerTiming = {
/** The full resync safety net. */
resyncMs: number;
/** How often local state is looked at for a change worth reporting. */
statusPollMs: number;
/** The least time between two status reports. */
statusMinGapMs: number;
/** Used until the server says otherwise. */
defaultReportWithinS: number;
streamIdleMs: number;
streamInitialBackoffMs: number;
/**
* The longest wait before reopening the controller stream. Kept short:
* every agent on the controller is offline while the stream is down.
*/
streamMaxBackoffMs: number;
/** The most events held per agent while its agent host is not taking them. */
eventBufferLimit: number;
};
export const DEFAULT_TIMING: ControllerTiming = {
resyncMs: 10 * 60 * 1000,
statusPollMs: 5_000,
statusMinGapMs: 1_000,
defaultReportWithinS: 60,
// Five missed pings, at the 2 s Switch sends them.
streamIdleMs: 10_000,
streamInitialBackoffMs: 1_000,
streamMaxBackoffMs: 8_000,
eventBufferLimit: 5_000,
};
/**
* How long an agent host restarted onto this controller's shared-host bundle
* is left before it is restarted again, when the restart failed or it still
* runs another bundle.
*/
export const OUTDATED_RESTART_RETRY_MS = 10 * 60 * 1000;
export type ControllerDeps = {
store: ControllerStore;
secrets: SecretStore;
/**
* Builds how agents run here, given the stream each agent's agent host hears
* its events on and the folder agents with no directory of their own work in.
*/
runtime: (openStream: (agentId: string) => OpenAgentStream, workspaces: string) => AgentRuntime;
/** Tries a provider's own login with a test prompt; absent where agents run as other users. */
verifyLogin?: (provider: Provider, binaryPath: string) => Promise<ProviderReadiness> | null;
locator: ProviderLocator;
/**
* Requests to check every provider again now, served while the controller
* runs. `refreshPath` runs first, so a CLI installed since is found.
*/
rechecks: ProviderRechecks;
refreshPath: () => Promise<void>;
/** Provider checks made before this controller started, taken as its own (`ProviderStatuses.seed`). */
providerSeed: ProviderSeed;
/**
* A session of an agent here started, and Switch took its host's report:
* passed to whoever runs this controller (`SessionStartedNotice`).
*/
sessionStarted: (started: Omit<SessionStartedNotice, 'kind'>) => void;
/** `git`, `gh` and vendor tools, as sessions here would be set up with them (`toolStatuses`). */
tools: () => Promise<ToolStatus[]>;
fetch: Fetch;
/** Opens the controller's connection to Switch. */
openWebSocket: OpenWebSocket;
log: Logger;
/** Where disk space is measured and provider checks run. */
dataDir: string;
/** Where agents' working directories go when their definition names none, for a server. */
workspacesFor: (server: string) => string;
version: string;
/** Who moves it onto another release, as its status reports to Switch. */
updatedBy: UpdatedBy;
/**
* Follows the release Switch names for this controller; null where Switch
* Console updates it instead.
*/
selfUpdate: SelfUpdater | null;
now: () => number;
random: () => number;
timing: ControllerTiming;
};
/**
* `taken_over`: another instance of this controller opened the controller
* stream after this one. Restarting would take it straight back, so this one
* ends instead.
*
* `upgrade_required` and `credential_invalid`: the server refused this
* controller's protocol, or knows no controller by its credential. Both are
* refusals the server marks not retryable, and asking again cannot change
* them, so this one ends rather than retrying forever.
*
* `restarting`: the release Switch names is installed, and the controller
* ends for its service to start it again on that release.
*/
export type ControllerExit =
| 'stopped'
| 'revoked'
| 'taken_over'
| 'upgrade_required'
| 'credential_invalid'
| 'restarting';
/**
* Makes an agent's credentials file name the relay, its hub, and a token the
* relay accepts: the token already in the file when it names both as they
* listen now, a new one otherwise. Resolves true when the file was (re)written,
* which a running agent host only reads when it starts, so it is restarted:
* that is also what moves an agent host from before the hub onto it.
*/
function sameLogin(a: GivenLogin | null, b: GivenLogin | null): boolean {
if (a === null || b === null) return a === b;
return (
a.provider === b.provider &&
a.revision === b.revision &&
a.kind === b.kind &&
a.credential === b.credential
);
}
export async function ensureRelayCredentials(
agentId: string,
providerLogin: GivenLogin | null,
deps: { runtime: AgentRuntime; relay: LocalRelay; log: Logger }
): Promise<boolean> {
const current = await deps.runtime.readCredentials(agentId);
const endpoint = deps.relay.endpoint;
const hub = deps.relay.hubUrl;
const relayCurrent =
current !== null &&
current.endpoint === endpoint &&
current.hub === hub &&
current.token.startsWith(RELAY_TOKEN_PREFIX);
const loginCurrent = sameLogin(current?.providerLogin ?? null, providerLogin);
if (current && relayCurrent && loginCurrent) {
if (!deps.relay.isRegistered(agentId, current.token))
deps.relay.register(agentId, current.token);
return false;
}
const token = current && relayCurrent ? current.token : deps.relay.mint(agentId);
if (current && relayCurrent && !deps.relay.isRegistered(agentId, token))
deps.relay.register(agentId, token);
await deps.runtime.writeCredentials(agentId, { endpoint, token, hub, providerLogin });
deps.log.info('Wrote the agent’s relay credentials', {
agentId,
why: !current
? 'it had none'
: !relayCurrent
? 'they named another endpoint, hub or a Switch key'
: providerLogin
? `the ${providerLogin.provider} login given to this machine changed`
: 'the login given to this machine is no longer used',
});
return true;
}
/** Runs one task at a time, in order; a failed task does not stop the next. */
class SerialQueue {
private tail: Promise<void> = Promise.resolve();
run<T>(task: () => Promise<T>): Promise<T> {
const next = this.tail.then(task);
this.tail = next.then(
() => {},
() => {}
);
return next;
}
drain(): Promise<void> {
return this.tail;
}
}
/**
* The controller's run loop, until `signal` fires, the server revokes it, or
* another instance takes its stream over.
*
* It starts the local relay its agents' agent hosts make their Switch calls
* through, exchanges its credential for an access token, and holds the
* controller stream open: agent frames go to each agent's agent host, which runs
* in this process (`AgentHub`), each connect and each `assignment.changed`
* pulls the assignment (with its ETag) and reconciles, each
* `operation.pending` runs the pending operations, and `credential.revoked`
* — or any request refused as `controller_revoked` — stops every agent,
* wipes the credential and ends the loop with `'revoked'`. A full resync runs
* every ten minutes regardless. Status goes out when something changes, and
* at least every `report_within_s`.
*
* Before any of that, the cached assignment is reconciled, so agents come back
* after a reboot even while the server is unreachable.
*/
export async function runController(
deps: ControllerDeps,
signal: AbortSignal
): Promise<ControllerExit> {
const { store, log, timing } = deps;
const identity = store.identity();
if (!identity)
throw new ConfigurationError(
`This controller is not enrolled: ${deps.dataDir} holds no identity. Run 'switch-agent-controller enroll --server <url> --code <code>' first.`
);
const credential = await deps.secrets.get(CONTROLLER_CREDENTIAL);
if (!credential) {
const revokedAt = store.revokedAt();
throw new ConfigurationError(
revokedAt
? `This controller was revoked at ${revokedAt} and its credential wiped. Enroll it again with a new code.`
: `The controller credential is missing from ${deps.secrets.description}. Enroll again with a new code.`
);
}
const warning = deps.secrets.startupWarning();
if (warning) log.warn(warning);
log.info('Agent controller starting', {
controllerId: identity.controllerId,
server: identity.server,
version: deps.version,
});
const stop = new AbortController();
const forward = () => stop.abort();
signal.addEventListener('abort', forward, { once: true });
if (signal.aborted) stop.abort();
const tokens = new AccessTokens({
fetch: deps.fetch,
server: identity.server,
controllerId: identity.controllerId,
credential: async () => credential,
now: deps.now,
log,
});
const client = new ControllerClient({
fetch: deps.fetch,
server: identity.server,
controllerId: identity.controllerId,
version: deps.version,
tokens,
openWebSocket: deps.openWebSocket,
});
const queue = new SerialQueue();
const cached = store.cachedAssignment();
if (cached.kind === 'unreadable') {
log.warn('The saved assignment is not one this version reads; discarding it to pull it again', {
detail: cached.detail,
});
store.discardAssignment();
}
let assignment: Assignment | null = cached.kind === 'saved' ? cached.assignment : null;
let etag: string | null = cached.kind === 'saved' ? cached.etag : null;
let reportWithinS = timing.defaultReportWithinS;
let revocation: Promise<void> | null = null;
let reporter: StatusReporter | null = null;
const hub = new AgentHub({
log,
bufferLimit: timing.eventBufferLimit,
onCursor: (agentId, cursor) =>
store.saveCursor(agentId, cursor, new Date(deps.now()).toISOString()),
onChange: () => reporter?.request(),
});
const workspaces = deps.workspacesFor(identity.server);
const runtime = deps.runtime(
(agentId) => (streamDeps) => hub.open(agentId, streamDeps),
workspaces
);
// The agents' operations go to Switch on the controller's socket while it takes them.
const calls = new SocketCalls();
const relay = new LocalRelay({
log,
roomFor: (agentId, sessionId) => hub.roomFor(agentId, sessionId),
hub: new HubSocket({ hub, log }),
forwarder: new UpstreamForwarder({
server: identity.server,
auth: {
token: () => tokens.get(),
invalidate: (token) => tokens.invalidate(token),
revoked: () => void revoke(),
},
log,
calls,
sessionStarted: ({ agentId, sessionId, startSource }) => {
const entry = assignment?.agents.find((held) => held.agent_id === agentId);
if (!entry) return;
deps.sessionStarted({ sessionId, provider: entry.definition.provider, startSource });
},
}),
});
/** The agents whose saved cursor the hub has been given. */
const placed = new Set<string>();
/** Resumes each newly assigned agent where this controller last confirmed it. */
const placeAgents = (held: Assignment) => {
const cursors = store.cursors();
for (const entry of held.agents) {
const agentId = entry.agent_id;
if (placed.has(agentId)) continue;
placed.add(agentId);
const cursor = cursors.get(agentId);
// Core may have attached the agent before the assignment that names it
// was read: where it attached it is then where the hub already stands.
if (cursor !== undefined && !hub.attachment(agentId)) hub.setCursor(agentId, cursor);
}
};
if (assignment) placeAgents(assignment);
const port = await relay.start(store.relayPort());
store.saveRelayPort(port);
log.info('Relay listening for this machine’s agents', { endpoint: relay.endpoint });
const revoke = (): Promise<void> => {
revocation ??= (async () => {
log.error(
'This controller has been revoked: stopping every agent, wiping its credential and exiting.'
);
const agentIds = new Set([
...store.agents().map((row) => row.agentId),
...(assignment?.agents.map((entry) => entry.agent_id) ?? []),
]);
for (const agentId of agentIds) {
if (!isSafeSegment(agentId)) continue;
relay.unregister(agentId);
try {
await runtime.stop(agentId, { wait: false });
hub.forget(agentId);
await runtime.deleteCredentials(agentId);
} catch (error) {
log.error('Could not stop an agent of the revoked controller', {
agentId,
error: errorMessage(error),
});
}
}
await deps.secrets.delete(CONTROLLER_CREDENTIAL);
store.markRevoked(new Date(deps.now()).toISOString());
stop.abort();
})();
return revocation;
};
/** Set when a refusal that cannot pass ends the run. */
let refused: 'upgrade_required' | 'credential_invalid' | null = null;
const refuse = (ending: 'upgrade_required' | 'credential_invalid') => {
refused ??= ending;
stop.abort();
};
let restarting = false;
const restart = () => {
restarting = true;
stop.abort();
};
const failed = (what: string, error: unknown) => {
if (isRevoked(error)) {
void revoke();
return;
}
if (isUpgradeRequired(error)) return refuse('upgrade_required');
if (isCredentialRefused(error)) return refuse('credential_invalid');
log.warn(`${what} failed; it is retried on the next nudge, reconnect or resync.`, {
error: errorMessage(error),
});
};
const keys = await sealingKeys(deps.secrets);
const sealed = keys
? new SealedLogins({ client, keys, controllerId: identity.controllerId })
: null;
if (!keys)
log.info(
'This controller keeps no key across starts (its credential is handed over), so provider logins given to the machine are not used; its own are.'
);
else
void registerSealingKey(client, keys, log).then((registered) => {
if (!registered) providers.disableGiven();
});
const providers = new ProviderStatuses({
locator: deps.locator,
runtime,
sealed,
probeCwd: deps.dataDir,
verify: deps.verifyLogin,
now: deps.now,
log,
onChange: () => reporter?.request(),
});
providers.seed(deps.providerSeed);
const collector = new StatusCollector({
store,
runtime,
providers,
tools: deps.tools,
attached: (agentId) => hub.attached(agentId),
dataDir: deps.dataDir,
workspacesDir: workspaces,
version: deps.version,
updatedBy: deps.updatedBy,
now: deps.now,
});
const reconcileDeps: ReconcileDeps = {
store,
runtime,
ensureCredentials: async (agentId, provider) =>
ensureRelayCredentials(
agentId,
isProvider(provider) ? await providers.givenLogin(provider) : null,
{ runtime, relay, log }
),
givenLogin: (provider) => providers.givenLogin(provider),
forgetAgent: (agentId) => {
relay.unregister(agentId);
hub.forget(agentId);
},
binaryPath: (provider) => providers.binaryPath(provider),
now: deps.now,
log,
};
let syncQueued = false;
const sync = (why: string): Promise<void> => {
if (syncQueued) return Promise.resolve();
syncQueued = true;
return queue.run(async () => {
syncQueued = false;
if (stop.signal.aborted) return;
try {
const pulled = await client.assignment(etag);
if (pulled.kind === 'changed') {
assignment = pulled.assignment;
etag = pulled.etag;
store.saveAssignment(assignment, etag, new Date(deps.now()).toISOString());
log.info('Pulled assignment', { revision: assignment.revision, why });
placeAgents(assignment);
}
if (assignment) await reconcile(assignment, reconcileDeps);
reporter?.request();
} catch (error) {
failed('Assignment sync', error);
}
});
};
let operationsQueued = false;
const runOperations = (): Promise<void> => {
if (operationsQueued) return Promise.resolve();
operationsQueued = true;
return queue.run(async () => {
operationsQueued = false;
if (stop.signal.aborted) return;
try {
const ran = await processPendingOperations({
client,
assignment: () => assignment,
restartAgent: async (entry) => {
const problem = definitionProblem(entry);
if (problem) return { reason: 'definition_invalid', detail: problem };
return startAgent(
{
kind: 'start',
agentId: entry.agent_id,
entry,
restart: true,
replaceIdentity: false,
clearTakenOver: true,
relaunch: true,
why: 'restart requested',
},
reconcileDeps
);
},
recheckProvider: async (provider) => {
// Asked for by someone fixing a login, like the Console recheck,
// so the login is tried for real.
const status = await providers.check(provider, { verify: true });
await reporter?.sendNow();
// Agents move onto a login given since, or off one withdrawn.
void reconcileLocally();
return status;
},
takeUpLogin: async (provider) => {
const status = await providers.check(provider, { verify: true });
await reporter?.sendNow();
void reconcileLocally();
return { status, problem: providers.loginProblem(provider) };
},
log,
});
if (ran) reporter?.request();
} catch (error) {
failed('Running operations', error);
}
});
};
const idle = async () =>
noSessionMidTurn(
await Promise.all((assignment?.agents ?? []).map((entry) => runtime.observe(entry.agent_id)))
);
/** What the last status response named; looked at again while an update waits for idle. */
let offeredRelease: unknown = null;
const offerRelease = () => {
if (deps.selfUpdate && !stop.signal.aborted)
void deps.selfUpdate.offer(offeredRelease, { idle, restart, signal: stop.signal });
};
reporter = new StatusReporter({
collect: async () => {
const collected = await collector.collect(assignment);
const update = deps.selfUpdate?.status();
return update ? { ...collected, update } : collected;
},
send: async (report) => {
const answer = await client.putStatus(report);
reportWithinS = answer.report_within_s;
if (assignment === null || answer.assignment_revision > assignment.revision)
void sync('status says the assignment moved on');
offeredRelease = answer.controller_release;
offerRelease();
},
seq: () => store.nextStatusSeq(deps.now()),
minGapMs: timing.statusMinGapMs,
failed: (error) => failed('Status report', error),
onLocalChange: () => void reconcileLocally(),
});
const stopServingRechecks = deps.rechecks.serve(async () => {
await deps.refreshPath();
// A recheck is asked for by someone setting up or fixing a login, so
// each login is tried for real, not only reported as there.
await providers.checkAll({ verify: true });
await reporter?.sendFresh();
// Agents waiting on a provider can start now.
void reconcileLocally();
});
let localQueued = false;
/**
* Reconciles the assignment already held, without asking the server: at
* startup, and whenever an agent's observed state changes, so an agent host
* that died is dealt with now rather than at the next nudge or resync.
*/
const reconcileLocally = (): Promise<void> => {
if (localQueued) return Promise.resolve();
localQueued = true;
return queue.run(async () => {
localQueued = false;
if (!assignment || stop.signal.aborted) return;
try {
await reconcile(assignment, reconcileDeps);
} catch (error) {
failed('Reconciling the held assignment', error);
}
});
};
/** When each outdated agent host was last restarted onto this controller's bundle. */
const outdatedRestarts = new Map<string, number>();
let outdatedQueued = false;
/**
* Restarts each agent host that outlived this controller on another
* shared-host bundle (`AgentObservation.outdated`) onto this controller's,
* once none of its sessions is mid-turn: idle sessions restart with it and
* resume their conversations. An agent host already restarted for it is
* left for `OUTDATED_RESTART_RETRY_MS` before another try.
*/
const restartOutdated = (): Promise<void> => {
if (outdatedQueued) return Promise.resolve();
outdatedQueued = true;
return queue.run(async () => {
outdatedQueued = false;
if (!assignment || stop.signal.aborted) return;
try {
for (const entry of assignment.agents) {
if (entry.desired_state !== 'running' || definitionProblem(entry)) continue;
const agentId = entry.agent_id;
const observation = await runtime.observe(agentId);
if (!observation.alive || !observation.outdated) continue;
const last = outdatedRestarts.get(agentId);
if (last !== undefined && deps.now() - last < OUTDATED_RESTART_RETRY_MS) continue;
if (!noSessionMidTurn([observation])) continue;
if (last !== undefined)
log.error(
'An agent host restarted onto this controller’s shared-host bundle is still on another (the restart failed, or it came back on another bundle); restarting it again',
{ agentId, restartedAt: new Date(last).toISOString() }
);
outdatedRestarts.set(agentId, deps.now());
log.warn(
'Restarting an agent host that outlived this controller on another shared-host bundle, now that none of its sessions is mid-turn',
{ agentId, version: deps.version }
);
await startAgent(
{
kind: 'start',
agentId,
entry,
restart: true,
replaceIdentity: false,
clearTakenOver: false,
relaunch: true,
why: 'agent host on another shared-host bundle',
},
reconcileDeps
);
}
} catch (error) {
failed('Restarting outdated agent hosts', error);
}
});
};
/** A hint that a chat with the agent was created or reopened: a failure costs only the head start. */
const warmAgent = async (agentId: string, roomId: string): Promise<void> => {
try {
await runtime.warm(agentId, roomId);
} catch (error) {
log.warn('Could not warm an agent’s session for a room; its first message starts it', {
agentId,
roomId,
error: errorMessage(error),
});
}
};
const onFrame = async (frame: ControllerFrame): Promise<void> => {
switch (frame.type) {
case 'connection_state':
reportWithinS = frame.data.report_within_s;
if (assignment === null || frame.data.assignment_revision > assignment.revision)
void sync('the stream says the assignment moved on');
return;
case 'evicted':
return;
case 'agent.event':
hub.ingest(frame.data);
return;
case 'agent.gap':
log.warn('Switch reports events an agent missed', {
agentId: frame.data.agent_id,
reason: frame.data.reason,
});
hub.gap(frame.data);
return;
case 'agent.session_command':
hub.sessionCommand(frame.data);
return;
case 'agent.warm':
void warmAgent(frame.data.agent_id, frame.data.room_id);
return;
case 'agent.approval_outcome':
hub.approvalOutcome(frame.data);
return;
case 'agent.attached':
hub.attach(frame.data.agent_id, frame.data.from_seq, frame.data.rooms);
return;
case 'agent.detached':
hub.detach(frame.data.agent_id, frame.data.reason);
return;
case 'agent.rooms':
hub.setRooms(frame.data.agent_id, frame.data.rooms);
return;
case 'assignment.changed':
void sync(`assignment.changed to revision ${frame.data.revision}`);
return;
case 'operation.pending':
void runOperations();
return;
case 'credential.revoked':
await revoke();
return;
}
};
await reconcileLocally();
relay.setReady();
void providers.refreshStale().catch((error: unknown) => failed('Checking providers', error));
const resync = setInterval(() => {
void sync('periodic resync');
void runOperations();
void providers.refreshStale().catch((error: unknown) => failed('Checking providers', error));
}, timing.resyncMs);
let lastPeriodic = deps.now();
const poll = setInterval(() => {
if (deps.now() - lastPeriodic >= reportWithinS * 1000) {
lastPeriodic = deps.now();
reporter?.request();
return;
}
void reporter?.requestIfChanged();
// A turn ending changes nothing reported, so it would otherwise wait for
// the next periodic report to let the update through.
if (deps.selfUpdate?.status()?.state === 'waiting_idle') offerRelease();
void restartOutdated();
}, timing.statusPollMs);
let ending: ControllerExit = 'stopped';
try {
ending = await runControllerStream({
client,
cursors: () => {
const cursors: Record<string, AgentCursor> = {};
for (const entry of assignment?.agents ?? []) cursors[entry.agent_id] = 'head';
return { ...cursors, ...hub.cursors() };
},
confirmed: () => hub.cursors(),
onOpened: (connection) =>
log.info('Switch has these agents bound to this controller', {
agents: connection.agents,
}),
onConnected: () => {
log.info('Connected to Switch');
hub.streamAttached();
void sync('connected');
void runOperations();
},
onDisconnected: () => {
hub.setUpstream(false);
},
onFrame,
signal: stop.signal,
log,
idleTimeoutMs: timing.streamIdleMs,
initialBackoffMs: timing.streamInitialBackoffMs,
maxBackoffMs: timing.streamMaxBackoffMs,
random: deps.random,
calls,
});
if (ending === 'revoked') await revoke();
if (ending === 'taken_over')
log.error(
'Another instance of this controller took over its connection to Switch; this one stops its agents and exits.'
);
if (ending === 'upgrade_required' || ending === 'credential_invalid') refused ??= ending;
if (refused === 'upgrade_required')
log.error(
'Switch does not speak this controller’s protocol; it stops its agents and exits. Update the controller.'
);
if (refused === 'credential_invalid')
log.error(
'Switch knows no controller by this credential; it stops its agents and exits. Enroll this machine again.'
);
if (restarting && !refused && !revocation)
log.info('Exiting for the service to start this controller on its new release.');
} finally {
stopServingRechecks();
clearInterval(resync);
clearInterval(poll);
signal.removeEventListener('abort', forward);
stop.abort();
try {
if (revocation) await revocation;
await queue.drain();
await reporter.idle();
await deps.selfUpdate?.settled();
} finally {
await runtime.close();
await relay.close();
}
}
if (revocation) return 'revoked';
return refused ?? (restarting ? 'restarting' : ending);
}
/**
* Sends status reports: on request, coalescing bursts so that at most one goes
* out per `minGapMs`, and only one is in flight at a time.
*/
class StatusReporter {
private lastSentFingerprint: string | null = null;
private lastObservedFingerprint: string | null = null;
private lastSentAt = 0;
private timer: ReturnType<typeof setTimeout> | null = null;
private inFlight: Promise<void> | null = null;
private again = false;
private closed = false;
constructor(
private readonly deps: {
collect: () => Promise<Omit<StatusReport, 'seq'>>;
send: (report: StatusReport) => Promise<void>;
seq: () => number;
minGapMs: number;
failed: (error: unknown) => void;
/** Something observed locally differs from the last look. */
onLocalChange: () => void;
}
) {}
request(): void {
if (this.closed) return;
if (this.inFlight) {
this.again = true;
return;
}
if (this.timer) return;
const wait = Math.max(0, this.lastSentAt + this.deps.minGapMs - Date.now());
this.timer = setTimeout(() => {
this.timer = null;
void this.sendNow();
}, wait);
}
async requestIfChanged(): Promise<void> {
if (this.inFlight || this.timer) return;
try {
const fingerprint = statusFingerprint(await this.deps.collect());
if (this.lastObservedFingerprint !== null && fingerprint !== this.lastObservedFingerprint)
this.deps.onLocalChange();
this.lastObservedFingerprint = fingerprint;
if (fingerprint !== this.lastSentFingerprint) this.request();
} catch (error) {
this.deps.failed(error);
}
}
sendNow(): Promise<void> {
if (this.closed) return Promise.resolve();
if (this.inFlight) {
this.again = true;
return this.inFlight;
}
this.inFlight = (async () => {
try {
const collected = await this.deps.collect();
const report = statusReportSchema.parse({ ...collected, seq: this.deps.seq() });
this.lastSentAt = Date.now();
await this.deps.send(report);
this.lastSentFingerprint = statusFingerprint(collected);
} catch (error) {
this.deps.failed(error);
}
})().finally(() => {
this.inFlight = null;
if (this.again) {
this.again = false;
this.request();
}
});
return this.inFlight;
}
/**
* Sends a report collected after this call, once any already on its way has
* gone, and fails when it could not be sent: for a caller that must know
* Switch holds what is true now.
*/
async sendFresh(): Promise<void> {
while (this.inFlight) await this.inFlight;
if (this.closed) throw new Error('The controller is stopping.');
let failure: unknown = null;
this.inFlight = (async () => {
try {
const collected = await this.deps.collect();
const report = statusReportSchema.parse({ ...collected, seq: this.deps.seq() });
this.lastSentAt = Date.now();
await this.deps.send(report);
this.lastSentFingerprint = statusFingerprint(collected);
} catch (error) {
failure = error;
}
})().finally(() => {
this.inFlight = null;
if (this.again) {
this.again = false;
this.request();
}
});
await this.inFlight;
if (failure !== null) throw failure;
}
/** Stops sending, and waits for a report already on its way. */
async idle(): Promise<void> {
this.closed = true;
if (this.timer) clearTimeout(this.timer);
this.timer = null;
this.again = false;
await this.inFlight;
}
}