Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Have the controller send its agents' operations on its socket (CHOO-3667)

Abel Dantas205ab69be7d3
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8b9791612..c6c000e8c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -85,6 +85,17 @@ version of their own to them without also giving them a release of their own.
Now a room the runner cannot read reads as missing.
#### Added
+- **An agent controller can send its agents' operations on its socket.**
+ Every operation a controller's agents call was an HTTP request of its own,
+ parsed, routed and authenticated with the controller's token, though the
+ controller already holds an authenticated WebSocket to the server. The
+ socket now takes them as `call` frames and answers each with a
+ `call.result` frame carrying the status and body the HTTP route
+ (`POST /agents/{id}/ops/{operation}`) would have answered with. A call is
+ authorized as that request is: the controller checked against its row, and
+ the agent's binding to it. The socket's first frame offers calls
+ (`calls.max_in_flight`, 64); a call over that is answered 429
+ `capacity_exceeded` without running, so the controller sends it over HTTP.
- **A reconnect storm is paced instead of taking the pool.** When a pod goes
away, every agent and controller it held opens a connection on the pods
left within seconds. On staging (two pods, 1,000 agents, one pod deleted)
@@ -4231,6 +4242,16 @@ tags; see RELEASING.md.
`--env-file`, not its user's own logins. `--agent-users` sets how many agents
can run (16 by default).
+#### Changed
+- **Agents' operations go to Switch on the controller's socket.** Every
+ operation an agent host calls through the relay
+ (`POST /agents/{id}/ops/{operation}`) is sent as a call on the socket the
+ controller already holds, when Switch offers calls, instead of as an HTTP
+ request of its own. Over HTTP as before against a server that does not
+ offer them, while the socket is down, and when as many calls are in flight
+ as Switch takes. A call the socket closes on is answered 502, as a request
+ whose connection drops.
+
#### Fixed
- **A Claude Code login that is there but does not work is reported as not
signed in.** `claude auth status` only says a login or key is present, so a
diff --git a/console/packages/agent-controller/src/controller.ts b/console/packages/agent-controller/src/controller.ts
index 4bdf949f8..6c4eca296 100644
--- a/console/packages/agent-controller/src/controller.ts
+++ b/console/packages/agent-controller/src/controller.ts
@@ -32,6 +32,7 @@ import {
import { type GivenLogin, registerSealingKey, SealedLogins, sealingKeys } from './sealed-logins';
import { CONTROLLER_CREDENTIAL, type SecretStore } from './secrets';
import type { SelfUpdater } from './self-update';
+import { SocketCalls } from './socket-calls';
import {
noSessionMidTurn,
type ProviderLocator,
@@ -308,6 +309,8 @@ export async function runController(
(agentId) => (streamDeps) => hub.open(agentId, streamDeps),
workspaces
);
+ // The agents' operations go to Switch on the controller's socket while it takes them.
+ const calls = new SocketCalls();
const relay = new LocalRelay({
log,
roomFor: (agentId, sessionId) => hub.roomFor(agentId, sessionId),
@@ -320,6 +323,7 @@ export async function runController(
revoked: () => void revoke(),
},
log,
+ calls,
}),
});
/** The agents whose saved cursor the hub has been given. */
@@ -758,6 +762,7 @@ export async function runController(
initialBackoffMs: timing.streamInitialBackoffMs,
maxBackoffMs: timing.streamMaxBackoffMs,
random: deps.random,
+ calls,
});
if (ending === 'revoked') await revoke();
if (ending === 'taken_over')
diff --git a/console/packages/agent-controller/src/relay-forward.ts b/console/packages/agent-controller/src/relay-forward.ts
index 336368a1b..d334240f1 100644
--- a/console/packages/agent-controller/src/relay-forward.ts
+++ b/console/packages/agent-controller/src/relay-forward.ts
@@ -11,6 +11,7 @@ import { isRevoked, PROTOCOL_HEADER } from './api';
import { errorMessage, type Logger } from './log';
import type { Forwarder } from './relay';
import { PROTOCOL_VERSION } from './schemas';
+import type { CallAnswer, SocketCalls } from './socket-calls';
/** The headers that describe one hop, not the request, and are never passed on. */
const HOP_BY_HOP = new Set([
@@ -50,6 +51,9 @@ const REPLACED = new Set([
export const REPLAYABLE_BODY_BYTES = 1024 * 1024;
const ERROR_BODY_BYTES = 64 * 1024;
+/** An agent operation, `POST /agents/{agent}/ops/{operation}`, which the socket can carry. */
+const OPERATION = /^\/agents\/([^/?]+)\/ops\/([^/?]+)$/;
+
export type UpstreamAuth = {
/** The controller's current access token. */
token: () => Promise<string>;
@@ -67,6 +71,8 @@ export class UpstreamForwarder implements Forwarder {
server: string;
auth: UpstreamAuth;
log: Logger;
+ /** Carries the agents' operations on the controller's socket, when it takes them. */
+ calls?: SocketCalls;
}
) {}
@@ -85,6 +91,24 @@ export class UpstreamForwarder implements Forwarder {
if (!bodyless && size !== null && Number.isSafeInteger(size) && size <= REPLAYABLE_BODY_BYTES)
buffered = await readAll(req, REPLAYABLE_BODY_BYTES);
const replayable = bodyless || buffered !== null;
+ if (method === 'POST' && buffered !== null) {
+ let answer: CallAnswer | null;
+ try {
+ answer = await this.overSocket(req, buffered, target);
+ } catch (error) {
+ return refuse(res, 502, `${errorMessage(error)} It may have run.`);
+ }
+ if (answer !== null) {
+ if (res.destroyed) return; // the agent host went away while Switch answered
+ const text = JSON.stringify(answer.body);
+ res.writeHead(answer.status, {
+ 'Content-Type': 'application/json',
+ 'Content-Length': Buffer.byteLength(text),
+ });
+ res.end(text);
+ return;
+ }
+ }
const headers = forwardedHeaders(req.headers, target);
for (let attempt = 0; ; attempt++) {
let token: string;
@@ -145,6 +169,45 @@ export class UpstreamForwarder implements Forwarder {
return;
}
}
+
+ /**
+ * An operation sent as a call on the controller's socket rather than as an
+ * HTTP request (`SocketCalls`), answered with the status and body the HTTP
+ * route would have. Null when it goes over HTTP instead: no socket that
+ * takes calls, as many calls in flight as Switch takes, or a body that is
+ * not a JSON object, which Switch refuses over HTTP in its own words.
+ */
+ private async overSocket(
+ req: IncomingMessage,
+ body: Buffer,
+ target: { agentId: string; roomId: string | null; path: string }
+ ): Promise<CallAnswer | null> {
+ const calls = this.deps.calls;
+ const operation = OPERATION.exec(target.path);
+ if (!calls || !operation || decodeURIComponent(operation[1]!) !== target.agentId) return null;
+ let args: unknown;
+ try {
+ args = body.length === 0 ? {} : JSON.parse(body.toString('utf8'));
+ } catch {
+ return null;
+ }
+ if (args === null || typeof args !== 'object' || Array.isArray(args)) return null;
+ const answered = calls.call({
+ agentId: target.agentId,
+ roomId: target.roomId,
+ operation: decodeURIComponent(operation[2]!),
+ args: args as Record<string, unknown>,
+ requestId: header(req, 'x-request-id'),
+ });
+ if (answered === null) return null;
+ const answer = await answered;
+ const code = errorCode(JSON.stringify(answer.body));
+ // Switch had as many calls in flight on the socket as it takes, so this
+ // one did not run.
+ if (answer.status === 429 && code === 'capacity_exceeded') return null;
+ if (answer.status === 401 && code === 'controller_revoked') this.deps.auth.revoked();
+ return answer;
+ }
}
function forwardedHeaders(
@@ -231,6 +294,12 @@ function errorCode(text: string): string | null {
}
}
+function header(req: IncomingMessage, name: string): string | null {
+ const value = req.headers[name];
+ const text = Array.isArray(value) ? value[0] : value;
+ return text ? text : null;
+}
+
function refuse(res: ServerResponse, status: number, detail: string): void {
const text = JSON.stringify({ detail });
res.writeHead(status, {
diff --git a/console/packages/agent-controller/src/relay.test.ts b/console/packages/agent-controller/src/relay.test.ts
index 3dbf65266..1f035f6f8 100644
--- a/console/packages/agent-controller/src/relay.test.ts
+++ b/console/packages/agent-controller/src/relay.test.ts
@@ -30,6 +30,7 @@ import { silentLogger } from './log';
import { LocalRelay } from './relay';
import { UpstreamForwarder } from './relay-forward';
import type { AgentAssignment } from './schemas';
+import { SocketCalls } from './socket-calls';
import { FakeCore } from './testing/fake-core';
const AGENT = 'agent-1';
@@ -89,6 +90,7 @@ async function waitFor(condition: () => boolean, what: string, timeoutMs = 15_00
let core: FakeCore;
let hub: AgentHub;
let relay: LocalRelay;
+let calls: SocketCalls;
let token: string;
let revokedCalls: number;
let cursorsSaved: [string, number][];
@@ -115,6 +117,7 @@ function newRelay(): LocalRelay {
revoked: () => revokedCalls++,
},
log: silentLogger,
+ calls,
}),
});
}
@@ -125,6 +128,7 @@ beforeEach(async () => {
core.setAssignment({ revision: 1, agents: [assigned(AGENT), assigned('agent-2')] });
revokedCalls = 0;
cursorsSaved = [];
+ calls = new SocketCalls();
hub = new AgentHub({
log: silentLogger,
bufferLimit: 100,
@@ -614,6 +618,76 @@ describe('forwarding to Switch', () => {
expect(unreachable.status).toBe(502);
});
+ it('sends an operation as a call on a socket that takes them, and passes its answer back', async () => {
+ const sent: Record<string, unknown>[] = [];
+ calls.open((text) => {
+ const frame = JSON.parse(text) as Record<string, unknown>;
+ sent.push(frame);
+ setImmediate(() =>
+ calls.answer({ id: frame.id as number, status: 403, body: { detail: 'not a member' } })
+ );
+ }, 4);
+ const response = await fetch(`${relay.endpoint}/agents/${AGENT}/ops/post_message`, {
+ method: 'POST',
+ headers: {
+ Authorization: `Bearer ${token}`,
+ 'Content-Type': 'application/json',
+ 'X-Request-Id': 'request-9',
+ },
+ body: JSON.stringify({ body: 'hi' }),
+ });
+
+ expect(response.status).toBe(403);
+ expect(await response.json()).toEqual({ detail: 'not a member' });
+ expect(sent).toEqual([
+ {
+ type: 'call',
+ id: 1,
+ agent_id: AGENT,
+ operation: 'post_message',
+ arguments: { body: 'hi' },
+ room_id: null,
+ request_id: 'request-9',
+ },
+ ]);
+ expect(core.requests.filter((r) => r.path.includes('/ops/'))).toEqual([]);
+ });
+
+ it('sends an operation over HTTP when the socket takes no calls, or no more of them', async () => {
+ await post(`/agents/${AGENT}/ops/post_message`, { body: 'no socket' });
+ calls.open((text) => {
+ const frame = JSON.parse(text) as Record<string, unknown>;
+ setImmediate(() =>
+ calls.answer({
+ id: frame.id as number,
+ status: 429,
+ body: { error: { code: 'capacity_exceeded', message: 'busy', retryable: true } },
+ })
+ );
+ }, 4);
+ const response = await post(`/agents/${AGENT}/ops/post_message`, { body: 'over capacity' });
+
+ expect(response.status).toBe(200);
+ expect(
+ core.requests.filter((r) => r.path === `/agents/${AGENT}/ops/post_message`)
+ ).toHaveLength(2);
+ });
+
+ it('sends what is not an operation over HTTP, socket or not', async () => {
+ const sent: unknown[] = [];
+ calls.open((text) => void sent.push(text), 4);
+ const response = await post('/agent-sessions/session-1/activity', { row: 1 });
+ expect(response.status).toBe(200);
+ expect(sent).toEqual([]);
+ });
+
+ it('answers 502 when the socket closes before Switch answers the call', async () => {
+ calls.open(() => setImmediate(() => calls.close('The controller socket closed.')), 4);
+ const response = await post(`/agents/${AGENT}/ops/post_message`, { body: 'lost' });
+ expect(response.status).toBe(502);
+ expect(((await response.json()) as { detail: string }).detail).toMatch(/may have run/);
+ });
+
it('streams an upload through without holding it', async () => {
const chunk = Buffer.alloc(64 * 1024, 7);
const answer = await new Promise<{ status: number; body: string }>((resolve, reject) => {
diff --git a/console/packages/agent-controller/src/schemas.test.ts b/console/packages/agent-controller/src/schemas.test.ts
index fb54c3aed..f520d8258 100644
--- a/console/packages/agent-controller/src/schemas.test.ts
+++ b/console/packages/agent-controller/src/schemas.test.ts
@@ -21,7 +21,7 @@ import {
tokenRequestSchema,
tokenResponseSchema,
} from './schemas';
-import { STREAM_FRAME_SCHEMAS } from './stream';
+import { SOCKET_FRAME_SCHEMAS } from './stream';
/**
* Core's contract fixtures: one JSON file per wire message. Core checks its
@@ -69,7 +69,7 @@ const SCHEMA_FOR: Record<string, z.ZodType> = {
/** `stream_frames.json` is a list of `{event, data}`; each is read as the stream reads it. */
const streamFramesSchema = z.array(
z.object({ event: z.string(), data: z.unknown() }).superRefine((frame, context) => {
- const schema = (STREAM_FRAME_SCHEMAS as Record<string, z.ZodType>)[frame.event];
+ const schema = (SOCKET_FRAME_SCHEMAS as Record<string, z.ZodType>)[frame.event];
if (!schema) {
context.addIssue({ code: 'custom', message: `No schema for stream event '${frame.event}'.` });
return;
@@ -108,7 +108,7 @@ describe('Core contract fixtures', () => {
event: string;
}[];
expect(new Set(frames.map((frame) => frame.event))).toEqual(
- new Set(Object.keys(STREAM_FRAME_SCHEMAS))
+ new Set(Object.keys(SOCKET_FRAME_SCHEMAS))
);
});
});
diff --git a/console/packages/agent-controller/src/schemas.ts b/console/packages/agent-controller/src/schemas.ts
index ee5801531..600c6fd9e 100644
--- a/console/packages/agent-controller/src/schemas.ts
+++ b/console/packages/agent-controller/src/schemas.ts
@@ -373,9 +373,19 @@ export const connectionStateSchema = z.object({
connection_id: id,
generation: z.number().int(),
heartbeat_interval_s: z.number().positive(),
+ /** The socket takes the agents' operations as calls; absent from a server from before them. */
+ calls: z.object({ max_in_flight: z.number().int().positive() }).optional(),
});
export type ConnectionState = z.infer<typeof connectionStateSchema>;
+/** A call's answer: the status and body the operation's HTTP route would have answered with. */
+export const callResultSchema = z.object({
+ id: z.union([z.string(), z.number().int()]),
+ status: z.number().int(),
+ body: z.unknown(),
+});
+export type CallResult = z.infer<typeof callResultSchema>;
+
/** The stream ends; `taken_over` is terminal, anything else is recovered by opening again. */
export const evictedSchema = z.object({ code: z.string().min(1), reason: z.string() });
export type Evicted = z.infer<typeof evictedSchema>;
diff --git a/console/packages/agent-controller/src/socket-calls.test.ts b/console/packages/agent-controller/src/socket-calls.test.ts
new file mode 100644
index 000000000..162af1cde
--- /dev/null
+++ b/console/packages/agent-controller/src/socket-calls.test.ts
@@ -0,0 +1,90 @@
+import { describe, expect, it } from 'vitest';
+import { type Call, SocketCallLost, SocketCalls } from './socket-calls';
+
+const call: Call = {
+ agentId: 'agent-1',
+ roomId: 'room-a',
+ operation: 'post_message',
+ args: { body: 'hi' },
+ requestId: 'request-1',
+};
+
+function opened(maxInFlight = 4) {
+ const calls = new SocketCalls();
+ const sent: Record<string, unknown>[] = [];
+ calls.open((text) => void sent.push(JSON.parse(text) as Record<string, unknown>), maxInFlight);
+ return { calls, sent };
+}
+
+describe('SocketCalls', () => {
+ it('sends a call frame and resolves with the answer that names its id', async () => {
+ const { calls, sent } = opened();
+ const first = calls.call(call)!;
+ const second = calls.call({ ...call, operation: 'read_context', roomId: null })!;
+
+ calls.answer({ id: 2, status: 404, body: { detail: 'no' } });
+ calls.answer({ id: 1, status: 200, body: { result: { event_id: '$e' } } });
+
+ expect(sent).toEqual([
+ {
+ type: 'call',
+ id: 1,
+ agent_id: 'agent-1',
+ operation: 'post_message',
+ arguments: { body: 'hi' },
+ room_id: 'room-a',
+ request_id: 'request-1',
+ },
+ {
+ type: 'call',
+ id: 2,
+ agent_id: 'agent-1',
+ operation: 'read_context',
+ arguments: { body: 'hi' },
+ room_id: null,
+ request_id: 'request-1',
+ },
+ ]);
+ expect(await first).toEqual({ status: 200, body: { result: { event_id: '$e' } } });
+ expect(await second).toEqual({ status: 404, body: { detail: 'no' } });
+ });
+
+ it('answers null while no socket that takes calls is open, so the call goes over HTTP', () => {
+ expect(new SocketCalls().call(call)).toBeNull();
+ });
+
+ it('answers null once as many calls are in flight as the server takes', () => {
+ const { calls, sent } = opened(1);
+ expect(calls.call(call)).not.toBeNull();
+ expect(calls.call(call)).toBeNull();
+ calls.answer({ id: 1, status: 200, body: { result: null } });
+ expect(calls.call(call)).not.toBeNull();
+ expect(sent).toHaveLength(2);
+ });
+
+ it('fails every call still waiting when the socket closes, and sends no more', async () => {
+ const { calls } = opened();
+ const waiting = calls.call(call)!;
+ calls.close('gone');
+
+ await expect(waiting).rejects.toBeInstanceOf(SocketCallLost);
+ expect(calls.call(call)).toBeNull();
+ });
+
+ it('answers null when the frame could not be sent', () => {
+ const calls = new SocketCalls();
+ calls.open(() => {
+ throw new Error('closing');
+ }, 4);
+ expect(calls.call(call)).toBeNull();
+ });
+
+ it('ignores an answer to no call it is waiting on', async () => {
+ const { calls } = opened();
+ const waiting = calls.call(call)!;
+ calls.answer({ id: 'c-7', status: 200, body: null });
+ calls.answer({ id: 9, status: 200, body: null });
+ calls.answer({ id: 1, status: 201, body: null });
+ expect(await waiting).toEqual({ status: 201, body: null });
+ });
+});
diff --git a/console/packages/agent-controller/src/socket-calls.ts b/console/packages/agent-controller/src/socket-calls.ts
new file mode 100644
index 000000000..e306dd8a4
--- /dev/null
+++ b/console/packages/agent-controller/src/socket-calls.ts
@@ -0,0 +1,92 @@
+import type { CallResult } from './schemas';
+
+/** A call's answer: the status and body the operation's HTTP route would have answered with. */
+export type CallAnswer = { status: number; body: unknown };
+
+export type Call = {
+ agentId: string;
+ /** The calling session's room, as `X-Switch-Room-Id` names it over HTTP. */
+ roomId: string | null;
+ operation: string;
+ args: Record<string, unknown>;
+ requestId: string | null;
+};
+
+/** The socket closed with the call unanswered: it may or may not have run. */
+export class SocketCallLost extends Error {}
+
+/**
+ * The agents' operations, sent as calls on the controller's socket.
+ *
+ * Every operation an agent host calls (`POST /agents/{id}/ops/{operation}`)
+ * reaches the relay, which would send it to Switch as an HTTP request of its
+ * own. A server whose socket takes calls says so in its first frame
+ * (`calls`); the relay then sends each operation here, as a `call` frame on
+ * the socket the controller already holds, and Switch answers it on the
+ * same socket with the status and body the HTTP route would have.
+ *
+ * `call` answers null when the operation has to go over HTTP instead: no
+ * socket that takes calls is open, or as many calls are in flight as the
+ * server takes. A call the socket closes on fails, as an HTTP request does
+ * when its connection drops.
+ */
+export class SocketCalls {
+ private send: ((text: string) => void) | null = null;
+ private maxInFlight = 0;
+ private nextId = 1;
+ private readonly waiting = new Map<
+ number,
+ { resolve: (answer: CallAnswer) => void; reject: (error: Error) => void }
+ >();
+
+ /** A socket that takes calls is open; `send` sends a text frame on it. */
+ open(send: (text: string) => void, maxInFlight: number): void {
+ this.send = send;
+ this.maxInFlight = maxInFlight;
+ }
+
+ /** The socket closed: every call still waiting on it fails. */
+ close(reason: string): void {
+ this.send = null;
+ const lost = new SocketCallLost(reason);
+ for (const { reject } of this.waiting.values()) reject(lost);
+ this.waiting.clear();
+ }
+
+ /** An answer arrived on the socket. */
+ answer(result: CallResult): void {
+ if (typeof result.id !== 'number') return;
+ const waiting = this.waiting.get(result.id);
+ if (!waiting) return;
+ this.waiting.delete(result.id);
+ waiting.resolve({ status: result.status, body: result.body });
+ }
+
+ /** Sends the call, or answers null when it has to go over HTTP instead. */
+ call(call: Call): Promise<CallAnswer> | null {
+ const send = this.send;
+ if (!send || this.waiting.size >= this.maxInFlight) return null;
+ const id = this.nextId++;
+ const answered = new Promise<CallAnswer>((resolve, reject) => {
+ this.waiting.set(id, { resolve, reject });
+ });
+ try {
+ send(
+ JSON.stringify({
+ type: 'call',
+ id,
+ agent_id: call.agentId,
+ operation: call.operation,
+ arguments: call.args,
+ room_id: call.roomId,
+ request_id: call.requestId,
+ })
+ );
+ } catch {
+ // The frame did not leave, so the call did not run.
+ this.waiting.delete(id);
+ return null;
+ }
+ return answered;
+ }
+}
diff --git a/console/packages/agent-controller/src/stream.test.ts b/console/packages/agent-controller/src/stream.test.ts
index 34660e475..6ee7abb8d 100644
--- a/console/packages/agent-controller/src/stream.test.ts
+++ b/console/packages/agent-controller/src/stream.test.ts
@@ -12,6 +12,7 @@ import {
import { DEFAULT_TIMING } from './controller';
import { silentLogger } from './log';
import type { AgentCursor, ControllerConnection } from './schemas';
+import { SocketCallLost, SocketCalls } from './socket-calls';
import { type ControllerFrame, type ControllerStreamOptions, runControllerStream } from './stream';
import { FakeCore } from './testing/fake-core';
@@ -138,6 +139,54 @@ function run(
}
describe('runControllerStream', () => {
+ it('takes calls on a socket that offers them, answers each as it arrives, and fails them when it closes', async () => {
+ const scripted = new ScriptedClient();
+ const offering = {
+ event: 'connection_state',
+ data: { connection_id: 'connection-1', calls: { max_in_flight: 2 } },
+ };
+ scripted.sockets = () => scriptedSocket([offering], 'open');
+ const calls = new SocketCalls();
+ const stop = new AbortController();
+ const { frames, states, ending } = run(scripted, stop.signal, { calls });
+ await waitFor(() => states.includes('connected'), 'the socket to attach');
+
+ const call = {
+ agentId: 'agent-1',
+ roomId: 'room-a',
+ operation: 'post_message',
+ args: { body: 'hi' },
+ requestId: null,
+ };
+ const answered = calls.call(call)!;
+ const socket = scripted.opened[0]!;
+ expect(socket.sent).toContainEqual(expect.objectContaining({ type: 'call', id: 1 }));
+ socket.deliver({ event: 'call.result', data: { id: 1, status: 200, body: { result: 'ok' } } });
+ expect(await answered).toEqual({ status: 200, body: { result: 'ok' } });
+
+ const unanswered = calls.call(call)!;
+ socket.end(1011);
+ await expect(unanswered).rejects.toBeInstanceOf(SocketCallLost);
+ stop.abort();
+ expect(await ending).toBe('stopped');
+ expect(frames.map((frame) => frame.type)).not.toContain('call.result');
+ });
+
+ it('takes no calls on a socket from a server that does not offer them', async () => {
+ const scripted = new ScriptedClient();
+ scripted.sockets = () => scriptedSocket([state], 'open');
+ const calls = new SocketCalls();
+ const stop = new AbortController();
+ const { states, ending } = run(scripted, stop.signal, { calls });
+ await waitFor(() => states.includes('connected'), 'the socket to attach');
+
+ expect(
+ calls.call({ agentId: 'agent-1', roomId: null, operation: 'x', args: {}, requestId: null })
+ ).toBeNull();
+ stop.abort();
+ expect(await ending).toBe('stopped');
+ });
+
it('opens with the cursors, attaches, and hands over typed frames in order, skipping bad ones', async () => {
const scripted = new ScriptedClient();
scripted.sockets = () =>
@@ -476,17 +525,20 @@ describe('runControllerStream', () => {
const stop = new AbortController();
const { frames, ending } = run(scripted, stop.signal);
await waitFor(
- () => frames.length === fixtureFrames.length || frames.at(-1)?.type === 'credential.revoked',
+ () =>
+ frames.length === fixtureFrames.length - 1 || frames.at(-1)?.type === 'credential.revoked',
'every fixture frame'
);
stop.abort();
await ending;
+ // An answer to a call goes to the calls waiting on it, not to `onFrame`.
+ const streamed = fixtureFrames.filter((frame) => frame.event !== 'call.result');
expect(frames.map((frame) => frame.type)).toEqual(
- fixtureFrames
+ streamed
.map((frame) => frame.event)
.slice(
0,
- fixtureFrames.findIndex((frame) => frame.event === 'credential.revoked') + 1 || undefined
+ streamed.findIndex((frame) => frame.event === 'credential.revoked') + 1 || undefined
)
);
});
diff --git a/console/packages/agent-controller/src/stream.ts b/console/packages/agent-controller/src/stream.ts
index 972133bcb..54d564085 100644
--- a/console/packages/agent-controller/src/stream.ts
+++ b/console/packages/agent-controller/src/stream.ts
@@ -29,6 +29,7 @@ import {
type AgentWarmFrame,
agentWarmFrameSchema,
assignmentChangedSchema,
+ callResultSchema,
type ConnectionState,
connectionStateSchema,
type ControllerConnection,
@@ -38,8 +39,9 @@ import {
type OperationPending,
operationPendingSchema,
} from './schemas';
+import type { SocketCalls } from './socket-calls';
-/** The schema for each frame the controller's socket carries, by its `event`. */
+/** The schema for each frame of the stream, by its `event`; each is handed to `onFrame`. */
export const STREAM_FRAME_SCHEMAS = {
connection_state: connectionStateSchema,
evicted: evictedSchema,
@@ -56,6 +58,12 @@ export const STREAM_FRAME_SCHEMAS = {
'credential.revoked': credentialRevokedSchema,
} as const satisfies Record<string, z.ZodType>;
+/** Every frame the controller's socket carries down: the stream's, and the answers to calls. */
+export const SOCKET_FRAME_SCHEMAS = {
+ ...STREAM_FRAME_SCHEMAS,
+ 'call.result': callResultSchema,
+} as const satisfies Record<string, z.ZodType>;
+
export type ControllerFrame =
| { type: 'connection_state'; data: ConnectionState }
| { type: 'evicted'; data: Evicted }
@@ -103,6 +111,8 @@ export type ControllerStreamOptions = {
maxBackoffMs: number;
/** Jitter source in [0, 1); injectable for tests. */
random: () => number;
+ /** Takes the agents' operations as calls while a socket that takes them is open. */
+ calls?: SocketCalls;
};
/**
@@ -310,6 +320,23 @@ class ControllerStream {
if (parsed.event === 'connection_state' && typeof beat === 'number' && beat > 0)
idleMs = Math.max(this.options.idleTimeoutMs, beat * 3000);
touch();
+ // Calls are answered as their answers arrive, not behind the stream's frames.
+ if (parsed.event === 'call.result') {
+ const result = callResultSchema.safeParse(parsed.data);
+ if (result.success) this.options.calls?.answer(result.data);
+ else
+ this.options.log.error('Dropped a call answer that does not match the protocol', {
+ error: result.error.message,
+ });
+ return;
+ }
+ if (parsed.event === 'connection_state') {
+ const offered = connectionStateSchema.shape.calls.safeParse(
+ (parsed.data as { calls?: unknown } | null | undefined)?.calls
+ );
+ if (offered.success && offered.data)
+ this.options.calls?.open((text) => socket.send(text), offered.data.max_in_flight);
+ }
if (parsed.event === 'ping') {
socket.send(JSON.stringify({ type: 'pong', cursors: this.options.confirmed() }));
return;
@@ -369,6 +396,7 @@ class ControllerStream {
if (!this.options.signal.aborted)
this.options.log.warn('The controller socket closed; reconnecting.', { code: closedWith });
} finally {
+ this.options.calls?.close('The controller socket closed before Switch answered the call.');
if (idle) clearTimeout(idle);
socket.removeEventListener('message', onMessage);
socket.removeEventListener('close', onClose);
diff --git a/core/switch_core/management/socket_calls.py b/core/switch_core/management/socket_calls.py
index a48d49c14..2a49628d1 100644
--- a/core/switch_core/management/socket_calls.py
+++ b/core/switch_core/management/socket_calls.py
@@ -125,12 +125,29 @@ async def answer(
authenticator: ManagementAuthenticator,
) -> dict[str, Any]:
"""Run one call a controller sent on its socket, and return the frame
- that answers it."""
+ that answers it: every call is answered, a failure with 500 as over
+ HTTP."""
try:
call = _parse(message, id_)
except _Malformed as exc:
return _result(id_, 400, {"detail": str(exc)})
+ try:
+ return await _run(
+ call, principal=principal, protocol=protocol, authenticator=authenticator
+ )
+ except Exception:
+ logger.exception("A call to %s on a controller's socket failed", call.operation)
+ return _result(id_, 500, {"detail": "Internal Server Error"})
+
+async def _run(
+ call: Call,
+ *,
+ principal: ControllerPrincipal,
+ protocol: AgentCore,
+ authenticator: ManagementAuthenticator,
+) -> dict[str, Any]:
+ id_ = call.id
try:
controller = await authenticator.current(principal)
except ControllerAuthError as exc:
@@ -175,10 +192,7 @@ async def answer(
except Exception as exc:
status = refusal_status(exc)
if status is None:
- logger.exception(
- "Operation %s failed on a controller's socket", call.operation
- )
- return _result(id_, 500, {"detail": "Internal Server Error"})
+ raise
return _result(id_, status, {"detail": str(exc)})
return _result(id_, 200, _BODY.dump_python({"result": result}, mode="json"))
diff --git a/core/tests/switch_core/fixtures/agent_controllers/stream_frames.json b/core/tests/switch_core/fixtures/agent_controllers/stream_frames.json
index 165f1e16a..cb364fd69 100644
--- a/core/tests/switch_core/fixtures/agent_controllers/stream_frames.json
+++ b/core/tests/switch_core/fixtures/agent_controllers/stream_frames.json
@@ -157,5 +157,15 @@
"code": "taken_over",
"reason": "another stream attached to this connection and took it over"
}
+ },
+ {
+ "event": "call.result",
+ "data": {
+ "id": 1,
+ "status": 404,
+ "body": {
+ "detail": "unknown operation 'no_such_operation'; expected one of: post_message, read_context"
+ }
+ }
}
]
diff --git a/core/tests/switch_core/management/test_contract_fixtures.py b/core/tests/switch_core/management/test_contract_fixtures.py
index 485b7f22b..db854fb32 100644
--- a/core/tests/switch_core/management/test_contract_fixtures.py
+++ b/core/tests/switch_core/management/test_contract_fixtures.py
@@ -21,12 +21,13 @@ from typing import Any
import pytest
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+from switch_core.bridges.agent.auth import ControllerPrincipal
from switch_core.bridges.agent.commands import room_control_frame
from switch_core.bridges.agent.protocol.types import AgentEvent, MessagePayload
from switch_core.config import ControllerRelease
from switch_core.db.models import TENANT_ZERO_ID
from switch_core.db.stores.agent_store import AgentStore
-from switch_core.management import controller_routes
+from switch_core.management import controller_routes, socket_calls
from switch_core.management.schemas import (
ControllerConnectionRequest,
ControllerInfoRequest,
@@ -363,6 +364,20 @@ class TestStreamFrames:
# before a restart), agent.approval_outcome
provoked += await take(stream, 4)
+ # The answer to a call made on the socket.
+ answer = await socket_calls.answer(
+ {"agent_id": agent_id, "operation": "no_such_operation"},
+ 1,
+ principal=ControllerPrincipal(
+ controller_id=controller.controller_id,
+ owner_id=owner.id,
+ tenant_id=TENANT_ZERO_ID,
+ ),
+ protocol=harness.protocol,
+ authenticator=harness.management.authenticator,
+ )
+ provoked.append((answer["event"], answer["data"]))
+
# Addressed to the agent: the row in the room's log, and the
# agent's own event for it. Its missed count runs from where
# attaching the agent began counting the room.
diff --git a/core/tests/switch_core/management/test_socket_calls.py b/core/tests/switch_core/management/test_socket_calls.py
index b3af9473c..8d1f659e0 100644
--- a/core/tests/switch_core/management/test_socket_calls.py
+++ b/core/tests/switch_core/management/test_socket_calls.py
@@ -9,7 +9,7 @@ from __future__ import annotations
from collections.abc import Iterator
from datetime import UTC, datetime
-from typing import Any
+from typing import Any, cast
import pytest
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
@@ -218,3 +218,28 @@ async def test_a_call_that_cannot_be_run_as_sent_is_a_400(
assert answer["data"]["status"] == 400
assert detail in answer["data"]["body"]["detail"]
+
+
+async def test_a_call_that_fails_is_still_answered() -> None:
+ """With a 500, as the HTTP route answers a failure, so the controller is
+ not left waiting on it until the socket closes."""
+
+ class _Failing:
+ async def current(self, principal: ControllerPrincipal) -> ControllerPrincipal:
+ raise RuntimeError("the database went away")
+
+ answer = await socket_calls.answer(
+ {"agent_id": "a1", "operation": "post_message"},
+ "c-1",
+ principal=ControllerPrincipal(
+ controller_id="c1", owner_id="o1", tenant_id=TENANT_ZERO_ID
+ ),
+ protocol=cast(Any, None),
+ authenticator=cast(Any, _Failing()),
+ )
+
+ assert answer["data"] == {
+ "id": "c-1",
+ "status": 500,
+ "body": {"detail": "Internal Server Error"},
+ }