Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Have the controller send its agents' operations on its socket (CHOO-3667)
Abel Dantas205ab69be7d3
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 8b9791612..c6c000e8c 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -85,6 +85,17 @@ version of their own to them without also giving them a release of their own.Now a room the runner cannot read reads as missing.#### Added+- **An agent controller can send its agents' operations on its socket.**+ Every operation a controller's agents call was an HTTP request of its own,+ parsed, routed and authenticated with the controller's token, though the+ controller already holds an authenticated WebSocket to the server. The+ socket now takes them as `call` frames and answers each with a+ `call.result` frame carrying the status and body the HTTP route+ (`POST /agents/{id}/ops/{operation}`) would have answered with. A call is+ authorized as that request is: the controller checked against its row, and+ the agent's binding to it. The socket's first frame offers calls+ (`calls.max_in_flight`, 64); a call over that is answered 429+ `capacity_exceeded` without running, so the controller sends it over HTTP.- **A reconnect storm is paced instead of taking the pool.** When a pod goesaway, every agent and controller it held opens a connection on the podsleft within seconds. On staging (two pods, 1,000 agents, one pod deleted)@@ -4231,6 +4242,16 @@ tags; see RELEASING.md.`--env-file`, not its user's own logins. `--agent-users` sets how many agentscan run (16 by default).+#### Changed+- **Agents' operations go to Switch on the controller's socket.** Every+ operation an agent host calls through the relay+ (`POST /agents/{id}/ops/{operation}`) is sent as a call on the socket the+ controller already holds, when Switch offers calls, instead of as an HTTP+ request of its own. Over HTTP as before against a server that does not+ offer them, while the socket is down, and when as many calls are in flight+ as Switch takes. A call the socket closes on is answered 502, as a request+ whose connection drops.+#### Fixed- **A Claude Code login that is there but does not work is reported as notsigned in.** `claude auth status` only says a login or key is present, so adiff --git a/console/packages/agent-controller/src/controller.ts b/console/packages/agent-controller/src/controller.tsindex 4bdf949f8..6c4eca296 100644--- a/console/packages/agent-controller/src/controller.ts+++ b/console/packages/agent-controller/src/controller.ts@@ -32,6 +32,7 @@ import {import { type GivenLogin, registerSealingKey, SealedLogins, sealingKeys } from './sealed-logins';import { CONTROLLER_CREDENTIAL, type SecretStore } from './secrets';import type { SelfUpdater } from './self-update';+import { SocketCalls } from './socket-calls';import {noSessionMidTurn,type ProviderLocator,@@ -308,6 +309,8 @@ export async function runController((agentId) => (streamDeps) => hub.open(agentId, streamDeps),workspaces);+ // The agents' operations go to Switch on the controller's socket while it takes them.+ const calls = new SocketCalls();const relay = new LocalRelay({log,roomFor: (agentId, sessionId) => hub.roomFor(agentId, sessionId),@@ -320,6 +323,7 @@ export async function runController(revoked: () => void revoke(),},log,+ calls,}),});/** The agents whose saved cursor the hub has been given. */@@ -758,6 +762,7 @@ export async function runController(initialBackoffMs: timing.streamInitialBackoffMs,maxBackoffMs: timing.streamMaxBackoffMs,random: deps.random,+ calls,});if (ending === 'revoked') await revoke();if (ending === 'taken_over')diff --git a/console/packages/agent-controller/src/relay-forward.ts b/console/packages/agent-controller/src/relay-forward.tsindex 336368a1b..d334240f1 100644--- a/console/packages/agent-controller/src/relay-forward.ts+++ b/console/packages/agent-controller/src/relay-forward.ts@@ -11,6 +11,7 @@ import { isRevoked, PROTOCOL_HEADER } from './api';import { errorMessage, type Logger } from './log';import type { Forwarder } from './relay';import { PROTOCOL_VERSION } from './schemas';+import type { CallAnswer, SocketCalls } from './socket-calls';/** The headers that describe one hop, not the request, and are never passed on. */const HOP_BY_HOP = new Set([@@ -50,6 +51,9 @@ const REPLACED = new Set([export const REPLAYABLE_BODY_BYTES = 1024 * 1024;const ERROR_BODY_BYTES = 64 * 1024;+/** An agent operation, `POST /agents/{agent}/ops/{operation}`, which the socket can carry. */+const OPERATION = /^\/agents\/([^/?]+)\/ops\/([^/?]+)$/;+export type UpstreamAuth = {/** The controller's current access token. */token: () => Promise<string>;@@ -67,6 +71,8 @@ export class UpstreamForwarder implements Forwarder {server: string;auth: UpstreamAuth;log: Logger;+ /** Carries the agents' operations on the controller's socket, when it takes them. */+ calls?: SocketCalls;}) {}@@ -85,6 +91,24 @@ export class UpstreamForwarder implements Forwarder {if (!bodyless && size !== null && Number.isSafeInteger(size) && size <= REPLAYABLE_BODY_BYTES)buffered = await readAll(req, REPLAYABLE_BODY_BYTES);const replayable = bodyless || buffered !== null;+ if (method === 'POST' && buffered !== null) {+ let answer: CallAnswer | null;+ try {+ answer = await this.overSocket(req, buffered, target);+ } catch (error) {+ return refuse(res, 502, `${errorMessage(error)} It may have run.`);+ }+ if (answer !== null) {+ if (res.destroyed) return; // the agent host went away while Switch answered+ const text = JSON.stringify(answer.body);+ res.writeHead(answer.status, {+ 'Content-Type': 'application/json',+ 'Content-Length': Buffer.byteLength(text),+ });+ res.end(text);+ return;+ }+ }const headers = forwardedHeaders(req.headers, target);for (let attempt = 0; ; attempt++) {let token: string;@@ -145,6 +169,45 @@ export class UpstreamForwarder implements Forwarder {return;}}++ /**+ * An operation sent as a call on the controller's socket rather than as an+ * HTTP request (`SocketCalls`), answered with the status and body the HTTP+ * route would have. Null when it goes over HTTP instead: no socket that+ * takes calls, as many calls in flight as Switch takes, or a body that is+ * not a JSON object, which Switch refuses over HTTP in its own words.+ */+ private async overSocket(+ req: IncomingMessage,+ body: Buffer,+ target: { agentId: string; roomId: string | null; path: string }+ ): Promise<CallAnswer | null> {+ const calls = this.deps.calls;+ const operation = OPERATION.exec(target.path);+ if (!calls || !operation || decodeURIComponent(operation[1]!) !== target.agentId) return null;+ let args: unknown;+ try {+ args = body.length === 0 ? {} : JSON.parse(body.toString('utf8'));+ } catch {+ return null;+ }+ if (args === null || typeof args !== 'object' || Array.isArray(args)) return null;+ const answered = calls.call({+ agentId: target.agentId,+ roomId: target.roomId,+ operation: decodeURIComponent(operation[2]!),+ args: args as Record<string, unknown>,+ requestId: header(req, 'x-request-id'),+ });+ if (answered === null) return null;+ const answer = await answered;+ const code = errorCode(JSON.stringify(answer.body));+ // Switch had as many calls in flight on the socket as it takes, so this+ // one did not run.+ if (answer.status === 429 && code === 'capacity_exceeded') return null;+ if (answer.status === 401 && code === 'controller_revoked') this.deps.auth.revoked();+ return answer;+ }}function forwardedHeaders(@@ -231,6 +294,12 @@ function errorCode(text: string): string | null {}}+function header(req: IncomingMessage, name: string): string | null {+ const value = req.headers[name];+ const text = Array.isArray(value) ? value[0] : value;+ return text ? text : null;+}+function refuse(res: ServerResponse, status: number, detail: string): void {const text = JSON.stringify({ detail });res.writeHead(status, {diff --git a/console/packages/agent-controller/src/relay.test.ts b/console/packages/agent-controller/src/relay.test.tsindex 3dbf65266..1f035f6f8 100644--- a/console/packages/agent-controller/src/relay.test.ts+++ b/console/packages/agent-controller/src/relay.test.ts@@ -30,6 +30,7 @@ import { silentLogger } from './log';import { LocalRelay } from './relay';import { UpstreamForwarder } from './relay-forward';import type { AgentAssignment } from './schemas';+import { SocketCalls } from './socket-calls';import { FakeCore } from './testing/fake-core';const AGENT = 'agent-1';@@ -89,6 +90,7 @@ async function waitFor(condition: () => boolean, what: string, timeoutMs = 15_00let core: FakeCore;let hub: AgentHub;let relay: LocalRelay;+let calls: SocketCalls;let token: string;let revokedCalls: number;let cursorsSaved: [string, number][];@@ -115,6 +117,7 @@ function newRelay(): LocalRelay {revoked: () => revokedCalls++,},log: silentLogger,+ calls,}),});}@@ -125,6 +128,7 @@ beforeEach(async () => {core.setAssignment({ revision: 1, agents: [assigned(AGENT), assigned('agent-2')] });revokedCalls = 0;cursorsSaved = [];+ calls = new SocketCalls();hub = new AgentHub({log: silentLogger,bufferLimit: 100,@@ -614,6 +618,76 @@ describe('forwarding to Switch', () => {expect(unreachable.status).toBe(502);});+ it('sends an operation as a call on a socket that takes them, and passes its answer back', async () => {+ const sent: Record<string, unknown>[] = [];+ calls.open((text) => {+ const frame = JSON.parse(text) as Record<string, unknown>;+ sent.push(frame);+ setImmediate(() =>+ calls.answer({ id: frame.id as number, status: 403, body: { detail: 'not a member' } })+ );+ }, 4);+ const response = await fetch(`${relay.endpoint}/agents/${AGENT}/ops/post_message`, {+ method: 'POST',+ headers: {+ Authorization: `Bearer ${token}`,+ 'Content-Type': 'application/json',+ 'X-Request-Id': 'request-9',+ },+ body: JSON.stringify({ body: 'hi' }),+ });++ expect(response.status).toBe(403);+ expect(await response.json()).toEqual({ detail: 'not a member' });+ expect(sent).toEqual([+ {+ type: 'call',+ id: 1,+ agent_id: AGENT,+ operation: 'post_message',+ arguments: { body: 'hi' },+ room_id: null,+ request_id: 'request-9',+ },+ ]);+ expect(core.requests.filter((r) => r.path.includes('/ops/'))).toEqual([]);+ });++ it('sends an operation over HTTP when the socket takes no calls, or no more of them', async () => {+ await post(`/agents/${AGENT}/ops/post_message`, { body: 'no socket' });+ calls.open((text) => {+ const frame = JSON.parse(text) as Record<string, unknown>;+ setImmediate(() =>+ calls.answer({+ id: frame.id as number,+ status: 429,+ body: { error: { code: 'capacity_exceeded', message: 'busy', retryable: true } },+ })+ );+ }, 4);+ const response = await post(`/agents/${AGENT}/ops/post_message`, { body: 'over capacity' });++ expect(response.status).toBe(200);+ expect(+ core.requests.filter((r) => r.path === `/agents/${AGENT}/ops/post_message`)+ ).toHaveLength(2);+ });++ it('sends what is not an operation over HTTP, socket or not', async () => {+ const sent: unknown[] = [];+ calls.open((text) => void sent.push(text), 4);+ const response = await post('/agent-sessions/session-1/activity', { row: 1 });+ expect(response.status).toBe(200);+ expect(sent).toEqual([]);+ });++ it('answers 502 when the socket closes before Switch answers the call', async () => {+ calls.open(() => setImmediate(() => calls.close('The controller socket closed.')), 4);+ const response = await post(`/agents/${AGENT}/ops/post_message`, { body: 'lost' });+ expect(response.status).toBe(502);+ expect(((await response.json()) as { detail: string }).detail).toMatch(/may have run/);+ });+it('streams an upload through without holding it', async () => {const chunk = Buffer.alloc(64 * 1024, 7);const answer = await new Promise<{ status: number; body: string }>((resolve, reject) => {diff --git a/console/packages/agent-controller/src/schemas.test.ts b/console/packages/agent-controller/src/schemas.test.tsindex fb54c3aed..f520d8258 100644--- a/console/packages/agent-controller/src/schemas.test.ts+++ b/console/packages/agent-controller/src/schemas.test.ts@@ -21,7 +21,7 @@ import {tokenRequestSchema,tokenResponseSchema,} from './schemas';-import { STREAM_FRAME_SCHEMAS } from './stream';+import { SOCKET_FRAME_SCHEMAS } from './stream';/*** Core's contract fixtures: one JSON file per wire message. Core checks its@@ -69,7 +69,7 @@ const SCHEMA_FOR: Record<string, z.ZodType> = {/** `stream_frames.json` is a list of `{event, data}`; each is read as the stream reads it. */const streamFramesSchema = z.array(z.object({ event: z.string(), data: z.unknown() }).superRefine((frame, context) => {- const schema = (STREAM_FRAME_SCHEMAS as Record<string, z.ZodType>)[frame.event];+ const schema = (SOCKET_FRAME_SCHEMAS as Record<string, z.ZodType>)[frame.event];if (!schema) {context.addIssue({ code: 'custom', message: `No schema for stream event '${frame.event}'.` });return;@@ -108,7 +108,7 @@ describe('Core contract fixtures', () => {event: string;}[];expect(new Set(frames.map((frame) => frame.event))).toEqual(- new Set(Object.keys(STREAM_FRAME_SCHEMAS))+ new Set(Object.keys(SOCKET_FRAME_SCHEMAS)));});});diff --git a/console/packages/agent-controller/src/schemas.ts b/console/packages/agent-controller/src/schemas.tsindex ee5801531..600c6fd9e 100644--- a/console/packages/agent-controller/src/schemas.ts+++ b/console/packages/agent-controller/src/schemas.ts@@ -373,9 +373,19 @@ export const connectionStateSchema = z.object({connection_id: id,generation: z.number().int(),heartbeat_interval_s: z.number().positive(),+ /** The socket takes the agents' operations as calls; absent from a server from before them. */+ calls: z.object({ max_in_flight: z.number().int().positive() }).optional(),});export type ConnectionState = z.infer<typeof connectionStateSchema>;+/** A call's answer: the status and body the operation's HTTP route would have answered with. */+export const callResultSchema = z.object({+ id: z.union([z.string(), z.number().int()]),+ status: z.number().int(),+ body: z.unknown(),+});+export type CallResult = z.infer<typeof callResultSchema>;+/** The stream ends; `taken_over` is terminal, anything else is recovered by opening again. */export const evictedSchema = z.object({ code: z.string().min(1), reason: z.string() });export type Evicted = z.infer<typeof evictedSchema>;diff --git a/console/packages/agent-controller/src/socket-calls.test.ts b/console/packages/agent-controller/src/socket-calls.test.tsnew file mode 100644index 000000000..162af1cde--- /dev/null+++ b/console/packages/agent-controller/src/socket-calls.test.ts@@ -0,0 +1,90 @@+import { describe, expect, it } from 'vitest';+import { type Call, SocketCallLost, SocketCalls } from './socket-calls';++const call: Call = {+ agentId: 'agent-1',+ roomId: 'room-a',+ operation: 'post_message',+ args: { body: 'hi' },+ requestId: 'request-1',+};++function opened(maxInFlight = 4) {+ const calls = new SocketCalls();+ const sent: Record<string, unknown>[] = [];+ calls.open((text) => void sent.push(JSON.parse(text) as Record<string, unknown>), maxInFlight);+ return { calls, sent };+}++describe('SocketCalls', () => {+ it('sends a call frame and resolves with the answer that names its id', async () => {+ const { calls, sent } = opened();+ const first = calls.call(call)!;+ const second = calls.call({ ...call, operation: 'read_context', roomId: null })!;++ calls.answer({ id: 2, status: 404, body: { detail: 'no' } });+ calls.answer({ id: 1, status: 200, body: { result: { event_id: '$e' } } });++ expect(sent).toEqual([+ {+ type: 'call',+ id: 1,+ agent_id: 'agent-1',+ operation: 'post_message',+ arguments: { body: 'hi' },+ room_id: 'room-a',+ request_id: 'request-1',+ },+ {+ type: 'call',+ id: 2,+ agent_id: 'agent-1',+ operation: 'read_context',+ arguments: { body: 'hi' },+ room_id: null,+ request_id: 'request-1',+ },+ ]);+ expect(await first).toEqual({ status: 200, body: { result: { event_id: '$e' } } });+ expect(await second).toEqual({ status: 404, body: { detail: 'no' } });+ });++ it('answers null while no socket that takes calls is open, so the call goes over HTTP', () => {+ expect(new SocketCalls().call(call)).toBeNull();+ });++ it('answers null once as many calls are in flight as the server takes', () => {+ const { calls, sent } = opened(1);+ expect(calls.call(call)).not.toBeNull();+ expect(calls.call(call)).toBeNull();+ calls.answer({ id: 1, status: 200, body: { result: null } });+ expect(calls.call(call)).not.toBeNull();+ expect(sent).toHaveLength(2);+ });++ it('fails every call still waiting when the socket closes, and sends no more', async () => {+ const { calls } = opened();+ const waiting = calls.call(call)!;+ calls.close('gone');++ await expect(waiting).rejects.toBeInstanceOf(SocketCallLost);+ expect(calls.call(call)).toBeNull();+ });++ it('answers null when the frame could not be sent', () => {+ const calls = new SocketCalls();+ calls.open(() => {+ throw new Error('closing');+ }, 4);+ expect(calls.call(call)).toBeNull();+ });++ it('ignores an answer to no call it is waiting on', async () => {+ const { calls } = opened();+ const waiting = calls.call(call)!;+ calls.answer({ id: 'c-7', status: 200, body: null });+ calls.answer({ id: 9, status: 200, body: null });+ calls.answer({ id: 1, status: 201, body: null });+ expect(await waiting).toEqual({ status: 201, body: null });+ });+});diff --git a/console/packages/agent-controller/src/socket-calls.ts b/console/packages/agent-controller/src/socket-calls.tsnew file mode 100644index 000000000..e306dd8a4--- /dev/null+++ b/console/packages/agent-controller/src/socket-calls.ts@@ -0,0 +1,92 @@+import type { CallResult } from './schemas';++/** A call's answer: the status and body the operation's HTTP route would have answered with. */+export type CallAnswer = { status: number; body: unknown };++export type Call = {+ agentId: string;+ /** The calling session's room, as `X-Switch-Room-Id` names it over HTTP. */+ roomId: string | null;+ operation: string;+ args: Record<string, unknown>;+ requestId: string | null;+};++/** The socket closed with the call unanswered: it may or may not have run. */+export class SocketCallLost extends Error {}++/**+ * The agents' operations, sent as calls on the controller's socket.+ *+ * Every operation an agent host calls (`POST /agents/{id}/ops/{operation}`)+ * reaches the relay, which would send it to Switch as an HTTP request of its+ * own. A server whose socket takes calls says so in its first frame+ * (`calls`); the relay then sends each operation here, as a `call` frame on+ * the socket the controller already holds, and Switch answers it on the+ * same socket with the status and body the HTTP route would have.+ *+ * `call` answers null when the operation has to go over HTTP instead: no+ * socket that takes calls is open, or as many calls are in flight as the+ * server takes. A call the socket closes on fails, as an HTTP request does+ * when its connection drops.+ */+export class SocketCalls {+ private send: ((text: string) => void) | null = null;+ private maxInFlight = 0;+ private nextId = 1;+ private readonly waiting = new Map<+ number,+ { resolve: (answer: CallAnswer) => void; reject: (error: Error) => void }+ >();++ /** A socket that takes calls is open; `send` sends a text frame on it. */+ open(send: (text: string) => void, maxInFlight: number): void {+ this.send = send;+ this.maxInFlight = maxInFlight;+ }++ /** The socket closed: every call still waiting on it fails. */+ close(reason: string): void {+ this.send = null;+ const lost = new SocketCallLost(reason);+ for (const { reject } of this.waiting.values()) reject(lost);+ this.waiting.clear();+ }++ /** An answer arrived on the socket. */+ answer(result: CallResult): void {+ if (typeof result.id !== 'number') return;+ const waiting = this.waiting.get(result.id);+ if (!waiting) return;+ this.waiting.delete(result.id);+ waiting.resolve({ status: result.status, body: result.body });+ }++ /** Sends the call, or answers null when it has to go over HTTP instead. */+ call(call: Call): Promise<CallAnswer> | null {+ const send = this.send;+ if (!send || this.waiting.size >= this.maxInFlight) return null;+ const id = this.nextId++;+ const answered = new Promise<CallAnswer>((resolve, reject) => {+ this.waiting.set(id, { resolve, reject });+ });+ try {+ send(+ JSON.stringify({+ type: 'call',+ id,+ agent_id: call.agentId,+ operation: call.operation,+ arguments: call.args,+ room_id: call.roomId,+ request_id: call.requestId,+ })+ );+ } catch {+ // The frame did not leave, so the call did not run.+ this.waiting.delete(id);+ return null;+ }+ return answered;+ }+}diff --git a/console/packages/agent-controller/src/stream.test.ts b/console/packages/agent-controller/src/stream.test.tsindex 34660e475..6ee7abb8d 100644--- a/console/packages/agent-controller/src/stream.test.ts+++ b/console/packages/agent-controller/src/stream.test.ts@@ -12,6 +12,7 @@ import {import { DEFAULT_TIMING } from './controller';import { silentLogger } from './log';import type { AgentCursor, ControllerConnection } from './schemas';+import { SocketCallLost, SocketCalls } from './socket-calls';import { type ControllerFrame, type ControllerStreamOptions, runControllerStream } from './stream';import { FakeCore } from './testing/fake-core';@@ -138,6 +139,54 @@ function run(}describe('runControllerStream', () => {+ it('takes calls on a socket that offers them, answers each as it arrives, and fails them when it closes', async () => {+ const scripted = new ScriptedClient();+ const offering = {+ event: 'connection_state',+ data: { connection_id: 'connection-1', calls: { max_in_flight: 2 } },+ };+ scripted.sockets = () => scriptedSocket([offering], 'open');+ const calls = new SocketCalls();+ const stop = new AbortController();+ const { frames, states, ending } = run(scripted, stop.signal, { calls });+ await waitFor(() => states.includes('connected'), 'the socket to attach');++ const call = {+ agentId: 'agent-1',+ roomId: 'room-a',+ operation: 'post_message',+ args: { body: 'hi' },+ requestId: null,+ };+ const answered = calls.call(call)!;+ const socket = scripted.opened[0]!;+ expect(socket.sent).toContainEqual(expect.objectContaining({ type: 'call', id: 1 }));+ socket.deliver({ event: 'call.result', data: { id: 1, status: 200, body: { result: 'ok' } } });+ expect(await answered).toEqual({ status: 200, body: { result: 'ok' } });++ const unanswered = calls.call(call)!;+ socket.end(1011);+ await expect(unanswered).rejects.toBeInstanceOf(SocketCallLost);+ stop.abort();+ expect(await ending).toBe('stopped');+ expect(frames.map((frame) => frame.type)).not.toContain('call.result');+ });++ it('takes no calls on a socket from a server that does not offer them', async () => {+ const scripted = new ScriptedClient();+ scripted.sockets = () => scriptedSocket([state], 'open');+ const calls = new SocketCalls();+ const stop = new AbortController();+ const { states, ending } = run(scripted, stop.signal, { calls });+ await waitFor(() => states.includes('connected'), 'the socket to attach');++ expect(+ calls.call({ agentId: 'agent-1', roomId: null, operation: 'x', args: {}, requestId: null })+ ).toBeNull();+ stop.abort();+ expect(await ending).toBe('stopped');+ });+it('opens with the cursors, attaches, and hands over typed frames in order, skipping bad ones', async () => {const scripted = new ScriptedClient();scripted.sockets = () =>@@ -476,17 +525,20 @@ describe('runControllerStream', () => {const stop = new AbortController();const { frames, ending } = run(scripted, stop.signal);await waitFor(- () => frames.length === fixtureFrames.length || frames.at(-1)?.type === 'credential.revoked',+ () =>+ frames.length === fixtureFrames.length - 1 || frames.at(-1)?.type === 'credential.revoked','every fixture frame');stop.abort();await ending;+ // An answer to a call goes to the calls waiting on it, not to `onFrame`.+ const streamed = fixtureFrames.filter((frame) => frame.event !== 'call.result');expect(frames.map((frame) => frame.type)).toEqual(- fixtureFrames+ streamed.map((frame) => frame.event).slice(0,- fixtureFrames.findIndex((frame) => frame.event === 'credential.revoked') + 1 || undefined+ streamed.findIndex((frame) => frame.event === 'credential.revoked') + 1 || undefined));});diff --git a/console/packages/agent-controller/src/stream.ts b/console/packages/agent-controller/src/stream.tsindex 972133bcb..54d564085 100644--- a/console/packages/agent-controller/src/stream.ts+++ b/console/packages/agent-controller/src/stream.ts@@ -29,6 +29,7 @@ import {type AgentWarmFrame,agentWarmFrameSchema,assignmentChangedSchema,+ callResultSchema,type ConnectionState,connectionStateSchema,type ControllerConnection,@@ -38,8 +39,9 @@ import {type OperationPending,operationPendingSchema,} from './schemas';+import type { SocketCalls } from './socket-calls';-/** The schema for each frame the controller's socket carries, by its `event`. */+/** The schema for each frame of the stream, by its `event`; each is handed to `onFrame`. */export const STREAM_FRAME_SCHEMAS = {connection_state: connectionStateSchema,evicted: evictedSchema,@@ -56,6 +58,12 @@ export const STREAM_FRAME_SCHEMAS = {'credential.revoked': credentialRevokedSchema,} as const satisfies Record<string, z.ZodType>;+/** Every frame the controller's socket carries down: the stream's, and the answers to calls. */+export const SOCKET_FRAME_SCHEMAS = {+ ...STREAM_FRAME_SCHEMAS,+ 'call.result': callResultSchema,+} as const satisfies Record<string, z.ZodType>;+export type ControllerFrame =| { type: 'connection_state'; data: ConnectionState }| { type: 'evicted'; data: Evicted }@@ -103,6 +111,8 @@ export type ControllerStreamOptions = {maxBackoffMs: number;/** Jitter source in [0, 1); injectable for tests. */random: () => number;+ /** Takes the agents' operations as calls while a socket that takes them is open. */+ calls?: SocketCalls;};/**@@ -310,6 +320,23 @@ class ControllerStream {if (parsed.event === 'connection_state' && typeof beat === 'number' && beat > 0)idleMs = Math.max(this.options.idleTimeoutMs, beat * 3000);touch();+ // Calls are answered as their answers arrive, not behind the stream's frames.+ if (parsed.event === 'call.result') {+ const result = callResultSchema.safeParse(parsed.data);+ if (result.success) this.options.calls?.answer(result.data);+ else+ this.options.log.error('Dropped a call answer that does not match the protocol', {+ error: result.error.message,+ });+ return;+ }+ if (parsed.event === 'connection_state') {+ const offered = connectionStateSchema.shape.calls.safeParse(+ (parsed.data as { calls?: unknown } | null | undefined)?.calls+ );+ if (offered.success && offered.data)+ this.options.calls?.open((text) => socket.send(text), offered.data.max_in_flight);+ }if (parsed.event === 'ping') {socket.send(JSON.stringify({ type: 'pong', cursors: this.options.confirmed() }));return;@@ -369,6 +396,7 @@ class ControllerStream {if (!this.options.signal.aborted)this.options.log.warn('The controller socket closed; reconnecting.', { code: closedWith });} finally {+ this.options.calls?.close('The controller socket closed before Switch answered the call.');if (idle) clearTimeout(idle);socket.removeEventListener('message', onMessage);socket.removeEventListener('close', onClose);diff --git a/core/switch_core/management/socket_calls.py b/core/switch_core/management/socket_calls.pyindex a48d49c14..2a49628d1 100644--- a/core/switch_core/management/socket_calls.py+++ b/core/switch_core/management/socket_calls.py@@ -125,12 +125,29 @@ async def answer(authenticator: ManagementAuthenticator,) -> dict[str, Any]:"""Run one call a controller sent on its socket, and return the frame- that answers it."""+ that answers it: every call is answered, a failure with 500 as over+ HTTP."""try:call = _parse(message, id_)except _Malformed as exc:return _result(id_, 400, {"detail": str(exc)})+ try:+ return await _run(+ call, principal=principal, protocol=protocol, authenticator=authenticator+ )+ except Exception:+ logger.exception("A call to %s on a controller's socket failed", call.operation)+ return _result(id_, 500, {"detail": "Internal Server Error"})++async def _run(+ call: Call,+ *,+ principal: ControllerPrincipal,+ protocol: AgentCore,+ authenticator: ManagementAuthenticator,+) -> dict[str, Any]:+ id_ = call.idtry:controller = await authenticator.current(principal)except ControllerAuthError as exc:@@ -175,10 +192,7 @@ async def answer(except Exception as exc:status = refusal_status(exc)if status is None:- logger.exception(- "Operation %s failed on a controller's socket", call.operation- )- return _result(id_, 500, {"detail": "Internal Server Error"})+ raisereturn _result(id_, status, {"detail": str(exc)})return _result(id_, 200, _BODY.dump_python({"result": result}, mode="json"))diff --git a/core/tests/switch_core/fixtures/agent_controllers/stream_frames.json b/core/tests/switch_core/fixtures/agent_controllers/stream_frames.jsonindex 165f1e16a..cb364fd69 100644--- a/core/tests/switch_core/fixtures/agent_controllers/stream_frames.json+++ b/core/tests/switch_core/fixtures/agent_controllers/stream_frames.json@@ -157,5 +157,15 @@"code": "taken_over","reason": "another stream attached to this connection and took it over"}+ },+ {+ "event": "call.result",+ "data": {+ "id": 1,+ "status": 404,+ "body": {+ "detail": "unknown operation 'no_such_operation'; expected one of: post_message, read_context"+ }+ }}]diff --git a/core/tests/switch_core/management/test_contract_fixtures.py b/core/tests/switch_core/management/test_contract_fixtures.pyindex 485b7f22b..db854fb32 100644--- a/core/tests/switch_core/management/test_contract_fixtures.py+++ b/core/tests/switch_core/management/test_contract_fixtures.py@@ -21,12 +21,13 @@ from typing import Anyimport pytestfrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker+from switch_core.bridges.agent.auth import ControllerPrincipalfrom switch_core.bridges.agent.commands import room_control_framefrom switch_core.bridges.agent.protocol.types import AgentEvent, MessagePayloadfrom switch_core.config import ControllerReleasefrom switch_core.db.models import TENANT_ZERO_IDfrom switch_core.db.stores.agent_store import AgentStore-from switch_core.management import controller_routes+from switch_core.management import controller_routes, socket_callsfrom switch_core.management.schemas import (ControllerConnectionRequest,ControllerInfoRequest,@@ -363,6 +364,20 @@ class TestStreamFrames:# before a restart), agent.approval_outcomeprovoked += await take(stream, 4)+ # The answer to a call made on the socket.+ answer = await socket_calls.answer(+ {"agent_id": agent_id, "operation": "no_such_operation"},+ 1,+ principal=ControllerPrincipal(+ controller_id=controller.controller_id,+ owner_id=owner.id,+ tenant_id=TENANT_ZERO_ID,+ ),+ protocol=harness.protocol,+ authenticator=harness.management.authenticator,+ )+ provoked.append((answer["event"], answer["data"]))+# Addressed to the agent: the row in the room's log, and the# agent's own event for it. Its missed count runs from where# attaching the agent began counting the room.diff --git a/core/tests/switch_core/management/test_socket_calls.py b/core/tests/switch_core/management/test_socket_calls.pyindex b3af9473c..8d1f659e0 100644--- a/core/tests/switch_core/management/test_socket_calls.py+++ b/core/tests/switch_core/management/test_socket_calls.py@@ -9,7 +9,7 @@ from __future__ import annotationsfrom collections.abc import Iteratorfrom datetime import UTC, datetime-from typing import Any+from typing import Any, castimport pytestfrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker@@ -218,3 +218,28 @@ async def test_a_call_that_cannot_be_run_as_sent_is_a_400(assert answer["data"]["status"] == 400assert detail in answer["data"]["body"]["detail"]+++async def test_a_call_that_fails_is_still_answered() -> None:+ """With a 500, as the HTTP route answers a failure, so the controller is+ not left waiting on it until the socket closes."""++ class _Failing:+ async def current(self, principal: ControllerPrincipal) -> ControllerPrincipal:+ raise RuntimeError("the database went away")++ answer = await socket_calls.answer(+ {"agent_id": "a1", "operation": "post_message"},+ "c-1",+ principal=ControllerPrincipal(+ controller_id="c1", owner_id="o1", tenant_id=TENANT_ZERO_ID+ ),+ protocol=cast(Any, None),+ authenticator=cast(Any, _Failing()),+ )++ assert answer["data"] == {+ "id": "c-1",+ "status": 500,+ "body": {"detail": "Internal Server Error"},+ }