Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Websockets (push) for Console<->core (CHOO-3323) (#717)

Abel Dantas36875e0c9919
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 11559e993..9f2b3efed 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -81,6 +81,13 @@ version of their own to them without also giving them a release of their own.
itself up, its rules with the people, agents and rooms they name. So an
agent can tell someone who may answer its questions and approvals instead
of guessing. Another agent sees only the kind.
+- **Console hears when your managed agents or machines change.** A signed-in
+ user's Console opens one WebSocket per workspace (`/gateway/changes/ws`), and
+ switch-core tells it when one of their managed agents or machines changed, so
+ Console reads it again instead of polling. A notice names a kind and an id
+ and carries nothing else. Notices travel between replicas on the signal bus,
+ so a change made through one pod reaches a Console connected to another
+ (#717).
- **The Airtable connection.** A person connects Airtable from Switch Console,
choosing at Airtable's consent screen which bases and workspaces to share,
and a granted agent reads and updates records and comments in them, as that
@@ -1975,6 +1982,10 @@ version of their own to them without also giving them a release of their own.
`GATEWAY_PUBLIC_URL`, and everyone else whom to ask.
#### Changed
+- **Managed agents and machines update when the server says they changed.**
+ Against a server that pushes change notices, Console reads those lists
+ again when told to, and otherwise only every five minutes instead of every
+ 15 to 30 seconds; against one that does not, it polls as before (#717).
- **One name for the controller.** Console, the Gateway and the docs say
"agent controller" everywhere, matching the `switch-agent-controller`
package, instead of a mix of "agent controller" and "agents controller".
diff --git a/console/apps/switch-console-desktop/src/main/core/switch-servers/gateway-client.ts b/console/apps/switch-console-desktop/src/main/core/switch-servers/gateway-client.ts
index 1f4bbf484..1fb327b41 100644
--- a/console/apps/switch-console-desktop/src/main/core/switch-servers/gateway-client.ts
+++ b/console/apps/switch-console-desktop/src/main/core/switch-servers/gateway-client.ts
@@ -305,6 +305,32 @@ async function silentLogin(server: SwitchServer): Promise<string | null> {
return (await getSessionCookie(server.id)) ?? minted;
}
+/** Where to open an authenticated gateway WebSocket, and the headers to open it with. */
+export type GatewaySocketTarget = { url: string; headers: Record<string, string> };
+
+/**
+ * The URL and headers for an authenticated WebSocket to the gateway: the
+ * session cookie (renewed if it is near expiry) and Console's identity, as
+ * every gateway call sends them. Node's WebSocket takes request headers,
+ * unlike a browser's. Refuses a server that is known to be unreachable or
+ * stopped, as `gatewayRequest` does.
+ */
+export async function gatewaySocketTarget(
+ server: SwitchServer,
+ path: string
+): Promise<GatewaySocketTarget> {
+ const blocked = managedServerHostBlocked(server);
+ if (blocked) throw new HostUnreachableError(blocked);
+ const stopped = managedServerStoppedPhase(server);
+ if (stopped) throw new ManagedServerStoppedError(server, stopped);
+ const identity = await consoleIdentityHeaders(server);
+ const cookie = await resolveAuthCookie(server);
+ return {
+ url: gatewayUrl(server, path).replace(/^http/, 'ws'),
+ headers: { ...identity, Cookie: `switch_auth=${cookie}` },
+ };
+}
+
/**
* One authenticated call to the gateway, answered with whatever it returned:
* a refusal is the caller's to read, and a streaming body is left open. Only
diff --git a/console/apps/switch-console-desktop/src/main/core/user-changes/controller.ts b/console/apps/switch-console-desktop/src/main/core/user-changes/controller.ts
new file mode 100644
index 000000000..5839ddeb7
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/main/core/user-changes/controller.ts
@@ -0,0 +1,13 @@
+import { createRPCController } from '@shared/lib/ipc/rpc';
+import { userChangesService } from './user-changes';
+
+/** The renderer's door to the change sockets (`user-changes.ts`). */
+export const userChangesController = createRPCController({
+ watch: async (serverId: string, workspaceId: string | null): Promise<boolean> => {
+ userChangesService.watch(serverId, workspaceId);
+ return userChangesService.isLive(serverId, workspaceId);
+ },
+ unwatch: async (serverId: string, workspaceId: string | null): Promise<void> => {
+ userChangesService.unwatch(serverId, workspaceId);
+ },
+});
diff --git a/console/apps/switch-console-desktop/src/main/core/user-changes/user-changes.test.ts b/console/apps/switch-console-desktop/src/main/core/user-changes/user-changes.test.ts
new file mode 100644
index 000000000..197a2855d
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/main/core/user-changes/user-changes.test.ts
@@ -0,0 +1,248 @@
+import { describe, expect, it, vi } from 'vitest';
+import type { UserChangesEvent } from '@shared/events/userChangesEvents';
+import { createUserChangesService, type UserChangesDeps } from './user-changes';
+
+vi.mock('@main/lib/events', () => ({ events: { emit: vi.fn() } }));
+vi.mock('@main/lib/logger', () => ({ log: { debug: vi.fn() } }));
+vi.mock('@main/core/switch-servers/gateway-client', () => ({ gatewaySocketTarget: vi.fn() }));
+vi.mock('@main/core/workspaces/workspace-session', () => ({
+ withReachableServerWorkspaceSession: vi.fn(),
+}));
+
+type Listener = (event: { data?: unknown; code?: number }) => void;
+
+class FakeSocket {
+ static opened: FakeSocket[] = [];
+ readonly sent: unknown[] = [];
+ closedWith: number | null = null;
+ private readonly listeners = new Map<string, Listener[]>();
+
+ constructor(
+ readonly url: string,
+ readonly init: { headers: Record<string, string> }
+ ) {
+ FakeSocket.opened.push(this);
+ }
+
+ addEventListener(type: string, listener: Listener): void {
+ this.listeners.set(type, [...(this.listeners.get(type) ?? []), listener]);
+ }
+
+ send(data: string): void {
+ this.sent.push(JSON.parse(data));
+ }
+
+ close(code = 1000): void {
+ this.closedWith = code;
+ }
+
+ fire(type: string, event: { data?: unknown; code?: number } = {}): void {
+ for (const listener of this.listeners.get(type) ?? []) listener(event);
+ }
+
+ frame(event: string, data: Record<string, unknown> = {}): void {
+ this.fire('message', { data: JSON.stringify({ event, data }) });
+ }
+}
+
+const clock = { now: 0 };
+
+function setup() {
+ FakeSocket.opened = [];
+ clock.now = 0;
+ const emitted: UserChangesEvent[] = [];
+ const timers: { fn: () => void; ms: number }[] = [];
+ const deps: UserChangesDeps = {
+ target: async (serverId, workspaceId) => ({
+ url: `ws://server/${serverId}/gateway/changes/ws${workspaceId ? `#${workspaceId}` : ''}`,
+ headers: { Cookie: 'switch_auth=abc' },
+ }),
+ socket: FakeSocket as unknown as UserChangesDeps['socket'],
+ emit: ((_channel: unknown, event: UserChangesEvent) => {
+ emitted.push(event);
+ }) as UserChangesDeps['emit'],
+ setTimer: (fn, ms) => {
+ timers.push({ fn, ms });
+ return timers.length as unknown as ReturnType<typeof setTimeout>;
+ },
+ clearTimer: () => {},
+ now: () => clock.now,
+ random: () => 0.999,
+ };
+ return { service: createUserChangesService(deps), emitted, timers };
+}
+
+const tick = () => new Promise((resolve) => setTimeout(resolve, 0));
+
+describe('user changes', () => {
+ it('opens one socket per watched server, with the session cookie', async () => {
+ const { service } = setup();
+ service.watch('s1');
+ service.watch('s1');
+ await tick();
+ expect(FakeSocket.opened).toHaveLength(1);
+ expect(FakeSocket.opened[0].url).toBe('ws://server/s1/gateway/changes/ws');
+ expect(FakeSocket.opened[0].init.headers.Cookie).toBe('switch_auth=abc');
+ });
+
+ it('is live after hello, and forwards notices', async () => {
+ const { service, emitted } = setup();
+ service.watch('s1');
+ await tick();
+ const socket = FakeSocket.opened[0];
+ socket.fire('open');
+ socket.frame('hello', { kinds: ['managed_agent', 'machine'], ping_interval_s: 25 });
+ socket.frame('changed', { changes: [{ kind: 'managed_agent', id: 'a1' }] });
+ expect(service.isLive('s1')).toBe(true);
+ expect(emitted).toEqual([
+ { serverId: 's1', workspaceId: null, type: 'live', kinds: ['managed_agent', 'machine'] },
+ {
+ serverId: 's1',
+ workspaceId: null,
+ type: 'changed',
+ changes: [{ kind: 'managed_agent', id: 'a1' }],
+ },
+ ]);
+ });
+
+ it('answers pings', async () => {
+ const { service } = setup();
+ service.watch('s1');
+ await tick();
+ const socket = FakeSocket.opened[0];
+ socket.frame('ping');
+ expect(socket.sent).toEqual([{ type: 'pong' }]);
+ });
+
+ it('goes down on close and opens again while watched', async () => {
+ const { service, emitted, timers } = setup();
+ service.watch('s1');
+ await tick();
+ const socket = FakeSocket.opened[0];
+ socket.fire('open');
+ socket.frame('hello', { kinds: ['machine'], ping_interval_s: 25 });
+ socket.fire('close', { code: 1012 });
+ expect(service.isLive('s1')).toBe(false);
+ expect(emitted.at(-1)).toEqual({ serverId: 's1', workspaceId: null, type: 'down' });
+ const retry = timers.at(-1)!;
+ expect(retry.ms).toBeLessThanOrEqual(1000);
+ retry.fn();
+ await tick();
+ expect(FakeSocket.opened).toHaveLength(2);
+ });
+
+ it('waits long before trying a server whose handshake never opened again', async () => {
+ const { service, timers } = setup();
+ service.watch('s1');
+ await tick();
+ FakeSocket.opened[0].fire('close', { code: 1006 });
+ expect(timers.at(-1)!.ms).toBe(5 * 60_000);
+ });
+
+ it('keeps retrying at the usual pace when a server that said hello is restarting', async () => {
+ const { service, timers } = setup();
+ service.watch('s1');
+ await tick();
+ const first = FakeSocket.opened[0];
+ first.fire('open');
+ first.frame('hello', { kinds: ['machine'], ping_interval_s: 25 });
+ first.fire('close', { code: 1012 });
+ timers.at(-1)!.fn();
+ await tick();
+ // The server is still down: the handshake fails without opening (a 502).
+ FakeSocket.opened[1].fire('close', { code: 1006 });
+ expect(timers.at(-1)!.ms).toBeLessThan(60_000);
+ timers.at(-1)!.fn();
+ await tick();
+ expect(FakeSocket.opened).toHaveLength(3);
+ });
+
+ it('retries at least every 5 s for two minutes after a known server goes away', async () => {
+ const { service, timers } = setup();
+ service.watch('s1');
+ await tick();
+ FakeSocket.opened[0].fire('open');
+ FakeSocket.opened[0].frame('hello', { kinds: ['machine'], ping_interval_s: 25 });
+ FakeSocket.opened[0].fire('close', { code: 1012 });
+ for (let attempt = 1; attempt <= 6; attempt += 1) {
+ clock.now += 5_000;
+ const retry = timers.at(-1)!;
+ expect(retry.ms).toBeLessThanOrEqual(5_000);
+ retry.fn();
+ await tick();
+ FakeSocket.opened.at(-1)!.fire('close', { code: 1006 });
+ }
+ // Past the window, the usual back-off grows again.
+ clock.now += 2 * 60_000;
+ timers.at(-1)!.fn();
+ await tick();
+ FakeSocket.opened.at(-1)!.fire('close', { code: 1006 });
+ expect(timers.at(-1)!.ms).toBeGreaterThan(5_000);
+ });
+
+ it('opens one socket per workspace on a server, each with its own session', async () => {
+ const { service, emitted } = setup();
+ service.watch('s1', 'w1');
+ service.watch('s1', 'w2');
+ await tick();
+ expect(FakeSocket.opened.map((s) => s.url)).toEqual([
+ 'ws://server/s1/gateway/changes/ws#w1',
+ 'ws://server/s1/gateway/changes/ws#w2',
+ ]);
+ FakeSocket.opened[1].frame('changed', { changes: [{ kind: 'machine', id: 'm1' }] });
+ expect(emitted.at(-1)).toMatchObject({ serverId: 's1', workspaceId: 'w2', type: 'changed' });
+ service.unwatch('s1', 'w1');
+ expect(FakeSocket.opened[0].closedWith).toBe(1000);
+ expect(FakeSocket.opened[1].closedWith).toBeNull();
+ });
+
+ it('closes the socket when the last watcher goes', async () => {
+ const { service } = setup();
+ service.watch('s1');
+ service.watch('s1');
+ await tick();
+ const socket = FakeSocket.opened[0];
+ service.unwatch('s1');
+ expect(socket.closedWith).toBeNull();
+ service.unwatch('s1');
+ expect(socket.closedWith).toBe(1000);
+ expect(service.isLive('s1')).toBe(false);
+ });
+
+ it('takes a server that stops pinging for dead', async () => {
+ const { service, timers } = setup();
+ service.watch('s1');
+ await tick();
+ const socket = FakeSocket.opened[0];
+ socket.fire('open');
+ socket.frame('hello', { kinds: ['machine'], ping_interval_s: 10 });
+ const watchdog = timers.at(-1)!;
+ expect(watchdog.ms).toBe(30_000);
+ watchdog.fn();
+ expect(socket.closedWith).toBe(4000);
+ });
+
+ it('opens a new socket after the watchdog fires, even if the old one never says closed', async () => {
+ const { service, emitted, timers } = setup();
+ service.watch('s1');
+ await tick();
+ const socket = FakeSocket.opened[0];
+ socket.fire('open');
+ socket.frame('hello', { kinds: ['machine'], ping_interval_s: 10 });
+ timers.at(-1)!.fn();
+ // No `close` event: the connection went silent.
+ expect(service.isLive('s1')).toBe(false);
+ expect(emitted.at(-1)).toMatchObject({ serverId: 's1', type: 'down' });
+ const retry = timers.at(-1)!;
+ expect(retry.ms).toBeLessThanOrEqual(5_000);
+ retry.fn();
+ await tick();
+ expect(FakeSocket.opened).toHaveLength(2);
+ // The old socket's close, if it ever comes, does not take the new one down.
+ FakeSocket.opened[1].fire('open');
+ FakeSocket.opened[1].frame('hello', { kinds: ['machine'], ping_interval_s: 10 });
+ socket.fire('close', { code: 1006 });
+ expect(service.isLive('s1')).toBe(true);
+ expect(FakeSocket.opened).toHaveLength(2);
+ });
+});
diff --git a/console/apps/switch-console-desktop/src/main/core/user-changes/user-changes.ts b/console/apps/switch-console-desktop/src/main/core/user-changes/user-changes.ts
new file mode 100644
index 000000000..31cacd32b
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/main/core/user-changes/user-changes.ts
@@ -0,0 +1,272 @@
+import {
+ gatewaySocketTarget,
+ type GatewaySocketTarget,
+} from '@main/core/switch-servers/gateway-client';
+import {
+ withReachableServerWorkspaceSession,
+ withReachableWorkspaceSession,
+} from '@main/core/workspaces/workspace-session';
+import { events } from '@main/lib/events';
+import { log } from '@main/lib/logger';
+import { type UserChange, userChangesChannel } from '@shared/events/userChangesEvents';
+
+/**
+ * One change socket per server whose lists a window is showing
+ * (`/gateway/changes/ws`), so those lists are read again when the server says
+ * something changed instead of on a timer.
+ *
+ * Watched by reference count: a list that is shown calls `watch`, and the
+ * socket closes when the last one is gone. While watched, a closed socket is
+ * opened again with back-off. A server without the socket (older than it)
+ * answers the handshake with an error and is tried again only rarely; until
+ * the socket says hello, the lists keep polling, so nothing goes stale.
+ */
+
+/** Node's WebSocket, which, unlike a browser's, takes request headers. */
+type HeaderedWebSocket = new (url: string, init: { headers: Record<string, string> }) => WebSocket;
+
+const MIN_RETRY_MS = 1_000;
+const MAX_RETRY_MS = 60_000;
+/** Back-off after a handshake that never opened: likely a server without the socket. */
+const UNSUPPORTED_RETRY_MS = 5 * 60_000;
+/**
+ * A server that has said hello and goes away is most likely restarting, which
+ * takes about half a minute: for this long, retry at least every
+ * `RESTART_RETRY_MS`, so the socket is back within seconds of the server.
+ */
+const RESTART_WINDOW_MS = 2 * 60_000;
+const RESTART_RETRY_MS = 5_000;
+/** Pings missed before the socket is taken for dead and opened again. */
+const MISSED_PINGS = 3;
+
+type Hello = { kinds: string[]; ping_interval_s: number };
+
+export type UserChangesDeps = {
+ target: (serverId: string, workspaceId: string | null) => Promise<GatewaySocketTarget>;
+ socket: HeaderedWebSocket | undefined;
+ emit: typeof events.emit;
+ setTimer: (fn: () => void, ms: number) => ReturnType<typeof setTimeout>;
+ clearTimer: (timer: ReturnType<typeof setTimeout>) => void;
+ now: () => number;
+ /** In [0, 1), to spread the retries of many Consoles. */
+ random: () => number;
+};
+
+const defaultDeps: UserChangesDeps = {
+ // The cookie of the workspace the lists show, so the socket's session is
+ // bound to that workspace's tenant.
+ target: (serverId, workspaceId) => {
+ const open = (server: Parameters<typeof gatewaySocketTarget>[0]) =>
+ gatewaySocketTarget(server, '/changes/ws');
+ return workspaceId === null
+ ? withReachableServerWorkspaceSession(serverId, open)
+ : withReachableWorkspaceSession(workspaceId, open);
+ },
+ socket: (globalThis as unknown as { WebSocket?: HeaderedWebSocket }).WebSocket,
+ emit: (...args) => events.emit(...args),
+ setTimer: (fn, ms) => setTimeout(fn, ms),
+ clearTimer: (timer) => clearTimeout(timer),
+ now: () => Date.now(),
+ random: () => Math.random(),
+};
+
+class ServerSocket {
+ watchers = 0;
+ live = false;
+ /** This server has said hello at least once, so it has the socket. */
+ private supported = false;
+ /** When the socket last went away, while it has not come back. */
+ private downSince: number | null = null;
+ private socket: WebSocket | null = null;
+ private retryMs = MIN_RETRY_MS;
+ private retry: ReturnType<typeof setTimeout> | null = null;
+ private watchdog: ReturnType<typeof setTimeout> | null = null;
+ private stopped = false;
+
+ constructor(
+ private readonly serverId: string,
+ private readonly workspaceId: string | null,
+ private readonly deps: UserChangesDeps
+ ) {}
+
+ start(): void {
+ this.stopped = false;
+ void this.open();
+ }
+
+ stop(): void {
+ this.stopped = true;
+ if (this.retry) this.deps.clearTimer(this.retry);
+ if (this.watchdog) this.deps.clearTimer(this.watchdog);
+ this.retry = null;
+ this.watchdog = null;
+ const socket = this.socket;
+ this.socket = null;
+ socket?.close(1000);
+ this.setDown();
+ }
+
+ private async open(): Promise<void> {
+ if (this.stopped || this.socket) return;
+ const Socket = this.deps.socket;
+ if (!Socket) return; // No WebSocket in this runtime: the lists keep polling.
+ let target: GatewaySocketTarget;
+ try {
+ target = await this.deps.target(this.serverId, this.workspaceId);
+ } catch (error) {
+ this.schedule(this.retryMs, `could not prepare: ${String(error)}`);
+ return;
+ }
+ if (this.stopped) return;
+ let opened = false;
+ const socket = new Socket(target.url, { headers: target.headers });
+ this.socket = socket;
+ socket.addEventListener('open', () => {
+ opened = true;
+ });
+ socket.addEventListener('message', (event) => this.onMessage(socket, event.data));
+ socket.addEventListener('close', (event) => {
+ // A socket the watchdog already gave up on has been replaced or is
+ // waiting for its retry: its late close changes nothing.
+ if (this.socket !== socket) return;
+ // A handshake that never opened, on a server that has never said hello,
+ // is most likely a server without the socket. One that has said hello
+ // is restarting or unreachable for now: keep trying at the usual pace.
+ this.drop(
+ opened || this.supported ? this.retryMs : UNSUPPORTED_RETRY_MS,
+ `closed (${event.code})`
+ );
+ });
+ }
+
+ /** Forget the current socket, tell the lists to poll, and open another later. */
+ private drop(retryMs: number, why: string): void {
+ this.socket = null;
+ this.downSince ??= this.deps.now();
+ if (this.watchdog) this.deps.clearTimer(this.watchdog);
+ this.watchdog = null;
+ this.setDown();
+ this.schedule(retryMs, why);
+ }
+
+ private onMessage(socket: WebSocket, data: unknown): void {
+ let frame: { event?: string; data?: Record<string, unknown> };
+ try {
+ frame = JSON.parse(String(data));
+ } catch {
+ return;
+ }
+ switch (frame.event) {
+ case 'hello': {
+ const hello = frame.data as unknown as Hello;
+ this.retryMs = MIN_RETRY_MS;
+ this.supported = true;
+ this.downSince = null;
+ this.live = true;
+ this.arm(socket, hello.ping_interval_s);
+ this.deps.emit(userChangesChannel, {
+ serverId: this.serverId,
+ workspaceId: this.workspaceId,
+ type: 'live',
+ kinds: hello.kinds,
+ });
+ return;
+ }
+ case 'ping':
+ socket.send(JSON.stringify({ type: 'pong' }));
+ this.arm(socket);
+ return;
+ case 'changed':
+ this.deps.emit(userChangesChannel, {
+ serverId: this.serverId,
+ workspaceId: this.workspaceId,
+ type: 'changed',
+ changes: (frame.data?.changes ?? []) as UserChange[],
+ });
+ return;
+ default:
+ return; // `refused` is followed by the close that schedules the retry.
+ }
+ }
+
+ private pingIntervalMs = 25_000;
+
+ /**
+ * Take the socket for dead when the server goes quiet for `MISSED_PINGS`
+ * pings. `close()` only starts the closing handshake, and on a connection
+ * that went silent (a laptop that slept, a NAT that dropped it) the `close`
+ * event may never come, so the socket is dropped here rather than when it
+ * does: the lists go back to polling and a new socket is opened.
+ */
+ private arm(socket: WebSocket, pingIntervalS?: number): void {
+ if (pingIntervalS !== undefined) this.pingIntervalMs = pingIntervalS * 1000;
+ if (this.watchdog) this.deps.clearTimer(this.watchdog);
+ this.watchdog = this.deps.setTimer(() => {
+ this.watchdog = null;
+ if (this.socket !== socket) return;
+ socket.close(4000);
+ this.drop(this.retryMs, 'heard no ping');
+ }, this.pingIntervalMs * MISSED_PINGS);
+ }
+
+ private setDown(): void {
+ if (!this.live) return;
+ this.live = false;
+ this.deps.emit(userChangesChannel, {
+ serverId: this.serverId,
+ workspaceId: this.workspaceId,
+ type: 'down',
+ });
+ }
+
+ private schedule(ms: number, why: string): void {
+ if (this.stopped) return;
+ const restarting =
+ this.supported &&
+ this.downSince !== null &&
+ this.deps.now() - this.downSince < RESTART_WINDOW_MS;
+ if (restarting) ms = Math.min(ms, RESTART_RETRY_MS) * (0.5 + this.deps.random() / 2);
+ log.debug(`Change socket for server ${this.serverId} ${why}; retrying in ${ms} ms`);
+ this.retry = this.deps.setTimer(() => {
+ this.retry = null;
+ void this.open();
+ }, ms);
+ this.retryMs = Math.min(this.retryMs * 2, MAX_RETRY_MS);
+ }
+}
+
+export function createUserChangesService(deps: UserChangesDeps = defaultDeps) {
+ const sockets = new Map<string, ServerSocket>();
+ const keyOf = (serverId: string, workspaceId: string | null) =>
+ `${serverId}\u0000${workspaceId ?? ''}`;
+ return {
+ /** A list for this server and workspace is shown: keep its change socket open. */
+ watch(serverId: string, workspaceId: string | null = null): void {
+ const key = keyOf(serverId, workspaceId);
+ let socket = sockets.get(key);
+ if (!socket) {
+ socket = new ServerSocket(serverId, workspaceId, deps);
+ sockets.set(key, socket);
+ }
+ socket.watchers += 1;
+ if (socket.watchers === 1) socket.start();
+ },
+ /** The list is gone; close the socket once nothing watches it. */
+ unwatch(serverId: string, workspaceId: string | null = null): void {
+ const key = keyOf(serverId, workspaceId);
+ const socket = sockets.get(key);
+ if (!socket) return;
+ socket.watchers = Math.max(0, socket.watchers - 1);
+ if (socket.watchers === 0) {
+ socket.stop();
+ sockets.delete(key);
+ }
+ },
+ /** Whether the server is pushing changes for this workspace right now. */
+ isLive(serverId: string, workspaceId: string | null = null): boolean {
+ return sockets.get(keyOf(serverId, workspaceId))?.live ?? false;
+ },
+ };
+}
+
+export const userChangesService = createUserChangesService();
diff --git a/console/apps/switch-console-desktop/src/main/rpc.ts b/console/apps/switch-console-desktop/src/main/rpc.ts
index 6eea4cefb..2bed369e9 100644
--- a/console/apps/switch-console-desktop/src/main/rpc.ts
+++ b/console/apps/switch-console-desktop/src/main/rpc.ts
@@ -26,6 +26,7 @@ import { switchRoomsController } from './core/switch-rooms/controller';
import { switchServersController } from './core/switch-servers/controller';
import { telemetryController } from './core/telemetry/controller';
import { updateController } from './core/updates/controller';
+import { userChangesController } from './core/user-changes/controller';
import { viewStateController } from './core/view-state/controller';
import { workspacesController } from './core/workspaces/controller';
@@ -58,6 +59,7 @@ export const rpcRouter = createRPCRouter({
hostControllers: hostControllersController,
managedAgents: managedAgentsController,
featureFlags: featureFlagsController,
+ userChanges: userChangesController,
fs: createRPCNamespace({
watch: filesController,
}),
diff --git a/console/apps/switch-console-desktop/src/renderer/features/locations/components/add-agent-modal/new-agent-form.tsx b/console/apps/switch-console-desktop/src/renderer/features/locations/components/add-agent-modal/new-agent-form.tsx
index 545e96e46..e0c1d76d6 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/locations/components/add-agent-modal/new-agent-form.tsx
+++ b/console/apps/switch-console-desktop/src/renderer/features/locations/components/add-agent-modal/new-agent-form.tsx
@@ -7,7 +7,11 @@ import { isMachineAsleep } from '@renderer/features/cloud-agents/cloud-machine-s
import { useCloudMachines } from '@renderer/features/cloud-agents/use-cloud-agents';
import { agentsStore } from '@renderer/features/locations/stores/agents-store';
import { getLocationManagerStore } from '@renderer/features/locations/stores/location-selectors';
-import { MANAGED_AGENTS_KEY } from '@renderer/features/managed-agents/use-managed-agents';
+import {
+ MANAGED_AGENTS_KEY,
+ refreshEvery,
+} from '@renderer/features/managed-agents/use-managed-agents';
+import { useUserChangesLive } from '@renderer/features/managed-agents/use-user-changes';
import { HostReachabilityNotice } from '@renderer/features/remote-hosts/host-reachability-notice';
import { hostReachabilityStore } from '@renderer/features/remote-hosts/host-reachability-store';
import {
@@ -162,6 +166,7 @@ export const NewAgentForm = observer(function NewAgentForm({
// every machine the user owns there. Null when it does not run management.
const askForMachines =
!!selectedServerId && !!workspacesStore.idOnServerInScope(selectedServerId);
+ const machinesPushed = useUserChangesLive(askForMachines ? selectedServerId : null);
const machinesQuery = useQuery({
queryKey: [
MANAGED_AGENTS_KEY,
@@ -171,7 +176,8 @@ export const NewAgentForm = observer(function NewAgentForm({
],
queryFn: () => rpc.managedAgents.machines(selectedServerId!),
enabled: askForMachines,
- refetchInterval: (query) => (query.state.data ? 5000 : false),
+ refetchInterval: (query) =>
+ query.state.data ? refreshEvery(5000, machinesPushed, query.state.status === 'error') : false,
});
const serverMachines = askForMachines ? (machinesQuery.data ?? null) : null;
const management = serverMachines !== null;
diff --git a/console/apps/switch-console-desktop/src/renderer/features/managed-agents/use-managed-agents-key.ts b/console/apps/switch-console-desktop/src/renderer/features/managed-agents/use-managed-agents-key.ts
new file mode 100644
index 000000000..59a8801c8
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/renderer/features/managed-agents/use-managed-agents-key.ts
@@ -0,0 +1,2 @@
+/** The react-query key prefix of every managed-agents list for a server. */
+export const MANAGED_AGENTS_KEY = 'managed-agents';
diff --git a/console/apps/switch-console-desktop/src/renderer/features/managed-agents/use-managed-agents.ts b/console/apps/switch-console-desktop/src/renderer/features/managed-agents/use-managed-agents.ts
index a4ce853b6..ce495ac1a 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/managed-agents/use-managed-agents.ts
+++ b/console/apps/switch-console-desktop/src/renderer/features/managed-agents/use-managed-agents.ts
@@ -8,16 +8,38 @@ import { events, rpc } from '@renderer/lib/ipc';
import { chatActivityChannel, chatMessageChannel } from '@shared/core/chats/chatEvents';
import type { ManagedAgentView } from '@shared/core/managed-agents/managed-agents';
import { agentMigrationChannel } from '@shared/events/agentMigrationEvents';
+import { MANAGED_AGENTS_KEY } from './use-managed-agents-key';
+import { useUserChangesLive } from './use-user-changes';
-export const MANAGED_AGENTS_KEY = 'managed-agents';
+export { MANAGED_AGENTS_KEY };
/**
- * How often the list is read again on its own. Changes made from this Console
- * (an edit, a new agent, a move) refresh it straight away, and react-query
- * reads it again when the window regains focus and pauses while it is hidden,
- * so this only bounds how long a change made elsewhere takes to show.
+ * How often the list is read again on its own, against a server that does not
+ * push changes (`useUserChangesLive`). One that does tells Console when to
+ * read it, so it is read only every `PUSHED_RECONCILE_MS`. Changes made from
+ * this Console (an edit, a new agent, a move) refresh it straight away either
+ * way, and react-query reads it again when the window regains focus.
*/
const MANAGED_AGENTS_REFRESH_MS = 30_000;
+/** The machines list's polling rate against a server that does not push changes. */
+const MACHINES_REFRESH_MS = 15_000;
+/**
+ * How often a list is read again while the server pushes changes. A notice
+ * can be lost without the socket closing: the pod that made the change died
+ * before it published it, or an older pod made it during a rolling deploy and
+ * publishes nothing. Nothing on the server can tell, so this bounds how long a
+ * list can stay stale.
+ */
+const PUSHED_RECONCILE_MS = 5 * 60_000;
+
+/**
+ * The polling rate for a list, given whether the server pushes changes. A read
+ * that failed is tried again at the polling rate even while the socket is up,
+ * since no notice may come to retry it.
+ */
+export function refreshEvery(pollMs: number, pushed: boolean, failed: boolean): number {
+ return pushed && !failed ? PUSHED_RECONCILE_MS : pollMs;
+}
/** How soon after an agent's activity in a chat the list is read again. */
const AGENT_ACTIVITY_REFRESH_MS = 1_500;
@@ -45,6 +67,7 @@ export function useManagedAgents(serverId: string | null) {
// The list is the workspace's, not the server's: switching workspace on the
// same server has to read it again rather than show the last one.
const workspaceId = workspacesStore.idOnServerInScope(serverId);
+ const pushed = useUserChangesLive(signedIn ? serverId : null);
const managementOn = useFeatureFlags(serverId).agent_management;
useEffect(
() =>
@@ -89,7 +112,7 @@ export function useManagedAgents(serverId: string | null) {
// switch-buddy included, until something remounted the list.
refetchInterval: (query) =>
query.state.data !== null
- ? MANAGED_AGENTS_REFRESH_MS
+ ? refreshEvery(MANAGED_AGENTS_REFRESH_MS, pushed, query.state.status === 'error')
: managementOn
? MANAGEMENT_UNAVAILABLE_RETRY_MS
: false,
@@ -99,6 +122,7 @@ export function useManagedAgents(serverId: string | null) {
/** The signed-in user's machines on the server, with what each reported; null without agent management. */
export function useOwnedMachines(serverId: string) {
+ const pushed = useUserChangesLive(serverId);
return useQuery({
queryKey: [
MANAGED_AGENTS_KEY,
@@ -107,7 +131,8 @@ export function useOwnedMachines(serverId: string) {
workspacesStore.idOnServerInScope(serverId),
],
queryFn: () => rpc.managedAgents.machines(serverId),
- refetchInterval: 15000,
+ refetchInterval: (query) =>
+ refreshEvery(MACHINES_REFRESH_MS, pushed, query.state.status === 'error'),
});
}
@@ -133,6 +158,7 @@ export function useMachineLabel(agent: ManagedAgentView): string | null {
/** The owner's machines on the server, with what each last reported. Null without agent management. */
export function useManagedMachines(serverId: string) {
+ const pushed = useUserChangesLive(serverId);
return useQuery({
queryKey: [
MANAGED_AGENTS_KEY,
@@ -141,7 +167,12 @@ export function useManagedMachines(serverId: string) {
workspacesStore.idOnServerInScope(serverId),
],
queryFn: () => rpc.managedAgents.machines(serverId),
- refetchInterval: (query) => (query.state.data ? 5000 : false),
+ // The same rate as `useOwnedMachines`: they share the key, so the faster
+ // of two different intervals would win for both.
+ refetchInterval: (query) =>
+ query.state.data
+ ? refreshEvery(MACHINES_REFRESH_MS, pushed, query.state.status === 'error')
+ : false,
retry: false,
});
}
diff --git a/console/apps/switch-console-desktop/src/renderer/features/managed-agents/use-user-changes.test.ts b/console/apps/switch-console-desktop/src/renderer/features/managed-agents/use-user-changes.test.ts
new file mode 100644
index 000000000..702e0b5cf
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/renderer/features/managed-agents/use-user-changes.test.ts
@@ -0,0 +1,78 @@
+// @vitest-environment jsdom
+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
+import { act, createElement, Fragment } from 'react';
+import { createRoot } from 'react-dom/client';
+import { describe, expect, it, vi } from 'vitest';
+import type { UserChangesEvent } from '@shared/events/userChangesEvents';
+
+const listeners = new Set<(event: UserChangesEvent) => void>();
+const watch = vi.fn(async () => false);
+const unwatch = vi.fn(async () => {});
+
+vi.mock('@renderer/lib/ipc', () => ({
+ events: {
+ on: (_channel: unknown, listener: (event: UserChangesEvent) => void) => {
+ listeners.add(listener);
+ return () => listeners.delete(listener);
+ },
+ },
+ rpc: { userChanges: { watch, unwatch } },
+}));
+vi.mock('@renderer/features/workspaces/workspaces-store', () => ({
+ workspacesStore: { idOnServerInScope: () => 'w1' },
+}));
+
+const { useUserChangesLive } = await import('./use-user-changes');
+
+(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true;
+
+const emit = (event: UserChangesEvent) => {
+ for (const listener of [...listeners]) listener(event);
+};
+
+describe('useUserChangesLive', () => {
+ it('reads the lists once per notice, however many rows use it', async () => {
+ const queryClient = new QueryClient();
+ const invalidate = vi.spyOn(queryClient, 'invalidateQueries').mockResolvedValue();
+ const seen: boolean[] = [];
+ function Row({ index }: { index: number }) {
+ seen[index] = useUserChangesLive('s1');
+ return null;
+ }
+ const rows = Array.from({ length: 21 }, (_, index) =>
+ createElement(Row, { key: index, index })
+ );
+ const root = createRoot(document.createElement('div'));
+ await act(async () => {
+ root.render(
+ createElement(
+ QueryClientProvider,
+ { client: queryClient },
+ createElement(Fragment, null, rows)
+ )
+ );
+ });
+ expect(watch).toHaveBeenCalledTimes(1);
+ expect(listeners.size).toBe(1);
+
+ await act(async () => {
+ emit({ serverId: 's1', workspaceId: 'w1', type: 'live', kinds: ['machine'] });
+ });
+ expect(seen.every(Boolean)).toBe(true);
+ invalidate.mockClear();
+
+ await act(async () => {
+ emit({
+ serverId: 's1',
+ workspaceId: 'w1',
+ type: 'changed',
+ changes: [{ kind: 'machine', id: 'm1' }],
+ });
+ });
+ expect(invalidate).toHaveBeenCalledTimes(1);
+
+ await act(async () => root.unmount());
+ expect(unwatch).toHaveBeenCalledTimes(1);
+ expect(listeners.size).toBe(0);
+ });
+});
diff --git a/console/apps/switch-console-desktop/src/renderer/features/managed-agents/use-user-changes.ts b/console/apps/switch-console-desktop/src/renderer/features/managed-agents/use-user-changes.ts
new file mode 100644
index 000000000..9ac3f8068
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/renderer/features/managed-agents/use-user-changes.ts
@@ -0,0 +1,95 @@
+import { type QueryClient, useQueryClient } from '@tanstack/react-query';
+import { useEffect, useState } from 'react';
+import { workspacesStore } from '@renderer/features/workspaces/workspaces-store';
+import { events, rpc } from '@renderer/lib/ipc';
+import { userChangesChannel } from '@shared/events/userChangesEvents';
+import { MANAGED_AGENTS_KEY } from './use-managed-agents-key';
+
+/**
+ * One listener per server and workspace in this window, however many
+ * components ask: every agent row reads the machines list through
+ * `useRunsOnThisComputer`, and a listener per row turned one notice into a
+ * read of the same lists per row. Each hook only follows whether the socket
+ * is live.
+ */
+type Feed = {
+ watchers: number;
+ live: boolean;
+ onLive: Set<(live: boolean) => void>;
+ stop: () => void;
+};
+
+const feeds = new Map<string, Feed>();
+
+function startFeed(queryClient: QueryClient, serverId: string, workspaceId: string | null): Feed {
+ const feed: Feed = { watchers: 0, live: false, onLive: new Set(), stop: () => {} };
+ const setLive = (live: boolean) => {
+ if (feed.live === live) return;
+ feed.live = live;
+ for (const notify of feed.onLive) notify(live);
+ };
+ const refresh = () =>
+ void queryClient.invalidateQueries({ queryKey: [MANAGED_AGENTS_KEY, serverId] });
+ let stopped = false;
+ const off = events.on(userChangesChannel, (event) => {
+ if (event.serverId !== serverId || event.workspaceId !== workspaceId) return;
+ if (event.type === 'live') {
+ setLive(true);
+ refresh();
+ } else if (event.type === 'down') {
+ setLive(false);
+ } else {
+ refresh();
+ }
+ });
+ void rpc.userChanges.watch(serverId, workspaceId).then((isLive) => {
+ if (!stopped && isLive) setLive(true);
+ });
+ feed.stop = () => {
+ stopped = true;
+ off();
+ setLive(false);
+ void rpc.userChanges.unwatch(serverId, workspaceId);
+ };
+ return feed;
+}
+
+/**
+ * Keep the change socket for this server and its workspace in scope open
+ * while the component is shown (a session is one workspace's, so switching
+ * workspace opens another), and
+ * read the managed-agents lists again whenever the server says one changed.
+ *
+ * Returns whether the server is pushing changes right now. While it is not
+ * (an older server, or the socket is between connections), callers keep
+ * polling; when it opens, everything is read once, so a change missed while
+ * it was closed shows up.
+ */
+export function useUserChangesLive(serverId: string | null): boolean {
+ const queryClient = useQueryClient();
+ const [live, setLive] = useState(false);
+ const workspaceId = workspacesStore.idOnServerInScope(serverId);
+ useEffect(() => {
+ if (serverId === null) return;
+ const key = `${serverId}\u0000${workspaceId ?? ''}`;
+ let feed = feeds.get(key);
+ if (!feed) {
+ feed = startFeed(queryClient, serverId, workspaceId);
+ feeds.set(key, feed);
+ }
+ feed.watchers += 1;
+ feed.onLive.add(setLive);
+ setLive(feed.live);
+ const held = feed;
+ return () => {
+ held.onLive.delete(setLive);
+ setLive(false);
+ held.watchers -= 1;
+ if (held.watchers === 0) {
+ feeds.delete(key);
+ held.stop();
+ }
+ };
+ }, [queryClient, serverId, workspaceId]);
+ return live;
+}
diff --git a/console/apps/switch-console-desktop/src/renderer/tests/browser/connectors-page.test.tsx b/console/apps/switch-console-desktop/src/renderer/tests/browser/connectors-page.test.tsx
index dfabcf20d..761ac3ca8 100644
--- a/console/apps/switch-console-desktop/src/renderer/tests/browser/connectors-page.test.tsx
+++ b/console/apps/switch-console-desktop/src/renderer/tests/browser/connectors-page.test.tsx
@@ -43,7 +43,12 @@ const featureFlags = vi.hoisted(() => ({
vi.mock('@renderer/lib/ipc', () => ({
events: { on: () => () => {} },
- rpc: { workspaces, featureFlags, switchServers },
+ rpc: {
+ workspaces,
+ featureFlags,
+ switchServers,
+ userChanges: { watch: async () => false, unwatch: async () => {} },
+ },
}));
vi.mock('@renderer/features/workspaces/workspaces-store', () => ({
diff --git a/console/apps/switch-console-desktop/src/renderer/tests/browser/managed-agent-page.test.tsx b/console/apps/switch-console-desktop/src/renderer/tests/browser/managed-agent-page.test.tsx
index 89ee27757..4a9c36c86 100644
--- a/console/apps/switch-console-desktop/src/renderer/tests/browser/managed-agent-page.test.tsx
+++ b/console/apps/switch-console-desktop/src/renderer/tests/browser/managed-agent-page.test.tsx
@@ -38,7 +38,12 @@ vi.hoisted(() => {
vi.mock('@renderer/lib/ipc', () => ({
events: { on: () => () => {} },
- rpc: { managedAgents, workspaces, agents: { modelCatalogue } },
+ rpc: {
+ managedAgents,
+ workspaces,
+ agents: { modelCatalogue },
+ userChanges: { watch: async () => false, unwatch: async () => {} },
+ },
}));
vi.mock(
'@renderer/features/locations/components/settings-view/sections/addressing-policy-settings-section',
diff --git a/console/apps/switch-console-desktop/src/renderer/tests/browser/new-agent-managed-machine.test.tsx b/console/apps/switch-console-desktop/src/renderer/tests/browser/new-agent-managed-machine.test.tsx
index 2abcac844..fd9c9b646 100644
--- a/console/apps/switch-console-desktop/src/renderer/tests/browser/new-agent-managed-machine.test.tsx
+++ b/console/apps/switch-console-desktop/src/renderer/tests/browser/new-agent-managed-machine.test.tsx
@@ -27,8 +27,9 @@ vi.hoisted(() => {
});
vi.mock('@renderer/lib/ipc', () => ({
- events: { on: vi.fn() },
+ events: { on: vi.fn(() => () => {}) },
rpc: {
+ userChanges: { watch: async () => false, unwatch: async () => {} },
embeddedController: { enable: embeddedEnable, defaultWorkspace },
hostControllers: { enable: hostEnable },
agents: { modelCatalogue },
diff --git a/console/apps/switch-console-desktop/src/shared/events/userChangesEvents.ts b/console/apps/switch-console-desktop/src/shared/events/userChangesEvents.ts
new file mode 100644
index 000000000..8995d6348
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/shared/events/userChangesEvents.ts
@@ -0,0 +1,27 @@
+import { defineEvent } from '@shared/lib/ipc/events';
+
+/** What changed for the signed-in user on a server, as the server pushes it. */
+export type UserChange = {
+ kind: 'managed_agent' | 'machine' | (string & {});
+ /** Null when the notice covers every one of its kind. */
+ id: string | null;
+};
+
+/**
+ * The socket's server and the workspace its session is bound to (null: the
+ * server's workspace in scope when it opened). A session is one tenant's, so
+ * each workspace on a server has a socket of its own.
+ */
+type SocketScope = { serverId: string; workspaceId: string | null };
+
+export type UserChangesEvent = SocketScope &
+ /** The server's change socket is open: lists it covers no longer need polling. */
+ (
+ | { type: 'live'; kinds: string[] }
+ /** The socket closed; poll until it is open again. */
+ | { type: 'down' }
+ | { type: 'changed'; changes: UserChange[] }
+ );
+
+/** Change notices from the servers Console is signed in to. */
+export const userChangesChannel = defineEvent<UserChangesEvent>('user-changes:event');
diff --git a/console/tooling-e2e/src/clients/switch-client.ts b/console/tooling-e2e/src/clients/switch-client.ts
index e29a3c64d..9e64ba2f8 100644
--- a/console/tooling-e2e/src/clients/switch-client.ts
+++ b/console/tooling-e2e/src/clients/switch-client.ts
@@ -1,3 +1,6 @@
+import { randomBytes } from "node:crypto";
+import { request as httpRequest } from "node:http";
+import { request as httpsRequest } from "node:https";
import type { Environment, SwitchEnv } from "../core/env.ts";
import { assertAllowed } from "../core/guard.ts";
import type { IssuedSecrets } from "../core/secrets.ts";
@@ -267,6 +270,7 @@ export class SwitchClient {
path: string,
cookie: string | null,
): Promise<RouteProbe> {
+ if (method === "WS") return this.upgradeProbe(path, cookie);
assertAllowed(this.environment, method, path);
const tag = this.requestId();
const response = await fetch(`${this.apiUrl}${path}`, {
@@ -287,6 +291,49 @@ export class SwitchClient {
};
}
+ /**
+ * A WebSocket route's answer to its opening handshake: `fetch` cannot send
+ * an upgrade, so this is the raw HTTP request a client's socket starts with.
+ * A refused handshake answers with an ordinary status (401 without a
+ * session); an accepted one answers 101, and the socket is dropped at once.
+ */
+ private upgradeProbe(path: string, cookie: string | null): Promise<RouteProbe> {
+ // The handshake is a GET, and a refused one changes nothing.
+ assertAllowed(this.environment, "GET", path);
+ const tag = this.requestId();
+ const url = new URL(`${this.apiUrl}${path}`);
+ const send = url.protocol === "https:" ? httpsRequest : httpRequest;
+ return new Promise((resolve, reject) => {
+ const request = send(url, {
+ headers: {
+ Connection: "Upgrade",
+ Upgrade: "websocket",
+ "Sec-WebSocket-Version": "13",
+ "Sec-WebSocket-Key": randomBytes(16).toString("base64"),
+ ...(cookie === null ? {} : { Cookie: cookie }),
+ ...tag,
+ },
+ timeout: 30_000,
+ });
+ const requestId = tag["X-Request-Id"] ?? null;
+ request.on("response", (response) => {
+ response.resume();
+ resolve({
+ status: response.statusCode ?? 0,
+ contentType: response.headers["content-type"] ?? "",
+ requestId,
+ });
+ });
+ request.on("upgrade", (response, socket) => {
+ socket.destroy();
+ resolve({ status: response.statusCode ?? 101, contentType: "", requestId });
+ });
+ request.on("timeout", () => request.destroy(new Error(`No answer to the handshake on ${path}`)));
+ request.on("error", reject);
+ request.end();
+ });
+ }
+
/**
* The same server, signed in as someone else. Shares this client's record of
* issued credentials, so the other session is scanned for too, and its
diff --git a/console/tooling-e2e/src/fast/session-required.ts b/console/tooling-e2e/src/fast/session-required.ts
index 0ddea5af1..2bb0a7c4d 100644
--- a/console/tooling-e2e/src/fast/session-required.ts
+++ b/console/tooling-e2e/src/fast/session-required.ts
@@ -17,6 +17,8 @@ import type { ServerSurface } from '../gate/gate.ts';
* A route behind a feature flag the deployment has off is not mounted, and
* answers 404 for that reason alone, so the probe skips exactly those: the
* snapshot names each route's flags, and the deployment says which it has on.
+ * A WebSocket route (`WS /gateway/...`) is probed with its opening handshake,
+ * which a session socket refuses with the same 401.
* Every other session route is probed, whether or not it appears in the
* deployment's OpenAPI document. Path parameters get a well-formed id that names nothing: the
* session check runs before the route ever looks the id up.
diff --git a/console/tooling-e2e/surface/exemptions.json b/console/tooling-e2e/surface/exemptions.json
index ce3f821fb..08048227b 100644
--- a/console/tooling-e2e/surface/exemptions.json
+++ b/console/tooling-e2e/surface/exemptions.json
@@ -14,7 +14,8 @@
"GET /gateway/service-connections/{service}/flows/callback": "Needs the third-party service's own sign-in page in a browser.",
"POST /gateway/auth/oidc/exchange": "Needs a code from a live OIDC sign-in; the harness signs in with a password.",
"POST /gateway/hosted-controller/machines/{machine_id}/observation": "Needs a hosted controller configured (HOSTED_CONTROLLER_CONFIG_PATH), which no stack the harness builds has.",
- "POST /gateway/hosted-controller/machines/{machine_id}/prepare": "Needs a hosted controller configured (HOSTED_CONTROLLER_CONFIG_PATH), which no stack the harness builds has."
+ "POST /gateway/hosted-controller/machines/{machine_id}/prepare": "Needs a hosted controller configured (HOSTED_CONTROLLER_CONFIG_PATH), which no stack the harness builds has.",
+ "WS /gateway/changes/ws": "Console's change-notice socket. What it sends is covered by core's test_changes_socket.py, and the session probe checks it refuses a handshake without a session; no scenario drives Console's lists yet."
},
"unauthenticated": {
"GET /gateway/agents/known-types": "The catalogue of agent types this server can register and their option schemas; no tenant data. Public since the initial import, and nothing records whether that is deliberate.",
diff --git a/console/tooling-e2e/surface/server-surface.json b/console/tooling-e2e/surface/server-surface.json
index 07c838125..b2632169c 100644
--- a/console/tooling-e2e/surface/server-surface.json
+++ b/console/tooling-e2e/surface/server-surface.json
@@ -1403,6 +1403,11 @@
"flags": [
"service_connections"
]
+ },
+ {
+ "route": "WS /gateway/changes/ws",
+ "auth": "session",
+ "flags": []
}
]
}
diff --git a/core/switch_core/gateway/app.py b/core/switch_core/gateway/app.py
index 2870b4e3c..282328c09 100644
--- a/core/switch_core/gateway/app.py
+++ b/core/switch_core/gateway/app.py
@@ -41,6 +41,7 @@ from switch_core.gateway.agent_sessions import router as agent_sessions_router
from switch_core.gateway.agents import router as agents_router
from switch_core.gateway.api_keys import router as api_keys_router
from switch_core.gateway.auth_routes import router as auth_router
+from switch_core.gateway.changes import router as changes_router
from switch_core.gateway.chats import chat_error_response
from switch_core.gateway.chats import router as chats_router
from switch_core.gateway.collaborations import router as collaborations_router
@@ -83,6 +84,7 @@ from switch_core.providers.hosted import HostedControllerSettings
from switch_core.room_service import RoomService
from switch_core.sessions.errors import SessionError
from switch_core.sessions.http import session_error_response
+from switch_core.user_changes import LocalUserChanges
def create_gateway_app(
@@ -110,6 +112,7 @@ def create_gateway_app(
protocol: AgentCore,
install_service: MessagingInstallService | None,
invite_mailer: InviteMailer | None,
+ user_changes: LocalUserChanges,
chat_service: ChatService,
connector_apps: ConnectorApps,
config: SwitchConfig,
@@ -138,6 +141,7 @@ def create_gateway_app(
protocol=protocol,
install_service=install_service,
invite_mailer=invite_mailer,
+ user_changes=user_changes,
config=config,
)
@@ -201,6 +205,7 @@ def create_gateway_app(
app.include_router(oidc_router, tags=["auth"])
app.include_router(tenants_router, tags=["tenants"])
app.include_router(rooms_router, prefix="/rooms", tags=["rooms"])
+ app.include_router(changes_router, tags=["changes"])
app.include_router(chats_router, prefix="/chats", tags=["chats"])
app.include_router(room_groups_router, prefix="/room-groups", tags=["room-groups"])
app.include_router(agents_router, prefix="/agents", tags=["agents"])
diff --git a/core/switch_core/gateway/auth.py b/core/switch_core/gateway/auth.py
index 812d14987..ac15568fe 100644
--- a/core/switch_core/gateway/auth.py
+++ b/core/switch_core/gateway/auth.py
@@ -12,6 +12,7 @@ import bcrypt
import jwt
from fastapi import Depends, HTTPException, Request, Response
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+from starlette.requests import HTTPConnection
from switch_core.authz import Action, Principal, require
from switch_core.config import SwitchConfig
@@ -193,7 +194,7 @@ async def _authenticate(
return payload
-def _session_claims(request: Request, config: SwitchConfig) -> dict:
+def _session_claims(request: HTTPConnection, config: SwitchConfig) -> dict:
token = request.cookies.get("switch_auth")
if token is None:
raise HTTPException(status_code=401, detail="Not authenticated")
@@ -655,6 +656,45 @@ async def _bound_user(
yield user
+@dataclass(frozen=True)
+class SocketCaller:
+ """Who opened a gateway WebSocket, and the tenant their session binds."""
+
+ user_id: str
+ tenant_id: str
+ # When the session cookie the socket was opened with expires, as a UNIX
+ # time: the socket is closed then, and the client opens it again with the
+ # cookie it has renewed meanwhile.
+ expires_at: float
+
+
+async def authenticate_socket(
+ connection: HTTPConnection,
+ session_factory: async_sessionmaker[AsyncSession],
+ user_store: UserStore,
+ config: SwitchConfig,
+) -> SocketCaller:
+ """`get_current_user`'s checks for a WebSocket, which has no request
+ session to load the user on and must not keep one for its lifetime.
+
+ The cookie, the account and the tenant resolution are exactly
+ `get_current_user`'s, each on a short session of its own that is closed
+ before this returns. Raises the same `HTTPException`s.
+ """
+ payload = _session_claims(connection, config)
+ user_id: str = payload["sub"]
+ tenant_id = await _resolve_tenant_id(
+ session_factory,
+ user_store,
+ user_id,
+ payload.get("tenant_id"),
+ config.gateway_tenant_choice_enabled,
+ )
+ return SocketCaller(
+ user_id=user_id, tenant_id=tenant_id, expires_at=float(payload["exp"])
+ )
+
+
async def get_current_user(
request: Request,
session: Annotated[AsyncSession, Depends(get_session)],
diff --git a/core/switch_core/gateway/changes.py b/core/switch_core/gateway/changes.py
new file mode 100644
index 000000000..192e020c8
--- /dev/null
+++ b/core/switch_core/gateway/changes.py
@@ -0,0 +1,197 @@
+"""The signed-in user's change notices, pushed over one WebSocket.
+
+Console keeps this open per server it is signed in to, and reads a list again
+when a notice names its kind, instead of polling it every few seconds (it
+still reads it every few minutes, for notices that were never sent). The frames are
+JSON `{"event", "data"}`, as on the agent and controller sockets:
+
+- `hello` once, on opening: `protocol`, the `kinds` this server sends notices
+ for, and `ping_interval_s`. A client stops polling only for those kinds.
+- `changed`: `changes`, a list of `{"kind", "id"}`; `id` null means any of
+ that kind (`switch_core.user_changes`).
+- `ping` every `ping_interval_s`; the client answers `{"type": "pong"}`. A
+ client that sends nothing for `_SILENT_PINGS` intervals is gone, and its
+ socket is closed. A client that hears no ping for as long should assume the
+ same of the server and reconnect.
+
+Authenticated once, during the handshake, by the same session cookie and
+tenant resolution as every gateway route (`authenticate_socket`). A caller
+without a valid session gets the HTTP 401 any gateway route answers and no
+socket: nothing is accepted, held or sent for it. Console treats that like any
+socket that would not open, and its next read of a list (which still polls
+every few minutes) gets the same 401 and asks the user to sign in again. The
+socket holds no database connection while it is open. It is closed with 4401
+when the cookie it was opened with expires, so a client reconnects with the
+cookie it has renewed since.
+
+A user holds at most `MAX_SOCKETS_PER_USER` of these at once. Console opens
+one per server, so the cap only meets a client opening sockets in a loop, which
+would otherwise hold a file handle and a subscription per socket. Past it the
+socket is refused with a `refused` frame and close 4429, and Console backs off and keeps
+polling, so a refused socket costs a delay, never a stale list.
+
+A client reads everything again each time the socket opens, so a notice lost
+while it was closed costs nothing more than that read.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from switch_core.config import SwitchConfig
+from switch_core.db.stores.user_store import UserStore
+from switch_core.gateway.auth import SocketCaller, authenticate_socket
+from switch_core.gateway.dependencies import (
+ get_config,
+ get_session_factory,
+ get_user_changes,
+ get_user_store,
+)
+from switch_core.user_changes import KINDS, LocalUserChanges, UserChangeSubscription
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+PROTOCOL = 1
+PING_INTERVAL_SECONDS = 25.0
+# Intervals without a word from the client before its socket is closed.
+_SILENT_PINGS = 3
+# The close code for a session cookie that has expired: 4000 plus 401.
+_EXPIRED = 4401
+# Sockets one user may hold on this process: one per Console is the norm, so this
+# leaves room for several machines and windows and stops a runaway client.
+MAX_SOCKETS_PER_USER = 16
+
+
+async def get_socket_caller(
+ websocket: WebSocket,
+ session_factory: Annotated[
+ async_sessionmaker[AsyncSession], Depends(get_session_factory)
+ ],
+ user_store: Annotated[UserStore, Depends(get_user_store)],
+ config: Annotated[SwitchConfig, Depends(get_config)],
+) -> SocketCaller:
+ """The signed-in caller, checked before the socket is accepted.
+
+ A dependency rather than a call in the handler, so its `HTTPException`
+ reaches the client as the handshake's HTTP response (Starlette's denial
+ response) instead of a frame on a socket that was accepted for nothing.
+ """
+ return await authenticate_socket(websocket, session_factory, user_store, config)
+
+
[email protected]("/changes/ws")
+async def changes_socket(
+ websocket: WebSocket,
+ caller: Annotated[SocketCaller, Depends(get_socket_caller)],
+ changes: Annotated[LocalUserChanges, Depends(get_user_changes)],
+) -> None:
+ await websocket.accept()
+ # No await between the count and the subscribe, so two sockets opening at
+ # once cannot both slip under the cap.
+ if (
+ changes.subscriber_count(caller.tenant_id, caller.user_id)
+ >= MAX_SOCKETS_PER_USER
+ ):
+ await _refuse(
+ websocket, 429, f"At most {MAX_SOCKETS_PER_USER} change sockets per user."
+ )
+ return
+ subscription = changes.subscribe(caller.tenant_id, caller.user_id)
+ heard = asyncio.Event()
+ listener = asyncio.create_task(_listen(websocket, heard))
+ try:
+ await websocket.send_json(
+ {
+ "event": "hello",
+ "data": {
+ "protocol": PROTOCOL,
+ "kinds": list(KINDS),
+ "ping_interval_s": PING_INTERVAL_SECONDS,
+ },
+ }
+ )
+ code = await _serve(websocket, subscription, heard, listener, caller.expires_at)
+ except (WebSocketDisconnect, RuntimeError):
+ # Gone, or closed under us by the server shutting down: nothing to send.
+ return
+ finally:
+ subscription.close()
+ listener.cancel()
+ try:
+ await websocket.close(code=code)
+ except RuntimeError:
+ pass # the client closed it first
+
+
+async def _refuse(websocket: WebSocket, status: int, detail: object) -> None:
+ await websocket.send_json(
+ {"event": "refused", "data": {"status": status, "detail": detail}}
+ )
+ await websocket.close(code=4000 + status)
+
+
+async def _serve(
+ websocket: WebSocket,
+ subscription: UserChangeSubscription,
+ heard: asyncio.Event,
+ listener: asyncio.Task[None],
+ expires_at: float,
+) -> int:
+ """Send notices as they come and a ping every interval, until the client
+ goes quiet, goes away or its cookie expires. Returns the close code.
+
+ One writer: a WebSocket must not be sent to from two tasks at once, so
+ notices and pings both go out from this loop."""
+ loop = asyncio.get_running_loop()
+ next_ping = loop.time() + PING_INTERVAL_SECONDS
+ silent = 0
+ while not listener.done():
+ if time.time() >= expires_at:
+ return _EXPIRED
+ timeout = min(next_ping - loop.time(), expires_at - time.time())
+ woken = asyncio.ensure_future(subscription.wake.wait())
+ await asyncio.wait(
+ {woken, listener},
+ timeout=max(0.0, timeout),
+ return_when=asyncio.FIRST_COMPLETED,
+ )
+ woken.cancel()
+ if listener.done():
+ break
+ drained = subscription.drain()
+ if drained:
+ await websocket.send_json(
+ {
+ "event": "changed",
+ "data": {
+ "changes": [{"kind": c.kind, "id": c.id} for c in drained]
+ },
+ }
+ )
+ if loop.time() >= next_ping:
+ silent = 0 if heard.is_set() else silent + 1
+ heard.clear()
+ if silent >= _SILENT_PINGS:
+ return 1001
+ await websocket.send_json({"event": "ping", "data": {}})
+ next_ping = loop.time() + PING_INTERVAL_SECONDS
+ return 1000
+
+
+async def _listen(websocket: WebSocket, heard: asyncio.Event) -> None:
+ """Note each message the client sends, until it goes. Its content is not
+ read: a pong and anything else both say the client is there."""
+ while True:
+ try:
+ await websocket.receive_text()
+ except (WebSocketDisconnect, RuntimeError):
+ return
+ heard.set()
diff --git a/core/switch_core/gateway/dependencies.py b/core/switch_core/gateway/dependencies.py
index ff6b06d16..b4941a833 100644
--- a/core/switch_core/gateway/dependencies.py
+++ b/core/switch_core/gateway/dependencies.py
@@ -37,6 +37,7 @@ from switch_core.gateway.invite_mail import InviteMailer
from switch_core.room_service import RoomService
from switch_core.rooms_yaml import RoomYamlService
from switch_core.telemetry import TelemetryService
+from switch_core.user_changes import LocalUserChanges
_state: dict[str, Any] = {}
@@ -66,6 +67,7 @@ def init_dependencies(
protocol: AgentCore,
install_service: MessagingInstallService | None,
invite_mailer: InviteMailer | None,
+ user_changes: LocalUserChanges,
config: SwitchConfig,
) -> None:
_state["agent_store"] = agent_store
@@ -91,6 +93,7 @@ def init_dependencies(
_state["protocol"] = protocol
_state["install_service"] = install_service
_state["invite_mailer"] = invite_mailer
+ _state["user_changes"] = user_changes
_state["config"] = config
@@ -230,6 +233,10 @@ def get_template_store() -> TemplateStore:
return _state["template_store"] # type: ignore[no-any-return]
+def get_user_changes() -> LocalUserChanges:
+ return _state["user_changes"] # type: ignore[no-any-return]
+
+
def get_config() -> SwitchConfig:
return _state["config"] # type: ignore[no-any-return]
diff --git a/core/switch_core/main.py b/core/switch_core/main.py
index 9d17aa22b..787571c8b 100644
--- a/core/switch_core/main.py
+++ b/core/switch_core/main.py
@@ -222,6 +222,7 @@ from switch_core.transport.invites import InviteBus
from switch_core.transport.membership_signals import MembershipSignals
from switch_core.transport.room_cache import RoomCacheLimits, RoomDeliveryCache
from switch_core.trust.setup import build_trust_client
+from switch_core.user_changes import LocalUserChanges, SignalUserChanges
from switch_core.version import switch_core_version
logger = logging.getLogger(__name__)
@@ -1013,8 +1014,17 @@ async def run(config: SwitchConfig, process_id: str) -> None:
# Built before the agent bridge app because its authenticator is the bearer
# middleware's controller branch; its routes are installed once both apps
# exist, below.
+ # Change notices for signed-in users' Consoles (`gateway/changes.py`): the
+ # sockets this process holds, reached from every process over the bus.
+ user_changes = LocalUserChanges()
+ user_change_publisher = SignalUserChanges(local=user_changes, bus=signal_bus)
management = create_management(
- config, session_factory, connections.controllers, signal_bus, process_lease
+ config,
+ session_factory,
+ connections.controllers,
+ signal_bus,
+ process_lease,
+ user_change_publisher,
)
# ── FastAPI apps ─────────────────────────────────────────────────────────
@@ -1226,6 +1236,7 @@ async def run(config: SwitchConfig, process_id: str) -> None:
if config.invite_email_enabled
else None
),
+ user_changes=user_changes,
chat_service=chat_service,
connector_apps=connector_apps,
config=config,
diff --git a/core/switch_core/management/service.py b/core/switch_core/management/service.py
index fef676bdc..06ed6fbda 100644
--- a/core/switch_core/management/service.py
+++ b/core/switch_core/management/service.py
@@ -101,6 +101,7 @@ from switch_core.management.sealed_logins import PublicKey, SealedLogin, key_id
from switch_core.management.signals import ControllerSignals
from switch_core.process_lease import ProcessLeases, read_leases
from switch_core.providers.registry import provider_ids
+from switch_core.user_changes import MACHINE, MANAGED_AGENT, UserChangePublisher
logger = logging.getLogger(__name__)
@@ -145,6 +146,7 @@ class ManagementService:
*,
settings: ManagementSettings,
signals: ControllerSignals,
+ user_changes: UserChangePublisher,
controllers: AgentControllerStore,
definitions: AgentDefinitionStore,
operations: AgentControllerOperationStore,
@@ -157,6 +159,7 @@ class ManagementService:
self.settings = settings
self.signals = signals
self.notifier = signals.notifier
+ self.user_changes = user_changes
self.presence = presence
self.controllers = controllers
self.definitions = definitions
@@ -225,6 +228,9 @@ class ManagementService:
api_key_id=key.id,
)
await session.commit()
+ self.user_changes.publish(
+ controller.tenant_id, owner_id, MACHINE, controller.id
+ )
logger.info(
"Enrolled agent controller %s (%s) for user %s",
controller.id,
@@ -366,6 +372,7 @@ class ManagementService:
len(moved),
controller.id,
)
+ self.user_changes.publish(tenant_id, consumed.owner_id, MACHINE, controller.id)
logger.info(
"Enrolled agent controller %s (%s) by code for user %s",
controller.id,
@@ -460,13 +467,21 @@ class ManagementService:
report: StatusReport,
) -> dict[str, Any]:
"""Store a status report unless a newer one is stored, and tell the
- controller its assignment revision either way."""
+ controller its assignment revision either way.
+
+ The owner's Console is told only when what the report says changed,
+ not on every report: free disk and memory, and the times a report and
+ its provider checks were taken, move on every one of them.
+ """
+ controller = await self._principal_controller(session, principal)
+ before = _status_signature(controller.status)
+ new_status = report.model_dump(mode="json", exclude_unset=True)
stored = await self.controllers.record_status(
session,
principal.tenant_id,
principal.controller_id,
seq=report.seq,
- status=report.model_dump(mode="json", exclude_unset=True),
+ status=new_status,
version=report.controller.version,
platform=report.machine.platform.model_dump(mode="json"),
seen_at=self.now(),
@@ -477,13 +492,21 @@ class ManagementService:
report.seq,
principal.controller_id,
)
- controller = await self._principal_controller(session, principal)
+ # Read again: the store wrote the row without touching the copy loaded
+ # above, and the release the controller is told depends on the status
+ # it just reported.
+ await session.refresh(controller)
revision = controller.assignment_revision
+ owner_id = controller.owner_id
if stored and controller.kind == "ec2":
await record_controller_status(
session, controller.id, report.machine.model_dump(mode="json")
)
await session.commit()
+ if stored and _status_signature(new_status) != before:
+ self.user_changes.publish(
+ principal.tenant_id, owner_id, MACHINE, principal.controller_id
+ )
return {
"assignment_revision": revision,
"report_within_s": self.settings.status_interval_seconds,
@@ -857,7 +880,9 @@ class ManagementService:
leases,
self.settings.controller_release,
)
+ owner_id = controller.owner_id
await session.commit()
+ self.user_changes.publish(tenant_id, owner_id, MACHINE, controller_id)
if "name" in changes:
self.presence.rename_controller(controller_id, controller.name)
self.signals.renamed(tenant_id, controller_id)
@@ -888,6 +913,7 @@ class ManagementService:
else None
)
await session.commit()
+ self.user_changes.publish(tenant_id, owner_id, MACHINE, controller_id)
logger.info("Revoked agent controller %s", controller_id)
if machine is not None:
logger.info(
@@ -1312,6 +1338,7 @@ class ManagementService:
session, tenant_id, {request.controller_id}
)
await session.commit()
+ self.user_changes.publish(tenant_id, owner_id, MANAGED_AGENT, result.agent_id)
await self._bind(session, tenant_id, row)
self._nudge(tenant_id, revisions)
logger.info(
@@ -1466,6 +1493,7 @@ class ManagementService:
)
revisions = await self._bump_and_collect(session, tenant_id, affected)
await session.commit()
+ self.user_changes.publish(tenant_id, owner_id, MANAGED_AGENT, agent.id)
await self._bind(session, tenant_id, row)
self._nudge(tenant_id, revisions)
agent = await self._owned_agent(session, owner_id, agent.id)
@@ -1490,6 +1518,7 @@ class ManagementService:
session, tenant_id, {row.controller_id}
)
await session.commit()
+ self.user_changes.publish(tenant_id, owner_id, MANAGED_AGENT, agent_id)
self.presence.unbind(agent_id, DETACH_UNASSIGNED)
self._nudge(tenant_id, revisions)
@@ -1551,6 +1580,7 @@ class ManagementService:
session, tenant_id, {row.controller_id}
)
await session.commit()
+ self.user_changes.publish(tenant_id, row.owner_id, MANAGED_AGENT, agent_id)
self.presence.unbind(agent_id, DETACH_DELETED)
self._nudge(tenant_id, revisions)
logger.info("Stopped managing agent %s, which is being deleted", agent_id)
@@ -1748,3 +1778,29 @@ def _known_agent_registration(
metadata["known_agent_type"] = known_type
metadata["known_agent_options"] = options.model_dump()
return spec, options, metadata
+
+
+# Status fields that change on every report whether or not anything happened.
+_VOLATILE_MACHINE = ("disk_free_bytes", "mem_free_bytes")
+
+
+def _status_signature(status: dict[str, Any] | None) -> dict[str, Any] | None:
+ """A status report without the fields that move on every report, so two
+ reports compare equal when nothing a person would see has changed."""
+ if status is None:
+ return None
+ signature = {k: v for k, v in status.items() if k not in ("seq", "observed_at")}
+ machine = signature.get("machine")
+ if isinstance(machine, dict):
+ signature["machine"] = {
+ k: v for k, v in machine.items() if k not in _VOLATILE_MACHINE
+ }
+ providers = signature.get("providers")
+ if isinstance(providers, list):
+ signature["providers"] = [
+ {k: v for k, v in p.items() if k != "checked_at"}
+ if isinstance(p, dict)
+ else p
+ for p in providers
+ ]
+ return signature
diff --git a/core/switch_core/management/signals.py b/core/switch_core/management/signals.py
index ceaec7138..5f2a1200c 100644
--- a/core/switch_core/management/signals.py
+++ b/core/switch_core/management/signals.py
@@ -22,6 +22,12 @@ run by a controller connected to any process then reads as live in every one,
and a connection this process holds that another process has since replaced
or taken up is closed `taken_over`.
+Each change to a controller's connection is also told to the controller's
+owner, whose Console shows the machine as connected or not
+(`switch_core.user_changes`). The process that made the change reads the
+owner from the controller's row once it has committed (`_tell_owner`), and
+publishes the notice for every process to hand to the owner's sockets.
+
Nothing here depends on a signal arriving. When the signal connection comes
back, every binding and every controller's connection is read again and every
controller held here is re-nudged from the database, so one that missed a
@@ -47,6 +53,7 @@ from switch_core.management.bindings import (
)
from switch_core.process_lease import ProcessLeases, read_leases
from switch_core.tenant_context import no_tenant
+from switch_core.user_changes import MACHINE
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
@@ -66,6 +73,7 @@ if TYPE_CHECKING:
)
from switch_core.management.notifier import ControllerNotifier
from switch_core.signals import SignalBus
+ from switch_core.user_changes import UserChangePublisher
logger = logging.getLogger(__name__)
@@ -95,6 +103,7 @@ class ControllerSignals:
rooms: RoomStore,
processes: SwitchCoreProcessStore,
clock: Callable[[], datetime],
+ user_changes: UserChangePublisher,
) -> None:
self.notifier = notifier
# The process this is, as signals and the rows naming a holder call it.
@@ -108,6 +117,7 @@ class ControllerSignals:
self._rooms = rooms
self._processes = processes
self._clock = clock
+ self._user_changes = user_changes
# What each controller has been nudged with from this process, so the
# periodic re-check tells a held controller only what is new to it.
self._nudged_revision: dict[str, int] = {}
@@ -122,6 +132,9 @@ class ControllerSignals:
self._connections_owed = bus.pending_work(
"controller connections", self._read_connection_of
)
+ # Controllers, by (tenant id, controller id), whose connection this
+ # process changed and whose owner has still to be told.
+ self._owners_owed = bus.pending_work("controller owners", self._tell_owner)
bus.handle(ASSIGNMENT_CHANGED_SIGNAL, self._on_assignment_changed)
bus.handle(OPERATION_PENDING_SIGNAL, self._on_operation_pending)
bus.handle(REVOKED_SIGNAL, self._on_revoked)
@@ -177,6 +190,19 @@ class ControllerSignals:
self._bus.publish(
CONNECTION_CHANGED_SIGNAL, tenant_id=tenant_id, controller_id=controller_id
)
+ self._owners_owed.add((tenant_id, controller_id))
+
+ async def _tell_owner(self, key: tuple[str, str]) -> None:
+ """Tell the controller's owner that its machine changed. Their
+ Console reads the machine again, so the notice carries nothing else."""
+ tenant_id, controller_id = key
+ async with tenant_session(self._session_factory, tenant_id) as session:
+ controller = await self._controllers.get(session, tenant_id, controller_id)
+ if controller is None:
+ return
+ self._user_changes.publish(
+ tenant_id, controller.owner_id, MACHINE, controller_id
+ )
# ── Receiving, in every other process ─────────────────────────────────────
diff --git a/core/switch_core/management/wiring.py b/core/switch_core/management/wiring.py
index ca6b49c04..c9d93e497 100644
--- a/core/switch_core/management/wiring.py
+++ b/core/switch_core/management/wiring.py
@@ -57,6 +57,7 @@ from switch_core.management.service import ManagementService, ManagementSettings
from switch_core.management.signals import ControllerSignals
from switch_core.process_lease import ProcessLease
from switch_core.signals import SignalBus
+from switch_core.user_changes import UserChangePublisher
logger = logging.getLogger(__name__)
@@ -141,6 +142,7 @@ def build_management(
clock: Callable[[], datetime],
signal_bus: SignalBus,
lease: ProcessLease,
+ user_changes: UserChangePublisher,
) -> Management:
process_id = lease.process_id
if signal_bus.process_id != process_id:
@@ -164,6 +166,7 @@ def build_management(
rooms=RoomStore(),
processes=processes,
clock=clock,
+ user_changes=user_changes,
)
ledger = ControllerConnectionLedger(
process_id=process_id,
@@ -182,6 +185,7 @@ def build_management(
controller_release=controller_release,
),
signals=signals,
+ user_changes=user_changes,
controllers=controllers,
definitions=definitions,
operations=operations,
@@ -228,6 +232,7 @@ def create_management(
presence: ControllerPresence,
signal_bus: SignalBus,
lease: ProcessLease,
+ user_changes: UserChangePublisher,
) -> Management | None:
if not config.agent_management_enabled:
return None
@@ -254,4 +259,5 @@ def create_management(
clock=utc_now,
signal_bus=signal_bus,
lease=lease,
+ user_changes=user_changes,
)
diff --git a/core/switch_core/user_changes.py b/core/switch_core/user_changes.py
new file mode 100644
index 000000000..497bab076
--- /dev/null
+++ b/core/switch_core/user_changes.py
@@ -0,0 +1,174 @@
+"""Telling a signed-in user that something of theirs changed, so their Console
+reads it again instead of polling for it.
+
+A change names a kind and an id (`managed_agent`, `machine`) and carries
+nothing else: the client already knows how to read the current state, and a
+notice that is only "read again" cannot leak what the reader would not be
+allowed to read. That also makes a lost notice cheap. A client reads
+everything again whenever its socket (re)connects, so a notice lost to a
+restart or to a socket that was between connections costs a delay, never
+correctness, and notices can coalesce: the same change twice before the
+socket wakes is sent once.
+
+Subscriptions are per (tenant, user), because a session is bound to one
+tenant: a change in another of the user's workspaces is not this session's.
+
+Each process holds its own Consoles' sockets, so a change committed in one
+process has to reach every other. `LocalUserChanges` fans a notice out in
+memory to the sockets this process holds. `SignalUserChanges` is what writers
+are given: it hands each notice to this process's `LocalUserChanges` at once
+and publishes it on the `SignalBus` (`switch_core.signals`), and every other
+process hands what it hears to its own. Writers depend on
+`UserChangePublisher` alone.
+
+A notice that the bus loses (its sender died between the commit and the send,
+or a receiver's listening connection was down) is not re-sent: there is
+nothing in the database to compare a socket against. Instead, when a
+process's listening connection comes back, every socket it holds is told that
+everything of every kind changed, so its Console reads it all again, as it
+would on reconnecting. That does not cover a notice that was never sent: the
+writer pod died after its commit, or an older pod without this module made the
+change during a rolling deploy. For those, Console also reads its lists again
+every few minutes while the socket is up, which bounds how stale one can get.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Protocol
+
+if TYPE_CHECKING:
+ from switch_core.signals import SignalBus
+
+logger = logging.getLogger(__name__)
+
+MANAGED_AGENT = "managed_agent"
+MACHINE = "machine"
+KINDS = (MANAGED_AGENT, MACHINE)
+
+USER_CHANGED_SIGNAL = "user.changed"
+
+# Notices a subscription holds before it collapses them into one per kind
+# (`id` None, "something of this kind changed"), so a socket that is not being
+# read cannot grow without bound.
+_MAX_PENDING = 64
+
+
+@dataclass(frozen=True)
+class UserChange:
+ kind: str
+ # None when the notice covers every one of its kind.
+ id: str | None
+
+
+class UserChangePublisher(Protocol):
+ def publish(self, tenant_id: str, user_id: str, kind: str, id: str) -> None:
+ """Tell `user_id`'s sessions in `tenant_id` that this changed. Call it
+ after the change has committed, and never wait on it."""
+ ...
+
+
+class UserChangeSubscription:
+ """One open socket's pending notices, and the event that wakes it."""
+
+ def __init__(self, feed: LocalUserChanges, key: tuple[str, str]) -> None:
+ self._feed = feed
+ self.key = key
+ self.wake = asyncio.Event()
+ # A dict as an ordered set: notices go out in the order they came.
+ self._pending: dict[UserChange, None] = {}
+
+ def _add(self, change: UserChange) -> None:
+ # A pending "any of this kind" already covers this one.
+ if UserChange(change.kind, None) in self._pending:
+ self.wake.set()
+ return
+ self._pending[change] = None
+ if len(self._pending) > _MAX_PENDING:
+ kinds = dict.fromkeys(c.kind for c in self._pending)
+ self._pending = {UserChange(kind, None): None for kind in kinds}
+ self.wake.set()
+
+ def drain(self) -> list[UserChange]:
+ self.wake.clear()
+ changes = list(self._pending)
+ self._pending.clear()
+ return changes
+
+ def close(self) -> None:
+ self._feed._unsubscribe(self)
+
+
+class LocalUserChanges:
+ """`UserChangePublisher` for one process, and the subscriptions it reaches."""
+
+ def __init__(self) -> None:
+ self._subscriptions: dict[tuple[str, str], set[UserChangeSubscription]] = {}
+
+ def subscribe(self, tenant_id: str, user_id: str) -> UserChangeSubscription:
+ key = (tenant_id, user_id)
+ subscription = UserChangeSubscription(self, key)
+ self._subscriptions.setdefault(key, set()).add(subscription)
+ return subscription
+
+ def _unsubscribe(self, subscription: UserChangeSubscription) -> None:
+ held = self._subscriptions.get(subscription.key)
+ if held is None:
+ return
+ held.discard(subscription)
+ if not held:
+ del self._subscriptions[subscription.key]
+
+ def subscriber_count(self, tenant_id: str, user_id: str) -> int:
+ return len(self._subscriptions.get((tenant_id, user_id), ()))
+
+ def publish(self, tenant_id: str, user_id: str, kind: str, id: str) -> None:
+ change = UserChange(kind, id)
+ for subscription in self._subscriptions.get((tenant_id, user_id), ()):
+ subscription._add(change)
+
+ def publish_everything(self) -> None:
+ """Tell every subscription here that anything of any kind may have
+ changed: for when notices may have been missed."""
+ everything = [UserChange(kind, None) for kind in KINDS]
+ for held in self._subscriptions.values():
+ for subscription in held:
+ for change in everything:
+ subscription._add(change)
+
+
+class SignalUserChanges:
+ """`UserChangePublisher` for every process: a notice published in one
+ reaches the subscriptions held in each of them."""
+
+ def __init__(self, *, local: LocalUserChanges, bus: SignalBus) -> None:
+ self._local = local
+ self._bus = bus
+ bus.handle(USER_CHANGED_SIGNAL, self._on_changed)
+ bus.on_resync(self._resync)
+
+ def publish(self, tenant_id: str, user_id: str, kind: str, id: str) -> None:
+ # The bus skips a process's own signals, so this one is told here.
+ self._local.publish(tenant_id, user_id, kind, id)
+ self._bus.publish(
+ USER_CHANGED_SIGNAL,
+ tenant_id=tenant_id,
+ user_id=user_id,
+ # `kind` is the signal's own.
+ change=kind,
+ id=id,
+ )
+
+ def _on_changed(self, signal: dict[str, Any]) -> None:
+ kind = signal.get("change")
+ if kind not in KINDS:
+ # A newer process may send kinds an older one does not know,
+ # during a rolling deploy; its Console polls for those anyway.
+ logger.debug("Ignoring a user change of unknown kind %r", kind)
+ return
+ self._local.publish(signal["tenant_id"], signal["user_id"], kind, signal["id"])
+
+ async def _resync(self) -> None:
+ self._local.publish_everything()
diff --git a/core/tests/switch_core/gateway/test_app_route_mounting.py b/core/tests/switch_core/gateway/test_app_route_mounting.py
index b119e19ae..17567ceba 100644
--- a/core/tests/switch_core/gateway/test_app_route_mounting.py
+++ b/core/tests/switch_core/gateway/test_app_route_mounting.py
@@ -31,6 +31,7 @@ def _app(**overrides: object) -> object:
event_buffer=MagicMock(),
session_factory=MagicMock(),
user_store=MagicMock(),
+ user_changes=MagicMock(),
external_user_store=MagicMock(),
api_key_store=MagicMock(),
invitation_store=MagicMock(),
diff --git a/core/tests/switch_core/gateway/test_changes_socket.py b/core/tests/switch_core/gateway/test_changes_socket.py
new file mode 100644
index 000000000..015f39647
--- /dev/null
+++ b/core/tests/switch_core/gateway/test_changes_socket.py
@@ -0,0 +1,191 @@
+"""The signed-in user's change socket: hello, notices, pings, refusal, expiry.
+
+Authentication is `get_current_user`'s own checks (`authenticate_socket`),
+covered with the gateway's auth tests; here it is replaced, so these tests
+are about the socket.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Iterator
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+from fastapi import FastAPI, HTTPException
+from fastapi.testclient import TestClient
+from starlette.testclient import WebSocketDenialResponse
+from starlette.websockets import WebSocketDisconnect
+
+from switch_core.gateway import changes
+from switch_core.gateway import dependencies as gw_deps
+from switch_core.gateway.auth import SocketCaller
+from switch_core.user_changes import MACHINE, MANAGED_AGENT, LocalUserChanges
+
+
+def _app(feed: LocalUserChanges) -> FastAPI:
+ app = FastAPI()
+ app.include_router(changes.router)
+ app.dependency_overrides[gw_deps.get_session_factory] = lambda: None
+ app.dependency_overrides[gw_deps.get_user_store] = lambda: None
+ app.dependency_overrides[gw_deps.get_config] = lambda: SimpleNamespace()
+ app.dependency_overrides[gw_deps.get_user_changes] = lambda: feed
+
+ # Notices are published on the app's own loop, as a committed change is.
+ @app.post("/publish/{user_id}/{kind}/{id}")
+ async def publish(user_id: str, kind: str, id: str) -> None:
+ feed.publish("t1", user_id, kind, id)
+
+ return app
+
+
+def feed() -> LocalUserChanges:
+ return LocalUserChanges()
+
+
+def _signed_in(
+ monkeypatch: pytest.MonkeyPatch, user_id: str = "ada", expires_in: float = 3600
+) -> None:
+ async def _authenticate(*_: Any) -> SocketCaller:
+ return SocketCaller(
+ user_id=user_id, tenant_id="t1", expires_at=time.time() + expires_in
+ )
+
+ monkeypatch.setattr(changes, "authenticate_socket", _authenticate)
+
+
+def client(feed: LocalUserChanges) -> Iterator[TestClient]:
+ with TestClient(_app(feed)) as client:
+ yield client
+
+
+def test_opens_with_hello_naming_the_kinds_it_sends(
+ client: TestClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ _signed_in(monkeypatch)
+ with client.websocket_connect("/changes/ws") as ws:
+ hello = ws.receive_json()
+ assert hello["event"] == "hello"
+ assert set(hello["data"]["kinds"]) == {MANAGED_AGENT, MACHINE}
+ assert hello["data"]["ping_interval_s"] == changes.PING_INTERVAL_SECONDS
+
+
+def test_a_change_to_the_user_arrives_as_a_notice(
+ client: TestClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ _signed_in(monkeypatch)
+ with client.websocket_connect("/changes/ws") as ws:
+ ws.receive_json()
+ client.post(f"/publish/ada/{MANAGED_AGENT}/a1")
+ frame = ws.receive_json()
+ assert frame == {
+ "event": "changed",
+ "data": {"changes": [{"kind": MANAGED_AGENT, "id": "a1"}]},
+ }
+
+
+def test_another_users_change_is_not_sent(
+ client: TestClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(changes, "PING_INTERVAL_SECONDS", 0.05)
+ _signed_in(monkeypatch)
+ with client.websocket_connect("/changes/ws") as ws:
+ ws.receive_json()
+ client.post(f"/publish/bob/{MANAGED_AGENT}/a1")
+ # The next frame is a ping, not bob's notice.
+ assert ws.receive_json()["event"] == "ping"
+
+
+def test_pings_and_closes_a_client_that_never_answers(
+ client: TestClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(changes, "PING_INTERVAL_SECONDS", 0.02)
+ _signed_in(monkeypatch)
+ with client.websocket_connect("/changes/ws") as ws:
+ ws.receive_json()
+ events = []
+ with pytest.raises(WebSocketDisconnect) as closed:
+ while True:
+ events.append(ws.receive_json()["event"])
+ assert "ping" in events
+ assert closed.value.code == 1001
+
+
+def test_a_client_that_answers_pings_stays(
+ client: TestClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(changes, "PING_INTERVAL_SECONDS", 0.02)
+ _signed_in(monkeypatch)
+ with client.websocket_connect("/changes/ws") as ws:
+ ws.receive_json()
+ for _ in range(8):
+ assert ws.receive_json()["event"] == "ping"
+ ws.send_json({"type": "pong"})
+
+
+def test_a_caller_without_a_session_gets_a_401_and_no_socket(
+ client: TestClient, feed: LocalUserChanges, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """Refused in the handshake, as any gateway route refuses: nothing is
+ accepted or held for a caller who is not signed in."""
+
+ async def _refuse(*_: Any) -> SocketCaller:
+ raise HTTPException(status_code=401, detail="Not authenticated")
+
+ monkeypatch.setattr(changes, "authenticate_socket", _refuse)
+ with pytest.raises(WebSocketDenialResponse) as denied:
+ with client.websocket_connect("/changes/ws"):
+ pass
+ assert denied.value.status_code == 401
+ assert denied.value.json() == {"detail": "Not authenticated"}
+ assert feed.subscriber_count("t1", "ada") == 0
+
+
+def test_closes_with_4401_when_the_session_cookie_expires(
+ client: TestClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ _signed_in(monkeypatch, expires_in=0.05)
+ with client.websocket_connect("/changes/ws") as ws:
+ ws.receive_json()
+ with pytest.raises(WebSocketDisconnect) as closed:
+ ws.receive_json()
+ assert closed.value.code == 4401
+
+
+def test_closing_the_socket_drops_its_subscription(
+ client: TestClient, feed: LocalUserChanges, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ _signed_in(monkeypatch)
+ with client.websocket_connect("/changes/ws") as ws:
+ ws.receive_json()
+ assert feed.subscriber_count("t1", "ada") == 1
+ for _ in range(100):
+ if feed.subscriber_count("t1", "ada") == 0:
+ break
+ time.sleep(0.01)
+ assert feed.subscriber_count("t1", "ada") == 0
+
+
+def test_a_user_past_the_socket_cap_is_refused_with_429(
+ client: TestClient, feed: LocalUserChanges, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """A client opening sockets in a loop stops at the cap; the user's open
+ sockets are untouched, and another user is not affected."""
+ _signed_in(monkeypatch)
+ for _ in range(changes.MAX_SOCKETS_PER_USER):
+ feed.subscribe("t1", "ada")
+ with client.websocket_connect("/changes/ws") as ws:
+ refused = ws.receive_json()
+ with pytest.raises(WebSocketDisconnect) as closed:
+ ws.receive_json()
+ assert refused["event"] == "refused"
+ assert refused["data"]["status"] == 429
+ assert closed.value.code == 4429
+ assert feed.subscriber_count("t1", "ada") == changes.MAX_SOCKETS_PER_USER
+
+ _signed_in(monkeypatch, user_id="grace")
+ with client.websocket_connect("/changes/ws") as ws:
+ assert ws.receive_json()["event"] == "hello"
diff --git a/core/tests/switch_core/management/harness.py b/core/tests/switch_core/management/harness.py
index c4ec871a6..595b86098 100644
--- a/core/tests/switch_core/management/harness.py
+++ b/core/tests/switch_core/management/harness.py
@@ -71,6 +71,7 @@ from switch_core.management.errors import ManagementError
from switch_core.management.wiring import Management, build_management
from switch_core.process_lease import ProcessLease
from switch_core.signals import SignalBus
+from switch_core.user_changes import LocalUserChanges, SignalUserChanges
from tests.db_buffer import db_buffer_on
from tests.switch_core.connections.fake_vendor import FakeVendor
from tests.switch_core.connections.github_seed import github_broker
@@ -184,6 +185,7 @@ class Harness:
controller_auth_cache: ControllerAuthCache
clock: Clock
session_factory: async_sessionmaker[AsyncSession]
+ user_changes: LocalUserChanges
broker: ServiceBroker
vendor: FakeVendor
@@ -229,6 +231,7 @@ def build_harness(
protocol = protocol_service(session_factory, cache)
process_id = str(uuid.uuid4())
signal_bus = SignalBus(session_factory, signal_engine, process_id=process_id)
+ user_changes = LocalUserChanges()
management = build_management(
token_secret=TOKEN_SECRET,
status_interval_seconds=STATUS_INTERVAL,
@@ -248,6 +251,7 @@ def build_harness(
processes=SwitchCoreProcessStore(),
key_ids=[key.id for key in TEST_KEYRING.keys],
),
+ user_changes=SignalUserChanges(local=user_changes, bus=signal_bus),
)
async def _session() -> AsyncIterator[AsyncSession]:
@@ -311,6 +315,7 @@ def build_harness(
controller_auth_cache=controller_auth_cache,
clock=clock,
session_factory=session_factory,
+ user_changes=user_changes,
broker=broker,
vendor=vendor,
)
diff --git a/core/tests/switch_core/management/test_controller_signals.py b/core/tests/switch_core/management/test_controller_signals.py
index 6dbe84c5a..6c6bca27a 100644
--- a/core/tests/switch_core/management/test_controller_signals.py
+++ b/core/tests/switch_core/management/test_controller_signals.py
@@ -30,6 +30,7 @@ from switch_core.management.notifier import (
)
from switch_core.management.signals import ControllerSignals
from switch_core.signals import SignalBus
+from switch_core.user_changes import LocalUserChanges, SignalUserChanges
from tests.switch_core.management.harness import (
Harness,
add_member,
@@ -73,6 +74,7 @@ def _process(
rooms=RoomStore(),
processes=SwitchCoreProcessStore(),
clock=lambda: datetime.now(UTC),
+ user_changes=SignalUserChanges(local=LocalUserChanges(), bus=bus),
)
return signals, registry, bus
diff --git a/core/tests/switch_core/management/test_controllers_across_processes.py b/core/tests/switch_core/management/test_controllers_across_processes.py
index 6a940c318..64fef5155 100644
--- a/core/tests/switch_core/management/test_controllers_across_processes.py
+++ b/core/tests/switch_core/management/test_controllers_across_processes.py
@@ -26,6 +26,7 @@ from switch_core.bridges.agent.protocol.types import AgentEvent, MessagePayload
from switch_core.db.models import TENANT_ZERO_ID, AgentController, SwitchCoreProcess
from switch_core.db.stores.agent_connection_store import AgentConnectionStore
from switch_core.process_lease import LEASE_TTL
+from switch_core.user_changes import MACHINE, UserChange
from tests.db_buffer import say
from tests.switch_core.management.harness import (
Harness,
@@ -699,3 +700,62 @@ class TestTheRowNamesTheConnectionOpenedLast:
)
== there.management.lease.process_id
)
+
+
+class TestOwnerNotices:
+ """A machine's owner hears that it changed, on whichever process holds
+ their Console's socket."""
+
+ async def _heard(self, subscription: Any, change: UserChange) -> None:
+ heard: list[UserChange] = []
+
+ def check() -> bool:
+ heard.extend(subscription.drain())
+ return change in heard
+
+ await _eventually(check)
+
+ async def test_enrolling_through_one_process_tells_the_owner_on_the_other(
+ self, processes: tuple[Harness, Harness]
+ ) -> None:
+ here, there = processes
+ owner = await add_member(here.session_factory, "ada")
+ subscription = there.user_changes.subscribe(TENANT_ZERO_ID, owner.id)
+ async with here.client() as client:
+ controller = await enroll_console(here, client, owner)
+ await self._heard(subscription, UserChange(MACHINE, controller.controller_id))
+
+ async def test_a_socket_attaching_and_going_tell_the_owner_on_the_other(
+ self, processes: tuple[Harness, Harness]
+ ) -> None:
+ here, there = processes
+ owner = await add_member(here.session_factory, "ada")
+ subscription = there.user_changes.subscribe(TENANT_ZERO_ID, owner.id)
+ async with here.client() as client:
+ controller = await enroll_console(here, client, owner)
+ machine = UserChange(MACHINE, controller.controller_id)
+ await self._heard(subscription, machine)
+
+ await connect(client, controller)
+ await self._heard(subscription, machine)
+
+ held = here.protocol.connections.controllers
+ conn = held.current_connection(controller.controller_id)
+ assert conn is not None
+ held.detach_stream(conn, conn.stream_token)
+ await here.management.ledger.flush_all()
+ await self._heard(subscription, machine)
+
+ async def test_nobody_else_hears_of_it(
+ self, processes: tuple[Harness, Harness]
+ ) -> None:
+ here, there = processes
+ owner = await add_member(here.session_factory, "ada")
+ other = await add_member(here.session_factory, "bob")
+ theirs = there.user_changes.subscribe(TENANT_ZERO_ID, owner.id)
+ bystander = there.user_changes.subscribe(TENANT_ZERO_ID, other.id)
+ async with here.client() as client:
+ controller = await enroll_console(here, client, owner)
+ await connect(client, controller)
+ await self._heard(theirs, UserChange(MACHINE, controller.controller_id))
+ assert bystander.drain() == []
diff --git a/core/tests/switch_core/management/test_middleware.py b/core/tests/switch_core/management/test_middleware.py
index 34413291b..14b5951bb 100644
--- a/core/tests/switch_core/management/test_middleware.py
+++ b/core/tests/switch_core/management/test_middleware.py
@@ -32,6 +32,7 @@ from switch_core.management.auth import ManagementAuthenticator
from switch_core.management.wiring import create_management
from switch_core.tenant_context import current_tenant_id
from switch_core.trust.client import NullTrustClient
+from switch_core.user_changes import LocalUserChanges
from tests.db_buffer import QuietRooms
from tests.switch_core.management.harness import (
TOKEN_SECRET,
@@ -286,6 +287,7 @@ class TestTheFlagOff:
AgentConnectionRegistry().controllers,
object(), # type: ignore[arg-type]
object(), # type: ignore[arg-type]
+ LocalUserChanges(),
)
is None
)
diff --git a/core/tests/switch_core/management/test_user_changes.py b/core/tests/switch_core/management/test_user_changes.py
new file mode 100644
index 000000000..2b5069d11
--- /dev/null
+++ b/core/tests/switch_core/management/test_user_changes.py
@@ -0,0 +1,300 @@
+"""Change notices reach the owner's Console when their managed agents or
+machines change, and only then."""
+
+from __future__ import annotations
+
+import asyncio
+import uuid
+from collections.abc import AsyncIterator
+
+import pytest
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+from sqlalchemy.pool import NullPool
+
+from switch_core.db.models import TENANT_ZERO_ID
+from switch_core.management.service import _status_signature
+from switch_core.signals import SignalBus
+from switch_core.user_changes import (
+ MACHINE,
+ MANAGED_AGENT,
+ USER_CHANGED_SIGNAL,
+ LocalUserChanges,
+ SignalUserChanges,
+ UserChange,
+)
+from tests.switch_core.management.harness import (
+ Harness,
+ add_member,
+ build_harness,
+ cookies_for,
+ create_managed_agent,
+ enroll_console,
+ provider,
+ report_status,
+ report_status_only,
+ status_report,
+)
+
+
+def harness(session_factory: async_sessionmaker[AsyncSession]) -> Harness:
+ return build_harness(session_factory)
+
+
+class TestLocalUserChanges:
+ def test_a_notice_reaches_only_that_users_subscriptions_in_that_tenant(
+ self,
+ ) -> None:
+ feed = LocalUserChanges()
+ mine = feed.subscribe("t1", "ada")
+ other_user = feed.subscribe("t1", "bob")
+ other_tenant = feed.subscribe("t2", "ada")
+ feed.publish("t1", "ada", MANAGED_AGENT, "a1")
+ assert mine.drain() == [UserChange(MANAGED_AGENT, "a1")]
+ assert other_user.drain() == []
+ assert other_tenant.drain() == []
+
+ def test_the_same_change_twice_is_sent_once(self) -> None:
+ feed = LocalUserChanges()
+ subscription = feed.subscribe("t1", "ada")
+ feed.publish("t1", "ada", MACHINE, "m1")
+ feed.publish("t1", "ada", MACHINE, "m1")
+ assert subscription.drain() == [UserChange(MACHINE, "m1")]
+ assert not subscription.wake.is_set()
+
+ def test_an_unread_subscription_collapses_to_one_notice_per_kind(self) -> None:
+ feed = LocalUserChanges()
+ subscription = feed.subscribe("t1", "ada")
+ for i in range(100):
+ feed.publish("t1", "ada", MANAGED_AGENT, f"a{i}")
+ feed.publish("t1", "ada", MACHINE, "m1")
+ drained = subscription.drain()
+ assert UserChange(MANAGED_AGENT, None) in drained
+ assert len(drained) <= 2
+
+ def test_a_closed_subscription_hears_nothing_and_is_forgotten(self) -> None:
+ feed = LocalUserChanges()
+ subscription = feed.subscribe("t1", "ada")
+ subscription.close()
+ feed.publish("t1", "ada", MACHINE, "m1")
+ assert subscription.drain() == []
+ assert feed.subscriber_count("t1", "ada") == 0
+
+ def test_everything_tells_every_subscription_every_kind(self) -> None:
+ feed = LocalUserChanges()
+ ada = feed.subscribe("t1", "ada")
+ bob = feed.subscribe("t2", "bob")
+ feed.publish("t1", "ada", MACHINE, "m1")
+ feed.publish_everything()
+ for subscription in (ada, bob):
+ drained = subscription.drain()
+ assert UserChange(MANAGED_AGENT, None) in drained
+ assert UserChange(MACHINE, None) in drained
+
+
+DELIVERY_TIMEOUT = 10.0
+
+
+class _Process:
+ """One switch-core process's share: its bus and the subscriptions it holds."""
+
+ def __init__(
+ self, postgres_url: str, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ self.bus = SignalBus(
+ session_factory,
+ lambda: create_async_engine(postgres_url, poolclass=NullPool),
+ process_id=str(uuid.uuid4()),
+ )
+ self.local = LocalUserChanges()
+ self.publisher = SignalUserChanges(local=self.local, bus=self.bus)
+
+
+async def two_processes(
+ postgres_url: str, session_factory: async_sessionmaker[AsyncSession]
+) -> AsyncIterator[tuple[_Process, _Process]]:
+ first = _Process(postgres_url, session_factory)
+ second = _Process(postgres_url, session_factory)
+ for process in (first, second):
+ await process.bus.start()
+ for process in (first, second):
+ await asyncio.wait_for(process.bus.connected.wait(), DELIVERY_TIMEOUT)
+ yield first, second
+ await first.bus.stop()
+ await second.bus.stop()
+
+
+async def _woken(subscription: object) -> None:
+ await asyncio.wait_for(subscription.wake.wait(), DELIVERY_TIMEOUT) # type: ignore[attr-defined]
+
+
+class TestAcrossProcesses:
+ async def test_a_notice_published_in_one_process_reaches_a_socket_in_another(
+ self, two_processes: tuple[_Process, _Process]
+ ) -> None:
+ sender, receiver = two_processes
+ there = receiver.local.subscribe("t1", "ada")
+ sender.publisher.publish("t1", "ada", MANAGED_AGENT, "a1")
+ await _woken(there)
+ assert there.drain() == [UserChange(MANAGED_AGENT, "a1")]
+
+ async def test_the_sending_process_tells_its_own_sockets_at_once(
+ self, two_processes: tuple[_Process, _Process]
+ ) -> None:
+ sender, receiver = two_processes
+ here = sender.local.subscribe("t1", "ada")
+ there = receiver.local.subscribe("t1", "ada")
+ sender.publisher.publish("t1", "ada", MACHINE, "m1")
+ # Before the bus has carried anything: told locally, not by the bus.
+ assert here.drain() == [UserChange(MACHINE, "m1")]
+ await _woken(there)
+ assert there.drain() == [UserChange(MACHINE, "m1")]
+ # The bus does not bring it back to the sender a second time.
+ await asyncio.sleep(0.2)
+ assert here.drain() == []
+
+ async def test_only_that_users_sockets_in_that_tenant_hear_it(
+ self, two_processes: tuple[_Process, _Process]
+ ) -> None:
+ sender, receiver = two_processes
+ mine = receiver.local.subscribe("t1", "ada")
+ other_user = receiver.local.subscribe("t1", "bob")
+ other_tenant = receiver.local.subscribe("t2", "ada")
+ sender.publisher.publish("t1", "ada", MACHINE, "m1")
+ await _woken(mine)
+ assert other_user.drain() == []
+ assert other_tenant.drain() == []
+
+ async def test_a_kind_this_process_does_not_know_is_ignored(
+ self, two_processes: tuple[_Process, _Process]
+ ) -> None:
+ sender, receiver = two_processes
+ there = receiver.local.subscribe("t1", "ada")
+ sender.bus.publish(
+ USER_CHANGED_SIGNAL, tenant_id="t1", user_id="ada", change="hat", id="h1"
+ )
+ sender.publisher.publish("t1", "ada", MACHINE, "m1")
+ await _woken(there)
+ assert there.drain() == [UserChange(MACHINE, "m1")]
+
+ async def test_a_listener_reconnecting_tells_its_sockets_to_read_everything(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """Notices sent while the listener was down are gone, so each socket
+ here is told that anything may have changed."""
+ bus = SignalBus(session_factory, lambda: None, process_id="p") # type: ignore[arg-type,return-value]
+ local = LocalUserChanges()
+ SignalUserChanges(local=local, bus=bus)
+ subscription = local.subscribe("t1", "ada")
+ await bus._resync_all()
+ assert set(subscription.drain()) == {
+ UserChange(MANAGED_AGENT, None),
+ UserChange(MACHINE, None),
+ }
+
+
+class TestStatusSignature:
+ def test_free_space_and_report_times_do_not_count_as_a_change(self) -> None:
+ first = status_report(1)
+ second = status_report(2)
+ second["observed_at"] = "2030-01-01T00:00:00Z"
+ second["machine"]["disk_free_bytes"] += 1
+ second["machine"]["mem_free_bytes"] += 1
+ for entry in second["providers"]:
+ entry["checked_at"] = "2030-01-01T00:00:00Z"
+ assert _status_signature(first) == _status_signature(second)
+
+ def test_a_provider_signing_out_counts(self) -> None:
+ first = status_report(1, providers=[provider("claude")])
+ second = status_report(2, providers=[provider("claude", auth="expired")])
+ assert _status_signature(first) != _status_signature(second)
+
+
+class TestManagementNotices:
+ async def test_creating_a_managed_agent_tells_its_owner(
+ self, harness: Harness
+ ) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ subscription = harness.user_changes.subscribe(TENANT_ZERO_ID, owner.id)
+ async with harness.client() as client:
+ response = await create_managed_agent(
+ client, owner, name="scout", controller_id=None
+ )
+ assert response.status_code == 201, response.text
+ agent_id = response.json()["agent_id"]
+ assert UserChange(MANAGED_AGENT, agent_id) in subscription.drain()
+
+ async def test_moving_and_deleting_a_managed_agent_tell_its_owner(
+ self, harness: Harness
+ ) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ await report_status(client, controller, 1)
+ created = await create_managed_agent(
+ client, owner, name="scout", controller_id=None
+ )
+ agent_id = created.json()["agent_id"]
+ subscription = harness.user_changes.subscribe(TENANT_ZERO_ID, owner.id)
+ moved = await client.patch(
+ f"/gateway/management/agents/{agent_id}",
+ json={"controller_id": controller.controller_id},
+ cookies=cookies_for(owner),
+ )
+ assert moved.status_code == 200, moved.text
+ assert UserChange(MANAGED_AGENT, agent_id) in subscription.drain()
+ deleted = await client.delete(
+ f"/gateway/management/agents/{agent_id}", cookies=cookies_for(owner)
+ )
+ assert deleted.status_code in (200, 204), deleted.text
+ assert UserChange(MANAGED_AGENT, agent_id) in subscription.drain()
+
+ async def test_enrolling_renaming_and_revoking_a_machine_tell_its_owner(
+ self, harness: Harness
+ ) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ subscription = harness.user_changes.subscribe(TENANT_ZERO_ID, owner.id)
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ machine = UserChange(MACHINE, controller.controller_id)
+ assert machine in subscription.drain()
+ renamed = await client.patch(
+ f"/gateway/management/controllers/{controller.controller_id}",
+ json={"name": "studio"},
+ cookies=cookies_for(owner),
+ )
+ assert renamed.status_code == 200, renamed.text
+ assert machine in subscription.drain()
+ revoked = await client.delete(
+ f"/gateway/management/controllers/{controller.controller_id}",
+ cookies=cookies_for(owner),
+ )
+ assert revoked.status_code in (200, 204), revoked.text
+ assert machine in subscription.drain()
+
+ async def test_a_status_report_tells_the_owner_only_when_something_changed(
+ self, harness: Harness
+ ) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ await report_status(client, controller, 1, providers=[provider("claude")])
+ subscription = harness.user_changes.subscribe(TENANT_ZERO_ID, owner.id)
+ await report_status_only(
+ client, controller, 2, providers=[provider("claude")]
+ )
+ assert subscription.drain() == []
+ await report_status_only(
+ client, controller, 3, providers=[provider("claude", auth="expired")]
+ )
+ assert subscription.drain() == [UserChange(MACHINE, controller.controller_id)]
+
+ async def test_nobody_else_hears_of_it(self, harness: Harness) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ other = await add_member(harness.session_factory, "bob")
+ subscription = harness.user_changes.subscribe(TENANT_ZERO_ID, other.id)
+ async with harness.client() as client:
+ await create_managed_agent(client, owner, name="scout", controller_id=None)
+ await enroll_console(harness, client, owner)
+ assert subscription.drain() == []
diff --git a/deploy/remote/helm/switch/templates/gateway/configmap-nginx.yaml b/deploy/remote/helm/switch/templates/gateway/configmap-nginx.yaml
index 412d2dc27..884fb27c6 100644
--- a/deploy/remote/helm/switch/templates/gateway/configmap-nginx.yaml
+++ b/deploy/remote/helm/switch/templates/gateway/configmap-nginx.yaml
@@ -20,6 +20,19 @@ data:
proxy_set_header X-Real-IP $remote_addr;
}
+ # The signed-in user's change socket (core gateway/changes.py): a WebSocket,
+ # so the upgrade headers are passed on, and an idle socket is kept for as
+ # long as its pings keep it alive.
+ location = /gateway/changes/ws {
+ proxy_pass http://{{ include "switch.switchCoreHost" . }}:{{ .Values.switchCore.service.port }};
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection "upgrade";
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_read_timeout 1h;
+ }
+
location /gateway {
proxy_pass http://{{ include "switch.switchCoreHost" . }}:{{ .Values.switchCore.service.port }};
proxy_set_header Host $host;
diff --git a/deploy/shared_resources/gateway-nginx.conf b/deploy/shared_resources/gateway-nginx.conf
index da08605cc..febc86225 100644
--- a/deploy/shared_resources/gateway-nginx.conf
+++ b/deploy/shared_resources/gateway-nginx.conf
@@ -11,6 +11,19 @@ server {
proxy_set_header X-Real-IP $remote_addr;
}
+ # The signed-in user's change socket (core gateway/changes.py): a WebSocket,
+ # so the upgrade headers are passed on, and an idle socket is kept for as
+ # long as its pings keep it alive.
+ location = /gateway/changes/ws {
+ proxy_pass http://switch:8000;
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection "upgrade";
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_read_timeout 1h;
+ }
+
location /gateway {
proxy_pass http://switch:8000;
proxy_set_header Host $host;
diff --git a/scripts/e2e_surface.py b/scripts/e2e_surface.py
index 927f8cb64..2ab645604 100644
--- a/scripts/e2e_surface.py
+++ b/scripts/e2e_surface.py
@@ -41,7 +41,7 @@ from unittest.mock import MagicMock
from fastapi import FastAPI
from fastapi.dependencies.models import Dependant
-from fastapi.routing import APIRoute
+from fastapi.routing import APIRoute, APIWebSocketRoute
from pydantic_settings import BaseSettings, PydanticBaseSettingsSource
from starlette.routing import Route
@@ -56,6 +56,7 @@ from switch_core.gateway.auth import (
get_current_user,
get_current_user_in_transaction,
)
+from switch_core.gateway.changes import get_socket_caller
from switch_core.gateway.hosted_controller import controller_session
from switch_core.management.wiring import mount_gateway_routes
@@ -72,6 +73,7 @@ SESSION_DEPENDENCIES: frozenset[Callable[..., Any]] = frozenset(
get_authenticated_user_id,
get_authenticated_session,
get_authenticated_caller,
+ get_socket_caller,
}
)
@@ -102,6 +104,7 @@ _COLLABORATORS = (
"budget_store",
"resource_service",
"protocol",
+ "user_changes",
)
@@ -168,8 +171,8 @@ OPERATOR_TOKEN_DEPENDENCIES: frozenset[Callable[..., Any]] = frozenset(
)
-def _auth(route: Route) -> str:
- if not isinstance(route, APIRoute):
+def _auth(route: Route | APIWebSocketRoute) -> str:
+ if not isinstance(route, (APIRoute, APIWebSocketRoute)):
return "none"
calls = set(_dependency_calls(route.dependant))
if calls & SESSION_DEPENDENCIES:
@@ -183,6 +186,14 @@ def _routes_by_key(app: FastAPI) -> dict[str, str]:
"""Each route's `METHOD /gateway/path`, and what it authenticates with."""
routes: dict[str, str] = {}
for route in app.routes:
+ if isinstance(route, APIWebSocketRoute):
+ # `WS`, not a method: the socket is opened by an HTTP upgrade, and
+ # a session socket refuses one without a session with the same 401.
+ key = f"WS /gateway{route.path_format}"
+ if key in routes:
+ raise ValueError(f"Gateway route {key} is mounted twice.")
+ routes[key] = _auth(route)
+ continue
if not isinstance(route, Route):
raise TypeError(
f"Gateway route {route!r} is a {type(route).__name__}, which this "