Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

feat(console): upgrade managed servers behind the pin and hold sessions until done (#544)

Louis Amaudruz8ba5b69c36d5
diff --git a/.env.example b/.env.example
index df892245d..7b287ce79 100644
--- a/.env.example
+++ b/.env.example
@@ -178,14 +178,6 @@ MATTERMOST_USER_PASSWORD=
# through the gateway (all credentials live in the bridge's stored
# connection_config), never in global config.
-# ── Session activity and request cards (local dev) ───────────────────────────
-# No agent host speaks the session interaction contract yet, so no session's
-# activity can reach a channel. With this set, `!session-demo` in a bridged
-# Slack channel posts a recorded turn there — what the agent did and said, and
-# then the permission it stopped to ask for, as a real card — so the answer
-# path can be exercised against a real workspace. Needs the repository
-# checkout, for the fixtures.
-# SESSION_DEMO_ENABLED=true
# ── OIDC (optional — MCP OAuth token validation; bring your own IdP) ─────────
# Enable OAuth/OIDC token auth on the MCP server by pointing at any standards
diff --git a/CHANGELOG.md b/CHANGELOG.md
index cfb3b0a5b..3e4fb5d33 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1317,6 +1317,19 @@ version of their own to them without also giving them a release of their own.
### [Unreleased]
+#### Changed
+- **Managed servers behind the app's switch-core pin are upgraded
+ automatically**, and sessions wait for it. The local server is upgraded at
+ startup and a remote managed server when its host is reconciled (at startup
+ and whenever the host becomes reachable); a stopped one is upgraded at its
+ next start rather than started at the old version. Every database is backed up
+ with `pg_dumpall` into a timestamped `backups/` directory next to the stack
+ first, and an `upgrade.json` journal resumes an interrupted upgrade or refuses
+ it loudly. Sessions and room watchers for agents on the server wait while it
+ updates and are refused with the error if it fails. The server page shows the
+ update's progress, or its error and a Retry button, in place of the optional
+ "Restart to update" banner.
+
### [0.35.0] - 2026-09-23
#### Added
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/compose.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/compose.ts
index 380e9460c..211073f8e 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/compose.ts
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/compose.ts
@@ -31,7 +31,11 @@ function baseArgs(host: ServerHost, globalFlags: string[], extraFiles: string[]
];
}
-async function runDocker(host: ServerHost, args: string[], timeout: number): Promise<string> {
+export async function runDocker(
+ host: ServerHost,
+ args: string[],
+ timeout: number
+): Promise<string> {
const full = [host.dockerBin, ...args];
try {
const { stdout } = await host.ctx.exec(host.dockerBin, args, {
@@ -136,6 +140,30 @@ export async function composeDown(host: ServerHost, removeVolumes: boolean): Pro
);
}
+/**
+ * Bring up one service of the stack as it is currently configured on the host
+ * — the compose file and `.env` already there, not this build's — without its
+ * dependents, and wait for it to report healthy.
+ */
+export async function composeUpService(host: ServerHost, service: string): Promise<void> {
+ log.info(`local-switch-server: docker compose up ${service} (${host.label})`);
+ await runDocker(
+ host,
+ [...baseArgs(host, []), 'up', '-d', '--no-deps', '--wait', service],
+ 5 * 60 * 1000
+ );
+}
+
+/** Id of the container running `service`. Throws when there is none. */
+export async function serviceContainerId(host: ServerHost, service: string): Promise<string> {
+ const stdout = await runDocker(host, [...baseArgs(host, []), 'ps', '-q', service], 60_000);
+ const id = stdout.trim();
+ if (!/^[a-f0-9]+$/.test(id)) {
+ throw new Error(`Could not identify the ${service} container on ${host.label}.`);
+ }
+ return id;
+}
+
/** Service names currently in the `running` state for the managed project. */
export async function runningServices(host: ServerHost): Promise<string[]> {
try {
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/env-file.test.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/env-file.test.ts
index 531a7598f..d527965aa 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/env-file.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/env-file.test.ts
@@ -29,7 +29,6 @@ describe('buildEnvFile', () => {
namespace: 'sandbox-quantum',
ports: { gateway: 51000, api: 51001, mattermost: 51002, postgres: 51003 },
secrets,
- sessionDemo: false,
telemetryEnabled: false,
});
const vars = Object.fromEntries(
@@ -95,11 +94,6 @@ describe('buildEnvFile', () => {
// An entry here must say why the stack is correct without it — leaving a
// var unset is a decision, not a default.
const intentionallyUnset = new Set<string>([
- // The recorded session demo, which posts a card into a real channel with
- // no session behind it. Off is the correct state for a released Console:
- // it is written only for a stack built from a local checkout, and this
- // `env` is the released one. Temporary — goes when the demo does.
- 'SESSION_DEMO_ENABLED',
// The four below configure switch-core as a distributed messaging app —
// one app we own, installed by a customer into their own workspace, with
// the platform posting events to URLs declared once in the app manifest.
@@ -124,21 +118,8 @@ describe('buildEnvFile', () => {
expect(missing, 'compose interpolates these but the .env does not set them').toEqual([]);
});
- it('turns the session demo on for a checkout build and off otherwise', () => {
- // The .env is regenerated on every start, so a developer has nowhere to set
- // this by hand — and a released Console runs pinned images that have never
- // heard of it. Temporary, and it goes when the demo does.
+ it('never passes the removed session demo switch, which older switch-core images reject when empty', () => {
expect(env).not.toContain('SESSION_DEMO_ENABLED');
- const fromCheckout = buildEnvFile({
- version: 'checkout',
- registry: 'ghcr.io',
- namespace: 'sandbox-quantum',
- ports: { gateway: 51000, api: 51001, mattermost: 51002, postgres: 51003 },
- secrets,
- sessionDemo: true,
- telemetryEnabled: false,
- });
- expect(fromCheckout).toContain('SESSION_DEMO_ENABLED=true');
});
it('carries the telemetry answer in both directions, never by omission', () => {
@@ -153,7 +134,6 @@ describe('buildEnvFile', () => {
namespace: 'sandbox-quantum',
ports: { gateway: 51000, api: 51001, mattermost: 51002, postgres: 51003 },
secrets,
- sessionDemo: false,
telemetryEnabled: true,
});
expect(sharing).toContain('TELEMETRY_ENABLED=true');
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/env-file.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/env-file.ts
index d9565e687..a554c0415 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/env-file.ts
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/env-file.ts
@@ -17,14 +17,6 @@ export type LocalServerEnvParams = {
/** Free host ports chosen for this machine (published on 127.0.0.1). */
ports: LocalServerPorts;
secrets: LocalServerSecrets;
- /**
- * Turn on the server's recorded session demo (`!session-demo` in a bridged
- * channel). True only for a stack built from a local checkout: this file is
- * regenerated on every start, so there is nowhere for a developer to set it
- * by hand, and a released Console runs pinned images that have never heard
- * of it. **Temporary** — goes when the demo does.
- */
- sessionDemo: boolean;
/**
* The user's "Share usage data" answer, applied to the server as well as to
* the app. One consent decision covers both, so this is the toggle's value
@@ -44,7 +36,7 @@ export const TELEMETRY_ENABLED_KEY = 'TELEMETRY_ENABLED';
* app's pin. Kept as a pure function so the exact output is unit-testable.
*/
export function buildEnvFile(params: LocalServerEnvParams): string {
- const { version, registry, namespace, ports, secrets, sessionDemo, telemetryEnabled } = params;
+ const { version, registry, namespace, ports, secrets, telemetryEnabled } = params;
return [
'# Generated by Switch Console local-server mode — do not edit by hand.',
'# Secrets originate in the app encrypted store; this file is derived.',
@@ -86,7 +78,6 @@ export function buildEnvFile(params: LocalServerEnvParams): string {
// served on the agent-bridge app, and this must match the `server` Switch Console
// reports in the deeplink it is rewriting.
`GATEWAY_PUBLIC_URL=${apiUrlFor(ports)}`,
- ...(sessionDemo ? ['SESSION_DEMO_ENABLED=true'] : []),
// Written on every start, in both directions. Omitting it when the answer
// is no would leave a stack that was started under a yes still reporting,
// because the compose file passes this key through only when the `.env`
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/local-host.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/local-host.ts
index a7a144c39..6bb96a871 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/local-host.ts
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/local-host.ts
@@ -1,5 +1,5 @@
import { execFile, spawn } from 'node:child_process';
-import { chmod, mkdir, readFile, writeFile } from 'node:fs/promises';
+import { chmod, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
import { dirname, join } from 'node:path';
import { promisify } from 'node:util';
import { LocalExecutionContext } from '@main/core/execution-context/local-execution-context';
@@ -68,8 +68,7 @@ export class LocalServerHost implements ServerHost {
await writeFile(path, content, mode !== undefined ? { encoding: 'utf8', mode } : 'utf8');
if (mode === undefined) return;
// writeFile only applies `mode` when creating; enforce it on rewrite too.
- await chmod(path, mode);
- if (process.platform === 'win32') await restrictWindowsFileToOwner(path);
+ await this.restrictMode(relPath, mode);
}
async readFile(relPath: string): Promise<string | null> {
@@ -81,6 +80,16 @@ export class LocalServerHost implements ServerHost {
}
}
+ async removeFile(relPath: string): Promise<void> {
+ await rm(join(this.workingDir, relPath), { force: true });
+ }
+
+ async restrictMode(relPath: string, mode: number): Promise<void> {
+ const path = join(this.workingDir, relPath);
+ await chmod(path, mode);
+ if (process.platform === 'win32') await restrictWindowsFileToOwner(path);
+ }
+
streamCommand(
command: string,
args: string[],
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/remote-host.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/remote-host.ts
index 622979b2c..75e0da953 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/remote-host.ts
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/remote-host.ts
@@ -67,10 +67,16 @@ export class RemoteServerHost implements ServerHost {
async writeFile(relPath: string, content: string, mode?: number): Promise<void> {
await this.fs.write(relPath, content);
- if (mode !== undefined) {
- // ctx is rooted at workingDir, so the relative path resolves there.
- await this.ctx.exec('chmod', [mode.toString(8).padStart(3, '0'), relPath]);
- }
+ if (mode !== undefined) await this.restrictMode(relPath, mode);
+ }
+
+ async removeFile(relPath: string): Promise<void> {
+ await this.ctx.exec('rm', ['-f', relPath]);
+ }
+
+ async restrictMode(relPath: string, mode: number): Promise<void> {
+ // ctx is rooted at workingDir, so the relative path resolves there.
+ await this.ctx.exec('chmod', [mode.toString(8).padStart(3, '0'), relPath]);
}
async readFile(relPath: string): Promise<string | null> {
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/types.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/types.ts
index 991e377d2..2930ec0e4 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/types.ts
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/types.ts
@@ -62,6 +62,13 @@ export interface ServerHost {
* unreadable file is not the same as an absent one. */
readFile(relPath: string): Promise<string | null>;
+ /** Delete `relPath` under {@link workingDir}. An absent file is not an error. */
+ removeFile(relPath: string): Promise<void>;
+
+ /** Set the permission bits of `relPath` under {@link workingDir} (a file or a
+ * directory), restricting it to the owner on Windows where there are none. */
+ restrictMode(relPath: string, mode: number): Promise<void>;
+
/**
* Run a command on the host streaming merged stdout+stderr line-by-line to
* `onLine`, rooted at {@link workingDir}. Used for the slow `compose up`
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.test.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.test.ts
new file mode 100644
index 000000000..4143140ef
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.test.ts
@@ -0,0 +1,257 @@
+import { beforeEach, expect, it, vi } from 'vitest';
+import type * as AppIdentity from '@shared/app-identity';
+import type { StartLocalServerResult } from '@shared/core/managed-switch-server/managed-switch-server';
+import type * as ManagedUpgrade from './managed-upgrade';
+import type { StartStackOptions } from './pipeline';
+
+const m = vi.hoisted(() => ({
+ managed: vi.fn(),
+ running: vi.fn(),
+ version: vi.fn(),
+ journal: vi.fn(),
+ start: vi.fn<(opts: StartStackOptions) => Promise<StartLocalServerResult>>(),
+ stop: vi.fn(),
+ emit: vi.fn(),
+}));
+vi.mock('@shared/app-identity', async (importOriginal) => ({
+ ...(await importOriginal<typeof AppIdentity>()),
+ COMPATIBLE_SWITCH_VERSION: '0.11.0',
+}));
+vi.mock('@main/lib/events', () => ({ events: { emit: m.emit } }));
+vi.mock('@main/lib/logger', () => ({ log: { warn: vi.fn(), error: vi.fn(), info: vi.fn() } }));
+vi.mock('@main/core/telemetry/managed-server', () => ({
+ reportManagedServerOutcome: vi.fn(),
+ reportManagedServerStart: vi.fn(),
+ reportManagedServerStartThrew: vi.fn(),
+}));
+vi.mock('@main/core/switch-servers/delete-server-agents', () => ({
+ deleteAgentsForServer: vi.fn(),
+}));
+vi.mock('@main/core/switch-servers/servers-store', () => ({ getManagedServer: m.managed }));
+vi.mock('./checkout-build', () => ({
+ findCoreCheckout: () => null,
+ isCheckoutBuildEnabled: vi.fn(),
+ setCheckoutBuildEnabled: vi.fn(),
+}));
+vi.mock('./compose', () => ({ isStackRunning: m.running }));
+vi.mock('./deployed-version', () => ({ readVersionStatus: m.version }));
+vi.mock('./telemetry-consent', () => ({ readDeployedTelemetry: vi.fn() }));
+vi.mock('./managed-upgrade', async (importOriginal) => ({
+ ...(await importOriginal<typeof ManagedUpgrade>()),
+ readUpgradeJournal: m.journal,
+}));
+vi.mock('./pipeline', () => ({ startStack: m.start, stopStack: m.stop, resetStack: vi.fn() }));
+vi.mock('./host/local-host', () => ({
+ LocalServerHost: class {
+ dispose() {}
+ },
+}));
+
+const { LocalServerService } = await import('./local-server-service');
+
+/** Boot, and wait out whatever the boot check started. */
+async function boot(service: InstanceType<typeof LocalServerService>): Promise<void> {
+ await service.initialize();
+ await service.ensureReady('Local').catch(() => {});
+}
+
+const behind = {
+ deployedVersion: '0.10.0',
+ drift: { deployed: '0.10.0', expected: '0.11.0', direction: 'upgrade' },
+};
+const started: StartLocalServerResult = {
+ kind: 'started',
+ serverId: 'local',
+ telemetryEnabled: false,
+};
+
+/** A start that runs the upgrade the way the pipeline does: announce, then work. */
+function upgradingStart(result: StartLocalServerResult = started) {
+ return async (opts: StartStackOptions) => {
+ opts.onUpgrade({ from: '0.10.0', to: '0.11.0' });
+ return result;
+ };
+}
+
+beforeEach(() => {
+ vi.resetAllMocks();
+ m.managed.mockResolvedValue({ id: 'local', name: 'Local' });
+ m.running.mockResolvedValue(true);
+ m.version.mockResolvedValue(behind);
+ m.journal.mockResolvedValue(null);
+ m.start.mockImplementation(upgradingStart());
+});
+
+it('upgrades a running server that is behind at boot, without switching to it', async () => {
+ const service = new LocalServerService();
+ await boot(service);
+
+ expect(m.start).toHaveBeenCalledOnce();
+ expect(m.start).toHaveBeenCalledWith(expect.objectContaining({ activate: false }));
+ expect(service.getStatus()).toMatchObject({ phase: 'running', upgrade: null, drift: null });
+ await expect(service.ensureReady('Local')).resolves.toBeUndefined();
+});
+
+it('shows the update while it runs and holds sessions until it has finished', async () => {
+ let finish: (result: StartLocalServerResult) => void = () => {};
+ m.start.mockImplementation(
+ (opts) =>
+ new Promise((resolve) => {
+ opts.onUpgrade({ from: '0.10.0', to: '0.11.0' });
+ finish = resolve;
+ })
+ );
+ const service = new LocalServerService();
+ void service.initialize();
+ let ready = false;
+ const session = service.ensureReady('Local').then(() => {
+ ready = true;
+ });
+
+ await vi.waitFor(() => expect(m.start).toHaveBeenCalledOnce());
+ expect(service.getStatus().upgrade).toEqual({ state: 'updating', from: '0.10.0', to: '0.11.0' });
+ expect(ready).toBe(false);
+
+ finish(started);
+ await session;
+ expect(ready).toBe(true);
+ expect(service.getStatus().upgrade).toBeNull();
+});
+
+it('marks a stopped server that is behind to update at its next start, never earlier', async () => {
+ m.running.mockResolvedValue(false);
+ const service = new LocalServerService();
+ await boot(service);
+
+ expect(m.start).not.toHaveBeenCalled();
+ expect(service.getStatus().upgrade).toEqual({ state: 'pending', from: '0.10.0', to: '0.11.0' });
+ await expect(service.ensureReady('Local')).rejects.toThrow(
+ /Local is stopped on switch-core 0\.10\.0.*Start it/
+ );
+
+ const upgraded = vi.fn();
+ service.onUpgradeFinished(upgraded);
+ expect(await service.start()).toEqual(started);
+ expect(m.start).toHaveBeenCalledWith(expect.objectContaining({ activate: true }));
+ await expect(service.ensureReady('Local')).resolves.toBeUndefined();
+ // Sessions were refused while it was stopped, so they are told it is back.
+ expect(upgraded).toHaveBeenCalledExactlyOnceWith('local');
+});
+
+it('resumes an interrupted upgrade even when no container survived it', async () => {
+ m.running.mockResolvedValue(false);
+ m.version.mockResolvedValue({ deployedVersion: '0.11.0', drift: null });
+ m.journal.mockResolvedValue({ from: '0.10.0', to: '0.11.0', backup: '/b' });
+ const service = new LocalServerService();
+ await boot(service);
+
+ expect(m.start).toHaveBeenCalledOnce();
+ expect(service.getStatus().upgrade).toBeNull();
+});
+
+it('surfaces a failed backup with its error and refuses sessions until a retry succeeds', async () => {
+ m.start.mockImplementationOnce(async (opts) => {
+ opts.onUpgrade({ from: '0.10.0', to: '0.11.0' });
+ throw new Error('pg_dumpall failed: No space left on device');
+ });
+ const service = new LocalServerService();
+ const upgraded = vi.fn();
+ service.onUpgradeFinished(upgraded);
+ await boot(service);
+
+ expect(service.getStatus()).toMatchObject({
+ phase: 'error',
+ upgrade: {
+ state: 'failed',
+ from: '0.10.0',
+ to: '0.11.0',
+ error: 'pg_dumpall failed: No space left on device',
+ },
+ });
+ await expect(service.ensureReady('Local')).rejects.toThrow(
+ /Updating Local from switch-core 0\.10\.0 to 0\.11\.0 failed: pg_dumpall failed/
+ );
+ expect(upgraded).not.toHaveBeenCalled();
+
+ await service.start();
+ await expect(service.ensureReady('Local')).resolves.toBeUndefined();
+ expect(upgraded).toHaveBeenCalledExactlyOnceWith('local');
+});
+
+it('records an upgrade that never turned healthy as failed', async () => {
+ m.start.mockImplementation(
+ upgradingStart({ kind: 'error', message: 'The server did not become healthy in time.' })
+ );
+ const service = new LocalServerService();
+ await boot(service);
+
+ expect(service.getStatus().upgrade).toMatchObject({
+ state: 'failed',
+ error: 'The server did not become healthy in time.',
+ });
+});
+
+it('fails loudly on an upgrade journal it cannot read', async () => {
+ m.journal.mockRejectedValue(new Error('journal cannot be read'));
+ const service = new LocalServerService();
+ await boot(service);
+
+ expect(m.start).not.toHaveBeenCalled();
+ await expect(service.ensureReady('Local')).rejects.toThrow('journal cannot be read');
+});
+
+it('neither upgrades nor holds sessions for a downgrade or an unreadable version', async () => {
+ for (const drift of [
+ { deployed: '0.12.0', expected: '0.11.0', direction: 'downgrade' },
+ { deployed: 'nightly', expected: '0.11.0', direction: 'unknown' },
+ { deployed: null, expected: '0.11.0', direction: 'unreadable', reason: 'daemon down' },
+ ]) {
+ m.version.mockResolvedValue({ deployedVersion: drift.deployed, drift });
+ const service = new LocalServerService();
+ await boot(service);
+ expect(service.getStatus()).toMatchObject({ drift, upgrade: null });
+ await expect(service.ensureReady('Local')).resolves.toBeUndefined();
+ }
+ expect(m.start).not.toHaveBeenCalled();
+});
+
+it('does nothing on an installation without a managed server', async () => {
+ m.managed.mockResolvedValue(null);
+ const service = new LocalServerService();
+ await boot(service);
+
+ expect(m.start).not.toHaveBeenCalled();
+ expect(m.version).not.toHaveBeenCalled();
+ await expect(service.ensureReady('Local')).resolves.toBeUndefined();
+});
+
+it('a Start click during the boot upgrade joins it instead of starting twice', async () => {
+ let finish: (result: StartLocalServerResult) => void = () => {};
+ m.start.mockImplementationOnce(
+ (opts) =>
+ new Promise((resolve) => {
+ opts.onUpgrade({ from: '0.10.0', to: '0.11.0' });
+ finish = resolve;
+ })
+ );
+ const service = new LocalServerService();
+ void service.initialize();
+ await vi.waitFor(() => expect(m.start).toHaveBeenCalledOnce());
+
+ const click = service.start();
+ finish(started);
+ expect(await click).toEqual(started);
+ expect(m.start).toHaveBeenCalledOnce();
+});
+
+it('keeps an owed upgrade pending across a stop', async () => {
+ m.start.mockImplementationOnce(upgradingStart({ kind: 'error', message: 'unhealthy' }));
+ const service = new LocalServerService();
+ await boot(service);
+ await service.stop();
+
+ expect(service.getStatus()).toMatchObject({
+ phase: 'stopped',
+ upgrade: { state: 'pending', from: '0.10.0', to: '0.11.0' },
+ });
+});
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.ts
index 688349450..7c683e056 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.ts
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.ts
@@ -14,6 +14,7 @@ import {
type DockerAvailability,
type LocalServerStatus,
type StartLocalServerResult,
+ managedServerUpgradeBlockedReason,
matrixMigrationFailedMessage,
switchVersionDowngradeMessage,
} from '@shared/core/managed-switch-server/managed-switch-server';
@@ -31,6 +32,12 @@ import { LOCAL_SERVER_NAME } from './constants';
import { readVersionStatus } from './deployed-version';
import { LocalServerHost } from './host/local-host';
import type { ServerHost } from './host/types';
+import {
+ owedUpgrade,
+ readUpgradeJournal,
+ type UpgradeJournal,
+ upgradeState,
+} from './managed-upgrade';
import { resetStack, startStack, stopStack } from './pipeline';
import { readDeployedTelemetry } from './telemetry-consent';
@@ -43,10 +50,15 @@ import { readDeployedTelemetry } from './telemetry-consent';
* Deliberately does NOT stop the containers on app quit — the local stack keeps
* running so its rooms stay live while Switch Console is closed, matching the remote
* sidecar model. `dispose()` only aborts an in-flight health wait.
+ *
+ * A stack behind this build's switch-core pin is upgraded on its own at boot
+ * (or at its next start, if it is stopped), and {@link ensureReady} holds
+ * sessions for its agents until that has finished.
*/
-class LocalServerService {
+export class LocalServerService {
private status: LocalServerStatus = {
phase: 'stopped',
+ upgrade: null,
serverId: null,
version: COMPATIBLE_SWITCH_VERSION,
deployedVersion: null,
@@ -59,6 +71,13 @@ class LocalServerService {
private busy = false;
private startAbort: AbortController | null = null;
+ private initialization: Promise<void> | null = null;
+ /** The start in flight, which an automatic upgrade and a Start click share. */
+ private starting: Promise<StartLocalServerResult> | null = null;
+ private readonly upgradeListeners = new Set<(serverId: string) => void>();
+ /** Whether {@link ensureReady} has turned anything away since the last
+ * upgrade finished, so that it can be told when the server is ready. */
+ private refused = false;
getStatus(): LocalServerStatus {
return this.status;
@@ -116,19 +135,31 @@ class LocalServerService {
/** Reconcile status at boot so a stack that survived the last quit shows as
* running without the user re-starting it, and so an app update that moved
- * the switch-core pin underneath it surfaces as drift instead of leaving the
- * user on a stale core indefinitely (CHOO-1736).
+ * the switch-core pin underneath it is acted on rather than leaving the user
+ * on a stale core (CHOO-1736): a running stack that is behind — or one whose
+ * last upgrade was interrupted — is upgraded now; a stopped one is marked to
+ * be upgraded when it is next started.
*
* The drift probe also runs for a stopped stack: its data volumes still hold
* whatever schema the last version migrated to, which is exactly what makes a
- * downgrade unsafe. */
- async initialize(): Promise<void> {
+ * downgrade unsafe.
+ *
+ * Memoised: {@link ensureReady} awaits the same reconcile, then the upgrade
+ * it started. */
+ initialize(): Promise<void> {
+ this.initialization ??= this.reconcile();
+ return this.initialization;
+ }
+
+ private async reconcile(): Promise<void> {
const host: ServerHost = new LocalServerHost();
+ let upgradeNow = false;
try {
this.setStatus({ checkoutBuild: await this.readCheckoutBuild(host) });
const managed = await getManagedServer();
if (!managed) return;
- if (await isStackRunning(host)) {
+ const running = await isStackRunning(host);
+ if (running) {
this.setStatus({ phase: 'running', serverId: managed.id, message: null, error: null });
// Only for a running stack: a stopped one sends nothing, so it cannot
// be out of step with the user's answer.
@@ -138,17 +169,75 @@ class LocalServerService {
// A checkout build is deliberately not a comparable version, so the pin
// comparison has nothing to say about it — the UI reports it as a
// checkout build instead of as unexplained drift.
- this.setStatus(
- version.deployedVersion === CHECKOUT_IMAGE_TAG ? { ...version, drift: null } : version
- );
+ const drift = version.deployedVersion === CHECKOUT_IMAGE_TAG ? null : version.drift;
+ let journal: UpgradeJournal | null;
+ try {
+ journal = await readUpgradeJournal(host);
+ } catch (error) {
+ this.setStatus({
+ ...version,
+ drift,
+ upgrade: {
+ state: 'failed',
+ from: version.deployedVersion ?? 'an unknown version',
+ to: COMPATIBLE_SWITCH_VERSION,
+ error: error instanceof Error ? error.message : String(error),
+ },
+ });
+ return;
+ }
+ const owed = owedUpgrade(drift, journal);
+ upgradeNow = owed !== null && (running || journal !== null);
+ this.setStatus({ ...version, drift, upgrade: owed && upgradeState(owed, upgradeNow) });
} catch (error) {
log.warn('local-switch-server: boot status reconcile failed', { error });
} finally {
host.dispose();
}
+ // Not awaited: the boot check is done once the upgrade is under way, and a
+ // Start click or a session joins the upgrade through `starting`.
+ if (upgradeNow) void this.beginStart(false);
+ }
+
+ /**
+ * Resolves once sessions may run against this server, waiting out the boot
+ * check and any upgrade in flight. Throws when the server still owes an
+ * upgrade — stopped, or failed — with the reason to show.
+ */
+ async ensureReady(serverName: string): Promise<void> {
+ await this.initialize();
+ while (this.starting) await this.starting;
+ const upgrade = this.status.upgrade;
+ if (upgrade === null) return;
+ if (upgrade.state === 'updating') {
+ throw new Error(`${serverName} reports an update in progress, but none is running.`);
+ }
+ this.refused = true;
+ throw new Error(managedServerUpgradeBlockedReason(serverName, upgrade));
+ }
+
+ /** Called with the server id when an upgrade finishes that sessions or
+ * watchers were turned away for (a failed one retried, or a stopped one
+ * started). Those that waited instead carry on by themselves. */
+ onUpgradeFinished(listener: (serverId: string) => void): void {
+ this.upgradeListeners.add(listener);
}
+ /** Start (or restart) the stack at this build's pin, upgrading it first if
+ * it is behind. Joins a start already in flight, such as the boot upgrade. */
async start(): Promise<StartLocalServerResult> {
+ await this.initialize();
+ return this.starting ?? this.beginStart(true);
+ }
+
+ private beginStart(activate: boolean): Promise<StartLocalServerResult> {
+ this.starting ??= this.runStart(activate).finally(() => {
+ this.starting = null;
+ });
+ return this.starting;
+ }
+
+ private async runStart(activate: boolean): Promise<StartLocalServerResult> {
if (this.busy) {
return { kind: 'error', message: 'A local-server operation is already in progress.' };
}
@@ -162,13 +251,16 @@ class LocalServerService {
host,
ref: { kind: 'local' },
serverName: LOCAL_SERVER_NAME,
+ activate,
onMessage: (message) => this.setStatus({ message }),
onLog: (line) => events.emit(localServerLogChannel, { line }),
+ onUpgrade: (owed) => this.setStatus({ upgrade: upgradeState(owed, true) }),
signal: this.startAbort.signal,
checkoutRoot,
});
if (result.kind === 'docker-unavailable') {
this.setStatus({ phase: 'error', error: result.detail });
+ this.failUpgrade(result.detail);
} else if (result.kind === 'version-downgrade') {
this.setStatus({
phase: 'error',
@@ -176,16 +268,15 @@ class LocalServerService {
error: switchVersionDowngradeMessage(result.deployed, result.expected),
deployedVersion: result.deployed,
drift: { deployed: result.deployed, expected: result.expected, direction: 'downgrade' },
+ upgrade: null,
});
} else if (result.kind === 'matrix-migration-failed') {
- this.setStatus({
- phase: 'error',
- message: null,
- error: matrixMigrationFailedMessage(result.deployed, result.expected),
- deployedVersion: result.deployed,
- });
+ const error = matrixMigrationFailedMessage(result.deployed, result.expected);
+ this.setStatus({ phase: 'error', message: null, error, deployedVersion: result.deployed });
+ this.failUpgrade(error);
} else if (result.kind === 'error') {
this.setStatus({ phase: 'error', error: result.message });
+ this.failUpgrade(result.message);
} else {
// The pipeline just wrote this build's pin and converged the containers
// onto it, so any drift the boot probe found is now resolved.
@@ -196,8 +287,13 @@ class LocalServerService {
error: null,
deployedVersion: checkoutRoot !== null ? CHECKOUT_IMAGE_TAG : COMPATIBLE_SWITCH_VERSION,
drift: null,
+ upgrade: null,
deployedTelemetry: { known: true, enabled: result.telemetryEnabled },
});
+ if (this.refused) {
+ this.refused = false;
+ for (const listener of this.upgradeListeners) listener(result.serverId);
+ }
}
reportManagedServerStart('local', result);
return result;
@@ -205,6 +301,7 @@ class LocalServerService {
const message = error instanceof Error ? error.message : String(error);
log.error('local-switch-server: start failed', { error });
this.setStatus({ phase: 'error', error: message });
+ this.failUpgrade(message);
reportManagedServerStartThrew('local');
return { kind: 'error', message };
} finally {
@@ -214,6 +311,14 @@ class LocalServerService {
}
}
+ /** Record why an upgrade did not finish. A start that was not an upgrade
+ * has nothing to record here; its error is on the status already. */
+ private failUpgrade(error: string): void {
+ const upgrade = this.status.upgrade;
+ if (!upgrade) return;
+ this.setStatus({ upgrade: { state: 'failed', from: upgrade.from, to: upgrade.to, error } });
+ }
+
async stop(): Promise<void> {
if (this.busy) throw new Error('A local-server operation is already in progress.');
this.busy = true;
@@ -221,7 +326,14 @@ class LocalServerService {
try {
this.setStatus({ phase: 'stopping', message: 'Stopping containers…' });
await stopStack(host);
- this.setStatus({ phase: 'stopped', message: null, error: null, deployedTelemetry: null });
+ const upgrade = this.status.upgrade;
+ this.setStatus({
+ phase: 'stopped',
+ message: null,
+ error: null,
+ deployedTelemetry: null,
+ upgrade: upgrade && upgradeState(upgrade, false),
+ });
reportManagedServerOutcome('stop', 'local', 'success');
} catch (error) {
this.setStatus({
@@ -253,7 +365,15 @@ class LocalServerService {
if (server) await deleteAgentsForServer(server.id);
this.setStatus({ phase: 'stopping', message: 'Destroying containers and data…' });
await resetStack(host);
- this.setStatus({ phase: 'stopped', message: null, error: null, deployedTelemetry: null });
+ this.setStatus({
+ phase: 'stopped',
+ message: null,
+ error: null,
+ deployedTelemetry: null,
+ deployedVersion: null,
+ drift: null,
+ upgrade: null,
+ });
reportManagedServerOutcome('reset', 'local', 'success');
} catch (error) {
this.setStatus({
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/managed-upgrade.test.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/managed-upgrade.test.ts
new file mode 100644
index 000000000..ff7212eb6
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/managed-upgrade.test.ts
@@ -0,0 +1,229 @@
+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import type * as AppIdentity from '@shared/app-identity';
+import type * as DeployedVersion from './deployed-version';
+import type { ServerHost } from './host/types';
+
+const deployed = vi.hoisted(() => vi.fn());
+vi.mock('@shared/app-identity', async (importOriginal) => ({
+ ...(await importOriginal<typeof AppIdentity>()),
+ COMPATIBLE_SWITCH_VERSION: '0.11.0',
+}));
+vi.mock('./deployed-version', async (importOriginal) => ({
+ ...(await importOriginal<typeof DeployedVersion>()),
+ readDeployedVersion: deployed,
+}));
+vi.mock('@main/lib/logger', () => ({ log: { warn: vi.fn(), info: vi.fn(), error: vi.fn() } }));
+
+const { finishUpgrade, owedUpgrade, prepareUpgrade, readUpgradeJournal, UPGRADE_JOURNAL_FILE } =
+ await import('./managed-upgrade');
+
+type Exec = (
+ command: string,
+ args: string[],
+ opts?: unknown
+) => Promise<{ stdout: string; stderr: string }>;
+
+let directory: string;
+let exec: ReturnType<typeof vi.fn<Exec>>;
+let modes: Map<string, number>;
+
+/** A host whose working dir is a temp directory and whose Docker is a fake
+ * that writes a dump where `docker cp` is asked to put it. */
+function fakeHost(kind: 'local' | 'remote'): ServerHost {
+ return {
+ kind,
+ label: kind === 'local' ? 'this computer' : 'builder',
+ workingDir: directory,
+ composeProjectName: 'test-stack',
+ dockerBin: 'docker',
+ ctx: { exec },
+ readFile: async (name: string) =>
+ readFile(join(directory, name), 'utf8').catch((error: NodeJS.ErrnoException) => {
+ if (error.code === 'ENOENT') return null;
+ throw error;
+ }),
+ writeFile: async (name: string, value: string, mode?: number) => {
+ await mkdir(dirname(join(directory, name)), { recursive: true });
+ await writeFile(join(directory, name), value);
+ if (mode !== undefined) modes.set(name, mode);
+ },
+ removeFile: async (name: string) => rm(join(directory, name), { force: true }),
+ restrictMode: async (name: string, mode: number) => {
+ modes.set(name, mode);
+ },
+ } as unknown as ServerHost;
+}
+
+beforeEach(async () => {
+ directory = await mkdtemp(join(tmpdir(), 'switch-upgrade-test-'));
+ modes = new Map();
+ exec = vi.fn<Exec>(async (_command, args) => {
+ if (args.includes('ps')) return { stdout: 'abc123\n', stderr: '' };
+ if (args[0] === 'cp') await writeFile(join(directory, args[2]), '-- PostgreSQL database dump');
+ return { stdout: '', stderr: '' };
+ });
+ deployed.mockResolvedValue({ kind: 'deployed', version: '0.10.0', source: 'container' });
+ const host = fakeHost('local');
+ await host.writeFile('.env', 'SWITCH_VERSION=0.10.0\n');
+ await host.writeFile('standalone-docker-compose.yml', 'services: {}\n');
+});
+
+afterEach(async () => {
+ await rm(directory, { recursive: true, force: true });
+});
+
+describe('prepareUpgrade', () => {
+ it.each(['local', 'remote'] as const)(
+ 'dumps the database and the old configuration before journalling the upgrade (%s host)',
+ async (kind) => {
+ const host = fakeHost(kind);
+ const onUpgrade = vi.fn();
+ const journal = await prepareUpgrade(host, null, vi.fn(), onUpgrade);
+
+ expect(onUpgrade).toHaveBeenCalledWith({ from: '0.10.0', to: '0.11.0' });
+ expect(journal).toMatchObject({ from: '0.10.0', to: '0.11.0' });
+ expect(journal!.backup.startsWith(directory)).toBe(true);
+ expect(JSON.parse((await host.readFile(UPGRADE_JOURNAL_FILE))!)).toEqual(journal);
+ expect(await readFile(join(journal!.backup, 'database.sql'), 'utf8')).toContain('PostgreSQL');
+ expect(await readFile(join(journal!.backup, '.env'), 'utf8')).toBe('SWITCH_VERSION=0.10.0\n');
+ // The stack's own configuration is left for the start pipeline to rewrite.
+ expect(await host.readFile('.env')).toBe('SWITCH_VERSION=0.10.0\n');
+
+ const relative = journal!.backup.slice(directory.length + 1);
+ expect(modes.get(relative)).toBe(0o700);
+ expect(modes.get(`${relative}/database.sql`)).toBe(0o600);
+ expect(modes.get(`${relative}/.env`)).toBe(0o600);
+
+ const commands = exec.mock.calls.map(([, args]) => args);
+ expect(commands[0]).toEqual(expect.arrayContaining(['up', '-d', '--no-deps', 'postgres']));
+ expect(
+ commands.some((args) => args[0] === 'exec' && args.join(' ').includes('pg_dumpall'))
+ ).toBe(true);
+ // The temporary dump is removed from the container afterwards.
+ expect(commands.at(-1)).toEqual(expect.arrayContaining(['exec', 'abc123', 'rm', '-f']));
+ }
+ );
+
+ it('does not journal an upgrade whose dump failed, and says why', async () => {
+ const host = fakeHost('remote');
+ exec.mockImplementation(async (_command, args) => {
+ if (args.includes('ps')) return { stdout: 'abc123\n', stderr: '' };
+ if (args[0] === 'exec' && args.join(' ').includes('pg_dumpall'))
+ throw Object.assign(new Error('exit 1'), { stderr: 'pg_dumpall: No space left on device' });
+ return { stdout: '', stderr: '' };
+ });
+
+ await expect(prepareUpgrade(host, null, vi.fn(), vi.fn())).rejects.toThrow(
+ 'No space left on device'
+ );
+ expect(await host.readFile(UPGRADE_JOURNAL_FILE)).toBeNull();
+ });
+
+ it('refuses to back up a stack whose configuration is missing', async () => {
+ const host = fakeHost('local');
+ await host.removeFile('standalone-docker-compose.yml');
+
+ await expect(prepareUpgrade(host, null, vi.fn(), vi.fn())).rejects.toThrow(
+ 'standalone-docker-compose.yml is missing'
+ );
+ expect(exec).not.toHaveBeenCalled();
+ expect(await host.readFile(UPGRADE_JOURNAL_FILE)).toBeNull();
+ });
+
+ it('resumes an interrupted upgrade without a second backup', async () => {
+ const host = fakeHost('local');
+ const first = await prepareUpgrade(host, null, vi.fn(), vi.fn());
+ exec.mockClear();
+ // By now the stack may already run the new version, half-migrated.
+ deployed.mockResolvedValue({ kind: 'deployed', version: '0.11.0', source: 'container' });
+ const onUpgrade = vi.fn();
+
+ expect(await prepareUpgrade(host, null, vi.fn(), onUpgrade)).toEqual(first);
+ expect(onUpgrade).toHaveBeenCalledWith({ from: '0.10.0', to: '0.11.0' });
+ expect(exec).not.toHaveBeenCalled();
+ });
+
+ it('carries an interrupted upgrade from an older app on to this pin', async () => {
+ const host = fakeHost('local');
+ await host.writeFile(
+ UPGRADE_JOURNAL_FILE,
+ JSON.stringify({ from: '0.9.0', to: '0.10.0', backup: '/backups/older' })
+ );
+
+ expect(await prepareUpgrade(host, null, vi.fn(), vi.fn())).toEqual({
+ from: '0.9.0',
+ to: '0.11.0',
+ backup: '/backups/older',
+ });
+ expect(JSON.parse((await host.readFile(UPGRADE_JOURNAL_FILE))!).to).toBe('0.11.0');
+ expect(exec).not.toHaveBeenCalled();
+ });
+
+ it('refuses an interrupted upgrade only a newer app can finish', async () => {
+ const host = fakeHost('local');
+ await host.writeFile(
+ UPGRADE_JOURNAL_FILE,
+ JSON.stringify({ from: '0.11.0', to: '0.12.0', backup: '/backups/newer' })
+ );
+
+ await expect(prepareUpgrade(host, null, vi.fn(), vi.fn())).rejects.toThrow(
+ /switch-core 0\.12\.0 or newer.*\/backups\/newer/
+ );
+ expect(exec).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ ['nothing deployed yet', { kind: 'absent' }],
+ ['an unreadable version', { kind: 'unreadable', reason: 'daemon down' }],
+ ['the same version', { kind: 'deployed', version: '0.11.0', source: 'container' }],
+ ['a newer version', { kind: 'deployed', version: '0.12.0', source: 'container' }],
+ ['an uncomparable version', { kind: 'deployed', version: 'nightly', source: 'env-file' }],
+ ['a checkout build', { kind: 'deployed', version: 'dev-checkout', source: 'container' }],
+ ])('is not an upgrade for %s', async (_label, version) => {
+ deployed.mockResolvedValue(version);
+ const onUpgrade = vi.fn();
+
+ expect(await prepareUpgrade(fakeHost('local'), null, vi.fn(), onUpgrade)).toBeNull();
+ expect(onUpgrade).not.toHaveBeenCalled();
+ expect(exec).not.toHaveBeenCalled();
+ });
+
+ it('does not back up when building from a checkout', async () => {
+ expect(await prepareUpgrade(fakeHost('local'), '/src/switch', vi.fn(), vi.fn())).toBeNull();
+ expect(exec).not.toHaveBeenCalled();
+ });
+});
+
+describe('the upgrade journal', () => {
+ it('is gone once the upgrade finishes; the backup stays', async () => {
+ const host = fakeHost('local');
+ const journal = await prepareUpgrade(host, null, vi.fn(), vi.fn());
+ await finishUpgrade(host);
+
+ expect(await readUpgradeJournal(host)).toBeNull();
+ expect(await readFile(join(journal!.backup, 'database.sql'), 'utf8')).toContain('PostgreSQL');
+ });
+
+ it('fails loudly when it cannot be read', async () => {
+ const host = fakeHost('remote');
+ await host.writeFile(UPGRADE_JOURNAL_FILE, '{"from": "0.10.0", "to"');
+
+ await expect(readUpgradeJournal(host)).rejects.toThrow(/interrupted.*cannot be read/);
+ });
+
+ it('names the upgrade a stack owes', () => {
+ const drift = { deployed: '0.10.0', expected: '0.11.0', direction: 'upgrade' } as const;
+ expect(owedUpgrade(drift, null)).toEqual({ from: '0.10.0', to: '0.11.0' });
+ expect(owedUpgrade(null, { from: '0.9.0', to: '0.10.0', backup: '/b' })).toEqual({
+ from: '0.9.0',
+ to: '0.11.0',
+ });
+ expect(
+ owedUpgrade({ deployed: '0.12.0', expected: '0.11.0', direction: 'downgrade' }, null)
+ ).toBeNull();
+ expect(owedUpgrade(null, null)).toBeNull();
+ });
+});
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/managed-upgrade.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/managed-upgrade.ts
new file mode 100644
index 000000000..8ca672435
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/managed-upgrade.ts
@@ -0,0 +1,233 @@
+import { randomUUID } from 'node:crypto';
+import { join, posix } from 'node:path';
+import { z } from 'zod';
+import { log } from '@main/lib/logger';
+import { compareVersions } from '@main/lib/semver';
+import { COMPATIBLE_SWITCH_VERSION } from '@shared/app-identity';
+import {
+ CHECKOUT_IMAGE_TAG,
+ type ManagedServerUpgrade,
+ type SwitchVersionDrift,
+} from '@shared/core/managed-switch-server/managed-switch-server';
+import { composeUpService, runDocker, serviceContainerId } from './compose';
+import { COMPOSE_FILE_NAME, ENV_FILE_NAME } from './constants';
+import { classifyVersionDrift, readDeployedVersion } from './deployed-version';
+import type { ServerHost } from './host/types';
+
+/**
+ * Moving a managed stack forward onto this build's switch-core pin.
+ *
+ * switch-core migrates its database on boot and has no way back, so before a
+ * start rewrites the stack at a newer pin the database is dumped next to the
+ * stack, on whichever host runs it. A journal records that an upgrade is under
+ * way: it is written once the backup is complete and removed once the stack is
+ * healthy at the pin, so an upgrade interrupted anywhere in between is resumed
+ * on the next start (without a second backup, which would capture a
+ * half-migrated database) or refused loudly when this app cannot finish it.
+ */
+
+/** Journal of an upgrade in progress, in the host working dir. */
+export const UPGRADE_JOURNAL_FILE = 'upgrade.json';
+const BACKUPS_DIR = 'backups';
+const DATABASE_SERVICE = 'postgres';
+/** A dump of a large install over a slow disk can take a while. */
+const DUMP_TIMEOUT_MS = 30 * 60 * 1000;
+
+const journalSchema = z.object({ from: z.string(), to: z.string(), backup: z.string() });
+export type UpgradeJournal = z.infer<typeof journalSchema>;
+
+/** `from` → `to` of an upgrade a stack still owes this build. */
+export type OwedUpgrade = { from: string; to: string };
+
+function hostPath(host: ServerHost, relPath: string): string {
+ return host.kind === 'local'
+ ? join(host.workingDir, relPath)
+ : posix.join(host.workingDir, relPath);
+}
+
+/**
+ * The journal of an interrupted upgrade on `host`, or null when none is under
+ * way. A journal that cannot be read throws: it means an upgrade was
+ * interrupted and there is no telling how far it got.
+ */
+export async function readUpgradeJournal(host: ServerHost): Promise<UpgradeJournal | null> {
+ const raw = await host.readFile(UPGRADE_JOURNAL_FILE);
+ if (raw === null) return null;
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(raw);
+ } catch {
+ parsed = null;
+ }
+ const journal = journalSchema.safeParse(parsed);
+ if (!journal.success) {
+ throw new Error(
+ `An earlier switch-core update on ${host.label} was interrupted and its journal ` +
+ `(${hostPath(host, UPGRADE_JOURNAL_FILE)}) cannot be read. Its database backup is under ` +
+ `${hostPath(host, BACKUPS_DIR)}. Check the stack, then delete the journal to update again.`
+ );
+ }
+ return journal.data;
+}
+
+/**
+ * The upgrade a stack owes this build: the interrupted one its journal names,
+ * or the one its deployed version being behind the pin calls for. Null when it
+ * is in step, ahead, or not comparable — those keep their own handling.
+ */
+export function owedUpgrade(
+ drift: SwitchVersionDrift | null,
+ journal: UpgradeJournal | null
+): OwedUpgrade | null {
+ if (journal) return { from: journal.from, to: COMPATIBLE_SWITCH_VERSION };
+ if (drift?.direction === 'upgrade') return { from: drift.deployed, to: drift.expected };
+ return null;
+}
+
+/** The status of an owed upgrade: running now, or waiting for the next start. */
+export function upgradeState(owed: OwedUpgrade, running: boolean): ManagedServerUpgrade {
+ return { state: running ? 'updating' : 'pending', from: owed.from, to: owed.to };
+}
+
+/**
+ * Get `host` ready to be rewritten at this build's pin: resume an interrupted
+ * upgrade, or back the database up and journal a new one. Returns the journal,
+ * or null when the start is not an upgrade (first install, same version, a
+ * checkout build, or a version that is unreadable or not comparable — the
+ * start pipeline guards those separately).
+ *
+ * `onUpgrade` fires before any work, so the caller can show the update and hold
+ * sessions for the whole of it. Any failure throws, leaving the stack's
+ * configuration untouched.
+ */
+export async function prepareUpgrade(
+ host: ServerHost,
+ checkoutRoot: string | null,
+ onMessage: (message: string) => void,
+ onUpgrade: (upgrade: OwedUpgrade) => void
+): Promise<UpgradeJournal | null> {
+ const pending = await readUpgradeJournal(host);
+ if (pending) {
+ const order = compareVersions(pending.to, COMPATIBLE_SWITCH_VERSION);
+ if (order === null || order > 0) {
+ throw new Error(
+ `An interrupted update on ${host.label} was moving switch-core to ${pending.to}, which ` +
+ `this app (switch-core ${COMPATIBLE_SWITCH_VERSION}) cannot finish. Install a Switch ` +
+ `Console with switch-core ${pending.to} or newer. The database backup taken before ` +
+ `that update is in ${pending.backup}.`
+ );
+ }
+ onUpgrade({ from: pending.from, to: COMPATIBLE_SWITCH_VERSION });
+ log.warn(
+ `managed-switch-server: resuming the interrupted switch-core update on ${host.label} ` +
+ `(${pending.from} → ${COMPATIBLE_SWITCH_VERSION}); backup in ${pending.backup}`
+ );
+ if (pending.to === COMPATIBLE_SWITCH_VERSION) return pending;
+ const resumed = { ...pending, to: COMPATIBLE_SWITCH_VERSION };
+ await writeJournal(host, resumed);
+ return resumed;
+ }
+ if (checkoutRoot !== null) return null;
+ const deployed = await readDeployedVersion(host);
+ if (deployed.kind !== 'deployed' || deployed.version === CHECKOUT_IMAGE_TAG) return null;
+ if (classifyVersionDrift(deployed.version, COMPATIBLE_SWITCH_VERSION)?.direction !== 'upgrade') {
+ return null;
+ }
+ onUpgrade({ from: deployed.version, to: COMPATIBLE_SWITCH_VERSION });
+ const backup = await backupDatabase(host, deployed.version, onMessage);
+ const journal = { from: deployed.version, to: COMPATIBLE_SWITCH_VERSION, backup };
+ await writeJournal(host, journal);
+ return journal;
+}
+
+/** Record that the upgrade finished: the stack is healthy at the pin. */
+export async function finishUpgrade(host: ServerHost): Promise<void> {
+ await host.removeFile(UPGRADE_JOURNAL_FILE);
+}
+
+async function writeJournal(host: ServerHost, journal: UpgradeJournal): Promise<void> {
+ await host.writeFile(UPGRADE_JOURNAL_FILE, `${JSON.stringify(journal, null, 2)}\n`, 0o600);
+}
+
+/**
+ * Dump every database of the stack as it is deployed now, plus the compose
+ * file and `.env` that describe it, into a new timestamped directory under
+ * `backups/` in the host working dir. Returns that directory's path on the
+ * host.
+ *
+ * The dump is written inside the database container and copied out with
+ * `docker cp`, so nothing is buffered through Console or piped through a shell
+ * that may not exist on the host; the same commands work locally and over SSH.
+ */
+async function backupDatabase(
+ host: ServerHost,
+ from: string,
+ onMessage: (message: string) => void
+): Promise<string> {
+ onMessage(
+ `Backing up the database before updating switch-core ${from} → ${COMPATIBLE_SWITCH_VERSION}…`
+ );
+ const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+ const directory = `${BACKUPS_DIR}/${stamp}-switch-core-${from}`;
+ await host.writeFile(`${directory}/README.txt`, backupReadme(from), 0o600);
+ await host.restrictMode(directory, 0o700);
+ for (const file of [COMPOSE_FILE_NAME, ENV_FILE_NAME]) {
+ const content = await host.readFile(file);
+ if (content === null) {
+ throw new Error(
+ `Cannot back up the Switch server on ${host.label} before updating it: ${file} is missing from ${host.workingDir}.`
+ );
+ }
+ await host.writeFile(`${directory}/${file}`, content, 0o600);
+ }
+
+ await composeUpService(host, DATABASE_SERVICE);
+ const container = await serviceContainerId(host, DATABASE_SERVICE);
+ const inContainer = `/tmp/switch-upgrade-${randomUUID()}.sql`;
+ const destination = `${directory}/database.sql`;
+ try {
+ await runDocker(
+ host,
+ [
+ 'exec',
+ container,
+ 'sh',
+ '-c',
+ 'umask 077 && pg_dumpall -U "$POSTGRES_USER" -f "$1" && test -s "$1"',
+ 'sh',
+ inContainer,
+ ],
+ DUMP_TIMEOUT_MS
+ );
+ await runDocker(host, ['cp', `${container}:${inContainer}`, destination], DUMP_TIMEOUT_MS);
+ } finally {
+ await runDocker(host, ['exec', container, 'rm', '-f', inContainer], 60_000).catch(
+ (error: unknown) => {
+ log.warn(
+ `managed-switch-server: could not remove the temporary dump ${inContainer} from the database container on ${host.label}`,
+ { error: String(error) }
+ );
+ }
+ );
+ }
+ await host.restrictMode(destination, 0o600);
+ const path = hostPath(host, directory);
+ log.info(`managed-switch-server: backed up ${host.label} to ${path} before updating`);
+ return path;
+}
+
+function backupReadme(from: string): string {
+ return [
+ `Switch Console took this backup before updating the Switch server from switch-core ${from} to ${COMPATIBLE_SWITCH_VERSION}.`,
+ '',
+ 'database.sql pg_dumpall of every database in the stack',
+ `${COMPOSE_FILE_NAME} the compose file the stack ran with`,
+ `${ENV_FILE_NAME} its environment, including credentials`,
+ '',
+ 'Keep these files private. switch-core migrates its database forward and cannot run',
+ `against it at ${from} again once the update has started. To go back, stop the stack and`,
+ `restore database.sql into a fresh database volume with switch-core ${from}.`,
+ 'Uploaded files stay in the Docker volumes; this dump does not copy them.',
+ '',
+ ].join('\n');
+}
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline-matrix-migration.test.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline-matrix-migration.test.ts
index 451b035ad..296037cbf 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline-matrix-migration.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline-matrix-migration.test.ts
@@ -15,6 +15,8 @@ const readDeployedVersionMock = vi.hoisted(() => vi.fn());
const composeUpMock = vi.hoisted(() => vi.fn(() => Promise.resolve()));
const dockerRunOneOffMock = vi.hoisted(() => vi.fn(() => Promise.resolve()));
+const prepareUpgradeMock = vi.hoisted(() => vi.fn(() => Promise.resolve(null)));
+const finishUpgradeMock = vi.hoisted(() => vi.fn(() => Promise.resolve()));
vi.mock('@shared/app-identity', async (importOriginal) => ({
...(await importOriginal<typeof AppIdentity>()),
COMPATIBLE_SWITCH_VERSION: '0.24.0',
@@ -62,6 +64,10 @@ vi.mock('@main/core/switch-servers/auth', () => ({
passwordLogin: () => Promise.resolve({ success: true }),
}));
vi.mock('@main/core/agents/resolve-servers', () => ({ resolveAgentServers: vi.fn() }));
+vi.mock('./managed-upgrade', () => ({
+ prepareUpgrade: prepareUpgradeMock,
+ finishUpgrade: finishUpgradeMock,
+}));
const { startStack } = await import('./pipeline');
@@ -83,6 +89,8 @@ function options(checkoutRoot: string | null = null) {
serverName: 'Local',
onMessage: vi.fn(),
onLog: vi.fn(),
+ activate: true,
+ onUpgrade: vi.fn(),
signal: new AbortController().signal,
checkoutRoot,
},
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline.test.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline.test.ts
index 6cee618fa..f5e827c8e 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline.test.ts
@@ -20,6 +20,10 @@ const telemetryConsentMock = vi.hoisted(() => vi.fn(() => Promise.resolve(false)
// Pin the build's expected version so the guard's arithmetic is not coupled to
// whatever the real pin happens to be.
+const prepareUpgradeMock = vi.hoisted(() =>
+ vi.fn<(...args: unknown[]) => Promise<unknown>>(() => Promise.resolve(null))
+);
+const finishUpgradeMock = vi.hoisted(() => vi.fn(() => Promise.resolve()));
vi.mock('@shared/app-identity', async (importOriginal) => ({
...(await importOriginal<typeof AppIdentity>()),
COMPATIBLE_SWITCH_VERSION: '0.11.0',
@@ -57,14 +61,19 @@ vi.mock('./telemetry-consent', () => ({
telemetryConsent: telemetryConsentMock,
readDeployedTelemetry: vi.fn(),
}));
+const setActiveServerIdMock = vi.hoisted(() => vi.fn());
vi.mock('@main/core/switch-servers/servers-store', () => ({
ensureManagedServer: () => Promise.resolve({ id: 'srv-1' }),
- setActiveServerId: vi.fn(),
+ setActiveServerId: setActiveServerIdMock,
}));
vi.mock('@main/core/switch-servers/auth', () => ({
passwordLogin: () => Promise.resolve({ success: true }),
}));
vi.mock('@main/core/agents/resolve-servers', () => ({ resolveAgentServers: vi.fn() }));
+vi.mock('./managed-upgrade', () => ({
+ prepareUpgrade: prepareUpgradeMock,
+ finishUpgrade: finishUpgradeMock,
+}));
const { startStack } = await import('./pipeline');
const { ENV_FILE_NAME } = await import('./constants');
@@ -87,6 +96,8 @@ function options() {
serverName: 'Local',
onMessage: vi.fn(),
onLog: vi.fn(),
+ activate: true,
+ onUpgrade: vi.fn(),
signal: new AbortController().signal,
checkoutRoot: null as string | null,
},
@@ -95,6 +106,7 @@ function options() {
beforeEach(() => {
vi.clearAllMocks();
+ prepareUpgradeMock.mockResolvedValue(null);
telemetryConsentMock.mockResolvedValue(false);
buildEnvFileMock.mockReturnValue('SWITCH_VERSION=0.11.0\n');
});
@@ -323,3 +335,88 @@ describe('startStack checkout build', () => {
expect(composeUpMock).toHaveBeenCalledWith(expect.anything(), expect.any(Function), false);
});
});
+
+describe('startStack upgrade', () => {
+ const journal = { from: '0.10.0', to: '0.11.0', backup: '/data/backups/x' };
+
+ it('backs the database up before rewriting the stack and closes the journal once healthy', async () => {
+ readDeployedVersionMock.mockResolvedValue({
+ kind: 'deployed',
+ version: '0.10.0',
+ source: 'container',
+ });
+ const order: string[] = [];
+ prepareUpgradeMock.mockImplementation(async () => {
+ order.push('backup');
+ return journal;
+ });
+ finishUpgradeMock.mockImplementation(async () => {
+ order.push('finish');
+ });
+ const { opts, writeFile } = options();
+ writeFile.mockImplementation(async (name: string) => {
+ order.push(`write ${name}`);
+ });
+
+ expect(await startStack(opts)).toMatchObject({ kind: 'started' });
+ expect(order[0]).toBe('backup');
+ expect(order.at(-1)).toBe('finish');
+ expect(prepareUpgradeMock).toHaveBeenCalledWith(
+ opts.host,
+ null,
+ opts.onMessage,
+ opts.onUpgrade
+ );
+ });
+
+ it('leaves the stack untouched when the backup fails', async () => {
+ readDeployedVersionMock.mockResolvedValue({
+ kind: 'deployed',
+ version: '0.10.0',
+ source: 'container',
+ });
+ prepareUpgradeMock.mockRejectedValue(new Error('pg_dumpall failed: disk full'));
+ const { opts, writeFile } = options();
+
+ await expect(startStack(opts)).rejects.toThrow('disk full');
+ expect(writeFile).not.toHaveBeenCalled();
+ expect(composeUpMock).not.toHaveBeenCalled();
+ expect(finishUpgradeMock).not.toHaveBeenCalled();
+ });
+
+ it('keeps the journal when the upgraded stack never turns healthy', async () => {
+ readDeployedVersionMock.mockResolvedValue({
+ kind: 'deployed',
+ version: '0.10.0',
+ source: 'container',
+ });
+ prepareUpgradeMock.mockResolvedValue(journal);
+ waitForHealthMock.mockResolvedValueOnce(false);
+ const { opts } = options();
+
+ expect(await startStack(opts)).toMatchObject({ kind: 'error' });
+ expect(finishUpgradeMock).not.toHaveBeenCalled();
+ });
+
+ it('does not back up a stack that is ahead of this build', async () => {
+ readDeployedVersionMock.mockResolvedValue({
+ kind: 'deployed',
+ version: '0.12.0',
+ source: 'container',
+ });
+ const { opts } = options();
+
+ expect(await startStack(opts)).toMatchObject({ kind: 'version-downgrade' });
+ expect(prepareUpgradeMock).not.toHaveBeenCalled();
+ });
+
+ it('does not switch the active server for a background upgrade', async () => {
+ readDeployedVersionMock.mockResolvedValue({ kind: 'absent' });
+ const { opts } = options();
+
+ await startStack({ ...opts, activate: false });
+ expect(setActiveServerIdMock).not.toHaveBeenCalled();
+ await startStack(opts);
+ expect(setActiveServerIdMock).toHaveBeenCalledWith('srv-1');
+ });
+});
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline.ts
index 161e9d2f5..0982bfa60 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline.ts
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline.ts
@@ -23,6 +23,7 @@ import { buildEnvFile } from './env-file';
import { apiUrlFor, gatewayUrlFor } from './free-port';
import { waitForHealth } from './health';
import type { ServerHost } from './host/types';
+import { finishUpgrade, type OwedUpgrade, prepareUpgrade } from './managed-upgrade';
import { crossesMatrixBoundary, runBackfill } from './matrix-migration';
import { clearPorts, resolvePorts } from './ports';
import { clearSecrets, loadOrCreateSecrets } from './secrets';
@@ -41,10 +42,16 @@ export type StartStackOptions = {
ref: ManagedServerRef;
/** Display name for the registered server record. */
serverName: string;
+ /** Whether to make this the active server. False for an upgrade Console runs
+ * on its own, which must not switch the user away from what they have open. */
+ activate: boolean;
/** Coarse step messages for the UI ("Pulling images…"). */
onMessage: (message: string) => void;
/** Live compose output lines for the UI log tail. */
onLog: (line: string) => void;
+ /** Fired before a start that moves the stack forward to this build's pin, so
+ * the supervisor can hold sessions until it finishes. */
+ onUpgrade: (upgrade: OwedUpgrade) => void;
/** Aborts an in-flight health wait (stop/cancel/quit). */
signal: AbortSignal;
/** Dev-only: root of the Switch checkout to build the stack's images from,
@@ -119,16 +126,18 @@ async function refuseDowngrade(
}
/**
- * Full start pipeline: detect Docker → refuse a downgrade → GHCR login →
- * materialise compose + `.env` → `compose up` → establish networking →
- * health-gate → register + activate → silent admin sign-in → reconcile agent
- * servers. Returns without registering anything if Docker is unavailable, the
- * stack is newer than this build, or it never turns healthy.
+ * Full start pipeline: detect Docker → refuse a downgrade → back up and
+ * journal an upgrade → GHCR login → materialise compose + `.env` → `compose up`
+ * → establish networking → health-gate → register (+ activate) → silent admin
+ * sign-in → reconcile agent servers → close the upgrade journal. Returns
+ * without registering anything if Docker is unavailable, the stack is newer
+ * than this build, or it never turns healthy.
*
* Doubles as the update path: the `.env` and compose file are re-materialised
* from this build every time, so `compose up -d` on an already-running stack
* re-pulls the newly pinned tags and recreates only the changed containers,
- * leaving the data volumes in place for switch-core to migrate forward.
+ * leaving the data volumes in place for switch-core to migrate forward. A
+ * stack behind the pin has its database dumped first (see managed-upgrade.ts).
*
* With `checkoutRoot` set (dev only) the images are built from that working
* tree on every start instead of pulled, and tagged {@link CHECKOUT_IMAGE_TAG}
@@ -182,7 +191,8 @@ async function migrateOffMatrix(
}
export async function startStack(opts: StartStackOptions): Promise<StartLocalServerResult> {
- const { host, ref, serverName, onMessage, onLog, signal, checkoutRoot } = opts;
+ const { host, ref, serverName, activate, onMessage, onLog, onUpgrade, signal, checkoutRoot } =
+ opts;
const docker = await host.detectDocker();
if (!docker.available) {
@@ -193,6 +203,10 @@ export async function startStack(opts: StartStackOptions): Promise<StartLocalSer
const downgrade = await refuseDowngrade(host, checkoutRoot);
if (downgrade) return downgrade;
+ // Back the database up before anything can migrate it: the Matrix backfill
+ // below already starts the stack, and the rewrite after it moves the pin.
+ const upgrade = await prepareUpgrade(host, checkoutRoot, onMessage, onUpgrade);
+
// Copy the homeserver's history across before the upgrade removes the only
// thing that can read it. Runs against the stack as currently deployed, so
// it must happen before the compose file and `.env` are re-materialised for
@@ -220,7 +234,6 @@ export async function startStack(opts: StartStackOptions): Promise<StartLocalSer
namespace: RELEASE_REPO_OWNER,
ports,
secrets,
- sessionDemo: checkoutRoot !== null,
telemetryEnabled,
}),
0o600
@@ -245,7 +258,7 @@ export async function startStack(opts: StartStackOptions): Promise<StartLocalSer
}
const server = await ensureManagedServer({ name: serverName, gatewayUrl, apiUrl }, ref);
- await setActiveServerId(server.id);
+ if (activate) await setActiveServerId(server.id);
// Switch Console generated the admin password, so sign in on the user's behalf
// rather than showing a login wall for a secret they never saw. A failure here
@@ -260,6 +273,7 @@ export async function startStack(opts: StartStackOptions): Promise<StartLocalSer
}
await resolveAgentServers();
+ if (upgrade) await finishUpgrade(host);
return { kind: 'started', serverId: server.id, telemetryEnabled };
}
@@ -276,4 +290,6 @@ export async function resetStack(host: ServerHost): Promise<void> {
await host.teardownNetworking();
await clearSecrets(host);
await clearPorts(host);
+ // Nothing is left to resume. The backups stay on disk.
+ await finishUpgrade(host);
}
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/remote-server-service.test.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/remote-server-service.test.ts
new file mode 100644
index 000000000..f77b13dfe
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/remote-server-service.test.ts
@@ -0,0 +1,277 @@
+import { beforeEach, expect, it, vi } from 'vitest';
+import type * as AppIdentity from '@shared/app-identity';
+import type { StartLocalServerResult } from '@shared/core/managed-switch-server/managed-switch-server';
+import type * as ManagedUpgrade from './managed-upgrade';
+import type { StartStackOptions } from './pipeline';
+
+type Change = { current: { sshHost: string; status: string } };
+const m = vi.hoisted(() => {
+ const listeners: ((change: Change) => void)[] = [];
+ const blocked = new Set<string>();
+ return {
+ listeners,
+ blocked,
+ reachability: {
+ on(_event: string, listener: (change: Change) => void) {
+ listeners.push(listener);
+ },
+ isBlocked: (sshHost: string) => blocked.has(sshHost),
+ requireReachable(sshHost: string) {
+ if (blocked.has(sshHost)) throw new Error(`${sshHost} is unreachable`);
+ },
+ },
+ servers: vi.fn(),
+ running: vi.fn(),
+ version: vi.fn(),
+ journal: vi.fn(),
+ ports: vi.fn(),
+ establish: vi.fn(),
+ start: vi.fn<(opts: StartStackOptions) => Promise<StartLocalServerResult>>(),
+ };
+});
+vi.mock('@shared/app-identity', async (importOriginal) => ({
+ ...(await importOriginal<typeof AppIdentity>()),
+ COMPATIBLE_SWITCH_VERSION: '0.11.0',
+}));
+vi.mock('@main/core/remote-hosts/production-host-reachability', () => ({
+ hostReachabilityService: m.reachability,
+}));
+vi.mock('@main/lib/events', () => ({ events: { emit: vi.fn() } }));
+vi.mock('@main/lib/logger', () => ({ log: { warn: vi.fn(), error: vi.fn(), info: vi.fn() } }));
+vi.mock('@main/core/telemetry/managed-server', () => ({
+ reportManagedServerOutcome: vi.fn(),
+ reportManagedServerStart: vi.fn(),
+ reportManagedServerStartThrew: vi.fn(),
+}));
+vi.mock('@main/core/switch-servers/delete-server-agents', () => ({
+ deleteAgentsForServer: vi.fn(),
+}));
+vi.mock('@main/core/switch-servers/servers-store', () => ({
+ listManagedServers: m.servers,
+ getRemoteManagedServer: vi.fn(),
+}));
+vi.mock('./compose', () => ({ isStackRunning: m.running }));
+vi.mock('./deployed-version', () => ({ readVersionStatus: m.version }));
+vi.mock('./ports', () => ({ readPersistedPorts: m.ports }));
+vi.mock('./telemetry-consent', () => ({ readDeployedTelemetry: vi.fn() }));
+vi.mock('./managed-upgrade', async (importOriginal) => ({
+ ...(await importOriginal<typeof ManagedUpgrade>()),
+ readUpgradeJournal: m.journal,
+}));
+vi.mock('./pipeline', () => ({ startStack: m.start, stopStack: vi.fn(), resetStack: vi.fn() }));
+vi.mock('./host/remote-host', () => ({
+ createRemoteServerHost: async (sshHost: string) => ({
+ sshHost,
+ establishNetworking: m.establish,
+ dispose: vi.fn(),
+ }),
+}));
+
+const { RemoteServerService } = await import('./remote-server-service');
+
+const behind = {
+ deployedVersion: '0.10.0',
+ drift: { deployed: '0.10.0', expected: '0.11.0', direction: 'upgrade' },
+};
+const inStep = { deployedVersion: '0.11.0', drift: null };
+
+function startedOn(serverId: string): StartLocalServerResult {
+ return { kind: 'started', serverId, telemetryEnabled: false };
+}
+
+/** A start that runs the upgrade the way the pipeline does: announce, then work. */
+function upgradingStart(result?: StartLocalServerResult) {
+ return async (opts: StartStackOptions) => {
+ opts.onUpgrade({ from: '0.10.0', to: '0.11.0' });
+ return result ?? startedOn(opts.ref.kind === 'remote' ? `srv-${opts.ref.sshHost}` : 'local');
+ };
+}
+
+function announce(sshHost: string, status: string): void {
+ for (const listener of m.listeners) listener({ current: { sshHost, status } });
+}
+
+beforeEach(() => {
+ vi.resetAllMocks();
+ m.listeners.length = 0;
+ m.blocked.clear();
+ m.servers.mockResolvedValue([
+ { id: 'srv-builder', name: 'Builder', managementKind: 'remote', sshHost: 'builder' },
+ ]);
+ m.running.mockResolvedValue(true);
+ m.version.mockResolvedValue(behind);
+ m.journal.mockResolvedValue(null);
+ m.ports.mockResolvedValue({ gateway: 1, api: 2, mattermost: 3, postgres: 4 });
+ m.start.mockImplementation(upgradingStart());
+});
+
+it('upgrades a running remote stack that is behind at boot, instead of adopting it', async () => {
+ const service = new RemoteServerService();
+ await service.initialize();
+ await service.ensureReady('builder', 'Builder');
+
+ expect(m.start).toHaveBeenCalledOnce();
+ expect(m.start).toHaveBeenCalledWith(
+ expect.objectContaining({
+ ref: { kind: 'remote', sshHost: 'builder' },
+ serverName: 'Builder',
+ activate: false,
+ })
+ );
+ expect(m.establish).not.toHaveBeenCalled();
+ expect(service.getStatus('builder')).toMatchObject({
+ phase: 'running',
+ serverId: 'srv-builder',
+ upgrade: null,
+ drift: null,
+ });
+});
+
+it('holds sessions for the host while its upgrade runs', async () => {
+ let finish: (result: StartLocalServerResult) => void = () => {};
+ m.start.mockImplementation(
+ (opts) =>
+ new Promise((resolve) => {
+ opts.onUpgrade({ from: '0.10.0', to: '0.11.0' });
+ finish = resolve;
+ })
+ );
+ const service = new RemoteServerService();
+ void service.initialize();
+ let ready = false;
+ const session = service.ensureReady('builder', 'Builder').then(() => {
+ ready = true;
+ });
+
+ await vi.waitFor(() => expect(m.start).toHaveBeenCalledOnce());
+ expect(service.getStatus('builder').upgrade).toEqual({
+ state: 'updating',
+ from: '0.10.0',
+ to: '0.11.0',
+ });
+ expect(ready).toBe(false);
+ finish(startedOn('srv-builder'));
+ await session;
+ expect(ready).toBe(true);
+});
+
+it('adopts a running stack that is in step without restarting it', async () => {
+ m.version.mockResolvedValue(inStep);
+ const service = new RemoteServerService();
+ await service.initialize();
+ await service.ensureReady('builder', 'Builder');
+
+ expect(m.start).not.toHaveBeenCalled();
+ expect(m.establish).toHaveBeenCalledOnce();
+ expect(service.getStatus('builder')).toMatchObject({
+ phase: 'running',
+ serverId: 'srv-builder',
+ upgrade: null,
+ });
+});
+
+it('upgrades a stopped stack that is behind at its next start, not before', async () => {
+ m.running.mockResolvedValue(false);
+ const service = new RemoteServerService();
+ const upgraded = vi.fn();
+ service.onUpgradeFinished(upgraded);
+ await service.initialize();
+
+ await expect(service.ensureReady('builder', 'Builder')).rejects.toThrow(
+ /Builder is stopped on switch-core 0\.10\.0/
+ );
+ expect(m.start).not.toHaveBeenCalled();
+
+ expect(await service.start('builder', 'Builder')).toMatchObject({ kind: 'started' });
+ expect(m.start).toHaveBeenCalledWith(expect.objectContaining({ activate: true }));
+ await expect(service.ensureReady('builder', 'Builder')).resolves.toBeUndefined();
+ expect(upgraded).toHaveBeenCalledExactlyOnceWith('srv-builder');
+});
+
+it('upgrades a host that was unreachable at boot once it comes back, holding sessions meanwhile', async () => {
+ m.blocked.add('builder');
+ const service = new RemoteServerService();
+ await service.initialize();
+ await service.ensureReady('builder', 'Builder');
+ expect(m.start).not.toHaveBeenCalled();
+
+ let finish: (result: StartLocalServerResult) => void = () => {};
+ m.start.mockImplementation(
+ (opts) =>
+ new Promise((resolve) => {
+ opts.onUpgrade({ from: '0.10.0', to: '0.11.0' });
+ finish = resolve;
+ })
+ );
+ m.blocked.delete('builder');
+ announce('builder', 'reachable');
+ // Asked straight after the recovery, as a watcher restoring its controller does.
+ let ready = false;
+ const session = service.ensureReady('builder', 'Builder').then(() => {
+ ready = true;
+ });
+
+ await vi.waitFor(() => expect(m.start).toHaveBeenCalledOnce());
+ expect(ready).toBe(false);
+ finish(startedOn('srv-builder'));
+ await session;
+ expect(service.getStatus('builder').upgrade).toBeNull();
+});
+
+it('surfaces a failed upgrade with its error and retries when the host is reconciled again', async () => {
+ m.start.mockImplementationOnce(
+ upgradingStart({ kind: 'error', message: 'The server did not become healthy in time.' })
+ );
+ const service = new RemoteServerService();
+ const upgraded = vi.fn();
+ service.onUpgradeFinished(upgraded);
+ await service.initialize();
+
+ await expect(service.ensureReady('builder', 'Builder')).rejects.toThrow(
+ /Updating Builder from switch-core 0\.10\.0 to 0\.11\.0 failed: The server did not become healthy/
+ );
+ expect(service.getStatus('builder')).toMatchObject({
+ phase: 'error',
+ upgrade: { state: 'failed', error: 'The server did not become healthy in time.' },
+ });
+
+ announce('builder', 'reachable');
+ await vi.waitFor(() => expect(m.start).toHaveBeenCalledTimes(2));
+ await expect(service.ensureReady('builder', 'Builder')).resolves.toBeUndefined();
+ expect(upgraded).toHaveBeenCalledExactlyOnceWith('srv-builder');
+});
+
+it('does not hold one host’s sessions behind another host’s upgrade', async () => {
+ m.servers.mockResolvedValue([
+ { id: 'srv-builder', name: 'Builder', managementKind: 'remote', sshHost: 'builder' },
+ { id: 'srv-other', name: 'Other', managementKind: 'remote', sshHost: 'other' },
+ ]);
+ m.start.mockImplementation(
+ (opts) =>
+ new Promise(() => {
+ opts.onUpgrade({ from: '0.10.0', to: '0.11.0' });
+ })
+ );
+ m.version.mockImplementation(async (host: { sshHost: string }) =>
+ host.sshHost === 'builder' ? behind : inStep
+ );
+ const service = new RemoteServerService();
+ await service.initialize();
+
+ await expect(service.ensureReady('other', 'Other')).resolves.toBeUndefined();
+ expect(service.getStatus('builder').upgrade?.state).toBe('updating');
+});
+
+it('refuses sessions when the host cannot be reached for its upgrade, rather than hanging', async () => {
+ const service = new RemoteServerService();
+ m.running.mockImplementation(async () => {
+ // The host drops between the probe and the start.
+ m.blocked.add('builder');
+ return true;
+ });
+ await service.initialize();
+
+ await expect(service.ensureReady('builder', 'Builder')).rejects.toThrow(
+ /failed: builder is unreachable/
+ );
+});
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/remote-server-service.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/remote-server-service.ts
index b8b044026..459be044c 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/remote-server-service.ts
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/remote-server-service.ts
@@ -16,6 +16,7 @@ import { COMPATIBLE_SWITCH_VERSION } from '@shared/app-identity';
import {
type DockerAvailability,
type StartLocalServerResult,
+ managedServerUpgradeBlockedReason,
matrixMigrationFailedMessage,
switchVersionDowngradeMessage,
} from '@shared/core/managed-switch-server/managed-switch-server';
@@ -27,6 +28,12 @@ import {
import { isStackRunning } from './compose';
import { readVersionStatus } from './deployed-version';
import { createRemoteServerHost, type RemoteServerHost } from './host/remote-host';
+import {
+ owedUpgrade,
+ readUpgradeJournal,
+ type UpgradeJournal,
+ upgradeState,
+} from './managed-upgrade';
import { resetStack, startStack, stopStack } from './pipeline';
import { readPersistedPorts } from './ports';
import { readDeployedTelemetry } from './telemetry-consent';
@@ -35,6 +42,7 @@ function initialStatus(sshHost: string): RemoteServerStatus {
return {
sshHost,
phase: 'stopped',
+ upgrade: null,
serverId: null,
version: COMPATIBLE_SWITCH_VERSION,
deployedVersion: null,
@@ -57,12 +65,25 @@ function initialStatus(sshHost: string): RemoteServerStatus {
* containers themselves run detached, so a remote stack (and its remote-host
* agents) stays up while Switch Console is closed — only the desktop-side forward
* goes away.
+ *
+ * A host whose stack is behind this build's switch-core pin is upgraded when it
+ * is reconciled — at boot, and whenever the host becomes reachable again — and
+ * {@link ensureReady} holds sessions for its agents until that has finished.
*/
-class RemoteServerService {
+export class RemoteServerService {
private readonly statuses = new Map<string, RemoteServerStatus>();
private readonly hosts = new Map<string, RemoteServerHost>();
private readonly busy = new Set<string>();
private readonly startAborts = new Map<string, AbortController>();
+ private initialization: Promise<void> | null = null;
+ /** The reconcile or start in flight per host, which {@link ensureReady} waits out. */
+ private readonly operations = new Map<string, Promise<void>>();
+ /** The start in flight per host, which an automatic upgrade and a Start click share. */
+ private readonly starting = new Map<string, Promise<StartLocalServerResult>>();
+ private readonly upgradeListeners = new Set<(serverId: string) => void>();
+ /** Hosts {@link ensureReady} has turned something away for since their last
+ * upgrade finished, so that it can be told when they are ready. */
+ private readonly refused = new Set<string>();
getStatuses(): RemoteServerStatus[] {
return [...this.statuses.values()];
@@ -89,32 +110,90 @@ class RemoteServerService {
}
/** Re-establish forwards + status for remote stacks that survived the last
- * quit, so their desktop reachability is restored on launch. Best-effort per
- * host: an unreachable host is left `stopped` rather than failing boot. */
- async initialize(): Promise<void> {
+ * quit, so their desktop reachability is restored on launch, and upgrade the
+ * ones that are behind. Hosts reconcile independently and in the background;
+ * an unreachable one is left `stopped` rather than failing boot.
+ *
+ * Memoised: {@link ensureReady} awaits the same call. It resolves once every
+ * host's reconcile is registered, not once they are done. */
+ initialize(): Promise<void> {
+ this.initialization ??= this.startReconciling();
+ return this.initialization;
+ }
+
+ private async startReconciling(): Promise<void> {
+ // Registered before the first await, so the host's reconcile is in flight
+ // before anything else that reacts to the same recovery asks whether the
+ // server is ready.
hostReachabilityService.on('change', ({ current }: HostReachabilityChange) => {
- if (current.status === 'reachable') void this.onHostReachable(current.sshHost);
+ if (current.status !== 'reachable') return;
+ void this.track(current.sshHost, () => this.onHostReachable(current.sshHost)).catch(
+ (error: unknown) => {
+ log.warn(`remote-switch-server: reconcile after recovery failed for ${current.sshHost}`, {
+ error,
+ });
+ }
+ );
});
- for (const [sshHost, serverId] of await this.remoteHosts()) {
+ for (const [sshHost, server] of await this.remoteHosts()) {
this.statuses.set(sshHost, initialStatus(sshHost));
- await this.reconcileHost(sshHost, serverId);
+ void this.track(sshHost, () => this.reconcileHost(sshHost, server));
}
}
- private async remoteHosts(): Promise<Map<string, string>> {
+ private async remoteHosts(): Promise<Map<string, { id: string; name: string }>> {
const remotes = (await listManagedServers()).filter(
(s) => s.managementKind === 'remote' && s.sshHost
);
- return new Map(remotes.map((s) => [s.sshHost!, s.id]));
+ return new Map(remotes.map((s) => [s.sshHost!, { id: s.id, name: s.name }]));
+ }
+
+ /** Remember `run` as the host's operation in flight until it settles. */
+ private track<T>(sshHost: string, run: () => Promise<T>): Promise<T> {
+ const promise = run();
+ const settled = promise.then(
+ () => undefined,
+ () => undefined
+ );
+ this.operations.set(sshHost, settled);
+ void settled.then(() => {
+ if (this.operations.get(sshHost) === settled) this.operations.delete(sshHost);
+ });
+ return promise;
+ }
+
+ /**
+ * Resolves once sessions may run against the stack on `sshHost`, waiting out
+ * its reconcile and any upgrade in flight. Throws when it still owes an
+ * upgrade — stopped, or failed — with the reason to show.
+ */
+ async ensureReady(sshHost: string, serverName: string): Promise<void> {
+ await this.initialize();
+ for (let op = this.operations.get(sshHost); op; op = this.operations.get(sshHost)) await op;
+ const upgrade = this.getStatus(sshHost).upgrade;
+ if (upgrade === null) return;
+ if (upgrade.state === 'updating') {
+ throw new Error(`${serverName} reports an update in progress, but none is running.`);
+ }
+ this.refused.add(sshHost);
+ throw new Error(managedServerUpgradeBlockedReason(serverName, upgrade));
+ }
+
+ /** Called with the server id when an upgrade finishes that sessions or
+ * watchers were turned away for (a failed one retried, or a stopped one
+ * started). Those that waited instead carry on by themselves. */
+ onUpgradeFinished(listener: (serverId: string) => void): void {
+ this.upgradeListeners.add(listener);
}
/** Pick a host's stack back up once its host is reachable again, so a
- * recovered host resumes without the user restarting anything. */
+ * recovered host resumes — and catches up on an owed upgrade — without the
+ * user restarting anything. */
private async onHostReachable(sshHost: string): Promise<void> {
if (this.busy.has(sshHost) || this.hosts.has(sshHost)) return;
- const serverId = (await this.remoteHosts()).get(sshHost);
- if (!serverId) return;
- await this.reconcileHost(sshHost, serverId);
+ const server = (await this.remoteHosts()).get(sshHost);
+ if (!server) return;
+ await this.reconcileHost(sshHost, server);
}
/** Adopt an already-running remote stack: re-open its forward and mark it
@@ -122,23 +201,47 @@ class RemoteServerService {
* call back through {@link onHostReachable} when it recovers.
*
* Also records how the host's deployed switch-core compares to this build's
- * pin, so an app update that moved the pin surfaces as drift rather than
- * leaving the host on a stale core (CHOO-1736). That check runs even when the
- * stack is down: its data volumes still hold the schema the last version
- * migrated to, which is what makes a downgrade unsafe. */
- private async reconcileHost(sshHost: string, serverId: string): Promise<void> {
+ * pin (CHOO-1736). A running stack that is behind, or one whose last upgrade
+ * was interrupted, is upgraded instead of adopted; a stopped one that is
+ * behind is marked to be upgraded at its next start. The check runs even
+ * when the stack is down: its data volumes still hold the schema the last
+ * version migrated to, which is what makes a downgrade unsafe. */
+ private async reconcileHost(
+ sshHost: string,
+ server: { id: string; name: string }
+ ): Promise<void> {
if (hostReachabilityService.isBlocked(sshHost)) return;
let host: RemoteServerHost | null = null;
let adopted = false;
+ let upgradeNow = false;
try {
host = await createRemoteServerHost(sshHost);
- if (await isStackRunning(host)) {
+ const running = await isStackRunning(host);
+ const version = await readVersionStatus(host, COMPATIBLE_SWITCH_VERSION);
+ let journal: UpgradeJournal | null;
+ try {
+ journal = await readUpgradeJournal(host);
+ } catch (error) {
+ this.setStatus(sshHost, {
+ ...version,
+ upgrade: {
+ state: 'failed',
+ from: version.deployedVersion ?? 'an unknown version',
+ to: COMPATIBLE_SWITCH_VERSION,
+ error: error instanceof Error ? error.message : String(error),
+ },
+ });
+ return;
+ }
+ const owed = owedUpgrade(version.drift, journal);
+ upgradeNow = owed !== null && (running || journal !== null);
+ if (running && !upgradeNow) {
const ports = await readPersistedPorts(host);
if (ports) {
await host.establishNetworking(ports);
this.hosts.set(sshHost, host);
adopted = true;
- this.setStatus(sshHost, { phase: 'running', serverId });
+ this.setStatus(sshHost, { phase: 'running', serverId: server.id });
// Only for a running stack: a stopped one sends nothing, so it
// cannot be out of step with the user's answer.
this.setStatus(sshHost, { deployedTelemetry: await readDeployedTelemetry(host) });
@@ -146,16 +249,48 @@ class RemoteServerService {
// Running but we don't know its ports — leave it stopped; the user can
// restart to re-derive them rather than forward to the wrong ports.
}
- this.setStatus(sshHost, await readVersionStatus(host, COMPATIBLE_SWITCH_VERSION));
+ this.setStatus(sshHost, { ...version, upgrade: owed && upgradeState(owed, upgradeNow) });
} catch (error) {
log.warn(`remote-switch-server: boot reconcile failed for ${sshHost}`, { error });
} finally {
// The adopted host owns the live port-forward; anything else is throwaway.
if (!adopted) host?.dispose();
}
+ if (!upgradeNow) return;
+ try {
+ await this.beginStart(sshHost, server.name, false);
+ } catch (error) {
+ // Only the reachability check throws rather than reporting a result.
+ log.warn(`remote-switch-server: could not upgrade ${sshHost}`, { error });
+ this.failUpgrade(sshHost, error instanceof Error ? error.message : String(error));
+ }
+ }
+
+ /** Start (or restart) the host's stack at this build's pin, upgrading it
+ * first if it is behind. Joins a start already in flight for the host. */
+ start(sshHost: string, serverName: string): Promise<StartLocalServerResult> {
+ return this.track(sshHost, () => this.beginStart(sshHost, serverName, true));
}
- async start(sshHost: string, serverName: string): Promise<StartLocalServerResult> {
+ private beginStart(
+ sshHost: string,
+ serverName: string,
+ activate: boolean
+ ): Promise<StartLocalServerResult> {
+ const inFlight = this.starting.get(sshHost);
+ if (inFlight) return inFlight;
+ const run = this.runStart(sshHost, serverName, activate).finally(() => {
+ this.starting.delete(sshHost);
+ });
+ this.starting.set(sshHost, run);
+ return run;
+ }
+
+ private async runStart(
+ sshHost: string,
+ serverName: string,
+ activate: boolean
+ ): Promise<StartLocalServerResult> {
if (this.busy.has(sshHost)) {
return { kind: 'error', message: `An operation is already in progress for ${sshHost}.` };
}
@@ -179,13 +314,16 @@ class RemoteServerService {
host,
ref: { kind: 'remote', sshHost },
serverName,
+ activate,
onMessage: (message) => this.setStatus(sshHost, { message }),
onLog: (line) => events.emit(remoteServerLogChannel, { sshHost, line }),
+ onUpgrade: (owed) => this.setStatus(sshHost, { upgrade: upgradeState(owed, true) }),
signal: abort.signal,
checkoutRoot: null,
});
if (result.kind === 'docker-unavailable') {
this.setStatus(sshHost, { phase: 'error', error: result.detail });
+ this.failUpgrade(sshHost, result.detail);
host.dispose();
} else if (result.kind === 'version-downgrade') {
this.setStatus(sshHost, {
@@ -194,17 +332,21 @@ class RemoteServerService {
error: switchVersionDowngradeMessage(result.deployed, result.expected),
deployedVersion: result.deployed,
drift: { deployed: result.deployed, expected: result.expected, direction: 'downgrade' },
+ upgrade: null,
});
host.dispose();
} else if (result.kind === 'matrix-migration-failed') {
+ const error = matrixMigrationFailedMessage(result.deployed, result.expected);
this.setStatus(sshHost, {
phase: 'error',
message: null,
- error: matrixMigrationFailedMessage(result.deployed, result.expected),
+ error,
deployedVersion: result.deployed,
});
+ this.failUpgrade(sshHost, error);
} else if (result.kind === 'error') {
this.setStatus(sshHost, { phase: 'error', error: result.message });
+ this.failUpgrade(sshHost, result.message);
host.dispose();
} else {
// Keep the host alive — it owns the port-forward.
@@ -218,8 +360,12 @@ class RemoteServerService {
error: null,
deployedVersion: COMPATIBLE_SWITCH_VERSION,
drift: null,
+ upgrade: null,
deployedTelemetry: { known: true, enabled: result.telemetryEnabled },
});
+ if (this.refused.delete(sshHost)) {
+ for (const listener of this.upgradeListeners) listener(result.serverId);
+ }
}
reportManagedServerStart('remote', result);
return result;
@@ -228,6 +374,7 @@ class RemoteServerService {
const message = error instanceof Error ? error.message : String(error);
log.error(`remote-switch-server: start failed for ${sshHost}`, { error });
this.setStatus(sshHost, { phase: 'error', error: message });
+ this.failUpgrade(sshHost, message);
reportManagedServerStartThrew('remote');
return { kind: 'error', message };
} finally {
@@ -236,6 +383,16 @@ class RemoteServerService {
}
}
+ /** Record why a host's upgrade did not finish. A start that was not an
+ * upgrade has nothing to record here; its error is on the status already. */
+ private failUpgrade(sshHost: string, error: string): void {
+ const upgrade = this.getStatus(sshHost).upgrade;
+ if (!upgrade) return;
+ this.setStatus(sshHost, {
+ upgrade: { state: 'failed', from: upgrade.from, to: upgrade.to, error },
+ });
+ }
+
async stop(sshHost: string): Promise<void> {
if (this.busy.has(sshHost))
throw new Error(`An operation is already in progress for ${sshHost}.`);
@@ -250,11 +407,13 @@ class RemoteServerService {
host = this.hosts.get(sshHost) ?? (await createRemoteServerHost(sshHost));
this.setStatus(sshHost, { phase: 'stopping', message: 'Stopping containers…' });
await stopStack(host);
+ const upgrade = this.getStatus(sshHost).upgrade;
this.setStatus(sshHost, {
phase: 'stopped',
message: null,
error: null,
deployedTelemetry: null,
+ upgrade: upgrade && upgradeState(upgrade, false),
});
reportManagedServerOutcome('stop', 'remote', 'success');
} catch (error) {
@@ -295,6 +454,9 @@ class RemoteServerService {
message: null,
error: null,
deployedTelemetry: null,
+ deployedVersion: null,
+ drift: null,
+ upgrade: null,
});
reportManagedServerOutcome('reset', 'remote', 'success');
} catch (error) {
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/resources/standalone-docker-compose.pinned.yml b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/resources/standalone-docker-compose.pinned.yml
index 5af90083f..5d1d13f64 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/resources/standalone-docker-compose.pinned.yml
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/resources/standalone-docker-compose.pinned.yml
@@ -220,9 +220,6 @@ services:
GATEWAY_ADMIN_PASSWORD: ${GATEWAY_ADMIN_PASSWORD}
FRONTEND_BASE_URL: ${FRONTEND_BASE_URL}
GATEWAY_PUBLIC_URL: ${GATEWAY_PUBLIC_URL:-}
- # Off by default. The recorded session behind it posts a card into a
- # real channel with no session behind it, so it is opt-in per server.
- SESSION_DEMO_ENABLED: ${SESSION_DEMO_ENABLED:-}
MESSAGING_PUBLIC_URL: ${MESSAGING_PUBLIC_URL:-}
SLACK_APP_CLIENT_ID: ${SLACK_APP_CLIENT_ID:-}
SLACK_APP_CLIENT_SECRET: ${SLACK_APP_CLIENT_SECRET:-}
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/session-readiness.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/session-readiness.ts
new file mode 100644
index 000000000..6e973696f
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/session-readiness.ts
@@ -0,0 +1,32 @@
+import type { SwitchServer } from '@shared/core/switch-servers/switch-servers';
+import { localServerService } from './local-server-service';
+import { remoteServerService } from './remote-server-service';
+
+/**
+ * The one gate between a managed Switch server and anything that runs agents
+ * against it — a session starting, or an agent's room watcher connecting.
+ *
+ * Resolves at once for a server Console only connects to, and for a managed
+ * one that is in step with this build. For a managed server Console is still
+ * checking or upgrading it waits until that has finished; for one that owes an
+ * upgrade it cannot run now (stopped, or a failed update) it throws the reason.
+ */
+export async function ensureServerSessionReady(server: SwitchServer): Promise<void> {
+ if (!server.managed) return;
+ if (server.managementKind === 'remote') {
+ // A remote row without a host is not one Console can supervise.
+ if (server.sshHost === null) return;
+ await remoteServerService.ensureReady(server.sshHost, server.name);
+ return;
+ }
+ await localServerService.ensureReady(server.name);
+}
+
+/**
+ * Called with a managed server's id when an upgrade that held its sessions
+ * back has finished, so whatever was refused can be started again.
+ */
+export function onManagedServerUpgraded(listener: (serverId: string) => void): void {
+ localServerService.onUpgradeFinished(listener);
+ remoteServerService.onUpgradeFinished(listener);
+}
diff --git a/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-agent-runtime.initial-prompt.test.ts b/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-agent-runtime.initial-prompt.test.ts
index 1e471a237..6f59eaaa9 100644
--- a/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-agent-runtime.initial-prompt.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-agent-runtime.initial-prompt.test.ts
@@ -15,6 +15,11 @@ const mocks = vi.hoisted(() => ({
readFailure: vi.fn(),
exec: vi.fn(),
specialization: vi.fn(),
+ ready: vi.fn(),
+}));
+
+vi.mock('@main/core/managed-switch-server/session-readiness', () => ({
+ ensureServerSessionReady: mocks.ready,
}));
vi.mock('./transcripts', () => ({ currentSnapshot: mocks.snapshot }));
@@ -105,6 +110,7 @@ beforeEach(() => {
serverId: 'server-1',
});
mocks.server.mockResolvedValue({ id: 'server-1' });
+ mocks.ready.mockResolvedValue(undefined);
mocks.exec.mockResolvedValue({ stdout: 'null' });
mocks.readFailure.mockResolvedValue(null);
// The launch reports an existing host, which is what a retry after a failed
@@ -356,3 +362,35 @@ it('passes the existing conversation room as a guarded migration hint', async ()
} as LocationTransport);
expect(other.roomConnection?.restoreRoomId).toBeUndefined();
});
+
+it('never starts a host or replays the initial prompt while the server is not ready', async () => {
+ mocks.ready.mockRejectedValueOnce(new Error('Updating Local from switch-core 0.1.0 failed'));
+ const agent = runtime();
+
+ await expect(agent.start(session, false, 'Do work')).rejects.toThrow('failed');
+ expect(mocks.ready).toHaveBeenCalledWith({ id: 'server-1' });
+ expect(mocks.runHost).not.toHaveBeenCalled();
+ expect(mocks.submit).not.toHaveBeenCalled();
+ expect(agent.startupStatus()).toMatchObject({ status: 'error' });
+});
+
+it('waits for the server’s update before starting the session', async () => {
+ let ready: () => void = () => {};
+ mocks.ready.mockReturnValueOnce(
+ new Promise<void>((resolve) => {
+ ready = resolve;
+ })
+ );
+ const agent = runtime();
+ const started = agent.start(session, false, 'Say hello');
+
+ await vi.waitFor(() => expect(mocks.ready).toHaveBeenCalled());
+ expect(agent.startupStatus()).toEqual({
+ status: 'starting',
+ message: 'Waiting for the Switch server to be ready…',
+ });
+ expect(mocks.runHost).not.toHaveBeenCalled();
+ ready();
+ await started;
+ expect(mocks.submit).toHaveBeenCalledTimes(1);
+});
diff --git a/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-agent-runtime.ts b/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-agent-runtime.ts
index 70f104222..14f2949a5 100644
--- a/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-agent-runtime.ts
+++ b/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-agent-runtime.ts
@@ -28,6 +28,7 @@ import { getAgentById } from '@main/core/agents/getAgentById';
import { agentSettingsRelativePath } from '@main/core/agents/switch-settings-paths';
import { hostDependencyStore } from '@main/core/dependencies/host-dependency-store';
import type { LocationTransport } from '@main/core/locations/location-transport';
+import { ensureServerSessionReady } from '@main/core/managed-switch-server/session-readiness';
import { getPlugin } from '@main/core/providers/plugin-registry';
import { AGENT_ENV_VARS } from '@main/core/sdk-host/agent-env';
import { setInitialPromptDelivery } from '@main/core/sessions/operations/set-initial-prompt-delivery';
@@ -126,6 +127,9 @@ export class SharedAgentRuntime implements AgentRuntimeProvider {
throw new Error('Link this agent to a Switch server before starting a session.');
this.server = await getServer(agent.serverId);
if (!this.server) throw new Error('The agent’s Switch server is missing.');
+ this.startupStage = 'Waiting for the Switch server to be ready…';
+ await ensureServerSessionReady(this.server);
+ this.startupStage = 'Preparing the session on its host…';
const intended = switchNotificationPoller.getSharedIntent(session.id, agent.switchAgentId);
const config = await buildSharedHostConfig(session, this.params, this.transport);
const previousEpoch = restart ? await journalEpoch(session.agentId, session.id) : null;
diff --git a/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-watcher.locality.test.ts b/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-watcher.locality.test.ts
index 2acdc4f45..893a4cf36 100644
--- a/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-watcher.locality.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-watcher.locality.test.ts
@@ -12,8 +12,15 @@ const mocks = vi.hoisted(() => ({
removeRoots: vi.fn(),
agentById: vi.fn(),
ssh: vi.fn(),
+ ready: vi.fn(),
+ server: vi.fn(),
}));
+vi.mock('@main/core/managed-switch-server/session-readiness', () => ({
+ ensureServerSessionReady: mocks.ready,
+}));
+vi.mock('@main/core/switch-servers/servers-store', () => ({ getServer: mocks.server }));
+
vi.mock('@main/core/switch-rooms/auto-session-store', () => ({
listStoppedControllerAgentIds: mocks.stopped,
listAutoSessionAgentIds: mocks.spawning,
@@ -75,7 +82,10 @@ beforeEach(() => {
name: 'scout',
switchAgentId: 'switch-agent-1',
providerId: 'claude',
+ serverId: 'server-1',
});
+ mocks.server.mockResolvedValue({ id: 'server-1', name: 'Local', managed: true });
+ mocks.ready.mockResolvedValue(undefined);
});
it('watches a local agent inside Console without deploying a host', async () => {
@@ -204,3 +214,48 @@ it('leaves a stopped controller off the air however auto-start is set', async ()
expect(mocks.stopLocal).toHaveBeenCalledWith('switch-agent-1');
expect(mocks.startLocal).not.toHaveBeenCalled();
});
+
+it('connects only once the agent’s managed server is ready', async () => {
+ mocks.location.mockResolvedValue({ id: 'local', dir: '/work', sshHost: null });
+ let ready: () => void = () => {};
+ mocks.ready.mockReturnValue(
+ new Promise<void>((resolve) => {
+ ready = resolve;
+ })
+ );
+ const connecting = configureSharedWatcher(
+ 'agent-1',
+ { connected: true, spawning: true },
+ 'restore'
+ );
+ await vi.waitFor(() =>
+ expect(mocks.ready).toHaveBeenCalledWith(expect.objectContaining({ id: 'server-1' }))
+ );
+ expect(mocks.startLocal).not.toHaveBeenCalled();
+ ready();
+ await connecting;
+ expect(mocks.startLocal).toHaveBeenCalled();
+});
+
+it('never connects to a server whose update failed', async () => {
+ mocks.location.mockResolvedValue({
+ id: 'remote',
+ dir: '/work',
+ sshHost: 'builder',
+ connectionId: 'connection-1',
+ });
+ mocks.ready.mockRejectedValue(new Error('Updating Local from switch-core 0.1.0 failed'));
+ await expect(
+ configureSharedWatcher('agent-1', { connected: true, spawning: true }, 'restore')
+ ).rejects.toThrow('failed');
+ expect(mocks.deploy).not.toHaveBeenCalled();
+ expect(mocks.startLocal).not.toHaveBeenCalled();
+});
+
+it('stands a watcher down without waiting for its server', async () => {
+ mocks.location.mockResolvedValue({ id: 'local', dir: '/work', sshHost: null });
+ mocks.ready.mockRejectedValue(new Error('update failed'));
+ await configureSharedWatcher('agent-1', { connected: false, spawning: false }, 'explicit');
+ expect(mocks.ready).not.toHaveBeenCalled();
+ expect(mocks.stopLocal).toHaveBeenCalledWith('switch-agent-1');
+});
diff --git a/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-watcher.ts b/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-watcher.ts
index 4af0da80e..dbc23e825 100644
--- a/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-watcher.ts
+++ b/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-watcher.ts
@@ -5,12 +5,14 @@ import { SshExecutionContext } from '@main/core/execution-context/ssh-execution-
import { locationManager } from '@main/core/locations/location-manager';
import { resolveSessionEnv } from '@main/core/locations/location-runtime-factory';
import { locationTransport, type LocationTransport } from '@main/core/locations/location-transport';
+import { ensureServerSessionReady } from '@main/core/managed-switch-server/session-readiness';
import { ensureSshConnected } from '@main/core/ssh/connect/connect-agent-ssh';
import {
listAutoSessionAgentIds,
listStoppedControllerAgentIds,
} from '@main/core/switch-rooms/auto-session-store';
import { controllerConnectionId } from '@main/core/switch-rooms/session-connection-id';
+import { getServer } from '@main/core/switch-servers/servers-store';
import { adoptSubagent } from './adopt-subagent';
import { stopLegacySidecar } from './legacy-sidecar';
import {
@@ -104,6 +106,12 @@ export async function configureSharedWatcher(
if (!state.connected) return;
throw new Error('Link the agent to Switch before it can hold a room connection.');
}
+ // Connecting waits for the agent's managed server to be in step with this
+ // build; standing down never does.
+ if (state.connected && agent.serverId) {
+ const server = await getServer(agent.serverId);
+ if (server) await ensureServerSessionReady(server);
+ }
const location = await getAgentLocation(agent);
const transport = locationTransport(location);
const identity = `watcher-${agent.switchAgentId}`;
diff --git a/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.test.ts b/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.test.ts
index 1a0ed1879..135c757d0 100644
--- a/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.test.ts
@@ -2,7 +2,9 @@ import { beforeEach, expect, it, vi } from 'vitest';
type Change = { current: { sshHost: string; status: string } };
const mocks = vi.hoisted(() => {
const listeners: ((change: Change) => void)[] = [];
+ const upgraded: ((serverId: string) => void)[] = [];
return {
+ upgraded,
apply: vi.fn(),
configure: vi.fn(),
dispose: vi.fn(async () => {}),
@@ -39,6 +41,9 @@ vi.mock('./auto-session-store', () => ({
setAutoSessionSubagent: vi.fn(),
}));
vi.mock('@main/lib/logger', () => ({ log: { error: vi.fn() } }));
+vi.mock('@main/core/managed-switch-server/session-readiness', () => ({
+ onManagedServerUpgraded: (listener: (serverId: string) => void) => mocks.upgraded.push(listener),
+}));
const { autoSessionWatcher } = await import('./auto-session-watcher');
beforeEach(() => {
@@ -163,3 +168,37 @@ it('stops everything it hosts when Console closes', async () => {
await autoSessionWatcher.dispose();
expect(mocks.dispose).toHaveBeenCalled();
});
+
+it('does not hold one server’s agents behind another server’s update', async () => {
+ mocks.agents.mockResolvedValue([
+ { id: 'updating', switchAgentId: 'switch-1', serverId: 'local' },
+ { id: 'elsewhere', switchAgentId: 'switch-2', serverId: 'other' },
+ ]);
+ let finish: () => void = () => {};
+ mocks.apply.mockImplementation(async (agentId: string) => {
+ if (agentId === 'updating')
+ await new Promise<void>((resolve) => {
+ finish = resolve;
+ });
+ });
+ const boot = autoSessionWatcher.initialize();
+
+ await vi.waitFor(() => expect(mocks.apply).toHaveBeenCalledWith('elsewhere', 'restore'));
+ finish();
+ await boot;
+});
+
+it('starts the controllers of a server once an update they were refused for finishes', async () => {
+ mocks.agents.mockResolvedValue([
+ { id: 'upgraded', switchAgentId: 'switch-1', serverId: 'local' },
+ { id: 'elsewhere', switchAgentId: 'switch-2', serverId: 'other' },
+ { id: 'unlinked', switchAgentId: null, serverId: 'local' },
+ ]);
+ await autoSessionWatcher.initialize();
+ mocks.apply.mockClear();
+
+ for (const listener of mocks.upgraded) listener('local');
+
+ await vi.waitFor(() => expect(mocks.apply).toHaveBeenCalledTimes(1));
+ expect(mocks.apply).toHaveBeenCalledWith('upgraded', 'restore');
+});
diff --git a/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.ts b/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.ts
index 03f98065b..1665e536f 100644
--- a/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.ts
+++ b/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.ts
@@ -1,11 +1,13 @@
import { getAgentLocation } from '@main/core/agents/agent-location';
import { getAgentById } from '@main/core/agents/getAgentById';
import { getAgents } from '@main/core/agents/getAgents';
+import { onManagedServerUpgraded } from '@main/core/managed-switch-server/session-readiness';
import type { HostReachabilityChange } from '@main/core/remote-hosts/host-reachability-service';
import { hostReachabilityService } from '@main/core/remote-hosts/production-host-reachability';
import { disposeLocalHosts } from '@main/core/sdk-host/local-host';
import { applyControllerState, configureSharedWatcher } from '@main/core/sdk-host/shared-watcher';
import { log } from '@main/lib/logger';
+import type { Agent } from '@shared/core/agents/agents';
import {
listAutoSessionAgentIds,
listAutoSessionSubagents,
@@ -13,8 +15,11 @@ import {
setAutoSessionSubagent,
} from './auto-session-store';
+type Subagent = { parentAgentId: string; name: string };
+
class AutoSessionWatcher {
private watchingHosts = false;
+ private watchingUpgrades = false;
private readonly recovering = new Map<string, { again: boolean }>();
/**
@@ -22,12 +27,58 @@ class AutoSessionWatcher {
* may start sessions: an agent is reachable because it exists, and the
* auto-start setting only decides what its controller does with a message it
* is addressed in.
+ *
+ * Each server's agents are brought up on their own, because a controller
+ * waits for its managed server to finish upgrading and that must not hold
+ * back the agents of every other server.
*/
async initialize(): Promise<void> {
this.watchHostRecovery();
+ this.watchServerUpgrades();
for (const agentId of await listAutoSessionAgentIds())
if (!(await getAgentById(agentId))) await setAutoSessionAgent(agentId, false);
- for (const agent of await getAgents()) {
+ const agents = await getAgents();
+ const subagents: Subagent[] = [];
+ for (const subagent of await listAutoSessionSubagents()) {
+ if (!(await getAgentById(subagent.parentAgentId))) {
+ await setAutoSessionSubagent(subagent.parentAgentId, subagent.name, false);
+ continue;
+ }
+ subagents.push(subagent);
+ }
+ const servers = new Set(agents.map((agent) => agent.serverId));
+ await Promise.all(
+ [...servers].map((serverId) => {
+ const members = agents.filter((agent) => agent.serverId === serverId);
+ return this.startControllers(members, subagentsOf(members, subagents));
+ })
+ );
+ }
+
+ /**
+ * Controllers refused while their managed server owed an upgrade — a failed
+ * one, or a stopped one — are started once it has finished.
+ */
+ private watchServerUpgrades(): void {
+ if (this.watchingUpgrades) return;
+ this.watchingUpgrades = true;
+ onManagedServerUpgraded((serverId) => void this.restoreServer(serverId));
+ }
+
+ private async restoreServer(serverId: string): Promise<void> {
+ try {
+ const members = (await getAgents()).filter((agent) => agent.serverId === serverId);
+ await this.startControllers(members, subagentsOf(members, await listAutoSessionSubagents()));
+ } catch (error) {
+ log.error('Shared SDK watchers could not start after their server updated', {
+ serverId,
+ error: String(error),
+ });
+ }
+ }
+
+ private async startControllers(agents: Agent[], subagents: Subagent[]): Promise<void> {
+ for (const agent of agents) {
if (!agent.switchAgentId) continue;
try {
// Restoring a controller an earlier run was already meant to be
@@ -42,11 +93,7 @@ class AutoSessionWatcher {
});
}
}
- for (const { parentAgentId, name } of await listAutoSessionSubagents()) {
- if (!(await getAgentById(parentAgentId))) {
- await setAutoSessionSubagent(parentAgentId, name, false);
- continue;
- }
+ for (const { parentAgentId, name } of subagents) {
try {
await this.startForSubagent(parentAgentId, name);
} catch (error) {
@@ -144,4 +191,9 @@ class AutoSessionWatcher {
return disposeLocalHosts();
}
}
+function subagentsOf(parents: Agent[], subagents: Subagent[]): Subagent[] {
+ const ids = new Set(parents.map((agent) => agent.id));
+ return subagents.filter((subagent) => ids.has(subagent.parentAgentId));
+}
+
export const autoSessionWatcher = new AutoSessionWatcher();
diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/VersionDriftNotice.tsx b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/VersionDriftNotice.tsx
index a03090307..d7d0a2c7b 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/VersionDriftNotice.tsx
+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/VersionDriftNotice.tsx
@@ -1,19 +1,23 @@
-import { ArrowUpCircle, RefreshCw, TriangleAlert } from 'lucide-react';
+import { ArrowUpCircle, Play, RefreshCw, TriangleAlert } from 'lucide-react';
import { Alert, AlertAction, AlertDescription, AlertTitle } from '@renderer/lib/ui/alert';
import { Button } from '@renderer/lib/ui/button';
+import { Spinner } from '@renderer/lib/ui/spinner';
import {
+ type ManagedServerUpgrade,
type SwitchVersionDrift,
switchVersionDowngradeMessage,
} from '@shared/core/managed-switch-server/managed-switch-server';
+const RESUME_NOTE = 'Sessions on this server resume once the update finishes.';
+
/**
* Surfaces a managed stack whose switch-core version no longer matches the one
* this build of Switch Console pins (CHOO-1736).
*
- * Updating the app moves the pin, but a stack that is already up is never
- * re-provisioned — without this the user silently keeps running the old core.
- * Restarting re-runs the start pipeline, which rewrites the `.env` at the new
- * pin and lets `compose up -d` recreate the changed containers.
+ * A stack that is behind the pin is upgraded by Console on its own, so for that
+ * case this is a status rather than an offer: the update in progress, the
+ * error it failed with (and a retry), or — for a stopped stack — that starting
+ * it will update it. Sessions on the server wait for it either way.
*
* The downgrade direction gets no action, only an explanation: the stack's
* database has already migrated forward and switch-core cannot roll back, so
@@ -21,14 +25,31 @@ import {
*/
export function VersionDriftNotice({
drift,
+ upgrade,
+ progress,
disabled,
onRestart,
}: {
drift: SwitchVersionDrift | null;
+ /** The upgrade the stack owes this build, if any. */
+ upgrade: ManagedServerUpgrade | null;
+ /** The current step of a start in flight ("Pulling images…"), or null. */
+ progress: string | null;
/** True while a lifecycle operation is in flight (or the host is unreachable). */
disabled: boolean;
+ /** Restart the stack; for an owed upgrade this is what runs it. */
onRestart: () => void;
}) {
+ if (upgrade) {
+ return (
+ <ServerUpgradeNotice
+ upgrade={upgrade}
+ progress={progress}
+ disabled={disabled}
+ onRetry={onRestart}
+ />
+ );
+ }
if (!drift) return null;
// We could not read what is deployed. No action offered: restarting on the
@@ -58,17 +79,26 @@ export function VersionDriftNotice({
);
}
- const unknown = drift.direction === 'unknown';
+ // Behind, but no update recorded yet: the check that starts it is still
+ // running.
+ if (drift.direction === 'upgrade') {
+ return (
+ <Alert variant="warning">
+ <ArrowUpCircle className="size-4" />
+ <AlertTitle>{`switch-core ${drift.expected} is required`}</AlertTitle>
+ <AlertDescription>
+ {`Still on switch-core ${drift.deployed}. Switch Console updates this server before running sessions on it. ${RESUME_NOTE}`}
+ </AlertDescription>
+ </Alert>
+ );
+ }
+
return (
<Alert variant="warning">
- {unknown ? <TriangleAlert className="size-4" /> : <ArrowUpCircle className="size-4" />}
- <AlertTitle>
- {unknown ? 'Version mismatch' : `switch-core ${drift.expected} is available`}
- </AlertTitle>
+ <TriangleAlert className="size-4" />
+ <AlertTitle>Version mismatch</AlertTitle>
<AlertDescription>
- {unknown
- ? `Runs switch-core ${drift.deployed}; this app expects ${drift.expected}. Can't tell which is newer — restart only if ${drift.expected} isn't older.`
- : `Still on switch-core ${drift.deployed}. Restart to pull ${drift.expected} and migrate; rooms and messages are kept.`}
+ {`Runs switch-core ${drift.deployed}; this app expects ${drift.expected}. Can't tell which is newer — restart only if ${drift.expected} isn't older.`}
</AlertDescription>
<AlertAction>
<Button size="sm" disabled={disabled} onClick={onRestart}>
@@ -79,3 +109,63 @@ export function VersionDriftNotice({
</Alert>
);
}
+
+/** The state of an upgrade Console is running (or owes) for a managed stack. */
+export function ServerUpgradeNotice({
+ upgrade,
+ progress,
+ disabled,
+ onRetry,
+}: {
+ upgrade: ManagedServerUpgrade;
+ progress: string | null;
+ disabled: boolean;
+ onRetry: () => void;
+}) {
+ const versions = `switch-core ${upgrade.from} → ${upgrade.to}`;
+ if (upgrade.state === 'updating') {
+ return (
+ <Alert role="status" aria-live="polite">
+ <Spinner size="sm" />
+ <AlertTitle>{`Updating ${versions}…`}</AlertTitle>
+ <AlertDescription>
+ <p>{`${RESUME_NOTE} Rooms and messages are kept, and the database is backed up before it is migrated.`}</p>
+ {progress && <p className="text-foreground-muted">{progress}</p>}
+ </AlertDescription>
+ </Alert>
+ );
+ }
+ if (upgrade.state === 'failed') {
+ return (
+ <Alert variant="destructive" role="alert">
+ <TriangleAlert className="size-4" />
+ <AlertTitle>{`Updating ${versions} failed`}</AlertTitle>
+ <AlertDescription>
+ <p className="break-words">{upgrade.error}</p>
+ <p>Sessions on this server stay paused until the update succeeds.</p>
+ </AlertDescription>
+ <AlertAction>
+ <Button size="sm" disabled={disabled} onClick={onRetry}>
+ <RefreshCw className="size-4" />
+ Retry
+ </Button>
+ </AlertAction>
+ </Alert>
+ );
+ }
+ return (
+ <Alert variant="warning">
+ <ArrowUpCircle className="size-4" />
+ <AlertTitle>{`switch-core ${upgrade.to} is required`}</AlertTitle>
+ <AlertDescription>
+ {`This server is stopped on switch-core ${upgrade.from}. Starting it updates it to ${upgrade.to} first. ${RESUME_NOTE}`}
+ </AlertDescription>
+ <AlertAction>
+ <Button size="sm" disabled={disabled} onClick={onRetry}>
+ <Play className="size-4" />
+ Start and update
+ </Button>
+ </AlertAction>
+ </Alert>
+ );
+}
diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/local-server-store.ts b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/local-server-store.ts
index 1bfc55600..4210c9e7e 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/local-server-store.ts
+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/local-server-store.ts
@@ -7,6 +7,7 @@ import type {
DeployedTelemetry,
DockerAvailability,
LocalServerStatus,
+ ManagedServerUpgrade,
SwitchVersionDrift,
} from '@shared/core/managed-switch-server/managed-switch-server';
import {
@@ -83,6 +84,11 @@ export class LocalServerStore {
return this.status?.drift ?? null;
}
+ /** Set while the stack is behind this build's pin and not yet upgraded. */
+ get upgrade(): ManagedServerUpgrade | null {
+ return this.status?.upgrade ?? null;
+ }
+
/** What the running stack is doing about usage data, or null when nothing is
* up to be doing anything. */
get deployedTelemetry(): DeployedTelemetry | null {
diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/remote-server-store.ts b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/remote-server-store.ts
index 39d52c0d3..c5f2972a0 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/remote-server-store.ts
+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/remote-server-store.ts
@@ -6,6 +6,7 @@ import { events, rpc } from '@renderer/lib/ipc';
import type {
DeployedTelemetry,
DockerAvailability,
+ ManagedServerUpgrade,
SwitchVersionDrift,
} from '@shared/core/managed-switch-server/managed-switch-server';
import {
@@ -25,6 +26,7 @@ function defaultStatus(sshHost: string): RemoteServerStatus {
return {
sshHost,
phase: 'stopped',
+ upgrade: null,
serverId: null,
version: '',
deployedVersion: null,
@@ -129,6 +131,11 @@ export class RemoteServerStore {
return this.statusFor(sshHost).drift;
}
+ /** Set while the host's stack is behind this build's pin and not yet upgraded. */
+ upgradeFor(sshHost: string): ManagedServerUpgrade | null {
+ return this.statusFor(sshHost).upgrade;
+ }
+
/** What the host's running stack is doing about usage data, or null when
* nothing is up there to be doing anything. */
deployedTelemetryFor(sshHost: string): DeployedTelemetry | null {
diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/server-presentation.tsx b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/server-presentation.tsx
index 5964cb3fb..8bc48fecb 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/server-presentation.tsx
+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/server-presentation.tsx
@@ -5,6 +5,7 @@ import { Tooltip, TooltipContent, TooltipTrigger } from '@renderer/lib/ui/toolti
import { cn } from '@renderer/utils/utils';
import type {
DeployedTelemetry,
+ ManagedServerUpgrade,
SwitchVersionDrift,
} from '@shared/core/managed-switch-server/managed-switch-server';
import type { SwitchServer } from '@shared/core/switch-servers/switch-servers';
@@ -93,6 +94,24 @@ export function serverDrift(server: SwitchServer): SwitchVersionDrift | null {
return localServerStore.drift;
}
+/** The upgrade a managed server's stack owes this build, or null. */
+export function serverUpgrade(server: SwitchServer): ManagedServerUpgrade | null {
+ if (!server.managed) return null;
+ if (server.managementKind === 'remote' && server.sshHost) {
+ return remoteServerStore.upgradeFor(server.sshHost);
+ }
+ return localServerStore.upgrade;
+}
+
+/** The current step of a managed server's lifecycle operation, or null. */
+export function serverProgress(server: SwitchServer): string | null {
+ if (!server.managed) return null;
+ if (server.managementKind === 'remote' && server.sshHost) {
+ return remoteServerStore.statusFor(server.sshHost).message;
+ }
+ return localServerStore.message;
+}
+
/**
* What a managed server's running stack is doing about usage data. Null for a
* server this app does not run: its operator decides, and the Console's toggle
@@ -180,8 +199,8 @@ export const ServerStatusPill = observer(function ServerStatusPill({
/**
* Flags a managed server whose switch-core no longer matches the version this
- * build pins (CHOO-1736), so an available update is visible from the switcher
- * rather than only on the server's own page.
+ * build pins (CHOO-1736), so an owed update is visible from the switcher rather
+ * than only on the server's own page.
*
* Sits beside the connection dot rather than recolouring it: the dot answers
* "can I reach this server", which stays true of a server running a stale core.
@@ -189,7 +208,7 @@ export const ServerStatusPill = observer(function ServerStatusPill({
export function ServerDriftIndicator({ drift }: { drift: SwitchVersionDrift }) {
const upgrade = drift.direction === 'upgrade';
const label = upgrade
- ? `switch-core ${drift.expected} is available (running ${drift.deployed})`
+ ? `Needs switch-core ${drift.expected} (running ${drift.deployed}); sessions wait for the update`
: drift.direction === 'downgrade'
? `Runs switch-core ${drift.deployed} — newer than this app expects (${drift.expected})`
: drift.direction === 'unreadable'
diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/version-drift-notice.test.ts b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/version-drift-notice.test.ts
new file mode 100644
index 000000000..ec6b5bcdb
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/version-drift-notice.test.ts
@@ -0,0 +1,136 @@
+import { createElement } from 'react';
+import { renderToStaticMarkup } from 'react-dom/server';
+import { describe, expect, it, vi } from 'vitest';
+import type {
+ ManagedServerUpgrade,
+ SwitchVersionDrift,
+} from '@shared/core/managed-switch-server/managed-switch-server';
+import { VersionDriftNotice } from './VersionDriftNotice';
+
+const behind: SwitchVersionDrift = { deployed: '0.10.0', expected: '0.11.0', direction: 'upgrade' };
+
+function render(props: {
+ drift: SwitchVersionDrift | null;
+ upgrade: ManagedServerUpgrade | null;
+ progress?: string | null;
+ disabled?: boolean;
+}): string {
+ return renderToStaticMarkup(
+ createElement(VersionDriftNotice, {
+ progress: null,
+ disabled: false,
+ onRestart: vi.fn(),
+ ...props,
+ })
+ );
+}
+
+function text(html: string): string {
+ return html
+ .replace(/<[^>]+>/g, ' ')
+ .replace(/&#x27;/g, "'")
+ .replace(/\s+/g, ' ')
+ .trim();
+}
+
+describe('VersionDriftNotice for a server behind the pin', () => {
+ it('shows the update in progress, with its current step, and no action', () => {
+ const html = render({
+ drift: behind,
+ upgrade: { state: 'updating', from: '0.10.0', to: '0.11.0' },
+ progress: 'Backing up the database before updating switch-core 0.10.0 → 0.11.0…',
+ });
+
+ expect(text(html)).toContain('Updating switch-core 0.10.0 → 0.11.0…');
+ expect(text(html)).toContain('Sessions on this server resume once the update finishes.');
+ expect(text(html)).toContain('Backing up the database');
+ expect(html).toContain('role="status"');
+ expect(html).not.toContain('<button');
+ expect(text(html)).not.toContain('Restart to update');
+ });
+
+ it('shows a failed update with its error and a Retry button', () => {
+ const html = render({
+ drift: behind,
+ upgrade: {
+ state: 'failed',
+ from: '0.10.0',
+ to: '0.11.0',
+ error: 'pg_dumpall failed: No space left on device',
+ },
+ });
+
+ expect(text(html)).toContain('Updating switch-core 0.10.0 → 0.11.0 failed');
+ expect(text(html)).toContain('pg_dumpall failed: No space left on device');
+ expect(html).toContain('role="alert"');
+ expect(html.match(/<button[^>]*>/g)).toHaveLength(1);
+ expect(text(html)).toContain('Retry');
+ expect(html).not.toMatch(/<button[^>]*\sdisabled=""/);
+ });
+
+ it('disables Retry while another operation is in flight', () => {
+ const html = render({
+ drift: behind,
+ upgrade: { state: 'failed', from: '0.10.0', to: '0.11.0', error: 'boom' },
+ disabled: true,
+ });
+
+ expect(html).toMatch(/<button[^>]*\sdisabled=""/);
+ });
+
+ it('tells a stopped server it will update when started', () => {
+ const html = render({
+ drift: behind,
+ upgrade: { state: 'pending', from: '0.10.0', to: '0.11.0' },
+ });
+
+ expect(text(html)).toContain('switch-core 0.11.0 is required');
+ expect(text(html)).toContain('stopped on switch-core 0.10.0');
+ expect(text(html)).toContain('Start and update');
+ });
+
+ it('never offers the old optional restart for a server that is behind', () => {
+ const html = render({ drift: behind, upgrade: null });
+
+ expect(text(html)).toContain('switch-core 0.11.0 is required');
+ expect(text(html)).not.toContain('Restart to update');
+ expect(html).not.toContain('<button');
+ });
+});
+
+describe('VersionDriftNotice for the cases it does not upgrade', () => {
+ it('still explains a downgrade with no action', () => {
+ const html = render({
+ drift: { deployed: '0.12.0', expected: '0.11.0', direction: 'downgrade' },
+ upgrade: null,
+ });
+
+ expect(text(html)).toContain('This server is newer than Switch Console');
+ expect(html).not.toContain('<button');
+ });
+
+ it('still reports an unreadable version with no action', () => {
+ const html = render({
+ drift: { deployed: null, expected: '0.11.0', direction: 'unreadable', reason: 'daemon down' },
+ upgrade: null,
+ });
+
+ expect(text(html)).toContain("Can't tell which switch-core this is running");
+ expect(text(html)).toContain('daemon down');
+ expect(html).not.toContain('<button');
+ });
+
+ it('still offers a restart for an uncomparable version', () => {
+ const html = render({
+ drift: { deployed: 'nightly', expected: '0.11.0', direction: 'unknown' },
+ upgrade: null,
+ });
+
+ expect(text(html)).toContain('Version mismatch');
+ expect(text(html)).toContain('Restart to update');
+ });
+
+ it('renders nothing for a server in step', () => {
+ expect(render({ drift: null, upgrade: null })).toBe('');
+ });
+});
diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/view.tsx b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/view.tsx
index 6752580c4..7cb17f9c0 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/view.tsx
+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/view.tsx
@@ -43,6 +43,8 @@ import {
serverDeployedTelemetry,
serverDrift,
serverPlacementLabel,
+ serverProgress,
+ serverUpgrade,
} from './server-presentation';
import { ServerResetSection } from './server-reset-section';
import { ServerSectionTitlebar } from './server-section-titlebar';
@@ -285,6 +287,8 @@ const ServerMainPanel = observer(function ServerMainPanel() {
rather than waiting inside the stack card further down. */}
<VersionDriftNotice
drift={drift}
+ upgrade={serverUpgrade(server)}
+ progress={serverProgress(server)}
disabled={stackTransitioning}
onRestart={() => restartStack(server)}
/>
diff --git a/console/apps/switch-console-desktop/src/renderer/tests/browser/version-drift-notice-layout.test.tsx b/console/apps/switch-console-desktop/src/renderer/tests/browser/version-drift-notice-layout.test.tsx
index 7da6f9e3d..ee136a25e 100644
--- a/console/apps/switch-console-desktop/src/renderer/tests/browser/version-drift-notice-layout.test.tsx
+++ b/console/apps/switch-console-desktop/src/renderer/tests/browser/version-drift-notice-layout.test.tsx
@@ -7,8 +7,9 @@ import '@renderer/index.css';
/**
* `AlertAction` used to be positioned absolutely over a fixed `pr-18` gutter,
* which only clears the text while the action stays narrower than that gutter.
- * "Restart to update" is roughly twice as wide, so the drift description ran
- * underneath the button (CHOO-1736).
+ * "Restart to update" was roughly twice as wide, so the drift description ran
+ * underneath the button (CHOO-1736). The stopped server's "Start and update"
+ * action is as wide, and shares the layout.
*
* These assertions are on measured geometry rather than class strings: the bug
* was invisible at the markup level — every class was present and correct — and
@@ -28,6 +29,8 @@ async function renderNotice(deployed: string, expected: string): Promise<HTMLDiv
root.render(
<VersionDriftNotice
drift={{ direction: 'upgrade', deployed, expected }}
+ upgrade={{ state: 'pending', from: deployed, to: expected }}
+ progress={null}
disabled={false}
onRestart={() => {}}
/>
diff --git a/console/apps/switch-console-desktop/src/shared/core/managed-switch-server/managed-switch-server.ts b/console/apps/switch-console-desktop/src/shared/core/managed-switch-server/managed-switch-server.ts
index 4099d8949..c1b864c36 100644
--- a/console/apps/switch-console-desktop/src/shared/core/managed-switch-server/managed-switch-server.ts
+++ b/console/apps/switch-console-desktop/src/shared/core/managed-switch-server/managed-switch-server.ts
@@ -124,9 +124,42 @@ export type DeployedTelemetry =
| { known: true; enabled: boolean }
| { known: false; reason: string };
+/**
+ * A managed stack that is behind this build's pin and is being moved onto it.
+ *
+ * Console upgrades such a stack on its own, and sessions for agents on the
+ * server wait while it does, so the state is visible to both the renderer and
+ * the session gate:
+ *
+ * - `updating` — the database is being backed up or the stack restarted at the
+ * pin. Sessions wait for it to finish.
+ * - `failed` — the backup or the restart failed. Sessions are refused with
+ * `error` until a retry succeeds.
+ * - `pending` — the stack is stopped. It is upgraded when it is next started,
+ * never started at the old version.
+ */
+export type ManagedServerUpgrade =
+ | { state: 'updating'; from: string; to: string }
+ | { state: 'failed'; from: string; to: string; error: string }
+ | { state: 'pending'; from: string; to: string };
+
+/** Why a session cannot start on a managed server whose upgrade has not
+ * finished. Shared so the thrown error and the UI say the same thing. */
+export function managedServerUpgradeBlockedReason(
+ serverName: string,
+ upgrade: Exclude<ManagedServerUpgrade, { state: 'updating' }>
+): string {
+ if (upgrade.state === 'pending') {
+ return `${serverName} is stopped on switch-core ${upgrade.from} and needs switch-core ${upgrade.to}. Start it from the server's page to update it; sessions on it start once the update finishes.`;
+ }
+ return `Updating ${serverName} from switch-core ${upgrade.from} to ${upgrade.to} failed: ${upgrade.error} Retry the update from the server's page; sessions on it stay paused until it succeeds.`;
+}
+
/** Snapshot of the managed local server, emitted on every transition. */
export type LocalServerStatus = {
phase: LocalServerPhase;
+ /** Set while the stack is behind this build's pin and not yet upgraded. */
+ upgrade: ManagedServerUpgrade | null;
/** The registered server's id once the stack is up, else null. */
serverId: string | null;
/** The pinned switch-core version this build runs. */
diff --git a/deploy/local/standalone-docker-compose.yml b/deploy/local/standalone-docker-compose.yml
index ded42cedd..9cfa0cc22 100644
--- a/deploy/local/standalone-docker-compose.yml
+++ b/deploy/local/standalone-docker-compose.yml
@@ -213,9 +213,6 @@ services:
GATEWAY_ADMIN_PASSWORD: ${GATEWAY_ADMIN_PASSWORD}
FRONTEND_BASE_URL: ${FRONTEND_BASE_URL}
GATEWAY_PUBLIC_URL: ${GATEWAY_PUBLIC_URL:-}
- # Off by default. The recorded session behind it posts a card into a
- # real channel with no session behind it, so it is opt-in per server.
- SESSION_DEMO_ENABLED: ${SESSION_DEMO_ENABLED:-}
MESSAGING_PUBLIC_URL: ${MESSAGING_PUBLIC_URL:-}
SLACK_APP_CLIENT_ID: ${SLACK_APP_CLIENT_ID:-}
SLACK_APP_CLIENT_SECRET: ${SLACK_APP_CLIENT_SECRET:-}