Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
feat(console): upgrade managed servers behind the pin and hold sessions until done (#544)
Louis Amaudruz8ba5b69c36d5
diff --git a/.env.example b/.env.exampleindex df892245d..7b287ce79 100644--- a/.env.example+++ b/.env.example@@ -178,14 +178,6 @@ MATTERMOST_USER_PASSWORD=# through the gateway (all credentials live in the bridge's stored# connection_config), never in global config.-# ── Session activity and request cards (local dev) ───────────────────────────-# No agent host speaks the session interaction contract yet, so no session's-# activity can reach a channel. With this set, `!session-demo` in a bridged-# Slack channel posts a recorded turn there — what the agent did and said, and-# then the permission it stopped to ask for, as a real card — so the answer-# path can be exercised against a real workspace. Needs the repository-# checkout, for the fixtures.-# SESSION_DEMO_ENABLED=true# ── OIDC (optional — MCP OAuth token validation; bring your own IdP) ─────────# Enable OAuth/OIDC token auth on the MCP server by pointing at any standardsdiff --git a/CHANGELOG.md b/CHANGELOG.mdindex cfb3b0a5b..3e4fb5d33 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1317,6 +1317,19 @@ version of their own to them without also giving them a release of their own.### [Unreleased]+#### Changed+- **Managed servers behind the app's switch-core pin are upgraded+ automatically**, and sessions wait for it. The local server is upgraded at+ startup and a remote managed server when its host is reconciled (at startup+ and whenever the host becomes reachable); a stopped one is upgraded at its+ next start rather than started at the old version. Every database is backed up+ with `pg_dumpall` into a timestamped `backups/` directory next to the stack+ first, and an `upgrade.json` journal resumes an interrupted upgrade or refuses+ it loudly. Sessions and room watchers for agents on the server wait while it+ updates and are refused with the error if it fails. The server page shows the+ update's progress, or its error and a Retry button, in place of the optional+ "Restart to update" banner.+### [0.35.0] - 2026-09-23#### Addeddiff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/compose.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/compose.tsindex 380e9460c..211073f8e 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/compose.ts+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/compose.ts@@ -31,7 +31,11 @@ function baseArgs(host: ServerHost, globalFlags: string[], extraFiles: string[]];}-async function runDocker(host: ServerHost, args: string[], timeout: number): Promise<string> {+export async function runDocker(+ host: ServerHost,+ args: string[],+ timeout: number+): Promise<string> {const full = [host.dockerBin, ...args];try {const { stdout } = await host.ctx.exec(host.dockerBin, args, {@@ -136,6 +140,30 @@ export async function composeDown(host: ServerHost, removeVolumes: boolean): Pro);}+/**+ * Bring up one service of the stack as it is currently configured on the host+ * — the compose file and `.env` already there, not this build's — without its+ * dependents, and wait for it to report healthy.+ */+export async function composeUpService(host: ServerHost, service: string): Promise<void> {+ log.info(`local-switch-server: docker compose up ${service} (${host.label})`);+ await runDocker(+ host,+ [...baseArgs(host, []), 'up', '-d', '--no-deps', '--wait', service],+ 5 * 60 * 1000+ );+}++/** Id of the container running `service`. Throws when there is none. */+export async function serviceContainerId(host: ServerHost, service: string): Promise<string> {+ const stdout = await runDocker(host, [...baseArgs(host, []), 'ps', '-q', service], 60_000);+ const id = stdout.trim();+ if (!/^[a-f0-9]+$/.test(id)) {+ throw new Error(`Could not identify the ${service} container on ${host.label}.`);+ }+ return id;+}+/** Service names currently in the `running` state for the managed project. */export async function runningServices(host: ServerHost): Promise<string[]> {try {diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/env-file.test.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/env-file.test.tsindex 531a7598f..d527965aa 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/env-file.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/env-file.test.ts@@ -29,7 +29,6 @@ describe('buildEnvFile', () => {namespace: 'sandbox-quantum',ports: { gateway: 51000, api: 51001, mattermost: 51002, postgres: 51003 },secrets,- sessionDemo: false,telemetryEnabled: false,});const vars = Object.fromEntries(@@ -95,11 +94,6 @@ describe('buildEnvFile', () => {// An entry here must say why the stack is correct without it — leaving a// var unset is a decision, not a default.const intentionallyUnset = new Set<string>([- // The recorded session demo, which posts a card into a real channel with- // no session behind it. Off is the correct state for a released Console:- // it is written only for a stack built from a local checkout, and this- // `env` is the released one. Temporary — goes when the demo does.- 'SESSION_DEMO_ENABLED',// The four below configure switch-core as a distributed messaging app —// one app we own, installed by a customer into their own workspace, with// the platform posting events to URLs declared once in the app manifest.@@ -124,21 +118,8 @@ describe('buildEnvFile', () => {expect(missing, 'compose interpolates these but the .env does not set them').toEqual([]);});- it('turns the session demo on for a checkout build and off otherwise', () => {- // The .env is regenerated on every start, so a developer has nowhere to set- // this by hand — and a released Console runs pinned images that have never- // heard of it. Temporary, and it goes when the demo does.+ it('never passes the removed session demo switch, which older switch-core images reject when empty', () => {expect(env).not.toContain('SESSION_DEMO_ENABLED');- const fromCheckout = buildEnvFile({- version: 'checkout',- registry: 'ghcr.io',- namespace: 'sandbox-quantum',- ports: { gateway: 51000, api: 51001, mattermost: 51002, postgres: 51003 },- secrets,- sessionDemo: true,- telemetryEnabled: false,- });- expect(fromCheckout).toContain('SESSION_DEMO_ENABLED=true');});it('carries the telemetry answer in both directions, never by omission', () => {@@ -153,7 +134,6 @@ describe('buildEnvFile', () => {namespace: 'sandbox-quantum',ports: { gateway: 51000, api: 51001, mattermost: 51002, postgres: 51003 },secrets,- sessionDemo: false,telemetryEnabled: true,});expect(sharing).toContain('TELEMETRY_ENABLED=true');diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/env-file.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/env-file.tsindex d9565e687..a554c0415 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/env-file.ts+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/env-file.ts@@ -17,14 +17,6 @@ export type LocalServerEnvParams = {/** Free host ports chosen for this machine (published on 127.0.0.1). */ports: LocalServerPorts;secrets: LocalServerSecrets;- /**- * Turn on the server's recorded session demo (`!session-demo` in a bridged- * channel). True only for a stack built from a local checkout: this file is- * regenerated on every start, so there is nowhere for a developer to set it- * by hand, and a released Console runs pinned images that have never heard- * of it. **Temporary** — goes when the demo does.- */- sessionDemo: boolean;/*** The user's "Share usage data" answer, applied to the server as well as to* the app. One consent decision covers both, so this is the toggle's value@@ -44,7 +36,7 @@ export const TELEMETRY_ENABLED_KEY = 'TELEMETRY_ENABLED';* app's pin. Kept as a pure function so the exact output is unit-testable.*/export function buildEnvFile(params: LocalServerEnvParams): string {- const { version, registry, namespace, ports, secrets, sessionDemo, telemetryEnabled } = params;+ const { version, registry, namespace, ports, secrets, telemetryEnabled } = params;return ['# Generated by Switch Console local-server mode — do not edit by hand.','# Secrets originate in the app encrypted store; this file is derived.',@@ -86,7 +78,6 @@ export function buildEnvFile(params: LocalServerEnvParams): string {// served on the agent-bridge app, and this must match the `server` Switch Console// reports in the deeplink it is rewriting.`GATEWAY_PUBLIC_URL=${apiUrlFor(ports)}`,- ...(sessionDemo ? ['SESSION_DEMO_ENABLED=true'] : []),// Written on every start, in both directions. Omitting it when the answer// is no would leave a stack that was started under a yes still reporting,// because the compose file passes this key through only when the `.env`diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/local-host.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/local-host.tsindex a7a144c39..6bb96a871 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/local-host.ts+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/local-host.ts@@ -1,5 +1,5 @@import { execFile, spawn } from 'node:child_process';-import { chmod, mkdir, readFile, writeFile } from 'node:fs/promises';+import { chmod, mkdir, readFile, rm, writeFile } from 'node:fs/promises';import { dirname, join } from 'node:path';import { promisify } from 'node:util';import { LocalExecutionContext } from '@main/core/execution-context/local-execution-context';@@ -68,8 +68,7 @@ export class LocalServerHost implements ServerHost {await writeFile(path, content, mode !== undefined ? { encoding: 'utf8', mode } : 'utf8');if (mode === undefined) return;// writeFile only applies `mode` when creating; enforce it on rewrite too.- await chmod(path, mode);- if (process.platform === 'win32') await restrictWindowsFileToOwner(path);+ await this.restrictMode(relPath, mode);}async readFile(relPath: string): Promise<string | null> {@@ -81,6 +80,16 @@ export class LocalServerHost implements ServerHost {}}+ async removeFile(relPath: string): Promise<void> {+ await rm(join(this.workingDir, relPath), { force: true });+ }++ async restrictMode(relPath: string, mode: number): Promise<void> {+ const path = join(this.workingDir, relPath);+ await chmod(path, mode);+ if (process.platform === 'win32') await restrictWindowsFileToOwner(path);+ }+streamCommand(command: string,args: string[],diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/remote-host.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/remote-host.tsindex 622979b2c..75e0da953 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/remote-host.ts+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/remote-host.ts@@ -67,10 +67,16 @@ export class RemoteServerHost implements ServerHost {async writeFile(relPath: string, content: string, mode?: number): Promise<void> {await this.fs.write(relPath, content);- if (mode !== undefined) {- // ctx is rooted at workingDir, so the relative path resolves there.- await this.ctx.exec('chmod', [mode.toString(8).padStart(3, '0'), relPath]);- }+ if (mode !== undefined) await this.restrictMode(relPath, mode);+ }++ async removeFile(relPath: string): Promise<void> {+ await this.ctx.exec('rm', ['-f', relPath]);+ }++ async restrictMode(relPath: string, mode: number): Promise<void> {+ // ctx is rooted at workingDir, so the relative path resolves there.+ await this.ctx.exec('chmod', [mode.toString(8).padStart(3, '0'), relPath]);}async readFile(relPath: string): Promise<string | null> {diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/types.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/types.tsindex 991e377d2..2930ec0e4 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/types.ts+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/host/types.ts@@ -62,6 +62,13 @@ export interface ServerHost {* unreadable file is not the same as an absent one. */readFile(relPath: string): Promise<string | null>;+ /** Delete `relPath` under {@link workingDir}. An absent file is not an error. */+ removeFile(relPath: string): Promise<void>;++ /** Set the permission bits of `relPath` under {@link workingDir} (a file or a+ * directory), restricting it to the owner on Windows where there are none. */+ restrictMode(relPath: string, mode: number): Promise<void>;+/*** Run a command on the host streaming merged stdout+stderr line-by-line to* `onLine`, rooted at {@link workingDir}. Used for the slow `compose up`diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.test.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.test.tsnew file mode 100644index 000000000..4143140ef--- /dev/null+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.test.ts@@ -0,0 +1,257 @@+import { beforeEach, expect, it, vi } from 'vitest';+import type * as AppIdentity from '@shared/app-identity';+import type { StartLocalServerResult } from '@shared/core/managed-switch-server/managed-switch-server';+import type * as ManagedUpgrade from './managed-upgrade';+import type { StartStackOptions } from './pipeline';++const m = vi.hoisted(() => ({+ managed: vi.fn(),+ running: vi.fn(),+ version: vi.fn(),+ journal: vi.fn(),+ start: vi.fn<(opts: StartStackOptions) => Promise<StartLocalServerResult>>(),+ stop: vi.fn(),+ emit: vi.fn(),+}));+vi.mock('@shared/app-identity', async (importOriginal) => ({+ ...(await importOriginal<typeof AppIdentity>()),+ COMPATIBLE_SWITCH_VERSION: '0.11.0',+}));+vi.mock('@main/lib/events', () => ({ events: { emit: m.emit } }));+vi.mock('@main/lib/logger', () => ({ log: { warn: vi.fn(), error: vi.fn(), info: vi.fn() } }));+vi.mock('@main/core/telemetry/managed-server', () => ({+ reportManagedServerOutcome: vi.fn(),+ reportManagedServerStart: vi.fn(),+ reportManagedServerStartThrew: vi.fn(),+}));+vi.mock('@main/core/switch-servers/delete-server-agents', () => ({+ deleteAgentsForServer: vi.fn(),+}));+vi.mock('@main/core/switch-servers/servers-store', () => ({ getManagedServer: m.managed }));+vi.mock('./checkout-build', () => ({+ findCoreCheckout: () => null,+ isCheckoutBuildEnabled: vi.fn(),+ setCheckoutBuildEnabled: vi.fn(),+}));+vi.mock('./compose', () => ({ isStackRunning: m.running }));+vi.mock('./deployed-version', () => ({ readVersionStatus: m.version }));+vi.mock('./telemetry-consent', () => ({ readDeployedTelemetry: vi.fn() }));+vi.mock('./managed-upgrade', async (importOriginal) => ({+ ...(await importOriginal<typeof ManagedUpgrade>()),+ readUpgradeJournal: m.journal,+}));+vi.mock('./pipeline', () => ({ startStack: m.start, stopStack: m.stop, resetStack: vi.fn() }));+vi.mock('./host/local-host', () => ({+ LocalServerHost: class {+ dispose() {}+ },+}));++const { LocalServerService } = await import('./local-server-service');++/** Boot, and wait out whatever the boot check started. */+async function boot(service: InstanceType<typeof LocalServerService>): Promise<void> {+ await service.initialize();+ await service.ensureReady('Local').catch(() => {});+}++const behind = {+ deployedVersion: '0.10.0',+ drift: { deployed: '0.10.0', expected: '0.11.0', direction: 'upgrade' },+};+const started: StartLocalServerResult = {+ kind: 'started',+ serverId: 'local',+ telemetryEnabled: false,+};++/** A start that runs the upgrade the way the pipeline does: announce, then work. */+function upgradingStart(result: StartLocalServerResult = started) {+ return async (opts: StartStackOptions) => {+ opts.onUpgrade({ from: '0.10.0', to: '0.11.0' });+ return result;+ };+}++beforeEach(() => {+ vi.resetAllMocks();+ m.managed.mockResolvedValue({ id: 'local', name: 'Local' });+ m.running.mockResolvedValue(true);+ m.version.mockResolvedValue(behind);+ m.journal.mockResolvedValue(null);+ m.start.mockImplementation(upgradingStart());+});++it('upgrades a running server that is behind at boot, without switching to it', async () => {+ const service = new LocalServerService();+ await boot(service);++ expect(m.start).toHaveBeenCalledOnce();+ expect(m.start).toHaveBeenCalledWith(expect.objectContaining({ activate: false }));+ expect(service.getStatus()).toMatchObject({ phase: 'running', upgrade: null, drift: null });+ await expect(service.ensureReady('Local')).resolves.toBeUndefined();+});++it('shows the update while it runs and holds sessions until it has finished', async () => {+ let finish: (result: StartLocalServerResult) => void = () => {};+ m.start.mockImplementation(+ (opts) =>+ new Promise((resolve) => {+ opts.onUpgrade({ from: '0.10.0', to: '0.11.0' });+ finish = resolve;+ })+ );+ const service = new LocalServerService();+ void service.initialize();+ let ready = false;+ const session = service.ensureReady('Local').then(() => {+ ready = true;+ });++ await vi.waitFor(() => expect(m.start).toHaveBeenCalledOnce());+ expect(service.getStatus().upgrade).toEqual({ state: 'updating', from: '0.10.0', to: '0.11.0' });+ expect(ready).toBe(false);++ finish(started);+ await session;+ expect(ready).toBe(true);+ expect(service.getStatus().upgrade).toBeNull();+});++it('marks a stopped server that is behind to update at its next start, never earlier', async () => {+ m.running.mockResolvedValue(false);+ const service = new LocalServerService();+ await boot(service);++ expect(m.start).not.toHaveBeenCalled();+ expect(service.getStatus().upgrade).toEqual({ state: 'pending', from: '0.10.0', to: '0.11.0' });+ await expect(service.ensureReady('Local')).rejects.toThrow(+ /Local is stopped on switch-core 0\.10\.0.*Start it/+ );++ const upgraded = vi.fn();+ service.onUpgradeFinished(upgraded);+ expect(await service.start()).toEqual(started);+ expect(m.start).toHaveBeenCalledWith(expect.objectContaining({ activate: true }));+ await expect(service.ensureReady('Local')).resolves.toBeUndefined();+ // Sessions were refused while it was stopped, so they are told it is back.+ expect(upgraded).toHaveBeenCalledExactlyOnceWith('local');+});++it('resumes an interrupted upgrade even when no container survived it', async () => {+ m.running.mockResolvedValue(false);+ m.version.mockResolvedValue({ deployedVersion: '0.11.0', drift: null });+ m.journal.mockResolvedValue({ from: '0.10.0', to: '0.11.0', backup: '/b' });+ const service = new LocalServerService();+ await boot(service);++ expect(m.start).toHaveBeenCalledOnce();+ expect(service.getStatus().upgrade).toBeNull();+});++it('surfaces a failed backup with its error and refuses sessions until a retry succeeds', async () => {+ m.start.mockImplementationOnce(async (opts) => {+ opts.onUpgrade({ from: '0.10.0', to: '0.11.0' });+ throw new Error('pg_dumpall failed: No space left on device');+ });+ const service = new LocalServerService();+ const upgraded = vi.fn();+ service.onUpgradeFinished(upgraded);+ await boot(service);++ expect(service.getStatus()).toMatchObject({+ phase: 'error',+ upgrade: {+ state: 'failed',+ from: '0.10.0',+ to: '0.11.0',+ error: 'pg_dumpall failed: No space left on device',+ },+ });+ await expect(service.ensureReady('Local')).rejects.toThrow(+ /Updating Local from switch-core 0\.10\.0 to 0\.11\.0 failed: pg_dumpall failed/+ );+ expect(upgraded).not.toHaveBeenCalled();++ await service.start();+ await expect(service.ensureReady('Local')).resolves.toBeUndefined();+ expect(upgraded).toHaveBeenCalledExactlyOnceWith('local');+});++it('records an upgrade that never turned healthy as failed', async () => {+ m.start.mockImplementation(+ upgradingStart({ kind: 'error', message: 'The server did not become healthy in time.' })+ );+ const service = new LocalServerService();+ await boot(service);++ expect(service.getStatus().upgrade).toMatchObject({+ state: 'failed',+ error: 'The server did not become healthy in time.',+ });+});++it('fails loudly on an upgrade journal it cannot read', async () => {+ m.journal.mockRejectedValue(new Error('journal cannot be read'));+ const service = new LocalServerService();+ await boot(service);++ expect(m.start).not.toHaveBeenCalled();+ await expect(service.ensureReady('Local')).rejects.toThrow('journal cannot be read');+});++it('neither upgrades nor holds sessions for a downgrade or an unreadable version', async () => {+ for (const drift of [+ { deployed: '0.12.0', expected: '0.11.0', direction: 'downgrade' },+ { deployed: 'nightly', expected: '0.11.0', direction: 'unknown' },+ { deployed: null, expected: '0.11.0', direction: 'unreadable', reason: 'daemon down' },+ ]) {+ m.version.mockResolvedValue({ deployedVersion: drift.deployed, drift });+ const service = new LocalServerService();+ await boot(service);+ expect(service.getStatus()).toMatchObject({ drift, upgrade: null });+ await expect(service.ensureReady('Local')).resolves.toBeUndefined();+ }+ expect(m.start).not.toHaveBeenCalled();+});++it('does nothing on an installation without a managed server', async () => {+ m.managed.mockResolvedValue(null);+ const service = new LocalServerService();+ await boot(service);++ expect(m.start).not.toHaveBeenCalled();+ expect(m.version).not.toHaveBeenCalled();+ await expect(service.ensureReady('Local')).resolves.toBeUndefined();+});++it('a Start click during the boot upgrade joins it instead of starting twice', async () => {+ let finish: (result: StartLocalServerResult) => void = () => {};+ m.start.mockImplementationOnce(+ (opts) =>+ new Promise((resolve) => {+ opts.onUpgrade({ from: '0.10.0', to: '0.11.0' });+ finish = resolve;+ })+ );+ const service = new LocalServerService();+ void service.initialize();+ await vi.waitFor(() => expect(m.start).toHaveBeenCalledOnce());++ const click = service.start();+ finish(started);+ expect(await click).toEqual(started);+ expect(m.start).toHaveBeenCalledOnce();+});++it('keeps an owed upgrade pending across a stop', async () => {+ m.start.mockImplementationOnce(upgradingStart({ kind: 'error', message: 'unhealthy' }));+ const service = new LocalServerService();+ await boot(service);+ await service.stop();++ expect(service.getStatus()).toMatchObject({+ phase: 'stopped',+ upgrade: { state: 'pending', from: '0.10.0', to: '0.11.0' },+ });+});diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.tsindex 688349450..7c683e056 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.ts+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.ts@@ -14,6 +14,7 @@ import {type DockerAvailability,type LocalServerStatus,type StartLocalServerResult,+ managedServerUpgradeBlockedReason,matrixMigrationFailedMessage,switchVersionDowngradeMessage,} from '@shared/core/managed-switch-server/managed-switch-server';@@ -31,6 +32,12 @@ import { LOCAL_SERVER_NAME } from './constants';import { readVersionStatus } from './deployed-version';import { LocalServerHost } from './host/local-host';import type { ServerHost } from './host/types';+import {+ owedUpgrade,+ readUpgradeJournal,+ type UpgradeJournal,+ upgradeState,+} from './managed-upgrade';import { resetStack, startStack, stopStack } from './pipeline';import { readDeployedTelemetry } from './telemetry-consent';@@ -43,10 +50,15 @@ import { readDeployedTelemetry } from './telemetry-consent';* Deliberately does NOT stop the containers on app quit — the local stack keeps* running so its rooms stay live while Switch Console is closed, matching the remote* sidecar model. `dispose()` only aborts an in-flight health wait.+ *+ * A stack behind this build's switch-core pin is upgraded on its own at boot+ * (or at its next start, if it is stopped), and {@link ensureReady} holds+ * sessions for its agents until that has finished.*/-class LocalServerService {+export class LocalServerService {private status: LocalServerStatus = {phase: 'stopped',+ upgrade: null,serverId: null,version: COMPATIBLE_SWITCH_VERSION,deployedVersion: null,@@ -59,6 +71,13 @@ class LocalServerService {private busy = false;private startAbort: AbortController | null = null;+ private initialization: Promise<void> | null = null;+ /** The start in flight, which an automatic upgrade and a Start click share. */+ private starting: Promise<StartLocalServerResult> | null = null;+ private readonly upgradeListeners = new Set<(serverId: string) => void>();+ /** Whether {@link ensureReady} has turned anything away since the last+ * upgrade finished, so that it can be told when the server is ready. */+ private refused = false;getStatus(): LocalServerStatus {return this.status;@@ -116,19 +135,31 @@ class LocalServerService {/** Reconcile status at boot so a stack that survived the last quit shows as* running without the user re-starting it, and so an app update that moved- * the switch-core pin underneath it surfaces as drift instead of leaving the- * user on a stale core indefinitely (CHOO-1736).+ * the switch-core pin underneath it is acted on rather than leaving the user+ * on a stale core (CHOO-1736): a running stack that is behind — or one whose+ * last upgrade was interrupted — is upgraded now; a stopped one is marked to+ * be upgraded when it is next started.** The drift probe also runs for a stopped stack: its data volumes still hold* whatever schema the last version migrated to, which is exactly what makes a- * downgrade unsafe. */- async initialize(): Promise<void> {+ * downgrade unsafe.+ *+ * Memoised: {@link ensureReady} awaits the same reconcile, then the upgrade+ * it started. */+ initialize(): Promise<void> {+ this.initialization ??= this.reconcile();+ return this.initialization;+ }++ private async reconcile(): Promise<void> {const host: ServerHost = new LocalServerHost();+ let upgradeNow = false;try {this.setStatus({ checkoutBuild: await this.readCheckoutBuild(host) });const managed = await getManagedServer();if (!managed) return;- if (await isStackRunning(host)) {+ const running = await isStackRunning(host);+ if (running) {this.setStatus({ phase: 'running', serverId: managed.id, message: null, error: null });// Only for a running stack: a stopped one sends nothing, so it cannot// be out of step with the user's answer.@@ -138,17 +169,75 @@ class LocalServerService {// A checkout build is deliberately not a comparable version, so the pin// comparison has nothing to say about it — the UI reports it as a// checkout build instead of as unexplained drift.- this.setStatus(- version.deployedVersion === CHECKOUT_IMAGE_TAG ? { ...version, drift: null } : version- );+ const drift = version.deployedVersion === CHECKOUT_IMAGE_TAG ? null : version.drift;+ let journal: UpgradeJournal | null;+ try {+ journal = await readUpgradeJournal(host);+ } catch (error) {+ this.setStatus({+ ...version,+ drift,+ upgrade: {+ state: 'failed',+ from: version.deployedVersion ?? 'an unknown version',+ to: COMPATIBLE_SWITCH_VERSION,+ error: error instanceof Error ? error.message : String(error),+ },+ });+ return;+ }+ const owed = owedUpgrade(drift, journal);+ upgradeNow = owed !== null && (running || journal !== null);+ this.setStatus({ ...version, drift, upgrade: owed && upgradeState(owed, upgradeNow) });} catch (error) {log.warn('local-switch-server: boot status reconcile failed', { error });} finally {host.dispose();}+ // Not awaited: the boot check is done once the upgrade is under way, and a+ // Start click or a session joins the upgrade through `starting`.+ if (upgradeNow) void this.beginStart(false);+ }++ /**+ * Resolves once sessions may run against this server, waiting out the boot+ * check and any upgrade in flight. Throws when the server still owes an+ * upgrade — stopped, or failed — with the reason to show.+ */+ async ensureReady(serverName: string): Promise<void> {+ await this.initialize();+ while (this.starting) await this.starting;+ const upgrade = this.status.upgrade;+ if (upgrade === null) return;+ if (upgrade.state === 'updating') {+ throw new Error(`${serverName} reports an update in progress, but none is running.`);+ }+ this.refused = true;+ throw new Error(managedServerUpgradeBlockedReason(serverName, upgrade));+ }++ /** Called with the server id when an upgrade finishes that sessions or+ * watchers were turned away for (a failed one retried, or a stopped one+ * started). Those that waited instead carry on by themselves. */+ onUpgradeFinished(listener: (serverId: string) => void): void {+ this.upgradeListeners.add(listener);}+ /** Start (or restart) the stack at this build's pin, upgrading it first if+ * it is behind. Joins a start already in flight, such as the boot upgrade. */async start(): Promise<StartLocalServerResult> {+ await this.initialize();+ return this.starting ?? this.beginStart(true);+ }++ private beginStart(activate: boolean): Promise<StartLocalServerResult> {+ this.starting ??= this.runStart(activate).finally(() => {+ this.starting = null;+ });+ return this.starting;+ }++ private async runStart(activate: boolean): Promise<StartLocalServerResult> {if (this.busy) {return { kind: 'error', message: 'A local-server operation is already in progress.' };}@@ -162,13 +251,16 @@ class LocalServerService {host,ref: { kind: 'local' },serverName: LOCAL_SERVER_NAME,+ activate,onMessage: (message) => this.setStatus({ message }),onLog: (line) => events.emit(localServerLogChannel, { line }),+ onUpgrade: (owed) => this.setStatus({ upgrade: upgradeState(owed, true) }),signal: this.startAbort.signal,checkoutRoot,});if (result.kind === 'docker-unavailable') {this.setStatus({ phase: 'error', error: result.detail });+ this.failUpgrade(result.detail);} else if (result.kind === 'version-downgrade') {this.setStatus({phase: 'error',@@ -176,16 +268,15 @@ class LocalServerService {error: switchVersionDowngradeMessage(result.deployed, result.expected),deployedVersion: result.deployed,drift: { deployed: result.deployed, expected: result.expected, direction: 'downgrade' },+ upgrade: null,});} else if (result.kind === 'matrix-migration-failed') {- this.setStatus({- phase: 'error',- message: null,- error: matrixMigrationFailedMessage(result.deployed, result.expected),- deployedVersion: result.deployed,- });+ const error = matrixMigrationFailedMessage(result.deployed, result.expected);+ this.setStatus({ phase: 'error', message: null, error, deployedVersion: result.deployed });+ this.failUpgrade(error);} else if (result.kind === 'error') {this.setStatus({ phase: 'error', error: result.message });+ this.failUpgrade(result.message);} else {// The pipeline just wrote this build's pin and converged the containers// onto it, so any drift the boot probe found is now resolved.@@ -196,8 +287,13 @@ class LocalServerService {error: null,deployedVersion: checkoutRoot !== null ? CHECKOUT_IMAGE_TAG : COMPATIBLE_SWITCH_VERSION,drift: null,+ upgrade: null,deployedTelemetry: { known: true, enabled: result.telemetryEnabled },});+ if (this.refused) {+ this.refused = false;+ for (const listener of this.upgradeListeners) listener(result.serverId);+ }}reportManagedServerStart('local', result);return result;@@ -205,6 +301,7 @@ class LocalServerService {const message = error instanceof Error ? error.message : String(error);log.error('local-switch-server: start failed', { error });this.setStatus({ phase: 'error', error: message });+ this.failUpgrade(message);reportManagedServerStartThrew('local');return { kind: 'error', message };} finally {@@ -214,6 +311,14 @@ class LocalServerService {}}+ /** Record why an upgrade did not finish. A start that was not an upgrade+ * has nothing to record here; its error is on the status already. */+ private failUpgrade(error: string): void {+ const upgrade = this.status.upgrade;+ if (!upgrade) return;+ this.setStatus({ upgrade: { state: 'failed', from: upgrade.from, to: upgrade.to, error } });+ }+async stop(): Promise<void> {if (this.busy) throw new Error('A local-server operation is already in progress.');this.busy = true;@@ -221,7 +326,14 @@ class LocalServerService {try {this.setStatus({ phase: 'stopping', message: 'Stopping containers…' });await stopStack(host);- this.setStatus({ phase: 'stopped', message: null, error: null, deployedTelemetry: null });+ const upgrade = this.status.upgrade;+ this.setStatus({+ phase: 'stopped',+ message: null,+ error: null,+ deployedTelemetry: null,+ upgrade: upgrade && upgradeState(upgrade, false),+ });reportManagedServerOutcome('stop', 'local', 'success');} catch (error) {this.setStatus({@@ -253,7 +365,15 @@ class LocalServerService {if (server) await deleteAgentsForServer(server.id);this.setStatus({ phase: 'stopping', message: 'Destroying containers and data…' });await resetStack(host);- this.setStatus({ phase: 'stopped', message: null, error: null, deployedTelemetry: null });+ this.setStatus({+ phase: 'stopped',+ message: null,+ error: null,+ deployedTelemetry: null,+ deployedVersion: null,+ drift: null,+ upgrade: null,+ });reportManagedServerOutcome('reset', 'local', 'success');} catch (error) {this.setStatus({diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/managed-upgrade.test.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/managed-upgrade.test.tsnew file mode 100644index 000000000..ff7212eb6--- /dev/null+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/managed-upgrade.test.ts@@ -0,0 +1,229 @@+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { dirname, join } from 'node:path';+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import type * as AppIdentity from '@shared/app-identity';+import type * as DeployedVersion from './deployed-version';+import type { ServerHost } from './host/types';++const deployed = vi.hoisted(() => vi.fn());+vi.mock('@shared/app-identity', async (importOriginal) => ({+ ...(await importOriginal<typeof AppIdentity>()),+ COMPATIBLE_SWITCH_VERSION: '0.11.0',+}));+vi.mock('./deployed-version', async (importOriginal) => ({+ ...(await importOriginal<typeof DeployedVersion>()),+ readDeployedVersion: deployed,+}));+vi.mock('@main/lib/logger', () => ({ log: { warn: vi.fn(), info: vi.fn(), error: vi.fn() } }));++const { finishUpgrade, owedUpgrade, prepareUpgrade, readUpgradeJournal, UPGRADE_JOURNAL_FILE } =+ await import('./managed-upgrade');++type Exec = (+ command: string,+ args: string[],+ opts?: unknown+) => Promise<{ stdout: string; stderr: string }>;++let directory: string;+let exec: ReturnType<typeof vi.fn<Exec>>;+let modes: Map<string, number>;++/** A host whose working dir is a temp directory and whose Docker is a fake+ * that writes a dump where `docker cp` is asked to put it. */+function fakeHost(kind: 'local' | 'remote'): ServerHost {+ return {+ kind,+ label: kind === 'local' ? 'this computer' : 'builder',+ workingDir: directory,+ composeProjectName: 'test-stack',+ dockerBin: 'docker',+ ctx: { exec },+ readFile: async (name: string) =>+ readFile(join(directory, name), 'utf8').catch((error: NodeJS.ErrnoException) => {+ if (error.code === 'ENOENT') return null;+ throw error;+ }),+ writeFile: async (name: string, value: string, mode?: number) => {+ await mkdir(dirname(join(directory, name)), { recursive: true });+ await writeFile(join(directory, name), value);+ if (mode !== undefined) modes.set(name, mode);+ },+ removeFile: async (name: string) => rm(join(directory, name), { force: true }),+ restrictMode: async (name: string, mode: number) => {+ modes.set(name, mode);+ },+ } as unknown as ServerHost;+}++beforeEach(async () => {+ directory = await mkdtemp(join(tmpdir(), 'switch-upgrade-test-'));+ modes = new Map();+ exec = vi.fn<Exec>(async (_command, args) => {+ if (args.includes('ps')) return { stdout: 'abc123\n', stderr: '' };+ if (args[0] === 'cp') await writeFile(join(directory, args[2]), '-- PostgreSQL database dump');+ return { stdout: '', stderr: '' };+ });+ deployed.mockResolvedValue({ kind: 'deployed', version: '0.10.0', source: 'container' });+ const host = fakeHost('local');+ await host.writeFile('.env', 'SWITCH_VERSION=0.10.0\n');+ await host.writeFile('standalone-docker-compose.yml', 'services: {}\n');+});++afterEach(async () => {+ await rm(directory, { recursive: true, force: true });+});++describe('prepareUpgrade', () => {+ it.each(['local', 'remote'] as const)(+ 'dumps the database and the old configuration before journalling the upgrade (%s host)',+ async (kind) => {+ const host = fakeHost(kind);+ const onUpgrade = vi.fn();+ const journal = await prepareUpgrade(host, null, vi.fn(), onUpgrade);++ expect(onUpgrade).toHaveBeenCalledWith({ from: '0.10.0', to: '0.11.0' });+ expect(journal).toMatchObject({ from: '0.10.0', to: '0.11.0' });+ expect(journal!.backup.startsWith(directory)).toBe(true);+ expect(JSON.parse((await host.readFile(UPGRADE_JOURNAL_FILE))!)).toEqual(journal);+ expect(await readFile(join(journal!.backup, 'database.sql'), 'utf8')).toContain('PostgreSQL');+ expect(await readFile(join(journal!.backup, '.env'), 'utf8')).toBe('SWITCH_VERSION=0.10.0\n');+ // The stack's own configuration is left for the start pipeline to rewrite.+ expect(await host.readFile('.env')).toBe('SWITCH_VERSION=0.10.0\n');++ const relative = journal!.backup.slice(directory.length + 1);+ expect(modes.get(relative)).toBe(0o700);+ expect(modes.get(`${relative}/database.sql`)).toBe(0o600);+ expect(modes.get(`${relative}/.env`)).toBe(0o600);++ const commands = exec.mock.calls.map(([, args]) => args);+ expect(commands[0]).toEqual(expect.arrayContaining(['up', '-d', '--no-deps', 'postgres']));+ expect(+ commands.some((args) => args[0] === 'exec' && args.join(' ').includes('pg_dumpall'))+ ).toBe(true);+ // The temporary dump is removed from the container afterwards.+ expect(commands.at(-1)).toEqual(expect.arrayContaining(['exec', 'abc123', 'rm', '-f']));+ }+ );++ it('does not journal an upgrade whose dump failed, and says why', async () => {+ const host = fakeHost('remote');+ exec.mockImplementation(async (_command, args) => {+ if (args.includes('ps')) return { stdout: 'abc123\n', stderr: '' };+ if (args[0] === 'exec' && args.join(' ').includes('pg_dumpall'))+ throw Object.assign(new Error('exit 1'), { stderr: 'pg_dumpall: No space left on device' });+ return { stdout: '', stderr: '' };+ });++ await expect(prepareUpgrade(host, null, vi.fn(), vi.fn())).rejects.toThrow(+ 'No space left on device'+ );+ expect(await host.readFile(UPGRADE_JOURNAL_FILE)).toBeNull();+ });++ it('refuses to back up a stack whose configuration is missing', async () => {+ const host = fakeHost('local');+ await host.removeFile('standalone-docker-compose.yml');++ await expect(prepareUpgrade(host, null, vi.fn(), vi.fn())).rejects.toThrow(+ 'standalone-docker-compose.yml is missing'+ );+ expect(exec).not.toHaveBeenCalled();+ expect(await host.readFile(UPGRADE_JOURNAL_FILE)).toBeNull();+ });++ it('resumes an interrupted upgrade without a second backup', async () => {+ const host = fakeHost('local');+ const first = await prepareUpgrade(host, null, vi.fn(), vi.fn());+ exec.mockClear();+ // By now the stack may already run the new version, half-migrated.+ deployed.mockResolvedValue({ kind: 'deployed', version: '0.11.0', source: 'container' });+ const onUpgrade = vi.fn();++ expect(await prepareUpgrade(host, null, vi.fn(), onUpgrade)).toEqual(first);+ expect(onUpgrade).toHaveBeenCalledWith({ from: '0.10.0', to: '0.11.0' });+ expect(exec).not.toHaveBeenCalled();+ });++ it('carries an interrupted upgrade from an older app on to this pin', async () => {+ const host = fakeHost('local');+ await host.writeFile(+ UPGRADE_JOURNAL_FILE,+ JSON.stringify({ from: '0.9.0', to: '0.10.0', backup: '/backups/older' })+ );++ expect(await prepareUpgrade(host, null, vi.fn(), vi.fn())).toEqual({+ from: '0.9.0',+ to: '0.11.0',+ backup: '/backups/older',+ });+ expect(JSON.parse((await host.readFile(UPGRADE_JOURNAL_FILE))!).to).toBe('0.11.0');+ expect(exec).not.toHaveBeenCalled();+ });++ it('refuses an interrupted upgrade only a newer app can finish', async () => {+ const host = fakeHost('local');+ await host.writeFile(+ UPGRADE_JOURNAL_FILE,+ JSON.stringify({ from: '0.11.0', to: '0.12.0', backup: '/backups/newer' })+ );++ await expect(prepareUpgrade(host, null, vi.fn(), vi.fn())).rejects.toThrow(+ /switch-core 0\.12\.0 or newer.*\/backups\/newer/+ );+ expect(exec).not.toHaveBeenCalled();+ });++ it.each([+ ['nothing deployed yet', { kind: 'absent' }],+ ['an unreadable version', { kind: 'unreadable', reason: 'daemon down' }],+ ['the same version', { kind: 'deployed', version: '0.11.0', source: 'container' }],+ ['a newer version', { kind: 'deployed', version: '0.12.0', source: 'container' }],+ ['an uncomparable version', { kind: 'deployed', version: 'nightly', source: 'env-file' }],+ ['a checkout build', { kind: 'deployed', version: 'dev-checkout', source: 'container' }],+ ])('is not an upgrade for %s', async (_label, version) => {+ deployed.mockResolvedValue(version);+ const onUpgrade = vi.fn();++ expect(await prepareUpgrade(fakeHost('local'), null, vi.fn(), onUpgrade)).toBeNull();+ expect(onUpgrade).not.toHaveBeenCalled();+ expect(exec).not.toHaveBeenCalled();+ });++ it('does not back up when building from a checkout', async () => {+ expect(await prepareUpgrade(fakeHost('local'), '/src/switch', vi.fn(), vi.fn())).toBeNull();+ expect(exec).not.toHaveBeenCalled();+ });+});++describe('the upgrade journal', () => {+ it('is gone once the upgrade finishes; the backup stays', async () => {+ const host = fakeHost('local');+ const journal = await prepareUpgrade(host, null, vi.fn(), vi.fn());+ await finishUpgrade(host);++ expect(await readUpgradeJournal(host)).toBeNull();+ expect(await readFile(join(journal!.backup, 'database.sql'), 'utf8')).toContain('PostgreSQL');+ });++ it('fails loudly when it cannot be read', async () => {+ const host = fakeHost('remote');+ await host.writeFile(UPGRADE_JOURNAL_FILE, '{"from": "0.10.0", "to"');++ await expect(readUpgradeJournal(host)).rejects.toThrow(/interrupted.*cannot be read/);+ });++ it('names the upgrade a stack owes', () => {+ const drift = { deployed: '0.10.0', expected: '0.11.0', direction: 'upgrade' } as const;+ expect(owedUpgrade(drift, null)).toEqual({ from: '0.10.0', to: '0.11.0' });+ expect(owedUpgrade(null, { from: '0.9.0', to: '0.10.0', backup: '/b' })).toEqual({+ from: '0.9.0',+ to: '0.11.0',+ });+ expect(+ owedUpgrade({ deployed: '0.12.0', expected: '0.11.0', direction: 'downgrade' }, null)+ ).toBeNull();+ expect(owedUpgrade(null, null)).toBeNull();+ });+});diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/managed-upgrade.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/managed-upgrade.tsnew file mode 100644index 000000000..8ca672435--- /dev/null+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/managed-upgrade.ts@@ -0,0 +1,233 @@+import { randomUUID } from 'node:crypto';+import { join, posix } from 'node:path';+import { z } from 'zod';+import { log } from '@main/lib/logger';+import { compareVersions } from '@main/lib/semver';+import { COMPATIBLE_SWITCH_VERSION } from '@shared/app-identity';+import {+ CHECKOUT_IMAGE_TAG,+ type ManagedServerUpgrade,+ type SwitchVersionDrift,+} from '@shared/core/managed-switch-server/managed-switch-server';+import { composeUpService, runDocker, serviceContainerId } from './compose';+import { COMPOSE_FILE_NAME, ENV_FILE_NAME } from './constants';+import { classifyVersionDrift, readDeployedVersion } from './deployed-version';+import type { ServerHost } from './host/types';++/**+ * Moving a managed stack forward onto this build's switch-core pin.+ *+ * switch-core migrates its database on boot and has no way back, so before a+ * start rewrites the stack at a newer pin the database is dumped next to the+ * stack, on whichever host runs it. A journal records that an upgrade is under+ * way: it is written once the backup is complete and removed once the stack is+ * healthy at the pin, so an upgrade interrupted anywhere in between is resumed+ * on the next start (without a second backup, which would capture a+ * half-migrated database) or refused loudly when this app cannot finish it.+ */++/** Journal of an upgrade in progress, in the host working dir. */+export const UPGRADE_JOURNAL_FILE = 'upgrade.json';+const BACKUPS_DIR = 'backups';+const DATABASE_SERVICE = 'postgres';+/** A dump of a large install over a slow disk can take a while. */+const DUMP_TIMEOUT_MS = 30 * 60 * 1000;++const journalSchema = z.object({ from: z.string(), to: z.string(), backup: z.string() });+export type UpgradeJournal = z.infer<typeof journalSchema>;++/** `from` → `to` of an upgrade a stack still owes this build. */+export type OwedUpgrade = { from: string; to: string };++function hostPath(host: ServerHost, relPath: string): string {+ return host.kind === 'local'+ ? join(host.workingDir, relPath)+ : posix.join(host.workingDir, relPath);+}++/**+ * The journal of an interrupted upgrade on `host`, or null when none is under+ * way. A journal that cannot be read throws: it means an upgrade was+ * interrupted and there is no telling how far it got.+ */+export async function readUpgradeJournal(host: ServerHost): Promise<UpgradeJournal | null> {+ const raw = await host.readFile(UPGRADE_JOURNAL_FILE);+ if (raw === null) return null;+ let parsed: unknown;+ try {+ parsed = JSON.parse(raw);+ } catch {+ parsed = null;+ }+ const journal = journalSchema.safeParse(parsed);+ if (!journal.success) {+ throw new Error(+ `An earlier switch-core update on ${host.label} was interrupted and its journal ` ++ `(${hostPath(host, UPGRADE_JOURNAL_FILE)}) cannot be read. Its database backup is under ` ++ `${hostPath(host, BACKUPS_DIR)}. Check the stack, then delete the journal to update again.`+ );+ }+ return journal.data;+}++/**+ * The upgrade a stack owes this build: the interrupted one its journal names,+ * or the one its deployed version being behind the pin calls for. Null when it+ * is in step, ahead, or not comparable — those keep their own handling.+ */+export function owedUpgrade(+ drift: SwitchVersionDrift | null,+ journal: UpgradeJournal | null+): OwedUpgrade | null {+ if (journal) return { from: journal.from, to: COMPATIBLE_SWITCH_VERSION };+ if (drift?.direction === 'upgrade') return { from: drift.deployed, to: drift.expected };+ return null;+}++/** The status of an owed upgrade: running now, or waiting for the next start. */+export function upgradeState(owed: OwedUpgrade, running: boolean): ManagedServerUpgrade {+ return { state: running ? 'updating' : 'pending', from: owed.from, to: owed.to };+}++/**+ * Get `host` ready to be rewritten at this build's pin: resume an interrupted+ * upgrade, or back the database up and journal a new one. Returns the journal,+ * or null when the start is not an upgrade (first install, same version, a+ * checkout build, or a version that is unreadable or not comparable — the+ * start pipeline guards those separately).+ *+ * `onUpgrade` fires before any work, so the caller can show the update and hold+ * sessions for the whole of it. Any failure throws, leaving the stack's+ * configuration untouched.+ */+export async function prepareUpgrade(+ host: ServerHost,+ checkoutRoot: string | null,+ onMessage: (message: string) => void,+ onUpgrade: (upgrade: OwedUpgrade) => void+): Promise<UpgradeJournal | null> {+ const pending = await readUpgradeJournal(host);+ if (pending) {+ const order = compareVersions(pending.to, COMPATIBLE_SWITCH_VERSION);+ if (order === null || order > 0) {+ throw new Error(+ `An interrupted update on ${host.label} was moving switch-core to ${pending.to}, which ` ++ `this app (switch-core ${COMPATIBLE_SWITCH_VERSION}) cannot finish. Install a Switch ` ++ `Console with switch-core ${pending.to} or newer. The database backup taken before ` ++ `that update is in ${pending.backup}.`+ );+ }+ onUpgrade({ from: pending.from, to: COMPATIBLE_SWITCH_VERSION });+ log.warn(+ `managed-switch-server: resuming the interrupted switch-core update on ${host.label} ` ++ `(${pending.from} → ${COMPATIBLE_SWITCH_VERSION}); backup in ${pending.backup}`+ );+ if (pending.to === COMPATIBLE_SWITCH_VERSION) return pending;+ const resumed = { ...pending, to: COMPATIBLE_SWITCH_VERSION };+ await writeJournal(host, resumed);+ return resumed;+ }+ if (checkoutRoot !== null) return null;+ const deployed = await readDeployedVersion(host);+ if (deployed.kind !== 'deployed' || deployed.version === CHECKOUT_IMAGE_TAG) return null;+ if (classifyVersionDrift(deployed.version, COMPATIBLE_SWITCH_VERSION)?.direction !== 'upgrade') {+ return null;+ }+ onUpgrade({ from: deployed.version, to: COMPATIBLE_SWITCH_VERSION });+ const backup = await backupDatabase(host, deployed.version, onMessage);+ const journal = { from: deployed.version, to: COMPATIBLE_SWITCH_VERSION, backup };+ await writeJournal(host, journal);+ return journal;+}++/** Record that the upgrade finished: the stack is healthy at the pin. */+export async function finishUpgrade(host: ServerHost): Promise<void> {+ await host.removeFile(UPGRADE_JOURNAL_FILE);+}++async function writeJournal(host: ServerHost, journal: UpgradeJournal): Promise<void> {+ await host.writeFile(UPGRADE_JOURNAL_FILE, `${JSON.stringify(journal, null, 2)}\n`, 0o600);+}++/**+ * Dump every database of the stack as it is deployed now, plus the compose+ * file and `.env` that describe it, into a new timestamped directory under+ * `backups/` in the host working dir. Returns that directory's path on the+ * host.+ *+ * The dump is written inside the database container and copied out with+ * `docker cp`, so nothing is buffered through Console or piped through a shell+ * that may not exist on the host; the same commands work locally and over SSH.+ */+async function backupDatabase(+ host: ServerHost,+ from: string,+ onMessage: (message: string) => void+): Promise<string> {+ onMessage(+ `Backing up the database before updating switch-core ${from} → ${COMPATIBLE_SWITCH_VERSION}…`+ );+ const stamp = new Date().toISOString().replace(/[:.]/g, '-');+ const directory = `${BACKUPS_DIR}/${stamp}-switch-core-${from}`;+ await host.writeFile(`${directory}/README.txt`, backupReadme(from), 0o600);+ await host.restrictMode(directory, 0o700);+ for (const file of [COMPOSE_FILE_NAME, ENV_FILE_NAME]) {+ const content = await host.readFile(file);+ if (content === null) {+ throw new Error(+ `Cannot back up the Switch server on ${host.label} before updating it: ${file} is missing from ${host.workingDir}.`+ );+ }+ await host.writeFile(`${directory}/${file}`, content, 0o600);+ }++ await composeUpService(host, DATABASE_SERVICE);+ const container = await serviceContainerId(host, DATABASE_SERVICE);+ const inContainer = `/tmp/switch-upgrade-${randomUUID()}.sql`;+ const destination = `${directory}/database.sql`;+ try {+ await runDocker(+ host,+ [+ 'exec',+ container,+ 'sh',+ '-c',+ 'umask 077 && pg_dumpall -U "$POSTGRES_USER" -f "$1" && test -s "$1"',+ 'sh',+ inContainer,+ ],+ DUMP_TIMEOUT_MS+ );+ await runDocker(host, ['cp', `${container}:${inContainer}`, destination], DUMP_TIMEOUT_MS);+ } finally {+ await runDocker(host, ['exec', container, 'rm', '-f', inContainer], 60_000).catch(+ (error: unknown) => {+ log.warn(+ `managed-switch-server: could not remove the temporary dump ${inContainer} from the database container on ${host.label}`,+ { error: String(error) }+ );+ }+ );+ }+ await host.restrictMode(destination, 0o600);+ const path = hostPath(host, directory);+ log.info(`managed-switch-server: backed up ${host.label} to ${path} before updating`);+ return path;+}++function backupReadme(from: string): string {+ return [+ `Switch Console took this backup before updating the Switch server from switch-core ${from} to ${COMPATIBLE_SWITCH_VERSION}.`,+ '',+ 'database.sql pg_dumpall of every database in the stack',+ `${COMPOSE_FILE_NAME} the compose file the stack ran with`,+ `${ENV_FILE_NAME} its environment, including credentials`,+ '',+ 'Keep these files private. switch-core migrates its database forward and cannot run',+ `against it at ${from} again once the update has started. To go back, stop the stack and`,+ `restore database.sql into a fresh database volume with switch-core ${from}.`,+ 'Uploaded files stay in the Docker volumes; this dump does not copy them.',+ '',+ ].join('\n');+}diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline-matrix-migration.test.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline-matrix-migration.test.tsindex 451b035ad..296037cbf 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline-matrix-migration.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline-matrix-migration.test.ts@@ -15,6 +15,8 @@ const readDeployedVersionMock = vi.hoisted(() => vi.fn());const composeUpMock = vi.hoisted(() => vi.fn(() => Promise.resolve()));const dockerRunOneOffMock = vi.hoisted(() => vi.fn(() => Promise.resolve()));+const prepareUpgradeMock = vi.hoisted(() => vi.fn(() => Promise.resolve(null)));+const finishUpgradeMock = vi.hoisted(() => vi.fn(() => Promise.resolve()));vi.mock('@shared/app-identity', async (importOriginal) => ({...(await importOriginal<typeof AppIdentity>()),COMPATIBLE_SWITCH_VERSION: '0.24.0',@@ -62,6 +64,10 @@ vi.mock('@main/core/switch-servers/auth', () => ({passwordLogin: () => Promise.resolve({ success: true }),}));vi.mock('@main/core/agents/resolve-servers', () => ({ resolveAgentServers: vi.fn() }));+vi.mock('./managed-upgrade', () => ({+ prepareUpgrade: prepareUpgradeMock,+ finishUpgrade: finishUpgradeMock,+}));const { startStack } = await import('./pipeline');@@ -83,6 +89,8 @@ function options(checkoutRoot: string | null = null) {serverName: 'Local',onMessage: vi.fn(),onLog: vi.fn(),+ activate: true,+ onUpgrade: vi.fn(),signal: new AbortController().signal,checkoutRoot,},diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline.test.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline.test.tsindex 6cee618fa..f5e827c8e 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline.test.ts@@ -20,6 +20,10 @@ const telemetryConsentMock = vi.hoisted(() => vi.fn(() => Promise.resolve(false)// Pin the build's expected version so the guard's arithmetic is not coupled to// whatever the real pin happens to be.+const prepareUpgradeMock = vi.hoisted(() =>+ vi.fn<(...args: unknown[]) => Promise<unknown>>(() => Promise.resolve(null))+);+const finishUpgradeMock = vi.hoisted(() => vi.fn(() => Promise.resolve()));vi.mock('@shared/app-identity', async (importOriginal) => ({...(await importOriginal<typeof AppIdentity>()),COMPATIBLE_SWITCH_VERSION: '0.11.0',@@ -57,14 +61,19 @@ vi.mock('./telemetry-consent', () => ({telemetryConsent: telemetryConsentMock,readDeployedTelemetry: vi.fn(),}));+const setActiveServerIdMock = vi.hoisted(() => vi.fn());vi.mock('@main/core/switch-servers/servers-store', () => ({ensureManagedServer: () => Promise.resolve({ id: 'srv-1' }),- setActiveServerId: vi.fn(),+ setActiveServerId: setActiveServerIdMock,}));vi.mock('@main/core/switch-servers/auth', () => ({passwordLogin: () => Promise.resolve({ success: true }),}));vi.mock('@main/core/agents/resolve-servers', () => ({ resolveAgentServers: vi.fn() }));+vi.mock('./managed-upgrade', () => ({+ prepareUpgrade: prepareUpgradeMock,+ finishUpgrade: finishUpgradeMock,+}));const { startStack } = await import('./pipeline');const { ENV_FILE_NAME } = await import('./constants');@@ -87,6 +96,8 @@ function options() {serverName: 'Local',onMessage: vi.fn(),onLog: vi.fn(),+ activate: true,+ onUpgrade: vi.fn(),signal: new AbortController().signal,checkoutRoot: null as string | null,},@@ -95,6 +106,7 @@ function options() {beforeEach(() => {vi.clearAllMocks();+ prepareUpgradeMock.mockResolvedValue(null);telemetryConsentMock.mockResolvedValue(false);buildEnvFileMock.mockReturnValue('SWITCH_VERSION=0.11.0\n');});@@ -323,3 +335,88 @@ describe('startStack checkout build', () => {expect(composeUpMock).toHaveBeenCalledWith(expect.anything(), expect.any(Function), false);});});++describe('startStack upgrade', () => {+ const journal = { from: '0.10.0', to: '0.11.0', backup: '/data/backups/x' };++ it('backs the database up before rewriting the stack and closes the journal once healthy', async () => {+ readDeployedVersionMock.mockResolvedValue({+ kind: 'deployed',+ version: '0.10.0',+ source: 'container',+ });+ const order: string[] = [];+ prepareUpgradeMock.mockImplementation(async () => {+ order.push('backup');+ return journal;+ });+ finishUpgradeMock.mockImplementation(async () => {+ order.push('finish');+ });+ const { opts, writeFile } = options();+ writeFile.mockImplementation(async (name: string) => {+ order.push(`write ${name}`);+ });++ expect(await startStack(opts)).toMatchObject({ kind: 'started' });+ expect(order[0]).toBe('backup');+ expect(order.at(-1)).toBe('finish');+ expect(prepareUpgradeMock).toHaveBeenCalledWith(+ opts.host,+ null,+ opts.onMessage,+ opts.onUpgrade+ );+ });++ it('leaves the stack untouched when the backup fails', async () => {+ readDeployedVersionMock.mockResolvedValue({+ kind: 'deployed',+ version: '0.10.0',+ source: 'container',+ });+ prepareUpgradeMock.mockRejectedValue(new Error('pg_dumpall failed: disk full'));+ const { opts, writeFile } = options();++ await expect(startStack(opts)).rejects.toThrow('disk full');+ expect(writeFile).not.toHaveBeenCalled();+ expect(composeUpMock).not.toHaveBeenCalled();+ expect(finishUpgradeMock).not.toHaveBeenCalled();+ });++ it('keeps the journal when the upgraded stack never turns healthy', async () => {+ readDeployedVersionMock.mockResolvedValue({+ kind: 'deployed',+ version: '0.10.0',+ source: 'container',+ });+ prepareUpgradeMock.mockResolvedValue(journal);+ waitForHealthMock.mockResolvedValueOnce(false);+ const { opts } = options();++ expect(await startStack(opts)).toMatchObject({ kind: 'error' });+ expect(finishUpgradeMock).not.toHaveBeenCalled();+ });++ it('does not back up a stack that is ahead of this build', async () => {+ readDeployedVersionMock.mockResolvedValue({+ kind: 'deployed',+ version: '0.12.0',+ source: 'container',+ });+ const { opts } = options();++ expect(await startStack(opts)).toMatchObject({ kind: 'version-downgrade' });+ expect(prepareUpgradeMock).not.toHaveBeenCalled();+ });++ it('does not switch the active server for a background upgrade', async () => {+ readDeployedVersionMock.mockResolvedValue({ kind: 'absent' });+ const { opts } = options();++ await startStack({ ...opts, activate: false });+ expect(setActiveServerIdMock).not.toHaveBeenCalled();+ await startStack(opts);+ expect(setActiveServerIdMock).toHaveBeenCalledWith('srv-1');+ });+});diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline.tsindex 161e9d2f5..0982bfa60 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline.ts+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/pipeline.ts@@ -23,6 +23,7 @@ import { buildEnvFile } from './env-file';import { apiUrlFor, gatewayUrlFor } from './free-port';import { waitForHealth } from './health';import type { ServerHost } from './host/types';+import { finishUpgrade, type OwedUpgrade, prepareUpgrade } from './managed-upgrade';import { crossesMatrixBoundary, runBackfill } from './matrix-migration';import { clearPorts, resolvePorts } from './ports';import { clearSecrets, loadOrCreateSecrets } from './secrets';@@ -41,10 +42,16 @@ export type StartStackOptions = {ref: ManagedServerRef;/** Display name for the registered server record. */serverName: string;+ /** Whether to make this the active server. False for an upgrade Console runs+ * on its own, which must not switch the user away from what they have open. */+ activate: boolean;/** Coarse step messages for the UI ("Pulling images…"). */onMessage: (message: string) => void;/** Live compose output lines for the UI log tail. */onLog: (line: string) => void;+ /** Fired before a start that moves the stack forward to this build's pin, so+ * the supervisor can hold sessions until it finishes. */+ onUpgrade: (upgrade: OwedUpgrade) => void;/** Aborts an in-flight health wait (stop/cancel/quit). */signal: AbortSignal;/** Dev-only: root of the Switch checkout to build the stack's images from,@@ -119,16 +126,18 @@ async function refuseDowngrade(}/**- * Full start pipeline: detect Docker → refuse a downgrade → GHCR login →- * materialise compose + `.env` → `compose up` → establish networking →- * health-gate → register + activate → silent admin sign-in → reconcile agent- * servers. Returns without registering anything if Docker is unavailable, the- * stack is newer than this build, or it never turns healthy.+ * Full start pipeline: detect Docker → refuse a downgrade → back up and+ * journal an upgrade → GHCR login → materialise compose + `.env` → `compose up`+ * → establish networking → health-gate → register (+ activate) → silent admin+ * sign-in → reconcile agent servers → close the upgrade journal. Returns+ * without registering anything if Docker is unavailable, the stack is newer+ * than this build, or it never turns healthy.** Doubles as the update path: the `.env` and compose file are re-materialised* from this build every time, so `compose up -d` on an already-running stack* re-pulls the newly pinned tags and recreates only the changed containers,- * leaving the data volumes in place for switch-core to migrate forward.+ * leaving the data volumes in place for switch-core to migrate forward. A+ * stack behind the pin has its database dumped first (see managed-upgrade.ts).** With `checkoutRoot` set (dev only) the images are built from that working* tree on every start instead of pulled, and tagged {@link CHECKOUT_IMAGE_TAG}@@ -182,7 +191,8 @@ async function migrateOffMatrix(}export async function startStack(opts: StartStackOptions): Promise<StartLocalServerResult> {- const { host, ref, serverName, onMessage, onLog, signal, checkoutRoot } = opts;+ const { host, ref, serverName, activate, onMessage, onLog, onUpgrade, signal, checkoutRoot } =+ opts;const docker = await host.detectDocker();if (!docker.available) {@@ -193,6 +203,10 @@ export async function startStack(opts: StartStackOptions): Promise<StartLocalSerconst downgrade = await refuseDowngrade(host, checkoutRoot);if (downgrade) return downgrade;+ // Back the database up before anything can migrate it: the Matrix backfill+ // below already starts the stack, and the rewrite after it moves the pin.+ const upgrade = await prepareUpgrade(host, checkoutRoot, onMessage, onUpgrade);+// Copy the homeserver's history across before the upgrade removes the only// thing that can read it. Runs against the stack as currently deployed, so// it must happen before the compose file and `.env` are re-materialised for@@ -220,7 +234,6 @@ export async function startStack(opts: StartStackOptions): Promise<StartLocalSernamespace: RELEASE_REPO_OWNER,ports,secrets,- sessionDemo: checkoutRoot !== null,telemetryEnabled,}),0o600@@ -245,7 +258,7 @@ export async function startStack(opts: StartStackOptions): Promise<StartLocalSer}const server = await ensureManagedServer({ name: serverName, gatewayUrl, apiUrl }, ref);- await setActiveServerId(server.id);+ if (activate) await setActiveServerId(server.id);// Switch Console generated the admin password, so sign in on the user's behalf// rather than showing a login wall for a secret they never saw. A failure here@@ -260,6 +273,7 @@ export async function startStack(opts: StartStackOptions): Promise<StartLocalSer}await resolveAgentServers();+ if (upgrade) await finishUpgrade(host);return { kind: 'started', serverId: server.id, telemetryEnabled };}@@ -276,4 +290,6 @@ export async function resetStack(host: ServerHost): Promise<void> {await host.teardownNetworking();await clearSecrets(host);await clearPorts(host);+ // Nothing is left to resume. The backups stay on disk.+ await finishUpgrade(host);}diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/remote-server-service.test.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/remote-server-service.test.tsnew file mode 100644index 000000000..f77b13dfe--- /dev/null+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/remote-server-service.test.ts@@ -0,0 +1,277 @@+import { beforeEach, expect, it, vi } from 'vitest';+import type * as AppIdentity from '@shared/app-identity';+import type { StartLocalServerResult } from '@shared/core/managed-switch-server/managed-switch-server';+import type * as ManagedUpgrade from './managed-upgrade';+import type { StartStackOptions } from './pipeline';++type Change = { current: { sshHost: string; status: string } };+const m = vi.hoisted(() => {+ const listeners: ((change: Change) => void)[] = [];+ const blocked = new Set<string>();+ return {+ listeners,+ blocked,+ reachability: {+ on(_event: string, listener: (change: Change) => void) {+ listeners.push(listener);+ },+ isBlocked: (sshHost: string) => blocked.has(sshHost),+ requireReachable(sshHost: string) {+ if (blocked.has(sshHost)) throw new Error(`${sshHost} is unreachable`);+ },+ },+ servers: vi.fn(),+ running: vi.fn(),+ version: vi.fn(),+ journal: vi.fn(),+ ports: vi.fn(),+ establish: vi.fn(),+ start: vi.fn<(opts: StartStackOptions) => Promise<StartLocalServerResult>>(),+ };+});+vi.mock('@shared/app-identity', async (importOriginal) => ({+ ...(await importOriginal<typeof AppIdentity>()),+ COMPATIBLE_SWITCH_VERSION: '0.11.0',+}));+vi.mock('@main/core/remote-hosts/production-host-reachability', () => ({+ hostReachabilityService: m.reachability,+}));+vi.mock('@main/lib/events', () => ({ events: { emit: vi.fn() } }));+vi.mock('@main/lib/logger', () => ({ log: { warn: vi.fn(), error: vi.fn(), info: vi.fn() } }));+vi.mock('@main/core/telemetry/managed-server', () => ({+ reportManagedServerOutcome: vi.fn(),+ reportManagedServerStart: vi.fn(),+ reportManagedServerStartThrew: vi.fn(),+}));+vi.mock('@main/core/switch-servers/delete-server-agents', () => ({+ deleteAgentsForServer: vi.fn(),+}));+vi.mock('@main/core/switch-servers/servers-store', () => ({+ listManagedServers: m.servers,+ getRemoteManagedServer: vi.fn(),+}));+vi.mock('./compose', () => ({ isStackRunning: m.running }));+vi.mock('./deployed-version', () => ({ readVersionStatus: m.version }));+vi.mock('./ports', () => ({ readPersistedPorts: m.ports }));+vi.mock('./telemetry-consent', () => ({ readDeployedTelemetry: vi.fn() }));+vi.mock('./managed-upgrade', async (importOriginal) => ({+ ...(await importOriginal<typeof ManagedUpgrade>()),+ readUpgradeJournal: m.journal,+}));+vi.mock('./pipeline', () => ({ startStack: m.start, stopStack: vi.fn(), resetStack: vi.fn() }));+vi.mock('./host/remote-host', () => ({+ createRemoteServerHost: async (sshHost: string) => ({+ sshHost,+ establishNetworking: m.establish,+ dispose: vi.fn(),+ }),+}));++const { RemoteServerService } = await import('./remote-server-service');++const behind = {+ deployedVersion: '0.10.0',+ drift: { deployed: '0.10.0', expected: '0.11.0', direction: 'upgrade' },+};+const inStep = { deployedVersion: '0.11.0', drift: null };++function startedOn(serverId: string): StartLocalServerResult {+ return { kind: 'started', serverId, telemetryEnabled: false };+}++/** A start that runs the upgrade the way the pipeline does: announce, then work. */+function upgradingStart(result?: StartLocalServerResult) {+ return async (opts: StartStackOptions) => {+ opts.onUpgrade({ from: '0.10.0', to: '0.11.0' });+ return result ?? startedOn(opts.ref.kind === 'remote' ? `srv-${opts.ref.sshHost}` : 'local');+ };+}++function announce(sshHost: string, status: string): void {+ for (const listener of m.listeners) listener({ current: { sshHost, status } });+}++beforeEach(() => {+ vi.resetAllMocks();+ m.listeners.length = 0;+ m.blocked.clear();+ m.servers.mockResolvedValue([+ { id: 'srv-builder', name: 'Builder', managementKind: 'remote', sshHost: 'builder' },+ ]);+ m.running.mockResolvedValue(true);+ m.version.mockResolvedValue(behind);+ m.journal.mockResolvedValue(null);+ m.ports.mockResolvedValue({ gateway: 1, api: 2, mattermost: 3, postgres: 4 });+ m.start.mockImplementation(upgradingStart());+});++it('upgrades a running remote stack that is behind at boot, instead of adopting it', async () => {+ const service = new RemoteServerService();+ await service.initialize();+ await service.ensureReady('builder', 'Builder');++ expect(m.start).toHaveBeenCalledOnce();+ expect(m.start).toHaveBeenCalledWith(+ expect.objectContaining({+ ref: { kind: 'remote', sshHost: 'builder' },+ serverName: 'Builder',+ activate: false,+ })+ );+ expect(m.establish).not.toHaveBeenCalled();+ expect(service.getStatus('builder')).toMatchObject({+ phase: 'running',+ serverId: 'srv-builder',+ upgrade: null,+ drift: null,+ });+});++it('holds sessions for the host while its upgrade runs', async () => {+ let finish: (result: StartLocalServerResult) => void = () => {};+ m.start.mockImplementation(+ (opts) =>+ new Promise((resolve) => {+ opts.onUpgrade({ from: '0.10.0', to: '0.11.0' });+ finish = resolve;+ })+ );+ const service = new RemoteServerService();+ void service.initialize();+ let ready = false;+ const session = service.ensureReady('builder', 'Builder').then(() => {+ ready = true;+ });++ await vi.waitFor(() => expect(m.start).toHaveBeenCalledOnce());+ expect(service.getStatus('builder').upgrade).toEqual({+ state: 'updating',+ from: '0.10.0',+ to: '0.11.0',+ });+ expect(ready).toBe(false);+ finish(startedOn('srv-builder'));+ await session;+ expect(ready).toBe(true);+});++it('adopts a running stack that is in step without restarting it', async () => {+ m.version.mockResolvedValue(inStep);+ const service = new RemoteServerService();+ await service.initialize();+ await service.ensureReady('builder', 'Builder');++ expect(m.start).not.toHaveBeenCalled();+ expect(m.establish).toHaveBeenCalledOnce();+ expect(service.getStatus('builder')).toMatchObject({+ phase: 'running',+ serverId: 'srv-builder',+ upgrade: null,+ });+});++it('upgrades a stopped stack that is behind at its next start, not before', async () => {+ m.running.mockResolvedValue(false);+ const service = new RemoteServerService();+ const upgraded = vi.fn();+ service.onUpgradeFinished(upgraded);+ await service.initialize();++ await expect(service.ensureReady('builder', 'Builder')).rejects.toThrow(+ /Builder is stopped on switch-core 0\.10\.0/+ );+ expect(m.start).not.toHaveBeenCalled();++ expect(await service.start('builder', 'Builder')).toMatchObject({ kind: 'started' });+ expect(m.start).toHaveBeenCalledWith(expect.objectContaining({ activate: true }));+ await expect(service.ensureReady('builder', 'Builder')).resolves.toBeUndefined();+ expect(upgraded).toHaveBeenCalledExactlyOnceWith('srv-builder');+});++it('upgrades a host that was unreachable at boot once it comes back, holding sessions meanwhile', async () => {+ m.blocked.add('builder');+ const service = new RemoteServerService();+ await service.initialize();+ await service.ensureReady('builder', 'Builder');+ expect(m.start).not.toHaveBeenCalled();++ let finish: (result: StartLocalServerResult) => void = () => {};+ m.start.mockImplementation(+ (opts) =>+ new Promise((resolve) => {+ opts.onUpgrade({ from: '0.10.0', to: '0.11.0' });+ finish = resolve;+ })+ );+ m.blocked.delete('builder');+ announce('builder', 'reachable');+ // Asked straight after the recovery, as a watcher restoring its controller does.+ let ready = false;+ const session = service.ensureReady('builder', 'Builder').then(() => {+ ready = true;+ });++ await vi.waitFor(() => expect(m.start).toHaveBeenCalledOnce());+ expect(ready).toBe(false);+ finish(startedOn('srv-builder'));+ await session;+ expect(service.getStatus('builder').upgrade).toBeNull();+});++it('surfaces a failed upgrade with its error and retries when the host is reconciled again', async () => {+ m.start.mockImplementationOnce(+ upgradingStart({ kind: 'error', message: 'The server did not become healthy in time.' })+ );+ const service = new RemoteServerService();+ const upgraded = vi.fn();+ service.onUpgradeFinished(upgraded);+ await service.initialize();++ await expect(service.ensureReady('builder', 'Builder')).rejects.toThrow(+ /Updating Builder from switch-core 0\.10\.0 to 0\.11\.0 failed: The server did not become healthy/+ );+ expect(service.getStatus('builder')).toMatchObject({+ phase: 'error',+ upgrade: { state: 'failed', error: 'The server did not become healthy in time.' },+ });++ announce('builder', 'reachable');+ await vi.waitFor(() => expect(m.start).toHaveBeenCalledTimes(2));+ await expect(service.ensureReady('builder', 'Builder')).resolves.toBeUndefined();+ expect(upgraded).toHaveBeenCalledExactlyOnceWith('srv-builder');+});++it('does not hold one host’s sessions behind another host’s upgrade', async () => {+ m.servers.mockResolvedValue([+ { id: 'srv-builder', name: 'Builder', managementKind: 'remote', sshHost: 'builder' },+ { id: 'srv-other', name: 'Other', managementKind: 'remote', sshHost: 'other' },+ ]);+ m.start.mockImplementation(+ (opts) =>+ new Promise(() => {+ opts.onUpgrade({ from: '0.10.0', to: '0.11.0' });+ })+ );+ m.version.mockImplementation(async (host: { sshHost: string }) =>+ host.sshHost === 'builder' ? behind : inStep+ );+ const service = new RemoteServerService();+ await service.initialize();++ await expect(service.ensureReady('other', 'Other')).resolves.toBeUndefined();+ expect(service.getStatus('builder').upgrade?.state).toBe('updating');+});++it('refuses sessions when the host cannot be reached for its upgrade, rather than hanging', async () => {+ const service = new RemoteServerService();+ m.running.mockImplementation(async () => {+ // The host drops between the probe and the start.+ m.blocked.add('builder');+ return true;+ });+ await service.initialize();++ await expect(service.ensureReady('builder', 'Builder')).rejects.toThrow(+ /failed: builder is unreachable/+ );+});diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/remote-server-service.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/remote-server-service.tsindex b8b044026..459be044c 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/remote-server-service.ts+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/remote-server-service.ts@@ -16,6 +16,7 @@ import { COMPATIBLE_SWITCH_VERSION } from '@shared/app-identity';import {type DockerAvailability,type StartLocalServerResult,+ managedServerUpgradeBlockedReason,matrixMigrationFailedMessage,switchVersionDowngradeMessage,} from '@shared/core/managed-switch-server/managed-switch-server';@@ -27,6 +28,12 @@ import {import { isStackRunning } from './compose';import { readVersionStatus } from './deployed-version';import { createRemoteServerHost, type RemoteServerHost } from './host/remote-host';+import {+ owedUpgrade,+ readUpgradeJournal,+ type UpgradeJournal,+ upgradeState,+} from './managed-upgrade';import { resetStack, startStack, stopStack } from './pipeline';import { readPersistedPorts } from './ports';import { readDeployedTelemetry } from './telemetry-consent';@@ -35,6 +42,7 @@ function initialStatus(sshHost: string): RemoteServerStatus {return {sshHost,phase: 'stopped',+ upgrade: null,serverId: null,version: COMPATIBLE_SWITCH_VERSION,deployedVersion: null,@@ -57,12 +65,25 @@ function initialStatus(sshHost: string): RemoteServerStatus {* containers themselves run detached, so a remote stack (and its remote-host* agents) stays up while Switch Console is closed — only the desktop-side forward* goes away.+ *+ * A host whose stack is behind this build's switch-core pin is upgraded when it+ * is reconciled — at boot, and whenever the host becomes reachable again — and+ * {@link ensureReady} holds sessions for its agents until that has finished.*/-class RemoteServerService {+export class RemoteServerService {private readonly statuses = new Map<string, RemoteServerStatus>();private readonly hosts = new Map<string, RemoteServerHost>();private readonly busy = new Set<string>();private readonly startAborts = new Map<string, AbortController>();+ private initialization: Promise<void> | null = null;+ /** The reconcile or start in flight per host, which {@link ensureReady} waits out. */+ private readonly operations = new Map<string, Promise<void>>();+ /** The start in flight per host, which an automatic upgrade and a Start click share. */+ private readonly starting = new Map<string, Promise<StartLocalServerResult>>();+ private readonly upgradeListeners = new Set<(serverId: string) => void>();+ /** Hosts {@link ensureReady} has turned something away for since their last+ * upgrade finished, so that it can be told when they are ready. */+ private readonly refused = new Set<string>();getStatuses(): RemoteServerStatus[] {return [...this.statuses.values()];@@ -89,32 +110,90 @@ class RemoteServerService {}/** Re-establish forwards + status for remote stacks that survived the last- * quit, so their desktop reachability is restored on launch. Best-effort per- * host: an unreachable host is left `stopped` rather than failing boot. */- async initialize(): Promise<void> {+ * quit, so their desktop reachability is restored on launch, and upgrade the+ * ones that are behind. Hosts reconcile independently and in the background;+ * an unreachable one is left `stopped` rather than failing boot.+ *+ * Memoised: {@link ensureReady} awaits the same call. It resolves once every+ * host's reconcile is registered, not once they are done. */+ initialize(): Promise<void> {+ this.initialization ??= this.startReconciling();+ return this.initialization;+ }++ private async startReconciling(): Promise<void> {+ // Registered before the first await, so the host's reconcile is in flight+ // before anything else that reacts to the same recovery asks whether the+ // server is ready.hostReachabilityService.on('change', ({ current }: HostReachabilityChange) => {- if (current.status === 'reachable') void this.onHostReachable(current.sshHost);+ if (current.status !== 'reachable') return;+ void this.track(current.sshHost, () => this.onHostReachable(current.sshHost)).catch(+ (error: unknown) => {+ log.warn(`remote-switch-server: reconcile after recovery failed for ${current.sshHost}`, {+ error,+ });+ }+ );});- for (const [sshHost, serverId] of await this.remoteHosts()) {+ for (const [sshHost, server] of await this.remoteHosts()) {this.statuses.set(sshHost, initialStatus(sshHost));- await this.reconcileHost(sshHost, serverId);+ void this.track(sshHost, () => this.reconcileHost(sshHost, server));}}- private async remoteHosts(): Promise<Map<string, string>> {+ private async remoteHosts(): Promise<Map<string, { id: string; name: string }>> {const remotes = (await listManagedServers()).filter((s) => s.managementKind === 'remote' && s.sshHost);- return new Map(remotes.map((s) => [s.sshHost!, s.id]));+ return new Map(remotes.map((s) => [s.sshHost!, { id: s.id, name: s.name }]));+ }++ /** Remember `run` as the host's operation in flight until it settles. */+ private track<T>(sshHost: string, run: () => Promise<T>): Promise<T> {+ const promise = run();+ const settled = promise.then(+ () => undefined,+ () => undefined+ );+ this.operations.set(sshHost, settled);+ void settled.then(() => {+ if (this.operations.get(sshHost) === settled) this.operations.delete(sshHost);+ });+ return promise;+ }++ /**+ * Resolves once sessions may run against the stack on `sshHost`, waiting out+ * its reconcile and any upgrade in flight. Throws when it still owes an+ * upgrade — stopped, or failed — with the reason to show.+ */+ async ensureReady(sshHost: string, serverName: string): Promise<void> {+ await this.initialize();+ for (let op = this.operations.get(sshHost); op; op = this.operations.get(sshHost)) await op;+ const upgrade = this.getStatus(sshHost).upgrade;+ if (upgrade === null) return;+ if (upgrade.state === 'updating') {+ throw new Error(`${serverName} reports an update in progress, but none is running.`);+ }+ this.refused.add(sshHost);+ throw new Error(managedServerUpgradeBlockedReason(serverName, upgrade));+ }++ /** Called with the server id when an upgrade finishes that sessions or+ * watchers were turned away for (a failed one retried, or a stopped one+ * started). Those that waited instead carry on by themselves. */+ onUpgradeFinished(listener: (serverId: string) => void): void {+ this.upgradeListeners.add(listener);}/** Pick a host's stack back up once its host is reachable again, so a- * recovered host resumes without the user restarting anything. */+ * recovered host resumes — and catches up on an owed upgrade — without the+ * user restarting anything. */private async onHostReachable(sshHost: string): Promise<void> {if (this.busy.has(sshHost) || this.hosts.has(sshHost)) return;- const serverId = (await this.remoteHosts()).get(sshHost);- if (!serverId) return;- await this.reconcileHost(sshHost, serverId);+ const server = (await this.remoteHosts()).get(sshHost);+ if (!server) return;+ await this.reconcileHost(sshHost, server);}/** Adopt an already-running remote stack: re-open its forward and mark it@@ -122,23 +201,47 @@ class RemoteServerService {* call back through {@link onHostReachable} when it recovers.** Also records how the host's deployed switch-core compares to this build's- * pin, so an app update that moved the pin surfaces as drift rather than- * leaving the host on a stale core (CHOO-1736). That check runs even when the- * stack is down: its data volumes still hold the schema the last version- * migrated to, which is what makes a downgrade unsafe. */- private async reconcileHost(sshHost: string, serverId: string): Promise<void> {+ * pin (CHOO-1736). A running stack that is behind, or one whose last upgrade+ * was interrupted, is upgraded instead of adopted; a stopped one that is+ * behind is marked to be upgraded at its next start. The check runs even+ * when the stack is down: its data volumes still hold the schema the last+ * version migrated to, which is what makes a downgrade unsafe. */+ private async reconcileHost(+ sshHost: string,+ server: { id: string; name: string }+ ): Promise<void> {if (hostReachabilityService.isBlocked(sshHost)) return;let host: RemoteServerHost | null = null;let adopted = false;+ let upgradeNow = false;try {host = await createRemoteServerHost(sshHost);- if (await isStackRunning(host)) {+ const running = await isStackRunning(host);+ const version = await readVersionStatus(host, COMPATIBLE_SWITCH_VERSION);+ let journal: UpgradeJournal | null;+ try {+ journal = await readUpgradeJournal(host);+ } catch (error) {+ this.setStatus(sshHost, {+ ...version,+ upgrade: {+ state: 'failed',+ from: version.deployedVersion ?? 'an unknown version',+ to: COMPATIBLE_SWITCH_VERSION,+ error: error instanceof Error ? error.message : String(error),+ },+ });+ return;+ }+ const owed = owedUpgrade(version.drift, journal);+ upgradeNow = owed !== null && (running || journal !== null);+ if (running && !upgradeNow) {const ports = await readPersistedPorts(host);if (ports) {await host.establishNetworking(ports);this.hosts.set(sshHost, host);adopted = true;- this.setStatus(sshHost, { phase: 'running', serverId });+ this.setStatus(sshHost, { phase: 'running', serverId: server.id });// Only for a running stack: a stopped one sends nothing, so it// cannot be out of step with the user's answer.this.setStatus(sshHost, { deployedTelemetry: await readDeployedTelemetry(host) });@@ -146,16 +249,48 @@ class RemoteServerService {// Running but we don't know its ports — leave it stopped; the user can// restart to re-derive them rather than forward to the wrong ports.}- this.setStatus(sshHost, await readVersionStatus(host, COMPATIBLE_SWITCH_VERSION));+ this.setStatus(sshHost, { ...version, upgrade: owed && upgradeState(owed, upgradeNow) });} catch (error) {log.warn(`remote-switch-server: boot reconcile failed for ${sshHost}`, { error });} finally {// The adopted host owns the live port-forward; anything else is throwaway.if (!adopted) host?.dispose();}+ if (!upgradeNow) return;+ try {+ await this.beginStart(sshHost, server.name, false);+ } catch (error) {+ // Only the reachability check throws rather than reporting a result.+ log.warn(`remote-switch-server: could not upgrade ${sshHost}`, { error });+ this.failUpgrade(sshHost, error instanceof Error ? error.message : String(error));+ }+ }++ /** Start (or restart) the host's stack at this build's pin, upgrading it+ * first if it is behind. Joins a start already in flight for the host. */+ start(sshHost: string, serverName: string): Promise<StartLocalServerResult> {+ return this.track(sshHost, () => this.beginStart(sshHost, serverName, true));}- async start(sshHost: string, serverName: string): Promise<StartLocalServerResult> {+ private beginStart(+ sshHost: string,+ serverName: string,+ activate: boolean+ ): Promise<StartLocalServerResult> {+ const inFlight = this.starting.get(sshHost);+ if (inFlight) return inFlight;+ const run = this.runStart(sshHost, serverName, activate).finally(() => {+ this.starting.delete(sshHost);+ });+ this.starting.set(sshHost, run);+ return run;+ }++ private async runStart(+ sshHost: string,+ serverName: string,+ activate: boolean+ ): Promise<StartLocalServerResult> {if (this.busy.has(sshHost)) {return { kind: 'error', message: `An operation is already in progress for ${sshHost}.` };}@@ -179,13 +314,16 @@ class RemoteServerService {host,ref: { kind: 'remote', sshHost },serverName,+ activate,onMessage: (message) => this.setStatus(sshHost, { message }),onLog: (line) => events.emit(remoteServerLogChannel, { sshHost, line }),+ onUpgrade: (owed) => this.setStatus(sshHost, { upgrade: upgradeState(owed, true) }),signal: abort.signal,checkoutRoot: null,});if (result.kind === 'docker-unavailable') {this.setStatus(sshHost, { phase: 'error', error: result.detail });+ this.failUpgrade(sshHost, result.detail);host.dispose();} else if (result.kind === 'version-downgrade') {this.setStatus(sshHost, {@@ -194,17 +332,21 @@ class RemoteServerService {error: switchVersionDowngradeMessage(result.deployed, result.expected),deployedVersion: result.deployed,drift: { deployed: result.deployed, expected: result.expected, direction: 'downgrade' },+ upgrade: null,});host.dispose();} else if (result.kind === 'matrix-migration-failed') {+ const error = matrixMigrationFailedMessage(result.deployed, result.expected);this.setStatus(sshHost, {phase: 'error',message: null,- error: matrixMigrationFailedMessage(result.deployed, result.expected),+ error,deployedVersion: result.deployed,});+ this.failUpgrade(sshHost, error);} else if (result.kind === 'error') {this.setStatus(sshHost, { phase: 'error', error: result.message });+ this.failUpgrade(sshHost, result.message);host.dispose();} else {// Keep the host alive — it owns the port-forward.@@ -218,8 +360,12 @@ class RemoteServerService {error: null,deployedVersion: COMPATIBLE_SWITCH_VERSION,drift: null,+ upgrade: null,deployedTelemetry: { known: true, enabled: result.telemetryEnabled },});+ if (this.refused.delete(sshHost)) {+ for (const listener of this.upgradeListeners) listener(result.serverId);+ }}reportManagedServerStart('remote', result);return result;@@ -228,6 +374,7 @@ class RemoteServerService {const message = error instanceof Error ? error.message : String(error);log.error(`remote-switch-server: start failed for ${sshHost}`, { error });this.setStatus(sshHost, { phase: 'error', error: message });+ this.failUpgrade(sshHost, message);reportManagedServerStartThrew('remote');return { kind: 'error', message };} finally {@@ -236,6 +383,16 @@ class RemoteServerService {}}+ /** Record why a host's upgrade did not finish. A start that was not an+ * upgrade has nothing to record here; its error is on the status already. */+ private failUpgrade(sshHost: string, error: string): void {+ const upgrade = this.getStatus(sshHost).upgrade;+ if (!upgrade) return;+ this.setStatus(sshHost, {+ upgrade: { state: 'failed', from: upgrade.from, to: upgrade.to, error },+ });+ }+async stop(sshHost: string): Promise<void> {if (this.busy.has(sshHost))throw new Error(`An operation is already in progress for ${sshHost}.`);@@ -250,11 +407,13 @@ class RemoteServerService {host = this.hosts.get(sshHost) ?? (await createRemoteServerHost(sshHost));this.setStatus(sshHost, { phase: 'stopping', message: 'Stopping containers…' });await stopStack(host);+ const upgrade = this.getStatus(sshHost).upgrade;this.setStatus(sshHost, {phase: 'stopped',message: null,error: null,deployedTelemetry: null,+ upgrade: upgrade && upgradeState(upgrade, false),});reportManagedServerOutcome('stop', 'remote', 'success');} catch (error) {@@ -295,6 +454,9 @@ class RemoteServerService {message: null,error: null,deployedTelemetry: null,+ deployedVersion: null,+ drift: null,+ upgrade: null,});reportManagedServerOutcome('reset', 'remote', 'success');} catch (error) {diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/resources/standalone-docker-compose.pinned.yml b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/resources/standalone-docker-compose.pinned.ymlindex 5af90083f..5d1d13f64 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/resources/standalone-docker-compose.pinned.yml+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/resources/standalone-docker-compose.pinned.yml@@ -220,9 +220,6 @@ services:GATEWAY_ADMIN_PASSWORD: ${GATEWAY_ADMIN_PASSWORD}FRONTEND_BASE_URL: ${FRONTEND_BASE_URL}GATEWAY_PUBLIC_URL: ${GATEWAY_PUBLIC_URL:-}- # Off by default. The recorded session behind it posts a card into a- # real channel with no session behind it, so it is opt-in per server.- SESSION_DEMO_ENABLED: ${SESSION_DEMO_ENABLED:-}MESSAGING_PUBLIC_URL: ${MESSAGING_PUBLIC_URL:-}SLACK_APP_CLIENT_ID: ${SLACK_APP_CLIENT_ID:-}SLACK_APP_CLIENT_SECRET: ${SLACK_APP_CLIENT_SECRET:-}diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/session-readiness.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/session-readiness.tsnew file mode 100644index 000000000..6e973696f--- /dev/null+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/session-readiness.ts@@ -0,0 +1,32 @@+import type { SwitchServer } from '@shared/core/switch-servers/switch-servers';+import { localServerService } from './local-server-service';+import { remoteServerService } from './remote-server-service';++/**+ * The one gate between a managed Switch server and anything that runs agents+ * against it — a session starting, or an agent's room watcher connecting.+ *+ * Resolves at once for a server Console only connects to, and for a managed+ * one that is in step with this build. For a managed server Console is still+ * checking or upgrading it waits until that has finished; for one that owes an+ * upgrade it cannot run now (stopped, or a failed update) it throws the reason.+ */+export async function ensureServerSessionReady(server: SwitchServer): Promise<void> {+ if (!server.managed) return;+ if (server.managementKind === 'remote') {+ // A remote row without a host is not one Console can supervise.+ if (server.sshHost === null) return;+ await remoteServerService.ensureReady(server.sshHost, server.name);+ return;+ }+ await localServerService.ensureReady(server.name);+}++/**+ * Called with a managed server's id when an upgrade that held its sessions+ * back has finished, so whatever was refused can be started again.+ */+export function onManagedServerUpgraded(listener: (serverId: string) => void): void {+ localServerService.onUpgradeFinished(listener);+ remoteServerService.onUpgradeFinished(listener);+}diff --git a/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-agent-runtime.initial-prompt.test.ts b/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-agent-runtime.initial-prompt.test.tsindex 1e471a237..6f59eaaa9 100644--- a/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-agent-runtime.initial-prompt.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-agent-runtime.initial-prompt.test.ts@@ -15,6 +15,11 @@ const mocks = vi.hoisted(() => ({readFailure: vi.fn(),exec: vi.fn(),specialization: vi.fn(),+ ready: vi.fn(),+}));++vi.mock('@main/core/managed-switch-server/session-readiness', () => ({+ ensureServerSessionReady: mocks.ready,}));vi.mock('./transcripts', () => ({ currentSnapshot: mocks.snapshot }));@@ -105,6 +110,7 @@ beforeEach(() => {serverId: 'server-1',});mocks.server.mockResolvedValue({ id: 'server-1' });+ mocks.ready.mockResolvedValue(undefined);mocks.exec.mockResolvedValue({ stdout: 'null' });mocks.readFailure.mockResolvedValue(null);// The launch reports an existing host, which is what a retry after a failed@@ -356,3 +362,35 @@ it('passes the existing conversation room as a guarded migration hint', async ()} as LocationTransport);expect(other.roomConnection?.restoreRoomId).toBeUndefined();});++it('never starts a host or replays the initial prompt while the server is not ready', async () => {+ mocks.ready.mockRejectedValueOnce(new Error('Updating Local from switch-core 0.1.0 failed'));+ const agent = runtime();++ await expect(agent.start(session, false, 'Do work')).rejects.toThrow('failed');+ expect(mocks.ready).toHaveBeenCalledWith({ id: 'server-1' });+ expect(mocks.runHost).not.toHaveBeenCalled();+ expect(mocks.submit).not.toHaveBeenCalled();+ expect(agent.startupStatus()).toMatchObject({ status: 'error' });+});++it('waits for the server’s update before starting the session', async () => {+ let ready: () => void = () => {};+ mocks.ready.mockReturnValueOnce(+ new Promise<void>((resolve) => {+ ready = resolve;+ })+ );+ const agent = runtime();+ const started = agent.start(session, false, 'Say hello');++ await vi.waitFor(() => expect(mocks.ready).toHaveBeenCalled());+ expect(agent.startupStatus()).toEqual({+ status: 'starting',+ message: 'Waiting for the Switch server to be ready…',+ });+ expect(mocks.runHost).not.toHaveBeenCalled();+ ready();+ await started;+ expect(mocks.submit).toHaveBeenCalledTimes(1);+});diff --git a/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-agent-runtime.ts b/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-agent-runtime.tsindex 70f104222..14f2949a5 100644--- a/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-agent-runtime.ts+++ b/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-agent-runtime.ts@@ -28,6 +28,7 @@ import { getAgentById } from '@main/core/agents/getAgentById';import { agentSettingsRelativePath } from '@main/core/agents/switch-settings-paths';import { hostDependencyStore } from '@main/core/dependencies/host-dependency-store';import type { LocationTransport } from '@main/core/locations/location-transport';+import { ensureServerSessionReady } from '@main/core/managed-switch-server/session-readiness';import { getPlugin } from '@main/core/providers/plugin-registry';import { AGENT_ENV_VARS } from '@main/core/sdk-host/agent-env';import { setInitialPromptDelivery } from '@main/core/sessions/operations/set-initial-prompt-delivery';@@ -126,6 +127,9 @@ export class SharedAgentRuntime implements AgentRuntimeProvider {throw new Error('Link this agent to a Switch server before starting a session.');this.server = await getServer(agent.serverId);if (!this.server) throw new Error('The agent’s Switch server is missing.');+ this.startupStage = 'Waiting for the Switch server to be ready…';+ await ensureServerSessionReady(this.server);+ this.startupStage = 'Preparing the session on its host…';const intended = switchNotificationPoller.getSharedIntent(session.id, agent.switchAgentId);const config = await buildSharedHostConfig(session, this.params, this.transport);const previousEpoch = restart ? await journalEpoch(session.agentId, session.id) : null;diff --git a/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-watcher.locality.test.ts b/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-watcher.locality.test.tsindex 2acdc4f45..893a4cf36 100644--- a/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-watcher.locality.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-watcher.locality.test.ts@@ -12,8 +12,15 @@ const mocks = vi.hoisted(() => ({removeRoots: vi.fn(),agentById: vi.fn(),ssh: vi.fn(),+ ready: vi.fn(),+ server: vi.fn(),}));+vi.mock('@main/core/managed-switch-server/session-readiness', () => ({+ ensureServerSessionReady: mocks.ready,+}));+vi.mock('@main/core/switch-servers/servers-store', () => ({ getServer: mocks.server }));+vi.mock('@main/core/switch-rooms/auto-session-store', () => ({listStoppedControllerAgentIds: mocks.stopped,listAutoSessionAgentIds: mocks.spawning,@@ -75,7 +82,10 @@ beforeEach(() => {name: 'scout',switchAgentId: 'switch-agent-1',providerId: 'claude',+ serverId: 'server-1',});+ mocks.server.mockResolvedValue({ id: 'server-1', name: 'Local', managed: true });+ mocks.ready.mockResolvedValue(undefined);});it('watches a local agent inside Console without deploying a host', async () => {@@ -204,3 +214,48 @@ it('leaves a stopped controller off the air however auto-start is set', async ()expect(mocks.stopLocal).toHaveBeenCalledWith('switch-agent-1');expect(mocks.startLocal).not.toHaveBeenCalled();});++it('connects only once the agent’s managed server is ready', async () => {+ mocks.location.mockResolvedValue({ id: 'local', dir: '/work', sshHost: null });+ let ready: () => void = () => {};+ mocks.ready.mockReturnValue(+ new Promise<void>((resolve) => {+ ready = resolve;+ })+ );+ const connecting = configureSharedWatcher(+ 'agent-1',+ { connected: true, spawning: true },+ 'restore'+ );+ await vi.waitFor(() =>+ expect(mocks.ready).toHaveBeenCalledWith(expect.objectContaining({ id: 'server-1' }))+ );+ expect(mocks.startLocal).not.toHaveBeenCalled();+ ready();+ await connecting;+ expect(mocks.startLocal).toHaveBeenCalled();+});++it('never connects to a server whose update failed', async () => {+ mocks.location.mockResolvedValue({+ id: 'remote',+ dir: '/work',+ sshHost: 'builder',+ connectionId: 'connection-1',+ });+ mocks.ready.mockRejectedValue(new Error('Updating Local from switch-core 0.1.0 failed'));+ await expect(+ configureSharedWatcher('agent-1', { connected: true, spawning: true }, 'restore')+ ).rejects.toThrow('failed');+ expect(mocks.deploy).not.toHaveBeenCalled();+ expect(mocks.startLocal).not.toHaveBeenCalled();+});++it('stands a watcher down without waiting for its server', async () => {+ mocks.location.mockResolvedValue({ id: 'local', dir: '/work', sshHost: null });+ mocks.ready.mockRejectedValue(new Error('update failed'));+ await configureSharedWatcher('agent-1', { connected: false, spawning: false }, 'explicit');+ expect(mocks.ready).not.toHaveBeenCalled();+ expect(mocks.stopLocal).toHaveBeenCalledWith('switch-agent-1');+});diff --git a/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-watcher.ts b/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-watcher.tsindex 4af0da80e..dbc23e825 100644--- a/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-watcher.ts+++ b/console/apps/switch-console-desktop/src/main/core/sdk-host/shared-watcher.ts@@ -5,12 +5,14 @@ import { SshExecutionContext } from '@main/core/execution-context/ssh-execution-import { locationManager } from '@main/core/locations/location-manager';import { resolveSessionEnv } from '@main/core/locations/location-runtime-factory';import { locationTransport, type LocationTransport } from '@main/core/locations/location-transport';+import { ensureServerSessionReady } from '@main/core/managed-switch-server/session-readiness';import { ensureSshConnected } from '@main/core/ssh/connect/connect-agent-ssh';import {listAutoSessionAgentIds,listStoppedControllerAgentIds,} from '@main/core/switch-rooms/auto-session-store';import { controllerConnectionId } from '@main/core/switch-rooms/session-connection-id';+import { getServer } from '@main/core/switch-servers/servers-store';import { adoptSubagent } from './adopt-subagent';import { stopLegacySidecar } from './legacy-sidecar';import {@@ -104,6 +106,12 @@ export async function configureSharedWatcher(if (!state.connected) return;throw new Error('Link the agent to Switch before it can hold a room connection.');}+ // Connecting waits for the agent's managed server to be in step with this+ // build; standing down never does.+ if (state.connected && agent.serverId) {+ const server = await getServer(agent.serverId);+ if (server) await ensureServerSessionReady(server);+ }const location = await getAgentLocation(agent);const transport = locationTransport(location);const identity = `watcher-${agent.switchAgentId}`;diff --git a/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.test.ts b/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.test.tsindex 1a0ed1879..135c757d0 100644--- a/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.test.ts@@ -2,7 +2,9 @@ import { beforeEach, expect, it, vi } from 'vitest';type Change = { current: { sshHost: string; status: string } };const mocks = vi.hoisted(() => {const listeners: ((change: Change) => void)[] = [];+ const upgraded: ((serverId: string) => void)[] = [];return {+ upgraded,apply: vi.fn(),configure: vi.fn(),dispose: vi.fn(async () => {}),@@ -39,6 +41,9 @@ vi.mock('./auto-session-store', () => ({setAutoSessionSubagent: vi.fn(),}));vi.mock('@main/lib/logger', () => ({ log: { error: vi.fn() } }));+vi.mock('@main/core/managed-switch-server/session-readiness', () => ({+ onManagedServerUpgraded: (listener: (serverId: string) => void) => mocks.upgraded.push(listener),+}));const { autoSessionWatcher } = await import('./auto-session-watcher');beforeEach(() => {@@ -163,3 +168,37 @@ it('stops everything it hosts when Console closes', async () => {await autoSessionWatcher.dispose();expect(mocks.dispose).toHaveBeenCalled();});++it('does not hold one server’s agents behind another server’s update', async () => {+ mocks.agents.mockResolvedValue([+ { id: 'updating', switchAgentId: 'switch-1', serverId: 'local' },+ { id: 'elsewhere', switchAgentId: 'switch-2', serverId: 'other' },+ ]);+ let finish: () => void = () => {};+ mocks.apply.mockImplementation(async (agentId: string) => {+ if (agentId === 'updating')+ await new Promise<void>((resolve) => {+ finish = resolve;+ });+ });+ const boot = autoSessionWatcher.initialize();++ await vi.waitFor(() => expect(mocks.apply).toHaveBeenCalledWith('elsewhere', 'restore'));+ finish();+ await boot;+});++it('starts the controllers of a server once an update they were refused for finishes', async () => {+ mocks.agents.mockResolvedValue([+ { id: 'upgraded', switchAgentId: 'switch-1', serverId: 'local' },+ { id: 'elsewhere', switchAgentId: 'switch-2', serverId: 'other' },+ { id: 'unlinked', switchAgentId: null, serverId: 'local' },+ ]);+ await autoSessionWatcher.initialize();+ mocks.apply.mockClear();++ for (const listener of mocks.upgraded) listener('local');++ await vi.waitFor(() => expect(mocks.apply).toHaveBeenCalledTimes(1));+ expect(mocks.apply).toHaveBeenCalledWith('upgraded', 'restore');+});diff --git a/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.ts b/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.tsindex 03f98065b..1665e536f 100644--- a/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.ts+++ b/console/apps/switch-console-desktop/src/main/core/switch-rooms/auto-session-watcher.ts@@ -1,11 +1,13 @@import { getAgentLocation } from '@main/core/agents/agent-location';import { getAgentById } from '@main/core/agents/getAgentById';import { getAgents } from '@main/core/agents/getAgents';+import { onManagedServerUpgraded } from '@main/core/managed-switch-server/session-readiness';import type { HostReachabilityChange } from '@main/core/remote-hosts/host-reachability-service';import { hostReachabilityService } from '@main/core/remote-hosts/production-host-reachability';import { disposeLocalHosts } from '@main/core/sdk-host/local-host';import { applyControllerState, configureSharedWatcher } from '@main/core/sdk-host/shared-watcher';import { log } from '@main/lib/logger';+import type { Agent } from '@shared/core/agents/agents';import {listAutoSessionAgentIds,listAutoSessionSubagents,@@ -13,8 +15,11 @@ import {setAutoSessionSubagent,} from './auto-session-store';+type Subagent = { parentAgentId: string; name: string };+class AutoSessionWatcher {private watchingHosts = false;+ private watchingUpgrades = false;private readonly recovering = new Map<string, { again: boolean }>();/**@@ -22,12 +27,58 @@ class AutoSessionWatcher {* may start sessions: an agent is reachable because it exists, and the* auto-start setting only decides what its controller does with a message it* is addressed in.+ *+ * Each server's agents are brought up on their own, because a controller+ * waits for its managed server to finish upgrading and that must not hold+ * back the agents of every other server.*/async initialize(): Promise<void> {this.watchHostRecovery();+ this.watchServerUpgrades();for (const agentId of await listAutoSessionAgentIds())if (!(await getAgentById(agentId))) await setAutoSessionAgent(agentId, false);- for (const agent of await getAgents()) {+ const agents = await getAgents();+ const subagents: Subagent[] = [];+ for (const subagent of await listAutoSessionSubagents()) {+ if (!(await getAgentById(subagent.parentAgentId))) {+ await setAutoSessionSubagent(subagent.parentAgentId, subagent.name, false);+ continue;+ }+ subagents.push(subagent);+ }+ const servers = new Set(agents.map((agent) => agent.serverId));+ await Promise.all(+ [...servers].map((serverId) => {+ const members = agents.filter((agent) => agent.serverId === serverId);+ return this.startControllers(members, subagentsOf(members, subagents));+ })+ );+ }++ /**+ * Controllers refused while their managed server owed an upgrade — a failed+ * one, or a stopped one — are started once it has finished.+ */+ private watchServerUpgrades(): void {+ if (this.watchingUpgrades) return;+ this.watchingUpgrades = true;+ onManagedServerUpgraded((serverId) => void this.restoreServer(serverId));+ }++ private async restoreServer(serverId: string): Promise<void> {+ try {+ const members = (await getAgents()).filter((agent) => agent.serverId === serverId);+ await this.startControllers(members, subagentsOf(members, await listAutoSessionSubagents()));+ } catch (error) {+ log.error('Shared SDK watchers could not start after their server updated', {+ serverId,+ error: String(error),+ });+ }+ }++ private async startControllers(agents: Agent[], subagents: Subagent[]): Promise<void> {+ for (const agent of agents) {if (!agent.switchAgentId) continue;try {// Restoring a controller an earlier run was already meant to be@@ -42,11 +93,7 @@ class AutoSessionWatcher {});}}- for (const { parentAgentId, name } of await listAutoSessionSubagents()) {- if (!(await getAgentById(parentAgentId))) {- await setAutoSessionSubagent(parentAgentId, name, false);- continue;- }+ for (const { parentAgentId, name } of subagents) {try {await this.startForSubagent(parentAgentId, name);} catch (error) {@@ -144,4 +191,9 @@ class AutoSessionWatcher {return disposeLocalHosts();}}+function subagentsOf(parents: Agent[], subagents: Subagent[]): Subagent[] {+ const ids = new Set(parents.map((agent) => agent.id));+ return subagents.filter((subagent) => ids.has(subagent.parentAgentId));+}+export const autoSessionWatcher = new AutoSessionWatcher();diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/VersionDriftNotice.tsx b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/VersionDriftNotice.tsxindex a03090307..d7d0a2c7b 100644--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/VersionDriftNotice.tsx+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/VersionDriftNotice.tsx@@ -1,19 +1,23 @@-import { ArrowUpCircle, RefreshCw, TriangleAlert } from 'lucide-react';+import { ArrowUpCircle, Play, RefreshCw, TriangleAlert } from 'lucide-react';import { Alert, AlertAction, AlertDescription, AlertTitle } from '@renderer/lib/ui/alert';import { Button } from '@renderer/lib/ui/button';+import { Spinner } from '@renderer/lib/ui/spinner';import {+ type ManagedServerUpgrade,type SwitchVersionDrift,switchVersionDowngradeMessage,} from '@shared/core/managed-switch-server/managed-switch-server';+const RESUME_NOTE = 'Sessions on this server resume once the update finishes.';+/*** Surfaces a managed stack whose switch-core version no longer matches the one* this build of Switch Console pins (CHOO-1736).*- * Updating the app moves the pin, but a stack that is already up is never- * re-provisioned — without this the user silently keeps running the old core.- * Restarting re-runs the start pipeline, which rewrites the `.env` at the new- * pin and lets `compose up -d` recreate the changed containers.+ * A stack that is behind the pin is upgraded by Console on its own, so for that+ * case this is a status rather than an offer: the update in progress, the+ * error it failed with (and a retry), or — for a stopped stack — that starting+ * it will update it. Sessions on the server wait for it either way.** The downgrade direction gets no action, only an explanation: the stack's* database has already migrated forward and switch-core cannot roll back, so@@ -21,14 +25,31 @@ import {*/export function VersionDriftNotice({drift,+ upgrade,+ progress,disabled,onRestart,}: {drift: SwitchVersionDrift | null;+ /** The upgrade the stack owes this build, if any. */+ upgrade: ManagedServerUpgrade | null;+ /** The current step of a start in flight ("Pulling images…"), or null. */+ progress: string | null;/** True while a lifecycle operation is in flight (or the host is unreachable). */disabled: boolean;+ /** Restart the stack; for an owed upgrade this is what runs it. */onRestart: () => void;}) {+ if (upgrade) {+ return (+ <ServerUpgradeNotice+ upgrade={upgrade}+ progress={progress}+ disabled={disabled}+ onRetry={onRestart}+ />+ );+ }if (!drift) return null;// We could not read what is deployed. No action offered: restarting on the@@ -58,17 +79,26 @@ export function VersionDriftNotice({);}- const unknown = drift.direction === 'unknown';+ // Behind, but no update recorded yet: the check that starts it is still+ // running.+ if (drift.direction === 'upgrade') {+ return (+ <Alert variant="warning">+ <ArrowUpCircle className="size-4" />+ <AlertTitle>{`switch-core ${drift.expected} is required`}</AlertTitle>+ <AlertDescription>+ {`Still on switch-core ${drift.deployed}. Switch Console updates this server before running sessions on it. ${RESUME_NOTE}`}+ </AlertDescription>+ </Alert>+ );+ }+return (<Alert variant="warning">- {unknown ? <TriangleAlert className="size-4" /> : <ArrowUpCircle className="size-4" />}- <AlertTitle>- {unknown ? 'Version mismatch' : `switch-core ${drift.expected} is available`}- </AlertTitle>+ <TriangleAlert className="size-4" />+ <AlertTitle>Version mismatch</AlertTitle><AlertDescription>- {unknown- ? `Runs switch-core ${drift.deployed}; this app expects ${drift.expected}. Can't tell which is newer — restart only if ${drift.expected} isn't older.`- : `Still on switch-core ${drift.deployed}. Restart to pull ${drift.expected} and migrate; rooms and messages are kept.`}+ {`Runs switch-core ${drift.deployed}; this app expects ${drift.expected}. Can't tell which is newer — restart only if ${drift.expected} isn't older.`}</AlertDescription><AlertAction><Button size="sm" disabled={disabled} onClick={onRestart}>@@ -79,3 +109,63 @@ export function VersionDriftNotice({</Alert>);}++/** The state of an upgrade Console is running (or owes) for a managed stack. */+export function ServerUpgradeNotice({+ upgrade,+ progress,+ disabled,+ onRetry,+}: {+ upgrade: ManagedServerUpgrade;+ progress: string | null;+ disabled: boolean;+ onRetry: () => void;+}) {+ const versions = `switch-core ${upgrade.from} → ${upgrade.to}`;+ if (upgrade.state === 'updating') {+ return (+ <Alert role="status" aria-live="polite">+ <Spinner size="sm" />+ <AlertTitle>{`Updating ${versions}…`}</AlertTitle>+ <AlertDescription>+ <p>{`${RESUME_NOTE} Rooms and messages are kept, and the database is backed up before it is migrated.`}</p>+ {progress && <p className="text-foreground-muted">{progress}</p>}+ </AlertDescription>+ </Alert>+ );+ }+ if (upgrade.state === 'failed') {+ return (+ <Alert variant="destructive" role="alert">+ <TriangleAlert className="size-4" />+ <AlertTitle>{`Updating ${versions} failed`}</AlertTitle>+ <AlertDescription>+ <p className="break-words">{upgrade.error}</p>+ <p>Sessions on this server stay paused until the update succeeds.</p>+ </AlertDescription>+ <AlertAction>+ <Button size="sm" disabled={disabled} onClick={onRetry}>+ <RefreshCw className="size-4" />+ Retry+ </Button>+ </AlertAction>+ </Alert>+ );+ }+ return (+ <Alert variant="warning">+ <ArrowUpCircle className="size-4" />+ <AlertTitle>{`switch-core ${upgrade.to} is required`}</AlertTitle>+ <AlertDescription>+ {`This server is stopped on switch-core ${upgrade.from}. Starting it updates it to ${upgrade.to} first. ${RESUME_NOTE}`}+ </AlertDescription>+ <AlertAction>+ <Button size="sm" disabled={disabled} onClick={onRetry}>+ <Play className="size-4" />+ Start and update+ </Button>+ </AlertAction>+ </Alert>+ );+}diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/local-server-store.ts b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/local-server-store.tsindex 1bfc55600..4210c9e7e 100644--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/local-server-store.ts+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/local-server-store.ts@@ -7,6 +7,7 @@ import type {DeployedTelemetry,DockerAvailability,LocalServerStatus,+ ManagedServerUpgrade,SwitchVersionDrift,} from '@shared/core/managed-switch-server/managed-switch-server';import {@@ -83,6 +84,11 @@ export class LocalServerStore {return this.status?.drift ?? null;}+ /** Set while the stack is behind this build's pin and not yet upgraded. */+ get upgrade(): ManagedServerUpgrade | null {+ return this.status?.upgrade ?? null;+ }+/** What the running stack is doing about usage data, or null when nothing is* up to be doing anything. */get deployedTelemetry(): DeployedTelemetry | null {diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/remote-server-store.ts b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/remote-server-store.tsindex 39d52c0d3..c5f2972a0 100644--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/remote-server-store.ts+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/remote-server-store.ts@@ -6,6 +6,7 @@ import { events, rpc } from '@renderer/lib/ipc';import type {DeployedTelemetry,DockerAvailability,+ ManagedServerUpgrade,SwitchVersionDrift,} from '@shared/core/managed-switch-server/managed-switch-server';import {@@ -25,6 +26,7 @@ function defaultStatus(sshHost: string): RemoteServerStatus {return {sshHost,phase: 'stopped',+ upgrade: null,serverId: null,version: '',deployedVersion: null,@@ -129,6 +131,11 @@ export class RemoteServerStore {return this.statusFor(sshHost).drift;}+ /** Set while the host's stack is behind this build's pin and not yet upgraded. */+ upgradeFor(sshHost: string): ManagedServerUpgrade | null {+ return this.statusFor(sshHost).upgrade;+ }+/** What the host's running stack is doing about usage data, or null when* nothing is up there to be doing anything. */deployedTelemetryFor(sshHost: string): DeployedTelemetry | null {diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/server-presentation.tsx b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/server-presentation.tsxindex 5964cb3fb..8bc48fecb 100644--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/server-presentation.tsx+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/server-presentation.tsx@@ -5,6 +5,7 @@ import { Tooltip, TooltipContent, TooltipTrigger } from '@renderer/lib/ui/tooltiimport { cn } from '@renderer/utils/utils';import type {DeployedTelemetry,+ ManagedServerUpgrade,SwitchVersionDrift,} from '@shared/core/managed-switch-server/managed-switch-server';import type { SwitchServer } from '@shared/core/switch-servers/switch-servers';@@ -93,6 +94,24 @@ export function serverDrift(server: SwitchServer): SwitchVersionDrift | null {return localServerStore.drift;}+/** The upgrade a managed server's stack owes this build, or null. */+export function serverUpgrade(server: SwitchServer): ManagedServerUpgrade | null {+ if (!server.managed) return null;+ if (server.managementKind === 'remote' && server.sshHost) {+ return remoteServerStore.upgradeFor(server.sshHost);+ }+ return localServerStore.upgrade;+}++/** The current step of a managed server's lifecycle operation, or null. */+export function serverProgress(server: SwitchServer): string | null {+ if (!server.managed) return null;+ if (server.managementKind === 'remote' && server.sshHost) {+ return remoteServerStore.statusFor(server.sshHost).message;+ }+ return localServerStore.message;+}+/*** What a managed server's running stack is doing about usage data. Null for a* server this app does not run: its operator decides, and the Console's toggle@@ -180,8 +199,8 @@ export const ServerStatusPill = observer(function ServerStatusPill({/*** Flags a managed server whose switch-core no longer matches the version this- * build pins (CHOO-1736), so an available update is visible from the switcher- * rather than only on the server's own page.+ * build pins (CHOO-1736), so an owed update is visible from the switcher rather+ * than only on the server's own page.** Sits beside the connection dot rather than recolouring it: the dot answers* "can I reach this server", which stays true of a server running a stale core.@@ -189,7 +208,7 @@ export const ServerStatusPill = observer(function ServerStatusPill({export function ServerDriftIndicator({ drift }: { drift: SwitchVersionDrift }) {const upgrade = drift.direction === 'upgrade';const label = upgrade- ? `switch-core ${drift.expected} is available (running ${drift.deployed})`+ ? `Needs switch-core ${drift.expected} (running ${drift.deployed}); sessions wait for the update`: drift.direction === 'downgrade'? `Runs switch-core ${drift.deployed} — newer than this app expects (${drift.expected})`: drift.direction === 'unreadable'diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/version-drift-notice.test.ts b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/version-drift-notice.test.tsnew file mode 100644index 000000000..ec6b5bcdb--- /dev/null+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/version-drift-notice.test.ts@@ -0,0 +1,136 @@+import { createElement } from 'react';+import { renderToStaticMarkup } from 'react-dom/server';+import { describe, expect, it, vi } from 'vitest';+import type {+ ManagedServerUpgrade,+ SwitchVersionDrift,+} from '@shared/core/managed-switch-server/managed-switch-server';+import { VersionDriftNotice } from './VersionDriftNotice';++const behind: SwitchVersionDrift = { deployed: '0.10.0', expected: '0.11.0', direction: 'upgrade' };++function render(props: {+ drift: SwitchVersionDrift | null;+ upgrade: ManagedServerUpgrade | null;+ progress?: string | null;+ disabled?: boolean;+}): string {+ return renderToStaticMarkup(+ createElement(VersionDriftNotice, {+ progress: null,+ disabled: false,+ onRestart: vi.fn(),+ ...props,+ })+ );+}++function text(html: string): string {+ return html+ .replace(/<[^>]+>/g, ' ')+ .replace(/'/g, "'")+ .replace(/\s+/g, ' ')+ .trim();+}++describe('VersionDriftNotice for a server behind the pin', () => {+ it('shows the update in progress, with its current step, and no action', () => {+ const html = render({+ drift: behind,+ upgrade: { state: 'updating', from: '0.10.0', to: '0.11.0' },+ progress: 'Backing up the database before updating switch-core 0.10.0 → 0.11.0…',+ });++ expect(text(html)).toContain('Updating switch-core 0.10.0 → 0.11.0…');+ expect(text(html)).toContain('Sessions on this server resume once the update finishes.');+ expect(text(html)).toContain('Backing up the database');+ expect(html).toContain('role="status"');+ expect(html).not.toContain('<button');+ expect(text(html)).not.toContain('Restart to update');+ });++ it('shows a failed update with its error and a Retry button', () => {+ const html = render({+ drift: behind,+ upgrade: {+ state: 'failed',+ from: '0.10.0',+ to: '0.11.0',+ error: 'pg_dumpall failed: No space left on device',+ },+ });++ expect(text(html)).toContain('Updating switch-core 0.10.0 → 0.11.0 failed');+ expect(text(html)).toContain('pg_dumpall failed: No space left on device');+ expect(html).toContain('role="alert"');+ expect(html.match(/<button[^>]*>/g)).toHaveLength(1);+ expect(text(html)).toContain('Retry');+ expect(html).not.toMatch(/<button[^>]*\sdisabled=""/);+ });++ it('disables Retry while another operation is in flight', () => {+ const html = render({+ drift: behind,+ upgrade: { state: 'failed', from: '0.10.0', to: '0.11.0', error: 'boom' },+ disabled: true,+ });++ expect(html).toMatch(/<button[^>]*\sdisabled=""/);+ });++ it('tells a stopped server it will update when started', () => {+ const html = render({+ drift: behind,+ upgrade: { state: 'pending', from: '0.10.0', to: '0.11.0' },+ });++ expect(text(html)).toContain('switch-core 0.11.0 is required');+ expect(text(html)).toContain('stopped on switch-core 0.10.0');+ expect(text(html)).toContain('Start and update');+ });++ it('never offers the old optional restart for a server that is behind', () => {+ const html = render({ drift: behind, upgrade: null });++ expect(text(html)).toContain('switch-core 0.11.0 is required');+ expect(text(html)).not.toContain('Restart to update');+ expect(html).not.toContain('<button');+ });+});++describe('VersionDriftNotice for the cases it does not upgrade', () => {+ it('still explains a downgrade with no action', () => {+ const html = render({+ drift: { deployed: '0.12.0', expected: '0.11.0', direction: 'downgrade' },+ upgrade: null,+ });++ expect(text(html)).toContain('This server is newer than Switch Console');+ expect(html).not.toContain('<button');+ });++ it('still reports an unreadable version with no action', () => {+ const html = render({+ drift: { deployed: null, expected: '0.11.0', direction: 'unreadable', reason: 'daemon down' },+ upgrade: null,+ });++ expect(text(html)).toContain("Can't tell which switch-core this is running");+ expect(text(html)).toContain('daemon down');+ expect(html).not.toContain('<button');+ });++ it('still offers a restart for an uncomparable version', () => {+ const html = render({+ drift: { deployed: 'nightly', expected: '0.11.0', direction: 'unknown' },+ upgrade: null,+ });++ expect(text(html)).toContain('Version mismatch');+ expect(text(html)).toContain('Restart to update');+ });++ it('renders nothing for a server in step', () => {+ expect(render({ drift: null, upgrade: null })).toBe('');+ });+});diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/view.tsx b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/view.tsxindex 6752580c4..7cb17f9c0 100644--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/view.tsx+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/view.tsx@@ -43,6 +43,8 @@ import {serverDeployedTelemetry,serverDrift,serverPlacementLabel,+ serverProgress,+ serverUpgrade,} from './server-presentation';import { ServerResetSection } from './server-reset-section';import { ServerSectionTitlebar } from './server-section-titlebar';@@ -285,6 +287,8 @@ const ServerMainPanel = observer(function ServerMainPanel() {rather than waiting inside the stack card further down. */}<VersionDriftNoticedrift={drift}+ upgrade={serverUpgrade(server)}+ progress={serverProgress(server)}disabled={stackTransitioning}onRestart={() => restartStack(server)}/>diff --git a/console/apps/switch-console-desktop/src/renderer/tests/browser/version-drift-notice-layout.test.tsx b/console/apps/switch-console-desktop/src/renderer/tests/browser/version-drift-notice-layout.test.tsxindex 7da6f9e3d..ee136a25e 100644--- a/console/apps/switch-console-desktop/src/renderer/tests/browser/version-drift-notice-layout.test.tsx+++ b/console/apps/switch-console-desktop/src/renderer/tests/browser/version-drift-notice-layout.test.tsx@@ -7,8 +7,9 @@ import '@renderer/index.css';/*** `AlertAction` used to be positioned absolutely over a fixed `pr-18` gutter,* which only clears the text while the action stays narrower than that gutter.- * "Restart to update" is roughly twice as wide, so the drift description ran- * underneath the button (CHOO-1736).+ * "Restart to update" was roughly twice as wide, so the drift description ran+ * underneath the button (CHOO-1736). The stopped server's "Start and update"+ * action is as wide, and shares the layout.** These assertions are on measured geometry rather than class strings: the bug* was invisible at the markup level — every class was present and correct — and@@ -28,6 +29,8 @@ async function renderNotice(deployed: string, expected: string): Promise<HTMLDivroot.render(<VersionDriftNoticedrift={{ direction: 'upgrade', deployed, expected }}+ upgrade={{ state: 'pending', from: deployed, to: expected }}+ progress={null}disabled={false}onRestart={() => {}}/>diff --git a/console/apps/switch-console-desktop/src/shared/core/managed-switch-server/managed-switch-server.ts b/console/apps/switch-console-desktop/src/shared/core/managed-switch-server/managed-switch-server.tsindex 4099d8949..c1b864c36 100644--- a/console/apps/switch-console-desktop/src/shared/core/managed-switch-server/managed-switch-server.ts+++ b/console/apps/switch-console-desktop/src/shared/core/managed-switch-server/managed-switch-server.ts@@ -124,9 +124,42 @@ export type DeployedTelemetry =| { known: true; enabled: boolean }| { known: false; reason: string };+/**+ * A managed stack that is behind this build's pin and is being moved onto it.+ *+ * Console upgrades such a stack on its own, and sessions for agents on the+ * server wait while it does, so the state is visible to both the renderer and+ * the session gate:+ *+ * - `updating` — the database is being backed up or the stack restarted at the+ * pin. Sessions wait for it to finish.+ * - `failed` — the backup or the restart failed. Sessions are refused with+ * `error` until a retry succeeds.+ * - `pending` — the stack is stopped. It is upgraded when it is next started,+ * never started at the old version.+ */+export type ManagedServerUpgrade =+ | { state: 'updating'; from: string; to: string }+ | { state: 'failed'; from: string; to: string; error: string }+ | { state: 'pending'; from: string; to: string };++/** Why a session cannot start on a managed server whose upgrade has not+ * finished. Shared so the thrown error and the UI say the same thing. */+export function managedServerUpgradeBlockedReason(+ serverName: string,+ upgrade: Exclude<ManagedServerUpgrade, { state: 'updating' }>+): string {+ if (upgrade.state === 'pending') {+ return `${serverName} is stopped on switch-core ${upgrade.from} and needs switch-core ${upgrade.to}. Start it from the server's page to update it; sessions on it start once the update finishes.`;+ }+ return `Updating ${serverName} from switch-core ${upgrade.from} to ${upgrade.to} failed: ${upgrade.error} Retry the update from the server's page; sessions on it stay paused until it succeeds.`;+}+/** Snapshot of the managed local server, emitted on every transition. */export type LocalServerStatus = {phase: LocalServerPhase;+ /** Set while the stack is behind this build's pin and not yet upgraded. */+ upgrade: ManagedServerUpgrade | null;/** The registered server's id once the stack is up, else null. */serverId: string | null;/** The pinned switch-core version this build runs. */diff --git a/deploy/local/standalone-docker-compose.yml b/deploy/local/standalone-docker-compose.ymlindex ded42cedd..9cfa0cc22 100644--- a/deploy/local/standalone-docker-compose.yml+++ b/deploy/local/standalone-docker-compose.yml@@ -213,9 +213,6 @@ services:GATEWAY_ADMIN_PASSWORD: ${GATEWAY_ADMIN_PASSWORD}FRONTEND_BASE_URL: ${FRONTEND_BASE_URL}GATEWAY_PUBLIC_URL: ${GATEWAY_PUBLIC_URL:-}- # Off by default. The recorded session behind it posts a card into a- # real channel with no session behind it, so it is opt-in per server.- SESSION_DEMO_ENABLED: ${SESSION_DEMO_ENABLED:-}MESSAGING_PUBLIC_URL: ${MESSAGING_PUBLIC_URL:-}SLACK_APP_CLIENT_ID: ${SLACK_APP_CLIENT_ID:-}SLACK_APP_CLIENT_SECRET: ${SLACK_APP_CLIENT_SECRET:-}