Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
fix(console): notice when the local server stops on its own
Simon Flacka28b50b311c9
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/health.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/health.tsindex 052e78914..53d7db84f 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/health.ts+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/health.ts@@ -20,6 +20,24 @@ const sleep = (ms: number, signal?: AbortSignal): Promise<void> =>);});+/** The readiness probe's address on a server. */+function healthUrl(serverUrl: string): string {+ return `${serverUrl}/gateway/auth/config`;+}++/**+ * One readiness probe: whether the gateway answers right now. For watching a+ * server that is already up, where a single miss is not yet a verdict.+ */+export async function probeHealth(serverUrl: string, timeoutMs = 5000): Promise<boolean> {+ try {+ const res = await fetch(healthUrl(serverUrl), { signal: AbortSignal.timeout(timeoutMs) });+ return res.ok;+ } catch {+ return false;+ }+}+/*** Poll the Switch core until its gateway answers, so we only register the server* once it can actually serve requests. `/gateway/auth/config` is unauthenticated@@ -32,7 +50,7 @@ export async function waitForHealth(): Promise<boolean> {const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;const deadline = Date.now() + timeoutMs;- const url = `${serverUrl}/gateway/auth/config`;+ const url = healthUrl(serverUrl);while (Date.now() < deadline) {try {diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-activity.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-activity.tsindex 7f4fa4fe1..ab6b2c7c6 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-activity.ts+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-activity.ts@@ -16,6 +16,7 @@ const ACTIONS: ReadonlySet<string> = new Set<LocalActivityAction>(['start-failed','updated','reset',+ 'crashed',]);function activityFile(): string {diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.test.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.test.tsindex 2f925ca7f..e4d3b21c5 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.test.ts@@ -107,7 +107,14 @@ const native = vi.hoisted(() => ({deployed: vi.fn(),launcher: vi.fn(),discard: vi.fn(async () => {}),+ failureLine: vi.fn(async (): Promise<string | null> => null),}));+const watch = vi.hoisted(() => ({+ health: vi.fn(async () => true),+ mattermostAnswers: vi.fn(async () => true),+ mattermostExpected: vi.fn(async () => false),+}));+vi.mock('./health', () => ({ probeHealth: watch.health }));vi.mock('./native/native-stack', () => ({startNativeStack: native.start,stopNativeStack: native.stop,@@ -118,6 +125,7 @@ vi.mock('./native/native-stack', () => ({readNativeTelemetry: vi.fn(async () => ({ known: true, enabled: false })),nativeMattermostLauncher: native.launcher,discardNativeData: native.discard,+ nativeCoreFailureLine: native.failureLine,}));const move = vi.hoisted(() => ({export: vi.fn(),@@ -135,6 +143,8 @@ vi.mock('./native/mattermost', () => ({readMattermostEnabled: async () => null,writeMattermostEnabled: mattermost.write,stopMattermost: mattermost.stop,+ mattermostAnswers: watch.mattermostAnswers,+ mattermostExpected: watch.mattermostExpected,}));const { LocalServerService } = await import('./local-server-service');@@ -178,6 +188,10 @@ beforeEach(() => {native.running.mockResolvedValue(false);native.deployed.mockResolvedValue({ kind: 'absent' });native.start.mockResolvedValue({ result: started, bringUpMattermost: null });+ native.failureLine.mockResolvedValue(null);+ watch.health.mockResolvedValue(true);+ watch.mattermostAnswers.mockResolvedValue(true);+ watch.mattermostExpected.mockResolvedValue(false);// Docker servers stay on Docker unless a test is about moving them.move.blocked.mockResolvedValue('not moved in this test');});@@ -717,3 +731,187 @@ it('runs the release the variable names, holding the checkout toggle off', asyncexpect(m.start).toHaveBeenCalledWith(expect.objectContaining({ checkoutRoot: null }));expect(service.getStatus().deployedVersion).toBe('0.12.0');});++/** A native server found running at boot, at this build's pin. */+async function bootRunningNative(): Promise<InstanceType<typeof LocalServerService>> {+ runtime.read.mockResolvedValue('native');+ native.running.mockResolvedValue(true);+ native.deployed.mockResolvedValue({ kind: 'deployed', version: '0.11.0', source: 'env-file' });+ const service = new LocalServerService();+ await boot(service);+ expect(service.getStatus().phase).toBe('running');+ return service;+}++it('notices a native server whose switch-core has exited, and says why', async () => {+ const service = await bootRunningNative();+ await service.watch();+ expect(service.getStatus().phase).toBe('running');++ native.running.mockResolvedValue(false);+ native.failureLine.mockResolvedValue('RuntimeError: database connection lost');+ await service.watch();++ expect(service.getStatus()).toMatchObject({+ phase: 'error',+ failedTo: 'run',+ deployedTelemetry: null,+ });+ expect(service.getStatus().error).toMatch(+ /^switch-core stopped unexpectedly: RuntimeError: database connection lost \(full log: .*switch-core\.log\)$/+ );+ expect(m.record).toHaveBeenCalledWith('crashed', service.getStatus().error);+ expect(m.emit).toHaveBeenLastCalledWith(+ expect.anything(),+ expect.objectContaining({ phase: 'error', failedTo: 'run' })+ );+});++it('marks a server that stops answering only after several misses, and back when it answers', async () => {+ const service = await bootRunningNative();+ watch.health.mockResolvedValue(false);++ await service.watch();+ await service.watch();+ expect(service.getStatus().phase).toBe('running');+ await service.watch();+ expect(service.getStatus()).toMatchObject({ phase: 'error', failedTo: 'run' });+ expect(service.getStatus().error).toMatch(/running but has not answered/);++ watch.health.mockResolvedValue(true);+ await service.watch();+ expect(service.getStatus()).toMatchObject({ phase: 'running', failedTo: null, error: null });+});++it('does not bring back a server whose process has gone', async () => {+ const service = await bootRunningNative();+ native.running.mockResolvedValue(false);+ await service.watch();+ expect(service.getStatus().phase).toBe('error');++ await service.watch();+ expect(service.getStatus()).toMatchObject({ phase: 'error', failedTo: 'run' });+});++it('restarts a server that stopped on its own like any other start', async () => {+ const service = await bootRunningNative();+ native.running.mockResolvedValue(false);+ await service.watch();++ expect(await service.start()).toEqual(started);+ expect(service.getStatus()).toMatchObject({ phase: 'running', failedTo: null, error: null });+});++it('tells a Docker server whose core container is gone from one Docker cannot be asked about', async () => {+ m.version.mockResolvedValue({ deployedVersion: '0.11.0', drift: null });+ const service = new LocalServerService();+ await boot(service);+ expect(service.getStatus().phase).toBe('running');+ watch.health.mockResolvedValue(false);++ // Docker not answering is not evidence the server has gone.+ images.mockRejectedValue(new Error('Cannot connect to the Docker daemon'));+ await service.watch();+ expect(service.getStatus().phase).toBe('running');++ images.mockResolvedValue(new Map([['postgres', 'postgres:16']]));+ await service.watch();+ expect(service.getStatus()).toMatchObject({ phase: 'error', failedTo: 'run' });+ expect(service.getStatus().error).toBe('switch-core stopped unexpectedly');+});++it('leaves a server alone while one of its own operations is running', async () => {+ const service = await bootRunningNative();+ let finish: (value: {+ result: StartLocalServerResult;+ bringUpMattermost: null;+ }) => void = () => {};+ native.start.mockImplementation(+ () =>+ new Promise((resolve) => {+ finish = resolve;+ })+ );+ const restart = service.start();+ await vi.waitFor(() => expect(native.start).toHaveBeenCalledOnce());+ native.running.mockResolvedValue(false);++ await service.watch();+ expect(watch.health).not.toHaveBeenCalled();++ native.running.mockResolvedValue(true);+ finish({ result: started, bringUpMattermost: null });+ await restart;+ expect(service.getStatus().phase).toBe('running');+});++it('ignores what it saw when a stop began while it looked', async () => {+ const service = await bootRunningNative();+ let answer: (healthy: boolean) => void = () => {};+ watch.health.mockImplementation(+ () =>+ new Promise((resolve) => {+ answer = resolve;+ })+ );+ native.running.mockResolvedValue(false);+ const looking = service.watch();+ await vi.waitFor(() => expect(watch.health).toHaveBeenCalledOnce());+ await service.stop();+ answer(false);+ await looking;++ expect(service.getStatus().phase).toBe('stopped');+ expect(m.record).not.toHaveBeenCalledWith('crashed', expect.anything());+});++it('checks the Mattermost of a server found running at launch, and notices it going', async () => {+ m.ports.mockResolvedValue({ api: 1, gateway: 2, postgres: 3, mattermost: 8065 });+ watch.mattermostExpected.mockResolvedValue(true);+ const service = await bootRunningNative();+ expect(service.getStatus().mattermostState).toBeNull();++ await service.watch();+ expect(watch.mattermostAnswers).toHaveBeenCalledWith(8065);+ expect(service.getStatus().mattermostState).toBe('running');++ watch.mattermostAnswers.mockResolvedValue(false);+ await service.watch();+ await service.watch();+ expect(service.getStatus().mattermostState).toBe('running');+ await service.watch();+ expect(service.getStatus()).toMatchObject({ phase: 'running', mattermostState: 'failed' });+ expect(service.getStatus().notice).toMatch(/Mattermost stopped answering/);++ watch.mattermostAnswers.mockResolvedValue(true);+ await service.watch();+ expect(service.getStatus()).toMatchObject({ mattermostState: 'running', notice: null });+});++it('does not look for a Mattermost that was never set up or is turned off', async () => {+ m.ports.mockResolvedValue({ api: 1, gateway: 2, postgres: 3, mattermost: 8065 });+ const service = await bootRunningNative();+ await service.watch();+ expect(watch.mattermostAnswers).not.toHaveBeenCalled();+ expect(service.getStatus().mattermostState).toBeNull();+});++it('leaves a Mattermost that failed during its start to the start to report', async () => {+ runtime.read.mockResolvedValue('native');+ m.ports.mockResolvedValue({ api: 1, gateway: 2, postgres: 3, mattermost: 8065 });+ native.start.mockResolvedValue({+ result: started,+ bringUpMattermost: async () => {+ throw new Error('seeding failed');+ },+ });+ const service = new LocalServerService();+ await boot(service);+ await service.start();+ native.running.mockResolvedValue(true);+ await vi.waitFor(() => expect(service.getStatus().mattermostState).toBe('failed'));++ await service.watch();+ expect(watch.mattermostAnswers).not.toHaveBeenCalled();+ expect(service.getStatus().notice).toMatch(/seeding failed/);+});diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.tsindex c26eecc7b..fd9e47f34 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.ts+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.ts@@ -1,3 +1,4 @@+import path from 'node:path';import { deleteAgentsForServer } from '@main/core/switch-servers/delete-server-agents';import { getManagedServer } from '@main/core/switch-servers/servers-store';import {@@ -38,6 +39,7 @@ import {} from './constants';import { readVersionStatus, versionStatusOf } from './deployed-version';import { findFreePort, type LocalServerPorts } from './free-port';+import { probeHealth } from './health';import { LocalServerHost } from './host/local-host';import type { ServerHost } from './host/types';import { readLocalActivity, recordLocalActivity } from './local-activity';@@ -48,7 +50,13 @@ import {type UpgradeJournal,upgradeState,} from './managed-upgrade';-import { readMattermostEnabled, stopMattermost, writeMattermostEnabled } from './native/mattermost';+import {+ mattermostAnswers,+ mattermostExpected,+ readMattermostEnabled,+ stopMattermost,+ writeMattermostEnabled,+} from './native/mattermost';import {type DockerExport,exportDockerStack,@@ -57,6 +65,7 @@ import {import { type NativeLayout, nativeLayout } from './native/native-paths';import {discardNativeData,+ nativeCoreFailureLine,type NativeStartOptions,type NativeStartResult,nativeMattermostLauncher,@@ -89,6 +98,16 @@ import { readDeployedTelemetry } from './telemetry-consent';* (or at its next start, if it is stopped), and {@link ensureReady} holds* sessions for its agents until that has finished.*/+/** How often a running server is looked at, to notice it stopping on its own. */+const WATCH_INTERVAL_MS = 5_000;+/** Missed probes in a row before something that is still running counts as not+ * answering: one miss is a busy moment, not a verdict. */+const MISSES_ALLOWED = 3;++const MATTERMOST_LOST_NOTICE =+ 'The server is running, but Mattermost stopped answering, so nothing reaches the server ' ++ 'from it. Restart the server to bring Mattermost back.';+function errorMessage(error: unknown): string {return error instanceof Error ? error.message : String(error);}@@ -138,6 +157,18 @@ export class LocalServerService {* upgrade finished, so that it can be told when the server is ready. */private refused = false;+ /** Watches the running server for it stopping on its own; see {@link watch}. */+ private watchTimer: ReturnType<typeof setInterval> | null = null;+ private watching = false;+ /** The {@link mattermostRun} the counters below were counted in: a start or+ * stop since begins them again. */+ private watchedRun = -1;+ private coreMisses = 0;+ private mattermostMisses = 0;+ /** Whether the watch, rather than a start, marked Mattermost failed: only+ * then does Mattermost answering again mean it is back. */+ private mattermostLost = false;+getStatus(): LocalServerStatus {return this.status;}@@ -362,11 +393,168 @@ export class LocalServerService {this.setStatus({ checked: true });host.dispose();}+ this.startWatching();// Not awaited: the boot check is done once the upgrade is under way, and a// Start click or a session joins the upgrade through `starting`.if (upgradeNow || moveNow) void this.beginStart(false);}+ private startWatching(): void {+ if (this.watchTimer !== null) return;+ this.watchTimer = setInterval(() => void this.watch(), WATCH_INTERVAL_MS);+ this.watchTimer.unref?.();+ }++ /**+ * Look at the server once, to notice it stopping without being asked.+ *+ * The phase is otherwise only ever set by this Console's own start, stop and+ * reset, so a switch-core that crashed, was killed or stopped answering would+ * go on showing as running everywhere while nothing it serves can be+ * reached. A server whose process is gone is marked so at once; one whose+ * process is there but whose gateway misses {@link MISSES_ALLOWED} probes in+ * a row is marked as not answering, and back to running if it answers again.+ * The Mattermost beside a native server is watched the same way, including+ * one found running at launch, which nothing else checks.+ *+ * Runs on a timer from boot; public so a test can drive it.+ */+ async watch(): Promise<void> {+ if (this.watching || this.busy || this.starting) return;+ const { phase, failedTo } = this.status;+ const lost = phase === 'error' && failedTo === 'run';+ if (phase !== 'running' && !lost) return;+ this.watching = true;+ const run = this.mattermostRun;+ if (run !== this.watchedRun) {+ this.watchedRun = run;+ this.coreMisses = 0;+ this.mattermostMisses = 0;+ this.mattermostLost = false;+ }+ // A start, stop or reset that began while this looked: what it saw is of+ // a server that is being replaced.+ const overtaken = () => run !== this.mattermostRun || this.busy || this.starting !== null;+ const host: ServerHost = new LocalServerHost();+ try {+ const server = await getManagedServer();+ if (!server) return;+ const native = (await this.runtime()) === 'native';+ const layout = nativeLayout(localServerDir());+ const answers = await probeHealth(server.url);+ // Null when it could not be told: Docker did not answer.+ let alive: boolean | null;+ if (native) alive = await nativeStackRunning(layout);+ else if (answers) alive = true;+ else {+ alive = await runningImages(host).then(+ (images) => images.has(CORE_SERVICE),+ () => null+ );+ }+ if (overtaken()) return;++ if (lost) {+ if (alive === true && answers) {+ log.info('local-switch-server: the server is answering again');+ this.coreMisses = 0;+ this.setStatus({+ phase: 'running',+ serverId: server.id,+ message: null,+ error: null,+ deployedTelemetry: native+ ? await readNativeTelemetry(layout)+ : await readDeployedTelemetry(host),+ });+ }+ return;+ }++ if (alive === false) {+ const reason = native ? await nativeCoreFailureLine(layout) : null;+ if (overtaken()) return;+ const where = native ? ` (full log: ${path.join(layout.logs, 'switch-core.log')})` : '';+ await this.markLost(+ `switch-core stopped unexpectedly${reason ? `: ${reason}` : ''}${where}`+ );+ return;+ }+ if (answers) {+ this.coreMisses = 0;+ } else if (++this.coreMisses >= MISSES_ALLOWED) {+ const seconds = Math.round((MISSES_ALLOWED * WATCH_INTERVAL_MS) / 1000);+ await this.markLost(+ `switch-core is running but has not answered for about ${seconds} seconds.`+ );+ return;+ } else {+ return;+ }+ if (native) await this.watchMattermost(host, layout, overtaken);+ } catch (error) {+ log.warn('local-switch-server: could not check on the server', { error: String(error) });+ } finally {+ host.dispose();+ this.watching = false;+ }+ }++ /** Say the running server has stopped, or stopped answering, on its own. */+ private async markLost(error: string): Promise<void> {+ log.warn(`local-switch-server: ${error}`);+ this.coreMisses = 0;+ this.setStatus({+ phase: 'error',+ failedTo: 'run',+ error,+ message: null,+ notice: null,+ mattermostState: null,+ deployedTelemetry: null,+ });+ await recordLocalActivity('crashed', error);+ }++ /** The Mattermost half of {@link watch}, for a native server that answers. */+ private async watchMattermost(+ host: ServerHost,+ layout: NativeLayout,+ overtaken: () => boolean+ ): Promise<void> {+ const state = this.status.mattermostState;+ // Null: running since before this launch, so nothing has looked yet.+ // Starting is the start's to report, and off is off; failed is watched+ // only when this is what said so.+ const watched =+ state === 'running' ||+ (state === null && (await mattermostExpected(layout, this.status.mattermost))) ||+ (state === 'failed' && this.mattermostLost);+ if (!watched) return;+ const ports = await readPersistedPorts(host);+ if (ports === null) return;+ const answers = await mattermostAnswers(ports.mattermost);+ if (overtaken() || this.status.phase !== 'running' || this.status.mattermostState !== state) {+ return;+ }+ if (answers) {+ this.mattermostMisses = 0;+ if (state === 'running') return;+ if (this.mattermostLost) log.info('local-switch-server: Mattermost is answering again');+ this.mattermostLost = false;+ this.setStatus({+ mattermostState: 'running',+ ...(state === 'failed' ? { notice: null } : {}),+ });+ return;+ }+ if (state === 'failed' || ++this.mattermostMisses < MISSES_ALLOWED) return;+ this.mattermostMisses = 0;+ this.mattermostLost = true;+ log.warn('local-switch-server: Mattermost is not answering');+ this.setStatus({ mattermostState: 'failed', notice: MATTERMOST_LOST_NOTICE });+ }+/*** Resolves once sessions may run against this server, waiting out the boot* check and any upgrade in flight. Throws when the server still owes an@@ -802,6 +990,8 @@ export class LocalServerService {dispose(): void {this.startAbort?.abort();this.startAbort = null;+ if (this.watchTimer !== null) clearInterval(this.watchTimer);+ this.watchTimer = null;}}diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/native/mattermost.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/native/mattermost.tsindex 2666e0b1a..5f50cd075 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/native/mattermost.ts+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/native/mattermost.ts@@ -243,6 +243,37 @@ export function pullMattermostImage(onLog: (line: string) => void): Promise<void});}+/**+ * Whether Mattermost has been brought up beside this server at some point and+ * is not turned off: what a server Console finds running at launch is+ * expected to have beside it. `enabled` is {@link readMattermostEnabled}'s+ * answer, null when the user never chose.+ */+export async function mattermostExpected(+ layout: NativeLayout,+ enabled: boolean | null+): Promise<boolean> {+ if (enabled === false) return false;+ try {+ await readFile(path.join(layout.root, COMPOSE_FILE), 'utf8');+ return true;+ } catch {+ return false;+ }+}++/** Whether the Mattermost on `port` answers. */+export async function mattermostAnswers(port: number, timeoutMs = 5000): Promise<boolean> {+ try {+ const res = await fetch(`http://localhost:${port}/api/v4/system/ping`, {+ signal: AbortSignal.timeout(timeoutMs),+ });+ return res.ok;+ } catch {+ return false;+ }+}+/** Stop Mattermost, keeping its data. A machine without Docker, or one that* never started it, has nothing to stop. */export async function stopMattermost(layout: NativeLayout, removeData: boolean): Promise<void> {diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/native/native-stack.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/native/native-stack.tsindex fde6efc09..0c8086e37 100644--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/native/native-stack.ts+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/native/native-stack.ts@@ -577,6 +577,12 @@ export async function nativeStackRunning(layout: NativeLayout): Promise<boolean>return (await runningCore(layout)) !== null;}+/** The line of switch-core's log that says why it stopped, or null. For a+ * server that stopped on its own, after it had started. */+export function nativeCoreFailureLine(layout: NativeLayout): Promise<string | null> {+ return coreFailureLine(layout);+}+export async function nativeStartedAt(layout: NativeLayout): Promise<string | null> {return (await runningCore(layout))?.startedAt ?? null;}diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/frozen-workspace.tsx b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/frozen-workspace.tsxindex 235752b8b..584c94d94 100644--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/frozen-workspace.tsx+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/frozen-workspace.tsx@@ -59,6 +59,8 @@ export function frozenSentence(server: SwitchServer, state: ManagedServerState):return `Docker isn't running, so ${name} can't start.`;case 'start-failed':return `${name} couldn't start, so agents and rooms are unavailable.`;+ case 'stopped-unexpectedly':+ return `${name} stopped unexpectedly, so agents and rooms are unavailable.`;case 'stop-failed':return `${name} didn't stop cleanly, so agents and rooms are unavailable.`;case 'reset-failed':@@ -71,8 +73,10 @@ export function frozenSentence(server: SwitchServer, state: ManagedServerState):/*** Across the top of a frozen workspace: what is wrong, the way to the server's* page, and Start when starting is all it takes. Nothing to press while it is- * already changing state, and no Start after a failure: the page says why it- * failed, and starting blind would only fail again.+ * already changing state, and no Start after a failed start: the page says why+ * it failed, and starting blind would only fail again. A server that was+ * running and stopped on its own is the exception: it has started before, so+ * Restart is the likeliest way back.*/export const FrozenBanner = observer(function FrozenBanner({ server }: { server: SwitchServer }) {const { navigate } = useNavigate();@@ -103,10 +107,10 @@ export const FrozenBanner = observer(function FrozenBanner({ server }: { server:>Server details</button>- {state.tone === 'stopped' && (+ {(state.tone === 'stopped' || state.reason === 'stopped-unexpectedly') && (<PrimaryButton onClick={() => startManagedServer(server)}><Power />- Start+ {state.reason === 'stopped-unexpectedly' ? 'Restart' : 'Start'}</PrimaryButton>)}</div>diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-state.test.ts b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-state.test.tsindex ff16e6c45..cc5fd5f43 100644--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-state.test.ts+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-state.test.ts@@ -93,4 +93,14 @@ describe('deriveManagedState', () => {'reset-failed');});++ it('says a running server that stopped on its own stopped unexpectedly, and freezes it', () => {+ expect(deriveManagedState({ ...base, phase: 'error', failedTo: 'run' })).toEqual({+ tone: 'failed',+ reason: 'stopped-unexpectedly',+ label: 'Stopped unexpectedly',+ answering: false,+ known: true,+ });+ });});diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-state.ts b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-state.tsindex 68a8b76fb..5ade194d4 100644--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-state.ts+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-state.ts@@ -35,6 +35,7 @@ export type ManagedReason =| 'downgrade'| 'docker-unavailable'| 'start-failed'+ | 'stopped-unexpectedly'| 'stop-failed'| 'reset-failed';@@ -55,6 +56,7 @@ const REASON: Record<ManagedReason, { tone: ManagedTone; label: string; answerindowngrade: { tone: 'failed', label: 'Newer than this app', answering: false },'docker-unavailable': { tone: 'failed', label: 'Docker not running', answering: false },'start-failed': { tone: 'failed', label: 'Failed to start', answering: false },+ 'stopped-unexpectedly': { tone: 'failed', label: 'Stopped unexpectedly', answering: false },'stop-failed': { tone: 'failed', label: "Couldn't stop", answering: false },'reset-failed': { tone: 'failed', label: 'Reset failed', answering: false },};@@ -88,8 +90,9 @@ export type ManagedStateInput = {/** Waiting for another Console to finish with the server. */waiting: boolean;dockerUnavailable: boolean;- /** What broke, when an error is from a stop or a reset rather than a start. */- failedTo: 'stop' | 'reset' | null;+ /** What broke, when an error is not from a start: a stop, a reset, or a+ * running server that stopped or stopped answering on its own. */+ failedTo: 'stop' | 'reset' | 'run' | null;};function stateFor(reason: ManagedReason, known: boolean): ManagedServerState {@@ -107,6 +110,7 @@ function reasonFor(input: ManagedStateInput): ManagedReason {if (phase === 'error') {if (input.failedTo === 'stop') return 'stop-failed';if (input.failedTo === 'reset') return 'reset-failed';+ if (input.failedTo === 'run') return 'stopped-unexpectedly';if (drift?.direction === 'downgrade') return 'downgrade';if (upgrade?.state === 'failed') return 'update-failed';if (input.dockerUnavailable) return 'docker-unavailable';diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-view.tsx b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-view.tsxindex 41a304704..baa4b54aa 100644--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-view.tsx+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-view.tsx@@ -75,6 +75,7 @@ const LOCAL_WHAT: Record<LocalActivityEntry['action'], (detail: string | null) =started: () => 'Started',stopped: () => 'Stopped','start-failed': () => 'Start failed',+ crashed: () => 'Stopped unexpectedly',updated: (detail) => (detail ? `Updated to ${detail}` : 'Updated'),reset: () => 'Reset',};@@ -504,6 +505,16 @@ const Actions = observer(function Actions({case 'downgrade':// Nothing safe to press: its database has already moved past this app.return logButton ? <div className="flex flex-wrap gap-2">{logButton}</div> : null;+ case 'stopped-unexpectedly':+ return (+ <div className="flex flex-wrap gap-2">+ <PrimaryButton onClick={() => startManagedServer(server)}>+ <Power />+ Restart+ </PrimaryButton>+ {logButton}+ </div>+ );case 'stop-failed':return (<div className="flex flex-wrap gap-2">@@ -680,6 +691,13 @@ const Messages = observer(function Messages({now={now}/>);+ case 'stopped-unexpectedly':+ return (+ <MessageBox tone="failed" title="The server stopped unexpectedly">+ {managedServerError(server) && <Mono>{managedServerError(server)}</Mono>}+ <p>Restart it to bring agents and rooms back. The server's data is kept.</p>+ </MessageBox>+ );case 'stop-failed':return (<MessageBox tone="failed" title="Couldn't stop">diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/view.tsx b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/view.tsxindex 274309d8f..0990e92a4 100644--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/view.tsx+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/view.tsx@@ -5,6 +5,7 @@ import {MoreVertical,Pencil,PlugZap,+ Power,RefreshCw,Trash2,Unplug,@@ -33,8 +34,12 @@ import {} from '@renderer/lib/ui/dropdown-menu';import { Spinner } from '@renderer/lib/ui/spinner';import { dashboardOrigin, type SwitchServer } from '@shared/core/switch-servers/switch-servers';+import { frozenSentence } from './frozen-workspace';import { GuardrailsSection } from './GuardrailsSection';import { localServerStore } from './local-server-store';+import { startManagedServer } from './managed-server-actions';+import { TONE_BOX, ToneDot } from './managed-server-dot';+import { managedServerState } from './managed-server-state';import { MessagingAppsCard } from './MessagingAppsCard';import { openServerPage } from './open-server-page';import { remoteServerStore } from './remote-server-store';@@ -123,7 +128,8 @@ const ServerMainPanel = observer(function ServerMainPanel() {// A managed server that isn't running has no gateway to reach — its// connection status, sign-in, and web-app links are meaningless until it's up,- // so we neither fetch nor render them (only the Start/Reset controls show).+ // so we neither fetch nor render them: a card saying it is down, with the way+ // to start it and to its status page, shows instead.const detailsVisible = !server?.managed || isManagedRunning(server);useEffect(() => {@@ -288,7 +294,9 @@ const ServerMainPanel = observer(function ServerMainPanel() {</Alert>)}- {detailsVisible && unreachable && <ServerUnreachableCard serverId={serverId} />}+ {!detailsVisible && <ManagedServerDownCard server={server} />}++ {detailsVisible && unreachable && <ServerUnreachableCard server={server} />}{detailsVisible && !unreachable && (<>@@ -400,12 +408,19 @@ const UNREACHABLE_RETRY_MS = 10_000;* gives them nothing to act on (CHOO-2042).*/const ServerUnreachableCard = observer(function ServerUnreachableCard({- serverId,+ server,}: {- serverId: string;+ server: SwitchServer;}) {const store = switchServersStore;+ const serverId = server.id;const retrying = store.refreshing.has(serverId);+ const { navigate } = useNavigate();+ // A server this Console runs is not a connection problem: the network+ // between here and it is this machine, or a host whose reachability is+ // modelled separately. Console watches its process, and the status page says+ // what that found.+ const managed = server.managed;// Keep probing while this is on screen. Waiting for the user to press// something would leave a server that recovered looking down. This only@@ -424,24 +439,84 @@ const ServerUnreachableCard = observer(function ServerUnreachableCard({<div className={`${card} flex items-center justify-between gap-3`}><div className="flex items-center gap-2 text-sm"><PlugZap className="size-4 text-amber-500" />- <span className="text-foreground">Cannot reach the server</span>+ <span className="text-foreground">+ {managed ? `${server.name} isn't answering` : 'Cannot reach the server'}+ </span><span className="text-foreground-muted">- {retrying ? 'Reconnecting…' : 'Check your connection.'}+ {retrying+ ? 'Reconnecting…'+ : managed+ ? 'Server status shows what it is doing.'+ : 'Check your connection.'}</span></div>- <Button- variant="outline"- size="sm"- disabled={retrying}- // A full refreshServer, not the lighter retryConnection the timer- // above uses: a click is one bounded request, not a forever-repeating- // one, so it can afford to also re-check sign-in options — the one- // way a persistently broken sign-in-options endpoint gets re-driven.- onClick={() => void store.refreshServer(serverId)}- >- <RefreshCw className={retrying ? 'size-4 animate-spin' : 'size-4'} />- Retry- </Button>+ <div className="flex shrink-0 items-center gap-2">+ {managed && (+ <Button+ variant="outline"+ size="sm"+ onClick={() => navigate('managedServer', { serverId })}+ >+ Server status+ </Button>+ )}+ <Button+ variant="outline"+ size="sm"+ disabled={retrying}+ // A full refreshServer, not the lighter retryConnection the timer+ // above uses: a click is one bounded request, not a forever-repeating+ // one, so it can afford to also re-check sign-in options — the one+ // way a persistently broken sign-in-options endpoint gets re-driven.+ onClick={() => void store.refreshServer(serverId)}+ >+ <RefreshCw className={retrying ? 'size-4 animate-spin' : 'size-4'} />+ Retry+ </Button>+ </div>+ </div>+ );+});++/**+ * In place of the page's details when a server this Console runs is not+ * running: what is wrong, in the words the workspace banner uses, Start (or+ * Restart, after it stopped on its own) when that is all it takes, and the way+ * to its status page, where the reason and the rest of its controls are.+ * Nothing until its state has been read: an unread state is not a stopped one.+ */+const ManagedServerDownCard = observer(function ManagedServerDownCard({+ server,+}: {+ server: SwitchServer;+}) {+ const { navigate } = useNavigate();+ const state = managedServerState(server);+ if (!state || !state.known || state.answering) return null;+ const crashed = state.reason === 'stopped-unexpectedly';+ return (+ <div+ className={`flex items-center justify-between gap-3 rounded-lg border p-4 ${TONE_BOX[state.tone]}`}+ >+ <div className="flex min-w-0 items-center gap-2 text-sm">+ <ToneDot tone={state.tone} size={7} />+ <span className="text-foreground">{frozenSentence(server, state)}</span>+ </div>+ <div className="flex shrink-0 items-center gap-2">+ <Button+ variant="outline"+ size="sm"+ onClick={() => navigate('managedServer', { serverId: server.id })}+ >+ Server status+ </Button>+ {(state.tone === 'stopped' || crashed) && (+ <Button size="sm" onClick={() => startManagedServer(server)}>+ <Power className="size-4" />+ {crashed ? 'Restart' : 'Start'}+ </Button>+ )}+ </div></div>);});diff --git a/console/apps/switch-console-desktop/src/renderer/tests/browser/server-view-local-down.test.tsx b/console/apps/switch-console-desktop/src/renderer/tests/browser/server-view-local-down.test.tsxnew file mode 100644index 000000000..67385804c--- /dev/null+++ b/console/apps/switch-console-desktop/src/renderer/tests/browser/server-view-local-down.test.tsx@@ -0,0 +1,193 @@+import type * as ReactQuery from '@tanstack/react-query';+import { act } from 'react';+import { createRoot, type Root } from 'react-dom/client';+import { afterEach, beforeEach, expect, it, vi } from 'vitest';++vi.hoisted(() => {+ window.electronAPI ??= {+ invoke: () => Promise.resolve(undefined),+ eventOn: () => () => {},+ eventSend: () => {},+ } as unknown as typeof window.electronAPI;+});++// The home page of the server on this computer when it is not answering: a+// crash shows as the server being down, with Restart and the way to its status+// page, rather than as a connection problem.+const state = vi.hoisted(() => ({+ status: null as unknown,+ unreachable: false,+ localStart: vi.fn(),+ navigate: vi.fn(),+}));++vi.mock('@renderer/lib/layout/navigation-provider', () => ({+ useParams: () => ({ params: { serverId: 'local' } }),+ useNavigate: () => ({ navigate: state.navigate }),+}));+vi.mock('@renderer/lib/modal/modal-provider', () => ({ useShowModal: () => () => {} }));+vi.mock('@tanstack/react-query', async (importOriginal) => ({+ ...(await importOriginal<typeof ReactQuery>()),+ useQueryClient: () => ({ invalidateQueries: () => Promise.resolve() }),+}));+vi.mock('@renderer/features/switch-servers/switch-servers-store', () => ({+ switchServersStore: {+ get servers() {+ return [LOCAL];+ },+ serverById: () => LOCAL,+ statusFor: () => ({ connected: true }),+ refreshing: new Set(),+ isUnreachable: () => state.unreachable,+ isHostBlocked: () => false,+ refreshStatus: () => Promise.resolve(),+ ensureAuthConfig: () => Promise.resolve(),+ refreshServer: () => Promise.resolve(),+ retryConnection: () => Promise.resolve(),+ error: null,+ errorDetail: null,+ },+}));+vi.mock('@renderer/features/switch-servers/local-server-store', () => ({+ localServerStore: {+ get status() {+ return state.status;+ },+ get isRunning() {+ return (state.status as { phase: string }).phase === 'running';+ },+ busy: false,+ docker: null,+ isNative: true,+ message: null,+ start: (...args: unknown[]) => state.localStart(...args),+ },+}));+vi.mock('@renderer/features/remote-hosts/host-reachability-store', () => ({+ hostReachabilityStore: {+ hydrate: () => Promise.resolve(),+ get: () => null,+ isBlocked: () => false,+ },+}));+vi.mock('@renderer/features/remote-hosts/host-unreachable-panel', () => ({+ HostUnreachablePanel: () => null,+}));+vi.mock('@renderer/features/locations/stores/agents-store', () => ({+ agentsStore: { load: () => Promise.resolve() },+}));+vi.mock('@renderer/features/switch-servers/switch-rooms-store', () => ({+ switchRoomsStore: { refreshRoomState: () => Promise.resolve() },+}));+vi.mock('@renderer/features/switch-servers/MessagingAppsCard', () => ({+ MessagingAppsCard: () => null,+}));+vi.mock('@renderer/features/switch-servers/GuardrailsSection', () => ({+ GuardrailsSection: () => null,+}));+vi.mock('@renderer/features/switch-servers/server-stat-tiles', () => ({+ ServerStatTiles: () => null,+}));+vi.mock('@renderer/features/switch-servers/server-sign-in', () => ({+ ServerSignInFields: () => null,+ useServerSignIn: () => ({}),+ machineUnavailableReason: () => null,+}));++import { serverView } from '@renderer/features/switch-servers/view';+import '@renderer/index.css';++const LOCAL = {+ id: 'local',+ name: 'Local server',+ url: 'http://localhost:41001',+ dashboardUrl: null,+ managed: true,+ managementKind: 'local' as const,+ sshHost: null,+ createdAt: '2026-01-01T00:00:00.000Z',+ updatedAt: '2026-01-01T00:00:00.000Z',+};++const base = {+ checked: true,+ upgrade: null,+ drift: null,+ runtime: 'native',+ error: null,+ failedTo: null,+};++let container: HTMLDivElement | null = null;+let root: Root | null = null;++beforeEach(() => {+ state.status = { ...base, phase: 'running' };+ state.unreachable = false;+ state.localStart.mockReset();+ state.navigate.mockReset();+});++afterEach(async () => {+ await act(async () => root?.unmount());+ container?.remove();+ container = null;+ root = null;+});++async function render() {+ const Page = serverView.MainPanel;+ container = document.createElement('div');+ document.body.appendChild(container);+ root = createRoot(container);+ await act(async () => root!.render(<Page />));+}++function button(label: RegExp): HTMLElement | undefined {+ return [...document.querySelectorAll('button')].find((b) =>+ label.test(b.textContent?.trim() ?? '')+ );+}++it('says the server stopped unexpectedly, and offers Restart and its status page', async () => {+ state.status = {+ ...base,+ phase: 'error',+ failedTo: 'run',+ error: 'switch-core stopped unexpectedly',+ };+ await render();++ expect(document.body.textContent).toContain('Local server stopped unexpectedly');+ expect(document.body.textContent).not.toContain('Check your connection');++ await act(async () => button(/^Restart$/)!.click());+ expect(state.localStart).toHaveBeenCalledOnce();+ await act(async () => button(/^Server status$/)!.click());+ expect(state.navigate).toHaveBeenCalledWith('managedServer', { serverId: 'local' });+});++it('offers Start for a server that was stopped', async () => {+ state.status = { ...base, phase: 'stopped' };+ await render();++ expect(document.body.textContent).toContain("Local server isn't running");+ expect(button(/^Start$/)).toBeDefined();+});++it('says nothing until the server has been looked at', async () => {+ state.status = { ...base, phase: 'stopped', checked: false };+ await render();++ expect(button(/^Start$/)).toBeUndefined();+ expect(button(/^Server status$/)).toBeUndefined();+});++it('does not blame the connection when the local server is not answering', async () => {+ state.unreachable = true;+ await render();++ expect(document.body.textContent).toContain("Local server isn't answering");+ expect(document.body.textContent).not.toContain('Check your connection');+ expect(button(/^Server status$/)).toBeDefined();+});diff --git a/console/apps/switch-console-desktop/src/shared/core/managed-switch-server/managed-switch-server.ts b/console/apps/switch-console-desktop/src/shared/core/managed-switch-server/managed-switch-server.tsindex d37b481e9..97cd3131a 100644--- a/console/apps/switch-console-desktop/src/shared/core/managed-switch-server/managed-switch-server.ts+++ b/console/apps/switch-console-desktop/src/shared/core/managed-switch-server/managed-switch-server.ts@@ -229,9 +229,10 @@ export type LocalServerStatus = {/** Populated only when `phase === 'error'`. */error: string | null;/** What failed when `phase === 'error'` and it was not a start: a stop or a- * reset that broke partway. Null for a failed start, and whenever the phase- * is anything else. */- failedTo: 'stop' | 'reset' | null;+ * reset that broke partway, or (`run`) a server that was running and then+ * stopped or stopped answering on its own. Null for a failed start, and+ * whenever the phase is anything else. */+ failedTo: 'stop' | 'reset' | 'run' | null;};/**@@ -259,7 +260,7 @@ export class ManagedServerStoppedError extends Error {/** Human-readable one-liner for a stopped managed stack, used in errors and the UI. */export function managedServerStoppedReason(serverName: string, phase: LocalServerPhase): string {if (phase === 'error') {- return `${serverName} isn't running: its last start or stop failed. See the server's page.`;+ return `${serverName} isn't running: it failed to start or stop, or stopped unexpectedly. See the server's page.`;}return `${serverName}'s Switch stack is not running. Start it from the server's page, then retry.`;}@@ -317,7 +318,14 @@ export type ConnectRemoteServerResult =| { kind: 'error'; message: string };/** Something that happened to the stack on this computer, as Console recorded it. */-export type LocalActivityAction = 'started' | 'stopped' | 'start-failed' | 'updated' | 'reset';+export type LocalActivityAction =+ | 'started'+ | 'stopped'+ | 'start-failed'+ | 'updated'+ | 'reset'+ /** The running server stopped, or stopped answering, without being asked. */+ | 'crashed';/** One entry of the local stack's history. Only this Console acts on it, so* there is no "who". */@@ -325,7 +333,8 @@ export type LocalActivityEntry = {/** ISO timestamp. */at: string;action: LocalActivityAction;- /** The version an update moved to, or why a start failed; else null. */+ /** The version an update moved to, or why a start failed or the server+ * stopped; else null. */detail: string | null;};