Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(console): notice when the local server stops on its own

Simon Flacka28b50b311c9
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/health.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/health.ts
index 052e78914..53d7db84f 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/health.ts
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/health.ts
@@ -20,6 +20,24 @@ const sleep = (ms: number, signal?: AbortSignal): Promise<void> =>
);
});
+/** The readiness probe's address on a server. */
+function healthUrl(serverUrl: string): string {
+ return `${serverUrl}/gateway/auth/config`;
+}
+
+/**
+ * One readiness probe: whether the gateway answers right now. For watching a
+ * server that is already up, where a single miss is not yet a verdict.
+ */
+export async function probeHealth(serverUrl: string, timeoutMs = 5000): Promise<boolean> {
+ try {
+ const res = await fetch(healthUrl(serverUrl), { signal: AbortSignal.timeout(timeoutMs) });
+ return res.ok;
+ } catch {
+ return false;
+ }
+}
+
/**
* Poll the Switch core until its gateway answers, so we only register the server
* once it can actually serve requests. `/gateway/auth/config` is unauthenticated
@@ -32,7 +50,7 @@ export async function waitForHealth(
): Promise<boolean> {
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
const deadline = Date.now() + timeoutMs;
- const url = `${serverUrl}/gateway/auth/config`;
+ const url = healthUrl(serverUrl);
while (Date.now() < deadline) {
try {
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-activity.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-activity.ts
index 7f4fa4fe1..ab6b2c7c6 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-activity.ts
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-activity.ts
@@ -16,6 +16,7 @@ const ACTIONS: ReadonlySet<string> = new Set<LocalActivityAction>([
'start-failed',
'updated',
'reset',
+ 'crashed',
]);
function activityFile(): string {
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.test.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.test.ts
index 2f925ca7f..e4d3b21c5 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.test.ts
@@ -107,7 +107,14 @@ const native = vi.hoisted(() => ({
deployed: vi.fn(),
launcher: vi.fn(),
discard: vi.fn(async () => {}),
+ failureLine: vi.fn(async (): Promise<string | null> => null),
}));
+const watch = vi.hoisted(() => ({
+ health: vi.fn(async () => true),
+ mattermostAnswers: vi.fn(async () => true),
+ mattermostExpected: vi.fn(async () => false),
+}));
+vi.mock('./health', () => ({ probeHealth: watch.health }));
vi.mock('./native/native-stack', () => ({
startNativeStack: native.start,
stopNativeStack: native.stop,
@@ -118,6 +125,7 @@ vi.mock('./native/native-stack', () => ({
readNativeTelemetry: vi.fn(async () => ({ known: true, enabled: false })),
nativeMattermostLauncher: native.launcher,
discardNativeData: native.discard,
+ nativeCoreFailureLine: native.failureLine,
}));
const move = vi.hoisted(() => ({
export: vi.fn(),
@@ -135,6 +143,8 @@ vi.mock('./native/mattermost', () => ({
readMattermostEnabled: async () => null,
writeMattermostEnabled: mattermost.write,
stopMattermost: mattermost.stop,
+ mattermostAnswers: watch.mattermostAnswers,
+ mattermostExpected: watch.mattermostExpected,
}));
const { LocalServerService } = await import('./local-server-service');
@@ -178,6 +188,10 @@ beforeEach(() => {
native.running.mockResolvedValue(false);
native.deployed.mockResolvedValue({ kind: 'absent' });
native.start.mockResolvedValue({ result: started, bringUpMattermost: null });
+ native.failureLine.mockResolvedValue(null);
+ watch.health.mockResolvedValue(true);
+ watch.mattermostAnswers.mockResolvedValue(true);
+ watch.mattermostExpected.mockResolvedValue(false);
// Docker servers stay on Docker unless a test is about moving them.
move.blocked.mockResolvedValue('not moved in this test');
});
@@ -717,3 +731,187 @@ it('runs the release the variable names, holding the checkout toggle off', async
expect(m.start).toHaveBeenCalledWith(expect.objectContaining({ checkoutRoot: null }));
expect(service.getStatus().deployedVersion).toBe('0.12.0');
});
+
+/** A native server found running at boot, at this build's pin. */
+async function bootRunningNative(): Promise<InstanceType<typeof LocalServerService>> {
+ runtime.read.mockResolvedValue('native');
+ native.running.mockResolvedValue(true);
+ native.deployed.mockResolvedValue({ kind: 'deployed', version: '0.11.0', source: 'env-file' });
+ const service = new LocalServerService();
+ await boot(service);
+ expect(service.getStatus().phase).toBe('running');
+ return service;
+}
+
+it('notices a native server whose switch-core has exited, and says why', async () => {
+ const service = await bootRunningNative();
+ await service.watch();
+ expect(service.getStatus().phase).toBe('running');
+
+ native.running.mockResolvedValue(false);
+ native.failureLine.mockResolvedValue('RuntimeError: database connection lost');
+ await service.watch();
+
+ expect(service.getStatus()).toMatchObject({
+ phase: 'error',
+ failedTo: 'run',
+ deployedTelemetry: null,
+ });
+ expect(service.getStatus().error).toMatch(
+ /^switch-core stopped unexpectedly: RuntimeError: database connection lost \(full log: .*switch-core\.log\)$/
+ );
+ expect(m.record).toHaveBeenCalledWith('crashed', service.getStatus().error);
+ expect(m.emit).toHaveBeenLastCalledWith(
+ expect.anything(),
+ expect.objectContaining({ phase: 'error', failedTo: 'run' })
+ );
+});
+
+it('marks a server that stops answering only after several misses, and back when it answers', async () => {
+ const service = await bootRunningNative();
+ watch.health.mockResolvedValue(false);
+
+ await service.watch();
+ await service.watch();
+ expect(service.getStatus().phase).toBe('running');
+ await service.watch();
+ expect(service.getStatus()).toMatchObject({ phase: 'error', failedTo: 'run' });
+ expect(service.getStatus().error).toMatch(/running but has not answered/);
+
+ watch.health.mockResolvedValue(true);
+ await service.watch();
+ expect(service.getStatus()).toMatchObject({ phase: 'running', failedTo: null, error: null });
+});
+
+it('does not bring back a server whose process has gone', async () => {
+ const service = await bootRunningNative();
+ native.running.mockResolvedValue(false);
+ await service.watch();
+ expect(service.getStatus().phase).toBe('error');
+
+ await service.watch();
+ expect(service.getStatus()).toMatchObject({ phase: 'error', failedTo: 'run' });
+});
+
+it('restarts a server that stopped on its own like any other start', async () => {
+ const service = await bootRunningNative();
+ native.running.mockResolvedValue(false);
+ await service.watch();
+
+ expect(await service.start()).toEqual(started);
+ expect(service.getStatus()).toMatchObject({ phase: 'running', failedTo: null, error: null });
+});
+
+it('tells a Docker server whose core container is gone from one Docker cannot be asked about', async () => {
+ m.version.mockResolvedValue({ deployedVersion: '0.11.0', drift: null });
+ const service = new LocalServerService();
+ await boot(service);
+ expect(service.getStatus().phase).toBe('running');
+ watch.health.mockResolvedValue(false);
+
+ // Docker not answering is not evidence the server has gone.
+ images.mockRejectedValue(new Error('Cannot connect to the Docker daemon'));
+ await service.watch();
+ expect(service.getStatus().phase).toBe('running');
+
+ images.mockResolvedValue(new Map([['postgres', 'postgres:16']]));
+ await service.watch();
+ expect(service.getStatus()).toMatchObject({ phase: 'error', failedTo: 'run' });
+ expect(service.getStatus().error).toBe('switch-core stopped unexpectedly');
+});
+
+it('leaves a server alone while one of its own operations is running', async () => {
+ const service = await bootRunningNative();
+ let finish: (value: {
+ result: StartLocalServerResult;
+ bringUpMattermost: null;
+ }) => void = () => {};
+ native.start.mockImplementation(
+ () =>
+ new Promise((resolve) => {
+ finish = resolve;
+ })
+ );
+ const restart = service.start();
+ await vi.waitFor(() => expect(native.start).toHaveBeenCalledOnce());
+ native.running.mockResolvedValue(false);
+
+ await service.watch();
+ expect(watch.health).not.toHaveBeenCalled();
+
+ native.running.mockResolvedValue(true);
+ finish({ result: started, bringUpMattermost: null });
+ await restart;
+ expect(service.getStatus().phase).toBe('running');
+});
+
+it('ignores what it saw when a stop began while it looked', async () => {
+ const service = await bootRunningNative();
+ let answer: (healthy: boolean) => void = () => {};
+ watch.health.mockImplementation(
+ () =>
+ new Promise((resolve) => {
+ answer = resolve;
+ })
+ );
+ native.running.mockResolvedValue(false);
+ const looking = service.watch();
+ await vi.waitFor(() => expect(watch.health).toHaveBeenCalledOnce());
+ await service.stop();
+ answer(false);
+ await looking;
+
+ expect(service.getStatus().phase).toBe('stopped');
+ expect(m.record).not.toHaveBeenCalledWith('crashed', expect.anything());
+});
+
+it('checks the Mattermost of a server found running at launch, and notices it going', async () => {
+ m.ports.mockResolvedValue({ api: 1, gateway: 2, postgres: 3, mattermost: 8065 });
+ watch.mattermostExpected.mockResolvedValue(true);
+ const service = await bootRunningNative();
+ expect(service.getStatus().mattermostState).toBeNull();
+
+ await service.watch();
+ expect(watch.mattermostAnswers).toHaveBeenCalledWith(8065);
+ expect(service.getStatus().mattermostState).toBe('running');
+
+ watch.mattermostAnswers.mockResolvedValue(false);
+ await service.watch();
+ await service.watch();
+ expect(service.getStatus().mattermostState).toBe('running');
+ await service.watch();
+ expect(service.getStatus()).toMatchObject({ phase: 'running', mattermostState: 'failed' });
+ expect(service.getStatus().notice).toMatch(/Mattermost stopped answering/);
+
+ watch.mattermostAnswers.mockResolvedValue(true);
+ await service.watch();
+ expect(service.getStatus()).toMatchObject({ mattermostState: 'running', notice: null });
+});
+
+it('does not look for a Mattermost that was never set up or is turned off', async () => {
+ m.ports.mockResolvedValue({ api: 1, gateway: 2, postgres: 3, mattermost: 8065 });
+ const service = await bootRunningNative();
+ await service.watch();
+ expect(watch.mattermostAnswers).not.toHaveBeenCalled();
+ expect(service.getStatus().mattermostState).toBeNull();
+});
+
+it('leaves a Mattermost that failed during its start to the start to report', async () => {
+ runtime.read.mockResolvedValue('native');
+ m.ports.mockResolvedValue({ api: 1, gateway: 2, postgres: 3, mattermost: 8065 });
+ native.start.mockResolvedValue({
+ result: started,
+ bringUpMattermost: async () => {
+ throw new Error('seeding failed');
+ },
+ });
+ const service = new LocalServerService();
+ await boot(service);
+ await service.start();
+ native.running.mockResolvedValue(true);
+ await vi.waitFor(() => expect(service.getStatus().mattermostState).toBe('failed'));
+
+ await service.watch();
+ expect(watch.mattermostAnswers).not.toHaveBeenCalled();
+ expect(service.getStatus().notice).toMatch(/seeding failed/);
+});
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.ts
index c26eecc7b..fd9e47f34 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.ts
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/local-server-service.ts
@@ -1,3 +1,4 @@
+import path from 'node:path';
import { deleteAgentsForServer } from '@main/core/switch-servers/delete-server-agents';
import { getManagedServer } from '@main/core/switch-servers/servers-store';
import {
@@ -38,6 +39,7 @@ import {
} from './constants';
import { readVersionStatus, versionStatusOf } from './deployed-version';
import { findFreePort, type LocalServerPorts } from './free-port';
+import { probeHealth } from './health';
import { LocalServerHost } from './host/local-host';
import type { ServerHost } from './host/types';
import { readLocalActivity, recordLocalActivity } from './local-activity';
@@ -48,7 +50,13 @@ import {
type UpgradeJournal,
upgradeState,
} from './managed-upgrade';
-import { readMattermostEnabled, stopMattermost, writeMattermostEnabled } from './native/mattermost';
+import {
+ mattermostAnswers,
+ mattermostExpected,
+ readMattermostEnabled,
+ stopMattermost,
+ writeMattermostEnabled,
+} from './native/mattermost';
import {
type DockerExport,
exportDockerStack,
@@ -57,6 +65,7 @@ import {
import { type NativeLayout, nativeLayout } from './native/native-paths';
import {
discardNativeData,
+ nativeCoreFailureLine,
type NativeStartOptions,
type NativeStartResult,
nativeMattermostLauncher,
@@ -89,6 +98,16 @@ import { readDeployedTelemetry } from './telemetry-consent';
* (or at its next start, if it is stopped), and {@link ensureReady} holds
* sessions for its agents until that has finished.
*/
+/** How often a running server is looked at, to notice it stopping on its own. */
+const WATCH_INTERVAL_MS = 5_000;
+/** Missed probes in a row before something that is still running counts as not
+ * answering: one miss is a busy moment, not a verdict. */
+const MISSES_ALLOWED = 3;
+
+const MATTERMOST_LOST_NOTICE =
+ 'The server is running, but Mattermost stopped answering, so nothing reaches the server ' +
+ 'from it. Restart the server to bring Mattermost back.';
+
function errorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
@@ -138,6 +157,18 @@ export class LocalServerService {
* upgrade finished, so that it can be told when the server is ready. */
private refused = false;
+ /** Watches the running server for it stopping on its own; see {@link watch}. */
+ private watchTimer: ReturnType<typeof setInterval> | null = null;
+ private watching = false;
+ /** The {@link mattermostRun} the counters below were counted in: a start or
+ * stop since begins them again. */
+ private watchedRun = -1;
+ private coreMisses = 0;
+ private mattermostMisses = 0;
+ /** Whether the watch, rather than a start, marked Mattermost failed: only
+ * then does Mattermost answering again mean it is back. */
+ private mattermostLost = false;
+
getStatus(): LocalServerStatus {
return this.status;
}
@@ -362,11 +393,168 @@ export class LocalServerService {
this.setStatus({ checked: true });
host.dispose();
}
+ this.startWatching();
// Not awaited: the boot check is done once the upgrade is under way, and a
// Start click or a session joins the upgrade through `starting`.
if (upgradeNow || moveNow) void this.beginStart(false);
}
+ private startWatching(): void {
+ if (this.watchTimer !== null) return;
+ this.watchTimer = setInterval(() => void this.watch(), WATCH_INTERVAL_MS);
+ this.watchTimer.unref?.();
+ }
+
+ /**
+ * Look at the server once, to notice it stopping without being asked.
+ *
+ * The phase is otherwise only ever set by this Console's own start, stop and
+ * reset, so a switch-core that crashed, was killed or stopped answering would
+ * go on showing as running everywhere while nothing it serves can be
+ * reached. A server whose process is gone is marked so at once; one whose
+ * process is there but whose gateway misses {@link MISSES_ALLOWED} probes in
+ * a row is marked as not answering, and back to running if it answers again.
+ * The Mattermost beside a native server is watched the same way, including
+ * one found running at launch, which nothing else checks.
+ *
+ * Runs on a timer from boot; public so a test can drive it.
+ */
+ async watch(): Promise<void> {
+ if (this.watching || this.busy || this.starting) return;
+ const { phase, failedTo } = this.status;
+ const lost = phase === 'error' && failedTo === 'run';
+ if (phase !== 'running' && !lost) return;
+ this.watching = true;
+ const run = this.mattermostRun;
+ if (run !== this.watchedRun) {
+ this.watchedRun = run;
+ this.coreMisses = 0;
+ this.mattermostMisses = 0;
+ this.mattermostLost = false;
+ }
+ // A start, stop or reset that began while this looked: what it saw is of
+ // a server that is being replaced.
+ const overtaken = () => run !== this.mattermostRun || this.busy || this.starting !== null;
+ const host: ServerHost = new LocalServerHost();
+ try {
+ const server = await getManagedServer();
+ if (!server) return;
+ const native = (await this.runtime()) === 'native';
+ const layout = nativeLayout(localServerDir());
+ const answers = await probeHealth(server.url);
+ // Null when it could not be told: Docker did not answer.
+ let alive: boolean | null;
+ if (native) alive = await nativeStackRunning(layout);
+ else if (answers) alive = true;
+ else {
+ alive = await runningImages(host).then(
+ (images) => images.has(CORE_SERVICE),
+ () => null
+ );
+ }
+ if (overtaken()) return;
+
+ if (lost) {
+ if (alive === true && answers) {
+ log.info('local-switch-server: the server is answering again');
+ this.coreMisses = 0;
+ this.setStatus({
+ phase: 'running',
+ serverId: server.id,
+ message: null,
+ error: null,
+ deployedTelemetry: native
+ ? await readNativeTelemetry(layout)
+ : await readDeployedTelemetry(host),
+ });
+ }
+ return;
+ }
+
+ if (alive === false) {
+ const reason = native ? await nativeCoreFailureLine(layout) : null;
+ if (overtaken()) return;
+ const where = native ? ` (full log: ${path.join(layout.logs, 'switch-core.log')})` : '';
+ await this.markLost(
+ `switch-core stopped unexpectedly${reason ? `: ${reason}` : ''}${where}`
+ );
+ return;
+ }
+ if (answers) {
+ this.coreMisses = 0;
+ } else if (++this.coreMisses >= MISSES_ALLOWED) {
+ const seconds = Math.round((MISSES_ALLOWED * WATCH_INTERVAL_MS) / 1000);
+ await this.markLost(
+ `switch-core is running but has not answered for about ${seconds} seconds.`
+ );
+ return;
+ } else {
+ return;
+ }
+ if (native) await this.watchMattermost(host, layout, overtaken);
+ } catch (error) {
+ log.warn('local-switch-server: could not check on the server', { error: String(error) });
+ } finally {
+ host.dispose();
+ this.watching = false;
+ }
+ }
+
+ /** Say the running server has stopped, or stopped answering, on its own. */
+ private async markLost(error: string): Promise<void> {
+ log.warn(`local-switch-server: ${error}`);
+ this.coreMisses = 0;
+ this.setStatus({
+ phase: 'error',
+ failedTo: 'run',
+ error,
+ message: null,
+ notice: null,
+ mattermostState: null,
+ deployedTelemetry: null,
+ });
+ await recordLocalActivity('crashed', error);
+ }
+
+ /** The Mattermost half of {@link watch}, for a native server that answers. */
+ private async watchMattermost(
+ host: ServerHost,
+ layout: NativeLayout,
+ overtaken: () => boolean
+ ): Promise<void> {
+ const state = this.status.mattermostState;
+ // Null: running since before this launch, so nothing has looked yet.
+ // Starting is the start's to report, and off is off; failed is watched
+ // only when this is what said so.
+ const watched =
+ state === 'running' ||
+ (state === null && (await mattermostExpected(layout, this.status.mattermost))) ||
+ (state === 'failed' && this.mattermostLost);
+ if (!watched) return;
+ const ports = await readPersistedPorts(host);
+ if (ports === null) return;
+ const answers = await mattermostAnswers(ports.mattermost);
+ if (overtaken() || this.status.phase !== 'running' || this.status.mattermostState !== state) {
+ return;
+ }
+ if (answers) {
+ this.mattermostMisses = 0;
+ if (state === 'running') return;
+ if (this.mattermostLost) log.info('local-switch-server: Mattermost is answering again');
+ this.mattermostLost = false;
+ this.setStatus({
+ mattermostState: 'running',
+ ...(state === 'failed' ? { notice: null } : {}),
+ });
+ return;
+ }
+ if (state === 'failed' || ++this.mattermostMisses < MISSES_ALLOWED) return;
+ this.mattermostMisses = 0;
+ this.mattermostLost = true;
+ log.warn('local-switch-server: Mattermost is not answering');
+ this.setStatus({ mattermostState: 'failed', notice: MATTERMOST_LOST_NOTICE });
+ }
+
/**
* Resolves once sessions may run against this server, waiting out the boot
* check and any upgrade in flight. Throws when the server still owes an
@@ -802,6 +990,8 @@ export class LocalServerService {
dispose(): void {
this.startAbort?.abort();
this.startAbort = null;
+ if (this.watchTimer !== null) clearInterval(this.watchTimer);
+ this.watchTimer = null;
}
}
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/native/mattermost.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/native/mattermost.ts
index 2666e0b1a..5f50cd075 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/native/mattermost.ts
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/native/mattermost.ts
@@ -243,6 +243,37 @@ export function pullMattermostImage(onLog: (line: string) => void): Promise<void
});
}
+/**
+ * Whether Mattermost has been brought up beside this server at some point and
+ * is not turned off: what a server Console finds running at launch is
+ * expected to have beside it. `enabled` is {@link readMattermostEnabled}'s
+ * answer, null when the user never chose.
+ */
+export async function mattermostExpected(
+ layout: NativeLayout,
+ enabled: boolean | null
+): Promise<boolean> {
+ if (enabled === false) return false;
+ try {
+ await readFile(path.join(layout.root, COMPOSE_FILE), 'utf8');
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+/** Whether the Mattermost on `port` answers. */
+export async function mattermostAnswers(port: number, timeoutMs = 5000): Promise<boolean> {
+ try {
+ const res = await fetch(`http://localhost:${port}/api/v4/system/ping`, {
+ signal: AbortSignal.timeout(timeoutMs),
+ });
+ return res.ok;
+ } catch {
+ return false;
+ }
+}
+
/** Stop Mattermost, keeping its data. A machine without Docker, or one that
* never started it, has nothing to stop. */
export async function stopMattermost(layout: NativeLayout, removeData: boolean): Promise<void> {
diff --git a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/native/native-stack.ts b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/native/native-stack.ts
index fde6efc09..0c8086e37 100644
--- a/console/apps/switch-console-desktop/src/main/core/managed-switch-server/native/native-stack.ts
+++ b/console/apps/switch-console-desktop/src/main/core/managed-switch-server/native/native-stack.ts
@@ -577,6 +577,12 @@ export async function nativeStackRunning(layout: NativeLayout): Promise<boolean>
return (await runningCore(layout)) !== null;
}
+/** The line of switch-core's log that says why it stopped, or null. For a
+ * server that stopped on its own, after it had started. */
+export function nativeCoreFailureLine(layout: NativeLayout): Promise<string | null> {
+ return coreFailureLine(layout);
+}
+
export async function nativeStartedAt(layout: NativeLayout): Promise<string | null> {
return (await runningCore(layout))?.startedAt ?? null;
}
diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/frozen-workspace.tsx b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/frozen-workspace.tsx
index 235752b8b..584c94d94 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/frozen-workspace.tsx
+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/frozen-workspace.tsx
@@ -59,6 +59,8 @@ export function frozenSentence(server: SwitchServer, state: ManagedServerState):
return `Docker isn't running, so ${name} can't start.`;
case 'start-failed':
return `${name} couldn't start, so agents and rooms are unavailable.`;
+ case 'stopped-unexpectedly':
+ return `${name} stopped unexpectedly, so agents and rooms are unavailable.`;
case 'stop-failed':
return `${name} didn't stop cleanly, so agents and rooms are unavailable.`;
case 'reset-failed':
@@ -71,8 +73,10 @@ export function frozenSentence(server: SwitchServer, state: ManagedServerState):
/**
* Across the top of a frozen workspace: what is wrong, the way to the server's
* page, and Start when starting is all it takes. Nothing to press while it is
- * already changing state, and no Start after a failure: the page says why it
- * failed, and starting blind would only fail again.
+ * already changing state, and no Start after a failed start: the page says why
+ * it failed, and starting blind would only fail again. A server that was
+ * running and stopped on its own is the exception: it has started before, so
+ * Restart is the likeliest way back.
*/
export const FrozenBanner = observer(function FrozenBanner({ server }: { server: SwitchServer }) {
const { navigate } = useNavigate();
@@ -103,10 +107,10 @@ export const FrozenBanner = observer(function FrozenBanner({ server }: { server:
>
Server details
</button>
- {state.tone === 'stopped' && (
+ {(state.tone === 'stopped' || state.reason === 'stopped-unexpectedly') && (
<PrimaryButton onClick={() => startManagedServer(server)}>
<Power />
- Start
+ {state.reason === 'stopped-unexpectedly' ? 'Restart' : 'Start'}
</PrimaryButton>
)}
</div>
diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-state.test.ts b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-state.test.ts
index ff16e6c45..cc5fd5f43 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-state.test.ts
+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-state.test.ts
@@ -93,4 +93,14 @@ describe('deriveManagedState', () => {
'reset-failed'
);
});
+
+ it('says a running server that stopped on its own stopped unexpectedly, and freezes it', () => {
+ expect(deriveManagedState({ ...base, phase: 'error', failedTo: 'run' })).toEqual({
+ tone: 'failed',
+ reason: 'stopped-unexpectedly',
+ label: 'Stopped unexpectedly',
+ answering: false,
+ known: true,
+ });
+ });
});
diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-state.ts b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-state.ts
index 68a8b76fb..5ade194d4 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-state.ts
+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-state.ts
@@ -35,6 +35,7 @@ export type ManagedReason =
| 'downgrade'
| 'docker-unavailable'
| 'start-failed'
+ | 'stopped-unexpectedly'
| 'stop-failed'
| 'reset-failed';
@@ -55,6 +56,7 @@ const REASON: Record<ManagedReason, { tone: ManagedTone; label: string; answerin
downgrade: { tone: 'failed', label: 'Newer than this app', answering: false },
'docker-unavailable': { tone: 'failed', label: 'Docker not running', answering: false },
'start-failed': { tone: 'failed', label: 'Failed to start', answering: false },
+ 'stopped-unexpectedly': { tone: 'failed', label: 'Stopped unexpectedly', answering: false },
'stop-failed': { tone: 'failed', label: "Couldn't stop", answering: false },
'reset-failed': { tone: 'failed', label: 'Reset failed', answering: false },
};
@@ -88,8 +90,9 @@ export type ManagedStateInput = {
/** Waiting for another Console to finish with the server. */
waiting: boolean;
dockerUnavailable: boolean;
- /** What broke, when an error is from a stop or a reset rather than a start. */
- failedTo: 'stop' | 'reset' | null;
+ /** What broke, when an error is not from a start: a stop, a reset, or a
+ * running server that stopped or stopped answering on its own. */
+ failedTo: 'stop' | 'reset' | 'run' | null;
};
function stateFor(reason: ManagedReason, known: boolean): ManagedServerState {
@@ -107,6 +110,7 @@ function reasonFor(input: ManagedStateInput): ManagedReason {
if (phase === 'error') {
if (input.failedTo === 'stop') return 'stop-failed';
if (input.failedTo === 'reset') return 'reset-failed';
+ if (input.failedTo === 'run') return 'stopped-unexpectedly';
if (drift?.direction === 'downgrade') return 'downgrade';
if (upgrade?.state === 'failed') return 'update-failed';
if (input.dockerUnavailable) return 'docker-unavailable';
diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-view.tsx b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-view.tsx
index 41a304704..baa4b54aa 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-view.tsx
+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/managed-server-view.tsx
@@ -75,6 +75,7 @@ const LOCAL_WHAT: Record<LocalActivityEntry['action'], (detail: string | null) =
started: () => 'Started',
stopped: () => 'Stopped',
'start-failed': () => 'Start failed',
+ crashed: () => 'Stopped unexpectedly',
updated: (detail) => (detail ? `Updated to ${detail}` : 'Updated'),
reset: () => 'Reset',
};
@@ -504,6 +505,16 @@ const Actions = observer(function Actions({
case 'downgrade':
// Nothing safe to press: its database has already moved past this app.
return logButton ? <div className="flex flex-wrap gap-2">{logButton}</div> : null;
+ case 'stopped-unexpectedly':
+ return (
+ <div className="flex flex-wrap gap-2">
+ <PrimaryButton onClick={() => startManagedServer(server)}>
+ <Power />
+ Restart
+ </PrimaryButton>
+ {logButton}
+ </div>
+ );
case 'stop-failed':
return (
<div className="flex flex-wrap gap-2">
@@ -680,6 +691,13 @@ const Messages = observer(function Messages({
now={now}
/>
);
+ case 'stopped-unexpectedly':
+ return (
+ <MessageBox tone="failed" title="The server stopped unexpectedly">
+ {managedServerError(server) && <Mono>{managedServerError(server)}</Mono>}
+ <p>Restart it to bring agents and rooms back. The server's data is kept.</p>
+ </MessageBox>
+ );
case 'stop-failed':
return (
<MessageBox tone="failed" title="Couldn't stop">
diff --git a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/view.tsx b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/view.tsx
index 274309d8f..0990e92a4 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/switch-servers/view.tsx
+++ b/console/apps/switch-console-desktop/src/renderer/features/switch-servers/view.tsx
@@ -5,6 +5,7 @@ import {
MoreVertical,
Pencil,
PlugZap,
+ Power,
RefreshCw,
Trash2,
Unplug,
@@ -33,8 +34,12 @@ import {
} from '@renderer/lib/ui/dropdown-menu';
import { Spinner } from '@renderer/lib/ui/spinner';
import { dashboardOrigin, type SwitchServer } from '@shared/core/switch-servers/switch-servers';
+import { frozenSentence } from './frozen-workspace';
import { GuardrailsSection } from './GuardrailsSection';
import { localServerStore } from './local-server-store';
+import { startManagedServer } from './managed-server-actions';
+import { TONE_BOX, ToneDot } from './managed-server-dot';
+import { managedServerState } from './managed-server-state';
import { MessagingAppsCard } from './MessagingAppsCard';
import { openServerPage } from './open-server-page';
import { remoteServerStore } from './remote-server-store';
@@ -123,7 +128,8 @@ const ServerMainPanel = observer(function ServerMainPanel() {
// A managed server that isn't running has no gateway to reach — its
// connection status, sign-in, and web-app links are meaningless until it's up,
- // so we neither fetch nor render them (only the Start/Reset controls show).
+ // so we neither fetch nor render them: a card saying it is down, with the way
+ // to start it and to its status page, shows instead.
const detailsVisible = !server?.managed || isManagedRunning(server);
useEffect(() => {
@@ -288,7 +294,9 @@ const ServerMainPanel = observer(function ServerMainPanel() {
</Alert>
)}
- {detailsVisible && unreachable && <ServerUnreachableCard serverId={serverId} />}
+ {!detailsVisible && <ManagedServerDownCard server={server} />}
+
+ {detailsVisible && unreachable && <ServerUnreachableCard server={server} />}
{detailsVisible && !unreachable && (
<>
@@ -400,12 +408,19 @@ const UNREACHABLE_RETRY_MS = 10_000;
* gives them nothing to act on (CHOO-2042).
*/
const ServerUnreachableCard = observer(function ServerUnreachableCard({
- serverId,
+ server,
}: {
- serverId: string;
+ server: SwitchServer;
}) {
const store = switchServersStore;
+ const serverId = server.id;
const retrying = store.refreshing.has(serverId);
+ const { navigate } = useNavigate();
+ // A server this Console runs is not a connection problem: the network
+ // between here and it is this machine, or a host whose reachability is
+ // modelled separately. Console watches its process, and the status page says
+ // what that found.
+ const managed = server.managed;
// Keep probing while this is on screen. Waiting for the user to press
// something would leave a server that recovered looking down. This only
@@ -424,24 +439,84 @@ const ServerUnreachableCard = observer(function ServerUnreachableCard({
<div className={`${card} flex items-center justify-between gap-3`}>
<div className="flex items-center gap-2 text-sm">
<PlugZap className="size-4 text-amber-500" />
- <span className="text-foreground">Cannot reach the server</span>
+ <span className="text-foreground">
+ {managed ? `${server.name} isn't answering` : 'Cannot reach the server'}
+ </span>
<span className="text-foreground-muted">
- {retrying ? 'Reconnecting…' : 'Check your connection.'}
+ {retrying
+ ? 'Reconnecting…'
+ : managed
+ ? 'Server status shows what it is doing.'
+ : 'Check your connection.'}
</span>
</div>
- <Button
- variant="outline"
- size="sm"
- disabled={retrying}
- // A full refreshServer, not the lighter retryConnection the timer
- // above uses: a click is one bounded request, not a forever-repeating
- // one, so it can afford to also re-check sign-in options — the one
- // way a persistently broken sign-in-options endpoint gets re-driven.
- onClick={() => void store.refreshServer(serverId)}
- >
- <RefreshCw className={retrying ? 'size-4 animate-spin' : 'size-4'} />
- Retry
- </Button>
+ <div className="flex shrink-0 items-center gap-2">
+ {managed && (
+ <Button
+ variant="outline"
+ size="sm"
+ onClick={() => navigate('managedServer', { serverId })}
+ >
+ Server status
+ </Button>
+ )}
+ <Button
+ variant="outline"
+ size="sm"
+ disabled={retrying}
+ // A full refreshServer, not the lighter retryConnection the timer
+ // above uses: a click is one bounded request, not a forever-repeating
+ // one, so it can afford to also re-check sign-in options — the one
+ // way a persistently broken sign-in-options endpoint gets re-driven.
+ onClick={() => void store.refreshServer(serverId)}
+ >
+ <RefreshCw className={retrying ? 'size-4 animate-spin' : 'size-4'} />
+ Retry
+ </Button>
+ </div>
+ </div>
+ );
+});
+
+/**
+ * In place of the page's details when a server this Console runs is not
+ * running: what is wrong, in the words the workspace banner uses, Start (or
+ * Restart, after it stopped on its own) when that is all it takes, and the way
+ * to its status page, where the reason and the rest of its controls are.
+ * Nothing until its state has been read: an unread state is not a stopped one.
+ */
+const ManagedServerDownCard = observer(function ManagedServerDownCard({
+ server,
+}: {
+ server: SwitchServer;
+}) {
+ const { navigate } = useNavigate();
+ const state = managedServerState(server);
+ if (!state || !state.known || state.answering) return null;
+ const crashed = state.reason === 'stopped-unexpectedly';
+ return (
+ <div
+ className={`flex items-center justify-between gap-3 rounded-lg border p-4 ${TONE_BOX[state.tone]}`}
+ >
+ <div className="flex min-w-0 items-center gap-2 text-sm">
+ <ToneDot tone={state.tone} size={7} />
+ <span className="text-foreground">{frozenSentence(server, state)}</span>
+ </div>
+ <div className="flex shrink-0 items-center gap-2">
+ <Button
+ variant="outline"
+ size="sm"
+ onClick={() => navigate('managedServer', { serverId: server.id })}
+ >
+ Server status
+ </Button>
+ {(state.tone === 'stopped' || crashed) && (
+ <Button size="sm" onClick={() => startManagedServer(server)}>
+ <Power className="size-4" />
+ {crashed ? 'Restart' : 'Start'}
+ </Button>
+ )}
+ </div>
</div>
);
});
diff --git a/console/apps/switch-console-desktop/src/renderer/tests/browser/server-view-local-down.test.tsx b/console/apps/switch-console-desktop/src/renderer/tests/browser/server-view-local-down.test.tsx
new file mode 100644
index 000000000..67385804c
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/renderer/tests/browser/server-view-local-down.test.tsx
@@ -0,0 +1,193 @@
+import type * as ReactQuery from '@tanstack/react-query';
+import { act } from 'react';
+import { createRoot, type Root } from 'react-dom/client';
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+
+vi.hoisted(() => {
+ window.electronAPI ??= {
+ invoke: () => Promise.resolve(undefined),
+ eventOn: () => () => {},
+ eventSend: () => {},
+ } as unknown as typeof window.electronAPI;
+});
+
+// The home page of the server on this computer when it is not answering: a
+// crash shows as the server being down, with Restart and the way to its status
+// page, rather than as a connection problem.
+const state = vi.hoisted(() => ({
+ status: null as unknown,
+ unreachable: false,
+ localStart: vi.fn(),
+ navigate: vi.fn(),
+}));
+
+vi.mock('@renderer/lib/layout/navigation-provider', () => ({
+ useParams: () => ({ params: { serverId: 'local' } }),
+ useNavigate: () => ({ navigate: state.navigate }),
+}));
+vi.mock('@renderer/lib/modal/modal-provider', () => ({ useShowModal: () => () => {} }));
+vi.mock('@tanstack/react-query', async (importOriginal) => ({
+ ...(await importOriginal<typeof ReactQuery>()),
+ useQueryClient: () => ({ invalidateQueries: () => Promise.resolve() }),
+}));
+vi.mock('@renderer/features/switch-servers/switch-servers-store', () => ({
+ switchServersStore: {
+ get servers() {
+ return [LOCAL];
+ },
+ serverById: () => LOCAL,
+ statusFor: () => ({ connected: true }),
+ refreshing: new Set(),
+ isUnreachable: () => state.unreachable,
+ isHostBlocked: () => false,
+ refreshStatus: () => Promise.resolve(),
+ ensureAuthConfig: () => Promise.resolve(),
+ refreshServer: () => Promise.resolve(),
+ retryConnection: () => Promise.resolve(),
+ error: null,
+ errorDetail: null,
+ },
+}));
+vi.mock('@renderer/features/switch-servers/local-server-store', () => ({
+ localServerStore: {
+ get status() {
+ return state.status;
+ },
+ get isRunning() {
+ return (state.status as { phase: string }).phase === 'running';
+ },
+ busy: false,
+ docker: null,
+ isNative: true,
+ message: null,
+ start: (...args: unknown[]) => state.localStart(...args),
+ },
+}));
+vi.mock('@renderer/features/remote-hosts/host-reachability-store', () => ({
+ hostReachabilityStore: {
+ hydrate: () => Promise.resolve(),
+ get: () => null,
+ isBlocked: () => false,
+ },
+}));
+vi.mock('@renderer/features/remote-hosts/host-unreachable-panel', () => ({
+ HostUnreachablePanel: () => null,
+}));
+vi.mock('@renderer/features/locations/stores/agents-store', () => ({
+ agentsStore: { load: () => Promise.resolve() },
+}));
+vi.mock('@renderer/features/switch-servers/switch-rooms-store', () => ({
+ switchRoomsStore: { refreshRoomState: () => Promise.resolve() },
+}));
+vi.mock('@renderer/features/switch-servers/MessagingAppsCard', () => ({
+ MessagingAppsCard: () => null,
+}));
+vi.mock('@renderer/features/switch-servers/GuardrailsSection', () => ({
+ GuardrailsSection: () => null,
+}));
+vi.mock('@renderer/features/switch-servers/server-stat-tiles', () => ({
+ ServerStatTiles: () => null,
+}));
+vi.mock('@renderer/features/switch-servers/server-sign-in', () => ({
+ ServerSignInFields: () => null,
+ useServerSignIn: () => ({}),
+ machineUnavailableReason: () => null,
+}));
+
+import { serverView } from '@renderer/features/switch-servers/view';
+import '@renderer/index.css';
+
+const LOCAL = {
+ id: 'local',
+ name: 'Local server',
+ url: 'http://localhost:41001',
+ dashboardUrl: null,
+ managed: true,
+ managementKind: 'local' as const,
+ sshHost: null,
+ createdAt: '2026-01-01T00:00:00.000Z',
+ updatedAt: '2026-01-01T00:00:00.000Z',
+};
+
+const base = {
+ checked: true,
+ upgrade: null,
+ drift: null,
+ runtime: 'native',
+ error: null,
+ failedTo: null,
+};
+
+let container: HTMLDivElement | null = null;
+let root: Root | null = null;
+
+beforeEach(() => {
+ state.status = { ...base, phase: 'running' };
+ state.unreachable = false;
+ state.localStart.mockReset();
+ state.navigate.mockReset();
+});
+
+afterEach(async () => {
+ await act(async () => root?.unmount());
+ container?.remove();
+ container = null;
+ root = null;
+});
+
+async function render() {
+ const Page = serverView.MainPanel;
+ container = document.createElement('div');
+ document.body.appendChild(container);
+ root = createRoot(container);
+ await act(async () => root!.render(<Page />));
+}
+
+function button(label: RegExp): HTMLElement | undefined {
+ return [...document.querySelectorAll('button')].find((b) =>
+ label.test(b.textContent?.trim() ?? '')
+ );
+}
+
+it('says the server stopped unexpectedly, and offers Restart and its status page', async () => {
+ state.status = {
+ ...base,
+ phase: 'error',
+ failedTo: 'run',
+ error: 'switch-core stopped unexpectedly',
+ };
+ await render();
+
+ expect(document.body.textContent).toContain('Local server stopped unexpectedly');
+ expect(document.body.textContent).not.toContain('Check your connection');
+
+ await act(async () => button(/^Restart$/)!.click());
+ expect(state.localStart).toHaveBeenCalledOnce();
+ await act(async () => button(/^Server status$/)!.click());
+ expect(state.navigate).toHaveBeenCalledWith('managedServer', { serverId: 'local' });
+});
+
+it('offers Start for a server that was stopped', async () => {
+ state.status = { ...base, phase: 'stopped' };
+ await render();
+
+ expect(document.body.textContent).toContain("Local server isn't running");
+ expect(button(/^Start$/)).toBeDefined();
+});
+
+it('says nothing until the server has been looked at', async () => {
+ state.status = { ...base, phase: 'stopped', checked: false };
+ await render();
+
+ expect(button(/^Start$/)).toBeUndefined();
+ expect(button(/^Server status$/)).toBeUndefined();
+});
+
+it('does not blame the connection when the local server is not answering', async () => {
+ state.unreachable = true;
+ await render();
+
+ expect(document.body.textContent).toContain("Local server isn't answering");
+ expect(document.body.textContent).not.toContain('Check your connection');
+ expect(button(/^Server status$/)).toBeDefined();
+});
diff --git a/console/apps/switch-console-desktop/src/shared/core/managed-switch-server/managed-switch-server.ts b/console/apps/switch-console-desktop/src/shared/core/managed-switch-server/managed-switch-server.ts
index d37b481e9..97cd3131a 100644
--- a/console/apps/switch-console-desktop/src/shared/core/managed-switch-server/managed-switch-server.ts
+++ b/console/apps/switch-console-desktop/src/shared/core/managed-switch-server/managed-switch-server.ts
@@ -229,9 +229,10 @@ export type LocalServerStatus = {
/** Populated only when `phase === 'error'`. */
error: string | null;
/** What failed when `phase === 'error'` and it was not a start: a stop or a
- * reset that broke partway. Null for a failed start, and whenever the phase
- * is anything else. */
- failedTo: 'stop' | 'reset' | null;
+ * reset that broke partway, or (`run`) a server that was running and then
+ * stopped or stopped answering on its own. Null for a failed start, and
+ * whenever the phase is anything else. */
+ failedTo: 'stop' | 'reset' | 'run' | null;
};
/**
@@ -259,7 +260,7 @@ export class ManagedServerStoppedError extends Error {
/** Human-readable one-liner for a stopped managed stack, used in errors and the UI. */
export function managedServerStoppedReason(serverName: string, phase: LocalServerPhase): string {
if (phase === 'error') {
- return `${serverName} isn't running: its last start or stop failed. See the server's page.`;
+ return `${serverName} isn't running: it failed to start or stop, or stopped unexpectedly. See the server's page.`;
}
return `${serverName}'s Switch stack is not running. Start it from the server's page, then retry.`;
}
@@ -317,7 +318,14 @@ export type ConnectRemoteServerResult =
| { kind: 'error'; message: string };
/** Something that happened to the stack on this computer, as Console recorded it. */
-export type LocalActivityAction = 'started' | 'stopped' | 'start-failed' | 'updated' | 'reset';
+export type LocalActivityAction =
+ | 'started'
+ | 'stopped'
+ | 'start-failed'
+ | 'updated'
+ | 'reset'
+ /** The running server stopped, or stopped answering, without being asked. */
+ | 'crashed';
/** One entry of the local stack's history. Only this Console acts on it, so
* there is no "who". */
@@ -325,7 +333,8 @@ export type LocalActivityEntry = {
/** ISO timestamp. */
at: string;
action: LocalActivityAction;
- /** The version an update moved to, or why a start failed; else null. */
+ /** The version an update moved to, or why a start failed or the server
+ * stopped; else null. */
detail: string | null;
};