Skip to content

armgit / mso

Public Imported from GitHub
0

Browser-based graphical shell and control plane for a Linux server you own — real terminal, file manager, live metrics and BYOK AI in one mobile-first workspace. Next.js 16 · React 19 · node-pty · Tailwind v4 · shadcn/ui. Public alpha, self-hosted.

fix(runtime): preserve fallback recovery readiness

Rahman925c1b558123
diff --git a/bin/mso b/bin/mso
index 3a338195..aef943f0 100755
--- a/bin/mso
+++ b/bin/mso
@@ -209,6 +209,24 @@ service_unit_local_url() {
printf 'http://127.0.0.1:%s' "$port"
}
+service_wait_ready() {
+ local local_url="$1" attempts="${MSO_SERVICE_READY_ATTEMPTS:-40}" body i
+ [[ "$attempts" =~ ^[0-9]+$ ]] && [ "$attempts" -gt 0 ] || attempts=40
+ for i in $(seq 1 "$attempts"); do
+ body="$(curl --noproxy '*' -fsS --connect-timeout 1 --max-time 2 "$local_url/api/health" 2>/dev/null || true)"
+ if printf '%s' "$body" | jq -e '
+ type == "object" and .status == "ok" and .service == "mso" and
+ (.buildId | type == "string" and length > 0) and
+ (.buildSha | type == "string" and test("^[0-9a-f]{7,40}$")) and
+ (.version | type == "string" and length > 0) and has("runtimeInstanceId")
+ ' >/dev/null 2>&1; then
+ return 0
+ fi
+ sleep 0.25
+ done
+ return 1
+}
+
service_handoff_marker() {
local canonical base key dir
canonical="$(realpath -e -- "$ROOT" 2>/dev/null || true)"
@@ -247,10 +265,13 @@ service_handoff_fallback_locked() {
service_restore_fallback_after_failure() {
local gateway="${MSO_GATEWAY_BIN:-$ROOT/scripts/mso-gateway}"
[ "${SERVICE_FALLBACK_QUIESCED:-0}" = 1 ] || return 0
- MSO_GATEWAY_ROOT="$ROOT" MSO_GATEWAY_ENV="$ENVF" MSO_GATEWAY_LOCAL_URL="$SERVICE_HANDOFF_URL" \
- "$gateway" local-start >/dev/null 2>&1 \
- || printf 'mso: service action failed and fallback restore also failed at %s\n' "$SERVICE_HANDOFF_URL" >&2
- mso_private_state_remove_file "$SERVICE_HANDOFF_MARKER" >/dev/null 2>&1 || true
+ if ! MSO_GATEWAY_ROOT="$ROOT" MSO_GATEWAY_ENV="$ENVF" MSO_GATEWAY_LOCAL_URL="$SERVICE_HANDOFF_URL" \
+ "$gateway" local-start >/dev/null 2>&1; then
+ printf 'mso: service action failed and fallback restore also failed at %s; recovery marker preserved\n' "$SERVICE_HANDOFF_URL" >&2
+ return 1
+ fi
+ mso_private_state_remove_file "$SERVICE_HANDOFF_MARKER" >/dev/null 2>&1 \
+ || { printf 'mso: fallback restored but recovery marker could not be cleared\n' >&2; return 1; }
}
service_lifecycle_safe() {
@@ -262,8 +283,18 @@ service_lifecycle_safe() {
SERVICE_FALLBACK_QUIESCED=0; SERVICE_HANDOFF_URL=''; SERVICE_HANDOFF_MARKER=''
[ "$active" = 1 ] || service_handoff_fallback_locked
service_manager_write "$verb" mso.service || rc=$?
+ if [ "$rc" -eq 0 ] && ! service_wait_ready "$(service_unit_local_url)"; then
+ printf 'mso: service %s returned success but /api/health did not become ready; restoring fallback if available\n' "$verb" >&2
+ rc=24
+ # A Type=simple unit can report a successful start before Next later fails.
+ # Stop any unhealthy survivor so the quiesced fallback can reclaim its port.
+ service_manager_write stop mso.service >/dev/null 2>&1 || true
+ fi
runtime_exclusion_release
- if [ "$rc" -ne 0 ]; then service_restore_fallback_after_failure; return "$rc"; fi
+ if [ "$rc" -ne 0 ]; then
+ service_restore_fallback_after_failure || return 1
+ return "$rc"
+ fi
[ -z "${SERVICE_HANDOFF_MARKER:-}" ] || mso_private_state_remove_file "$SERVICE_HANDOFF_MARKER" >/dev/null 2>&1 || true
}
diff --git a/bin/mso-service-lifecycle.test.ts b/bin/mso-service-lifecycle.test.ts
index 6e49b0ac..8380fa6b 100644
--- a/bin/mso-service-lifecycle.test.ts
+++ b/bin/mso-service-lifecycle.test.ts
@@ -12,6 +12,7 @@ function fixture() {
const root = fs.mkdtempSync(path.join(os.tmpdir(), "mso-service-cli-")); roots.push(root);
const bin = path.join(root, "bin"), capture = path.join(root, "capture"), home = path.join(root, "home");
fs.mkdirSync(bin); fs.mkdirSync(home);
+ fs.writeFileSync(path.join(bin, "curl"), `#!/bin/sh\n[ \"\${MSO_TEST_HEALTH_FAIL:-0}\" = 1 ] && exit 7\nprintf '%s\\n' '{\"status\":\"ok\",\"service\":\"mso\",\"buildId\":\"fixture\",\"buildSha\":\"abcdef1\",\"runtimeInstanceId\":null,\"version\":\"0.2.1\"}'\n`, { mode: 0o755 });
const manager = path.join(bin, "manager");
fs.writeFileSync(manager, `#!/bin/sh\nprintf 'manager %s\\n' "$*" >> "${capture}"\ncase "$*" in\n 'show -p WorkingDirectory --value mso.service') printf '%s\\n' "${process.cwd()}" ;;\n 'show -p Environment --value mso.service') printf 'PORT=4005\\n' ;;\n 'is-active --quiet mso.service') [ "\${MSO_TEST_SERVICE_ACTIVE:-1}" = 1 ] && exit 0; exit 3 ;;\n 'start mso.service'|'restart mso.service') [ "\${MSO_TEST_MANAGER_FAIL:-0}" = 1 ] && exit 23; exit 0 ;;\nesac\n`, { mode: 0o755 });
const helper = path.join(root, "service-update");
@@ -19,7 +20,7 @@ function fixture() {
const gateway = path.join(root, "gateway");
fs.writeFileSync(gateway, `#!/bin/sh\nprintf 'gateway %s marker=%s local=%s\\n' "$*" "\${MSO_GATEWAY_RECOVERY_MARKER:-}" "\${MSO_GATEWAY_LOCAL_URL:-}" >> "${capture}"\ncase "$1" in\n runtime-assert-update-safe) [ "\${MSO_TEST_FALLBACK:-0}" = 1 ] && echo 'runtime: update-owned' || echo 'runtime: update-safe' ;;\n runtime-stop) [ -n "\${MSO_GATEWAY_RECOVERY_MARKER:-}" ] || exit 9; printf '1\\n' > "\$MSO_GATEWAY_RECOVERY_MARKER"; chmod 600 "\$MSO_GATEWAY_RECOVERY_MARKER"; echo 'runtime: stopped-owned' ;;\n local-start) echo 'runtime: healthy MSO at fixture' ;;\n *) exit 2 ;;\nesac\n`, { mode: 0o755 });
const exclusion = path.join(root, "runtime-exclusion");
- const env = { ...process.env, HOME: home, MSO_ENV: "/dev/null", MSO_GATEWAY_LOCAL_URL: "http://127.0.0.1:4005",
+ const env = { ...process.env, HOME: home, PATH: `${bin}:${process.env.PATH}`, MSO_ENV: "/dev/null", MSO_GATEWAY_LOCAL_URL: "http://127.0.0.1:4005",
MSO_SYSTEMCTL_BIN: manager, MSO_SERVICE_UPDATE_BIN: helper, MSO_GATEWAY_BIN: gateway, MSO_RUNTIME_EXCLUSION_DIR: exclusion, MSO_SERVICE_HANDOFF_DIR: path.join(root, "handoff") };
return { root, capture, env, exclusion };
}
@@ -37,9 +38,10 @@ afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursiv
describe("mso service/deploy runtime exclusion", () => {
it.each(["start", "restart"])("refuses service %s while an offline build owns the checkout", async (verb) => {
- const f = fixture(), lock = lockPath(f.exclusion);
- const holder = spawn("flock", ["-x", lock, "-c", "sleep 0.8"], { stdio: "ignore" });
- await new Promise((r) => setTimeout(r, 80));
+ const f = fixture(), lock = lockPath(f.exclusion), held = path.join(f.root, "lock-held");
+ const holder = spawn("flock", ["-x", lock, "-c", `touch ${JSON.stringify(held)}; sleep 0.8`], { stdio: "ignore" });
+ for (let i = 0; i < 50 && !fs.existsSync(held); i++) await new Promise((r) => setTimeout(r, 20));
+ expect(fs.existsSync(held)).toBe(true);
const out = spawnSync(CLI, ["service", verb], {
env: { ...f.env, MSO_RUNTIME_EXCLUSION_TIMEOUT_SECONDS: "0.1" }, encoding: "utf8",
});
@@ -76,6 +78,17 @@ describe("mso service/deploy runtime exclusion", () => {
expect(calls).toContain("gateway local-start");
});
+ it("restores the gateway fallback when Type=simple start returns before MSO becomes healthy", () => {
+ const f = fixture();
+ const out = spawnSync(CLI, ["service", "start"], { env: { ...f.env, MSO_TEST_SERVICE_ACTIVE: "0", MSO_TEST_FALLBACK: "1", MSO_TEST_HEALTH_FAIL: "1", MSO_SERVICE_READY_ATTEMPTS: "1" }, encoding: "utf8" });
+ expect(out.status).not.toBe(0);
+ expect(out.stderr).toContain("/api/health did not become ready");
+ const calls = fs.readFileSync(f.capture, "utf8");
+ expect(calls).toContain("manager start mso.service");
+ expect(calls).toContain("manager stop mso.service");
+ expect(calls).toContain("gateway local-start");
+ });
+
it("refuses lifecycle actions from a checkout that does not own the service", () => {
const f = fixture(), other = path.join(f.root, "other"); fs.mkdirSync(other);
const manager = path.join(f.root, "wrong-manager");
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 7d109020..cf8d54d9 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -8,6 +8,7 @@ this is the *what*, and it is what Settings → About shows as “What's new”.
**Fixed**
+- `runtime` preserve fallback recovery readiness
- `runtime` keep service starts on shared exclusion
- `settings` hide premature update warning
- `update` enforce main and raise MCP daily limit
diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md
index d04599ac..1ef10399 100644
--- a/docs/TROUBLESHOOTING.md
+++ b/docs/TROUBLESHOOTING.md
@@ -45,8 +45,11 @@ response with source `HEAD`; source equality alone is not treated as proof that
MSO takes the checkout runtime exclusion exclusively before service handoff. If the unit is inactive and
its loopback port is occupied by a **gateway-owned** fallback, MSO quiesces only that recorded process,
-preserves any public tunnel, then starts the system service on the freed port. If the service action fails,
-the fallback is restored. An unowned/manual responder is never killed automatically.
+preserves any public tunnel, then starts the system service on the freed port. Exit status alone is not
+readiness: MSO waits for the loopback `/api/health` identity before clearing recovery state; if readiness
+fails, the unhealthy service is stopped and the fallback is restored. A gateway-owned fallback that exited
+naturally is also reconciled into durable restore intent on the next update/deploy. An unowned/manual
+responder is never killed automatically.
### `mso update` says local main is ahead/diverged from `origin/main`
diff --git a/scripts/lib/gateway-runtime.sh b/scripts/lib/gateway-runtime.sh
index 472751d9..4ec5cdf9 100644
--- a/scripts/lib/gateway-runtime.sh
+++ b/scripts/lib/gateway-runtime.sh
@@ -161,11 +161,11 @@ gateway_cmd_runtime_stop_locked() {
gateway_info "runtime: stopped-owned"
return 0
fi
- if gateway_recovery_pending && ! gateway_runtime_process_matches "$runtime" && ! gateway_health_ok; then
- gateway_persist_quiesced_state "$state" \
+ if ! gateway_runtime_process_matches "$runtime" && ! gateway_health_ok; then
+ # Natural exit: persist restore intent before dropping ownership.
+ gateway_mark_recovery_pending; gateway_persist_quiesced_state "$state" \
|| gateway_fail "could not reconcile stale owned runtime; recovery marker preserved"
- gateway_info "runtime: recovered-stale-owned"
- return 0
+ gateway_info "runtime: recovered-stale-owned"; return 0
fi
gateway_fail "recorded runtime ownership no longer matches a safely recoverable MSO instance; refusing offline rebuild"
fi
diff --git a/scripts/mso-gateway-natural-exit.test.ts b/scripts/mso-gateway-natural-exit.test.ts
new file mode 100644
index 00000000..6c3f8e26
--- /dev/null
+++ b/scripts/mso-gateway-natural-exit.test.ts
@@ -0,0 +1,49 @@
+import { createHash } from "node:crypto";
+import { execFileSync, spawn } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+import { expect, it } from "vitest";
+import { fixture, GATEWAY, ROOT } from "./mso-gateway-test-fixture";
+
+function identity(pid: number) {
+ const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8");
+ const rest = stat.slice(stat.lastIndexOf(") ") + 2).trim().split(/\s+/);
+ return {
+ pid,
+ startTicks: rest[19],
+ exe: fs.realpathSync(`/proc/${pid}/exe`),
+ cmdHash: null,
+ instanceId: "fixture",
+ };
+}
+
+it("persists recovery intent when an owned fallback exited naturally before update", async () => {
+ const f = fixture();
+ const child = spawn(process.execPath, ["-e", "setInterval(()=>{},1000)"], { stdio: "ignore" });
+ await new Promise((r) => setTimeout(r, 100));
+ const runtimeIdentity = identity(child.pid!);
+ child.kill("SIGTERM");
+ await new Promise<void>((resolve) => child.once("close", () => resolve()));
+
+ const localUrl = "http://127.0.0.1:4005", canonical = fs.realpathSync(ROOT);
+ const scopeId = createHash("sha256").update(`${canonical}\n${localUrl}`).digest("hex");
+ fs.mkdirSync(f.state, { recursive: true, mode: 0o700 });
+ fs.writeFileSync(path.join(f.state, "state.json"), JSON.stringify({
+ scopeId, root: canonical, provider: "local", mode: "local", url: localUrl, localUrl,
+ tunnelIdentity: null, runtimeIdentity, runtimeOwned: true, startedAt: "2026-01-01T00:00:00Z",
+ }) + "\n", { mode: 0o600 });
+ fs.writeFileSync(f.curl, "#!/bin/sh\nexit 7\n", { mode: 0o700 });
+
+ const recoveryDir = path.join(f.dir, "recovery");
+ fs.mkdirSync(recoveryDir, { mode: 0o700 });
+ const marker = path.join(recoveryDir, "restart-runtime");
+ const out = execFileSync(GATEWAY, ["runtime-stop"], {
+ env: { ...f.baseEnv, MSO_GATEWAY_RECOVERY_MARKER: marker }, encoding: "utf8",
+ });
+ expect(out).toContain("runtime: recovered-stale-owned");
+ const state = JSON.parse(fs.readFileSync(path.join(f.state, "state.json"), "utf8"));
+ expect(state.runtimeOwned).toBe(false);
+ expect(state.runtimeIdentity).toBeNull();
+ expect(fs.statSync(marker).mode & 0o777).toBe(0o600);
+ expect(fs.readFileSync(marker, "utf8").trim()).toBe("1");
+});