Python
fix(runtime): clear leases that predate the current boot
Rahul Ranjana17230dadb5d
diff --git a/memhub/packages/runtime-control/src/runtime-service.test.ts b/memhub/packages/runtime-control/src/runtime-service.test.tsindex 382c206..6629888 100644--- a/memhub/packages/runtime-control/src/runtime-service.test.ts+++ b/memhub/packages/runtime-control/src/runtime-service.test.ts@@ -94,6 +94,83 @@ describe("RuntimeService", () => {);});+ it("reclaims a lease older than the current boot even when a recycled PID answers the liveness probe", async () => {+ // Issue #184: a forced power-off leaves runtime-service.lock behind;+ // after the reboot the owner's PID is another process, so the old+ // liveness-only check reports "already running" forever and launchd's+ // KeepAlive loops the wrapper. A lease from a previous boot can never+ // have a living owner, regardless of what the PID table says.+ const dataDir = await directory("runtime-service-prevboot-");+ const lock = path.join(dataDir, "runtime-service.lock");+ await import("node:fs/promises").then(({ mkdir }) =>+ mkdir(lock, { mode: 0o700 }),+ );+ await writeFile(+ path.join(lock, "owner.json"),+ JSON.stringify({ pid: 777 }),+ {+ mode: 0o600,+ },+ );+ await chmod(lock, 0o700);+ const leaseTime = Date.now();+ const service = new RuntimeService({+ dataDir,+ createSupervisor: supervisor,+ // The recycled PID answers the probe: the liveness check alone+ // would refuse forever.+ processIsAlive: () => true,+ // This boot began minutes after the lease was taken.+ bootTime: () => leaseTime + 10 * 60_000,+ });++ await service.start();+ await service.stop();+ });++ it("reclaims a lease whose owner.json was never written once the machine has rebooted", async () => {+ // Issue #184's hard case: a power cut inside the mkdir → privateWrite+ // window leaves the lock directory with no owner.json at all. That used+ // to be runtime_service_lock_invalid forever; after a reboot it is+ // provably from a previous boot and is cleared.+ const dataDir = await directory("runtime-service-prevboot- corrupt-");+ const lock = path.join(dataDir, "runtime-service.lock");+ await import("node:fs/promises").then(({ mkdir }) =>+ mkdir(lock, { mode: 0o700 }),+ );+ await chmod(lock, 0o700);+ const leaseTime = Date.now();+ const service = new RuntimeService({+ dataDir,+ createSupervisor: supervisor,+ processIsAlive: () => true,+ bootTime: () => leaseTime + 10 * 60_000,+ });++ await service.start();+ await service.stop();+ });++ it("still refuses a corrupt lease from the current boot", async () => {+ // Same-boot corruption stays refused: the boot-time shortcut only+ // applies when the lease provably predates the boot.+ const dataDir = await directory("runtime-service-corrupt-sameboot-");+ const lock = path.join(dataDir, "runtime-service.lock");+ await import("node:fs/promises").then(({ mkdir }) =>+ mkdir(lock, { mode: 0o700 }),+ );+ await chmod(lock, 0o700);+ const service = new RuntimeService({+ dataDir,+ createSupervisor: supervisor,+ bootTime: () => Date.now() - 60 * 60_000, // booted an hour ago+ });++ await expect(service.start()).rejects.toThrow(+ "runtime_service_lock_invalid",+ );+ });+it("reclaims a stale validated lease", async () => {const dataDir = await directory("runtime-service-stale-");const lock = path.join(dataDir, "runtime-service.lock");diff --git a/memhub/packages/runtime-control/src/runtime-service.ts b/memhub/packages/runtime-control/src/runtime-service.tsindex d3bdc1d..ee6afdc 100644--- a/memhub/packages/runtime-control/src/runtime-service.ts+++ b/memhub/packages/runtime-control/src/runtime-service.ts@@ -9,6 +9,7 @@ import {rmdir,writeFile,} from "node:fs/promises";+import os from "node:os";import path from "node:path";import { processIsAlive } from "./core-supervisor.js";@@ -35,6 +36,10 @@ export interface RuntimeServiceOptions {): ManagedRuntime | Promise<ManagedRuntime>;pid?: number;processIsAlive?: (pid: number) => boolean;+ /** Wall-clock epoch (ms) of the current boot. The default is derived from+ * `os.uptime()`; injectable so tests can put a lease before or after the+ * boot without moving the machine's clock. */+ bootTime?: () => number;diagnostics?: DiagnosticSink;}@@ -45,6 +50,9 @@ interface LeaseIdentity {/** Publishes one outer Runtime identity around the Core+sidecar supervisor. */export class RuntimeService {+ /** See removeStaleLease: absorbs a forward clock step since boot. */+ private static readonly BOOT_SLACK_MS = 2 * 60_000;+private readonly options: RuntimeServiceOptions;private lease: LeaseIdentity | null = null;private supervisor: ManagedRuntime | null = null;@@ -223,6 +231,23 @@ export class RuntimeService {) {throw new Error("runtime_service_lock_unsafe");}+ // A lease created before the current boot cannot have a living owner:+ // every process from that boot is gone, and a PID that answers the+ // liveness probe now is a different process that merely reused the+ // number. Before this check, a forced power-off (issue #184) left+ // runtime-service.lock behind, the reboot recycled the owner's PID, and+ // launchd's KeepAlive restarted the wrapper into+ // runtime_service_already_running without end.+ //+ // The slack absorbs a forward clock step since boot: an NTP correction+ // would otherwise make a lease taken moments after boot look older than+ // the boot itself. With the slack, only a jump larger than two minutes+ // within that window could mistake a live lease for a stale one.+ const bootTime = this.options.bootTime ?? defaultBootTime;+ if (info.mtimeMs < bootTime() - RuntimeService.BOOT_SLACK_MS) {+ await this.clearLeaseDir(lock);+ return;+ }let owner: unknown;try {owner = JSON.parse(await readFile(path.join(lock, "owner.json"), "utf8"));@@ -239,6 +264,10 @@ export class RuntimeService {if ((this.options.processIsAlive ?? processIsAlive)(pid)) {throw new Error("runtime_service_already_running");}+ await this.clearLeaseDir(lock);+ }++ private async clearLeaseDir(lock: string): Promise<void> {const stale = `${lock}.stale.${randomUUID()}`;await rename(lock, stale);await rm(stale, { recursive: true });@@ -302,6 +331,12 @@ export class RuntimeService {}}+function defaultBootTime(): number {+ // Wall-clock epoch of the current boot, as far as the clock admits today.+ // os.uptime() is boot-based (includes suspend) on every supported platform.+ return Date.now() - os.uptime() * 1000;+}+async function privateWrite(target: string, value: string): Promise<void> {const temporary = `${target}.${randomUUID()}.tmp`;try {