Skip to content
Python

fix(runtime): clear leases that predate the current boot

Rahul Ranjana17230dadb5d
diff --git a/memhub/packages/runtime-control/src/runtime-service.test.ts b/memhub/packages/runtime-control/src/runtime-service.test.ts
index 382c206..6629888 100644
--- a/memhub/packages/runtime-control/src/runtime-service.test.ts
+++ b/memhub/packages/runtime-control/src/runtime-service.test.ts
@@ -94,6 +94,83 @@ describe("RuntimeService", () => {
);
});
+ it("reclaims a lease older than the current boot even when a recycled PID answers the liveness probe", async () => {
+ // Issue #184: a forced power-off leaves runtime-service.lock behind;
+ // after the reboot the owner's PID is another process, so the old
+ // liveness-only check reports "already running" forever and launchd's
+ // KeepAlive loops the wrapper. A lease from a previous boot can never
+ // have a living owner, regardless of what the PID table says.
+ const dataDir = await directory("runtime-service-prevboot-");
+ const lock = path.join(dataDir, "runtime-service.lock");
+ await import("node:fs/promises").then(({ mkdir }) =>
+ mkdir(lock, { mode: 0o700 }),
+ );
+ await writeFile(
+ path.join(lock, "owner.json"),
+ JSON.stringify({ pid: 777 }),
+ {
+ mode: 0o600,
+ },
+ );
+ await chmod(lock, 0o700);
+ const leaseTime = Date.now();
+ const service = new RuntimeService({
+ dataDir,
+ createSupervisor: supervisor,
+ // The recycled PID answers the probe: the liveness check alone
+ // would refuse forever.
+ processIsAlive: () => true,
+ // This boot began minutes after the lease was taken.
+ bootTime: () => leaseTime + 10 * 60_000,
+ });
+
+ await service.start();
+ await service.stop();
+ });
+
+ it("reclaims a lease whose owner.json was never written once the machine has rebooted", async () => {
+ // Issue #184's hard case: a power cut inside the mkdir → privateWrite
+ // window leaves the lock directory with no owner.json at all. That used
+ // to be runtime_service_lock_invalid forever; after a reboot it is
+ // provably from a previous boot and is cleared.
+ const dataDir = await directory("runtime-service-prevboot- corrupt-");
+ const lock = path.join(dataDir, "runtime-service.lock");
+ await import("node:fs/promises").then(({ mkdir }) =>
+ mkdir(lock, { mode: 0o700 }),
+ );
+ await chmod(lock, 0o700);
+ const leaseTime = Date.now();
+ const service = new RuntimeService({
+ dataDir,
+ createSupervisor: supervisor,
+ processIsAlive: () => true,
+ bootTime: () => leaseTime + 10 * 60_000,
+ });
+
+ await service.start();
+ await service.stop();
+ });
+
+ it("still refuses a corrupt lease from the current boot", async () => {
+ // Same-boot corruption stays refused: the boot-time shortcut only
+ // applies when the lease provably predates the boot.
+ const dataDir = await directory("runtime-service-corrupt-sameboot-");
+ const lock = path.join(dataDir, "runtime-service.lock");
+ await import("node:fs/promises").then(({ mkdir }) =>
+ mkdir(lock, { mode: 0o700 }),
+ );
+ await chmod(lock, 0o700);
+ const service = new RuntimeService({
+ dataDir,
+ createSupervisor: supervisor,
+ bootTime: () => Date.now() - 60 * 60_000, // booted an hour ago
+ });
+
+ await expect(service.start()).rejects.toThrow(
+ "runtime_service_lock_invalid",
+ );
+ });
+
it("reclaims a stale validated lease", async () => {
const dataDir = await directory("runtime-service-stale-");
const lock = path.join(dataDir, "runtime-service.lock");
diff --git a/memhub/packages/runtime-control/src/runtime-service.ts b/memhub/packages/runtime-control/src/runtime-service.ts
index d3bdc1d..ee6afdc 100644
--- a/memhub/packages/runtime-control/src/runtime-service.ts
+++ b/memhub/packages/runtime-control/src/runtime-service.ts
@@ -9,6 +9,7 @@ import {
rmdir,
writeFile,
} from "node:fs/promises";
+import os from "node:os";
import path from "node:path";
import { processIsAlive } from "./core-supervisor.js";
@@ -35,6 +36,10 @@ export interface RuntimeServiceOptions {
): ManagedRuntime | Promise<ManagedRuntime>;
pid?: number;
processIsAlive?: (pid: number) => boolean;
+ /** Wall-clock epoch (ms) of the current boot. The default is derived from
+ * `os.uptime()`; injectable so tests can put a lease before or after the
+ * boot without moving the machine's clock. */
+ bootTime?: () => number;
diagnostics?: DiagnosticSink;
}
@@ -45,6 +50,9 @@ interface LeaseIdentity {
/** Publishes one outer Runtime identity around the Core+sidecar supervisor. */
export class RuntimeService {
+ /** See removeStaleLease: absorbs a forward clock step since boot. */
+ private static readonly BOOT_SLACK_MS = 2 * 60_000;
+
private readonly options: RuntimeServiceOptions;
private lease: LeaseIdentity | null = null;
private supervisor: ManagedRuntime | null = null;
@@ -223,6 +231,23 @@ export class RuntimeService {
) {
throw new Error("runtime_service_lock_unsafe");
}
+ // A lease created before the current boot cannot have a living owner:
+ // every process from that boot is gone, and a PID that answers the
+ // liveness probe now is a different process that merely reused the
+ // number. Before this check, a forced power-off (issue #184) left
+ // runtime-service.lock behind, the reboot recycled the owner's PID, and
+ // launchd's KeepAlive restarted the wrapper into
+ // runtime_service_already_running without end.
+ //
+ // The slack absorbs a forward clock step since boot: an NTP correction
+ // would otherwise make a lease taken moments after boot look older than
+ // the boot itself. With the slack, only a jump larger than two minutes
+ // within that window could mistake a live lease for a stale one.
+ const bootTime = this.options.bootTime ?? defaultBootTime;
+ if (info.mtimeMs < bootTime() - RuntimeService.BOOT_SLACK_MS) {
+ await this.clearLeaseDir(lock);
+ return;
+ }
let owner: unknown;
try {
owner = JSON.parse(await readFile(path.join(lock, "owner.json"), "utf8"));
@@ -239,6 +264,10 @@ export class RuntimeService {
if ((this.options.processIsAlive ?? processIsAlive)(pid)) {
throw new Error("runtime_service_already_running");
}
+ await this.clearLeaseDir(lock);
+ }
+
+ private async clearLeaseDir(lock: string): Promise<void> {
const stale = `${lock}.stale.${randomUUID()}`;
await rename(lock, stale);
await rm(stale, { recursive: true });
@@ -302,6 +331,12 @@ export class RuntimeService {
}
}
+function defaultBootTime(): number {
+ // Wall-clock epoch of the current boot, as far as the clock admits today.
+ // os.uptime() is boot-based (includes suspend) on every supported platform.
+ return Date.now() - os.uptime() * 1000;
+}
+
async function privateWrite(target: string, value: string): Promise<void> {
const temporary = `${target}.${randomUUID()}.tmp`;
try {