Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

fix(runtime): keep durable builds progressing under load

kashorinbd39ef50ca7c
diff --git a/packages/driver-node/test/scheduler.test.ts b/packages/driver-node/test/scheduler.test.ts
index 6c60114c..310b89a5 100644
--- a/packages/driver-node/test/scheduler.test.ts
+++ b/packages/driver-node/test/scheduler.test.ts
@@ -246,6 +246,51 @@ test("independent paid commands inside one Build may fill the same resource with
record.type.name === types.generated.name).length, 2);
});
+test("a terminal failure keeps its diagnostic while an in-flight sibling settles", async () => {
+ const operations = memoryOperations();
+ const producers = new ProducerRegistry();
+ const endpoints = new EndpointRegistry();
+ registerGreetingProducers(producers);
+ let calls = 0;
+ const endpoint: AsyncEndpoint = {
+ async start() {
+ calls += 1;
+ if (calls === 1) {
+ return {
+ status: "failed",
+ failure: { code: "PROVIDER_REJECTED", message: "the first provider request was rejected" },
+ };
+ }
+ await new Promise<void>((resolve) => setTimeout(resolve, 20));
+ return {
+ status: "completed",
+ result: { value: { kind: "inline", value: "late sibling result" } },
+ };
+ },
+ poll() {
+ throw new Error("completed fixture operations are not polled");
+ },
+ };
+ endpoints.registerAsyncEndpoint(
+ "generation.parallel",
+ capabilities.generation,
+ types.generated,
+ endpoint,
+ { scheduling: { resources: [{ id: "pool:fixture.account", limit: 2 }] } },
+ );
+
+ const [result] = await new LocalBuildScheduler(new NodeDriver({ producers, endpoints, operations })).run([{
+ id: "parallel-failure",
+ state: createParallelGreetingBuild(),
+ }]);
+
+ assert.equal(calls, 2);
+ assert.equal(result?.status, "failed");
+ assert.equal(result?.state.diagnostics.at(-1)?.code, "PROVIDER_REJECTED");
+ assert.match(result?.state.diagnostics.at(-1)?.message ?? "", /first provider request was rejected/);
+ assert.equal(result?.outcomes.some((outcome) => outcome.status === "error"), false);
+});
+
test("an asynchronous Endpoint starts once and is polled until complete", async () => {
const operations = memoryOperations();
let starts = 0;
diff --git a/packages/provider-hyperframes-local/README.md b/packages/provider-hyperframes-local/README.md
index 718de158..0612996c 100644
--- a/packages/provider-hyperframes-local/README.md
+++ b/packages/provider-hyperframes-local/README.md
@@ -42,10 +42,12 @@ const visual = await renderHyperframesVisual(
Omit `range` for the complete document. Ranges are zero-based and half-open on the original
programme clock. At 30 fps, `[240, 360)` returns 120 frames covering seconds 8–12.
`onProgress` reports preparation, worker ranges, browser PIDs, completion and elapsed time.
-`signal` cancels capture and encoding; `processTimeoutMs` bounds the operation.
-`initializationTimeoutMs` (30 seconds by default) and `frameTimeoutMs` (15 seconds) bound each
-worker's initialization and individual frame capture. Errors name the worker and stage/frame.
-A completed worker closes its Chrome immediately. A timeout aborts the other workers and awaits cleanup.
+`signal` cancels capture and encoding; `processTimeoutMs` bounds the complete operation and defaults
+to 30 minutes. Deployments may additionally set `initializationTimeoutMs` or `frameTimeoutMs` when
+they have a measured stage deadline. Stage deadlines are otherwise unset: a busy machine must not
+turn one slow Chrome start or frame into a false render failure while the operation deadline remains.
+An explicit stage-timeout error names the worker and stage/frame, aborts sibling workers and awaits
+cleanup. A completed worker closes its Chrome immediately.
One call stages the HTML and assets once, merges overlapping source-frame extraction windows,
and shares decoded PNGs across workers. Each worker initializes its own page and captures its
diff --git a/packages/provider-hyperframes-local/src/render.ts b/packages/provider-hyperframes-local/src/render.ts
index bac225ef..6640a02b 100644
--- a/packages/provider-hyperframes-local/src/render.ts
+++ b/packages/provider-hyperframes-local/src/render.ts
@@ -40,8 +40,12 @@ export function resolveExecutionOptions(options: HyperframesExecutionOptions) {
quality, browserGpu,
ffmpegPath: options.ffmpegPath ?? "ffmpeg",
ffprobePath: options.ffprobePath ?? "ffprobe",
- initializationTimeoutMs: positiveInteger(options.initializationTimeoutMs ?? 30_000, "initializationTimeoutMs"),
- frameTimeoutMs: positiveInteger(options.frameTimeoutMs ?? 15_000, "frameTimeoutMs"),
+ initializationTimeoutMs: options.initializationTimeoutMs === undefined
+ ? undefined
+ : positiveInteger(options.initializationTimeoutMs, "initializationTimeoutMs"),
+ frameTimeoutMs: options.frameTimeoutMs === undefined
+ ? undefined
+ : positiveInteger(options.frameTimeoutMs, "frameTimeoutMs"),
processTimeoutMs: positiveInteger(options.processTimeoutMs ?? 30 * 60_000, "processTimeoutMs"),
maxProcessOutputBytes: positiveInteger(options.maxProcessOutputBytes ?? 4 * 1024 * 1024, "maxProcessOutputBytes"),
maxRenderedBytes: positiveInteger(options.maxRenderedBytes ?? 16 * 1024 * 1024 * 1024, "maxRenderedBytes"),
@@ -79,16 +83,21 @@ export async function renderHyperframesVisual(
}
return promise;
};
- const stage = async <T>(subject: string, timeoutMs: number, run: () => Promise<T>): Promise<T> => {
+ const stage = async <T>(subject: string, timeoutMs: number | undefined, run: () => Promise<T>): Promise<T> => {
signal.throwIfAborted();
- const timeout = setTimeout(() => controller.abort(new Error(`HyperFrames ${subject} timed out after ${timeoutMs} ms`)), timeoutMs);
+ const timeout = timeoutMs === undefined
+ ? undefined
+ : setTimeout(() => controller.abort(new Error(`HyperFrames ${subject} timed out after ${timeoutMs} ms`)), timeoutMs);
let onAbort: () => void = () => {};
const stopped = new Promise<never>((_, reject) => {
onAbort = () => reject(signal.reason);
signal.addEventListener("abort", onAbort, { once: true });
});
try { return await Promise.race([run(), stopped]); }
- finally { clearTimeout(timeout); signal.removeEventListener("abort", onAbort); }
+ finally {
+ if (timeout !== undefined) clearTimeout(timeout);
+ signal.removeEventListener("abort", onAbort);
+ }
};
let server: Awaited<ReturnType<typeof createFileServer>> | undefined;
// Closing pages interrupts an in-flight capture as well as the next loop iteration.
diff --git a/packages/provider-hyperframes-local/test/range.test.ts b/packages/provider-hyperframes-local/test/range.test.ts
index faabd9aa..96b72663 100644
--- a/packages/provider-hyperframes-local/test/range.test.ts
+++ b/packages/provider-hyperframes-local/test/range.test.ts
@@ -11,6 +11,7 @@ import { compileHyperframesDocument } from "@hypit/hyperframes";
import { sealProgramSpace } from "@hypit/program-space";
import type { BlobRef } from "@hypit/protocol";
import { renderHyperframesVisual } from "../src/index.js";
+import { resolveExecutionOptions } from "../src/render.js";
import { hypitPackage } from "../src/activation.js";
import type { RuntimeEndpointAdapterImplementation } from "@hypit/runtime-kit";
import { endpointResourceClaims } from "@hypit/endpoint-kit";
@@ -42,6 +43,17 @@ function documentFor(artifact: BlobRef) {
canvas: { width: 64, height: 64, clearColor: "#000000" }, tracks: [track] }), space);
}
+test("the render deadline is global while stage deadlines are explicit deployment policy", () => {
+ const defaults = resolveExecutionOptions({});
+ assert.equal(defaults.processTimeoutMs, 30 * 60_000);
+ assert.equal(defaults.initializationTimeoutMs, undefined);
+ assert.equal(defaults.frameTimeoutMs, undefined);
+
+ const explicit = resolveExecutionOptions({ initializationTimeoutMs: 31_000, frameTimeoutMs: 16_000 });
+ assert.equal(explicit.initializationTimeoutMs, 31_000);
+ assert.equal(explicit.frameTimeoutMs, 16_000);
+});
+
test("source selection retains loop, hold and fractional-speed sampling and shares decoded frames", () => {
const document = documentFor({ kind: "blob", resource: "res_range_source", size: 1, mediaType: "video/mp4" });
const slots = videoSlots(document.html);
diff --git a/packages/runtime-local/src/config.ts b/packages/runtime-local/src/config.ts
index 51fb9b99..a6fbe0be 100644
--- a/packages/runtime-local/src/config.ts
+++ b/packages/runtime-local/src/config.ts
@@ -108,8 +108,6 @@ export type LoadRuntimeConfigOptions = {
readonly distributionPackageRoot?: string;
/** Persistent machine/user state. Defaults to the platform Hypit state root. */
readonly hostStateRoot?: string;
- /** Worker-only ownership check performed before reclaiming or claiming execution. */
- readonly assertExecutionOwner?: () => Promise<void> | void;
};
export type RuntimeConfigDoctorResult = {
@@ -988,7 +986,6 @@ export async function createRuntimeFromConfig(
commandExecutionStore: state.commandExecutions,
assertEnvironment: async () => {
await state.environment.assert(environmentConfig);
- await options.assertExecutionOwner?.();
},
executionStore: state.execution,
removeActiveBuild: async (build) => await state.removeActiveBuild(build),
diff --git a/packages/runtime-local/src/host.ts b/packages/runtime-local/src/host.ts
index 4d09f5a4..9265456d 100644
--- a/packages/runtime-local/src/host.ts
+++ b/packages/runtime-local/src/host.ts
@@ -4,8 +4,7 @@ import type {
RuntimeWorkerLaunch,
} from "@hypit/runtime-host-node";
import { hypitHostStateRoot } from "@hypit/runtime-host-node";
-import { join, resolve } from "node:path";
-import { SqliteRuntimeState } from "@hypit/store-sqlite";
+import { resolve } from "node:path";
import {
createRuntimeControlFromConfig,
@@ -28,7 +27,6 @@ import {
import {
ensureRuntimeProcess,
markRuntimeProcessReady,
- runtimeProcessOwnsCurrentProfile,
runtimeProcessLogs,
runtimeProcessStatus,
stopRuntimeProcess,
@@ -146,67 +144,25 @@ export async function openLocalRuntimeHost(
...distribution,
}),
runWorker: async (readyFile, owner) => {
- const selected = await resolveRuntimeConfigPaths(profile, { packageRoot: basePackageRoot, ...distribution });
- const leaseState = new SqliteRuntimeState(join(selected.dataRoot, "runtime.sqlite"));
- const leaseMs = 30_000;
- const acquiredAt = Date.now();
- const acquired = await leaseState.workerLease.acquire({
- owner,
- pid: process.pid,
- acquiredAt,
- expiresAt: acquiredAt + leaseMs,
- });
- if (!acquired) {
- leaseState.close();
- throw new Error("another Runtime Worker owns this Runtime");
- }
let runtime: Awaited<ReturnType<typeof createRuntimeFromConfig>> | undefined;
const abort = new AbortController();
const stop = (): void => abort.abort();
process.once("SIGTERM", stop);
process.once("SIGINT", stop);
- let renewal = Promise.resolve();
- const heartbeat = setInterval(() => {
- renewal = renewal.then(async () => {
- if (!await runtimeProcessOwnsCurrentProfile(profile, selected.dataRoot, owner)) {
- abort.abort(new Error("Runtime Profile changed; this Worker stopped before using mixed configuration"));
- return;
- }
- const renewed = await leaseState.workerLease.renew(owner, process.pid, Date.now() + leaseMs);
- if (!renewed) abort.abort(new Error("Runtime Worker ownership was lost"));
- }).catch((error: unknown) => abort.abort(error));
- }, 5_000);
try {
- const assertExecutionOwner = async (): Promise<void> => {
- const lease = await leaseState.workerLease.read();
- if (lease?.owner !== owner || lease.pid !== process.pid || lease.expiresAt <= Date.now()) {
- throw new Error("Runtime Worker no longer owns this Runtime");
- }
- };
runtime = await createRuntimeFromConfig(profile, {
packageRoot: basePackageRoot,
...distribution,
- assertExecutionOwner,
});
await runtime.work({
idlePollMs: 250,
signal: abort.signal,
- ready: async () => {
- await assertExecutionOwner();
- if (!await runtimeProcessOwnsCurrentProfile(profile, selected.dataRoot, owner)) {
- throw new Error("Runtime Profile changed while the Worker was starting");
- }
- await markRuntimeProcessReady(readyFile, owner);
- },
+ ready: async () => await markRuntimeProcessReady(readyFile, owner),
});
} finally {
- clearInterval(heartbeat);
- await renewal.catch(() => undefined);
process.removeListener("SIGTERM", stop);
process.removeListener("SIGINT", stop);
await runtime?.close();
- await leaseState.workerLease.release(owner, process.pid).catch(() => undefined);
- leaseState.close();
}
},
};
diff --git a/packages/runtime-local/src/worker-process.ts b/packages/runtime-local/src/worker-process.ts
index 6966831a..38075c71 100644
--- a/packages/runtime-local/src/worker-process.ts
+++ b/packages/runtime-local/src/worker-process.ts
@@ -3,7 +3,6 @@ import { randomUUID } from "node:crypto";
import { mkdir, open, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
import { dirname, join, resolve } from "node:path";
-import { SqliteRuntimeState } from "@hypit/store-sqlite";
import { canonicalStringify } from "@hypit/protocol";
import { processAlive, stopProcessTree } from "./process-control.js";
@@ -54,16 +53,6 @@ function paths(dataRoot: string) {
};
}
-async function exists(path: string): Promise<boolean> {
- try {
- await stat(path);
- return true;
- } catch (error) {
- if (error instanceof Error && "code" in error && error.code === "ENOENT") return false;
- throw error;
- }
-}
-
async function rotateLog(path: string): Promise<void> {
let size = 0;
try {
@@ -99,27 +88,6 @@ async function profileConfig(profile: string): Promise<string> {
return canonicalStringify(JSON.parse(await readFile(resolve(profile), "utf8")));
}
-async function activeLease(dataRoot: string) {
- const state = new SqliteRuntimeState(join(resolve(dataRoot), "runtime.sqlite"), { readOnly: true });
- try {
- return await state.workerLease.read();
- } catch (error) {
- if (error instanceof Error && error.message.includes("no such table: hypit_worker_lease")) return undefined;
- throw error;
- } finally {
- state.close();
- }
-}
-
-async function releaseLease(dataRoot: string, owner: string, pid: number): Promise<void> {
- const state = new SqliteRuntimeState(join(resolve(dataRoot), "runtime.sqlite"));
- try {
- await state.workerLease.release(owner, pid);
- } finally {
- state.close();
- }
-}
-
async function readyOwner(path: string): Promise<string | undefined> {
try {
const value = (await readFile(path, "utf8")).trim();
@@ -136,11 +104,7 @@ export async function runtimeProcessStatus(
): Promise<RuntimeProcessState> {
const location = paths(dataRoot);
const current = await record(profile, dataRoot);
- const lease = current === undefined ? undefined : await activeLease(dataRoot);
- const ready = current === undefined ? undefined : await readyOwner(location.ready);
- if (current === undefined || !processAlive(current.pid)
- || lease?.owner !== current.owner || lease.pid !== current.pid || lease.expiresAt <= Date.now()
- || ready !== current.owner) {
+ if (current === undefined || !processAlive(current.pid)) {
return { state: "stopped", profile: resolve(profile), logPath: location.log };
}
return {
@@ -187,6 +151,7 @@ async function acquireLaunch(path: string, timeoutMs: number): Promise<() => Pro
async function waitForReady(
profile: string,
dataRoot: string,
+ owner: string,
timeoutMs: number,
): Promise<RuntimeProcessState> {
const location = paths(dataRoot);
@@ -202,43 +167,46 @@ async function waitForReady(
}
throw new Error(`Runtime Worker exited before becoming ready${log.length === 0 ? "" : `: ${log.trim().split("\n").at(-1)}`}`);
}
- if (await exists(location.ready)) {
- const state = await runtimeProcessStatus(profile, dataRoot);
- if (state.state === "running") return state;
+ if (current.owner !== owner) {
+ throw new Error("Runtime Worker record changed while waiting for the Worker to become ready");
+ }
+ if (await readyOwner(location.ready) === owner) {
+ return await runtimeProcessStatus(profile, dataRoot);
}
await new Promise((resolveWait) => setTimeout(resolveWait, 10));
}
throw new Error(`Runtime Worker did not become ready within ${timeoutMs}ms; log: ${location.log}`);
}
+async function waitForProcessExit(pid: number, timeoutMs: number): Promise<boolean> {
+ const deadline = Date.now() + timeoutMs;
+ while (Date.now() <= deadline) {
+ if (!processAlive(pid)) return true;
+ await new Promise((resolveWait) => setTimeout(resolveWait, 20));
+ }
+ return !processAlive(pid);
+}
+
export async function ensureRuntimeProcess(
profile: string,
dataRoot: string,
launch: RuntimeWorkerLaunch,
timeoutMs = 10_000,
): Promise<RuntimeProcessState> {
- const current = await runtimeProcessStatus(profile, dataRoot);
- if (current.state === "running") {
- if (current.configuration === "changed") {
- throw new Error("Runtime Profile changed while its Worker is running; stop and start the Worker to activate it");
- }
- return current;
- }
const absolute = resolve(profile);
const location = paths(dataRoot);
await mkdir(location.root, { recursive: true });
const releaseLaunch = await acquireLaunch(location.launch, timeoutMs);
try {
- const raced = await runtimeProcessStatus(profile, dataRoot);
- if (raced.state === "running") {
- if (raced.configuration === "changed") {
+ const current = await runtimeProcessStatus(profile, dataRoot);
+ if (current.state === "running") {
+ if (current.configuration === "changed") {
throw new Error("Runtime Profile changed while its Worker is running; stop and start the Worker to activate it");
}
- return raced;
+ return current;
}
const stale = await record(profile, dataRoot).catch(() => undefined);
if (stale !== undefined && !processAlive(stale.pid)) {
- await releaseLease(dataRoot, stale.owner, stale.pid).catch(() => undefined);
await rm(location.pid, { force: true });
await rm(location.ready, { force: true });
}
@@ -278,10 +246,19 @@ export async function ensureRuntimeProcess(
} satisfies ProcessRecord), "utf8");
child.unref();
try {
- return await waitForReady(absolute, dataRoot, timeoutMs);
+ const ready = await waitForReady(absolute, dataRoot, owner, timeoutMs);
+ if (ready.configuration === "changed") {
+ throw new Error("Runtime Profile changed while its Worker was starting; start it again to activate the new Profile");
+ }
+ return ready;
} catch (error) {
- await stopProcessTree(child.pid, true).catch(() => "gone");
- await releaseLease(dataRoot, owner, child.pid).catch(() => undefined);
+ const stopped = await stopProcessTree(child.pid, true).catch(() => "denied" as const);
+ if (stopped === "denied" || !await waitForProcessExit(child.pid, 2_000)) {
+ throw new Error(
+ `Runtime Worker ${child.pid} failed to become ready and is still running; stop it explicitly before retrying`,
+ { cause: error },
+ );
+ }
const saved = await record(profile, dataRoot).catch(() => undefined);
if (saved?.owner === owner) {
await rm(location.pid, { force: true });
@@ -297,21 +274,10 @@ export async function ensureRuntimeProcess(
}
}
-export async function runtimeProcessOwnsCurrentProfile(
- profile: string,
- dataRoot: string,
- owner: string,
-): Promise<boolean> {
- const current = await record(profile, dataRoot);
- return current?.owner === owner && current.profileConfig === await profileConfig(profile);
-}
-
async function stopRuntimeProcessUnlocked(profile: string, dataRoot: string, timeoutMs: number): Promise<RuntimeProcessState> {
const current = await record(profile, dataRoot);
const location = paths(dataRoot);
- const lease = current === undefined ? undefined : await activeLease(dataRoot);
- if (current === undefined || !processAlive(current.pid)
- || lease?.owner !== current.owner || lease.pid !== current.pid) {
+ if (current === undefined || !processAlive(current.pid)) {
await rm(location.pid, { force: true });
await rm(location.ready, { force: true });
return { state: "stopped", profile: resolve(profile), logPath: location.log };
@@ -339,12 +305,18 @@ async function stopRuntimeProcessUnlocked(profile: string, dataRoot: string, tim
}
await rm(location.pid, { force: true });
await rm(location.ready, { force: true });
- await releaseLease(dataRoot, current.owner, current.pid).catch(() => undefined);
return { state: "stopped", profile: resolve(profile), logPath: location.log };
}
export async function stopRuntimeProcess(profile: string, dataRoot: string, timeoutMs = 10_000): Promise<RuntimeProcessState> {
- return await stopRuntimeProcessUnlocked(profile, dataRoot, timeoutMs);
+ const location = paths(dataRoot);
+ await mkdir(location.root, { recursive: true });
+ const releaseLaunch = await acquireLaunch(location.launch, timeoutMs);
+ try {
+ return await stopRuntimeProcessUnlocked(profile, dataRoot, timeoutMs);
+ } finally {
+ await releaseLaunch();
+ }
}
export async function runtimeProcessLogs(dataRoot: string): Promise<{ readonly path: string; readonly text: string }> {
diff --git a/packages/runtime-local/test/local.test.ts b/packages/runtime-local/test/local.test.ts
index 36c68755..0d8e1fb1 100644
--- a/packages/runtime-local/test/local.test.ts
+++ b/packages/runtime-local/test/local.test.ts
@@ -102,19 +102,19 @@ function durableBuildRequest(
} as const;
}
-test("a Worker that lost Runtime ownership cannot claim a ready Build", async () => {
+test("a Worker does not claim a Build when the pinned Runtime environment differs", async () => {
const directory = await mkdtemp(join(tmpdir(), "hypit-local-worker-owner-"));
const initial = createGreetingBuild();
try {
const runtime = await createLocalRuntime({
...projectRuntimeFixture(directory),
assertEnvironment: () => {
- throw new Error("Runtime Worker no longer owns this Runtime");
+ throw new Error("Runtime Profile no longer matches this Worker's active environment");
},
});
await runtime.build(durableBuildRequest(directory, "bld_20260902T120000000Z_0000000001", initial));
- await assert.rejects(runtime.workOnce(), /no longer owns this Runtime/u);
+ await assert.rejects(runtime.workOnce(), /no longer matches this Worker's active environment/u);
const status = await runtime.inspect("bld_20260902T120000000Z_0000000001");
assert.equal(status?.activity, "ready");
assert.deepEqual(status?.requests, { total: 1, completed: 0 });
diff --git a/packages/runtime-local/test/worker-process.test.ts b/packages/runtime-local/test/worker-process.test.ts
index d87a92b6..44d3542e 100644
--- a/packages/runtime-local/test/worker-process.test.ts
+++ b/packages/runtime-local/test/worker-process.test.ts
@@ -11,7 +11,7 @@ import {
stopRuntimeProcess,
} from "../src/worker-process.js";
-test("one detached Runtime Worker can be started, observed and stopped", async () => {
+test("one live detached Runtime Worker survives repeated starts and stale startup markers", async () => {
const root = await mkdtemp(join(tmpdir(), "hypit-runtime-process-"));
const profile = join(root, "runtime.json");
const dataRoot = join(root, ".hypit", "runtimes", "local");
@@ -21,27 +21,18 @@ test("one detached Runtime Worker can be started, observed and stopped", async (
const program = `
const fs = require("node:fs");
const path = require("node:path");
- const { DatabaseSync } = require("node:sqlite");
const index = process.argv.indexOf("--ready-file");
const ready = process.argv[index + 1];
const ownerIndex = process.argv.indexOf("--worker-owner");
const owner = process.argv[ownerIndex + 1];
- const databasePath = ${JSON.stringify(join(dataRoot, "runtime.sqlite"))};
- fs.mkdirSync(path.dirname(databasePath), { recursive: true });
- const database = new DatabaseSync(databasePath);
- database.exec("CREATE TABLE IF NOT EXISTS hypit_worker_lease (singleton INTEGER PRIMARY KEY CHECK (singleton = 1), owner_id TEXT NOT NULL, pid INTEGER NOT NULL, acquired_at INTEGER NOT NULL, expires_at INTEGER NOT NULL) STRICT");
- const now = Date.now();
- database.prepare("INSERT OR REPLACE INTO hypit_worker_lease (singleton, owner_id, pid, acquired_at, expires_at) VALUES (1, ?, ?, ?, ?)").run(owner, process.pid, now, now + 60000);
fs.mkdirSync(path.dirname(ready), { recursive: true });
// Say it in the log before claiming to be ready. The ready file is what the parent waits on,
// so anything written after it is a race the parent can win.
process.stdout.write("worker-ready\\n");
fs.writeFileSync(ready, owner);
- const heartbeat = setInterval(() => database.prepare("UPDATE hypit_worker_lease SET expires_at = ? WHERE singleton = 1 AND owner_id = ? AND pid = ?").run(Date.now() + 60000, owner, process.pid), 1000);
+ const keepAlive = setInterval(() => undefined, 1000);
process.on("SIGTERM", () => {
- clearInterval(heartbeat);
- database.prepare("DELETE FROM hypit_worker_lease WHERE singleton = 1 AND owner_id = ? AND pid = ?").run(owner, process.pid);
- database.close();
+ clearInterval(keepAlive);
process.exit(0);
});
`;
@@ -75,6 +66,15 @@ test("one detached Runtime Worker can be started, observed and stopped", async (
assert.equal((await runtimeProcessStatus(profile, dataRoot)).state, "running");
assert.match((await runtimeProcessLogs(dataRoot)).text, /worker-ready/u);
+ await rm(join(dataRoot, "worker", "ready"));
+ const afterMissingStartupMarker = await ensureRuntimeProcess(
+ profile,
+ dataRoot,
+ { command: "must-not-run", args: [] },
+ 5_000,
+ );
+ assert.equal(afterMissingStartupMarker.pid, first.pid);
+
await writeFile(profile, JSON.stringify({ format: "hypit.runtime-local@1", changed: true }), "utf8");
assert.equal((await runtimeProcessStatus(profile, dataRoot)).configuration, "changed");
await assert.rejects(
diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts
index 64d78d6e..4eb52c80 100644
--- a/packages/runtime/src/index.ts
+++ b/packages/runtime/src/index.ts
@@ -20,4 +20,3 @@ export {
isStreamingResourceStore,
} from "./types.js";
export type * from "./types.js";
-export type * from "./worker.js";
diff --git a/packages/runtime/src/scheduler.ts b/packages/runtime/src/scheduler.ts
index b2856e02..00ace433 100644
--- a/packages/runtime/src/scheduler.ts
+++ b/packages/runtime/src/scheduler.ts
@@ -191,6 +191,10 @@ export class LocalBuildScheduler {
const settled = await Promise.race([...active.values()].map((item) => item.promise));
active.delete(settled.key);
const build = settled.build;
+ // A command may finish after a concurrently running sibling has already made the
+ // Build terminal. Its capacity has been released, but no later completion, failure or
+ // deferral belongs to Core's outstanding commands or may replace the first outcome.
+ if (build.state.status === "failed" || build.state.status === "complete") continue;
if (settled.execution === undefined) {
build.stopped = true;
build.outcomes.push({
diff --git a/packages/runtime/src/worker.ts b/packages/runtime/src/worker.ts
deleted file mode 100644
index b28157e5..00000000
--- a/packages/runtime/src/worker.ts
+++ /dev/null
@@ -1,16 +0,0 @@
-export type RuntimeWorkerLease = {
- readonly owner: string;
- readonly pid: number;
- readonly acquiredAt: number;
- readonly expiresAt: number;
-};
-
-export type RuntimeWorkerLeaseStore = {
- read(): Promise<RuntimeWorkerLease | undefined>;
- /** Acquire the one live Runtime worker slot, replacing only an expired lease. */
- acquire(request: RuntimeWorkerLease): Promise<boolean>;
- /** Renew only the lease held by this exact owner and process. */
- renew(owner: string, pid: number, expiresAt: number): Promise<boolean>;
- /** Release only the caller's own lease. */
- release(owner: string, pid: number): Promise<void>;
-};
diff --git a/packages/store-sqlite/src/store.ts b/packages/store-sqlite/src/store.ts
index dc642115..55b504de 100644
--- a/packages/store-sqlite/src/store.ts
+++ b/packages/store-sqlite/src/store.ts
@@ -34,8 +34,6 @@ import type {
OperationSnapshot,
OperationStore,
OperationUpdate,
- RuntimeWorkerLease,
- RuntimeWorkerLeaseStore,
RuntimeEnvironmentStore,
} from "@hypit/runtime";
@@ -527,71 +525,6 @@ class SqliteCommandExecutionStore implements CommandExecutionStore {
}
}
-class SqliteRuntimeWorkerLeaseStore implements RuntimeWorkerLeaseStore {
- readonly #database: DatabaseSync;
-
- constructor(database: DatabaseSync) {
- this.#database = database;
- }
-
- async read(): Promise<RuntimeWorkerLease | undefined> {
- const row = this.#database.prepare("SELECT * FROM hypit_worker_lease WHERE singleton = 1").get() as Row | undefined;
- if (row === undefined) return undefined;
- assert(typeof row.owner_id === "string" && typeof row.pid === "number"
- && typeof row.acquired_at === "number" && typeof row.expires_at === "number",
- "SQLite Runtime Worker lease is invalid");
- return {
- owner: row.owner_id,
- pid: row.pid,
- acquiredAt: row.acquired_at,
- expiresAt: row.expires_at,
- };
- }
-
- async acquire(request: RuntimeWorkerLease): Promise<boolean> {
- positiveInteger(request.pid, "Runtime Worker pid");
- nonNegativeInteger(request.acquiredAt, "Runtime Worker acquisition time");
- assert(request.owner.trim().length > 0, "Runtime Worker owner is empty");
- assert(Number.isSafeInteger(request.expiresAt) && request.expiresAt > request.acquiredAt,
- "Runtime Worker lease expiration is invalid");
- return transaction(this.#database, () => {
- const row = this.#database.prepare(
- "SELECT owner_id, pid, expires_at FROM hypit_worker_lease WHERE singleton = 1",
- ).get() as Row | undefined;
- if (row !== undefined) {
- assert(typeof row.owner_id === "string" && typeof row.pid === "number"
- && typeof row.expires_at === "number", "SQLite Runtime Worker lease is invalid");
- if (row.owner_id === request.owner && row.pid === request.pid) return true;
- if (row.expires_at > request.acquiredAt) return false;
- }
- this.#database.prepare(`
- INSERT INTO hypit_worker_lease (singleton, owner_id, pid, acquired_at, expires_at)
- VALUES (1, ?, ?, ?, ?)
- ON CONFLICT(singleton) DO UPDATE SET
- owner_id = excluded.owner_id,
- pid = excluded.pid,
- acquired_at = excluded.acquired_at,
- expires_at = excluded.expires_at
- `).run(request.owner, request.pid, request.acquiredAt, request.expiresAt);
- return true;
- });
- }
-
- async renew(owner: string, pid: number, expiresAt: number): Promise<boolean> {
- const result = this.#database.prepare(`
- UPDATE hypit_worker_lease SET expires_at = ?
- WHERE singleton = 1 AND owner_id = ? AND pid = ?
- `).run(expiresAt, owner, pid);
- return result.changes === 1;
- }
-
- async release(owner: string, pid: number): Promise<void> {
- this.#database.prepare(
- "DELETE FROM hypit_worker_lease WHERE singleton = 1 AND owner_id = ? AND pid = ?",
- ).run(owner, pid);
- }
-}
-
class SqliteRuntimeEnvironmentStore implements RuntimeEnvironmentStore {
readonly #database: DatabaseSync;
@@ -1011,7 +944,6 @@ export class SqliteRuntimeState {
readonly commandExecutions: CommandExecutionStore;
readonly execution: BuildExecutionStore;
readonly catalog: BuildCatalog;
- readonly workerLease: RuntimeWorkerLeaseStore;
readonly environment: RuntimeEnvironmentStore;
readonly submissions: PendingBuildStore;
readonly #database: DatabaseSync;
@@ -1102,13 +1034,6 @@ export class SqliteRuntimeState {
created_at INTEGER NOT NULL,
PRIMARY KEY (build_id, command_id)
) STRICT;
- CREATE TABLE IF NOT EXISTS hypit_worker_lease (
- singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
- owner_id TEXT NOT NULL,
- pid INTEGER NOT NULL,
- acquired_at INTEGER NOT NULL,
- expires_at INTEGER NOT NULL
- ) STRICT;
CREATE TABLE IF NOT EXISTS hypit_runtime_environment (
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
config_json TEXT NOT NULL
@@ -1119,7 +1044,6 @@ export class SqliteRuntimeState {
this.commandExecutions = new SqliteCommandExecutionStore(database);
this.execution = new SqliteBuildExecutionStore(database);
this.catalog = new SqliteBuildCatalog(database);
- this.workerLease = new SqliteRuntimeWorkerLeaseStore(database);
this.environment = new SqliteRuntimeEnvironmentStore(database);
this.submissions = new SqlitePendingBuildStore(database);
}