Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
fix(runtime): keep durable builds progressing under load
kashorinbd39ef50ca7c
diff --git a/packages/driver-node/test/scheduler.test.ts b/packages/driver-node/test/scheduler.test.tsindex 6c60114c..310b89a5 100644--- a/packages/driver-node/test/scheduler.test.ts+++ b/packages/driver-node/test/scheduler.test.ts@@ -246,6 +246,51 @@ test("independent paid commands inside one Build may fill the same resource withrecord.type.name === types.generated.name).length, 2);});+test("a terminal failure keeps its diagnostic while an in-flight sibling settles", async () => {+ const operations = memoryOperations();+ const producers = new ProducerRegistry();+ const endpoints = new EndpointRegistry();+ registerGreetingProducers(producers);+ let calls = 0;+ const endpoint: AsyncEndpoint = {+ async start() {+ calls += 1;+ if (calls === 1) {+ return {+ status: "failed",+ failure: { code: "PROVIDER_REJECTED", message: "the first provider request was rejected" },+ };+ }+ await new Promise<void>((resolve) => setTimeout(resolve, 20));+ return {+ status: "completed",+ result: { value: { kind: "inline", value: "late sibling result" } },+ };+ },+ poll() {+ throw new Error("completed fixture operations are not polled");+ },+ };+ endpoints.registerAsyncEndpoint(+ "generation.parallel",+ capabilities.generation,+ types.generated,+ endpoint,+ { scheduling: { resources: [{ id: "pool:fixture.account", limit: 2 }] } },+ );++ const [result] = await new LocalBuildScheduler(new NodeDriver({ producers, endpoints, operations })).run([{+ id: "parallel-failure",+ state: createParallelGreetingBuild(),+ }]);++ assert.equal(calls, 2);+ assert.equal(result?.status, "failed");+ assert.equal(result?.state.diagnostics.at(-1)?.code, "PROVIDER_REJECTED");+ assert.match(result?.state.diagnostics.at(-1)?.message ?? "", /first provider request was rejected/);+ assert.equal(result?.outcomes.some((outcome) => outcome.status === "error"), false);+});+test("an asynchronous Endpoint starts once and is polled until complete", async () => {const operations = memoryOperations();let starts = 0;diff --git a/packages/provider-hyperframes-local/README.md b/packages/provider-hyperframes-local/README.mdindex 718de158..0612996c 100644--- a/packages/provider-hyperframes-local/README.md+++ b/packages/provider-hyperframes-local/README.md@@ -42,10 +42,12 @@ const visual = await renderHyperframesVisual(Omit `range` for the complete document. Ranges are zero-based and half-open on the originalprogramme clock. At 30 fps, `[240, 360)` returns 120 frames covering seconds 8–12.`onProgress` reports preparation, worker ranges, browser PIDs, completion and elapsed time.-`signal` cancels capture and encoding; `processTimeoutMs` bounds the operation.-`initializationTimeoutMs` (30 seconds by default) and `frameTimeoutMs` (15 seconds) bound each-worker's initialization and individual frame capture. Errors name the worker and stage/frame.-A completed worker closes its Chrome immediately. A timeout aborts the other workers and awaits cleanup.+`signal` cancels capture and encoding; `processTimeoutMs` bounds the complete operation and defaults+to 30 minutes. Deployments may additionally set `initializationTimeoutMs` or `frameTimeoutMs` when+they have a measured stage deadline. Stage deadlines are otherwise unset: a busy machine must not+turn one slow Chrome start or frame into a false render failure while the operation deadline remains.+An explicit stage-timeout error names the worker and stage/frame, aborts sibling workers and awaits+cleanup. A completed worker closes its Chrome immediately.One call stages the HTML and assets once, merges overlapping source-frame extraction windows,and shares decoded PNGs across workers. Each worker initializes its own page and captures itsdiff --git a/packages/provider-hyperframes-local/src/render.ts b/packages/provider-hyperframes-local/src/render.tsindex bac225ef..6640a02b 100644--- a/packages/provider-hyperframes-local/src/render.ts+++ b/packages/provider-hyperframes-local/src/render.ts@@ -40,8 +40,12 @@ export function resolveExecutionOptions(options: HyperframesExecutionOptions) {quality, browserGpu,ffmpegPath: options.ffmpegPath ?? "ffmpeg",ffprobePath: options.ffprobePath ?? "ffprobe",- initializationTimeoutMs: positiveInteger(options.initializationTimeoutMs ?? 30_000, "initializationTimeoutMs"),- frameTimeoutMs: positiveInteger(options.frameTimeoutMs ?? 15_000, "frameTimeoutMs"),+ initializationTimeoutMs: options.initializationTimeoutMs === undefined+ ? undefined+ : positiveInteger(options.initializationTimeoutMs, "initializationTimeoutMs"),+ frameTimeoutMs: options.frameTimeoutMs === undefined+ ? undefined+ : positiveInteger(options.frameTimeoutMs, "frameTimeoutMs"),processTimeoutMs: positiveInteger(options.processTimeoutMs ?? 30 * 60_000, "processTimeoutMs"),maxProcessOutputBytes: positiveInteger(options.maxProcessOutputBytes ?? 4 * 1024 * 1024, "maxProcessOutputBytes"),maxRenderedBytes: positiveInteger(options.maxRenderedBytes ?? 16 * 1024 * 1024 * 1024, "maxRenderedBytes"),@@ -79,16 +83,21 @@ export async function renderHyperframesVisual(}return promise;};- const stage = async <T>(subject: string, timeoutMs: number, run: () => Promise<T>): Promise<T> => {+ const stage = async <T>(subject: string, timeoutMs: number | undefined, run: () => Promise<T>): Promise<T> => {signal.throwIfAborted();- const timeout = setTimeout(() => controller.abort(new Error(`HyperFrames ${subject} timed out after ${timeoutMs} ms`)), timeoutMs);+ const timeout = timeoutMs === undefined+ ? undefined+ : setTimeout(() => controller.abort(new Error(`HyperFrames ${subject} timed out after ${timeoutMs} ms`)), timeoutMs);let onAbort: () => void = () => {};const stopped = new Promise<never>((_, reject) => {onAbort = () => reject(signal.reason);signal.addEventListener("abort", onAbort, { once: true });});try { return await Promise.race([run(), stopped]); }- finally { clearTimeout(timeout); signal.removeEventListener("abort", onAbort); }+ finally {+ if (timeout !== undefined) clearTimeout(timeout);+ signal.removeEventListener("abort", onAbort);+ }};let server: Awaited<ReturnType<typeof createFileServer>> | undefined;// Closing pages interrupts an in-flight capture as well as the next loop iteration.diff --git a/packages/provider-hyperframes-local/test/range.test.ts b/packages/provider-hyperframes-local/test/range.test.tsindex faabd9aa..96b72663 100644--- a/packages/provider-hyperframes-local/test/range.test.ts+++ b/packages/provider-hyperframes-local/test/range.test.ts@@ -11,6 +11,7 @@ import { compileHyperframesDocument } from "@hypit/hyperframes";import { sealProgramSpace } from "@hypit/program-space";import type { BlobRef } from "@hypit/protocol";import { renderHyperframesVisual } from "../src/index.js";+import { resolveExecutionOptions } from "../src/render.js";import { hypitPackage } from "../src/activation.js";import type { RuntimeEndpointAdapterImplementation } from "@hypit/runtime-kit";import { endpointResourceClaims } from "@hypit/endpoint-kit";@@ -42,6 +43,17 @@ function documentFor(artifact: BlobRef) {canvas: { width: 64, height: 64, clearColor: "#000000" }, tracks: [track] }), space);}+test("the render deadline is global while stage deadlines are explicit deployment policy", () => {+ const defaults = resolveExecutionOptions({});+ assert.equal(defaults.processTimeoutMs, 30 * 60_000);+ assert.equal(defaults.initializationTimeoutMs, undefined);+ assert.equal(defaults.frameTimeoutMs, undefined);++ const explicit = resolveExecutionOptions({ initializationTimeoutMs: 31_000, frameTimeoutMs: 16_000 });+ assert.equal(explicit.initializationTimeoutMs, 31_000);+ assert.equal(explicit.frameTimeoutMs, 16_000);+});+test("source selection retains loop, hold and fractional-speed sampling and shares decoded frames", () => {const document = documentFor({ kind: "blob", resource: "res_range_source", size: 1, mediaType: "video/mp4" });const slots = videoSlots(document.html);diff --git a/packages/runtime-local/src/config.ts b/packages/runtime-local/src/config.tsindex 51fb9b99..a6fbe0be 100644--- a/packages/runtime-local/src/config.ts+++ b/packages/runtime-local/src/config.ts@@ -108,8 +108,6 @@ export type LoadRuntimeConfigOptions = {readonly distributionPackageRoot?: string;/** Persistent machine/user state. Defaults to the platform Hypit state root. */readonly hostStateRoot?: string;- /** Worker-only ownership check performed before reclaiming or claiming execution. */- readonly assertExecutionOwner?: () => Promise<void> | void;};export type RuntimeConfigDoctorResult = {@@ -988,7 +986,6 @@ export async function createRuntimeFromConfig(commandExecutionStore: state.commandExecutions,assertEnvironment: async () => {await state.environment.assert(environmentConfig);- await options.assertExecutionOwner?.();},executionStore: state.execution,removeActiveBuild: async (build) => await state.removeActiveBuild(build),diff --git a/packages/runtime-local/src/host.ts b/packages/runtime-local/src/host.tsindex 4d09f5a4..9265456d 100644--- a/packages/runtime-local/src/host.ts+++ b/packages/runtime-local/src/host.ts@@ -4,8 +4,7 @@ import type {RuntimeWorkerLaunch,} from "@hypit/runtime-host-node";import { hypitHostStateRoot } from "@hypit/runtime-host-node";-import { join, resolve } from "node:path";-import { SqliteRuntimeState } from "@hypit/store-sqlite";+import { resolve } from "node:path";import {createRuntimeControlFromConfig,@@ -28,7 +27,6 @@ import {import {ensureRuntimeProcess,markRuntimeProcessReady,- runtimeProcessOwnsCurrentProfile,runtimeProcessLogs,runtimeProcessStatus,stopRuntimeProcess,@@ -146,67 +144,25 @@ export async function openLocalRuntimeHost(...distribution,}),runWorker: async (readyFile, owner) => {- const selected = await resolveRuntimeConfigPaths(profile, { packageRoot: basePackageRoot, ...distribution });- const leaseState = new SqliteRuntimeState(join(selected.dataRoot, "runtime.sqlite"));- const leaseMs = 30_000;- const acquiredAt = Date.now();- const acquired = await leaseState.workerLease.acquire({- owner,- pid: process.pid,- acquiredAt,- expiresAt: acquiredAt + leaseMs,- });- if (!acquired) {- leaseState.close();- throw new Error("another Runtime Worker owns this Runtime");- }let runtime: Awaited<ReturnType<typeof createRuntimeFromConfig>> | undefined;const abort = new AbortController();const stop = (): void => abort.abort();process.once("SIGTERM", stop);process.once("SIGINT", stop);- let renewal = Promise.resolve();- const heartbeat = setInterval(() => {- renewal = renewal.then(async () => {- if (!await runtimeProcessOwnsCurrentProfile(profile, selected.dataRoot, owner)) {- abort.abort(new Error("Runtime Profile changed; this Worker stopped before using mixed configuration"));- return;- }- const renewed = await leaseState.workerLease.renew(owner, process.pid, Date.now() + leaseMs);- if (!renewed) abort.abort(new Error("Runtime Worker ownership was lost"));- }).catch((error: unknown) => abort.abort(error));- }, 5_000);try {- const assertExecutionOwner = async (): Promise<void> => {- const lease = await leaseState.workerLease.read();- if (lease?.owner !== owner || lease.pid !== process.pid || lease.expiresAt <= Date.now()) {- throw new Error("Runtime Worker no longer owns this Runtime");- }- };runtime = await createRuntimeFromConfig(profile, {packageRoot: basePackageRoot,...distribution,- assertExecutionOwner,});await runtime.work({idlePollMs: 250,signal: abort.signal,- ready: async () => {- await assertExecutionOwner();- if (!await runtimeProcessOwnsCurrentProfile(profile, selected.dataRoot, owner)) {- throw new Error("Runtime Profile changed while the Worker was starting");- }- await markRuntimeProcessReady(readyFile, owner);- },+ ready: async () => await markRuntimeProcessReady(readyFile, owner),});} finally {- clearInterval(heartbeat);- await renewal.catch(() => undefined);process.removeListener("SIGTERM", stop);process.removeListener("SIGINT", stop);await runtime?.close();- await leaseState.workerLease.release(owner, process.pid).catch(() => undefined);- leaseState.close();}},};diff --git a/packages/runtime-local/src/worker-process.ts b/packages/runtime-local/src/worker-process.tsindex 6966831a..38075c71 100644--- a/packages/runtime-local/src/worker-process.ts+++ b/packages/runtime-local/src/worker-process.ts@@ -3,7 +3,6 @@ import { randomUUID } from "node:crypto";import { mkdir, open, readFile, rename, rm, stat, writeFile } from "node:fs/promises";import { dirname, join, resolve } from "node:path";-import { SqliteRuntimeState } from "@hypit/store-sqlite";import { canonicalStringify } from "@hypit/protocol";import { processAlive, stopProcessTree } from "./process-control.js";@@ -54,16 +53,6 @@ function paths(dataRoot: string) {};}-async function exists(path: string): Promise<boolean> {- try {- await stat(path);- return true;- } catch (error) {- if (error instanceof Error && "code" in error && error.code === "ENOENT") return false;- throw error;- }-}-async function rotateLog(path: string): Promise<void> {let size = 0;try {@@ -99,27 +88,6 @@ async function profileConfig(profile: string): Promise<string> {return canonicalStringify(JSON.parse(await readFile(resolve(profile), "utf8")));}-async function activeLease(dataRoot: string) {- const state = new SqliteRuntimeState(join(resolve(dataRoot), "runtime.sqlite"), { readOnly: true });- try {- return await state.workerLease.read();- } catch (error) {- if (error instanceof Error && error.message.includes("no such table: hypit_worker_lease")) return undefined;- throw error;- } finally {- state.close();- }-}--async function releaseLease(dataRoot: string, owner: string, pid: number): Promise<void> {- const state = new SqliteRuntimeState(join(resolve(dataRoot), "runtime.sqlite"));- try {- await state.workerLease.release(owner, pid);- } finally {- state.close();- }-}-async function readyOwner(path: string): Promise<string | undefined> {try {const value = (await readFile(path, "utf8")).trim();@@ -136,11 +104,7 @@ export async function runtimeProcessStatus(): Promise<RuntimeProcessState> {const location = paths(dataRoot);const current = await record(profile, dataRoot);- const lease = current === undefined ? undefined : await activeLease(dataRoot);- const ready = current === undefined ? undefined : await readyOwner(location.ready);- if (current === undefined || !processAlive(current.pid)- || lease?.owner !== current.owner || lease.pid !== current.pid || lease.expiresAt <= Date.now()- || ready !== current.owner) {+ if (current === undefined || !processAlive(current.pid)) {return { state: "stopped", profile: resolve(profile), logPath: location.log };}return {@@ -187,6 +151,7 @@ async function acquireLaunch(path: string, timeoutMs: number): Promise<() => Proasync function waitForReady(profile: string,dataRoot: string,+ owner: string,timeoutMs: number,): Promise<RuntimeProcessState> {const location = paths(dataRoot);@@ -202,43 +167,46 @@ async function waitForReady(}throw new Error(`Runtime Worker exited before becoming ready${log.length === 0 ? "" : `: ${log.trim().split("\n").at(-1)}`}`);}- if (await exists(location.ready)) {- const state = await runtimeProcessStatus(profile, dataRoot);- if (state.state === "running") return state;+ if (current.owner !== owner) {+ throw new Error("Runtime Worker record changed while waiting for the Worker to become ready");+ }+ if (await readyOwner(location.ready) === owner) {+ return await runtimeProcessStatus(profile, dataRoot);}await new Promise((resolveWait) => setTimeout(resolveWait, 10));}throw new Error(`Runtime Worker did not become ready within ${timeoutMs}ms; log: ${location.log}`);}+async function waitForProcessExit(pid: number, timeoutMs: number): Promise<boolean> {+ const deadline = Date.now() + timeoutMs;+ while (Date.now() <= deadline) {+ if (!processAlive(pid)) return true;+ await new Promise((resolveWait) => setTimeout(resolveWait, 20));+ }+ return !processAlive(pid);+}+export async function ensureRuntimeProcess(profile: string,dataRoot: string,launch: RuntimeWorkerLaunch,timeoutMs = 10_000,): Promise<RuntimeProcessState> {- const current = await runtimeProcessStatus(profile, dataRoot);- if (current.state === "running") {- if (current.configuration === "changed") {- throw new Error("Runtime Profile changed while its Worker is running; stop and start the Worker to activate it");- }- return current;- }const absolute = resolve(profile);const location = paths(dataRoot);await mkdir(location.root, { recursive: true });const releaseLaunch = await acquireLaunch(location.launch, timeoutMs);try {- const raced = await runtimeProcessStatus(profile, dataRoot);- if (raced.state === "running") {- if (raced.configuration === "changed") {+ const current = await runtimeProcessStatus(profile, dataRoot);+ if (current.state === "running") {+ if (current.configuration === "changed") {throw new Error("Runtime Profile changed while its Worker is running; stop and start the Worker to activate it");}- return raced;+ return current;}const stale = await record(profile, dataRoot).catch(() => undefined);if (stale !== undefined && !processAlive(stale.pid)) {- await releaseLease(dataRoot, stale.owner, stale.pid).catch(() => undefined);await rm(location.pid, { force: true });await rm(location.ready, { force: true });}@@ -278,10 +246,19 @@ export async function ensureRuntimeProcess(} satisfies ProcessRecord), "utf8");child.unref();try {- return await waitForReady(absolute, dataRoot, timeoutMs);+ const ready = await waitForReady(absolute, dataRoot, owner, timeoutMs);+ if (ready.configuration === "changed") {+ throw new Error("Runtime Profile changed while its Worker was starting; start it again to activate the new Profile");+ }+ return ready;} catch (error) {- await stopProcessTree(child.pid, true).catch(() => "gone");- await releaseLease(dataRoot, owner, child.pid).catch(() => undefined);+ const stopped = await stopProcessTree(child.pid, true).catch(() => "denied" as const);+ if (stopped === "denied" || !await waitForProcessExit(child.pid, 2_000)) {+ throw new Error(+ `Runtime Worker ${child.pid} failed to become ready and is still running; stop it explicitly before retrying`,+ { cause: error },+ );+ }const saved = await record(profile, dataRoot).catch(() => undefined);if (saved?.owner === owner) {await rm(location.pid, { force: true });@@ -297,21 +274,10 @@ export async function ensureRuntimeProcess(}}-export async function runtimeProcessOwnsCurrentProfile(- profile: string,- dataRoot: string,- owner: string,-): Promise<boolean> {- const current = await record(profile, dataRoot);- return current?.owner === owner && current.profileConfig === await profileConfig(profile);-}-async function stopRuntimeProcessUnlocked(profile: string, dataRoot: string, timeoutMs: number): Promise<RuntimeProcessState> {const current = await record(profile, dataRoot);const location = paths(dataRoot);- const lease = current === undefined ? undefined : await activeLease(dataRoot);- if (current === undefined || !processAlive(current.pid)- || lease?.owner !== current.owner || lease.pid !== current.pid) {+ if (current === undefined || !processAlive(current.pid)) {await rm(location.pid, { force: true });await rm(location.ready, { force: true });return { state: "stopped", profile: resolve(profile), logPath: location.log };@@ -339,12 +305,18 @@ async function stopRuntimeProcessUnlocked(profile: string, dataRoot: string, tim}await rm(location.pid, { force: true });await rm(location.ready, { force: true });- await releaseLease(dataRoot, current.owner, current.pid).catch(() => undefined);return { state: "stopped", profile: resolve(profile), logPath: location.log };}export async function stopRuntimeProcess(profile: string, dataRoot: string, timeoutMs = 10_000): Promise<RuntimeProcessState> {- return await stopRuntimeProcessUnlocked(profile, dataRoot, timeoutMs);+ const location = paths(dataRoot);+ await mkdir(location.root, { recursive: true });+ const releaseLaunch = await acquireLaunch(location.launch, timeoutMs);+ try {+ return await stopRuntimeProcessUnlocked(profile, dataRoot, timeoutMs);+ } finally {+ await releaseLaunch();+ }}export async function runtimeProcessLogs(dataRoot: string): Promise<{ readonly path: string; readonly text: string }> {diff --git a/packages/runtime-local/test/local.test.ts b/packages/runtime-local/test/local.test.tsindex 36c68755..0d8e1fb1 100644--- a/packages/runtime-local/test/local.test.ts+++ b/packages/runtime-local/test/local.test.ts@@ -102,19 +102,19 @@ function durableBuildRequest(} as const;}-test("a Worker that lost Runtime ownership cannot claim a ready Build", async () => {+test("a Worker does not claim a Build when the pinned Runtime environment differs", async () => {const directory = await mkdtemp(join(tmpdir(), "hypit-local-worker-owner-"));const initial = createGreetingBuild();try {const runtime = await createLocalRuntime({...projectRuntimeFixture(directory),assertEnvironment: () => {- throw new Error("Runtime Worker no longer owns this Runtime");+ throw new Error("Runtime Profile no longer matches this Worker's active environment");},});await runtime.build(durableBuildRequest(directory, "bld_20260902T120000000Z_0000000001", initial));- await assert.rejects(runtime.workOnce(), /no longer owns this Runtime/u);+ await assert.rejects(runtime.workOnce(), /no longer matches this Worker's active environment/u);const status = await runtime.inspect("bld_20260902T120000000Z_0000000001");assert.equal(status?.activity, "ready");assert.deepEqual(status?.requests, { total: 1, completed: 0 });diff --git a/packages/runtime-local/test/worker-process.test.ts b/packages/runtime-local/test/worker-process.test.tsindex d87a92b6..44d3542e 100644--- a/packages/runtime-local/test/worker-process.test.ts+++ b/packages/runtime-local/test/worker-process.test.ts@@ -11,7 +11,7 @@ import {stopRuntimeProcess,} from "../src/worker-process.js";-test("one detached Runtime Worker can be started, observed and stopped", async () => {+test("one live detached Runtime Worker survives repeated starts and stale startup markers", async () => {const root = await mkdtemp(join(tmpdir(), "hypit-runtime-process-"));const profile = join(root, "runtime.json");const dataRoot = join(root, ".hypit", "runtimes", "local");@@ -21,27 +21,18 @@ test("one detached Runtime Worker can be started, observed and stopped", async (const program = `const fs = require("node:fs");const path = require("node:path");- const { DatabaseSync } = require("node:sqlite");const index = process.argv.indexOf("--ready-file");const ready = process.argv[index + 1];const ownerIndex = process.argv.indexOf("--worker-owner");const owner = process.argv[ownerIndex + 1];- const databasePath = ${JSON.stringify(join(dataRoot, "runtime.sqlite"))};- fs.mkdirSync(path.dirname(databasePath), { recursive: true });- const database = new DatabaseSync(databasePath);- database.exec("CREATE TABLE IF NOT EXISTS hypit_worker_lease (singleton INTEGER PRIMARY KEY CHECK (singleton = 1), owner_id TEXT NOT NULL, pid INTEGER NOT NULL, acquired_at INTEGER NOT NULL, expires_at INTEGER NOT NULL) STRICT");- const now = Date.now();- database.prepare("INSERT OR REPLACE INTO hypit_worker_lease (singleton, owner_id, pid, acquired_at, expires_at) VALUES (1, ?, ?, ?, ?)").run(owner, process.pid, now, now + 60000);fs.mkdirSync(path.dirname(ready), { recursive: true });// Say it in the log before claiming to be ready. The ready file is what the parent waits on,// so anything written after it is a race the parent can win.process.stdout.write("worker-ready\\n");fs.writeFileSync(ready, owner);- const heartbeat = setInterval(() => database.prepare("UPDATE hypit_worker_lease SET expires_at = ? WHERE singleton = 1 AND owner_id = ? AND pid = ?").run(Date.now() + 60000, owner, process.pid), 1000);+ const keepAlive = setInterval(() => undefined, 1000);process.on("SIGTERM", () => {- clearInterval(heartbeat);- database.prepare("DELETE FROM hypit_worker_lease WHERE singleton = 1 AND owner_id = ? AND pid = ?").run(owner, process.pid);- database.close();+ clearInterval(keepAlive);process.exit(0);});`;@@ -75,6 +66,15 @@ test("one detached Runtime Worker can be started, observed and stopped", async (assert.equal((await runtimeProcessStatus(profile, dataRoot)).state, "running");assert.match((await runtimeProcessLogs(dataRoot)).text, /worker-ready/u);+ await rm(join(dataRoot, "worker", "ready"));+ const afterMissingStartupMarker = await ensureRuntimeProcess(+ profile,+ dataRoot,+ { command: "must-not-run", args: [] },+ 5_000,+ );+ assert.equal(afterMissingStartupMarker.pid, first.pid);+await writeFile(profile, JSON.stringify({ format: "hypit.runtime-local@1", changed: true }), "utf8");assert.equal((await runtimeProcessStatus(profile, dataRoot)).configuration, "changed");await assert.rejects(diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.tsindex 64d78d6e..4eb52c80 100644--- a/packages/runtime/src/index.ts+++ b/packages/runtime/src/index.ts@@ -20,4 +20,3 @@ export {isStreamingResourceStore,} from "./types.js";export type * from "./types.js";-export type * from "./worker.js";diff --git a/packages/runtime/src/scheduler.ts b/packages/runtime/src/scheduler.tsindex b2856e02..00ace433 100644--- a/packages/runtime/src/scheduler.ts+++ b/packages/runtime/src/scheduler.ts@@ -191,6 +191,10 @@ export class LocalBuildScheduler {const settled = await Promise.race([...active.values()].map((item) => item.promise));active.delete(settled.key);const build = settled.build;+ // A command may finish after a concurrently running sibling has already made the+ // Build terminal. Its capacity has been released, but no later completion, failure or+ // deferral belongs to Core's outstanding commands or may replace the first outcome.+ if (build.state.status === "failed" || build.state.status === "complete") continue;if (settled.execution === undefined) {build.stopped = true;build.outcomes.push({diff --git a/packages/runtime/src/worker.ts b/packages/runtime/src/worker.tsdeleted file mode 100644index b28157e5..00000000--- a/packages/runtime/src/worker.ts+++ /dev/null@@ -1,16 +0,0 @@-export type RuntimeWorkerLease = {- readonly owner: string;- readonly pid: number;- readonly acquiredAt: number;- readonly expiresAt: number;-};--export type RuntimeWorkerLeaseStore = {- read(): Promise<RuntimeWorkerLease | undefined>;- /** Acquire the one live Runtime worker slot, replacing only an expired lease. */- acquire(request: RuntimeWorkerLease): Promise<boolean>;- /** Renew only the lease held by this exact owner and process. */- renew(owner: string, pid: number, expiresAt: number): Promise<boolean>;- /** Release only the caller's own lease. */- release(owner: string, pid: number): Promise<void>;-};diff --git a/packages/store-sqlite/src/store.ts b/packages/store-sqlite/src/store.tsindex dc642115..55b504de 100644--- a/packages/store-sqlite/src/store.ts+++ b/packages/store-sqlite/src/store.ts@@ -34,8 +34,6 @@ import type {OperationSnapshot,OperationStore,OperationUpdate,- RuntimeWorkerLease,- RuntimeWorkerLeaseStore,RuntimeEnvironmentStore,} from "@hypit/runtime";@@ -527,71 +525,6 @@ class SqliteCommandExecutionStore implements CommandExecutionStore {}}-class SqliteRuntimeWorkerLeaseStore implements RuntimeWorkerLeaseStore {- readonly #database: DatabaseSync;-- constructor(database: DatabaseSync) {- this.#database = database;- }-- async read(): Promise<RuntimeWorkerLease | undefined> {- const row = this.#database.prepare("SELECT * FROM hypit_worker_lease WHERE singleton = 1").get() as Row | undefined;- if (row === undefined) return undefined;- assert(typeof row.owner_id === "string" && typeof row.pid === "number"- && typeof row.acquired_at === "number" && typeof row.expires_at === "number",- "SQLite Runtime Worker lease is invalid");- return {- owner: row.owner_id,- pid: row.pid,- acquiredAt: row.acquired_at,- expiresAt: row.expires_at,- };- }-- async acquire(request: RuntimeWorkerLease): Promise<boolean> {- positiveInteger(request.pid, "Runtime Worker pid");- nonNegativeInteger(request.acquiredAt, "Runtime Worker acquisition time");- assert(request.owner.trim().length > 0, "Runtime Worker owner is empty");- assert(Number.isSafeInteger(request.expiresAt) && request.expiresAt > request.acquiredAt,- "Runtime Worker lease expiration is invalid");- return transaction(this.#database, () => {- const row = this.#database.prepare(- "SELECT owner_id, pid, expires_at FROM hypit_worker_lease WHERE singleton = 1",- ).get() as Row | undefined;- if (row !== undefined) {- assert(typeof row.owner_id === "string" && typeof row.pid === "number"- && typeof row.expires_at === "number", "SQLite Runtime Worker lease is invalid");- if (row.owner_id === request.owner && row.pid === request.pid) return true;- if (row.expires_at > request.acquiredAt) return false;- }- this.#database.prepare(`- INSERT INTO hypit_worker_lease (singleton, owner_id, pid, acquired_at, expires_at)- VALUES (1, ?, ?, ?, ?)- ON CONFLICT(singleton) DO UPDATE SET- owner_id = excluded.owner_id,- pid = excluded.pid,- acquired_at = excluded.acquired_at,- expires_at = excluded.expires_at- `).run(request.owner, request.pid, request.acquiredAt, request.expiresAt);- return true;- });- }-- async renew(owner: string, pid: number, expiresAt: number): Promise<boolean> {- const result = this.#database.prepare(`- UPDATE hypit_worker_lease SET expires_at = ?- WHERE singleton = 1 AND owner_id = ? AND pid = ?- `).run(expiresAt, owner, pid);- return result.changes === 1;- }-- async release(owner: string, pid: number): Promise<void> {- this.#database.prepare(- "DELETE FROM hypit_worker_lease WHERE singleton = 1 AND owner_id = ? AND pid = ?",- ).run(owner, pid);- }-}-class SqliteRuntimeEnvironmentStore implements RuntimeEnvironmentStore {readonly #database: DatabaseSync;@@ -1011,7 +944,6 @@ export class SqliteRuntimeState {readonly commandExecutions: CommandExecutionStore;readonly execution: BuildExecutionStore;readonly catalog: BuildCatalog;- readonly workerLease: RuntimeWorkerLeaseStore;readonly environment: RuntimeEnvironmentStore;readonly submissions: PendingBuildStore;readonly #database: DatabaseSync;@@ -1102,13 +1034,6 @@ export class SqliteRuntimeState {created_at INTEGER NOT NULL,PRIMARY KEY (build_id, command_id)) STRICT;- CREATE TABLE IF NOT EXISTS hypit_worker_lease (- singleton INTEGER PRIMARY KEY CHECK (singleton = 1),- owner_id TEXT NOT NULL,- pid INTEGER NOT NULL,- acquired_at INTEGER NOT NULL,- expires_at INTEGER NOT NULL- ) STRICT;CREATE TABLE IF NOT EXISTS hypit_runtime_environment (singleton INTEGER PRIMARY KEY CHECK (singleton = 1),config_json TEXT NOT NULL@@ -1119,7 +1044,6 @@ export class SqliteRuntimeState {this.commandExecutions = new SqliteCommandExecutionStore(database);this.execution = new SqliteBuildExecutionStore(database);this.catalog = new SqliteBuildCatalog(database);- this.workerLease = new SqliteRuntimeWorkerLeaseStore(database);this.environment = new SqliteRuntimeEnvironmentStore(database);this.submissions = new SqlitePendingBuildStore(database);}