Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

fix runtime worker cold-start races

rpone39f37e53e677
diff --git a/packages/driver-node/src/driver.ts b/packages/driver-node/src/driver.ts
index eab66623..1d090bde 100644
--- a/packages/driver-node/src/driver.ts
+++ b/packages/driver-node/src/driver.ts
@@ -1,4 +1,4 @@
-import { randomUUID } from "node:crypto";
+import { createHash } from "node:crypto";
import { reduce, resolveProducer } from "@hypit/core";
import type { ProducerHandlerResult } from "@hypit/component-kit";
@@ -85,6 +85,11 @@ function failureMessage(error: unknown): string {
return [...new Set(parts)].join("; caused by: ");
}
+function operationId(identity: Pick<OperationSnapshot, "build" | "command" | "endpoint" | "pool" | "lane">): string {
+ const key = [identity.build, identity.command, identity.endpoint, identity.pool, identity.lane].join("\u0000");
+ return `op_${createHash("sha256").update(key).digest("hex")}`;
+}
+
export class NodeDriver {
readonly producers: ProducerRegistry;
readonly endpoints: EndpointRegistry;
@@ -312,7 +317,7 @@ export class NodeDriver {
}
const fresh = latest === undefined;
const identity = fresh
- ? { id: `op_${randomUUID()}`, ...base }
+ ? { id: operationId(base), ...base }
: latest;
const endpointContext = {
command: structuredClone(executable.command),
@@ -333,8 +338,19 @@ export class NodeDriver {
handle: structuredClone(latest.handle),
});
}
- const write = async (update: OperationUpdate): Promise<OperationSnapshot> =>
- fresh ? await operations.create({ ...identity, ...update }) : await operations.update(identity.id, update);
+ const write = async (update: OperationUpdate): Promise<OperationSnapshot> => {
+ if (!fresh) return await operations.update(identity.id, update);
+ try {
+ return await operations.create({ ...identity, ...update });
+ } catch (error) {
+ // Another Worker may have started the same Build/Command concurrently. The deterministic
+ // identity makes that write a harmless race: reuse the winner's durable Operation rather
+ // than issuing a second logical Operation to the caller.
+ const existing = await operations.read(identity.id);
+ if (existing !== undefined) return existing;
+ throw error;
+ }
+ };
if (outcome.status === "pending") {
const written = await write({
status: "pending",
diff --git a/packages/driver-node/test/scheduler.test.ts b/packages/driver-node/test/scheduler.test.ts
index 2e05e82e..9d14af11 100644
--- a/packages/driver-node/test/scheduler.test.ts
+++ b/packages/driver-node/test/scheduler.test.ts
@@ -26,6 +26,7 @@ function memoryOperations(): OperationStore {
const values = new Map<string, OperationSnapshot>();
return {
async create(operation) {
+ if (values.has(operation.id)) throw new Error(`Operation ${operation.id} already exists`);
values.set(operation.id, structuredClone(operation));
return structuredClone(operation);
},
@@ -295,6 +296,37 @@ test("an asynchronous Endpoint starts once and is polled until complete", async
assert.equal((await operations.read(pending.operation))?.status, "completed");
});
+test("concurrent first attempts use one stable Operation identity", async () => {
+ const operations = memoryOperations();
+ const operationIds: string[] = [];
+ let starts = 0;
+ let release!: () => void;
+ const bothStarted = new Promise<void>((resolve) => { release = resolve; });
+ const endpoint: AsyncEndpoint = {
+ async start({ operation }) {
+ operationIds.push(operation);
+ starts += 1;
+ if (starts === 2) release();
+ await bothStarted;
+ return { status: "pending", handle: { remoteJob: operation } };
+ },
+ poll() {
+ throw new Error("poll is not part of this race test");
+ },
+ };
+ const first = asyncExecutor(endpoint, operations);
+ const second = asyncExecutor(endpoint, operations);
+ const results = await Promise.all([
+ new LocalBuildScheduler(first).run([{ id: "same-video", state: createGreetingBuild() }]),
+ new LocalBuildScheduler(second).run([{ id: "same-video", state: createGreetingBuild() }]),
+ ]);
+ assert.equal(starts, 2);
+ assert.equal(operationIds.length, 2);
+ assert.equal(operationIds[0], operationIds[1]);
+ assert.equal((await operations.list({ build: "same-video" })).length, 1);
+ assert.equal(results.every(([result]) => result?.status === "paused"), true);
+});
+
test("wakeAt prevents early polling and Runtime cancellation becomes a terminal Core failure", async () => {
const operations = memoryOperations();
let polls = 0;
diff --git a/packages/runtime-local/src/worker-process.ts b/packages/runtime-local/src/worker-process.ts
index c6caab68..6076e37b 100644
--- a/packages/runtime-local/src/worker-process.ts
+++ b/packages/runtime-local/src/worker-process.ts
@@ -36,11 +36,73 @@ function paths(dataRoot: string) {
return {
root,
pid: join(root, "worker.json"),
+ lock: join(root, "worker.lock"),
ready: join(root, "ready"),
log: join(root, "worker.log"),
};
}
+async function acquireProcessLock(dataRoot: string, timeoutMs: number) {
+ const location = paths(dataRoot);
+ const deadline = Date.now() + timeoutMs;
+ await mkdir(location.root, { recursive: true });
+ while (true) {
+ let lock;
+ try {
+ lock = await open(location.lock, "wx");
+ } catch (error) {
+ if (!nodeError(error, "EEXIST")) throw error;
+ // A crashed host can leave the lock file behind. Only remove a lock whose owner is
+ // positively known to be gone; an empty/invalid file is treated as held while its writer
+ // finishes publishing the owner record.
+ let owner: unknown;
+ let ownerReadable = true;
+ try {
+ owner = JSON.parse(await readFile(location.lock, "utf8"));
+ } catch (readError) {
+ if (nodeError(readError, "ENOENT")) continue;
+ ownerReadable = false;
+ }
+ const ownerPid = owner !== null && typeof owner === "object" && "pid" in owner
+ && typeof owner.pid === "number" ? owner.pid : undefined;
+ if (ownerPid !== undefined && !processAlive(ownerPid)) {
+ await rm(location.lock, { force: true });
+ continue;
+ }
+ if (!ownerReadable) {
+ // A crash while publishing the tiny owner record can leave malformed JSON. Once that
+ // partial file is older than the lock wait budget it cannot belong to a live acquisition.
+ const age = Date.now() - (await stat(location.lock)).mtimeMs;
+ if (age > timeoutMs) {
+ await rm(location.lock, { force: true });
+ continue;
+ }
+ }
+ if (Date.now() >= deadline) throw new Error(`Timed out waiting for Runtime Worker lock: ${location.lock}`);
+ await new Promise((resolveWait) => setTimeout(resolveWait, 10));
+ continue;
+ }
+ try {
+ await lock.writeFile(JSON.stringify({ pid: process.pid, startedAt: Date.now() }), "utf8");
+ return lock;
+ } catch (error) {
+ await lock.close().catch(() => undefined);
+ await rm(location.lock, { force: true }).catch(() => undefined);
+ throw error;
+ }
+ }
+}
+
+async function withProcessLock<T>(dataRoot: string, timeoutMs: number, action: () => Promise<T>): Promise<T> {
+ const lock = await acquireProcessLock(dataRoot, timeoutMs);
+ try {
+ return await action();
+ } finally {
+ await lock.close();
+ await rm(paths(dataRoot).lock, { force: true });
+ }
+}
+
async function exists(path: string): Promise<boolean> {
try {
await stat(path);
@@ -128,40 +190,41 @@ export async function ensureRuntimeProcess(
launch: RuntimeWorkerLaunch,
timeoutMs = 10_000,
): Promise<RuntimeProcessState> {
- const current = await runtimeProcessStatus(profile, dataRoot);
- if (current.state === "running") return current;
- const absolute = resolve(profile);
- const location = paths(dataRoot);
- await mkdir(location.root, { recursive: true });
- await rm(location.ready, { force: true });
- await rotateLog(location.log);
- const log = await open(location.log, "a");
- const child = spawn(launch.command, [
- ...launch.args,
- "_worker",
- absolute,
- "--ready-file",
- location.ready,
- ...(launch.workerArgs ?? []),
- ], {
- cwd: process.cwd(),
- // See the managed program start in `programs.ts`: on Windows, detaching costs the console and
- // every console descendant then gets a window of its own.
- detached: process.platform !== "win32",
- windowsHide: true,
- stdio: ["ignore", log.fd, log.fd],
- env: process.env,
+ return await withProcessLock(dataRoot, timeoutMs + 1_000, async () => {
+ const current = await runtimeProcessStatus(profile, dataRoot);
+ if (current.state === "running") return current;
+ const absolute = resolve(profile);
+ const location = paths(dataRoot);
+ await rm(location.ready, { force: true });
+ await rotateLog(location.log);
+ const log = await open(location.log, "a");
+ const child = spawn(launch.command, [
+ ...launch.args,
+ "_worker",
+ absolute,
+ "--ready-file",
+ location.ready,
+ ...(launch.workerArgs ?? []),
+ ], {
+ cwd: process.cwd(),
+ // See the managed program start in `programs.ts`: on Windows, detaching costs the console and
+ // every console descendant then gets a window of its own.
+ detached: process.platform !== "win32",
+ windowsHide: true,
+ stdio: ["ignore", log.fd, log.fd],
+ env: process.env,
+ });
+ if (child.pid === undefined) throw new Error("Runtime Worker process has no pid");
+ const startedAt = Date.now();
+ await writeFile(location.pid, JSON.stringify({
+ profile: absolute,
+ pid: child.pid,
+ startedAt,
+ } satisfies ProcessRecord), "utf8");
+ child.unref();
+ await log.close();
+ return await waitForReady(absolute, dataRoot, timeoutMs);
});
- if (child.pid === undefined) throw new Error("Runtime Worker process has no pid");
- const startedAt = Date.now();
- await writeFile(location.pid, JSON.stringify({
- profile: absolute,
- pid: child.pid,
- startedAt,
- } satisfies ProcessRecord), "utf8");
- child.unref();
- await log.close();
- return await waitForReady(absolute, dataRoot, timeoutMs);
}
async function stopRuntimeProcessUnlocked(profile: string, dataRoot: string, timeoutMs: number): Promise<RuntimeProcessState> {
@@ -199,7 +262,8 @@ async function stopRuntimeProcessUnlocked(profile: string, dataRoot: string, tim
}
export async function stopRuntimeProcess(profile: string, dataRoot: string, timeoutMs = 10_000): Promise<RuntimeProcessState> {
- return await stopRuntimeProcessUnlocked(profile, dataRoot, timeoutMs);
+ return await withProcessLock(dataRoot, timeoutMs + 1_000,
+ () => stopRuntimeProcessUnlocked(profile, dataRoot, timeoutMs));
}
export async function runtimeProcessLogs(dataRoot: string): Promise<{ readonly path: string; readonly text: string }> {
diff --git a/packages/runtime-local/test/worker-process.test.ts b/packages/runtime-local/test/worker-process.test.ts
index 1e936caa..952e144e 100644
--- a/packages/runtime-local/test/worker-process.test.ts
+++ b/packages/runtime-local/test/worker-process.test.ts
@@ -59,3 +59,33 @@ test("one detached Runtime Worker can be started, observed and stopped", async (
await rm(root, { recursive: true, force: true });
}
});
+
+test("concurrent cold starts publish and reuse one Runtime Worker", async () => {
+ const root = await mkdtemp(join(tmpdir(), "hypit-runtime-process-race-"));
+ const profile = join(root, "runtime.json");
+ const dataRoot = join(root, ".hypit", "runtimes", "local");
+ await writeFile(profile, JSON.stringify({ format: "hypit.runtime-profile@1" }), "utf8");
+ const program = `
+ const fs = require("node:fs");
+ const path = require("node:path");
+ const index = process.argv.indexOf("--ready-file");
+ const ready = process.argv[index + 1];
+ const count = path.join(path.dirname(ready), "starts");
+ fs.appendFileSync(count, String(process.pid) + "\\n");
+ fs.mkdirSync(path.dirname(ready), { recursive: true });
+ setTimeout(() => fs.writeFileSync(ready, String(process.pid)), 50);
+ process.on("SIGTERM", () => process.exit(0));
+ setInterval(() => {}, 1000);
+ `;
+ try {
+ const results = await Promise.all([
+ ensureRuntimeProcess(profile, dataRoot, { command: process.execPath, args: ["-e", program] }, 5_000),
+ ensureRuntimeProcess(profile, dataRoot, { command: process.execPath, args: ["-e", program] }, 5_000),
+ ]);
+ assert.equal(results[0]?.pid, results[1]?.pid);
+ assert.equal((await readFile(join(dataRoot, "worker", "starts"), "utf8")).trim().split("\n").length, 1);
+ } finally {
+ await stopRuntimeProcess(profile, dataRoot, 5_000).catch(() => undefined);
+ await rm(root, { recursive: true, force: true });
+ }
+});