Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
fix(runtime): harden durable execution control
kashorin439b1503f765
diff --git a/package.json b/package.jsonindex 2a968dc9..b7a2a736 100644--- a/package.json+++ b/package.json@@ -26,6 +26,7 @@},"devDependencies": {"@narratage/artifact": "workspace:*",+ "@narratage/artifact-store-fs": "workspace:*","@narratage/audio-track": "workspace:*","@narratage/background-removal": "workspace:*","@narratage/caption": "workspace:*",@@ -33,6 +34,8 @@"@narratage/caption-gemini": "workspace:*","@narratage/comment-sticker": "workspace:*","@narratage/composition": "workspace:*",+ "@narratage/credential-store-env": "workspace:*",+ "@narratage/credential-store-keychain": "workspace:*","@narratage/deck-track": "workspace:*","@narratage/estimate": "workspace:*","@narratage/film": "workspace:*",@@ -72,6 +75,7 @@"@narratage/speech-basis": "workspace:*","@narratage/speech-evidence": "workspace:*","@narratage/speech-spine": "workspace:*",+ "@narratage/store-sqlite": "workspace:*","@narratage/svs": "workspace:*","@narratage/typography-track": "workspace:*","@narratage/temporal": "workspace:*",diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.tsindex de941adc..c1a0ccdb 100644--- a/packages/cli/src/main.ts+++ b/packages/cli/src/main.ts@@ -345,6 +345,7 @@ function usage(): string {" narratage operations <build-id> --runtime profile.json|./svml.runtime.ts"," narratage operation <operation-id> --runtime profile.json|./svml.runtime.ts"," narratage cancel build <build-id> --runtime profile.json|./svml.runtime.ts [--reason text]",+ " narratage cancel operation <operation-id> --runtime profile.json|./svml.runtime.ts [--reason text]"," narratage auth status|login|logout <endpoint-instance> --runtime profile.json [--slot name] [--from secret-file]","","output:",@@ -505,6 +506,11 @@ export async function runCli(if (!operational || (!fileOptional && args.file === undefined)) {throw new Error(usage());}+ if (args.watch && args.command !== "queue") throw new Error("--watch applies only to queue");+ if (args.jsonl && (args.command !== "queue" || !args.watch)) {+ throw new Error("--jsonl applies only to queue --watch");+ }+ if (args.watch && args.json) throw new Error("queue --watch is a stream; use --jsonl instead of --json");if (args.command === "lock-packages") {if (args.packages.length === 0) throw new Error("lock-packages requires at least one --package");if (args.packageLock !== undefined) throw new Error("lock-packages does not accept --package-lock");@@ -590,8 +596,16 @@ export async function runCli(}if (args.action === "logs") {const logs = await runtimeProcessLogs(profile);- if (!args.json && !args.jsonl) io.write(logs.text);- else writeOperational(logs, "Runtime logs");+ const lines = logs.text.length === 0 ? [] : logs.text.replace(/\n$/u, "").split("\n");+ const shown = args.verbose ? lines : lines.slice(-100);+ writeOperational({+ format: "narratage.cli-runtime-logs@1",+ path: logs.path,+ text: logs.text,+ }, "Runtime logs", "info", [+ ["Path", logs.path],+ ["Lines", String(lines.length)],+ ], shown.length === 0 ? ["No log output."] : shown);return;}if (args.action === "down") {@@ -884,19 +898,41 @@ export async function runCli(]);}} else {- if (args.action !== "build") throw new Error("cancel must name its scope: cancel build <build-id>");- const result = await runtime.cancel(args.file!, args.reason);- const machine = {- build: args.file,- cancelled: result !== undefined,- phase: result?.phase,- admission: result?.admission,- terminal: result?.terminal,- };- writeOperational(machine, result === undefined ? "Build not found" : "Cancellation requested",- result === undefined ? "warning" : "success", [- ["Build", args.file!], ["Admission", result?.admission ?? "missing"], ["Phase", result?.phase ?? "missing"],- ]);+ if (args.action === "build") {+ const result = await runtime.cancel(args.file!, args.reason);+ const machine = {+ scope: "build",+ build: args.file,+ requested: result !== undefined,+ phase: result?.phase,+ admission: result?.admission,+ terminal: result?.terminal,+ };+ writeOperational(machine, result === undefined ? "Build not found" : "Build cancellation requested",+ result === undefined ? "warning" : "success", [+ ["Build", args.file!], ["Admission", result?.admission ?? "missing"], ["Phase", result?.phase ?? "missing"],+ ]);+ } else if (args.action === "operation") {+ if (args.file === undefined || !isDigest(args.file)) {+ throw new Error("cancel operation requires an Operation digest");+ }+ const result = await runtime.cancelOperation(args.file!, args.reason);+ const machine = {+ scope: "operation",+ operation: args.file,+ requested: result !== undefined,+ build: result?.build,+ execution: result?.status,+ control: result?.cancellation?.status,+ };+ writeOperational(machine, result === undefined ? "Operation not found" : "Operation cancellation requested",+ result === undefined ? "warning" : "success", [+ ["Operation", args.file!], ["Build", result?.build ?? "missing"],+ ["Execution", result?.status ?? "missing"], ["Control", result?.cancellation?.status ?? "missing"],+ ]);+ } else {+ throw new Error("cancel must name its scope: cancel build <build-id> or cancel operation <operation-id>");+ }}} finally {await runtime.close();diff --git a/packages/cli/src/output.ts b/packages/cli/src/output.tsindex 581bb16a..c3e2df6a 100644--- a/packages/cli/src/output.ts+++ b/packages/cli/src/output.ts@@ -379,6 +379,7 @@ export function writeCliHelp(io: CliIo): void {" inspect <build-id> inspect accepted Records and demanded outputs"," get <build-id> read or materialize one archived result"," cancel build <build-id> close admission and request honest cancellation",+ " cancel operation <digest> suppress one exact Operation realization"," operations <build-id> inspect external Operation facts"," operation <digest> inspect one external Operation","",diff --git a/packages/cli/src/runtime-process.ts b/packages/cli/src/runtime-process.tsindex aa3ba5da..0842b09a 100644--- a/packages/cli/src/runtime-process.ts+++ b/packages/cli/src/runtime-process.ts@@ -9,10 +9,12 @@ export type RuntimeWorkerLaunch = {};export type RuntimeProcessState = {- readonly state: "running" | "stopped";+ readonly state: "running" | "stale" | "stopped";readonly profile: string;readonly pid?: number;readonly startedAt?: number;+ readonly profileDigest?: string;+ readonly currentProfileDigest?: string;readonly logPath: string;};@@ -21,6 +23,8 @@ type ProcessRecord = {readonly profile: string;readonly pid: number;readonly startedAt: number;+ /** Optional only so a pre-digest development record can still be stopped safely. */+ readonly profileDigest?: string;};function paths(profile: string) {@@ -54,6 +58,31 @@ function alive(pid: number): boolean {}}+async function digestProfile(profile: string): Promise<string> {+ const absolute = resolve(profile);+ const bytes = await readFile(absolute);+ const hash = createHash("sha256");+ hash.update("profile\0");+ hash.update(bytes);+ if (absolute.endsWith(".json")) {+ const document = JSON.parse(bytes.toString("utf8")) as {+ readonly root?: unknown;+ readonly packageLock?: unknown;+ readonly runtimePackageLock?: unknown;+ };+ const root = resolve(dirname(absolute), typeof document.root === "string" ? document.root : ".");+ for (const [name, value] of [+ ["packageLock", document.packageLock],+ ["runtimePackageLock", document.runtimePackageLock],+ ] as const) {+ if (typeof value !== "string") continue;+ hash.update(`\0${name}\0${resolve(root, value)}\0`);+ hash.update(await readFile(resolve(root, value)));+ }+ }+ return `sha256:${hash.digest("hex")}`;+}+async function record(profile: string): Promise<ProcessRecord | undefined> {const path = paths(profile).pid;try {@@ -73,11 +102,25 @@ export async function runtimeProcessStatus(profile: string): Promise<RuntimeProcif (current === undefined || !alive(current.pid)) {return { state: "stopped", profile: resolve(profile), logPath: location.log };}+ const currentProfileDigest = await digestProfile(profile).catch(() => undefined);+ if (current.profileDigest === undefined || current.profileDigest !== currentProfileDigest) {+ return {+ state: "stale",+ profile: current.profile,+ pid: current.pid,+ startedAt: current.startedAt,+ ...(current.profileDigest === undefined ? {} : { profileDigest: current.profileDigest }),+ ...(currentProfileDigest === undefined ? {} : { currentProfileDigest }),+ logPath: location.log,+ };+ }return {state: "running",profile: current.profile,pid: current.pid,startedAt: current.startedAt,+ profileDigest: current.profileDigest,+ currentProfileDigest,logPath: location.log,};}@@ -92,7 +135,7 @@ async function waitForReady(profile: string, timeoutMs: number): Promise<Runtimethrow new Error(`Runtime Worker exited before becoming ready${log.length === 0 ? "" : `: ${log.trim().split("\n").at(-1)}`}`);}if (await exists(location.ready)) return state;- await new Promise((resolveWait) => setTimeout(resolveWait, 50));+ await new Promise((resolveWait) => setTimeout(resolveWait, 10));}throw new Error(`Runtime Worker did not become ready within ${timeoutMs}ms; log: ${location.log}`);}@@ -104,8 +147,12 @@ export async function ensureRuntimeProcess(): Promise<RuntimeProcessState> {const current = await runtimeProcessStatus(profile);if (current.state === "running") return current;+ if (current.state === "stale") {+ await stopRuntimeProcess(profile, timeoutMs);+ }const absolute = resolve(profile);const location = paths(absolute);+ const profileDigest = await digestProfile(absolute);await mkdir(location.root, { recursive: true });await unlink(location.ready).catch(() => undefined);const log = await open(location.log, "a");@@ -128,6 +175,7 @@ export async function ensureRuntimeProcess(profile: absolute,pid: child.pid,startedAt,+ profileDigest,} satisfies ProcessRecord), "utf8");child.unref();await log.close();diff --git a/packages/cli/test/runtime-build-process.test.ts b/packages/cli/test/runtime-build-process.test.tsnew file mode 100644index 00000000..7536eb11--- /dev/null+++ b/packages/cli/test/runtime-build-process.test.ts@@ -0,0 +1,217 @@+import assert from "node:assert/strict";+import { execFile, spawn } from "node:child_process";+import { mkdtemp, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { promisify } from "node:util";+import test from "node:test";++import {+ createNodePackageLock,+ writeNodePackageLock,+} from "@narratage/package-loader-node";++import {+ ensureRuntimeProcess,+ runtimeProcessStatus,+ stopRuntimeProcess,+} from "../src/runtime-process.js";++const execute = promisify(execFile);+const cliEntry = join(process.cwd(), "packages", "video-cli", "src", "cli.ts");+const tsxImport = import.meta.resolve("tsx");++async function cli(project: string, argv: readonly string[]): Promise<Record<string, unknown>> {+ const result = await execute(process.execPath, ["--import", tsxImport, cliEntry, ...argv, "--json"], {+ cwd: project,+ maxBuffer: 8 * 1024 * 1024,+ });+ return JSON.parse(result.stdout) as Record<string, unknown>;+}++async function cliHuman(project: string, argv: readonly string[]): Promise<string> {+ const result = await execute(process.execPath, ["--import", tsxImport, cliEntry, ...argv, "--no-color"], {+ cwd: project,+ maxBuffer: 8 * 1024 * 1024,+ });+ return result.stdout;+}++function authorSource(length: number): string {+ const chain = Array.from({ length }, (_, index) => {+ const id = `message-${String(index).padStart(3, "0")}`;+ const previous = index === 0 ? "seed" : `message-${String(index - 1).padStart(3, "0")}`;+ return ` <text:Render id="${id}" template={kit.pass}>\n`+ + ` <text:Set name="input" text={${previous}}/>\n`+ + " </text:Render>";+ }).join("\n");+ return `<?svml using="@narratage/markup@1"?>++<svml>+ <import as="text" from="@narratage/text@1"/>+ <import as="kit" source="./pass.svs"/>+ <text:Value id="seed">durable execution</text:Value>+${chain}+</svml>+`;+}++async function waitForBuild(+ project: string,+ profile: string,+ build: string,+ timeoutMs = 20_000,+): Promise<Record<string, unknown>> {+ const deadline = Date.now() + timeoutMs;+ let last: Record<string, unknown> | undefined;+ while (Date.now() <= deadline) {+ last = await cli(project, ["status", build, "--runtime", profile]);+ const dispatch = last.dispatch as { readonly phase?: string } | undefined;+ if (dispatch?.phase === "terminal") return last;+ await new Promise((resolve) => setTimeout(resolve, 200));+ }+ throw new Error(`Build ${build} did not finish: ${JSON.stringify(last)}`);+}++test("CLI exits after durable submission and a restarted detached Worker completes a non-video Build", async () => {+ const project = await mkdtemp(join(tmpdir(), "narratage-runtime-build-process-"));+ const authorLockPath = join(project, "svml.packages.lock");+ const runtimeLockPath = join(project, "svml.runtime-packages.lock");+ const profile = join(project, "svml.runtime.json");+ const source = join(project, "main.svml");+ const run = join(project, "build.svrun");+ const build = "process-recovery-build";+ try {+ await writeNodePackageLock(authorLockPath, await createNodePackageLock([+ "@narratage/run-markup",+ "@narratage/text",+ ], process.cwd()));+ await writeNodePackageLock(runtimeLockPath, await createNodePackageLock([+ "@narratage/local",+ "@narratage/store-sqlite",+ "@narratage/artifact-store-fs",+ "@narratage/credential-store-env",+ ], process.cwd()));+ await writeFile(join(project, "pass.svs"), `<?svml using="@narratage/text/svs@1"?>++<sheet version="1" id="pass">+ text-template.pass { separator: paragraph; }+ text-template.pass.block.input {+ kind: slot;+ order: 10;+ slot: input;+ }+</sheet>+`, "utf8");+ await writeFile(source, authorSource(5), "utf8");+ await writeFile(run, `<?svml using="@narratage/run-markup@1"?>++<svrun version="1" targets="delivery">+ <author source="./main.svml"/>+ <target-set id="delivery">+ <target output="message-004" accepts="exact"/>+ </target-set>+</svrun>+`, "utf8");+ await writeFile(profile, JSON.stringify({+ format: "svml.runtime-config@1",+ root: ".",+ packageRoot: process.cwd(),+ packageLock: "./svml.packages.lock",+ runtimePackageLock: "./svml.runtime-packages.lock",+ runtimeServices: [+ { use: "@narratage/local", instance: "execution" },+ { use: "@narratage/store-sqlite", instance: "state", config: { path: ".svml/runtime.sqlite" } },+ { use: "@narratage/artifact-store-fs", instance: "artifacts", config: { path: ".svml/artifacts" } },+ { use: "@narratage/credential-store-env", instance: "credentials.env", config: {} },+ ],+ services: {+ scheduler: "execution.scheduler",+ worker: "execution.worker",+ stores: {+ build: "state.builds",+ operations: "state.operations",+ dispatch: "state.dispatch",+ journal: "state.journal",+ artifacts: "artifacts",+ credentials: ["credentials.env"],+ },+ },+ endpoints: [],+ permissions: ["filesystem:state", "filesystem:artifacts", "environment:credentials"],+ scheduling: { maxConcurrency: 2 },+ }, null, 2), "utf8");++ const parked = await ensureRuntimeProcess(profile, {+ command: process.execPath,+ args: ["-e", `+ const fs = require("node:fs");+ const path = require("node:path");+ const index = process.argv.indexOf("--ready-file");+ const ready = process.argv[index + 1];+ fs.mkdirSync(path.dirname(ready), { recursive: true });+ fs.writeFileSync(ready, String(process.pid));+ process.on("SIGTERM", () => process.exit(0));+ setInterval(() => {}, 1000);+ `],+ }, 5_000);+ const submitted = await cli(project, [+ "build", run,+ "--build-id", build,+ "--runtime", profile,+ "--package-lock", authorLockPath,+ "--package-root", process.cwd(),+ "--root", project,+ "--no-services",+ ]);+ assert.equal(submitted.status, "queued", "the foreground CLI reports durable admission, not execution ownership");+ assert.equal((submitted.dispatch as { readonly phase?: string }).phase, "queued");+ const originalProcess = await runtimeProcessStatus(profile);+ assert.equal(originalProcess.state, "running", "the detached Runtime process outlives the submitting CLI process");+ assert.equal(originalProcess.pid, parked.pid, "build reuses one profile-scoped Runtime process");++ const follower = spawn(process.execPath, [+ "--import", tsxImport, cliEntry,+ "build", run,+ "--build-id", build,+ "--runtime", profile,+ "--package-lock", authorLockPath,+ "--package-root", process.cwd(),+ "--root", project,+ "--no-services",+ "--follow",+ "--json",+ ], { cwd: project, stdio: "ignore" });+ await new Promise((resolve) => setTimeout(resolve, 500));+ assert.equal(follower.exitCode, null, "--follow is observing the still-queued Build");+ const followerExit = new Promise<void>((resolve, reject) => {+ follower.once("exit", () => resolve());+ follower.once("error", reject);+ });+ follower.kill("SIGINT");+ await followerExit;+ const afterInterrupt = await cli(project, ["status", build, "--runtime", profile]);+ assert.equal((afterInterrupt.dispatch as { readonly admission?: string }).admission, "open",+ "interrupting --follow does not translate into Build cancellation");++ await stopRuntimeProcess(profile, 10_000);+ const interrupted = await cli(project, ["status", build, "--runtime", profile]);+ assert.notEqual((interrupted.dispatch as { readonly phase?: string }).phase, "terminal",+ "the long Build is durably incomplete when its first Worker is killed");++ const restarted = await cli(project, ["runtime", "up", profile]);+ assert.equal((restarted.worker as { readonly state?: string }).state, "running");+ assert.notEqual((restarted.worker as { readonly pid?: number }).pid, originalProcess.pid);+ const logs = await cli(project, ["runtime", "logs", profile]);+ assert.equal(logs.format, "narratage.cli-runtime-logs@1");+ assert.equal(typeof logs.path, "string");+ assert.equal(typeof logs.text, "string");+ assert.match(await cliHuman(project, ["runtime", "logs", profile]), /Runtime logs[\s\S]*Path/u);+ const completed = await waitForBuild(project, profile, build);+ assert.equal((completed.build as { readonly status?: string }).status, "complete");+ assert.equal((completed.dispatch as { readonly terminal?: string }).terminal, "complete");+ } finally {+ await stopRuntimeProcess(profile, 10_000).catch(() => undefined);+ await rm(project, { recursive: true, force: true });+ }+});diff --git a/packages/cli/test/runtime-process.test.ts b/packages/cli/test/runtime-process.test.tsindex 20ecb79a..9dc36732 100644--- a/packages/cli/test/runtime-process.test.ts+++ b/packages/cli/test/runtime-process.test.ts@@ -14,7 +14,14 @@ import {test("one detached Runtime Worker is observable, reusable and explicitly stoppable", async () => {const root = await mkdtemp(join(tmpdir(), "narratage-runtime-process-"));const profile = join(root, "runtime.json");- await writeFile(profile, "{}\n", "utf8");+ const authorLock = join(root, "author.lock");+ const runtimeLock = join(root, "runtime.lock");+ await writeFile(authorLock, "author-v1\n", "utf8");+ await writeFile(runtimeLock, "runtime-v1\n", "utf8");+ await writeFile(profile, JSON.stringify({+ packageLock: "./author.lock",+ runtimePackageLock: "./runtime.lock",+ }), "utf8");const program = `const fs = require("node:fs");const path = require("node:path");@@ -34,6 +41,20 @@ test("one detached Runtime Worker is observable, reusable and explicitly stoppabassert.equal(second.pid, first.pid);assert.equal((await runtimeProcessStatus(profile)).state, "running");assert.match((await runtimeProcessLogs(profile)).text, /worker-ready/u);+ await writeFile(runtimeLock, "runtime-v2\n", "utf8");+ assert.equal((await runtimeProcessStatus(profile)).state, "stale");+ const replaced = await ensureRuntimeProcess(profile, { command: process.execPath, args: ["-e", program] }, 5_000);+ assert.notEqual(replaced.pid, first.pid);+ assert.equal(replaced.state, "running");+ await writeFile(profile, JSON.stringify({+ root: ".",+ packageLock: "./author.lock",+ runtimePackageLock: "./runtime.lock",+ }), "utf8");+ assert.equal((await runtimeProcessStatus(profile)).state, "stale");+ const replacedAgain = await ensureRuntimeProcess(profile, { command: process.execPath, args: ["-e", program] }, 5_000);+ assert.notEqual(replacedAgain.pid, replaced.pid);+ assert.equal(replacedAgain.state, "running");assert.equal((await stopRuntimeProcess(profile, 5_000)).state, "stopped");assert.equal((await runtimeProcessStatus(profile)).state, "stopped");} finally {diff --git a/packages/core/test/greeting-fixture.ts b/packages/core/test/greeting-fixture.tsindex f7cb93d7..a9bc1568 100644--- a/packages/core/test/greeting-fixture.ts+++ b/packages/core/test/greeting-fixture.ts@@ -126,7 +126,7 @@ export const manifest: ModuleManifest = {],};-export function greetingGraph(program: LinkedProgram): CompiledGraph {+export function greetingGraph(program: LinkedProgram, includeSide = false): CompiledGraph {return sealCompiledGraph({program: program.semanticDigest,outputs: [@@ -148,12 +148,27 @@ export function greetingGraph(program: LinkedProgram): CompiledGraph {primary: "assemble",semanticInputs: [{ kind: "logical-output", id: "generated" }],},+ ...(includeSide ? [{+ id: "side-generated",+ type: types.generated,+ primary: "side-placeholder",+ semanticInputs: [{ kind: "logical-output" as const, id: "prompt" }],+ }, {+ id: "side-document",+ type: types.document,+ primary: "side-assemble",+ semanticInputs: [{ kind: "logical-output" as const, id: "side-generated" }],+ }] : []),],candidates: [{ id: "make-prompt", type: types.prompt, root: { kind: "operation", result: { kind: "operation-result", operation: "make-prompt" } } },{ id: "request-text", type: types.generated, root: { kind: "operation", result: { kind: "operation-result", operation: "request-text" } } },{ id: "placeholder-text", type: types.generated, root: { kind: "operation", result: { kind: "operation-result", operation: "placeholder-text" } } },{ id: "assemble", type: types.document, root: { kind: "operation", result: { kind: "operation-result", operation: "assemble" } } },+ ...(includeSide ? [+ { id: "side-placeholder", type: types.generated, root: { kind: "operation" as const, result: { kind: "operation-result" as const, operation: "side-placeholder" } } },+ { id: "side-assemble", type: types.document, root: { kind: "operation" as const, result: { kind: "operation-result" as const, operation: "side-assemble" } } },+ ] : []),],operations: [{@@ -180,6 +195,17 @@ export function greetingGraph(program: LinkedProgram): CompiledGraph {inputs: { generated: { kind: "logical-output", id: "generated" } },result: { kind: "output", name: "document", record: "document:root" },},+ ...(includeSide ? [{+ id: "side-placeholder",+ producer: producers.placeholderText,+ inputs: { prompt: { kind: "logical-output" as const, id: "prompt" } },+ result: { kind: "output" as const, name: "generated", record: "generated:side" },+ }, {+ id: "side-assemble",+ producer: producers.assemble,+ inputs: { generated: { kind: "logical-output" as const, id: "side-generated" } },+ result: { kind: "output" as const, name: "document", record: "document:side" },+ }] : []),],});}@@ -188,6 +214,7 @@ export function createGreetingBuild(options?: {readonly generationRealization?: "primary" | "placeholder";readonly goalAccepts?: "exact" | "substitute";readonly implementationClosure?: import("@narratage/protocol").Digest;+ readonly includeSideTarget?: boolean;}): BuildState {const closure = createResolvedClosure([manifest]);const authored = sealRecord({@@ -208,13 +235,13 @@ export function createGreetingBuild(options?: {records: [authored],});const program = link(closure, [typedModule]);- const graph = greetingGraph(program);+ const graph = greetingGraph(program, options?.includeSideTarget ?? false);const request = sealBuildRequest({graph: graph.id,...(options?.implementationClosure === undefined? {}: { implementationClosure: options.implementationClosure }),- targets: [{+ targets: [...(options?.includeSideTarget ? [{ output: "side-document", accepts: "exact" as const }] : []), {output: "document",accepts: options?.goalAccepts ?? "exact",}],diff --git a/packages/driver-node/src/driver.ts b/packages/driver-node/src/driver.tsindex 3656cc75..8c7b90d4 100644--- a/packages/driver-node/src/driver.ts+++ b/packages/driver-node/src/driver.ts@@ -564,21 +564,14 @@ export class NodeDriver {return result.status === "stored" ? result.snapshot : result.current;};let current = await operations.read(operation.id) ?? operation;- if (current.status === "completed" || current.status === "failed" || current.status === "cancelled") {- if (current.cancellation === undefined) {- current = await write(current, {- status: "control",- cancellation: { requestedAt, requestId, status: "too-late", attempts: 0 },- });- }- return current;- }- if (current.cancellation === undefined) {+ while (current.cancellation === undefined) {+ const terminal = current.status === "completed" || current.status === "failed" || current.status === "cancelled";current = await write(current, {status: "control",- cancellation: { requestedAt, requestId, status: "requested", attempts: 0 },+ cancellation: { requestedAt, requestId, status: terminal ? "too-late" : "requested", attempts: 0 },});}+ if (current.status === "completed" || current.status === "failed" || current.status === "cancelled") return current;const control = current.cancellation;if (control === undefined || control.requestId !== requestId) {throw new Error(`Operation ${operation.id} already has another cancellation request`);@@ -613,7 +606,7 @@ export class NodeDriver {operation: identity,checkpoint: current.status === "pending" ? structuredClone(current.checkpoint) : undefined,};- if (control.status === "requested") {+ if (control.status === "requested" || control.status === "accepted") {if (executable.registration.endpoint.cancel === undefined) {current = await write(current, {status: "control",@@ -652,10 +645,27 @@ export class NodeDriver {// Acknowledgement is not a terminal fact. Reconcile the same remote submission until it// either confirms cancellation through `cancel`, or naturally completes/fails.- const observed = await executable.registration.endpoint.resume({- ...endpointContext,- checkpoint: current.status === "pending" ? structuredClone(current.checkpoint) : undefined,- });+ let observed: Awaited<ReturnType<typeof executable.registration.endpoint.resume>>;+ try {+ observed = await executable.registration.endpoint.resume({+ ...endpointContext,+ checkpoint: current.status === "pending" ? structuredClone(current.checkpoint) : undefined,+ });+ } catch (error) {+ const cancellation = current.cancellation;+ if (cancellation === undefined) throw error;+ return await write(current, {+ status: "control",+ cancellation: {+ ...cancellation,+ retryAt: Date.now() + 1_000,+ lastError: {+ code: "CANCEL_RECONCILE_FAILED",+ message: error instanceof Error ? error.message : String(error),+ },+ },+ });+ }if (observed.status === "pending") {return await write(current, {status: "pending",diff --git a/packages/driver-node/test/scheduler.test.ts b/packages/driver-node/test/scheduler.test.tsindex d199d2f4..601453b3 100644--- a/packages/driver-node/test/scheduler.test.ts+++ b/packages/driver-node/test/scheduler.test.ts@@ -810,6 +810,7 @@ test("wakeAt prevents early polling and Runtime cancellation becomes a terminaltest("cancellation acknowledgement never invents a remote terminal fact", async () => {const operations = new MemoryOperationStore();const runtime = resolvedRuntime(1, "recoverable");+ let cancelCalls = 0;const endpoint: RecoverableEndpoint = {start() {return { status: "pending", checkpoint: { remoteJob: "still-running" } };@@ -818,7 +819,8 @@ test("cancellation acknowledgement never invents a remote terminal fact", asyncreturn { status: "pending", checkpoint: checkpoint ?? { remoteJob: "still-running" } };},cancel() {- return { status: "accepted" };+ cancelCalls += 1;+ return cancelCalls === 1 ? { status: "accepted" } : { status: "confirmed" };},};const executor = recoverableExecutor(endpoint, operations, runtime).executor;@@ -831,6 +833,63 @@ test("cancellation acknowledgement never invents a remote terminal fact", asyncconst controlled = await executor.cancelOperation(first!.state, operation, 123);assert.equal(controlled.status, "pending");assert.equal(controlled.cancellation?.status, "accepted");+ const confirmed = await executor.cancelOperation(first!.state, controlled, 123);+ assert.equal(confirmed.status, "cancelled");+ assert.equal(confirmed.cancellation?.status, "confirmed");+ assert.equal(cancelCalls, 2, "accepted acknowledgement is reconciled until a terminal fact exists");+});++test("a completion racing with cancellation is recorded as too late instead of overwritten", async () => {+ const stored = new MemoryOperationStore();+ let race = false;+ const operations: OperationStore = {+ create: (identity) => stored.create(identity),+ read: (id) => stored.read(id),+ list: (query) => stored.list(query),+ async compareAndSwap(id, revision, update) {+ if (race && update.status === "control") {+ race = false;+ const completed = await stored.compareAndSwap(id, revision, {+ status: "completed",+ completion: {+ value: { kind: "inline", value: "remote fact won the race" },+ conformance: "exact",+ delivery: "executed",+ metadata: {},+ },+ });+ assert.equal(completed.status, "stored");+ return { status: "conflict", current: completed.snapshot };+ }+ return await stored.compareAndSwap(id, revision, update);+ },+ };+ let cancelCalls = 0;+ const endpoint: RecoverableEndpoint = {+ start() {+ return { status: "pending", checkpoint: { remoteJob: "finishing" } };+ },+ resume() {+ throw new Error("the raced terminal fact must be observed before resume");+ },+ cancel() {+ cancelCalls += 1;+ return { status: "confirmed" };+ },+ };+ const runtime = resolvedRuntime(1, "recoverable");+ const executor = recoverableExecutor(endpoint, operations, runtime).executor;+ const scheduler = new LocalBuildScheduler(executor, localSchedulerOptionsFromClosure(runtime.closure));+ const [first] = await scheduler.run([{ id: "cancel-completion-race", state: createGreetingBuild() }]);+ const operationId = first?.journal.find((item) => item.status === "pending")?.operation;+ assert.ok(operationId);+ const operation = await operations.read(operationId);+ assert.ok(operation);+ race = true;+ const observed = await executor.cancelOperation(first!.state, operation, 789);+ assert.equal(observed.status, "completed");+ assert.equal(observed.cancellation?.status, "too-late");+ assert.equal(cancelCalls, 0);});test("an Endpoint without cancellation stays observable instead of changing Candidate or fact", async () => {diff --git a/packages/local/src/runtime.ts b/packages/local/src/runtime.tsindex df6cba1d..d3cc3486 100644--- a/packages/local/src/runtime.ts+++ b/packages/local/src/runtime.ts@@ -20,6 +20,7 @@ import {isEnumerableBuildStore,isManagedArtifactStore,isStreamingArtifactStore,+ operationCancellationRequestId,resolveRuntimeProfile,sealRuntimeProfile,writableCredentialStore,@@ -338,6 +339,37 @@ export async function createLocalRuntime(});return dispatch;},+ async cancelOperation(id, reason) {+ let current = await options.operationStore.read(id);+ if (current === undefined) return undefined;+ if (current.cancellation === undefined) {+ const requestedAt = Date.now();+ while (current.cancellation === undefined) {+ const status = current.status === "completed" || current.status === "failed" || current.status === "cancelled"+ ? "too-late" as const+ : "requested" as const;+ const written = await options.operationStore.compareAndSwap(current.id, current.revision, {+ status: "control",+ cancellation: {+ requestedAt,+ requestId: operationCancellationRequestId(current.id, requestedAt),+ status,+ attempts: 0,+ },+ });+ current = written.status === "stored" ? written.snapshot : written.current;+ }+ await options.journal.append({+ at: requestedAt,+ kind: "cancellation-requested",+ build: current.build,+ operation: current.id,+ detail: { scope: "operation", ...(reason === undefined ? {} : { reason }) },+ });+ }+ await options.dispatchStore.wake(current.build);+ return current;+ },async workOnce(workOptions) {return await worker.runOnce(workOptions);},@@ -379,8 +411,9 @@ export async function createLocalRuntime(}/**- * Zero-service local distribution. The Runtime authority stays in this process while configured- * Capability Endpoints may execute locally, in a vendor API, in Lambda, or on a hosted service.+ * Node project assembly over only the Runtime service packages selected by the caller. Capability+ * Endpoints may execute locally, in a vendor API, in Lambda, or on a hosted service; none is+ * inferred from this local process boundary.*/export async function createProjectLocalRuntime(options: ProjectLocalRuntimeOptions,@@ -469,6 +502,7 @@ export async function createProjectLocalRuntime(deleteCredential: runtime.deleteCredential,builds: runtime.builds,cancel: runtime.cancel,+ cancelOperation: runtime.cancelOperation,workOnce: runtime.workOnce,work: runtime.work,readArtifact: runtime.readArtifact,diff --git a/packages/local/src/types.ts b/packages/local/src/types.tsindex 7a93d21d..72ba52ab 100644--- a/packages/local/src/types.ts+++ b/packages/local/src/types.ts@@ -144,6 +144,8 @@ export type LocalRuntime = {deleteCredential(endpoint: string, slot: string): Promise<{ readonly deleted: boolean; readonly credential: LocalCredentialStatus }>;builds(): Promise<readonly BuildCatalogEntry[]>;cancel(build: string, reason?: string): Promise<BuildDispatchSnapshot | undefined>;+ /** Request control of one exact Operation attempt without closing the whole Build. */+ cancelOperation(id: Digest, reason?: string): Promise<OperationSnapshot | undefined>;workOnce(options: { readonly owner: string; readonly leaseMs: number }): Promise<BuildDispatchSnapshot | undefined>;work(options: import("@narratage/runtime").RuntimeWorkerRunOptions): Promise<void>;readArtifact(digest: Digest): Promise<Uint8Array | undefined>;diff --git a/packages/local/src/worker.ts b/packages/local/src/worker.tsindex 8e5dccad..5eef7a8f 100644--- a/packages/local/src/worker.ts+++ b/packages/local/src/worker.ts@@ -46,6 +46,7 @@ class CapacityExecutor implements RuntimeCommandExecutor {readonly #build: string;readonly #dispatchLease: DispatchLease;readonly #leaseMs: number;+ readonly #suppressedCommands: ReadonlySet<string>;constructor(delegate: RuntimeCommandExecutor,@@ -53,16 +54,31 @@ class CapacityExecutor implements RuntimeCommandExecutor {build: string,dispatchLease: DispatchLease,leaseMs: number,+ suppressedCommands: ReadonlySet<string> = new Set(),) {this.#delegate = delegate;this.#options = options;this.#build = build;this.#dispatchLease = dispatchLease;this.#leaseMs = leaseMs;+ this.#suppressedCommands = suppressedCommands;}prepare(state: BuildState): RuntimePreparation {- return this.#delegate.prepare(state);+ const prepared = this.#delegate.prepare(state);+ const suppressed = prepared.runnable.filter((item) => this.#suppressedCommands.has(item.command.id));+ return {+ state: prepared.state,+ runnable: prepared.runnable.filter((item) => !this.#suppressedCommands.has(item.command.id)),+ blocked: [+ ...prepared.blocked,+ ...suppressed.map((item) => ({+ command: item.command.id,+ reason: "the exact Operation was cancelled by Runtime control",+ subject: item.command.id,+ })),+ ],+ };}async #assertAuthority(): Promise<void> {@@ -112,8 +128,23 @@ class CapacityExecutor implements RuntimeCommandExecutor {const reservation = await this.#acquire(descriptor);const lease = reservation.active;assert(lease !== undefined, `Capacity reservation ${reservation.id} has no active lease`);+ let heartbeatError: unknown;+ let heartbeat = Promise.resolve();+ const heartbeatTimer = setInterval(() => {+ heartbeat = heartbeat.then(async () => {+ await this.#options.stores.dispatch.heartbeatCapacity(+ reservation.id,+ lease,+ Date.now(),+ this.#leaseMs,+ );+ }).catch((error: unknown) => { heartbeatError = error; });+ }, Math.max(1, Math.floor(this.#leaseMs / 3)));try {const result = await this.#delegate.executeCommand(state, commandId, context);+ clearInterval(heartbeatTimer);+ await heartbeat;+ if (heartbeatError !== undefined) throw heartbeatError;await this.#assertAuthority();await this.#options.stores.dispatch.heartbeatCapacity(reservation.id, lease, Date.now(), this.#leaseMs);if (result.status === "pending" && (descriptor.capacityMode ?? "active") === "recoverable") {@@ -121,10 +152,25 @@ class CapacityExecutor implements RuntimeCommandExecutor {} else {await this.#options.stores.dispatch.releaseCapacity(reservation.id, lease);}+ const controlled = (await this.#options.stores.operations.list({+ build: this.#build,+ command: commandId,+ })).find((item) => item.cancellation !== undefined);+ if (controlled !== undefined) {+ return {+ status: "pending",+ operation: controlled.id,+ ...(controlled.cancellation?.retryAt === undefined ? {} : { wakeAt: controlled.cancellation.retryAt }),+ };+ }return result;} catch (error) {+ clearInterval(heartbeatTimer);+ await heartbeat;await this.#options.stores.dispatch.releaseCapacity(reservation.id, lease).catch(() => undefined);throw error;+ } finally {+ clearInterval(heartbeatTimer);}}@@ -134,7 +180,9 @@ class CapacityExecutor implements RuntimeCommandExecutor {requestedAt: number,) {assert(this.#delegate.cancelOperation !== undefined, "selected executor cannot cancel Operations");- return await this.#delegate.cancelOperation(state, operation, requestedAt);+ const observed = await this.#delegate.cancelOperation(state, operation, requestedAt);+ await this.#assertAuthority();+ return observed;}}@@ -206,6 +254,43 @@ class DurableLocalWorker implements RuntimeWorker {);}+ async #reconcileOperationControl(+ state: BuildState,+ executor: CapacityExecutor,+ dispatch: BuildDispatchSnapshot,+ lease: DispatchLease,+ suppressed: Set<string>,+ ): Promise<void> {+ const operations = await this.#options.stores.operations.list({ build: dispatch.build });+ for (const operation of operations) {+ const control = operation.cancellation;+ if (control === undefined) continue;+ suppressed.add(operation.command);+ if (operation.status === "completed" || operation.status === "failed" || operation.status === "cancelled") {+ await this.#options.stores.dispatch.clearCapacity(+ capacityReservationId(dispatch.build, operation.command),+ dispatch.build,+ lease,+ );+ continue;+ }+ if (control.retryAt !== undefined && control.retryAt > Date.now()) continue;+ const observed = await executor.cancelOperation(state, operation, control.requestedAt);+ await this.#journal("operation-control", dispatch.build, lease.owner, {+ operation: observed.id,+ execution: observed.status,+ control: observed.cancellation?.status ?? "none",+ });+ if (observed.status === "completed" || observed.status === "failed" || observed.status === "cancelled") {+ await this.#options.stores.dispatch.clearCapacity(+ capacityReservationId(dispatch.build, observed.command),+ dispatch.build,+ lease,+ );+ }+ }+ }+async runOnce(options: { readonly owner: string; readonly leaseMs: number }): Promise<BuildDispatchSnapshot | undefined> {assert(options.owner.trim().length > 0, "Worker owner is empty");const leaseMs = positive(options.leaseMs, "Worker leaseMs");@@ -228,7 +313,16 @@ class DurableLocalWorker implements RuntimeWorker {assert(this.#options.runtimeClosure.digest === dispatch.runtimeClosure,`Dispatch ${dispatch.build} Runtime Closure differs`);}- const controlled = new CapacityExecutor(this.#executor, this.#options, dispatch.build, lease, leaseMs);+ const suppressedCommands = new Set<string>();+ const controlled = new CapacityExecutor(+ this.#executor,+ this.#options,+ dispatch.build,+ lease,+ leaseMs,+ suppressedCommands,+ );+ await this.#reconcileOperationControl(stored.state, controlled, dispatch, lease, suppressedCommands);const scheduler = this.#options.scheduler.create(controlled, {...this.#options.scheduling,buildStore: this.#options.stores.builds,@@ -240,7 +334,7 @@ class DurableLocalWorker implements RuntimeWorker {heartbeat = heartbeat.then(async () => {await this.#options.stores.dispatch.heartbeat(dispatch.build, lease, Date.now(), leaseMs);}).catch((error: unknown) => { heartbeatError = error; });- }, Math.max(50, Math.floor(leaseMs / 3)));+ }, Math.max(1, Math.floor(leaseMs / 3)));try {const [result] = await scheduler.run([{ id: dispatch.build, state: stored.state }]);assert(result !== undefined, `Scheduler returned no result for ${dispatch.build}`);@@ -263,11 +357,24 @@ class DurableLocalWorker implements RuntimeWorker {return terminal;}const pending = result.journal.filter((item) => item.status === "pending");- const wakeAt = pending.length === 0- ? Date.now()- : Math.min(...pending.map((item) => item.wakeAt ?? Date.now() + 1_000));+ const controlledOperations = (await this.#options.stores.operations.list({ build: dispatch.build }))+ .filter((item) => item.cancellation !== undefined)+ .filter((item) => item.status === "created" || item.status === "pending");+ const now = Date.now();+ const wakeTimes = [+ ...pending.map((item) => item.wakeAt ?? now + 1_000),+ ...controlledOperations.map((item) => item.cancellation?.status === "requested"+ && item.cancellation.retryAt === undefined+ ? now+ : item.cancellation?.retryAt ?? item.wakeAt ?? now + 1_000),+ ];+ const wakeAt = wakeTimes.length === 0+ ? result.blocked.length > 0 ? Number.MAX_SAFE_INTEGER : Date.now()+ : Math.min(...wakeTimes);const released = await this.#options.stores.dispatch.release(dispatch.build, lease, {- phase: pending.length > 0 ? "waiting" : result.blocked.length > 0 ? "blocked" : "queued",+ phase: pending.length > 0 || controlledOperations.length > 0+ ? "waiting"+ : result.blocked.length > 0 ? "blocked" : "queued",availableAt: wakeAt,...(result.blocked.length === 0 ? {} : { reason: result.blocked.map((item) => item.reason).join(", ") }),});diff --git a/packages/local/test/config.test.ts b/packages/local/test/config.test.tsindex d5e330c8..5722a892 100644--- a/packages/local/test/config.test.ts+++ b/packages/local/test/config.test.ts@@ -39,6 +39,25 @@ test("declarative Runtime config has no implicit local services", () => {}), /runtimeServices/u);});+test("an explicit empty Runtime service set fails instead of manufacturing local defaults", async () => {+ const root = await mkdtemp(join(tmpdir(), "svml-runtime-empty-services-"));+ const path = join(root, "svml.runtime.json");+ try {+ await writeFile(path, JSON.stringify({+ format: "svml.runtime-config@1",+ ...required,+ endpoints: [],+ permissions: [],+ }));+ await assert.rejects(+ async () => await createRuntimeFromConfig(path, { registry: new RuntimeAdapterRegistry() }),+ /unknown instance execution\.scheduler/u,+ );+ } finally {+ await rm(root, { recursive: true, force: true });+ }+});+test("Runtime config is closed data and rejects unknown environment authority", () => {const parsed = parseRuntimeConfig({format: "svml.runtime-config@1",diff --git a/packages/local/test/local.test.ts b/packages/local/test/local.test.tsindex 0aaa30e6..140cb8c5 100644--- a/packages/local/test/local.test.ts+++ b/packages/local/test/local.test.ts@@ -189,9 +189,14 @@ test("project local runtime resumes durable work while component and endpoint palet promptCalls = 0;let requestCalls = 0;let assembleCalls = 0;+ let placeholderCalls = 0;let starts = 0;let resumes = 0;let cancels = 0;+ let cancellationMode: "confirmed" | "unsupported" = "confirmed";+ let holdNextStart = false;+ let startEntered: (() => void) | undefined;+ let releaseStart: (() => void) | undefined;const components: ComponentPackage = {name: "example.components",producers: [@@ -215,6 +220,16 @@ test("project local runtime resumes durable work while component and endpoint pareturn { outputs: {}, needs: { generation: { prompt: inputs.prompt.value.value } } };},},+ {+ producer: producers.placeholderText,+ implementationDigest: implementationDigests.placeholderText,+ handler: async ({ inputs }) => {+ placeholderCalls += 1;+ assert.equal(inputs.prompt?.value.kind, "inline");+ await new Promise((resolve) => setTimeout(resolve, 100));+ return { outputs: { generated: { kind: "inline", value: `Preview: ${inputs.prompt.value.value}` } }, needs: {} };+ },+ },{producer: producers.assemble,implementationDigest: implementationDigests.assemble,@@ -232,7 +247,11 @@ test("project local runtime resumes durable work while component and endpoint paconst recoverableEndpoint: RecoverableEndpoint = {start({ operation }) {starts += 1;- return { status: "pending", checkpoint: { remoteJob: operation.submissionKey }, wakeAt: Date.now() };+ const pending = { status: "pending" as const, checkpoint: { remoteJob: operation.submissionKey }, wakeAt: Date.now() };+ if (!holdNextStart) return pending;+ holdNextStart = false;+ startEntered?.();+ return new Promise((resolve) => { releaseStart = () => resolve(pending); });},resume({ checkpoint }) {resumes += 1;@@ -249,7 +268,7 @@ test("project local runtime resumes durable work while component and endpoint pa},cancel() {cancels += 1;- return { status: "confirmed" };+ return { status: cancellationMode };},};const endpointPackage: EndpointPackage = {@@ -325,6 +344,9 @@ test("project local runtime resumes durable work while component and endpoint paconst followedStatus = await secondRuntime.status("greeting-follow");assert.equal(followedStatus.build?.state.status, "complete");assert.equal(followedStatus.operations.length, 1);+ const tooLate = await secondRuntime.cancelOperation(followedStatus.operations[0]!.id);+ assert.equal(tooLate?.status, "completed");+ assert.equal(tooLate?.cancellation?.status, "too-late");const waiting = await secondRuntime.build({ id: "greeting-cancel", state: createGreetingBuild() });assert.equal(waiting.status, "queued");@@ -334,12 +356,156 @@ test("project local runtime resumes durable work while component and endpoint paconst cancelled = await secondRuntime.workOnce({ owner: "worker-two", leaseMs: 5_000 });assert.equal(cancelled?.terminal, "cancelled");assert.equal(cancels, 1);++ await secondRuntime.build({ id: "greeting-operation-cancel", state: createGreetingBuild() });+ assert.equal(+ (await secondRuntime.workOnce({ owner: "worker-two", leaseMs: 5_000 }))?.phase,+ "waiting",+ );+ const [operation] = (await secondRuntime.status("greeting-operation-cancel")).operations;+ assert.ok(operation);+ const operationRequest = await secondRuntime.cancelOperation(operation.id, "stop only this realization");+ assert.equal(operationRequest?.cancellation?.status, "requested");+ const stranded = await secondRuntime.workOnce({ owner: "worker-two", leaseMs: 5_000 });+ assert.equal(stranded?.phase, "blocked");+ assert.equal(stranded?.admission, "open", "Operation control does not close unrelated Build admission");+ assert.equal(stranded?.terminal, undefined);+ const controlled = await secondRuntime.operation(operation.id);+ assert.equal(controlled?.status, "cancelled");+ assert.equal(controlled?.cancellation?.status, "confirmed");+ assert.notEqual((await secondRuntime.status("greeting-operation-cancel")).build?.state.status, "failed");+ assert.equal(cancels, 2);++ cancellationMode = "unsupported";+ await secondRuntime.build({ id: "greeting-operation-late", state: createGreetingBuild() });+ await secondRuntime.workOnce({ owner: "worker-two", leaseMs: 5_000 });+ const [lateOperation] = (await secondRuntime.status("greeting-operation-late")).operations;+ assert.ok(lateOperation);+ await secondRuntime.cancelOperation(lateOperation.id);+ const lateDispatch = await secondRuntime.workOnce({ owner: "worker-two", leaseMs: 5_000 });+ assert.equal(lateDispatch?.phase, "blocked");+ const lateFact = await secondRuntime.operation(lateOperation.id);+ assert.equal(lateFact?.status, "completed", "an unsupported remote stop keeps the factual result");+ assert.equal(lateFact?.cancellation?.status, "unsupported");+ assert.ok(lateFact?.completion, "late paid output remains archived in OperationStore");+ const lateBuild = await secondRuntime.status("greeting-operation-late");+ assert.notEqual(lateBuild.build?.state.status, "complete", "late output is not reduced into Core");+ assert.notEqual(lateBuild.build?.state.status, "failed");++ cancellationMode = "confirmed";+ const assembledBeforeSide = assembleCalls;+ holdNextStart = true;+ const entered = new Promise<void>((resolve) => { startEntered = resolve; });+ await secondRuntime.build({+ id: "greeting-operation-side",+ state: createGreetingBuild({ includeSideTarget: true }),+ });+ const firstSideTurn = secondRuntime.workOnce({ owner: "worker-two", leaseMs: 5_000 });+ await entered;+ const sidePending = await secondRuntime.status("greeting-operation-side");+ const [sideOperation] = sidePending.operations;+ assert.ok(sideOperation);+ await secondRuntime.cancelOperation(sideOperation.id);+ releaseStart?.();+ const sideDispatch = await firstSideTurn;+ assert.equal(placeholderCalls, 1);+ // The Scheduler may finish the independent side branch in this same leased turn; cancellation+ // constrains only the exact remote Command and deliberately imposes no ordering on siblings.+ assert.equal(sideDispatch?.phase, "waiting");+ assert.ok(sideDispatch.availableAt <= Date.now(), JSON.stringify(sideDispatch));+ const reconciledSide = await secondRuntime.workOnce({ owner: "worker-two", leaseMs: 5_000 });+ assert.equal(reconciledSide?.phase, "blocked");+ const sideStatus = await secondRuntime.status("greeting-operation-side");+ assert.equal(sideStatus.build?.state.records.some((item) => item.id === "document:side"), true,+ "an unrelated branch keeps advancing after one Operation is suppressed");+ assert.equal(assembleCalls, assembledBeforeSide + 1);+ assert.notEqual(sideStatus.build?.state.status, "failed");await secondRuntime.close();} finally {await rm(directory, { recursive: true, force: true });}});+test("two durable Workers share one SQLite capacity limit across Runtime instances", async () => {+ const directory = await mkdtemp(join(tmpdir(), "svml-local-shared-capacity-"));+ let promptCalls = 0;+ let entered: (() => void) | undefined;+ let release: (() => void) | undefined;+ const firstEntered = new Promise<void>((resolve) => { entered = resolve; });+ const holdFirst = new Promise<void>((resolve) => { release = resolve; });+ const components: ComponentPackage = {+ name: "example.shared-capacity-components",+ producers: [{+ producer: producers.makePrompt,+ implementationDigest: implementationDigests.makePrompt,+ handler: async ({ inputs }) => {+ promptCalls += 1;+ if (promptCalls === 1) {+ entered?.();+ await holdFirst;+ }+ assert.equal(inputs.intent?.value.kind, "inline");+ return { outputs: { prompt: { kind: "inline", value: "Greet Ada" } }, needs: {} };+ },+ }, {+ producer: producers.requestText,+ implementationDigest: implementationDigests.requestText,+ handler: ({ inputs }) => {+ assert.equal(inputs.prompt?.value.kind, "inline");+ return { outputs: {}, needs: { generation: { prompt: inputs.prompt.value.value } } };+ },+ }, {+ producer: producers.placeholderText,+ implementationDigest: implementationDigests.placeholderText,+ handler: ({ inputs }) => ({+ outputs: { generated: { kind: "inline", value: `Preview: ${String(inputs.prompt?.value.kind === "inline" ? inputs.prompt.value.value : "")}` } },+ needs: {},+ }),+ }, {+ producer: producers.assemble,+ implementationDigest: implementationDigests.assemble,+ handler: ({ inputs }) => ({+ outputs: { document: { kind: "inline", value: { text: String(inputs.generated?.value.kind === "inline" ? inputs.generated.value.value : "") } } },+ needs: {},+ }),+ }],+ };+ const options = () => ({+ ...projectRuntimeFixture(directory),+ scheduling: { maxConcurrency: 1 },+ components: [components],+ } as const);+ let first: Awaited<ReturnType<typeof createProjectLocalRuntime>> | undefined;+ let second: Awaited<ReturnType<typeof createProjectLocalRuntime>> | undefined;+ try {+ first = await createProjectLocalRuntime({ root: directory, ...options() });+ second = await createProjectLocalRuntime({ root: directory, ...options() });+ await first.build({+ id: "capacity-a",+ state: createGreetingBuild({ generationRealization: "placeholder", goalAccepts: "substitute" }),+ });+ await second.build({+ id: "capacity-b",+ state: createGreetingBuild({ generationRealization: "placeholder", goalAccepts: "substitute" }),+ });+ const firstTurn = first.workOnce({ owner: "worker-a", leaseMs: 120 });+ await firstEntered;+ const secondTurn = second.workOnce({ owner: "worker-b", leaseMs: 120 });+ await new Promise((resolve) => setTimeout(resolve, 200));+ assert.equal(promptCalls, 1,+ "the second Worker cannot enter after the original capacity expiry while its peer keeps heartbeating");+ assert.equal((await first.queue()).capacity.length, 1);+ release?.();+ await Promise.all([firstTurn, secondTurn]);+ assert.equal(promptCalls, 2);+ } finally {+ release?.();+ await second?.close();+ await first?.close();+ await rm(directory, { recursive: true, force: true });+ }+});+test("project local runtime activates locked compute facets without deployment source registration", async () => {const directory = await mkdtemp(join(tmpdir(), "svml-local-locked-components-"));const runtimeRoot = join(directory, "external-project");diff --git a/packages/runtime/src/dispatch.ts b/packages/runtime/src/dispatch.tsindex 99329cb1..eab2af1f 100644--- a/packages/runtime/src/dispatch.ts+++ b/packages/runtime/src/dispatch.ts@@ -114,6 +114,8 @@ export type BuildDispatchStore = {release(build: string, lease: DispatchLease, update: BuildDispatchRelease, now?: number): Promise<BuildDispatchSnapshot>;finish(build: string, lease: DispatchLease, terminal: DispatchTerminal, reason?: string, now?: number): Promise<BuildDispatchSnapshot>;requestCancellation(build: string, reason?: string, now?: number): Promise<BuildDispatchSnapshot>;+ /** Make non-terminal work immediately claimable without changing admission or creative intent. */+ wake(build: string, now?: number): Promise<BuildDispatchSnapshot>;acquireCapacity(request: CapacityAcquireRequest): Promise<CapacityAcquire>;heartbeatCapacity(id: Digest, lease: DispatchLease, now: number, leaseMs: number): Promise<CapacityReservation>;parkCapacity(id: Digest, lease: DispatchLease, inFlight: boolean, now?: number): Promise<CapacityReservation>;diff --git a/packages/runtime/src/services.ts b/packages/runtime/src/services.tsindex 711ea4ab..3e9cdd5b 100644--- a/packages/runtime/src/services.ts+++ b/packages/runtime/src/services.ts@@ -156,7 +156,7 @@ function verifyServicePort(service: RuntimeService): void {break;case "dispatch-store":for (const method of [- "create", "read", "list", "claim", "heartbeat", "release", "finish", "requestCancellation",+ "create", "read", "list", "claim", "heartbeat", "release", "finish", "requestCancellation", "wake","acquireCapacity", "heartbeatCapacity", "parkCapacity", "releaseCapacity", "clearCapacity", "listCapacity",]) callable(service.service, method, service.instance.id);break;diff --git a/packages/runtime/test/services.test.ts b/packages/runtime/test/services.test.tsindex d95c2e67..2d74dbb2 100644--- a/packages/runtime/test/services.test.ts+++ b/packages/runtime/test/services.test.ts@@ -67,7 +67,7 @@ function servicePackage(close?: () => void) {instance: "dispatch.example",implementation: { locator: "example.runtime-services/dispatch", digest: digestOf("dispatch") },service: Object.fromEntries([- "create", "read", "list", "claim", "heartbeat", "release", "finish", "requestCancellation",+ "create", "read", "list", "wake", "claim", "heartbeat", "release", "finish", "requestCancellation","acquireCapacity", "heartbeatCapacity", "parkCapacity", "releaseCapacity", "clearCapacity", "listCapacity",].map((name) => [name, async () => undefined])) as never,},diff --git a/packages/store-sqlite/src/store.ts b/packages/store-sqlite/src/store.tsindex e31019cb..ff1b0778 100644--- a/packages/store-sqlite/src/store.ts+++ b/packages/store-sqlite/src/store.ts@@ -57,7 +57,7 @@ import type {RuntimeServicePackage,} from "@narratage/runtime";-const databaseSchemaVersion = 3;+const databaseSchemaVersion = 4;export const sqliteStoreModuleRef = {name: "@narratage/store-sqlite",@@ -578,11 +578,13 @@ class SqliteBuildDispatchStore implements BuildDispatchStore {nonNegativeInteger(update.availableAt, "Dispatch release availableAt");const updated = this.#database.prepare(`UPDATE svml_dispatches- SET revision = revision + 1, updated_at = ?, phase = ?, available_at = ?, reason = ?,+ SET revision = revision + 1, updated_at = ?, phase = ?,+ available_at = CASE WHEN wake_at IS NULL THEN ? ELSE MIN(?, wake_at) END,+ wake_at = NULL, reason = ?,lease_owner = NULL, lease_token = NULL, lease_expires_at = NULLWHERE build_id = ? AND phase = 'leased'AND lease_owner = ? AND lease_token = ? AND lease_fence = ?- `).run(now, update.phase, update.availableAt, update.reason ?? null,+ `).run(now, update.phase, update.availableAt, update.availableAt, update.reason ?? null,build, lease.owner, lease.token, lease.fence);assert(updated.changes === 1, `Dispatch ${build} lease is stale`);return (await this.read(build))!;@@ -599,7 +601,8 @@ class SqliteBuildDispatchStore implements BuildDispatchStore {const updated = this.#database.prepare(`UPDATE svml_dispatchesSET revision = revision + 1, updated_at = ?, phase = 'terminal', admission = 'closed',- terminal = ?, reason = ?, lease_owner = NULL, lease_token = NULL, lease_expires_at = NULL+ terminal = ?, reason = ?, wake_at = NULL,+ lease_owner = NULL, lease_token = NULL, lease_expires_at = NULLWHERE build_id = ? AND phase = 'leased'AND lease_owner = ? AND lease_token = ? AND lease_fence = ?`).run(now, terminal, reason ?? null, build, lease.owner, lease.token, lease.fence);@@ -613,9 +616,28 @@ class SqliteBuildDispatchStore implements BuildDispatchStore {UPDATE svml_dispatchesSET revision = revision + 1, updated_at = ?, admission = 'closing',cancel_requested_at = COALESCE(cancel_requested_at, ?),- cancel_reason = COALESCE(cancel_reason, ?), available_at = MIN(available_at, ?)+ cancel_reason = COALESCE(cancel_reason, ?),+ available_at = CASE WHEN phase = 'leased' THEN available_at ELSE MIN(available_at, ?) END,+ wake_at = CASE WHEN phase = 'leased' THEN MIN(COALESCE(wake_at, ?), ?) ELSE wake_at END+ WHERE build_id = ? AND phase != 'terminal'+ `).run(now, now, reason ?? null, now, now, now, build);+ if (updated.changes === 0) {+ const current = await this.read(build);+ if (current === undefined) throw new Error(`Dispatch ${build} does not exist`);+ return current;+ }+ return (await this.read(build))!;+ }++ async wake(build: string, now = Date.now()): Promise<BuildDispatchSnapshot> {+ nonNegativeInteger(now, "Dispatch wake time");+ const updated = this.#database.prepare(`+ UPDATE svml_dispatches+ SET revision = revision + 1, updated_at = ?,+ available_at = CASE WHEN phase = 'leased' THEN available_at ELSE MIN(available_at, ?) END,+ wake_at = CASE WHEN phase = 'leased' THEN MIN(COALESCE(wake_at, ?), ?) ELSE wake_at ENDWHERE build_id = ? AND phase != 'terminal'- `).run(now, now, reason ?? null, now, build);+ `).run(now, now, now, now, build);if (updated.changes === 0) {const current = await this.read(build);if (current === undefined) throw new Error(`Dispatch ${build} does not exist`);@@ -888,6 +910,7 @@ export class SqliteRuntimeState {updated_at INTEGER NOT NULL,priority INTEGER NOT NULL,available_at INTEGER NOT NULL,+ wake_at INTEGER,admission TEXT NOT NULL CHECK (admission IN ('open', 'closing', 'closed')),phase TEXT NOT NULL CHECK (phase IN ('queued', 'leased', 'waiting', 'blocked', 'settling', 'terminal')),lease_owner TEXT,diff --git a/packages/store-sqlite/test/store.test.ts b/packages/store-sqlite/test/store.test.tsindex c7696550..4a913cf8 100644--- a/packages/store-sqlite/test/store.test.ts+++ b/packages/store-sqlite/test/store.test.ts@@ -159,7 +159,7 @@ test("Host Catalog schema changes do not change the execution Runtime Closure",assert.deepEqual(services.services.map((item) => item.instance.configurationDigest),[- ...Array(4).fill(digestOf({ path: join(directory, "runtime.sqlite"), schemaVersion: 3, busyTimeoutMs: 5_000 })),+ ...Array(4).fill(digestOf({ path: join(directory, "runtime.sqlite"), schemaVersion: 4, busyTimeoutMs: 5_000 })),],);await services.close?.();@@ -202,6 +202,18 @@ test("SQLite fences expired Workers and shares capacity across Build dispatches"});assert.equal(reserved.status, "acquired");+ // A control request can race with the leased Worker's final release. The Store must retain the+ // wake independently of the old ready timestamp so that release cannot overwrite it.+ await state.dispatch.wake("build-a", 113);+ const woken = await state.dispatch.release(+ "build-a",+ second.lease,+ { phase: "waiting", availableAt: 10_000 },+ 114,+ );+ assert.equal(woken.availableAt, 113);+ assert.equal(woken.phase, "waiting");+await state.dispatch.create(createBuildDispatchIdentity({build: "build-b", core: digestOf("core:b"), runtimeClosure: closure,}), { now: 112 });@@ -213,6 +225,22 @@ test("SQLite fences expired Workers and shares capacity across Build dispatches"limits: { globalActive: 1, laneActive: 1, laneInFlight: 1 },});assert.equal(blocked.status, "blocked");++ await state.dispatch.create(createBuildDispatchIdentity({+ build: "build-c", core: digestOf("core:c"), runtimeClosure: closure,+ }), { now: 200, priority: 100 });+ const fourth = await state.dispatch.claim({ owner: "worker-d", token: "lease-d", now: 200, leaseMs: 10 });+ assert.ok(fourth?.lease);+ await state.dispatch.requestCancellation("build-c", "stop", 201);+ const cancelledWake = await state.dispatch.release(+ "build-c",+ fourth.lease,+ { phase: "waiting", availableAt: 20_000 },+ 202,+ );+ assert.equal(cancelledWake.admission, "closing");+ assert.equal(cancelledWake.availableAt, 201,+ "a cancellation racing with release cannot be delayed by the Worker's stale schedule");state.close();} finally {await rm(directory, { recursive: true, force: true });diff --git a/packages/video-cli/test/cli.test.ts b/packages/video-cli/test/cli.test.tsindex c49b3c94..329f064b 100644--- a/packages/video-cli/test/cli.test.ts+++ b/packages/video-cli/test/cli.test.ts@@ -487,6 +487,7 @@ test("CLI inspect and get read the durable Build archive independently of buildconst recordDigest = `sha256:${"1".repeat(64)}`;const buildDigest = `sha256:${"2".repeat(64)}`;const requestDigest = `sha256:${"3".repeat(64)}`;+ const operationId = `sha256:${"7".repeat(64)}`;const state = {id: buildDigest,status: "active",@@ -544,16 +545,30 @@ test("CLI inspect and get read the durable Build archive independently of buildcreatedAt: 100,updatedAt: 100,};+ const operation = {+ id: operationId,+ build: "archive-1",+ command: `sha256:${"8".repeat(64)}`,+ endpoint: "example.endpoint",+ attempt: 1,+ status: "pending",+ };+ const dispatch = { build: "archive-1", phase: "waiting", admission: "open" };await writeFile(runtimePath, `const bytes = Buffer.from(${JSON.stringify(bytes.toString("base64"))}, "base64");const rawBytes = Buffer.from(${JSON.stringify(rawBytes.toString("base64"))}, "base64");const catalog = ${JSON.stringify(catalog)};+const operation = ${JSON.stringify(operation)};+const dispatch = ${JSON.stringify(dispatch)};export default {async build() { throw new Error("not used"); },async builds() { return [catalog]; },async credentials() { return []; },async putCredential() { throw new Error("not used"); },async deleteCredential() { throw new Error("not used"); },- async status(id) { return { build: id === "archive-1" ? { build: id, revision: 7, state: ${JSON.stringify(state)} } : undefined, catalog: id === "archive-1" ? catalog : undefined, operations: [] }; },+ async queue() { return { dispatches: [dispatch], capacity: [] }; },+ async operation(id) { return id === operation.id ? operation : undefined; },+ async cancelOperation(id) { return id === operation.id ? { ...operation, cancellation: { status: "requested" } } : undefined; },+ async status(id) { return { build: id === "archive-1" ? { build: id, revision: 7, state: ${JSON.stringify(state)} } : undefined, catalog: id === "archive-1" ? catalog : undefined, operations: id === "archive-1" ? [operation] : [], dispatch: id === "archive-1" ? dispatch : undefined }; },async readArtifact(digest) {if (digest === ${JSON.stringify(artifactDigest)}) return bytes;if (digest === ${JSON.stringify(rawDigest)}) return rawBytes;@@ -617,6 +632,17 @@ export default {targets: ["final.video"],});+ let queueOutput = "";+ await runCli(["queue", "--runtime", runtimePath], { write: (text) => { queueOutput += text; } });+ const queue = JSON.parse(queueOutput) as { readonly dispatches: readonly { readonly phase: string }[] };+ assert.equal(queue.dispatches[0]?.phase, "waiting");++ let operationOutput = "";+ await runCli(["operation", operationId, "--runtime", runtimePath], {+ write: (text) => { operationOutput += text; },+ });+ assert.equal((JSON.parse(operationOutput) as { readonly operation?: { readonly id: string } }).operation?.id, operationId);+let getOutput = "";await runCli(["get", "archive-1", "--runtime", runtimePath,@@ -632,4 +658,47 @@ export default {"--artifact", rawDigest, "--to", rawDestination,], { write() {} });assert.deepEqual(await readFile(rawDestination), rawBytes);++ let cancellationOutput = "";+ await runCli([+ "cancel", "operation", operationId, "--runtime", runtimePath,+ "--reason", "stop this attempt",+ ], { write: (text) => { cancellationOutput += text; } });+ const cancellation = JSON.parse(cancellationOutput) as {+ readonly scope: string;+ readonly requested: boolean;+ readonly build: string;+ readonly control: string;+ };+ assert.deepEqual(cancellation, {+ scope: "operation",+ operation: operationId,+ requested: true,+ build: "archive-1",+ execution: "pending",+ control: "requested",+ });++ const human = async (argv: readonly string[]) => {+ let output = "";+ await runVideoCli([...argv, "--color", "never"], { write: (text) => { output += text; } });+ return output;+ };+ assert.match(await human(["queue", "--runtime", runtimePath]), /Runtime queue[\s\S]*archive-1: waiting/u);+ assert.match(await human(["status", "archive-1", "--runtime", runtimePath]), /Build status[\s\S]*Dispatch\s+waiting/u);+ assert.match(await human(["operation", operationId, "--runtime", runtimePath]), /Operation detail[\s\S]*Status\s+pending/u);+ assert.match(await human(["cancel", "operation", operationId, "--runtime", runtimePath]),+ /Operation cancellation requested[\s\S]*Control\s+requested/u);+ await assert.rejects(+ runCli(["status", "archive-1", "--runtime", runtimePath, "--watch"], { write() {} }),+ /--watch applies only to queue/u,+ );+ await assert.rejects(+ runCli(["queue", "--runtime", runtimePath, "--jsonl"], { write() {} }),+ /--jsonl applies only to queue --watch/u,+ );+ await assert.rejects(+ runCli(["queue", "--runtime", runtimePath, "--watch", "--json"], { write() {} }),+ /use --jsonl instead of --json/u,+ );});diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex be85bba2..d828eb55 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -11,6 +11,9 @@ importers:'@narratage/artifact':specifier: workspace:*version: link:packages/artifact+ '@narratage/artifact-store-fs':+ specifier: workspace:*+ version: link:packages/artifact-store-fs'@narratage/audio-track':specifier: workspace:*version: link:packages/audio-track@@ -32,6 +35,12 @@ importers:'@narratage/composition':specifier: workspace:*version: link:packages/composition+ '@narratage/credential-store-env':+ specifier: workspace:*+ version: link:packages/credential-store-env+ '@narratage/credential-store-keychain':+ specifier: workspace:*+ version: link:packages/credential-store-keychain'@narratage/deck-track':specifier: workspace:*version: link:packages/deck-track@@ -152,6 +161,9 @@ importers:'@narratage/speech-spine':specifier: workspace:*version: link:packages/speech-spine+ '@narratage/store-sqlite':+ specifier: workspace:*+ version: link:packages/store-sqlite'@narratage/svs':specifier: workspace:*version: link:packages/svs@@ -200,6 +212,9 @@ importers:'@narratage/runtime':specifier: workspace:*version: link:../runtime+ '@narratage/runtime-adapter':+ specifier: workspace:*+ version: link:../runtime-adapterpackages/artifact-store-s3:dependencies:@@ -563,6 +578,9 @@ importers:'@narratage/runtime':specifier: workspace:*version: link:../runtime+ '@narratage/runtime-adapter':+ specifier: workspace:*+ version: link:../runtime-adapterpackages/credential-store-keychain:dependencies:@@ -1267,15 +1285,9 @@ importers:packages/local:dependencies:- '@narratage/artifact-store-fs':- specifier: workspace:*- version: link:../artifact-store-fs'@narratage/component-kit':specifier: workspace:*version: link:../component-kit- '@narratage/credential-store-env':- specifier: workspace:*- version: link:../credential-store-env'@narratage/driver-node':specifier: workspace:*version: link:../driver-node@@ -1297,16 +1309,22 @@ importers:'@narratage/runtime-adapter':specifier: workspace:*version: link:../runtime-adapter- '@narratage/store-sqlite':- specifier: workspace:*- version: link:../store-sqlite'@narratage/validation':specifier: workspace:*version: link:../validationdevDependencies:+ '@narratage/artifact-store-fs':+ specifier: workspace:*+ version: link:../artifact-store-fs'@narratage/core':specifier: workspace:*version: link:../core+ '@narratage/credential-store-env':+ specifier: workspace:*+ version: link:../credential-store-env+ '@narratage/store-sqlite':+ specifier: workspace:*+ version: link:../store-sqlitepackages/markup:dependencies:@@ -1761,9 +1779,6 @@ importers:'@narratage/runtime-adapter':specifier: workspace:*version: link:../runtime-adapter- '@narratage/runtime-adapter-node':- specifier: workspace:*- version: link:../runtime-adapter-nodedevDependencies:'@narratage/artifact-store-fs':specifier: workspace:*@@ -1774,6 +1789,9 @@ importers:'@narratage/core':specifier: workspace:*version: link:../core+ '@narratage/credential-store-env':+ specifier: workspace:*+ version: link:../credential-store-env'@narratage/driver-node':specifier: workspace:*version: link:../driver-node@@ -1813,6 +1831,9 @@ importers:'@narratage/seedream':specifier: workspace:*version: link:../seedream+ '@narratage/store-sqlite':+ specifier: workspace:*+ version: link:../store-sqlite'@narratage/validation':specifier: workspace:*version: link:../validation@@ -1948,9 +1969,6 @@ importers:'@narratage/runtime-adapter':specifier: workspace:*version: link:../runtime-adapter- '@narratage/runtime-adapter-node':- specifier: workspace:*- version: link:../runtime-adapter-nodepackages/ranking:dependencies:@@ -2431,6 +2449,9 @@ importers:'@narratage/runtime':specifier: workspace:*version: link:../runtime+ '@narratage/runtime-adapter':+ specifier: workspace:*+ version: link:../runtime-adapterpackages/svs:dependencies: