Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

feat(runtime): add durable local execution control

kashorinc04e0eb08a0a
diff --git a/packages/artifact-store-fs/package.json b/packages/artifact-store-fs/package.json
index 5bb9e885..59ac83a6 100644
--- a/packages/artifact-store-fs/package.json
+++ b/packages/artifact-store-fs/package.json
@@ -6,8 +6,12 @@
"exports": {
".": "./src/index.ts"
},
+ "svml": {
+ "activation": "./src/activation.ts"
+ },
"dependencies": {
"@narratage/protocol": "workspace:*",
- "@narratage/runtime": "workspace:*"
+ "@narratage/runtime": "workspace:*",
+ "@narratage/runtime-adapter": "workspace:*"
}
}
diff --git a/packages/artifact-store-fs/src/activation.ts b/packages/artifact-store-fs/src/activation.ts
new file mode 100644
index 00000000..df3aac8d
--- /dev/null
+++ b/packages/artifact-store-fs/src/activation.ts
@@ -0,0 +1,36 @@
+import { resolve } from "node:path";
+
+import {
+ createRuntimeServiceAdapterFacet,
+ runtimeConfigExact,
+ runtimeConfigObject,
+ runtimeConfigString,
+} from "@narratage/runtime-adapter";
+
+import { createFileArtifactStorePackage } from "./store.js";
+
+const fileArtifactStoreAdapter = createRuntimeServiceAdapterFacet({
+ use: "@narratage/artifact-store-fs",
+ validate(context) {
+ const config = runtimeConfigObject(context.config, "filesystem ArtifactStore");
+ runtimeConfigExact(config, ["path"], "filesystem ArtifactStore");
+ if (runtimeConfigString(config.path, "Artifact path") === undefined) throw new Error("Artifact path is required");
+ },
+ create(context) {
+ const config = runtimeConfigObject(context.config, "filesystem ArtifactStore");
+ const path = runtimeConfigString(config.path, "Artifact path");
+ if (path === undefined) throw new Error("Artifact path is required");
+ return createFileArtifactStorePackage({
+ root: resolve(context.root, path),
+ instance: context.instance,
+ });
+ },
+});
+
+export const svmlPackage = {
+ format: "svml.node-package@1" as const,
+ name: "@narratage/artifact-store-fs",
+ hostFacets: [fileArtifactStoreAdapter],
+};
+
+export default svmlPackage;
diff --git a/packages/cli/src/distribution.ts b/packages/cli/src/distribution.ts
index 28be0afc..91964be0 100644
--- a/packages/cli/src/distribution.ts
+++ b/packages/cli/src/distribution.ts
@@ -25,6 +25,8 @@ export type CliDistribution = {
readonly runFrontends: readonly RunFrontend[];
createCompiler(options: CliCompilerOptions): NodeCompiler;
createRuntimeFromConfig(path: string): Promise<LocalRuntime>;
+ /** Re-enter this exact Distribution as the hidden durable Worker process. */
+ runtimeWorkerLaunch(): { readonly command: string; readonly args: readonly string[] };
doctorRuntimeConfig(path: string): Promise<RuntimeConfigDoctorResult>;
/** The external programs a Runtime Profile implies: probe, prepare and start them. */
readonly externalServices: {
diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts
index 516e970e..de941adc 100644
--- a/packages/cli/src/main.ts
+++ b/packages/cli/src/main.ts
@@ -1,9 +1,11 @@
import { dirname, extname, resolve } from "node:path";
import { pathToFileURL } from "node:url";
+import { readFile } from "node:fs/promises";
import type { ExternalServiceReport, LocalRuntime } from "@narratage/local";
import type { NodeCompiledSourceClosure } from "@narratage/compiler-node";
import type { BuildCatalogDescriptor } from "@narratage/runtime";
+import { isDigest } from "@narratage/protocol";
import { parseSourceHeader } from "@narratage/source";
import {
createNodePackageLock,
@@ -24,10 +26,17 @@ import { collectRunFrontends, loadRunFile } from "./run-file.js";
import type { CliDistribution } from "./distribution.js";
import { writeCliHelp, writeCliOutput } from "./output.js";
import type { CliColorMode, CliIo } from "./output.js";
+import {
+ ensureRuntimeProcess,
+ markRuntimeProcessReady,
+ runtimeProcessLogs,
+ runtimeProcessStatus,
+ stopRuntimeProcess,
+} from "./runtime-process.js";
type ParsedArgs = {
readonly command: string | undefined;
- /** Second command word. Only `services` takes one. */
+ /** Second command word for scoped commands such as runtime and cancel. */
readonly action: string | undefined;
readonly file: string | undefined;
/** Canonical containment boundary for Author and Run Sources plus source assets. */
@@ -54,14 +63,22 @@ type ParsedArgs = {
readonly json: boolean;
readonly color: CliColorMode;
readonly verbose: boolean;
+ readonly watch: boolean;
+ readonly jsonl: boolean;
+ readonly readyFile: string | undefined;
+ readonly reason: string | undefined;
+ readonly slot: string | undefined;
+ readonly from: string | undefined;
};
function parseArgs(argv: readonly string[]): ParsedArgs {
const [command, ...tail] = argv;
- const action = command === "services" ? tail[0] : undefined;
- const positional = command === "services" ? tail.slice(1) : tail;
- const file = command === "builds" ? undefined : positional[0];
- const rest = command === "builds" ? positional : positional.slice(1);
+ const scoped = command === "services" || command === "runtime" || command === "cancel" || command === "auth";
+ const action = scoped ? tail[0] : undefined;
+ const positional = scoped ? tail.slice(1) : tail;
+ const noFile = command === "builds" || command === "queue";
+ const file = noFile ? undefined : positional[0];
+ const rest = noFile ? positional : positional.slice(1);
const targets: string[] = [];
let workspaceRoot: string | undefined;
let packageRoot: string | undefined;
@@ -83,12 +100,26 @@ function parseArgs(argv: readonly string[]): ParsedArgs {
let json = false;
let color: CliColorMode = "auto";
let verbose = false;
+ let watch = false;
+ let jsonl = false;
+ let readyFile: string | undefined;
+ let reason: string | undefined;
+ let slot: string | undefined;
+ let from: string | undefined;
for (let index = 0; index < rest.length; index += 1) {
const item = rest[index];
if (item === "--json") {
json = true;
continue;
}
+ if (item === "--jsonl") {
+ jsonl = true;
+ continue;
+ }
+ if (item === "--watch") {
+ watch = true;
+ continue;
+ }
if (item === "--verbose") {
verbose = true;
continue;
@@ -231,6 +262,34 @@ function parseArgs(argv: readonly string[]): ParsedArgs {
index += 1;
continue;
}
+ if (item === "--ready-file") {
+ const value = rest[index + 1];
+ if (value === undefined || value.startsWith("--")) throw new Error("--ready-file requires a path");
+ readyFile = resolve(value);
+ index += 1;
+ continue;
+ }
+ if (item === "--reason") {
+ const value = rest[index + 1];
+ if (value === undefined || value.startsWith("--")) throw new Error("--reason requires text");
+ reason = value;
+ index += 1;
+ continue;
+ }
+ if (item === "--slot") {
+ const value = rest[index + 1];
+ if (value === undefined || value.startsWith("--")) throw new Error("--slot requires a credential slot");
+ slot = value;
+ index += 1;
+ continue;
+ }
+ if (item === "--from") {
+ const value = rest[index + 1];
+ if (value === undefined || value.startsWith("--")) throw new Error("--from requires a credential file");
+ from = resolve(value);
+ index += 1;
+ continue;
+ }
throw new Error(`unknown option ${item}`);
}
return {
@@ -258,6 +317,12 @@ function parseArgs(argv: readonly string[]): ParsedArgs {
json,
color,
verbose,
+ watch,
+ jsonl,
+ readyFile,
+ reason,
+ slot,
+ from,
};
}
@@ -267,6 +332,8 @@ function usage(): string {
" narratage lock-packages <svml.packages.lock> --package installed-name [--package installed-name] [--package-root directory]",
" narratage doctor <runtime-profile.json>",
" narratage services up|down|status <runtime-profile.json> [--max-wait-ms milliseconds]",
+ " narratage runtime up|status|logs|down <runtime-profile.json>",
+ " narratage queue --runtime profile.json [--watch]",
" narratage gc <runtime-profile.json> [--apply]",
" narratage check <self-described-source> [--runtime profile.json] [--package-lock file] [--package-root directory] [--root workspace]",
" narratage plan <run-source> [--runtime profile.json] [--package-lock file] [--package-root directory] [--root workspace]",
@@ -275,7 +342,10 @@ function usage(): string {
" narratage builds --runtime profile.json|./svml.runtime.ts",
" narratage inspect <build-id> --runtime profile.json|./svml.runtime.ts",
" narratage get <build-id> --runtime profile.json|./svml.runtime.ts [--name source-name|--record record-id|--output logical-output-id|--artifact digest] [--to path]",
- " narratage cancel <build-id> --runtime profile.json|./svml.runtime.ts",
+ " narratage operations <build-id> --runtime profile.json|./svml.runtime.ts",
+ " narratage operation <operation-id> --runtime profile.json|./svml.runtime.ts",
+ " narratage cancel build <build-id> --runtime profile.json|./svml.runtime.ts [--reason text]",
+ " narratage auth status|login|logout <endpoint-instance> --runtime profile.json [--slot name] [--from secret-file]",
"",
"output:",
" --json --verbose --color auto|always|never --no-color --debug",
@@ -291,6 +361,12 @@ function isLocalRuntime(value: unknown): value is LocalRuntime {
&& typeof value.builds === "function"
&& "readArtifact" in value
&& typeof value.readArtifact === "function"
+ && "credentials" in value
+ && typeof value.credentials === "function"
+ && "putCredential" in value
+ && typeof value.putCredential === "function"
+ && "deleteCredential" in value
+ && typeof value.deleteCredential === "function"
&& "close" in value
&& typeof value.close === "function";
}
@@ -384,11 +460,49 @@ export async function runCli(
return;
}
const args = parseArgs(argv);
+ const writeOperational = (
+ machine: unknown,
+ title: string,
+ status: "success" | "warning" | "error" | "info" = "info",
+ facts: readonly (readonly [string, string])[] = [],
+ lines: readonly string[] = [],
+ ): void => writeCliOutput(io, {
+ json: args.json || args.jsonl,
+ jsonl: args.jsonl,
+ color: args.color,
+ verbose: args.verbose,
+ }, { kind: "operational", machine, title, status, facts, lines });
+ if (args.command === "_worker") {
+ if (args.file === undefined || args.readyFile === undefined) throw new Error("internal Worker launch is incomplete");
+ const runtime = await loadLocalRuntime(resolve(args.file), distribution);
+ const abort = new AbortController();
+ const stop = (): void => abort.abort();
+ process.once("SIGTERM", stop);
+ process.once("SIGINT", stop);
+ try {
+ await markRuntimeProcessReady(args.readyFile);
+ await runtime.work({
+ owner: `worker-${process.pid}`,
+ leaseMs: 30_000,
+ idlePollMs: 250,
+ signal: abort.signal,
+ });
+ } finally {
+ process.removeListener("SIGTERM", stop);
+ process.removeListener("SIGINT", stop);
+ await runtime.close();
+ }
+ return;
+ }
const known = args.command === "lock-packages" || args.command === "check" || args.command === "plan"
|| args.command === "build" || args.command === "status" || args.command === "builds"
|| args.command === "inspect" || args.command === "get" || args.command === "cancel"
- || args.command === "doctor" || args.command === "gc" || args.command === "services";
- if (!known || (args.command !== "builds" && args.file === undefined)) {
+ || args.command === "doctor" || args.command === "gc" || args.command === "services"
+ || args.command === "runtime" || args.command === "queue" || args.command === "operations"
+ || args.command === "operation";
+ const operational = known || args.command === "auth";
+ const fileOptional = args.command === "builds" || args.command === "queue";
+ if (!operational || (!fileOptional && args.file === undefined)) {
throw new Error(usage());
}
if (args.command === "lock-packages") {
@@ -401,7 +515,10 @@ export async function runCli(
const packageRoot = args.packageRoot ?? distribution.packageRoot ?? dirname(output);
const lock = await createNodePackageLock(args.packages, packageRoot);
await writeNodePackageLock(output, lock);
- io.write(`${JSON.stringify({ ok: true, packageLock: output, digest: lock.digest, packages: lock.packages }, null, 2)}\n`);
+ writeOperational({ ok: true, packageLock: output, digest: lock.digest, packages: lock.packages },
+ "Package lock written", "success", [
+ ["Path", output], ["Packages", String(lock.packages.length)], ["Digest", lock.digest],
+ ]);
return;
}
if (args.command === "doctor") {
@@ -439,11 +556,76 @@ export async function runCli(
: args.action === "down"
? await distribution.externalServices.down(profile)
: await distribution.externalServices.report(profile);
- io.write(`${JSON.stringify({
+ const machine = {
ok: result.services.every((item) => (args.action === "down" ? item.state.state !== "ready" : item.state.state === "ready")),
root: result.root,
services: result.services,
- }, null, 2)}\n`);
+ };
+ writeOperational(machine, `External services ${args.action}`, machine.ok ? "success" : "warning", [
+ ["Root", result.root],
+ ["Services", String(result.services.length)],
+ ["Ready", String(result.services.filter((item) => item.state.state === "ready").length)],
+ ]);
+ return;
+ }
+ if (args.command === "runtime") {
+ if (args.action !== "up" && args.action !== "down" && args.action !== "status" && args.action !== "logs") {
+ throw new Error("runtime takes up, down, status or logs");
+ }
+ const profile = resolve(args.file!);
+ if (args.action === "up") {
+ const external = await distribution.externalServices.up(profile, {
+ ...(args.maxWaitMs === undefined ? {} : { maxWaitMs: args.maxWaitMs }),
+ });
+ const processState = await ensureRuntimeProcess(
+ profile,
+ distribution.runtimeWorkerLaunch(),
+ args.maxWaitMs ?? 10_000,
+ );
+ writeOperational({ ok: true, worker: processState, services: external.services }, "Runtime is up", "success", [
+ ["Worker", String(processState.pid)],
+ ["External services", String(external.services.length)],
+ ]);
+ return;
+ }
+ if (args.action === "logs") {
+ const logs = await runtimeProcessLogs(profile);
+ if (!args.json && !args.jsonl) io.write(logs.text);
+ else writeOperational(logs, "Runtime logs");
+ return;
+ }
+ if (args.action === "down") {
+ const worker = await stopRuntimeProcess(profile, args.maxWaitMs ?? 10_000);
+ const external = await distribution.externalServices.down(profile);
+ writeOperational({ ok: true, worker, services: external.services }, "Runtime is down", "success", [
+ ["Worker", worker.state],
+ ["External services", String(external.services.length)],
+ ]);
+ return;
+ }
+ const worker = await runtimeProcessStatus(profile);
+ const external = await distribution.externalServices.report(profile);
+ const runtime = await loadLocalRuntime(profile, distribution);
+ try {
+ const queue = await runtime.queue();
+ const counts = Object.fromEntries(["queued", "leased", "waiting", "blocked", "settling", "terminal"]
+ .map((phase) => [phase, queue.dispatches.filter((item) => item.phase === phase).length]));
+ const machine = {
+ ok: worker.state === "running" && external.services.every((item) => item.state.state === "ready"),
+ worker,
+ queue: { counts, capacity: queue.capacity },
+ services: external.services,
+ };
+ writeOperational(machine, "Runtime status", machine.ok ? "success" : "warning", [
+ ["Worker", worker.state],
+ ["Queued", String(counts.queued ?? 0)],
+ ["Running", String(counts.leased ?? 0)],
+ ["Waiting", String(counts.waiting ?? 0)],
+ ["Capacity", String(queue.capacity.length)],
+ ]);
+ } finally {
+ await runtime.close();
+ }
return;
}
if (args.command === "gc") {
@@ -457,12 +639,67 @@ export async function runCli(
throw new Error("selected Runtime does not expose Artifact maintenance");
}
const report = await runtime.garbageCollectArtifacts({ apply: args.apply });
- io.write(`${JSON.stringify({
+ const machine = {
applied: args.apply,
reachable: report.reachable,
unreachable: report.unreachable,
deleted: report.deleted,
- }, null, 2)}\n`);
+ };
+ writeOperational(machine, args.apply ? "Artifact garbage collection applied" : "Artifact garbage collection preview",
+ report.unreachable.length === 0 ? "success" : "warning", [
+ ["Reachable", String(report.reachable.length)],
+ ["Unreachable", String(report.unreachable.length)],
+ ["Deleted", String(report.deleted.length)],
+ ]);
+ } finally {
+ await runtime.close();
+ }
+ return;
+ }
+ if (args.command === "auth") {
+ if (args.action !== "status" && args.action !== "login" && args.action !== "logout") {
+ throw new Error("auth takes status, login or logout");
+ }
+ if (args.runtime === undefined) throw new Error("auth requires --runtime");
+ if (args.from !== undefined && args.action !== "login") throw new Error("--from applies only to auth login");
+ const runtime = await loadLocalRuntime(args.runtime, distribution);
+ try {
+ let credentials = await runtime.credentials(args.file!);
+ if (args.slot !== undefined) credentials = credentials.filter((item) => item.slot === args.slot);
+ if (credentials.length === 0) throw new Error(`Endpoint ${args.file} has no matching credential`);
+ if (args.slot === undefined && credentials.length > 1 && args.action !== "status") {
+ throw new Error(`Endpoint ${args.file} has several credentials; select one with --slot`);
+ }
+ if (args.action === "status") {
+ writeOperational({ endpoint: args.file, credentials }, "Credential status", "info", [
+ ["Endpoint", args.file!],
+ ["Configured", `${credentials.filter((item) => item.configured).length}/${credentials.length}`],
+ ], credentials.map((item) => `${item.slot}: ${item.configured ? "configured" : "missing"} · ${item.writable ? "writable" : "read-only"}`));
+ } else if (args.action === "login") {
+ const [item] = credentials;
+ if (item === undefined) throw new Error(`Endpoint ${args.file} has no matching credential`);
+ const raw = args.from === undefined
+ ? await io.readSecret?.(`${item.label}: `)
+ : await readFile(args.from, "utf8");
+ if (raw === undefined) throw new Error("interactive credential input is unavailable; use --from <file>");
+ const secret = raw.trim();
+ if (secret.length === 0) throw new Error("credential input is empty");
+ if (item.kind === "json") {
+ try { JSON.parse(secret); } catch { throw new Error(`${item.label} is not valid JSON`); }
+ }
+ const stored = await runtime.putCredential(item.endpoint, item.slot, secret);
+ writeOperational({ endpoint: args.file, stored }, "Credential stored", "success", [
+ ["Endpoint", args.file!], ["Slot", stored.slot], ["Store", stored.ref.store],
+ ]);
+ } else {
+ const [item] = credentials;
+ if (item === undefined) throw new Error(`Endpoint ${args.file} has no matching credential`);
+ const removed = await runtime.deleteCredential(item.endpoint, item.slot);
+ writeOperational({ endpoint: args.file, ...removed }, removed.deleted ? "Credential removed" : "Credential was absent",
+ removed.deleted ? "success" : "warning", [
+ ["Endpoint", args.file!], ["Slot", removed.credential.slot], ["Store", removed.credential.ref.store],
+ ]);
+ }
} finally {
await runtime.close();
}
@@ -481,11 +718,46 @@ export async function runCli(
throw new Error("Targets, Candidate selections and fidelity belong in a self-described Run Source; CLI --target, --pin and --accept-substitute are not supported");
}
if (args.command === "status" || args.command === "builds" || args.command === "inspect"
- || args.command === "get" || args.command === "cancel") {
+ || args.command === "get" || args.command === "cancel" || args.command === "queue"
+ || args.command === "operations" || args.command === "operation") {
if (args.runtime === undefined) throw new Error(`${args.command} requires --runtime`);
const runtime = await loadLocalRuntime(args.runtime, distribution);
try {
- if (args.command === "builds") {
+ if (args.command === "queue") {
+ const writeQueue = async (): Promise<boolean> => {
+ const queue = await runtime.queue();
+ const value = {
+ format: "narratage.cli-queue@1",
+ at: Date.now(),
+ dispatches: queue.dispatches,
+ capacity: queue.capacity,
+ };
+ const active = queue.dispatches.filter((item) => item.phase !== "terminal");
+ writeOperational(value, "Runtime queue", active.length === 0 ? "success" : "info", [
+ ["Active Builds", String(active.length)],
+ ["Total Builds", String(queue.dispatches.length)],
+ ["Reserved capacity", String(queue.capacity.length)],
+ ], active.slice(0, args.verbose ? undefined : 12).map((item) =>
+ `${item.build}: ${item.phase}${item.admission === "open" ? "" : ` · ${item.admission}`}`));
+ return queue.dispatches.every((item) => item.phase === "terminal");
+ };
+ if (!args.watch) await writeQueue();
+ else while (!(await writeQueue())) await new Promise((resolveWait) => setTimeout(resolveWait, 1_000));
+ } else if (args.command === "operations") {
+ const status = await runtime.status(args.file!);
+ const machine = { build: args.file, operations: status.operations };
+ writeOperational(machine, "Build operations", "info", [
+ ["Build", args.file!], ["Operations", String(status.operations.length)],
+ ], status.operations.map((item) => `${item.id}: ${item.status} · ${item.endpoint}`));
+ } else if (args.command === "operation") {
+ if (!isDigest(args.file!)) throw new Error("operation id must be a content digest");
+ const operation = await runtime.operation(args.file!);
+ writeOperational({ operation }, operation === undefined ? "Operation not found" : "Operation detail",
+ operation === undefined ? "warning" : "info", operation === undefined ? [] : [
+ ["Operation", operation.id], ["Status", operation.status], ["Endpoint", operation.endpoint],
+ ["Attempt", String(operation.attempt)],
+ ]);
+ } else if (args.command === "builds") {
const entries = await runtime.builds();
const builds = await Promise.all(entries.map(async (entry) => {
const status = await runtime.status(entry.build);
@@ -498,10 +770,11 @@ export async function runCli(
...summarizeBuildCatalog(entry, status.build?.state),
};
}));
- io.write(`${JSON.stringify({ builds }, null, 2)}\n`);
+ writeOperational({ builds }, "Build archive", "info", [["Builds", String(builds.length)]],
+ builds.slice(0, args.verbose ? undefined : 20).map((item) => `${item.build}: ${item.status ?? "unknown"}`));
} else if (args.command === "status") {
const status = await runtime.status(args.file!);
- io.write(`${JSON.stringify({
+ const machine = {
build: status.build === undefined ? undefined : {
id: status.build.build,
revision: status.build.revision,
@@ -520,12 +793,22 @@ export async function runCli(
status: operation.status,
...(operation.wakeAt === undefined ? {} : { wakeAt: operation.wakeAt }),
...(operation.failure === undefined ? {} : { failure: operation.failure }),
+ ...(operation.cancellation === undefined ? {} : { cancellation: operation.cancellation }),
})),
- }, null, 2)}\n`);
+ dispatch: status.dispatch,
+ };
+ const phase = status.dispatch?.phase ?? "missing";
+ writeOperational(machine, status.build === undefined ? "Build not found" : "Build status",
+ status.build === undefined ? "warning" : status.dispatch?.terminal === "failed" ? "error" : "info", [
+ ["Build", args.file!],
+ ["Core", status.build?.state.status ?? "missing"],
+ ["Dispatch", phase],
+ ["Operations", String(status.operations.length)],
+ ]);
} else if (args.command === "inspect") {
const status = await runtime.status(args.file!);
if (status.build === undefined) throw new Error(`Build ${args.file} does not exist`);
- io.write(`${JSON.stringify({
+ const machine = {
build: status.build.build,
revision: status.build.revision,
archive: inspectBuild(status.build.state, status.catalog),
@@ -538,7 +821,11 @@ export async function runCli(
...(operation.wakeAt === undefined ? {} : { wakeAt: operation.wakeAt }),
...(operation.failure === undefined ? {} : { failure: operation.failure }),
})),
- }, null, 2)}\n`);
+ };
+ writeOperational(machine, "Build archive detail", "info", [
+ ["Build", status.build.build], ["Revision", String(status.build.revision)],
+ ["Operations", String(status.operations.length)],
+ ]);
} else if (args.command === "get") {
const status = await runtime.status(args.file!);
if (status.build === undefined) throw new Error(`Build ${args.file} does not exist`);
@@ -549,17 +836,22 @@ export async function runCli(
const references = findArchivedArtifact(status.build.state, args.artifact);
if (references.length === 0) throw new Error(`Build ${args.file} does not reference Artifact ${args.artifact}`);
if (args.to === undefined) {
- io.write(`${JSON.stringify({ build: status.build.build, artifact: args.artifact, references }, null, 2)}\n`);
+ writeOperational({ build: status.build.build, artifact: args.artifact, references }, "Artifact references", "info", [
+ ["Build", status.build.build], ["Artifact", args.artifact], ["References", String(references.length)],
+ ]);
} else {
const [first] = references;
if (first === undefined) throw new Error(`Build ${args.file} does not reference Artifact ${args.artifact}`);
const materialized = await materializeArtifact(runtime, first, args.to, `Build ${args.file}`);
- io.write(`${JSON.stringify({
+ const machine = {
build: status.build.build,
artifact: args.artifact,
references,
materialized,
- }, null, 2)}\n`);
+ };
+ writeOperational(machine, "Artifact materialized", "success", [
+ ["Build", status.build.build], ["Artifact", args.artifact], ["Path", materialized.path],
+ ]);
}
return;
}
@@ -570,28 +862,41 @@ export async function runCli(
...(status.catalog === undefined ? {} : { catalog: status.catalog }),
});
if (args.to === undefined) {
- io.write(`${JSON.stringify({
+ const machine = {
build: status.build.build,
revision: status.build.revision,
record,
artifacts: collectArtifacts(record.value.kind === "blob" ? record.value : record.value.value),
- }, null, 2)}\n`);
+ };
+ writeOperational(machine, "Archived Record", "info", [
+ ["Build", status.build.build], ["Record", record.id],
+ ["Artifacts", String(machine.artifacts.length)],
+ ]);
} else {
const materialized = await materializeRecord(runtime, record, args.to);
- io.write(`${JSON.stringify({
+ const machine = {
build: status.build.build,
record: record.id,
materialized,
- }, null, 2)}\n`);
+ };
+ writeOperational(machine, "Record materialized", "success", [
+ ["Build", status.build.build], ["Record", record.id], ["Path", materialized.path],
+ ]);
}
} else {
- const result = await runtime.cancel(args.file!);
- io.write(`${JSON.stringify({
+ if (args.action !== "build") throw new Error("cancel must name its scope: cancel build <build-id>");
+ const result = await runtime.cancel(args.file!, args.reason);
+ const machine = {
build: args.file,
cancelled: result !== undefined,
- status: result?.status,
- diagnostics: result?.state.diagnostics ?? [],
- }, null, 2)}\n`);
+ phase: result?.phase,
+ admission: result?.admission,
+ terminal: result?.terminal,
+ };
+ writeOperational(machine, result === undefined ? "Build not found" : "Cancellation requested",
+ result === undefined ? "warning" : "success", [
+ ["Build", args.file!], ["Admission", result?.admission ?? "missing"], ["Phase", result?.phase ?? "missing"],
+ ]);
}
} finally {
await runtime.close();
@@ -695,7 +1000,6 @@ export async function runCli(
}
if (args.command === "build") {
if (args.runtime === undefined) throw new Error("build requires --runtime with a Runtime Profile or trusted local config module");
- const services = args.noServices ? undefined : await startDeclaredServices(args.runtime, distribution);
const runtime = await loadLocalRuntime(args.runtime, distribution);
try {
const loadedRun = await loadRunFile({
@@ -706,7 +1010,7 @@ export async function runCli(
runtime,
});
const result = loadedRun.compiler.planCompilation(loadedRun, loadedPackageSet?.lock.digest);
- const built = await runtime.build({
+ const request = {
id: args.buildId ?? result.state.id,
state: result.state,
catalog: createCatalogDescriptor({
@@ -719,14 +1023,25 @@ export async function runCli(
},
}),
attachments: result.compilation.attachments,
- }, {
- follow: args.follow,
- ...(args.maxWaitMs === undefined ? {} : { maxWaitMs: args.maxWaitMs }),
- });
- io.write(`${JSON.stringify({
+ } as const;
+ const services = args.noServices ? undefined : await startDeclaredServices(args.runtime, distribution);
+ let built = await runtime.build(request);
+ const worker = await ensureRuntimeProcess(
+ args.runtime,
+ distribution.runtimeWorkerLaunch(),
+ args.maxWaitMs ?? 10_000,
+ );
+ if (args.follow) {
+ built = await runtime.build(request, {
+ follow: true,
+ ...(args.maxWaitMs === undefined ? {} : { maxWaitMs: args.maxWaitMs }),
+ });
+ }
+ const machine = {
build: built.id,
core: built.state.id,
status: built.status,
+ worker,
...(services === undefined ? {} : {
services: services.map((item) => ({ id: item.id, action: item.action })),
}),
@@ -739,9 +1054,19 @@ export async function runCli(
...(record === undefined ? {} : { digest: record.digest, value: record.value }),
};
}),
- blocked: built.blocked,
- journal: built.journal,
- }, null, 2)}\n`);
+ dispatch: {
+ phase: built.dispatch.phase,
+ admission: built.dispatch.admission,
+ ...(built.dispatch.reason === undefined ? {} : { reason: built.dispatch.reason }),
+ },
+ };
+ writeOperational(machine, args.follow ? "Build observation finished" : "Build submitted",
+ built.status === "failed" ? "error" : built.status === "cancelled" ? "warning" : "success", [
+ ["Build", built.id],
+ ["Status", built.status],
+ ["Worker", String(worker.pid)],
+ ["Goals", String(machine.goals.length)],
+ ]);
} finally {
await runtime.close();
}
diff --git a/packages/cli/src/output.ts b/packages/cli/src/output.ts
index 725d6f62..581bb16a 100644
--- a/packages/cli/src/output.ts
+++ b/packages/cli/src/output.ts
@@ -11,6 +11,8 @@ export type CliTerminal = {
export type CliIo = {
readonly write: (text: string) => void;
+ /** Interactive secret input supplied by the concrete CLI shell; never echoed or logged. */
+ readonly readSecret?: (prompt: string) => Promise<string>;
readonly terminal?: CliTerminal;
};
@@ -18,6 +20,7 @@ export type CliColorMode = "auto" | "always" | "never";
export type CliOutputOptions = {
readonly json: boolean;
+ readonly jsonl?: boolean;
readonly color: CliColorMode;
readonly verbose: boolean;
};
@@ -95,6 +98,14 @@ export type CliPresentation =
readonly machine: BuildPlan;
readonly run: string;
readonly targetSet: string;
+ }
+ | {
+ readonly kind: "operational";
+ readonly machine: unknown;
+ readonly title: string;
+ readonly status?: "success" | "warning" | "error" | "info";
+ readonly facts?: readonly (readonly [string, string])[];
+ readonly lines?: readonly string[];
};
type Palette = {
@@ -318,13 +329,24 @@ function renderPlan(
return `${lines.join("\n")}\n`;
}
+function renderOperational(
+ view: Extract<CliPresentation, { kind: "operational" }>,
+ io: CliIo,
+ colors: Palette,
+): string {
+ const lines = [heading(view.status ?? "info", view.title, io, colors)];
+ if ((view.facts?.length ?? 0) > 0) lines.push("", ...facts(view.facts!, colors));
+ if ((view.lines?.length ?? 0) > 0) lines.push("", ...view.lines!);
+ return `${lines.join("\n")}\n`;
+}
+
export function writeCliOutput(
io: CliIo,
options: CliOutputOptions,
presentation: CliPresentation,
): void {
if (options.json) {
- io.write(`${JSON.stringify(presentation.machine, null, 2)}\n`);
+ io.write(`${JSON.stringify(presentation.machine, null, options.jsonl === true ? 0 : 2)}\n`);
return;
}
const colors = palette(colorEnabled(io, options.color));
@@ -334,7 +356,9 @@ export function writeCliOutput(
? renderAuthorCheck(presentation, io, colors, options.verbose)
: presentation.kind === "check-run"
? renderRunCheck(presentation, io, colors, options.verbose)
- : renderPlan(presentation, io, colors, options.verbose);
+ : presentation.kind === "plan"
+ ? renderPlan(presentation, io, colors, options.verbose)
+ : renderOperational(presentation, io, colors);
io.write(output);
}
@@ -354,11 +378,16 @@ export function writeCliHelp(io: CliIo): void {
" status <build-id> show Build and Operation status",
" inspect <build-id> inspect accepted Records and demanded outputs",
" get <build-id> read or materialize one archived result",
- " cancel <build-id> request cancellation (transitional command)",
+ " cancel build <build-id> close admission and request honest cancellation",
+ " operations <build-id> inspect external Operation facts",
+ " operation <digest> inspect one external Operation",
"",
colors.strong("Runtime"),
" doctor <profile> validate deployment without executing",
- " services up|status|down manage current local dependencies",
+ " runtime up|status|logs|down manage the durable Worker and dependencies",
+ " services up|status|down manage declared external programs only",
+ " queue [--watch] inspect durable dispatch and shared capacity",
+ " auth status|login|logout manage Endpoint-declared credential references",
" gc <profile> report unreachable Artifacts; --apply deletes",
"",
colors.strong("Packages"),
diff --git a/packages/cli/src/runtime-process.ts b/packages/cli/src/runtime-process.ts
new file mode 100644
index 00000000..aa3ba5da
--- /dev/null
+++ b/packages/cli/src/runtime-process.ts
@@ -0,0 +1,164 @@
+import { createHash } from "node:crypto";
+import { spawn } from "node:child_process";
+import { mkdir, open, readFile, stat, unlink, writeFile } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+
+export type RuntimeWorkerLaunch = {
+ readonly command: string;
+ readonly args: readonly string[];
+};
+
+export type RuntimeProcessState = {
+ readonly state: "running" | "stopped";
+ readonly profile: string;
+ readonly pid?: number;
+ readonly startedAt?: number;
+ readonly logPath: string;
+};
+
+type ProcessRecord = {
+ readonly format: "narratage.runtime-process@1";
+ readonly profile: string;
+ readonly pid: number;
+ readonly startedAt: number;
+};
+
+function paths(profile: string) {
+ const absolute = resolve(profile);
+ const id = createHash("sha256").update(absolute).digest("hex").slice(0, 16);
+ const root = join(dirname(absolute), ".svml", "runtime", id);
+ return {
+ root,
+ pid: join(root, "worker.json"),
+ ready: join(root, "ready"),
+ log: join(root, "worker.log"),
+ };
+}
+
+async function exists(path: string): Promise<boolean> {
+ try {
+ await stat(path);
+ return true;
+ } catch (error) {
+ if (error instanceof Error && "code" in error && error.code === "ENOENT") return false;
+ throw error;
+ }
+}
+
+function alive(pid: number): boolean {
+ try {
+ process.kill(pid, 0);
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+async function record(profile: string): Promise<ProcessRecord | undefined> {
+ const path = paths(profile).pid;
+ try {
+ const value = JSON.parse(await readFile(path, "utf8")) as ProcessRecord;
+ if (value.format !== "narratage.runtime-process@1" || value.profile !== resolve(profile)
+ || !Number.isSafeInteger(value.pid) || value.pid < 1) return undefined;
+ return value;
+ } catch (error) {
+ if (error instanceof Error && "code" in error && error.code === "ENOENT") return undefined;
+ return undefined;
+ }
+}
+
+export async function runtimeProcessStatus(profile: string): Promise<RuntimeProcessState> {
+ const location = paths(profile);
+ const current = await record(profile);
+ if (current === undefined || !alive(current.pid)) {
+ return { state: "stopped", profile: resolve(profile), logPath: location.log };
+ }
+ return {
+ state: "running",
+ profile: current.profile,
+ pid: current.pid,
+ startedAt: current.startedAt,
+ logPath: location.log,
+ };
+}
+
+async function waitForReady(profile: string, timeoutMs: number): Promise<RuntimeProcessState> {
+ const location = paths(profile);
+ const deadline = Date.now() + timeoutMs;
+ while (Date.now() <= deadline) {
+ const state = await runtimeProcessStatus(profile);
+ if (state.state === "stopped") {
+ const log = await readFile(location.log, "utf8").catch(() => "");
+ throw new Error(`Runtime Worker exited before becoming ready${log.length === 0 ? "" : `: ${log.trim().split("\n").at(-1)}`}`);
+ }
+ if (await exists(location.ready)) return state;
+ await new Promise((resolveWait) => setTimeout(resolveWait, 50));
+ }
+ throw new Error(`Runtime Worker did not become ready within ${timeoutMs}ms; log: ${location.log}`);
+}
+
+export async function ensureRuntimeProcess(
+ profile: string,
+ launch: RuntimeWorkerLaunch,
+ timeoutMs = 10_000,
+): Promise<RuntimeProcessState> {
+ const current = await runtimeProcessStatus(profile);
+ if (current.state === "running") return current;
+ const absolute = resolve(profile);
+ const location = paths(absolute);
+ await mkdir(location.root, { recursive: true });
+ await unlink(location.ready).catch(() => undefined);
+ const log = await open(location.log, "a");
+ const child = spawn(launch.command, [
+ ...launch.args,
+ "_worker",
+ absolute,
+ "--ready-file",
+ location.ready,
+ ], {
+ cwd: process.cwd(),
+ detached: true,
+ stdio: ["ignore", log.fd, log.fd],
+ env: process.env,
+ });
+ if (child.pid === undefined) throw new Error("Runtime Worker process has no pid");
+ const startedAt = Date.now();
+ await writeFile(location.pid, JSON.stringify({
+ format: "narratage.runtime-process@1",
+ profile: absolute,
+ pid: child.pid,
+ startedAt,
+ } satisfies ProcessRecord), "utf8");
+ child.unref();
+ await log.close();
+ return await waitForReady(absolute, timeoutMs);
+}
+
+export async function stopRuntimeProcess(profile: string, timeoutMs = 10_000): Promise<RuntimeProcessState> {
+ const current = await record(profile);
+ const location = paths(profile);
+ if (current === undefined || !alive(current.pid)) {
+ await unlink(location.pid).catch(() => undefined);
+ await unlink(location.ready).catch(() => undefined);
+ return { state: "stopped", profile: resolve(profile), logPath: location.log };
+ }
+ process.kill(current.pid, "SIGTERM");
+ const deadline = Date.now() + timeoutMs;
+ while (Date.now() <= deadline && alive(current.pid)) {
+ await new Promise((resolveWait) => setTimeout(resolveWait, 50));
+ }
+ if (alive(current.pid)) throw new Error(`Runtime Worker ${current.pid} did not stop within ${timeoutMs}ms`);
+ await unlink(location.pid).catch(() => undefined);
+ await unlink(location.ready).catch(() => undefined);
+ return { state: "stopped", profile: resolve(profile), logPath: location.log };
+}
+
+export async function runtimeProcessLogs(profile: string): Promise<{ readonly path: string; readonly text: string }> {
+ const path = paths(profile).log;
+ return { path, text: await readFile(path, "utf8").catch(() => "") };
+}
+
+export async function markRuntimeProcessReady(path: string): Promise<void> {
+ await mkdir(dirname(resolve(path)), { recursive: true });
+ await writeFile(resolve(path), `${process.pid}\n`, "utf8");
+}
diff --git a/packages/cli/test/build-services.test.ts b/packages/cli/test/build-services.test.ts
index 17436ccd..393ec290 100644
--- a/packages/cli/test/build-services.test.ts
+++ b/packages/cli/test/build-services.test.ts
@@ -46,7 +46,7 @@ async function runSource(): Promise<string> {
return path;
}
-test("a declared service that is not ready stops the Build before it spends anything", async () => {
+test("a Build that cannot construct its Runtime starts no declared external service", async () => {
const calls: string[] = [];
const source = await runSource();
await assert.rejects(
@@ -61,15 +61,9 @@ test("a declared service that is not ready stops the Build before it spends anyt
detail: "uv failed: no such project",
}]),
),
- // Both reasons reach the operator: what the probe saw, and why the attempt
- // to fix it fell short. Each names a different repair.
- (error: Error) => /1 external service is not ready/u.test(error.message)
- && /whisperx: down/u.test(error.message)
- && /nothing is answering/u.test(error.message)
- && /uv failed: no such project/u.test(error.message)
- && /--no-services/u.test(error.message),
+ /createRuntimeFromConfig/u,
);
- assert.deepEqual(calls, ["up /p/svml.runtime.json"]);
+ assert.deepEqual(calls, []);
});
test("--no-services leaves the declared programs alone", async () => {
@@ -93,25 +87,6 @@ test("--no-services leaves the declared programs alone", async () => {
assert.deepEqual(calls, []);
});
-test("a Build whose services are all ready proceeds to compile", async () => {
- const calls: string[] = [];
- const source = await runSource();
- await assert.rejects(
- async () => await runCli(
- ["build", source, "--runtime", "/p/svml.runtime.json"],
- io,
- distribution(calls, [{
- id: "whisperx",
- instances: ["whisperx.local"],
- action: "already-running",
- state: { state: "ready" },
- }]),
- ),
- /createRuntimeFromConfig/u,
- );
- assert.deepEqual(calls, ["up /p/svml.runtime.json"]);
-});
-
test("--no-services belongs to build, the only command that starts a program", async () => {
await assert.rejects(
async () => await runCli(
diff --git a/packages/cli/test/runtime-process.test.ts b/packages/cli/test/runtime-process.test.ts
new file mode 100644
index 00000000..20ecb79a
--- /dev/null
+++ b/packages/cli/test/runtime-process.test.ts
@@ -0,0 +1,43 @@
+import assert from "node:assert/strict";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+
+import {
+ ensureRuntimeProcess,
+ runtimeProcessLogs,
+ runtimeProcessStatus,
+ stopRuntimeProcess,
+} from "../src/runtime-process.js";
+
+test("one detached Runtime Worker is observable, reusable and explicitly stoppable", async () => {
+ const root = await mkdtemp(join(tmpdir(), "narratage-runtime-process-"));
+ const profile = join(root, "runtime.json");
+ await writeFile(profile, "{}\n", "utf8");
+ const program = `
+ const fs = require("node:fs");
+ const path = require("node:path");
+ const index = process.argv.indexOf("--ready-file");
+ const ready = process.argv[index + 1];
+ fs.mkdirSync(path.dirname(ready), { recursive: true });
+ fs.writeFileSync(ready, String(process.pid));
+ process.stdout.write("worker-ready\\n");
+ process.on("SIGTERM", () => process.exit(0));
+ setInterval(() => {}, 1000);
+ `;
+ try {
+ const first = await ensureRuntimeProcess(profile, { command: process.execPath, args: ["-e", program] }, 5_000);
+ assert.equal(first.state, "running");
+ assert.ok(first.pid);
+ const second = await ensureRuntimeProcess(profile, { command: "must-not-run", args: [] }, 5_000);
+ assert.equal(second.pid, first.pid);
+ assert.equal((await runtimeProcessStatus(profile)).state, "running");
+ assert.match((await runtimeProcessLogs(profile)).text, /worker-ready/u);
+ assert.equal((await stopRuntimeProcess(profile, 5_000)).state, "stopped");
+ assert.equal((await runtimeProcessStatus(profile)).state, "stopped");
+ } finally {
+ await stopRuntimeProcess(profile, 1_000).catch(() => undefined);
+ await rm(root, { recursive: true, force: true });
+ }
+});
diff --git a/packages/credential-store-env/package.json b/packages/credential-store-env/package.json
index ccdf8974..19ec99c9 100644
--- a/packages/credential-store-env/package.json
+++ b/packages/credential-store-env/package.json
@@ -6,8 +6,12 @@
"exports": {
".": "./src/index.ts"
},
+ "svml": {
+ "activation": "./src/activation.ts"
+ },
"dependencies": {
"@narratage/protocol": "workspace:*",
- "@narratage/runtime": "workspace:*"
+ "@narratage/runtime": "workspace:*",
+ "@narratage/runtime-adapter": "workspace:*"
}
}
diff --git a/packages/credential-store-env/src/activation.ts b/packages/credential-store-env/src/activation.ts
new file mode 100644
index 00000000..8a1f0d77
--- /dev/null
+++ b/packages/credential-store-env/src/activation.ts
@@ -0,0 +1,26 @@
+import {
+ createRuntimeServiceAdapterFacet,
+ runtimeConfigExact,
+ runtimeConfigObject,
+} from "@narratage/runtime-adapter";
+
+import { createEnvironmentCredentialStorePackage } from "./index.js";
+
+const environmentCredentialStoreAdapter = createRuntimeServiceAdapterFacet({
+ use: "@narratage/credential-store-env",
+ validate(context) {
+ const config = runtimeConfigObject(context.config, "environment CredentialStore");
+ runtimeConfigExact(config, [], "environment CredentialStore");
+ },
+ create(context) {
+ return createEnvironmentCredentialStorePackage({ instance: context.instance });
+ },
+});
+
+export const svmlPackage = {
+ format: "svml.node-package@1" as const,
+ name: "@narratage/credential-store-env",
+ hostFacets: [environmentCredentialStoreAdapter],
+};
+
+export default svmlPackage;
diff --git a/packages/credential-store-keychain/src/index.ts b/packages/credential-store-keychain/src/index.ts
index 396c1bb6..d602d4fe 100644
--- a/packages/credential-store-keychain/src/index.ts
+++ b/packages/credential-store-keychain/src/index.ts
@@ -7,5 +7,7 @@ export {
} from "./store.js";
export type {
CreateKeychainCredentialStorePackageOptions,
+ KeychainDeleter,
KeychainReader,
+ KeychainWriter,
} from "./store.js";
diff --git a/packages/credential-store-keychain/src/store.ts b/packages/credential-store-keychain/src/store.ts
index eba471db..8e52196b 100644
--- a/packages/credential-store-keychain/src/store.ts
+++ b/packages/credential-store-keychain/src/store.ts
@@ -4,9 +4,9 @@ import { digestOf } from "@narratage/protocol";
import { defineRuntimeServicePackage, verifyCredentialRef } from "@narratage/runtime";
import type {
CredentialRef,
- CredentialStore,
CredentialValue,
RuntimeServicePackage,
+ WritableCredentialStore,
} from "@narratage/runtime";
export const keychainCredentialStoreModuleRef = {
@@ -20,12 +20,16 @@ export const keychainCredentialStoreImplementationDigest = digestOf(
/** Reads one named secret. Substituted in tests so no test touches a real keychain. */
export type KeychainReader = (service: string, account: string) => Promise<string | undefined>;
+export type KeychainWriter = (service: string, account: string, secret: string) => Promise<void>;
+export type KeychainDeleter = (service: string, account: string) => Promise<boolean>;
export type CreateKeychainCredentialStorePackageOptions = {
readonly instance?: string;
/** Keychain service name every lookup is scoped to. */
readonly service?: string;
readonly read?: KeychainReader;
+ readonly write?: KeychainWriter;
+ readonly remove?: KeychainDeleter;
};
const DEFAULT_SERVICE = "narratage";
@@ -54,6 +58,28 @@ function macosSecurity(service: string): KeychainReader {
});
}
+function macosWriter(service: string): KeychainWriter {
+ return async (_service, account, secret) => await new Promise((resolve, reject) => {
+ execFile("/usr/bin/security",
+ ["add-generic-password", "-U", "-s", service, "-a", account, "-w", secret],
+ { timeout: 10_000, shell: false },
+ (error) => error === null ? resolve() : reject(new Error(`keychain write for ${account} failed`)));
+ });
+}
+
+function macosDeleter(service: string): KeychainDeleter {
+ return async (_service, account) => await new Promise((resolve, reject) => {
+ execFile("/usr/bin/security",
+ ["delete-generic-password", "-s", service, "-a", account],
+ { timeout: 10_000, shell: false },
+ (error) => {
+ if (error === null) resolve(true);
+ else if ((error as { code?: number }).code === 44) resolve(false);
+ else reject(new Error(`keychain delete for ${account} failed`));
+ });
+ });
+}
+
/**
* A CredentialStore backed by the operating system keychain.
*
@@ -61,13 +87,26 @@ function macosSecurity(service: string): KeychainReader {
* environment store answers only for `store: "env"`. Anything else is not its
* business and it says so by returning undefined rather than guessing.
*/
-export class KeychainCredentialStore implements CredentialStore {
+export class KeychainCredentialStore implements WritableCredentialStore {
readonly #read: KeychainReader;
+ readonly #write: KeychainWriter;
+ readonly #remove: KeychainDeleter;
readonly #service: string;
- constructor(options: { readonly service?: string; readonly read?: KeychainReader } = {}) {
+ constructor(options: {
+ readonly service?: string;
+ readonly read?: KeychainReader;
+ readonly write?: KeychainWriter;
+ readonly remove?: KeychainDeleter;
+ } = {}) {
this.#service = options.service ?? DEFAULT_SERVICE;
this.#read = options.read ?? macosSecurity(this.#service);
+ this.#write = options.write ?? macosWriter(this.#service);
+ this.#remove = options.remove ?? macosDeleter(this.#service);
+ }
+
+ owns(ref: CredentialRef): boolean {
+ return ref.store === "keychain";
}
async resolve(ref: CredentialRef): Promise<CredentialValue | undefined> {
@@ -77,6 +116,19 @@ export class KeychainCredentialStore implements CredentialStore {
if (secret === undefined || secret.length === 0) return undefined;
return { secret };
}
+
+ async put(ref: CredentialRef, value: CredentialValue): Promise<void> {
+ verifyCredentialRef(ref);
+ if (!this.owns(ref)) throw new Error(`keychain CredentialStore does not own ${ref.store}`);
+ if (value.secret.length === 0) throw new Error("credential secret is empty");
+ await this.#write(this.#service, ref.key, value.secret);
+ }
+
+ async delete(ref: CredentialRef): Promise<boolean> {
+ verifyCredentialRef(ref);
+ if (!this.owns(ref)) throw new Error(`keychain CredentialStore does not own ${ref.store}`);
+ return await this.#remove(this.#service, ref.key);
+ }
}
/** The command that puts a secret where this store will find it. */
@@ -105,6 +157,8 @@ export function createKeychainCredentialStorePackage(
service: new KeychainCredentialStore({
service,
...(options.read === undefined ? {} : { read: options.read }),
+ ...(options.write === undefined ? {} : { write: options.write }),
+ ...(options.remove === undefined ? {} : { remove: options.remove }),
}),
}],
});
diff --git a/packages/credential-store-keychain/test/store.test.ts b/packages/credential-store-keychain/test/store.test.ts
index d26a414d..04478c9b 100644
--- a/packages/credential-store-keychain/test/store.test.ts
+++ b/packages/credential-store-keychain/test/store.test.ts
@@ -54,6 +54,22 @@ test("it can say how to put a secret where it will be found", () => {
assert.equal(keychainAddCommand("KIE_API_KEY"), "security add-generic-password -s narratage -a KIE_API_KEY -w");
});
+test("writable facet stores and removes only keychain-owned references", async () => {
+ const values = new Map<string, string>();
+ const store = new KeychainCredentialStore({
+ read: async (_service, account) => values.get(account),
+ write: async (_service, account, secret) => { values.set(account, secret); },
+ remove: async (_service, account) => values.delete(account),
+ });
+ const ref = credentialRef("keychain", "provider.api-key");
+ assert.equal(store.owns(ref), true);
+ await store.put(ref, { secret: "new-secret" });
+ assert.deepEqual(await store.resolve(ref), { secret: "new-secret" });
+ assert.equal(await store.delete(ref), true);
+ assert.equal(await store.resolve(ref), undefined);
+ await assert.rejects(store.put(credentialRef("env", "PROVIDER_KEY"), { secret: "x" }), /does not own/u);
+});
+
/**
* The reason a second implementation was written.
*
diff --git a/packages/driver-node/src/driver.ts b/packages/driver-node/src/driver.ts
index 141f2372..3656cc75 100644
--- a/packages/driver-node/src/driver.ts
+++ b/packages/driver-node/src/driver.ts
@@ -16,10 +16,12 @@ import {
} from "@narratage/validation";
import type { TypeValidatorRegistryLike } from "@narratage/validation";
import {
+ operationCancellationRequestId,
sealOperationIdentity,
verifyOperationSnapshot,
} from "@narratage/runtime";
import type {
+ OperationCancellationControl,
OperationIdentity,
CredentialStore,
CredentialValue,
@@ -305,6 +307,18 @@ export class NodeDriver {
event: { ...content, id: `event:${digestOf(content)}` },
};
}
+ if (snapshot.status === "cancelled") {
+ const content = {
+ kind: "command-failed",
+ command: executable.command.id,
+ code: "CANCELLED",
+ message: `Operation ${snapshot.id} was cancelled by the Runtime authority`,
+ } as const;
+ return {
+ status: "completed",
+ event: { ...content, id: `event:${digestOf(content)}` },
+ };
+ }
return {
status: "pending",
operation: snapshot.id,
@@ -360,7 +374,7 @@ export class NodeDriver {
});
this.#assertOperationHistory(history, base);
let latest = history.at(-1);
- if (latest?.status === "completed") {
+ if (latest?.status === "completed" || latest?.status === "cancelled") {
return await this.#completedOperation(state, executable, latest, latest.id, maxAttempts);
}
if (latest?.status === "pending" && latest.wakeAt !== undefined && latest.wakeAt > Date.now()) {
@@ -384,7 +398,7 @@ export class NodeDriver {
if (current.id !== identity.id) {
throw new Error(`OperationStore returned ${current.id} for ${identity.id}`);
}
- if (current.status === "completed" || current.status === "failed") {
+ if (current.status === "completed" || current.status === "failed" || current.status === "cancelled") {
return await this.#completedOperation(state, executable, current, identity.id, maxAttempts);
}
const endpointContext = {
@@ -510,6 +524,10 @@ export class NodeDriver {
command: executable.command,
lane: executable.lane,
maxConcurrency: executable.maxConcurrency,
+ capacityMode: "endpointId" in executable && executable.registration.kind === "recoverable"
+ ? "recoverable"
+ : "active",
+ maxInFlight: executable.maxConcurrency,
}]),
blocked: classifications
.map((item) => item.blocked)
@@ -532,11 +550,39 @@ export class NodeDriver {
}
/** Cancel one journaled external Operation without trusting serialized Command content. */
- async cancelOperation(initial: BuildState, operation: OperationSnapshot): Promise<OperationSnapshot> {
+ async cancelOperation(
+ initial: BuildState,
+ operation: OperationSnapshot,
+ requestedAt: number,
+ ): Promise<OperationSnapshot> {
verifyOperationSnapshot(operation);
- if (operation.status === "completed" || operation.status === "failed") return operation;
const operations = this.operations;
if (operations === undefined) throw new Error("cancelling an Operation requires OperationStore");
+ const requestId = operationCancellationRequestId(operation.id, requestedAt);
+ const write = async (current: OperationSnapshot, update: import("@narratage/runtime").OperationUpdate) => {
+ const result = await operations.compareAndSwap(current.id, current.revision, update);
+ return result.status === "stored" ? result.snapshot : result.current;
+ };
+ let current = await operations.read(operation.id) ?? operation;
+ if (current.status === "completed" || current.status === "failed" || current.status === "cancelled") {
+ if (current.cancellation === undefined) {
+ current = await write(current, {
+ status: "control",
+ cancellation: { requestedAt, requestId, status: "too-late", attempts: 0 },
+ });
+ }
+ return current;
+ }
+ if (current.cancellation === undefined) {
+ current = await write(current, {
+ status: "control",
+ cancellation: { requestedAt, requestId, status: "requested", attempts: 0 },
+ });
+ }
+ const control = current.cancellation;
+ if (control === undefined || control.requestId !== requestId) {
+ throw new Error(`Operation ${operation.id} already has another cancellation request`);
+ }
const prepared = this.prepare(initial);
const descriptor = prepared.runnable.find((item) => item.command.id === operation.command);
if (descriptor === undefined) throw new Error(`Operation ${operation.id} Command is not currently runnable`);
@@ -547,9 +593,6 @@ export class NodeDriver {
|| executable.registration.kind !== "recoverable") {
throw new Error(`Operation ${operation.id} does not match the regenerated Endpoint Command`);
}
- if (executable.registration.endpoint.cancel === undefined) {
- throw new Error(`Endpoint ${executable.endpointId} does not support cancellation`);
- }
const identity: OperationIdentity = {
format: operation.format,
id: operation.id,
@@ -562,23 +605,68 @@ export class NodeDriver {
attempt: operation.attempt,
submissionKey: operation.submissionKey,
};
- await executable.registration.endpoint.cancel({
+ const endpointContext = {
command: structuredClone(executable.command),
need: structuredClone(executable.command.need),
artifacts: this.artifacts,
credentials: await this.#endpointCredentials(executable.registration),
operation: identity,
- checkpoint: operation.status === "pending" ? structuredClone(operation.checkpoint) : undefined,
- });
- const written = await operations.compareAndSwap(operation.id, operation.revision, {
- status: "failed",
- failure: {
- code: "CANCELLED",
- message: `Operation ${operation.id} was cancelled by the Runtime authority`,
- retryable: false,
- },
+ checkpoint: current.status === "pending" ? structuredClone(current.checkpoint) : undefined,
+ };
+ if (control.status === "requested") {
+ if (executable.registration.endpoint.cancel === undefined) {
+ current = await write(current, {
+ status: "control",
+ cancellation: { ...control, status: "unsupported", attempts: control.attempts + 1 },
+ });
+ } else if (control.retryAt === undefined || control.retryAt <= Date.now()) {
+ try {
+ const outcome = await executable.registration.endpoint.cancel(endpointContext);
+ const nextControl: OperationCancellationControl = {
+ requestedAt,
+ requestId,
+ status: outcome.status,
+ attempts: control.attempts + 1,
+ ...(outcome.status === "accepted" && outcome.wakeAt !== undefined ? { retryAt: outcome.wakeAt } : {}),
+ };
+ current = outcome.status === "confirmed"
+ ? await write(current, { status: "cancelled", cancellation: nextControl })
+ : await write(current, { status: "control", cancellation: nextControl });
+ } catch (error) {
+ current = await write(current, {
+ status: "control",
+ cancellation: {
+ ...control,
+ attempts: control.attempts + 1,
+ retryAt: Date.now() + 1_000,
+ lastError: {
+ code: "CANCEL_REQUEST_FAILED",
+ message: error instanceof Error ? error.message : String(error),
+ },
+ },
+ });
+ }
+ }
+ }
+ if (current.status === "cancelled" || current.cancellation?.status === "requested") return current;
+
+ // Acknowledgement is not a terminal fact. Reconcile the same remote submission until it
+ // either confirms cancellation through `cancel`, or naturally completes/fails.
+ const observed = await executable.registration.endpoint.resume({
+ ...endpointContext,
+ checkpoint: current.status === "pending" ? structuredClone(current.checkpoint) : undefined,
});
- return written.status === "stored" ? written.snapshot : written.current;
+ if (observed.status === "pending") {
+ return await write(current, {
+ status: "pending",
+ checkpoint: observed.checkpoint,
+ ...(observed.wakeAt === undefined ? {} : { wakeAt: observed.wakeAt }),
+ });
+ }
+ if (observed.status === "failed") {
+ return await write(current, { status: "failed", failure: observed.failure });
+ }
+ return await write(current, { status: "completed", completion: observed.result });
}
async run(initial: BuildState, context?: RuntimeExecutionContext): Promise<DriverRunResult> {
diff --git a/packages/driver-node/test/scheduler.test.ts b/packages/driver-node/test/scheduler.test.ts
index f6f0a927..d199d2f4 100644
--- a/packages/driver-node/test/scheduler.test.ts
+++ b/packages/driver-node/test/scheduler.test.ts
@@ -216,6 +216,22 @@ function resolvedRuntime(laneLimit: number, lifecycle: "immediate" | "recoverabl
},
permissions: [],
},
+ ...([
+ ["worker", "worker"],
+ ["builds", "build-store"],
+ ["dispatch", "dispatch-store"],
+ ["journal", "runtime-journal"],
+ ["artifacts", "artifact-store"],
+ ["credentials", "credential-store"],
+ ] as const).map(([name, role]) => ({
+ name,
+ role,
+ implementation: {
+ locator: `example.scheduler-runtime/${name}`,
+ digest: digestOf(`example.scheduler-runtime/${name}@1`),
+ },
+ permissions: [],
+ })),
{
name: providerFacet.name,
role: "capability-endpoint",
@@ -236,11 +252,25 @@ function resolvedRuntime(laneLimit: number, lifecycle: "immediate" | "recoverabl
name: "scheduler-test",
instances: [
{ id: "scheduler.local", facet: { module: runtimeModule, name: "scheduler" } },
+ { id: "worker.local", facet: { module: runtimeModule, name: "worker" } },
+ { id: "builds.memory", facet: { module: runtimeModule, name: "builds" } },
{ id: "operations.memory", facet: { module: runtimeModule, name: "operations" } },
+ { id: "dispatch.memory", facet: { module: runtimeModule, name: "dispatch" } },
+ { id: "journal.memory", facet: { module: runtimeModule, name: "journal" } },
+ { id: "artifacts.memory", facet: { module: runtimeModule, name: "artifacts" } },
+ { id: "credentials.memory", facet: { module: runtimeModule, name: "credentials" } },
{ id: "generation.local", facet: providerFacet, lane: "endpoint:generation.local" },
],
scheduler: "scheduler.local",
- stores: { operations: "operations.memory" },
+ worker: "worker.local",
+ stores: {
+ build: "builds.memory",
+ operations: "operations.memory",
+ dispatch: "dispatch.memory",
+ journal: "journal.memory",
+ artifacts: "artifacts.memory",
+ credentials: ["credentials.memory"],
+ },
endpoints: [{
capability: capabilities.generation,
returns: types.generated,
@@ -755,6 +785,7 @@ test("wakeAt prevents early polling and Runtime cancellation becomes a terminal
cancel({ checkpoint }) {
cancels += 1;
assert.deepEqual(checkpoint, { remoteJob: "job-wait" });
+ return { status: "confirmed" };
},
};
const executor = recoverableExecutor(endpoint, operations, runtime).executor;
@@ -769,9 +800,66 @@ test("wakeAt prevents early polling and Runtime cancellation becomes a terminal
const operation = await operations.read(operationId);
assert.ok(operation);
- await executor.cancelOperation(early!.state, operation);
+ await executor.cancelOperation(early!.state, operation, Date.now());
assert.equal(cancels, 1);
const [cancelled] = await scheduler.run([{ id: "cancel-video", state: early!.state }]);
assert.equal(cancelled?.status, "failed");
assert.equal(cancelled?.state.diagnostics.at(-1)?.code, "CANCELLED");
});
+
+test("cancellation acknowledgement never invents a remote terminal fact", async () => {
+ const operations = new MemoryOperationStore();
+ const runtime = resolvedRuntime(1, "recoverable");
+ const endpoint: RecoverableEndpoint = {
+ start() {
+ return { status: "pending", checkpoint: { remoteJob: "still-running" } };
+ },
+ resume({ checkpoint }) {
+ return { status: "pending", checkpoint: checkpoint ?? { remoteJob: "still-running" } };
+ },
+ cancel() {
+ return { status: "accepted" };
+ },
+ };
+ const executor = recoverableExecutor(endpoint, operations, runtime).executor;
+ const scheduler = new LocalBuildScheduler(executor, localSchedulerOptionsFromClosure(runtime.closure));
+ const [first] = await scheduler.run([{ id: "accepted-cancel", state: createGreetingBuild() }]);
+ const operationId = first?.journal.find((item) => item.status === "pending")?.operation;
+ assert.ok(operationId);
+ const operation = await operations.read(operationId);
+ assert.ok(operation);
+ const controlled = await executor.cancelOperation(first!.state, operation, 123);
+ assert.equal(controlled.status, "pending");
+ assert.equal(controlled.cancellation?.status, "accepted");
+});
+
+test("an Endpoint without cancellation stays observable instead of changing Candidate or fact", async () => {
+ const operations = new MemoryOperationStore();
+ const runtime = resolvedRuntime(1, "recoverable");
+ const endpoint: RecoverableEndpoint = {
+ start() {
+ return { status: "pending", checkpoint: { remoteJob: "cannot-cancel" } };
+ },
+ resume() {
+ return {
+ status: "completed",
+ result: {
+ value: { kind: "inline", value: "late but real" },
+ conformance: "exact",
+ delivery: "executed",
+ metadata: {},
+ },
+ };
+ },
+ };
+ const executor = recoverableExecutor(endpoint, operations, runtime).executor;
+ const scheduler = new LocalBuildScheduler(executor, localSchedulerOptionsFromClosure(runtime.closure));
+ const [first] = await scheduler.run([{ id: "unsupported-cancel", state: createGreetingBuild() }]);
+ const operationId = first?.journal.find((item) => item.status === "pending")?.operation;
+ assert.ok(operationId);
+ const operation = await operations.read(operationId);
+ assert.ok(operation);
+ const observed = await executor.cancelOperation(first!.state, operation, 456);
+ assert.equal(observed.status, "completed");
+ assert.equal(observed.cancellation?.status, "unsupported");
+});
diff --git a/packages/endpoint-kit/src/index.ts b/packages/endpoint-kit/src/index.ts
index 571219d0..acdf34fd 100644
--- a/packages/endpoint-kit/src/index.ts
+++ b/packages/endpoint-kit/src/index.ts
@@ -63,10 +63,21 @@ export type EndpointResumeContext = EndpointStartContext & {
readonly checkpoint: CanonicalValue | undefined;
};
+/**
+ * Provider-side acknowledgement is deliberately separate from the Operation's final state.
+ * `accepted` means the remote system accepted a request but may still finish normally; only
+ * `confirmed` proves that the work itself is cancelled.
+ */
+export type EndpointCancelOutcome =
+ | { readonly status: "confirmed" }
+ | { readonly status: "accepted"; readonly wakeAt?: number }
+ | { readonly status: "unsupported" }
+ | { readonly status: "too-late" };
+
export type RecoverableEndpoint = {
start(context: EndpointStartContext): Awaitable<EndpointOutcome>;
resume(context: EndpointResumeContext): Awaitable<EndpointOutcome>;
- cancel?(context: EndpointResumeContext): Awaitable<void>;
+ cancel?(context: EndpointResumeContext): Awaitable<EndpointCancelOutcome>;
};
/** Endpoint/implementation scheduling metadata; it never changes Core demand or command identity. */
@@ -118,9 +129,19 @@ export type EndpointPackage = {
readonly manifest: RuntimeModuleManifest;
readonly instance: RuntimeProfileInstance;
readonly bindings: readonly RuntimeEndpointBinding[];
+ /** Host-facing login material declared by this exact configured Endpoint instance. */
+ readonly credentials: readonly EndpointCredentialDescription[];
install(registry: EndpointRegistrar): Awaitable<void>;
};
+export type EndpointCredentialDescription = {
+ readonly endpoint: string;
+ readonly slot: string;
+ readonly label: string;
+ readonly kind: "secret" | "json";
+ readonly ref: CredentialRef;
+};
+
type EndpointCapabilityBase = {
readonly capability: CapabilityRef;
readonly returns: TypeRef;
@@ -153,6 +174,10 @@ export type DefineEndpointPackageOptions = {
/** Non-secret deployment facts such as base URL, region and credential references. */
readonly configuration?: CanonicalValue;
readonly credentials?: Readonly<Record<string, CredentialRef>>;
+ readonly credentialInputs?: Readonly<Record<string, {
+ readonly label: string;
+ readonly kind?: "secret" | "json";
+ }>>;
readonly defaultConcurrency?: number;
readonly capabilities: readonly EndpointCapability[];
};
@@ -194,6 +219,24 @@ export function defineEndpointPackage(options: DefineEndpointPackageOptions): En
verifyCredentialRef(ref);
return [slot, structuredClone(ref)];
}));
+ const credentialInputs = options.credentialInputs ?? {};
+ for (const slot of Object.keys(credentialInputs)) {
+ assert(slot in credentials, `Endpoint credential input ${slot} has no configured CredentialRef`);
+ }
+ const credentialDescriptions = Object.entries(credentials).map(([slot, ref]) => {
+ const input = credentialInputs[slot];
+ const label = input?.label ?? slot;
+ assert(label.trim().length > 0, `Endpoint credential ${slot} label is empty`);
+ const kind = input?.kind ?? "secret";
+ assert(kind === "secret" || kind === "json", `Endpoint credential ${slot} kind is invalid`);
+ return {
+ endpoint: options.instance,
+ slot,
+ label,
+ kind,
+ ref: structuredClone(ref),
+ } satisfies EndpointCredentialDescription;
+ });
const configuration = canonicalize(options.configuration ?? null);
const executionPolicy = options.capabilities.map((capability) => ({
capability: refKey(capability.capability),
@@ -239,6 +282,7 @@ export function defineEndpointPackage(options: DefineEndpointPackageOptions): En
manifest,
instance,
bindings,
+ credentials: credentialDescriptions,
install(registry) {
for (const capability of options.capabilities) {
const common: EndpointRegistrationOptions = {
diff --git a/packages/local/package.json b/packages/local/package.json
index 16401079..ae6a4345 100644
--- a/packages/local/package.json
+++ b/packages/local/package.json
@@ -8,10 +8,11 @@
"./config": "./src/config.ts",
"./external-services": "./src/external-services.ts"
},
+ "svml": {
+ "activation": "./src/activation.ts"
+ },
"dependencies": {
- "@narratage/artifact-store-fs": "workspace:*",
"@narratage/component-kit": "workspace:*",
- "@narratage/credential-store-env": "workspace:*",
"@narratage/driver-node": "workspace:*",
"@narratage/endpoint-kit": "workspace:*",
"@narratage/host": "workspace:*",
@@ -19,10 +20,12 @@
"@narratage/protocol": "workspace:*",
"@narratage/runtime": "workspace:*",
"@narratage/runtime-adapter": "workspace:*",
- "@narratage/store-sqlite": "workspace:*",
"@narratage/validation": "workspace:*"
},
"devDependencies": {
- "@narratage/core": "workspace:*"
+ "@narratage/artifact-store-fs": "workspace:*",
+ "@narratage/credential-store-env": "workspace:*",
+ "@narratage/core": "workspace:*",
+ "@narratage/store-sqlite": "workspace:*"
}
}
diff --git a/packages/local/src/activation.ts b/packages/local/src/activation.ts
new file mode 100644
index 00000000..2dadaafe
--- /dev/null
+++ b/packages/local/src/activation.ts
@@ -0,0 +1,26 @@
+import {
+ createRuntimeServiceAdapterFacet,
+ runtimeConfigExact,
+ runtimeConfigObject,
+} from "@narratage/runtime-adapter";
+
+import { createLocalExecutionPackage } from "./scheduler-package.js";
+
+const localExecutionAdapter = createRuntimeServiceAdapterFacet({
+ use: "@narratage/local",
+ validate(context) {
+ const config = runtimeConfigObject(context.config, "local execution");
+ runtimeConfigExact(config, [], "local execution");
+ },
+ create(context) {
+ return createLocalExecutionPackage(context.instance);
+ },
+});
+
+export const svmlPackage = {
+ format: "svml.node-package@1" as const,
+ name: "@narratage/local",
+ hostFacets: [localExecutionAdapter],
+};
+
+export default svmlPackage;
diff --git a/packages/local/src/config.ts b/packages/local/src/config.ts
index f4cf08db..dea60ae0 100644
--- a/packages/local/src/config.ts
+++ b/packages/local/src/config.ts
@@ -14,6 +14,7 @@ import type {
RuntimeExternalService,
RuntimeServiceState,
} from "@narratage/runtime-adapter";
+import type { RuntimeServiceSelection } from "@narratage/runtime";
import type { LocalRuntime } from "./types.js";
@@ -30,18 +31,16 @@ export type RuntimeConfigDocument = {
readonly root?: string;
/** Resolved relative to the configuration file. Locates installed Node packages, not Runtime data. */
readonly packageRoot?: string;
- readonly statePath?: string;
- readonly catalogPath?: string;
- readonly artifactPath?: string;
readonly packageLock?: string;
/** Locked physical packages allowed to configure privileged Runtime adapters. */
readonly runtimePackageLock?: string;
/** Replaceable parts of the Runtime itself. One package may fill several roles. */
readonly runtimeServices: readonly RuntimeConfigEntry[];
+ readonly services: RuntimeServiceSelection;
readonly endpoints: readonly RuntimeConfigEntry[];
readonly permissions: readonly string[];
- readonly scheduling?: {
- readonly maxConcurrency?: number;
+ readonly scheduling: {
+ readonly maxConcurrency: number;
readonly lanes?: Readonly<Record<string, number>>;
readonly maxEventsPerBuild?: number;
};
@@ -97,7 +96,7 @@ function entry(value: unknown, subject: string, laneAllowed: boolean): RuntimeCo
function scheduling(value: unknown): RuntimeConfigDocument["scheduling"] {
- if (value === undefined) return undefined;
+ if (value === undefined) throw new Error("$runtime.scheduling is required");
const item = object(value, "$runtime.scheduling");
exactKeys(item, ["maxConcurrency", "lanes", "maxEventsPerBuild"], "$runtime.scheduling");
const lanes = item.lanes === undefined ? undefined : object(item.lanes, "$runtime.scheduling.lanes");
@@ -105,10 +104,10 @@ function scheduling(value: unknown): RuntimeConfigDocument["scheduling"] {
if (name.trim().length === 0) throw new Error("Runtime lane name must not be empty");
return [name, positiveInteger(limit, `$runtime.scheduling.lanes.${name}`)!];
}));
+ const maxConcurrency = positiveInteger(item.maxConcurrency, "$runtime.scheduling.maxConcurrency");
+ if (maxConcurrency === undefined) throw new Error("$runtime.scheduling.maxConcurrency is required");
return {
- ...(positiveInteger(item.maxConcurrency, "$runtime.scheduling.maxConcurrency") === undefined
- ? {}
- : { maxConcurrency: item.maxConcurrency as number }),
+ maxConcurrency,
...(normalizedLanes === undefined ? {} : { lanes: normalizedLanes }),
...(positiveInteger(item.maxEventsPerBuild, "$runtime.scheduling.maxEventsPerBuild") === undefined
? {}
@@ -116,18 +115,39 @@ function scheduling(value: unknown): RuntimeConfigDocument["scheduling"] {
};
}
+function serviceSelection(value: unknown): RuntimeServiceSelection {
+ const item = object(value, "$runtime.services");
+ exactKeys(item, ["scheduler", "worker", "stores"], "$runtime.services");
+ const stores = object(item.stores, "$runtime.services.stores");
+ exactKeys(stores, ["build", "operations", "dispatch", "journal", "artifacts", "credentials"],
+ "$runtime.services.stores");
+ const credentials = stringList(stores.credentials, "$runtime.services.stores.credentials");
+ if (credentials.length === 0) throw new Error("$runtime.services.stores.credentials must select at least one store");
+ return {
+ scheduler: requiredString(item.scheduler, "$runtime.services.scheduler"),
+ worker: requiredString(item.worker, "$runtime.services.worker"),
+ stores: {
+ build: requiredString(stores.build, "$runtime.services.stores.build"),
+ operations: requiredString(stores.operations, "$runtime.services.stores.operations"),
+ dispatch: requiredString(stores.dispatch, "$runtime.services.stores.dispatch"),
+ journal: requiredString(stores.journal, "$runtime.services.stores.journal"),
+ artifacts: requiredString(stores.artifacts, "$runtime.services.stores.artifacts"),
+ credentials,
+ },
+ };
+}
+
export function parseRuntimeConfig(value: unknown): RuntimeConfigDocument {
const item = object(value, "$runtime");
exactKeys(item, [
- "format", "root", "packageRoot", "statePath", "catalogPath", "artifactPath", "packageLock", "runtimePackageLock",
- "runtimeServices", "endpoints",
+ "format", "root", "packageRoot", "packageLock", "runtimePackageLock",
+ "runtimeServices", "services", "endpoints",
"permissions", "scheduling",
], "$runtime");
if (item.format !== "svml.runtime-config@1") throw new Error("$runtime.format must be svml.runtime-config@1");
- const runtimeServices = item.runtimeServices === undefined ? [] : (() => {
- if (!Array.isArray(item.runtimeServices)) throw new Error("$runtime.runtimeServices must be an array");
- return item.runtimeServices.map((value, index) => entry(value, `$runtime.runtimeServices[${index}]`, false));
- })();
+ if (!Array.isArray(item.runtimeServices)) throw new Error("$runtime.runtimeServices must be an array");
+ const runtimeServices = item.runtimeServices.map((value, index) =>
+ entry(value, `$runtime.runtimeServices[${index}]`, false));
const endpoints = item.endpoints === undefined ? [] : (() => {
if (!Array.isArray(item.endpoints)) throw new Error("$runtime.endpoints must be an array");
return item.endpoints.map((value, index) => entry(value, `$runtime.endpoints[${index}]`, true));
@@ -141,17 +161,15 @@ export function parseRuntimeConfig(value: unknown): RuntimeConfigDocument {
...(optionalString(item.packageRoot, "$runtime.packageRoot") === undefined
? {}
: { packageRoot: item.packageRoot as string }),
- ...(optionalString(item.statePath, "$runtime.statePath") === undefined ? {} : { statePath: item.statePath as string }),
- ...(optionalString(item.catalogPath, "$runtime.catalogPath") === undefined ? {} : { catalogPath: item.catalogPath as string }),
- ...(optionalString(item.artifactPath, "$runtime.artifactPath") === undefined ? {} : { artifactPath: item.artifactPath as string }),
...(optionalString(item.packageLock, "$runtime.packageLock") === undefined ? {} : { packageLock: item.packageLock as string }),
...(optionalString(item.runtimePackageLock, "$runtime.runtimePackageLock") === undefined
? {}
: { runtimePackageLock: item.runtimePackageLock as string }),
runtimeServices,
+ services: serviceSelection(item.services),
endpoints,
permissions: stringList(item.permissions, "$runtime.permissions"),
- ...(scheduled === undefined ? {} : { scheduling: scheduled }),
+ scheduling: scheduled,
};
}
@@ -383,14 +401,12 @@ export async function createRuntimeFromConfig(
return await createProjectLocalRuntime({
root,
packageRoot,
- ...(document.statePath === undefined ? {} : { statePath: document.statePath }),
- ...(document.catalogPath === undefined ? {} : { catalogPath: document.catalogPath }),
- ...(document.artifactPath === undefined ? {} : { artifactPath: document.artifactPath }),
...(document.packageLock === undefined ? {} : { packageLock: document.packageLock }),
runtimeServices,
+ runtimeSelection: document.services,
components: options.components ?? [],
endpoints,
allowedPermissions: document.permissions,
- ...(document.scheduling === undefined ? {} : { scheduling: document.scheduling }),
+ scheduling: document.scheduling,
});
}
diff --git a/packages/local/src/index.ts b/packages/local/src/index.ts
index 1389574f..364b43fb 100644
--- a/packages/local/src/index.ts
+++ b/packages/local/src/index.ts
@@ -1,7 +1,9 @@
export {
createLocalSchedulerPackage,
+ createLocalExecutionPackage,
localRuntimeModuleRef,
localSchedulerImplementationDigest,
+ localWorkerImplementationDigest,
} from "./scheduler-package.js";
export {
createLocalRuntime,
diff --git a/packages/local/src/project-services.ts b/packages/local/src/project-services.ts
index 9b106ec0..0a8aff4c 100644
--- a/packages/local/src/project-services.ts
+++ b/packages/local/src/project-services.ts
@@ -1,46 +1,22 @@
-import { resolve } from "node:path";
-
-import { createFileArtifactStorePackage } from "@narratage/artifact-store-fs";
-import { createEnvironmentCredentialStorePackage } from "@narratage/credential-store-env";
import {
assembleRuntimeServices,
verifyRuntimeServicePackage,
} from "@narratage/runtime";
import type {
- ArtifactStore,
BuildCatalog,
- BuildStore,
- CredentialStore,
- OperationStore,
RuntimeServiceAssembly,
- RuntimeServiceFacetRole,
RuntimeServicePackage,
+ RuntimeServiceSelection,
} from "@narratage/runtime";
-import { createSqliteRuntimeServicePackage, SqliteRuntimeState } from "@narratage/store-sqlite";
-import { createLocalSchedulerPackage } from "./scheduler-package.js";
import type { ProjectLocalRuntimeOptions } from "./types.js";
-export type ProjectRuntimeServiceSelection = {
- readonly scheduler: string;
- readonly stores: {
- readonly build: string;
- readonly operations: string;
- readonly artifacts: string;
- readonly credentials: string;
- };
-};
-
-export type ProjectRuntimeServiceAssembly = RuntimeServiceAssembly & {
- readonly buildStore: BuildStore;
- readonly operationStore: OperationStore;
- readonly artifactStore: ArtifactStore;
- readonly credentialStore: CredentialStore;
-};
+export type ProjectRuntimeServiceSelection = RuntimeServiceSelection;
+export type ProjectRuntimeServiceAssembly = RuntimeServiceAssembly;
export type AssembledProjectRuntimeServices = {
readonly assembly: ProjectRuntimeServiceAssembly;
- readonly catalog: BuildCatalog;
+ readonly catalog: BuildCatalog | undefined;
readonly selection: ProjectRuntimeServiceSelection;
readonly allowedPermissions: readonly string[];
close(): Promise<void>;
@@ -50,177 +26,45 @@ function assert(condition: unknown, message: string): asserts condition {
if (!condition) throw new Error(message);
}
-function configuredServiceIds(
+function packageCatalog(
packages: readonly RuntimeServicePackage[],
- role: RuntimeServiceFacetRole,
-): readonly string[] {
- return packages.flatMap((item) => item.services
- .filter((service) => service.role === role)
- .map((service) => service.instance.id));
+ buildStore: string,
+): BuildCatalog | undefined {
+ const owner = packages.find((item) => item.services.some((service) =>
+ service.role === "build-store" && service.instance.id === buildStore));
+ if (owner === undefined || !("catalog" in owner)) return undefined;
+ const catalog = (owner as { readonly catalog?: unknown }).catalog;
+ if (catalog === undefined) return undefined;
+ assert(catalog !== null && typeof catalog === "object"
+ && "record" in catalog && typeof catalog.record === "function"
+ && "read" in catalog && typeof catalog.read === "function"
+ && "list" in catalog && typeof catalog.list === "function",
+ `Runtime package ${owner.name} exposes an invalid Build Catalog`);
+ return catalog as BuildCatalog;
}
/**
- * A Runtime fills each role exactly once. One package may fill several roles —
- * the SQLite package supplies both stores from one file — but two packages
- * filling the same role is a configuration to correct, not an ambiguity to
- * resolve after the fact.
+ * Assemble only the services named by the Runtime Profile. This layer never manufactures a
+ * Scheduler, Worker, Store, path, permission or credential source on the project's behalf.
*/
-function chooseService(
- packages: readonly RuntimeServicePackage[],
- role: RuntimeServiceFacetRole,
- fallback: string,
-): string {
- const configured = configuredServiceIds(packages, role);
- if (configured.length === 0) return fallback;
- if (configured.length === 1) return configured[0]!;
- throw new Error(
- `runtimeServices configures ${configured.length} ${role} implementations (${configured.join(", ")}); a Runtime uses one`,
- );
-}
-
-function hasConfiguredService(
- packages: readonly RuntimeServicePackage[],
- id: string,
-): boolean {
- return packages.some((item) => item.services.some((service) => service.instance.id === id));
-}
-
-function manifestPermissions(packages: readonly RuntimeServicePackage[]): readonly string[] {
- return [...new Set(packages.flatMap((item) => item.manifest.facets.flatMap((facet) => facet.permissions)))];
-}
-
-async function closeServicePackages(packages: readonly RuntimeServicePackage[]): Promise<void> {
- const closed = new Set<RuntimeServicePackage>();
- for (const item of [...packages].reverse()) {
- if (closed.has(item)) continue;
- closed.add(item);
- await item.close?.();
- }
-}
-
-async function closeAfterFailure(
- packages: readonly RuntimeServicePackage[],
- error: unknown,
-): Promise<never> {
- try {
- await closeServicePackages(packages);
- } catch (closeError) {
- throw new AggregateError([error, closeError], "Runtime service assembly and cleanup both failed");
- }
- throw error;
-}
-
-function projectSelection(
- packages: readonly RuntimeServicePackage[],
- options: ProjectLocalRuntimeOptions,
-): ProjectRuntimeServiceSelection {
- return {
- scheduler: chooseService(packages, "scheduler", "scheduler.local"),
- stores: {
- build: chooseService(packages, "build-store", "builds.sqlite"),
- operations: chooseService(packages, "operation-store", "operations.sqlite"),
- artifacts: chooseService(packages, "artifact-store", "artifacts.fs"),
- credentials: chooseService(packages, "credential-store", "credentials.env"),
- },
- };
-}
-
-/** Local-distribution defaults and exact service selection, separate from Build execution. */
export async function createProjectRuntimeServices(
- root: string,
+ _root: string,
options: ProjectLocalRuntimeOptions,
): Promise<AssembledProjectRuntimeServices> {
- const configured = [...(options.runtimeServices ?? [])];
- let selection: ProjectRuntimeServiceSelection;
+ const packages = [...options.runtimeServices];
try {
- for (const item of configured) verifyRuntimeServicePackage(item);
- selection = projectSelection(configured, options);
- } catch (error) {
- return await closeAfterFailure(configured, error);
- }
-
- const defaults: RuntimeServicePackage[] = [];
- let defaultCatalog: BuildCatalog | undefined;
- let standaloneCatalog: SqliteRuntimeState | undefined;
- try {
- if (!hasConfiguredService(configured, selection.scheduler)) {
- assert(selection.scheduler === "scheduler.local", `unknown selected scheduler ${selection.scheduler}`);
- defaults.push(createLocalSchedulerPackage(selection.scheduler));
- }
- const needsDefaultBuild = !hasConfiguredService(configured, selection.stores.build);
- const needsDefaultOperations = !hasConfiguredService(configured, selection.stores.operations);
- if (needsDefaultBuild || needsDefaultOperations) {
- assert((!needsDefaultBuild || selection.stores.build === "builds.sqlite")
- && (!needsDefaultOperations || selection.stores.operations === "operations.sqlite"),
- "unknown selected BuildStore or OperationStore");
- const sqlite = createSqliteRuntimeServicePackage({
- path: resolve(root, options.statePath ?? ".svml/runtime.sqlite"),
- buildInstance: "builds.sqlite",
- operationInstance: "operations.sqlite",
- });
- defaults.push(sqlite);
- defaultCatalog = sqlite.catalog;
- } else if (options.statePath !== undefined) {
- throw new Error("statePath configures the default SQLite services, but neither was selected");
- }
- if (!hasConfiguredService(configured, selection.stores.artifacts)) {
- assert(selection.stores.artifacts === "artifacts.fs",
- `unknown selected ArtifactStore ${selection.stores.artifacts}`);
- defaults.push(createFileArtifactStorePackage({
- root: resolve(root, options.artifactPath ?? ".svml/artifacts"),
- instance: selection.stores.artifacts,
- }));
- } else if (options.artifactPath !== undefined) {
- throw new Error("artifactPath configures the default filesystem ArtifactStore, but it was not selected");
- }
- if (!hasConfiguredService(configured, selection.stores.credentials)) {
- assert(selection.stores.credentials === "credentials.env",
- `unknown selected CredentialStore ${selection.stores.credentials}`);
- defaults.push(createEnvironmentCredentialStorePackage({ instance: selection.stores.credentials }));
- }
- if (options.buildCatalog !== undefined && options.catalogPath !== undefined) {
- throw new Error("buildCatalog and catalogPath are mutually exclusive");
- }
- let catalog = options.buildCatalog;
- if (catalog === undefined && options.catalogPath !== undefined) {
- standaloneCatalog = new SqliteRuntimeState(resolve(root, options.catalogPath));
- catalog = standaloneCatalog.catalog;
- }
- if (catalog === undefined) catalog = defaultCatalog;
- if (catalog === undefined) {
- standaloneCatalog = new SqliteRuntimeState(resolve(root, ".svml/catalog.sqlite"));
- catalog = standaloneCatalog.catalog;
- }
- const assembly = assembleRuntimeServices([...configured, ...defaults], selection);
- assert(assembly.buildStore !== undefined, "project Runtime requires a BuildStore");
- assert(assembly.operationStore !== undefined, "project Runtime requires an OperationStore");
- assert(assembly.artifactStore !== undefined, "project Runtime requires an ArtifactStore");
- assert(assembly.credentialStore !== undefined, "project Runtime requires a CredentialStore");
- const complete: ProjectRuntimeServiceAssembly = {
- ...assembly,
- buildStore: assembly.buildStore,
- operationStore: assembly.operationStore,
- artifactStore: assembly.artifactStore,
- credentialStore: assembly.credentialStore,
- };
+ for (const item of packages) verifyRuntimeServicePackage(item);
+ const assembly = assembleRuntimeServices(packages, options.runtimeSelection);
+ const catalog = options.buildCatalog ?? packageCatalog(packages, options.runtimeSelection.stores.build);
return {
- assembly: complete,
+ assembly,
catalog,
- selection,
- allowedPermissions: [
- ...manifestPermissions(defaults),
- ...(options.allowedPermissions ?? []),
- ],
- async close() {
- try {
- await complete.close();
- } finally {
- standaloneCatalog?.close();
- }
- },
+ selection: options.runtimeSelection,
+ allowedPermissions: [...options.allowedPermissions],
+ close: () => assembly.close(),
};
} catch (error) {
- standaloneCatalog?.close();
- return await closeAfterFailure([...configured, ...defaults], error);
+ for (const item of [...packages].reverse()) await item.close?.();
+ throw error;
}
}
diff --git a/packages/local/src/runtime.ts b/packages/local/src/runtime.ts
index f4b8de07..df6cba1d 100644
--- a/packages/local/src/runtime.ts
+++ b/packages/local/src/runtime.ts
@@ -15,15 +15,14 @@ import {
} from "@narratage/package-loader-node";
import { isDigest } from "@narratage/protocol";
import {
- LocalBuildScheduler,
- MemoryBuildCatalog,
RuntimeModuleRegistry,
+ createBuildDispatchIdentity,
isEnumerableBuildStore,
isManagedArtifactStore,
isStreamingArtifactStore,
- localSchedulerOptionsFromClosure,
resolveRuntimeProfile,
sealRuntimeProfile,
+ writableCredentialStore,
} from "@narratage/runtime";
import { TypeValidatorRegistry } from "@narratage/validation";
import type {
@@ -36,6 +35,7 @@ import type {
CreateLocalRuntimeOptions,
LocalBuildOptions,
LocalBuildRequest,
+ LocalBuildSubmission,
LocalRuntime,
EndpointPackage,
ProjectLocalRuntimeOptions,
@@ -141,10 +141,8 @@ function verifyEndpointPackages(packages: readonly EndpointPackage[]): void {
export async function createLocalRuntime(
options: CreateLocalRuntimeOptions,
): Promise<LocalRuntime> {
- const buildCatalog = options.buildCatalog ?? new MemoryBuildCatalog();
- if (options.closure !== undefined
- && (options.scheduling?.maxConcurrency !== undefined
- || options.scheduling?.laneLimits !== undefined)) {
+ const buildCatalog = options.buildCatalog;
+ if (options.scheduling?.maxConcurrency !== undefined || options.scheduling?.laneLimits !== undefined) {
throw new Error("a locked Runtime Closure owns maxConcurrency and lane limits");
}
const producers = new ProducerRegistry();
@@ -159,35 +157,52 @@ export async function createLocalRuntime(
registerProducerFacets(producers, component.producers ?? []);
}
for (const endpoint of options.endpoints ?? []) await endpoint.install(endpoints);
- if (options.closure !== undefined) {
- endpoints.applyRuntimeClosure(
- options.closure.value,
- options.closure.modules,
- { allowedPermissions: options.closure.allowedPermissions ?? [] },
- );
- }
+ endpoints.applyRuntimeClosure(
+ options.closure.value,
+ options.closure.modules,
+ { allowedPermissions: options.closure.allowedPermissions ?? [] },
+ );
const driver = new NodeDriver({
producers,
endpoints,
artifacts: options.artifactStore,
- ...(options.credentialStore === undefined ? {} : { credentials: options.credentialStore }),
- ...(options.operationStore === undefined ? {} : { operations: options.operationStore }),
+ credentials: options.credentialStore,
+ operations: options.operationStore,
validators,
...(options.implementationClosure === undefined
? {}
: { implementationClosure: options.implementationClosure }),
});
- const closureScheduling = options.closure === undefined
- ? {}
- : localSchedulerOptionsFromClosure(options.closure.value);
- const scheduler = (options.scheduler ?? {
- create(executor, schedulerOptions) {
- return new LocalBuildScheduler(executor, schedulerOptions);
+ const scheduling = {
+ maxConcurrency: options.closure.value.scheduling.maxConcurrency,
+ laneLimits: Object.fromEntries(options.closure.value.scheduling.lanes.map((lane) => [lane.name, lane.maxConcurrency])),
+ ...(options.scheduling?.maxEventsPerBuild === undefined
+ ? {} : { maxEventsPerBuild: options.scheduling.maxEventsPerBuild }),
+ };
+ const worker = options.worker.create(driver, {
+ scheduler: options.scheduler,
+ stores: {
+ builds: options.buildStore,
+ operations: options.operationStore,
+ dispatch: options.dispatchStore,
+ journal: options.journal,
+ artifacts: options.artifactStore,
},
- }).create(driver, {
- ...closureScheduling,
- ...(options.scheduling ?? {}),
- buildStore: options.buildStore,
+ scheduling,
+ runtimeClosure: options.closure.value,
+ });
+ const credentialDescriptions = (options.endpoints ?? []).flatMap((item) => item.credentials);
+ const credential = (endpoint: string, slot: string) => {
+ const matches = credentialDescriptions.filter((item) => item.endpoint === endpoint && item.slot === slot);
+ assert(matches.length === 1, matches.length === 0
+ ? `Endpoint ${endpoint} has no credential slot ${slot}`
+ : `Endpoint ${endpoint} repeats credential slot ${slot}`);
+ return matches[0]!;
+ };
+ const credentialStatus = async (item: typeof credentialDescriptions[number]) => ({
+ ...structuredClone(item),
+ configured: await options.credentialStore.resolve(item.ref) !== undefined,
+ writable: await writableCredentialStore(options.credentialStore, item.ref) !== undefined,
});
const stageAttachments = async (request: LocalBuildRequest): Promise<void> => {
for (const item of request.attachments ?? []) {
@@ -200,72 +215,134 @@ export async function createLocalRuntime(
);
}
};
- const scheduled = (request: LocalBuildRequest) => ({ id: request.id, state: request.state });
- const runBuild = async (
- request: LocalBuildRequest,
- follow: LocalBuildOptions = {},
- ) => {
+ const presentation = async (build: string): Promise<LocalBuildSubmission> => {
+ const [snapshot, dispatch] = await Promise.all([
+ options.buildStore.read(build),
+ options.dispatchStore.read(build),
+ ]);
+ assert(snapshot !== undefined && dispatch !== undefined, `Build ${build} has no durable Runtime state`);
+ const status: LocalBuildSubmission["status"] = dispatch.phase === "terminal"
+ ? dispatch.terminal!
+ : dispatch.phase === "leased" ? "running" : dispatch.phase;
+ return { id: build, state: snapshot.state, status, dispatch };
+ };
+
+ const submit = async (request: LocalBuildRequest): Promise<LocalBuildSubmission> => {
+ assert(request.id.trim().length > 0, "Build id must not be empty");
await stageAttachments(request);
+ let stored = await options.buildStore.read(request.id);
+ if (stored === undefined) {
+ try {
+ stored = await options.buildStore.create(request.id, request.state);
+ } catch (error) {
+ stored = await options.buildStore.read(request.id);
+ if (stored === undefined) throw error;
+ }
+ }
+ assert(stored.state.id === request.state.id,
+ `Build ${request.id} already names another Core Build`);
+ const created = await options.dispatchStore.create(createBuildDispatchIdentity({
+ build: request.id,
+ core: request.state.id,
+ runtimeClosure: options.closure.value.digest,
+ }));
+ if (created.status === "created") {
+ await options.journal.append({
+ at: Date.now(),
+ kind: "dispatch-created",
+ build: request.id,
+ detail: { dispatch: created.snapshot.id },
+ });
+ }
if (request.catalog !== undefined) {
+ assert(buildCatalog !== undefined, "Build supplied Host catalog metadata but no BuildCatalog was selected");
assert(request.catalog.core === request.state.id,
`Build Catalog Core ${request.catalog.core} differs from Build ${request.state.id}`);
+ await buildCatalog.record(request.id, request.catalog);
}
+ return await presentation(request.id);
+ };
+
+ const runBuild = async (
+ request: LocalBuildRequest,
+ follow: LocalBuildOptions = {},
+ ): Promise<LocalBuildSubmission> => {
+ let result = await submit(request);
const startedAt = Date.now();
const pollIntervalMs = nonNegativeInteger(follow.pollIntervalMs ?? 1_000, "pollIntervalMs");
const maxWaitMs = follow.maxWaitMs === undefined
? undefined
: nonNegativeInteger(follow.maxWaitMs, "maxWaitMs");
- let [result] = await scheduler.run([scheduled(request)]);
- if (result === undefined) throw new Error(`Local Scheduler returned no result for ${request.id}`);
- if (request.catalog !== undefined) await buildCatalog.record(request.id, request.catalog);
- while (true) {
- if (follow.follow !== true || result.status !== "paused") return result;
- const pending = result.journal.filter((item) => item.status === "pending");
- if (pending.length === 0) return result;
- const now = Date.now();
- const wakeAt = Math.min(...pending.map((item) => item.wakeAt ?? now + pollIntervalMs));
- const delay = Math.max(0, wakeAt - now);
- if (maxWaitMs !== undefined && now - startedAt + delay > maxWaitMs) return result;
- await wait(delay, follow.signal);
- [result] = await scheduler.run([scheduled(request)]);
- if (result === undefined) throw new Error(`Local Scheduler returned no result for ${request.id}`);
+ while (follow.follow === true && !["complete", "failed", "cancelled"].includes(result.status)) {
+ if (maxWaitMs !== undefined && Date.now() - startedAt + pollIntervalMs > maxWaitMs) return result;
+ await wait(pollIntervalMs, follow.signal);
+ result = await presentation(request.id);
}
+ return result;
};
return {
build: runBuild,
async buildMany(requests) {
- await Promise.all(requests.map(stageAttachments));
- for (const request of requests) {
- if (request.catalog === undefined) continue;
- assert(request.catalog.core === request.state.id,
- `Build Catalog Core ${request.catalog.core} differs from Build ${request.state.id}`);
- }
- const results = await scheduler.run(requests.map(scheduled));
- await Promise.all(requests.map(async (request) => {
- if (request.catalog !== undefined) await buildCatalog.record(request.id, request.catalog);
- }));
- return results;
+ return await Promise.all(requests.map(submit));
},
async status(build) {
return {
build: await options.buildStore.read(build),
- catalog: await buildCatalog.read(build),
- operations: options.operationStore === undefined ? [] : await options.operationStore.list({ build }),
+ catalog: await buildCatalog?.read(build),
+ operations: await options.operationStore.list({ build }),
+ dispatch: await options.dispatchStore.read(build),
};
},
+ async queue() {
+ return {
+ dispatches: await options.dispatchStore.list(),
+ capacity: await options.dispatchStore.listCapacity(),
+ };
+ },
+ async operation(id) {
+ return await options.operationStore.read(id);
+ },
+ async journal(query) {
+ return await options.journal.list(query);
+ },
+ async credentials(endpoint) {
+ const selected = credentialDescriptions.filter((item) => endpoint === undefined || item.endpoint === endpoint);
+ return await Promise.all(selected.map(credentialStatus));
+ },
+ async putCredential(endpoint, slot, secret) {
+ assert(secret.length > 0, "credential secret is empty");
+ const item = credential(endpoint, slot);
+ const store = await writableCredentialStore(options.credentialStore, item.ref);
+ assert(store !== undefined, `CredentialStore ${item.ref.store} is not writable`);
+ await store.put(item.ref, { secret });
+ return await credentialStatus(item);
+ },
+ async deleteCredential(endpoint, slot) {
+ const item = credential(endpoint, slot);
+ const store = await writableCredentialStore(options.credentialStore, item.ref);
+ assert(store !== undefined, `CredentialStore ${item.ref.store} is not writable`);
+ const deleted = await store.delete(item.ref);
+ return { deleted, credential: await credentialStatus(item) };
+ },
async builds() {
- return await buildCatalog.list();
+ return await buildCatalog?.list() ?? [];
},
- async cancel(build) {
- if (options.operationStore === undefined) throw new Error("Local Runtime has no OperationStore");
- const snapshot = await options.buildStore.read(build);
- if (snapshot === undefined) return undefined;
- const active = (await options.operationStore.list({ build }))
- .filter((operation) => operation.status === "created" || operation.status === "pending");
- if (active.length === 0) return undefined;
- for (const operation of active) await driver.cancelOperation(snapshot.state, operation);
- const [result] = await scheduler.run([{ id: build, state: snapshot.state }]);
- return result;
+ async cancel(build, reason) {
+ if (await options.dispatchStore.read(build) === undefined) return undefined;
+ const dispatch = await options.dispatchStore.requestCancellation(build, reason);
+ await options.journal.append({
+ at: Date.now(),
+ kind: "cancellation-requested",
+ build,
+ detail: { ...(reason === undefined ? {} : { reason }) },
+ });
+ return dispatch;
+ },
+ async workOnce(workOptions) {
+ return await worker.runOnce(workOptions);
+ },
+ async work(workOptions) {
+ await worker.run(workOptions);
},
async readArtifact(digest) {
return await options.artifactStore.get(digest);
@@ -282,9 +359,7 @@ export async function createLocalRuntime(
"selected BuildStore does not expose the maintenance index required for Artifact GC");
const reachable = new Set<import("@narratage/protocol").Digest>();
for (const snapshot of await options.buildStore.list()) collectArtifactDigests(snapshot.state, reachable);
- if (options.operationStore !== undefined) {
- for (const operation of await options.operationStore.list({})) collectArtifactDigests(operation, reachable);
- }
+ for (const operation of await options.operationStore.list({})) collectArtifactDigests(operation, reachable);
const stored = await options.artifactStore.list();
const unreachable = stored.filter((digest) => !reachable.has(digest)).sort();
const deleted: import("@narratage/protocol").Digest[] = [];
@@ -308,7 +383,7 @@ export async function createLocalRuntime(
* Capability Endpoints may execute locally, in a vendor API, in Lambda, or on a hosted service.
*/
export async function createProjectLocalRuntime(
- options: ProjectLocalRuntimeOptions = {},
+ options: ProjectLocalRuntimeOptions,
): Promise<LocalRuntime> {
const root = resolve(options.root ?? process.cwd());
const packageRoot = resolve(options.packageRoot ?? root);
@@ -338,16 +413,19 @@ export async function createProjectLocalRuntime(
...endpointPackages.map((item) => item.instance),
],
scheduler: selection.scheduler,
+ worker: selection.worker,
stores: {
build: selection.stores.build,
operations: selection.stores.operations,
+ dispatch: selection.stores.dispatch,
+ journal: selection.stores.journal,
artifacts: selection.stores.artifacts,
credentials: selection.stores.credentials,
},
endpoints: endpointPackages.flatMap((item) => item.bindings),
scheduling: {
- maxConcurrency: options.scheduling?.maxConcurrency ?? 4,
- lanes: Object.entries(options.scheduling?.lanes ?? {}).map(([name, maxConcurrency]) => ({
+ maxConcurrency: options.scheduling.maxConcurrency,
+ lanes: Object.entries(options.scheduling.lanes ?? {}).map(([name, maxConcurrency]) => ({
name,
maxConcurrency,
})),
@@ -358,18 +436,21 @@ export async function createProjectLocalRuntime(
});
const runtime = await createLocalRuntime({
buildStore: services.buildStore,
- buildCatalog: projectServices.catalog,
+ ...(projectServices.catalog === undefined ? {} : { buildCatalog: projectServices.catalog }),
operationStore: services.operationStore,
+ dispatchStore: services.dispatchStore,
+ journal: services.journal,
artifactStore: services.artifactStore,
credentialStore: services.credentialStore,
scheduler: services.scheduler,
+ worker: services.worker,
...(configuredComponents.length === 0
? {}
: { components: configuredComponents }),
endpoints: endpointPackages,
closure: { modules, value: closure, allowedPermissions: projectServices.allowedPermissions },
scheduling: {
- ...(options.scheduling?.maxEventsPerBuild === undefined
+ ...(options.scheduling.maxEventsPerBuild === undefined
? {}
: { maxEventsPerBuild: options.scheduling.maxEventsPerBuild }),
},
@@ -380,8 +461,16 @@ export async function createProjectLocalRuntime(
build: runtime.build,
buildMany: runtime.buildMany,
status: runtime.status,
+ queue: runtime.queue,
+ operation: runtime.operation,
+ journal: runtime.journal,
+ credentials: runtime.credentials,
+ putCredential: runtime.putCredential,
+ deleteCredential: runtime.deleteCredential,
builds: runtime.builds,
cancel: runtime.cancel,
+ workOnce: runtime.workOnce,
+ work: runtime.work,
readArtifact: runtime.readArtifact,
openArtifact: runtime.openArtifact,
garbageCollectArtifacts: runtime.garbageCollectArtifacts,
diff --git a/packages/local/src/scheduler-package.ts b/packages/local/src/scheduler-package.ts
index 719c1ca0..89108eb7 100644
--- a/packages/local/src/scheduler-package.ts
+++ b/packages/local/src/scheduler-package.ts
@@ -5,12 +5,15 @@ import {
} from "@narratage/runtime";
import type { RuntimeServicePackage } from "@narratage/runtime";
+import { durableLocalWorkerFactory } from "./worker.js";
+
export const localRuntimeModuleRef = {
name: "@narratage/local",
version: "1",
} as const;
export const localSchedulerImplementationDigest = digestOf("@narratage/local/scheduler@1");
+export const localWorkerImplementationDigest = digestOf("@narratage/local/worker@1");
export function createLocalSchedulerPackage(
instance = "scheduler.local",
@@ -35,3 +38,40 @@ export function createLocalSchedulerPackage(
}],
});
}
+
+export function createLocalExecutionPackage(
+ instance: string,
+): RuntimeServicePackage {
+ return defineRuntimeServicePackage({
+ name: instance,
+ module: localRuntimeModuleRef,
+ services: [
+ {
+ role: "scheduler",
+ facet: "scheduler",
+ instance: `${instance}.scheduler`,
+ implementation: {
+ locator: "@narratage/local/scheduler",
+ digest: localSchedulerImplementationDigest,
+ },
+ configuration: { algorithm: "fair-lanes", version: 1 },
+ service: {
+ create(executor, options) {
+ return new LocalBuildScheduler(executor, options);
+ },
+ },
+ },
+ {
+ role: "worker",
+ facet: "worker",
+ instance: `${instance}.worker`,
+ implementation: {
+ locator: "@narratage/local/worker",
+ digest: localWorkerImplementationDigest,
+ },
+ configuration: { dispatch: "leased", capacity: "shared", version: 1 },
+ service: durableLocalWorkerFactory,
+ },
+ ],
+ });
+}
diff --git a/packages/local/src/types.ts b/packages/local/src/types.ts
index 94e23156..7a93d21d 100644
--- a/packages/local/src/types.ts
+++ b/packages/local/src/types.ts
@@ -12,12 +12,15 @@ import type {
BuildSchedulerOptions,
BuildSnapshot,
BuildStore,
+ BuildDispatchSnapshot,
CredentialStore,
OperationStore,
OperationSnapshot,
RuntimeModuleRegistry,
RuntimeServicePackage,
- ScheduledBuildResult,
+ RuntimeServiceSelection,
+ RuntimeJournal,
+ RuntimeWorkerFactory,
} from "@narratage/runtime";
import type { TypeValidatorRegistrar, TypeValidatorRegistryLike } from "@narratage/validation";
@@ -38,13 +41,16 @@ export type CreateLocalRuntimeOptions = {
readonly buildStore: BuildStore;
/** Host presentation metadata only; never part of Runtime Closure or Core state. */
readonly buildCatalog?: BuildCatalog;
- readonly operationStore?: OperationStore;
+ readonly operationStore: OperationStore;
+ readonly dispatchStore: import("@narratage/runtime").BuildDispatchStore;
+ readonly journal: RuntimeJournal;
readonly artifactStore: ArtifactStore;
- readonly credentialStore?: CredentialStore;
- readonly scheduler?: BuildSchedulerFactory;
+ readonly credentialStore: CredentialStore;
+ readonly scheduler: BuildSchedulerFactory;
+ readonly worker: RuntimeWorkerFactory;
readonly components?: readonly ComponentPackage[];
readonly endpoints?: readonly EndpointPackage[];
- readonly closure?: LocalRuntimeClosureOptions;
+ readonly closure: LocalRuntimeClosureOptions;
readonly scheduling?: Omit<BuildSchedulerOptions, "buildStore" | "runtimeClosure">;
readonly validators?: LocalTypeValidatorRegistry;
/** Expected implementation package closure already bound into BuildRequest. */
@@ -56,24 +62,22 @@ export type ProjectLocalRuntimeOptions = {
readonly root?: string;
/** Host directory whose node_modules contains the packages named by packageLock. Defaults to root. */
readonly packageRoot?: string;
- readonly statePath?: string;
- /** Optional separate Host catalog database. Defaults to statePath when using local SQLite state. */
- readonly catalogPath?: string;
- readonly artifactPath?: string;
readonly buildCatalog?: BuildCatalog;
/** Exact installed implementation package lock. Source imports cannot change this selection. */
readonly packageLock?: string;
/**
* Replaceable parts of the Runtime itself. One package may fill several roles;
- * a supplied role replaces that role's local default, and two packages filling
- * one role is a configuration error rather than a choice made here.
+ * every selected role is explicit, and two packages exposing the same instance
+ * id are a configuration error rather than a choice made here.
*/
- readonly runtimeServices?: readonly RuntimeServicePackage[];
+ readonly runtimeServices: readonly RuntimeServicePackage[];
+ /** Exact service instances selected from runtimeServices; no role is inferred by uniqueness. */
+ readonly runtimeSelection: RuntimeServiceSelection;
readonly components?: readonly ComponentPackage[];
readonly endpoints?: readonly EndpointPackage[];
- readonly allowedPermissions?: readonly string[];
- readonly scheduling?: {
- readonly maxConcurrency?: number;
+ readonly allowedPermissions: readonly string[];
+ readonly scheduling: {
+ readonly maxConcurrency: number;
readonly lanes?: Readonly<Record<string, number>>;
readonly maxEventsPerBuild?: number;
};
@@ -91,7 +95,7 @@ export type LocalBuildRequest = {
};
export type LocalBuildOptions = {
- /** Keep resuming pending Operations according to wakeAt until completion, pause, abort or deadline. */
+ /** Observe the durable Dispatch until terminal; execution remains owned by a Worker process. */
readonly follow?: boolean;
readonly pollIntervalMs?: number;
readonly maxWaitMs?: number;
@@ -102,6 +106,24 @@ export type LocalRuntimeStatus = {
readonly build: BuildSnapshot | undefined;
readonly catalog: BuildCatalogEntry | undefined;
readonly operations: readonly OperationSnapshot[];
+ readonly dispatch: BuildDispatchSnapshot | undefined;
+};
+
+export type LocalRuntimeQueue = {
+ readonly dispatches: readonly BuildDispatchSnapshot[];
+ readonly capacity: readonly import("@narratage/runtime").CapacityReservation[];
+};
+
+export type LocalCredentialStatus = import("@narratage/endpoint-kit").EndpointCredentialDescription & {
+ readonly configured: boolean;
+ readonly writable: boolean;
+};
+
+export type LocalBuildSubmission = {
+ readonly id: string;
+ readonly state: BuildState;
+ readonly status: "queued" | "running" | "waiting" | "blocked" | "settling" | "complete" | "failed" | "cancelled";
+ readonly dispatch: BuildDispatchSnapshot;
};
export type ArtifactGarbageCollection = {
@@ -111,11 +133,19 @@ export type ArtifactGarbageCollection = {
};
export type LocalRuntime = {
- build(request: LocalBuildRequest, options?: LocalBuildOptions): Promise<ScheduledBuildResult>;
- buildMany(requests: readonly LocalBuildRequest[]): Promise<readonly ScheduledBuildResult[]>;
+ build(request: LocalBuildRequest, options?: LocalBuildOptions): Promise<LocalBuildSubmission>;
+ buildMany(requests: readonly LocalBuildRequest[]): Promise<readonly LocalBuildSubmission[]>;
status(build: string): Promise<LocalRuntimeStatus>;
+ queue(): Promise<LocalRuntimeQueue>;
+ operation(id: Digest): Promise<OperationSnapshot | undefined>;
+ journal(query?: import("@narratage/runtime").RuntimeJournalQuery): Promise<readonly import("@narratage/runtime").RuntimeJournalEntry[]>;
+ credentials(endpoint?: string): Promise<readonly LocalCredentialStatus[]>;
+ putCredential(endpoint: string, slot: string, secret: string): Promise<LocalCredentialStatus>;
+ deleteCredential(endpoint: string, slot: string): Promise<{ readonly deleted: boolean; readonly credential: LocalCredentialStatus }>;
builds(): Promise<readonly BuildCatalogEntry[]>;
- cancel(build: string): Promise<ScheduledBuildResult | undefined>;
+ cancel(build: string, reason?: string): Promise<BuildDispatchSnapshot | undefined>;
+ workOnce(options: { readonly owner: string; readonly leaseMs: number }): Promise<BuildDispatchSnapshot | undefined>;
+ work(options: import("@narratage/runtime").RuntimeWorkerRunOptions): Promise<void>;
readArtifact(digest: Digest): Promise<Uint8Array | undefined>;
openArtifact(digest: Digest): Promise<AsyncIterable<Uint8Array> | undefined>;
/** Explicit maintenance only. apply=false is a read-only reachability report. */
diff --git a/packages/local/src/worker.ts b/packages/local/src/worker.ts
new file mode 100644
index 00000000..8e5dccad
--- /dev/null
+++ b/packages/local/src/worker.ts
@@ -0,0 +1,314 @@
+import { randomUUID } from "node:crypto";
+
+import type { BuildState, Digest } from "@narratage/protocol";
+import { capacityReservationId } from "@narratage/runtime";
+import type {
+ BuildDispatchSnapshot,
+ CapacityReservation,
+ DispatchLease,
+ RuntimeCommandExecutor,
+ RuntimeExecutionResult,
+ RuntimePreparation,
+ RuntimeRunnableCommand,
+ RuntimeWorker,
+ RuntimeWorkerFactory,
+ RuntimeWorkerFactoryOptions,
+ RuntimeWorkerRunOptions,
+} from "@narratage/runtime";
+
+function assert(condition: unknown, message: string): asserts condition {
+ if (!condition) throw new Error(message);
+}
+
+function positive(value: number, subject: string): number {
+ assert(Number.isSafeInteger(value) && value > 0, `${subject} must be a positive safe integer`);
+ return value;
+}
+
+function sameLease(left: DispatchLease | undefined, right: DispatchLease): boolean {
+ return left?.owner === right.owner && left.token === right.token && left.fence === right.fence;
+}
+
+async function pause(ms: number, signal?: AbortSignal): Promise<void> {
+ if (signal?.aborted === true) throw signal.reason ?? new Error("Worker stopped");
+ await new Promise<void>((resolve, reject) => {
+ const timer = setTimeout(resolve, ms);
+ signal?.addEventListener("abort", () => {
+ clearTimeout(timer);
+ reject(signal.reason ?? new Error("Worker stopped"));
+ }, { once: true });
+ });
+}
+
+class CapacityExecutor implements RuntimeCommandExecutor {
+ readonly #delegate: RuntimeCommandExecutor;
+ readonly #options: RuntimeWorkerFactoryOptions;
+ readonly #build: string;
+ readonly #dispatchLease: DispatchLease;
+ readonly #leaseMs: number;
+
+ constructor(
+ delegate: RuntimeCommandExecutor,
+ options: RuntimeWorkerFactoryOptions,
+ build: string,
+ dispatchLease: DispatchLease,
+ leaseMs: number,
+ ) {
+ this.#delegate = delegate;
+ this.#options = options;
+ this.#build = build;
+ this.#dispatchLease = dispatchLease;
+ this.#leaseMs = leaseMs;
+ }
+
+ prepare(state: BuildState): RuntimePreparation {
+ return this.#delegate.prepare(state);
+ }
+
+ async #assertAuthority(): Promise<void> {
+ const dispatch = await this.#options.stores.dispatch.read(this.#build);
+ assert(dispatch?.admission === "open", `Build ${this.#build} no longer admits new Operations`);
+ assert(dispatch.phase === "leased" && sameLease(dispatch.lease, this.#dispatchLease),
+ `Build ${this.#build} Worker lease is stale`);
+ }
+
+ async #acquire(descriptor: RuntimeRunnableCommand): Promise<CapacityReservation> {
+ while (true) {
+ await this.#assertAuthority();
+ const now = Date.now();
+ const acquired = await this.#options.stores.dispatch.acquireCapacity({
+ build: this.#build,
+ command: descriptor.command.id,
+ lane: descriptor.lane,
+ mode: descriptor.capacityMode ?? "active",
+ buildLease: this.#dispatchLease,
+ owner: this.#dispatchLease.owner,
+ token: randomUUID(),
+ now,
+ leaseMs: this.#leaseMs,
+ limits: {
+ globalActive: positive(this.#options.scheduling.maxConcurrency ?? 1, "global active capacity"),
+ laneActive: positive(
+ this.#options.scheduling.laneLimits?.[descriptor.lane] ?? descriptor.maxConcurrency,
+ `lane ${descriptor.lane} active capacity`,
+ ),
+ laneInFlight: positive(descriptor.maxInFlight ?? descriptor.maxConcurrency,
+ `lane ${descriptor.lane} in-flight capacity`),
+ },
+ });
+ if (acquired.status === "acquired") return acquired.reservation;
+ await pause(Math.max(1, acquired.retryAt - now));
+ }
+ }
+
+ async executeCommand(
+ state: BuildState,
+ commandId: string,
+ context: { readonly build: string },
+ ): Promise<RuntimeExecutionResult> {
+ assert(context.build === this.#build, "Capacity executor received another Build identity");
+ const descriptor = this.#delegate.prepare(state).runnable.find((item) => item.command.id === commandId);
+ assert(descriptor !== undefined, `command ${commandId} is not currently executable`);
+ const reservation = await this.#acquire(descriptor);
+ const lease = reservation.active;
+ assert(lease !== undefined, `Capacity reservation ${reservation.id} has no active lease`);
+ try {
+ const result = await this.#delegate.executeCommand(state, commandId, context);
+ await this.#assertAuthority();
+ await this.#options.stores.dispatch.heartbeatCapacity(reservation.id, lease, Date.now(), this.#leaseMs);
+ if (result.status === "pending" && (descriptor.capacityMode ?? "active") === "recoverable") {
+ await this.#options.stores.dispatch.parkCapacity(reservation.id, lease, true);
+ } else {
+ await this.#options.stores.dispatch.releaseCapacity(reservation.id, lease);
+ }
+ return result;
+ } catch (error) {
+ await this.#options.stores.dispatch.releaseCapacity(reservation.id, lease).catch(() => undefined);
+ throw error;
+ }
+ }
+
+ async cancelOperation(
+ state: BuildState,
+ operation: import("@narratage/runtime").OperationSnapshot,
+ requestedAt: number,
+ ) {
+ assert(this.#delegate.cancelOperation !== undefined, "selected executor cannot cancel Operations");
+ return await this.#delegate.cancelOperation(state, operation, requestedAt);
+ }
+}
+
+class DurableLocalWorker implements RuntimeWorker {
+ readonly #executor: RuntimeCommandExecutor;
+ readonly #options: RuntimeWorkerFactoryOptions;
+
+ constructor(executor: RuntimeCommandExecutor, options: RuntimeWorkerFactoryOptions) {
+ this.#executor = executor;
+ this.#options = options;
+ }
+
+ async #journal(
+ kind: import("@narratage/runtime").RuntimeJournalKind,
+ build: string | undefined,
+ worker: string,
+ detail: import("@narratage/protocol").CanonicalValue,
+ ): Promise<void> {
+ await this.#options.stores.journal.append({ at: Date.now(), kind, worker, ...(build === undefined ? {} : { build }), detail });
+ }
+
+ async #cancel(dispatch: BuildDispatchSnapshot, lease: DispatchLease): Promise<BuildDispatchSnapshot> {
+ const snapshot = await this.#options.stores.builds.read(dispatch.build);
+ assert(snapshot !== undefined, `Build ${dispatch.build} has no durable state`);
+ const active = (await this.#options.stores.operations.list({ build: dispatch.build }))
+ .filter((item) => item.status === "created" || item.status === "pending");
+ if (active.length > 0 && this.#executor.cancelOperation === undefined) {
+ return await this.#options.stores.dispatch.release(dispatch.build, lease, {
+ phase: "settling",
+ availableAt: Date.now() + 1_000,
+ reason: "cancellation awaits an Operation controller",
+ });
+ }
+ const requestedAt = dispatch.cancellation?.requestedAt;
+ assert(requestedAt !== undefined, `closing Build ${dispatch.build} has no cancellation request`);
+ for (const operation of active) {
+ const observed = await this.#executor.cancelOperation!(snapshot.state, operation, requestedAt);
+ await this.#journal("operation-control", dispatch.build, lease.owner, {
+ operation: observed.id,
+ execution: observed.status,
+ control: observed.cancellation?.status ?? "none",
+ });
+ }
+ const operations = await this.#options.stores.operations.list({ build: dispatch.build });
+ const terminal = operations.filter((item) => item.status === "completed"
+ || item.status === "failed" || item.status === "cancelled");
+ for (const operation of terminal) {
+ await this.#options.stores.dispatch.clearCapacity(
+ capacityReservationId(dispatch.build, operation.command),
+ dispatch.build,
+ lease,
+ );
+ }
+ const remaining = operations.filter((item) => item.status === "created" || item.status === "pending");
+ if (remaining.length > 0) {
+ const wakeAt = Math.min(...remaining.map((item) => item.cancellation?.retryAt
+ ?? item.wakeAt ?? Date.now() + 1_000));
+ return await this.#options.stores.dispatch.release(dispatch.build, lease, {
+ phase: "settling",
+ availableAt: wakeAt,
+ reason: "cancellation is awaiting remote terminal facts",
+ });
+ }
+ return await this.#options.stores.dispatch.finish(
+ dispatch.build,
+ lease,
+ "cancelled",
+ dispatch.cancellation?.reason ?? "cancelled by Runtime authority",
+ );
+ }
+
+ async runOnce(options: { readonly owner: string; readonly leaseMs: number }): Promise<BuildDispatchSnapshot | undefined> {
+ assert(options.owner.trim().length > 0, "Worker owner is empty");
+ const leaseMs = positive(options.leaseMs, "Worker leaseMs");
+ const token = randomUUID();
+ const dispatch = await this.#options.stores.dispatch.claim({
+ owner: options.owner,
+ token,
+ now: Date.now(),
+ leaseMs,
+ });
+ if (dispatch === undefined) return undefined;
+ const lease = dispatch.lease;
+ assert(lease !== undefined, `claimed Dispatch ${dispatch.build} has no lease`);
+ await this.#journal("dispatch-claimed", dispatch.build, options.owner, { fence: lease.fence });
+ if (dispatch.admission !== "open") return await this.#cancel(dispatch, lease);
+ const stored = await this.#options.stores.builds.read(dispatch.build);
+ assert(stored !== undefined, `Dispatch ${dispatch.build} has no BuildState`);
+ assert(stored.state.id === dispatch.core, `Dispatch ${dispatch.build} Core identity differs`);
+ if (this.#options.runtimeClosure !== undefined) {
+ assert(this.#options.runtimeClosure.digest === dispatch.runtimeClosure,
+ `Dispatch ${dispatch.build} Runtime Closure differs`);
+ }
+ const controlled = new CapacityExecutor(this.#executor, this.#options, dispatch.build, lease, leaseMs);
+ const scheduler = this.#options.scheduler.create(controlled, {
+ ...this.#options.scheduling,
+ buildStore: this.#options.stores.builds,
+ ...(this.#options.runtimeClosure === undefined ? {} : { runtimeClosure: this.#options.runtimeClosure }),
+ });
+ let heartbeatError: unknown;
+ let heartbeat = Promise.resolve();
+ const timer = setInterval(() => {
+ heartbeat = heartbeat.then(async () => {
+ await this.#options.stores.dispatch.heartbeat(dispatch.build, lease, Date.now(), leaseMs);
+ }).catch((error: unknown) => { heartbeatError = error; });
+ }, Math.max(50, Math.floor(leaseMs / 3)));
+ try {
+ const [result] = await scheduler.run([{ id: dispatch.build, state: stored.state }]);
+ assert(result !== undefined, `Scheduler returned no result for ${dispatch.build}`);
+ clearInterval(timer);
+ await heartbeat;
+ if (heartbeatError !== undefined) throw heartbeatError;
+ const current = await this.#options.stores.dispatch.read(dispatch.build);
+ assert(current?.phase === "leased" && sameLease(current.lease, lease),
+ `Dispatch ${dispatch.build} lease was fenced before settlement`);
+ if (current.admission !== "open") return await this.#cancel(current, lease);
+ if (result.status === "complete") {
+ const terminal = await this.#options.stores.dispatch.finish(dispatch.build, lease, "complete");
+ await this.#journal("dispatch-terminal", dispatch.build, options.owner, { terminal: "complete" });
+ return terminal;
+ }
+ if (result.status === "failed" || result.journal.some((item) => item.status === "error")) {
+ const reason = result.journal.find((item) => item.status === "error")?.message ?? "Core Build failed";
+ const terminal = await this.#options.stores.dispatch.finish(dispatch.build, lease, "failed", reason);
+ await this.#journal("dispatch-terminal", dispatch.build, options.owner, { terminal: "failed", reason });
+ return terminal;
+ }
+ const pending = result.journal.filter((item) => item.status === "pending");
+ const wakeAt = pending.length === 0
+ ? Date.now()
+ : Math.min(...pending.map((item) => item.wakeAt ?? Date.now() + 1_000));
+ const released = await this.#options.stores.dispatch.release(dispatch.build, lease, {
+ phase: pending.length > 0 ? "waiting" : result.blocked.length > 0 ? "blocked" : "queued",
+ availableAt: wakeAt,
+ ...(result.blocked.length === 0 ? {} : { reason: result.blocked.map((item) => item.reason).join(", ") }),
+ });
+ await this.#journal("dispatch-released", dispatch.build, options.owner, {
+ phase: released.phase,
+ availableAt: released.availableAt,
+ });
+ return released;
+ } catch (error) {
+ clearInterval(timer);
+ await heartbeat;
+ const current = await this.#options.stores.dispatch.read(dispatch.build);
+ if (current?.phase !== "leased" || !sameLease(current.lease, lease)) throw error;
+ if (current.admission !== "open") return await this.#cancel(current, lease);
+ const reason = error instanceof Error ? error.message : String(error);
+ const terminal = await this.#options.stores.dispatch.finish(dispatch.build, lease, "failed", reason);
+ await this.#journal("dispatch-terminal", dispatch.build, options.owner, { terminal: "failed", reason });
+ return terminal;
+ } finally {
+ clearInterval(timer);
+ }
+ }
+
+ async run(options: RuntimeWorkerRunOptions): Promise<void> {
+ positive(options.idlePollMs, "Worker idlePollMs");
+ await this.#journal("worker-started", undefined, options.owner, {});
+ try {
+ while (options.signal?.aborted !== true) {
+ const dispatch = await this.runOnce(options);
+ if (dispatch === undefined) await pause(options.idlePollMs, options.signal).catch((error: unknown) => {
+ if (options.signal?.aborted !== true) throw error;
+ });
+ }
+ } finally {
+ await this.#journal("worker-stopped", undefined, options.owner, {});
+ }
+ }
+}
+
+export const durableLocalWorkerFactory: RuntimeWorkerFactory = {
+ create(executor, options) {
+ return new DurableLocalWorker(executor, options);
+ },
+};
diff --git a/packages/local/test/config.test.ts b/packages/local/test/config.test.ts
index 284bf2f3..d5e330c8 100644
--- a/packages/local/test/config.test.ts
+++ b/packages/local/test/config.test.ts
@@ -16,35 +16,41 @@ import {
RuntimeAdapterRegistry,
} from "@narratage/local";
-test("declarative Runtime config starts the domain-neutral local defaults", async () => {
- const root = await mkdtemp(join(tmpdir(), "svml-runtime-config-"));
- const path = join(root, "svml.runtime.json");
- await writeFile(path, JSON.stringify({
+const services = {
+ scheduler: "execution.scheduler",
+ worker: "execution.worker",
+ stores: {
+ build: "state.builds",
+ operations: "state.operations",
+ dispatch: "state.dispatch",
+ journal: "state.journal",
+ artifacts: "artifacts",
+ credentials: ["credentials"],
+ },
+};
+
+const required = { runtimeServices: [], services, scheduling: { maxConcurrency: 3 } } as const;
+
+test("declarative Runtime config has no implicit local services", () => {
+ assert.throws(() => parseRuntimeConfig({
format: "svml.runtime-config@1",
endpoints: [],
permissions: [],
- scheduling: { maxConcurrency: 3, lanes: { generation: 2 } },
- }));
- const runtime = await createRuntimeFromConfig(path, { registry: new RuntimeAdapterRegistry() });
- try {
- assert.equal((await runtime.status("absent")).build, undefined);
- } finally {
- await runtime.close();
- }
+ }), /runtimeServices/u);
});
test("Runtime config is closed data and rejects unknown environment authority", () => {
const parsed = parseRuntimeConfig({
format: "svml.runtime-config@1",
+ ...required,
packageRoot: "/opt/narratage",
- catalogPath: ".svml/catalog.sqlite",
endpoints: [],
permissions: [],
});
- assert.equal(parsed.catalogPath, ".svml/catalog.sqlite");
assert.equal(parsed.packageRoot, "/opt/narratage");
assert.throws(() => parseRuntimeConfig({
format: "svml.runtime-config@1",
+ ...required,
endpoints: [],
permissions: [],
apiKey: "must-not-live-here",
@@ -56,6 +62,7 @@ test("declarative adapters are explicit and never guessed", async () => {
const path = join(root, "svml.runtime.json");
await writeFile(path, JSON.stringify({
format: "svml.runtime-config@1",
+ ...required,
endpoints: [{ use: "example.missing", instance: "missing", config: {} }],
permissions: [],
}));
@@ -70,6 +77,7 @@ test("runtimeServices names the Runtime's own replaceable parts, apart from exte
const path = join(root, "svml.runtime.json");
await writeFile(path, JSON.stringify({
format: "svml.runtime-config@1",
+ ...required,
runtimeServices: [{ use: "@example/store", instance: "artifacts.example" }],
endpoints: [],
permissions: [],
@@ -77,17 +85,7 @@ test("runtimeServices names the Runtime's own replaceable parts, apart from exte
const document = parseRuntimeConfig(JSON.parse(await readFile(path, "utf8")));
assert.deepEqual(document.runtimeServices, [{ use: "@example/store", instance: "artifacts.example" }]);
- await writeFile(path, JSON.stringify({
- format: "svml.runtime-config@1",
- services: [],
- endpoints: [],
- permissions: [],
- }));
- await assert.rejects(
- async () => parseRuntimeConfig(JSON.parse(await readFile(path, "utf8"))),
- /does not accept services/u,
- "the word services now belongs to the external processes a deployment must have running",
- );
+ assert.deepEqual(document.services, services);
await rm(root, { recursive: true, force: true });
});
@@ -96,6 +94,7 @@ test("doctor names the external program a Provider needs, and the command that s
const path = join(root, "svml.runtime.json");
await writeFile(path, JSON.stringify({
format: "svml.runtime-config@1",
+ ...required,
endpoints: [
{ use: "example.absent", instance: "absent", config: {} },
{ use: "example.wrong", instance: "wrong", config: {} },
@@ -146,6 +145,7 @@ test("doctor validates closed config without constructing adapters or cascading
const path = join(root, "svml.runtime.json");
await writeFile(path, JSON.stringify({
format: "svml.runtime-config@1",
+ ...required,
runtimeServices: [{ use: "example.store", instance: "store", config: { mode: "valid" } }],
endpoints: [
{ use: "example.invalid", instance: "invalid", config: { mode: "bad" } },
diff --git a/packages/local/test/external-services.test.ts b/packages/local/test/external-services.test.ts
index 1b44c520..05589051 100644
--- a/packages/local/test/external-services.test.ts
+++ b/packages/local/test/external-services.test.ts
@@ -25,12 +25,26 @@ async function project(service: (root: string) => RuntimeExternalService) {
const path = join(root, "svml.runtime.json");
await writeFile(path, JSON.stringify({
format: "svml.runtime-config@1",
+ runtimeServices: [],
+ services: {
+ scheduler: "execution.scheduler",
+ worker: "execution.worker",
+ stores: {
+ build: "state.builds",
+ operations: "state.operations",
+ dispatch: "state.dispatch",
+ journal: "state.journal",
+ artifacts: "artifacts",
+ credentials: ["credentials"],
+ },
+ },
endpoints: [
{ use: "example.program", instance: "one", config: {} },
// A second Endpoint driving the same program: it is brought up once.
{ use: "example.program", instance: "two", config: {} },
],
permissions: [],
+ scheduling: { maxConcurrency: 1 },
}));
const registry = new RuntimeAdapterRegistry();
registry.registerFacet(createRuntimeEndpointAdapterFacet({
diff --git a/packages/local/test/local.test.ts b/packages/local/test/local.test.ts
index be3406b9..0aaa30e6 100644
--- a/packages/local/test/local.test.ts
+++ b/packages/local/test/local.test.ts
@@ -10,21 +10,25 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";
-import { FileArtifactStore } from "@narratage/artifact-store-fs";
+import { FileArtifactStore, createFileArtifactStorePackage } from "@narratage/artifact-store-fs";
+import { createEnvironmentCredentialStorePackage } from "@narratage/credential-store-env";
import { MemoryArtifactStore } from "@narratage/driver-node";
+import { defineEndpointPackage } from "@narratage/endpoint-kit";
import type { RecoverableEndpoint } from "@narratage/endpoint-kit";
import type {
ComponentPackage,
EndpointPackage,
} from "@narratage/local";
-import { createLocalRuntime, createProjectLocalRuntime } from "@narratage/local";
+import { createLocalExecutionPackage, createProjectLocalRuntime } from "@narratage/local";
import { digestOf } from "@narratage/core";
import {
createNodePackageLock,
writeNodePackageLock,
} from "@narratage/package-loader-node";
-import { LocalBuildScheduler, MemoryBuildStore, defineRuntimeServicePackage } from "@narratage/runtime";
+import { LocalBuildScheduler, credentialRef, defineRuntimeServicePackage } from "@narratage/runtime";
+import type { CredentialValue, WritableCredentialStore } from "@narratage/runtime";
import type { RuntimeModuleManifest } from "@narratage/runtime";
+import { createSqliteRuntimeServicePackage } from "@narratage/store-sqlite";
import {
capabilities,
@@ -53,15 +57,46 @@ const providerManifest: RuntimeModuleManifest = {
}],
};
+function projectRuntimeFixture(directory: string) {
+ const execution = createLocalExecutionPackage("execution.local");
+ const state = createSqliteRuntimeServicePackage({
+ path: join(directory, ".svml", "runtime.sqlite"),
+ name: "state.sqlite",
+ buildInstance: "state.builds",
+ operationInstance: "state.operations",
+ dispatchInstance: "state.dispatch",
+ journalInstance: "state.journal",
+ });
+ const artifacts = createFileArtifactStorePackage({
+ root: join(directory, ".svml", "artifacts"),
+ instance: "artifacts.fs",
+ });
+ const credentials = createEnvironmentCredentialStorePackage({ instance: "credentials.env" });
+ return {
+ runtimeServices: [execution, state, artifacts, credentials],
+ runtimeSelection: {
+ scheduler: "execution.local.scheduler",
+ worker: "execution.local.worker",
+ stores: {
+ build: "state.builds",
+ operations: "state.operations",
+ dispatch: "state.dispatch",
+ journal: "state.journal",
+ artifacts: "artifacts.fs",
+ credentials: ["credentials.env"],
+ },
+ },
+ allowedPermissions: ["filesystem:state", "filesystem:artifacts", "environment:credentials"],
+ scheduling: { maxConcurrency: 4 },
+ } as const;
+}
+
test("Artifact GC is explicit, dry-run by default, and only removes unreachable managed bytes", async () => {
const directory = await mkdtemp(join(tmpdir(), "svml-local-artifact-gc-"));
try {
- const artifacts = new FileArtifactStore(join(directory, "artifacts"));
+ const artifacts = new FileArtifactStore(join(directory, ".svml", "artifacts"));
const orphan = await artifacts.put(new TextEncoder().encode("orphan"), "application/octet-stream");
- const runtime = await createLocalRuntime({
- buildStore: new MemoryBuildStore(),
- artifactStore: artifacts,
- });
+ const runtime = await createProjectLocalRuntime({ root: directory, ...projectRuntimeFixture(directory) });
const preview = await runtime.garbageCollectArtifacts();
assert.deepEqual(preview.unreachable, [orphan.digest]);
assert.deepEqual(preview.deleted, []);
@@ -74,6 +109,70 @@ test("Artifact GC is explicit, dry-run by default, and only removes unreachable
}
});
+test("Endpoint-declared credentials use the selected writable Store without a Provider switch", async () => {
+ const directory = await mkdtemp(join(tmpdir(), "svml-local-auth-"));
+ const values = new Map<string, CredentialValue>();
+ const credentialStore: WritableCredentialStore = {
+ owns(ref) { return ref.store === "memory"; },
+ async resolve(ref) { return ref.store === "memory" ? values.get(ref.key) : undefined; },
+ async put(ref, value) { values.set(ref.key, value); },
+ async delete(ref) { return values.delete(ref.key); },
+ };
+ const memoryCredentials = defineRuntimeServicePackage({
+ name: "credentials.memory",
+ module: { name: "example.credentials-memory", version: "1" },
+ services: [{
+ role: "credential-store",
+ facet: "credential-store",
+ instance: "credentials.memory",
+ implementation: { locator: "example.credentials-memory", digest: digestOf("credentials-memory@1") },
+ service: credentialStore,
+ }],
+ });
+ const endpoint = defineEndpointPackage({
+ module: providerModule,
+ facet: "generation",
+ instance: "generation.auth-test",
+ implementation: { locator: "example.local-endpoint/generation", digest: providerDigest },
+ credentials: { apiKey: credentialRef("memory", "generation.api-key") },
+ credentialInputs: { apiKey: { label: "Generation API key" } },
+ capabilities: [{
+ capability: capabilities.generation,
+ returns: types.generated,
+ lifecycle: "recoverable",
+ endpoint: {
+ start() { throw new Error("unused"); },
+ resume() { throw new Error("unused"); },
+ },
+ }],
+ });
+ try {
+ const base = projectRuntimeFixture(directory);
+ const runtime = await createProjectLocalRuntime({
+ root: directory,
+ ...base,
+ runtimeServices: [...base.runtimeServices.slice(0, -1), memoryCredentials],
+ runtimeSelection: {
+ ...base.runtimeSelection,
+ stores: { ...base.runtimeSelection.stores, credentials: ["credentials.memory"] },
+ },
+ allowedPermissions: ["filesystem:state", "filesystem:artifacts"],
+ endpoints: [endpoint],
+ });
+ assert.deepEqual((await runtime.credentials("generation.auth-test")).map((item) => ({
+ slot: item.slot, configured: item.configured, writable: item.writable,
+ })), [{ slot: "apiKey", configured: false, writable: true }]);
+ const stored = await runtime.putCredential("generation.auth-test", "apiKey", "secret");
+ assert.equal(stored.configured, true);
+ const removed = await runtime.deleteCredential("generation.auth-test", "apiKey");
+ assert.equal(removed.deleted, true);
+ assert.equal(removed.credential.configured, false);
+ await runtime.close();
+ } finally {
+ await rm(directory, { recursive: true, force: true });
+ }
+});
+
test("project local runtime resumes durable work while component and endpoint packages stay replaceable", async () => {
const directory = await mkdtemp(join(tmpdir(), "svml-local-"));
const initial = createGreetingBuild();
@@ -133,7 +232,7 @@ test("project local runtime resumes durable work while component and endpoint pa
const recoverableEndpoint: RecoverableEndpoint = {
start({ operation }) {
starts += 1;
- return { status: "pending", checkpoint: { remoteJob: operation.submissionKey } };
+ return { status: "pending", checkpoint: { remoteJob: operation.submissionKey }, wakeAt: Date.now() };
},
resume({ checkpoint }) {
resumes += 1;
@@ -150,6 +249,7 @@ test("project local runtime resumes durable work while component and endpoint pa
},
cancel() {
cancels += 1;
+ return { status: "confirmed" };
},
};
const endpointPackage: EndpointPackage = {
@@ -161,6 +261,7 @@ test("project local runtime resumes durable work while component and endpoint pa
returns: types.generated,
endpoint: "generation.personal",
}],
+ credentials: [],
install(registry) {
registry.registerRecoverableEndpoint(
"generation.personal",
@@ -181,22 +282,26 @@ test("project local runtime resumes durable work while component and endpoint pa
try {
const firstRuntime = await createProjectLocalRuntime({
root: directory,
+ ...projectRuntimeFixture(directory),
components: [components],
endpoints: [endpointPackage],
});
const first = await firstRuntime.build({ id: "greeting-build", state: initial, catalog });
- assert.equal(first.status, "paused");
+ assert.equal(first.status, "queued");
+ assert.equal((await firstRuntime.workOnce({ owner: "worker-one", leaseMs: 5_000 }))?.phase, "waiting");
assert.equal(starts, 1);
assert.equal(resumes, 0);
await firstRuntime.close();
const secondRuntime = await createProjectLocalRuntime({
root: directory,
+ ...projectRuntimeFixture(directory),
components: [components],
endpoints: [endpointPackage],
});
const second = await secondRuntime.build({ id: "greeting-build", state: createGreetingBuild(), catalog });
- assert.equal(second.status, "complete");
+ assert.equal(second.status, "waiting");
+ assert.equal((await secondRuntime.workOnce({ owner: "worker-two", leaseMs: 5_000 }))?.terminal, "complete");
assert.equal(starts, 1);
assert.equal(resumes, 1);
assert.equal(promptCalls, 1, "persisted Core facts stop deterministic upstream replay");
@@ -207,6 +312,9 @@ test("project local runtime resumes durable work while component and endpoint pa
assert.equal(clientStatus.catalog?.aliases[0]?.name, "final.document");
assert.deepEqual((await secondRuntime.builds()).map((item) => item.build), ["greeting-build"]);
+ await secondRuntime.build({ id: "greeting-follow", state: createGreetingBuild() });
+ await secondRuntime.workOnce({ owner: "worker-two", leaseMs: 5_000 });
+ await secondRuntime.workOnce({ owner: "worker-two", leaseMs: 5_000 });
const followed = await secondRuntime.build(
{ id: "greeting-follow", state: createGreetingBuild() },
{ follow: true, pollIntervalMs: 1, maxWaitMs: 1_000 },
@@ -219,9 +327,12 @@ test("project local runtime resumes durable work while component and endpoint pa
assert.equal(followedStatus.operations.length, 1);
const waiting = await secondRuntime.build({ id: "greeting-cancel", state: createGreetingBuild() });
- assert.equal(waiting.status, "paused");
- const cancelled = await secondRuntime.cancel("greeting-cancel");
- assert.equal(cancelled?.status, "failed");
+ assert.equal(waiting.status, "queued");
+ await secondRuntime.workOnce({ owner: "worker-two", leaseMs: 5_000 });
+ const requested = await secondRuntime.cancel("greeting-cancel");
+ assert.equal(requested?.admission, "closing");
+ const cancelled = await secondRuntime.workOnce({ owner: "worker-two", leaseMs: 5_000 });
+ assert.equal(cancelled?.terminal, "cancelled");
assert.equal(cancels, 1);
await secondRuntime.close();
} finally {
@@ -287,16 +398,17 @@ test("project local runtime activates locked compute facets without deployment s
await writeNodePackageLock(lockPath, lock);
const runtime = await createProjectLocalRuntime({
root: runtimeRoot,
+ ...projectRuntimeFixture(runtimeRoot),
packageRoot: installedRoot,
packageLock: "../svml.packages.lock",
});
- await assert.rejects(
- async () => await runtime.build({
- id: "unlocked-preview",
- state: createGreetingBuild({ generationRealization: "placeholder", goalAccepts: "substitute" }),
- }),
- /does not bind this Host's implementation package closure/u,
- );
+ await runtime.build({
+ id: "unlocked-preview",
+ state: createGreetingBuild({ generationRealization: "placeholder", goalAccepts: "substitute" }),
+ });
+ const refused = await runtime.workOnce({ owner: "locked-test", leaseMs: 5_000 });
+ assert.equal(refused?.terminal, "failed");
+ assert.match(refused?.reason ?? "", /does not bind this Host's implementation package closure/u);
const result = await runtime.build({
id: "locked-preview",
state: createGreetingBuild({
@@ -305,15 +417,18 @@ test("project local runtime activates locked compute facets without deployment s
implementationClosure: lock.digest,
}),
});
- assert.equal(result.status, "complete");
- assert.equal(result.state.records.some((record) => record.id === "document:root"), true);
+ assert.equal(result.status, "queued");
+ const completed = await runtime.workOnce({ owner: "locked-test", leaseMs: 5_000 });
+ assert.equal(completed?.terminal, "complete");
+ assert.equal((await runtime.status("locked-preview")).build?.state.records
+ .some((record) => record.id === "document:root"), true);
await runtime.close();
} finally {
await rm(directory, { recursive: true, force: true });
}
});
-test("project local runtime infers one supplied Scheduler service without knowing its package", async () => {
+test("project local runtime uses only the explicitly selected Scheduler", async () => {
const directory = await mkdtemp(join(tmpdir(), "svml-local-scheduler-service-"));
let creates = 0;
const scheduler = defineRuntimeServicePackage({
@@ -335,10 +450,15 @@ test("project local runtime infers one supplied Scheduler service without knowin
}],
});
try {
+ const base = projectRuntimeFixture(directory);
const runtime = await createProjectLocalRuntime({
root: directory,
- runtimeServices: [scheduler],
+ ...base,
+ runtimeServices: [...base.runtimeServices, scheduler],
+ runtimeSelection: { ...base.runtimeSelection, scheduler: "scheduler.example" },
});
+ await runtime.build({ id: "scheduler-selection", state: createGreetingBuild() });
+ await runtime.workOnce({ owner: "scheduler-test", leaseMs: 5_000 });
assert.equal(creates, 1);
await runtime.close();
} finally {
@@ -346,7 +466,7 @@ test("project local runtime infers one supplied Scheduler service without knowin
}
});
-test("two Scheduler services are refused and neither gains authority", async () => {
+test("two installed Schedulers are unambiguous because the Profile selects one", async () => {
const directory = await mkdtemp(join(tmpdir(), "svml-local-scheduler-selection-"));
const creates = { one: 0, two: 0 };
const closes = { one: 0, two: 0 };
@@ -371,23 +491,18 @@ test("two Scheduler services are refused and neither gains authority", async ()
close() { closes[name] += 1; },
});
try {
- // Two schedulers is a configuration to correct, not a choice to make later.
- await assert.rejects(
- createProjectLocalRuntime({
- root: directory,
- runtimeServices: [scheduler("one"), scheduler("two")],
- }),
- /configures 2 scheduler implementations/u,
- );
- assert.deepEqual(closes, { one: 1, two: 1 });
-
+ const base = projectRuntimeFixture(directory);
const runtime = await createProjectLocalRuntime({
root: directory,
- runtimeServices: [scheduler("two")],
+ ...base,
+ runtimeServices: [...base.runtimeServices, scheduler("one"), scheduler("two")],
+ runtimeSelection: { ...base.runtimeSelection, scheduler: "scheduler.two" },
});
+ await runtime.build({ id: "scheduler-two", state: createGreetingBuild() });
+ await runtime.workOnce({ owner: "scheduler-test", leaseMs: 5_000 });
assert.deepEqual(creates, { one: 0, two: 1 });
await runtime.close();
- assert.deepEqual(closes, { one: 1, two: 2 });
+ assert.deepEqual(closes, { one: 1, two: 1 });
} finally {
await rm(directory, { recursive: true, force: true });
}
@@ -414,14 +529,29 @@ test("project local runtime accepts a permission-checked replacement ArtifactSto
}],
});
try {
+ const denied = projectRuntimeFixture(directory);
await assert.rejects(
- createProjectLocalRuntime({ root: directory, runtimeServices: [artifacts] }),
+ createProjectLocalRuntime({
+ root: directory,
+ ...denied,
+ runtimeServices: [...denied.runtimeServices, artifacts],
+ runtimeSelection: {
+ ...denied.runtimeSelection,
+ stores: { ...denied.runtimeSelection.stores, artifacts: "artifacts.remote" },
+ },
+ }),
/disallowed Runtime permission network:remote-artifacts/u,
);
+ const allowed = projectRuntimeFixture(directory);
const runtime = await createProjectLocalRuntime({
root: directory,
- runtimeServices: [artifacts],
- allowedPermissions: ["network:remote-artifacts"],
+ ...allowed,
+ runtimeServices: [...allowed.runtimeServices, artifacts],
+ runtimeSelection: {
+ ...allowed.runtimeSelection,
+ stores: { ...allowed.runtimeSelection.stores, artifacts: "artifacts.remote" },
+ },
+ allowedPermissions: [...allowed.allowedPermissions, "network:remote-artifacts"],
});
const bytes = new Uint8Array([7, 8, 9]);
const sourceArtifact = {
@@ -431,14 +561,14 @@ test("project local runtime accepts a permission-checked replacement ArtifactSto
mediaType: "application/octet-stream",
};
assert.equal(await artifactStore.has(sourceArtifact.digest), false);
- await assert.rejects(
- runtime.build({
- id: "source-artifact-staging",
- state: createGreetingBuild(),
- attachments: [{ artifact: sourceArtifact, bytes }],
- }),
- /does not bind demanded capability/u,
- );
+ await runtime.build({
+ id: "source-artifact-staging",
+ state: createGreetingBuild(),
+ attachments: [{ artifact: sourceArtifact, bytes }],
+ });
+ const blocked = await runtime.workOnce({ owner: "artifact-test", leaseMs: 5_000 });
+ assert.equal(blocked?.terminal, "failed");
+ assert.match(blocked?.reason ?? "", /does not bind demanded capability/u);
assert.deepEqual(await artifactStore.get(sourceArtifact.digest), bytes);
await assert.rejects(
runtime.build({
diff --git a/packages/provider-google-vertex/src/activation.ts b/packages/provider-google-vertex/src/activation.ts
index 77b1e97c..24cb589f 100644
--- a/packages/provider-google-vertex/src/activation.ts
+++ b/packages/provider-google-vertex/src/activation.ts
@@ -1,6 +1,6 @@
-import { credentialRef } from "@narratage/runtime";
import {
createRuntimeEndpointAdapterFacet,
+ runtimeConfigCredentialRef,
runtimeConfigExact,
runtimeConfigObject,
runtimeConfigPositiveInteger,
@@ -15,16 +15,18 @@ const googleVertexRuntimeAdapter = createRuntimeEndpointAdapterFacet({
validate(context) {
const config = runtimeConfigObject(context.config, "Google Vertex");
runtimeConfigExact(config, [
- "project", "projectEnv", "location", "credentialsEnv", "defaultConcurrency",
+ "project", "projectEnv", "location", "credentials", "defaultConcurrency",
"requestTimeoutMs", "maxResponseBytes",
], "Google Vertex");
const project = runtimeConfigString(config.project, "Google Vertex project");
const projectEnv = runtimeConfigString(config.projectEnv, "Google Vertex projectEnv");
- if (project !== undefined && projectEnv !== undefined) {
- throw new Error("Google Vertex accepts project or projectEnv, not both");
+ if ((project === undefined) === (projectEnv === undefined)) {
+ throw new Error("Google Vertex requires exactly one of project or projectEnv");
}
runtimeConfigString(config.location, "Google Vertex location");
- runtimeConfigString(config.credentialsEnv, "Google Vertex credentialsEnv");
+ if (runtimeConfigCredentialRef(config.credentials, "Google Vertex credentials") === undefined) {
+ throw new Error("Google Vertex credentials CredentialRef is required");
+ }
runtimeConfigPositiveInteger(config.defaultConcurrency, "Google Vertex defaultConcurrency");
runtimeConfigPositiveInteger(config.requestTimeoutMs, "Google Vertex requestTimeoutMs");
runtimeConfigPositiveInteger(config.maxResponseBytes, "Google Vertex maxResponseBytes");
@@ -32,24 +34,25 @@ const googleVertexRuntimeAdapter = createRuntimeEndpointAdapterFacet({
create(context) {
const config = runtimeConfigObject(context.config, "Google Vertex");
runtimeConfigExact(config, [
- "project", "projectEnv", "location", "credentialsEnv", "defaultConcurrency",
+ "project", "projectEnv", "location", "credentials", "defaultConcurrency",
"requestTimeoutMs", "maxResponseBytes",
], "Google Vertex");
const projectValue = runtimeConfigString(config.project, "Google Vertex project");
const projectEnv = runtimeConfigString(config.projectEnv, "Google Vertex projectEnv");
- if (projectValue !== undefined && projectEnv !== undefined) {
- throw new Error("Google Vertex accepts project or projectEnv, not both");
+ if ((projectValue === undefined) === (projectEnv === undefined)) {
+ throw new Error("Google Vertex requires exactly one of project or projectEnv");
}
- const project = projectValue ?? (projectEnv === undefined ? undefined : process.env[projectEnv]?.trim());
- if (project === undefined || project.length === 0) throw new Error("Google Vertex project is required");
- const credentialsEnv = runtimeConfigString(config.credentialsEnv, "Google Vertex credentialsEnv");
+ const project = projectValue ?? process.env[projectEnv!]?.trim();
+ if (project === undefined || project.length === 0) throw new Error(`Google Vertex project environment ${projectEnv} is empty`);
+ const credentials = runtimeConfigCredentialRef(config.credentials, "Google Vertex credentials");
+ if (credentials === undefined) throw new Error("Google Vertex credentials CredentialRef is required");
return createGoogleVertexCaptionProvider({
project,
instance: context.instance,
...(context.lane === undefined ? {} : { lane: context.lane }),
...(runtimeConfigString(config.location, "Google Vertex location") === undefined
? {} : { location: config.location as string }),
- ...(credentialsEnv === undefined ? {} : { credentialsJson: credentialRef("env", credentialsEnv) }),
+ credentialsJson: credentials,
...(runtimeConfigPositiveInteger(config.defaultConcurrency, "Google Vertex defaultConcurrency") === undefined
? {} : { defaultConcurrency: config.defaultConcurrency as number }),
...(runtimeConfigPositiveInteger(config.requestTimeoutMs, "Google Vertex requestTimeoutMs") === undefined
@@ -60,16 +63,8 @@ const googleVertexRuntimeAdapter = createRuntimeEndpointAdapterFacet({
},
doctor(context) {
const config = runtimeConfigObject(context.config, "Google Vertex");
- const project = runtimeConfigString(config.project, "Google Vertex project");
const projectEnv = runtimeConfigString(config.projectEnv, "Google Vertex projectEnv");
- const credentialsEnv = runtimeConfigString(config.credentialsEnv, "Google Vertex credentialsEnv")
- ?? "GOOGLE_APPLICATION_CREDENTIALS_JSON";
- return [
- ...(project !== undefined
- ? []
- : diagnoseRuntimeEnvironmentCredential(projectEnv ?? "GOOGLE_CLOUD_PROJECT", "Google Vertex project")),
- ...diagnoseRuntimeEnvironmentCredential(credentialsEnv, "Google Vertex credentials"),
- ];
+ return projectEnv === undefined ? [] : diagnoseRuntimeEnvironmentCredential(projectEnv, "Google Vertex project");
},
});
diff --git a/packages/provider-google-vertex/src/provider.ts b/packages/provider-google-vertex/src/provider.ts
index 01577aa6..e2913de0 100644
--- a/packages/provider-google-vertex/src/provider.ts
+++ b/packages/provider-google-vertex/src/provider.ts
@@ -217,6 +217,9 @@ export function createGoogleVertexCaptionProvider(options: CreateGoogleVertexCap
credentials: {
googleCredentials: options.credentialsJson ?? credentialRef("env", "GOOGLE_APPLICATION_CREDENTIALS_JSON"),
},
+ credentialInputs: {
+ googleCredentials: { label: "Google Cloud credentials JSON", kind: "json" },
+ },
defaultConcurrency: options.defaultConcurrency ?? 2,
capabilities: [{
capability: captionGeminiCapabilities.plan,
diff --git a/packages/provider-hyperframes-aws-lambda/src/provider.ts b/packages/provider-hyperframes-aws-lambda/src/provider.ts
index 5f960484..c318068e 100644
--- a/packages/provider-hyperframes-aws-lambda/src/provider.ts
+++ b/packages/provider-hyperframes-aws-lambda/src/provider.ts
@@ -654,6 +654,7 @@ export function createAwsLambdaHyperframesProvider(config: CreateAwsLambdaHyperf
} catch (error) {
if (!alreadyStopped(error)) throw error;
}
+ return { status: "accepted" };
},
};
diff --git a/packages/provider-kie/package.json b/packages/provider-kie/package.json
index 9b9a0c52..0dde2d5d 100644
--- a/packages/provider-kie/package.json
+++ b/packages/provider-kie/package.json
@@ -16,13 +16,13 @@
"@narratage/generation": "workspace:*",
"@narratage/protocol": "workspace:*",
"@narratage/runtime": "workspace:*",
- "@narratage/runtime-adapter": "workspace:*",
- "@narratage/runtime-adapter-node": "workspace:*"
+ "@narratage/runtime-adapter": "workspace:*"
},
"devDependencies": {
"@narratage/artifact-store-fs": "workspace:*",
"@narratage/component-kit": "workspace:*",
"@narratage/core": "workspace:*",
+ "@narratage/credential-store-env": "workspace:*",
"@narratage/driver-node": "workspace:*",
"@narratage/elaborator": "workspace:*",
"@narratage/gemini-omni": "workspace:*",
@@ -36,6 +36,7 @@
"@narratage/narrative": "workspace:*",
"@narratage/seedance": "workspace:*",
"@narratage/seedream": "workspace:*",
+ "@narratage/store-sqlite": "workspace:*",
"@narratage/validation": "workspace:*"
}
}
diff --git a/packages/provider-kie/src/activation.ts b/packages/provider-kie/src/activation.ts
index a82c8761..ece34993 100644
--- a/packages/provider-kie/src/activation.ts
+++ b/packages/provider-kie/src/activation.ts
@@ -1,12 +1,11 @@
-import { credentialRef } from "@narratage/runtime";
import {
createRuntimeEndpointAdapterFacet,
+ runtimeConfigCredentialRef,
runtimeConfigExact,
runtimeConfigObject,
runtimeConfigPositiveInteger,
runtimeConfigString,
} from "@narratage/runtime-adapter";
-import { diagnoseRuntimeEnvironmentCredential } from "@narratage/runtime-adapter-node";
import { createKieProvider } from "./provider.js";
@@ -15,7 +14,7 @@ const kieRuntimeAdapter = createRuntimeEndpointAdapterFacet({
validate(context) {
const config = runtimeConfigObject(context.config, "KIE");
runtimeConfigExact(config, [
- "apiBaseUrl", "uploadBaseUrl", "apiKeyEnv", "defaultConcurrency", "pollIntervalMs",
+ "apiBaseUrl", "uploadBaseUrl", "apiKey", "defaultConcurrency", "pollIntervalMs",
"submissionIntervalMs", "requestTimeoutMs", "maxOperationMs", "maxArtifactBytes",
], "KIE");
for (const [key, subject] of [["apiBaseUrl", "KIE apiBaseUrl"], ["uploadBaseUrl", "KIE uploadBaseUrl"]] as const) {
@@ -26,7 +25,9 @@ const kieRuntimeAdapter = createRuntimeEndpointAdapterFacet({
throw new Error(`${subject} must use HTTPS or localhost`);
}
}
- runtimeConfigString(config.apiKeyEnv, "KIE apiKeyEnv");
+ if (runtimeConfigCredentialRef(config.apiKey, "KIE apiKey") === undefined) {
+ throw new Error("KIE apiKey CredentialRef is required");
+ }
runtimeConfigPositiveInteger(config.defaultConcurrency, "KIE defaultConcurrency");
runtimeConfigPositiveInteger(config.pollIntervalMs, "KIE pollIntervalMs");
runtimeConfigPositiveInteger(config.submissionIntervalMs, "KIE submissionIntervalMs");
@@ -37,10 +38,11 @@ const kieRuntimeAdapter = createRuntimeEndpointAdapterFacet({
create(context) {
const config = runtimeConfigObject(context.config, "KIE");
runtimeConfigExact(config, [
- "apiBaseUrl", "uploadBaseUrl", "apiKeyEnv", "defaultConcurrency", "pollIntervalMs",
+ "apiBaseUrl", "uploadBaseUrl", "apiKey", "defaultConcurrency", "pollIntervalMs",
"submissionIntervalMs", "requestTimeoutMs", "maxOperationMs", "maxArtifactBytes",
], "KIE");
- const apiKeyEnv = runtimeConfigString(config.apiKeyEnv, "KIE apiKeyEnv");
+ const apiKey = runtimeConfigCredentialRef(config.apiKey, "KIE apiKey");
+ if (apiKey === undefined) throw new Error("KIE apiKey CredentialRef is required");
return createKieProvider({
instance: context.instance,
...(context.lane === undefined ? {} : { lane: context.lane }),
@@ -48,7 +50,7 @@ const kieRuntimeAdapter = createRuntimeEndpointAdapterFacet({
? {} : { apiBaseUrl: config.apiBaseUrl as string }),
...(runtimeConfigString(config.uploadBaseUrl, "KIE uploadBaseUrl") === undefined
? {} : { uploadBaseUrl: config.uploadBaseUrl as string }),
- ...(apiKeyEnv === undefined ? {} : { apiKey: credentialRef("env", apiKeyEnv) }),
+ apiKey,
...(runtimeConfigPositiveInteger(config.defaultConcurrency, "KIE defaultConcurrency") === undefined
? {} : { defaultConcurrency: config.defaultConcurrency as number }),
...(runtimeConfigPositiveInteger(config.pollIntervalMs, "KIE pollIntervalMs") === undefined
@@ -63,11 +65,6 @@ const kieRuntimeAdapter = createRuntimeEndpointAdapterFacet({
? {} : { maxArtifactBytes: config.maxArtifactBytes as number }),
});
},
- doctor(context) {
- const config = runtimeConfigObject(context.config, "KIE");
- const apiKeyEnv = runtimeConfigString(config.apiKeyEnv, "KIE apiKeyEnv") ?? "KIE_API_KEY";
- return diagnoseRuntimeEnvironmentCredential(apiKeyEnv, "KIE");
- },
});
export const svmlPackage = {
diff --git a/packages/provider-kie/src/provider.ts b/packages/provider-kie/src/provider.ts
index d370dbb4..ab1ac24e 100644
--- a/packages/provider-kie/src/provider.ts
+++ b/packages/provider-kie/src/provider.ts
@@ -581,6 +581,7 @@ export function createKieProvider(config: CreateKieProviderOptions = {}) {
fetchImplementationDigest,
}),
credentials: { apiKey: config.apiKey ?? credentialRef("env", "KIE_API_KEY") },
+ credentialInputs: { apiKey: { label: "KIE API key" } },
defaultConcurrency: config.defaultConcurrency ?? 2,
capabilities: kieRoutes.map((route) => ({
capability: route.capability,
diff --git a/packages/provider-kie/test/live-smoke.ts b/packages/provider-kie/test/live-smoke.ts
index 7a8e8631..d16df1ef 100644
--- a/packages/provider-kie/test/live-smoke.ts
+++ b/packages/provider-kie/test/live-smoke.ts
@@ -1,7 +1,8 @@
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
-import { FileArtifactStore } from "@narratage/artifact-store-fs";
+import { FileArtifactStore, createFileArtifactStorePackage } from "@narratage/artifact-store-fs";
+import { createEnvironmentCredentialStorePackage } from "@narratage/credential-store-env";
import { registerTypeValidatorFacets } from "@narratage/component-kit";
import {
createResolvedClosure,
@@ -39,7 +40,8 @@ import {
grokImagineManifest,
sealGrokImagineRequest,
} from "@narratage/grok-imagine";
-import { createProjectLocalRuntime } from "@narratage/local";
+import { createLocalExecutionPackage, createProjectLocalRuntime } from "@narratage/local";
+import type { LocalBuildSubmission } from "@narratage/local";
import {
minimaxH3Component,
minimaxH3Endpoints,
@@ -61,7 +63,7 @@ import type {
TypedRecord,
} from "@narratage/protocol";
import { credentialRef } from "@narratage/runtime";
-import type { ScheduledBuildResult } from "@narratage/runtime";
+import { createSqliteRuntimeServicePackage } from "@narratage/store-sqlite";
import {
sealSeedanceRequest,
seedanceComponent,
@@ -396,10 +398,9 @@ async function inspectArtifact(root: string, item: SmokeCase, artifact: BlobRef)
return inspectPath;
}
-function failureMessage(item: SmokeCase, build: ScheduledBuildResult): string {
+function failureMessage(item: SmokeCase, build: LocalBuildSubmission): string {
const diagnostic = build.state.diagnostics.map((entry) => `${entry.code}: ${entry.message}`).join("; ");
- const journal = build.journal.map((entry) => `${entry.status}:${entry.message ?? entry.command}`).join("; ");
- return `${item.key} KIE smoke Build ${build.status}: ${diagnostic || journal || "no diagnostic"}`;
+ return `${item.key} KIE smoke Build ${build.status}: ${diagnostic || build.dispatch.reason || "no diagnostic"}`;
}
async function main(): Promise<void> {
@@ -421,16 +422,51 @@ async function main(): Promise<void> {
defaultConcurrency: 1,
pollIntervalMs: 3_000,
});
+ const execution = createLocalExecutionPackage("execution.local");
+ const state = createSqliteRuntimeServicePackage({
+ path: join(root, ".svml", "runtime.sqlite"),
+ name: "state.sqlite",
+ buildInstance: "state.builds",
+ operationInstance: "state.operations",
+ dispatchInstance: "state.dispatch",
+ journalInstance: "state.journal",
+ });
+ const artifacts = createFileArtifactStorePackage({ root: join(root, ".svml", "artifacts"), instance: "artifacts.fs" });
+ const credentials = createEnvironmentCredentialStorePackage({ instance: "credentials.env" });
const runtime = await createProjectLocalRuntime({
root,
+ runtimeServices: [execution, state, artifacts, credentials],
+ runtimeSelection: {
+ scheduler: "execution.local.scheduler",
+ worker: "execution.local.worker",
+ stores: {
+ build: "state.builds",
+ operations: "state.operations",
+ dispatch: "state.dispatch",
+ journal: "state.journal",
+ artifacts: "artifacts.fs",
+ credentials: ["credentials.env"],
+ },
+ },
components: [
generationComponent,
...[...new Map(selected.map((item) => [item.component.name, item.component])).values()],
],
endpoints: [provider],
- allowedPermissions: provider.manifest.facets.flatMap((facet) => facet.permissions),
+ allowedPermissions: [
+ "filesystem:state", "filesystem:artifacts", "environment:credentials",
+ ...provider.manifest.facets.flatMap((facet) => facet.permissions),
+ ],
+ scheduling: { maxConcurrency: 1 },
});
const failures: string[] = [];
+ const workerAbort = new AbortController();
+ const workerRun = runtime.work({
+ owner: `kie-smoke-${process.pid}`,
+ leaseMs: 30_000,
+ idlePollMs: 100,
+ signal: workerAbort.signal,
+ });
try {
for (const item of selected) {
const before = await accountCredits(baseUrl, key);
@@ -455,6 +491,8 @@ async function main(): Promise<void> {
console.log(`[${item.key}] credits after: ${after}; consumed: ${before - after}`);
}
} finally {
+ workerAbort.abort();
+ await workerRun;
await runtime.close();
}
if (failures.length > 0) throw new Error(`KIE live smoke failures:\n${failures.join("\n")}`);
diff --git a/packages/provider-xiaomi-mimo/package.json b/packages/provider-xiaomi-mimo/package.json
index 1cdf3ef3..79035e6b 100644
--- a/packages/provider-xiaomi-mimo/package.json
+++ b/packages/provider-xiaomi-mimo/package.json
@@ -14,7 +14,6 @@
"@narratage/generation": "workspace:*",
"@narratage/protocol": "workspace:*",
"@narratage/runtime": "workspace:*",
- "@narratage/runtime-adapter": "workspace:*",
- "@narratage/runtime-adapter-node": "workspace:*"
+ "@narratage/runtime-adapter": "workspace:*"
}
}
diff --git a/packages/provider-xiaomi-mimo/src/activation.ts b/packages/provider-xiaomi-mimo/src/activation.ts
index e351fecc..5ce74adc 100644
--- a/packages/provider-xiaomi-mimo/src/activation.ts
+++ b/packages/provider-xiaomi-mimo/src/activation.ts
@@ -1,12 +1,11 @@
-import { credentialRef } from "@narratage/runtime";
import {
createRuntimeEndpointAdapterFacet,
+ runtimeConfigCredentialRef,
runtimeConfigExact,
runtimeConfigObject,
runtimeConfigPositiveInteger,
runtimeConfigString,
} from "@narratage/runtime-adapter";
-import { diagnoseRuntimeEnvironmentCredential } from "@narratage/runtime-adapter-node";
import { createXiaomiMimoProvider } from "./provider.js";
@@ -15,7 +14,7 @@ const adapter = createRuntimeEndpointAdapterFacet({
validate(context) {
const config = runtimeConfigObject(context.config, "Xiaomi MiMo");
runtimeConfigExact(config, [
- "apiBaseUrl", "apiKeyEnv", "defaultConcurrency", "requestTimeoutMs",
+ "apiBaseUrl", "apiKey", "defaultConcurrency", "requestTimeoutMs",
"maxResponseBytes", "maxVoiceSampleBase64Bytes",
], "Xiaomi MiMo");
const base = runtimeConfigString(config.apiBaseUrl, "Xiaomi MiMo apiBaseUrl");
@@ -25,7 +24,9 @@ const adapter = createRuntimeEndpointAdapterFacet({
throw new Error("Xiaomi MiMo apiBaseUrl must use HTTPS or loopback");
}
}
- runtimeConfigString(config.apiKeyEnv, "Xiaomi MiMo apiKeyEnv");
+ if (runtimeConfigCredentialRef(config.apiKey, "Xiaomi MiMo apiKey") === undefined) {
+ throw new Error("Xiaomi MiMo apiKey CredentialRef is required");
+ }
runtimeConfigPositiveInteger(config.defaultConcurrency, "Xiaomi MiMo defaultConcurrency");
runtimeConfigPositiveInteger(config.requestTimeoutMs, "Xiaomi MiMo requestTimeoutMs");
runtimeConfigPositiveInteger(config.maxResponseBytes, "Xiaomi MiMo maxResponseBytes");
@@ -33,13 +34,14 @@ const adapter = createRuntimeEndpointAdapterFacet({
},
create(context) {
const config = runtimeConfigObject(context.config, "Xiaomi MiMo");
- const apiKeyEnv = runtimeConfigString(config.apiKeyEnv, "Xiaomi MiMo apiKeyEnv");
+ const apiKey = runtimeConfigCredentialRef(config.apiKey, "Xiaomi MiMo apiKey");
+ if (apiKey === undefined) throw new Error("Xiaomi MiMo apiKey CredentialRef is required");
return createXiaomiMimoProvider({
instance: context.instance,
...(context.lane === undefined ? {} : { lane: context.lane }),
...(runtimeConfigString(config.apiBaseUrl, "Xiaomi MiMo apiBaseUrl") === undefined
? {} : { apiBaseUrl: config.apiBaseUrl as string }),
- ...(apiKeyEnv === undefined ? {} : { apiKey: credentialRef("env", apiKeyEnv) }),
+ apiKey,
...(runtimeConfigPositiveInteger(config.defaultConcurrency, "Xiaomi MiMo defaultConcurrency") === undefined
? {} : { defaultConcurrency: config.defaultConcurrency as number }),
...(runtimeConfigPositiveInteger(config.requestTimeoutMs, "Xiaomi MiMo requestTimeoutMs") === undefined
@@ -50,13 +52,6 @@ const adapter = createRuntimeEndpointAdapterFacet({
? {} : { maxVoiceSampleBase64Bytes: config.maxVoiceSampleBase64Bytes as number }),
});
},
- doctor(context) {
- const config = runtimeConfigObject(context.config, "Xiaomi MiMo");
- return diagnoseRuntimeEnvironmentCredential(
- runtimeConfigString(config.apiKeyEnv, "Xiaomi MiMo apiKeyEnv") ?? "MIMO_API_KEY",
- "Xiaomi MiMo",
- );
- },
});
export const svmlPackage = {
diff --git a/packages/provider-xiaomi-mimo/src/provider.ts b/packages/provider-xiaomi-mimo/src/provider.ts
index 782a9fcd..77665ab5 100644
--- a/packages/provider-xiaomi-mimo/src/provider.ts
+++ b/packages/provider-xiaomi-mimo/src/provider.ts
@@ -253,6 +253,7 @@ export function createXiaomiMimoProvider(options: CreateXiaomiMimoProviderOption
apiBaseUrl, requestTimeoutMs, maxResponseBytes, maxVoiceSampleBase64Bytes, transportDigest,
}),
credentials: { apiKey: options.apiKey ?? credentialRef("env", "MIMO_API_KEY") },
+ credentialInputs: { apiKey: { label: "Xiaomi MiMo API key" } },
defaultConcurrency: options.defaultConcurrency ?? 2,
capabilities: endpointCapabilities,
});
diff --git a/packages/runtime-adapter/src/index.ts b/packages/runtime-adapter/src/index.ts
index a59b8d02..e04cc546 100644
--- a/packages/runtime-adapter/src/index.ts
+++ b/packages/runtime-adapter/src/index.ts
@@ -7,6 +7,8 @@ import type { HostFacet } from "@narratage/host";
import { canonicalize, digestOf, isDigest } from "@narratage/protocol";
import type { CanonicalValue, Digest } from "@narratage/protocol";
import type { RuntimeServicePackage } from "@narratage/runtime";
+import { credentialRef } from "@narratage/runtime";
+import type { CredentialRef } from "@narratage/runtime";
export const runtimeAdapterHostAbi = "svml.runtime-adapter-host@1";
@@ -396,3 +398,17 @@ export function runtimeConfigBoolean(value: CanonicalValue | undefined, subject:
assert(typeof value === "boolean", `${subject} must be a boolean`);
return value;
}
+
+/** Closed deployment syntax for a CredentialRef; secret bytes never enter the Profile. */
+export function runtimeConfigCredentialRef(
+ value: CanonicalValue | undefined,
+ subject: string,
+): CredentialRef | undefined {
+ if (value === undefined) return undefined;
+ const object = runtimeConfigObject(value, subject);
+ runtimeConfigExact(object, ["store", "key"], subject);
+ const store = runtimeConfigString(object.store, `${subject}.store`);
+ const key = runtimeConfigString(object.key, `${subject}.key`);
+ assert(store !== undefined && key !== undefined, `${subject} requires store and key`);
+ return credentialRef(store, key);
+}
diff --git a/packages/runtime/src/credentials.ts b/packages/runtime/src/credentials.ts
index c2878676..8a51ca7f 100644
--- a/packages/runtime/src/credentials.ts
+++ b/packages/runtime/src/credentials.ts
@@ -16,6 +16,12 @@ export type CredentialStore = {
resolve(ref: CredentialRef): Promise<CredentialValue | undefined>;
};
+export type WritableCredentialStore = CredentialStore & {
+ owns(ref: CredentialRef): boolean;
+ put(ref: CredentialRef, value: CredentialValue): Promise<void>;
+ delete(ref: CredentialRef): Promise<boolean>;
+};
+
function assert(condition: unknown, message: string): asserts condition {
if (!condition) throw new Error(message);
}
@@ -31,3 +37,52 @@ export function verifyCredentialRef(ref: CredentialRef): void {
assert(ref.store.trim().length > 0, "CredentialRef store is empty");
assert(ref.key.trim().length > 0, "CredentialRef key is empty");
}
+
+export function isWritableCredentialStore(value: CredentialStore): value is WritableCredentialStore {
+ return "owns" in value && typeof value.owns === "function"
+ && "put" in value && typeof value.put === "function"
+ && "delete" in value && typeof value.delete === "function";
+}
+
+export async function writableCredentialStore(
+ store: CredentialStore,
+ ref: CredentialRef,
+): Promise<WritableCredentialStore | undefined> {
+ verifyCredentialRef(ref);
+ if (store instanceof CompositeCredentialStore) return await store.writable(ref);
+ return isWritableCredentialStore(store) && store.owns(ref) ? store : undefined;
+}
+
+/**
+ * Compose independently selected stores without a central store-name registry. Every implementation
+ * sees the exact CredentialRef and must decline names it does not own. Ambiguous ownership fails.
+ */
+export class CompositeCredentialStore implements CredentialStore {
+ readonly #stores: readonly CredentialStore[];
+
+ constructor(stores: readonly CredentialStore[]) {
+ assert(stores.length > 0, "CredentialStore composition is empty");
+ this.#stores = [...stores];
+ }
+
+ async resolve(ref: CredentialRef): Promise<CredentialValue | undefined> {
+ verifyCredentialRef(ref);
+ const resolved = (await Promise.all(this.#stores.map(async (store) => await store.resolve(ref))))
+ .filter((value): value is CredentialValue => value !== undefined);
+ assert(resolved.length <= 1, `CredentialRef store ${ref.store} is implemented more than once`);
+ return resolved[0];
+ }
+
+ async writable(ref: CredentialRef): Promise<WritableCredentialStore | undefined> {
+ verifyCredentialRef(ref);
+ const candidates: WritableCredentialStore[] = [];
+ for (const store of this.#stores) {
+ if (!isWritableCredentialStore(store)) continue;
+ // A read cannot prove ownership for a missing key. Writable implementations therefore expose
+ // an optional structural ownership predicate rather than a framework registry.
+ if (store.owns(ref)) candidates.push(store);
+ }
+ assert(candidates.length <= 1, `CredentialRef store ${ref.store} has several writable owners`);
+ return candidates[0];
+ }
+}
diff --git a/packages/runtime/src/dispatch.ts b/packages/runtime/src/dispatch.ts
new file mode 100644
index 00000000..99329cb1
--- /dev/null
+++ b/packages/runtime/src/dispatch.ts
@@ -0,0 +1,253 @@
+import { canonicalize, digestOf, isDigest } from "@narratage/protocol";
+import type { CanonicalValue, Digest } from "@narratage/protocol";
+
+export type DispatchAdmission = "open" | "closing" | "closed";
+export type DispatchPhase = "queued" | "leased" | "waiting" | "blocked" | "settling" | "terminal";
+export type DispatchTerminal = "complete" | "failed" | "cancelled";
+
+export type DispatchLease = {
+ readonly owner: string;
+ readonly token: string;
+ /** Monotonic fencing value. A stale owner can never commit with an older fence. */
+ readonly fence: number;
+ readonly expiresAt: number;
+};
+
+export type BuildDispatchIdentity = {
+ readonly format: "svml.build-dispatch-identity@1";
+ readonly id: Digest;
+ readonly build: string;
+ readonly core: Digest;
+ readonly runtimeClosure: Digest;
+};
+
+export type BuildDispatchSnapshot = BuildDispatchIdentity & {
+ readonly revision: number;
+ readonly createdAt: number;
+ readonly updatedAt: number;
+ readonly priority: number;
+ readonly availableAt: number;
+ readonly admission: DispatchAdmission;
+ readonly phase: DispatchPhase;
+ readonly lease?: DispatchLease;
+ readonly reason?: string;
+ readonly cancellation?: {
+ readonly requestedAt: number;
+ readonly reason?: string;
+ };
+ readonly terminal?: DispatchTerminal;
+};
+
+export type BuildDispatchCreate =
+ | { readonly status: "created"; readonly snapshot: BuildDispatchSnapshot }
+ | { readonly status: "existing"; readonly snapshot: BuildDispatchSnapshot };
+
+export type BuildDispatchClaim = {
+ readonly owner: string;
+ readonly token: string;
+ readonly now: number;
+ readonly leaseMs: number;
+};
+
+export type BuildDispatchRelease = {
+ readonly phase: "queued" | "waiting" | "blocked" | "settling";
+ readonly availableAt: number;
+ readonly reason?: string;
+};
+
+export type DispatchQuery = {
+ readonly phases?: readonly DispatchPhase[];
+ readonly admission?: DispatchAdmission;
+};
+
+export type CapacityMode = "active" | "recoverable";
+
+export type CapacityLimits = {
+ readonly globalActive: number;
+ readonly laneActive: number;
+ readonly laneInFlight: number;
+};
+
+export type CapacityReservation = {
+ readonly format: "svml.capacity-reservation@1";
+ readonly id: Digest;
+ readonly build: string;
+ readonly command: string;
+ readonly lane: string;
+ readonly mode: CapacityMode;
+ readonly inFlight: boolean;
+ readonly active?: DispatchLease;
+ readonly createdAt: number;
+ readonly updatedAt: number;
+};
+
+export type CapacityAcquireRequest = {
+ readonly build: string;
+ readonly command: string;
+ readonly lane: string;
+ readonly mode: CapacityMode;
+ /** Current Build authority; a stale Worker cannot reserve capacity after being fenced. */
+ readonly buildLease: DispatchLease;
+ readonly owner: string;
+ readonly token: string;
+ readonly now: number;
+ readonly leaseMs: number;
+ readonly limits: CapacityLimits;
+};
+
+export type CapacityAcquire =
+ | { readonly status: "acquired"; readonly reservation: CapacityReservation }
+ | { readonly status: "blocked"; readonly retryAt: number; readonly reason: "global-active" | "lane-active" | "lane-in-flight" };
+
+/**
+ * Durable execution control for Build identities, leases and shared capacity.
+ *
+ * It deliberately stores no Core Command body. `command` is only the identity regenerated from
+ * verified BuildState by the Worker. Provider checkpoints remain in OperationStore.
+ */
+export type BuildDispatchStore = {
+ create(identity: BuildDispatchIdentity, options?: { readonly priority?: number; readonly availableAt?: number; readonly now?: number }): Promise<BuildDispatchCreate>;
+ read(build: string): Promise<BuildDispatchSnapshot | undefined>;
+ list(query?: DispatchQuery): Promise<readonly BuildDispatchSnapshot[]>;
+ claim(request: BuildDispatchClaim): Promise<BuildDispatchSnapshot | undefined>;
+ heartbeat(build: string, lease: DispatchLease, now: number, leaseMs: number): Promise<BuildDispatchSnapshot>;
+ release(build: string, lease: DispatchLease, update: BuildDispatchRelease, now?: number): Promise<BuildDispatchSnapshot>;
+ finish(build: string, lease: DispatchLease, terminal: DispatchTerminal, reason?: string, now?: number): Promise<BuildDispatchSnapshot>;
+ requestCancellation(build: string, reason?: string, now?: number): Promise<BuildDispatchSnapshot>;
+ acquireCapacity(request: CapacityAcquireRequest): Promise<CapacityAcquire>;
+ heartbeatCapacity(id: Digest, lease: DispatchLease, now: number, leaseMs: number): Promise<CapacityReservation>;
+ parkCapacity(id: Digest, lease: DispatchLease, inFlight: boolean, now?: number): Promise<CapacityReservation>;
+ releaseCapacity(id: Digest, lease: DispatchLease, now?: number): Promise<void>;
+ /** Remove a parked reservation only while holding the owning Build's current fenced lease. */
+ clearCapacity(id: Digest, build: string, buildLease: DispatchLease): Promise<void>;
+ listCapacity(): Promise<readonly CapacityReservation[]>;
+};
+
+export type RuntimeJournalKind =
+ | "dispatch-created"
+ | "dispatch-claimed"
+ | "dispatch-released"
+ | "dispatch-terminal"
+ | "cancellation-requested"
+ | "worker-started"
+ | "worker-stopped"
+ | "capacity-acquired"
+ | "capacity-released"
+ | "operation-control";
+
+export type RuntimeJournalEntry = {
+ readonly format: "svml.runtime-journal-entry@1";
+ readonly sequence: number;
+ readonly at: number;
+ readonly kind: RuntimeJournalKind;
+ readonly build?: string;
+ readonly operation?: Digest;
+ readonly worker?: string;
+ readonly detail: CanonicalValue;
+};
+
+export type RuntimeJournalQuery = {
+ readonly after?: number;
+ readonly build?: string;
+ readonly operation?: Digest;
+ readonly worker?: string;
+ readonly limit?: number;
+};
+
+export type RuntimeJournal = {
+ append(entry: Omit<RuntimeJournalEntry, "format" | "sequence">): Promise<RuntimeJournalEntry>;
+ list(query?: RuntimeJournalQuery): Promise<readonly RuntimeJournalEntry[]>;
+};
+
+function assert(condition: unknown, message: string): asserts condition {
+ if (!condition) throw new Error(message);
+}
+
+function safeNonNegative(value: number, subject: string): number {
+ assert(Number.isSafeInteger(value) && value >= 0, `${subject} must be a non-negative safe integer`);
+ return value;
+}
+
+function positive(value: number, subject: string): number {
+ assert(Number.isSafeInteger(value) && value > 0, `${subject} must be a positive safe integer`);
+ return value;
+}
+
+export function createBuildDispatchIdentity(input: {
+ readonly build: string;
+ readonly core: Digest;
+ readonly runtimeClosure: Digest;
+}): BuildDispatchIdentity {
+ assert(input.build.trim().length > 0, "Build Dispatch build id is empty");
+ assert(isDigest(input.core), "Build Dispatch Core digest is invalid");
+ assert(isDigest(input.runtimeClosure), "Build Dispatch Runtime Closure digest is invalid");
+ const content = {
+ format: "svml.build-dispatch-identity@1" as const,
+ build: input.build,
+ core: input.core,
+ runtimeClosure: input.runtimeClosure,
+ };
+ return { ...content, id: digestOf(content) };
+}
+
+export function verifyBuildDispatchIdentity(value: BuildDispatchIdentity): void {
+ const expected = createBuildDispatchIdentity(value);
+ assert(value.format === expected.format && value.id === expected.id, "Build Dispatch identity differs");
+}
+
+export function verifyDispatchLease(value: DispatchLease, subject = "Dispatch lease"): void {
+ assert(value.owner.trim().length > 0, `${subject} owner is empty`);
+ assert(value.token.trim().length > 0, `${subject} token is empty`);
+ positive(value.fence, `${subject} fence`);
+ safeNonNegative(value.expiresAt, `${subject} expiry`);
+}
+
+export function verifyBuildDispatchSnapshot(value: BuildDispatchSnapshot): void {
+ verifyBuildDispatchIdentity(value);
+ safeNonNegative(value.revision, "Build Dispatch revision");
+ safeNonNegative(value.createdAt, "Build Dispatch createdAt");
+ safeNonNegative(value.updatedAt, "Build Dispatch updatedAt");
+ assert(Number.isSafeInteger(value.priority), "Build Dispatch priority must be a safe integer");
+ safeNonNegative(value.availableAt, "Build Dispatch availableAt");
+ assert(["open", "closing", "closed"].includes(value.admission), "Build Dispatch admission is invalid");
+ assert(["queued", "leased", "waiting", "blocked", "settling", "terminal"].includes(value.phase), "Build Dispatch phase is invalid");
+ if (value.lease !== undefined) verifyDispatchLease(value.lease);
+ assert((value.phase === "leased") === (value.lease !== undefined), "only a leased Dispatch may carry a lease");
+ assert((value.phase === "terminal") === (value.terminal !== undefined), "only a terminal Dispatch may carry a terminal outcome");
+ if (value.phase === "terminal") assert(value.admission === "closed", "terminal Dispatch admission must be closed");
+ if (value.cancellation !== undefined) safeNonNegative(value.cancellation.requestedAt, "cancellation requestedAt");
+}
+
+export function capacityReservationId(build: string, command: string): Digest {
+ assert(build.trim().length > 0 && command.trim().length > 0, "Capacity reservation identity is empty");
+ return digestOf({ format: "svml.capacity-reservation-identity@1", build, command });
+}
+
+export function verifyCapacityLimits(value: CapacityLimits): void {
+ positive(value.globalActive, "global active capacity");
+ positive(value.laneActive, "lane active capacity");
+ positive(value.laneInFlight, "lane in-flight capacity");
+}
+
+export function verifyCapacityReservation(value: CapacityReservation): void {
+ assert(value.format === "svml.capacity-reservation@1", "Capacity reservation format is unsupported");
+ assert(value.id === capacityReservationId(value.build, value.command), "Capacity reservation identity differs");
+ assert(value.lane.trim().length > 0, "Capacity reservation lane is empty");
+ assert(value.mode === "active" || value.mode === "recoverable", "Capacity reservation mode is invalid");
+ if (value.active !== undefined) verifyDispatchLease(value.active, "Capacity lease");
+ safeNonNegative(value.createdAt, "Capacity reservation createdAt");
+ safeNonNegative(value.updatedAt, "Capacity reservation updatedAt");
+}
+
+export function verifyRuntimeJournalEntry(value: RuntimeJournalEntry): void {
+ assert(value.format === "svml.runtime-journal-entry@1", "Runtime Journal entry format is unsupported");
+ positive(value.sequence, "Runtime Journal sequence");
+ safeNonNegative(value.at, "Runtime Journal timestamp");
+ assert([
+ "dispatch-created", "dispatch-claimed", "dispatch-released", "dispatch-terminal",
+ "cancellation-requested", "worker-started", "worker-stopped", "capacity-acquired",
+ "capacity-released", "operation-control",
+ ].includes(value.kind), "Runtime Journal kind is invalid");
+ if (value.operation !== undefined) assert(isDigest(value.operation), "Runtime Journal Operation digest is invalid");
+ canonicalize(value.detail);
+}
diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts
index 8244de97..555dd886 100644
--- a/packages/runtime/src/index.ts
+++ b/packages/runtime/src/index.ts
@@ -6,15 +6,31 @@ export {
} from "./catalog.js";
export type * from "./catalog.js";
export {
+ CompositeCredentialStore,
credentialRef,
+ isWritableCredentialStore,
+ writableCredentialStore,
verifyCredentialRef,
} from "./credentials.js";
export type * from "./credentials.js";
+export {
+ capacityReservationId,
+ createBuildDispatchIdentity,
+ verifyBuildDispatchIdentity,
+ verifyBuildDispatchSnapshot,
+ verifyCapacityLimits,
+ verifyCapacityReservation,
+ verifyDispatchLease,
+ verifyRuntimeJournalEntry,
+} from "./dispatch.js";
+export type * from "./dispatch.js";
export {
MemoryOperationStore,
+ operationCancellationRequestId,
sealOperationCompletion,
sealOperationIdentity,
verifyOperationIdentity,
+ verifyOperationCancellationControl,
verifyOperationSnapshot,
} from "./operations.js";
export type * from "./operations.js";
diff --git a/packages/runtime/src/operations.ts b/packages/runtime/src/operations.ts
index 9d6f1d7e..798a65b5 100644
--- a/packages/runtime/src/operations.ts
+++ b/packages/runtime/src/operations.ts
@@ -36,13 +36,23 @@ export type OperationFailure = {
readonly retryAt?: number;
};
+export type OperationCancellationControl = {
+ readonly requestedAt: number;
+ readonly requestId: Digest;
+ readonly status: "requested" | "accepted" | "confirmed" | "unsupported" | "too-late";
+ readonly attempts: number;
+ readonly retryAt?: number;
+ readonly lastError?: { readonly code: string; readonly message: string };
+};
+
export type OperationSnapshot = OperationIdentity & {
readonly revision: number;
- readonly status: "created" | "pending" | "completed" | "failed";
+ readonly status: "created" | "pending" | "completed" | "failed" | "cancelled";
readonly checkpoint?: CanonicalValue;
readonly wakeAt?: number;
readonly completion?: OperationCompletion;
readonly failure?: OperationFailure;
+ readonly cancellation?: OperationCancellationControl;
};
export type OperationUpdate =
@@ -51,7 +61,9 @@ export type OperationUpdate =
readonly status: "completed";
readonly completion: Omit<OperationCompletion, "digest">;
}
- | { readonly status: "failed"; readonly failure: OperationFailure };
+ | { readonly status: "failed"; readonly failure: OperationFailure }
+ | { readonly status: "cancelled"; readonly cancellation: OperationCancellationControl }
+ | { readonly status: "control"; readonly cancellation: OperationCancellationControl };
export type OperationCreate =
| { readonly status: "created"; readonly snapshot: OperationSnapshot }
@@ -192,6 +204,34 @@ export function verifyOperationSnapshot(snapshot: OperationSnapshot): void {
epochMillisecond(snapshot.failure.retryAt, "Operation failure retryAt");
}
}
+ if (snapshot.status === "cancelled") {
+ assert(presence === 0, "cancelled Operation cannot carry execution result state");
+ assert(snapshot.cancellation?.status === "confirmed", "cancelled Operation requires confirmed cancellation");
+ }
+ if (snapshot.cancellation !== undefined) verifyOperationCancellationControl(snapshot.id, snapshot.cancellation);
+}
+
+export function operationCancellationRequestId(operation: Digest, requestedAt: number): Digest {
+ epochMillisecond(requestedAt, "Operation cancellation requestedAt");
+ return digestOf({ format: "svml.operation-cancellation-request@1", operation, requestedAt });
+}
+
+export function verifyOperationCancellationControl(
+ operation: Digest,
+ control: OperationCancellationControl,
+): void {
+ epochMillisecond(control.requestedAt, "Operation cancellation requestedAt");
+ assert(control.requestId === operationCancellationRequestId(operation, control.requestedAt),
+ "Operation cancellation request identity differs");
+ assert(["requested", "accepted", "confirmed", "unsupported", "too-late"].includes(control.status),
+ "Operation cancellation status is invalid");
+ assert(Number.isSafeInteger(control.attempts) && control.attempts >= 0,
+ "Operation cancellation attempts is invalid");
+ if (control.retryAt !== undefined) epochMillisecond(control.retryAt, "Operation cancellation retryAt");
+ if (control.lastError !== undefined) {
+ assert(control.lastError.code.trim().length > 0 && control.lastError.message.trim().length > 0,
+ "Operation cancellation error is invalid");
+ }
}
function copy(snapshot: OperationSnapshot): OperationSnapshot {
@@ -241,16 +281,30 @@ export class MemoryOperationStore implements OperationStore {
const current = this.#operations.get(id);
assert(current !== undefined, `Operation ${id} does not exist`);
if (current.revision !== expectedRevision) return { status: "conflict", current: copy(current) };
- assert(current.status !== "completed" && current.status !== "failed", `Operation ${id} is already terminal`);
- const mutable = update.status === "pending"
+ if (update.status !== "control") {
+ assert(current.status !== "completed" && current.status !== "failed" && current.status !== "cancelled",
+ `Operation ${id} is already terminal`);
+ }
+ const mutable = update.status === "control"
+ ? {
+ status: current.status,
+ ...(current.checkpoint === undefined ? {} : { checkpoint: structuredClone(current.checkpoint) }),
+ ...(current.wakeAt === undefined ? {} : { wakeAt: current.wakeAt }),
+ ...(current.completion === undefined ? {} : { completion: structuredClone(current.completion) }),
+ ...(current.failure === undefined ? {} : { failure: structuredClone(current.failure) }),
+ cancellation: structuredClone(update.cancellation),
+ }
+ : update.status === "pending"
? {
status: "pending" as const,
checkpoint: structuredClone(update.checkpoint),
...(update.wakeAt === undefined ? {} : { wakeAt: epochMillisecond(update.wakeAt, "Operation wakeAt") }),
}
- : update.status === "completed"
+ : update.status === "completed"
? { status: "completed" as const, completion: sealOperationCompletion(update.completion) }
- : { status: "failed" as const, failure: structuredClone(update.failure) };
+ : update.status === "failed"
+ ? { status: "failed" as const, failure: structuredClone(update.failure) }
+ : { status: "cancelled" as const, cancellation: structuredClone(update.cancellation) };
const snapshot: OperationSnapshot = {
format: current.format,
id: current.id,
@@ -264,6 +318,8 @@ export class MemoryOperationStore implements OperationStore {
submissionKey: current.submissionKey,
revision: current.revision + 1,
...mutable,
+ ...(update.status === "control" || update.status === "cancelled" || current.cancellation === undefined
+ ? {} : { cancellation: structuredClone(current.cancellation) }),
};
verifyOperationSnapshot(snapshot);
this.#operations.set(id, snapshot);
diff --git a/packages/runtime/src/profile.ts b/packages/runtime/src/profile.ts
index a26b2aa6..98415c5e 100644
--- a/packages/runtime/src/profile.ts
+++ b/packages/runtime/src/profile.ts
@@ -10,8 +10,11 @@ import type {
export type RuntimeFacetRole =
| "scheduler"
+ | "worker"
| "build-store"
| "operation-store"
+ | "dispatch-store"
+ | "runtime-journal"
| "artifact-store"
| "credential-store"
| "capability-endpoint";
@@ -81,11 +84,14 @@ export type RuntimeProfile = {
readonly name: string;
readonly instances: readonly RuntimeProfileInstance[];
readonly scheduler: string;
+ readonly worker: string;
readonly stores: {
- readonly build?: string;
- readonly operations?: string;
- readonly artifacts?: string;
- readonly credentials?: string;
+ readonly build: string;
+ readonly operations: string;
+ readonly dispatch: string;
+ readonly journal: string;
+ readonly artifacts: string;
+ readonly credentials: readonly string[];
};
readonly endpoints: readonly RuntimeEndpointBinding[];
readonly scheduling: {
@@ -126,6 +132,7 @@ export type RuntimeClosure = {
readonly modules: readonly { readonly module: ModuleRef; readonly digest: Digest }[];
readonly instances: readonly ResolvedRuntimeInstance[];
readonly scheduler: string;
+ readonly worker: string;
readonly stores: RuntimeProfile["stores"];
readonly endpoints: readonly RuntimeEndpointBinding[];
readonly scheduling: RuntimeProfile["scheduling"];
@@ -311,11 +318,14 @@ function profileContent(profile: RuntimeProfile): Omit<RuntimeProfile, "digest">
}))
.sort((left, right) => left.id.localeCompare(right.id)),
scheduler: profile.scheduler,
+ worker: profile.worker,
stores: {
- ...(profile.stores.build === undefined ? {} : { build: profile.stores.build }),
- ...(profile.stores.operations === undefined ? {} : { operations: profile.stores.operations }),
- ...(profile.stores.artifacts === undefined ? {} : { artifacts: profile.stores.artifacts }),
- ...(profile.stores.credentials === undefined ? {} : { credentials: profile.stores.credentials }),
+ build: profile.stores.build,
+ operations: profile.stores.operations,
+ dispatch: profile.stores.dispatch,
+ journal: profile.stores.journal,
+ artifacts: profile.stores.artifacts,
+ credentials: [...profile.stores.credentials].sort(),
},
endpoints: [...profile.endpoints]
.map((binding) => ({ ...normalizeCapability(binding), endpoint: binding.endpoint }))
@@ -333,6 +343,18 @@ function verifyProfileShape(profile: RuntimeProfile): void {
assert(profile.format === "svml.runtime-profile@1", "unsupported Runtime Profile format");
assert(profile.name.trim().length > 0, "Runtime Profile name is empty");
assert(profile.scheduler.trim().length > 0, "Runtime Profile scheduler is empty");
+ assert(profile.worker.trim().length > 0, "Runtime Profile worker is empty");
+ for (const [name, id] of Object.entries({
+ build: profile.stores.build,
+ operations: profile.stores.operations,
+ dispatch: profile.stores.dispatch,
+ journal: profile.stores.journal,
+ artifacts: profile.stores.artifacts,
+ })) assert(id.trim().length > 0, `Runtime Profile ${name} store is empty`);
+ assert(profile.stores.credentials.length > 0, "Runtime Profile selects no CredentialStore");
+ for (const id of profile.stores.credentials) assert(id.trim().length > 0, "Runtime Profile CredentialStore id is empty");
+ assert(new Set(profile.stores.credentials).size === profile.stores.credentials.length,
+ "Runtime Profile repeats a CredentialStore");
positiveInteger(profile.scheduling.maxConcurrency, "Runtime Profile maxConcurrency");
const ids = profile.instances.map((instance) => {
assert(instance.id.trim().length > 0, "Runtime Profile instance id is empty");
@@ -385,6 +407,7 @@ function closureContent(closure: RuntimeClosure): Omit<RuntimeClosure, "digest">
instances: [...closure.instances].map((instance) => structuredClone(instance))
.sort((left, right) => left.id.localeCompare(right.id)),
scheduler: closure.scheduler,
+ worker: closure.worker,
stores: { ...closure.stores },
endpoints: [...closure.endpoints].map((binding) => structuredClone(binding))
.sort((left, right) => bindingKey(left).localeCompare(bindingKey(right))),
@@ -431,15 +454,21 @@ export function verifyRuntimeClosure(closure: RuntimeClosure): void {
}
const schedulers = closure.instances.filter((instance) => instance.role === "scheduler");
assert(schedulers.length === 1 && schedulers[0]?.id === closure.scheduler, "Runtime Closure must select exactly one scheduler");
+ const workers = closure.instances.filter((instance) => instance.role === "worker");
+ assert(workers.length === 1 && workers[0]?.id === closure.worker, "Runtime Closure must select exactly one worker");
const roles = {
build: "build-store",
operations: "operation-store",
+ dispatch: "dispatch-store",
+ journal: "runtime-journal",
artifacts: "artifact-store",
- credentials: "credential-store",
} as const;
for (const [name, role] of Object.entries(roles) as [keyof typeof roles, typeof roles[keyof typeof roles]][]) {
const id = closure.stores[name];
- if (id !== undefined) assert(instances.get(id)?.role === role, `Runtime Closure ${name} store has the wrong role`);
+ assert(instances.get(id)?.role === role, `Runtime Closure ${name} store has the wrong role`);
+ }
+ for (const id of closure.stores.credentials) {
+ assert(instances.get(id)?.role === "credential-store", `Runtime Closure credential store ${id} has the wrong role`);
}
for (const binding of closure.endpoints) {
const endpoint = instances.get(binding.endpoint);
@@ -449,10 +478,10 @@ export function verifyRuntimeClosure(closure: RuntimeClosure): void {
assert(new Set(closure.endpoints.map(bindingKey)).size === closure.endpoints.length,
"Runtime Closure repeats an Endpoint binding");
if (closure.instances.some((instance) => instance.role === "capability-endpoint" && instance.lifecycle === "recoverable")) {
- assert(closure.stores.operations !== undefined, "recoverable Endpoints require an OperationStore");
+ assert(instances.get(closure.stores.operations)?.role === "operation-store", "recoverable Endpoints require an OperationStore");
}
if (closure.instances.some((instance) => instance.role === "capability-endpoint" && instance.credentialSlots.length > 0)) {
- assert(closure.stores.credentials !== undefined, "credentialed Endpoints require a CredentialStore");
+ assert(closure.stores.credentials.length > 0, "credentialed Endpoints require a CredentialStore");
}
}
@@ -495,15 +524,22 @@ export function resolveRuntimeProfile(
const schedulers = instances.filter((instance) => instance.role === "scheduler");
assert(schedulers.length === 1, "Runtime Profile must activate exactly one scheduler instance");
assert(schedulers[0]?.id === profile.scheduler, `Runtime Profile selects unknown scheduler ${profile.scheduler}`);
+ const workers = instances.filter((instance) => instance.role === "worker");
+ assert(workers.length === 1, "Runtime Profile must activate exactly one worker instance");
+ assert(workers[0]?.id === profile.worker, `Runtime Profile selects unknown worker ${profile.worker}`);
const storeRoles = {
build: "build-store",
operations: "operation-store",
+ dispatch: "dispatch-store",
+ journal: "runtime-journal",
artifacts: "artifact-store",
- credentials: "credential-store",
} as const;
for (const [name, role] of Object.entries(storeRoles) as [keyof typeof storeRoles, typeof storeRoles[keyof typeof storeRoles]][]) {
const id = profile.stores[name];
- if (id !== undefined) assert(byId.get(id)?.role === role, `Runtime Profile ${name} store ${id} has the wrong role`);
+ assert(byId.get(id)?.role === role, `Runtime Profile ${name} store ${id} has the wrong role`);
+ }
+ for (const id of profile.stores.credentials) {
+ assert(byId.get(id)?.role === "credential-store", `Runtime Profile credential store ${id} has the wrong role`);
}
for (const binding of profile.endpoints) {
const endpoint = byId.get(binding.endpoint);
@@ -511,10 +547,10 @@ export function resolveRuntimeProfile(
assert(endpoint.fulfills.some((item) => sameCapability(item, binding)), `${binding.endpoint} does not fulfill ${bindingKey(binding)}`);
}
if (instances.some((instance) => instance.role === "capability-endpoint" && instance.lifecycle === "recoverable")) {
- assert(profile.stores.operations !== undefined, "recoverable Endpoints require an OperationStore");
+ assert(byId.get(profile.stores.operations)?.role === "operation-store", "recoverable Endpoints require an OperationStore");
}
if (instances.some((instance) => instance.role === "capability-endpoint" && instance.credentialSlots.length > 0)) {
- assert(profile.stores.credentials !== undefined, "credentialed Endpoints require a CredentialStore");
+ assert(profile.stores.credentials.length > 0, "credentialed Endpoints require a CredentialStore");
}
const draft: RuntimeClosure = {
format: "svml.runtime-closure@1",
@@ -523,6 +559,7 @@ export function resolveRuntimeProfile(
modules: [...modules.values()],
instances,
scheduler: profile.scheduler,
+ worker: profile.worker,
stores: { ...profile.stores },
endpoints: profile.endpoints.map((binding) => structuredClone(binding)),
scheduling: structuredClone(profile.scheduling),
diff --git a/packages/runtime/src/services.ts b/packages/runtime/src/services.ts
index 20aa048d..711ea4ab 100644
--- a/packages/runtime/src/services.ts
+++ b/packages/runtime/src/services.ts
@@ -1,7 +1,9 @@
import { canonicalize, digestOf, isDigest } from "@narratage/protocol";
import type { CanonicalValue, Digest, ModuleRef } from "@narratage/protocol";
+import { CompositeCredentialStore } from "./credentials.js";
import type { CredentialStore } from "./credentials.js";
+import type { BuildDispatchStore, RuntimeJournal } from "./dispatch.js";
import type { OperationStore } from "./operations.js";
import {
RuntimeModuleRegistry,
@@ -15,6 +17,7 @@ import type {
ArtifactStore,
BuildSchedulerFactory,
BuildStore,
+ RuntimeWorkerFactory,
} from "./types.js";
type RuntimeServiceBase<Role extends RuntimeServiceFacetRole, Service> = {
@@ -24,15 +27,21 @@ type RuntimeServiceBase<Role extends RuntimeServiceFacetRole, Service> = {
};
export type RuntimeSchedulerService = RuntimeServiceBase<"scheduler", BuildSchedulerFactory>;
+export type RuntimeWorkerService = RuntimeServiceBase<"worker", RuntimeWorkerFactory>;
export type RuntimeBuildStoreService = RuntimeServiceBase<"build-store", BuildStore>;
export type RuntimeOperationStoreService = RuntimeServiceBase<"operation-store", OperationStore>;
+export type RuntimeDispatchStoreService = RuntimeServiceBase<"dispatch-store", BuildDispatchStore>;
+export type RuntimeJournalService = RuntimeServiceBase<"runtime-journal", RuntimeJournal>;
export type RuntimeArtifactStoreService = RuntimeServiceBase<"artifact-store", ArtifactStore>;
export type RuntimeCredentialStoreService = RuntimeServiceBase<"credential-store", CredentialStore>;
export type RuntimeService =
| RuntimeSchedulerService
+ | RuntimeWorkerService
| RuntimeBuildStoreService
| RuntimeOperationStoreService
+ | RuntimeDispatchStoreService
+ | RuntimeJournalService
| RuntimeArtifactStoreService
| RuntimeCredentialStoreService;
@@ -59,8 +68,11 @@ type RuntimeServiceDefinitionBase<Role extends RuntimeServiceFacetRole, Service>
export type RuntimeServiceDefinition =
| RuntimeServiceDefinitionBase<"scheduler", BuildSchedulerFactory>
+ | RuntimeServiceDefinitionBase<"worker", RuntimeWorkerFactory>
| RuntimeServiceDefinitionBase<"build-store", BuildStore>
| RuntimeServiceDefinitionBase<"operation-store", OperationStore>
+ | RuntimeServiceDefinitionBase<"dispatch-store", BuildDispatchStore>
+ | RuntimeServiceDefinitionBase<"runtime-journal", RuntimeJournal>
| RuntimeServiceDefinitionBase<"artifact-store", ArtifactStore>
| RuntimeServiceDefinitionBase<"credential-store", CredentialStore>;
@@ -73,11 +85,14 @@ export type DefineRuntimeServicePackageOptions = {
export type RuntimeServiceSelection = {
readonly scheduler: string;
+ readonly worker: string;
readonly stores: {
- readonly build?: string;
- readonly operations?: string;
- readonly artifacts?: string;
- readonly credentials?: string;
+ readonly build: string;
+ readonly operations: string;
+ readonly dispatch: string;
+ readonly journal: string;
+ readonly artifacts: string;
+ readonly credentials: readonly string[];
};
};
@@ -85,10 +100,13 @@ export type RuntimeServiceAssembly = {
readonly manifests: readonly RuntimeModuleManifest[];
readonly instances: readonly RuntimeProfileInstance[];
readonly scheduler: BuildSchedulerFactory;
- readonly buildStore?: BuildStore;
- readonly operationStore?: OperationStore;
- readonly artifactStore?: ArtifactStore;
- readonly credentialStore?: CredentialStore;
+ readonly worker: RuntimeWorkerFactory;
+ readonly buildStore: BuildStore;
+ readonly operationStore: OperationStore;
+ readonly dispatchStore: BuildDispatchStore;
+ readonly journal: RuntimeJournal;
+ readonly artifactStore: ArtifactStore;
+ readonly credentialStore: CredentialStore;
close(): Promise<void>;
};
@@ -122,6 +140,9 @@ function verifyServicePort(service: RuntimeService): void {
case "scheduler":
callable(service.service, "create", service.instance.id);
break;
+ case "worker":
+ callable(service.service, "create", service.instance.id);
+ break;
case "build-store":
callable(service.service, "create", service.instance.id);
callable(service.service, "read", service.instance.id);
@@ -133,6 +154,16 @@ function verifyServicePort(service: RuntimeService): void {
callable(service.service, "list", service.instance.id);
callable(service.service, "compareAndSwap", service.instance.id);
break;
+ case "dispatch-store":
+ for (const method of [
+ "create", "read", "list", "claim", "heartbeat", "release", "finish", "requestCancellation",
+ "acquireCapacity", "heartbeatCapacity", "parkCapacity", "releaseCapacity", "clearCapacity", "listCapacity",
+ ]) callable(service.service, method, service.instance.id);
+ break;
+ case "runtime-journal":
+ callable(service.service, "append", service.instance.id);
+ callable(service.service, "list", service.instance.id);
+ break;
case "artifact-store":
callable(service.service, "put", service.instance.id);
callable(service.service, "get", service.instance.id);
@@ -255,11 +286,15 @@ export function assembleRuntimeServices(
}
}
const scheduler = selectedService(services, selection.scheduler, "scheduler")!;
+ const worker = selectedService(services, selection.worker, "worker")!;
const build = selectedService(services, selection.stores.build, "build-store");
const operations = selectedService(services, selection.stores.operations, "operation-store");
+ const dispatch = selectedService(services, selection.stores.dispatch, "dispatch-store");
+ const journal = selectedService(services, selection.stores.journal, "runtime-journal");
const artifacts = selectedService(services, selection.stores.artifacts, "artifact-store");
- const credentials = selectedService(services, selection.stores.credentials, "credential-store");
- const selected = [scheduler, build, operations, artifacts, credentials]
+ assert(selection.stores.credentials.length > 0, "Runtime must explicitly select at least one CredentialStore");
+ const credentialServices = selection.stores.credentials.map((id) => selectedService(services, id, "credential-store")!);
+ const selected = [scheduler, worker, build, operations, dispatch, journal, artifacts, ...credentialServices]
.filter((item): item is NonNullable<typeof item> => item !== undefined);
const selectedPackages = [...new Set(selected.map((item) => item.package))];
let closed = false;
@@ -267,10 +302,13 @@ export function assembleRuntimeServices(
manifests: selectedPackages.map((item) => item.manifest),
instances: selected.map((item) => item.service.instance),
scheduler: scheduler.service.service,
- ...(build === undefined ? {} : { buildStore: build.service.service }),
- ...(operations === undefined ? {} : { operationStore: operations.service.service }),
- ...(artifacts === undefined ? {} : { artifactStore: artifacts.service.service }),
- ...(credentials === undefined ? {} : { credentialStore: credentials.service.service }),
+ worker: worker.service.service,
+ buildStore: build!.service.service,
+ operationStore: operations!.service.service,
+ dispatchStore: dispatch!.service.service,
+ journal: journal!.service.service,
+ artifactStore: artifacts!.service.service,
+ credentialStore: new CompositeCredentialStore(credentialServices.map((item) => item.service.service)),
async close() {
if (closed) return;
closed = true;
diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts
index 69665e26..82278352 100644
--- a/packages/runtime/src/types.ts
+++ b/packages/runtime/src/types.ts
@@ -6,6 +6,8 @@ import type {
Digest,
} from "@narratage/protocol";
import type { RuntimeClosure } from "./profile.js";
+import type { BuildDispatchSnapshot, BuildDispatchStore, RuntimeJournal } from "./dispatch.js";
+import type { OperationSnapshot } from "./operations.js";
export type RuntimeBlockedCommand = {
readonly command: string;
@@ -19,6 +21,47 @@ export type RuntimeRunnableCommand = {
readonly lane: string;
/** Registration default; a Runtime Profile may explicitly override it for this Scheduler. */
readonly maxConcurrency: number;
+ /** Recoverable work retains one shared in-flight reservation while polling. */
+ readonly capacityMode?: "active" | "recoverable";
+ readonly maxInFlight?: number;
+};
+
+export type RuntimeExecutionStores = {
+ readonly builds: BuildStore;
+ readonly operations: import("./operations.js").OperationStore;
+ readonly dispatch: BuildDispatchStore;
+ readonly journal: RuntimeJournal;
+ readonly artifacts: ArtifactStore;
+};
+
+export type RuntimeWorkerRunOptions = {
+ readonly owner: string;
+ readonly leaseMs: number;
+ readonly idlePollMs: number;
+ readonly signal?: AbortSignal;
+};
+
+export type RuntimeWorker = {
+ /** Claim and advance at most one Build. Undefined means no Dispatch was ready. */
+ runOnce(options: Omit<RuntimeWorkerRunOptions, "idlePollMs" | "signal">): Promise<BuildDispatchSnapshot | undefined>;
+ /** Continue until the caller-owned process signal is aborted. */
+ run(options: RuntimeWorkerRunOptions): Promise<void>;
+};
+
+export type RuntimeWorkerFactoryOptions = {
+ readonly scheduler: BuildSchedulerFactory;
+ readonly stores: RuntimeExecutionStores;
+ readonly scheduling: BuildSchedulerOptions;
+ readonly runtimeClosure?: RuntimeClosure;
+};
+
+/** Selected execution strategy. Process supervision remains a generic Host concern. */
+export type RuntimeWorkerFactory = {
+ create(executor: RuntimeCommandExecutor, options: RuntimeWorkerFactoryOptions): RuntimeWorker;
+};
+
+export type RuntimeQueueStatus = {
+ readonly dispatches: readonly BuildDispatchSnapshot[];
};
export type RuntimePreparation = {
@@ -47,6 +90,11 @@ export type RuntimeCommandExecutor = {
commandId: string,
context: RuntimeExecutionContext,
): Promise<RuntimeExecutionResult>;
+ cancelOperation?(
+ state: BuildState,
+ operation: OperationSnapshot,
+ requestedAt: number,
+ ): Promise<OperationSnapshot>;
};
/** Content-addressed bytes. Location, retention and remote transport are adapter policy. */
@@ -146,4 +194,3 @@ export type BuildScheduler = {
export type BuildSchedulerFactory = {
create(executor: RuntimeCommandExecutor, options?: BuildSchedulerOptions): BuildScheduler;
};
-
diff --git a/packages/runtime/test/profile.test.ts b/packages/runtime/test/profile.test.ts
index 9ed66ad0..69991312 100644
--- a/packages/runtime/test/profile.test.ts
+++ b/packages/runtime/test/profile.test.ts
@@ -38,24 +38,23 @@ function manifest(permissions: readonly string[] = []): RuntimeModuleManifest {
},
permissions: [],
},
- {
- name: "memory-build-store",
- role: "build-store",
+ ...([
+ ["local-worker", "worker"],
+ ["memory-build-store", "build-store"],
+ ["memory-operation-store", "operation-store"],
+ ["memory-dispatch-store", "dispatch-store"],
+ ["memory-runtime-journal", "runtime-journal"],
+ ["memory-artifact-store", "artifact-store"],
+ ["memory-credential-store", "credential-store"],
+ ] as const).map(([name, role]) => ({
+ name,
+ role,
implementation: {
- locator: "example.runtime-fixture/memory-build-store",
- digest: digestOf("example.runtime-fixture/memory-build-store@1"),
+ locator: `example.runtime-fixture/${name}`,
+ digest: digestOf(`example.runtime-fixture/${name}@1`),
},
permissions: [],
- },
- {
- name: "memory-operation-store",
- role: "operation-store",
- implementation: {
- locator: "example.runtime-fixture/memory-operation-store",
- digest: digestOf("example.runtime-fixture/memory-operation-store@1"),
- },
- permissions: [],
- },
+ })),
{
name: endpointFacet.name,
role: "capability-endpoint",
@@ -77,11 +76,16 @@ function profile(options: { readonly operations?: boolean; readonly lane?: numbe
name: "fixture.local",
instances: [
{ id: "scheduler.local", facet: { module: runtimeModule, name: "local-scheduler" } },
+ { id: "worker.local", facet: { module: runtimeModule, name: "local-worker" } },
{ id: "build.memory", facet: { module: runtimeModule, name: "memory-build-store" } },
...(options.operations === false ? [] : [{
id: "operations.memory",
facet: { module: runtimeModule, name: "memory-operation-store" },
}]),
+ { id: "dispatch.memory", facet: { module: runtimeModule, name: "memory-dispatch-store" } },
+ { id: "journal.memory", facet: { module: runtimeModule, name: "memory-runtime-journal" } },
+ { id: "artifacts.memory", facet: { module: runtimeModule, name: "memory-artifact-store" } },
+ { id: "credentials.memory", facet: { module: runtimeModule, name: "memory-credential-store" } },
{
id: "greeting.local",
facet: endpointFacet,
@@ -89,9 +93,14 @@ function profile(options: { readonly operations?: boolean; readonly lane?: numbe
},
],
scheduler: "scheduler.local",
+ worker: "worker.local",
stores: {
build: "build.memory",
- ...(options.operations === false ? {} : { operations: "operations.memory" }),
+ operations: "operations.memory",
+ dispatch: "dispatch.memory",
+ journal: "journal.memory",
+ artifacts: "artifacts.memory",
+ credentials: ["credentials.memory"],
},
endpoints: [{
capability: capabilities.generation,
@@ -165,11 +174,11 @@ test("recoverable Endpoints require an OperationStore before any paid execution"
registry.register(manifest());
assert.throws(
() => resolveRuntimeProfile(registry, profile({ operations: false })),
- /require an OperationStore/u,
+ /operations store/u,
);
});
-test("credentialed Endpoints require an explicitly selected CredentialStore", () => {
+test("a selected CredentialStore is part of every explicit Runtime Profile", () => {
const registry = new RuntimeModuleRegistry();
const configured = manifest();
registry.register({
@@ -178,10 +187,7 @@ test("credentialed Endpoints require an explicitly selected CredentialStore", ()
? { ...facet, credentialSlots: ["apiKey"] }
: facet),
});
- assert.throws(
- () => resolveRuntimeProfile(registry, profile()),
- /require a CredentialStore/u,
- );
+ assert.doesNotThrow(() => resolveRuntimeProfile(registry, profile()));
});
test("Runtime permissions require an explicit Host allowlist", () => {
diff --git a/packages/runtime/test/services.test.ts b/packages/runtime/test/services.test.ts
index 05102120..d95c2e67 100644
--- a/packages/runtime/test/services.test.ts
+++ b/packages/runtime/test/services.test.ts
@@ -32,6 +32,13 @@ function servicePackage(close?: () => void) {
},
},
},
+ {
+ role: "worker",
+ facet: "worker",
+ instance: "worker.example",
+ implementation: { locator: "example.runtime-services/worker", digest: digestOf("worker") },
+ service: { create() { return { async runOnce() { return undefined; }, async run() {} }; } },
+ },
{
role: "build-store",
facet: "build-store",
@@ -54,6 +61,41 @@ function servicePackage(close?: () => void) {
configuration: { database: "fixture" },
service: new MemoryOperationStore(),
},
+ {
+ role: "dispatch-store",
+ facet: "dispatch-store",
+ instance: "dispatch.example",
+ implementation: { locator: "example.runtime-services/dispatch", digest: digestOf("dispatch") },
+ service: Object.fromEntries([
+ "create", "read", "list", "claim", "heartbeat", "release", "finish", "requestCancellation",
+ "acquireCapacity", "heartbeatCapacity", "parkCapacity", "releaseCapacity", "clearCapacity", "listCapacity",
+ ].map((name) => [name, async () => undefined])) as never,
+ },
+ {
+ role: "runtime-journal",
+ facet: "runtime-journal",
+ instance: "journal.example",
+ implementation: { locator: "example.runtime-services/journal", digest: digestOf("journal") },
+ service: { async append() { throw new Error("unused"); }, async list() { return []; } },
+ },
+ {
+ role: "artifact-store",
+ facet: "artifact-store",
+ instance: "artifacts.example",
+ implementation: { locator: "example.runtime-services/artifacts", digest: digestOf("artifacts") },
+ service: {
+ async put() { throw new Error("unused"); },
+ async get() { return undefined; },
+ async has() { return false; },
+ },
+ },
+ {
+ role: "credential-store",
+ facet: "credential-store",
+ instance: "credentials.example",
+ implementation: { locator: "example.runtime-services/credentials", digest: digestOf("credentials") },
+ service: { async resolve() { return undefined; } },
+ },
],
...(close === undefined ? {} : { close }),
});
@@ -64,20 +106,30 @@ test("one physical Runtime package exposes separately selected Scheduler and Sto
const configured = servicePackage(() => { closes += 1; });
const assembly = assembleRuntimeServices([configured], {
scheduler: "scheduler.example",
+ worker: "worker.example",
stores: {
build: "builds.example",
operations: "operations.example",
+ dispatch: "dispatch.example",
+ journal: "journal.example",
+ artifacts: "artifacts.example",
+ credentials: ["credentials.example"],
},
});
assert.equal(assembly.manifests.length, 1);
assert.deepEqual(assembly.instances.map((item) => item.id), [
"scheduler.example",
+ "worker.example",
"builds.example",
"operations.example",
+ "dispatch.example",
+ "journal.example",
+ "artifacts.example",
+ "credentials.example",
]);
assert.ok(assembly.buildStore);
assert.ok(assembly.operationStore);
- assert.equal(assembly.artifactStore, undefined);
+ assert.ok(assembly.artifactStore);
assert.equal((await assembly.scheduler.create({
prepare() { throw new Error("unused"); },
async executeCommand() { throw new Error("unused"); },
@@ -93,7 +145,15 @@ test("Runtime service configuration is identity-bound and role selection is exac
assert.throws(
() => assembleRuntimeServices([configured], {
scheduler: "builds.example",
- stores: {},
+ worker: "worker.example",
+ stores: {
+ build: "builds.example",
+ operations: "operations.example",
+ dispatch: "dispatch.example",
+ journal: "journal.example",
+ artifacts: "artifacts.example",
+ credentials: ["credentials.example"],
+ },
}),
/not scheduler/u,
);
diff --git a/packages/store-sqlite/package.json b/packages/store-sqlite/package.json
index f4f547c2..b1ff805f 100644
--- a/packages/store-sqlite/package.json
+++ b/packages/store-sqlite/package.json
@@ -9,9 +9,13 @@
"exports": {
".": "./src/index.ts"
},
+ "svml": {
+ "activation": "./src/activation.ts"
+ },
"dependencies": {
"@narratage/core": "workspace:*",
"@narratage/protocol": "workspace:*",
- "@narratage/runtime": "workspace:*"
+ "@narratage/runtime": "workspace:*",
+ "@narratage/runtime-adapter": "workspace:*"
}
}
diff --git a/packages/store-sqlite/src/activation.ts b/packages/store-sqlite/src/activation.ts
new file mode 100644
index 00000000..86eb2257
--- /dev/null
+++ b/packages/store-sqlite/src/activation.ts
@@ -0,0 +1,43 @@
+import {
+ createRuntimeServiceAdapterFacet,
+ runtimeConfigExact,
+ runtimeConfigObject,
+ runtimeConfigPositiveInteger,
+ runtimeConfigString,
+} from "@narratage/runtime-adapter";
+import { resolve } from "node:path";
+
+import { createSqliteRuntimeServicePackage } from "./store.js";
+
+const sqliteStateAdapter = createRuntimeServiceAdapterFacet({
+ use: "@narratage/store-sqlite",
+ validate(context) {
+ const config = runtimeConfigObject(context.config, "SQLite Runtime state");
+ runtimeConfigExact(config, ["path", "busyTimeoutMs"], "SQLite Runtime state");
+ if (runtimeConfigString(config.path, "SQLite path") === undefined) throw new Error("SQLite path is required");
+ runtimeConfigPositiveInteger(config.busyTimeoutMs, "SQLite busyTimeoutMs");
+ },
+ create(context) {
+ const config = runtimeConfigObject(context.config, "SQLite Runtime state");
+ const path = runtimeConfigString(config.path, "SQLite path");
+ if (path === undefined) throw new Error("SQLite path is required");
+ return createSqliteRuntimeServicePackage({
+ name: context.instance,
+ path: resolve(context.root, path),
+ buildInstance: `${context.instance}.builds`,
+ operationInstance: `${context.instance}.operations`,
+ dispatchInstance: `${context.instance}.dispatch`,
+ journalInstance: `${context.instance}.journal`,
+ ...(runtimeConfigPositiveInteger(config.busyTimeoutMs, "SQLite busyTimeoutMs") === undefined
+ ? {} : { busyTimeoutMs: config.busyTimeoutMs as number }),
+ });
+ },
+});
+
+export const svmlPackage = {
+ format: "svml.node-package@1" as const,
+ name: "@narratage/store-sqlite",
+ hostFacets: [sqliteStateAdapter],
+};
+
+export default svmlPackage;
diff --git a/packages/store-sqlite/src/index.ts b/packages/store-sqlite/src/index.ts
index 0727379c..5737b8de 100644
--- a/packages/store-sqlite/src/index.ts
+++ b/packages/store-sqlite/src/index.ts
@@ -2,7 +2,9 @@ export {
SqliteRuntimeState,
createSqliteRuntimeServicePackage,
sqliteBuildStoreImplementationDigest,
+ sqliteDispatchStoreImplementationDigest,
sqliteOperationStoreImplementationDigest,
+ sqliteRuntimeJournalImplementationDigest,
sqliteStoreModuleRef,
} from "./store.js";
export type * from "./store.js";
diff --git a/packages/store-sqlite/src/store.ts b/packages/store-sqlite/src/store.ts
index d1e7183d..e31019cb 100644
--- a/packages/store-sqlite/src/store.ts
+++ b/packages/store-sqlite/src/store.ts
@@ -6,7 +6,7 @@ import {
canonicalStringify,
verifyBuildState,
} from "@narratage/core";
-import { digestOf } from "@narratage/protocol";
+import { canonicalize, digestOf } from "@narratage/protocol";
import type {
BuildState,
Digest,
@@ -15,17 +15,35 @@ import {
verifyBuildCatalogDescriptor,
verifyBuildCatalogEntry,
defineRuntimeServicePackage,
+ capacityReservationId,
sealOperationCompletion,
+ verifyBuildDispatchIdentity,
+ verifyBuildDispatchSnapshot,
+ verifyCapacityLimits,
+ verifyCapacityReservation,
+ verifyDispatchLease,
verifyOperationIdentity,
verifyOperationSnapshot,
+ verifyRuntimeJournalEntry,
} from "@narratage/runtime";
import type {
+ BuildDispatchClaim,
+ BuildDispatchCreate,
+ BuildDispatchIdentity,
+ BuildDispatchRelease,
+ BuildDispatchSnapshot,
+ BuildDispatchStore,
BuildCatalog,
BuildCatalogDescriptor,
BuildCatalogEntry,
BuildSnapshot,
BuildStore,
BuildStoreWrite,
+ CapacityAcquire,
+ CapacityAcquireRequest,
+ CapacityReservation,
+ DispatchLease,
+ DispatchQuery,
OperationCreate,
OperationIdentity,
OperationQuery,
@@ -33,10 +51,13 @@ import type {
OperationStore,
OperationStoreWrite,
OperationUpdate,
+ RuntimeJournal,
+ RuntimeJournalEntry,
+ RuntimeJournalQuery,
RuntimeServicePackage,
} from "@narratage/runtime";
-const databaseSchemaVersion = 1;
+const databaseSchemaVersion = 3;
export const sqliteStoreModuleRef = {
name: "@narratage/store-sqlite",
@@ -51,6 +72,14 @@ export const sqliteOperationStoreImplementationDigest = digestOf(
"@narratage/store-sqlite/operation-store@1",
);
+export const sqliteDispatchStoreImplementationDigest = digestOf(
+ "@narratage/store-sqlite/dispatch-store@1",
+);
+
+export const sqliteRuntimeJournalImplementationDigest = digestOf(
+ "@narratage/store-sqlite/runtime-journal@1",
+);
+
export type SqliteRuntimeStateOptions = {
readonly busyTimeoutMs?: number;
};
@@ -60,6 +89,8 @@ export type CreateSqliteRuntimeServicePackageOptions = SqliteRuntimeStateOptions
readonly name?: string;
readonly buildInstance?: string;
readonly operationInstance?: string;
+ readonly dispatchInstance?: string;
+ readonly journalInstance?: string;
};
type Row = Record<string, unknown>;
@@ -73,6 +104,23 @@ function positiveInteger(value: number, subject: string): number {
return value;
}
+function nonNegativeInteger(value: number, subject: string): number {
+ assert(Number.isSafeInteger(value) && value >= 0, `${subject} must be a non-negative safe integer`);
+ return value;
+}
+
+function transaction<T>(database: DatabaseSync, body: () => T): T {
+ database.exec("BEGIN IMMEDIATE");
+ try {
+ const result = body();
+ database.exec("COMMIT");
+ return result;
+ } catch (error) {
+ database.exec("ROLLBACK");
+ throw error;
+ }
+}
+
function durableBuildState(state: BuildState): BuildState {
verifyBuildState(state);
const normalized = { ...structuredClone(state), outstanding: [] };
@@ -129,11 +177,15 @@ function parseOperationSnapshot(row: Row): OperationSnapshot {
: row.status === "failed"
? { failure: JSON.parse(String(row.failure_json)) }
: {};
+ const cancellation = typeof row.cancellation_json === "string"
+ ? { cancellation: JSON.parse(row.cancellation_json) }
+ : {};
const snapshot = {
...identity,
revision: row.revision,
status: row.status,
...mutable,
+ ...cancellation,
} as OperationSnapshot;
verifyOperationSnapshot(snapshot);
return snapshot;
@@ -279,8 +331,8 @@ class SqliteOperationStore implements OperationStore {
const result = this.#database.prepare(`
INSERT OR IGNORE INTO svml_operations (
operation_id, revision, status, identity_json,
- checkpoint_json, completion_json, failure_json
- ) VALUES (?, 0, 'created', ?, NULL, NULL, NULL)
+ checkpoint_json, completion_json, failure_json, cancellation_json
+ ) VALUES (?, 0, 'created', ?, NULL, NULL, NULL, NULL)
`).run(identity.id, canonicalStringify(identity));
const snapshot: OperationSnapshot = {
...copy(identity),
@@ -297,7 +349,7 @@ class SqliteOperationStore implements OperationStore {
async read(id: Digest): Promise<OperationSnapshot | undefined> {
const row = this.#database.prepare(`
- SELECT identity_json, revision, status, checkpoint_json, completion_json, failure_json
+ SELECT identity_json, revision, status, checkpoint_json, completion_json, failure_json, cancellation_json
FROM svml_operations
WHERE operation_id = ?
`).get(id) as Row | undefined;
@@ -306,7 +358,7 @@ class SqliteOperationStore implements OperationStore {
async list(query: OperationQuery): Promise<readonly OperationSnapshot[]> {
const rows = this.#database.prepare(`
- SELECT identity_json, revision, status, checkpoint_json, completion_json, failure_json
+ SELECT identity_json, revision, status, checkpoint_json, completion_json, failure_json, cancellation_json
FROM svml_operations
ORDER BY operation_id ASC
`).all() as Row[];
@@ -328,7 +380,25 @@ class SqliteOperationStore implements OperationStore {
const current = await this.read(id);
if (current === undefined) throw new Error(`Operation ${id} does not exist`);
if (current.revision !== expectedRevision) return { status: "conflict", current };
- assert(current.status !== "completed" && current.status !== "failed", `Operation ${id} is already terminal`);
+ if (update.status !== "control") {
+ assert(current.status !== "completed" && current.status !== "failed" && current.status !== "cancelled",
+ `Operation ${id} is already terminal`);
+ }
+ if (update.status === "control") {
+ const result = this.#database.prepare(`
+ UPDATE svml_operations
+ SET revision = ?, cancellation_json = ?
+ WHERE operation_id = ? AND revision = ?
+ `).run(expectedRevision + 1, canonicalStringify(update.cancellation), id, expectedRevision);
+ if (result.changes !== 1) {
+ const conflict = await this.read(id);
+ if (conflict === undefined) throw new Error(`Operation ${id} disappeared during update`);
+ return { status: "conflict", current: conflict };
+ }
+ const stored = await this.read(id);
+ if (stored === undefined) throw new Error(`Operation ${id} disappeared after update`);
+ return { status: "stored", snapshot: stored };
+ }
const checkpoint = update.status === "pending" ? canonicalStringify({
format: "svml.operation-pending@1",
checkpoint: update.checkpoint,
@@ -338,11 +408,14 @@ class SqliteOperationStore implements OperationStore {
? canonicalStringify(sealOperationCompletion(update.completion))
: null;
const failure = update.status === "failed" ? canonicalStringify(update.failure) : null;
+ const cancellation = update.status === "cancelled"
+ ? canonicalStringify(update.cancellation)
+ : current.cancellation === undefined ? null : canonicalStringify(current.cancellation);
const result = this.#database.prepare(`
UPDATE svml_operations
- SET revision = ?, status = ?, checkpoint_json = ?, completion_json = ?, failure_json = ?
- WHERE operation_id = ? AND revision = ? AND status NOT IN ('completed', 'failed')
- `).run(expectedRevision + 1, update.status, checkpoint, completion, failure, id, expectedRevision);
+ SET revision = ?, status = ?, checkpoint_json = ?, completion_json = ?, failure_json = ?, cancellation_json = ?
+ WHERE operation_id = ? AND revision = ? AND status NOT IN ('completed', 'failed', 'cancelled')
+ `).run(expectedRevision + 1, update.status, checkpoint, completion, failure, cancellation, id, expectedRevision);
if (result.changes !== 1) {
const conflict = await this.read(id);
if (conflict === undefined) throw new Error(`Operation ${id} disappeared during update`);
@@ -354,11 +427,419 @@ class SqliteOperationStore implements OperationStore {
}
}
-/** One local database, two execution stores, one Host catalog, and no durable ready queue. */
+function dispatchLease(row: Row): DispatchLease | undefined {
+ if (row.lease_owner === null || row.lease_owner === undefined) return undefined;
+ assert(typeof row.lease_owner === "string", "SQLite Dispatch lease owner is invalid");
+ assert(typeof row.lease_token === "string", "SQLite Dispatch lease token is invalid");
+ assert(typeof row.lease_fence === "number", "SQLite Dispatch lease fence is invalid");
+ assert(typeof row.lease_expires_at === "number", "SQLite Dispatch lease expiry is invalid");
+ return {
+ owner: row.lease_owner,
+ token: row.lease_token,
+ fence: row.lease_fence,
+ expiresAt: row.lease_expires_at,
+ };
+}
+
+function parseDispatchSnapshot(row: Row): BuildDispatchSnapshot {
+ assert(typeof row.identity_json === "string", "SQLite Dispatch row has no identity");
+ assert(typeof row.revision === "number", "SQLite Dispatch row has no revision");
+ assert(typeof row.created_at === "number" && typeof row.updated_at === "number", "SQLite Dispatch timestamps are invalid");
+ assert(typeof row.priority === "number" && typeof row.available_at === "number", "SQLite Dispatch schedule is invalid");
+ assert(typeof row.admission === "string" && typeof row.phase === "string", "SQLite Dispatch state is invalid");
+ const identity = JSON.parse(row.identity_json) as BuildDispatchIdentity;
+ const lease = dispatchLease(row);
+ const snapshot = {
+ ...identity,
+ revision: row.revision,
+ createdAt: row.created_at,
+ updatedAt: row.updated_at,
+ priority: row.priority,
+ availableAt: row.available_at,
+ admission: row.admission,
+ phase: row.phase,
+ ...(lease === undefined ? {} : { lease }),
+ ...(typeof row.reason === "string" ? { reason: row.reason } : {}),
+ ...(typeof row.cancel_requested_at === "number"
+ ? { cancellation: {
+ requestedAt: row.cancel_requested_at,
+ ...(typeof row.cancel_reason === "string" ? { reason: row.cancel_reason } : {}),
+ } }
+ : {}),
+ ...(typeof row.terminal === "string" ? { terminal: row.terminal } : {}),
+ } as BuildDispatchSnapshot;
+ verifyBuildDispatchSnapshot(snapshot);
+ return snapshot;
+}
+
+function leaseMatches(row: Row, lease: DispatchLease): boolean {
+ return row.lease_owner === lease.owner
+ && row.lease_token === lease.token
+ && row.lease_fence === lease.fence;
+}
+
+class SqliteBuildDispatchStore implements BuildDispatchStore {
+ readonly #database: DatabaseSync;
+
+ constructor(database: DatabaseSync) {
+ this.#database = database;
+ }
+
+ async create(
+ identity: BuildDispatchIdentity,
+ options: { readonly priority?: number; readonly availableAt?: number; readonly now?: number } = {},
+ ): Promise<BuildDispatchCreate> {
+ verifyBuildDispatchIdentity(identity);
+ const now = nonNegativeInteger(options.now ?? Date.now(), "Dispatch creation time");
+ const priority = options.priority ?? 0;
+ assert(Number.isSafeInteger(priority), "Dispatch priority must be a safe integer");
+ const availableAt = nonNegativeInteger(options.availableAt ?? now, "Dispatch availableAt");
+ const result = this.#database.prepare(`
+ INSERT OR IGNORE INTO svml_dispatches (
+ build_id, identity_json, revision, created_at, updated_at, priority, available_at,
+ admission, phase, lease_owner, lease_token, lease_fence, lease_expires_at,
+ reason, cancel_requested_at, cancel_reason, terminal
+ ) VALUES (?, ?, 0, ?, ?, ?, ?, 'open', 'queued', NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL)
+ `).run(identity.build, canonicalStringify(identity), now, now, priority, availableAt);
+ const snapshot = await this.read(identity.build);
+ if (snapshot === undefined) throw new Error(`Dispatch ${identity.build} disappeared after create`);
+ assert(canonicalStringify({ format: snapshot.format, id: snapshot.id, build: snapshot.build, core: snapshot.core, runtimeClosure: snapshot.runtimeClosure })
+ === canonicalStringify(identity), `Dispatch ${identity.build} already names another Build or Runtime Closure`);
+ return { status: result.changes === 1 ? "created" : "existing", snapshot };
+ }
+
+ async read(build: string): Promise<BuildDispatchSnapshot | undefined> {
+ const row = this.#database.prepare("SELECT * FROM svml_dispatches WHERE build_id = ?").get(build) as Row | undefined;
+ return row === undefined ? undefined : parseDispatchSnapshot(row);
+ }
+
+ async list(query: DispatchQuery = {}): Promise<readonly BuildDispatchSnapshot[]> {
+ const rows = this.#database.prepare(`
+ SELECT * FROM svml_dispatches
+ ORDER BY priority DESC, available_at ASC, created_at ASC, build_id ASC
+ `).all() as Row[];
+ return rows.map(parseDispatchSnapshot)
+ .filter((item) => query.phases === undefined || query.phases.includes(item.phase))
+ .filter((item) => query.admission === undefined || item.admission === query.admission);
+ }
+
+ async claim(request: BuildDispatchClaim): Promise<BuildDispatchSnapshot | undefined> {
+ assert(request.owner.trim().length > 0 && request.token.trim().length > 0, "Dispatch claim identity is empty");
+ const now = nonNegativeInteger(request.now, "Dispatch claim time");
+ const leaseMs = positiveInteger(request.leaseMs, "Dispatch leaseMs");
+ return transaction(this.#database, () => {
+ const row = this.#database.prepare(`
+ SELECT * FROM svml_dispatches
+ WHERE phase != 'terminal'
+ AND available_at <= ?
+ AND (lease_owner IS NULL OR lease_expires_at <= ?)
+ ORDER BY priority DESC, available_at ASC, created_at ASC, build_id ASC
+ LIMIT 1
+ `).get(now, now) as Row | undefined;
+ if (row === undefined) return undefined;
+ assert(typeof row.build_id === "string" && typeof row.revision === "number" && typeof row.lease_fence === "number",
+ "SQLite Dispatch claim row is invalid");
+ const fence = row.lease_fence + 1;
+ const updated = this.#database.prepare(`
+ UPDATE svml_dispatches
+ SET revision = revision + 1, updated_at = ?, phase = 'leased',
+ lease_owner = ?, lease_token = ?, lease_fence = ?, lease_expires_at = ?
+ WHERE build_id = ? AND revision = ?
+ AND phase != 'terminal' AND (lease_owner IS NULL OR lease_expires_at <= ?)
+ `).run(now, request.owner, request.token, fence, now + leaseMs, row.build_id, row.revision, now);
+ assert(updated.changes === 1, `Dispatch ${row.build_id} claim lost its transaction`);
+ const claimed = this.#database.prepare("SELECT * FROM svml_dispatches WHERE build_id = ?").get(row.build_id) as Row;
+ return parseDispatchSnapshot(claimed);
+ });
+ }
+
+ async heartbeat(build: string, lease: DispatchLease, now: number, leaseMs: number): Promise<BuildDispatchSnapshot> {
+ verifyDispatchLease(lease);
+ nonNegativeInteger(now, "Dispatch heartbeat time");
+ positiveInteger(leaseMs, "Dispatch heartbeat leaseMs");
+ const updated = this.#database.prepare(`
+ UPDATE svml_dispatches
+ SET revision = revision + 1, updated_at = ?, lease_expires_at = ?
+ WHERE build_id = ? AND phase = 'leased'
+ AND lease_owner = ? AND lease_token = ? AND lease_fence = ? AND lease_expires_at > ?
+ `).run(now, now + leaseMs, build, lease.owner, lease.token, lease.fence, now);
+ assert(updated.changes === 1, `Dispatch ${build} lease is stale`);
+ return (await this.read(build))!;
+ }
+
+ async release(
+ build: string,
+ lease: DispatchLease,
+ update: BuildDispatchRelease,
+ now = Date.now(),
+ ): Promise<BuildDispatchSnapshot> {
+ verifyDispatchLease(lease);
+ nonNegativeInteger(now, "Dispatch release time");
+ nonNegativeInteger(update.availableAt, "Dispatch release availableAt");
+ const updated = this.#database.prepare(`
+ UPDATE svml_dispatches
+ SET revision = revision + 1, updated_at = ?, phase = ?, available_at = ?, reason = ?,
+ lease_owner = NULL, lease_token = NULL, lease_expires_at = NULL
+ WHERE build_id = ? AND phase = 'leased'
+ AND lease_owner = ? AND lease_token = ? AND lease_fence = ?
+ `).run(now, update.phase, update.availableAt, update.reason ?? null,
+ build, lease.owner, lease.token, lease.fence);
+ assert(updated.changes === 1, `Dispatch ${build} lease is stale`);
+ return (await this.read(build))!;
+ }
+
+ async finish(
+ build: string,
+ lease: DispatchLease,
+ terminal: "complete" | "failed" | "cancelled",
+ reason?: string,
+ now = Date.now(),
+ ): Promise<BuildDispatchSnapshot> {
+ verifyDispatchLease(lease);
+ const updated = this.#database.prepare(`
+ UPDATE svml_dispatches
+ SET revision = revision + 1, updated_at = ?, phase = 'terminal', admission = 'closed',
+ terminal = ?, reason = ?, lease_owner = NULL, lease_token = NULL, lease_expires_at = NULL
+ WHERE build_id = ? AND phase = 'leased'
+ AND lease_owner = ? AND lease_token = ? AND lease_fence = ?
+ `).run(now, terminal, reason ?? null, build, lease.owner, lease.token, lease.fence);
+ assert(updated.changes === 1, `Dispatch ${build} lease is stale`);
+ return (await this.read(build))!;
+ }
+
+ async requestCancellation(build: string, reason?: string, now = Date.now()): Promise<BuildDispatchSnapshot> {
+ nonNegativeInteger(now, "cancellation request time");
+ const updated = this.#database.prepare(`
+ UPDATE svml_dispatches
+ SET revision = revision + 1, updated_at = ?, admission = 'closing',
+ cancel_requested_at = COALESCE(cancel_requested_at, ?),
+ cancel_reason = COALESCE(cancel_reason, ?), available_at = MIN(available_at, ?)
+ WHERE build_id = ? AND phase != 'terminal'
+ `).run(now, now, reason ?? null, now, build);
+ if (updated.changes === 0) {
+ const current = await this.read(build);
+ if (current === undefined) throw new Error(`Dispatch ${build} does not exist`);
+ return current;
+ }
+ return (await this.read(build))!;
+ }
+
+ async acquireCapacity(request: CapacityAcquireRequest): Promise<CapacityAcquire> {
+ assert(request.owner.trim().length > 0 && request.token.trim().length > 0, "Capacity claimant is empty");
+ assert(request.lane.trim().length > 0, "Capacity lane is empty");
+ nonNegativeInteger(request.now, "Capacity acquisition time");
+ positiveInteger(request.leaseMs, "Capacity leaseMs");
+ verifyCapacityLimits(request.limits);
+ verifyDispatchLease(request.buildLease);
+ const id = capacityReservationId(request.build, request.command);
+ return transaction(this.#database, () => {
+ const dispatch = this.#database.prepare(`
+ SELECT lease_owner, lease_token, lease_fence FROM svml_dispatches
+ WHERE build_id = ? AND phase = 'leased'
+ `).get(request.build) as Row | undefined;
+ assert(dispatch !== undefined && leaseMatches(dispatch, request.buildLease),
+ `Dispatch ${request.build} lease is stale`);
+ const existing = this.#database.prepare("SELECT * FROM svml_capacity WHERE reservation_id = ?").get(id) as Row | undefined;
+ if (existing !== undefined && typeof existing.active_expires_at === "number" && existing.active_expires_at > request.now) {
+ return { status: "blocked", retryAt: existing.active_expires_at, reason: "lane-active" };
+ }
+ const activeGlobal = this.#database.prepare(`
+ SELECT COUNT(*) AS count FROM svml_capacity
+ WHERE active_owner IS NOT NULL AND active_expires_at > ? AND reservation_id != ?
+ `).get(request.now, id) as Row;
+ const activeLane = this.#database.prepare(`
+ SELECT COUNT(*) AS count FROM svml_capacity
+ WHERE lane = ? AND active_owner IS NOT NULL AND active_expires_at > ? AND reservation_id != ?
+ `).get(request.lane, request.now, id) as Row;
+ const inFlightLane = this.#database.prepare(`
+ SELECT COUNT(*) AS count FROM svml_capacity
+ WHERE lane = ? AND in_flight = 1 AND reservation_id != ?
+ `).get(request.lane, id) as Row;
+ assert(typeof activeGlobal.count === "number" && typeof activeLane.count === "number" && typeof inFlightLane.count === "number",
+ "SQLite Capacity counts are invalid");
+ const retryAtRow = this.#database.prepare(`
+ SELECT MIN(active_expires_at) AS retry_at FROM svml_capacity
+ WHERE active_owner IS NOT NULL AND active_expires_at > ?
+ `).get(request.now) as Row;
+ const retryAt = typeof retryAtRow.retry_at === "number" ? retryAtRow.retry_at : request.now + 100;
+ if (activeGlobal.count >= request.limits.globalActive) return { status: "blocked", retryAt, reason: "global-active" };
+ if (activeLane.count >= request.limits.laneActive) return { status: "blocked", retryAt, reason: "lane-active" };
+ const alreadyInFlight = existing?.in_flight === 1;
+ if (request.mode === "recoverable" && !alreadyInFlight && inFlightLane.count >= request.limits.laneInFlight) {
+ return { status: "blocked", retryAt, reason: "lane-in-flight" };
+ }
+ const fence = typeof existing?.active_fence === "number" ? existing.active_fence + 1 : 1;
+ if (existing === undefined) {
+ this.#database.prepare(`
+ INSERT INTO svml_capacity (
+ reservation_id, build_id, command_id, lane, mode, in_flight,
+ active_owner, active_token, active_fence, active_expires_at, created_at, updated_at
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ `).run(id, request.build, request.command, request.lane, request.mode,
+ request.mode === "recoverable" ? 1 : 0,
+ request.owner, request.token, fence, request.now + request.leaseMs, request.now, request.now);
+ } else {
+ assert(existing.build_id === request.build && existing.command_id === request.command
+ && existing.lane === request.lane && existing.mode === request.mode,
+ `Capacity reservation ${id} identity differs`);
+ this.#database.prepare(`
+ UPDATE svml_capacity
+ SET in_flight = MAX(in_flight, ?), active_owner = ?, active_token = ?,
+ active_fence = ?, active_expires_at = ?, updated_at = ?
+ WHERE reservation_id = ?
+ `).run(request.mode === "recoverable" ? 1 : 0, request.owner, request.token,
+ fence, request.now + request.leaseMs, request.now, id);
+ }
+ return { status: "acquired", reservation: parseCapacityReservation(
+ this.#database.prepare("SELECT * FROM svml_capacity WHERE reservation_id = ?").get(id) as Row,
+ ) };
+ });
+ }
+
+ async heartbeatCapacity(id: Digest, lease: DispatchLease, now: number, leaseMs: number): Promise<CapacityReservation> {
+ verifyDispatchLease(lease, "Capacity lease");
+ const updated = this.#database.prepare(`
+ UPDATE svml_capacity SET active_expires_at = ?, updated_at = ?
+ WHERE reservation_id = ? AND active_owner = ? AND active_token = ? AND active_fence = ? AND active_expires_at > ?
+ `).run(now + leaseMs, now, id, lease.owner, lease.token, lease.fence, now);
+ assert(updated.changes === 1, `Capacity reservation ${id} lease is stale`);
+ return parseCapacityReservation(this.#database.prepare("SELECT * FROM svml_capacity WHERE reservation_id = ?").get(id) as Row);
+ }
+
+ async parkCapacity(id: Digest, lease: DispatchLease, inFlight: boolean, now = Date.now()): Promise<CapacityReservation> {
+ verifyDispatchLease(lease, "Capacity lease");
+ const updated = this.#database.prepare(`
+ UPDATE svml_capacity
+ SET in_flight = ?, active_owner = NULL, active_token = NULL, active_expires_at = NULL, updated_at = ?
+ WHERE reservation_id = ? AND active_owner = ? AND active_token = ? AND active_fence = ?
+ `).run(inFlight ? 1 : 0, now, id, lease.owner, lease.token, lease.fence);
+ assert(updated.changes === 1, `Capacity reservation ${id} lease is stale`);
+ return parseCapacityReservation(this.#database.prepare("SELECT * FROM svml_capacity WHERE reservation_id = ?").get(id) as Row);
+ }
+
+ async releaseCapacity(id: Digest, lease: DispatchLease): Promise<void> {
+ verifyDispatchLease(lease, "Capacity lease");
+ const removed = this.#database.prepare(`
+ DELETE FROM svml_capacity
+ WHERE reservation_id = ? AND active_owner = ? AND active_token = ? AND active_fence = ?
+ `).run(id, lease.owner, lease.token, lease.fence);
+ assert(removed.changes === 1, `Capacity reservation ${id} lease is stale`);
+ }
+
+ async clearCapacity(id: Digest, build: string, buildLease: DispatchLease): Promise<void> {
+ verifyDispatchLease(buildLease);
+ transaction(this.#database, () => {
+ const dispatch = this.#database.prepare(`
+ SELECT lease_owner, lease_token, lease_fence FROM svml_dispatches
+ WHERE build_id = ? AND phase = 'leased'
+ `).get(build) as Row | undefined;
+ assert(dispatch !== undefined && leaseMatches(dispatch, buildLease),
+ `Dispatch ${build} lease is stale`);
+ this.#database.prepare(`
+ DELETE FROM svml_capacity WHERE reservation_id = ? AND build_id = ?
+ `).run(id, build);
+ });
+ }
+
+ async listCapacity(): Promise<readonly CapacityReservation[]> {
+ return (this.#database.prepare("SELECT * FROM svml_capacity ORDER BY created_at ASC, reservation_id ASC").all() as Row[])
+ .map(parseCapacityReservation);
+ }
+}
+
+function parseCapacityReservation(row: Row): CapacityReservation {
+ assert(typeof row.reservation_id === "string" && typeof row.build_id === "string"
+ && typeof row.command_id === "string" && typeof row.lane === "string" && typeof row.mode === "string",
+ "SQLite Capacity row identity is invalid");
+ assert(typeof row.in_flight === "number" && typeof row.created_at === "number" && typeof row.updated_at === "number",
+ "SQLite Capacity row state is invalid");
+ const active = row.active_owner === null || row.active_owner === undefined ? undefined : (() => {
+ assert(typeof row.active_owner === "string" && typeof row.active_token === "string"
+ && typeof row.active_fence === "number" && typeof row.active_expires_at === "number",
+ "SQLite Capacity active lease is invalid");
+ return { owner: row.active_owner, token: row.active_token, fence: row.active_fence, expiresAt: row.active_expires_at };
+ })();
+ const value = {
+ format: "svml.capacity-reservation@1" as const,
+ id: row.reservation_id as Digest,
+ build: row.build_id,
+ command: row.command_id,
+ lane: row.lane,
+ mode: row.mode,
+ inFlight: row.in_flight === 1,
+ ...(active === undefined ? {} : { active }),
+ createdAt: row.created_at,
+ updatedAt: row.updated_at,
+ } as CapacityReservation;
+ verifyCapacityReservation(value);
+ return value;
+}
+
+class SqliteRuntimeJournal implements RuntimeJournal {
+ readonly #database: DatabaseSync;
+
+ constructor(database: DatabaseSync) {
+ this.#database = database;
+ }
+
+ async append(entry: Omit<RuntimeJournalEntry, "format" | "sequence">): Promise<RuntimeJournalEntry> {
+ nonNegativeInteger(entry.at, "Runtime Journal timestamp");
+ const detail = canonicalize(entry.detail);
+ const result = this.#database.prepare(`
+ INSERT INTO svml_runtime_journal (at, kind, build_id, operation_id, worker_id, detail_json)
+ VALUES (?, ?, ?, ?, ?, ?)
+ `).run(entry.at, entry.kind, entry.build ?? null, entry.operation ?? null, entry.worker ?? null,
+ canonicalStringify(detail));
+ assert(typeof result.lastInsertRowid === "number" || typeof result.lastInsertRowid === "bigint",
+ "SQLite Runtime Journal returned no sequence");
+ const sequence = Number(result.lastInsertRowid);
+ const value: RuntimeJournalEntry = {
+ format: "svml.runtime-journal-entry@1",
+ sequence,
+ ...entry,
+ detail,
+ };
+ verifyRuntimeJournalEntry(value);
+ return value;
+ }
+
+ async list(query: RuntimeJournalQuery = {}): Promise<readonly RuntimeJournalEntry[]> {
+ const after = query.after ?? 0;
+ const limit = query.limit ?? 1_000;
+ nonNegativeInteger(after, "Runtime Journal after");
+ positiveInteger(limit, "Runtime Journal limit");
+ const rows = this.#database.prepare(`
+ SELECT sequence, at, kind, build_id, operation_id, worker_id, detail_json
+ FROM svml_runtime_journal WHERE sequence > ? ORDER BY sequence ASC LIMIT ?
+ `).all(after, limit) as Row[];
+ return rows.map((row) => {
+ assert(typeof row.sequence === "number" && typeof row.at === "number" && typeof row.kind === "string"
+ && typeof row.detail_json === "string", "SQLite Runtime Journal row is invalid");
+ const entry = {
+ format: "svml.runtime-journal-entry@1" as const,
+ sequence: row.sequence,
+ at: row.at,
+ kind: row.kind,
+ ...(typeof row.build_id === "string" ? { build: row.build_id } : {}),
+ ...(typeof row.operation_id === "string" ? { operation: row.operation_id as Digest } : {}),
+ ...(typeof row.worker_id === "string" ? { worker: row.worker_id } : {}),
+ detail: JSON.parse(row.detail_json),
+ } as RuntimeJournalEntry;
+ verifyRuntimeJournalEntry(entry);
+ return entry;
+ }).filter((entry) => query.build === undefined || entry.build === query.build)
+ .filter((entry) => query.operation === undefined || entry.operation === query.operation)
+ .filter((entry) => query.worker === undefined || entry.worker === query.worker);
+ }
+}
+
+/** One local database with domain-neutral execution facts and Host presentation catalog. */
export class SqliteRuntimeState {
readonly path: string;
readonly builds: BuildStore;
readonly operations: OperationStore;
+ readonly dispatch: BuildDispatchStore;
+ readonly journal: RuntimeJournal;
readonly catalog: BuildCatalog;
readonly #database: DatabaseSync;
@@ -385,11 +866,12 @@ export class SqliteRuntimeState {
CREATE TABLE IF NOT EXISTS svml_operations (
operation_id TEXT PRIMARY KEY,
revision INTEGER NOT NULL,
- status TEXT NOT NULL CHECK (status IN ('created', 'pending', 'completed', 'failed')),
+ status TEXT NOT NULL CHECK (status IN ('created', 'pending', 'completed', 'failed', 'cancelled')),
identity_json TEXT NOT NULL,
checkpoint_json TEXT,
completion_json TEXT,
- failure_json TEXT
+ failure_json TEXT,
+ cancellation_json TEXT
) STRICT;
CREATE TABLE IF NOT EXISTS svml_build_catalog (
build_id TEXT PRIMARY KEY,
@@ -398,6 +880,54 @@ export class SqliteRuntimeState {
updated_at INTEGER NOT NULL,
descriptor_json TEXT NOT NULL
) STRICT;
+ CREATE TABLE IF NOT EXISTS svml_dispatches (
+ build_id TEXT PRIMARY KEY,
+ identity_json TEXT NOT NULL,
+ revision INTEGER NOT NULL,
+ created_at INTEGER NOT NULL,
+ updated_at INTEGER NOT NULL,
+ priority INTEGER NOT NULL,
+ available_at INTEGER NOT NULL,
+ admission TEXT NOT NULL CHECK (admission IN ('open', 'closing', 'closed')),
+ phase TEXT NOT NULL CHECK (phase IN ('queued', 'leased', 'waiting', 'blocked', 'settling', 'terminal')),
+ lease_owner TEXT,
+ lease_token TEXT,
+ lease_fence INTEGER NOT NULL,
+ lease_expires_at INTEGER,
+ reason TEXT,
+ cancel_requested_at INTEGER,
+ cancel_reason TEXT,
+ terminal TEXT CHECK (terminal IS NULL OR terminal IN ('complete', 'failed', 'cancelled'))
+ ) STRICT;
+ CREATE INDEX IF NOT EXISTS svml_dispatch_ready
+ ON svml_dispatches (phase, available_at, priority DESC);
+ CREATE TABLE IF NOT EXISTS svml_capacity (
+ reservation_id TEXT PRIMARY KEY,
+ build_id TEXT NOT NULL,
+ command_id TEXT NOT NULL,
+ lane TEXT NOT NULL,
+ mode TEXT NOT NULL CHECK (mode IN ('active', 'recoverable')),
+ in_flight INTEGER NOT NULL CHECK (in_flight IN (0, 1)),
+ active_owner TEXT,
+ active_token TEXT,
+ active_fence INTEGER NOT NULL,
+ active_expires_at INTEGER,
+ created_at INTEGER NOT NULL,
+ updated_at INTEGER NOT NULL,
+ UNIQUE (build_id, command_id)
+ ) STRICT;
+ CREATE INDEX IF NOT EXISTS svml_capacity_lane ON svml_capacity (lane, in_flight, active_expires_at);
+ CREATE TABLE IF NOT EXISTS svml_runtime_journal (
+ sequence INTEGER PRIMARY KEY AUTOINCREMENT,
+ at INTEGER NOT NULL,
+ kind TEXT NOT NULL,
+ build_id TEXT,
+ operation_id TEXT,
+ worker_id TEXT,
+ detail_json TEXT NOT NULL
+ ) STRICT;
+ CREATE INDEX IF NOT EXISTS svml_runtime_journal_build ON svml_runtime_journal (build_id, sequence);
+ CREATE INDEX IF NOT EXISTS svml_runtime_journal_operation ON svml_runtime_journal (operation_id, sequence);
`);
const version = this.#database.prepare("SELECT schema_version FROM svml_store_meta WHERE singleton = 1").get() as Row | undefined;
if (version === undefined) {
@@ -408,6 +938,8 @@ export class SqliteRuntimeState {
}
this.builds = new SqliteBuildStore(database);
this.operations = new SqliteOperationStore(database);
+ this.dispatch = new SqliteBuildDispatchStore(database);
+ this.journal = new SqliteRuntimeJournal(database);
this.catalog = new SqliteBuildCatalog(database);
}
@@ -424,6 +956,8 @@ export function createSqliteRuntimeServicePackage(
});
const buildInstance = options.buildInstance ?? "builds.sqlite";
const operationInstance = options.operationInstance ?? "operations.sqlite";
+ const dispatchInstance = options.dispatchInstance ?? "dispatch.sqlite";
+ const journalInstance = options.journalInstance ?? "journal.sqlite";
try {
return Object.assign(defineRuntimeServicePackage({
name: options.name ?? "state.sqlite",
@@ -461,6 +995,38 @@ export function createSqliteRuntimeServicePackage(
},
service: state.operations,
},
+ {
+ role: "dispatch-store",
+ facet: "dispatch-store",
+ instance: dispatchInstance,
+ implementation: {
+ locator: "@narratage/store-sqlite/dispatch-store",
+ digest: sqliteDispatchStoreImplementationDigest,
+ },
+ permissions: ["filesystem:state"],
+ configuration: {
+ path: state.path,
+ schemaVersion: databaseSchemaVersion,
+ busyTimeoutMs: options.busyTimeoutMs ?? 5_000,
+ },
+ service: state.dispatch,
+ },
+ {
+ role: "runtime-journal",
+ facet: "runtime-journal",
+ instance: journalInstance,
+ implementation: {
+ locator: "@narratage/store-sqlite/runtime-journal",
+ digest: sqliteRuntimeJournalImplementationDigest,
+ },
+ permissions: ["filesystem:state"],
+ configuration: {
+ path: state.path,
+ schemaVersion: databaseSchemaVersion,
+ busyTimeoutMs: options.busyTimeoutMs ?? 5_000,
+ },
+ service: state.journal,
+ },
],
close: () => state.close(),
}), { catalog: state.catalog });
diff --git a/packages/store-sqlite/test/store.test.ts b/packages/store-sqlite/test/store.test.ts
index fa3781eb..c7696550 100644
--- a/packages/store-sqlite/test/store.test.ts
+++ b/packages/store-sqlite/test/store.test.ts
@@ -6,6 +6,8 @@ import test from "node:test";
import { digestOf } from "@narratage/core";
import {
+ createBuildDispatchIdentity,
+ operationCancellationRequestId,
sealOperationIdentity,
} from "@narratage/runtime";
import {
@@ -111,6 +113,45 @@ test("SQLite Operation CAS preserves a terminal completion", async () => {
}
});
+test("SQLite keeps cancellation control independent from execution state", async () => {
+ const directory = await mkdtemp(join(tmpdir(), "svml-sqlite-cancel-"));
+ try {
+ const state = new SqliteRuntimeState(join(directory, "runtime.sqlite"));
+ const identity = sealOperationIdentity({
+ build: "cancel-build",
+ command: "command:cancel",
+ endpoint: "endpoint.cancel",
+ implementationDigest: digestOf("implementation.cancel"),
+ runtimeClosure: digestOf("runtime.cancel"),
+ requestDigest: digestOf("request.cancel"),
+ attempt: 1,
+ });
+ await state.operations.create(identity);
+ const requestId = operationCancellationRequestId(identity.id, 100);
+ const controlled = await state.operations.compareAndSwap(identity.id, 0, {
+ status: "control",
+ cancellation: { requestedAt: 100, requestId, status: "unsupported", attempts: 1 },
+ });
+ assert.equal(controlled.status, "stored");
+ const completed = await state.operations.compareAndSwap(identity.id, 1, {
+ status: "completed",
+ completion: {
+ value: { kind: "inline", value: "late result" },
+ conformance: "exact",
+ delivery: "executed",
+ metadata: {},
+ },
+ });
+ assert.equal(completed.status, "stored");
+ const reopened = await state.operations.read(identity.id);
+ assert.equal(reopened?.status, "completed");
+ assert.equal(reopened?.cancellation?.status, "unsupported");
+ state.close();
+ } finally {
+ await rm(directory, { recursive: true, force: true });
+ }
+});
+
test("Host Catalog schema changes do not change the execution Runtime Closure", async () => {
const directory = await mkdtemp(join(tmpdir(), "svml-sqlite-closure-"));
try {
@@ -118,8 +159,7 @@ test("Host Catalog schema changes do not change the execution Runtime Closure",
assert.deepEqual(
services.services.map((item) => item.instance.configurationDigest),
[
- digestOf({ path: join(directory, "runtime.sqlite"), schemaVersion: 1, busyTimeoutMs: 5_000 }),
- digestOf({ path: join(directory, "runtime.sqlite"), schemaVersion: 1, busyTimeoutMs: 5_000 }),
+ ...Array(4).fill(digestOf({ path: join(directory, "runtime.sqlite"), schemaVersion: 3, busyTimeoutMs: 5_000 })),
],
);
await services.close?.();
@@ -127,3 +167,54 @@ test("Host Catalog schema changes do not change the execution Runtime Closure",
await rm(directory, { recursive: true, force: true });
}
});
+
+test("SQLite fences expired Workers and shares capacity across Build dispatches", async () => {
+ const directory = await mkdtemp(join(tmpdir(), "svml-sqlite-dispatch-"));
+ try {
+ const state = new SqliteRuntimeState(join(directory, "runtime.sqlite"));
+ const closure = digestOf("runtime:dispatch-test");
+ await state.dispatch.create(createBuildDispatchIdentity({
+ build: "build-a", core: digestOf("core:a"), runtimeClosure: closure,
+ }), { now: 100 });
+ const first = await state.dispatch.claim({ owner: "worker-a", token: "lease-a", now: 100, leaseMs: 10 });
+ assert.ok(first?.lease);
+ assert.equal(first.lease.fence, 1);
+ assert.equal(await state.dispatch.claim({ owner: "worker-b", token: "early", now: 105, leaseMs: 10 }), undefined);
+ const second = await state.dispatch.claim({ owner: "worker-b", token: "lease-b", now: 111, leaseMs: 10 });
+ assert.ok(second?.lease);
+ assert.equal(second.lease.fence, 2);
+ await assert.rejects(
+ state.dispatch.release("build-a", first.lease, { phase: "waiting", availableAt: 120 }, 112),
+ /stale/u,
+ );
+ await assert.rejects(
+ state.dispatch.acquireCapacity({
+ build: "build-a", command: "command:a", lane: "paid", mode: "recoverable",
+ buildLease: first.lease, owner: "worker-a", token: "capacity-a", now: 112, leaseMs: 10,
+ limits: { globalActive: 1, laneActive: 1, laneInFlight: 1 },
+ }),
+ /stale/u,
+ );
+ const reserved = await state.dispatch.acquireCapacity({
+ build: "build-a", command: "command:a", lane: "paid", mode: "recoverable",
+ buildLease: second.lease, owner: "worker-b", token: "capacity-b", now: 112, leaseMs: 10,
+ limits: { globalActive: 1, laneActive: 1, laneInFlight: 1 },
+ });
+ assert.equal(reserved.status, "acquired");
+
+ await state.dispatch.create(createBuildDispatchIdentity({
+ build: "build-b", core: digestOf("core:b"), runtimeClosure: closure,
+ }), { now: 112 });
+ const third = await state.dispatch.claim({ owner: "worker-c", token: "lease-c", now: 112, leaseMs: 10 });
+ assert.ok(third?.lease);
+ const blocked = await state.dispatch.acquireCapacity({
+ build: "build-b", command: "command:b", lane: "paid", mode: "recoverable",
+ buildLease: third.lease, owner: "worker-c", token: "capacity-c", now: 112, leaseMs: 10,
+ limits: { globalActive: 1, laneActive: 1, laneInFlight: 1 },
+ });
+ assert.equal(blocked.status, "blocked");
+ state.close();
+ } finally {
+ await rm(directory, { recursive: true, force: true });
+ }
+});
diff --git a/packages/video-cli/src/cli.ts b/packages/video-cli/src/cli.ts
index f9697bfa..bc7d1091 100644
--- a/packages/video-cli/src/cli.ts
+++ b/packages/video-cli/src/cli.ts
@@ -13,8 +13,40 @@ const color = !json && colorMode !== "never" && process.env.TERM !== "dumb"
&& (colorMode === "always" || (process.env.NO_COLOR === undefined && process.stdout.isTTY === true));
const unicode = process.env.TERM !== "dumb";
+async function readSecret(prompt: string): Promise<string> {
+ if (process.stdin.isTTY !== true || typeof process.stdin.setRawMode !== "function") {
+ const chunks: Buffer[] = [];
+ for await (const chunk of process.stdin) chunks.push(Buffer.from(chunk));
+ return Buffer.concat(chunks).toString("utf8");
+ }
+ process.stderr.write(prompt);
+ return await new Promise<string>((resolve, reject) => {
+ let value = "";
+ const finish = (error?: Error): void => {
+ process.stdin.off("data", input);
+ process.stdin.setRawMode(false);
+ process.stdin.pause();
+ process.stderr.write("\n");
+ if (error === undefined) resolve(value);
+ else reject(error);
+ };
+ const input = (chunk: Buffer): void => {
+ for (const byte of chunk) {
+ if (byte === 3) { finish(new Error("credential input cancelled")); return; }
+ if (byte === 10 || byte === 13) { finish(); return; }
+ if (byte === 8 || byte === 127) { value = value.slice(0, -1); continue; }
+ value += String.fromCharCode(byte);
+ }
+ };
+ process.stdin.setRawMode(true);
+ process.stdin.resume();
+ process.stdin.on("data", input);
+ });
+}
+
const io: CliIo = {
write: (text) => process.stdout.write(text),
+ readSecret,
terminal: {
isTTY: process.stdout.isTTY === true,
color,
diff --git a/packages/video-cli/src/distribution.ts b/packages/video-cli/src/distribution.ts
index 09cd84e2..11dfcb73 100644
--- a/packages/video-cli/src/distribution.ts
+++ b/packages/video-cli/src/distribution.ts
@@ -1,4 +1,5 @@
import type { CliDistribution } from "@narratage/cli";
+import { fileURLToPath } from "node:url";
import {
createVideoCompiler,
videoBuiltInPackageContributions,
@@ -13,6 +14,10 @@ export const videoCliDistribution: CliDistribution = {
builtInPackageContributions: videoBuiltInPackageContributions,
runFrontends: [],
createCompiler: createVideoCompiler,
+ runtimeWorkerLaunch: () => ({
+ command: process.execPath,
+ args: [...process.execArgv, fileURLToPath(new URL("./cli.ts", import.meta.url))],
+ }),
createRuntimeFromConfig: async (path) => {
const { createVideoRuntimeFromConfig } = await import("./runtime-config.js");
return await createVideoRuntimeFromConfig(path);
diff --git a/packages/video-cli/test/cli.test.ts b/packages/video-cli/test/cli.test.ts
index aec64881..c49b3c94 100644
--- a/packages/video-cli/test/cli.test.ts
+++ b/packages/video-cli/test/cli.test.ts
@@ -267,23 +267,47 @@ test("CLI accepts a declarative Runtime Profile without an executable config mod
const runtimePackageLock = join(root, "runtime.packages.lock.json");
await writeNodePackageLock(
runtimePackageLock,
- await createNodePackageLock(["@narratage/provider-kie"], process.cwd()),
+ await createNodePackageLock([
+ "@narratage/local",
+ "@narratage/store-sqlite",
+ "@narratage/artifact-store-fs",
+ "@narratage/credential-store-env",
+ "@narratage/provider-kie",
+ ], process.cwd()),
);
await writeFile(profile, JSON.stringify({
format: "svml.runtime-config@1",
root,
- statePath: join(root, "state.sqlite"),
- catalogPath: join(root, "catalog.sqlite"),
- artifactPath: join(root, "artifacts"),
runtimePackageLock,
+ runtimeServices: [
+ { use: "@narratage/local", instance: "execution" },
+ { use: "@narratage/store-sqlite", instance: "state", config: { path: "state.sqlite" } },
+ { use: "@narratage/artifact-store-fs", instance: "artifacts", config: { path: "artifacts" } },
+ { use: "@narratage/credential-store-env", instance: "credentials", config: {} },
+ ],
+ services: {
+ scheduler: "execution.scheduler",
+ worker: "execution.worker",
+ stores: {
+ build: "state.builds",
+ operations: "state.operations",
+ dispatch: "state.dispatch",
+ journal: "state.journal",
+ artifacts: "artifacts",
+ credentials: ["credentials"],
+ },
+ },
endpoints: [{
use: "@narratage/provider-kie",
instance: "kie.cli-test",
lane: "generation",
- config: { apiKeyEnv: "SVML_TEST_MISSING_KIE_KEY", defaultConcurrency: 2 },
+ config: { apiKey: { store: "env", key: "SVML_TEST_MISSING_KIE_KEY" }, defaultConcurrency: 2 },
}],
- permissions: ["network:api.kie.ai", "network:kieai.redpandaai.co"],
- scheduling: { lanes: { generation: 2 } },
+ permissions: [
+ "filesystem:state", "filesystem:artifacts", "environment:credentials",
+ "network:api.kie.ai", "network:kieai.redpandaai.co",
+ ],
+ scheduling: { maxConcurrency: 4, lanes: { generation: 2 } },
}), "utf8");
let output = "";
await runCli(["status", "missing-build", "--runtime", profile], {
@@ -297,9 +321,18 @@ test("CLI accepts a declarative Runtime Profile without an executable config mod
readonly ok: boolean;
readonly diagnostics: readonly { readonly code: string; readonly subject?: string }[];
};
- assert.equal(diagnosis.ok, false);
- assert.equal(diagnosis.diagnostics.some((item) =>
- item.code === "RUNTIME_CREDENTIAL_MISSING" && item.subject === "SVML_TEST_MISSING_KIE_KEY"), true);
+ assert.equal(diagnosis.ok, true);
+ assert.equal(diagnosis.diagnostics.length, 0);
+ output = "";
+ await runCli(["auth", "status", "kie.cli-test", "--runtime", profile], {
+ write: (text) => { output += text; },
+ });
+ const auth = JSON.parse(output) as {
+ readonly credentials: readonly { readonly slot: string; readonly configured: boolean; readonly writable: boolean }[];
+ };
+ assert.deepEqual(auth.credentials.map((item) => ({
+ slot: item.slot, configured: item.configured, writable: item.writable,
+ })), [{ slot: "apiKey", configured: false, writable: false }]);
});
test("one checked-in fixture closes the complete provider-free video plan", async () => {
@@ -517,6 +550,9 @@ const catalog = ${JSON.stringify(catalog)};
export default {
async build() { throw new Error("not used"); },
async builds() { return [catalog]; },
+ async credentials() { return []; },
+ async putCredential() { throw new Error("not used"); },
+ async deleteCredential() { throw new Error("not used"); },
async status(id) { return { build: id === "archive-1" ? { build: id, revision: 7, state: ${JSON.stringify(state)} } : undefined, catalog: id === "archive-1" ? catalog : undefined, operations: [] }; },
async readArtifact(digest) {
if (digest === ${JSON.stringify(artifactDigest)}) return bytes;