Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

refactor(runtime): separate endpoint actions from durable operations

kashorin1ed40edd6f71
diff --git a/packages/build-result-s3/src/repository.ts b/packages/build-result-s3/src/repository.ts
index 9e6d5bde..c4406ae0 100644
--- a/packages/build-result-s3/src/repository.ts
+++ b/packages/build-result-s3/src/repository.ts
@@ -119,6 +119,7 @@ class S3BuildResultWriter implements BuildResultWriter {
const updated: BuildResultManifest = {
...manifest,
outcome: input.outcome,
+ ...(input.operations === undefined ? {} : { operations: input.operations }),
finishedAt: manifest.finishedAt ?? now,
...(input.failure === undefined ? {} : { failure: input.failure }),
};
diff --git a/packages/build-result/src/decode.ts b/packages/build-result/src/decode.ts
index 7c6122a9..b4202ccd 100644
--- a/packages/build-result/src/decode.ts
+++ b/packages/build-result/src/decode.ts
@@ -129,6 +129,55 @@ export function decodeBuildResultJson(value: Uint8Array, subject: string): unkno
}
}
+function progressNumber(value: unknown, subject: string): number {
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new Error(`${subject} must be non-negative`);
+ return value;
+}
+
+function operationReceipts(value: unknown, subject: string): NonNullable<BuildResultManifest["operations"]> {
+ if (!Array.isArray(value)) throw new Error(`${subject} must be an array`);
+ return value.map((raw) => {
+ const item = object(raw, subject);
+ const status = item.status;
+ if (status !== "pending" && status !== "completed" && status !== "failed" && status !== "cancelled") throw new Error(`${subject} has an invalid Operation status`);
+ const need = item.need === undefined ? undefined : object(item.need, `${subject}.need`);
+ const progress = item.progress === undefined ? undefined : object(item.progress, `${subject}.progress`);
+ const cancellation = item.cancellation === undefined ? undefined : object(item.cancellation, `${subject}.cancellation`);
+ if (cancellation !== undefined && !["confirmed", "accepted", "unsupported", "too-late", "failed"].includes(String(cancellation.outcome))) {
+ throw new Error(`${subject} has an invalid cancellation acknowledgement`);
+ }
+ const receipt = item.receipt === undefined ? undefined : object(item.receipt, `${subject}.receipt`);
+ const failure = item.failure === undefined ? undefined : object(item.failure, `${subject}.failure`);
+ const credentials = item.credentials === undefined ? undefined : Object.fromEntries(Object.entries(object(item.credentials, `${subject}.credentials`)).map(([slot, rawRef]) => {
+ const ref = object(rawRef, `${subject}.credential`);
+ return [slot, { store: text(ref.store, "credential store"), key: text(ref.key, "credential reference") }];
+ }));
+ return {
+ ...(need === undefined ? {} : { need: { id: text(need.id, "Need id"), capability: typeRef(need.capability, "Need capability") } }),
+ ...(item.createdAt === undefined ? {} : { createdAt: optionalTime(item.createdAt, "Operation createdAt")! }),
+ ...(item.acknowledgedAt === undefined ? {} : { acknowledgedAt: optionalTime(item.acknowledgedAt, "Operation acknowledgedAt")! }),
+ ...(item.endedAt === undefined ? {} : { endedAt: optionalTime(item.endedAt, "Operation endedAt")! }),
+ ...(progress === undefined ? {} : { progress: {
+ phase: text(progress.phase, "Operation phase"),
+ ...(progress.completed === undefined ? {} : { completed: progressNumber(progress.completed, "Operation completed units") }),
+ ...(progress.total === undefined ? {} : { total: progressNumber(progress.total, "Operation total units") }),
+ ...(progress.unit === undefined ? {} : { unit: text(progress.unit, "Operation unit") }),
+ } }),
+ ...(cancellation === undefined ? {} : { cancellation: {
+ outcome: cancellation.outcome as "confirmed" | "accepted" | "unsupported" | "too-late" | "failed",
+ ...(cancellation.message === undefined ? {} : { message: text(cancellation.message, "Cancellation error") }),
+ } }),
+ operation: text(item.operation, "Operation id"), command: text(item.command, "Operation command"),
+ endpoint: text(item.endpoint, "Operation Endpoint"), status,
+ ...(item.pool === undefined ? {} : { pool: text(item.pool, "Operation pool") }),
+ ...(credentials === undefined ? {} : { credentials }),
+ ...(receipt === undefined ? {} : { receipt: { id: text(receipt.id, "Remote receipt id"),
+ ...(receipt.url === undefined ? {} : { url: text(receipt.url, "Remote receipt URL") }) } }),
+ ...(failure === undefined ? {} : { failure: { code: text(failure.code, "Operation failure code"), message: text(failure.message, "Operation failure message") } }),
+ };
+ });
+}
+
/** Decode one Result document and attach the Build identity supplied by its repository address. */
export function decodeBuildResultManifest(
value: unknown,
@@ -170,6 +219,7 @@ export function decodeBuildResultManifest(
...(outcome === undefined ? {} : { outcome }),
...(failure === undefined ? {} : { failure }),
outputs: outputs(item.outputs, `${subject}.outputs`),
+ ...(item.operations === undefined ? {} : { operations: operationReceipts(item.operations, `${subject}.operations`) }),
};
}
diff --git a/packages/build-result/src/store.ts b/packages/build-result/src/store.ts
index 53f0eb8b..856000a1 100644
--- a/packages/build-result/src/store.ts
+++ b/packages/build-result/src/store.ts
@@ -422,6 +422,7 @@ export class FileBuildResult {
const updated: BuildResultManifest = {
...manifest,
outcome: input.outcome,
+ ...(input.operations === undefined ? {} : { operations: input.operations }),
finishedAt: manifest.finishedAt ?? now,
...(input.failure === undefined ? {} : { failure: input.failure }),
};
diff --git a/packages/build-result/src/types.ts b/packages/build-result/src/types.ts
index ba7544a6..f0e3d7b2 100644
--- a/packages/build-result/src/types.ts
+++ b/packages/build-result/src/types.ts
@@ -154,6 +154,24 @@ export type BuildResultOutput = {
readonly value: BuildResultOutputValue;
};
+/** Non-secret execution facts at the end of this attempt. Pending means the remote outcome was not observed. */
+export type BuildResultOperation = {
+ readonly operation: string;
+ readonly need?: { readonly id: string; readonly capability: TypeRef };
+ readonly createdAt?: number;
+ readonly acknowledgedAt?: number;
+ readonly endedAt?: number;
+ readonly progress?: { readonly phase: string; readonly completed?: number; readonly total?: number; readonly unit?: string };
+ readonly cancellation?: { readonly outcome: "confirmed" | "accepted" | "unsupported" | "too-late" | "failed"; readonly message?: string };
+ readonly command: string;
+ readonly endpoint: string;
+ readonly pool?: string;
+ readonly credentials?: Readonly<Record<string, { readonly store: string; readonly key: string }>>;
+ readonly receipt?: { readonly id: string; readonly url?: string };
+ readonly status: "pending" | "completed" | "failed" | "cancelled";
+ readonly failure?: { readonly code: string; readonly message: string };
+};
+
export type BuildResultManifest = {
readonly format: "hypit.build-result@2";
/** Injected from the Repository address; `result.json` does not repeat its containing Build id. */
@@ -171,6 +189,7 @@ export type BuildResultManifest = {
readonly outcome?: BuildResultOutcome;
readonly failure?: string;
readonly outputs: Readonly<Record<string, BuildResultOutput>>;
+ readonly operations?: readonly BuildResultOperation[];
};
export type FinishedBuildResultManifest = BuildResultManifest & {
@@ -250,6 +269,7 @@ export type BuildResultSync = {
};
export type BuildResultFinish = {
+ readonly operations?: readonly BuildResultOperation[];
readonly outcome: BuildResultOutcome;
readonly failure?: string;
};
diff --git a/packages/cli/src/commands/execution.ts b/packages/cli/src/commands/execution.ts
index dee7f825..787cef17 100644
--- a/packages/cli/src/commands/execution.ts
+++ b/packages/cli/src/commands/execution.ts
@@ -111,7 +111,7 @@ export async function runExecutionCommand(input: {
worker: worker.state,
builds,
...(activity.builds.length <= args.limit ? {} : { omittedBuilds: activity.builds.length - args.limit }),
- activeRequests: activity.capacity.length,
+ capacity: activity.capacity,
};
const buildLines = activity.builds.slice(0, args.limit).map((item) => {
const requestProgress = item.requests === undefined || item.requests.total === 0
@@ -230,10 +230,10 @@ export async function runExecutionCommand(input: {
? [] : [["Outputs", String(build.result.outputCount)] as const]),
] : [["State", build?.work.state ?? "unknown"] as const]),
], (build?.operations ?? []).map((operation) => operation.failure !== undefined
- ? `${operation.endpoint}: ${operation.failure.code} — ${operation.failure.message}`
+ ? `${operation.endpoint}${operation.receipt === undefined ? "" : ` · task ${operation.receipt.id}`}: ${operation.failure.code} — ${operation.failure.message}`
: operation.progress === undefined
- ? `${operation.endpoint}: ${operation.state}`
- : `${operation.endpoint}: ${formatOperationProgress(operation.progress)}`)
+ ? `${operation.endpoint}${operation.receipt === undefined ? "" : ` · task ${operation.receipt.id}`}: ${operation.state}`
+ : `${operation.endpoint}${operation.receipt === undefined ? "" : ` · task ${operation.receipt.id}`}: ${formatOperationProgress(operation.progress)}`)
.concat(attention === undefined ? [] : [
`Attention ${attention.message}`,
...(attention.action === undefined ? [] : [`Action ${attention.action}`]),
diff --git a/packages/cli/src/commands/results.ts b/packages/cli/src/commands/results.ts
index 25d584e3..7c3acc34 100644
--- a/packages/cli/src/commands/results.ts
+++ b/packages/cli/src/commands/results.ts
@@ -121,6 +121,10 @@ export async function runProjectResultCommand(input: {
] : []),
], [
...(build.failure === undefined ? [] : [`Reason ${build.failure}`]),
+ ...(build.operations ?? []).map((operation) =>
+ `${operation.endpoint}: ${operation.status}${operation.receipt === undefined ? "" : ` · task ${operation.receipt.id}`}`
+ + (operation.failure === undefined ? "" : ` · ${operation.failure.code}: ${operation.failure.message}`)),
+ ...(build.omittedOperations === undefined ? [] : [`${build.omittedOperations} more execution receipts · use --limit <count>`]),
...(build.note === undefined ? [] : [`Note ${build.note}`]),
...visibleOutputs.map((item) => `${item.highlighted ? "★" : item.target ? "Target" : "Output"} ${item.name}`
+ (!args.presentation.verbose ? "" : ` · ${item.type} · ${item.kind}`
diff --git a/packages/cli/src/machine-view.ts b/packages/cli/src/machine-view.ts
index 07df77a4..e13da247 100644
--- a/packages/cli/src/machine-view.ts
+++ b/packages/cli/src/machine-view.ts
@@ -21,7 +21,7 @@ export type OperationalMachineView =
| { readonly format: "hypit.cli-result-edit@3"; readonly build: string; readonly title: string | null; readonly note: string | null; readonly highlightedOutputCount: number; readonly highlightedOutputs: readonly string[]; readonly omittedHighlightedOutputs?: number }
| { readonly format: "hypit.cli-get@4"; readonly build: string; readonly output: string; readonly type: string; readonly kind: "scalar" | "resource" | "composite"; readonly path: string }
| { readonly format: "hypit.cli-status@3"; readonly build: CliBuildStatusView | null }
- | { readonly format: "hypit.cli-activity@2"; readonly at: number; readonly worker: string; readonly builds: readonly { readonly id: string; readonly work: CliBuildStatusView["work"]; readonly outcome?: string; readonly attention?: AttentionView }[]; readonly omittedBuilds?: number; readonly activeRequests: number }
+ | { readonly format: "hypit.cli-activity@2"; readonly at: number; readonly worker: string; readonly builds: readonly { readonly id: string; readonly work: CliBuildStatusView["work"]; readonly outcome?: string; readonly attention?: AttentionView }[]; readonly omittedBuilds?: number; readonly capacity: readonly import("@hypit/runtime").CapacityReservation[] }
| { readonly format: "hypit.cli-result-discard@2"; readonly build: string; readonly discarded: boolean }
| { readonly format: "hypit.cli-result-finish@2"; readonly build: string; readonly found?: false; readonly outcome?: string; readonly attention?: AttentionView }
| { readonly format: "hypit.cli-cancel@3"; readonly requested: boolean; readonly build: CliBuildStatusView | null }
diff --git a/packages/cli/src/output.ts b/packages/cli/src/output.ts
index 19b1f4c2..a891c6ce 100644
--- a/packages/cli/src/output.ts
+++ b/packages/cli/src/output.ts
@@ -698,7 +698,7 @@ function commandHelp(topic: string, colors: Palette): readonly string[] | undefi
],
cancel: [
colors.accent(colors.strong("hypit cancel")),
- colors.dim("Withdraw one Build and honestly reconcile work already submitted to Providers."),
+ colors.dim("Stop one Build and request cancellation of submitted work when supported."),
"",
" hypit cancel <build-id> [--runtime <profile>] [--reason <text>]",
],
diff --git a/packages/cli/src/view.ts b/packages/cli/src/view.ts
index d5013268..41a71d95 100644
--- a/packages/cli/src/view.ts
+++ b/packages/cli/src/view.ts
@@ -48,6 +48,8 @@ export type CliBuildResultView = {
readonly outputs: readonly CliOutputView[];
readonly omittedOutputs?: number;
readonly failure?: string;
+ readonly operations?: BuildResultManifest["operations"];
+ readonly omittedOperations?: number;
};
export type CliBuildStatusView = {
@@ -66,6 +68,8 @@ export type CliBuildStatusView = {
};
readonly attention?: { readonly message: string; readonly action?: string };
readonly operations?: readonly {
+ readonly id?: string;
+ readonly receipt?: { readonly id: string; readonly url?: string };
readonly endpoint: string;
readonly state: string;
readonly progress?: {
@@ -172,11 +176,13 @@ export function buildStatusView(options: {
}
: { message: options.resultReadError! },
}),
- ...(!options.verbose || options.runtime === undefined ? {} : {
- operations: options.runtime.operations
+ ...(!options.verbose ? {} : {
+ operations: (options.runtime?.operations ?? options.result?.operations ?? [])
.filter((item) => item.status !== "completed")
.slice(0, options.operationLimit ?? 20)
.map((item) => ({
+ ...("operation" in item ? { id: item.operation } : item.id === undefined ? {} : { id: item.id }),
+ ...(item.receipt === undefined ? {} : { receipt: item.receipt }),
endpoint: item.endpoint,
state: item.status,
...(item.progress === undefined ? {} : { progress: item.progress }),
@@ -216,5 +222,7 @@ export async function buildResultView(
outputs,
...(selected.length === allNames.length ? {} : { omittedOutputs: allNames.length - selected.length }),
...(manifest.failure === undefined ? {} : { failure: manifest.failure }),
+ ...(manifest.operations === undefined ? {} : { operations: manifest.operations.slice(0, options.limit) }),
+ ...((manifest.operations?.length ?? 0) <= options.limit ? {} : { omittedOperations: manifest.operations!.length - options.limit }),
};
}
diff --git a/packages/cli/test/build-results.test.ts b/packages/cli/test/build-results.test.ts
index ec666374..140606e3 100644
--- a/packages/cli/test/build-results.test.ts
+++ b/packages/cli/test/build-results.test.ts
@@ -496,3 +496,30 @@ test("build-record selects one exact Result Output without leaking its storage a
await rm(root, { recursive: true, force: true });
}
});
+
+test("a failed Result exposes task receipts and credential references without a Runtime", async () => {
+ const root = await mkdtemp(join(tmpdir(), "hypit-receipts-"));
+ const id = "bld_20260906T110000000Z_0000000001";
+ try {
+ const result = await FileBuildResult.create(join(root, ".hypit", "results"), {
+ id, source: { path: "main.svml" }, targets: [], publishedOutputs: [],
+ });
+ const operation = {
+ operation: "op-1", command: "need-1", endpoint: "selected-account", pool: "my-pool",
+ need: { id: "generation-1", capability: valueType },
+ credentials: { apiKey: { store: "os", key: "personal-api-key" } },
+ receipt: { id: "remote-task-1" }, status: "failed" as const,
+ failure: { code: "DOWNLOAD_FAILED", message: "network timed out" },
+ createdAt: 1, acknowledgedAt: 2,
+ };
+ await result.finish({ outcome: "failed", failure: "download failed", operations: [operation] });
+ const inspected = await jsonCommand(["inspect", id], root) as {
+ build: { outcome: string; operations: unknown[] };
+ };
+ assert.equal(inspected.build.outcome, "failed");
+ assert.deepEqual(inspected.build.operations, [operation]);
+ const human = await humanCommand(["inspect", id], root);
+ assert.match(human, /remote-task-1/);
+ assert.match(human, /DOWNLOAD_FAILED/);
+ } finally { await rm(root, { recursive: true, force: true }); }
+});
diff --git a/packages/core/test/greeting-fixture.ts b/packages/core/test/greeting-fixture.ts
index 9b96a6b7..a6e2ed5d 100644
--- a/packages/core/test/greeting-fixture.ts
+++ b/packages/core/test/greeting-fixture.ts
@@ -188,3 +188,65 @@ export function createGreetingBuild(options?: {
});
return start(program, graph, request);
}
+
+export function createParallelGreetingBuild(generationCount = 2) {
+ const generations = ["a", "b", "c"].slice(0, generationCount);
+ const closure = createResolvedClosure([manifest]);
+ const authored = sealRecord({
+ id: "intent:root",
+ type: types.intent,
+ value: { kind: "inline", value: { name: "Ada" } },
+ });
+ const program = link(closure, [authored]);
+ const graph = sealCompiledGraph({
+ outputs: [
+ {
+ id: "prompt",
+ type: types.prompt,
+ primary: "make-prompt",
+ },
+ ...generations.map((suffix) => ({
+ id: `generated-${suffix}`,
+ type: types.generated,
+ primary: `generate-${suffix}`,
+ })),
+ ],
+ candidates: [
+ {
+ id: "make-prompt",
+ type: types.prompt,
+ root: { kind: "operation", result: { kind: "operation-result", operation: "make-prompt" } },
+ },
+ ...generations.map((suffix) => ({
+ id: `generate-${suffix}`,
+ type: types.generated,
+ root: {
+ kind: "operation" as const,
+ result: { kind: "operation-result" as const, operation: `generate-${suffix}` },
+ },
+ })),
+ ],
+ operations: [
+ {
+ id: "make-prompt",
+ producer: producers.makePrompt,
+ inputs: { intent: { kind: "record", id: "intent:root" } },
+ result: { kind: "output", name: "prompt", record: "prompt:root" },
+ },
+ ...generations.map((suffix) => ({
+ id: `generate-${suffix}`,
+ producer: producers.requestText,
+ inputs: { prompt: { kind: "logical-output" as const, id: "prompt" } },
+ result: {
+ kind: "need" as const,
+ name: "generation",
+ id: `need:generation-${suffix}`,
+ record: `generated:${suffix}`,
+ },
+ })),
+ ],
+ });
+ return start(program, graph, sealBuildRequest({
+ targets: generations.map((suffix) => ({ output: `generated-${suffix}` })),
+ }));
+}
diff --git a/packages/driver-node/README.md b/packages/driver-node/README.md
index 9e41865e..e9826e96 100644
--- a/packages/driver-node/README.md
+++ b/packages/driver-node/README.md
@@ -11,8 +11,13 @@ and fulfilled by an exact endpoint. When several endpoints offer the same capabi
configuration must bind one explicitly.
Immediate endpoints run once. Asynchronous endpoints start a task, store its handle in an
-`OperationStore`, and poll it until completion. Credentials are resolved only for slots declared by
-the selected endpoint. Cancellation is best effort and never rolls back completed work.
+`OperationStore`, and poll it until completion or failure. An optional `collect` action obtains already-generated
+artifacts after remote completion. Polling uses the stored Need and handle without loading its whole
+Build. Credentials are resolved only for slots declared by the selected endpoint; simultaneous reads
+of the same credential are coalesced. Cancellation is best effort and never rolls back completed work.
+A failed action retains its receipt and error for the Result; the Driver does not retry or reconcile it.
+`acceptOperation()` validates an already received value without making a remote call. Runtime can
+retain such a sibling result when another Need fails, without polling unfinished jobs to completion.
The Driver validates returned values before offering a command result to Core. It does not load
packages, parse source files, own the active Build work set or know any video vocabulary.
diff --git a/packages/driver-node/src/driver.ts b/packages/driver-node/src/driver.ts
index a47192fa..473371c0 100644
--- a/packages/driver-node/src/driver.ts
+++ b/packages/driver-node/src/driver.ts
@@ -23,7 +23,6 @@ import type {
CredentialValue,
OperationSnapshot,
OperationStore,
- OperationUpdate,
RuntimeExecutionContext,
RuntimeExecutionResult,
RuntimePreparation,
@@ -54,6 +53,7 @@ export type NodeDriverOptions = {
readonly operations?: OperationStore;
readonly credentials?: CredentialStore;
readonly validators?: TypeValidatorRegistryLike;
+ readonly actions?: import("@hypit/runtime").RuntimeActionExecutor;
};
type Executable =
@@ -95,6 +95,8 @@ export class NodeDriver {
readonly operations: OperationStore | undefined;
readonly credentials: CredentialStore | undefined;
readonly validators: TypeValidatorRegistryLike;
+ readonly #actions: import("@hypit/runtime").RuntimeActionExecutor | undefined;
+ readonly #credentialReads = new Map<string, Promise<CredentialValue | undefined>>();
constructor(options: NodeDriverOptions = {}) {
this.producers = options.producers ?? new ProducerRegistry();
@@ -104,6 +106,7 @@ export class NodeDriver {
this.operations = options.operations;
this.credentials = options.credentials;
this.validators = options.validators ?? new TypeValidatorRegistry();
+ this.#actions = options.actions;
}
#resourceStore(build?: string): ResourceStore {
@@ -121,7 +124,13 @@ export class NodeDriver {
const resolved: Record<string, CredentialValue> = {};
for (const slot of Object.keys(requested).sort()) {
const ref = requested[slot]!;
- const value = await this.credentials.resolve(ref);
+ const key = JSON.stringify([ref.store, ref.key]);
+ let read = this.#credentialReads.get(key);
+ if (read === undefined) {
+ read = this.credentials.resolve(ref).finally(() => this.#credentialReads.delete(key));
+ this.#credentialReads.set(key, read);
+ }
+ const value = await read;
if (value === undefined) {
throw new Error(`Endpoint ${registration.id} credential ${slot} is unavailable from ${ref.store}:${ref.key}`);
}
@@ -237,52 +246,40 @@ export class NodeDriver {
} as const;
}
- async #completedOperation(
+ async acceptOperation(
state: BuildState,
- executable: Extract<Executable, { readonly endpointId: string }>,
snapshot: OperationSnapshot,
- expectedOperation: string,
- ): Promise<RuntimeExecutionResult> {
- if (snapshot.id !== expectedOperation) {
- throw new Error(`OperationStore returned ${snapshot.id} for ${expectedOperation}`);
- }
- if (snapshot.status === "completed" && snapshot.completion !== undefined) {
- return {
- status: "completed",
- event: await this.#endpointEvent(state, executable, snapshot.completion),
- };
+ ): Promise<CommandResult | undefined> {
+ if ((snapshot.status === "completed" || snapshot.status === "pending") && snapshot.completion !== undefined) {
+ const command = state.outstanding.find((item) => item.id === snapshot.command);
+ if (command?.kind !== "fulfill-need") return undefined;
+ await validateValue(state.program.closure, command.need.returns, snapshot.completion.value, this.validators);
+ if (snapshot.status === "pending") {
+ await this.operations!.update(snapshot.id, { status: "completed", completion: snapshot.completion });
+ }
+ return { kind: "need-fulfilled", command: snapshot.command, value: snapshot.completion.value };
}
if (snapshot.status === "failed") {
const failure = snapshot.failure;
if (failure === undefined) throw new Error(`Operation ${snapshot.id} has no failure`);
const content = {
kind: "command-failed",
- command: executable.command.id,
+ command: snapshot.command,
code: failure.code,
message: failure.message,
} as const;
- return {
- status: "completed",
- event: content,
- };
+ return content;
}
if (snapshot.status === "cancelled") {
const content = {
kind: "command-failed",
- command: executable.command.id,
+ command: snapshot.command,
code: "CANCELLED",
message: `Operation ${snapshot.id} was cancelled by the Runtime capacity controller`,
} as const;
- return {
- status: "completed",
- event: content,
- };
+ return content;
}
- return {
- status: "pending",
- operation: snapshot.id,
- ...(snapshot.wakeAt === undefined ? {} : { wakeAt: snapshot.wakeAt }),
- };
+ return undefined;
}
async #executeEndpoint(
@@ -293,138 +290,123 @@ export class NodeDriver {
if (executable.registration.kind !== "asynchronous") throw new Error("Endpoint is not asynchronous");
const operations = this.operations;
if (operations === undefined) throw new Error("asynchronous Endpoint requires OperationStore");
- const base = {
- build: context.build,
- command: executable.command.id,
- endpoint: executable.endpointId,
- } as const;
- const history = await operations.list({
- build: base.build,
- command: base.command,
- endpoint: base.endpoint,
- });
- if (history.length > 1) {
- throw new Error(`Command ${base.command} has multiple asynchronous Operations; refusing to submit or choose between them`);
+ const history = await operations.list({ build: context.build, command: executable.command.id });
+ if (history.length > 1) throw new Error(`Command ${executable.command.id} has multiple Operations`);
+ let operation = history[0];
+ if (operation !== undefined && operation.endpoint !== executable.endpointId) {
+ throw new Error(`Operation ${operation.id} belongs to ${operation.endpoint}; its execution source cannot change`);
}
- const latest = history[0];
- if (latest?.status === "completed" || latest?.status === "cancelled") {
- return await this.#completedOperation(state, executable, latest, latest.id);
- }
- if (latest?.status === "pending" && latest.cancellationRequested) {
- const settled = await this.cancelOperation(state, latest);
- return await this.#completedOperation(state, executable, settled, latest.id);
- }
- if (latest?.status === "pending" && latest.wakeAt !== undefined && latest.wakeAt > Date.now()) {
- return { status: "pending", operation: latest.id, wakeAt: latest.wakeAt };
- }
- if (latest?.status === "failed") {
- return await this.#completedOperation(state, executable, latest, latest.id);
+ operation ??= await operations.create({
+ id: `op_${randomUUID()}`, build: context.build, command: executable.command.id,
+ endpoint: executable.endpointId, status: "pending", submission: "queued", createdAt: Date.now(),
+ request: structuredClone(executable.command),
+ credentials: structuredClone(executable.registration.credentials ?? {}),
+ ...(executable.registration.pool === undefined ? {} : { pool: executable.registration.pool }),
+ wakeAt: Date.now(), progress: { phase: "ready-to-submit" },
+ });
+ if (operation.status === "pending" && operation.completion === undefined) {
+ operation = await this.#advanceOperation(operation, executable.registration, context);
}
- if (latest?.status === "pending" && latest.handle === undefined) {
- const unknown = await operations.update(latest.id, {
- status: "pending",
- wakeAt: Date.now() + 30_000,
- progress: { phase: "submission-unknown" },
- failure: {
- code: "SUBMISSION_UNKNOWN",
- message: `Operation ${latest.id} stopped before its Endpoint acknowledgement was stored; the same Build will not submit it again`,
- },
+ if (operation.remoteEnded) await context.releaseOperationCapacity?.();
+ const event = await this.acceptOperation(state, operation);
+ return event === undefined ? {
+ status: "pending", operation: operation.id,
+ ...(operation.wakeAt === undefined ? {} : { wakeAt: operation.wakeAt }),
+ } : { status: "completed", event };
+ }
+
+ async #advanceOperation(
+ operation: OperationSnapshot,
+ registration: EndpointRegistration,
+ runtimeContext: RuntimeExecutionContext,
+ ): Promise<OperationSnapshot> {
+ const operations = this.operations!;
+ if (registration.kind !== "asynchronous") throw new Error("Operation Endpoint is not asynchronous");
+ if (operation.status !== "pending" || operation.completion !== undefined) return operation;
+ if ((operation.wakeAt ?? 0) > Date.now()) return operation;
+ const command = operation.request;
+ if (command === undefined) throw new Error(`Operation ${operation.id} has no stored request`);
+ const endpoint = registration.endpoint;
+ const action: import("@hypit/endpoint-kit").EndpointAction = operation.remoteEnded ? "collect"
+ : operation.handle !== undefined ? "poll" : "submit";
+ if (action === "submit" && operation.submission !== "queued") {
+ return await operations.update(operation.id, {
+ status: "failed", failure: { code: "SUBMISSION_INTERRUPTED",
+ message: "Submission ended without a task receipt; this execution attempt has failed" },
});
- return await this.#completedOperation(state, executable, unknown, latest.id);
}
- const fresh = latest === undefined;
- const identity = fresh
- ? { id: `op_${randomUUID()}`, ...base }
- : latest;
- const endpointContext = {
- command: structuredClone(executable.command),
- need: structuredClone(executable.command.need),
- resources: this.#resourceStore(context.build),
- credentials: await this.#endpointCredentials(executable.registration),
- operation: identity.id,
- };
- let outcome: EndpointOutcome;
- if (latest === undefined) {
- await operations.create({
- ...identity,
- status: "pending",
- progress: { phase: "submitting" },
- });
- try {
- outcome = await executable.registration.endpoint.start(endpointContext);
- } catch (error) {
- const unknown = await operations.update(identity.id, {
- status: "pending",
- wakeAt: Date.now() + 30_000,
- progress: { phase: "submission-unknown" },
- failure: {
- code: "SUBMISSION_UNKNOWN",
- message: `Endpoint start ended without a stored acknowledgement: ${failureMessage(error)}`,
- },
- });
- return await this.#completedOperation(state, executable, unknown, identity.id);
+ const invoke = async (): Promise<EndpointOutcome> => {
+ const context = {
+ command: structuredClone(command), need: structuredClone(command.need),
+ resources: this.#resourceStore(operation.build), operation: operation.id,
+ credentials: await this.#endpointCredentials({ ...registration, credentials: operation.credentials ?? registration.credentials ?? {} }),
+ checkpoint: async (checkpoint: import("@hypit/endpoint-kit").EndpointCheckpoint) => {
+ await operations.update(operation.id, { status: "pending", ...checkpoint,
+ submission: "accepted", acknowledgedAt: operation.acknowledgedAt ?? Date.now(), wakeAt: Date.now(), progress: { phase: checkpoint.remoteEnded ? "collecting" : "submitted" } });
+ if (checkpoint.remoteEnded) await runtimeContext.releaseOperationCapacity?.();
+ },
+ };
+ if (action === "submit") {
+ await operations.update(operation.id, { status: "pending", submission: "started", progress: { phase: "submitting" } });
+ return await endpoint.start(context);
}
- } else {
- if (latest.status !== "pending" || latest.handle === undefined) {
- throw new Error(`Operation ${latest.id} cannot be polled`);
+ const pollContext = { ...context, handle: structuredClone(operation.handle!) };
+ if (action === "collect") {
+ if (endpoint.collect === undefined) throw new Error(`Endpoint ${registration.id} declared artifacts ready but has no collect action`);
+ return await endpoint.collect(pollContext);
}
- try {
- outcome = await executable.registration.endpoint.poll({
- ...endpointContext,
- handle: structuredClone(latest.handle),
- ...(latest.failure === undefined ? {} : { settling: true as const }),
+ return await endpoint.poll(pollContext);
+ };
+ let outcome: EndpointOutcome;
+ try {
+ const resources = registration.scheduling?.actions?.[action] ?? [];
+ if (resources.length > 0) {
+ if (this.#actions === undefined) throw new Error(`Endpoint ${registration.id} requires Runtime action admission`);
+ const result = await this.#actions.run({ build: operation.build, command: operation.command, action, resources }, invoke);
+ if (result.status === "deferred") return await operations.update(operation.id, {
+ status: "pending", ...(result.wakeAt === undefined ? {} : { wakeAt: result.wakeAt }),
+ progress: { phase: "waiting-resource" },
});
- } catch {
- // A transport error says nothing about whether the submitted work has ended.
- outcome = { status: "pending", handle: latest.handle, wakeAt: Date.now() + 1_000,
- progress: { phase: "poll-unavailable" } };
- }
+ outcome = result.value;
+ } else outcome = await invoke();
+ } catch (error) {
+ return await operations.update(operation.id, {
+ status: "failed", failure: { code: "ENDPOINT_ACTION_FAILED",
+ message: `${action} failed: ${failureMessage(error)}` },
+ });
}
- const write = async (update: OperationUpdate): Promise<OperationSnapshot> =>
- await operations.update(identity.id, update);
+ const recorded = await operations.read(operation.id) ?? operation;
+ const receipt = outcome.receipt ?? recorded.receipt;
+ const facts = receipt === undefined ? {} : {
+ receipt, acknowledgedAt: recorded.acknowledgedAt ?? Date.now(),
+ };
if (outcome.status === "pending") {
- const handle = outcome.handle === undefined ? latest?.handle : outcome.handle;
- const failure = latest?.failure ?? outcome.failure;
- const written = await write({
- status: "pending",
- ...(handle === undefined ? {} : { handle }),
- ...(outcome.wakeAt === undefined ? {} : { wakeAt: outcome.wakeAt }),
+ return await operations.update(operation.id, {
+ status: "pending", ...facts, submission: "accepted", handle: outcome.handle,
+ wakeAt: outcome.wakeAt ?? Date.now(),
...(outcome.progress === undefined ? {} : { progress: outcome.progress }),
- ...(failure === undefined ? {} : { failure }),
- });
- return await this.#completedOperation(
- state,
- executable,
- written,
- identity.id,
- );
- }
- if (latest?.failure !== undefined || outcome.status === "settled") {
- const written = await write({ status: "failed", failure: latest?.failure ?? {
- code: "ENDPOINT_SETTLED_WITHOUT_RESULT", message: "Endpoint ended without a fulfillment result",
- } });
- return await this.#completedOperation(state, executable, written, identity.id);
- }
- if (outcome.status === "failed") {
- const written = await write({
- status: "failed",
- failure: outcome.failure,
});
- return await this.#completedOperation(
- state,
- executable,
- written,
- identity.id,
- );
}
- // Validate before the terminal completion becomes durable. Result writing may later
- // accept this stored value without loading or calling the Endpoint package again.
- const event = await this.#endpointEvent(state, executable, outcome.result);
- await write({
- status: "completed",
- completion: outcome.result,
+ if (outcome.status === "ready") return await operations.update(operation.id, {
+ status: "pending", ...facts, remoteEnded: true, endedAt: Date.now(), handle: outcome.handle,
+ wakeAt: Date.now(), progress: { phase: "ready-to-collect" },
+ });
+ if (outcome.status === "failed") return await operations.update(operation.id, {
+ status: "failed", ...facts, failure: outcome.failure,
});
- return { status: "completed", event };
+ return await operations.update(operation.id, {
+ status: "pending", ...facts, remoteEnded: true, endedAt: recorded.endedAt ?? Date.now(),
+ completion: outcome.result, wakeAt: Date.now(), progress: { phase: "ready-to-accept" },
+ });
+ }
+
+ async advanceOperation(operation: OperationSnapshot): Promise<OperationSnapshot> {
+ if (operation.request === undefined) throw new Error(`Operation ${operation.id} has no stored request`);
+ const selected = this.endpoints.resolve(operation.request.need);
+ if (selected.status !== "resolved" || selected.registration.id !== operation.endpoint) {
+ throw new Error(`Operation ${operation.id} cannot change its selected Endpoint ${operation.endpoint}`);
+ }
+ return await this.#advanceOperation(operation, selected.registration, { build: operation.build });
}
async #execute(
@@ -524,49 +506,31 @@ export class NodeDriver {
return await this.#execute(state, classified.executable, context);
}
- /** Cancel once, then observe the same remote work until termination is confirmed. */
+ /** Stop observing this Operation and make at most one explicit cancellation request. */
async cancelOperation(initial: BuildState, operation: OperationSnapshot): Promise<OperationSnapshot> {
const operations = this.operations;
if (operations === undefined) throw new Error("cancelling an Operation requires OperationStore");
const current = await operations.read(operation.id) ?? operation;
if (current.status !== "pending") return current;
- if (current.cancellationRequested && (current.wakeAt ?? 0) > Date.now()) return current;
- const pending = (phase: string, wakeAt = Date.now() + 1_000, handle = current.handle) => operations.update(current.id, {
- status: "pending", ...(handle === undefined ? {} : { handle }),
- wakeAt, progress: { phase }, cancellationRequested: true,
- ...(current.failure === undefined ? {} : { failure: current.failure }),
- });
- if (current.handle === undefined) return await pending("submission-unknown", Date.now() + 30_000);
- const command = resolveNeedCommand(initial, current.command);
- if (command === undefined) throw new Error(`Operation ${current.id} has no matching Need Command`);
- const { need } = command;
- const resolution = this.endpoints.resolve(need);
- if (resolution.status !== "resolved" || resolution.registration.id !== current.endpoint
- || resolution.registration.kind !== "asynchronous") {
- throw new Error(`Operation ${current.id} does not match its bound Endpoint`);
- }
- const endpoint = resolution.registration.endpoint;
- try {
- const context = {
- command: structuredClone(command), need: structuredClone(need),
- resources: this.#resourceStore(current.build),
- credentials: await this.#endpointCredentials(resolution.registration),
- operation: current.id, handle: structuredClone(current.handle), settling: true as const,
- };
- if (!current.cancellationRequested) {
- await pending("cancelling");
- const cancelled = await endpoint.cancel?.(context);
- if (cancelled?.status === "confirmed") return await operations.update(current.id, { status: "cancelled" });
- if (cancelled?.status === "accepted" && cancelled.wakeAt !== undefined && cancelled.wakeAt > Date.now()) {
- return await pending("cancelling", cancelled.wakeAt);
- }
+ let cancellation: NonNullable<OperationSnapshot["cancellation"]> = { outcome: "unsupported" };
+ if (current.handle !== undefined && !current.remoteEnded) {
+ try {
+ const command = current.request ?? resolveNeedCommand(initial, current.command);
+ if (command === undefined) throw new Error(`Operation ${current.id} has no Need Command`);
+ const selected = this.endpoints.resolve(command.need);
+ if (selected.status !== "resolved" || selected.registration.id !== current.endpoint
+ || selected.registration.kind !== "asynchronous") throw new Error("Operation Endpoint changed");
+ const result = await selected.registration.endpoint.cancel?.({
+ command, need: command.need, operation: current.id, handle: current.handle,
+ resources: this.#resourceStore(current.build),
+ credentials: await this.#endpointCredentials(selected.registration),
+ });
+ cancellation = { outcome: result?.status ?? "unsupported" };
+ } catch (error) {
+ cancellation = { outcome: "failed", message: failureMessage(error) };
}
- const outcome = await endpoint.poll(context);
- if (outcome.status === "pending") return await pending("cancelling", outcome.wakeAt ?? Date.now() + 1_000, outcome.handle);
- return await operations.update(current.id, { status: "cancelled" });
- } catch {
- return await pending("cancellation-unconfirmed");
}
+ return await operations.update(current.id, { status: "cancelled", cancellation });
}
async run(initial: BuildState, context?: RuntimeExecutionContext): Promise<DriverRunResult> {
@@ -623,7 +587,11 @@ export class NodeDriver {
status: "error",
message: error instanceof Error ? error.message : String(error),
});
- return { status: "paused", state, outcomes, blocked: [] };
+ state = reduce(state, {
+ kind: "command-failed", command: selected.command.id, code: "EXECUTION_FAILED",
+ message: failureMessage(error),
+ });
+ return { status: "failed", state, outcomes, blocked: [] };
}
}
}
diff --git a/packages/driver-node/src/registry.ts b/packages/driver-node/src/registry.ts
index 3fffd0d5..038546c4 100644
--- a/packages/driver-node/src/registry.ts
+++ b/packages/driver-node/src/registry.ts
@@ -39,15 +39,16 @@ function sameRef(
function verifyScheduling(scheduling: EndpointScheduling | undefined): void {
if (scheduling === undefined) return;
if (scheduling.resources.length === 0) throw new Error("scheduling resources must not be empty");
- const ids = scheduling.resources.map((resource) => {
- capacityUnits(resource);
- if (resource.id.trim().length === 0) throw new Error("scheduling resource id must not be empty");
- if (!Number.isSafeInteger(resource.limit) || resource.limit < 1) {
- throw new Error(`scheduling resource ${resource.id} limit must be a positive safe integer`);
- }
- return resource.id;
- });
- if (new Set(ids).size !== ids.length) throw new Error("scheduling resources contain duplicate ids");
+ for (const resources of [scheduling.resources, ...Object.values(scheduling.actions ?? {})]) {
+ const ids = resources.map((resource) => {
+ capacityUnits(resource);
+ return resource.id;
+ });
+ if (new Set(ids).size !== ids.length) throw new Error("scheduling resources contain duplicate ids");
+ }
+ if (scheduling.resources.some((resource) => resource.periodMs !== undefined)) {
+ throw new Error("Rate budgets apply to Endpoint actions; whole-operation resources declare occupancy");
+ }
}
export function producerRegistryKey(ref: ProducerRef): string {
@@ -120,25 +121,25 @@ export class EndpointRegistry implements EndpointRegistrar {
* Shared capacity resources that different registrations size differently. Two Endpoints in one
* pool with different concurrency would otherwise only collide inside the scheduler, mid-Build.
*/
- capacityConflicts(): readonly { readonly resource: string; readonly limits: readonly number[]; readonly endpointIds: readonly string[] }[] {
- const limits = new Map<string, Map<number, Set<string>>>();
+ capacityConflicts(): readonly { readonly resource: string; readonly limits: readonly number[]; readonly settings: readonly string[]; readonly endpointIds: readonly string[] }[] {
+ const resources = new Map<string, Map<string, { limit: number; endpoints: Set<string> }>>();
for (const registration of this.#registrations) {
- for (const resource of registration.scheduling?.resources ?? []) {
- const byLimit = limits.get(resource.id) ?? new Map<number, Set<string>>();
- const ids = byLimit.get(resource.limit) ?? new Set<string>();
- ids.add(registration.id);
- byLimit.set(resource.limit, ids);
- limits.set(resource.id, byLimit);
+ const scheduling = registration.scheduling;
+ for (const resource of [...(scheduling?.resources ?? []), ...Object.values(scheduling?.actions ?? {}).flat()]) {
+ const settings = resource.periodMs === undefined ? `${resource.limit} concurrent` : `${resource.limit} per ${resource.periodMs} ms`;
+ const variants = resources.get(resource.id) ?? new Map();
+ const variant = variants.get(settings) ?? { limit: resource.limit, endpoints: new Set<string>() };
+ variant.endpoints.add(registration.id);
+ variants.set(settings, variant);
+ resources.set(resource.id, variants);
}
}
- return [...limits.entries()]
- .filter(([, byLimit]) => byLimit.size > 1)
- .map(([resource, byLimit]) => ({
- resource,
- limits: [...byLimit.keys()].sort((left, right) => left - right),
- endpointIds: [...new Set([...byLimit.values()].flatMap((ids) => [...ids]))].sort(),
- }))
- .sort((left, right) => left.resource.localeCompare(right.resource));
+ return [...resources.entries()].filter(([, variants]) => variants.size > 1).map(([resource, variants]) => ({
+ resource,
+ limits: [...new Set([...variants.values()].map((variant) => variant.limit))].sort((a, b) => a - b),
+ settings: [...variants.keys()].sort((a, b) => a.localeCompare(b, "en", { numeric: true })),
+ endpointIds: [...new Set([...variants.values()].flatMap((variant) => [...variant.endpoints]))].sort(),
+ })).sort((a, b) => a.resource.localeCompare(b.resource));
}
/** Capabilities offered by more than one Endpoint, with the ids, for a deployment to bind. */
diff --git a/packages/driver-node/test/driver.test.ts b/packages/driver-node/test/driver.test.ts
index d804accf..77db795c 100644
--- a/packages/driver-node/test/driver.test.ts
+++ b/packages/driver-node/test/driver.test.ts
@@ -230,7 +230,7 @@ test("Core still owns scheduling when Driver has every implementation", async ()
assert.equal(result.status, "complete");
});
-test("a direct Driver caller can retry one failed Handler without replaying completed producers", async () => {
+test("a direct Driver failure ends its state and a new attempt starts independently", async () => {
const { producers, endpoints, calls } = configuredRegistry();
let attempts = 0;
endpoints.registerImmediateEndpoint("example:unstable", capabilities.generation, types.generated, () => {
@@ -244,11 +244,28 @@ test("a direct Driver caller can retry one failed Handler without replaying comp
const driver = new NodeDriver({ producers, endpoints });
const paused = await driver.run(createGreetingBuild());
- assert.equal(paused.status, "paused");
+ assert.equal(paused.status, "failed");
assert.match(paused.outcomes.at(-1)?.message ?? "", /temporary outage/u);
assert.deepEqual(calls, { prompt: 1, request: 1, assemble: 0, fulfill: 1 });
- const completed = await driver.run(paused.state);
+ assert.equal((await driver.run(paused.state)).status, "failed");
+ assert.equal(calls.fulfill, 1);
+ const completed = await driver.run(createGreetingBuild());
assert.equal(completed.status, "complete");
- assert.deepEqual(calls, { prompt: 1, request: 1, assemble: 1, fulfill: 2 });
+ assert.deepEqual(calls, { prompt: 2, request: 2, assemble: 1, fulfill: 2 });
+});
+
+test("shared action-rate declarations conflict when their periods differ", () => {
+ const registry = new EndpointRegistry();
+ for (const [id, periodMs] of [["a", 100], ["b", 200]] as const) {
+ registry.registerAsyncEndpoint(id, capabilities.generation, types.generated, {
+ start: () => ({ status: "pending", handle: { id } }),
+ poll: () => ({ status: "completed", result: { value: { kind: "inline", value: "done" } } }),
+ }, { scheduling: { resources: [{ id: "pool:shared", limit: 10 }],
+ actions: { submit: [{ id: "rate:shared/submit", limit: 1, periodMs }] } } });
+ }
+ const [conflict] = registry.capacityConflicts();
+ assert.equal(conflict?.resource, "rate:shared/submit");
+ assert.deepEqual(conflict?.endpointIds, ["a", "b"]);
+ assert.deepEqual(conflict?.settings, ["1 per 100 ms", "1 per 200 ms"]);
});
diff --git a/packages/driver-node/test/scheduler.test.ts b/packages/driver-node/test/scheduler.test.ts
index 310b89a5..266e1078 100644
--- a/packages/driver-node/test/scheduler.test.ts
+++ b/packages/driver-node/test/scheduler.test.ts
@@ -11,16 +11,7 @@ import {
LocalBuildScheduler,
} from "@hypit/runtime";
import type { OperationSnapshot, OperationStore, OperationUpdate } from "@hypit/runtime";
-import {
- createResolvedClosure,
- link,
- sealBuildRequest,
- sealCompiledGraph,
- sealRecord,
- start,
-} from "@hypit/core";
-
-import { capabilities, createGreetingBuild, manifest, producers as greetingProducers, types } from "../../core/test/greeting-fixture.js";
+import { capabilities, createGreetingBuild, createParallelGreetingBuild, producers as greetingProducers, types } from "../../core/test/greeting-fixture.js";
function memoryOperations(): OperationStore {
const values = new Map<string, OperationSnapshot>();
@@ -44,79 +35,20 @@ function memoryOperations(): OperationStore {
if (current === undefined) throw new Error(`Operation ${id} does not exist`);
if (["completed", "failed", "cancelled"].includes(current.status)) return structuredClone(current);
const next = {
+ ...current,
id: current.id,
build: current.build,
command: current.command,
endpoint: current.endpoint,
...structuredClone(update),
} as OperationSnapshot;
+ if (update.status !== "pending" || update.wakeAt === undefined) delete (next as { wakeAt?: number }).wakeAt;
values.set(id, next);
return structuredClone(next);
},
};
}
-function createParallelGreetingBuild(generationCount = 2) {
- const generations = ["a", "b", "c"].slice(0, generationCount);
- const closure = createResolvedClosure([manifest]);
- const authored = sealRecord({
- id: "intent:root",
- type: types.intent,
- value: { kind: "inline", value: { name: "Ada" } },
- });
- const program = link(closure, [authored]);
- const graph = sealCompiledGraph({
- outputs: [
- {
- id: "prompt",
- type: types.prompt,
- primary: "make-prompt",
- },
- ...generations.map((suffix) => ({
- id: `generated-${suffix}`,
- type: types.generated,
- primary: `generate-${suffix}`,
- })),
- ],
- candidates: [
- {
- id: "make-prompt",
- type: types.prompt,
- root: { kind: "operation", result: { kind: "operation-result", operation: "make-prompt" } },
- },
- ...generations.map((suffix) => ({
- id: `generate-${suffix}`,
- type: types.generated,
- root: {
- kind: "operation" as const,
- result: { kind: "operation-result" as const, operation: `generate-${suffix}` },
- },
- })),
- ],
- operations: [
- {
- id: "make-prompt",
- producer: greetingProducers.makePrompt,
- inputs: { intent: { kind: "record", id: "intent:root" } },
- result: { kind: "output", name: "prompt", record: "prompt:root" },
- },
- ...generations.map((suffix) => ({
- id: `generate-${suffix}`,
- producer: greetingProducers.requestText,
- inputs: { prompt: { kind: "logical-output" as const, id: "prompt" } },
- result: {
- kind: "need" as const,
- name: "generation",
- id: `need:generation-${suffix}`,
- record: `generated:${suffix}`,
- },
- })),
- ],
- });
- return start(program, graph, sealBuildRequest({
- targets: generations.map((suffix) => ({ output: `generated-${suffix}` })),
- }));
-}
function configuredExecutor(options: {
readonly resource: string;
@@ -246,7 +178,7 @@ test("independent paid commands inside one Build may fill the same resource with
record.type.name === types.generated.name).length, 2);
});
-test("a terminal failure keeps its diagnostic while an in-flight sibling settles", async () => {
+test("a failure stops new work and preserves a result from an already running sibling", async () => {
const operations = memoryOperations();
const producers = new ProducerRegistry();
const endpoints = new EndpointRegistry();
@@ -281,7 +213,7 @@ test("a terminal failure keeps its diagnostic while an in-flight sibling settles
const [result] = await new LocalBuildScheduler(new NodeDriver({ producers, endpoints, operations })).run([{
id: "parallel-failure",
- state: createParallelGreetingBuild(),
+ state: createParallelGreetingBuild(3),
}]);
assert.equal(calls, 2);
@@ -289,6 +221,8 @@ test("a terminal failure keeps its diagnostic while an in-flight sibling settles
assert.equal(result?.state.diagnostics.at(-1)?.code, "PROVIDER_REJECTED");
assert.match(result?.state.diagnostics.at(-1)?.message ?? "", /first provider request was rejected/);
assert.equal(result?.outcomes.some((outcome) => outcome.status === "error"), false);
+ assert.deepEqual(result?.state.records.find((record) => record.type.name === types.generated.name)?.value,
+ { kind: "inline", value: "late sibling result" });
});
test("an asynchronous Endpoint starts once and is polled until complete", async () => {
@@ -336,45 +270,27 @@ test("an asynchronous Endpoint starts once and is polled until complete", async
assert.equal((await operations.read(pending.operation))?.status, "completed");
});
-test("a persisted submitting Operation is never submitted again after its caller stops", async () => {
- const durable = memoryOperations();
- let interruptCreate = true;
- const operations: OperationStore = {
- ...durable,
- async create(operation) {
- const stored = await durable.create(operation);
- if (interruptCreate) {
- interruptCreate = false;
- throw new Error("caller stopped after the Operation row was stored");
- }
- return stored;
- },
- };
+test("a submission error ends its attempt and a new Build can submit normally", async () => {
+ const operations = memoryOperations();
let starts = 0;
const endpoint: AsyncEndpoint = {
start() {
- starts += 1;
- return { status: "pending", handle: { remoteJob: "must-not-exist" } };
- },
- poll() {
- throw new Error("an unacknowledged Operation cannot be polled");
+ starts++;
+ if (starts === 1) throw new Error("submission timed out without a receipt");
+ return { status: "pending", handle: { job: "second" }, receipt: { id: "second" } };
},
+ poll() { return { status: "completed", result: { value: { kind: "inline", value: "done" } } }; },
};
- const [interrupted] = await new LocalBuildScheduler(asyncExecutor(endpoint, operations))
- .run([{ id: "submission-window", state: createGreetingBuild() }]);
- assert.equal(interrupted?.status, "paused");
- assert.equal(starts, 0);
- const [stored] = await operations.list({ build: "submission-window" });
- assert.equal(stored?.status, "pending");
- assert.equal(stored?.handle, undefined);
-
- const [resumed] = await new LocalBuildScheduler(asyncExecutor(endpoint, operations))
- .run([{ id: "submission-window", state: interrupted!.state }]);
- assert.equal(resumed?.status, "paused");
- const [unknown] = await operations.list({ build: "submission-window" });
- assert.equal(unknown?.status, "pending");
- assert.equal(unknown?.failure?.code, "SUBMISSION_UNKNOWN");
- assert.equal(starts, 0);
+ const scheduler = new LocalBuildScheduler(asyncExecutor(endpoint, operations));
+ const [failed] = await scheduler.run([{ id: "first", state: createGreetingBuild() }]);
+ assert.equal(failed?.status, "failed");
+ assert.equal((await operations.list({ build: "first" }))[0]?.status, "failed");
+ await scheduler.run([{ id: "first", state: failed!.state }]);
+ assert.equal(starts, 1);
+ const [next] = await scheduler.run([{ id: "second", state: createGreetingBuild() }]);
+ const [completed] = await scheduler.run([{ id: "second", state: next!.state }]);
+ assert.equal(completed?.status, "complete");
+ assert.equal(starts, 2);
});
test("wakeAt prevents early polling and Runtime cancellation becomes a terminal Core failure", async () => {
@@ -432,25 +348,22 @@ test("request quantities govern concurrent admission, independent of whole-Need
assert.equal(maximum, 4, "two four-worker requests cannot fit in six slots");
});
-test("a poll transport error keeps the same Operation until its remote work finishes", async () => {
+test("a poll transport error fails once and retains the acknowledged receipt", async () => {
const operations = memoryOperations();
- let starts = 0, unavailable = true;
+ let starts = 0, polls = 0;
const endpoint: AsyncEndpoint = {
- start() { starts++; return { status: "pending", handle: { job: "one" } }; },
- poll() {
- if (unavailable) throw new Error("connection lost");
- return { status: "completed", result: { value: { kind: "inline", value: "done" } } };
- },
+ start() { starts++; return { status: "pending", handle: { job: "one" }, receipt: { id: "one" } }; },
+ poll() { polls++; throw new Error("connection lost"); },
};
const scheduler = new LocalBuildScheduler(asyncExecutor(endpoint, operations));
const [first] = await scheduler.run([{ id: "poll-error", state: createGreetingBuild() }]);
const [second] = await scheduler.run([{ id: "poll-error", state: first!.state }]);
const [operation] = await operations.list({ build: "poll-error" });
- assert.equal(operation?.status, "pending");
- assert.equal(operation?.progress?.phase, "poll-unavailable");
- unavailable = false;
- await operations.update(operation!.id, { status: "pending", handle: operation!.handle! });
- const [third] = await scheduler.run([{ id: "poll-error", state: second!.state }]);
- assert.equal(third?.status, "complete");
+ assert.equal(second?.status, "failed");
+ assert.equal(operation?.status, "failed");
+ assert.equal(operation?.receipt?.id, "one");
+ assert.match(operation!.failure!.message, /connection lost/);
+ await scheduler.run([{ id: "poll-error", state: second!.state }]);
assert.equal(starts, 1);
+ assert.equal(polls, 1);
});
diff --git a/packages/driver-node/test/validation.test.ts b/packages/driver-node/test/validation.test.ts
index 70287daf..beff7fca 100644
--- a/packages/driver-node/test/validation.test.ts
+++ b/packages/driver-node/test/validation.test.ts
@@ -210,7 +210,7 @@ test("a structurally valid but semantically invalid Producer value never enters
validators: registry(),
}).run(outputBuild(linked));
- assert.equal(result.status, "paused");
+ assert.equal(result.status, "failed");
assert.match(result.outcomes[0]?.message ?? "", /measurement must be even/u);
assert.equal(result.state.records.some((record) => record.id === "measurement:root"), false);
});
@@ -273,7 +273,7 @@ test("Endpoint results pass the same Type-owner validation gate as Producer resu
assert.equal(valid.state.records[0]?.value.kind, "inline");
const invalid = await providerBuild(7);
- assert.equal(invalid.status, "paused");
+ assert.equal(invalid.status, "failed");
assert.match(invalid.outcomes.at(-1)?.message ?? "", /measurement must be even/u);
assert.equal(invalid.state.records.some((record) => record.id === "measurement:endpoint"), false);
});
diff --git a/packages/endpoint-kit/README.md b/packages/endpoint-kit/README.md
index 385f8ce8..5cc67128 100644
--- a/packages/endpoint-kit/README.md
+++ b/packages/endpoint-kit/README.md
@@ -12,13 +12,34 @@ without changing Core demand. Each resource declares a `limit` and optional `uni
A capability may add `resources` and a pure `unitsForRequest(request)` resolver for quantities of
already declared resources. Runtime admits all claims atomically for one `fulfill-need` Command.
-For asynchronous work, `failed` means the external job has ended or submission was definitely rejected.
-Transport errors and local deadlines cannot prove that. Return `pending` with the same handle;
-a pending `failure` records a local deadline while Runtime waits for remote termination. A poll with
-`settling: true` observes the existing job and returns `settled` when it ends, without downloading output.
-`cancel` returning `accepted`, `unsupported` or `too-late` does not release capacity. Only confirmed
-cancellation or observed termination does. A lost submission acknowledgement may remain pending without
-a handle; it must not trigger automatic resubmission or an automatic capacity release.
+Asynchronous execution moves forward through `start`, `poll`, and optional `collect`. `start` returns
+a task handle; `pending` means an acknowledged task is still running. `ready` records remote completion
+and hands its artifacts to `collect`, allowing download capacity to differ from task capacity.
+An Endpoint may return `completed` directly when no separate collection is needed.
+
+A `failed` outcome or thrown error ends the local execution attempt. This does not assert that the
+remote job ended. A submission timeout with no receipt is a failure, not a pending task to reconcile.
+`context.checkpoint` saves an acknowledgement before subsequent work; public `receipt` fields contain
+non-secret task identifiers suitable for inspection and Result retention. Opaque `handle` remains
+Provider-owned execution data. Explicit cancellation is best effort; `accepted`, `unsupported` and
+`too-late` are recorded acknowledgements, not proof of remote termination.
+
+For asynchronous capabilities, `actionLimits` configures `submit`, `poll` and `collect` separately:
+
+```ts
+actionLimits: {
+ submit: { concurrency: 2, rate: { limit: 1, periodMs: 200 } },
+ poll: { concurrency: 8 },
+ collect: { concurrency: 2 },
+}
+```
+
+These are example deployment choices, not model limits. `defineEndpointPackage` scopes their resource
+identities to the declared pool; custom `actions` can instead supply explicit shared resource claims.
+`concurrency` holds units until that action returns. A rate budget starts with `limit` permits,
+replenishes `limit` permits per `periodMs`, and spends one permit per admitted action. One action may
+make several HTTP requests; Provider transport policy owns those individual requests. Whole-operation
+resources describe occupancy. Core contains neither these phases nor any Provider names.
An individual immediate capability may declare `transient: true`. That permits a Runtime to use the
same handler in a disposable authoring execution with no Build, Result or recoverable Operation. It is a
diff --git a/packages/endpoint-kit/src/index.ts b/packages/endpoint-kit/src/index.ts
index a0c0ece0..a5f1427e 100644
--- a/packages/endpoint-kit/src/index.ts
+++ b/packages/endpoint-kit/src/index.ts
@@ -15,6 +15,7 @@ import type {
OperationFailure,
OperationProgress,
CapacityResourceClaim,
+ OperationReceipt,
} from "@hypit/runtime";
import { capacityUnits, verifyCredentialRef } from "@hypit/runtime";
@@ -54,29 +55,54 @@ export type ImmediateEndpointHandler = (
context: EndpointInvocationContext,
) => Awaitable<EndpointFulfillment>;
-export type EndpointOutcome =
+export type EndpointCheckpoint = {
+ readonly handle: CanonicalValue;
+ readonly receipt?: OperationReceipt;
+ readonly remoteEnded?: true;
+};
+
+export const endpointActions = ["submit", "poll", "collect"] as const;
+export type EndpointAction = typeof endpointActions[number];
+export type EndpointActionLimits = Partial<Readonly<Record<EndpointAction, {
+ readonly concurrency?: number;
+ readonly rate?: { readonly limit: number; readonly periodMs: number };
+}>>>;
+
+export function actionResourceClaims(pool: string, limits: EndpointActionLimits): NonNullable<EndpointScheduling["actions"]> {
+ return Object.fromEntries(Object.entries(limits).map(([action, limit]) => {
+ if (!endpointActions.includes(action as EndpointAction)) throw new Error(`Unknown Endpoint action ${action}`);
+ const resources: CapacityResourceClaim[] = [
+ ...(limit.concurrency === undefined ? [] : [{ id: `action:${pool}/${action}`, limit: limit.concurrency }]),
+ ...(limit.rate === undefined ? [] : [{ id: `rate:${pool}/${action}`, ...limit.rate }]),
+ ];
+ resources.forEach(capacityUnits);
+ return [action, resources];
+ }));
+}
+
+export type EndpointOutcome = (
| {
readonly status: "pending";
- readonly handle?: CanonicalValue;
+ readonly handle: CanonicalValue;
readonly wakeAt?: number;
readonly progress?: OperationProgress;
- readonly failure?: OperationFailure;
}
- /** Only returned to a settling poll, after the external work has stopped. No output is downloaded. */
- | { readonly status: "settled" }
| { readonly status: "completed"; readonly result: EndpointFulfillment }
- | { readonly status: "failed"; readonly failure: OperationFailure };
+ | { readonly status: "failed"; readonly failure: OperationFailure }
+ /** Remote work ended; collect its existing artifacts in a separate short action. */
+ | { readonly status: "ready"; readonly handle: CanonicalValue }
+) & { readonly receipt?: OperationReceipt };
export type EndpointStartContext = EndpointInvocationContext & {
/** Runtime-local identifier used to poll or cancel this submission. */
readonly operation: string;
+ /** Persist acknowledgement before doing further work. Secret values never belong in a receipt. */
+ readonly checkpoint?: (checkpoint: EndpointCheckpoint) => Promise<void>;
};
export type EndpointPollContext = EndpointStartContext & {
/** Provider task state returned by start(). */
readonly handle: CanonicalValue;
- /** Observe termination without creating work, enforcing a local deadline, or downloading results. */
- readonly settling?: true;
};
/**
@@ -86,7 +112,7 @@ export type EndpointPollContext = EndpointStartContext & {
*/
export type EndpointCancelOutcome =
| { readonly status: "confirmed" }
- | { readonly status: "accepted"; readonly wakeAt?: number }
+ | { readonly status: "accepted" }
| { readonly status: "unsupported" }
| { readonly status: "too-late" };
@@ -94,11 +120,13 @@ export type AsyncEndpoint = {
start(context: EndpointStartContext): Awaitable<EndpointOutcome>;
poll(context: EndpointPollContext): Awaitable<EndpointOutcome>;
cancel?(context: EndpointPollContext): Awaitable<EndpointCancelOutcome>;
+ collect?(context: EndpointPollContext): Awaitable<EndpointOutcome>;
};
/** Endpoint scheduling. It never changes Core demand. */
export type EndpointScheduling = {
readonly resources: readonly CapacityResourceClaim[];
+ readonly actions?: Partial<Readonly<Record<EndpointAction, readonly CapacityResourceClaim[]>>>;
/** Pure request-dependent quantities for already declared resources. */
readonly unitsForRequest?: (request: EndpointRequest) => Readonly<Record<string, number>>;
};
@@ -117,6 +145,7 @@ export function endpointResourceClaims(scheduling: EndpointScheduling, request:
}
export type EndpointRegistrationOptions = {
+ readonly pool?: string;
readonly supports?: (request: EndpointRequest) => boolean;
readonly scheduling?: EndpointScheduling;
readonly credentials?: Readonly<Record<string, CredentialRef>>;
@@ -229,6 +258,8 @@ export type DefineEndpointPackageOptions = {
}>>;
/** Total capacity shared by every capability under this configured Provider pool. */
readonly defaultConcurrency?: number;
+ readonly actions?: EndpointScheduling["actions"];
+ readonly actionLimits?: EndpointActionLimits;
/** The Provider's own price page, or `local` for work that runs on this machine without a charge. */
readonly pricing?: EndpointPricing;
readonly capabilities: readonly EndpointCapability[];
@@ -255,6 +286,8 @@ export function defineEndpointPackage(options: DefineEndpointPackageOptions): En
assert(options.instance.trim().length > 0, "Endpoint instance is empty");
assert(options.pool.trim().length > 0, "Endpoint Provider Pool is empty");
assert(options.capabilities.length > 0, "Endpoint package declares no capability");
+ assert(options.actions === undefined || options.actionLimits === undefined,
+ "Endpoint package must choose actions or actionLimits, not both");
const keys = options.capabilities.map((item) => refKey(item.capability));
assert(new Set(keys).size === keys.length, "Endpoint package repeats a capability");
const capacities = options.capabilities.map((item) => item.capacity ?? item.capability.name);
@@ -314,6 +347,7 @@ export function defineEndpointPackage(options: DefineEndpointPackageOptions): En
id: options.instance,
pool: options.pool,
};
+ const actions = options.actions ?? (options.actionLimits === undefined ? undefined : actionResourceClaims(options.pool, options.actionLimits));
const offers: readonly EndpointOffer[] = fulfills.map((item) => ({
...item,
endpoint: options.instance,
@@ -332,10 +366,12 @@ export function defineEndpointPackage(options: DefineEndpointPackageOptions): En
`${capacity} maxConcurrency`,
);
const common: EndpointRegistrationOptions = {
+ pool: options.pool,
...(capability.supports === undefined ? {} : { supports: capability.supports }),
...(capability.transient === true ? { transient: true } : {}),
credentials,
scheduling: {
+ ...(capability.lifecycle !== "asynchronous" || actions === undefined ? {} : { actions }),
...(capability.unitsForRequest === undefined ? {} : { unitsForRequest: capability.unitsForRequest }),
resources: [
{
diff --git a/packages/provider-hyperframes-aws-lambda/src/provider.ts b/packages/provider-hyperframes-aws-lambda/src/provider.ts
index 3459484a..e6510f38 100644
--- a/packages/provider-hyperframes-aws-lambda/src/provider.ts
+++ b/packages/provider-hyperframes-aws-lambda/src/provider.ts
@@ -409,10 +409,11 @@ export function createAwsLambdaHyperframesProvider(config: CreateAwsLambdaHyperf
executionName: executionName(context.operation),
});
const handle = makeHandle(context, site, render, startedAt);
- return wakeAfter(canonicalize(handle), pollIntervalMs, now(), { phase: "submitted" });
+ const receipt = { id: handle.executionArn };
+ await context.checkpoint?.({ handle: canonicalize(handle), receipt });
+ return { ...wakeAfter(canonicalize(handle), pollIntervalMs, now(), { phase: "submitted" }), receipt };
} catch (error) {
- return { status: "pending", wakeAt: now() + 30_000, progress: { phase: "submission-unknown" },
- failure: { code: "SUBMISSION_UNKNOWN", message: "HyperFrames submission acknowledgement is unknown; the same Build will not submit again" } };
+ return implementationFailure("HYPERFRAMES_SUBMISSION_FAILED", error);
}
};
@@ -435,9 +436,8 @@ export function createAwsLambdaHyperframesProvider(config: CreateAwsLambdaHyperf
} catch (error) {
throw error;
}
- if (!context.settling && now() - handle.startedAt >= maxOperationMs) {
- return { ...wakeAfter(canonicalize(handle), pollIntervalMs, now(), { phase: "settling" }),
- failure: { code: "HYPERFRAMES_OPERATION_TIMEOUT", message: "HyperFrames deadline exceeded; waiting for remote termination" } };
+ if (now() - handle.startedAt >= maxOperationMs) {
+ return implementationFailure("HYPERFRAMES_OPERATION_TIMEOUT", new Error("HyperFrames deadline exceeded"));
}
let document: HyperframesDocument;
let progress: HyperframesLambdaProgress;
@@ -449,12 +449,12 @@ export function createAwsLambdaHyperframesProvider(config: CreateAwsLambdaHyperf
region,
});
} catch (error) {
- return wakeAfter(canonicalize(handle), pollIntervalMs, now(), { phase: "poll-unavailable" });
+ return implementationFailure("HYPERFRAMES_POLL_FAILED", error);
}
try {
verifyProgress(progress, document);
} catch (error) {
- return wakeAfter(canonicalize(handle), pollIntervalMs, now(), { phase: "poll-unavailable" });
+ return implementationFailure("HYPERFRAMES_POLL_FAILED", error);
}
const next = { ...handle, polls: handle.polls + 1 };
if (progress.status === "RUNNING") {
@@ -465,7 +465,6 @@ export function createAwsLambdaHyperframesProvider(config: CreateAwsLambdaHyperf
unit: "frames",
});
}
- if (context.settling) return { status: "settled" };
if (progress.status !== "SUCCEEDED") {
const details = progress.errors.map((item) => `${item.state}: ${item.error}: ${item.cause}`).join("; ");
return implementationFailure(`HYPERFRAMES_${progress.status}`,
diff --git a/packages/provider-hyperframes-local/README.md b/packages/provider-hyperframes-local/README.md
index 0612996c..bbf4fd98 100644
--- a/packages/provider-hyperframes-local/README.md
+++ b/packages/provider-hyperframes-local/README.md
@@ -22,6 +22,26 @@ The Runtime Profile configures these controls through the Provider because they
renders program audio; `@hypit/provider-media-local` separately prepares `TimelineAudio` and muxes
the final media.
+For a Runtime Profile, place `pool` on the Endpoint entry and these Provider settings inside
+`config`. This is an illustrative entry to merge into the Profile's existing `endpoints`:
+
+```json
+"hyperframes.local": {
+ "use": "@hypit/provider-hyperframes-local",
+ "pool": "local-render",
+ "config": {
+ "workers": 4,
+ "defaultConcurrency": 2,
+ "browserCapacity": 6
+ }
+}
+```
+
+This configuration admits one 4-worker request at a time under its 6-browser budget even though
+the whole-request limit is 2. Two 4-worker requests would need 8 browser slots. Choose budgets from
+the actual machine and shared workloads; the example is not an automatic tuning recommendation.
+Reload the Runtime Worker after changing its Profile and restart Studio if it uses that Profile.
+
`browserGpu` picks Chrome's rasterizer and defaults to `hardware`. Set `software` without a usable
GPU, or `auto` to let the engine decide. Capture uses screenshots and independent browser processes;
the CLI's automatic worker and drawElement policies do not override the count.
diff --git a/packages/provider-hypihub/README.md b/packages/provider-hypihub/README.md
index 1b5e99f8..6640093e 100644
--- a/packages/provider-hypihub/README.md
+++ b/packages/provider-hypihub/README.md
@@ -73,12 +73,18 @@ sets narrower group limits, for example `{ "seedance-2-mini": 2, "gemini": 3, "t
Image/video/speech groups use the exact capability name; all Gemini capabilities share `gemini`, and
WhisperX uses `transcription`. These limits coordinate this Runtime's requests; HypiHub remains
responsible for service-wide account limits. Immediate speech/Gemini/transcription slots cover the
-active HTTP invocation, while asynchronous image/video slots remain held until the remote job ends.
+active HTTP invocation, while asynchronous image/video slots cover remote work until completion or local execution failure.
-For asynchronous jobs, a polling error retains the same job and its capacity. An operation deadline
-records failure but continues observing remote termination without downloading the output. Unknown
-submission acknowledgements are never automatically resubmitted. Runtime bindings never switch from
-a user's own Provider to HypiHub after a key, quota or transport failure.
+For asynchronous image/video jobs, `actionLimits` configures the common `submit`, `poll` and `collect`
+admission budgets. Each accepts `concurrency` and `rate: { limit, periodMs }`, shared by the pool.
+These limits count lifecycle actions; Provider-specific upload parts and HTTP requests remain inside
+those actions. Synchronous speech, Gemini and transcription retain their ordinary request capacity.
+
+A submission, polling or collection error ends the local attempt. Known job IDs and credential
+references remain available in Result receipts; a timeout with no ID is recorded as such. A job can
+be inspected at `/jobs/<id>` and its generated assets at `/jobs/<id>/assets` on the selected API base.
+The next production attempt uses a new Run and Build. Runtime bindings never switch from a user's
+own Provider to HypiHub after a key, quota or transport failure.
Multipart concurrency is different: HypiHub
selects it for one upload session, and it does not create a Build queue or a second Runtime pool.
diff --git a/packages/provider-hypihub/src/activation.ts b/packages/provider-hypihub/src/activation.ts
index 62779e5b..9e832921 100644
--- a/packages/provider-hypihub/src/activation.ts
+++ b/packages/provider-hypihub/src/activation.ts
@@ -1,6 +1,7 @@
import {
createRuntimeEndpointAdapterFacet,
runtimeConfigCredentialRef,
+ runtimeConfigActionLimits,
runtimeConfigExact,
runtimeConfigObject,
runtimeConfigPositiveInteger,
@@ -18,6 +19,7 @@ const adapter = createRuntimeEndpointAdapterFacet({
"baseUrl",
"apiKey",
"defaultConcurrency",
+ "actionLimits",
"capabilityConcurrency",
"pollIntervalMs",
"requestTimeoutMs",
@@ -38,6 +40,7 @@ const adapter = createRuntimeEndpointAdapterFacet({
}
const apiKey = runtimeConfigCredentialRef(config.apiKey, "HypiHub apiKey");
if (apiKey === undefined) throw new Error("HypiHub apiKey CredentialRef is required");
+ const actionLimits = runtimeConfigActionLimits(config.actionLimits);
const defaultConcurrency = runtimeConfigPositiveInteger(config.defaultConcurrency, "HypiHub defaultConcurrency");
const pollIntervalMs = runtimeConfigPositiveInteger(config.pollIntervalMs, "HypiHub pollIntervalMs");
const requestTimeoutMs = runtimeConfigPositiveInteger(config.requestTimeoutMs, "HypiHub requestTimeoutMs");
@@ -58,6 +61,7 @@ const adapter = createRuntimeEndpointAdapterFacet({
...(baseUrl === undefined ? {} : { baseUrl }),
apiKey,
...(defaultConcurrency === undefined ? {} : { defaultConcurrency }),
+ ...(actionLimits === undefined ? {} : { actionLimits }),
...(capabilityConcurrency === undefined ? {} : { capabilityConcurrency }),
...(pollIntervalMs === undefined ? {} : { pollIntervalMs }),
...(requestTimeoutMs === undefined ? {} : { requestTimeoutMs }),
diff --git a/packages/provider-hypihub/src/provider.ts b/packages/provider-hypihub/src/provider.ts
index 130a3675..a0d8b141 100644
--- a/packages/provider-hypihub/src/provider.ts
+++ b/packages/provider-hypihub/src/provider.ts
@@ -1,3 +1,4 @@
+import { requestDeadline } from "@hypit/runtime-kit";
import type { AsyncEndpoint, EndpointFulfillment, EndpointInvocationContext, EndpointPollContext, EndpointStartContext, EndpointOutcome, ImmediateEndpointHandler } from "@hypit/endpoint-kit";
import { defineEndpointPackage, wakeAfter } from "@hypit/endpoint-kit";
import { geminiCapabilities, geminiModels, geminiTypes, sealVisualObservation, verifyGeminiRequest } from "@hypit/gemini";
@@ -27,6 +28,7 @@ export type CreateHypiHubProviderOptions = {
readonly baseUrl?: string;
readonly apiKey?: CredentialRef;
readonly defaultConcurrency?: number;
+ readonly actionLimits?: import("@hypit/endpoint-kit").EndpointActionLimits;
readonly capabilityConcurrency?: Readonly<Record<string, number>>;
readonly pollIntervalMs?: number;
readonly requestTimeoutMs?: number;
@@ -124,28 +126,27 @@ class HypiHubClient {
});
}
async json(path: string, apiKey: string, init: RequestInit = {}): Promise<Record<string, unknown>> {
- const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), this.timeout);
+ const deadline = requestDeadline(this.timeout);
try {
- const response = await this.fetcher(`${this.baseUrl}${path}`, { ...init, signal: controller.signal, headers: { authorization: `Bearer ${apiKey}`, ...(init.headers ?? {}) } });
- const text = await response.text(); let body: unknown = {};
+ const response = await deadline.wait(this.fetcher(`${this.baseUrl}${path}`, { ...init, signal: deadline.signal, headers: { authorization: `Bearer ${apiKey}`, ...(init.headers ?? {}) } }));
+ const text = await deadline.wait(response.text()); let body: unknown = {};
if (!response.ok) throw new HypiHubHttpError(response.status, `HypiHub returned HTTP ${response.status}: ${text.slice(0, 300)}`);
try { body = text.length === 0 ? {} : JSON.parse(text); } catch { throw new Error(`HypiHub returned invalid JSON (${response.status})`); }
return object(body, "HypiHub response");
- } finally { clearTimeout(timer); }
+ } finally { deadline.finish(); }
}
async download(url: string): Promise<{ readonly bytes: Uint8Array; readonly mediaType: string }> {
let lastError: unknown;
for (let attempt = 0; attempt < this.downloadAttempts; attempt += 1) {
- const controller = new AbortController();
- const timer = setTimeout(() => controller.abort(), this.timeout);
+ const deadline = requestDeadline(this.timeout);
try {
- const response = await this.fetcher(url, { signal: controller.signal });
+ const response = await deadline.wait(this.fetcher(url, { signal: deadline.signal }));
if (!response.ok) throw new Error(`HypiHub asset returned HTTP ${response.status}`);
- return { bytes: new Uint8Array(await response.arrayBuffer()), mediaType: response.headers.get("content-type")?.split(";", 1)[0] ?? "application/octet-stream" };
+ return { bytes: new Uint8Array(await deadline.wait(response.arrayBuffer())), mediaType: response.headers.get("content-type")?.split(";", 1)[0] ?? "application/octet-stream" };
} catch (error) {
lastError = error;
} finally {
- clearTimeout(timer);
+ deadline.finish();
}
}
throw lastError instanceof Error ? lastError : new Error(String(lastError));
@@ -164,16 +165,15 @@ class HypiHubClient {
});
}
async speech(apiKey: string, body: Record<string, unknown>): Promise<readonly { readonly bytes: Uint8Array; readonly mediaType: string }[]> {
- const controller = new AbortController();
- const timer = setTimeout(() => controller.abort(), this.timeout);
+ const deadline = requestDeadline(this.timeout);
try {
- const response = await this.fetcher(`${this.baseUrl}/audio/speech`, {
+ const response = await deadline.wait(this.fetcher(`${this.baseUrl}/audio/speech`, {
method: "POST",
headers: { authorization: `Bearer ${apiKey}`, "content-type": "application/json" },
body: JSON.stringify({ ...body, output: "b64_json" }),
- signal: controller.signal,
- });
- const bytes = new Uint8Array(await response.arrayBuffer());
+ signal: deadline.signal,
+ }));
+ const bytes = new Uint8Array(await deadline.wait(response.arrayBuffer()));
if (!response.ok) {
throw new Error(`HypiHub returned HTTP ${response.status}: ${Buffer.from(bytes).toString("utf8").slice(0, 300)}`);
}
@@ -205,7 +205,7 @@ class HypiHubClient {
return await this.download(item.url);
}));
} finally {
- clearTimeout(timer);
+ deadline.finish();
}
}
}
@@ -303,8 +303,6 @@ async function synthesizeAudio(client: HypiHubClient, context: EndpointInvocatio
function endpoint(client: HypiHubClient, pollIntervalMs: number, maxOperationMs: number, publicAssetUrl: CreateHypiHubProviderOptions["publicAssetUrl"]): AsyncEndpoint {
return {
async start(context: EndpointStartContext) {
- let submitting = false;
- let remoteEnded = false;
try {
const route = hypiHubRouteForCapability(context.need.capability);
assert(route !== undefined, "HypiHub does not implement this exact capability"); const apiKey = credential(context);
@@ -325,46 +323,46 @@ function endpoint(client: HypiHubClient, pollIntervalMs: number, maxOperationMs:
const path = operation === "image_edits" ? "/images/edits"
: operation === "images" ? "/images/generations" : "/videos";
await verifyModelRoute(client, apiKey, compiled.model, operation);
- submitting = true;
const response = await client.json(path, apiKey, { method: "POST", headers: { "content-type": "application/json", "idempotency-key": context.operation }, body: JSON.stringify({ model: compiled.model, ...input }) });
const status = response.status;
- remoteEnded = status === "succeeded" || status === "completed";
- if (remoteEnded) return await complete(client, apiKey, route, jobId(response), context.resources);
+ const remoteEnded = status === "succeeded" || status === "completed";
const handle: Handle = { contract: "hypit.hypihub-operation@1", jobId: jobId(response), route: capabilityKey(route.capability), startedAt: Date.now() };
- return wakeAfter(canonicalize(handle), pollIntervalMs, Date.now(), { phase: "submitted" });
+ const receipt = { id: handle.jobId };
+ await context.checkpoint?.({ handle: canonicalize(handle), receipt, ...(remoteEnded ? { remoteEnded: true as const } : {}) });
+ return remoteEnded ? { status: "ready", handle: canonicalize(handle), receipt }
+ : { ...wakeAfter(canonicalize(handle), pollIntervalMs, Date.now(), { phase: "submitted" }), receipt };
} catch (error) {
- if (submitting && !remoteEnded && !(error instanceof HypiHubHttpError && error.status >= 400 && error.status < 500)) {
- return { status: "pending", wakeAt: Date.now() + 30_000, progress: { phase: "submission-unknown" },
- failure: { code: "SUBMISSION_UNKNOWN", message: "HypiHub submission acknowledgement is unknown; the same Build will not submit again" } };
- }
return failure(error);
}
},
async poll(context: EndpointPollContext) {
- let remoteEnded = false;
try {
const handle = object(context.handle, "HypiHub handle") as unknown as Handle; const route = hypiHubRouteForCapability(context.need.capability);
assert(route !== undefined && handle.contract === "hypit.hypihub-operation@1" && handle.route === capabilityKey(route.capability), "HypiHub handle is invalid");
- if (!context.settling && Date.now() - handle.startedAt > maxOperationMs) {
- return { ...wakeAfter(canonicalize(handle), pollIntervalMs, Date.now(), { phase: "settling" }),
- failure: { code: "HYPIHUB_OPERATION_TIMEOUT", message: "HypiHub operation timed out; waiting for remote termination" } };
+ if (Date.now() - handle.startedAt > maxOperationMs) {
+ return { status: "failed",
+ failure: { code: "HYPIHUB_OPERATION_TIMEOUT", message: "HypiHub operation timed out" } };
}
const job = await client.json(`/jobs/${encodeURIComponent(handle.jobId)}`, credential(context)); const status = job.status;
if (status === "queued" || status === "running" || status === "in_progress") return wakeAfter(canonicalize(handle), pollIntervalMs, Date.now(), { phase: String(status) });
- remoteEnded = ["failed", "queue_expired", "cancelled", "succeeded", "completed"].includes(String(status));
- if (remoteEnded && context.settling) return { status: "settled" };
if (status === "failed" || status === "queue_expired" || status === "cancelled") {
const detail = [job.error, job.message, job.reason, job.detail]
.find((value) => typeof value === "string" && value.length > 0);
throw new Error(`HypiHub job ${status}${typeof detail === "string" ? `: ${detail}` : ""}`);
}
if (status !== "succeeded" && status !== "completed") throw new Error(`HypiHub returned unknown job status ${String(status)}`);
- return await complete(client, credential(context), route, handle.jobId, context.resources);
+ return { status: "ready", handle: context.handle, receipt: { id: handle.jobId } };
} catch (error) {
- if (remoteEnded) return failure(error);
- return wakeAfter(context.handle, pollIntervalMs, Date.now(), { phase: "poll-unavailable" });
+ return failure(error);
}
},
+ async collect(context) {
+ const handle = object(context.handle, "HypiHub handle") as unknown as Handle;
+ const route = hypiHubRouteForCapability(context.need.capability);
+ assert(route !== undefined && handle.route === capabilityKey(route.capability), "HypiHub collection route differs");
+ return await complete(client, credential(context), route, handle.jobId, context.resources);
+ },
+
};
}
@@ -481,6 +479,7 @@ export function createHypiHubProvider(options: CreateHypiHubProviderOptions = {}
},
} },
defaultConcurrency: options.defaultConcurrency ?? 4,
+ ...(options.actionLimits === undefined ? {} : { actionLimits: options.actionLimits }),
capabilities: [
...hypiHubRoutes
.map((route) => route.media === "audio"
diff --git a/packages/provider-hypihub/src/upload.ts b/packages/provider-hypihub/src/upload.ts
index 4cc4e0bc..ba6b046d 100644
--- a/packages/provider-hypihub/src/upload.ts
+++ b/packages/provider-hypihub/src/upload.ts
@@ -1,3 +1,4 @@
+import { requestDeadline } from "@hypit/runtime-kit";
import { createHash } from "node:crypto";
function assert(condition: unknown, message: string): asserts condition {
@@ -92,15 +93,14 @@ export class HypiHubUploader {
}
async #json(path: string, apiKey: string, init: RequestInit = {}): Promise<Record<string, unknown>> {
- const controller = new AbortController();
- const timer = setTimeout(() => controller.abort(), this.#requestTimeoutMs);
+ const deadline = requestDeadline(this.#requestTimeoutMs);
try {
- const response = await this.#fetch(`${this.#baseUrl}${path}`, {
+ const response = await deadline.wait(this.#fetch(`${this.#baseUrl}${path}`, {
...init,
- signal: controller.signal,
+ signal: deadline.signal,
headers: { authorization: `Bearer ${apiKey}`, ...(init.headers ?? {}) },
- });
- const text = await response.text();
+ }));
+ const text = await deadline.wait(response.text());
let body: unknown = {};
try { body = text.length === 0 ? {} : JSON.parse(text); }
catch { throw new Error(`HypiHub returned invalid JSON (${response.status})`); }
@@ -109,7 +109,7 @@ export class HypiHubUploader {
}
return object(body, "HypiHub response");
} finally {
- clearTimeout(timer);
+ deadline.finish();
}
}
@@ -162,19 +162,18 @@ export class HypiHubUploader {
if (attempt > 1) {
signed = (await this.#signParts(uploadId, [declaration], apiKey)).get(declaration.part_number) ?? {};
}
- const controller = new AbortController();
- const timer = setTimeout(() => controller.abort(), this.#uploadPartTimeoutMs);
+ const deadline = requestDeadline(this.#uploadPartTimeoutMs);
try {
const url = requiredString(signed.url, "HypiHub signed upload URL");
assertHTTPS(url, "HypiHub signed upload URL");
const body = new ArrayBuffer(bytes.byteLength);
new Uint8Array(body).set(bytes);
- const response = await this.#fetch(url, {
+ const response = await deadline.wait(this.#fetch(url, {
method: "PUT",
headers: this.#signedHeaders(signed.headers, declaration),
body,
- signal: controller.signal,
- });
+ signal: deadline.signal,
+ }));
if (!response.ok) throw new Error(`S3 rejected upload part with HTTP ${response.status}`);
const etag = response.headers.get("etag");
assert(etag !== null && etag.length > 0,
@@ -192,7 +191,7 @@ export class HypiHubUploader {
throw new Error(`HypiHub upload part ${declaration.part_number} failed after ${attempt} attempts`);
}
} finally {
- clearTimeout(timer);
+ deadline.finish();
}
}
throw new Error(`HypiHub upload part ${declaration.part_number} failed`);
diff --git a/packages/provider-hypihub/test/provider.test.ts b/packages/provider-hypihub/test/provider.test.ts
index d5011978..fbed3261 100644
--- a/packages/provider-hypihub/test/provider.test.ts
+++ b/packages/provider-hypihub/test/provider.test.ts
@@ -282,7 +282,10 @@ test("HypiHub uploads one referenced Resource once and submits its HTTPS URL", a
assert.deepEqual(signedBatches, [[1, 2, 3], [2]]);
assert.deepEqual([...partAttempts.entries()].sort(), [[1, 1], [2, 2], [3, 1]]);
if (started.status !== "pending") return;
- const completed = await endpoint.poll({ ...common, handle: started.handle! });
+ const ready = await endpoint.poll({ ...common, handle: started.handle! });
+ assert.equal(ready.status, "ready");
+ if (ready.status !== "ready") return;
+ const completed = await endpoint.collect!({ ...common, handle: ready.handle });
assert.equal(completed.status, "completed");
});
@@ -544,15 +547,14 @@ test("HypiHub exposes model groups beneath its own total capacity", async () =>
assert.throws(() => createHypiHubProvider({ capabilityConcurrency: { invented: 1 } }), /unknown HypiHub capacity/);
});
-test("HypiHub polling errors and local deadlines retain remote work until settlement", async () => {
+test("HypiHub polling errors and operation deadlines fail without settlement polling", async () => {
const request = need({});
- let unavailable = true, status = "running", requests = 0;
+ let requests = 0;
const registry = new EndpointRegistry();
- await createHypiHubProvider({ pollIntervalMs: 0, operationTimeoutMs: 10,
+ await createHypiHubProvider({ pollIntervalMs: 0, operationTimeoutMs: 60_000,
fetch: async (url) => {
requests++; assert.match(String(url), /\/jobs\/test$/);
- if (unavailable) throw new Error("offline");
- return Response.json({ status });
+ throw new Error("offline");
} }).install(registry);
const selected = registry.resolve(request);
assert.equal(selected.status, "resolved"); assert.equal(selected.registration.kind, "asynchronous");
@@ -562,13 +564,11 @@ test("HypiHub polling errors and local deadlines retain remote work until settle
handle: { contract: "hypit.hypihub-operation@1", jobId: "test",
route: `${request.capability.module.name}@${request.capability.module.version}#${request.capability.name}`, startedAt: 0 } };
const timedOut = await endpoint.poll(common);
- assert.equal(timedOut.status, "pending");
- assert.equal(timedOut.status === "pending" && timedOut.failure?.code, "HYPIHUB_OPERATION_TIMEOUT");
+ assert.equal(timedOut.status, "failed");
+ assert.equal(timedOut.status === "failed" && timedOut.failure.code, "HYPIHUB_OPERATION_TIMEOUT");
assert.equal(requests, 0);
- assert.equal((await endpoint.poll({ ...common, settling: true })).status, "pending");
- unavailable = false;
- assert.equal((await endpoint.poll({ ...common, settling: true })).status, "pending");
- status = "completed";
- assert.deepEqual(await endpoint.poll({ ...common, settling: true }), { status: "settled" });
- assert.equal(requests, 3, "settling must only query the existing job and never download or submit");
+ const offline = await endpoint.poll({ ...common, handle: { ...common.handle, startedAt: Date.now() } });
+ assert.equal(offline.status, "failed");
+ assert.match(offline.status === "failed" ? offline.failure.message : "", /offline/);
+ assert.equal(requests, 1);
});
diff --git a/packages/provider-kie/README.md b/packages/provider-kie/README.md
index 1d91172d..0954d7d9 100644
--- a/packages/provider-kie/README.md
+++ b/packages/provider-kie/README.md
@@ -87,30 +87,29 @@ unless the Profile explicitly gives it that same real-resource identity.
One Need execution acquires its total and model-specific capacity claims atomically. These are shared-resource
limits, not parent and child queues.
-`submissionIntervalMs` spaces submissions within one Provider instance. It is a frequency limit,
-not a concurrency slot, and is not shared across separate instances by the `pool` setting.
-Polling errors retain the task and its capacity. A local operation deadline records failure and
-continues observing termination; unsupported cancellation cannot release a still-running task.
+Optional `actionLimits` uses the shared Endpoint vocabulary for `submit`, `poll` and `collect`.
+For example, `submit: { rate: { limit: 1, periodMs: 200 } }` admits one start action every 200 ms,
+with an initial burst of one. It is shared by instances in the same pool. A start includes reference
+preparation and submission; this is an action-admission limit, not a timestamp guarantee for each
+HTTP request. `collect: { concurrency: 2 }` separately bounds concurrent result downloads.
+Polling transport errors and operation deadlines fail the attempt and retain the task receipt.
An advanced embedding adds `kie` to its Endpoint list beside a complete, explicit set of Runtime
service packages and selections. The `.svml` Module Closure separately contains only the model
Manifests actually imported by the author document; installing KIE does not add author intent.
-## Paid-operation law
-
-1. Reference `BlobRef`s are read from the current Build's working byte area and uploaded through KIE's file
- stream API. KIE temporary URLs never enter author source or generated Product identity.
-2. A successful `createTask` response is persisted as one asynchronous Operation. Because KIE does not document an
- idempotency key, an ambiguous network/5xx submission is not automatically retried.
-3. Once a `taskId` exists, later Worker polling continues only that same task. Poll/download errors cannot create a new
- paid generation.
-4. Successful result URLs are converted to short-lived download URLs, bounded while streaming,
- immediately written into that Build's working byte area, and removed from durable
- result metadata.
-5. The selected Runtime Execution Store owns shared capacity across Builds. This Provider contributes
- one KIE-total claim plus narrower exact-capability claims and a conservative create-task interval; it does not introduce Redis or another source of
- Build truth.
-
-The automated suite uses an adversarial fake KIE service. It covers submission ambiguity,
-continuation of accepted tasks, polling, bounded downloads and failures without spending money or
-requiring deployment credentials.
+## Task execution and receipts
+
+Reference Resources are uploaded before `createTask`. An acknowledged task ID is saved immediately;
+normal polling uses that same ID. Submission, polling or collection errors end the attempt with the
+actual error. A timeout without an ID records the missing acknowledgement; the next attempt can be a
+new Build. No failure changes the selected Endpoint or account.
+
+On success, poll reports artifacts ready and `collect` downloads them into the Build's working byte
+area. The Result retains completed public Outputs and non-secret task receipts after active Runtime
+state is removed. A task receipt identifies `/api/v1/jobs/recordInfo?taskId=<id>` on the configured
+KIE API base URL; the selected credential reference supplies access without storing the key in Result.
+These records support inspection and explicit reuse in a new Run, not restarting the old Build.
+
+The automated tests exercise submission errors, accepted-task polling, full-response deadlines and
+bounded downloads against fake services, without paid generation.
diff --git a/packages/provider-kie/src/activation.ts b/packages/provider-kie/src/activation.ts
index dfd3fd9d..578ba1dc 100644
--- a/packages/provider-kie/src/activation.ts
+++ b/packages/provider-kie/src/activation.ts
@@ -1,6 +1,7 @@
import {
createRuntimeEndpointAdapterFacet,
runtimeConfigCredentialRef,
+ runtimeConfigActionLimits,
runtimeConfigExact,
runtimeConfigObject,
runtimeConfigPositiveInteger,
@@ -16,7 +17,7 @@ const kieRuntimeAdapter = createRuntimeEndpointAdapterFacet({
const config = runtimeConfigObject(context.config, "KIE");
runtimeConfigExact(config, [
"apiBaseUrl", "uploadBaseUrl", "apiKey", "defaultConcurrency", "pollIntervalMs",
- "capabilityConcurrency", "submissionIntervalMs", "requestTimeoutMs", "maxOperationMs", "maxArtifactBytes",
+ "capabilityConcurrency", "actionLimits", "requestTimeoutMs", "maxOperationMs", "maxArtifactBytes",
], "KIE");
const apiBaseUrl = runtimeConfigString(config.apiBaseUrl, "KIE apiBaseUrl");
const uploadBaseUrl = runtimeConfigString(config.uploadBaseUrl, "KIE uploadBaseUrl");
@@ -39,7 +40,7 @@ const kieRuntimeAdapter = createRuntimeEndpointAdapterFacet({
return [capability, concurrency];
}));
const pollIntervalMs = runtimeConfigPositiveInteger(config.pollIntervalMs, "KIE pollIntervalMs");
- const submissionIntervalMs = runtimeConfigPositiveInteger(config.submissionIntervalMs, "KIE submissionIntervalMs");
+ const actionLimits = runtimeConfigActionLimits(config.actionLimits);
const requestTimeoutMs = runtimeConfigPositiveInteger(config.requestTimeoutMs, "KIE requestTimeoutMs");
const maxOperationMs = runtimeConfigPositiveInteger(config.maxOperationMs, "KIE maxOperationMs");
const maxArtifactBytes = runtimeConfigPositiveInteger(config.maxArtifactBytes, "KIE maxArtifactBytes");
@@ -53,7 +54,7 @@ const kieRuntimeAdapter = createRuntimeEndpointAdapterFacet({
...(defaultConcurrency === undefined ? {} : { defaultConcurrency }),
...(capabilityConcurrency === undefined ? {} : { capabilityConcurrency }),
...(pollIntervalMs === undefined ? {} : { pollIntervalMs }),
- ...(submissionIntervalMs === undefined ? {} : { submissionIntervalMs }),
+ ...(actionLimits === undefined ? {} : { actionLimits }),
...(requestTimeoutMs === undefined ? {} : { requestTimeoutMs }),
...(maxOperationMs === undefined ? {} : { maxOperationMs }),
...(maxArtifactBytes === undefined ? {} : { maxArtifactBytes }),
diff --git a/packages/provider-kie/src/provider.ts b/packages/provider-kie/src/provider.ts
index 49cf6be5..b0745af9 100644
--- a/packages/provider-kie/src/provider.ts
+++ b/packages/provider-kie/src/provider.ts
@@ -1,3 +1,4 @@
+import { requestDeadline } from "@hypit/runtime-kit";
import type {
AsyncEndpoint,
EndpointPollContext,
@@ -41,7 +42,7 @@ export type CreateKieProviderOptions = {
/** Optional KIE lane limits keyed by capability name, for example seedance-2.5. */
readonly capabilityConcurrency?: Readonly<Record<string, number>>;
readonly pollIntervalMs?: number;
- readonly submissionIntervalMs?: number;
+ readonly actionLimits?: import("@hypit/endpoint-kit").EndpointActionLimits;
readonly requestTimeoutMs?: number;
readonly maxOperationMs?: number;
readonly maxArtifactBytes?: number;
@@ -112,36 +113,6 @@ function mediaExtension(mediaType: string): string {
return known[mediaType] ?? "bin";
}
-function delay(milliseconds: number): Promise<void> {
- return new Promise((resolve) => setTimeout(resolve, milliseconds));
-}
-
-class IntervalGate {
- #tail = Promise.resolve();
- #nextAt = 0;
- readonly #intervalMs: number;
- readonly #now: () => number;
-
- constructor(intervalMs: number, now: () => number) {
- this.#intervalMs = intervalMs;
- this.#now = now;
- }
-
- async enter(): Promise<void> {
- const previous = this.#tail;
- let release!: () => void;
- this.#tail = new Promise<void>((resolve) => { release = resolve; });
- await previous;
- try {
- const wait = Math.max(0, this.#nextAt - this.#now());
- if (wait > 0) await delay(wait);
- this.#nextAt = this.#now() + this.#intervalMs;
- } finally {
- release();
- }
- }
-}
-
type KieClientOptions = {
readonly apiBaseUrl: string;
readonly uploadBaseUrl: string;
@@ -157,55 +128,68 @@ class KieClient {
this.#options = options;
}
- async #fetch(url: string, init: RequestInit): Promise<Response> {
- const controller = new AbortController();
- const timer = setTimeout(() => controller.abort(new Error("KIE request timed out")), this.#options.requestTimeoutMs);
+ async #open(url: string, init: RequestInit): Promise<{
+ readonly response: Response;
+ readonly wait: <T>(work: Promise<T>) => Promise<T>;
+ readonly finish: () => void;
+ }> {
+ const deadline = requestDeadline(this.#options.requestTimeoutMs, () => new KieError("KIE_REQUEST_TIMEOUT", "KIE request timed out"));
+ const { wait, finish } = deadline;
try {
- return await this.#options.fetch(url, { ...init, signal: controller.signal });
+ const response = await wait(this.#options.fetch(url, { ...init, signal: deadline.signal }));
+ return { response, wait, finish };
} catch (error) {
+ finish();
+ if (error instanceof KieError) throw error;
throw new KieError("KIE_NETWORK_ERROR", error instanceof Error ? error.message : "KIE network request failed");
- } finally {
- clearTimeout(timer);
}
}
async #json(url: string, init: RequestInit): Promise<Record<string, unknown>> {
- const response = await this.#fetch(url, init);
- if (!response.ok) {
- throw new KieError("KIE_HTTP_ERROR", `KIE returned HTTP ${response.status}`, { status: response.status });
- }
- const limit = 2_000_000;
- const reader = response.body?.getReader();
- let text: string;
- if (reader === undefined) {
- const bytes = new Uint8Array(await response.arrayBuffer());
- if (bytes.byteLength > limit) throw new KieError("KIE_RESPONSE_TOO_LARGE", "KIE JSON response is too large");
- text = new TextDecoder().decode(bytes);
- } else {
- const decoder = new TextDecoder();
- let size = 0;
- let decoded = "";
- try {
- while (true) {
- const item = await reader.read();
- if (item.done) break;
- size += item.value.byteLength;
- if (size > limit) {
- await reader.cancel();
- throw new KieError("KIE_RESPONSE_TOO_LARGE", "KIE JSON response is too large");
+ const opened = await this.#open(url, init);
+ try {
+ const response = opened.response;
+ if (!response.ok) {
+ throw new KieError("KIE_HTTP_ERROR", `KIE returned HTTP ${response.status}`, { status: response.status });
+ }
+ const limit = 2_000_000;
+ const reader = response.body?.getReader();
+ let text: string;
+ if (reader === undefined) {
+ const bytes = new Uint8Array(await opened.wait(response.arrayBuffer()));
+ if (bytes.byteLength > limit) throw new KieError("KIE_RESPONSE_TOO_LARGE", "KIE JSON response is too large");
+ text = new TextDecoder().decode(bytes);
+ } else {
+ const decoder = new TextDecoder();
+ let size = 0;
+ let decoded = "";
+ try {
+ while (true) {
+ const item = await opened.wait(reader.read());
+ if (item.done) break;
+ size += item.value.byteLength;
+ if (size > limit) {
+ await reader.cancel();
+ throw new KieError("KIE_RESPONSE_TOO_LARGE", "KIE JSON response is too large");
+ }
+ decoded += decoder.decode(item.value, { stream: true });
}
- decoded += decoder.decode(item.value, { stream: true });
+ text = decoded + decoder.decode();
+ } finally {
+ reader.releaseLock();
}
- text = decoded + decoder.decode();
- } finally {
- reader.releaseLock();
}
- }
- try {
- return object(JSON.parse(text), "KIE response");
+ try {
+ return object(JSON.parse(text), "KIE response");
+ } catch (error) {
+ if (error instanceof KieError) throw error;
+ throw new KieError("KIE_INVALID_JSON", "KIE returned invalid JSON");
+ }
} catch (error) {
if (error instanceof KieError) throw error;
- throw new KieError("KIE_INVALID_JSON", "KIE returned invalid JSON");
+ throw new KieError("KIE_NETWORK_ERROR", error instanceof Error ? error.message : "KIE network request failed");
+ } finally {
+ opened.finish();
}
}
@@ -346,44 +330,63 @@ class KieClient {
mediaType: string;
}> {
const url = await this.#downloadUrl(original, apiKey);
- const response = await this.#fetch(url, { method: "GET" });
- if (!response.ok) throw new KieError("KIE_DOWNLOAD_FAILED", `KIE artifact download returned HTTP ${response.status}`);
- const declared = Number(response.headers.get("content-length"));
- if (Number.isFinite(declared) && declared > this.#options.maxArtifactBytes) {
- throw new KieError("KIE_ARTIFACT_TOO_LARGE", "KIE result exceeds the configured artifact limit");
- }
- const header = response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase();
- const fallback = expected === "image" ? "image/png" : "video/mp4";
- const mediaType = header === undefined || header === "application/octet-stream" ? fallback : header;
- if (!mediaType.startsWith(`${expected}/`)) {
- throw new KieError("KIE_RESULT_MEDIA_MISMATCH", `KIE returned ${mediaType} for ${expected} generation`);
- }
- const maximum = this.#options.maxArtifactBytes;
- const chunks = response.body === null
- ? (async function* () {
- const bytes = new Uint8Array(await response.arrayBuffer());
- if (bytes.byteLength > maximum) throw new KieError("KIE_ARTIFACT_TOO_LARGE", "KIE result is too large");
- yield bytes;
- })()
- : (async function* () {
- const reader = response.body!.getReader();
- let size = 0;
- try {
- while (true) {
- const item = await reader.read();
- if (item.done) break;
- size += item.value.byteLength;
- if (size > maximum) {
- await reader.cancel();
- throw new KieError("KIE_ARTIFACT_TOO_LARGE", "KIE result is too large");
+ const opened = await this.#open(url, { method: "GET" });
+ let handedOff = false;
+ try {
+ const response = opened.response;
+ if (!response.ok) throw new KieError("KIE_DOWNLOAD_FAILED", `KIE artifact download returned HTTP ${response.status}`);
+ const declared = Number(response.headers.get("content-length"));
+ if (Number.isFinite(declared) && declared > this.#options.maxArtifactBytes) {
+ throw new KieError("KIE_ARTIFACT_TOO_LARGE", "KIE result exceeds the configured artifact limit");
+ }
+ const header = response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase();
+ const fallback = expected === "image" ? "image/png" : "video/mp4";
+ const mediaType = header === undefined || header === "application/octet-stream" ? fallback : header;
+ if (!mediaType.startsWith(`${expected}/`)) {
+ throw new KieError("KIE_RESULT_MEDIA_MISMATCH", `KIE returned ${mediaType} for ${expected} generation`);
+ }
+ const maximum = this.#options.maxArtifactBytes;
+ const finish = opened.finish;
+ const chunks = response.body === null
+ ? (async function* () {
+ try {
+ const bytes = new Uint8Array(await opened.wait(response.arrayBuffer()));
+ if (bytes.byteLength > maximum) throw new KieError("KIE_ARTIFACT_TOO_LARGE", "KIE result is too large");
+ yield bytes;
+ } catch (error) {
+ if (error instanceof KieError) throw error;
+ throw new KieError("KIE_NETWORK_ERROR", error instanceof Error ? error.message : "KIE result download failed");
+ } finally {
+ finish();
+ }
+ })()
+ : (async function* () {
+ const reader = response.body!.getReader();
+ let size = 0;
+ try {
+ while (true) {
+ const item = await opened.wait(reader.read());
+ if (item.done) break;
+ size += item.value.byteLength;
+ if (size > maximum) {
+ await reader.cancel();
+ throw new KieError("KIE_ARTIFACT_TOO_LARGE", "KIE result is too large");
+ }
+ yield item.value;
}
- yield item.value;
+ } catch (error) {
+ if (error instanceof KieError) throw error;
+ throw new KieError("KIE_NETWORK_ERROR", error instanceof Error ? error.message : "KIE result download failed");
+ } finally {
+ reader.releaseLock();
+ finish();
}
- } finally {
- reader.releaseLock();
- }
- })();
- return { chunks, mediaType };
+ })();
+ handedOff = true;
+ return { chunks, mediaType };
+ } finally {
+ if (!handedOff) opened.finish();
+ }
}
}
@@ -424,7 +427,6 @@ function readHandle(value: CanonicalValue | undefined, context: EndpointPollCont
function endpoint(options: {
readonly client: KieClient;
- readonly gate: IntervalGate;
readonly pollIntervalMs: number;
readonly maxOperationMs: number;
readonly now: () => number;
@@ -441,7 +443,6 @@ function endpoint(options: {
};
return {
async start(context) {
- let submitting = false;
try {
const route = kieRouteForCapability(context.need.capability);
if (route === undefined) throw new KieError("KIE_UNSUPPORTED_CAPABILITY", "KIE does not implement this exact capability");
@@ -455,23 +456,16 @@ function endpoint(options: {
return promise;
};
const task = await route.compile(context.need.constraints, resolve);
- await options.gate.enter();
- submitting = true;
const taskId = await options.client.createTask(task, key);
const handle: KieHandle = {
taskId,
routeKey: route.key,
startedAt: options.now(),
};
- return wakeAfter(canonicalize(handle), options.pollIntervalMs, options.now(), {
- phase: "submitted",
- });
+ const receipt = { id: taskId };
+ await context.checkpoint?.({ handle: canonicalize(handle), receipt });
+ return { ...wakeAfter(canonicalize(handle), options.pollIntervalMs, options.now(), { phase: "submitted" }), receipt };
} catch (error) {
- if (submitting && !(error instanceof KieError && ["KIE_SUBMISSION_REJECTED", "KIE_RATE_LIMITED"].includes(error.code))) {
- return { status: "pending", wakeAt: options.now() + 30_000,
- progress: { phase: "submission-unknown" }, failure: { code: "SUBMISSION_UNKNOWN",
- message: "KIE submission acknowledgement is unknown; the same Build will not submit again" } };
- }
return failure(error);
}
},
@@ -482,15 +476,13 @@ function endpoint(options: {
} catch (error) {
throw error;
}
- let remoteEnded = false;
try {
const route = kieRouteForCapability(context.need.capability);
if (route === undefined || route.key !== handle.routeKey) {
throw new KieError("KIE_HANDLE_INVALID", "KIE handle capability differs");
}
- if (!context.settling && options.now() - handle.startedAt >= options.maxOperationMs) {
- return { ...wakeAfter(canonicalize(handle), options.pollIntervalMs, options.now(), { phase: "settling" }),
- failure: { code: "KIE_OPERATION_TIMEOUT", message: "KIE task exceeded its operation deadline; waiting for remote termination" } };
+ if (options.now() - handle.startedAt >= options.maxOperationMs) {
+ throw new KieError("KIE_OPERATION_TIMEOUT", "KIE task exceeded its operation deadline");
}
const key = secret(context);
const data = await options.client.taskInfo(handle.taskId, key);
@@ -500,8 +492,6 @@ function endpoint(options: {
phase: state,
});
}
- remoteEnded = state === "fail" || state === "success";
- if (remoteEnded && context.settling) return { status: "settled" };
if (state === "fail") {
const vendorCode = typeof data.failCode === "string" && data.failCode.length > 0 ? data.failCode : "unknown";
const vendorMessage = typeof data.failMsg === "string" && data.failMsg.length > 0
@@ -511,37 +501,46 @@ function endpoint(options: {
if (state !== "success") throw new KieError("KIE_TASK_STATE_INVALID", "KIE returned an unknown task state");
const urls = resultUrls(data);
if (urls.length > route.maxResults) throw new KieError("KIE_RESULT_COUNT_EXCEEDED", "KIE returned too many result artifacts");
- const artifacts: BlobRef[] = [];
- for (const url of urls) {
- const downloaded = await options.client.download(url, route.media, key);
- if (isStreamingResourceStore(context.resources)) {
- artifacts.push(await context.resources.putStream(downloaded.chunks, downloaded.mediaType));
- } else {
- const chunks: Uint8Array[] = [];
- let size = 0;
- for await (const chunk of downloaded.chunks) {
- chunks.push(Uint8Array.from(chunk));
- size += chunk.byteLength;
- }
- const bytes = new Uint8Array(size);
- let offset = 0;
- for (const chunk of chunks) {
- bytes.set(chunk, offset);
- offset += chunk.byteLength;
- }
- artifacts.push(await context.resources.put(bytes, downloaded.mediaType));
+ return { status: "ready", handle: canonicalize({ ...handle, urls }), receipt: { id: handle.taskId } };
+ } catch (error) {
+ return failure(error);
+ }
+ },
+ async collect(context) {
+ readHandle(context.handle, context);
+ const route = kieRouteForCapability(context.need.capability)!;
+ const raw = object(context.handle, "KIE handle");
+ const urls = raw.urls;
+ if (!Array.isArray(urls) || !urls.every((url) => typeof url === "string")) throw new KieError("KIE_HANDLE_INVALID", "KIE collection has no artifact addresses");
+ const key = secret(context);
+ const artifacts: BlobRef[] = [];
+ for (const url of urls) {
+ const downloaded = await options.client.download(url, route.media, key);
+ if (isStreamingResourceStore(context.resources)) {
+ artifacts.push(await context.resources.putStream(downloaded.chunks, downloaded.mediaType));
+ } else {
+ const chunks: Uint8Array[] = [];
+ let size = 0;
+ for await (const chunk of downloaded.chunks) {
+ chunks.push(Uint8Array.from(chunk));
+ size += chunk.byteLength;
+ }
+ const bytes = new Uint8Array(size);
+ let offset = 0;
+ for (const chunk of chunks) {
+ bytes.set(chunk, offset);
+ offset += chunk.byteLength;
}
+ artifacts.push(await context.resources.put(bytes, downloaded.mediaType));
}
- const result = route.packageResult(artifacts);
- return {
- status: "completed",
- result: { value: result },
- };
- } catch (error) {
- if (remoteEnded) return failure(error);
- return wakeAfter(canonicalize(handle), options.pollIntervalMs, options.now(), { phase: "poll-unavailable" });
}
+ const result = route.packageResult(artifacts);
+ return {
+ status: "completed",
+ result: { value: result },
+ };
},
+
};
}
@@ -555,7 +554,6 @@ export function createKieProvider(config: CreateKieProviderOptions) {
const apiBaseUrl = baseUrl(config.apiBaseUrl ?? "https://api.kie.ai", "apiBaseUrl");
const uploadBaseUrl = baseUrl(config.uploadBaseUrl ?? "https://kieai.redpandaai.co", "uploadBaseUrl");
const pollIntervalMs = nonNegativeInteger(config.pollIntervalMs ?? 3_000, "pollIntervalMs");
- const submissionIntervalMs = nonNegativeInteger(config.submissionIntervalMs ?? 500, "submissionIntervalMs");
const requestTimeoutMs = positiveInteger(config.requestTimeoutMs ?? 30_000, "requestTimeoutMs");
const maxOperationMs = positiveInteger(config.maxOperationMs ?? 20 * 60_000, "maxOperationMs");
const maxArtifactBytes = positiveInteger(config.maxArtifactBytes ?? 512 * 1024 * 1024, "maxArtifactBytes");
@@ -569,7 +567,6 @@ export function createKieProvider(config: CreateKieProviderOptions) {
});
const providerEndpoint = endpoint({
client,
- gate: new IntervalGate(submissionIntervalMs, now),
pollIntervalMs,
maxOperationMs,
now,
@@ -583,6 +580,7 @@ export function createKieProvider(config: CreateKieProviderOptions) {
credentials: { apiKey: config.apiKey ?? credentialRef("env", "KIE_API_KEY") },
credentialInputs: { apiKey: { label: "KIE API key" } },
defaultConcurrency: config.defaultConcurrency ?? 10,
+ ...(config.actionLimits === undefined ? {} : { actionLimits: config.actionLimits }),
capabilities: kieRoutes.map((route) => ({
capability: route.capability,
returns: route.returns,
diff --git a/packages/provider-kie/test/provider.test.ts b/packages/provider-kie/test/provider.test.ts
index c154b532..5a7c4f4b 100644
--- a/packages/provider-kie/test/provider.test.ts
+++ b/packages/provider-kie/test/provider.test.ts
@@ -35,14 +35,14 @@ async function endpointFor(
request: Need,
fetch: typeof globalThis.fetch,
now = () => 1_000,
+ requestTimeoutMs = 1_000,
): Promise<{ endpoint: AsyncEndpoint; registration: EndpointRegistration }> {
const registry = new EndpointRegistry();
const provider = createKieProvider({
fetch,
now,
pollIntervalMs: 0,
- submissionIntervalMs: 0,
- requestTimeoutMs: 1_000,
+ requestTimeoutMs,
maxOperationMs: 60_000,
});
await provider.install(registry);
@@ -161,10 +161,13 @@ test("KIE uploads referenced resources, polls one task, and persists generated b
assert.deepEqual(started.status === "pending" && started.progress, { phase: "submitted" });
assert.equal(calls.length, 2);
assert.equal(started.status === "pending" && (started.handle as Record<string, unknown>).taskId, "task_seedance_test");
- const completed = await endpoint.poll({
+ const ready = await endpoint.poll({
...common,
handle: started.handle!,
});
+ assert.equal(ready.status, "ready");
+ if (ready.status !== "ready") return;
+ const completed = await endpoint.collect!({ ...common, handle: ready.handle });
assert.equal(completed.status, "completed");
assert.equal(calls.length, 5);
if (completed.status !== "completed") return;
@@ -215,9 +218,12 @@ test("KIE fulfills generic Background Removal with the documented Recraft wire m
const started = await endpoint.start(common);
assert.equal(started.status, "pending");
if (started.status !== "pending") return;
- const completed = await endpoint.poll({
+ const ready = await endpoint.poll({
...common, handle: started.handle!,
});
+ assert.equal(ready.status, "ready");
+ if (ready.status !== "ready") return;
+ const completed = await endpoint.collect!({ ...common, handle: ready.handle });
assert.equal(completed.status, "completed");
if (completed.status !== "completed") return;
assert.equal(completed.result.value.kind, "blob");
@@ -225,29 +231,49 @@ test("KIE fulfills generic Background Removal with the documented Recraft wire m
assert.equal(calls.length, 5);
});
-test("KIE keeps a timed-out task reserved while observing its actual remote end", async () => {
+test("KIE polling errors and operation deadlines are terminal request failures", async () => {
const request = need({});
- let now = 0, unavailable = false, state = "generating", submissions = 0;
- const { endpoint } = await endpointFor(request, async (url) => {
- if (String(url).includes("createTask")) { submissions++; throw new Error("no submission expected"); }
- if (unavailable) throw new Error("offline");
- return Response.json({ code: 200, data: { state } });
+ let now = 0, requests = 0;
+ const { endpoint } = await endpointFor(request, async () => {
+ requests++; throw new Error("offline");
}, () => now);
+ const { kieRouteForCapability } = await import("../src/routes.js");
const common = { command: { kind: "fulfill-need", id: "need:test", need: request } as const,
need: request, resources: new MemoryResourceStore(), credentials: { apiKey: { secret: "test" } }, operation: "op:test",
- handle: { taskId: "existing", routeKey: "seedance-2-mini", startedAt: 0 } };
- // Use the route's actual private handle format, obtained without any external call.
- const { kieRouteForCapability } = await import("../src/routes.js");
- common.handle.routeKey = kieRouteForCapability(request.capability)!.key;
- unavailable = true;
- assert.equal((await endpoint.poll(common)).status, "pending");
+ handle: { taskId: "existing", routeKey: kieRouteForCapability(request.capability)!.key, startedAt: 0 } };
+ const offline = await endpoint.poll(common);
+ assert.equal(offline.status, "failed");
+ assert.match(offline.status === "failed" ? offline.failure.message : "", /offline/);
now = 60_001;
const timedOut = await endpoint.poll(common);
- assert.equal(timedOut.status, "pending");
- assert.equal(timedOut.status === "pending" && timedOut.failure?.code, "KIE_OPERATION_TIMEOUT");
- unavailable = false;
- assert.equal((await endpoint.poll({ ...common, settling: true })).status, "pending");
- state = "success";
- assert.deepEqual(await endpoint.poll({ ...common, settling: true }), { status: "settled" });
- assert.equal(submissions, 0);
+ assert.equal(timedOut.status, "failed");
+ assert.equal(timedOut.status === "failed" && timedOut.failure.code, "KIE_OPERATION_TIMEOUT");
+ assert.equal(requests, 1, "an expired operation does no further network work");
+});
+
+test("KIE request timeout covers a response body that never finishes", async () => {
+ const request = need({});
+ const { endpoint } = await endpointFor(request, async () => {
+ // Some gateways return headers and then leave a body open without reacting
+ // to AbortSignal. The Provider deadline must still release the Build turn.
+ const stream = new ReadableStream<Uint8Array>({ start() {} });
+ return new Response(stream, { status: 200, headers: { "content-type": "application/json" } });
+ }, () => 1_000, 20);
+ const { kieRouteForCapability } = await import("../src/routes.js");
+ const startedAt = Date.now();
+ const outcome = await endpoint.poll({
+ command: { kind: "fulfill-need", id: "need:body-timeout", need: request },
+ need: request,
+ resources: new MemoryResourceStore(),
+ credentials: { apiKey: { secret: "test" } },
+ operation: "op:body-timeout",
+ handle: {
+ taskId: "existing",
+ routeKey: kieRouteForCapability(request.capability)!.key,
+ startedAt: 1_000,
+ },
+ });
+ assert.equal(outcome.status, "failed");
+ assert.equal(outcome.status === "failed" && outcome.failure.code, "KIE_REQUEST_TIMEOUT");
+ assert.ok(Date.now() - startedAt < 500, "poll returns after the configured network timeout");
});
diff --git a/packages/render-hyperframes/test/render.test.ts b/packages/render-hyperframes/test/render.test.ts
index 19965ede..2a9a6a59 100644
--- a/packages/render-hyperframes/test/render.test.ts
+++ b/packages/render-hyperframes/test/render.test.ts
@@ -305,7 +305,7 @@ test("a render Product with another frame domain is rejected by the explicit dow
endpoints,
validators: validatorRegistry(),
}).run(build());
- assert.equal(result.status, "paused");
+ assert.equal(result.status, "failed");
assert.match(result.outcomes.at(-1)?.message ?? "", /different presentation durations/u);
});
diff --git a/packages/runtime-host-node/src/index.ts b/packages/runtime-host-node/src/index.ts
index dd867247..c987f8b4 100644
--- a/packages/runtime-host-node/src/index.ts
+++ b/packages/runtime-host-node/src/index.ts
@@ -37,6 +37,9 @@ export type RuntimeHostBuildSubmission = RuntimeHostActiveBuildSubmission | Runt
export type BuildActivity = "submitting" | "ready" | "running" | "waiting" | "saving-result";
export type BuildOperationView = {
+ readonly id?: string;
+ readonly receipt?: import("@hypit/runtime").OperationReceipt;
+ readonly wakeAt?: number;
readonly endpoint: string;
readonly status: OperationSnapshot["status"];
readonly progress?: OperationSnapshot["progress"];
diff --git a/packages/runtime-kit/src/index.ts b/packages/runtime-kit/src/index.ts
index ce97edf7..1ac64635 100644
--- a/packages/runtime-kit/src/index.ts
+++ b/packages/runtime-kit/src/index.ts
@@ -1,3 +1,4 @@
+import { endpointActions } from "@hypit/endpoint-kit";
import type { EndpointPackage } from "@hypit/endpoint-kit";
import type { HostFacet } from "@hypit/host";
import type { CanonicalValue } from "@hypit/protocol";
@@ -280,3 +281,27 @@ export function runtimeConfigCredentialRef(
assert(store !== undefined && key !== undefined, `${subject} requires store and key`);
return credentialRef(store, key);
}
+
+/** Common Profile representation for short action occupancy and rate budgets. */
+export function runtimeConfigActionLimits(value: CanonicalValue | undefined): import("@hypit/endpoint-kit").EndpointActionLimits | undefined {
+ if (value === undefined) return undefined;
+ const actions = runtimeConfigObject(value, "actionLimits");
+ runtimeConfigExact(actions, endpointActions, "actionLimits");
+ return Object.fromEntries(Object.entries(actions).map(([action, raw]) => {
+ const limit = runtimeConfigObject(raw, `actionLimits.${action}`);
+ runtimeConfigExact(limit, ["concurrency", "rate"], `actionLimits.${action}`);
+ const concurrency = runtimeConfigPositiveInteger(limit.concurrency, `${action} concurrency`);
+ let rate: { limit: number; periodMs: number } | undefined;
+ if (limit.rate !== undefined) {
+ const item = runtimeConfigObject(limit.rate, `${action} rate`);
+ runtimeConfigExact(item, ["limit", "periodMs"], `${action} rate`);
+ const units = runtimeConfigPositiveInteger(item.limit, `${action} rate.limit`);
+ const periodMs = runtimeConfigPositiveInteger(item.periodMs, `${action} rate.periodMs`);
+ if (units === undefined || periodMs === undefined) throw new Error(`${action} rate requires limit and periodMs`);
+ rate = { limit: units, periodMs };
+ }
+ return [action, { ...(concurrency === undefined ? {} : { concurrency }), ...(rate === undefined ? {} : { rate }) }];
+ }));
+}
+
+export { requestDeadline } from "./request-deadline.js";
diff --git a/packages/runtime-kit/src/request-deadline.ts b/packages/runtime-kit/src/request-deadline.ts
new file mode 100644
index 00000000..502ded46
--- /dev/null
+++ b/packages/runtime-kit/src/request-deadline.ts
@@ -0,0 +1,17 @@
+/** A deadline covers headers and body consumption even when a transport ignores abort. */
+export function requestDeadline(timeoutMs: number, error: () => Error = () => new Error("Request timed out")) {
+ if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1) throw new Error("Request timeout must be a positive integer");
+ const controller = new AbortController();
+ let reject!: (reason: Error) => void;
+ const expired = new Promise<never>((_resolve, fail) => { reject = fail; });
+ const timer = setTimeout(() => {
+ const reason = error();
+ reject(reason);
+ controller.abort(reason);
+ }, timeoutMs);
+ return {
+ signal: controller.signal,
+ wait: async <T>(work: Promise<T>): Promise<T> => await Promise.race([work, expired]),
+ finish: (): void => clearTimeout(timer),
+ };
+}
diff --git a/packages/runtime-local/README.md b/packages/runtime-local/README.md
index 5fe230e4..18f1badc 100644
--- a/packages/runtime-local/README.md
+++ b/packages/runtime-local/README.md
@@ -62,17 +62,27 @@ Claims may carry `units` (default 1): one render Need can occupy one request slo
slots at once. The same admission rule applies to total Provider capacity, exact models, and local
compute resources. SQLite retains the weighted reservation across Worker turns and restarts. This
coordinates Builds sharing this Runtime's Execution Store; it does not enforce a service's quotas
-across other machines. Request-frequency limits are a separate Provider policy.
-
-Runtime first persists a `stop` request for either user cancellation or execution failure. Its first
-cause and reason are retained; repeated cancellation cannot replace a recorded failure. The Worker
-resumes that stop after a restart without starting new Needs or storing control flags in Operation
-failure codes. It requests cancellation of already submitted Operations once.
-If cancellation is only accepted or unsupported, Runtime polls the same remote work until it ends,
-retaining its capacity and working Resources. A failed Build also settles unfinished Operations before
-writing its final Result and cleaning up. Completed output is never rolled back. An unknown submission
-acknowledgement remains visible as `submission-unknown`; it needs external confirmation, not an automatic
-retry, expiry or forced release.
+across other machines. Short-action limits are described below.
+
+A failed request ends that Build's execution attempt. Runtime saves completed public Outputs and
+non-secret Operation receipts with the failed Result, including any remote status still unknown.
+It accepts results from calls already in progress before finalizing, without starting further work
+or polling unfinished remote jobs.
+It then releases local reservations. Submission timeouts with no receipt remain recorded failures;
+a new Run and Build can request the remaining work. No original-Build reconciliation is required.
+An explicit cancellation stops new work and makes one best-effort remote cancellation call when
+supported; its acknowledgement is recorded separately from the local cancelled outcome.
+
+`defaultConcurrency` and exact-model limits govern managed Need occupancy. Asynchronous Providers can
+also expose `actionLimits` for `submit`, `poll` and `collect`; each action supports `concurrency` and
+`rate: { limit, periodMs }`. These action budgets share the same Runtime store and real `pool` identity.
+Rate permits replenish with time and are not returned when an action finishes. Rate counts admitted
+actions, not every HTTP request a Provider may make inside one action. Cloud services own their actual
+account-wide limits, including tasks submitted elsewhere or still running after a local failure.
+
+The Worker yields to sockets and timers between graph reads. Resource waiters are awakened as capacity
+becomes available; known Operations are polled from their own lightweight records. Graph hydration is
+serialized, while network actions and local work remain concurrent under their declared limits.
HypiHub can be the explicitly selected gateway for users without their own service keys. A bound
Provider's authentication, quota or transport error never changes that selection. Separate Kie and
diff --git a/packages/runtime-local/src/actions.ts b/packages/runtime-local/src/actions.ts
new file mode 100644
index 00000000..32ccb54c
--- /dev/null
+++ b/packages/runtime-local/src/actions.ts
@@ -0,0 +1,20 @@
+import type { BuildExecutionStore, RuntimeActionExecutor } from "@hypit/runtime";
+
+/** All short Endpoint actions use the Runtime's existing shared resource authority. */
+export function createActionExecutor(store: BuildExecutionStore): RuntimeActionExecutor {
+ return {
+ async run(request, execute) {
+ const command = `action:${JSON.stringify([request.command, request.action])}`;
+ const acquired = await store.acquireCapacity({
+ build: request.build, command, lifetime: "action", resources: request.resources, now: Date.now(),
+ });
+ if (acquired.status === "blocked") return {
+ status: "deferred",
+ ...(acquired.availableAt === undefined ? {} : { wakeAt: acquired.availableAt }),
+ reason: `${acquired.reason}:${acquired.resource}`,
+ };
+ try { return { status: "completed", value: await execute() }; }
+ finally { await store.releaseCapacity(request.build, command); }
+ },
+ };
+}
diff --git a/packages/runtime-local/src/config.ts b/packages/runtime-local/src/config.ts
index a6fbe0be..7861e893 100644
--- a/packages/runtime-local/src/config.ts
+++ b/packages/runtime-local/src/config.ts
@@ -871,7 +871,7 @@ async function inspectRuntimeConfig(
diagnostics.push({
severity: "error",
code: "RUNTIME_POOL_CONFLICT",
- message: `${conflict.endpointIds.join(", ")} share ${conflict.resource} but size it ${conflict.limits.join(" and ")}; use the same limit for this shared resource or different pools`,
+ message: `${conflict.endpointIds.join(", ")} share ${conflict.resource} but size it ${conflict.settings.join(" and ")}; use the same limit and period for this shared resource or different pools`,
subject: conflict.resource,
});
}
diff --git a/packages/runtime-local/src/control.ts b/packages/runtime-local/src/control.ts
index d9a98c70..9bb68a18 100644
--- a/packages/runtime-local/src/control.ts
+++ b/packages/runtime-local/src/control.ts
@@ -58,7 +58,10 @@ function buildView(input: {
acceptedRecords: input.snapshot?.state.records.length ?? 0,
outstandingCommands: input.snapshot?.state.outstanding.length ?? 0,
operations: input.operations.map((operation) => ({
+ id: operation.id,
endpoint: operation.endpoint,
+ ...(operation.receipt === undefined ? {} : { receipt: operation.receipt }),
+ ...(operation.wakeAt === undefined ? {} : { wakeAt: operation.wakeAt }),
status: operation.status,
...(operation.progress === undefined ? {} : { progress: operation.progress }),
...(operation.failure === undefined ? {} : { failure: operation.failure }),
diff --git a/packages/runtime-local/src/result-writer.ts b/packages/runtime-local/src/result-writer.ts
index 1e5ea284..375d6a9a 100644
--- a/packages/runtime-local/src/result-writer.ts
+++ b/packages/runtime-local/src/result-writer.ts
@@ -87,13 +87,14 @@ export function createLocalResultWriter(
await options.buildStore.append(build, fact);
machine.commit();
};
- for (const receipt of await options.commandExecutionStore.list(build)) {
- if (receipt.status === "completed" && receipt.event !== undefined) await accept(receipt.event);
- }
- for (const operation of await options.operationStore.list({ build })) {
- const event = operationEvent(operation);
- if (event !== undefined) await accept(event);
- }
+ const events = [
+ ...(await options.commandExecutionStore.list(build)).flatMap((receipt) =>
+ receipt.status === "completed" && receipt.event !== undefined ? [receipt.event] : []),
+ ...(await options.operationStore.list({ build })).flatMap((operation) => operationEvent(operation) ?? []),
+ ];
+ // Preserve already validated sibling outputs before a failure closes Core's outstanding work.
+ for (const event of events.filter((event) => event.kind !== "command-failed")) await accept(event);
+ for (const event of events.filter((event) => event.kind === "command-failed")) await accept(event);
const accepted = await options.buildStore.read(build);
assert(accepted !== undefined, `Build ${build} disappeared while accepting stored results`);
return accepted;
@@ -134,7 +135,23 @@ export function createLocalResultWriter(
await syncResult(execution, snapshot.state);
const reopened = await openWriter(execution);
try {
+ const operations = (await options.operationStore.list({ build: execution.build })).map((operation) => {
+ return {
+ ...(operation.request === undefined ? {} : { need: { id: operation.request.need.id, capability: operation.request.need.capability } }),
+ ...(operation.createdAt === undefined ? {} : { createdAt: operation.createdAt }),
+ ...(operation.acknowledgedAt === undefined ? {} : { acknowledgedAt: operation.acknowledgedAt }),
+ ...(operation.endedAt === undefined ? {} : { endedAt: operation.endedAt }),
+ ...(operation.progress === undefined ? {} : { progress: operation.progress }),
+ ...(operation.cancellation === undefined ? {} : { cancellation: operation.cancellation }),
+ operation: operation.id, command: operation.command, endpoint: operation.endpoint, status: operation.status,
+ ...(operation.pool === undefined ? {} : { pool: operation.pool }),
+ ...(operation.credentials === undefined ? {} : { credentials: operation.credentials }),
+ ...(operation.receipt === undefined ? {} : { receipt: operation.receipt }),
+ ...(operation.failure === undefined ? {} : { failure: operation.failure }),
+ };
+ });
await reopened.writer.finish({
+ operations,
outcome: execution.decision.outcome,
...(execution.decision.reason === undefined ? {} : { failure: execution.decision.reason }),
});
@@ -158,9 +175,6 @@ export function createLocalResultWriter(
`Execution ${execution.build} Result writer is not owned by this process`);
let step: "result" | "cleanup" = "result";
try {
- const operations = await options.operationStore.list({ build: execution.build });
- assert(!operations.some((operation) => operation.status === "pending"),
- `Build ${execution.build} still has unsettled external work`);
await finishResult(execution);
step = "cleanup";
await Promise.resolve(options.clearBuildResources?.(execution.build));
diff --git a/packages/runtime-local/src/runtime.ts b/packages/runtime-local/src/runtime.ts
index 49fd7548..d2c22fb4 100644
--- a/packages/runtime-local/src/runtime.ts
+++ b/packages/runtime-local/src/runtime.ts
@@ -1,3 +1,4 @@
+import { createActionExecutor } from "./actions.js";
import { isAbsolute, relative, resolve, sep } from "node:path";
import {
@@ -148,6 +149,7 @@ export async function createLocalRuntime(
...(options.resourceStoreForBuild === undefined ? {} : { resourcesForBuild: options.resourceStoreForBuild }),
credentials: options.credentialStore,
operations: options.operationStore,
+ actions: createActionExecutor(options.executionStore),
validators,
});
const resultWriter = createLocalResultWriter({
diff --git a/packages/runtime-local/src/worker.ts b/packages/runtime-local/src/worker.ts
index 27688aa6..3c2f6b17 100644
--- a/packages/runtime-local/src/worker.ts
+++ b/packages/runtime-local/src/worker.ts
@@ -1,4 +1,5 @@
import { randomUUID } from "node:crypto";
+import { setImmediate } from "node:timers/promises";
import type { BuildState } from "@hypit/protocol";
import type {
ResourceStore,
@@ -12,6 +13,7 @@ import type {
RuntimeWorkerRunOptions,
} from "@hypit/runtime";
import { LocalBuildScheduler } from "@hypit/runtime";
+import { BuildMachine } from "@hypit/core";
type LocalWorkerOptions = {
readonly stores: {
@@ -86,7 +88,9 @@ class CapacityExecutor implements RuntimeCommandExecutor {
): Promise<RuntimeExecutionResult> {
const store = this.#options.stores.executions;
if (descriptor.capacityMode === "asynchronous") {
- return await this.#delegate.executeCommand(state, descriptor, context);
+ return await this.#delegate.executeCommand(state, descriptor, {
+ ...context, releaseOperationCapacity: () => this.#options.stores.execution.releaseCapacity(context.build, descriptor.command.id),
+ });
}
const begun = await store.begin(context.build, descriptor.command.id);
if (!begun.created) {
@@ -120,6 +124,12 @@ class CapacityExecutor implements RuntimeCommandExecutor {
}
}
+ async #releaseAfter(result: RuntimeExecutionResult, build: string, command: string): Promise<boolean> {
+ if (result.status !== "pending") return true;
+ const [operation] = await this.#options.stores.operations.list({ build, command });
+ return operation?.remoteEnded === true || operation?.submission === "queued";
+ }
+
async executeCommand(
state: BuildState,
descriptor: RuntimeRunnableCommand,
@@ -127,22 +137,14 @@ class CapacityExecutor implements RuntimeCommandExecutor {
): Promise<RuntimeExecutionResult> {
const resources = descriptor.resources;
await this.#assertRunning(context.build);
+ if (descriptor.capacityMode === "asynchronous") {
+ const [operation] = await this.#options.stores.operations.list({ build: context.build, command: descriptor.command.id });
+ if (operation?.remoteEnded) return await this.#executeOnce(state, descriptor, context);
+ }
if (resources.length === 0) {
return await this.#executeOnce(state, descriptor, context);
}
- // An asynchronous command keeps its capacity reservation while it is
- // being polled. Do not count that same build/command against itself on a
- // later execution turn; the reservation is intentionally retained until
- // the Operation reaches a terminal state.
- const reservation = (await this.#options.stores.execution.listCapacity())
- .find((item) => item.build === context.build && item.command === descriptor.command.id);
- if (reservation !== undefined) {
- const result = await this.#executeOnce(state, descriptor, context);
- if (result.status !== "pending") {
- await this.#options.stores.execution.releaseCapacity(context.build, descriptor.command.id);
- }
- return result;
- }
+ // acquireCapacity also returns the reservation already held by this command.
const acquired = await this.#options.stores.execution.acquireCapacity({
build: context.build,
command: descriptor.command.id,
@@ -152,13 +154,13 @@ class CapacityExecutor implements RuntimeCommandExecutor {
if (acquired.status === "blocked") {
return {
status: "deferred",
- wakeAt: acquired.availableAt,
+ ...(acquired.availableAt === undefined ? {} : { wakeAt: acquired.availableAt }),
reason: `${acquired.reason}:${acquired.resource}`,
};
}
try {
const result = await this.#executeOnce(state, descriptor, context);
- if (result.status !== "pending") {
+ if (await this.#releaseAfter(result, context.build, descriptor.command.id)) {
await this.#options.stores.execution.releaseCapacity(
acquired.reservation.build,
acquired.reservation.command,
@@ -188,6 +190,7 @@ class DurableLocalWorker {
readonly #options: LocalWorkerOptions;
readonly #executorWithCapacity: CapacityExecutor;
readonly #owner = randomUUID();
+ #buildRead = Promise.resolve();
constructor(executor: RuntimeCommandExecutor, options: LocalWorkerOptions) {
this.#executor = executor;
@@ -195,43 +198,73 @@ class DurableLocalWorker {
this.#executorWithCapacity = new CapacityExecutor(executor, options, this.#owner);
}
- async #finishResult(execution: BuildExecutionSnapshot, outcome: import("@hypit/runtime").BuildOutcome): Promise<WorkerTurnResult> {
+ async #readBuild(build: string): Promise<import("@hypit/runtime").BuildSnapshot | undefined> {
+ const previous = this.#buildRead;
+ let release!: () => void;
+ this.#buildRead = new Promise<void>((resolve) => { release = resolve; });
+ await previous;
+ try {
+ // Hydrating a persisted Definition + Facts performs substantial JSON parsing on
+ // Node's one JavaScript thread. Serializing only this hydration avoids a thundering
+ // memory spike; work started from the snapshots remains concurrent under its own
+ // declared capacity.
+ return await this.#options.stores.builds.read(build);
+ } finally {
+ release();
+ }
+ }
+
+ async #finishResult(execution: BuildExecutionSnapshot, outcome: import("@hypit/runtime").BuildOutcome, reason?: string): Promise<WorkerTurnResult> {
await this.#options.stores.execution.releaseBuildCapacity(execution.build);
- const decided = await this.#options.stores.execution.decide(execution.build, this.#owner, outcome);
+ const decided = await this.#options.stores.execution.decide(execution.build, this.#owner, outcome, reason);
return await this.#options.resultWriter.completeResult(decided);
}
- async #settle(execution: BuildExecutionSnapshot): Promise<WorkerTurnResult> {
+ async #finishStopped(execution: BuildExecutionSnapshot): Promise<WorkerTurnResult> {
assert(execution.stop !== undefined, `Build ${execution.build} has no stop request`);
- const snapshot = await this.#options.stores.builds.read(execution.build);
- assert(snapshot !== undefined, `Build ${execution.build} has no durable execution state`);
- for (const operation of await this.#options.stores.operations.list({ build: execution.build })) {
- if (operation.status !== "pending") continue;
- const settled = await this.#executor.cancelOperation?.(snapshot.state, operation).catch(() => undefined);
- if (settled !== undefined && settled.status !== "pending") {
- await this.#options.stores.execution.releaseCapacity(execution.build, operation.command);
+ const operations = await this.#options.stores.operations.list({ build: execution.build });
+ const received = operations.filter((operation) => operation.status === "pending" && operation.completion !== undefined);
+ if (received.length > 0 || execution.stop.cause === "user-cancelled") {
+ const snapshot = await this.#readBuild(execution.build);
+ assert(snapshot !== undefined, `Build ${execution.build} has no execution state`);
+ const machine = new BuildMachine(snapshot.definition, snapshot.facts);
+ for (const operation of received) {
+ let event: import("@hypit/protocol").CommandResult | undefined;
+ try {
+ event = await this.#executor.acceptOperation?.(machine.view(), operation);
+ } catch (error) {
+ await this.#options.stores.operations.update(operation.id, {
+ status: "failed", failure: { code: "INVALID_ENDPOINT_RESULT", message: error instanceof Error ? error.message : String(error) },
+ });
+ continue;
+ }
+ if (event === undefined || event.kind === "command-failed") continue;
+ const fact = machine.evaluate(event);
+ if (fact !== undefined) {
+ await this.#options.stores.builds.append(execution.build, fact);
+ machine.commit();
+ }
+ }
+ if (execution.stop.cause === "user-cancelled") {
+ for (const operation of await this.#options.stores.operations.list({ build: execution.build })) {
+ if (operation.status === "pending") await this.#executor.cancelOperation?.(machine.view(), operation);
+ }
}
}
- const pending = (await this.#options.stores.operations.list({ build: execution.build }))
- .filter((operation) => operation.status === "pending");
- if (pending.length > 0) {
- const now = Date.now();
- return await this.#options.stores.execution.releaseTurn(execution.build, this.#owner,
- Math.min(...pending.map((operation) => Math.max(now + 100, operation.wakeAt ?? now + 1_000))), execution.stop);
- }
- return await this.#finishResult(execution, execution.stop.cause === "user-cancelled" ? "cancelled" : "failed");
+ return await this.#finishResult(execution,
+ execution.stop.cause === "user-cancelled" ? "cancelled" : "failed", execution.stop.reason);
}
async #stopFailed(execution: BuildExecutionSnapshot, reason: string): Promise<WorkerTurnResult> {
const stopped = await this.#options.stores.execution.requestStop(execution.build, { cause: "execution-failed", reason });
- return await this.#settle(stopped);
+ return await this.#finishStopped(stopped);
}
async #finish(execution: BuildExecutionSnapshot, result: ScheduledBuildResult): Promise<WorkerTurnResult> {
const current = await this.#options.stores.execution.read(execution.build);
assert(current?.turn?.owner === this.#owner && current.decision === undefined,
`Execution ${execution.build} is no longer owned by this Worker turn`);
- if (current.stop !== undefined) return await this.#settle(current);
+ if (current.stop !== undefined) return await this.#finishStopped(current);
if (result.status === "complete") {
return await this.#finishResult(execution, "complete");
}
@@ -247,15 +280,19 @@ class DurableLocalWorker {
const reason = result.blocked.map((item) => `${item.reason}: ${item.subject}`).join(", ");
return await this.#stopFailed(execution, reason);
}
- const now = Date.now();
const wakeAt = Math.min(...[
- ...pending.map((item) => item.wakeAt ?? now + 1_000),
- ...deferred.map((item) => item.wakeAt ?? now + 1_000),
+ ...pending.flatMap((item) => item.wakeAt === undefined ? [] : [item.wakeAt]),
+ ...deferred.flatMap((item) => item.wakeAt === undefined ? [] : [item.wakeAt]),
]);
+ const operations = await this.#options.stores.operations.list({ build: execution.build });
+ const waiting = operations.filter((item) => item.status === "pending" && item.request !== undefined && item.submission !== "queued");
+ const commands = new Set(waiting.map((item) => item.command));
+ const onlyOperations = this.#executor.prepare(result.state).runnable.every((item) => commands.has(item.command.id));
const released = await this.#options.stores.execution.releaseTurn(
execution.build,
this.#owner,
- Number.isFinite(wakeAt) ? wakeAt : now,
+ Number.isFinite(wakeAt) ? wakeAt : undefined,
+ onlyOperations && waiting.length > 0 ? waiting.map((item) => item.id) : undefined,
);
return released;
}
@@ -265,23 +302,56 @@ class DurableLocalWorker {
return await this.#stopFailed(execution, reason);
}
- async #runClaimed(executions: readonly BuildExecutionSnapshot[]): Promise<readonly WorkerTurnResult[]> {
+ async #runClaimed(
+ executions: readonly BuildExecutionSnapshot[],
+ hydrated?: () => void,
+ ): Promise<readonly WorkerTurnResult[]> {
const finished: WorkerTurnResult[] = [];
const runnable: { readonly execution: BuildExecutionSnapshot; readonly snapshot: import("@hypit/runtime").BuildSnapshot }[] = [];
for (const execution of executions) {
assert(execution.turn?.owner === this.#owner && execution.decision === undefined,
`Execution ${execution.build} was not claimed by this Worker turn`);
if (execution.stop !== undefined) {
- finished.push(await this.#settle(execution));
+ finished.push(await this.#finishStopped(execution));
continue;
}
try {
+ if (execution.operationWait !== undefined && this.#executor.advanceOperation !== undefined) {
+ const pending = await Promise.all(execution.operationWait.map((id) => this.#options.stores.operations.read(id)));
+ // Only snapshot hydration is serialized; network actions can overlap across Builds.
+ hydrated?.();
+ const actions = await Promise.allSettled(pending.map(async (operation) => {
+ if (operation === undefined) throw new Error(`Build ${execution.build} lost a waiting Operation`);
+ const result = await this.#executor.advanceOperation!(operation);
+ if (result.remoteEnded || result.status !== "pending") {
+ await this.#options.stores.execution.releaseCapacity(execution.build, result.command);
+ }
+ return result;
+ }));
+ const advanced = actions.map((action) => {
+ if (action.status === "rejected") throw action.reason;
+ return action.value;
+ });
+ const failed = advanced.find((item) => item.status === "failed");
+ if (failed !== undefined) {
+ finished.push(await this.#stopFailed(execution, failed.failure!.message));
+ continue;
+ }
+ if (advanced.every((item) => item.status === "pending" && item.completion === undefined)) {
+ const wake = Math.min(...advanced.flatMap((item) => item.wakeAt === undefined ? [] : [item.wakeAt]));
+ finished.push(await this.#options.stores.execution.releaseTurn(execution.build, this.#owner,
+ Number.isFinite(wake) ? wake : undefined, execution.operationWait));
+ continue;
+ }
+ }
await this.#options.installComponentPackages(execution.componentPackages);
- const snapshot = await this.#options.stores.builds.read(execution.build);
+ const snapshot = await this.#readBuild(execution.build);
assert(snapshot !== undefined, `Execution ${execution.build} has no Build Definition`);
runnable.push({ execution, snapshot });
} catch (error) {
finished.push(await this.#fail(execution, error));
+ } finally {
+ hydrated?.();
}
}
if (runnable.length === 0) return finished;
@@ -324,6 +394,7 @@ class DurableLocalWorker {
// The Runtime Host admits one Worker process. Any stored turn owner therefore belongs to the
// previous process; clearing it changes no external fact and performs no Result action.
await this.#options.assertEnvironment?.();
+ await this.#options.stores.execution.reclaimActionCapacity();
await this.#options.stores.execution.reclaimTurns(Date.now());
await this.#options.stores.execution.reclaimResultWrites();
for (const execution of await this.#options.stores.execution.list()) {
@@ -338,21 +409,31 @@ class DurableLocalWorker {
try {
while (options.signal?.aborted !== true) {
await this.#options.assertEnvironment?.();
- while (true) {
+ while (!Boolean(options.signal?.aborted)) {
const execution = await this.#options.stores.execution.claim(this.#owner, Date.now());
if (execution === undefined) break;
+ let markHydrated!: () => void;
+ const hydration = new Promise<void>((resolve) => { markHydrated = resolve; });
let task: Promise<void>;
- task = this.#runClaimed([execution])
+ task = this.#runClaimed([execution], markHydrated)
.then(() => undefined)
.finally(() => active.delete(task));
active.add(task);
+ // The next Build may execute concurrently, but its persisted snapshot is
+ // not hydrated until this one has left the single-threaded JSON boundary.
+ await Promise.race([hydration, task]);
+ // Awaiting synchronous stores only drains microtasks. Let sockets, timers and
+ // stop signals run before claiming another piece of ready work.
+ await setImmediate();
}
- await Promise.race([
- ...active,
- pause(options.idlePollMs, options.signal),
- ]).catch((error: unknown) => {
- if (options.signal?.aborted !== true) throw error;
- });
+ const idle = new AbortController();
+ const signal = options.signal === undefined ? idle.signal : AbortSignal.any([idle.signal, options.signal]);
+ try {
+ await Promise.race([...active, pause(options.idlePollMs, signal)]);
+ } catch (error) {
+ if (!Boolean(options.signal?.aborted)) throw error;
+ } finally { idle.abort(); }
+
}
} finally {
await Promise.allSettled(active);
diff --git a/packages/runtime-local/test/config.test.ts b/packages/runtime-local/test/config.test.ts
index 60bad876..9aab1147 100644
--- a/packages/runtime-local/test/config.test.ts
+++ b/packages/runtime-local/test/config.test.ts
@@ -771,8 +771,9 @@ test("doctor reports inconsistent limits for shared pools and capacity resources
for (const resource of ["pool:generation", "capacity:generation/browsers"]) {
const conflict = conflicts.find((item) => item.subject === resource);
assert.ok(conflict, `the shared resource ${resource} is reported`);
- assert.equal(conflict.message,
- `four, ten share ${resource} but size it 4 and 10; use the same limit for this shared resource or different pools`);
+ assert.match(conflict.message, /4 concurrent and 10 concurrent/);
+ assert.match(conflict.message, /same limit.*different pools/);
+ assert.doesNotMatch(conflict.message, /defaultConcurrency/);
}
} finally {
await rm(root, { recursive: true, force: true });
diff --git a/packages/runtime-local/test/local.test.ts b/packages/runtime-local/test/local.test.ts
index 0d8e1fb1..1c960989 100644
--- a/packages/runtime-local/test/local.test.ts
+++ b/packages/runtime-local/test/local.test.ts
@@ -31,6 +31,7 @@ import { SqliteRuntimeState } from "@hypit/store-sqlite";
import {
capabilities,
createGreetingBuild,
+ createParallelGreetingBuild,
manifest as greetingManifest,
producers,
types,
@@ -457,6 +458,49 @@ test("a failed Build keeps public Outputs completed before removing active Runti
}
});
+for (const phase of ["submit", "poll"] as const) {
+ test(`a ${phase} failure retains an already received sibling Output in the final Result`, async () => {
+ const directory = await mkdtemp(join(tmpdir(), "hypit-local-sibling-result-"));
+ const initial = createParallelGreetingBuild();
+ let starts = 0;
+ let polls = 0;
+ const components: ComponentPackage = { producers: [
+ { producer: producers.makePrompt, handler: () => ({ outputs: { prompt: { kind: "inline", value: "hello" } }, needs: {} }) },
+ { producer: producers.requestText, handler: () => ({ outputs: {}, needs: { generation: {} } }) },
+ ] };
+ const finish = async (id: number) => {
+ if (id === 1) return { status: "failed" as const, failure: { code: "EXAMPLE_FAILURE", message: "first request failed" } };
+ await new Promise<void>((resolve) => setTimeout(resolve, 10));
+ return { status: "completed" as const, result: { value: { kind: "inline" as const, value: "retained sibling output" } } };
+ };
+ const endpoint = defineEndpointPackage({
+ module: providerModule, facet: "generation", instance: "generation.siblings", pool: "generation.siblings", defaultConcurrency: 2,
+ capabilities: [{ lifecycle: "asynchronous", capability: capabilities.generation, returns: types.generated, endpoint: {
+ start() {
+ const id = ++starts;
+ return phase === "submit" ? finish(id) : { status: "pending", handle: { id }, receipt: { id: `task-${id}` }, wakeAt: Date.now() };
+ },
+ poll({ handle }) { polls++; return finish((handle as { id: number }).id); },
+ } }],
+ });
+ const runtime = await createLocalRuntime({ ...projectRuntimeFixture(directory), components: [components], endpoints: [endpoint] });
+ try {
+ const id = "bld_20260906T110000000Z_0000000001";
+ await runtime.build(durableBuildRequest(directory, id, initial));
+ assert.equal((await finishClaimedBuild(runtime)).outcome, "failed");
+ const result = await new FileBuildResultRepository(join(directory, "results")).read(id);
+ assert.equal(result?.failure, "first request failed");
+ assert.deepEqual(result?.outputs["target.2"]?.value, { kind: "inline", value: "retained sibling output" });
+ assert.equal(starts, 2);
+ assert.equal(polls, phase === "submit" ? 0 : 2);
+ assert.equal((await runtime.activity()).builds.length, 0);
+ } finally {
+ await runtime.close();
+ await rm(directory, { recursive: true, force: true });
+ }
+ });
+}
+
test("an interrupted Result write finishes explicitly without rerunning the Build", async () => {
const directory = await mkdtemp(join(tmpdir(), "hypit-local-result-retry-"));
const initial = createGreetingBuild({ generationRealization: "placeholder" });
@@ -718,6 +762,9 @@ test("a selected historical file is staged once before its Build becomes active"
test("one local Worker admits later Builds while preserving shared Endpoint capacity", async () => {
const directory = await mkdtemp(join(tmpdir(), "hypit-local-bld_20260902T120000011Z_0000000001uilds-"));
+ const fixture = projectRuntimeFixture(directory);
+ let activeBuildReads = 0;
+ let mostBuildReads = 0;
let unrestrictedActive = 0;
let mostUnrestricted = 0;
let limitedActive = 0;
@@ -783,7 +830,22 @@ test("one local Worker admits later Builds while preserving shared Endpoint capa
});
try {
const runtime = await createLocalRuntime({
- ...projectRuntimeFixture(directory),
+ ...fixture,
+ buildStore: {
+ create: (...args) => fixture.buildStore.create(...args),
+ append: (...args) => fixture.buildStore.append(...args),
+ read: async (...args) => {
+ activeBuildReads += 1;
+ mostBuildReads = Math.max(mostBuildReads, activeBuildReads);
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ try {
+ return await fixture.buildStore.read(...args);
+ } finally {
+ activeBuildReads -= 1;
+ }
+ },
+ remove: (...args) => fixture.buildStore.remove!(...args),
+ },
components: [components],
endpoints: [endpoint],
});
@@ -947,20 +1009,16 @@ test("project local runtime remembers the complete package closure across increm
});
for (const cancellation of ["accepted", "unsupported", "failed-build", "host-failure"] as const) {
- test(`unconfirmed ${cancellation} retains capacity and work until remote termination`, async () => {
+ test(`${cancellation} finishes the Build with receipts without waiting for remote termination`, async () => {
const directory = await mkdtemp(join(tmpdir(), "hypit-settlement-"));
let fixture = projectRuntimeFixture(directory);
let failRead = false;
- let remoteActive = 0, starts = 0, cancels = 0, finished = false;
+ let remoteActive = 0, starts = 0, cancels = 0, polls = 0;
const provider = defineEndpointPackage({ module: providerModule, facet: "generation", instance: "settlement",
pool: "shared", defaultConcurrency: 1, capabilities: [{ capability: capabilities.generation, returns: types.generated,
lifecycle: "asynchronous", endpoint: {
- start() { starts++; remoteActive++; return { status: "pending", handle: { job: starts }, wakeAt: Date.now() + 60_000 }; },
- poll(context) {
- assert.equal(context.settling, true);
- if (!finished) return { status: "pending", handle: context.handle, wakeAt: Date.now() };
- remoteActive--; return { status: "settled" };
- },
+ start() { starts++; remoteActive++; return { status: "pending", handle: { job: starts }, receipt: { id: `job-${starts}` }, wakeAt: Date.now() + 60_000 }; },
+ poll() { polls++; throw new Error("stopped Builds must not poll"); },
cancel() { cancels++; return { status: cancellation === "accepted" ? "accepted" : "unsupported" }; },
} }] });
const openRuntime = () => createLocalRuntime({ ...fixture, endpoints: [provider],
@@ -994,40 +1052,132 @@ for (const cancellation of ["accepted", "unsupported", "failed-build", "host-fai
await fixture.executionStore.claim("test-wake", Date.now() + 60_001);
await fixture.executionStore.releaseTurn(first, "test-wake", Date.now());
} else { await runtime.cancel(first); }
- await runtime.workOnce();
- assert.equal((await fixture.operationStore.list({ build: first }))[0]?.status, "pending");
- assert.equal((await fixture.executionStore.listCapacity()).length, 1);
- assert.equal((await fixture.executionStore.read(first))?.decision, undefined);
- const stopped = (await fixture.executionStore.read(first))!.stop!;
- const isFailure = cancellation === "failed-build" || cancellation === "host-failure";
- assert.equal(stopped.cause, isFailure ? "execution-failed" : "user-cancelled");
- assert.equal((await fixture.operationStore.list({ build: first }))[0]?.failure, undefined,
- "Build stopping must not overwrite the remote Operation's own failure");
- await runtime.cancel(first, "second cancellation must not replace the first stop");
- await runtime.cancel(first, "third cancellation");
- assert.deepEqual((await fixture.executionStore.read(first))?.stop, stopped);
- await runtime.close();
- fixture = projectRuntimeFixture(directory);
- runtime = await openRuntime();
- assert.deepEqual((await fixture.executionStore.read(first))?.stop, stopped);
- assert.deepEqual((await runtime.inspect(first))?.stop, stopped);
- await runtime.build(durableBuildRequest(directory, second, initial));
- await runtime.workOnce();
- assert.equal(starts, 1, "the next Need must not start while the cancelled remote work still occupies its slot");
- assert.equal(remoteActive, 1);
- finished = true;
- await new Promise((resolve) => setTimeout(resolve, 120));
const completion = await runtime.workOnce();
+ const isFailure = cancellation === "failed-build" || cancellation === "host-failure";
assert.ok(completion !== undefined && "outcome" in completion);
assert.equal(completion.outcome, isFailure ? "failed" : "cancelled");
- assert.equal(completion.reason, stopped.reason);
- assert.equal((await fixture.operationStore.list({ build: first })).length, 0);
- assert.equal(remoteActive, 0);
- assert.equal(cancels, 1, "cancel is attempted once across polling turns");
- await new Promise((resolve) => setTimeout(resolve, 260));
+ assert.equal((await fixture.executionStore.listCapacity()).length, 0);
+ assert.equal(await runtime.inspect(first), undefined);
+ const result = await new FileBuildResultRepository(join(directory, "results")).read(first);
+ assert.equal(result?.outcome, completion.outcome);
+ assert.equal(result?.operations?.[0]?.receipt?.id, "job-1");
+ assert.equal(result?.operations?.[0]?.status, isFailure ? "pending" : "cancelled");
+ assert.equal(polls, 0);
+ assert.equal(cancels, isFailure ? 0 : 1);
+ assert.equal(remoteActive, 1, "local termination does not pretend the cloud task stopped");
+ await runtime.build(durableBuildRequest(directory, second, initial));
await runtime.workOnce();
- assert.equal(starts, 2);
- assert.equal(remoteActive, 1);
+ assert.equal(starts, 2, "a new Build can perform its own execution attempt");
} finally { await runtime.close(); await rm(directory, { recursive: true, force: true }); }
});
}
+
+test("100 durable Builds share task and download capacity without spinning on waiting graphs", { timeout: 30_000 }, async (t) => {
+ const directory = await mkdtemp(join(tmpdir(), "hypit-concurrent-builds-"));
+ const fixture = projectRuntimeFixture(directory);
+ let reads = 0, starts = 0, polls = 0, activeTasks = 0, peakTasks = 0, downloads = 0, peakDownloads = 0;
+ const checks = new Map<number, number>();
+ const provider = defineEndpointPackage({
+ module: providerModule, facet: "generation", instance: "concurrent", pool: "shared-account",
+ defaultConcurrency: 8,
+ actionLimits: { submit: { concurrency: 2 }, collect: { concurrency: 2 } },
+ capabilities: [{ capability: capabilities.generation, returns: types.generated, lifecycle: "asynchronous", endpoint: {
+ async start() {
+ const id = starts++;
+ activeTasks++;
+ peakTasks = Math.max(peakTasks, activeTasks);
+ await new Promise((resolve) => setTimeout(resolve, 2));
+ if (id % 10 === 0) {
+ activeTasks--;
+ throw new Error("submission timeout without receipt");
+ }
+ return { status: "pending", handle: { id }, receipt: { id: `remote-${id}` }, wakeAt: Date.now() + 3 };
+ },
+ poll({ handle }) {
+ polls++;
+ const { id } = handle as { id: number };
+ const count = (checks.get(id) ?? 0) + 1;
+ checks.set(id, count);
+ if (id % 10 === 1) {
+ activeTasks--;
+ throw new Error("poll transport failed");
+ }
+ if (count < 10) return { status: "pending", handle, wakeAt: Date.now() + 3 };
+ activeTasks--;
+ return { status: "ready", handle };
+ },
+ async collect({ handle }) {
+ const { id } = handle as { id: number };
+ downloads++;
+ peakDownloads = Math.max(peakDownloads, downloads);
+ try {
+ await new Promise((resolve) => setTimeout(resolve, 40));
+ if (id % 10 === 2) throw new Error("download failed");
+ return { status: "completed", result: { value: { kind: "inline", value: `generated-${id}` } } };
+ } finally { downloads--; }
+ },
+ } }],
+ });
+ const runtime = await createLocalRuntime({
+ ...fixture,
+ buildStore: {
+ ...fixture.buildStore,
+ create: (...args) => fixture.buildStore.create(...args),
+ append: (...args) => fixture.buildStore.append(...args),
+ read: (...args) => { reads++; return fixture.buildStore.read(...args); },
+ },
+ endpoints: [provider],
+ components: [{ producers: [
+ { producer: producers.makePrompt, handler: () => ({ outputs: { prompt: { kind: "inline", value: "hello" } }, needs: {} }) },
+ { producer: producers.requestText, handler: () => ({ outputs: {}, needs: { generation: { prompt: "hello" } } }) },
+ ] }],
+ });
+ const controller = new AbortController();
+ let work: Promise<void> | undefined;
+ try {
+ const ids: string[] = [];
+ for (let index = 0; index < 100; index++) {
+ const id = `bld_20260906T120000000Z_${String(index).padStart(10, "0")}`;
+ ids.push(id);
+ const request = durableBuildRequest(directory, id, createGreetingBuild({ targetOutputs: ["generated"] }));
+ // Persist a sizeable authored input so polling a whole graph has a measurable cost.
+ const program = request.definition.program;
+ await runtime.build({ ...request, definition: { ...request.definition, program: { ...program,
+ records: program.records.map((record) => record.id === "intent:root"
+ ? { ...record, value: { kind: "inline" as const, value: { name: "x".repeat(64 * 1024) } } } : record),
+ } } });
+ }
+ reads = 0;
+ const begun = Date.now();
+ let timerAt = 0;
+ const timer = setTimeout(() => { timerAt = Date.now() - begun; }, 100);
+ const deadline = setTimeout(() => controller.abort(), 20_000);
+ try {
+ work = runtime.work({ idlePollMs: 2, signal: controller.signal });
+ while ((await fixture.executionStore.list()).length > 0 && !controller.signal.aborted) {
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ }
+ controller.abort();
+ await work;
+ } finally { clearTimeout(deadline); clearTimeout(timer); }
+ assert.equal((await fixture.executionStore.list()).length, 0, "every attempt must finish, including failures");
+ assert.equal((await fixture.executionStore.listCapacity()).length, 0);
+ assert.equal(starts, 100, "each Build submits once");
+ assert.ok(peakTasks <= 8 && peakTasks > 1, `remote task concurrency: ${peakTasks}`);
+ assert.ok(peakDownloads === 2, `download concurrency: ${peakDownloads}`);
+ assert.ok(timerAt > 0 && timerAt < 1_000, `100 ms timer ran at ${timerAt} ms`);
+ assert.ok(reads < 800, `waiting Builds should not reload their graphs on every poll: ${reads} reads`);
+ const results = new FileBuildResultRepository(join(directory, "results"));
+ const manifests = await Promise.all(ids.map((id) => results.read(id)));
+ assert.equal(manifests.filter((item) => item?.outcome === "complete").length, 70);
+ assert.equal(manifests.filter((item) => item?.outcome === "failed").length, 30);
+ assert.equal(manifests.filter((item) => item?.operations?.[0]?.receipt !== undefined).length, 90);
+ assert.ok(manifests.filter((item) => item?.outcome === "failed").every((item) => item!.failure !== undefined));
+ t.diagnostic(`100 Builds; ${reads} graph reads; ${polls} polls; task peak ${peakTasks}; download peak ${peakDownloads}; 100 ms timer at ${timerAt} ms; ${Date.now() - begun} ms total`);
+ } finally {
+ controller.abort();
+ await work;
+ await runtime.close();
+ await rm(directory, { recursive: true, force: true });
+ }
+});
diff --git a/packages/runtime/README.md b/packages/runtime/README.md
index 1b1f7620..2f4b4d68 100644
--- a/packages/runtime/README.md
+++ b/packages/runtime/README.md
@@ -21,8 +21,13 @@ The scheduler never changes graph meaning. Targets and candidates are fixed befo
resource limits only control when an already selected command may run.
-A Build's `stop` records `user-cancelled` or `execution-failed` and an optional reason. The first request
-is retained across turns and restarts; repeated requests cannot replace it. It prohibits new work,
-but says nothing about whether an external Operation has ended. Only after those Operations settle
-does the Runtime freeze `decision` and finish the Result. Operation failures keep their own meaning;
-error codes do not carry Build control state.
+A Build's `stop` records `user-cancelled` or `execution-failed` and an optional reason. The first
+cause is retained. Runtime stops launching work, saves the attempt's outcome, completed Outputs and
+available Operation receipts, then releases its local reservations. Pending remote work can remain
+pending in the Result: ending local execution does not assert that a cloud task was cancelled.
+A later attempt is a new Build with explicit Run Candidates for reusable Outputs or retrieved files.
+
+Capacity claims describe occupancy; action rate budgets describe admissions over time. An asynchronous
+Operation holds its declared occupancy while Runtime manages the remote task. Its submit, poll and
+collect actions can have separate concurrency and rate limits. Waiting claims park until a resource
+release or their next eligible time; polling does not require rematerializing the whole Build.
diff --git a/packages/runtime/src/capacity.ts b/packages/runtime/src/capacity.ts
index 72fb62a1..851a417b 100644
--- a/packages/runtime/src/capacity.ts
+++ b/packages/runtime/src/capacity.ts
@@ -1,9 +1,11 @@
-/** Provider capacity belongs to active Runtime execution, not to a Build lifecycle. */
+/** A shared resource: occupancy when periodMs is absent, a replenishing rate budget otherwise. */
export type CapacityResourceClaim = {
readonly id: string;
readonly limit: number;
/** Number of units held by this Command; ordinary requests occupy one. */
readonly units?: number;
+ /** Replenish `limit` units per period, with a burst of at most `limit`. Rate units are not returned. */
+ readonly periodMs?: number;
};
export function capacityUnits(resource: CapacityResourceClaim): number {
@@ -15,6 +17,9 @@ export function capacityUnits(resource: CapacityResourceClaim): number {
if (units > resource.limit) {
throw new Error(`Capacity ${resource.id} request needs ${units} units but its limit is ${resource.limit}`);
}
+ if (resource.periodMs !== undefined && (!Number.isSafeInteger(resource.periodMs) || resource.periodMs < 1)) {
+ throw new Error(`Resource ${resource.id} periodMs must be a positive integer`);
+ }
return units;
}
@@ -24,9 +29,11 @@ export type CapacityReservation = {
readonly command: string;
readonly resources: readonly CapacityResourceClaim[];
readonly createdAt: number;
+ readonly lifetime?: "action" | "operation";
};
export type CapacityAcquireRequest = {
+ readonly lifetime?: "action" | "operation";
readonly build: string;
readonly command: string;
readonly resources: readonly CapacityResourceClaim[];
@@ -37,7 +44,8 @@ export type CapacityAcquire =
| { readonly status: "acquired"; readonly reservation: CapacityReservation }
| {
readonly status: "blocked";
- readonly availableAt: number;
- readonly reason: "resource-in-flight";
+ /** Absent when only a resource release can make this work ready. */
+ readonly availableAt?: number;
+ readonly reason: "resource-in-flight" | "rate-limit";
readonly resource: string;
};
diff --git a/packages/runtime/src/execution.ts b/packages/runtime/src/execution.ts
index cbd8be64..db2a4f1b 100644
--- a/packages/runtime/src/execution.ts
+++ b/packages/runtime/src/execution.ts
@@ -44,11 +44,14 @@ export type BuildResultWriteLease = BuildExecutionTurn;
*
* It has no persisted lifecycle phase. Scheduling is derived from `wakeAt` and `turn`; the one
* immutable execution conclusion is `decision`; `stop` records why new work must stop while
- * existing Operations settle. Operator attention is an independent fact.
+ * the Result is written. Operator attention is an independent fact.
*/
export type BuildExecutionSnapshot = BuildExecutionRequest & {
readonly createdAt: number;
- readonly wakeAt: number;
+ /** Absent when waiting for a resource release. */
+ readonly wakeAt?: number;
+ /** Only these Operations can advance the Build until their next outcome. */
+ readonly operationWait?: readonly string[];
readonly turn?: BuildExecutionTurn;
readonly resultWrite?: BuildResultWriteLease;
readonly stop?: BuildExecutionStop;
@@ -64,7 +67,7 @@ export function buildExecutionActivity(
): BuildExecutionActivity {
if (execution.decision !== undefined) return "saving-result";
if (execution.turn !== undefined) return "running";
- return execution.wakeAt > now ? "waiting" : "ready";
+ return execution.wakeAt === undefined || execution.wakeAt > now ? "waiting" : "ready";
}
/** Ephemeral acknowledgement returned after active Runtime state has been removed. */
@@ -80,8 +83,8 @@ export type BuildExecutionStore = {
claim(owner: string, now?: number): Promise<BuildExecutionSnapshot | undefined>;
/** A new owning Worker clears turns left by the previous process; no external action is repeated here. */
reclaimTurns(now?: number): Promise<readonly string[]>;
- /** An unobserved stop wakes the next turn immediately; a settling turn may wait for its next poll. */
- releaseTurn(build: string, owner: string, wakeAt: number, observedStop?: BuildExecutionStop): Promise<BuildExecutionSnapshot>;
+ /** An unobserved stop wakes the next turn immediately to finish the attempt. */
+ releaseTurn(build: string, owner: string, wakeAt: number | undefined, operationWait?: readonly string[]): Promise<BuildExecutionSnapshot>;
/** Freeze the one conclusion. An execution with a decision can never be claimed again. */
decide(build: string, owner: string, outcome: BuildOutcome, reason?: string): Promise<BuildExecutionSnapshot>;
/** Record the first stop request. Repeated requests cannot replace its cause or reason. */
@@ -94,6 +97,7 @@ export type BuildExecutionStore = {
acquireCapacity(request: CapacityAcquireRequest): Promise<CapacityAcquire>;
releaseCapacity(build: string, command: string): Promise<void>;
releaseBuildCapacity(build: string): Promise<void>;
+ reclaimActionCapacity(): Promise<void>;
listCapacity(): Promise<readonly CapacityReservation[]>;
};
diff --git a/packages/runtime/src/operations.ts b/packages/runtime/src/operations.ts
index 514d2cc7..66809105 100644
--- a/packages/runtime/src/operations.ts
+++ b/packages/runtime/src/operations.ts
@@ -1,7 +1,28 @@
import type {
CanonicalValue,
StoredValue,
+ FulfillNeedCommand,
} from "@hypit/protocol";
+import type { CredentialRef } from "./credentials.js";
+
+/** Provider-authored, non-secret acknowledgement suitable for Results and user inspection. */
+export type OperationReceipt = {
+ readonly id: string;
+ readonly url?: string;
+};
+
+export type OperationFacts = {
+ readonly createdAt?: number;
+ readonly acknowledgedAt?: number;
+ readonly endedAt?: number;
+ /** The exact request needed to advance this Operation without materializing its Build. */
+ readonly request?: FulfillNeedCommand;
+ readonly credentials?: Readonly<Record<string, CredentialRef>>;
+ readonly pool?: string;
+ readonly receipt?: OperationReceipt;
+ readonly remoteEnded?: true;
+ readonly submission?: "queued" | "started" | "accepted";
+};
export type OperationIdentity = {
readonly id: string;
@@ -33,34 +54,33 @@ export type OperationProgress = {
readonly unit?: string;
};
-export type OperationSnapshot = OperationIdentity & {
+export type OperationSnapshot = OperationIdentity & OperationFacts & {
readonly status: OperationUpdate["status"];
readonly handle?: CanonicalValue;
readonly wakeAt?: number;
readonly progress?: OperationProgress;
readonly completion?: OperationCompletion;
readonly failure?: OperationFailure;
- /** A cancel call was attempted; subsequent turns only observe the same external work. */
- readonly cancellationRequested?: true;
+ /** Last acknowledgement of an explicit, best-effort cancellation request. */
+ readonly cancellation?: { readonly outcome: "confirmed" | "accepted" | "unsupported" | "too-late" | "failed"; readonly message?: string };
};
-export type OperationUpdate =
+export type OperationUpdate = OperationFacts & (
| {
readonly status: "pending";
- /** Missing only when submission acknowledgement is unknown; never resubmit automatically. */
+ /** Candidate awaits Build-boundary validation. */
+ readonly completion?: OperationCompletion;
+ /** Provider-owned task state once submission is acknowledged. */
readonly handle?: CanonicalValue;
readonly wakeAt?: number;
readonly progress?: OperationProgress;
- /** Local execution failed, but remote settlement has not yet been confirmed. */
- readonly failure?: OperationFailure;
- readonly cancellationRequested?: true;
}
| {
readonly status: "completed";
readonly completion: OperationCompletion;
}
| { readonly status: "failed"; readonly failure: OperationFailure }
- | { readonly status: "cancelled" };
+ | { readonly status: "cancelled"; readonly cancellation?: OperationSnapshot["cancellation"] });
export type OperationQuery = {
readonly build?: string;
diff --git a/packages/runtime/src/scheduler.ts b/packages/runtime/src/scheduler.ts
index 00ace433..70255914 100644
--- a/packages/runtime/src/scheduler.ts
+++ b/packages/runtime/src/scheduler.ts
@@ -1,5 +1,5 @@
import { BuildMachine, reduce } from "@hypit/core";
-import type { BuildState } from "@hypit/protocol";
+import type { BuildState, CommandResult } from "@hypit/protocol";
import { capacityUnits } from "./capacity.js";
import type {
@@ -19,6 +19,8 @@ type MutableBuild = {
readonly outcomes: SchedulerExecutionOutcome[];
blocked: ScheduledBuildResult["blocked"];
stopped: boolean;
+ failure?: Extract<CommandResult, { readonly kind: "command-failed" }>;
+ readonly attempted: Set<string>;
};
type ActiveResult = {
@@ -90,6 +92,7 @@ export class LocalBuildScheduler {
outcomes: [],
blocked: [],
stopped: false,
+ attempted: new Set(),
});
}
const active = new Map<string, ActiveCommand>();
@@ -107,7 +110,7 @@ export class LocalBuildScheduler {
return proposed;
};
- const accept = async (build: MutableBuild, event: import("@hypit/protocol").CommandResult): Promise<void> => {
+ const accept = async (build: MutableBuild, event: CommandResult): Promise<void> => {
if (this.#buildStore === undefined || build.machine === undefined) {
build.state = reduce(build.state, event);
await this.#onStateChange?.(build.id, build.state);
@@ -135,7 +138,7 @@ export class LocalBuildScheduler {
build.state = prepared.state;
build.blocked = prepared.blocked;
ready.set(build.id, prepared.runnable.filter((item) =>
- !active.has(buildCommandKey(build.id, item.command.id))));
+ !active.has(buildCommandKey(build.id, item.command.id)) && !build.attempted.has(item.command.id)));
}
return ready;
};
@@ -191,9 +194,6 @@ export class LocalBuildScheduler {
const settled = await Promise.race([...active.values()].map((item) => item.promise));
active.delete(settled.key);
const build = settled.build;
- // A command may finish after a concurrently running sibling has already made the
- // Build terminal. Its capacity has been released, but no later completion, failure or
- // deferral belongs to Core's outstanding commands or may replace the first outcome.
if (build.state.status === "failed" || build.state.status === "complete") continue;
if (settled.execution === undefined) {
build.stopped = true;
@@ -207,7 +207,7 @@ export class LocalBuildScheduler {
continue;
}
if (settled.execution.status === "pending") {
- build.stopped = true;
+ build.attempted.add(settled.command.command.id);
build.outcomes.push({
command: settled.command.command.id,
kind: settled.command.command.kind,
@@ -219,20 +219,27 @@ export class LocalBuildScheduler {
continue;
}
if (settled.execution.status === "deferred") {
- build.stopped = true;
+ build.attempted.add(settled.command.command.id);
build.outcomes.push({
command: settled.command.command.id,
kind: settled.command.command.kind,
resources: settled.command.resources.map((resource) => resource.id),
status: "deferred",
- wakeAt: settled.execution.wakeAt,
+ ...(settled.execution.wakeAt === undefined ? {} : { wakeAt: settled.execution.wakeAt }),
message: settled.execution.reason,
});
continue;
}
const event = settled.execution.event;
try {
- await accept(build, event);
+ if (event.kind === "command-failed") {
+ // Stop launching work immediately, but accept results of calls already running
+ // before the failure clears Core's outstanding commands. No pending job is polled.
+ build.stopped = true;
+ build.failure ??= event;
+ } else {
+ await accept(build, event);
+ }
build.outcomes.push({
command: settled.command.command.id,
kind: settled.command.command.kind,
@@ -251,6 +258,10 @@ export class LocalBuildScheduler {
}
}
+ for (const build of builds) {
+ if (build.failure !== undefined) await accept(build, build.failure);
+ }
+
} catch (error) {
// Do not let a classification/configuration error release capacity beneath other active calls.
await Promise.allSettled([...active.values()].map((item) => item.promise));
diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts
index fb3c38ec..3336740e 100644
--- a/packages/runtime/src/types.ts
+++ b/packages/runtime/src/types.ts
@@ -41,12 +41,24 @@ export type RuntimePreparation = {
export type RuntimeExecutionContext = {
/** Stable Run-local identity; two identical BuildRequests may still be distinct Builds. */
readonly build: string;
+ /** Release only the asynchronous Operation's occupancy after its remote end is confirmed. */
+ readonly releaseOperationCapacity?: () => Promise<void>;
+};
+
+export type RuntimeActionResult<T> =
+ | { readonly status: "completed"; readonly value: T }
+ | Extract<RuntimeExecutionResult, { readonly status: "deferred" }>;
+
+/** One short action; occupancy ends with the call, while rate consumption remains. */
+export type RuntimeActionExecutor = {
+ run<T>(request: { readonly build: string; readonly command: string; readonly action: string;
+ readonly resources: readonly RuntimeResourceClaim[] }, execute: () => Promise<T>): Promise<RuntimeActionResult<T>>;
};
export type RuntimeExecutionResult =
| { readonly status: "completed"; readonly event: CommandResult }
| { readonly status: "pending"; readonly operation: string; readonly wakeAt?: number }
- | { readonly status: "deferred"; readonly wakeAt: number; readonly reason: string };
+ | { readonly status: "deferred"; readonly wakeAt?: number; readonly reason: string };
/** Minimal execution port used by a Scheduler. `prepare` is the sole command-generation boundary. */
export type RuntimeCommandExecutor = {
@@ -60,6 +72,9 @@ export type RuntimeCommandExecutor = {
state: BuildState,
operation: OperationSnapshot,
): Promise<OperationSnapshot>;
+ advanceOperation?(operation: OperationSnapshot): Promise<OperationSnapshot>;
+ /** Validate an already received result without invoking a Producer or contacting an Endpoint. */
+ acceptOperation?(state: BuildState, operation: OperationSnapshot): Promise<CommandResult | undefined>;
};
/** Build-local byte resources. Location, retention and transport are Runtime policy. */
diff --git a/packages/store-sqlite/README.md b/packages/store-sqlite/README.md
index a4bbab23..25050874 100644
--- a/packages/store-sqlite/README.md
+++ b/packages/store-sqlite/README.md
@@ -3,9 +3,12 @@
Private durable state used by `@hypit/runtime-local`.
One SQLite file stores Build facts, external Operations, the active Build work set, capacity reservations
-and a small output catalog. No ready-command queue is stored: Core derives ready work from the saved
+a small output catalog, resource waiters and action rate budgets. No ready-command queue is stored: Core derives ready work from the saved
Build facts whenever the Worker advances it.
-Artifacts and credentials never enter this database. Components and Endpoint packages do not import
+Operation receipts retain task IDs, original Need identity and credential references until the Result
+has recorded them. Unknown remote outcomes do not keep a failed Build active.
+
+Artifact bytes and secret credential values never enter this database. Components and Endpoint packages do not import
SQLite. `SqliteRuntimeState` exposes the narrow stores needed by the local Runtime and owns their shared
connection.
diff --git a/packages/store-sqlite/src/store.ts b/packages/store-sqlite/src/store.ts
index 55b504de..8fd3b001 100644
--- a/packages/store-sqlite/src/store.ts
+++ b/packages/store-sqlite/src/store.ts
@@ -24,6 +24,7 @@ import type {
BuildSnapshot,
BuildStore,
CapacityAcquire,
+ CapacityResourceClaim,
CapacityAcquireRequest,
CapacityReservation,
CommandExecutionBegin,
@@ -98,36 +99,18 @@ function parseOperationSnapshot(row: Row): OperationSnapshot {
}
assert(typeof row.status === "string", "SQLite Operation row has no status");
const payload = typeof row.payload_json === "string" ? JSON.parse(row.payload_json) as unknown : undefined;
- const pending = row.status === "pending" ? payload as {
- readonly handle: unknown;
- readonly wakeAt?: number;
- readonly progress?: OperationSnapshot["progress"];
- readonly failure?: OperationSnapshot["failure"];
- readonly cancellationRequested?: true;
- } : undefined;
- const mutable = row.status === "pending"
- ? {
- handle: pending!.handle,
- ...(pending!.wakeAt === undefined ? {} : { wakeAt: pending!.wakeAt }),
- ...(pending!.progress === undefined ? {} : { progress: pending!.progress }),
- ...(pending!.failure === undefined ? {} : { failure: pending!.failure }),
- ...(pending!.cancellationRequested === undefined ? {} : { cancellationRequested: true as const }),
- }
- : row.status === "completed"
- ? { completion: payload }
- : row.status === "failed"
- ? { failure: payload }
- : {};
return {
- id: row.operation_id,
- build: row.build_id,
- command: row.command_id,
- endpoint: row.endpoint_id,
- status: row.status,
- ...mutable,
+ id: row.operation_id, build: row.build_id, command: row.command_id,
+ endpoint: row.endpoint_id, status: row.status,
+ ...(payload as object ?? {}),
} as OperationSnapshot;
}
+function operationPayload(operation: OperationSnapshot): string {
+ const { id: _id, build: _build, command: _command, endpoint: _endpoint, status: _status, ...facts } = operation;
+ return JSON.stringify(facts);
+}
+
function copy<T>(value: T): T {
return structuredClone(value);
}
@@ -360,17 +343,7 @@ class SqliteOperationStore implements OperationStore {
async create(operation: OperationSnapshot): Promise<OperationSnapshot> {
const identity = operationIdentityFrom(operation);
- const payload = operation.status === "pending"
- ? JSON.stringify({
- handle: operation.handle,
- ...(operation.wakeAt === undefined ? {} : { wakeAt: operation.wakeAt }),
- ...(operation.progress === undefined ? {} : { progress: operation.progress }),
- ...(operation.failure === undefined ? {} : { failure: operation.failure }),
- ...(operation.cancellationRequested === undefined ? {} : { cancellationRequested: true as const }),
- })
- : operation.status === "completed"
- ? JSON.stringify(operation.completion)
- : operation.status === "failed" ? JSON.stringify(operation.failure) : null;
+ const payload = operationPayload(operation);
const result = this.#database.prepare(`
INSERT INTO hypit_operations (
operation_id, build_id, command_id, endpoint_id, status,
@@ -422,39 +395,17 @@ class SqliteOperationStore implements OperationStore {
const current = await this.read(id);
if (current === undefined) throw new Error(`Operation ${id} does not exist`);
if (current.status === "completed" || current.status === "failed" || current.status === "cancelled") return current;
- const payload = update.status === "pending"
- ? JSON.stringify({
- handle: update.handle,
- ...(update.wakeAt === undefined ? {} : { wakeAt: update.wakeAt }),
- ...(update.progress === undefined ? {} : { progress: update.progress }),
- ...(update.failure === undefined ? {} : { failure: update.failure }),
- ...(update.cancellationRequested === undefined ? {} : { cancellationRequested: true as const }),
- })
- : update.status === "completed"
- ? JSON.stringify(update.completion)
- : update.status === "failed" ? JSON.stringify(update.failure) : null;
+ const next = { ...current, ...copy(update) } as OperationSnapshot;
+ // Progress and timers describe the next action; acknowledgement and request
+ // facts survive success/failure until the Result has saved the receipt.
+ if (update.status !== "pending" || update.wakeAt === undefined) delete (next as { wakeAt?: number }).wakeAt;
const result = this.#database.prepare(`
- UPDATE hypit_operations
- SET status = ?, payload_json = ?
- WHERE operation_id = ?
- `).run(update.status, payload, id);
+ UPDATE hypit_operations SET status = ?, payload_json = ? WHERE operation_id = ?
+ `).run(next.status, operationPayload(next), id);
if (result.changes !== 1) throw new Error(`Operation ${id} disappeared during update`);
- const mutable = update.status === "pending"
- ? {
- status: "pending" as const,
- ...(update.handle === undefined ? {} : { handle: copy(update.handle) }),
- ...(update.wakeAt === undefined ? {} : { wakeAt: update.wakeAt }),
- ...(update.progress === undefined ? {} : { progress: copy(update.progress) }),
- ...(update.failure === undefined ? {} : { failure: copy(update.failure) }),
- ...(update.cancellationRequested === undefined ? {} : { cancellationRequested: true as const }),
- }
- : update.status === "completed"
- ? { status: "completed" as const, completion: copy(update.completion) }
- : update.status === "failed"
- ? { status: "failed" as const, failure: copy(update.failure) }
- : { status: "cancelled" as const };
- return { ...operationIdentityFrom(current), ...mutable };
+ return next;
}
+
}
function parseCommandExecution(row: Row): CommandExecutionReceipt {
@@ -570,7 +521,7 @@ class SqliteRuntimeEnvironmentStore implements RuntimeEnvironmentStore {
function parseExecutionSnapshot(row: Row): BuildExecutionSnapshot {
assert(typeof row.created_at === "number", "SQLite Execution creation time is invalid");
- assert(typeof row.wake_at === "number", "SQLite Execution wake time is invalid");
+ assert(row.wake_at === null || typeof row.wake_at === "number", "SQLite Execution wake time is invalid");
assert((row.turn_owner === null && row.turn_acquired_at === null)
|| (typeof row.turn_owner === "string" && typeof row.turn_acquired_at === "number"),
"SQLite Execution turn is invalid");
@@ -587,7 +538,8 @@ function parseExecutionSnapshot(row: Row): BuildExecutionSnapshot {
return {
...parseExecutionRequest(row, "Execution"),
createdAt: row.created_at,
- wakeAt: row.wake_at,
+ ...(typeof row.wake_at === "number" ? { wakeAt: row.wake_at } : {}),
+ ...(typeof row.operation_wait_json === "string" ? { operationWait: JSON.parse(row.operation_wait_json) as string[] } : {}),
...(typeof row.turn_owner === "string" ? {
turn: { owner: row.turn_owner, acquiredAt: row.turn_acquired_at as number },
} : {}),
@@ -661,9 +613,12 @@ class SqliteBuildExecutionStore implements BuildExecutionStore {
nonNegativeInteger(now, "Execution claim time");
return transaction(this.#database, () => {
const row = this.#database.prepare(`
- SELECT build_id FROM hypit_executions
+ SELECT execution.build_id FROM hypit_executions AS execution
WHERE decision_outcome IS NULL AND turn_owner IS NULL AND wake_at <= ?
- ORDER BY wake_at ASC, created_at ASC, build_id ASC
+ ORDER BY EXISTS (
+ SELECT 1 FROM hypit_capacity AS capacity
+ WHERE capacity.build_id = execution.build_id
+ ) DESC, wake_at ASC, created_at ASC, execution.build_id ASC
LIMIT 1
`).get(now) as Row | undefined;
if (row === undefined) return undefined;
@@ -680,19 +635,27 @@ class SqliteBuildExecutionStore implements BuildExecutionStore {
});
}
- async releaseTurn(build: string, owner: string, wakeAt: number, observedStop?: BuildExecutionStop): Promise<BuildExecutionSnapshot> {
- nonNegativeInteger(wakeAt, "Execution wakeAt");
+ async releaseTurn(build: string, owner: string, wakeAt: number | undefined, operationWait?: readonly string[]): Promise<BuildExecutionSnapshot> {
+ if (wakeAt !== undefined) nonNegativeInteger(wakeAt, "Execution wakeAt");
return transaction(this.#database, () => {
const row = this.#database.prepare("SELECT * FROM hypit_executions WHERE build_id = ?").get(build) as Row | undefined;
if (row === undefined) throw new Error(`Execution ${build} does not exist`);
const current = parseExecutionSnapshot(row);
assert(current.decision === undefined, `Execution ${build} already has a decision`);
assert(current.turn?.owner === owner, `Execution ${build} is not owned by ${owner}`);
+ // A release can race this turn. Recheck parked claims in this transaction so
+ // that a notification received before releaseTurn is not lost.
+ const now = Date.now();
+ const readyWait = current.stop === undefined && (this.#database.prepare(
+ "SELECT resources_json FROM hypit_resource_waits WHERE build_id = ?",
+ ).all(build) as Row[]).some((wait) => this.#blocked(JSON.parse(wait.resources_json as string), now) === undefined);
+ const next = current.stop !== undefined || readyWait
+ ? Math.min(wakeAt ?? now, now) : wakeAt;
this.#database.prepare(`
UPDATE hypit_executions
- SET turn_owner = NULL, turn_acquired_at = NULL, wake_at = ?
+ SET turn_owner = NULL, turn_acquired_at = NULL, wake_at = ?, operation_wait_json = ?
WHERE build_id = ? AND turn_owner = ?
- `).run(current.stop !== undefined && observedStop === undefined ? Math.min(wakeAt, Date.now()) : wakeAt, build, owner);
+ `).run(next ?? null, readyWait || operationWait === undefined ? null : JSON.stringify(operationWait), build, owner);
return parseExecutionSnapshot(
this.#database.prepare("SELECT * FROM hypit_executions WHERE build_id = ?").get(build) as Row,
);
@@ -827,63 +790,116 @@ class SqliteBuildExecutionStore implements BuildExecutionStore {
});
}
+ #rate(resource: CapacityResourceClaim, now: number): number {
+ const row = this.#database.prepare("SELECT * FROM hypit_rates WHERE resource_id = ?").get(resource.id) as Row | undefined;
+ if (row === undefined) return resource.limit;
+ assert(row.limit_units === resource.limit && row.period_ms === resource.periodMs,
+ `Shared rate ${resource.id} has conflicting limit or periodMs`);
+ return Math.min(resource.limit, (row.tokens as number)
+ + Math.max(0, now - (row.updated_at as number)) * resource.limit / resource.periodMs!);
+ }
+
+ #blocked(resources: readonly CapacityResourceClaim[], now: number): Extract<CapacityAcquire, { status: "blocked" }> | undefined {
+ for (const resource of resources) {
+ const units = capacityUnits(resource);
+ if (resource.periodMs !== undefined) {
+ const tokens = this.#rate(resource, now);
+ if (tokens + 1e-9 < units) return {
+ status: "blocked", availableAt: now + Math.ceil((units - tokens) * resource.periodMs / resource.limit),
+ reason: "rate-limit", resource: resource.id,
+ };
+ } else {
+ const count = this.#database.prepare(`
+ SELECT COALESCE(SUM(COALESCE(json_extract(claim.value, '$.units'), 1)), 0) AS count
+ FROM hypit_capacity AS capacity, json_each(capacity.resources_json) AS claim
+ WHERE json_extract(claim.value, '$.id') = ?
+ `).get(resource.id) as Row;
+ if ((count.count as number) + units > resource.limit) return {
+ status: "blocked", reason: "resource-in-flight", resource: resource.id,
+ };
+ }
+ }
+ return undefined;
+ }
+
async acquireCapacity(request: CapacityAcquireRequest): Promise<CapacityAcquire> {
- nonNegativeInteger(request.now, "Capacity acquisition time");
- assert(request.resources.length > 0, "Capacity resources are empty");
+ nonNegativeInteger(request.now, "Resource acquisition time");
+ assert(request.resources.length > 0, "Resources are empty");
const resources = [...request.resources].sort((left, right) => left.id.localeCompare(right.id));
- assert(new Set(resources.map((resource) => resource.id)).size === resources.length, "Capacity resources repeat an id");
+ assert(new Set(resources.map((resource) => resource.id)).size === resources.length, "Resources repeat an id");
for (const resource of resources) capacityUnits(resource);
- const reservation: CapacityReservation = {
- build: request.build,
- command: request.command,
- resources,
- createdAt: request.now,
- };
return transaction(this.#database, () => {
const existing = this.#database.prepare(
"SELECT * FROM hypit_capacity WHERE build_id = ? AND command_id = ?",
- ).get(reservation.build, reservation.command) as Row | undefined;
- if (existing !== undefined) {
- return { status: "acquired", reservation: parseCapacityReservation(existing) };
+ ).get(request.build, request.command) as Row | undefined;
+ if (existing !== undefined) return { status: "acquired", reservation: parseCapacityReservation(existing) };
+ const blocked = this.#blocked(resources, request.now);
+ if (blocked !== undefined) {
+ this.#database.prepare(`
+ INSERT INTO hypit_resource_waits (build_id, command_id, resource_id, resources_json) VALUES (?, ?, ?, ?)
+ ON CONFLICT(build_id, command_id) DO UPDATE SET resource_id = excluded.resource_id, resources_json = excluded.resources_json
+ `).run(request.build, request.command, blocked.resource, JSON.stringify(resources));
+ return blocked;
}
for (const resource of resources) {
- const count = this.#database.prepare(`
- SELECT COALESCE(SUM(COALESCE(json_extract(claim.value, '$.units'), 1)), 0) AS count
- FROM hypit_capacity AS capacity, json_each(capacity.resources_json) AS claim
- WHERE json_extract(claim.value, '$.id') = ?
- `).get(resource.id) as Row;
- assert(typeof count.count === "number", `SQLite Capacity count for ${resource.id} is invalid`);
- if (count.count + capacityUnits(resource) > resource.limit) {
- return {
- status: "blocked",
- availableAt: request.now + 250,
- reason: "resource-in-flight",
- resource: resource.id,
- };
- }
+ if (resource.periodMs === undefined) continue;
+ this.#database.prepare(`
+ INSERT INTO hypit_rates (resource_id, limit_units, period_ms, tokens, updated_at) VALUES (?, ?, ?, ?, ?)
+ ON CONFLICT(resource_id) DO UPDATE SET tokens = excluded.tokens, updated_at = excluded.updated_at
+ `).run(resource.id, resource.limit, resource.periodMs, this.#rate(resource, request.now) - capacityUnits(resource), request.now);
}
- this.#database.prepare(`
- INSERT INTO hypit_capacity (
- build_id, command_id, resources_json, created_at
- ) VALUES (?, ?, ?, ?)
- `).run(
- reservation.build,
- reservation.command,
- JSON.stringify(reservation.resources),
- reservation.createdAt,
- );
+ const reservation: CapacityReservation = {
+ build: request.build, command: request.command, createdAt: request.now,
+ resources: resources.filter((resource) => resource.periodMs === undefined),
+ lifetime: request.lifetime ?? "operation",
+ };
+ if (reservation.resources.length > 0) this.#database.prepare(`
+ INSERT INTO hypit_capacity (build_id, command_id, resources_json, created_at, lifetime) VALUES (?, ?, ?, ?, ?)
+ `).run(request.build, request.command, JSON.stringify(reservation.resources), request.now, reservation.lifetime!);
+ this.#database.prepare("DELETE FROM hypit_resource_waits WHERE build_id = ? AND command_id = ?").run(request.build, request.command);
+ this.#wakeAvailableWaiter();
return { status: "acquired", reservation };
});
}
+ #release(build: string, command?: string): void {
+ const where = command === undefined ? "build_id = ?" : "build_id = ? AND command_id = ?";
+ const args = command === undefined ? [build] : [build, command];
+ this.#database.prepare(`DELETE FROM hypit_capacity WHERE ${where}`).run(...args);
+ if (command === undefined) this.#database.prepare("DELETE FROM hypit_resource_waits WHERE build_id = ?").run(build);
+ this.#wakeAvailableWaiter();
+ }
+
+ #wakeAvailableWaiter(): void {
+ const now = Date.now();
+ const waits = this.#database.prepare(`
+ SELECT waits.build_id, waits.resources_json FROM hypit_resource_waits AS waits
+ JOIN hypit_executions AS execution ON execution.build_id = waits.build_id
+ WHERE execution.decision_outcome IS NULL AND execution.stop_cause IS NULL
+ AND execution.turn_owner IS NULL AND (execution.wake_at IS NULL OR execution.wake_at > ?)
+ ORDER BY execution.created_at, waits.rowid
+ `).all(now) as Row[];
+ for (const wait of waits) {
+ if (this.#blocked(JSON.parse(wait.resources_json as string), now) !== undefined) continue;
+ this.#database.prepare("UPDATE hypit_executions SET wake_at = ?, operation_wait_json = NULL WHERE build_id = ?")
+ .run(now, wait.build_id as string);
+ break;
+ }
+ }
+
async releaseCapacity(build: string, command: string): Promise<void> {
- this.#database.prepare(
- "DELETE FROM hypit_capacity WHERE build_id = ? AND command_id = ?",
- ).run(build, command);
+ transaction(this.#database, () => this.#release(build, command));
+ }
+
+ async reclaimActionCapacity(): Promise<void> {
+ transaction(this.#database, () => {
+ const rows = this.#database.prepare("SELECT build_id, command_id FROM hypit_capacity WHERE lifetime = 'action'").all() as Row[];
+ for (const row of rows) this.#release(row.build_id as string, row.command_id as string);
+ });
}
async releaseBuildCapacity(build: string): Promise<void> {
- this.#database.prepare("DELETE FROM hypit_capacity WHERE build_id = ?").run(build);
+ transaction(this.#database, () => this.#release(build));
}
async reclaimTurns(now = Date.now()): Promise<readonly string[]> {
@@ -932,6 +948,7 @@ function parseCapacityReservation(row: Row): CapacityReservation {
build: row.build_id,
command: row.command_id,
resources,
+ lifetime: row.lifetime as "action" | "operation",
createdAt: row.created_at,
};
return value;
@@ -1008,7 +1025,8 @@ export class SqliteRuntimeState {
component_packages_json TEXT NOT NULL,
result_location_json TEXT NOT NULL,
created_at INTEGER NOT NULL,
- wake_at INTEGER NOT NULL,
+ wake_at INTEGER,
+ operation_wait_json TEXT,
turn_owner TEXT,
turn_acquired_at INTEGER,
result_writer_owner TEXT,
@@ -1032,8 +1050,18 @@ export class SqliteRuntimeState {
command_id TEXT NOT NULL,
resources_json TEXT NOT NULL,
created_at INTEGER NOT NULL,
+ lifetime TEXT NOT NULL CHECK (lifetime IN ('action', 'operation')),
PRIMARY KEY (build_id, command_id)
) STRICT;
+ CREATE TABLE IF NOT EXISTS hypit_resource_waits (
+ build_id TEXT NOT NULL, command_id TEXT NOT NULL, resource_id TEXT NOT NULL,
+ resources_json TEXT NOT NULL, PRIMARY KEY (build_id, command_id)
+ ) STRICT;
+ CREATE INDEX IF NOT EXISTS hypit_resource_waiters ON hypit_resource_waits (resource_id);
+ CREATE TABLE IF NOT EXISTS hypit_rates (
+ resource_id TEXT PRIMARY KEY, limit_units INTEGER NOT NULL, period_ms INTEGER NOT NULL,
+ tokens REAL NOT NULL, updated_at INTEGER NOT NULL
+ ) STRICT;
CREATE TABLE IF NOT EXISTS hypit_runtime_environment (
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
config_json TEXT NOT NULL
@@ -1055,9 +1083,8 @@ export class SqliteRuntimeState {
if (row === undefined) throw new Error(`Execution ${build} does not exist`);
const current = parseExecutionSnapshot(row);
assert(current.decision !== undefined, `Execution ${build} has no decision`);
- const pending = this.#database.prepare("SELECT operation_id FROM hypit_operations WHERE build_id = ? AND status = 'pending' LIMIT 1").get(build);
- assert(pending === undefined, `Build ${build} still has unsettled external work`);
this.#database.prepare("DELETE FROM hypit_capacity WHERE build_id = ?").run(build);
+ this.#database.prepare("DELETE FROM hypit_resource_waits WHERE build_id = ?").run(build);
this.#database.prepare("DELETE FROM hypit_operations WHERE build_id = ?").run(build);
this.#database.prepare("DELETE FROM hypit_command_executions WHERE build_id = ?").run(build);
this.#database.prepare("DELETE FROM hypit_build_facts WHERE build_id = ?").run(build);
diff --git a/packages/store-sqlite/test/store.test.ts b/packages/store-sqlite/test/store.test.ts
index 48fa0a84..12295818 100644
--- a/packages/store-sqlite/test/store.test.ts
+++ b/packages/store-sqlite/test/store.test.ts
@@ -274,7 +274,6 @@ test("shared capacity resources are acquired atomically across Builds", async ()
});
assert.deepEqual(sameRoute, {
status: "blocked",
- availableAt: 352,
reason: "resource-in-flight",
resource: seedance.id,
});
@@ -335,8 +334,27 @@ test("weighted capacity admits 4 + 2 workers atomically and survives reopening",
} finally { state.close(); await rm(directory, { recursive: true, force: true }); }
});
+test("an execution already holding capacity is claimed before fresh work", async () => {
+ const directory = await mkdtemp(join(tmpdir(), "hypit-capacity-claim-order-"));
+ const state = new SqliteRuntimeState(join(directory, "runtime.sqlite"));
+ try {
+ await state.execution.create({ build: "fresh", componentPackages: [], result: resultLocation }, { now: 100 });
+ await state.execution.create({ build: "in-flight", componentPackages: [], result: resultLocation }, { now: 101 });
+ assert.equal((await state.execution.acquireCapacity({
+ build: "in-flight",
+ command: "remote",
+ resources: [{ id: "pool:generation", limit: 2 }],
+ now: 102,
+ })).status, "acquired");
+ assert.equal((await state.execution.claim("worker", 200))?.build, "in-flight");
+ } finally {
+ state.close();
+ await rm(directory, { recursive: true, force: true });
+ }
+});
-test("stop requests preserve their first cause, wake an unaware turn, and retain polling delays", async () => {
+
+test("stop requests preserve their first cause and promptly wake the final execution turn", async () => {
const directory = await mkdtemp(join(tmpdir(), "hypit-stop-request-"));
const state = new SqliteRuntimeState(join(directory, "runtime.sqlite"));
try {
@@ -347,14 +365,59 @@ test("stop requests preserve their first cause, wake an unaware turn, and retain
assert.deepEqual((await state.execution.requestStop("stopping", { cause: "user-cancelled", reason: "later" })).stop, stop);
const wakeAt = Date.now() + 60_000;
const unaware = await state.execution.releaseTurn("stopping", "worker", wakeAt);
- assert.ok(unaware.wakeAt <= Date.now(), "a stop received during execution must be processed promptly");
+ assert.ok(unaware.wakeAt! <= Date.now(), "a stop received during execution must be processed promptly");
await state.execution.claim("worker");
- const settling = await state.execution.releaseTurn("stopping", "worker", wakeAt, stop);
- assert.equal(settling.wakeAt, wakeAt, "a turn that already handled stop must respect its next poll time");
- await state.execution.requestStop("stopping", { cause: "user-cancelled" });
- assert.equal((await state.execution.read("stopping"))?.wakeAt, wakeAt);
- await state.execution.claim("worker", wakeAt);
assert.deepEqual((await state.execution.decide("stopping", "worker", "cancelled", "later")).decision,
{ outcome: "failed", reason: "original failure" });
} finally { state.close(); await rm(directory, { recursive: true, force: true }); }
});
+
+test("shared action rates spend tokens on admission, independently of occupancy and pool", async () => {
+ const directory = await mkdtemp(join(tmpdir(), "hypit-rates-"));
+ const state = new SqliteRuntimeState(join(directory, "runtime.sqlite"));
+ try {
+ const rate = { id: "rate:account/submit", limit: 2, periodMs: 1_000 };
+ const slot = { id: "action:account/submit", limit: 1 };
+ const acquire = (build: string, now: number) => state.execution.acquireCapacity({ build, command: "submit", resources: [slot, rate], now });
+ assert.equal((await acquire("a", 0)).status, "acquired");
+ assert.equal((await acquire("blocked", 0)).status, "blocked");
+ await state.execution.releaseCapacity("a", "submit");
+ assert.equal((await acquire("b", 0)).status, "acquired", "blocked work spends no rate tokens");
+ await state.execution.releaseCapacity("b", "submit");
+ const limited = await acquire("c", 0);
+ assert.equal(limited.status, "blocked");
+ assert.equal(limited.status === "blocked" && limited.availableAt, 500);
+ assert.equal((await state.execution.acquireCapacity({ build: "other", command: "submit",
+ resources: [{ ...rate, id: "rate:other-account/submit" }], now: 0 })).status, "acquired");
+ assert.equal((await acquire("c", 500)).status, "acquired", "rate replenishes with time, not release");
+ } finally { state.close(); await rm(directory, { recursive: true, force: true }); }
+});
+
+test("a resource release before parking is observed and later releases wake a fitting waiter", async () => {
+ const directory = await mkdtemp(join(tmpdir(), "hypit-capacity-wakeup-"));
+ const state = new SqliteRuntimeState(join(directory, "runtime.sqlite"));
+ const resource = { id: "browsers", limit: 2 };
+ const destination = { root: directory, selection: { use: "@hypit/build-result-fs" } };
+ try {
+ for (const [now, build] of ["holder", "waiter", "later"].entries()) {
+ await state.execution.create({ build, componentPackages: [], result: destination }, { now });
+ }
+ await state.execution.acquireCapacity({ build: "holder", command: "render", resources: [{ ...resource, units: 2 }], now: Date.now() });
+ assert.equal((await state.execution.claim("owner"))?.build, "holder");
+ await state.execution.releaseTurn("holder", "owner", Date.now() + 60_000);
+ assert.equal((await state.execution.claim("owner"))?.build, "waiter");
+ assert.equal((await state.execution.acquireCapacity({ build: "waiter", command: "render", resources: [resource], now: Date.now() })).status, "blocked");
+ await state.execution.releaseCapacity("holder", "render");
+ const parked = await state.execution.releaseTurn("waiter", "owner", undefined);
+ assert.ok(parked.wakeAt !== undefined && parked.wakeAt <= Date.now());
+ assert.equal((await state.execution.acquireCapacity({ build: "waiter", command: "render", resources: [{ ...resource, units: 2 }], now: Date.now() })).status, "acquired");
+ await state.execution.claim("waiter-owner");
+ await state.execution.releaseTurn("waiter", "waiter-owner", Date.now() + 60_000);
+ assert.equal((await state.execution.claim("later-owner"))?.build, "later");
+ assert.equal((await state.execution.acquireCapacity({ build: "later", command: "render", resources: [resource], now: Date.now() })).status, "blocked");
+ await state.execution.releaseTurn("later", "later-owner", undefined);
+ assert.equal(await state.execution.claim("idle"), undefined);
+ await state.execution.releaseCapacity("waiter", "render");
+ assert.equal((await state.execution.claim("next"))?.build, "later");
+ } finally { state.close(); await rm(directory, { recursive: true, force: true }); }
+});