Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

fix: admit concurrent builds to compatible workers

kashorin9903588b26d7
diff --git a/.agents/skills/narratage/references/runtime.md b/.agents/skills/narratage/references/runtime.md
index baaf7d87..2b9e7647 100644
--- a/.agents/skills/narratage/references/runtime.md
+++ b/.agents/skills/narratage/references/runtime.md
@@ -29,7 +29,8 @@ Builds remain archived and neither command cancels remote Provider work.
## Preserve durable semantics
- `runtime up` starts or reuses the detached Worker plus declared external programs. A direct
- `build` also ensures the selected Runtime is running before submission.
+ `build` durably submits first, then ensures a Worker containing every queued Build's implementation
+ packages is running. A Worker skips work whose packages it has not loaded.
- `programs up/status/down` manages external programs only. Do not use it as the normal Build
bootstrap because it does not own the Worker lifecycle.
- A Build continues after durable submission. `--follow` only observes progress; interrupting the
diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts
index 5fa9f43c..0baa153f 100644
--- a/packages/cli/src/main.ts
+++ b/packages/cli/src/main.ts
@@ -1724,10 +1724,15 @@ export async function runCli(
reportProgramProgress,
);
runtime = await loadRuntime(await runtimeHost(args.runtime), loadedPackageSet);
- worker = await controller.worker.up({
- ...(args.maxWaitMs === undefined ? {} : { maxWaitMs: args.maxWaitMs }),
- });
let built = await runtime.build(request);
+ try {
+ worker = await controller.worker.up({
+ ...(args.maxWaitMs === undefined ? {} : { maxWaitMs: args.maxWaitMs }),
+ });
+ } catch (error) {
+ const detail = error instanceof Error ? error.message : String(error);
+ throw new Error(`Build ${built.id} is queued, but the Runtime Worker could not start: ${detail}`);
+ }
if (args.follow && runtime !== undefined) {
built = await observeBuild(runtime, built, {
...(args.maxWaitMs === undefined ? {} : { maxWaitMs: args.maxWaitMs }),
diff --git a/packages/runtime-local/src/config.ts b/packages/runtime-local/src/config.ts
index f2877f43..79a3814a 100644
--- a/packages/runtime-local/src/config.ts
+++ b/packages/runtime-local/src/config.ts
@@ -451,6 +451,7 @@ export async function createRuntimeFromConfig(
...collectNodePackageComponents((options.implementationPackages ?? []).map((item) => item.contribution)),
...(options.components ?? []),
],
+ implementationPackages: (options.implementationPackages ?? []).map((item) => item.specifier),
endpoints,
scheduling: { maxConcurrency: document.concurrency },
close: async () => {
diff --git a/packages/runtime-local/src/control.ts b/packages/runtime-local/src/control.ts
index edc1d1df..fd92cb84 100644
--- a/packages/runtime-local/src/control.ts
+++ b/packages/runtime-local/src/control.ts
@@ -128,6 +128,11 @@ export function createLocalRuntimeControl(
"selected ArtifactStore does not expose explicit retention management");
assert(isEnumerableBuildStore(options.buildStore),
"selected BuildStore does not expose the maintenance index required for Artifact GC");
+ if (gc.apply === true) {
+ const active = await options.dispatchStore.list({ phases: ["queued", "running", "waiting", "blocked"] });
+ assert(active.length === 0,
+ `Artifact GC cannot delete while ${active.length} Build${active.length === 1 ? " is" : "s are"} active`);
+ }
const reachable = new Set<import("@narratage/protocol").Digest>();
for (const snapshot of await options.buildStore.list()) collectArtifactDigests(snapshot.state, reachable);
for (const operation of await options.operationStore.list({})) collectArtifactDigests(operation, reachable);
diff --git a/packages/runtime-local/src/runtime.ts b/packages/runtime-local/src/runtime.ts
index b3e71a82..7062b7ae 100644
--- a/packages/runtime-local/src/runtime.ts
+++ b/packages/runtime-local/src/runtime.ts
@@ -94,6 +94,7 @@ export async function createLocalRuntime(
artifacts: options.artifactStore,
},
scheduling,
+ implementationPackages: [...new Set(options.implementationPackages ?? [])],
});
const credentialControl = createLocalCredentialControl({
credentialStore: options.credentialStore,
diff --git a/packages/runtime-local/src/types.ts b/packages/runtime-local/src/types.ts
index 9ec83444..8d2b91df 100644
--- a/packages/runtime-local/src/types.ts
+++ b/packages/runtime-local/src/types.ts
@@ -34,6 +34,8 @@ export type CreateLocalRuntimeOptions = {
readonly artifactStore: ArtifactStore;
readonly credentialStore: CredentialStore;
readonly components?: readonly ComponentPackage[];
+ /** Package specifiers already loaded into this Worker process. */
+ readonly implementationPackages?: readonly string[];
readonly endpoints?: readonly EndpointPackage[];
readonly scheduling: Omit<BuildSchedulerOptions, "buildStore">;
readonly validators?: LocalTypeValidatorRegistry;
diff --git a/packages/runtime-local/src/worker.ts b/packages/runtime-local/src/worker.ts
index 72a8298c..3c76756e 100644
--- a/packages/runtime-local/src/worker.ts
+++ b/packages/runtime-local/src/worker.ts
@@ -16,6 +16,7 @@ import { LocalBuildScheduler } from "@narratage/runtime";
type LocalWorkerOptions = {
readonly stores: RuntimeExecutionStores;
readonly scheduling: BuildSchedulerOptions;
+ readonly implementationPackages: readonly string[];
};
function assert(condition: unknown, message: string): asserts condition {
@@ -267,7 +268,10 @@ class DurableLocalWorker implements RuntimeWorker {
}
async runOnce(): Promise<BuildDispatchSnapshot | undefined> {
- const dispatch = await this.#options.stores.dispatch.claim();
+ const dispatch = await this.#options.stores.dispatch.claim(
+ Date.now(),
+ this.#options.implementationPackages,
+ );
return dispatch === undefined ? undefined : await this.#runClaimed(dispatch);
}
@@ -282,7 +286,10 @@ class DurableLocalWorker implements RuntimeWorker {
};
while (options.signal?.aborted !== true) {
while (active.size < maxBuilds) {
- const dispatch = await this.#options.stores.dispatch.claim();
+ const dispatch = await this.#options.stores.dispatch.claim(
+ Date.now(),
+ this.#options.implementationPackages,
+ );
if (dispatch === undefined) break;
launch(dispatch);
}
diff --git a/packages/runtime-local/test/local.test.ts b/packages/runtime-local/test/local.test.ts
index cd9ff78d..069bae78 100644
--- a/packages/runtime-local/test/local.test.ts
+++ b/packages/runtime-local/test/local.test.ts
@@ -64,6 +64,12 @@ test("Artifact GC is explicit, dry-run by default, and only removes unreachable
assert.deepEqual(preview.unreachable, [orphan.digest]);
assert.deepEqual(preview.deleted, []);
assert.equal(await artifacts.has(orphan.digest), true);
+ await runtime.build({ id: "active-build", definition: buildDefinition(createGreetingBuild()) });
+ await assert.rejects(
+ async () => await runtime.garbageCollectArtifacts({ apply: true }),
+ /Artifact GC cannot delete while 1 Build is active/u,
+ );
+ await runtime.cancel("active-build");
const applied = await runtime.garbageCollectArtifacts({ apply: true });
assert.deepEqual(applied.deleted, [orphan.digest]);
assert.equal(await artifacts.has(orphan.digest), false);
@@ -304,10 +310,12 @@ test("one local Worker advances independent Builds concurrently under one comman
const runtime = await createLocalRuntime({
...projectRuntimeFixture(directory),
components: [components],
+ implementationPackages: ["@example/parallel-a", "@example/parallel-b"],
});
await runtime.buildMany(["parallel-a", "parallel-b"].map((id) => ({
id,
definition: buildDefinition(createGreetingBuild({ generationRealization: "placeholder" })),
+ implementationPackages: [`@example/${id}`],
})));
const controller = new AbortController();
const work = runtime.work({ idlePollMs: 5, signal: controller.signal });
diff --git a/packages/runtime/src/dispatch.ts b/packages/runtime/src/dispatch.ts
index 271a0d07..1596b07b 100644
--- a/packages/runtime/src/dispatch.ts
+++ b/packages/runtime/src/dispatch.ts
@@ -79,8 +79,8 @@ export type BuildDispatchStore = {
}): Promise<BuildDispatchSnapshot>;
read(build: string): Promise<BuildDispatchSnapshot | undefined>;
list(query?: DispatchQuery): Promise<readonly BuildDispatchSnapshot[]>;
- /** Claim one ready Build for the Runtime's single Worker process. */
- claim(now?: number): Promise<BuildDispatchSnapshot | undefined>;
+ /** Claim one ready Build that this Worker has the implementation packages to execute. */
+ claim(now?: number, implementationPackages?: readonly string[]): Promise<BuildDispatchSnapshot | undefined>;
release(build: string, update: BuildDispatchRelease): Promise<BuildDispatchSnapshot>;
finish(build: string, terminal: DispatchTerminal, reason?: string): Promise<BuildDispatchSnapshot>;
requestCancellation(build: string, reason?: string, now?: number): Promise<BuildDispatchSnapshot>;
diff --git a/packages/store-sqlite/src/store.ts b/packages/store-sqlite/src/store.ts
index ef557b47..cbbb0029 100644
--- a/packages/store-sqlite/src/store.ts
+++ b/packages/store-sqlite/src/store.ts
@@ -411,15 +411,23 @@ class SqliteBuildDispatchStore implements BuildDispatchStore {
.filter((item) => query.phases === undefined || query.phases.includes(item.phase));
}
- async claim(now = Date.now()): Promise<BuildDispatchSnapshot | undefined> {
+ async claim(
+ now = Date.now(),
+ implementationPackages: readonly string[] = [],
+ ): Promise<BuildDispatchSnapshot | undefined> {
nonNegativeInteger(now, "Dispatch claim time");
+ const available = new Set(implementationPackages);
return transaction(this.#database, () => {
- const row = this.#database.prepare(`
- SELECT build_id FROM narratage_dispatches
+ const rows = this.#database.prepare(`
+ SELECT build_id, identity_json FROM narratage_dispatches
WHERE phase IN ('queued', 'waiting', 'blocked') AND available_at <= ?
ORDER BY available_at ASC, created_at ASC, build_id ASC
- LIMIT 1
- `).get(now) as Row | undefined;
+ `).all(now) as Row[];
+ const row = rows.find((candidate) => {
+ assert(typeof candidate.identity_json === "string", "SQLite ready Dispatch has no identity");
+ const identity = JSON.parse(candidate.identity_json) as BuildDispatchIdentity;
+ return identity.implementationPackages.every((item) => available.has(item));
+ });
if (row === undefined) return undefined;
assert(typeof row.build_id === "string", "SQLite ready Dispatch has no Build id");
const updated = this.#database.prepare(`
diff --git a/packages/store-sqlite/test/store.test.ts b/packages/store-sqlite/test/store.test.ts
index d0f2ea52..241ecc55 100644
--- a/packages/store-sqlite/test/store.test.ts
+++ b/packages/store-sqlite/test/store.test.ts
@@ -136,6 +136,27 @@ test("cancelling a never-claimed Build atomically withdraws it from dispatch", a
}
});
+test("a Worker skips ready Builds whose implementation packages it has not loaded", async () => {
+ const directory = await mkdtemp(join(tmpdir(), "narratage-sqlite-package-admission-"));
+ try {
+ const state = new SqliteRuntimeState(join(directory, "runtime.sqlite"));
+ await state.dispatch.create(createBuildDispatchIdentity({
+ build: "a-needs-image",
+ implementationPackages: ["@example/image"],
+ }), { now: 100 });
+ await state.dispatch.create(createBuildDispatchIdentity({
+ build: "b-needs-nothing",
+ }), { now: 100 });
+
+ assert.equal((await state.dispatch.claim(101, []))?.build, "b-needs-nothing");
+ assert.equal(await state.dispatch.claim(102, []), undefined);
+ assert.equal((await state.dispatch.claim(103, ["@example/image"]))?.build, "a-needs-image");
+ state.close();
+ } finally {
+ await rm(directory, { recursive: true, force: true });
+ }
+});
+
test("Pool and Lane limits count only asynchronous Operations still in flight", async () => {
const directory = await mkdtemp(join(tmpdir(), "narratage-sqlite-hierarchy-"));
try {