AI media harness for planning, generating, and editing locally
TypeScript
import { isTauri } from "./persistence";
import { albumCoverGenerator } from "./referenceIconSettings";
import { restoreGeneratedImage, saveImageDraft } from "./imageHistory";
import { compileImagePrompt } from "./imagePrompt";
import { compileImageEdits } from "./imageEditing";
import { imageScenePrompt } from "./imageScene";
import { longShotBlocker } from "./longShot";
import { continuationBlocker, continuationSceneId, generationAssetId, sceneGenerationPending, sceneGenerationSnapshot, sceneTypeFor } from "./project";
import type { ProjectSession } from "./projectSession";
import { sceneLoraBlocker, sceneGenerationRequest, sendBlocker, templateSceneBlocker } from "./sceneGeneration";
import { templateWithSceneLoras, generatorPathFields, loadGeneratorTemplateSettings, templateSupportsMusic, templateUsable } from "./settings";
import { isWorkActive, projectQueueKey, type WorkQueue } from "./workQueue";
export interface SceneGenerateCommand { op: "scene.generate"; scene: string; template: string }
export interface ImageGenerateCommand { op: "image.generate"; image: string; template: string }
export interface MusicGenerateCommand { op: "music.generate"; song: string; template: string }
export interface AlbumCoverGenerateCommand { op: "music.cover.generate"; song: string }
export type AgentGenerateCommand = SceneGenerateCommand | ImageGenerateCommand | MusicGenerateCommand | AlbumCoverGenerateCommand;
export interface GenerationObservation {
scene?: string;
image?: string;
song?: string;
sourceSong?: string;
template?: string;
status: "completed" | "failed" | "cancelled" | "rejected";
workId?: string;
detail: string;
error?: string | null;
needsSave?: boolean;
assetId?: string;
outputRelativePath?: string | null;
}
export function generateAgentMedia(queue: WorkQueue, session: ProjectSession, command: AgentGenerateCommand,
signal: AbortSignal, onProgress: (text: string) => void): Promise<GenerationObservation> {
return command.op === "image.generate" ? generateAgentImage(queue, session, command, signal, onProgress)
: command.op === "music.generate" ? generateAgentMusic(queue, session, command, signal, onProgress)
: command.op === "music.cover.generate" ? generateAgentAlbumCover(queue, session, command, signal, onProgress)
: generateAgentScene(queue, session, command, signal, onProgress);
}
export async function generateAgentAlbumCover(queue: WorkQueue, session: ProjectSession, command: AlbumCoverGenerateCommand,
signal: AbortSignal, onProgress: (text: string) => void): Promise<GenerationObservation> {
signal.throwIfAborted();
const song = session.getSnapshot().config.musicSongs?.find((song) => song.id === command.song);
if (!song) throw new Error(`Song '${command.song}' was not found. Use music.add first.`);
const template = albumCoverGenerator();
const id = queue.enqueueAlbumCover(session, song.id);
if (!id) throw new Error(`Song '${song.title}' already has album cover generation in progress.`);
const cancel = () => { void queue.cancel(id).catch(() => undefined); };
signal.addEventListener("abort", cancel, { once: true });
try {
if (signal.aborted) { cancel(); signal.throwIfAborted(); }
let lastDetail = "";
const item = await queue.waitFor(id, signal, (work) => {
if (lastDetail !== work.detail) { lastDetail = work.detail; onProgress(`${song.title}: ${work.detail}`); }
});
signal.throwIfAborted();
const current = session.getSnapshot().config.musicSongs?.find((song) => song.id === command.song);
return { song: song.id, template: template?.id, workId: id,
status: item.status === "completed" ? "completed" : item.status === "cancelled" ? "cancelled" : "failed",
detail: item.detail, error: item.error, needsSave: item.needsSave,
...(item.status === "completed" || item.needsSave ? { outputRelativePath: current?.coverRelativePath } : {}) };
} finally { signal.removeEventListener("abort", cancel); }
}
export async function generateAgentMusic(queue: WorkQueue, session: ProjectSession, command: MusicGenerateCommand,
signal: AbortSignal, onProgress: (text: string) => void): Promise<GenerationObservation> {
signal.throwIfAborted();
if (!isTauri()) throw new Error("Music generation requires the desktop app.");
const source = session.getSnapshot().config.musicSongs?.find((song) => song.id === command.song);
if (!source) throw new Error(`Song '${command.song}' was not found. Use music.add first.`);
const template = loadGeneratorTemplateSettings().templates.find((item) => item.id === command.template);
if (!template || !templateSupportsMusic(template)) throw new Error("Choose an existing music generator template.");
if (!templateUsable(template) || !generatorPathFields(template).every(({ id, required }) => !required || template.paths[id].trim() || template.sources?.[id]?.length))
throw new Error(`Generator '${template.name}' needs its weights prepared before generation.`);
if (queue.getSnapshot().some((item) => item.projectKey === projectQueueKey(session.record) && item.kind === "music" && item.sceneId === source.id && isWorkActive(item)))
throw new Error(`Song '${source.title}' already has generation in progress.`);
const songId = queue.enqueueMusic(session, source.id, template);
const work = queue.getSnapshot().find((item) => item.projectKey === projectQueueKey(session.record) && item.kind === "music" && item.sceneId === songId && isWorkActive(item));
if (!work) throw new Error("Music could not be queued.");
const cancel = () => { void queue.cancel(work.id).catch(() => undefined); };
signal.addEventListener("abort", cancel, { once: true });
try {
if (signal.aborted) { cancel(); signal.throwIfAborted(); }
let lastDetail = "";
const item = await queue.waitFor(work.id, signal, (progress) => {
if (lastDetail !== progress.detail) { lastDetail = progress.detail; onProgress(`${source.title}: ${progress.detail}`); }
});
signal.throwIfAborted();
const config = session.getSnapshot().config;
const song = config.musicSongs?.find((song) => song.id === songId);
const asset = config.assets.find((asset) => asset.id === song?.assetId);
return { song: songId, sourceSong: source.id, template: template.id, workId: work.id,
status: item.status === "completed" ? "completed" : item.status === "cancelled" ? "cancelled" : "failed",
detail: item.detail, error: item.error, needsSave: item.needsSave,
...(item.status === "completed" || item.needsSave ? { assetId: asset?.id, outputRelativePath: asset?.relativePath } : {}) };
} finally { signal.removeEventListener("abort", cancel); }
}
export async function generateAgentImage(queue: WorkQueue, session: ProjectSession, command: ImageGenerateCommand,
signal: AbortSignal, onProgress: (text: string) => void): Promise<GenerationObservation> {
signal.throwIfAborted();
if (!isTauri()) throw new Error("Image generation requires the desktop app.");
const config = session.getSnapshot().config;
const draft = config.assets.find((asset) => asset.id === command.image && asset.kind === "image" && asset.imageDraft && asset.imageGeneration && !asset.imageGeneration.template);
if (!draft) throw new Error(`Editable image draft '${command.image}' was not found.`);
const template = loadGeneratorTemplateSettings().templates.find((candidate) => candidate.id === command.template);
if (!template) throw new Error(`Generator template '${command.template}' no longer exists.`);
if (template.mode === "animate") throw new Error("Choose a prompt generator for images.");
if (!templateUsable(template)) throw new Error(`Generator '${template.name}' needs its weights or LoRAs prepared before generation.`);
if (queue.getSnapshot().some((item) => item.projectKey === projectQueueKey(session.record) && (item.imageAssetId === draft.id || item.imageDraftId === draft.id) && isWorkActive(item))) {
throw new Error(`Image '${draft.name}' already has generation in progress. Do not submit it again.`);
}
const restored = restoreGeneratedImage(saveImageDraft(config), draft.id);
// Validate before switching the editor or submitting work.
if (!restored.imageScene || !imageScenePrompt(restored.imageScene)) throw new Error("Describe the image or add an object before generating.");
if (restored.imageScene?.rootType === "image") compileImageEdits(restored);
else compileImagePrompt(restored);
session.update(restored);
const id = queue.enqueueImage(session, template);
if (!id) throw new Error("This image is already queued.");
const cancel = () => { void queue.cancel(id).catch(() => undefined); };
signal.addEventListener("abort", cancel, { once: true });
try {
if (signal.aborted) { cancel(); signal.throwIfAborted(); }
let lastDetail = "";
const item = await queue.waitFor(id, signal, (work) => {
const detail = `${draft.name} · ${template.name}: ${work.detail}`;
if (detail !== lastDetail) { lastDetail = detail; onProgress(detail); }
});
signal.throwIfAborted();
const asset = session.getSnapshot().config.assets.find((asset) => asset.id === draft.id);
return { image: draft.id, template: template.id, workId: id,
status: item.status === "completed" ? "completed" : item.status === "cancelled" ? "cancelled" : "failed",
detail: item.detail, error: item.error, needsSave: item.needsSave,
...(item.status === "completed" || item.needsSave ? { assetId: asset?.id, outputRelativePath: asset?.relativePath } : {}),
};
} finally { signal.removeEventListener("abort", cancel); }
}
/** Choose an engine for this run without changing the user's global selection.
* The queue snapshots the inputs and owns the output throughout encode/save. */
export async function generateAgentScene(queue: WorkQueue, session: ProjectSession, command: SceneGenerateCommand,
signal: AbortSignal, onProgress: (text: string) => void): Promise<GenerationObservation> {
signal.throwIfAborted();
if (!isTauri()) throw new Error("Scene generation requires the desktop app.");
const config = session.getSnapshot().config;
const job = config.generationJobs.find((candidate) => candidate.id === command.scene);
if (!job) throw new Error(`Scene '${command.scene}' no longer exists.`);
const template = loadGeneratorTemplateSettings().templates.find((candidate) => candidate.id === command.template);
if (!template) throw new Error(`Generator template '${command.template}' no longer exists.`);
const loraBlocker = sceneLoraBlocker(job, template);
if (loraBlocker) throw new Error(loraBlocker);
if (!templateUsable(templateWithSceneLoras(template, job))) throw new Error(`Generator '${template.name}' needs its weights or LoRAs prepared before generation.`);
if (queue.getSnapshot().some((item) => item.projectKey === projectQueueKey(session.record) && item.sceneId === job.id && isWorkActive(item))) {
throw new Error(`Scene '${job.title}' already has generation in progress. Do not submit it again.`);
}
const source = config.generationJobs.find((candidate) => candidate.id === continuationSceneId(job, config.generationJobs));
const context = job.sceneType === "single-shot" && config.generationJobs.find((candidate) => candidate.id === job.contextSceneId);
const blocker = continuationBlocker(job, config.generationJobs)
?? longShotBlocker(job, config, true, true)
?? (context && !sceneGenerationPending(context) && (!context.latentRelativePath || context.status !== "completed") ? "Generate or queue the Context video first." : null)
?? (source && !sceneGenerationPending(source) && (!source.latentRelativePath || source.status !== "completed") ? "Generate or queue the selected source scene first." : null)
?? templateSceneBlocker(sceneTypeFor(job), template) ?? sendBlocker(job, config.references, config);
if (blocker) throw new Error(blocker);
const request = sceneGenerationRequest(job, config, session.record.folderPath, template.defaultSteps, template);
const [id] = queue.enqueue(session, [{ job, request, snapshot: sceneGenerationSnapshot(job, request) }], template);
if (!id) throw new Error("This scene is already queued.");
// Stop cancels this run and its downstream dependents. Encoding already in
// progress is allowed to finish safely, but the stopped agent will not resume.
const cancel = () => { void queue.cancel(id).catch(() => undefined); };
signal.addEventListener("abort", cancel, { once: true });
try {
if (signal.aborted) { cancel(); signal.throwIfAborted(); }
let lastDetail = "";
const item = await queue.waitFor(id, signal, (work) => {
const detail = `${job.title} · ${template.name}: ${work.detail}`;
if (detail !== lastDetail) { lastDetail = detail; onProgress(detail); }
});
signal.throwIfAborted();
const current = session.getSnapshot().config;
const asset = current.assets.find((asset) => asset.id === generationAssetId(job.id));
return { scene: job.id, template: template.id, workId: id,
status: item.status === "completed" ? "completed" : item.status === "cancelled" ? "cancelled" : "failed",
detail: item.detail, error: item.error, needsSave: item.needsSave,
...(item.status === "completed" || item.needsSave ? { assetId: asset?.id, outputRelativePath: asset?.relativePath } : {}),
};
} finally { signal.removeEventListener("abort", cancel); }
}