AI media harness for planning, generating, and editing locally
TypeScript
import { beforeEach, expect, it, vi } from "vitest";
import { createProjectConfig, type AgentMessage, type ProjectRecord } from "./project";
import { runAgentTurn, type AgentTurnResponse, type AgentTurnResult } from "./runtime";
import { runAgentWorkflow } from "./agentWorkflow";
import type { GenerationObservation } from "./agentGeneration";
vi.mock("./runtime", () => ({ runAgentTurn: vi.fn() }));
beforeEach(() => vi.resetAllMocks());
const project = (): ProjectRecord => ({ folderPath: "D:/Project", config: createProjectConfig({ name: "Original", prompt: "A quiet scene", aspectRatio: "16:9", resolution: "416p", targetDurationSeconds: 10 }) });
const command = { op: "scene.generate" as const, scene: "job-initial-brief", template: "chosen" };
const response = (result: AgentTurnResult): AgentTurnResponse => ({ result, events: [], messages: [] });
const generate = response({ kind: "generation", summary: "Generate this scene", command });
const answer = response({ kind: "answer", content: "Reviewed the finished scene." });
const outcome: GenerationObservation = { scene: command.scene, template: command.template, status: "completed", workId: "work-one", detail: "Video saved" };
const input = (content: string): AgentMessage => ({ id: crypto.randomUUID(), role: "user", content, createdAt: new Date().toISOString() });
it.each([generate, response({ kind: "commands", summary: "Old plan", commands: [{ op: "project.set", name: "Old name" }] })])(
"reconsiders a pending proposal with ordered user updates before returning actions", async (proposal) => {
const record = project();
let current = record;
const pending: AgentMessage[] = [];
const host = { getRecord: () => current, generate: vi.fn() };
vi.mocked(runAgentTurn).mockImplementationOnce(async () => {
pending.push(input("Use a warmer style"), input("Only update the plan; do not generate"));
current = { ...record, config: { ...record.config, name: "Edited while thinking" } };
return proposal;
}).mockResolvedValueOnce(answer);
const result = await runAgentWorkflow(record, "local", "Plan and generate", "turn", [], new AbortController().signal,
host, undefined, "project", () => pending.splice(0));
expect(host.generate).not.toHaveBeenCalled();
expect(runAgentTurn).toHaveBeenCalledTimes(2);
const retry = vi.mocked(runAgentTurn).mock.calls[1];
expect(retry[0].config.name).toBe("Edited while thinking");
expect(retry[2]).toContain('["Use a warmer style","Only update the plan; do not generate"]');
expect(retry[2]).toContain("its commands have not been applied");
expect(result.result).toEqual(answer.result);
expect(result.messages.map(({ content }) => content)).toEqual([
"Plan and generate", "Use a warmer style", "Only update the plan; do not generate", "Reviewed the finished scene.",
]);
},
);
it("includes messages received during generation alongside its result when resuming", async () => {
const record = project();
const pending: AgentMessage[] = [];
const host = { getRecord: () => record, generate: vi.fn(async () => {
pending.push(input("Review the lighting too"), input("Then stop"));
return outcome;
}) };
vi.mocked(runAgentTurn).mockResolvedValueOnce(generate).mockResolvedValueOnce(answer);
await runAgentWorkflow(record, "local", "Generate and review", "turn", [], new AbortController().signal,
host, undefined, "project", () => pending.splice(0));
const resumed = vi.mocked(runAgentTurn).mock.calls[1][2];
expect(resumed).toContain('"status":"completed"');
expect(resumed).toContain('["Review the lighting too","Then stop"]');
expect(host.generate).toHaveBeenCalledOnce();
});
it("prepares and saves image drafts before generation and never reapplies duplicate preparation", async () => {
const record = project();
let prepared = false;
const host = { getRecord: () => record, prepare: vi.fn(async () => { prepared = true; }),
generate: vi.fn(async () => { expect(prepared).toBe(true); return { image: "poster", template: "chosen", status: "completed" as const, detail: "Image saved" }; }) };
const prepare = [{ op: "image.draft.add" as const, id: "poster", name: "Poster", prompt: "A red balloon" }];
const image = response({ kind: "generation", summary: "Prepare and generate Poster", prepare, command: { op: "image.generate", image: "poster", template: "chosen" } });
vi.mocked(runAgentTurn).mockResolvedValueOnce(image).mockResolvedValueOnce(image).mockResolvedValueOnce(answer);
await runAgentWorkflow(record, "local", "Prepare and generate", "turn", [], new AbortController().signal, host);
expect(host.prepare).toHaveBeenCalledExactlyOnceWith(prepare);
expect(host.generate).toHaveBeenCalledOnce();
expect(vi.mocked(runAgentTurn).mock.calls[1][2]).toContain('"image":"poster"');
});
it("reports failed preparation without generating or retrying edits", async () => {
const record = project();
const host = { getRecord: () => record, prepare: vi.fn().mockRejectedValue(new Error("Save failed")), generate: vi.fn() };
const image = response({ kind: "generation", summary: "Prepare", prepare: [{ op: "image.draft.add", id: "poster", name: "Poster", prompt: "A balloon" }],
command: { op: "image.generate", image: "poster", template: "chosen" } });
vi.mocked(runAgentTurn).mockResolvedValueOnce(image).mockResolvedValueOnce(image).mockResolvedValueOnce(answer);
await runAgentWorkflow(record, "local", "Generate", "turn", [], new AbortController().signal, host);
expect(host.generate).not.toHaveBeenCalled();
expect(host.prepare).toHaveBeenCalledOnce();
expect(vi.mocked(runAgentTurn).mock.calls[1][2]).toContain("Save failed");
});
it("waits for the exact generation result, then resumes with fresh edits and clean chat", async () => {
const record = project();
let current = structuredClone(record);
let finish!: (result: GenerationObservation) => void;
const host = { getRecord: () => current, generate: vi.fn(() => new Promise<GenerationObservation>((resolve) => { finish = resolve; })) };
vi.mocked(runAgentTurn).mockResolvedValueOnce(generate).mockResolvedValueOnce(answer);
const pending = runAgentWorkflow(record, "codex", "Generate and review", "turn", [], new AbortController().signal, host);
await vi.waitFor(() => expect(host.generate).toHaveBeenCalledOnce());
expect(runAgentTurn).toHaveBeenCalledOnce();
current = { ...current, config: { ...current.config, name: "User renamed while rendering", assets: [{ id: "asset-result", name: "Rendered", kind: "generated", relativePath: "media/result.mp4", mimeType: "video/mp4", createdAt: current.config.createdAt }] } };
finish(outcome);
const result = await pending;
expect(vi.mocked(runAgentTurn).mock.calls[1][0]).toMatchObject({ config: { name: "User renamed while rendering", assets: [{ id: "asset-result" }] } });
expect(vi.mocked(runAgentTurn).mock.calls[1][2]).toContain('"status":"completed"');
expect(result.messages.map((message) => message.content)).toEqual(["Generate and review", "Reviewed the finished scene."]);
});
it("deduplicates repeated generation commands after resuming", async () => {
const record = project();
const host = { getRecord: () => record, generate: vi.fn().mockResolvedValue(outcome) };
vi.mocked(runAgentTurn).mockResolvedValueOnce(generate).mockResolvedValueOnce(generate).mockResolvedValueOnce(answer);
await runAgentWorkflow(record, "local", "Generate", "turn", [], new AbortController().signal, host);
expect(host.generate).toHaveBeenCalledOnce();
expect(runAgentTurn).toHaveBeenCalledTimes(3);
});
it("prepares music once and resumes with the new song receipt without generating duplicate variations", async () => {
const record = project();
const host = { getRecord: () => record, prepare: vi.fn().mockResolvedValue(undefined), generate: vi.fn().mockResolvedValue({ song: "new-song", sourceSong: "old-song", template: "chosen", status: "completed", assetId: "audio", detail: "Saved" }) };
const music = response({ kind: "generation", summary: "Music", prepare: [{ op: "music.set", id: "old-song", style: "Jazz" }],
command: { op: "music.generate", song: "old-song", template: "chosen" } });
vi.mocked(runAgentTurn).mockResolvedValueOnce(music).mockResolvedValueOnce(music).mockResolvedValueOnce(answer);
await runAgentWorkflow(record, "local", "Generate music", "turn", [], new AbortController().signal, host);
expect(host.generate).toHaveBeenCalledOnce(); expect(host.prepare).toHaveBeenCalledOnce();
expect(vi.mocked(runAgentTurn).mock.calls[1][2]).toContain('"song":"new-song"');
expect(vi.mocked(runAgentTurn).mock.calls[1][2]).toContain("music.analyze");
});
it.each([false, true])("deduplicates album cover requests and resumes with their receipt (rejected: %s)", async (rejected) => {
const record = project();
const host = { getRecord: () => record, prepare: vi.fn().mockResolvedValue(undefined), generate: vi.fn() };
if (rejected) host.generate.mockRejectedValue(new Error("Choose an album cover generator in Settings"));
else host.generate.mockResolvedValue({ song: "song", template: "cover-model", status: "completed", detail: "Album cover saved", outputRelativePath: "media/cover.jpg" });
const cover = response({ kind: "generation", summary: "Cover", prepare: [{ op: "music.add", id: "song", title: "Evening", style: "Jazz" }],
command: { op: "music.cover.generate", song: "song" } });
vi.mocked(runAgentTurn).mockResolvedValueOnce(cover).mockResolvedValueOnce(cover).mockResolvedValueOnce(answer);
await runAgentWorkflow(record, "local", "Generate album cover", "turn", [], new AbortController().signal, host);
expect(host.generate).toHaveBeenCalledOnce(); expect(host.prepare).toHaveBeenCalledOnce();
expect(vi.mocked(runAgentTurn).mock.calls[1][2]).toContain('"song":"song"');
expect(vi.mocked(runAgentTurn).mock.calls[1][2]).toContain(rejected ? '"status":"rejected"' : '"outputRelativePath":"media/cover.jpg"');
});
it.each(["failed", "cancelled"] as const)("returns %s to the agent without automatically generating again", async (status) => {
const record = project();
const host = { getRecord: () => record, generate: vi.fn().mockResolvedValue({ ...outcome, status, error: "Stopped", needsSave: true }) };
vi.mocked(runAgentTurn).mockResolvedValueOnce(generate).mockResolvedValueOnce(answer);
await runAgentWorkflow(record, "local", "Generate", "turn", [], new AbortController().signal, host);
expect(vi.mocked(runAgentTurn).mock.calls[1][2]).toContain(`"status":"${status}"`);
expect(vi.mocked(runAgentTurn).mock.calls[1][2]).toContain('"needsSave":true');
expect(host.generate).toHaveBeenCalledOnce();
});
it("reports startup rejection and does not retry that command", async () => {
const record = project();
const host = { getRecord: () => record, generate: vi.fn().mockRejectedValue(new Error("Template was removed")) };
vi.mocked(runAgentTurn).mockResolvedValueOnce(generate).mockResolvedValueOnce(generate).mockResolvedValueOnce(answer);
await runAgentWorkflow(record, "local", "Generate", "turn", [], new AbortController().signal, host);
expect(vi.mocked(runAgentTurn).mock.calls[1][2]).toContain("Template was removed");
expect(host.generate).toHaveBeenCalledOnce();
});
it("does not resume after the user stops while generation is running", async () => {
const record = project();
const controller = new AbortController();
const host = { getRecord: () => record, generate: vi.fn(async () => { controller.abort(new Error("Stopped")); return outcome; }) };
vi.mocked(runAgentTurn).mockResolvedValue(generate);
await expect(runAgentWorkflow(record, "local", "Generate", "turn", [], controller.signal, host)).rejects.toThrow("Stopped");
expect(runAgentTurn).toHaveBeenCalledOnce();
});
it("does not start generation from a late response to a cancelled turn", async () => {
const record = project();
const controller = new AbortController();
const host = { getRecord: () => record, generate: vi.fn() };
vi.mocked(runAgentTurn).mockImplementation(async () => { controller.abort(new Error("Stopped")); return generate; });
await expect(runAgentWorkflow(record, "local", "Generate", "turn", [], controller.signal, host)).rejects.toThrow("Stopped");
expect(host.generate).not.toHaveBeenCalled();
});
it("bounds a provider that keeps requesting generation", async () => {
const record = project();
const host = { getRecord: () => record, generate: vi.fn().mockResolvedValue(outcome) };
vi.mocked(runAgentTurn).mockResolvedValue(generate);
await expect(runAgentWorkflow(record, "local", "Generate", "turn", [], new AbortController().signal, host)).rejects.toThrow("8 generation requests");
expect(host.generate).toHaveBeenCalledOnce();
expect(runAgentTurn).toHaveBeenCalledTimes(9);
});