AI media harness for planning, generating, and editing locally
TypeScript
// @vitest-environment jsdom
import { invoke } from "@tauri-apps/api/core";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { audioSamplesFromIpc, bytesFromIpc, releaseRendered, renderedSummary, resampleInterleaved, saveGeneratedScene, saveGenerationPreview } from "./generatedVideo";
import { encodeGenerationPreview } from "./generationPreview";
vi.mock("@tauri-apps/api/core", () => ({ invoke: vi.fn() }));
vi.mock("./generationPreview", () => ({ encodeGenerationPreview: vi.fn() }));
const muxed = vi.hoisted(() => ({
configs: [] as Array<Record<string, unknown>>,
audioChunks: 0,
videoChunks: 0,
finalized: false,
}));
vi.mock("mp4-muxer", () => ({
ArrayBufferTarget: class { buffer = new Uint8Array([1, 2, 3, 4]).buffer; },
Muxer: class {
constructor(config: Record<string, unknown>) { muxed.configs.push(config); }
addVideoChunk() { muxed.videoChunks += 1; }
addAudioChunk() { muxed.audioChunks += 1; }
finalize() { muxed.finalized = true; }
},
}));
const invoked = vi.mocked(invoke);
const scope = globalThis as unknown as Record<string, unknown>;
/** A render waiting in Rust's memory, as `generated_summary` reports it. */
const summary = (over: Record<string, unknown> = {}) => ({
jobId: "job-01", width: 1344, height: 768, frameCount: 120, fps: 24,
audioChannels: 2, audioSampleRate: 48_000, audioSamples: 480_000, ...over,
});
const desktop = () => { scope.__TAURI_INTERNALS__ = {}; };
it("writes a cancelled preview as a silent video with its actual dimensions and duration", async () => {
desktop();
const preview = { jobId: "work-preview", step: 3, width: 168, height: 96, fps: 2, frames: ["a", "b", "c"] };
const bytes = new Uint8Array([1, 2, 3, 4]);
vi.mocked(encodeGenerationPreview).mockResolvedValue({ arrayBuffer: async () => bytes.buffer } as Blob);
invoked.mockResolvedValue({ relativePath: "media/generated/work-preview.mp4", bytes: 4 });
await expect(saveGenerationPreview("C:/My Project", "work-preview", preview)).resolves.toEqual({
relativePath: "media/generated/work-preview.mp4", bytes: 4, hasAudio: false, note: null,
width: 168, height: 96, durationMs: 1500,
});
expect(encodeGenerationPreview).toHaveBeenCalledWith(preview, expect.any(AbortSignal));
expect(invoked).toHaveBeenCalledExactlyOnceWith("write_generated_video", bytes, {
headers: { "x-generated-folder": "C%3A%2FMy%20Project", "x-generated-job": "work-preview" },
});
});
it("does not write a file when preview encoding fails", async () => {
desktop();
vi.mocked(encodeGenerationPreview).mockRejectedValue(new Error("Preview encoding is unavailable."));
await expect(saveGenerationPreview("C:/Project", "work-preview", {
jobId: "work-preview", step: 0, width: 2, height: 2, fps: 2, frames: ["jpeg"],
})).rejects.toThrow("Preview encoding is unavailable");
expect(invoked).not.toHaveBeenCalled();
});
beforeEach(() => {
invoked.mockReset().mockResolvedValue(true);
vi.mocked(encodeGenerationPreview).mockReset();
muxed.configs.length = 0;
muxed.audioChunks = 0;
muxed.videoChunks = 0;
muxed.finalized = false;
delete scope.__TAURI_INTERNALS__;
delete scope.VideoEncoder;
delete scope.VideoFrame;
delete scope.AudioEncoder;
delete scope.AudioData;
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null);
});
afterEach(() => {
vi.restoreAllMocks();
delete scope.__TAURI_INTERNALS__;
delete scope.VideoEncoder;
delete scope.VideoFrame;
delete scope.AudioEncoder;
delete scope.AudioData;
});
/* jsdom has no WebCodecs, so nothing here encodes anything — and that is
exactly the half worth pinning in a test. What a save does BEFORE the
encoder is where it can be wrong in ways that cost the user a render: asking
the wrong side of the app, keeping hold of half a gigabyte of frames after a
failure, or reporting a file that was never written. */
describe("saving a rendered scene", () => {
it.each(["configure", "encode"])("closes video resources when %s throws", async (operation) => {
desktop();
const closeFrame = vi.fn();
const closeEncoder = vi.fn();
scope.VideoFrame = class { close = closeFrame; };
scope.VideoEncoder = class {
static isConfigSupported = async () => ({ supported: true });
state = "configured";
configure() { if (operation === "configure") throw new Error("Encoder failed"); }
encode() { throw new Error("Encoder failed"); }
close = closeEncoder;
};
invoked.mockImplementation(async (command: string) => {
if (command === "generated_summary") return summary({ width: 1, height: 1, frameCount: 1, audioSamples: 0 });
if (command === "generated_frame") return [0, 0, 0, 255];
return true;
});
await expect(saveGeneratedScene({ folderPath: "/p", jobId: "job-01" })).rejects.toThrow("Encoder failed");
expect(closeEncoder).toHaveBeenCalledOnce();
expect(closeFrame).toHaveBeenCalledTimes(operation === "encode" ? 1 : 0);
expect(invoked).toHaveBeenCalledWith("release_generated_frames", { jobId: "job-01" });
});
it.each(["configure", "encode"])("closes audio resources when %s throws and saves the video", async (operation) => {
desktop();
const closeData = vi.fn();
const closeEncoder = vi.fn();
scope.AudioData = class { close = closeData; };
scope.AudioEncoder = class {
static isConfigSupported = async () => ({ supported: true });
state = "configured";
configure() { if (operation === "configure") throw new Error("Audio failed"); }
encode() { throw new Error("Audio failed"); }
close = closeEncoder;
};
scope.VideoFrame = class { close() {} };
scope.VideoEncoder = class {
static isConfigSupported = async () => ({ supported: true });
state = "configured";
encodeQueueSize = 0;
constructor(private init: VideoEncoderInit) {}
configure() {}
encode() { this.init.output({} as EncodedVideoChunk); }
async flush() {}
close() {}
};
invoked.mockImplementation(async (command: string) => {
if (command === "generated_summary") return summary({ width: 1, height: 1, frameCount: 1, audioSamples: 2 });
if (command === "generated_frame") return [0, 0, 0, 255];
if (command === "generated_audio") return new Float32Array([0, 0]).buffer;
if (command === "release_generated_frames") expect(muxed.finalized).toBe(false);
if (command === "write_generated_video") return { relativePath: "media/result.mp4", bytes: 4 };
return true;
});
await expect(saveGeneratedScene({ folderPath: "/p", jobId: "job-01" })).resolves.toMatchObject({ note: "Saved without sound: Audio failed" });
expect(closeEncoder).toHaveBeenCalledOnce();
expect(closeData).toHaveBeenCalledTimes(operation === "encode" ? 1 : 0);
const commands = invoked.mock.calls.map(([command]) => command);
expect(commands.filter((command) => command === "release_generated_frames")).toHaveLength(1);
expect(commands.indexOf("release_generated_frames")).toBeLessThan(commands.indexOf("write_generated_video"));
});
it("cannot save from the browser preview, and says which half is missing", async () => {
await expect(saveGeneratedScene({ folderPath: "/p", jobId: "job-01" })).rejects.toThrow(/desktop app/);
expect(invoked).not.toHaveBeenCalled();
});
it("says a webview with no encoder cannot make a video file", async () => {
desktop();
await expect(saveGeneratedScene({ folderPath: "/p", jobId: "job-01" })).rejects.toThrow(/no WebCodecs VideoEncoder/);
expect(invoked).toHaveBeenCalledExactlyOnceWith("release_generated_frames", { jobId: "job-01" });
});
it("says when the frames are gone rather than writing an empty file", async () => {
desktop();
scope.VideoEncoder = class {};
scope.VideoFrame = class {};
invoked.mockResolvedValueOnce(null);
await expect(saveGeneratedScene({ folderPath: "/p", jobId: "job-01" })).rejects.toThrow(/no longer in memory/);
expect(invoked).toHaveBeenCalledWith("generated_summary", { jobId: "job-01" });
});
it("hands the frames back even when the encode never starts", async () => {
desktop();
/* An encoder that refuses every configuration. Hundreds of megabytes are
waiting in Rust at this point, and a save that gives up without
releasing them leaks the lot for the life of the app. */
scope.VideoEncoder = { isConfigSupported: async () => ({ supported: false }) };
scope.VideoFrame = class {};
invoked.mockImplementation(async (command: string) => (command === "generated_summary" ? summary() : true));
await expect(saveGeneratedScene({ folderPath: "/p", jobId: "job-01" })).rejects.toThrow(/refused every H.264/);
expect(invoked).toHaveBeenCalledWith("release_generated_frames", { jobId: "job-01" });
});
it("refuses a render with no pictures in it", async () => {
desktop();
scope.VideoEncoder = class {};
scope.VideoFrame = class {};
invoked.mockResolvedValueOnce(summary({ frameCount: 0 }));
await expect(saveGeneratedScene({ folderPath: "/p", jobId: "job-01" })).rejects.toThrow(/no pictures/);
expect(invoked).toHaveBeenCalledWith("release_generated_frames", { jobId: "job-01" });
});
});
describe("what the app asks Rust about a render", () => {
it("reports nothing waiting in the browser preview instead of calling a command that is not there", async () => {
await expect(renderedSummary("job-01")).resolves.toBeNull();
await expect(releaseRendered("job-01")).resolves.toBe(false);
expect(invoked).not.toHaveBeenCalled();
});
it("never lets a failed release become a failed save", async () => {
desktop();
invoked.mockRejectedValueOnce(new Error("the queue lock is poisoned"));
await expect(releaseRendered("job-01")).resolves.toBe(false);
});
});
describe("generated audio at the Rust/webview boundary", () => {
it("reads raw response bytes as little-endian float samples, not as byte-valued samples", () => {
const original = new Float32Array([0.25, -0.5, 0.75, -1]);
const rawBytes = Array.from(new Uint8Array(original.buffer));
expect(Array.from(audioSamplesFromIpc(rawBytes, 4))).toEqual(Array.from(original));
});
it("keeps the exact window of an IPC byte view", () => {
const backing = new Uint8Array([99, 1, 2, 3, 4, 88]);
expect(Array.from(bytesFromIpc(backing.subarray(1, 5)))).toEqual([1, 2, 3, 4]);
});
it("refuses a truncated or mismatched PCM body", () => {
expect(() => audioSamplesFromIpc([0, 0, 0], 1)).toThrow(/whole 32-bit PCM/);
expect(() => audioSamplesFromIpc([0, 0, 0, 0], 2)).toThrow(/promised 2 audio samples/);
});
it("resamples every interleaved channel without crossing channel boundaries", () => {
const stereo = new Float32Array([0, 10, 1, 11, 2, 12]);
expect(Array.from(resampleInterleaved(stereo, 2, 3, 6))).toEqual([
0, 10,
0.5, 10.5,
1, 11,
1.5, 11.5,
2, 12,
2, 12,
]);
});
it("still saves the pictures when this webview cannot encode AAC", async () => {
desktop();
class FakeVideoFrame { close() { /* owned by the test */ } }
class FakeVideoEncoder {
static isConfigSupported = async (config: unknown) => ({ supported: true, config });
state = "configured";
encodeQueueSize = 0;
private output: (chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata) => void;
constructor(init: VideoEncoderInit) { this.output = init.output; }
configure() { /* accepted */ }
encode() { this.output({} as EncodedVideoChunk); }
async flush() { /* synchronous fake output */ }
close() { this.state = "closed"; }
}
class FakeAudioEncoder {
static isConfigSupported = async () => ({ supported: false });
}
scope.VideoFrame = FakeVideoFrame;
scope.VideoEncoder = FakeVideoEncoder;
scope.AudioData = class {};
scope.AudioEncoder = FakeAudioEncoder;
invoked.mockImplementation(async (command: string) => {
if (command === "generated_summary") return summary({ width: 1, height: 1, frameCount: 1, audioSampleRate: 32_000, audioSamples: 4 });
if (command === "generated_frame") return [1, 2, 3, 255];
if (command === "write_generated_video") return { relativePath: "media/generated/job-01.mp4", bytes: 4 };
return true;
});
await expect(saveGeneratedScene({ folderPath: "C:\\Project", jobId: "job-01" })).resolves.toMatchObject({
relativePath: "media/generated/job-01.mp4",
note: expect.stringMatching(/without sound/),
hasAudio: false,
});
expect(invoked).not.toHaveBeenCalledWith("generated_audio", expect.anything());
expect(muxed.configs[0]).not.toHaveProperty("audio");
expect(muxed.videoChunks).toBe(1);
expect(invoked).toHaveBeenCalledWith("release_generated_frames", { jobId: "job-01" });
});
it("encodes slopfab's interleaved PCM and adds an AAC track to the saved MP4", async () => {
desktop();
const audioData: Array<Record<string, unknown>> = [];
class FakeVideoFrame { close() { /* owned by the test */ } }
class FakeVideoEncoder {
static isConfigSupported = async (config: unknown) => ({ supported: true, config });
state = "configured";
encodeQueueSize = 0;
private output: (chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata) => void;
constructor(init: VideoEncoderInit) { this.output = init.output; }
configure() { /* accepted */ }
encode() { this.output({} as EncodedVideoChunk); }
async flush() { /* synchronous fake output */ }
close() { this.state = "closed"; }
}
class FakeAudioData {
constructor(init: AudioDataInit) { audioData.push(init as unknown as Record<string, unknown>); }
close() { /* owned by the test */ }
}
class FakeAudioEncoder {
static isConfigSupported = async (config: unknown) => ({ supported: true, config });
state = "configured";
encodeQueueSize = 0;
private output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => void;
constructor(init: AudioEncoderInit) { this.output = init.output; }
configure() { /* accepted */ }
encode() { this.output({} as EncodedAudioChunk); }
async flush() { /* synchronous fake output */ }
close() { this.state = "closed"; }
}
scope.VideoFrame = FakeVideoFrame;
scope.VideoEncoder = FakeVideoEncoder;
scope.AudioData = FakeAudioData;
scope.AudioEncoder = FakeAudioEncoder;
const pcm = new Float32Array([0.25, -0.5, 0.75, -1]);
invoked.mockImplementation(async (command: string) => {
if (command === "generated_summary") return summary({ width: 1, height: 1, frameCount: 1, audioSampleRate: 32_000, audioSamples: pcm.length });
if (command === "generated_frame") return [1, 2, 3, 255];
if (command === "generated_audio") return Array.from(new Uint8Array(pcm.buffer));
if (command === "write_generated_video") return { relativePath: "media/generated/job-01.mp4", bytes: 4 };
return true;
});
await expect(saveGeneratedScene({ folderPath: "C:\\Project", jobId: "job-01" })).resolves.toMatchObject({
relativePath: "media/generated/job-01.mp4",
note: null,
hasAudio: true,
});
expect(muxed.videoChunks).toBe(1);
expect(muxed.audioChunks).toBeGreaterThan(0);
expect(muxed.configs[0]).toMatchObject({ audio: { codec: "aac", numberOfChannels: 2, sampleRate: 32_000 } });
expect(audioData[0]).toMatchObject({ format: "f32", numberOfChannels: 2, numberOfFrames: 2, sampleRate: 32_000, timestamp: 0 });
expect(Array.from(audioData[0].data as Float32Array)).toEqual(Array.from(pcm));
expect(invoked).toHaveBeenCalledWith("release_generated_frames", { jobId: "job-01" });
});
});
it("encodes grid-aligned upscaled frames at the selected output size", async () => {
desktop();
const frames: VideoFrameBufferInit[] = [];
const configured: VideoEncoderConfig[] = [];
class FakeVideoFrame {
constructor(_pixels: unknown, init: VideoFrameBufferInit) { frames.push(init); }
close() { /* released */ }
}
class FakeVideoEncoder {
static isConfigSupported = async (config: VideoEncoderConfig) => ({ supported: true, config });
state = "configured";
encodeQueueSize = 0;
constructor(private init: VideoEncoderInit) {}
configure(config: VideoEncoderConfig) { configured.push(config); }
encode() { this.init.output({} as EncodedVideoChunk); }
async flush() { /* complete */ }
close() { this.state = "closed"; }
}
scope.VideoFrame = FakeVideoFrame;
scope.VideoEncoder = FakeVideoEncoder;
invoked.mockImplementation(async (command: string) => {
if (command === "generated_summary") return summary({ width: 768, height: 448, frameCount: 2, audioSamples: 0 });
if (command === "generated_frame") return new Uint8Array(768 * 448 * 4);
if (command === "write_generated_video") return { relativePath: "media/generated/job-01.mp4", bytes: 4 };
return true;
});
const saved = await saveGeneratedScene({ folderPath: "C:/Project", jobId: "job-01", outputSize: { width: 736, height: 416 } });
expect(saved).toMatchObject({ width: 736, height: 416, durationMs: 83 });
expect(configured[0]).toMatchObject({ width: 736, height: 416 });
expect(muxed.configs[0].video).toMatchObject({ width: 736, height: 416 });
expect(frames).toHaveLength(2);
expect(frames[0]).toMatchObject({ codedWidth: 768, codedHeight: 448, timestamp: 0 });
expect(frames[1].timestamp).toBe(41667);
expect(muxed.videoChunks).toBe(2);
expect(invoked).toHaveBeenCalledWith("release_generated_frames", { jobId: "job-01" });
});