AI media harness for planning, generating, and editing locally
TypeScript
Allow reference video decoder fallback
Bittia4051cecf679
diff --git a/docs/video-references.md b/docs/video-references.mdindex 85521fc..6545380 100644--- a/docs/video-references.md+++ b/docs/video-references.md@@ -29,6 +29,11 @@ the video updates the icon. MP4, M4V, and MOV containers are accepted; the compudecoder must support the video codec. Video files remain at their originallocations. The project stores their paths and clip settings.+Reference preparation tries hardware decoding first, then automatic WebCodecs+decoder selection if that configuration is unavailable. This allows the webview+to use its software decoder for formats such as AV1 when hardware decoding is+unavailable.+Set the clip's start and duration, and choose whether to include its sound.Each clip must last 2–15 seconds. A generation can use up to three video clipstotaling 15 seconds, alongside up to nine images. Cite the reference in a shotdiff --git a/src/lib/referenceVideo.test.ts b/src/lib/referenceVideo.test.tsindex 73be448..c9f3833 100644--- a/src/lib/referenceVideo.test.ts+++ b/src/lib/referenceVideo.test.ts@@ -23,12 +23,13 @@ function decoderMocks() {frames.push(frame); return frame;};const close = vi.fn();+ const configure = vi.fn();vi.stubGlobal("EncodedVideoChunk", class { constructor(public init: EncodedVideoChunkInit) {} });vi.stubGlobal("VideoDecoder", class {static async isConfigSupported(config: VideoDecoderConfig) { return { supported: true, config }; }state = "configured"; decodeQueueSize = 0;constructor(private init: VideoDecoderInit) {}- configure() {}+ configure = configure;decode(chunk: { init: EncodedVideoChunkInit }) { this.init.output(make(chunk.init.timestamp) as unknown as VideoFrame); }async flush() {}close = close;@@ -37,9 +38,47 @@ function decoderMocks() {constructor(public width: number, public height: number) {}getContext() { return { drawImage: vi.fn(), getImageData: () => ({ data: new Uint8ClampedArray(this.width * this.height * 4) }) }; }});- return { frames, close };+ return { frames, close, configure };}+it.each(["unsupported", "rejected"])("falls back when AV1 hardware decoding is %s", async (failure) => {+ const { configure, frames } = decoderMocks();+ const av1 = { ...source, config: { codec: "av01.0.08M.08", codedWidth: 544, codedHeight: 960,+ description: new Uint8Array([129, 8, 12, 0]) } };+ const probe = vi.spyOn(VideoDecoder, "isConfigSupported").mockImplementation(async (config) => {+ if (config.hardwareAcceleration === "prefer-hardware") {+ if (failure === "rejected") throw new DOMException("No hardware decoder", "NotSupportedError");+ return { supported: false, config };+ }+ return { supported: true, config };+ });+ await uploadReferenceVideoFrames(av1, "video-1", 1, 2, () => false);+ expect(probe.mock.calls.map(([config]) => config.hardwareAcceleration)).toEqual(["prefer-hardware", "no-preference"]);+ expect(configure).toHaveBeenCalledWith({ ...av1.config, hardwareAcceleration: "no-preference" });+ expect(invoke).toHaveBeenCalledWith("append_reference_video", expect.any(Uint8Array), expect.anything());+ expect(frames.every((frame) => frame.close.mock.calls.length === 1)).toBe(true);+});++it("keeps hardware decoding when supported", async () => {+ const { configure } = decoderMocks();+ const probe = vi.spyOn(VideoDecoder, "isConfigSupported");+ await uploadReferenceVideoFrames(source, "video-1", 1, 2, () => false);+ expect(probe).toHaveBeenCalledOnce();+ expect(configure).toHaveBeenCalledWith({ ...source.config, hardwareAcceleration: "prefer-hardware" });+});++it("reports unavailable codecs only after both probes and releases the native reference", async () => {+ const { configure } = decoderMocks();+ const probe = vi.spyOn(VideoDecoder, "isConfigSupported").mockImplementation(async (config) => ({ supported: false, config }));+ vi.mocked(demux).mockResolvedValue(source);+ const config = createProjectConfig({ name: "Test", prompt: "", aspectRatio: "16:9", resolution: "1080p", targetDurationSeconds: 20 });+ const request = { jobId: "job", prompt: "", frames: 120, steps: 4, seed: 1, canvasWidth: 736, canvasHeight: 416, referencePaths: [], referenceVideos: [input] };+ await expect(prepareReferenceVideos("C:/project", request, config, () => false, () => undefined)).rejects.toThrow("hardware or automatic decoder selection");+ expect(probe).toHaveBeenCalledTimes(2);+ expect(configure).not.toHaveBeenCalled();+ expect(invoke).toHaveBeenCalledWith("release_reference_videos", { ids: ["video-1"] });+});+it("validates trim bounds and uses at most 15 seconds for an unspecified clip", () => {expect(referenceVideoRange(input, 4)).toEqual({ start: 1, duration: 2 });expect(referenceVideoRange({ ...input, durationSeconds: undefined }, 30).duration).toBe(15);diff --git a/src/lib/referenceVideo.ts b/src/lib/referenceVideo.tsindex a0611e2..33c9213 100644--- a/src/lib/referenceVideo.ts+++ b/src/lib/referenceVideo.ts@@ -32,13 +32,27 @@ export async function inspectReferenceVideo(folderPath: string, sourcePath: strireturn { durationSeconds: Math.min(15, duration), includeAudio: Boolean(source.hasAudio) };}+async function referenceDecoderConfig(config: VideoDecoderConfig): Promise<VideoDecoderConfig> {+ // A rejected hardware preference does not mean the codec is unavailable.+ // In particular, AV1 may still be decoded by the webview's software decoder.+ for (const hardwareAcceleration of ["prefer-hardware", "no-preference"] as const) {+ const candidate = { ...config, hardwareAcceleration };+ try {+ const support = await VideoDecoder.isConfigSupported(candidate);+ if (support.supported) return support.config ?? candidate;+ } catch (error) {+ if (!(error instanceof DOMException) || error.name !== "NotSupportedError") throw error;+ }+ }+ throw new Error(`This webview cannot decode reference codec ${config.codec} with hardware or automatic decoder selection.`);+}+/** Decode only the selected interval. At most a small decoder batch and one* retained frame are alive; binary IPC avoids JSON/base64 copies of pixels.* Slopfab consumes host RGB, so readback is confined to this ingestion step. */export async function uploadReferenceVideoFrames(source: DemuxedSource, id: string, start: number, duration: number, cancelled: () => boolean) {if (typeof VideoDecoder === "undefined") throw new Error("This webview does not support WebCodecs video decoding.");- const support = await VideoDecoder.isConfigSupported({ ...source.config, hardwareAcceleration: "prefer-hardware" });- if (!support.supported) throw new Error(`This computer cannot decode reference codec ${source.config.codec}.`);+ const config = await referenceDecoderConfig(source.config);const origin = source.samples.reduce((value, sample) => Math.min(value, sample.timestampUs), Infinity);const beginUs = origin + Math.round(start * 1_000_000);const endUs = beginUs + Math.round(duration * 1_000_000);@@ -99,7 +113,7 @@ export async function uploadReferenceVideoFrames(source: DemuxedSource, id: strierror: (error) => { state.error = error; },});try {- decoder.configure(support.config ?? source.config);+ decoder.configure(config);for (let index = first; index < last; index++) {check();const sample = source.samples[index];