Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Allow reference video decoder fallback

Bittia4051cecf679
diff --git a/docs/video-references.md b/docs/video-references.md
index 85521fc..6545380 100644
--- a/docs/video-references.md
+++ b/docs/video-references.md
@@ -29,6 +29,11 @@ the video updates the icon. MP4, M4V, and MOV containers are accepted; the compu
decoder must support the video codec. Video files remain at their original
locations. The project stores their paths and clip settings.
+Reference preparation tries hardware decoding first, then automatic WebCodecs
+decoder selection if that configuration is unavailable. This allows the webview
+to use its software decoder for formats such as AV1 when hardware decoding is
+unavailable.
+
Set the clip's start and duration, and choose whether to include its sound.
Each clip must last 2–15 seconds. A generation can use up to three video clips
totaling 15 seconds, alongside up to nine images. Cite the reference in a shot
diff --git a/src/lib/referenceVideo.test.ts b/src/lib/referenceVideo.test.ts
index 73be448..c9f3833 100644
--- a/src/lib/referenceVideo.test.ts
+++ b/src/lib/referenceVideo.test.ts
@@ -23,12 +23,13 @@ function decoderMocks() {
frames.push(frame); return frame;
};
const close = vi.fn();
+ const configure = vi.fn();
vi.stubGlobal("EncodedVideoChunk", class { constructor(public init: EncodedVideoChunkInit) {} });
vi.stubGlobal("VideoDecoder", class {
static async isConfigSupported(config: VideoDecoderConfig) { return { supported: true, config }; }
state = "configured"; decodeQueueSize = 0;
constructor(private init: VideoDecoderInit) {}
- configure() {}
+ configure = configure;
decode(chunk: { init: EncodedVideoChunkInit }) { this.init.output(make(chunk.init.timestamp) as unknown as VideoFrame); }
async flush() {}
close = close;
@@ -37,9 +38,47 @@ function decoderMocks() {
constructor(public width: number, public height: number) {}
getContext() { return { drawImage: vi.fn(), getImageData: () => ({ data: new Uint8ClampedArray(this.width * this.height * 4) }) }; }
});
- return { frames, close };
+ return { frames, close, configure };
}
+it.each(["unsupported", "rejected"])("falls back when AV1 hardware decoding is %s", async (failure) => {
+ const { configure, frames } = decoderMocks();
+ const av1 = { ...source, config: { codec: "av01.0.08M.08", codedWidth: 544, codedHeight: 960,
+ description: new Uint8Array([129, 8, 12, 0]) } };
+ const probe = vi.spyOn(VideoDecoder, "isConfigSupported").mockImplementation(async (config) => {
+ if (config.hardwareAcceleration === "prefer-hardware") {
+ if (failure === "rejected") throw new DOMException("No hardware decoder", "NotSupportedError");
+ return { supported: false, config };
+ }
+ return { supported: true, config };
+ });
+ await uploadReferenceVideoFrames(av1, "video-1", 1, 2, () => false);
+ expect(probe.mock.calls.map(([config]) => config.hardwareAcceleration)).toEqual(["prefer-hardware", "no-preference"]);
+ expect(configure).toHaveBeenCalledWith({ ...av1.config, hardwareAcceleration: "no-preference" });
+ expect(invoke).toHaveBeenCalledWith("append_reference_video", expect.any(Uint8Array), expect.anything());
+ expect(frames.every((frame) => frame.close.mock.calls.length === 1)).toBe(true);
+});
+
+it("keeps hardware decoding when supported", async () => {
+ const { configure } = decoderMocks();
+ const probe = vi.spyOn(VideoDecoder, "isConfigSupported");
+ await uploadReferenceVideoFrames(source, "video-1", 1, 2, () => false);
+ expect(probe).toHaveBeenCalledOnce();
+ expect(configure).toHaveBeenCalledWith({ ...source.config, hardwareAcceleration: "prefer-hardware" });
+});
+
+it("reports unavailable codecs only after both probes and releases the native reference", async () => {
+ const { configure } = decoderMocks();
+ const probe = vi.spyOn(VideoDecoder, "isConfigSupported").mockImplementation(async (config) => ({ supported: false, config }));
+ vi.mocked(demux).mockResolvedValue(source);
+ const config = createProjectConfig({ name: "Test", prompt: "", aspectRatio: "16:9", resolution: "1080p", targetDurationSeconds: 20 });
+ const request = { jobId: "job", prompt: "", frames: 120, steps: 4, seed: 1, canvasWidth: 736, canvasHeight: 416, referencePaths: [], referenceVideos: [input] };
+ await expect(prepareReferenceVideos("C:/project", request, config, () => false, () => undefined)).rejects.toThrow("hardware or automatic decoder selection");
+ expect(probe).toHaveBeenCalledTimes(2);
+ expect(configure).not.toHaveBeenCalled();
+ expect(invoke).toHaveBeenCalledWith("release_reference_videos", { ids: ["video-1"] });
+});
+
it("validates trim bounds and uses at most 15 seconds for an unspecified clip", () => {
expect(referenceVideoRange(input, 4)).toEqual({ start: 1, duration: 2 });
expect(referenceVideoRange({ ...input, durationSeconds: undefined }, 30).duration).toBe(15);
diff --git a/src/lib/referenceVideo.ts b/src/lib/referenceVideo.ts
index a0611e2..33c9213 100644
--- a/src/lib/referenceVideo.ts
+++ b/src/lib/referenceVideo.ts
@@ -32,13 +32,27 @@ export async function inspectReferenceVideo(folderPath: string, sourcePath: stri
return { durationSeconds: Math.min(15, duration), includeAudio: Boolean(source.hasAudio) };
}
+async function referenceDecoderConfig(config: VideoDecoderConfig): Promise<VideoDecoderConfig> {
+ // A rejected hardware preference does not mean the codec is unavailable.
+ // In particular, AV1 may still be decoded by the webview's software decoder.
+ for (const hardwareAcceleration of ["prefer-hardware", "no-preference"] as const) {
+ const candidate = { ...config, hardwareAcceleration };
+ try {
+ const support = await VideoDecoder.isConfigSupported(candidate);
+ if (support.supported) return support.config ?? candidate;
+ } catch (error) {
+ if (!(error instanceof DOMException) || error.name !== "NotSupportedError") throw error;
+ }
+ }
+ throw new Error(`This webview cannot decode reference codec ${config.codec} with hardware or automatic decoder selection.`);
+}
+
/** Decode only the selected interval. At most a small decoder batch and one
* retained frame are alive; binary IPC avoids JSON/base64 copies of pixels.
* Slopfab consumes host RGB, so readback is confined to this ingestion step. */
export async function uploadReferenceVideoFrames(source: DemuxedSource, id: string, start: number, duration: number, cancelled: () => boolean) {
if (typeof VideoDecoder === "undefined") throw new Error("This webview does not support WebCodecs video decoding.");
- const support = await VideoDecoder.isConfigSupported({ ...source.config, hardwareAcceleration: "prefer-hardware" });
- if (!support.supported) throw new Error(`This computer cannot decode reference codec ${source.config.codec}.`);
+ const config = await referenceDecoderConfig(source.config);
const origin = source.samples.reduce((value, sample) => Math.min(value, sample.timestampUs), Infinity);
const beginUs = origin + Math.round(start * 1_000_000);
const endUs = beginUs + Math.round(duration * 1_000_000);
@@ -99,7 +113,7 @@ export async function uploadReferenceVideoFrames(source: DemuxedSource, id: stri
error: (error) => { state.error = error; },
});
try {
- decoder.configure(support.config ?? source.config);
+ decoder.configure(config);
for (let index = first; index < last; index++) {
check();
const sample = source.samples[index];