Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Add the export plan maths and the webcodecs pipeline

Bittid288638f3c3e
diff --git a/package-lock.json b/package-lock.json
index 80ff292..6222aed 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -23,6 +23,7 @@
"@types/react": "^18.3.18",
"@types/react-dom": "^18.3.5",
"@vitejs/plugin-react": "^4.3.4",
+ "@webgpu/types": "^0.1.72",
"jsdom": "^25.0.1",
"typescript": "~5.7.2",
"vite": "^6.0.5",
@@ -1887,6 +1888,13 @@
"url": "https://opencollective.com/vitest"
}
},
+ "node_modules/@webgpu/types": {
+ "version": "0.1.72",
+ "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.72.tgz",
+ "integrity": "sha512-0cF7RFM2edNoiIS1ODJp0/Gzv4/xSXhwoR0YCza+OWpJWtn4wmo9DvK91aLlH9+uUnwIriP7ZiC3WitmyhuzBw==",
+ "dev": true,
+ "license": "BSD-3-Clause"
+ },
"node_modules/agent-base": {
"version": "7.1.4",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz",
diff --git a/package.json b/package.json
index 08f9c84..d307d91 100644
--- a/package.json
+++ b/package.json
@@ -26,6 +26,7 @@
"@types/react": "^18.3.18",
"@types/react-dom": "^18.3.5",
"@vitejs/plugin-react": "^4.3.4",
+ "@webgpu/types": "^0.1.72",
"jsdom": "^25.0.1",
"typescript": "~5.7.2",
"vite": "^6.0.5",
diff --git a/src/lib/export.ts b/src/lib/export.ts
new file mode 100644
index 0000000..64ba372
--- /dev/null
+++ b/src/lib/export.ts
@@ -0,0 +1,397 @@
+/* ============================================================================
+ export.ts — what an export IS, worked out with arithmetic only.
+
+ Everything in this file is pure: it takes a project and the user's export
+ settings and works out the frame-by-frame shape of the output file. It never
+ touches WebCodecs, WebGPU, mp4box or the filesystem — that is
+ exportPipeline.ts, which consumes the plan this module produces.
+
+ The split exists so the timing maths can be tested without a GPU: which clip
+ is on screen at time T, and which moment of the source file that output frame
+ has to come from, are the two questions an editor gets wrong silently.
+ ========================================================================== */
+
+import type { AspectRatio, ProjectConfig, Resolution, TimelineClip, TimelineTrack } from "./project";
+
+export type FrameRate = ProjectConfig["settings"]["frameRate"];
+export type OutputCodecId = "h264" | "vp9" | "av1";
+export type QualityId = "draft" | "balanced" | "high";
+
+export interface ExportSettings {
+ resolution: Resolution;
+ frameRate: FrameRate;
+ codec: OutputCodecId;
+ quality: QualityId;
+}
+
+export interface OutputCodec {
+ id: OutputCodecId;
+ label: string;
+ /** The track codec name mp4-muxer expects. */
+ muxer: "avc" | "vp9" | "av1";
+ /** Codec strings tried in order against `VideoEncoder.isConfigSupported`. */
+ candidates: (width: number, height: number, frameRate: number) => string[];
+ detail: string;
+}
+
+/* H.264 levels, as the byte that ends an `avc1.` codec string. The encoder
+ rejects a configuration whose level cannot hold the frame size and rate, so
+ the level is derived from the actual output rather than hard-coded. */
+function avcLevel(width: number, height: number, frameRate: number): string {
+ const macroblocks = Math.ceil(width / 16) * Math.ceil(height / 16);
+ const rate = macroblocks * frameRate;
+ if (macroblocks <= 3600 && rate <= 108_000) return "1f"; // 3.1
+ if (macroblocks <= 8192 && rate <= 245_760) return "28"; // 4.0
+ if (macroblocks <= 8704 && rate <= 522_240) return "2a"; // 4.2
+ if (macroblocks <= 22_080 && rate <= 589_824) return "33"; // 5.1
+ return "34"; // 5.2
+}
+
+export const OUTPUT_CODECS: readonly OutputCodec[] = [
+ {
+ id: "h264",
+ label: "H.264 / AVC",
+ muxer: "avc",
+ /* High, then Main, then Baseline at the same level. Hardware encoders
+ differ on which profiles they expose, so the first one this machine
+ actually accepts is the one used — never a guess. */
+ candidates: (width, height, frameRate) => {
+ const level = avcLevel(width, height, frameRate);
+ return [`avc1.6400${level}`, `avc1.4d00${level}`, `avc1.4200${level}`];
+ },
+ detail: "Plays everywhere. The baseline choice for an .mp4.",
+ },
+ {
+ id: "vp9",
+ label: "VP9",
+ muxer: "vp9",
+ candidates: () => ["vp09.00.10.08"],
+ detail: "Smaller files at the same quality; not every player reads VP9 inside .mp4.",
+ },
+ {
+ id: "av1",
+ label: "AV1",
+ muxer: "av1",
+ candidates: () => ["av01.0.08M.08", "av01.0.04M.08"],
+ detail: "Newest codec, smallest files, slowest to encode and least widely supported.",
+ },
+];
+
+export function outputCodec(id: OutputCodecId): OutputCodec {
+ const codec = OUTPUT_CODECS.find((candidate) => candidate.id === id);
+ if (!codec) throw new Error(`Unknown output codec: ${id}`);
+ return codec;
+}
+
+export interface QualityPreset {
+ id: QualityId;
+ label: string;
+ /** Bits spent per pixel per frame. Bitrate is derived, never typed in. */
+ bitsPerPixel: number;
+ detail: string;
+}
+
+export const QUALITY_PRESETS: readonly QualityPreset[] = [
+ { id: "draft", label: "Draft", bitsPerPixel: 0.04, detail: "Fastest and smallest. Visible compression in motion." },
+ { id: "balanced", label: "Balanced", bitsPerPixel: 0.09, detail: "The usual choice for a delivery file." },
+ { id: "high", label: "High", bitsPerPixel: 0.16, detail: "Large files. Keeps detail in grain and fast motion." },
+];
+
+export function qualityPreset(id: QualityId): QualityPreset {
+ const preset = QUALITY_PRESETS.find((candidate) => candidate.id === id);
+ if (!preset) throw new Error(`Unknown quality preset: ${id}`);
+ return preset;
+}
+
+/* The resolution names the SHORT edge, so one setting means the same amount of
+ picture in every aspect ratio the project offers: 1080p is 1920×1080 wide,
+ 1080×1920 tall, 1080×1080 square and 1080×1350 at 4:5. */
+const SHORT_EDGE: Record<Resolution, number> = { "720p": 720, "1080p": 1080, "4k": 2160 };
+const RATIO: Record<AspectRatio, readonly [number, number]> = {
+ "16:9": [16, 9],
+ "9:16": [9, 16],
+ "1:1": [1, 1],
+ "4:5": [4, 5],
+};
+
+/** Encoders reject odd dimensions in 4:2:0, so both edges land on an even number. */
+const even = (value: number) => Math.max(2, Math.round(value / 2) * 2);
+
+export function outputDimensions(resolution: Resolution, aspectRatio: AspectRatio): { width: number; height: number } {
+ const base = SHORT_EDGE[resolution];
+ const [ratioWidth, ratioHeight] = RATIO[aspectRatio];
+ return ratioWidth >= ratioHeight
+ ? { width: even((base * ratioWidth) / ratioHeight), height: even(base) }
+ : { width: even(base), height: even((base * ratioHeight) / ratioWidth) };
+}
+
+/** The export starts as the project already describes itself. Nothing here is
+ * invented: resolution and frame rate are the project's own settings. */
+export function defaultExportSettings(config: ProjectConfig): ExportSettings {
+ return {
+ resolution: config.settings.resolution,
+ frameRate: config.settings.frameRate,
+ codec: "h264",
+ quality: "balanced",
+ };
+}
+
+export function bitrateFor(width: number, height: number, frameRate: number, quality: QualityId): number {
+ return Math.round(width * height * frameRate * qualityPreset(quality).bitsPerPixel);
+}
+
+/** Bytes the encoder is being ASKED to produce. A rate-controlled encoder lands
+ * near its target rather than on it, so this is labelled an estimate wherever
+ * it is shown. */
+export function estimatedBytes(bitrate: number, durationMs: number): number {
+ return Math.round((bitrate / 8) * (durationMs / 1000));
+}
+
+/* ---------------------------------------------------------------------------
+ The plan
+ --------------------------------------------------------------------------- */
+
+export type ExportSegment =
+ | {
+ kind: "clip";
+ clipId: string;
+ assetId: string;
+ label: string;
+ /** First output frame index this clip fills. */
+ startFrame: number;
+ /** One past the last output frame index it fills. */
+ endFrame: number;
+ /** The clip's own position on the timeline, in ms. */
+ clipStartMs: number;
+ /** Where inside the source file the clip begins, in ms. */
+ clipSourceStartMs: number;
+ }
+ | { kind: "gap"; startFrame: number; endFrame: number };
+
+export interface ExportPlan {
+ width: number;
+ height: number;
+ frameRate: FrameRate;
+ durationMs: number;
+ frameCount: number;
+ backgroundColor: string;
+ segments: ExportSegment[];
+ /** Video clips that contribute at least one frame to the file. */
+ clipCount: number;
+ /** Frames with nothing over them; they get the project's background colour. */
+ gapFrames: number;
+ /** Clips on audio tracks. They are NOT in the file — see `notes`. */
+ audioClipCount: number;
+ /** Reasons this export cannot run at all. Empty means it can. */
+ blockers: string[];
+ /** True things about the output the user has to be told before they press go. */
+ notes: string[];
+}
+
+/** Container formats mp4box can demux today. webm and mkv import fine and play
+ * in the app, but nothing here can take them apart yet, so a clip built on one
+ * blocks the export instead of quietly rendering as background. */
+const DEMUXABLE_VIDEO = new Set(["video/mp4", "video/quicktime"]);
+const DRAWABLE_IMAGE = new Set(["image/png", "image/jpeg", "image/webp", "image/gif"]);
+
+/** The moment output frame `frameIndex` has to be taken from inside the source
+ * file, in milliseconds. This is the whole of the trim maths: an output frame
+ * sits at `frameIndex / frameRate` on the timeline, and the source moment is
+ * that far past the clip's start, offset by where the clip begins in its file. */
+export function sourceTimeMsForFrame(
+ segment: Extract<ExportSegment, { kind: "clip" }>,
+ frameIndex: number,
+ frameRate: number,
+): number {
+ return segment.clipSourceStartMs + ((frameIndex * 1000) / frameRate - segment.clipStartMs);
+}
+
+/** The clip a viewer sees at `timeMs`.
+ *
+ * Tracks are drawn top to bottom in array order, so the FIRST video track that
+ * has something at this moment wins — an overlay covers the story beneath it.
+ * Within one track two clips can overlap after a drag; the later one is the
+ * more recent edit, so it is the one on top. */
+export function visibleClipAt(tracks: TimelineTrack[], timeMs: number): TimelineClip | null {
+ for (const track of tracks) {
+ if (track.kind !== "video") continue;
+ let best: TimelineClip | null = null;
+ for (const clip of track.clips) {
+ if (timeMs < clip.startMs || timeMs >= clip.startMs + clip.durationMs) continue;
+ if (!best || clip.startMs >= best.startMs) best = clip;
+ }
+ if (best) return best;
+ }
+ return null;
+}
+
+/** Where the picture ends. Audio is not muxed yet, so a soundtrack that runs
+ * past the last shot cannot extend the file — claiming otherwise would promise
+ * frames that do not exist. */
+export function videoDurationMs(config: ProjectConfig): number {
+ return config.timeline.tracks
+ .filter((track) => track.kind === "video")
+ .flatMap((track) => track.clips)
+ .reduce((end, clip) => Math.max(end, clip.startMs + clip.durationMs), 0);
+}
+
+export function buildExportPlan(config: ProjectConfig, settings: ExportSettings): ExportPlan {
+ const { width, height } = outputDimensions(settings.resolution, config.settings.aspectRatio);
+ const frameRate = settings.frameRate;
+ const tracks = config.timeline.tracks;
+ const videoTracks = tracks.filter((track) => track.kind === "video");
+ const durationMs = videoDurationMs(config);
+ const frameCount = Math.round((durationMs * frameRate) / 1000);
+
+ /* Resolve every output frame first, then run the identical answers together.
+ Deriving segments from clip boundaries instead would disagree with the
+ encoder loop the moment two clips overlap. */
+ const segments: ExportSegment[] = [];
+ let gapFrames = 0;
+ for (let frame = 0; frame < frameCount; frame += 1) {
+ const clip = visibleClipAt(videoTracks, (frame * 1000) / frameRate);
+ const last = segments[segments.length - 1];
+ if (!clip) {
+ gapFrames += 1;
+ if (last && last.kind === "gap") last.endFrame = frame + 1;
+ else segments.push({ kind: "gap", startFrame: frame, endFrame: frame + 1 });
+ continue;
+ }
+ if (last && last.kind === "clip" && last.clipId === clip.id) {
+ last.endFrame = frame + 1;
+ continue;
+ }
+ segments.push({
+ kind: "clip",
+ clipId: clip.id,
+ assetId: clip.assetId,
+ label: clip.label,
+ startFrame: frame,
+ endFrame: frame + 1,
+ clipStartMs: clip.startMs,
+ clipSourceStartMs: clip.sourceStartMs,
+ });
+ }
+
+ const usedClipIds = new Set(segments.flatMap((segment) => (segment.kind === "clip" ? [segment.clipId] : [])));
+ const audioClipCount = tracks
+ .filter((track) => track.kind === "audio")
+ .reduce((total, track) => total + track.clips.length, 0);
+
+ const blockers: string[] = [];
+ if (frameCount === 0) {
+ blockers.push("There are no video clips on the timeline, so there is nothing to render.");
+ }
+ const missing: string[] = [];
+ const undecodable: string[] = [];
+ for (const segment of segments) {
+ if (segment.kind !== "clip") continue;
+ const asset = config.assets.find((candidate) => candidate.id === segment.assetId);
+ if (!asset) {
+ missing.push(segment.label);
+ continue;
+ }
+ const drawable = DEMUXABLE_VIDEO.has(asset.mimeType) || DRAWABLE_IMAGE.has(asset.mimeType);
+ if (!drawable) undecodable.push(`${segment.label} (${asset.name}, ${asset.mimeType})`);
+ }
+ if (missing.length > 0) {
+ blockers.push(`These clips point at media that is not in this project: ${missing.join(", ")}.`);
+ }
+ if (undecodable.length > 0) {
+ blockers.push(
+ `PolStudio can only take apart .mp4 and .mov video and still images so far. These clips use something else: ${undecodable.join(", ")}.`,
+ );
+ }
+
+ const notes: string[] = [];
+ if (audioClipCount > 0) {
+ notes.push(
+ `The file will have no sound. ${audioClipCount} audio ${audioClipCount === 1 ? "clip is" : "clips are"} on the timeline and none of them are muxed yet.`,
+ );
+ }
+ if (gapFrames > 0) {
+ notes.push(
+ `${gapFrames} of ${frameCount} frames have no clip over them and will be solid ${config.settings.backgroundColor}.`,
+ );
+ }
+ const unusedClips = videoTracks
+ .flatMap((track) => track.clips)
+ .filter((clip) => !usedClipIds.has(clip.id));
+ if (unusedClips.length > 0) {
+ notes.push(
+ `${unusedClips.length} video ${unusedClips.length === 1 ? "clip is" : "clips are"} hidden underneath another track and will not appear.`,
+ );
+ }
+
+ return {
+ width,
+ height,
+ frameRate,
+ durationMs,
+ frameCount,
+ backgroundColor: config.settings.backgroundColor,
+ segments,
+ clipCount: usedClipIds.size,
+ gapFrames,
+ audioClipCount,
+ blockers,
+ notes,
+ };
+}
+
+/* ---------------------------------------------------------------------------
+ Geometry shared by the preview and the compositor
+ --------------------------------------------------------------------------- */
+
+export interface FittedRect {
+ x: number;
+ y: number;
+ width: number;
+ height: number;
+}
+
+/** The source frame, scaled to fit inside the output and centred. Nothing is
+ * cropped; whatever is left over is background. The preview and the export
+ * compositor both call this, so what you see is the frame you get. */
+export function fitRect(sourceWidth: number, sourceHeight: number, outWidth: number, outHeight: number): FittedRect {
+ if (sourceWidth <= 0 || sourceHeight <= 0) return { x: 0, y: 0, width: outWidth, height: outHeight };
+ const scale = Math.min(outWidth / sourceWidth, outHeight / sourceHeight);
+ const width = sourceWidth * scale;
+ const height = sourceHeight * scale;
+ return { x: (outWidth - width) / 2, y: (outHeight - height) / 2, width, height };
+}
+
+/* ---------------------------------------------------------------------------
+ Formatting
+ --------------------------------------------------------------------------- */
+
+export function formatBytes(bytes: number): string {
+ if (bytes < 1000) return `${bytes} B`;
+ const units = ["kB", "MB", "GB"];
+ let value = bytes / 1000;
+ let unit = 0;
+ while (value >= 1000 && unit < units.length - 1) {
+ value /= 1000;
+ unit += 1;
+ }
+ return `${value.toFixed(value >= 100 ? 0 : 1)} ${units[unit]}`;
+}
+
+/** mm:ss.mmm — an editor needs the fraction, and hours would be noise for a
+ * timeline measured in seconds. */
+export function formatDuration(ms: number): string {
+ const total = Math.max(0, Math.round(ms));
+ const minutes = Math.floor(total / 60_000);
+ const seconds = Math.floor((total % 60_000) / 1000);
+ const millis = total % 1000;
+ return `${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}.${String(millis).padStart(3, "0")}`;
+}
+
+/** A file name the OS will accept, derived from the project's own name. */
+export function suggestedFileName(projectName: string): string {
+ const cleaned = projectName
+ .replace(/[\\/:*?"<>|]/g, " ")
+ .replace(/\s+/g, " ")
+ .trim();
+ return `${cleaned || "PolStudio export"}.mp4`;
+}
diff --git a/src/lib/exportPipeline.ts b/src/lib/exportPipeline.ts
new file mode 100644
index 0000000..59446cb
--- /dev/null
+++ b/src/lib/exportPipeline.ts
@@ -0,0 +1,740 @@
+/// <reference types="@webgpu/types" />
+
+/* ============================================================================
+ exportPipeline.ts — the five stages CLAUDE.md describes, in order.
+
+ demux (mp4box.js) -> decode (WebCodecs VideoDecoder, OS hardware)
+ -> process (WebGPU, importExternalTexture, no readback)
+ -> encode (WebCodecs VideoEncoder) -> mux (mp4-muxer)
+
+ No FFmpeg anywhere, deliberately: the whole point of this pipeline is to stay
+ clear of GPL/LGPL binaries. The heavy libraries are imported dynamically so
+ that neither the tests nor the first paint of the app pay for them.
+
+ What this module will NOT do: pretend. If WebCodecs is missing, if the codec
+ is unsupported, if a source file cannot be taken apart — it throws with the
+ real reason and the view shows that reason verbatim.
+ ========================================================================== */
+
+import { invoke } from "@tauri-apps/api/core";
+import { isTauri } from "./persistence";
+import {
+ fitRect,
+ outputCodec,
+ sourceTimeMsForFrame,
+ type ExportPlan,
+ type ExportSegment,
+ type ExportSettings,
+ type OutputCodecId,
+} from "./export";
+import type { ProjectAsset, ProjectConfig } from "./project";
+
+/* ---------------------------------------------------------------------------
+ What this machine can do
+ --------------------------------------------------------------------------- */
+
+export interface ExportSupport {
+ /** WebCodecs encode. Present in WebView2/Chromium, absent in older engines. */
+ encoder: boolean;
+ decoder: boolean;
+ /** The GPU compositor. Absent is not fatal — there is a 2D canvas fallback. */
+ webgpu: boolean;
+ /** Only the desktop shell has files to read and a place to write one. */
+ desktop: boolean;
+}
+
+export function detectExportSupport(): ExportSupport {
+ const scope = globalThis as unknown as Record<string, unknown>;
+ return {
+ encoder: typeof scope.VideoEncoder === "function",
+ decoder: typeof scope.VideoDecoder === "function",
+ webgpu: typeof navigator !== "undefined" && "gpu" in navigator && Boolean(navigator.gpu),
+ desktop: isTauri(),
+ };
+}
+
+export interface CodecProbe {
+ id: OutputCodecId;
+ supported: boolean;
+ /** The codec string this machine accepted, when it accepted one. */
+ codecString: string | null;
+ /** Why not, in the encoder's own terms, when it did not. */
+ detail: string;
+}
+
+function encoderConfig(codecString: string, plan: ExportPlan, bitrate: number): VideoEncoderConfig {
+ return {
+ codec: codecString,
+ width: plan.width,
+ height: plan.height,
+ bitrate,
+ framerate: plan.frameRate,
+ // Length-prefixed AVC with the parameter sets in the sample description,
+ // which is the only shape mp4-muxer can put in an .mp4.
+ avc: { format: "avc" },
+ };
+}
+
+/** Asks the encoder itself, one candidate codec string at a time. Nothing here
+ * is assumed from the codec name. */
+export async function probeCodec(id: OutputCodecId, plan: ExportPlan, bitrate: number): Promise<CodecProbe> {
+ const codec = outputCodec(id);
+ if (typeof VideoEncoder === "undefined") {
+ return { id, supported: false, codecString: null, detail: "This webview has no WebCodecs VideoEncoder." };
+ }
+ const tried: string[] = [];
+ for (const candidate of codec.candidates(plan.width, plan.height, plan.frameRate)) {
+ tried.push(candidate);
+ try {
+ const support = await VideoEncoder.isConfigSupported(encoderConfig(candidate, plan, bitrate));
+ if (support.supported) return { id, supported: true, codecString: candidate, detail: candidate };
+ } catch (reason) {
+ return {
+ id,
+ supported: false,
+ codecString: null,
+ detail: reason instanceof Error ? reason.message : String(reason),
+ };
+ }
+ }
+ return {
+ id,
+ supported: false,
+ codecString: null,
+ detail: `This computer's encoder refused every profile tried at ${plan.width}×${plan.height}: ${tried.join(", ")}.`,
+ };
+}
+
+export async function probeAllCodecs(plan: ExportPlan, bitrate: number): Promise<CodecProbe[]> {
+ return Promise.all((["h264", "vp9", "av1"] as OutputCodecId[]).map((id) => probeCodec(id, plan, bitrate)));
+}
+
+/* ---------------------------------------------------------------------------
+ Stage 3 — process. WebGPU first, 2D canvas only if the GPU path is missing.
+ --------------------------------------------------------------------------- */
+
+export type CompositorKind = "webgpu" | "canvas2d";
+
+interface Compositor {
+ readonly kind: CompositorKind;
+ /** Paints the background alone — what a frame with no clip over it looks like. */
+ clear(): void;
+ /** Paints the background, then the frame scaled to fit inside it. */
+ draw(frame: VideoFrame): void;
+ /** The composited picture, ready for the encoder. Caller closes it. */
+ take(timestampUs: number, durationUs: number): VideoFrame;
+ dispose(): void;
+}
+
+function parseColor(hex: string): { r: number; g: number; b: number } {
+ const match = /^#([0-9a-fA-F]{6})$/.exec(hex.trim());
+ if (!match) return { r: 0, g: 0, b: 0 };
+ const value = Number.parseInt(match[1], 16);
+ return { r: (value >> 16) & 255, g: (value >> 8) & 255, b: value & 255 };
+}
+
+const COMPOSITOR_SHADER = /* wgsl */ `
+struct VertexOut {
+ @builtin(position) position: vec4f,
+ @location(0) uv: vec2f,
+};
+
+// x0, y0, x1, y1 of the fitted picture in clip space.
+@group(0) @binding(0) var<uniform> rect: vec4f;
+@group(0) @binding(1) var source: texture_external;
+@group(0) @binding(2) var source_sampler: sampler;
+
+@vertex
+fn vs(@builtin(vertex_index) index: u32) -> VertexOut {
+ var corners = array<vec2f, 6>(
+ vec2f(0.0, 0.0), vec2f(1.0, 0.0), vec2f(0.0, 1.0),
+ vec2f(0.0, 1.0), vec2f(1.0, 0.0), vec2f(1.0, 1.0),
+ );
+ let corner = corners[index];
+ var out: VertexOut;
+ out.position = vec4f(mix(rect.x, rect.z, corner.x), mix(rect.y, rect.w, corner.y), 0.0, 1.0);
+ out.uv = vec2f(corner.x, 1.0 - corner.y);
+ return out;
+}
+
+@fragment
+fn fs(@location(0) uv: vec2f) -> @location(0) vec4f {
+ return textureSampleBaseClampToEdge(source, source_sampler, uv);
+}
+`;
+
+async function createWebGpuCompositor(width: number, height: number, background: string): Promise<Compositor | null> {
+ if (typeof navigator === "undefined" || !("gpu" in navigator) || !navigator.gpu) return null;
+ const adapter = await navigator.gpu.requestAdapter();
+ if (!adapter) return null;
+ const device = await adapter.requestDevice();
+ const canvas = new OffscreenCanvas(width, height);
+ const context = canvas.getContext("webgpu");
+ if (!context) return null;
+ const format = navigator.gpu.getPreferredCanvasFormat();
+ context.configure({ device, format, alphaMode: "opaque" });
+ const shader = device.createShaderModule({ code: COMPOSITOR_SHADER });
+ const pipeline = device.createRenderPipeline({
+ layout: "auto",
+ vertex: { module: shader, entryPoint: "vs" },
+ fragment: { module: shader, entryPoint: "fs", targets: [{ format }] },
+ primitive: { topology: "triangle-list" },
+ });
+ const uniform = device.createBuffer({ size: 16, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST });
+ const sampler = device.createSampler({ magFilter: "linear", minFilter: "linear" });
+ const { r, g, b } = parseColor(background);
+ /* The canvas format is not an -srgb one, so the clear value lands in the
+ texture as written: the project's own background colour, unconverted. */
+ const clearValue = { r: r / 255, g: g / 255, b: b / 255, a: 1 };
+
+ const pass = (view: GPUTextureView, bindGroup: GPUBindGroup | null) => {
+ const encoder = device.createCommandEncoder();
+ const renderPass = encoder.beginRenderPass({
+ colorAttachments: [{ view, clearValue, loadOp: "clear", storeOp: "store" }],
+ });
+ if (bindGroup) {
+ renderPass.setPipeline(pipeline);
+ renderPass.setBindGroup(0, bindGroup);
+ renderPass.draw(6);
+ }
+ renderPass.end();
+ device.queue.submit([encoder.finish()]);
+ };
+
+ return {
+ kind: "webgpu",
+ clear() {
+ pass(context.getCurrentTexture().createView(), null);
+ },
+ draw(frame) {
+ const box = fitRect(frame.displayWidth, frame.displayHeight, width, height);
+ const x0 = (box.x / width) * 2 - 1;
+ const x1 = ((box.x + box.width) / width) * 2 - 1;
+ const yTop = 1 - (box.y / height) * 2;
+ const yBottom = 1 - ((box.y + box.height) / height) * 2;
+ device.queue.writeBuffer(uniform, 0, new Float32Array([x0, yBottom, x1, yTop]));
+ // Zero copy: the decoded frame is sampled where it already lives.
+ const external = device.importExternalTexture({ source: frame });
+ const bindGroup = device.createBindGroup({
+ layout: pipeline.getBindGroupLayout(0),
+ entries: [
+ { binding: 0, resource: { buffer: uniform } },
+ { binding: 1, resource: external },
+ { binding: 2, resource: sampler },
+ ],
+ });
+ pass(context.getCurrentTexture().createView(), bindGroup);
+ },
+ take(timestamp, duration) {
+ return new VideoFrame(canvas, { timestamp, duration, alpha: "discard" });
+ },
+ dispose() {
+ device.destroy();
+ },
+ };
+}
+
+function createCanvasCompositor(width: number, height: number, background: string): Compositor {
+ const canvas = new OffscreenCanvas(width, height);
+ const context = canvas.getContext("2d", { alpha: false });
+ if (!context) throw new Error("This computer gave no 2D canvas to composite into.");
+ const paintBackground = () => {
+ context.fillStyle = background;
+ context.fillRect(0, 0, width, height);
+ };
+ return {
+ kind: "canvas2d",
+ clear: paintBackground,
+ draw(frame) {
+ paintBackground();
+ const box = fitRect(frame.displayWidth, frame.displayHeight, width, height);
+ context.drawImage(frame, box.x, box.y, box.width, box.height);
+ },
+ take(timestamp, duration) {
+ return new VideoFrame(canvas, { timestamp, duration, alpha: "discard" });
+ },
+ dispose() {},
+ };
+}
+
+/* ---------------------------------------------------------------------------
+ Stage 1 — demux
+ --------------------------------------------------------------------------- */
+
+interface DemuxedSource {
+ config: VideoDecoderConfig;
+ samples: Array<{ data: Uint8Array; timestampUs: number; durationUs: number; key: boolean }>;
+}
+
+async function readAssetBytes(folderPath: string, asset: ProjectAsset): Promise<ArrayBuffer> {
+ if (!isTauri()) {
+ throw new Error("Reading project media needs the desktop app; the browser preview has no project folder.");
+ }
+ return invoke<ArrayBuffer>("read_project_file", { folderPath, relativePath: asset.relativePath });
+}
+
+/** Any of the codec configuration boxes. They all serialise themselves the
+ * same way; the declared parameter type is only the stream subclass mp4box
+ * happens to use internally, and `write` touches nothing beyond DataStream. */
+type ConfigurationBox = { write(stream: unknown): void };
+
+async function demux(bytes: ArrayBuffer, name: string): Promise<DemuxedSource> {
+ const { createFile, DataStream, Endianness, MP4BoxBuffer } = await import("mp4box");
+ type Movie = import("mp4box").Movie;
+ type Sample = import("mp4box").Sample;
+ const file = createFile();
+ const raw: Sample[] = [];
+ /* Held on an object rather than in `let`s: these are written from callbacks,
+ and TypeScript would otherwise narrow them to their initial value. */
+ const state: { movie: Movie | null; failure: string | null } = { movie: null, failure: null };
+
+ file.onError = (module, message) => {
+ state.failure = `${module}: ${message}`;
+ };
+ file.onReady = (info) => {
+ state.movie = info;
+ const track = info.videoTracks[0];
+ if (!track) return;
+ file.setExtractionOptions(track.id, null, { nbSamples: Number.MAX_SAFE_INTEGER });
+ file.start();
+ };
+ file.onSamples = (_id, _user, samples) => {
+ for (const sample of samples) raw.push(sample);
+ };
+ file.appendBuffer(MP4BoxBuffer.fromArrayBuffer(bytes, 0), true);
+ file.flush();
+
+ if (state.failure) throw new Error(`${name} could not be read: ${state.failure}`);
+ if (!state.movie) throw new Error(`${name} has no MP4 header PolStudio could parse.`);
+ const track = state.movie.videoTracks[0];
+ if (!track) throw new Error(`${name} has no video track in it.`);
+ if (raw.length === 0) throw new Error(`${name} has a video track with no readable frames in it.`);
+
+ /* The decoder needs the codec's own configuration record (avcC/hvcC/…) byte
+ for byte. Ask the parsed box to write itself and drop the 8-byte box
+ header, which is exactly what a VideoDecoderConfig description is. */
+ let description: Uint8Array | undefined;
+ const trak = file.getTrackById(track.id);
+ for (const entry of trak.mdia.minf.stbl.stsd.entries) {
+ const visual = entry as import("mp4box").VisualSampleEntry;
+ const box = (visual.avcC ?? visual.hvcC ?? visual.vpcC ?? visual.av1C) as unknown as ConfigurationBox | undefined;
+ if (!box) continue;
+ const stream = new DataStream(undefined, 0, Endianness.BIG_ENDIAN);
+ box.write(stream);
+ description = new Uint8Array(stream.buffer.slice(8));
+ break;
+ }
+
+ const samples = raw.map((sample) => {
+ if (!sample.data) throw new Error(`${name} has frames whose data could not be read out of the file.`);
+ return {
+ data: sample.data,
+ timestampUs: Math.round((sample.cts * 1_000_000) / sample.timescale),
+ durationUs: Math.round((sample.duration * 1_000_000) / sample.timescale),
+ key: sample.is_sync,
+ };
+ });
+
+ return {
+ config: {
+ codec: track.codec,
+ codedWidth: track.video?.width ?? track.track_width,
+ codedHeight: track.video?.height ?? track.track_height,
+ ...(description ? { description } : {}),
+ },
+ samples,
+ };
+}
+
+/* ---------------------------------------------------------------------------
+ The run
+ --------------------------------------------------------------------------- */
+
+export type ExportPhase = "preparing" | "rendering" | "finishing" | "writing" | "done";
+
+export interface ExportProgress {
+ phase: ExportPhase;
+ framesDone: number;
+ frameCount: number;
+ detail: string;
+}
+
+export class ExportCancelled extends Error {
+ constructor() {
+ super("Export cancelled.");
+ this.name = "ExportCancelled";
+ }
+}
+
+export interface ExportResult {
+ bytes: Uint8Array;
+ compositor: CompositorKind;
+ codecString: string;
+}
+
+export interface ExportRunOptions {
+ folderPath: string;
+ config: ProjectConfig;
+ settings: ExportSettings;
+ plan: ExportPlan;
+ bitrate: number;
+ onProgress: (progress: ExportProgress) => void;
+ /** Polled once per frame. Returning true aborts with `ExportCancelled`. */
+ cancelled: () => boolean;
+}
+
+const tick = () => new Promise<void>((resolve) => setTimeout(resolve, 0));
+
+export async function runExport(options: ExportRunOptions): Promise<ExportResult> {
+ const { plan, settings, config, folderPath, bitrate, onProgress, cancelled } = options;
+ const support = detectExportSupport();
+ if (!support.encoder || !support.decoder) {
+ throw new Error("This webview has no WebCodecs VideoEncoder/VideoDecoder, so PolStudio cannot encode video here.");
+ }
+ if (plan.blockers.length > 0) throw new Error(plan.blockers.join(" "));
+ if (plan.frameCount === 0) throw new Error("There is nothing on the timeline to render.");
+
+ const probe = await probeCodec(settings.codec, plan, bitrate);
+ if (!probe.supported || !probe.codecString) throw new Error(probe.detail);
+
+ const report = (phase: ExportPhase, framesDone: number, detail: string) =>
+ onProgress({ phase, framesDone, frameCount: plan.frameCount, detail });
+ const stopIfCancelled = () => {
+ if (cancelled()) throw new ExportCancelled();
+ };
+
+ report("preparing", 0, "Setting up the encoder…");
+ const { ArrayBufferTarget, Muxer } = await import("mp4-muxer");
+ const target = new ArrayBufferTarget();
+ const muxer = new Muxer({
+ target,
+ video: {
+ codec: outputCodec(settings.codec).muxer,
+ width: plan.width,
+ height: plan.height,
+ frameRate: plan.frameRate,
+ },
+ fastStart: "in-memory",
+ });
+
+ /* On an object because the encoder writes it from its own callback, where
+ TypeScript's flow analysis cannot follow the assignment. */
+ const encoderFailure: { reason: Error | null } = { reason: null };
+ const encoder = new VideoEncoder({
+ output: (chunk, meta) => muxer.addVideoChunk(chunk, meta),
+ error: (reason) => {
+ encoderFailure.reason = reason instanceof Error ? reason : new Error(String(reason));
+ },
+ });
+ encoder.configure(encoderConfig(probe.codecString, plan, bitrate));
+
+ const compositor =
+ (await createWebGpuCompositor(plan.width, plan.height, plan.backgroundColor).catch(() => null)) ??
+ createCanvasCompositor(plan.width, plan.height, plan.backgroundColor);
+
+ const frameDurationUs = Math.round(1_000_000 / plan.frameRate);
+ /* A keyframe every two seconds. Without them a player cannot seek, and a
+ 30-minute delta chain is also where quality quietly collapses. */
+ const keyFrameInterval = Math.max(1, Math.round(plan.frameRate * 2));
+ let framesDone = 0;
+
+ const emit = async (frame: VideoFrame | null, outputIndex: number) => {
+ stopIfCancelled();
+ if (encoderFailure.reason) throw encoderFailure.reason;
+ if (frame) compositor.draw(frame);
+ else compositor.clear();
+ const composited = compositor.take(outputIndex * frameDurationUs, frameDurationUs);
+ encoder.encode(composited, { keyFrame: outputIndex % keyFrameInterval === 0 });
+ composited.close();
+ framesDone += 1;
+ if (framesDone % 5 === 0 || framesDone === plan.frameCount) {
+ report("rendering", framesDone, `Rendering frame ${framesDone} of ${plan.frameCount}.`);
+ }
+ // Yield so the encoder's own queue drains and the window stays alive.
+ while (encoder.encodeQueueSize > 8) await tick();
+ if (framesDone % 12 === 0) await tick();
+ };
+
+ /* One decoded source at a time. Holding every clip's samples would mean
+ holding every clip's file, which for 4K footage is gigabytes. Kept on an
+ object so the closure below can replace it without TypeScript pinning the
+ fields to their initial null. */
+ const loaded: {
+ assetId: string | null;
+ source: DemuxedSource | null;
+ image: VideoFrame | null;
+ bitmap: ImageBitmap | null;
+ } = { assetId: null, source: null, image: null, bitmap: null };
+
+ const releaseLoaded = () => {
+ loaded.image?.close();
+ loaded.bitmap?.close();
+ loaded.image = null;
+ loaded.bitmap = null;
+ loaded.source = null;
+ };
+
+ const assetFor = (assetId: string) => config.assets.find((asset) => asset.id === assetId);
+
+ const loadAsset = async (asset: ProjectAsset) => {
+ if (loaded.assetId === asset.id) return;
+ releaseLoaded();
+ loaded.assetId = asset.id;
+ report("rendering", framesDone, `Reading ${asset.name}…`);
+ const bytes = await readAssetBytes(folderPath, asset);
+ if (asset.mimeType.startsWith("image/")) {
+ /* The bitmap stays alive as long as the frame made from it does: a
+ VideoFrame built from a CanvasImageSource is not guaranteed to own a
+ copy of the pixels. */
+ loaded.bitmap = await createImageBitmap(new Blob([bytes], { type: asset.mimeType }));
+ loaded.image = new VideoFrame(loaded.bitmap, { timestamp: 0 });
+ return;
+ }
+ loaded.source = await demux(bytes, asset.name);
+ };
+
+ const renderClipSegment = async (segment: Extract<ExportSegment, { kind: "clip" }>) => {
+ const asset = assetFor(segment.assetId);
+ if (!asset) throw new Error(`Clip “${segment.label}” points at media that is not in this project.`);
+ await loadAsset(asset);
+
+ const still = loaded.image;
+ if (still) {
+ for (let index = segment.startFrame; index < segment.endFrame; index += 1) await emit(still, index);
+ return;
+ }
+ const source = loaded.source;
+ if (!source) throw new Error(`${asset.name} produced no decodable video.`);
+
+ const startUs = sourceTimeMsForFrame(segment, segment.startFrame, plan.frameRate) * 1000;
+ const endUs = sourceTimeMsForFrame(segment, segment.endFrame - 1, plan.frameRate) * 1000;
+ /* Decoding has to begin at the keyframe at or before the trim point: a
+ delta frame on its own decodes to garbage. Everything decoded before the
+ trim point is fed to the decoder and then thrown away, never shown.
+
+ Samples are in DECODE order, so their timestamps are not monotonic once a
+ file uses B-frames. Hence a scan for the best keyframe rather than a
+ break on the first late sample, and a margin past the last wanted frame. */
+ let first = 0;
+ let keyframeUs = Number.NEGATIVE_INFINITY;
+ for (let index = 0; index < source.samples.length; index += 1) {
+ const sample = source.samples[index];
+ if (sample.key && sample.timestampUs <= startUs && sample.timestampUs > keyframeUs) {
+ keyframeUs = sample.timestampUs;
+ first = index;
+ }
+ }
+ const REORDER_MARGIN = 8;
+ let last = source.samples.length - 1;
+ for (let index = first; index < source.samples.length; index += 1) {
+ if (source.samples[index].timestampUs > endUs) {
+ last = Math.min(source.samples.length - 1, index + REORDER_MARGIN);
+ break;
+ }
+ }
+
+ let nextOutput = segment.startFrame;
+ /* The frame currently on screen. On an object for the same reason as the
+ failure below: it is replaced inside `drain`, and the cleanup that has to
+ close it lives outside. */
+ const on: { screen: VideoFrame | null } = { screen: null };
+ const failure: { reason: Error | null } = { reason: null };
+ const wantUs = (index: number) => sourceTimeMsForFrame(segment, index, plan.frameRate) * 1000;
+
+ const pending: VideoFrame[] = [];
+ const decoder = new VideoDecoder({
+ output: (frame) => pending.push(frame),
+ error: (reason) => {
+ failure.reason = reason instanceof Error ? reason : new Error(String(reason));
+ },
+ });
+ decoder.configure(source.config);
+
+ /* Frames arrive in presentation order, so the frame showing at output time
+ W is the last one whose timestamp is at or before W. Each decoded frame
+ therefore settles the output frames that came before IT, and the source
+ is only released once its successor has arrived. */
+ const drain = async (final: boolean) => {
+ while (pending.length > 0) {
+ const frame = pending.shift() as VideoFrame;
+ if (on.screen) {
+ while (nextOutput < segment.endFrame && wantUs(nextOutput) < frame.timestamp) {
+ await emit(on.screen, nextOutput);
+ nextOutput += 1;
+ }
+ on.screen.close();
+ }
+ on.screen = frame;
+ if (nextOutput >= segment.endFrame) break;
+ }
+ if (final) {
+ if (!on.screen && nextOutput < segment.endFrame) {
+ throw new Error(`${asset.name} decoded no frames, so clip “${segment.label}” has no picture to export.`);
+ }
+ while (nextOutput < segment.endFrame) {
+ await emit(on.screen, nextOutput);
+ nextOutput += 1;
+ }
+ }
+ };
+
+ try {
+ for (let index = first; index <= last && nextOutput < segment.endFrame; index += 1) {
+ if (failure.reason) throw failure.reason;
+ const sample = source.samples[index];
+ decoder.decode(
+ new EncodedVideoChunk({
+ type: sample.key ? "key" : "delta",
+ timestamp: sample.timestampUs,
+ duration: sample.durationUs,
+ data: sample.data,
+ }),
+ );
+ while (decoder.decodeQueueSize > 12) await tick();
+ await drain(false);
+ }
+ if (nextOutput < segment.endFrame) {
+ await decoder.flush();
+ await drain(false);
+ }
+ if (failure.reason) throw failure.reason;
+ await drain(true);
+ } finally {
+ on.screen?.close();
+ for (const frame of pending) frame.close();
+ if (decoder.state !== "closed") decoder.close();
+ }
+ };
+
+ try {
+ for (const segment of plan.segments) {
+ stopIfCancelled();
+ if (segment.kind === "gap") {
+ for (let index = segment.startFrame; index < segment.endFrame; index += 1) await emit(null, index);
+ } else {
+ await renderClipSegment(segment);
+ }
+ }
+
+ report("finishing", framesDone, "Flushing the encoder and writing the MP4 index…");
+ await encoder.flush();
+ if (encoderFailure.reason) throw encoderFailure.reason;
+ muxer.finalize();
+ report("done", framesDone, "Encoded.");
+ return {
+ bytes: new Uint8Array(target.buffer),
+ compositor: compositor.kind,
+ codecString: probe.codecString,
+ };
+ } finally {
+ releaseLoaded();
+ if (encoder.state !== "closed") encoder.close();
+ compositor.dispose();
+ }
+}
+
+/* ---------------------------------------------------------------------------
+ Saving the result
+ --------------------------------------------------------------------------- */
+
+/** The OS save dialog, run in Rust. Null means the user cancelled. */
+export async function chooseExportDestination(suggestedName: string): Promise<string | null> {
+ if (!isTauri()) return null;
+ return invoke<string | null>("choose_export_destination", { suggestedName });
+}
+
+/** Writes the muxed bytes to the chosen path and returns how many landed.
+ * The payload rides as a raw IPC body — a JSON array of numbers for a
+ * 100 MB file would be a gigabyte of text. */
+export async function writeExportFile(path: string, bytes: Uint8Array): Promise<number> {
+ return invoke<number>("write_export_file", bytes, { headers: { "x-export-path": encodeURIComponent(path) } });
+}
+
+/* ---------------------------------------------------------------------------
+ Preview
+ --------------------------------------------------------------------------- */
+
+/** Blob URLs for the project's own media, made once and revoked together.
+ * A leaked blob URL pins the whole file in memory for the life of the page. */
+export class PreviewSources {
+ private readonly urls = new Map<string, Promise<string>>();
+
+ constructor(private readonly folderPath: string) {}
+
+ url(asset: ProjectAsset): Promise<string> {
+ const existing = this.urls.get(asset.id);
+ if (existing) return existing;
+ const created = readAssetBytes(this.folderPath, asset).then((bytes) =>
+ URL.createObjectURL(new Blob([bytes], { type: asset.mimeType })),
+ );
+ this.urls.set(asset.id, created);
+ return created;
+ }
+
+ dispose(): void {
+ for (const pending of this.urls.values()) {
+ void pending.then(URL.revokeObjectURL).catch(() => undefined);
+ }
+ this.urls.clear();
+ }
+}
+
+const seeked = (video: HTMLVideoElement, event: string) =>
+ new Promise<void>((resolve, reject) => {
+ video.addEventListener(event, () => resolve(), { once: true });
+ video.addEventListener("error", () => reject(new Error("This file could not be decoded for preview.")), {
+ once: true,
+ });
+ });
+
+/** Draws the frame the export would produce at `timeMs` into `canvas`, using
+ * the same fit and the same background as the encoder. The picture is decoded
+ * from the real file by the webview's own hardware decoder — nothing here is
+ * drawn from imagination, and a source that cannot be read throws. */
+export async function drawPreviewFrame(options: {
+ canvas: HTMLCanvasElement;
+ plan: ExportPlan;
+ config: ProjectConfig;
+ sources: PreviewSources;
+ clip: { assetId: string; sourceTimeMs: number } | null;
+}): Promise<void> {
+ const { canvas, plan, config, sources, clip } = options;
+ const context = canvas.getContext("2d");
+ if (!context) throw new Error("This computer gave no 2D canvas to preview into.");
+ context.fillStyle = plan.backgroundColor;
+ context.fillRect(0, 0, canvas.width, canvas.height);
+ if (!clip) return;
+
+ const asset = config.assets.find((candidate) => candidate.id === clip.assetId);
+ if (!asset) throw new Error("This clip points at media that is not in this project.");
+ const url = await sources.url(asset);
+
+ if (asset.mimeType.startsWith("image/")) {
+ const image = new Image();
+ image.src = url;
+ await image.decode();
+ const box = fitRect(image.naturalWidth, image.naturalHeight, canvas.width, canvas.height);
+ context.drawImage(image, box.x, box.y, box.width, box.height);
+ return;
+ }
+
+ const video = document.createElement("video");
+ video.muted = true;
+ video.preload = "auto";
+ video.playsInline = true;
+ video.src = url;
+ try {
+ await seeked(video, "loadeddata");
+ const target = Math.max(0, clip.sourceTimeMs / 1000);
+ if (target > 0 && Number.isFinite(video.duration)) {
+ const landed = seeked(video, "seeked");
+ video.currentTime = Math.min(target, Math.max(0, video.duration - 1 / plan.frameRate));
+ await landed;
+ }
+ const box = fitRect(video.videoWidth, video.videoHeight, canvas.width, canvas.height);
+ context.drawImage(video, box.x, box.y, box.width, box.height);
+ } finally {
+ video.removeAttribute("src");
+ video.load();
+ }
+}