AI media harness for planning, generating, and editing locally
TypeScript
Add the export plan maths and the webcodecs pipeline
Bittid288638f3c3e
diff --git a/package-lock.json b/package-lock.jsonindex 80ff292..6222aed 100644--- a/package-lock.json+++ b/package-lock.json@@ -23,6 +23,7 @@"@types/react": "^18.3.18","@types/react-dom": "^18.3.5","@vitejs/plugin-react": "^4.3.4",+ "@webgpu/types": "^0.1.72","jsdom": "^25.0.1","typescript": "~5.7.2","vite": "^6.0.5",@@ -1887,6 +1888,13 @@"url": "https://opencollective.com/vitest"}},+ "node_modules/@webgpu/types": {+ "version": "0.1.72",+ "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.72.tgz",+ "integrity": "sha512-0cF7RFM2edNoiIS1ODJp0/Gzv4/xSXhwoR0YCza+OWpJWtn4wmo9DvK91aLlH9+uUnwIriP7ZiC3WitmyhuzBw==",+ "dev": true,+ "license": "BSD-3-Clause"+ },"node_modules/agent-base": {"version": "7.1.4","resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz",diff --git a/package.json b/package.jsonindex 08f9c84..d307d91 100644--- a/package.json+++ b/package.json@@ -26,6 +26,7 @@"@types/react": "^18.3.18","@types/react-dom": "^18.3.5","@vitejs/plugin-react": "^4.3.4",+ "@webgpu/types": "^0.1.72","jsdom": "^25.0.1","typescript": "~5.7.2","vite": "^6.0.5",diff --git a/src/lib/export.ts b/src/lib/export.tsnew file mode 100644index 0000000..64ba372--- /dev/null+++ b/src/lib/export.ts@@ -0,0 +1,397 @@+/* ============================================================================+ export.ts — what an export IS, worked out with arithmetic only.++ Everything in this file is pure: it takes a project and the user's export+ settings and works out the frame-by-frame shape of the output file. It never+ touches WebCodecs, WebGPU, mp4box or the filesystem — that is+ exportPipeline.ts, which consumes the plan this module produces.++ The split exists so the timing maths can be tested without a GPU: which clip+ is on screen at time T, and which moment of the source file that output frame+ has to come from, are the two questions an editor gets wrong silently.+ ========================================================================== */++import type { AspectRatio, ProjectConfig, Resolution, TimelineClip, TimelineTrack } from "./project";++export type FrameRate = ProjectConfig["settings"]["frameRate"];+export type OutputCodecId = "h264" | "vp9" | "av1";+export type QualityId = "draft" | "balanced" | "high";++export interface ExportSettings {+ resolution: Resolution;+ frameRate: FrameRate;+ codec: OutputCodecId;+ quality: QualityId;+}++export interface OutputCodec {+ id: OutputCodecId;+ label: string;+ /** The track codec name mp4-muxer expects. */+ muxer: "avc" | "vp9" | "av1";+ /** Codec strings tried in order against `VideoEncoder.isConfigSupported`. */+ candidates: (width: number, height: number, frameRate: number) => string[];+ detail: string;+}++/* H.264 levels, as the byte that ends an `avc1.` codec string. The encoder+ rejects a configuration whose level cannot hold the frame size and rate, so+ the level is derived from the actual output rather than hard-coded. */+function avcLevel(width: number, height: number, frameRate: number): string {+ const macroblocks = Math.ceil(width / 16) * Math.ceil(height / 16);+ const rate = macroblocks * frameRate;+ if (macroblocks <= 3600 && rate <= 108_000) return "1f"; // 3.1+ if (macroblocks <= 8192 && rate <= 245_760) return "28"; // 4.0+ if (macroblocks <= 8704 && rate <= 522_240) return "2a"; // 4.2+ if (macroblocks <= 22_080 && rate <= 589_824) return "33"; // 5.1+ return "34"; // 5.2+}++export const OUTPUT_CODECS: readonly OutputCodec[] = [+ {+ id: "h264",+ label: "H.264 / AVC",+ muxer: "avc",+ /* High, then Main, then Baseline at the same level. Hardware encoders+ differ on which profiles they expose, so the first one this machine+ actually accepts is the one used — never a guess. */+ candidates: (width, height, frameRate) => {+ const level = avcLevel(width, height, frameRate);+ return [`avc1.6400${level}`, `avc1.4d00${level}`, `avc1.4200${level}`];+ },+ detail: "Plays everywhere. The baseline choice for an .mp4.",+ },+ {+ id: "vp9",+ label: "VP9",+ muxer: "vp9",+ candidates: () => ["vp09.00.10.08"],+ detail: "Smaller files at the same quality; not every player reads VP9 inside .mp4.",+ },+ {+ id: "av1",+ label: "AV1",+ muxer: "av1",+ candidates: () => ["av01.0.08M.08", "av01.0.04M.08"],+ detail: "Newest codec, smallest files, slowest to encode and least widely supported.",+ },+];++export function outputCodec(id: OutputCodecId): OutputCodec {+ const codec = OUTPUT_CODECS.find((candidate) => candidate.id === id);+ if (!codec) throw new Error(`Unknown output codec: ${id}`);+ return codec;+}++export interface QualityPreset {+ id: QualityId;+ label: string;+ /** Bits spent per pixel per frame. Bitrate is derived, never typed in. */+ bitsPerPixel: number;+ detail: string;+}++export const QUALITY_PRESETS: readonly QualityPreset[] = [+ { id: "draft", label: "Draft", bitsPerPixel: 0.04, detail: "Fastest and smallest. Visible compression in motion." },+ { id: "balanced", label: "Balanced", bitsPerPixel: 0.09, detail: "The usual choice for a delivery file." },+ { id: "high", label: "High", bitsPerPixel: 0.16, detail: "Large files. Keeps detail in grain and fast motion." },+];++export function qualityPreset(id: QualityId): QualityPreset {+ const preset = QUALITY_PRESETS.find((candidate) => candidate.id === id);+ if (!preset) throw new Error(`Unknown quality preset: ${id}`);+ return preset;+}++/* The resolution names the SHORT edge, so one setting means the same amount of+ picture in every aspect ratio the project offers: 1080p is 1920×1080 wide,+ 1080×1920 tall, 1080×1080 square and 1080×1350 at 4:5. */+const SHORT_EDGE: Record<Resolution, number> = { "720p": 720, "1080p": 1080, "4k": 2160 };+const RATIO: Record<AspectRatio, readonly [number, number]> = {+ "16:9": [16, 9],+ "9:16": [9, 16],+ "1:1": [1, 1],+ "4:5": [4, 5],+};++/** Encoders reject odd dimensions in 4:2:0, so both edges land on an even number. */+const even = (value: number) => Math.max(2, Math.round(value / 2) * 2);++export function outputDimensions(resolution: Resolution, aspectRatio: AspectRatio): { width: number; height: number } {+ const base = SHORT_EDGE[resolution];+ const [ratioWidth, ratioHeight] = RATIO[aspectRatio];+ return ratioWidth >= ratioHeight+ ? { width: even((base * ratioWidth) / ratioHeight), height: even(base) }+ : { width: even(base), height: even((base * ratioHeight) / ratioWidth) };+}++/** The export starts as the project already describes itself. Nothing here is+ * invented: resolution and frame rate are the project's own settings. */+export function defaultExportSettings(config: ProjectConfig): ExportSettings {+ return {+ resolution: config.settings.resolution,+ frameRate: config.settings.frameRate,+ codec: "h264",+ quality: "balanced",+ };+}++export function bitrateFor(width: number, height: number, frameRate: number, quality: QualityId): number {+ return Math.round(width * height * frameRate * qualityPreset(quality).bitsPerPixel);+}++/** Bytes the encoder is being ASKED to produce. A rate-controlled encoder lands+ * near its target rather than on it, so this is labelled an estimate wherever+ * it is shown. */+export function estimatedBytes(bitrate: number, durationMs: number): number {+ return Math.round((bitrate / 8) * (durationMs / 1000));+}++/* ---------------------------------------------------------------------------+ The plan+ --------------------------------------------------------------------------- */++export type ExportSegment =+ | {+ kind: "clip";+ clipId: string;+ assetId: string;+ label: string;+ /** First output frame index this clip fills. */+ startFrame: number;+ /** One past the last output frame index it fills. */+ endFrame: number;+ /** The clip's own position on the timeline, in ms. */+ clipStartMs: number;+ /** Where inside the source file the clip begins, in ms. */+ clipSourceStartMs: number;+ }+ | { kind: "gap"; startFrame: number; endFrame: number };++export interface ExportPlan {+ width: number;+ height: number;+ frameRate: FrameRate;+ durationMs: number;+ frameCount: number;+ backgroundColor: string;+ segments: ExportSegment[];+ /** Video clips that contribute at least one frame to the file. */+ clipCount: number;+ /** Frames with nothing over them; they get the project's background colour. */+ gapFrames: number;+ /** Clips on audio tracks. They are NOT in the file — see `notes`. */+ audioClipCount: number;+ /** Reasons this export cannot run at all. Empty means it can. */+ blockers: string[];+ /** True things about the output the user has to be told before they press go. */+ notes: string[];+}++/** Container formats mp4box can demux today. webm and mkv import fine and play+ * in the app, but nothing here can take them apart yet, so a clip built on one+ * blocks the export instead of quietly rendering as background. */+const DEMUXABLE_VIDEO = new Set(["video/mp4", "video/quicktime"]);+const DRAWABLE_IMAGE = new Set(["image/png", "image/jpeg", "image/webp", "image/gif"]);++/** The moment output frame `frameIndex` has to be taken from inside the source+ * file, in milliseconds. This is the whole of the trim maths: an output frame+ * sits at `frameIndex / frameRate` on the timeline, and the source moment is+ * that far past the clip's start, offset by where the clip begins in its file. */+export function sourceTimeMsForFrame(+ segment: Extract<ExportSegment, { kind: "clip" }>,+ frameIndex: number,+ frameRate: number,+): number {+ return segment.clipSourceStartMs + ((frameIndex * 1000) / frameRate - segment.clipStartMs);+}++/** The clip a viewer sees at `timeMs`.+ *+ * Tracks are drawn top to bottom in array order, so the FIRST video track that+ * has something at this moment wins — an overlay covers the story beneath it.+ * Within one track two clips can overlap after a drag; the later one is the+ * more recent edit, so it is the one on top. */+export function visibleClipAt(tracks: TimelineTrack[], timeMs: number): TimelineClip | null {+ for (const track of tracks) {+ if (track.kind !== "video") continue;+ let best: TimelineClip | null = null;+ for (const clip of track.clips) {+ if (timeMs < clip.startMs || timeMs >= clip.startMs + clip.durationMs) continue;+ if (!best || clip.startMs >= best.startMs) best = clip;+ }+ if (best) return best;+ }+ return null;+}++/** Where the picture ends. Audio is not muxed yet, so a soundtrack that runs+ * past the last shot cannot extend the file — claiming otherwise would promise+ * frames that do not exist. */+export function videoDurationMs(config: ProjectConfig): number {+ return config.timeline.tracks+ .filter((track) => track.kind === "video")+ .flatMap((track) => track.clips)+ .reduce((end, clip) => Math.max(end, clip.startMs + clip.durationMs), 0);+}++export function buildExportPlan(config: ProjectConfig, settings: ExportSettings): ExportPlan {+ const { width, height } = outputDimensions(settings.resolution, config.settings.aspectRatio);+ const frameRate = settings.frameRate;+ const tracks = config.timeline.tracks;+ const videoTracks = tracks.filter((track) => track.kind === "video");+ const durationMs = videoDurationMs(config);+ const frameCount = Math.round((durationMs * frameRate) / 1000);++ /* Resolve every output frame first, then run the identical answers together.+ Deriving segments from clip boundaries instead would disagree with the+ encoder loop the moment two clips overlap. */+ const segments: ExportSegment[] = [];+ let gapFrames = 0;+ for (let frame = 0; frame < frameCount; frame += 1) {+ const clip = visibleClipAt(videoTracks, (frame * 1000) / frameRate);+ const last = segments[segments.length - 1];+ if (!clip) {+ gapFrames += 1;+ if (last && last.kind === "gap") last.endFrame = frame + 1;+ else segments.push({ kind: "gap", startFrame: frame, endFrame: frame + 1 });+ continue;+ }+ if (last && last.kind === "clip" && last.clipId === clip.id) {+ last.endFrame = frame + 1;+ continue;+ }+ segments.push({+ kind: "clip",+ clipId: clip.id,+ assetId: clip.assetId,+ label: clip.label,+ startFrame: frame,+ endFrame: frame + 1,+ clipStartMs: clip.startMs,+ clipSourceStartMs: clip.sourceStartMs,+ });+ }++ const usedClipIds = new Set(segments.flatMap((segment) => (segment.kind === "clip" ? [segment.clipId] : [])));+ const audioClipCount = tracks+ .filter((track) => track.kind === "audio")+ .reduce((total, track) => total + track.clips.length, 0);++ const blockers: string[] = [];+ if (frameCount === 0) {+ blockers.push("There are no video clips on the timeline, so there is nothing to render.");+ }+ const missing: string[] = [];+ const undecodable: string[] = [];+ for (const segment of segments) {+ if (segment.kind !== "clip") continue;+ const asset = config.assets.find((candidate) => candidate.id === segment.assetId);+ if (!asset) {+ missing.push(segment.label);+ continue;+ }+ const drawable = DEMUXABLE_VIDEO.has(asset.mimeType) || DRAWABLE_IMAGE.has(asset.mimeType);+ if (!drawable) undecodable.push(`${segment.label} (${asset.name}, ${asset.mimeType})`);+ }+ if (missing.length > 0) {+ blockers.push(`These clips point at media that is not in this project: ${missing.join(", ")}.`);+ }+ if (undecodable.length > 0) {+ blockers.push(+ `PolStudio can only take apart .mp4 and .mov video and still images so far. These clips use something else: ${undecodable.join(", ")}.`,+ );+ }++ const notes: string[] = [];+ if (audioClipCount > 0) {+ notes.push(+ `The file will have no sound. ${audioClipCount} audio ${audioClipCount === 1 ? "clip is" : "clips are"} on the timeline and none of them are muxed yet.`,+ );+ }+ if (gapFrames > 0) {+ notes.push(+ `${gapFrames} of ${frameCount} frames have no clip over them and will be solid ${config.settings.backgroundColor}.`,+ );+ }+ const unusedClips = videoTracks+ .flatMap((track) => track.clips)+ .filter((clip) => !usedClipIds.has(clip.id));+ if (unusedClips.length > 0) {+ notes.push(+ `${unusedClips.length} video ${unusedClips.length === 1 ? "clip is" : "clips are"} hidden underneath another track and will not appear.`,+ );+ }++ return {+ width,+ height,+ frameRate,+ durationMs,+ frameCount,+ backgroundColor: config.settings.backgroundColor,+ segments,+ clipCount: usedClipIds.size,+ gapFrames,+ audioClipCount,+ blockers,+ notes,+ };+}++/* ---------------------------------------------------------------------------+ Geometry shared by the preview and the compositor+ --------------------------------------------------------------------------- */++export interface FittedRect {+ x: number;+ y: number;+ width: number;+ height: number;+}++/** The source frame, scaled to fit inside the output and centred. Nothing is+ * cropped; whatever is left over is background. The preview and the export+ * compositor both call this, so what you see is the frame you get. */+export function fitRect(sourceWidth: number, sourceHeight: number, outWidth: number, outHeight: number): FittedRect {+ if (sourceWidth <= 0 || sourceHeight <= 0) return { x: 0, y: 0, width: outWidth, height: outHeight };+ const scale = Math.min(outWidth / sourceWidth, outHeight / sourceHeight);+ const width = sourceWidth * scale;+ const height = sourceHeight * scale;+ return { x: (outWidth - width) / 2, y: (outHeight - height) / 2, width, height };+}++/* ---------------------------------------------------------------------------+ Formatting+ --------------------------------------------------------------------------- */++export function formatBytes(bytes: number): string {+ if (bytes < 1000) return `${bytes} B`;+ const units = ["kB", "MB", "GB"];+ let value = bytes / 1000;+ let unit = 0;+ while (value >= 1000 && unit < units.length - 1) {+ value /= 1000;+ unit += 1;+ }+ return `${value.toFixed(value >= 100 ? 0 : 1)} ${units[unit]}`;+}++/** mm:ss.mmm — an editor needs the fraction, and hours would be noise for a+ * timeline measured in seconds. */+export function formatDuration(ms: number): string {+ const total = Math.max(0, Math.round(ms));+ const minutes = Math.floor(total / 60_000);+ const seconds = Math.floor((total % 60_000) / 1000);+ const millis = total % 1000;+ return `${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}.${String(millis).padStart(3, "0")}`;+}++/** A file name the OS will accept, derived from the project's own name. */+export function suggestedFileName(projectName: string): string {+ const cleaned = projectName+ .replace(/[\\/:*?"<>|]/g, " ")+ .replace(/\s+/g, " ")+ .trim();+ return `${cleaned || "PolStudio export"}.mp4`;+}diff --git a/src/lib/exportPipeline.ts b/src/lib/exportPipeline.tsnew file mode 100644index 0000000..59446cb--- /dev/null+++ b/src/lib/exportPipeline.ts@@ -0,0 +1,740 @@+/// <reference types="@webgpu/types" />++/* ============================================================================+ exportPipeline.ts — the five stages CLAUDE.md describes, in order.++ demux (mp4box.js) -> decode (WebCodecs VideoDecoder, OS hardware)+ -> process (WebGPU, importExternalTexture, no readback)+ -> encode (WebCodecs VideoEncoder) -> mux (mp4-muxer)++ No FFmpeg anywhere, deliberately: the whole point of this pipeline is to stay+ clear of GPL/LGPL binaries. The heavy libraries are imported dynamically so+ that neither the tests nor the first paint of the app pay for them.++ What this module will NOT do: pretend. If WebCodecs is missing, if the codec+ is unsupported, if a source file cannot be taken apart — it throws with the+ real reason and the view shows that reason verbatim.+ ========================================================================== */++import { invoke } from "@tauri-apps/api/core";+import { isTauri } from "./persistence";+import {+ fitRect,+ outputCodec,+ sourceTimeMsForFrame,+ type ExportPlan,+ type ExportSegment,+ type ExportSettings,+ type OutputCodecId,+} from "./export";+import type { ProjectAsset, ProjectConfig } from "./project";++/* ---------------------------------------------------------------------------+ What this machine can do+ --------------------------------------------------------------------------- */++export interface ExportSupport {+ /** WebCodecs encode. Present in WebView2/Chromium, absent in older engines. */+ encoder: boolean;+ decoder: boolean;+ /** The GPU compositor. Absent is not fatal — there is a 2D canvas fallback. */+ webgpu: boolean;+ /** Only the desktop shell has files to read and a place to write one. */+ desktop: boolean;+}++export function detectExportSupport(): ExportSupport {+ const scope = globalThis as unknown as Record<string, unknown>;+ return {+ encoder: typeof scope.VideoEncoder === "function",+ decoder: typeof scope.VideoDecoder === "function",+ webgpu: typeof navigator !== "undefined" && "gpu" in navigator && Boolean(navigator.gpu),+ desktop: isTauri(),+ };+}++export interface CodecProbe {+ id: OutputCodecId;+ supported: boolean;+ /** The codec string this machine accepted, when it accepted one. */+ codecString: string | null;+ /** Why not, in the encoder's own terms, when it did not. */+ detail: string;+}++function encoderConfig(codecString: string, plan: ExportPlan, bitrate: number): VideoEncoderConfig {+ return {+ codec: codecString,+ width: plan.width,+ height: plan.height,+ bitrate,+ framerate: plan.frameRate,+ // Length-prefixed AVC with the parameter sets in the sample description,+ // which is the only shape mp4-muxer can put in an .mp4.+ avc: { format: "avc" },+ };+}++/** Asks the encoder itself, one candidate codec string at a time. Nothing here+ * is assumed from the codec name. */+export async function probeCodec(id: OutputCodecId, plan: ExportPlan, bitrate: number): Promise<CodecProbe> {+ const codec = outputCodec(id);+ if (typeof VideoEncoder === "undefined") {+ return { id, supported: false, codecString: null, detail: "This webview has no WebCodecs VideoEncoder." };+ }+ const tried: string[] = [];+ for (const candidate of codec.candidates(plan.width, plan.height, plan.frameRate)) {+ tried.push(candidate);+ try {+ const support = await VideoEncoder.isConfigSupported(encoderConfig(candidate, plan, bitrate));+ if (support.supported) return { id, supported: true, codecString: candidate, detail: candidate };+ } catch (reason) {+ return {+ id,+ supported: false,+ codecString: null,+ detail: reason instanceof Error ? reason.message : String(reason),+ };+ }+ }+ return {+ id,+ supported: false,+ codecString: null,+ detail: `This computer's encoder refused every profile tried at ${plan.width}×${plan.height}: ${tried.join(", ")}.`,+ };+}++export async function probeAllCodecs(plan: ExportPlan, bitrate: number): Promise<CodecProbe[]> {+ return Promise.all((["h264", "vp9", "av1"] as OutputCodecId[]).map((id) => probeCodec(id, plan, bitrate)));+}++/* ---------------------------------------------------------------------------+ Stage 3 — process. WebGPU first, 2D canvas only if the GPU path is missing.+ --------------------------------------------------------------------------- */++export type CompositorKind = "webgpu" | "canvas2d";++interface Compositor {+ readonly kind: CompositorKind;+ /** Paints the background alone — what a frame with no clip over it looks like. */+ clear(): void;+ /** Paints the background, then the frame scaled to fit inside it. */+ draw(frame: VideoFrame): void;+ /** The composited picture, ready for the encoder. Caller closes it. */+ take(timestampUs: number, durationUs: number): VideoFrame;+ dispose(): void;+}++function parseColor(hex: string): { r: number; g: number; b: number } {+ const match = /^#([0-9a-fA-F]{6})$/.exec(hex.trim());+ if (!match) return { r: 0, g: 0, b: 0 };+ const value = Number.parseInt(match[1], 16);+ return { r: (value >> 16) & 255, g: (value >> 8) & 255, b: value & 255 };+}++const COMPOSITOR_SHADER = /* wgsl */ `+struct VertexOut {+ @builtin(position) position: vec4f,+ @location(0) uv: vec2f,+};++// x0, y0, x1, y1 of the fitted picture in clip space.+@group(0) @binding(0) var<uniform> rect: vec4f;+@group(0) @binding(1) var source: texture_external;+@group(0) @binding(2) var source_sampler: sampler;++@vertex+fn vs(@builtin(vertex_index) index: u32) -> VertexOut {+ var corners = array<vec2f, 6>(+ vec2f(0.0, 0.0), vec2f(1.0, 0.0), vec2f(0.0, 1.0),+ vec2f(0.0, 1.0), vec2f(1.0, 0.0), vec2f(1.0, 1.0),+ );+ let corner = corners[index];+ var out: VertexOut;+ out.position = vec4f(mix(rect.x, rect.z, corner.x), mix(rect.y, rect.w, corner.y), 0.0, 1.0);+ out.uv = vec2f(corner.x, 1.0 - corner.y);+ return out;+}++@fragment+fn fs(@location(0) uv: vec2f) -> @location(0) vec4f {+ return textureSampleBaseClampToEdge(source, source_sampler, uv);+}+`;++async function createWebGpuCompositor(width: number, height: number, background: string): Promise<Compositor | null> {+ if (typeof navigator === "undefined" || !("gpu" in navigator) || !navigator.gpu) return null;+ const adapter = await navigator.gpu.requestAdapter();+ if (!adapter) return null;+ const device = await adapter.requestDevice();+ const canvas = new OffscreenCanvas(width, height);+ const context = canvas.getContext("webgpu");+ if (!context) return null;+ const format = navigator.gpu.getPreferredCanvasFormat();+ context.configure({ device, format, alphaMode: "opaque" });+ const shader = device.createShaderModule({ code: COMPOSITOR_SHADER });+ const pipeline = device.createRenderPipeline({+ layout: "auto",+ vertex: { module: shader, entryPoint: "vs" },+ fragment: { module: shader, entryPoint: "fs", targets: [{ format }] },+ primitive: { topology: "triangle-list" },+ });+ const uniform = device.createBuffer({ size: 16, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST });+ const sampler = device.createSampler({ magFilter: "linear", minFilter: "linear" });+ const { r, g, b } = parseColor(background);+ /* The canvas format is not an -srgb one, so the clear value lands in the+ texture as written: the project's own background colour, unconverted. */+ const clearValue = { r: r / 255, g: g / 255, b: b / 255, a: 1 };++ const pass = (view: GPUTextureView, bindGroup: GPUBindGroup | null) => {+ const encoder = device.createCommandEncoder();+ const renderPass = encoder.beginRenderPass({+ colorAttachments: [{ view, clearValue, loadOp: "clear", storeOp: "store" }],+ });+ if (bindGroup) {+ renderPass.setPipeline(pipeline);+ renderPass.setBindGroup(0, bindGroup);+ renderPass.draw(6);+ }+ renderPass.end();+ device.queue.submit([encoder.finish()]);+ };++ return {+ kind: "webgpu",+ clear() {+ pass(context.getCurrentTexture().createView(), null);+ },+ draw(frame) {+ const box = fitRect(frame.displayWidth, frame.displayHeight, width, height);+ const x0 = (box.x / width) * 2 - 1;+ const x1 = ((box.x + box.width) / width) * 2 - 1;+ const yTop = 1 - (box.y / height) * 2;+ const yBottom = 1 - ((box.y + box.height) / height) * 2;+ device.queue.writeBuffer(uniform, 0, new Float32Array([x0, yBottom, x1, yTop]));+ // Zero copy: the decoded frame is sampled where it already lives.+ const external = device.importExternalTexture({ source: frame });+ const bindGroup = device.createBindGroup({+ layout: pipeline.getBindGroupLayout(0),+ entries: [+ { binding: 0, resource: { buffer: uniform } },+ { binding: 1, resource: external },+ { binding: 2, resource: sampler },+ ],+ });+ pass(context.getCurrentTexture().createView(), bindGroup);+ },+ take(timestamp, duration) {+ return new VideoFrame(canvas, { timestamp, duration, alpha: "discard" });+ },+ dispose() {+ device.destroy();+ },+ };+}++function createCanvasCompositor(width: number, height: number, background: string): Compositor {+ const canvas = new OffscreenCanvas(width, height);+ const context = canvas.getContext("2d", { alpha: false });+ if (!context) throw new Error("This computer gave no 2D canvas to composite into.");+ const paintBackground = () => {+ context.fillStyle = background;+ context.fillRect(0, 0, width, height);+ };+ return {+ kind: "canvas2d",+ clear: paintBackground,+ draw(frame) {+ paintBackground();+ const box = fitRect(frame.displayWidth, frame.displayHeight, width, height);+ context.drawImage(frame, box.x, box.y, box.width, box.height);+ },+ take(timestamp, duration) {+ return new VideoFrame(canvas, { timestamp, duration, alpha: "discard" });+ },+ dispose() {},+ };+}++/* ---------------------------------------------------------------------------+ Stage 1 — demux+ --------------------------------------------------------------------------- */++interface DemuxedSource {+ config: VideoDecoderConfig;+ samples: Array<{ data: Uint8Array; timestampUs: number; durationUs: number; key: boolean }>;+}++async function readAssetBytes(folderPath: string, asset: ProjectAsset): Promise<ArrayBuffer> {+ if (!isTauri()) {+ throw new Error("Reading project media needs the desktop app; the browser preview has no project folder.");+ }+ return invoke<ArrayBuffer>("read_project_file", { folderPath, relativePath: asset.relativePath });+}++/** Any of the codec configuration boxes. They all serialise themselves the+ * same way; the declared parameter type is only the stream subclass mp4box+ * happens to use internally, and `write` touches nothing beyond DataStream. */+type ConfigurationBox = { write(stream: unknown): void };++async function demux(bytes: ArrayBuffer, name: string): Promise<DemuxedSource> {+ const { createFile, DataStream, Endianness, MP4BoxBuffer } = await import("mp4box");+ type Movie = import("mp4box").Movie;+ type Sample = import("mp4box").Sample;+ const file = createFile();+ const raw: Sample[] = [];+ /* Held on an object rather than in `let`s: these are written from callbacks,+ and TypeScript would otherwise narrow them to their initial value. */+ const state: { movie: Movie | null; failure: string | null } = { movie: null, failure: null };++ file.onError = (module, message) => {+ state.failure = `${module}: ${message}`;+ };+ file.onReady = (info) => {+ state.movie = info;+ const track = info.videoTracks[0];+ if (!track) return;+ file.setExtractionOptions(track.id, null, { nbSamples: Number.MAX_SAFE_INTEGER });+ file.start();+ };+ file.onSamples = (_id, _user, samples) => {+ for (const sample of samples) raw.push(sample);+ };+ file.appendBuffer(MP4BoxBuffer.fromArrayBuffer(bytes, 0), true);+ file.flush();++ if (state.failure) throw new Error(`${name} could not be read: ${state.failure}`);+ if (!state.movie) throw new Error(`${name} has no MP4 header PolStudio could parse.`);+ const track = state.movie.videoTracks[0];+ if (!track) throw new Error(`${name} has no video track in it.`);+ if (raw.length === 0) throw new Error(`${name} has a video track with no readable frames in it.`);++ /* The decoder needs the codec's own configuration record (avcC/hvcC/…) byte+ for byte. Ask the parsed box to write itself and drop the 8-byte box+ header, which is exactly what a VideoDecoderConfig description is. */+ let description: Uint8Array | undefined;+ const trak = file.getTrackById(track.id);+ for (const entry of trak.mdia.minf.stbl.stsd.entries) {+ const visual = entry as import("mp4box").VisualSampleEntry;+ const box = (visual.avcC ?? visual.hvcC ?? visual.vpcC ?? visual.av1C) as unknown as ConfigurationBox | undefined;+ if (!box) continue;+ const stream = new DataStream(undefined, 0, Endianness.BIG_ENDIAN);+ box.write(stream);+ description = new Uint8Array(stream.buffer.slice(8));+ break;+ }++ const samples = raw.map((sample) => {+ if (!sample.data) throw new Error(`${name} has frames whose data could not be read out of the file.`);+ return {+ data: sample.data,+ timestampUs: Math.round((sample.cts * 1_000_000) / sample.timescale),+ durationUs: Math.round((sample.duration * 1_000_000) / sample.timescale),+ key: sample.is_sync,+ };+ });++ return {+ config: {+ codec: track.codec,+ codedWidth: track.video?.width ?? track.track_width,+ codedHeight: track.video?.height ?? track.track_height,+ ...(description ? { description } : {}),+ },+ samples,+ };+}++/* ---------------------------------------------------------------------------+ The run+ --------------------------------------------------------------------------- */++export type ExportPhase = "preparing" | "rendering" | "finishing" | "writing" | "done";++export interface ExportProgress {+ phase: ExportPhase;+ framesDone: number;+ frameCount: number;+ detail: string;+}++export class ExportCancelled extends Error {+ constructor() {+ super("Export cancelled.");+ this.name = "ExportCancelled";+ }+}++export interface ExportResult {+ bytes: Uint8Array;+ compositor: CompositorKind;+ codecString: string;+}++export interface ExportRunOptions {+ folderPath: string;+ config: ProjectConfig;+ settings: ExportSettings;+ plan: ExportPlan;+ bitrate: number;+ onProgress: (progress: ExportProgress) => void;+ /** Polled once per frame. Returning true aborts with `ExportCancelled`. */+ cancelled: () => boolean;+}++const tick = () => new Promise<void>((resolve) => setTimeout(resolve, 0));++export async function runExport(options: ExportRunOptions): Promise<ExportResult> {+ const { plan, settings, config, folderPath, bitrate, onProgress, cancelled } = options;+ const support = detectExportSupport();+ if (!support.encoder || !support.decoder) {+ throw new Error("This webview has no WebCodecs VideoEncoder/VideoDecoder, so PolStudio cannot encode video here.");+ }+ if (plan.blockers.length > 0) throw new Error(plan.blockers.join(" "));+ if (plan.frameCount === 0) throw new Error("There is nothing on the timeline to render.");++ const probe = await probeCodec(settings.codec, plan, bitrate);+ if (!probe.supported || !probe.codecString) throw new Error(probe.detail);++ const report = (phase: ExportPhase, framesDone: number, detail: string) =>+ onProgress({ phase, framesDone, frameCount: plan.frameCount, detail });+ const stopIfCancelled = () => {+ if (cancelled()) throw new ExportCancelled();+ };++ report("preparing", 0, "Setting up the encoder…");+ const { ArrayBufferTarget, Muxer } = await import("mp4-muxer");+ const target = new ArrayBufferTarget();+ const muxer = new Muxer({+ target,+ video: {+ codec: outputCodec(settings.codec).muxer,+ width: plan.width,+ height: plan.height,+ frameRate: plan.frameRate,+ },+ fastStart: "in-memory",+ });++ /* On an object because the encoder writes it from its own callback, where+ TypeScript's flow analysis cannot follow the assignment. */+ const encoderFailure: { reason: Error | null } = { reason: null };+ const encoder = new VideoEncoder({+ output: (chunk, meta) => muxer.addVideoChunk(chunk, meta),+ error: (reason) => {+ encoderFailure.reason = reason instanceof Error ? reason : new Error(String(reason));+ },+ });+ encoder.configure(encoderConfig(probe.codecString, plan, bitrate));++ const compositor =+ (await createWebGpuCompositor(plan.width, plan.height, plan.backgroundColor).catch(() => null)) ??+ createCanvasCompositor(plan.width, plan.height, plan.backgroundColor);++ const frameDurationUs = Math.round(1_000_000 / plan.frameRate);+ /* A keyframe every two seconds. Without them a player cannot seek, and a+ 30-minute delta chain is also where quality quietly collapses. */+ const keyFrameInterval = Math.max(1, Math.round(plan.frameRate * 2));+ let framesDone = 0;++ const emit = async (frame: VideoFrame | null, outputIndex: number) => {+ stopIfCancelled();+ if (encoderFailure.reason) throw encoderFailure.reason;+ if (frame) compositor.draw(frame);+ else compositor.clear();+ const composited = compositor.take(outputIndex * frameDurationUs, frameDurationUs);+ encoder.encode(composited, { keyFrame: outputIndex % keyFrameInterval === 0 });+ composited.close();+ framesDone += 1;+ if (framesDone % 5 === 0 || framesDone === plan.frameCount) {+ report("rendering", framesDone, `Rendering frame ${framesDone} of ${plan.frameCount}.`);+ }+ // Yield so the encoder's own queue drains and the window stays alive.+ while (encoder.encodeQueueSize > 8) await tick();+ if (framesDone % 12 === 0) await tick();+ };++ /* One decoded source at a time. Holding every clip's samples would mean+ holding every clip's file, which for 4K footage is gigabytes. Kept on an+ object so the closure below can replace it without TypeScript pinning the+ fields to their initial null. */+ const loaded: {+ assetId: string | null;+ source: DemuxedSource | null;+ image: VideoFrame | null;+ bitmap: ImageBitmap | null;+ } = { assetId: null, source: null, image: null, bitmap: null };++ const releaseLoaded = () => {+ loaded.image?.close();+ loaded.bitmap?.close();+ loaded.image = null;+ loaded.bitmap = null;+ loaded.source = null;+ };++ const assetFor = (assetId: string) => config.assets.find((asset) => asset.id === assetId);++ const loadAsset = async (asset: ProjectAsset) => {+ if (loaded.assetId === asset.id) return;+ releaseLoaded();+ loaded.assetId = asset.id;+ report("rendering", framesDone, `Reading ${asset.name}…`);+ const bytes = await readAssetBytes(folderPath, asset);+ if (asset.mimeType.startsWith("image/")) {+ /* The bitmap stays alive as long as the frame made from it does: a+ VideoFrame built from a CanvasImageSource is not guaranteed to own a+ copy of the pixels. */+ loaded.bitmap = await createImageBitmap(new Blob([bytes], { type: asset.mimeType }));+ loaded.image = new VideoFrame(loaded.bitmap, { timestamp: 0 });+ return;+ }+ loaded.source = await demux(bytes, asset.name);+ };++ const renderClipSegment = async (segment: Extract<ExportSegment, { kind: "clip" }>) => {+ const asset = assetFor(segment.assetId);+ if (!asset) throw new Error(`Clip “${segment.label}” points at media that is not in this project.`);+ await loadAsset(asset);++ const still = loaded.image;+ if (still) {+ for (let index = segment.startFrame; index < segment.endFrame; index += 1) await emit(still, index);+ return;+ }+ const source = loaded.source;+ if (!source) throw new Error(`${asset.name} produced no decodable video.`);++ const startUs = sourceTimeMsForFrame(segment, segment.startFrame, plan.frameRate) * 1000;+ const endUs = sourceTimeMsForFrame(segment, segment.endFrame - 1, plan.frameRate) * 1000;+ /* Decoding has to begin at the keyframe at or before the trim point: a+ delta frame on its own decodes to garbage. Everything decoded before the+ trim point is fed to the decoder and then thrown away, never shown.++ Samples are in DECODE order, so their timestamps are not monotonic once a+ file uses B-frames. Hence a scan for the best keyframe rather than a+ break on the first late sample, and a margin past the last wanted frame. */+ let first = 0;+ let keyframeUs = Number.NEGATIVE_INFINITY;+ for (let index = 0; index < source.samples.length; index += 1) {+ const sample = source.samples[index];+ if (sample.key && sample.timestampUs <= startUs && sample.timestampUs > keyframeUs) {+ keyframeUs = sample.timestampUs;+ first = index;+ }+ }+ const REORDER_MARGIN = 8;+ let last = source.samples.length - 1;+ for (let index = first; index < source.samples.length; index += 1) {+ if (source.samples[index].timestampUs > endUs) {+ last = Math.min(source.samples.length - 1, index + REORDER_MARGIN);+ break;+ }+ }++ let nextOutput = segment.startFrame;+ /* The frame currently on screen. On an object for the same reason as the+ failure below: it is replaced inside `drain`, and the cleanup that has to+ close it lives outside. */+ const on: { screen: VideoFrame | null } = { screen: null };+ const failure: { reason: Error | null } = { reason: null };+ const wantUs = (index: number) => sourceTimeMsForFrame(segment, index, plan.frameRate) * 1000;++ const pending: VideoFrame[] = [];+ const decoder = new VideoDecoder({+ output: (frame) => pending.push(frame),+ error: (reason) => {+ failure.reason = reason instanceof Error ? reason : new Error(String(reason));+ },+ });+ decoder.configure(source.config);++ /* Frames arrive in presentation order, so the frame showing at output time+ W is the last one whose timestamp is at or before W. Each decoded frame+ therefore settles the output frames that came before IT, and the source+ is only released once its successor has arrived. */+ const drain = async (final: boolean) => {+ while (pending.length > 0) {+ const frame = pending.shift() as VideoFrame;+ if (on.screen) {+ while (nextOutput < segment.endFrame && wantUs(nextOutput) < frame.timestamp) {+ await emit(on.screen, nextOutput);+ nextOutput += 1;+ }+ on.screen.close();+ }+ on.screen = frame;+ if (nextOutput >= segment.endFrame) break;+ }+ if (final) {+ if (!on.screen && nextOutput < segment.endFrame) {+ throw new Error(`${asset.name} decoded no frames, so clip “${segment.label}” has no picture to export.`);+ }+ while (nextOutput < segment.endFrame) {+ await emit(on.screen, nextOutput);+ nextOutput += 1;+ }+ }+ };++ try {+ for (let index = first; index <= last && nextOutput < segment.endFrame; index += 1) {+ if (failure.reason) throw failure.reason;+ const sample = source.samples[index];+ decoder.decode(+ new EncodedVideoChunk({+ type: sample.key ? "key" : "delta",+ timestamp: sample.timestampUs,+ duration: sample.durationUs,+ data: sample.data,+ }),+ );+ while (decoder.decodeQueueSize > 12) await tick();+ await drain(false);+ }+ if (nextOutput < segment.endFrame) {+ await decoder.flush();+ await drain(false);+ }+ if (failure.reason) throw failure.reason;+ await drain(true);+ } finally {+ on.screen?.close();+ for (const frame of pending) frame.close();+ if (decoder.state !== "closed") decoder.close();+ }+ };++ try {+ for (const segment of plan.segments) {+ stopIfCancelled();+ if (segment.kind === "gap") {+ for (let index = segment.startFrame; index < segment.endFrame; index += 1) await emit(null, index);+ } else {+ await renderClipSegment(segment);+ }+ }++ report("finishing", framesDone, "Flushing the encoder and writing the MP4 index…");+ await encoder.flush();+ if (encoderFailure.reason) throw encoderFailure.reason;+ muxer.finalize();+ report("done", framesDone, "Encoded.");+ return {+ bytes: new Uint8Array(target.buffer),+ compositor: compositor.kind,+ codecString: probe.codecString,+ };+ } finally {+ releaseLoaded();+ if (encoder.state !== "closed") encoder.close();+ compositor.dispose();+ }+}++/* ---------------------------------------------------------------------------+ Saving the result+ --------------------------------------------------------------------------- */++/** The OS save dialog, run in Rust. Null means the user cancelled. */+export async function chooseExportDestination(suggestedName: string): Promise<string | null> {+ if (!isTauri()) return null;+ return invoke<string | null>("choose_export_destination", { suggestedName });+}++/** Writes the muxed bytes to the chosen path and returns how many landed.+ * The payload rides as a raw IPC body — a JSON array of numbers for a+ * 100 MB file would be a gigabyte of text. */+export async function writeExportFile(path: string, bytes: Uint8Array): Promise<number> {+ return invoke<number>("write_export_file", bytes, { headers: { "x-export-path": encodeURIComponent(path) } });+}++/* ---------------------------------------------------------------------------+ Preview+ --------------------------------------------------------------------------- */++/** Blob URLs for the project's own media, made once and revoked together.+ * A leaked blob URL pins the whole file in memory for the life of the page. */+export class PreviewSources {+ private readonly urls = new Map<string, Promise<string>>();++ constructor(private readonly folderPath: string) {}++ url(asset: ProjectAsset): Promise<string> {+ const existing = this.urls.get(asset.id);+ if (existing) return existing;+ const created = readAssetBytes(this.folderPath, asset).then((bytes) =>+ URL.createObjectURL(new Blob([bytes], { type: asset.mimeType })),+ );+ this.urls.set(asset.id, created);+ return created;+ }++ dispose(): void {+ for (const pending of this.urls.values()) {+ void pending.then(URL.revokeObjectURL).catch(() => undefined);+ }+ this.urls.clear();+ }+}++const seeked = (video: HTMLVideoElement, event: string) =>+ new Promise<void>((resolve, reject) => {+ video.addEventListener(event, () => resolve(), { once: true });+ video.addEventListener("error", () => reject(new Error("This file could not be decoded for preview.")), {+ once: true,+ });+ });++/** Draws the frame the export would produce at `timeMs` into `canvas`, using+ * the same fit and the same background as the encoder. The picture is decoded+ * from the real file by the webview's own hardware decoder — nothing here is+ * drawn from imagination, and a source that cannot be read throws. */+export async function drawPreviewFrame(options: {+ canvas: HTMLCanvasElement;+ plan: ExportPlan;+ config: ProjectConfig;+ sources: PreviewSources;+ clip: { assetId: string; sourceTimeMs: number } | null;+}): Promise<void> {+ const { canvas, plan, config, sources, clip } = options;+ const context = canvas.getContext("2d");+ if (!context) throw new Error("This computer gave no 2D canvas to preview into.");+ context.fillStyle = plan.backgroundColor;+ context.fillRect(0, 0, canvas.width, canvas.height);+ if (!clip) return;++ const asset = config.assets.find((candidate) => candidate.id === clip.assetId);+ if (!asset) throw new Error("This clip points at media that is not in this project.");+ const url = await sources.url(asset);++ if (asset.mimeType.startsWith("image/")) {+ const image = new Image();+ image.src = url;+ await image.decode();+ const box = fitRect(image.naturalWidth, image.naturalHeight, canvas.width, canvas.height);+ context.drawImage(image, box.x, box.y, box.width, box.height);+ return;+ }++ const video = document.createElement("video");+ video.muted = true;+ video.preload = "auto";+ video.playsInline = true;+ video.src = url;+ try {+ await seeked(video, "loadeddata");+ const target = Math.max(0, clip.sourceTimeMs / 1000);+ if (target > 0 && Number.isFinite(video.duration)) {+ const landed = seeked(video, "seeked");+ video.currentTime = Math.min(target, Math.max(0, video.duration - 1 / plan.frameRate));+ await landed;+ }+ const box = fitRect(video.videoWidth, video.videoHeight, canvas.width, canvas.height);+ context.drawImage(video, box.x, box.y, box.width, box.height);+ } finally {+ video.removeAttribute("src");+ video.load();+ }+}