AI media harness for planning, generating, and editing locally
TypeScript
Add optional clip effects
Bittibd25ae2d2df8
diff --git a/src-tauri/src/agent_commands.rs b/src-tauri/src/agent_commands.rsindex 3dc6829..7512416 100644--- a/src-tauri/src/agent_commands.rs+++ b/src-tauri/src/agent_commands.rs@@ -822,6 +822,7 @@ fn apply_command(status: status.into(),transform: None,look: None,+ chroma_key: None,transition: None,});}diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rsindex f21e837..4bad7b7 100644--- a/src-tauri/src/lib.rs+++ b/src-tauri/src/lib.rs@@ -150,6 +150,8 @@ struct TimelineClip {#[serde(default, skip_serializing_if = "Option::is_none")]look: Option<ClipLook>,#[serde(default, skip_serializing_if = "Option::is_none")]+ chroma_key: Option<ClipChromaKey>,+ #[serde(default, skip_serializing_if = "Option::is_none")]transition: Option<ClipTransition>,}@@ -168,6 +170,12 @@ struct ClipLook {temperature: f64,}+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+struct ClipChromaKey {+ color: String,+ tolerance: f64,+}+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]#[serde(rename_all = "camelCase")]struct ClipTransition {@@ -964,6 +972,16 @@ fn validate_and_normalize_config(mut config: ProjectConfig) -> Result<ProjectConreturn Err(format!("Clip '{}' has an invalid look.", clip.id));}}+ if let Some(key) = &clip.chroma_key {+ if key.color.len() != 7+ || !key.color.starts_with('#')+ || !key.color.as_bytes()[1..].iter().all(u8::is_ascii_hexdigit)+ || !key.tolerance.is_finite()+ || !(0.0..=100.0).contains(&key.tolerance)+ {+ return Err(format!("Clip '{}' has an invalid chroma key.", clip.id));+ }+ }if let Some(transition) = &clip.transition {if !matches!(transition.r#type.as_str(),@@ -3305,6 +3323,32 @@ mod tests {assert_eq!(files, [PROJECT_FILE_NAME]);}+ #[test]+ fn chroma_key_survives_save_and_reopen_and_validates_settings() {+ let mut config = fixture();+ config.timeline.tracks[0].clips[0].chroma_key = Some(ClipChromaKey {+ color: "#12ABef".into(),+ tolerance: 27.0,+ });+ let config = without_derived_project_state(validate_and_normalize_config(config).unwrap());+ let root = tempfile::tempdir().unwrap();+ let created = create_project_in(root.path(), &config).unwrap();+ let folder = PathBuf::from(&created.folder_path);+ write_project(&folder, &config).unwrap();+ assert_eq!(read_project(&folder).unwrap().config, config);+ let json = serde_json::to_value(&config).unwrap();+ assert_eq!(json["timeline"]["tracks"][0]["clips"][0]["chromaKey"]["color"], "#12ABef");+ for (color, tolerance) in [("green", 20.0), ("#12345g", 20.0), ("#00ff00", -1.0), ("#00ff00", 101.0), ("#00ff00", f64::NAN)] {+ let mut invalid = config.clone();+ invalid.timeline.tracks[0].clips[0].chroma_key = Some(ClipChromaKey { color: color.into(), tolerance });+ assert!(validate_and_normalize_config(invalid).unwrap_err().contains("chroma key"));+ }+ let mut removed = config.clone();+ removed.timeline.tracks[0].clips[0].chroma_key = None;+ write_project(&folder, &removed).unwrap();+ assert!(read_project(&folder).unwrap().config.timeline.tracks[0].clips[0].chroma_key.is_none());+ }+#[test]fn deleting_a_project_removes_its_folder_and_every_file_inside_it() {let parent = tempfile::tempdir().unwrap();diff --git a/src/components/workspace/ChromaKeyPreview.tsx b/src/components/workspace/ChromaKeyPreview.tsxnew file mode 100644index 0000000..ec350c5--- /dev/null+++ b/src/components/workspace/ChromaKeyPreview.tsx@@ -0,0 +1,46 @@+import { useEffect, useRef, type CSSProperties, type RefObject } from "react";+import { createChromaKeyRenderer } from "../../lib/chromaKeyPreview";+import type { ClipChromaKey } from "../../lib/project";++export function ChromaKeyPreview({ source, sourceUrl, effect, playing, style, onError }: {+ source: RefObject<HTMLVideoElement | HTMLImageElement>;+ sourceUrl: string;+ effect: ClipChromaKey;+ playing: boolean;+ style?: CSSProperties;+ onError: (message: string) => void;+}) {+ const canvas = useRef<HTMLCanvasElement>(null);+ const settings = useRef(effect);+ const errorHandler = useRef(onError);+ const redraw = useRef<(() => void) | null>(null);+ settings.current = effect;+ errorHandler.current = onError;+ useEffect(() => {+ const media = source.current;+ if (!canvas.current || !media) return;+ let frame = 0;+ let failed = false;+ let renderer: ReturnType<typeof createChromaKeyRenderer>;+ try { renderer = createChromaKeyRenderer(canvas.current); }+ catch (reason) { errorHandler.current(String(reason)); return; }+ const draw = () => {+ if (failed) return;+ try { renderer.draw(media, settings.current); }+ catch (reason) { failed = true; errorHandler.current(String(reason)); }+ };+ const animate = () => { draw(); if (!failed) frame = requestAnimationFrame(animate); };+ redraw.current = draw;+ ["loadeddata", "seeked", "load"].forEach((event) => media.addEventListener(event, draw));+ draw();+ if (playing) frame = requestAnimationFrame(animate);+ return () => {+ cancelAnimationFrame(frame);+ redraw.current = null;+ ["loadeddata", "seeked", "load"].forEach((event) => media.removeEventListener(event, draw));+ renderer.dispose();+ };+ }, [source, sourceUrl, playing]);+ useEffect(() => redraw.current?.(), [effect]);+ return <canvas ref={canvas} className="program-keyed-picture" style={style} aria-label="Chroma Key preview" />;+}diff --git a/src/components/workspace/ClipEffects.test.tsx b/src/components/workspace/ClipEffects.test.tsxnew file mode 100644index 0000000..902b433--- /dev/null+++ b/src/components/workspace/ClipEffects.test.tsx@@ -0,0 +1,65 @@+// @vitest-environment jsdom+import { cleanup, fireEvent, render, screen, within } from "@testing-library/react";+import { useState } from "react";+import { afterEach, describe, expect, it, vi } from "vitest";+import { timelineClipSchema, type TimelineClip } from "../../lib/project";+import { ClipEffects } from "./ClipEffects";++afterEach(cleanup);+const original: TimelineClip = {+ id: "clip", assetId: "asset", trackId: "video", label: "Clip", startMs: 0,+ durationMs: 2000, sourceStartMs: 0, status: "approved",+};+function Harness() {+ const [clip, setClip] = useState(original);+ return <><ClipEffects clip={clip} disabled={false} onChange={(patch) => setClip((current) => timelineClipSchema.parse({ ...current, ...patch }))} /><output data-testid="saved">{JSON.stringify(clip)}</output></>;+}++describe("clip effects", () => {+ it("adds only chosen effects, persists edits, and removes them without changing other effects", () => {+ render(<Harness />);+ expect(screen.queryByRole("heading", { name: "Look" })).toBeNull();+ expect(screen.queryByRole("heading", { name: "Transition" })).toBeNull();+ fireEvent.click(screen.getByRole("button", { name: "Add Effect" }));+ fireEvent.click(screen.getByRole("button", { name: /Chroma Key Make/ }));+ fireEvent.change(screen.getByLabelText("Chroma Key color"), { target: { value: "#123456" } });+ fireEvent.change(screen.getByLabelText("Chroma Key tolerance"), { target: { value: "43" } });+ fireEvent.click(screen.getByRole("button", { name: "Add Effect" }));+ const picker = screen.getByRole("group", { name: "Available effects" });+ expect(within(picker).queryByText("Chroma Key")).toBeNull();+ fireEvent.click(within(picker).getByRole("button", { name: /Look Opacity/ }));+ fireEvent.change(screen.getByLabelText("Clip opacity"), { target: { value: "65" } });+ expect(JSON.parse(screen.getByTestId("saved").textContent!)).toMatchObject({ chromaKey: { color: "#123456", tolerance: 43 }, look: { opacity: 65 } });+ fireEvent.click(screen.getByRole("button", { name: "Remove Chroma Key effect" }));+ const saved = JSON.parse(screen.getByTestId("saved").textContent!);+ expect(saved.chromaKey).toBeUndefined();+ expect(saved.look.opacity).toBe(65);+ fireEvent.click(screen.getByRole("button", { name: "Add Effect" }));+ expect(screen.getByRole("button", { name: /Chroma Key Make/ })).toBeTruthy();+ });++ it("keeps existing effects editable and closes the keyboard picker with Escape", () => {+ render(<ClipEffects clip={{ ...original, transition: { type: "wipe-left", durationMs: 800 } }} disabled={false} onChange={vi.fn()} />);+ expect((screen.getByLabelText("Clip transition") as HTMLSelectElement).value).toBe("wipe-left");+ fireEvent.click(screen.getByRole("button", { name: "Add Effect" }));+ expect(document.activeElement?.textContent).toContain("Look");+ fireEvent.keyDown(document.activeElement!, { key: "ArrowDown" });+ expect(document.activeElement?.textContent).toContain("Chroma Key");+ fireEvent.keyDown(document.activeElement!, { key: "Escape" });+ expect(screen.queryByRole("group", { name: "Available effects" })).toBeNull();+ expect(document.activeElement).toBe(screen.getByRole("button", { name: "Add Effect" }));+ });++ it("disables effects for locked or nonvisual clips and closes the picker when selection changes", () => {+ const props = { onChange: vi.fn(), clip: original, disabled: false };+ const view = render(<ClipEffects {...props} />);+ fireEvent.click(screen.getByRole("button", { name: "Add Effect" }));+ view.rerender(<ClipEffects {...props} clip={{ ...original, id: "next" }} />);+ expect(screen.queryByRole("group", { name: "Available effects" })).toBeNull();+ view.rerender(<ClipEffects {...props} disabled clip={{ ...original, chromaKey: { color: "#00ff00", tolerance: 20 } }} />);+ expect((screen.getByRole("button", { name: "Add Effect" }) as HTMLButtonElement).disabled).toBe(true);+ expect((screen.getByRole("button", { name: "Remove Chroma Key effect" }) as HTMLButtonElement).disabled).toBe(true);+ fireEvent.change(screen.getByLabelText("Chroma Key tolerance"), { target: { value: "40" } });+ expect(props.onChange).not.toHaveBeenCalled();+ });+});diff --git a/src/components/workspace/ClipEffects.tsx b/src/components/workspace/ClipEffects.tsxnew file mode 100644index 0000000..ba3c8f3--- /dev/null+++ b/src/components/workspace/ClipEffects.tsx@@ -0,0 +1,95 @@+import { Plus, X } from "lucide-react";+import { useEffect, useId, useRef, useState, type ReactNode } from "react";+import { DEFAULT_CLIP_CHROMA_KEY, DEFAULT_CLIP_LOOK, type TimelineClip } from "../../lib/project";++type EffectId = "look" | "transition" | "chromaKey";+type EditorProps = { clip: TimelineClip; update: (patch: Partial<TimelineClip>) => void };+type EffectDefinition = {+ id: EffectId;+ name: string;+ description: string;+ defaults: Partial<TimelineClip>;+ editor: (props: EditorProps) => ReactNode;+};++// One catalogue drives the picker, defaults and editors. Missing settings mean+// the effect is absent; older projects with Look/Transition keep their edits.+const EFFECTS: readonly EffectDefinition[] = [+ {+ id: "look", name: "Look", description: "Opacity and color temperature",+ defaults: { look: DEFAULT_CLIP_LOOK },+ editor: ({ clip, update }) => {+ const look = clip.look!;+ return <>+ <label className="range-field"><span>Opacity <b>{look.opacity}%</b></span><input aria-label="Clip opacity" type="range" min="0" max="100" step="1" value={look.opacity} onChange={(event) => update({ look: { ...look, opacity: Number(event.target.value) } })} /></label>+ <label className="range-field"><span>Temperature <b>{look.temperature > 0 ? "+" : ""}{look.temperature}</b></span><input aria-label="Clip temperature" type="range" min="-100" max="100" step="1" value={look.temperature} onChange={(event) => update({ look: { ...look, temperature: Number(event.target.value) } })} /></label>+ </>;+ },+ },+ {+ id: "transition", name: "Transition", description: "Fade or wipe into the clip",+ defaults: { transition: { type: "fade", durationMs: 500 } },+ editor: ({ clip, update }) => {+ const transition = clip.transition!;+ return <>+ <label><span>When this clip begins</span><select aria-label="Clip transition" value={transition.type} onChange={(event) => update({ transition: { ...transition, type: event.target.value as typeof transition.type } })}>+ <option value="cut">Cut</option><option value="fade">Fade in</option><option value="wipe-left">Wipe from left</option><option value="wipe-right">Wipe from right</option>+ </select></label>+ <label className="range-field"><span>Duration <b>{transition.durationMs} ms</b></span><input aria-label="Transition duration" type="range" min="100" max="3000" step="100" value={transition.durationMs} disabled={transition.type === "cut"} onChange={(event) => update({ transition: { ...transition, durationMs: Number(event.target.value) } })} /></label>+ </>;+ },+ },+ {+ id: "chromaKey", name: "Chroma Key", description: "Make a selected color transparent",+ defaults: { chromaKey: DEFAULT_CLIP_CHROMA_KEY },+ editor: ({ clip, update }) => {+ const key = clip.chromaKey!;+ return <>+ <label className="effect-color"><span>Color</span><input aria-label="Chroma Key color" type="color" value={key.color} onChange={(event) => update({ chromaKey: { ...key, color: event.target.value } })} /><code>{key.color.toUpperCase()}</code></label>+ <label className="range-field"><span>Tolerance <b>{key.tolerance}%</b></span><input aria-label="Chroma Key tolerance" type="range" min="0" max="100" step="1" value={key.tolerance} onChange={(event) => update({ chromaKey: { ...key, tolerance: Number(event.target.value) } })} /></label>+ </>;+ },+ },+];++export function ClipEffects({ clip, disabled, onChange }: {+ clip: TimelineClip;+ disabled: boolean;+ onChange: (patch: Partial<TimelineClip>) => void;+}) {+ const [open, setOpen] = useState(false);+ const root = useRef<HTMLDivElement>(null);+ const trigger = useRef<HTMLButtonElement>(null);+ const pickerId = useId();+ const available = EFFECTS.filter((effect) => !clip[effect.id]);+ const update = (patch: Partial<TimelineClip>) => { if (!disabled) onChange(patch); };+ useEffect(() => setOpen(false), [clip.id, disabled]);+ useEffect(() => {+ if (!open) return;+ const closeOutside = (event: PointerEvent) => { if (!root.current?.contains(event.target as Node)) setOpen(false); };+ document.addEventListener("pointerdown", closeOutside);+ root.current?.querySelector<HTMLButtonElement>(".clip-effects__picker button")?.focus();+ return () => document.removeEventListener("pointerdown", closeOutside);+ }, [open]);+ return <>+ {EFFECTS.filter((effect) => clip[effect.id]).map((effect) => <section className="inspector-section clip-effect" key={effect.id} aria-label={`${effect.name} effect`}>+ <header><h3>{effect.name}</h3><button type="button" className="clip-effect__remove" aria-label={`Remove ${effect.name} effect`} disabled={disabled} onClick={() => update({ [effect.id]: undefined })}><X size={14} /></button></header>+ <fieldset disabled={disabled}>{effect.editor({ clip, update })}</fieldset>+ </section>)}+ <div className="clip-effects" ref={root} onKeyDown={(event) => {+ if (event.key === "Escape" && open) { event.stopPropagation(); setOpen(false); trigger.current?.focus(); }+ if (open && (event.key === "ArrowDown" || event.key === "ArrowUp" || event.key === "Home" || event.key === "End")) {+ event.preventDefault();+ const buttons = Array.from(root.current?.querySelectorAll<HTMLButtonElement>(".clip-effects__picker button") ?? []);+ const index = buttons.indexOf(document.activeElement as HTMLButtonElement);+ const next = event.key === "Home" ? 0 : event.key === "End" ? buttons.length - 1 : (index + (event.key === "ArrowUp" ? -1 : 1) + buttons.length) % buttons.length;+ buttons[next]?.focus();+ }+ }} onBlur={(event) => { if (!event.currentTarget.contains(event.relatedTarget)) setOpen(false); }}>+ <button ref={trigger} type="button" className="clip-effects__add" disabled={disabled || available.length === 0} aria-expanded={open} aria-controls={pickerId} onClick={() => setOpen((value) => !value)}><Plus size={16} /> Add Effect</button>+ {open && <div id={pickerId} className="clip-effects__picker" role="group" aria-label="Available effects">+ {available.map((effect) => <button type="button" key={effect.id} onClick={() => { update(effect.defaults); setOpen(false); trigger.current?.focus(); }}><strong>{effect.name}</strong><span>{effect.description}</span></button>)}+ </div>}+ </div>+ </>;+}diff --git a/src/components/workspace/ProgramLayer.tsx b/src/components/workspace/ProgramLayer.tsxnew file mode 100644index 0000000..0f1843a--- /dev/null+++ b/src/components/workspace/ProgramLayer.tsx@@ -0,0 +1,61 @@+import { useEffect, useRef, useState, type CSSProperties } from "react";+import { clipFrameStyle, clipVisualSettings, type ClipFrameStyle } from "../../lib/export";+import { PreviewSources } from "../../lib/exportPipeline";+import type { ProjectAsset, TimelineClip } from "../../lib/project";+import { ChromaKeyPreview } from "./ChromaKeyPreview";++export function previewMediaStyle({ transform, look, opacity, revealStart, revealEnd }: ClipFrameStyle): CSSProperties {+ const warmth = look.temperature / 100;+ return {+ transform: `translate(${transform.positionX}%, ${transform.positionY}%) scale(${transform.scale / 100}) rotate(${transform.rotation}deg)`,+ opacity,+ filter: warmth === 0 ? "none" : `sepia(${Math.abs(warmth) * 0.22}) saturate(${1 + Math.abs(warmth) * 0.3}) hue-rotate(${warmth > 0 ? -8 : 172}deg)`,+ clipPath: `inset(0 ${(1 - revealEnd) * 100}% 0 ${revealStart * 100}%)`,+ };+}++/** Lower picture tracks follow the monitor clock. Their audio is mixed by the+ * monitor's existing audio elements, so these video elements stay muted. */+export function ProgramLayer({ clip, asset, sources, playheadMs, playing }: {+ clip: TimelineClip;+ asset: ProjectAsset | undefined;+ sources: PreviewSources;+ playheadMs: number;+ playing: boolean;+}) {+ const [url, setUrl] = useState<string | null>(null);+ const [error, setError] = useState<string | null>(null);+ const video = useRef<HTMLVideoElement>(null);+ const image = useRef<HTMLImageElement>(null);+ const position = useRef({ clip, playheadMs, playing });+ position.current = { clip, playheadMs, playing };+ const isImage = asset?.mimeType.startsWith("image/");+ useEffect(() => {+ let live = true;+ setUrl(null);+ setError(null);+ if (asset && (asset.relativePath || asset.sourcePath)) {+ void sources.url(asset).then((value) => { if (live) setUrl(value); }, (reason) => { if (live) setError(String(reason)); });+ } else setError(`Media for ${clip.label} is unavailable.`);+ return () => { live = false; };+ }, [asset, sources, clip.label]);+ const sync = () => {+ const element = video.current;+ if (!element || element.readyState < 1) return;+ const state = position.current;+ const target = Math.max(0, (state.clip.sourceStartMs + state.playheadMs - state.clip.startMs) / 1000);+ if (Math.abs(element.currentTime - target) > (state.playing ? 0.25 : 0.033)) element.currentTime = target;+ if (state.playing) { if (element.paused) void element.play().catch(() => undefined); }+ else element.pause();+ };+ useEffect(sync, [playheadMs, playing, clip, url]);+ const style = previewMediaStyle(clipFrameStyle(clipVisualSettings(clip), playheadMs - clip.startMs));+ const sourceStyle: CSSProperties = clip.chromaKey ? { ...style, visibility: "hidden", position: "absolute" } : style;+ return <div className="program-layer" aria-label={`Background clip: ${clip.label}`}>+ {url && (isImage+ ? <img ref={image} src={url} alt={clip.label} style={sourceStyle} onError={() => setError(`Could not decode ${clip.label}.`)} />+ : <video ref={video} src={url} style={sourceStyle} muted playsInline preload="auto" onLoadedMetadata={sync} onError={() => setError(`Could not decode ${clip.label}.`)} />)}+ {url && clip.chromaKey && <ChromaKeyPreview source={isImage ? image : video} sourceUrl={url} effect={clip.chromaKey} playing={playing && !isImage} style={style} onError={setError} />}+ {error && <div className="program-note program-note--error" role="alert">{error}</div>}+ </div>;+}diff --git a/src/components/workspace/ProgramMonitor.tsx b/src/components/workspace/ProgramMonitor.tsxindex 09c5a34..4e374b6 100644--- a/src/components/workspace/ProgramMonitor.tsx+++ b/src/components/workspace/ProgramMonitor.tsx@@ -1,6 +1,8 @@+import { ChromaKeyPreview } from "./ChromaKeyPreview";+import { ProgramLayer, previewMediaStyle } from "./ProgramLayer";import { Film, TriangleAlert } from "lucide-react";import { useCallback, useEffect, useMemo, useRef, useState } from "react";-import { clipFrameStyle, clipVisualSettings, visibleClipAt } from "../../lib/export";+import { clipFrameStyle, clipVisualSettings, visibleClipsAt } from "../../lib/export";import { isTauri } from "../../lib/persistence";import type { ClipTransform, ProjectAsset, ProjectConfig, TimelineClip } from "../../lib/project";import { clipEndMs } from "../../lib/timeline";@@ -101,7 +103,8 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek}) {const tracks = config.timeline.tracks;const assetsById = useMemo(() => new Map(config.assets.map((candidate) => [candidate.id, candidate])), [config.assets]);- const clip = useMemo(() => visibleClipAt(tracks, playheadMs, assetsById), [tracks, playheadMs, assetsById]);+ const layers = useMemo(() => visibleClipsAt(tracks, playheadMs, assetsById), [tracks, playheadMs, assetsById]);+ const clip = layers[0] ?? null;const asset = useMemo(() => (clip ? config.assets.find((candidate) => candidate.id === clip.assetId) : undefined),[clip, config.assets],@@ -110,20 +113,8 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek() => clip ? clipFrameStyle(clipVisualSettings(clip), playheadMs - clip.startMs) : null,[clip, playheadMs],);- const mediaStyle = useMemo<React.CSSProperties | undefined>(() => {- if (!frameStyle) return undefined;- const { transform, look, opacity, revealStart, revealEnd } = frameStyle;- const warmth = look.temperature / 100;- const filter = warmth === 0- ? "none"- : `sepia(${Math.abs(warmth) * 0.22}) saturate(${1 + Math.abs(warmth) * 0.3}) hue-rotate(${warmth > 0 ? -8 : 172}deg)`;- return {- transform: `translate(${transform.positionX}%, ${transform.positionY}%) scale(${transform.scale / 100}) rotate(${transform.rotation}deg)`,- opacity,- filter,- clipPath: `inset(0 ${(1 - revealEnd) * 100}% 0 ${revealStart * 100}%)`,- };- }, [frameStyle]);+ const mediaStyle = frameStyle ? previewMediaStyle(frameStyle) : undefined;+ const sourceStyle: React.CSSProperties | undefined = clip?.chromaKey ? { ...mediaStyle, visibility: "hidden", position: "absolute" } : mediaStyle;/* Sound that should be audible at this moment: audio-only files and embeddedsound from every active video, minus muted tracks. The visible video playsits own stream, so a second audio element is only needed for the rest. */@@ -146,6 +137,7 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek);const videoRef = useRef<HTMLVideoElement>(null);+ const imageRef = useRef<HTMLImageElement>(null);const pictureRef = useRef<HTMLDivElement>(null);const transformGesture = useRef<TransformGesture | null>(null);const audioRefs = useRef(new Map<string, HTMLAudioElement>());@@ -422,19 +414,21 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeekonSelectClip(clip.id);}}>+ {layers.slice(1).reverse().map((layer) => <ProgramLayer key={layer.id} clip={layer} asset={assetsById.get(layer.assetId)} sources={sources} playheadMs={playheadMs} playing={playing} />)}{/* One element, re-pointed at whatever the playhead is over. Rebuilding itper clip would drop the decoder and re-open the file on every cut. */}{asset && !isImage(asset) && url && <videoref={videoRef}src={url}- style={mediaStyle}+ style={sourceStyle}muted={videoTrackMuted}playsInlinepreload="auto"onLoadedData={() => setReady(true)}onError={() => setError("This file could not be decoded.")}/>}- {asset && isImage(asset) && url && <img src={url} alt={clip?.label ?? ""} style={mediaStyle} />}+ {asset && isImage(asset) && url && <img ref={imageRef} src={url} alt={clip?.label ?? ""} style={sourceStyle} />}+ {asset && url && clip?.chromaKey && <ChromaKeyPreview source={isImage(asset) ? imageRef : videoRef} sourceUrl={url} effect={clip.chromaKey} playing={playing && !isImage(asset)} style={mediaStyle} onError={setError} />}{clip && frameStyle && editingClipId === clip.id && onTransformChange && !transformEditingDisabled && <divclassName="program-transform"style={{ transform: `translate(${frameStyle.transform.positionX}%, ${frameStyle.transform.positionY}%) scale(${frameStyle.transform.scale / 100}) rotate(${frameStyle.transform.rotation}deg)` }}diff --git a/src/components/workspace/TimelineView.tsx b/src/components/workspace/TimelineView.tsxindex 252907a..252dcda 100644--- a/src/components/workspace/TimelineView.tsx+++ b/src/components/workspace/TimelineView.tsx@@ -1,3 +1,4 @@+import { ClipEffects } from "./ClipEffects";import {Copy, Film, Layers3, LayoutGrid, List, Lock, LockOpen,Pause, Play, Plus, Scissors, SkipBack, SkipForward, Trash2, Upload,@@ -8,9 +9,7 @@ import { resolutionLabel } from "../../lib/export";import { importMediaFiles, isTauri } from "../../lib/persistence";import { loadMediaLayout, saveMediaLayout, type MediaLayout } from "../../lib/settings";import {- clipLook,clipTransform,- clipTransition,generationAssetId,roundClipTransform,sceneDurationSeconds,@@ -198,8 +197,6 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {const selectedTrack = tracks.find((track) => track.id === selected?.trackId);const selectedMediaAsset = config.assets.find((asset) => asset.id === selected?.assetId);const selectedTransform = selected ? clipTransform(selected) : null;- const selectedLook = selected ? clipLook(selected) : null;- const selectedTransition = selected ? clipTransition(selected) : null;const visualControlsDisabled = selectedMediaAsset?.kind === "audio" || selectedTrack?.kind !== "video" || selectedTrack.locked;const clipCount = tracks.reduce((total, track) => total + track.clips.length, 0);/** Where the last clip ends — where playback stops, which is not the same as@@ -298,12 +295,6 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {const [minimum, maximum] = limits[key];updateClip(selected.id, { transform: roundClipTransform({ ...selectedTransform, [key]: Math.max(minimum, Math.min(maximum, value)) }) });};- const updateLook = (key: keyof NonNullable<TimelineClip["look"]>, value: number) => {- if (!selected || !selectedLook || !Number.isFinite(value)) return;- const limits = { opacity: [0, 100], temperature: [-100, 100] } as const;- const [minimum, maximum] = limits[key];- updateClip(selected.id, { look: { ...selectedLook, [key]: Math.max(minimum, Math.min(maximum, value)) } });- };const toggleTrack = (trackId: string, key: "muted" | "locked") => updateTracks(tracks.map((track) => track.id === trackId ? { ...track, [key]: !track[key] } : track));/* Every removal — the toolbar, the key, the × on the clip itself — goesthrough one function, so a locked track refuses all three. */@@ -881,21 +872,7 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {<label><span>Position Y (%)</span><CommittedNumberInput minimum={-100} maximum={100} step="1" value={selectedTransform?.positionY ?? 0} disabled={visualControlsDisabled} onCommit={(value) => updateTransform("positionY", value)} /></label></div></section>- <section className="inspector-section">- <h3>Look</h3>- <label className="range-field"><span>Opacity <b>{selectedLook?.opacity ?? 100}%</b></span><input aria-label="Clip opacity" type="range" min="0" max="100" step="1" value={selectedLook?.opacity ?? 100} disabled={visualControlsDisabled} onChange={(event) => updateLook("opacity", Number(event.target.value))} /></label>- <label className="range-field"><span>Temperature <b>{(selectedLook?.temperature ?? 0) > 0 ? "+" : ""}{selectedLook?.temperature ?? 0}</b></span><input aria-label="Clip temperature" type="range" min="-100" max="100" step="1" value={selectedLook?.temperature ?? 0} disabled={visualControlsDisabled} onChange={(event) => updateLook("temperature", Number(event.target.value))} /></label>- </section>- <section className="inspector-section">- <h3>Transition</h3>- <label><span>When this clip begins</span><select aria-label="Clip transition" value={selectedTransition?.type ?? "cut"} disabled={visualControlsDisabled} onChange={(event) => selected && selectedTransition && updateClip(selected.id, { transition: { ...selectedTransition, type: event.target.value as NonNullable<TimelineClip["transition"]>["type"] } })}>- <option value="cut">Cut</option>- <option value="fade">Fade in</option>- <option value="wipe-left">Wipe from left</option>- <option value="wipe-right">Wipe from right</option>- </select></label>- <label className="range-field"><span>Duration <b>{selectedTransition?.durationMs ?? 500} ms</b></span><input aria-label="Transition duration" type="range" min="100" max="3000" step="100" value={selectedTransition?.durationMs ?? 500} disabled={visualControlsDisabled || selectedTransition?.type === "cut"} onChange={(event) => selected && selectedTransition && updateClip(selected.id, { transition: { ...selectedTransition, durationMs: Number(event.target.value) } })} /></label>- </section>+ <ClipEffects clip={selected} disabled={visualControlsDisabled} onChange={(patch) => updateClip(selected.id, patch)} /></> : clipCount === 0? <div className="inspector-empty"><MouseSelection /><b>Nothing to edit yet</b><span>Once a scene lands on the timeline, select it here to rename it and check its timing.</span></div>: <div className="inspector-empty"><MouseSelection /><b>Select a clip to edit it</b><span>Pick any clip on the timeline below to rename it, check its timing, and see how it was made.</span></div>}diff --git a/src/lib/chromaKey.test.ts b/src/lib/chromaKey.test.tsnew file mode 100644index 0000000..aaa4065--- /dev/null+++ b/src/lib/chromaKey.test.ts@@ -0,0 +1,31 @@+import { describe, expect, it } from "vitest";+import { applyChromaKey, keyAlpha } from "./chromaKey";+import { clipChromaKeySchema, timelineClipSchema } from "./project";++describe("Chroma Key", () => {+ it("removes the chosen color, keeps other colors and preserves existing alpha", () => {+ const pixels = new Uint8ClampedArray([0, 255, 0, 255, 255, 0, 0, 128, 0, 0, 255, 255]);+ applyChromaKey(pixels, { color: "#00ff00", tolerance: 0 });+ expect(Array.from(pixels)).toEqual([0, 255, 0, 0, 255, 0, 0, 128, 0, 0, 255, 255]);+ });+ it("widens the transparent range as tolerance increases and feathers its edge", () => {+ const narrow = new Uint8ClampedArray([30, 210, 30, 255]);+ const wide = narrow.slice();+ applyChromaKey(narrow, { color: "#00ff00", tolerance: 1 });+ applyChromaKey(wide, { color: "#00ff00", tolerance: 20 });+ expect(narrow[3]).toBe(255);+ expect(wide[3]).toBe(0);+ expect(keyAlpha(0.21, 20)).toBeCloseTo(0.5);+ expect(keyAlpha(1, 100)).toBe(0);+ });+ it("rejects malformed colors and out-of-range tolerances while accepting old clips", () => {+ for (const key of [{ color: "green", tolerance: 20 }, { color: "#12345g", tolerance: 20 }, { color: "#00ff00", tolerance: -1 }, { color: "#00ff00", tolerance: 101 }]) {+ expect(clipChromaKeySchema.safeParse(key).success).toBe(false);+ }+ const old = { id: "clip", assetId: "a", trackId: "t", label: "Clip", startMs: 0, durationMs: 1000 };+ expect(timelineClipSchema.parse(old).chromaKey).toBeUndefined();+ expect(timelineClipSchema.parse({ ...old, chromaKey: null }).chromaKey).toBeNull();+ const edited = { ...old, chromaKey: { color: "#12ABef", tolerance: 30 } };+ expect(timelineClipSchema.parse(JSON.parse(JSON.stringify(edited))).chromaKey).toEqual(edited.chromaKey);+ });+});diff --git a/src/lib/chromaKey.ts b/src/lib/chromaKey.tsnew file mode 100644index 0000000..fdee206--- /dev/null+++ b/src/lib/chromaKey.ts@@ -0,0 +1,24 @@+import type { ClipChromaKey } from "./project";++export function keyColor(color: string): [number, number, number] {+ const value = Number.parseInt(color.slice(1), 16);+ return [((value >> 16) & 255) / 255, ((value >> 8) & 255) / 255, (value & 255) / 255];+}++// RGB distance normalized to 0–1. A narrow feather avoids jagged compressed+// edges. These constants/formula are shared by the preview and export shaders.+export const KEY_FEATHER = 0.02;+export const RGB_DISTANCE_SCALE = Math.sqrt(3);+export function keyAlpha(distance: number, tolerance: number): number {+ const edge = Math.max(0, Math.min(1, (distance - tolerance / 100) / KEY_FEATHER));+ return edge * edge * (3 - 2 * edge);+}++/** CPU fallback only; the normal video path performs this on the GPU. */+export function applyChromaKey(pixels: Uint8ClampedArray, key: ClipChromaKey): void {+ const [r, g, b] = keyColor(key.color);+ for (let index = 0; index < pixels.length; index += 4) {+ const distance = Math.hypot(pixels[index] / 255 - r, pixels[index + 1] / 255 - g, pixels[index + 2] / 255 - b) / RGB_DISTANCE_SCALE;+ pixels[index + 3] *= keyAlpha(distance, key.tolerance);+ }+}diff --git a/src/lib/chromaKeyPreview.ts b/src/lib/chromaKeyPreview.tsnew file mode 100644index 0000000..e8a7eb5--- /dev/null+++ b/src/lib/chromaKeyPreview.ts@@ -0,0 +1,110 @@+import { applyChromaKey, keyColor, KEY_FEATHER, RGB_DISTANCE_SCALE } from "./chromaKey";+import type { ClipChromaKey } from "./project";++type Picture = HTMLVideoElement | HTMLImageElement;+export interface ChromaKeyRenderer {+ draw(source: Picture, key: ClipChromaKey): void;+ dispose(): void;+}++/** Upload media directly as a texture; no pixel readback during GPU playback. */+export function createChromaKeyRenderer(canvas: HTMLCanvasElement): ChromaKeyRenderer {+ const gl = canvas.getContext("webgl", { alpha: true, premultipliedAlpha: false });+ if (!gl) {+ const context = canvas.getContext("2d", { willReadFrequently: true });+ if (!context) throw new Error("This computer could not create a Chroma Key preview.");+ return {+ draw(source, key) {+ const [width, height] = dimensions(source);+ if (!width || !height) return;+ // Bound CPU work to a monitor-sized picture on machines without WebGL.+ const scale = Math.min(1, 1280 / width);+ canvas.width = Math.round(width * scale);+ canvas.height = Math.round(height * scale);+ context.drawImage(source, 0, 0, canvas.width, canvas.height);+ const pixels = context.getImageData(0, 0, canvas.width, canvas.height);+ applyChromaKey(pixels.data, key);+ context.putImageData(pixels, 0, 0);+ },+ dispose() {},+ };+ }+ const shaders: WebGLShader[] = [];+ const program = gl.createProgram();+ const vertices = gl.createBuffer();+ const texture = gl.createTexture();+ const dispose = () => {+ shaders.forEach((shader) => gl.deleteShader(shader));+ gl.deleteProgram(program);+ gl.deleteBuffer(vertices);+ gl.deleteTexture(texture);+ };+ try {+ if (!program || !vertices || !texture) throw new Error("Could not allocate the Chroma Key preview.");+ const compile = (type: number, code: string) => {+ const shader = gl.createShader(type);+ if (!shader) throw new Error("Could not allocate the Chroma Key shader.");+ shaders.push(shader);+ gl.shaderSource(shader, code);+ gl.compileShader(shader);+ if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) throw new Error(gl.getShaderInfoLog(shader) ?? "Chroma Key shader failed.");+ gl.attachShader(program, shader);+ };+ compile(gl.VERTEX_SHADER, `+ attribute vec2 position;+ varying vec2 uv;+ void main() { uv = (position + 1.0) * 0.5; gl_Position = vec4(position, 0.0, 1.0); }+ `);+ compile(gl.FRAGMENT_SHADER, `+ precision mediump float;+ varying vec2 uv;+ uniform sampler2D source;+ uniform vec4 key;+ void main() {+ vec4 sampled = texture2D(source, uv);+ float difference = distance(sampled.rgb, key.rgb) / ${RGB_DISTANCE_SCALE};+ float alpha = smoothstep(key.a, key.a + ${KEY_FEATHER}, difference);+ gl_FragColor = vec4(sampled.rgb, sampled.a * alpha);+ }+ `);+ gl.linkProgram(program);+ if (!gl.getProgramParameter(program, gl.LINK_STATUS)) throw new Error(gl.getProgramInfoLog(program) ?? "Chroma Key shader failed to link.");+ gl.useProgram(program);+ gl.bindBuffer(gl.ARRAY_BUFFER, vertices);+ gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]), gl.STATIC_DRAW);+ const position = gl.getAttribLocation(program, "position");+ gl.enableVertexAttribArray(position);+ gl.vertexAttribPointer(position, 2, gl.FLOAT, false, 0, 0);+ gl.bindTexture(gl.TEXTURE_2D, texture);+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);+ gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);+ gl.uniform1i(gl.getUniformLocation(program, "source"), 0);+ const keyUniform = gl.getUniformLocation(program, "key");+ return {+ draw(source, key) {+ const [width, height] = dimensions(source);+ if (!width || !height) return;+ if (gl.isContextLost()) throw new Error("The GPU stopped responding during Chroma Key preview.");+ if (canvas.width !== width) canvas.width = width;+ if (canvas.height !== height) canvas.height = height;+ gl.viewport(0, 0, width, height);+ gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, source);+ gl.uniform4f(keyUniform, ...keyColor(key.color), key.tolerance / 100);+ gl.drawArrays(gl.TRIANGLES, 0, 6);+ },+ dispose,+ };+ } catch (error) {+ dispose();+ throw error;+ }+}++function dimensions(source: Picture): [number, number] {+ return source instanceof HTMLVideoElement+ ? source.readyState >= 2 ? [source.videoWidth, source.videoHeight] : [0, 0]+ : source.complete ? [source.naturalWidth, source.naturalHeight] : [0, 0];+}diff --git a/src/lib/export.test.ts b/src/lib/export.test.tsindex e4a12d2..fca7be6 100644--- a/src/lib/export.test.ts+++ b/src/lib/export.test.ts@@ -16,6 +16,7 @@ import {suggestedFileName,videoDurationMs,visibleClipAt,+ visibleClipsAt,type ExportSegment,type ExportSettings,} from "./export";@@ -71,6 +72,35 @@ const settings = (overrides: Partial<ExportSettings> = {}): ExportSettings => ({const clipSegments = (segments: ExportSegment[]) => segments.flatMap((segment) => (segment.kind === "clip" ? [segment] : []));+describe("effect compositing", () => {+ it("splits keyed segments at background cuts, retaining each layer's trim", () => {+ const foreground = clip("green", "track-story", 0, 2000, { chromaKey: { color: "#00ff00", tolerance: 25 } });+ const first = clip("first", "track-v2", 0, 1000, { sourceStartMs: 4000 });+ const second = clip("second", "track-v2", 1000, 1000, { sourceStartMs: 7000 });+ const config = project([foreground, first, second], [asset("asset-green"), asset("asset-first"), asset("asset-second")]);+ expect(visibleClipsAt(config.timeline.tracks, 500).map((item) => item.id)).toEqual(["green", "first"]);+ const plan = buildExportPlan(config, settings());+ const segments = clipSegments(plan.segments);+ expect(segments.map((segment) => [segment.startFrame, segment.endFrame, segment.underlays?.[0].clipId])).toEqual([[0, 30, "first"], [30, 60, "second"]]);+ expect(sourceTimeMsForFrame(segments[1].underlays![0], 45, 30)).toBe(7500);+ expect(segments[0].visual.chromaKey).toEqual(foreground.chromaKey);+ expect(plan.clipCount).toBe(3);+ expect(plan.notes.join(" ")).not.toContain("hidden underneath");+ expect(plan.blockers).toEqual([]);+ });++ it("validates exposed background media and excludes it when the key is removed", () => {+ const foreground = clip("green", "track-story", 0, 2000, { chromaKey: { color: "#00ff00", tolerance: 20 } });+ const background = clip("missing", "track-v2", 0, 2000);+ const config = project([foreground, background], [asset("asset-green")]);+ expect(buildExportPlan(config, settings()).blockers.join(" ")).toContain("missing");+ foreground.chromaKey = undefined;+ const opaque = buildExportPlan(config, settings());+ expect(opaque.blockers).toEqual([]);+ expect(opaque.clipCount).toBe(1);+ });+});+describe("output geometry", () => {it("offers the widescreen sizes MiniMax H3 actually generates at", () => {// The ladder, verbatim. These are not 16:9 rounded off — 2432×1344 is 1.810diff --git a/src/lib/export.ts b/src/lib/export.tsindex c75dc5f..b8dc529 100644--- a/src/lib/export.ts+++ b/src/lib/export.ts@@ -17,6 +17,7 @@ import {clipTransition,type AspectRatio,type ClipLook,+ type ClipChromaKey,type ClipTransform,type ClipTransition,type ProjectAsset,@@ -40,6 +41,7 @@ export interface ExportSettings {export interface ClipVisualSettings {transform: ClipTransform;look: ClipLook;+ chromaKey?: ClipChromaKey | null;transition: ClipTransition;}@@ -54,6 +56,7 @@ export interface ClipFrameStyle extends ClipVisualSettings {export const clipVisualSettings = (clip: TimelineClip): ClipVisualSettings => ({transform: clipTransform(clip),look: clipLook(clip),+ chromaKey: clip.chromaKey,transition: clipTransition(clip),});@@ -230,6 +233,15 @@ export function estimatedBytes(bitrate: number, durationMs: number): number {The plan--------------------------------------------------------------------------- */+export interface ExportClipLayer {+ clipId: string;+ assetId: string;+ label: string;+ clipStartMs: number;+ clipSourceStartMs: number;+ visual: ClipVisualSettings;+}+export type ExportSegment =| {kind: "clip";@@ -245,6 +257,8 @@ export type ExportSegment =/** Where inside the source file the clip begins, in ms. */clipSourceStartMs: number;visual: ClipVisualSettings;+ /** Lower visible tracks, nearest first. Paint in reverse order. */+ underlays?: ExportClipLayer[];}| { kind: "gap"; startFrame: number; endFrame: number };@@ -370,7 +384,7 @@ const DRAWABLE_IMAGE = new Set(["image/png", "image/jpeg", "image/webp", "image/* sits at `frameIndex / frameRate` on the timeline, and the source moment is* that far past the clip's start, offset by where the clip begins in its file. */export function sourceTimeMsForFrame(- segment: Extract<ExportSegment, { kind: "clip" }>,+ segment: ExportClipLayer,frameIndex: number,frameRate: number,): number {@@ -388,6 +402,16 @@ export function visibleClipAt(timeMs: number,assetsById?: ReadonlyMap<string, ProjectAsset>,): TimelineClip | null {+ return visibleClipsAt(tracks, timeMs, assetsById)[0] ?? null;+}++/** Top to bottom, stopping once an opaque clip covers the remaining tracks. */+export function visibleClipsAt(+ tracks: TimelineTrack[],+ timeMs: number,+ assetsById?: ReadonlyMap<string, ProjectAsset>,+): TimelineClip[] {+ const visible: TimelineClip[] = [];for (const track of tracks) {if (track.kind !== "video") continue;let best: TimelineClip | null = null;@@ -396,9 +420,14 @@ export function visibleClipAt(if (timeMs < clip.startMs || timeMs >= clip.startMs + clip.durationMs) continue;if (!best || clip.startMs >= best.startMs) best = clip;}- if (best) return best;+ if (best) {+ visible.push(best);+ const revealing = best.chromaKey || (best.look?.opacity ?? 100) < 100 ||+ (best.transition && best.transition.type !== "cut" && timeMs - best.startMs < best.transition.durationMs);+ if (!revealing) break;+ }}- return null;+ return visible;}/** Where the picture ends. Audio is not muxed yet, so a soundtrack that runs@@ -428,7 +457,7 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)const segments: ExportSegment[] = [];let gapFrames = 0;for (let frame = 0; frame < frameCount; frame += 1) {- const clip = visibleClipAt(videoTracks, (frame * 1000) / frameRate, assetsById);+ const [clip, ...underlays] = visibleClipsAt(videoTracks, (frame * 1000) / frameRate, assetsById);const last = segments[segments.length - 1];if (!clip) {gapFrames += 1;@@ -436,7 +465,8 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)else segments.push({ kind: "gap", startFrame: frame, endFrame: frame + 1 });continue;}- if (last && last.kind === "clip" && last.clipId === clip.id) {+ if (last && last.kind === "clip" && last.clipId === clip.id &&+ (last.underlays?.length ?? 0) === underlays.length && underlays.every((layer, index) => last.underlays?.[index].clipId === layer.id)) {last.endFrame = frame + 1;continue;}@@ -450,10 +480,16 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)clipStartMs: clip.startMs,clipSourceStartMs: clip.sourceStartMs,visual: clipVisualSettings(clip),+ ...(underlays.length ? { underlays: underlays.map((layer) => ({+ clipId: layer.id, assetId: layer.assetId, label: layer.label,+ clipStartMs: layer.startMs, clipSourceStartMs: layer.sourceStartMs,+ visual: clipVisualSettings(layer),+ })) } : {}),});}- const usedClipIds = new Set(segments.flatMap((segment) => (segment.kind === "clip" ? [segment.clipId] : [])));+ const pictureLayers = segments.flatMap((segment) => segment.kind === "clip" ? [segment, ...(segment.underlays ?? [])] : []);+ const usedClipIds = new Set(pictureLayers.map((layer) => layer.clipId));const audioClipCount = audioTimelineClips(config, false).length + audioTimelineClips(config, true).length;const blockers: string[] = [];@@ -463,8 +499,7 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)const missing: string[] = [];const locationless: string[] = [];const undecodable: string[] = [];- for (const segment of segments) {- if (segment.kind !== "clip") continue;+ for (const segment of pictureLayers) {const asset = config.assets.find((candidate) => candidate.id === segment.assetId);if (!asset) {missing.push(segment.label);diff --git a/src/lib/exportPipeline.ts b/src/lib/exportPipeline.tsindex d3c04d6..1555e76 100644--- a/src/lib/exportPipeline.ts+++ b/src/lib/exportPipeline.ts@@ -28,11 +28,13 @@ import {type AudioSegment,type ClipFrameStyle,type ExportPlan,+ type ExportClipLayer,type ExportSegment,type ExportSettings,type OutputCodecId,} from "./export";import type { ProjectAsset, ProjectConfig } from "./project";+import { applyChromaKey, keyColor, KEY_FEATHER, RGB_DISTANCE_SCALE } from "./chromaKey";/* ---------------------------------------------------------------------------What this machine can do@@ -135,7 +137,7 @@ interface Compositor {readonly kind: CompositorKind;/** Paints the background alone — what a frame with no clip over it looks like. */clear(): void;- /** Paints the background, then the frame scaled to fit inside it. */+ /** Blends a frame over the current picture. Call clear before each output. */draw(frame: VideoFrame, style: ClipFrameStyle): void;/** The composited picture, ready for the encoder. Caller closes it. */take(timestampUs: number, durationUs: number): VideoFrame;@@ -161,6 +163,7 @@ struct VertexOut {struct Style {primary: vec4f,secondary: vec4f,+ key: vec4f,};@group(0) @binding(1) var<uniform> style: Style;@group(0) @binding(2) var source: texture_external;@@ -193,7 +196,12 @@ fn fs(@location(0) uv: vec2f) -> @location(0) vec4f {let sampled = textureSampleBaseClampToEdge(source, source_sampler, uv);let temperature = style.primary.z;let rgb = clamp(sampled.rgb + vec3f(temperature * 0.1, 0.0, -temperature * 0.1), vec3f(0.0), vec3f(1.0));- return vec4f(rgb, sampled.a * style.primary.y);+ var alpha = sampled.a * style.primary.y;+ if (style.secondary.y > 0.0) {+ let difference = distance(sampled.rgb, style.key.rgb) / ${RGB_DISTANCE_SCALE};+ alpha *= smoothstep(style.key.a, style.key.a + ${KEY_FEATHER}, difference);+ }+ return vec4f(rgb, alpha);}`;@@ -239,7 +247,7 @@ async function createWebGpuCompositor(width: number, height: number, background:primitive: { topology: "triangle-list" },});const uniform = device.createBuffer({ size: 16, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST });- const styleUniform = device.createBuffer({ size: 32, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST });+ const styleUniform = device.createBuffer({ size: 48, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST });const sampler = device.createSampler({ magFilter: "linear", minFilter: "linear" });/* A device that dies mid-export would keep handing back an untouched canvas,which is a black frame that looks like a rendered one. Record the loss and@@ -261,7 +269,7 @@ async function createWebGpuCompositor(width: number, height: number, background:if (lost.reason) throw new Error(`This computer's GPU stopped responding during the export: ${lost.reason}`);const encoder = device.createCommandEncoder();const renderPass = encoder.beginRenderPass({- colorAttachments: [{ view, clearValue, loadOp: "clear", storeOp: "store" }],+ colorAttachments: [{ view, clearValue, loadOp: bindGroup ? "load" : "clear", storeOp: "store" }],});if (bindGroup) {renderPass.setPipeline(pipeline);@@ -297,9 +305,11 @@ async function createWebGpuCompositor(width: number, height: number, background:style.look.temperature / 100,style.revealStart,style.revealEnd,+ style.chromaKey ? 1 : 0,0,0,- 0,+ ...(style.chromaKey ? keyColor(style.chromaKey.color) : [0, 0, 0]),+ (style.chromaKey?.tolerance ?? 0) / 100,]));// Zero copy: the decoded frame is sampled where it already lives.const external = device.importExternalTexture({ source: frame });@@ -328,6 +338,7 @@ function createCanvasCompositor(width: number, height: number, background: strinconst canvas = new OffscreenCanvas(width, height);const context = canvas.getContext("2d", { alpha: false });if (!context) throw new Error("This computer gave no 2D canvas to composite into.");+ let keyCanvas: OffscreenCanvas | null = null;const paintBackground = () => {context.fillStyle = background;context.fillRect(0, 0, width, height);@@ -336,7 +347,20 @@ function createCanvasCompositor(width: number, height: number, background: strinkind: "canvas2d",clear: paintBackground,draw(frame, style) {- paintBackground();+ let picture: VideoFrame | OffscreenCanvas = frame;+ if (style.chromaKey) {+ keyCanvas ??= new OffscreenCanvas(frame.displayWidth, frame.displayHeight);+ if (keyCanvas.width !== frame.displayWidth) keyCanvas.width = frame.displayWidth;+ if (keyCanvas.height !== frame.displayHeight) keyCanvas.height = frame.displayHeight;+ const keyContext = keyCanvas.getContext("2d", { willReadFrequently: true });+ if (!keyContext) throw new Error("Could not create the Chroma Key canvas.");+ keyContext.clearRect(0, 0, keyCanvas.width, keyCanvas.height);+ keyContext.drawImage(frame, 0, 0);+ const pixels = keyContext.getImageData(0, 0, keyCanvas.width, keyCanvas.height);+ applyChromaKey(pixels.data, style.chromaKey);+ keyContext.putImageData(pixels, 0, 0);+ picture = keyCanvas;+ }const fitted = fitRect(frame.displayWidth, frame.displayHeight, width, height);const scale = style.transform.scale / 100;const drawWidth = fitted.width * scale;@@ -354,7 +378,7 @@ function createCanvasCompositor(width: number, height: number, background: strin: `sepia(${Math.abs(warmth) * 0.22}) saturate(${1 + Math.abs(warmth) * 0.3}) hue-rotate(${warmth > 0 ? -8 : 172}deg)`;context.translate(centreX, centreY);context.rotate((style.transform.rotation * Math.PI) / 180);- context.drawImage(frame, -drawWidth / 2, -drawHeight / 2, drawWidth, drawHeight);+ context.drawImage(picture, -drawWidth / 2, -drawHeight / 2, drawWidth, drawHeight);context.restore();},take(timestamp, duration) {@@ -805,6 +829,86 @@ export interface ExportRunOptions {const tick = () => new Promise<void>((resolve) => setTimeout(resolve, 0));+interface LayerFrameReader {+ /** Borrowed frame, valid until the next call or dispose. Times must increase. */+ frameAt(timestampUs: number): Promise<VideoFrame>;+ dispose(): void;+}++/** One bounded decoder queue per visible layer. Decoded pictures stay in VRAM;+ * only compressed samples and the current/next pictures are held per source. */+async function openLayerFrameReader(+ folderPath: string, asset: ProjectAsset, startUs: number, stopIfCancelled: () => void,+): Promise<LayerFrameReader> {+ const bytes = await readAssetBytes(folderPath, asset);+ stopIfCancelled();+ if (asset.mimeType.startsWith("image/")) {+ const bitmap = await createImageBitmap(new Blob([bytes], { type: asset.mimeType }));+ try {+ const frame = new VideoFrame(bitmap, { timestamp: 0 });+ return { frameAt: async () => { stopIfCancelled(); return frame; }, dispose: () => { frame.close(); bitmap.close(); } };+ } catch (reason) { bitmap.close(); throw reason; }+ }+ const source = await demux(bytes, asset.name);+ let nextSample = 0;+ let keyframeUs = Number.NEGATIVE_INFINITY;+ source.samples.forEach((sample, index) => {+ if (sample.key && sample.timestampUs <= startUs && sample.timestampUs > keyframeUs) {+ keyframeUs = sample.timestampUs;+ nextSample = index;+ }+ });+ const pending: VideoFrame[] = [];+ let screen: VideoFrame | null = null;+ let flushed = false;+ const failure: { reason: Error | null } = { reason: null };+ const decoder = new VideoDecoder({+ output: (frame) => pending.push(frame),+ error: (reason) => { failure.reason = reason; },+ });+ const dispose = () => {+ if (decoder.state !== "closed") decoder.close();+ screen?.close();+ pending.forEach((frame) => frame.close());+ pending.length = 0;+ screen = null;+ };+ try { decoder.configure(source.config); }+ catch (reason) { dispose(); throw reason; }+ return {+ async frameAt(wantUs) {+ while (true) {+ stopIfCancelled();+ if (failure.reason) throw failure.reason;+ // VideoDecoder outputs presentation order, including reordered B frames.+ while (pending.length && (!screen || pending[0].timestamp <= wantUs)) {+ screen?.close();+ screen = pending.shift()!;+ }+ if (screen && (pending.length || flushed)) return screen;+ if (flushed) throw new Error(`${asset.name} decoded no frames.`);+ if (nextSample >= source.samples.length) {+ await decoder.flush();+ flushed = true;+ } else {+ // Yield between small batches so output can arrive before feeding+ // more samples. This bounds decoded frames even for long sources.+ for (let count = 0; count < 4 && nextSample < source.samples.length; count += 1) {+ const sample = source.samples[nextSample++];+ decoder.decode(new EncodedVideoChunk({+ type: sample.key ? "key" : "delta", timestamp: sample.timestampUs,+ duration: sample.durationUs, data: sample.data,+ }));+ }+ while (decoder.decodeQueueSize > 8) { stopIfCancelled(); if (failure.reason) throw failure.reason; await tick(); }+ await tick();+ }+ }+ },+ dispose,+ };+}+/** One sentence about the sound in the finished file, true in every case —* including the cases where there is none. */function describeSoundtrack(@@ -925,10 +1029,13 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResultcompositor: createCanvasCompositor(plan.width, plan.height, plan.backgroundColor),detail: attempt.reason,};- const paint = (frame: VideoFrame | null, style: ClipFrameStyle | null) => {+ const paint = (pictures: { frame: VideoFrame; style: ClipFrameStyle }[]) => {+ const draw = () => {+ stage.compositor.clear();+ for (const picture of pictures) stage.compositor.draw(picture.frame, picture.style);+ };try {- if (frame && style) stage.compositor.draw(frame, style);- else stage.compositor.clear();+ draw();return;} catch (reason) {if (stage.compositor.kind === "canvas2d") throw reason;@@ -936,8 +1043,7 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResultstage.compositor = createCanvasCompositor(plan.width, plan.height, plan.backgroundColor);stage.detail = `The GPU compositor gave out ${framesDone} frames in and the rest was composited on a 2D canvas: ${reason instanceof Error ? reason.message : String(reason)}`;}- if (frame && style) stage.compositor.draw(frame, style);- else stage.compositor.clear();+ draw();};/* Timestamps are computed from the frame index every time rather than@@ -953,12 +1059,17 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResultframe: VideoFrame | null,outputIndex: number,segment: Extract<ExportSegment, { kind: "clip" }> | null = null,+ underlays: { frame: VideoFrame; layer: ExportClipLayer }[] = [],) => {stopIfCancelled();if (encoderFailure.reason) throw encoderFailure.reason;const timelineMs = (outputIndex * 1000) / plan.frameRate;const style = segment ? clipFrameStyle(segment.visual, timelineMs - segment.clipStartMs) : null;- paint(frame, style);+ const pictures = underlays.slice().reverse().map((picture) => ({+ frame: picture.frame, style: clipFrameStyle(picture.layer.visual, timelineMs - picture.layer.clipStartMs),+ }));+ if (frame && style) pictures.push({ frame, style });+ paint(pictures);const composited = stage.compositor.take(timestampUs(outputIndex),timestampUs(outputIndex + 1) - timestampUs(outputIndex),@@ -1125,11 +1236,40 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResult}};+ const renderLayeredSegment = async (segment: Extract<ExportSegment, { kind: "clip" }>) => {+ releaseLoaded();+ loaded.assetId = null;+ const layers = [segment, ...(segment.underlays ?? [])];+ const readers: LayerFrameReader[] = [];+ try {+ for (const layer of layers) {+ stopIfCancelled();+ const asset = assetFor(layer.assetId);+ if (!asset) throw new Error(`Media for ${layer.label} is missing.`);+ report("rendering", framesDone, `Reading ${asset.name}…`);+ readers.push(await openLayerFrameReader(folderPath, asset, sourceTimeMsForFrame(layer, segment.startFrame, plan.frameRate) * 1000, stopIfCancelled));+ }+ for (let index = segment.startFrame; index < segment.endFrame; index += 1) {+ const pictures: { frame: VideoFrame; layer: ExportClipLayer }[] = [];+ for (let layerIndex = 0; layerIndex < layers.length; layerIndex += 1) {+ const layer = layers[layerIndex];+ const frame = await readers[layerIndex].frameAt(sourceTimeMsForFrame(layer, index, plan.frameRate) * 1000);+ pictures.push({ frame, layer });+ }+ await emit(pictures[0].frame, index, segment, pictures.slice(1));+ }+ } finally {+ readers.forEach((reader) => reader.dispose());+ }+ };+try {for (const segment of plan.segments) {stopIfCancelled();if (segment.kind === "gap") {for (let index = segment.startFrame; index < segment.endFrame; index += 1) await emit(null, index);+ } else if (segment.underlays?.length) {+ await renderLayeredSegment(segment);} else {await renderClipSegment(segment);}diff --git a/src/lib/project.ts b/src/lib/project.tsindex c8d3de7..e75db98 100644--- a/src/lib/project.ts+++ b/src/lib/project.ts@@ -180,6 +180,11 @@ export const clipTransformSchema = z.object({positionY: z.number().min(-100).max(100),});+export const clipChromaKeySchema = z.object({+ color: z.string().regex(/^#[0-9a-fA-F]{6}$/),+ tolerance: z.number().min(0).max(100),+});+export const clipLookSchema = z.object({opacity: z.number().min(0).max(100),/** Cool at -100, neutral at 0, warm at +100. */@@ -205,6 +210,7 @@ export const timelineClipSchema = z.object({/** Optional so projects written before clip styling keep their exact shape. */transform: clipTransformSchema.nullish(),look: clipLookSchema.nullish(),+ chromaKey: clipChromaKeySchema.nullish(),/** Applied at this clip's head when the cut changes to it. */transition: clipTransitionSchema.nullish(),});@@ -509,10 +515,12 @@ export type TimelineClip = z.infer<typeof timelineClipSchema>;export type TimelineTrack = z.infer<typeof timelineTrackSchema>;export type ClipTransform = z.infer<typeof clipTransformSchema>;export type ClipLook = z.infer<typeof clipLookSchema>;+export type ClipChromaKey = z.infer<typeof clipChromaKeySchema>;export type ClipTransition = z.infer<typeof clipTransitionSchema>;export const DEFAULT_CLIP_TRANSFORM: ClipTransform = { scale: 100, rotation: 0, positionX: 0, positionY: 0 };export const DEFAULT_CLIP_LOOK: ClipLook = { opacity: 100, temperature: 0 };+export const DEFAULT_CLIP_CHROMA_KEY: ClipChromaKey = { color: "#00ff00", tolerance: 20 };export const DEFAULT_CLIP_TRANSITION: ClipTransition = { type: "cut", durationMs: 500 };const roundTransformValue = (value: number): number => {diff --git a/src/styles/timeline.css b/src/styles/timeline.cssindex fabaa1b..dbc9215 100644--- a/src/styles/timeline.css+++ b/src/styles/timeline.css@@ -529,7 +529,8 @@}.program-picture video,-.program-picture img {+.program-picture img,+.program-picture canvas {display: block;width: 100%;height: 100%;@@ -539,7 +540,8 @@}.program-picture--selectable video,-.program-picture--selectable img {+.program-picture--selectable img,+.program-picture--selectable canvas {cursor: pointer;}@@ -1500,3 +1502,25 @@--track-column: 168px;}}++/* Optional clip effects share the compact inspector controls. */+.clip-effect > header { display: flex; align-items: center; justify-content: space-between; }+.clip-effect fieldset { margin: 0; padding: 0; border: 0; min-width: 0; }+.clip-effect__remove { display: grid; place-items: center; padding: 4px; border: 0; border-radius: var(--radius-sm); color: var(--text-muted); background: transparent; }+.clip-effect__remove:hover:not(:disabled) { color: var(--text-primary); background: var(--panel-3); }+.clip-effect .effect-color { display: flex; align-items: center; gap: var(--space-2); }+.clip-effect .effect-color > span { margin: 0; margin-right: auto; }+.clip-effect .effect-color input { width: 36px; padding: 3px; cursor: pointer; }+.effect-color code { font-size: var(--text-xs); color: var(--text-muted); }+.clip-effects { position: relative; padding: var(--space-3) var(--space-4); }+.clip-effects__add { display: flex; align-items: center; justify-content: center; gap: var(--space-2); width: 100%; padding: var(--space-2); border: 1px solid var(--line-strong); border-radius: var(--radius-sm); background: var(--panel-2); color: var(--text-primary); font-size: var(--text-sm); }+.clip-effects__add:hover:not(:disabled) { border-color: var(--accent); background: var(--panel-3); }+.clip-effects__add:disabled, .clip-effect__remove:disabled { opacity: .45; cursor: not-allowed; }+.clip-effects__picker { position: absolute; z-index: 5; bottom: calc(100% - var(--space-2)); left: var(--space-4); right: var(--space-4); padding: 4px; border: 1px solid var(--line-strong); border-radius: var(--radius-md); background: var(--panel-2); box-shadow: var(--shadow-sm); max-height: min(280px, 40vh); overflow-y: auto; }+.clip-effects__picker button { display: flex; flex-direction: column; gap: 3px; width: 100%; border: 0; border-radius: var(--radius-sm); padding: var(--space-2); background: transparent; color: var(--text-primary); text-align: left; }+.clip-effects__picker button:hover, .clip-effects__picker button:focus-visible { outline: none; background: var(--panel-3); }+.clip-effects__picker strong { font-size: var(--text-sm); font-weight: var(--weight-medium); }+.clip-effects__picker span { font-size: var(--text-xs); color: var(--text-muted); }++.program-layer { position: absolute; inset: 0; pointer-events: none; }+.program-picture > video, .program-picture > img, .program-keyed-picture { position: absolute; inset: 0; }