Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Add optional clip effects

Bittibd25ae2d2df8
diff --git a/src-tauri/src/agent_commands.rs b/src-tauri/src/agent_commands.rs
index 3dc6829..7512416 100644
--- a/src-tauri/src/agent_commands.rs
+++ b/src-tauri/src/agent_commands.rs
@@ -822,6 +822,7 @@ fn apply_command(
status: status.into(),
transform: None,
look: None,
+ chroma_key: None,
transition: None,
});
}
diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs
index f21e837..4bad7b7 100644
--- a/src-tauri/src/lib.rs
+++ b/src-tauri/src/lib.rs
@@ -150,6 +150,8 @@ struct TimelineClip {
#[serde(default, skip_serializing_if = "Option::is_none")]
look: Option<ClipLook>,
#[serde(default, skip_serializing_if = "Option::is_none")]
+ chroma_key: Option<ClipChromaKey>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
transition: Option<ClipTransition>,
}
@@ -168,6 +170,12 @@ struct ClipLook {
temperature: f64,
}
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+struct ClipChromaKey {
+ color: String,
+ tolerance: f64,
+}
+
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
struct ClipTransition {
@@ -964,6 +972,16 @@ fn validate_and_normalize_config(mut config: ProjectConfig) -> Result<ProjectCon
return Err(format!("Clip '{}' has an invalid look.", clip.id));
}
}
+ if let Some(key) = &clip.chroma_key {
+ if key.color.len() != 7
+ || !key.color.starts_with('#')
+ || !key.color.as_bytes()[1..].iter().all(u8::is_ascii_hexdigit)
+ || !key.tolerance.is_finite()
+ || !(0.0..=100.0).contains(&key.tolerance)
+ {
+ return Err(format!("Clip '{}' has an invalid chroma key.", clip.id));
+ }
+ }
if let Some(transition) = &clip.transition {
if !matches!(
transition.r#type.as_str(),
@@ -3305,6 +3323,32 @@ mod tests {
assert_eq!(files, [PROJECT_FILE_NAME]);
}
+ #[test]
+ fn chroma_key_survives_save_and_reopen_and_validates_settings() {
+ let mut config = fixture();
+ config.timeline.tracks[0].clips[0].chroma_key = Some(ClipChromaKey {
+ color: "#12ABef".into(),
+ tolerance: 27.0,
+ });
+ let config = without_derived_project_state(validate_and_normalize_config(config).unwrap());
+ let root = tempfile::tempdir().unwrap();
+ let created = create_project_in(root.path(), &config).unwrap();
+ let folder = PathBuf::from(&created.folder_path);
+ write_project(&folder, &config).unwrap();
+ assert_eq!(read_project(&folder).unwrap().config, config);
+ let json = serde_json::to_value(&config).unwrap();
+ assert_eq!(json["timeline"]["tracks"][0]["clips"][0]["chromaKey"]["color"], "#12ABef");
+ for (color, tolerance) in [("green", 20.0), ("#12345g", 20.0), ("#00ff00", -1.0), ("#00ff00", 101.0), ("#00ff00", f64::NAN)] {
+ let mut invalid = config.clone();
+ invalid.timeline.tracks[0].clips[0].chroma_key = Some(ClipChromaKey { color: color.into(), tolerance });
+ assert!(validate_and_normalize_config(invalid).unwrap_err().contains("chroma key"));
+ }
+ let mut removed = config.clone();
+ removed.timeline.tracks[0].clips[0].chroma_key = None;
+ write_project(&folder, &removed).unwrap();
+ assert!(read_project(&folder).unwrap().config.timeline.tracks[0].clips[0].chroma_key.is_none());
+ }
+
#[test]
fn deleting_a_project_removes_its_folder_and_every_file_inside_it() {
let parent = tempfile::tempdir().unwrap();
diff --git a/src/components/workspace/ChromaKeyPreview.tsx b/src/components/workspace/ChromaKeyPreview.tsx
new file mode 100644
index 0000000..ec350c5
--- /dev/null
+++ b/src/components/workspace/ChromaKeyPreview.tsx
@@ -0,0 +1,46 @@
+import { useEffect, useRef, type CSSProperties, type RefObject } from "react";
+import { createChromaKeyRenderer } from "../../lib/chromaKeyPreview";
+import type { ClipChromaKey } from "../../lib/project";
+
+export function ChromaKeyPreview({ source, sourceUrl, effect, playing, style, onError }: {
+ source: RefObject<HTMLVideoElement | HTMLImageElement>;
+ sourceUrl: string;
+ effect: ClipChromaKey;
+ playing: boolean;
+ style?: CSSProperties;
+ onError: (message: string) => void;
+}) {
+ const canvas = useRef<HTMLCanvasElement>(null);
+ const settings = useRef(effect);
+ const errorHandler = useRef(onError);
+ const redraw = useRef<(() => void) | null>(null);
+ settings.current = effect;
+ errorHandler.current = onError;
+ useEffect(() => {
+ const media = source.current;
+ if (!canvas.current || !media) return;
+ let frame = 0;
+ let failed = false;
+ let renderer: ReturnType<typeof createChromaKeyRenderer>;
+ try { renderer = createChromaKeyRenderer(canvas.current); }
+ catch (reason) { errorHandler.current(String(reason)); return; }
+ const draw = () => {
+ if (failed) return;
+ try { renderer.draw(media, settings.current); }
+ catch (reason) { failed = true; errorHandler.current(String(reason)); }
+ };
+ const animate = () => { draw(); if (!failed) frame = requestAnimationFrame(animate); };
+ redraw.current = draw;
+ ["loadeddata", "seeked", "load"].forEach((event) => media.addEventListener(event, draw));
+ draw();
+ if (playing) frame = requestAnimationFrame(animate);
+ return () => {
+ cancelAnimationFrame(frame);
+ redraw.current = null;
+ ["loadeddata", "seeked", "load"].forEach((event) => media.removeEventListener(event, draw));
+ renderer.dispose();
+ };
+ }, [source, sourceUrl, playing]);
+ useEffect(() => redraw.current?.(), [effect]);
+ return <canvas ref={canvas} className="program-keyed-picture" style={style} aria-label="Chroma Key preview" />;
+}
diff --git a/src/components/workspace/ClipEffects.test.tsx b/src/components/workspace/ClipEffects.test.tsx
new file mode 100644
index 0000000..902b433
--- /dev/null
+++ b/src/components/workspace/ClipEffects.test.tsx
@@ -0,0 +1,65 @@
+// @vitest-environment jsdom
+import { cleanup, fireEvent, render, screen, within } from "@testing-library/react";
+import { useState } from "react";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { timelineClipSchema, type TimelineClip } from "../../lib/project";
+import { ClipEffects } from "./ClipEffects";
+
+afterEach(cleanup);
+const original: TimelineClip = {
+ id: "clip", assetId: "asset", trackId: "video", label: "Clip", startMs: 0,
+ durationMs: 2000, sourceStartMs: 0, status: "approved",
+};
+function Harness() {
+ const [clip, setClip] = useState(original);
+ return <><ClipEffects clip={clip} disabled={false} onChange={(patch) => setClip((current) => timelineClipSchema.parse({ ...current, ...patch }))} /><output data-testid="saved">{JSON.stringify(clip)}</output></>;
+}
+
+describe("clip effects", () => {
+ it("adds only chosen effects, persists edits, and removes them without changing other effects", () => {
+ render(<Harness />);
+ expect(screen.queryByRole("heading", { name: "Look" })).toBeNull();
+ expect(screen.queryByRole("heading", { name: "Transition" })).toBeNull();
+ fireEvent.click(screen.getByRole("button", { name: "Add Effect" }));
+ fireEvent.click(screen.getByRole("button", { name: /Chroma Key Make/ }));
+ fireEvent.change(screen.getByLabelText("Chroma Key color"), { target: { value: "#123456" } });
+ fireEvent.change(screen.getByLabelText("Chroma Key tolerance"), { target: { value: "43" } });
+ fireEvent.click(screen.getByRole("button", { name: "Add Effect" }));
+ const picker = screen.getByRole("group", { name: "Available effects" });
+ expect(within(picker).queryByText("Chroma Key")).toBeNull();
+ fireEvent.click(within(picker).getByRole("button", { name: /Look Opacity/ }));
+ fireEvent.change(screen.getByLabelText("Clip opacity"), { target: { value: "65" } });
+ expect(JSON.parse(screen.getByTestId("saved").textContent!)).toMatchObject({ chromaKey: { color: "#123456", tolerance: 43 }, look: { opacity: 65 } });
+ fireEvent.click(screen.getByRole("button", { name: "Remove Chroma Key effect" }));
+ const saved = JSON.parse(screen.getByTestId("saved").textContent!);
+ expect(saved.chromaKey).toBeUndefined();
+ expect(saved.look.opacity).toBe(65);
+ fireEvent.click(screen.getByRole("button", { name: "Add Effect" }));
+ expect(screen.getByRole("button", { name: /Chroma Key Make/ })).toBeTruthy();
+ });
+
+ it("keeps existing effects editable and closes the keyboard picker with Escape", () => {
+ render(<ClipEffects clip={{ ...original, transition: { type: "wipe-left", durationMs: 800 } }} disabled={false} onChange={vi.fn()} />);
+ expect((screen.getByLabelText("Clip transition") as HTMLSelectElement).value).toBe("wipe-left");
+ fireEvent.click(screen.getByRole("button", { name: "Add Effect" }));
+ expect(document.activeElement?.textContent).toContain("Look");
+ fireEvent.keyDown(document.activeElement!, { key: "ArrowDown" });
+ expect(document.activeElement?.textContent).toContain("Chroma Key");
+ fireEvent.keyDown(document.activeElement!, { key: "Escape" });
+ expect(screen.queryByRole("group", { name: "Available effects" })).toBeNull();
+ expect(document.activeElement).toBe(screen.getByRole("button", { name: "Add Effect" }));
+ });
+
+ it("disables effects for locked or nonvisual clips and closes the picker when selection changes", () => {
+ const props = { onChange: vi.fn(), clip: original, disabled: false };
+ const view = render(<ClipEffects {...props} />);
+ fireEvent.click(screen.getByRole("button", { name: "Add Effect" }));
+ view.rerender(<ClipEffects {...props} clip={{ ...original, id: "next" }} />);
+ expect(screen.queryByRole("group", { name: "Available effects" })).toBeNull();
+ view.rerender(<ClipEffects {...props} disabled clip={{ ...original, chromaKey: { color: "#00ff00", tolerance: 20 } }} />);
+ expect((screen.getByRole("button", { name: "Add Effect" }) as HTMLButtonElement).disabled).toBe(true);
+ expect((screen.getByRole("button", { name: "Remove Chroma Key effect" }) as HTMLButtonElement).disabled).toBe(true);
+ fireEvent.change(screen.getByLabelText("Chroma Key tolerance"), { target: { value: "40" } });
+ expect(props.onChange).not.toHaveBeenCalled();
+ });
+});
diff --git a/src/components/workspace/ClipEffects.tsx b/src/components/workspace/ClipEffects.tsx
new file mode 100644
index 0000000..ba3c8f3
--- /dev/null
+++ b/src/components/workspace/ClipEffects.tsx
@@ -0,0 +1,95 @@
+import { Plus, X } from "lucide-react";
+import { useEffect, useId, useRef, useState, type ReactNode } from "react";
+import { DEFAULT_CLIP_CHROMA_KEY, DEFAULT_CLIP_LOOK, type TimelineClip } from "../../lib/project";
+
+type EffectId = "look" | "transition" | "chromaKey";
+type EditorProps = { clip: TimelineClip; update: (patch: Partial<TimelineClip>) => void };
+type EffectDefinition = {
+ id: EffectId;
+ name: string;
+ description: string;
+ defaults: Partial<TimelineClip>;
+ editor: (props: EditorProps) => ReactNode;
+};
+
+// One catalogue drives the picker, defaults and editors. Missing settings mean
+// the effect is absent; older projects with Look/Transition keep their edits.
+const EFFECTS: readonly EffectDefinition[] = [
+ {
+ id: "look", name: "Look", description: "Opacity and color temperature",
+ defaults: { look: DEFAULT_CLIP_LOOK },
+ editor: ({ clip, update }) => {
+ const look = clip.look!;
+ return <>
+ <label className="range-field"><span>Opacity <b>{look.opacity}%</b></span><input aria-label="Clip opacity" type="range" min="0" max="100" step="1" value={look.opacity} onChange={(event) => update({ look: { ...look, opacity: Number(event.target.value) } })} /></label>
+ <label className="range-field"><span>Temperature <b>{look.temperature > 0 ? "+" : ""}{look.temperature}</b></span><input aria-label="Clip temperature" type="range" min="-100" max="100" step="1" value={look.temperature} onChange={(event) => update({ look: { ...look, temperature: Number(event.target.value) } })} /></label>
+ </>;
+ },
+ },
+ {
+ id: "transition", name: "Transition", description: "Fade or wipe into the clip",
+ defaults: { transition: { type: "fade", durationMs: 500 } },
+ editor: ({ clip, update }) => {
+ const transition = clip.transition!;
+ return <>
+ <label><span>When this clip begins</span><select aria-label="Clip transition" value={transition.type} onChange={(event) => update({ transition: { ...transition, type: event.target.value as typeof transition.type } })}>
+ <option value="cut">Cut</option><option value="fade">Fade in</option><option value="wipe-left">Wipe from left</option><option value="wipe-right">Wipe from right</option>
+ </select></label>
+ <label className="range-field"><span>Duration <b>{transition.durationMs} ms</b></span><input aria-label="Transition duration" type="range" min="100" max="3000" step="100" value={transition.durationMs} disabled={transition.type === "cut"} onChange={(event) => update({ transition: { ...transition, durationMs: Number(event.target.value) } })} /></label>
+ </>;
+ },
+ },
+ {
+ id: "chromaKey", name: "Chroma Key", description: "Make a selected color transparent",
+ defaults: { chromaKey: DEFAULT_CLIP_CHROMA_KEY },
+ editor: ({ clip, update }) => {
+ const key = clip.chromaKey!;
+ return <>
+ <label className="effect-color"><span>Color</span><input aria-label="Chroma Key color" type="color" value={key.color} onChange={(event) => update({ chromaKey: { ...key, color: event.target.value } })} /><code>{key.color.toUpperCase()}</code></label>
+ <label className="range-field"><span>Tolerance <b>{key.tolerance}%</b></span><input aria-label="Chroma Key tolerance" type="range" min="0" max="100" step="1" value={key.tolerance} onChange={(event) => update({ chromaKey: { ...key, tolerance: Number(event.target.value) } })} /></label>
+ </>;
+ },
+ },
+];
+
+export function ClipEffects({ clip, disabled, onChange }: {
+ clip: TimelineClip;
+ disabled: boolean;
+ onChange: (patch: Partial<TimelineClip>) => void;
+}) {
+ const [open, setOpen] = useState(false);
+ const root = useRef<HTMLDivElement>(null);
+ const trigger = useRef<HTMLButtonElement>(null);
+ const pickerId = useId();
+ const available = EFFECTS.filter((effect) => !clip[effect.id]);
+ const update = (patch: Partial<TimelineClip>) => { if (!disabled) onChange(patch); };
+ useEffect(() => setOpen(false), [clip.id, disabled]);
+ useEffect(() => {
+ if (!open) return;
+ const closeOutside = (event: PointerEvent) => { if (!root.current?.contains(event.target as Node)) setOpen(false); };
+ document.addEventListener("pointerdown", closeOutside);
+ root.current?.querySelector<HTMLButtonElement>(".clip-effects__picker button")?.focus();
+ return () => document.removeEventListener("pointerdown", closeOutside);
+ }, [open]);
+ return <>
+ {EFFECTS.filter((effect) => clip[effect.id]).map((effect) => <section className="inspector-section clip-effect" key={effect.id} aria-label={`${effect.name} effect`}>
+ <header><h3>{effect.name}</h3><button type="button" className="clip-effect__remove" aria-label={`Remove ${effect.name} effect`} disabled={disabled} onClick={() => update({ [effect.id]: undefined })}><X size={14} /></button></header>
+ <fieldset disabled={disabled}>{effect.editor({ clip, update })}</fieldset>
+ </section>)}
+ <div className="clip-effects" ref={root} onKeyDown={(event) => {
+ if (event.key === "Escape" && open) { event.stopPropagation(); setOpen(false); trigger.current?.focus(); }
+ if (open && (event.key === "ArrowDown" || event.key === "ArrowUp" || event.key === "Home" || event.key === "End")) {
+ event.preventDefault();
+ const buttons = Array.from(root.current?.querySelectorAll<HTMLButtonElement>(".clip-effects__picker button") ?? []);
+ const index = buttons.indexOf(document.activeElement as HTMLButtonElement);
+ const next = event.key === "Home" ? 0 : event.key === "End" ? buttons.length - 1 : (index + (event.key === "ArrowUp" ? -1 : 1) + buttons.length) % buttons.length;
+ buttons[next]?.focus();
+ }
+ }} onBlur={(event) => { if (!event.currentTarget.contains(event.relatedTarget)) setOpen(false); }}>
+ <button ref={trigger} type="button" className="clip-effects__add" disabled={disabled || available.length === 0} aria-expanded={open} aria-controls={pickerId} onClick={() => setOpen((value) => !value)}><Plus size={16} /> Add Effect</button>
+ {open && <div id={pickerId} className="clip-effects__picker" role="group" aria-label="Available effects">
+ {available.map((effect) => <button type="button" key={effect.id} onClick={() => { update(effect.defaults); setOpen(false); trigger.current?.focus(); }}><strong>{effect.name}</strong><span>{effect.description}</span></button>)}
+ </div>}
+ </div>
+ </>;
+}
diff --git a/src/components/workspace/ProgramLayer.tsx b/src/components/workspace/ProgramLayer.tsx
new file mode 100644
index 0000000..0f1843a
--- /dev/null
+++ b/src/components/workspace/ProgramLayer.tsx
@@ -0,0 +1,61 @@
+import { useEffect, useRef, useState, type CSSProperties } from "react";
+import { clipFrameStyle, clipVisualSettings, type ClipFrameStyle } from "../../lib/export";
+import { PreviewSources } from "../../lib/exportPipeline";
+import type { ProjectAsset, TimelineClip } from "../../lib/project";
+import { ChromaKeyPreview } from "./ChromaKeyPreview";
+
+export function previewMediaStyle({ transform, look, opacity, revealStart, revealEnd }: ClipFrameStyle): CSSProperties {
+ const warmth = look.temperature / 100;
+ return {
+ transform: `translate(${transform.positionX}%, ${transform.positionY}%) scale(${transform.scale / 100}) rotate(${transform.rotation}deg)`,
+ opacity,
+ filter: warmth === 0 ? "none" : `sepia(${Math.abs(warmth) * 0.22}) saturate(${1 + Math.abs(warmth) * 0.3}) hue-rotate(${warmth > 0 ? -8 : 172}deg)`,
+ clipPath: `inset(0 ${(1 - revealEnd) * 100}% 0 ${revealStart * 100}%)`,
+ };
+}
+
+/** Lower picture tracks follow the monitor clock. Their audio is mixed by the
+ * monitor's existing audio elements, so these video elements stay muted. */
+export function ProgramLayer({ clip, asset, sources, playheadMs, playing }: {
+ clip: TimelineClip;
+ asset: ProjectAsset | undefined;
+ sources: PreviewSources;
+ playheadMs: number;
+ playing: boolean;
+}) {
+ const [url, setUrl] = useState<string | null>(null);
+ const [error, setError] = useState<string | null>(null);
+ const video = useRef<HTMLVideoElement>(null);
+ const image = useRef<HTMLImageElement>(null);
+ const position = useRef({ clip, playheadMs, playing });
+ position.current = { clip, playheadMs, playing };
+ const isImage = asset?.mimeType.startsWith("image/");
+ useEffect(() => {
+ let live = true;
+ setUrl(null);
+ setError(null);
+ if (asset && (asset.relativePath || asset.sourcePath)) {
+ void sources.url(asset).then((value) => { if (live) setUrl(value); }, (reason) => { if (live) setError(String(reason)); });
+ } else setError(`Media for ${clip.label} is unavailable.`);
+ return () => { live = false; };
+ }, [asset, sources, clip.label]);
+ const sync = () => {
+ const element = video.current;
+ if (!element || element.readyState < 1) return;
+ const state = position.current;
+ const target = Math.max(0, (state.clip.sourceStartMs + state.playheadMs - state.clip.startMs) / 1000);
+ if (Math.abs(element.currentTime - target) > (state.playing ? 0.25 : 0.033)) element.currentTime = target;
+ if (state.playing) { if (element.paused) void element.play().catch(() => undefined); }
+ else element.pause();
+ };
+ useEffect(sync, [playheadMs, playing, clip, url]);
+ const style = previewMediaStyle(clipFrameStyle(clipVisualSettings(clip), playheadMs - clip.startMs));
+ const sourceStyle: CSSProperties = clip.chromaKey ? { ...style, visibility: "hidden", position: "absolute" } : style;
+ return <div className="program-layer" aria-label={`Background clip: ${clip.label}`}>
+ {url && (isImage
+ ? <img ref={image} src={url} alt={clip.label} style={sourceStyle} onError={() => setError(`Could not decode ${clip.label}.`)} />
+ : <video ref={video} src={url} style={sourceStyle} muted playsInline preload="auto" onLoadedMetadata={sync} onError={() => setError(`Could not decode ${clip.label}.`)} />)}
+ {url && clip.chromaKey && <ChromaKeyPreview source={isImage ? image : video} sourceUrl={url} effect={clip.chromaKey} playing={playing && !isImage} style={style} onError={setError} />}
+ {error && <div className="program-note program-note--error" role="alert">{error}</div>}
+ </div>;
+}
diff --git a/src/components/workspace/ProgramMonitor.tsx b/src/components/workspace/ProgramMonitor.tsx
index 09c5a34..4e374b6 100644
--- a/src/components/workspace/ProgramMonitor.tsx
+++ b/src/components/workspace/ProgramMonitor.tsx
@@ -1,6 +1,8 @@
+import { ChromaKeyPreview } from "./ChromaKeyPreview";
+import { ProgramLayer, previewMediaStyle } from "./ProgramLayer";
import { Film, TriangleAlert } from "lucide-react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
-import { clipFrameStyle, clipVisualSettings, visibleClipAt } from "../../lib/export";
+import { clipFrameStyle, clipVisualSettings, visibleClipsAt } from "../../lib/export";
import { isTauri } from "../../lib/persistence";
import type { ClipTransform, ProjectAsset, ProjectConfig, TimelineClip } from "../../lib/project";
import { clipEndMs } from "../../lib/timeline";
@@ -101,7 +103,8 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek
}) {
const tracks = config.timeline.tracks;
const assetsById = useMemo(() => new Map(config.assets.map((candidate) => [candidate.id, candidate])), [config.assets]);
- const clip = useMemo(() => visibleClipAt(tracks, playheadMs, assetsById), [tracks, playheadMs, assetsById]);
+ const layers = useMemo(() => visibleClipsAt(tracks, playheadMs, assetsById), [tracks, playheadMs, assetsById]);
+ const clip = layers[0] ?? null;
const asset = useMemo(
() => (clip ? config.assets.find((candidate) => candidate.id === clip.assetId) : undefined),
[clip, config.assets],
@@ -110,20 +113,8 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek
() => clip ? clipFrameStyle(clipVisualSettings(clip), playheadMs - clip.startMs) : null,
[clip, playheadMs],
);
- const mediaStyle = useMemo<React.CSSProperties | undefined>(() => {
- if (!frameStyle) return undefined;
- const { transform, look, opacity, revealStart, revealEnd } = frameStyle;
- const warmth = look.temperature / 100;
- const filter = warmth === 0
- ? "none"
- : `sepia(${Math.abs(warmth) * 0.22}) saturate(${1 + Math.abs(warmth) * 0.3}) hue-rotate(${warmth > 0 ? -8 : 172}deg)`;
- return {
- transform: `translate(${transform.positionX}%, ${transform.positionY}%) scale(${transform.scale / 100}) rotate(${transform.rotation}deg)`,
- opacity,
- filter,
- clipPath: `inset(0 ${(1 - revealEnd) * 100}% 0 ${revealStart * 100}%)`,
- };
- }, [frameStyle]);
+ const mediaStyle = frameStyle ? previewMediaStyle(frameStyle) : undefined;
+ const sourceStyle: React.CSSProperties | undefined = clip?.chromaKey ? { ...mediaStyle, visibility: "hidden", position: "absolute" } : mediaStyle;
/* Sound that should be audible at this moment: audio-only files and embedded
sound from every active video, minus muted tracks. The visible video plays
its own stream, so a second audio element is only needed for the rest. */
@@ -146,6 +137,7 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek
);
const videoRef = useRef<HTMLVideoElement>(null);
+ const imageRef = useRef<HTMLImageElement>(null);
const pictureRef = useRef<HTMLDivElement>(null);
const transformGesture = useRef<TransformGesture | null>(null);
const audioRefs = useRef(new Map<string, HTMLAudioElement>());
@@ -422,19 +414,21 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek
onSelectClip(clip.id);
}}
>
+ {layers.slice(1).reverse().map((layer) => <ProgramLayer key={layer.id} clip={layer} asset={assetsById.get(layer.assetId)} sources={sources} playheadMs={playheadMs} playing={playing} />)}
{/* One element, re-pointed at whatever the playhead is over. Rebuilding it
per clip would drop the decoder and re-open the file on every cut. */}
{asset && !isImage(asset) && url && <video
ref={videoRef}
src={url}
- style={mediaStyle}
+ style={sourceStyle}
muted={videoTrackMuted}
playsInline
preload="auto"
onLoadedData={() => setReady(true)}
onError={() => setError("This file could not be decoded.")}
/>}
- {asset && isImage(asset) && url && <img src={url} alt={clip?.label ?? ""} style={mediaStyle} />}
+ {asset && isImage(asset) && url && <img ref={imageRef} src={url} alt={clip?.label ?? ""} style={sourceStyle} />}
+ {asset && url && clip?.chromaKey && <ChromaKeyPreview source={isImage(asset) ? imageRef : videoRef} sourceUrl={url} effect={clip.chromaKey} playing={playing && !isImage(asset)} style={mediaStyle} onError={setError} />}
{clip && frameStyle && editingClipId === clip.id && onTransformChange && !transformEditingDisabled && <div
className="program-transform"
style={{ transform: `translate(${frameStyle.transform.positionX}%, ${frameStyle.transform.positionY}%) scale(${frameStyle.transform.scale / 100}) rotate(${frameStyle.transform.rotation}deg)` }}
diff --git a/src/components/workspace/TimelineView.tsx b/src/components/workspace/TimelineView.tsx
index 252907a..252dcda 100644
--- a/src/components/workspace/TimelineView.tsx
+++ b/src/components/workspace/TimelineView.tsx
@@ -1,3 +1,4 @@
+import { ClipEffects } from "./ClipEffects";
import {
Copy, Film, Layers3, LayoutGrid, List, Lock, LockOpen,
Pause, Play, Plus, Scissors, SkipBack, SkipForward, Trash2, Upload,
@@ -8,9 +9,7 @@ import { resolutionLabel } from "../../lib/export";
import { importMediaFiles, isTauri } from "../../lib/persistence";
import { loadMediaLayout, saveMediaLayout, type MediaLayout } from "../../lib/settings";
import {
- clipLook,
clipTransform,
- clipTransition,
generationAssetId,
roundClipTransform,
sceneDurationSeconds,
@@ -198,8 +197,6 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
const selectedTrack = tracks.find((track) => track.id === selected?.trackId);
const selectedMediaAsset = config.assets.find((asset) => asset.id === selected?.assetId);
const selectedTransform = selected ? clipTransform(selected) : null;
- const selectedLook = selected ? clipLook(selected) : null;
- const selectedTransition = selected ? clipTransition(selected) : null;
const visualControlsDisabled = selectedMediaAsset?.kind === "audio" || selectedTrack?.kind !== "video" || selectedTrack.locked;
const clipCount = tracks.reduce((total, track) => total + track.clips.length, 0);
/** Where the last clip ends — where playback stops, which is not the same as
@@ -298,12 +295,6 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
const [minimum, maximum] = limits[key];
updateClip(selected.id, { transform: roundClipTransform({ ...selectedTransform, [key]: Math.max(minimum, Math.min(maximum, value)) }) });
};
- const updateLook = (key: keyof NonNullable<TimelineClip["look"]>, value: number) => {
- if (!selected || !selectedLook || !Number.isFinite(value)) return;
- const limits = { opacity: [0, 100], temperature: [-100, 100] } as const;
- const [minimum, maximum] = limits[key];
- updateClip(selected.id, { look: { ...selectedLook, [key]: Math.max(minimum, Math.min(maximum, value)) } });
- };
const toggleTrack = (trackId: string, key: "muted" | "locked") => updateTracks(tracks.map((track) => track.id === trackId ? { ...track, [key]: !track[key] } : track));
/* Every removal — the toolbar, the key, the × on the clip itself — goes
through one function, so a locked track refuses all three. */
@@ -881,21 +872,7 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
<label><span>Position Y (%)</span><CommittedNumberInput minimum={-100} maximum={100} step="1" value={selectedTransform?.positionY ?? 0} disabled={visualControlsDisabled} onCommit={(value) => updateTransform("positionY", value)} /></label>
</div>
</section>
- <section className="inspector-section">
- <h3>Look</h3>
- <label className="range-field"><span>Opacity <b>{selectedLook?.opacity ?? 100}%</b></span><input aria-label="Clip opacity" type="range" min="0" max="100" step="1" value={selectedLook?.opacity ?? 100} disabled={visualControlsDisabled} onChange={(event) => updateLook("opacity", Number(event.target.value))} /></label>
- <label className="range-field"><span>Temperature <b>{(selectedLook?.temperature ?? 0) > 0 ? "+" : ""}{selectedLook?.temperature ?? 0}</b></span><input aria-label="Clip temperature" type="range" min="-100" max="100" step="1" value={selectedLook?.temperature ?? 0} disabled={visualControlsDisabled} onChange={(event) => updateLook("temperature", Number(event.target.value))} /></label>
- </section>
- <section className="inspector-section">
- <h3>Transition</h3>
- <label><span>When this clip begins</span><select aria-label="Clip transition" value={selectedTransition?.type ?? "cut"} disabled={visualControlsDisabled} onChange={(event) => selected && selectedTransition && updateClip(selected.id, { transition: { ...selectedTransition, type: event.target.value as NonNullable<TimelineClip["transition"]>["type"] } })}>
- <option value="cut">Cut</option>
- <option value="fade">Fade in</option>
- <option value="wipe-left">Wipe from left</option>
- <option value="wipe-right">Wipe from right</option>
- </select></label>
- <label className="range-field"><span>Duration <b>{selectedTransition?.durationMs ?? 500} ms</b></span><input aria-label="Transition duration" type="range" min="100" max="3000" step="100" value={selectedTransition?.durationMs ?? 500} disabled={visualControlsDisabled || selectedTransition?.type === "cut"} onChange={(event) => selected && selectedTransition && updateClip(selected.id, { transition: { ...selectedTransition, durationMs: Number(event.target.value) } })} /></label>
- </section>
+ <ClipEffects clip={selected} disabled={visualControlsDisabled} onChange={(patch) => updateClip(selected.id, patch)} />
</> : clipCount === 0
? <div className="inspector-empty"><MouseSelection /><b>Nothing to edit yet</b><span>Once a scene lands on the timeline, select it here to rename it and check its timing.</span></div>
: <div className="inspector-empty"><MouseSelection /><b>Select a clip to edit it</b><span>Pick any clip on the timeline below to rename it, check its timing, and see how it was made.</span></div>}
diff --git a/src/lib/chromaKey.test.ts b/src/lib/chromaKey.test.ts
new file mode 100644
index 0000000..aaa4065
--- /dev/null
+++ b/src/lib/chromaKey.test.ts
@@ -0,0 +1,31 @@
+import { describe, expect, it } from "vitest";
+import { applyChromaKey, keyAlpha } from "./chromaKey";
+import { clipChromaKeySchema, timelineClipSchema } from "./project";
+
+describe("Chroma Key", () => {
+ it("removes the chosen color, keeps other colors and preserves existing alpha", () => {
+ const pixels = new Uint8ClampedArray([0, 255, 0, 255, 255, 0, 0, 128, 0, 0, 255, 255]);
+ applyChromaKey(pixels, { color: "#00ff00", tolerance: 0 });
+ expect(Array.from(pixels)).toEqual([0, 255, 0, 0, 255, 0, 0, 128, 0, 0, 255, 255]);
+ });
+ it("widens the transparent range as tolerance increases and feathers its edge", () => {
+ const narrow = new Uint8ClampedArray([30, 210, 30, 255]);
+ const wide = narrow.slice();
+ applyChromaKey(narrow, { color: "#00ff00", tolerance: 1 });
+ applyChromaKey(wide, { color: "#00ff00", tolerance: 20 });
+ expect(narrow[3]).toBe(255);
+ expect(wide[3]).toBe(0);
+ expect(keyAlpha(0.21, 20)).toBeCloseTo(0.5);
+ expect(keyAlpha(1, 100)).toBe(0);
+ });
+ it("rejects malformed colors and out-of-range tolerances while accepting old clips", () => {
+ for (const key of [{ color: "green", tolerance: 20 }, { color: "#12345g", tolerance: 20 }, { color: "#00ff00", tolerance: -1 }, { color: "#00ff00", tolerance: 101 }]) {
+ expect(clipChromaKeySchema.safeParse(key).success).toBe(false);
+ }
+ const old = { id: "clip", assetId: "a", trackId: "t", label: "Clip", startMs: 0, durationMs: 1000 };
+ expect(timelineClipSchema.parse(old).chromaKey).toBeUndefined();
+ expect(timelineClipSchema.parse({ ...old, chromaKey: null }).chromaKey).toBeNull();
+ const edited = { ...old, chromaKey: { color: "#12ABef", tolerance: 30 } };
+ expect(timelineClipSchema.parse(JSON.parse(JSON.stringify(edited))).chromaKey).toEqual(edited.chromaKey);
+ });
+});
diff --git a/src/lib/chromaKey.ts b/src/lib/chromaKey.ts
new file mode 100644
index 0000000..fdee206
--- /dev/null
+++ b/src/lib/chromaKey.ts
@@ -0,0 +1,24 @@
+import type { ClipChromaKey } from "./project";
+
+export function keyColor(color: string): [number, number, number] {
+ const value = Number.parseInt(color.slice(1), 16);
+ return [((value >> 16) & 255) / 255, ((value >> 8) & 255) / 255, (value & 255) / 255];
+}
+
+// RGB distance normalized to 0–1. A narrow feather avoids jagged compressed
+// edges. These constants/formula are shared by the preview and export shaders.
+export const KEY_FEATHER = 0.02;
+export const RGB_DISTANCE_SCALE = Math.sqrt(3);
+export function keyAlpha(distance: number, tolerance: number): number {
+ const edge = Math.max(0, Math.min(1, (distance - tolerance / 100) / KEY_FEATHER));
+ return edge * edge * (3 - 2 * edge);
+}
+
+/** CPU fallback only; the normal video path performs this on the GPU. */
+export function applyChromaKey(pixels: Uint8ClampedArray, key: ClipChromaKey): void {
+ const [r, g, b] = keyColor(key.color);
+ for (let index = 0; index < pixels.length; index += 4) {
+ const distance = Math.hypot(pixels[index] / 255 - r, pixels[index + 1] / 255 - g, pixels[index + 2] / 255 - b) / RGB_DISTANCE_SCALE;
+ pixels[index + 3] *= keyAlpha(distance, key.tolerance);
+ }
+}
diff --git a/src/lib/chromaKeyPreview.ts b/src/lib/chromaKeyPreview.ts
new file mode 100644
index 0000000..e8a7eb5
--- /dev/null
+++ b/src/lib/chromaKeyPreview.ts
@@ -0,0 +1,110 @@
+import { applyChromaKey, keyColor, KEY_FEATHER, RGB_DISTANCE_SCALE } from "./chromaKey";
+import type { ClipChromaKey } from "./project";
+
+type Picture = HTMLVideoElement | HTMLImageElement;
+export interface ChromaKeyRenderer {
+ draw(source: Picture, key: ClipChromaKey): void;
+ dispose(): void;
+}
+
+/** Upload media directly as a texture; no pixel readback during GPU playback. */
+export function createChromaKeyRenderer(canvas: HTMLCanvasElement): ChromaKeyRenderer {
+ const gl = canvas.getContext("webgl", { alpha: true, premultipliedAlpha: false });
+ if (!gl) {
+ const context = canvas.getContext("2d", { willReadFrequently: true });
+ if (!context) throw new Error("This computer could not create a Chroma Key preview.");
+ return {
+ draw(source, key) {
+ const [width, height] = dimensions(source);
+ if (!width || !height) return;
+ // Bound CPU work to a monitor-sized picture on machines without WebGL.
+ const scale = Math.min(1, 1280 / width);
+ canvas.width = Math.round(width * scale);
+ canvas.height = Math.round(height * scale);
+ context.drawImage(source, 0, 0, canvas.width, canvas.height);
+ const pixels = context.getImageData(0, 0, canvas.width, canvas.height);
+ applyChromaKey(pixels.data, key);
+ context.putImageData(pixels, 0, 0);
+ },
+ dispose() {},
+ };
+ }
+ const shaders: WebGLShader[] = [];
+ const program = gl.createProgram();
+ const vertices = gl.createBuffer();
+ const texture = gl.createTexture();
+ const dispose = () => {
+ shaders.forEach((shader) => gl.deleteShader(shader));
+ gl.deleteProgram(program);
+ gl.deleteBuffer(vertices);
+ gl.deleteTexture(texture);
+ };
+ try {
+ if (!program || !vertices || !texture) throw new Error("Could not allocate the Chroma Key preview.");
+ const compile = (type: number, code: string) => {
+ const shader = gl.createShader(type);
+ if (!shader) throw new Error("Could not allocate the Chroma Key shader.");
+ shaders.push(shader);
+ gl.shaderSource(shader, code);
+ gl.compileShader(shader);
+ if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) throw new Error(gl.getShaderInfoLog(shader) ?? "Chroma Key shader failed.");
+ gl.attachShader(program, shader);
+ };
+ compile(gl.VERTEX_SHADER, `
+ attribute vec2 position;
+ varying vec2 uv;
+ void main() { uv = (position + 1.0) * 0.5; gl_Position = vec4(position, 0.0, 1.0); }
+ `);
+ compile(gl.FRAGMENT_SHADER, `
+ precision mediump float;
+ varying vec2 uv;
+ uniform sampler2D source;
+ uniform vec4 key;
+ void main() {
+ vec4 sampled = texture2D(source, uv);
+ float difference = distance(sampled.rgb, key.rgb) / ${RGB_DISTANCE_SCALE};
+ float alpha = smoothstep(key.a, key.a + ${KEY_FEATHER}, difference);
+ gl_FragColor = vec4(sampled.rgb, sampled.a * alpha);
+ }
+ `);
+ gl.linkProgram(program);
+ if (!gl.getProgramParameter(program, gl.LINK_STATUS)) throw new Error(gl.getProgramInfoLog(program) ?? "Chroma Key shader failed to link.");
+ gl.useProgram(program);
+ gl.bindBuffer(gl.ARRAY_BUFFER, vertices);
+ gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]), gl.STATIC_DRAW);
+ const position = gl.getAttribLocation(program, "position");
+ gl.enableVertexAttribArray(position);
+ gl.vertexAttribPointer(position, 2, gl.FLOAT, false, 0, 0);
+ gl.bindTexture(gl.TEXTURE_2D, texture);
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+ gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
+ gl.uniform1i(gl.getUniformLocation(program, "source"), 0);
+ const keyUniform = gl.getUniformLocation(program, "key");
+ return {
+ draw(source, key) {
+ const [width, height] = dimensions(source);
+ if (!width || !height) return;
+ if (gl.isContextLost()) throw new Error("The GPU stopped responding during Chroma Key preview.");
+ if (canvas.width !== width) canvas.width = width;
+ if (canvas.height !== height) canvas.height = height;
+ gl.viewport(0, 0, width, height);
+ gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, source);
+ gl.uniform4f(keyUniform, ...keyColor(key.color), key.tolerance / 100);
+ gl.drawArrays(gl.TRIANGLES, 0, 6);
+ },
+ dispose,
+ };
+ } catch (error) {
+ dispose();
+ throw error;
+ }
+}
+
+function dimensions(source: Picture): [number, number] {
+ return source instanceof HTMLVideoElement
+ ? source.readyState >= 2 ? [source.videoWidth, source.videoHeight] : [0, 0]
+ : source.complete ? [source.naturalWidth, source.naturalHeight] : [0, 0];
+}
diff --git a/src/lib/export.test.ts b/src/lib/export.test.ts
index e4a12d2..fca7be6 100644
--- a/src/lib/export.test.ts
+++ b/src/lib/export.test.ts
@@ -16,6 +16,7 @@ import {
suggestedFileName,
videoDurationMs,
visibleClipAt,
+ visibleClipsAt,
type ExportSegment,
type ExportSettings,
} from "./export";
@@ -71,6 +72,35 @@ const settings = (overrides: Partial<ExportSettings> = {}): ExportSettings => ({
const clipSegments = (segments: ExportSegment[]) => segments.flatMap((segment) => (segment.kind === "clip" ? [segment] : []));
+describe("effect compositing", () => {
+ it("splits keyed segments at background cuts, retaining each layer's trim", () => {
+ const foreground = clip("green", "track-story", 0, 2000, { chromaKey: { color: "#00ff00", tolerance: 25 } });
+ const first = clip("first", "track-v2", 0, 1000, { sourceStartMs: 4000 });
+ const second = clip("second", "track-v2", 1000, 1000, { sourceStartMs: 7000 });
+ const config = project([foreground, first, second], [asset("asset-green"), asset("asset-first"), asset("asset-second")]);
+ expect(visibleClipsAt(config.timeline.tracks, 500).map((item) => item.id)).toEqual(["green", "first"]);
+ const plan = buildExportPlan(config, settings());
+ const segments = clipSegments(plan.segments);
+ expect(segments.map((segment) => [segment.startFrame, segment.endFrame, segment.underlays?.[0].clipId])).toEqual([[0, 30, "first"], [30, 60, "second"]]);
+ expect(sourceTimeMsForFrame(segments[1].underlays![0], 45, 30)).toBe(7500);
+ expect(segments[0].visual.chromaKey).toEqual(foreground.chromaKey);
+ expect(plan.clipCount).toBe(3);
+ expect(plan.notes.join(" ")).not.toContain("hidden underneath");
+ expect(plan.blockers).toEqual([]);
+ });
+
+ it("validates exposed background media and excludes it when the key is removed", () => {
+ const foreground = clip("green", "track-story", 0, 2000, { chromaKey: { color: "#00ff00", tolerance: 20 } });
+ const background = clip("missing", "track-v2", 0, 2000);
+ const config = project([foreground, background], [asset("asset-green")]);
+ expect(buildExportPlan(config, settings()).blockers.join(" ")).toContain("missing");
+ foreground.chromaKey = undefined;
+ const opaque = buildExportPlan(config, settings());
+ expect(opaque.blockers).toEqual([]);
+ expect(opaque.clipCount).toBe(1);
+ });
+});
+
describe("output geometry", () => {
it("offers the widescreen sizes MiniMax H3 actually generates at", () => {
// The ladder, verbatim. These are not 16:9 rounded off — 2432×1344 is 1.810
diff --git a/src/lib/export.ts b/src/lib/export.ts
index c75dc5f..b8dc529 100644
--- a/src/lib/export.ts
+++ b/src/lib/export.ts
@@ -17,6 +17,7 @@ import {
clipTransition,
type AspectRatio,
type ClipLook,
+ type ClipChromaKey,
type ClipTransform,
type ClipTransition,
type ProjectAsset,
@@ -40,6 +41,7 @@ export interface ExportSettings {
export interface ClipVisualSettings {
transform: ClipTransform;
look: ClipLook;
+ chromaKey?: ClipChromaKey | null;
transition: ClipTransition;
}
@@ -54,6 +56,7 @@ export interface ClipFrameStyle extends ClipVisualSettings {
export const clipVisualSettings = (clip: TimelineClip): ClipVisualSettings => ({
transform: clipTransform(clip),
look: clipLook(clip),
+ chromaKey: clip.chromaKey,
transition: clipTransition(clip),
});
@@ -230,6 +233,15 @@ export function estimatedBytes(bitrate: number, durationMs: number): number {
The plan
--------------------------------------------------------------------------- */
+export interface ExportClipLayer {
+ clipId: string;
+ assetId: string;
+ label: string;
+ clipStartMs: number;
+ clipSourceStartMs: number;
+ visual: ClipVisualSettings;
+}
+
export type ExportSegment =
| {
kind: "clip";
@@ -245,6 +257,8 @@ export type ExportSegment =
/** Where inside the source file the clip begins, in ms. */
clipSourceStartMs: number;
visual: ClipVisualSettings;
+ /** Lower visible tracks, nearest first. Paint in reverse order. */
+ underlays?: ExportClipLayer[];
}
| { kind: "gap"; startFrame: number; endFrame: number };
@@ -370,7 +384,7 @@ const DRAWABLE_IMAGE = new Set(["image/png", "image/jpeg", "image/webp", "image/
* sits at `frameIndex / frameRate` on the timeline, and the source moment is
* that far past the clip's start, offset by where the clip begins in its file. */
export function sourceTimeMsForFrame(
- segment: Extract<ExportSegment, { kind: "clip" }>,
+ segment: ExportClipLayer,
frameIndex: number,
frameRate: number,
): number {
@@ -388,6 +402,16 @@ export function visibleClipAt(
timeMs: number,
assetsById?: ReadonlyMap<string, ProjectAsset>,
): TimelineClip | null {
+ return visibleClipsAt(tracks, timeMs, assetsById)[0] ?? null;
+}
+
+/** Top to bottom, stopping once an opaque clip covers the remaining tracks. */
+export function visibleClipsAt(
+ tracks: TimelineTrack[],
+ timeMs: number,
+ assetsById?: ReadonlyMap<string, ProjectAsset>,
+): TimelineClip[] {
+ const visible: TimelineClip[] = [];
for (const track of tracks) {
if (track.kind !== "video") continue;
let best: TimelineClip | null = null;
@@ -396,9 +420,14 @@ export function visibleClipAt(
if (timeMs < clip.startMs || timeMs >= clip.startMs + clip.durationMs) continue;
if (!best || clip.startMs >= best.startMs) best = clip;
}
- if (best) return best;
+ if (best) {
+ visible.push(best);
+ const revealing = best.chromaKey || (best.look?.opacity ?? 100) < 100 ||
+ (best.transition && best.transition.type !== "cut" && timeMs - best.startMs < best.transition.durationMs);
+ if (!revealing) break;
+ }
}
- return null;
+ return visible;
}
/** Where the picture ends. Audio is not muxed yet, so a soundtrack that runs
@@ -428,7 +457,7 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)
const segments: ExportSegment[] = [];
let gapFrames = 0;
for (let frame = 0; frame < frameCount; frame += 1) {
- const clip = visibleClipAt(videoTracks, (frame * 1000) / frameRate, assetsById);
+ const [clip, ...underlays] = visibleClipsAt(videoTracks, (frame * 1000) / frameRate, assetsById);
const last = segments[segments.length - 1];
if (!clip) {
gapFrames += 1;
@@ -436,7 +465,8 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)
else segments.push({ kind: "gap", startFrame: frame, endFrame: frame + 1 });
continue;
}
- if (last && last.kind === "clip" && last.clipId === clip.id) {
+ if (last && last.kind === "clip" && last.clipId === clip.id &&
+ (last.underlays?.length ?? 0) === underlays.length && underlays.every((layer, index) => last.underlays?.[index].clipId === layer.id)) {
last.endFrame = frame + 1;
continue;
}
@@ -450,10 +480,16 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)
clipStartMs: clip.startMs,
clipSourceStartMs: clip.sourceStartMs,
visual: clipVisualSettings(clip),
+ ...(underlays.length ? { underlays: underlays.map((layer) => ({
+ clipId: layer.id, assetId: layer.assetId, label: layer.label,
+ clipStartMs: layer.startMs, clipSourceStartMs: layer.sourceStartMs,
+ visual: clipVisualSettings(layer),
+ })) } : {}),
});
}
- const usedClipIds = new Set(segments.flatMap((segment) => (segment.kind === "clip" ? [segment.clipId] : [])));
+ const pictureLayers = segments.flatMap((segment) => segment.kind === "clip" ? [segment, ...(segment.underlays ?? [])] : []);
+ const usedClipIds = new Set(pictureLayers.map((layer) => layer.clipId));
const audioClipCount = audioTimelineClips(config, false).length + audioTimelineClips(config, true).length;
const blockers: string[] = [];
@@ -463,8 +499,7 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)
const missing: string[] = [];
const locationless: string[] = [];
const undecodable: string[] = [];
- for (const segment of segments) {
- if (segment.kind !== "clip") continue;
+ for (const segment of pictureLayers) {
const asset = config.assets.find((candidate) => candidate.id === segment.assetId);
if (!asset) {
missing.push(segment.label);
diff --git a/src/lib/exportPipeline.ts b/src/lib/exportPipeline.ts
index d3c04d6..1555e76 100644
--- a/src/lib/exportPipeline.ts
+++ b/src/lib/exportPipeline.ts
@@ -28,11 +28,13 @@ import {
type AudioSegment,
type ClipFrameStyle,
type ExportPlan,
+ type ExportClipLayer,
type ExportSegment,
type ExportSettings,
type OutputCodecId,
} from "./export";
import type { ProjectAsset, ProjectConfig } from "./project";
+import { applyChromaKey, keyColor, KEY_FEATHER, RGB_DISTANCE_SCALE } from "./chromaKey";
/* ---------------------------------------------------------------------------
What this machine can do
@@ -135,7 +137,7 @@ interface Compositor {
readonly kind: CompositorKind;
/** Paints the background alone — what a frame with no clip over it looks like. */
clear(): void;
- /** Paints the background, then the frame scaled to fit inside it. */
+ /** Blends a frame over the current picture. Call clear before each output. */
draw(frame: VideoFrame, style: ClipFrameStyle): void;
/** The composited picture, ready for the encoder. Caller closes it. */
take(timestampUs: number, durationUs: number): VideoFrame;
@@ -161,6 +163,7 @@ struct VertexOut {
struct Style {
primary: vec4f,
secondary: vec4f,
+ key: vec4f,
};
@group(0) @binding(1) var<uniform> style: Style;
@group(0) @binding(2) var source: texture_external;
@@ -193,7 +196,12 @@ fn fs(@location(0) uv: vec2f) -> @location(0) vec4f {
let sampled = textureSampleBaseClampToEdge(source, source_sampler, uv);
let temperature = style.primary.z;
let rgb = clamp(sampled.rgb + vec3f(temperature * 0.1, 0.0, -temperature * 0.1), vec3f(0.0), vec3f(1.0));
- return vec4f(rgb, sampled.a * style.primary.y);
+ var alpha = sampled.a * style.primary.y;
+ if (style.secondary.y > 0.0) {
+ let difference = distance(sampled.rgb, style.key.rgb) / ${RGB_DISTANCE_SCALE};
+ alpha *= smoothstep(style.key.a, style.key.a + ${KEY_FEATHER}, difference);
+ }
+ return vec4f(rgb, alpha);
}
`;
@@ -239,7 +247,7 @@ async function createWebGpuCompositor(width: number, height: number, background:
primitive: { topology: "triangle-list" },
});
const uniform = device.createBuffer({ size: 16, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST });
- const styleUniform = device.createBuffer({ size: 32, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST });
+ const styleUniform = device.createBuffer({ size: 48, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST });
const sampler = device.createSampler({ magFilter: "linear", minFilter: "linear" });
/* A device that dies mid-export would keep handing back an untouched canvas,
which is a black frame that looks like a rendered one. Record the loss and
@@ -261,7 +269,7 @@ async function createWebGpuCompositor(width: number, height: number, background:
if (lost.reason) throw new Error(`This computer's GPU stopped responding during the export: ${lost.reason}`);
const encoder = device.createCommandEncoder();
const renderPass = encoder.beginRenderPass({
- colorAttachments: [{ view, clearValue, loadOp: "clear", storeOp: "store" }],
+ colorAttachments: [{ view, clearValue, loadOp: bindGroup ? "load" : "clear", storeOp: "store" }],
});
if (bindGroup) {
renderPass.setPipeline(pipeline);
@@ -297,9 +305,11 @@ async function createWebGpuCompositor(width: number, height: number, background:
style.look.temperature / 100,
style.revealStart,
style.revealEnd,
+ style.chromaKey ? 1 : 0,
0,
0,
- 0,
+ ...(style.chromaKey ? keyColor(style.chromaKey.color) : [0, 0, 0]),
+ (style.chromaKey?.tolerance ?? 0) / 100,
]));
// Zero copy: the decoded frame is sampled where it already lives.
const external = device.importExternalTexture({ source: frame });
@@ -328,6 +338,7 @@ function createCanvasCompositor(width: number, height: number, background: strin
const canvas = new OffscreenCanvas(width, height);
const context = canvas.getContext("2d", { alpha: false });
if (!context) throw new Error("This computer gave no 2D canvas to composite into.");
+ let keyCanvas: OffscreenCanvas | null = null;
const paintBackground = () => {
context.fillStyle = background;
context.fillRect(0, 0, width, height);
@@ -336,7 +347,20 @@ function createCanvasCompositor(width: number, height: number, background: strin
kind: "canvas2d",
clear: paintBackground,
draw(frame, style) {
- paintBackground();
+ let picture: VideoFrame | OffscreenCanvas = frame;
+ if (style.chromaKey) {
+ keyCanvas ??= new OffscreenCanvas(frame.displayWidth, frame.displayHeight);
+ if (keyCanvas.width !== frame.displayWidth) keyCanvas.width = frame.displayWidth;
+ if (keyCanvas.height !== frame.displayHeight) keyCanvas.height = frame.displayHeight;
+ const keyContext = keyCanvas.getContext("2d", { willReadFrequently: true });
+ if (!keyContext) throw new Error("Could not create the Chroma Key canvas.");
+ keyContext.clearRect(0, 0, keyCanvas.width, keyCanvas.height);
+ keyContext.drawImage(frame, 0, 0);
+ const pixels = keyContext.getImageData(0, 0, keyCanvas.width, keyCanvas.height);
+ applyChromaKey(pixels.data, style.chromaKey);
+ keyContext.putImageData(pixels, 0, 0);
+ picture = keyCanvas;
+ }
const fitted = fitRect(frame.displayWidth, frame.displayHeight, width, height);
const scale = style.transform.scale / 100;
const drawWidth = fitted.width * scale;
@@ -354,7 +378,7 @@ function createCanvasCompositor(width: number, height: number, background: strin
: `sepia(${Math.abs(warmth) * 0.22}) saturate(${1 + Math.abs(warmth) * 0.3}) hue-rotate(${warmth > 0 ? -8 : 172}deg)`;
context.translate(centreX, centreY);
context.rotate((style.transform.rotation * Math.PI) / 180);
- context.drawImage(frame, -drawWidth / 2, -drawHeight / 2, drawWidth, drawHeight);
+ context.drawImage(picture, -drawWidth / 2, -drawHeight / 2, drawWidth, drawHeight);
context.restore();
},
take(timestamp, duration) {
@@ -805,6 +829,86 @@ export interface ExportRunOptions {
const tick = () => new Promise<void>((resolve) => setTimeout(resolve, 0));
+interface LayerFrameReader {
+ /** Borrowed frame, valid until the next call or dispose. Times must increase. */
+ frameAt(timestampUs: number): Promise<VideoFrame>;
+ dispose(): void;
+}
+
+/** One bounded decoder queue per visible layer. Decoded pictures stay in VRAM;
+ * only compressed samples and the current/next pictures are held per source. */
+async function openLayerFrameReader(
+ folderPath: string, asset: ProjectAsset, startUs: number, stopIfCancelled: () => void,
+): Promise<LayerFrameReader> {
+ const bytes = await readAssetBytes(folderPath, asset);
+ stopIfCancelled();
+ if (asset.mimeType.startsWith("image/")) {
+ const bitmap = await createImageBitmap(new Blob([bytes], { type: asset.mimeType }));
+ try {
+ const frame = new VideoFrame(bitmap, { timestamp: 0 });
+ return { frameAt: async () => { stopIfCancelled(); return frame; }, dispose: () => { frame.close(); bitmap.close(); } };
+ } catch (reason) { bitmap.close(); throw reason; }
+ }
+ const source = await demux(bytes, asset.name);
+ let nextSample = 0;
+ let keyframeUs = Number.NEGATIVE_INFINITY;
+ source.samples.forEach((sample, index) => {
+ if (sample.key && sample.timestampUs <= startUs && sample.timestampUs > keyframeUs) {
+ keyframeUs = sample.timestampUs;
+ nextSample = index;
+ }
+ });
+ const pending: VideoFrame[] = [];
+ let screen: VideoFrame | null = null;
+ let flushed = false;
+ const failure: { reason: Error | null } = { reason: null };
+ const decoder = new VideoDecoder({
+ output: (frame) => pending.push(frame),
+ error: (reason) => { failure.reason = reason; },
+ });
+ const dispose = () => {
+ if (decoder.state !== "closed") decoder.close();
+ screen?.close();
+ pending.forEach((frame) => frame.close());
+ pending.length = 0;
+ screen = null;
+ };
+ try { decoder.configure(source.config); }
+ catch (reason) { dispose(); throw reason; }
+ return {
+ async frameAt(wantUs) {
+ while (true) {
+ stopIfCancelled();
+ if (failure.reason) throw failure.reason;
+ // VideoDecoder outputs presentation order, including reordered B frames.
+ while (pending.length && (!screen || pending[0].timestamp <= wantUs)) {
+ screen?.close();
+ screen = pending.shift()!;
+ }
+ if (screen && (pending.length || flushed)) return screen;
+ if (flushed) throw new Error(`${asset.name} decoded no frames.`);
+ if (nextSample >= source.samples.length) {
+ await decoder.flush();
+ flushed = true;
+ } else {
+ // Yield between small batches so output can arrive before feeding
+ // more samples. This bounds decoded frames even for long sources.
+ for (let count = 0; count < 4 && nextSample < source.samples.length; count += 1) {
+ const sample = source.samples[nextSample++];
+ decoder.decode(new EncodedVideoChunk({
+ type: sample.key ? "key" : "delta", timestamp: sample.timestampUs,
+ duration: sample.durationUs, data: sample.data,
+ }));
+ }
+ while (decoder.decodeQueueSize > 8) { stopIfCancelled(); if (failure.reason) throw failure.reason; await tick(); }
+ await tick();
+ }
+ }
+ },
+ dispose,
+ };
+}
+
/** One sentence about the sound in the finished file, true in every case —
* including the cases where there is none. */
function describeSoundtrack(
@@ -925,10 +1029,13 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResult
compositor: createCanvasCompositor(plan.width, plan.height, plan.backgroundColor),
detail: attempt.reason,
};
- const paint = (frame: VideoFrame | null, style: ClipFrameStyle | null) => {
+ const paint = (pictures: { frame: VideoFrame; style: ClipFrameStyle }[]) => {
+ const draw = () => {
+ stage.compositor.clear();
+ for (const picture of pictures) stage.compositor.draw(picture.frame, picture.style);
+ };
try {
- if (frame && style) stage.compositor.draw(frame, style);
- else stage.compositor.clear();
+ draw();
return;
} catch (reason) {
if (stage.compositor.kind === "canvas2d") throw reason;
@@ -936,8 +1043,7 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResult
stage.compositor = createCanvasCompositor(plan.width, plan.height, plan.backgroundColor);
stage.detail = `The GPU compositor gave out ${framesDone} frames in and the rest was composited on a 2D canvas: ${reason instanceof Error ? reason.message : String(reason)}`;
}
- if (frame && style) stage.compositor.draw(frame, style);
- else stage.compositor.clear();
+ draw();
};
/* Timestamps are computed from the frame index every time rather than
@@ -953,12 +1059,17 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResult
frame: VideoFrame | null,
outputIndex: number,
segment: Extract<ExportSegment, { kind: "clip" }> | null = null,
+ underlays: { frame: VideoFrame; layer: ExportClipLayer }[] = [],
) => {
stopIfCancelled();
if (encoderFailure.reason) throw encoderFailure.reason;
const timelineMs = (outputIndex * 1000) / plan.frameRate;
const style = segment ? clipFrameStyle(segment.visual, timelineMs - segment.clipStartMs) : null;
- paint(frame, style);
+ const pictures = underlays.slice().reverse().map((picture) => ({
+ frame: picture.frame, style: clipFrameStyle(picture.layer.visual, timelineMs - picture.layer.clipStartMs),
+ }));
+ if (frame && style) pictures.push({ frame, style });
+ paint(pictures);
const composited = stage.compositor.take(
timestampUs(outputIndex),
timestampUs(outputIndex + 1) - timestampUs(outputIndex),
@@ -1125,11 +1236,40 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResult
}
};
+ const renderLayeredSegment = async (segment: Extract<ExportSegment, { kind: "clip" }>) => {
+ releaseLoaded();
+ loaded.assetId = null;
+ const layers = [segment, ...(segment.underlays ?? [])];
+ const readers: LayerFrameReader[] = [];
+ try {
+ for (const layer of layers) {
+ stopIfCancelled();
+ const asset = assetFor(layer.assetId);
+ if (!asset) throw new Error(`Media for ${layer.label} is missing.`);
+ report("rendering", framesDone, `Reading ${asset.name}…`);
+ readers.push(await openLayerFrameReader(folderPath, asset, sourceTimeMsForFrame(layer, segment.startFrame, plan.frameRate) * 1000, stopIfCancelled));
+ }
+ for (let index = segment.startFrame; index < segment.endFrame; index += 1) {
+ const pictures: { frame: VideoFrame; layer: ExportClipLayer }[] = [];
+ for (let layerIndex = 0; layerIndex < layers.length; layerIndex += 1) {
+ const layer = layers[layerIndex];
+ const frame = await readers[layerIndex].frameAt(sourceTimeMsForFrame(layer, index, plan.frameRate) * 1000);
+ pictures.push({ frame, layer });
+ }
+ await emit(pictures[0].frame, index, segment, pictures.slice(1));
+ }
+ } finally {
+ readers.forEach((reader) => reader.dispose());
+ }
+ };
+
try {
for (const segment of plan.segments) {
stopIfCancelled();
if (segment.kind === "gap") {
for (let index = segment.startFrame; index < segment.endFrame; index += 1) await emit(null, index);
+ } else if (segment.underlays?.length) {
+ await renderLayeredSegment(segment);
} else {
await renderClipSegment(segment);
}
diff --git a/src/lib/project.ts b/src/lib/project.ts
index c8d3de7..e75db98 100644
--- a/src/lib/project.ts
+++ b/src/lib/project.ts
@@ -180,6 +180,11 @@ export const clipTransformSchema = z.object({
positionY: z.number().min(-100).max(100),
});
+export const clipChromaKeySchema = z.object({
+ color: z.string().regex(/^#[0-9a-fA-F]{6}$/),
+ tolerance: z.number().min(0).max(100),
+});
+
export const clipLookSchema = z.object({
opacity: z.number().min(0).max(100),
/** Cool at -100, neutral at 0, warm at +100. */
@@ -205,6 +210,7 @@ export const timelineClipSchema = z.object({
/** Optional so projects written before clip styling keep their exact shape. */
transform: clipTransformSchema.nullish(),
look: clipLookSchema.nullish(),
+ chromaKey: clipChromaKeySchema.nullish(),
/** Applied at this clip's head when the cut changes to it. */
transition: clipTransitionSchema.nullish(),
});
@@ -509,10 +515,12 @@ export type TimelineClip = z.infer<typeof timelineClipSchema>;
export type TimelineTrack = z.infer<typeof timelineTrackSchema>;
export type ClipTransform = z.infer<typeof clipTransformSchema>;
export type ClipLook = z.infer<typeof clipLookSchema>;
+export type ClipChromaKey = z.infer<typeof clipChromaKeySchema>;
export type ClipTransition = z.infer<typeof clipTransitionSchema>;
export const DEFAULT_CLIP_TRANSFORM: ClipTransform = { scale: 100, rotation: 0, positionX: 0, positionY: 0 };
export const DEFAULT_CLIP_LOOK: ClipLook = { opacity: 100, temperature: 0 };
+export const DEFAULT_CLIP_CHROMA_KEY: ClipChromaKey = { color: "#00ff00", tolerance: 20 };
export const DEFAULT_CLIP_TRANSITION: ClipTransition = { type: "cut", durationMs: 500 };
const roundTransformValue = (value: number): number => {
diff --git a/src/styles/timeline.css b/src/styles/timeline.css
index fabaa1b..dbc9215 100644
--- a/src/styles/timeline.css
+++ b/src/styles/timeline.css
@@ -529,7 +529,8 @@
}
.program-picture video,
-.program-picture img {
+.program-picture img,
+.program-picture canvas {
display: block;
width: 100%;
height: 100%;
@@ -539,7 +540,8 @@
}
.program-picture--selectable video,
-.program-picture--selectable img {
+.program-picture--selectable img,
+.program-picture--selectable canvas {
cursor: pointer;
}
@@ -1500,3 +1502,25 @@
--track-column: 168px;
}
}
+
+/* Optional clip effects share the compact inspector controls. */
+.clip-effect > header { display: flex; align-items: center; justify-content: space-between; }
+.clip-effect fieldset { margin: 0; padding: 0; border: 0; min-width: 0; }
+.clip-effect__remove { display: grid; place-items: center; padding: 4px; border: 0; border-radius: var(--radius-sm); color: var(--text-muted); background: transparent; }
+.clip-effect__remove:hover:not(:disabled) { color: var(--text-primary); background: var(--panel-3); }
+.clip-effect .effect-color { display: flex; align-items: center; gap: var(--space-2); }
+.clip-effect .effect-color > span { margin: 0; margin-right: auto; }
+.clip-effect .effect-color input { width: 36px; padding: 3px; cursor: pointer; }
+.effect-color code { font-size: var(--text-xs); color: var(--text-muted); }
+.clip-effects { position: relative; padding: var(--space-3) var(--space-4); }
+.clip-effects__add { display: flex; align-items: center; justify-content: center; gap: var(--space-2); width: 100%; padding: var(--space-2); border: 1px solid var(--line-strong); border-radius: var(--radius-sm); background: var(--panel-2); color: var(--text-primary); font-size: var(--text-sm); }
+.clip-effects__add:hover:not(:disabled) { border-color: var(--accent); background: var(--panel-3); }
+.clip-effects__add:disabled, .clip-effect__remove:disabled { opacity: .45; cursor: not-allowed; }
+.clip-effects__picker { position: absolute; z-index: 5; bottom: calc(100% - var(--space-2)); left: var(--space-4); right: var(--space-4); padding: 4px; border: 1px solid var(--line-strong); border-radius: var(--radius-md); background: var(--panel-2); box-shadow: var(--shadow-sm); max-height: min(280px, 40vh); overflow-y: auto; }
+.clip-effects__picker button { display: flex; flex-direction: column; gap: 3px; width: 100%; border: 0; border-radius: var(--radius-sm); padding: var(--space-2); background: transparent; color: var(--text-primary); text-align: left; }
+.clip-effects__picker button:hover, .clip-effects__picker button:focus-visible { outline: none; background: var(--panel-3); }
+.clip-effects__picker strong { font-size: var(--text-sm); font-weight: var(--weight-medium); }
+.clip-effects__picker span { font-size: var(--text-xs); color: var(--text-muted); }
+
+.program-layer { position: absolute; inset: 0; pointer-events: none; }
+.program-picture > video, .program-picture > img, .program-keyed-picture { position: absolute; inset: 0; }