Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Add clip effects

Bitti9cc0970807c3
diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs
index b43061c..5556bba 100644
--- a/src-tauri/src/lib.rs
+++ b/src-tauri/src/lib.rs
@@ -134,6 +134,34 @@ struct TimelineClip {
color: Option<String>,
#[serde(default = "default_clip_status")]
status: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ transform: Option<ClipTransform>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ look: Option<ClipLook>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ transition: Option<ClipTransition>,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct ClipTransform {
+ scale: f64,
+ rotation: f64,
+ position_x: f64,
+ position_y: f64,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+struct ClipLook {
+ opacity: f64,
+ temperature: f64,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct ClipTransition {
+ r#type: String,
+ duration_ms: u64,
}
fn default_clip_status() -> String {
@@ -793,6 +821,37 @@ fn validate_and_normalize_config(mut config: ProjectConfig) -> Result<ProjectCon
if !matches!(clip.status.as_str(), "draft" | "generated" | "approved") {
return Err(format!("Unsupported clip status '{}'.", clip.status));
}
+ if let Some(transform) = &clip.transform {
+ if !transform.scale.is_finite()
+ || !(10.0..=400.0).contains(&transform.scale)
+ || !transform.rotation.is_finite()
+ || !(-180.0..=180.0).contains(&transform.rotation)
+ || !transform.position_x.is_finite()
+ || !(-100.0..=100.0).contains(&transform.position_x)
+ || !transform.position_y.is_finite()
+ || !(-100.0..=100.0).contains(&transform.position_y)
+ {
+ return Err(format!("Clip '{}' has an invalid transform.", clip.id));
+ }
+ }
+ if let Some(look) = &clip.look {
+ if !look.opacity.is_finite()
+ || !(0.0..=100.0).contains(&look.opacity)
+ || !look.temperature.is_finite()
+ || !(-100.0..=100.0).contains(&look.temperature)
+ {
+ return Err(format!("Clip '{}' has an invalid look.", clip.id));
+ }
+ }
+ if let Some(transition) = &clip.transition {
+ if !matches!(
+ transition.r#type.as_str(),
+ "cut" | "fade" | "wipe-left" | "wipe-right"
+ ) || !(100..=3000).contains(&transition.duration_ms)
+ {
+ return Err(format!("Clip '{}' has an invalid transition.", clip.id));
+ }
+ }
}
}
for job in &mut config.generation_jobs {
diff --git a/src/components/workspace/ProgramMonitor.tsx b/src/components/workspace/ProgramMonitor.tsx
index 5b849c9..91fb913 100644
--- a/src/components/workspace/ProgramMonitor.tsx
+++ b/src/components/workspace/ProgramMonitor.tsx
@@ -1,6 +1,6 @@
import { Film, TriangleAlert } from "lucide-react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
-import { visibleClipAt } from "../../lib/export";
+import { clipFrameStyle, clipVisualSettings, visibleClipAt } from "../../lib/export";
import { isTauri } from "../../lib/persistence";
import type { ProjectAsset, ProjectConfig, TimelineClip } from "../../lib/project";
import { clipEndMs } from "../../lib/timeline";
@@ -84,6 +84,24 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek
() => (clip ? config.assets.find((candidate) => candidate.id === clip.assetId) : undefined),
[clip, config.assets],
);
+ const frameStyle = useMemo(
+ () => clip ? clipFrameStyle(clipVisualSettings(clip), playheadMs - clip.startMs) : null,
+ [clip, playheadMs],
+ );
+ const mediaStyle = useMemo<React.CSSProperties | undefined>(() => {
+ if (!frameStyle) return undefined;
+ const { transform, look, opacity, revealStart, revealEnd } = frameStyle;
+ const warmth = look.temperature / 100;
+ const filter = warmth === 0
+ ? "none"
+ : `sepia(${Math.abs(warmth) * 0.22}) saturate(${1 + Math.abs(warmth) * 0.3}) hue-rotate(${warmth > 0 ? -8 : 172}deg)`;
+ return {
+ transform: `translate(${transform.positionX}%, ${transform.positionY}%) scale(${transform.scale / 100}) rotate(${transform.rotation}deg)`,
+ opacity,
+ filter,
+ clipPath: `inset(0 ${(1 - revealEnd) * 100}% 0 ${revealStart * 100}%)`,
+ };
+ }, [frameStyle]);
/* Sound that should be audible at this moment: every audio clip the playhead
is inside, minus the tracks the user has muted. A muted track is dropped
here rather than played silently, so nothing is decoded for it at all. */
@@ -315,13 +333,14 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek
{asset && !isImage(asset) && url && <video
ref={videoRef}
src={url}
+ style={mediaStyle}
muted={videoTrackMuted}
playsInline
preload="auto"
onLoadedData={() => setReady(true)}
onError={() => setError("This file could not be decoded.")}
/>}
- {asset && isImage(asset) && url && <img src={url} alt={clip?.label ?? ""} />}
+ {asset && isImage(asset) && url && <img src={url} alt={clip?.label ?? ""} style={mediaStyle} />}
{audioClips.map((audioClip) => {
const audioUrl = audioUrls[audioClip.assetId];
return audioUrl
diff --git a/src/components/workspace/TimelineView.tsx b/src/components/workspace/TimelineView.tsx
index 6584e80..115ce7e 100644
--- a/src/components/workspace/TimelineView.tsx
+++ b/src/components/workspace/TimelineView.tsx
@@ -8,6 +8,9 @@ import { resolutionLabel } from "../../lib/export";
import { importMediaFiles, isTauri } from "../../lib/persistence";
import { loadMediaLayout, saveMediaLayout, type MediaLayout } from "../../lib/settings";
import {
+ clipLook,
+ clipTransform,
+ clipTransition,
generationAssetId,
sceneDurationSeconds,
type GenerationJob,
@@ -26,7 +29,6 @@ import { sceneShape, STATUS_WORD } from "./sceneStatus";
import { ShotThumbnail } from "./ShotThumbnail";
const MIN_DURATION = 10_000;
-const NOT_YET = "Not available yet. This control doesn’t change your project.";
/* How long a dropped clip is when the file itself cannot say.
A drop normally lasts as long as the footage: the media panel decodes each
file to draw its thumbnail and records the duration it read there (see
@@ -185,6 +187,10 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
const tracks = config.timeline.tracks;
const selected = useMemo(() => tracks.flatMap((track) => track.clips).find((clip) => clip.id === selectedId), [tracks, selectedId]);
const selectedTrack = tracks.find((track) => track.id === selected?.trackId);
+ const selectedTransform = selected ? clipTransform(selected) : null;
+ const selectedLook = selected ? clipLook(selected) : null;
+ const selectedTransition = selected ? clipTransition(selected) : null;
+ const visualControlsDisabled = selectedTrack?.kind !== "video";
const clipCount = tracks.reduce((total, track) => total + track.clips.length, 0);
/** Where the last clip ends — where playback stops, which is not the same as
* where the ruler stops (the canvas is at least as long as the film the user
@@ -269,6 +275,18 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
onChange((current) => ({ ...current, timeline: { tracks: nextTracks } }));
const renameTrack = (trackId: string, name: string) => updateTracks(tracks.map((track) => track.id === trackId ? { ...track, name } : track));
const updateClip = (clipId: string, updates: Partial<TimelineClip>) => updateTracks(tracks.map((track) => ({ ...track, clips: track.clips.map((clip) => clip.id === clipId ? { ...clip, ...updates } : clip) })));
+ const updateTransform = (key: keyof NonNullable<TimelineClip["transform"]>, value: number) => {
+ if (!selected || !selectedTransform || !Number.isFinite(value)) return;
+ const limits = { scale: [10, 400], rotation: [-180, 180], positionX: [-100, 100], positionY: [-100, 100] } as const;
+ const [minimum, maximum] = limits[key];
+ updateClip(selected.id, { transform: { ...selectedTransform, [key]: Math.max(minimum, Math.min(maximum, value)) } });
+ };
+ const updateLook = (key: keyof NonNullable<TimelineClip["look"]>, value: number) => {
+ if (!selected || !selectedLook || !Number.isFinite(value)) return;
+ const limits = { opacity: [0, 100], temperature: [-100, 100] } as const;
+ const [minimum, maximum] = limits[key];
+ updateClip(selected.id, { look: { ...selectedLook, [key]: Math.max(minimum, Math.min(maximum, value)) } });
+ };
const toggleTrack = (trackId: string, key: "muted" | "locked") => updateTracks(tracks.map((track) => track.id === trackId ? { ...track, [key]: !track[key] } : track));
/* Every removal — the toolbar, the key, the × on the clip itself — goes
through one function, so a locked track refuses all three. */
@@ -297,7 +315,7 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
const splitSelected = () => {
if (!selected || !selectedTrack || selectedTrack.locked || playhead <= selected.startMs || playhead >= selected.startMs + selected.durationMs) return;
const leftDuration = playhead - selected.startMs;
- const right: TimelineClip = { ...selected, id: `${selected.id}-split-${Date.now()}`, startMs: playhead, durationMs: selected.durationMs - leftDuration, sourceStartMs: selected.sourceStartMs + leftDuration, label: `${selected.label} · B` };
+ const right: TimelineClip = { ...selected, id: `${selected.id}-split-${Date.now()}`, startMs: playhead, durationMs: selected.durationMs - leftDuration, sourceStartMs: selected.sourceStartMs + leftDuration, label: `${selected.label} · B`, transition: undefined };
updateTracks(tracks.map((track) => track.id === selected.trackId ? { ...track, clips: track.clips.flatMap((clip) => clip.id === selected.id ? [{ ...clip, durationMs: leftDuration, label: `${clip.label} · A` }, right] : [clip]) } : track));
setSelectedId(right.id);
};
@@ -834,21 +852,31 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
/></label>
</div>
</section>
- <section className="inspector-section inspector-section--pending">
- <h3>Transform <em>Not available yet</em></h3>
+ <section className="inspector-section">
+ <h3>Transform</h3>
<div className="field-pair">
- <label><span>Scale</span><input defaultValue="100%" disabled title={NOT_YET} /></label>
- <label><span>Rotation</span><input defaultValue="0°" disabled title={NOT_YET} /></label>
+ <label><span>Scale (%)</span><input type="number" min="10" max="400" step="1" value={selectedTransform?.scale ?? 100} disabled={visualControlsDisabled} onChange={(event) => updateTransform("scale", Number(event.target.value))} /></label>
+ <label><span>Rotation (°)</span><input type="number" min="-180" max="180" step="1" value={selectedTransform?.rotation ?? 0} disabled={visualControlsDisabled} onChange={(event) => updateTransform("rotation", Number(event.target.value))} /></label>
</div>
<div className="field-pair">
- <label><span>Position X</span><input defaultValue="0" disabled title={NOT_YET} /></label>
- <label><span>Position Y</span><input defaultValue="0" disabled title={NOT_YET} /></label>
+ <label><span>Position X (%)</span><input type="number" min="-100" max="100" step="1" value={selectedTransform?.positionX ?? 0} disabled={visualControlsDisabled} onChange={(event) => updateTransform("positionX", Number(event.target.value))} /></label>
+ <label><span>Position Y (%)</span><input type="number" min="-100" max="100" step="1" value={selectedTransform?.positionY ?? 0} disabled={visualControlsDisabled} onChange={(event) => updateTransform("positionY", Number(event.target.value))} /></label>
</div>
</section>
- <section className="inspector-section inspector-section--pending">
- <h3>Look <em>Not available yet</em></h3>
- <label className="range-field"><span>Opacity <b>100%</b></span><input type="range" defaultValue="100" disabled title={NOT_YET} /></label>
- <label className="range-field"><span>Temperature <b>+4</b></span><input type="range" defaultValue="54" disabled title={NOT_YET} /></label>
+ <section className="inspector-section">
+ <h3>Look</h3>
+ <label className="range-field"><span>Opacity <b>{selectedLook?.opacity ?? 100}%</b></span><input aria-label="Clip opacity" type="range" min="0" max="100" step="1" value={selectedLook?.opacity ?? 100} disabled={visualControlsDisabled} onChange={(event) => updateLook("opacity", Number(event.target.value))} /></label>
+ <label className="range-field"><span>Temperature <b>{(selectedLook?.temperature ?? 0) > 0 ? "+" : ""}{selectedLook?.temperature ?? 0}</b></span><input aria-label="Clip temperature" type="range" min="-100" max="100" step="1" value={selectedLook?.temperature ?? 0} disabled={visualControlsDisabled} onChange={(event) => updateLook("temperature", Number(event.target.value))} /></label>
+ </section>
+ <section className="inspector-section">
+ <h3>Transition</h3>
+ <label><span>When this clip begins</span><select aria-label="Clip transition" value={selectedTransition?.type ?? "cut"} disabled={visualControlsDisabled} onChange={(event) => selected && selectedTransition && updateClip(selected.id, { transition: { ...selectedTransition, type: event.target.value as NonNullable<TimelineClip["transition"]>["type"] } })}>
+ <option value="cut">Cut</option>
+ <option value="fade">Fade in</option>
+ <option value="wipe-left">Wipe from left</option>
+ <option value="wipe-right">Wipe from right</option>
+ </select></label>
+ <label className="range-field"><span>Duration <b>{selectedTransition?.durationMs ?? 500} ms</b></span><input aria-label="Transition duration" type="range" min="100" max="3000" step="100" value={selectedTransition?.durationMs ?? 500} disabled={visualControlsDisabled || selectedTransition?.type === "cut"} onChange={(event) => selected && selectedTransition && updateClip(selected.id, { transition: { ...selectedTransition, durationMs: Number(event.target.value) } })} /></label>
</section>
</> : clipCount === 0
? <div className="inspector-empty"><MouseSelection /><b>Nothing to edit yet</b><span>Once a scene lands on the timeline, select it here to rename it and check its timing.</span></div>
diff --git a/src/lib/export.test.ts b/src/lib/export.test.ts
index 81089c9..cf2e3d9 100644
--- a/src/lib/export.test.ts
+++ b/src/lib/export.test.ts
@@ -2,6 +2,8 @@ import { describe, expect, it } from "vitest";
import {
bitrateFor,
buildExportPlan,
+ clipFrameStyle,
+ clipVisualSettings,
defaultExportSettings,
estimatedBytes,
fitRect,
@@ -201,6 +203,33 @@ describe("which clip is on screen at time T", () => {
});
describe("the plan", () => {
+ it("carries clip transforms, looks, and transitions into every rendered segment", () => {
+ const treated = clip("a", "track-story", 0, 2_000, {
+ transform: { scale: 125, rotation: 8, positionX: -12, positionY: 6 },
+ look: { opacity: 80, temperature: 35 },
+ transition: { type: "fade", durationMs: 500 },
+ });
+ const plan = buildExportPlan(project([treated], [asset("asset-a")]), settings());
+ const [segment] = clipSegments(plan.segments);
+
+ expect(segment.visual).toEqual(clipVisualSettings(treated));
+ expect(clipFrameStyle(segment.visual, 250)).toMatchObject({
+ transform: treated.transform,
+ look: treated.look,
+ opacity: 0.4,
+ revealStart: 0,
+ revealEnd: 1,
+ });
+ });
+
+ it("reveals wipe transitions in the selected direction", () => {
+ const base = clipVisualSettings(clip("a", "track-story", 0, 1_000));
+ expect(clipFrameStyle({ ...base, transition: { type: "wipe-left", durationMs: 1_000 } }, 250))
+ .toMatchObject({ revealStart: 0, revealEnd: 0.25 });
+ expect(clipFrameStyle({ ...base, transition: { type: "wipe-right", durationMs: 1_000 } }, 250))
+ .toMatchObject({ revealStart: 0.75, revealEnd: 1 });
+ });
+
it("turns two back-to-back clips into two segments of the right length", () => {
const config = project(
[clip("a", "track-story", 0, 2_000), clip("b", "track-story", 2_000, 1_000)],
diff --git a/src/lib/export.ts b/src/lib/export.ts
index e5e1eca..a16d5cc 100644
--- a/src/lib/export.ts
+++ b/src/lib/export.ts
@@ -11,7 +11,19 @@
has to come from, are the two questions an editor gets wrong silently.
========================================================================== */
-import type { AspectRatio, ProjectConfig, Resolution, TimelineClip, TimelineTrack } from "./project";
+import {
+ clipLook,
+ clipTransform,
+ clipTransition,
+ type AspectRatio,
+ type ClipLook,
+ type ClipTransform,
+ type ClipTransition,
+ type ProjectConfig,
+ type Resolution,
+ type TimelineClip,
+ type TimelineTrack,
+} from "./project";
export type FrameRate = ProjectConfig["settings"]["frameRate"];
export type OutputCodecId = "h264" | "vp9" | "av1";
@@ -24,6 +36,40 @@ export interface ExportSettings {
quality: QualityId;
}
+export interface ClipVisualSettings {
+ transform: ClipTransform;
+ look: ClipLook;
+ transition: ClipTransition;
+}
+
+export interface ClipFrameStyle extends ClipVisualSettings {
+ /** Opacity after both Look and the transition at this moment are applied. */
+ opacity: number;
+ /** Horizontal source reveal, used by wipe transitions. */
+ revealStart: number;
+ revealEnd: number;
+}
+
+export const clipVisualSettings = (clip: TimelineClip): ClipVisualSettings => ({
+ transform: clipTransform(clip),
+ look: clipLook(clip),
+ transition: clipTransition(clip),
+});
+
+/** Resolve the time-varying part of a clip's visual treatment. */
+export function clipFrameStyle(visual: ClipVisualSettings, elapsedMs: number): ClipFrameStyle {
+ const transition = visual.transition;
+ const progress = transition.type === "cut"
+ ? 1
+ : Math.max(0, Math.min(1, elapsedMs / transition.durationMs));
+ return {
+ ...visual,
+ opacity: (visual.look.opacity / 100) * (transition.type === "fade" ? progress : 1),
+ revealStart: transition.type === "wipe-right" ? 1 - progress : 0,
+ revealEnd: transition.type === "wipe-left" ? progress : 1,
+ };
+}
+
export interface OutputCodec {
id: OutputCodecId;
label: string;
@@ -186,6 +232,7 @@ export type ExportSegment =
clipStartMs: number;
/** Where inside the source file the clip begins, in ms. */
clipSourceStartMs: number;
+ visual: ClipVisualSettings;
}
| { kind: "gap"; startFrame: number; endFrame: number };
@@ -365,6 +412,7 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)
endFrame: frame + 1,
clipStartMs: clip.startMs,
clipSourceStartMs: clip.sourceStartMs,
+ visual: clipVisualSettings(clip),
});
}
diff --git a/src/lib/exportPipeline.ts b/src/lib/exportPipeline.ts
index 7f336d4..8351766 100644
--- a/src/lib/exportPipeline.ts
+++ b/src/lib/exportPipeline.ts
@@ -21,10 +21,12 @@ import { isTauri, readMediaFileBytes } from "./persistence";
import {
AUDIO_CHANNELS,
AUDIO_SAMPLE_RATE,
+ clipFrameStyle,
fitRect,
outputCodec,
sourceTimeMsForFrame,
type AudioSegment,
+ type ClipFrameStyle,
type ExportPlan,
type ExportSegment,
type ExportSettings,
@@ -134,7 +136,7 @@ interface Compositor {
/** Paints the background alone — what a frame with no clip over it looks like. */
clear(): void;
/** Paints the background, then the frame scaled to fit inside it. */
- draw(frame: VideoFrame): void;
+ draw(frame: VideoFrame, style: ClipFrameStyle): void;
/** The composited picture, ready for the encoder. Caller closes it. */
take(timestampUs: number, durationUs: number): VideoFrame;
dispose(): void;
@@ -153,10 +155,16 @@ struct VertexOut {
@location(0) uv: vec2f,
};
-// x0, y0, x1, y1 of the fitted picture in clip space.
+// x0, y0, x1, y1 of the fitted and transformed picture in clip space.
@group(0) @binding(0) var<uniform> rect: vec4f;
-@group(0) @binding(1) var source: texture_external;
-@group(0) @binding(2) var source_sampler: sampler;
+// rotation, opacity, temperature, reveal start / reveal end.
+struct Style {
+ primary: vec4f,
+ secondary: vec4f,
+};
+@group(0) @binding(1) var<uniform> style: Style;
+@group(0) @binding(2) var source: texture_external;
+@group(0) @binding(3) var source_sampler: sampler;
@vertex
fn vs(@builtin(vertex_index) index: u32) -> VertexOut {
@@ -166,14 +174,26 @@ fn vs(@builtin(vertex_index) index: u32) -> VertexOut {
);
let corner = corners[index];
var out: VertexOut;
- out.position = vec4f(mix(rect.x, rect.z, corner.x), mix(rect.y, rect.w, corner.y), 0.0, 1.0);
+ let centre = vec2f((rect.x + rect.z) * 0.5, (rect.y + rect.w) * 0.5);
+ let point = vec2f(mix(rect.x, rect.z, corner.x), mix(rect.y, rect.w, corner.y));
+ let delta = point - centre;
+ let angle = style.primary.x;
+ let rotated = vec2f(
+ delta.x * cos(angle) - delta.y * sin(angle),
+ delta.x * sin(angle) + delta.y * cos(angle),
+ );
+ out.position = vec4f(centre + rotated, 0.0, 1.0);
out.uv = vec2f(corner.x, 1.0 - corner.y);
return out;
}
@fragment
fn fs(@location(0) uv: vec2f) -> @location(0) vec4f {
- return textureSampleBaseClampToEdge(source, source_sampler, uv);
+ if (uv.x < style.primary.w || uv.x > style.secondary.x) { discard; }
+ let sampled = textureSampleBaseClampToEdge(source, source_sampler, uv);
+ let temperature = style.primary.z;
+ let rgb = clamp(sampled.rgb + vec3f(temperature * 0.1, 0.0, -temperature * 0.1), vec3f(0.0), vec3f(1.0));
+ return vec4f(rgb, sampled.a * style.primary.y);
}
`;
@@ -205,10 +225,21 @@ async function createWebGpuCompositor(width: number, height: number, background:
const pipeline = device.createRenderPipeline({
layout: "auto",
vertex: { module: shader, entryPoint: "vs" },
- fragment: { module: shader, entryPoint: "fs", targets: [{ format }] },
+ fragment: {
+ module: shader,
+ entryPoint: "fs",
+ targets: [{
+ format,
+ blend: {
+ color: { srcFactor: "src-alpha", dstFactor: "one-minus-src-alpha", operation: "add" },
+ alpha: { srcFactor: "one", dstFactor: "one-minus-src-alpha", operation: "add" },
+ },
+ }],
+ },
primitive: { topology: "triangle-list" },
});
const uniform = device.createBuffer({ size: 16, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST });
+ const styleUniform = device.createBuffer({ size: 32, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST });
const sampler = device.createSampler({ magFilter: "linear", minFilter: "linear" });
/* A device that dies mid-export would keep handing back an untouched canvas,
which is a black frame that looks like a rendered one. Record the loss and
@@ -246,21 +277,39 @@ async function createWebGpuCompositor(width: number, height: number, background:
clear() {
pass(context.getCurrentTexture().createView(), null);
},
- draw(frame) {
- const box = fitRect(frame.displayWidth, frame.displayHeight, width, height);
+ draw(frame, style) {
+ const fitted = fitRect(frame.displayWidth, frame.displayHeight, width, height);
+ const scale = style.transform.scale / 100;
+ const box = {
+ width: fitted.width * scale,
+ height: fitted.height * scale,
+ x: fitted.x + (fitted.width * (1 - scale)) / 2 + (style.transform.positionX / 100) * width,
+ y: fitted.y + (fitted.height * (1 - scale)) / 2 + (style.transform.positionY / 100) * height,
+ };
const x0 = (box.x / width) * 2 - 1;
const x1 = ((box.x + box.width) / width) * 2 - 1;
const yTop = 1 - (box.y / height) * 2;
const yBottom = 1 - ((box.y + box.height) / height) * 2;
device.queue.writeBuffer(uniform, 0, new Float32Array([x0, yBottom, x1, yTop]));
+ device.queue.writeBuffer(styleUniform, 0, new Float32Array([
+ (style.transform.rotation * Math.PI) / 180,
+ style.opacity,
+ style.look.temperature / 100,
+ style.revealStart,
+ style.revealEnd,
+ 0,
+ 0,
+ 0,
+ ]));
// Zero copy: the decoded frame is sampled where it already lives.
const external = device.importExternalTexture({ source: frame });
const bindGroup = device.createBindGroup({
layout: pipeline.getBindGroupLayout(0),
entries: [
{ binding: 0, resource: { buffer: uniform } },
- { binding: 1, resource: external },
- { binding: 2, resource: sampler },
+ { binding: 1, resource: { buffer: styleUniform } },
+ { binding: 2, resource: external },
+ { binding: 3, resource: sampler },
],
});
pass(context.getCurrentTexture().createView(), bindGroup);
@@ -286,10 +335,27 @@ function createCanvasCompositor(width: number, height: number, background: strin
return {
kind: "canvas2d",
clear: paintBackground,
- draw(frame) {
+ draw(frame, style) {
paintBackground();
- const box = fitRect(frame.displayWidth, frame.displayHeight, width, height);
- context.drawImage(frame, box.x, box.y, box.width, box.height);
+ const fitted = fitRect(frame.displayWidth, frame.displayHeight, width, height);
+ const scale = style.transform.scale / 100;
+ const drawWidth = fitted.width * scale;
+ const drawHeight = fitted.height * scale;
+ const centreX = fitted.x + fitted.width / 2 + (style.transform.positionX / 100) * width;
+ const centreY = fitted.y + fitted.height / 2 + (style.transform.positionY / 100) * height;
+ const warmth = style.look.temperature / 100;
+ context.save();
+ context.beginPath();
+ context.rect(style.revealStart * width, 0, (style.revealEnd - style.revealStart) * width, height);
+ context.clip();
+ context.globalAlpha = style.opacity;
+ context.filter = warmth === 0
+ ? "none"
+ : `sepia(${Math.abs(warmth) * 0.22}) saturate(${1 + Math.abs(warmth) * 0.3}) hue-rotate(${warmth > 0 ? -8 : 172}deg)`;
+ context.translate(centreX, centreY);
+ context.rotate((style.transform.rotation * Math.PI) / 180);
+ context.drawImage(frame, -drawWidth / 2, -drawHeight / 2, drawWidth, drawHeight);
+ context.restore();
},
take(timestamp, duration) {
return new VideoFrame(canvas, { timestamp, duration, alpha: "discard" });
@@ -859,9 +925,9 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResult
compositor: createCanvasCompositor(plan.width, plan.height, plan.backgroundColor),
detail: attempt.reason,
};
- const paint = (frame: VideoFrame | null) => {
+ const paint = (frame: VideoFrame | null, style: ClipFrameStyle | null) => {
try {
- if (frame) stage.compositor.draw(frame);
+ if (frame && style) stage.compositor.draw(frame, style);
else stage.compositor.clear();
return;
} catch (reason) {
@@ -870,7 +936,7 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResult
stage.compositor = createCanvasCompositor(plan.width, plan.height, plan.backgroundColor);
stage.detail = `The GPU compositor gave out ${framesDone} frames in and the rest was composited on a 2D canvas: ${reason instanceof Error ? reason.message : String(reason)}`;
}
- if (frame) stage.compositor.draw(frame);
+ if (frame && style) stage.compositor.draw(frame, style);
else stage.compositor.clear();
};
@@ -883,10 +949,16 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResult
const keyFrameInterval = Math.max(1, Math.round(plan.frameRate * 2));
let framesDone = 0;
- const emit = async (frame: VideoFrame | null, outputIndex: number) => {
+ const emit = async (
+ frame: VideoFrame | null,
+ outputIndex: number,
+ segment: Extract<ExportSegment, { kind: "clip" }> | null = null,
+ ) => {
stopIfCancelled();
if (encoderFailure.reason) throw encoderFailure.reason;
- paint(frame);
+ const timelineMs = (outputIndex * 1000) / plan.frameRate;
+ const style = segment ? clipFrameStyle(segment.visual, timelineMs - segment.clipStartMs) : null;
+ paint(frame, style);
const composited = stage.compositor.take(
timestampUs(outputIndex),
timestampUs(outputIndex + 1) - timestampUs(outputIndex),
@@ -947,7 +1019,7 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResult
const still = loaded.image;
if (still) {
- for (let index = segment.startFrame; index < segment.endFrame; index += 1) await emit(still, index);
+ for (let index = segment.startFrame; index < segment.endFrame; index += 1) await emit(still, index, segment);
return;
}
const source = loaded.source;
@@ -1006,7 +1078,7 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResult
const frame = pending.shift() as VideoFrame;
if (on.screen) {
while (nextOutput < segment.endFrame && wantUs(nextOutput) < frame.timestamp) {
- await emit(on.screen, nextOutput);
+ await emit(on.screen, nextOutput, segment);
nextOutput += 1;
}
on.screen.close();
@@ -1019,7 +1091,7 @@ export async function runExport(options: ExportRunOptions): Promise<ExportResult
throw new Error(`${asset.name} decoded no frames, so clip “${segment.label}” has no picture to export.`);
}
while (nextOutput < segment.endFrame) {
- await emit(on.screen, nextOutput);
+ await emit(on.screen, nextOutput, segment);
nextOutput += 1;
}
}
diff --git a/src/lib/project.test.ts b/src/lib/project.test.ts
index 55a7f8e..7070aba 100644
--- a/src/lib/project.test.ts
+++ b/src/lib/project.test.ts
@@ -102,6 +102,36 @@ describe("project schema", () => {
expect(project.providerSettings).toEqual({});
});
+ it("persists clip transforms, looks, and transitions while old clips keep defaults", () => {
+ const project = createProjectConfig({
+ name: "Treated cut",
+ prompt: "A graded product cut",
+ aspectRatio: "16:9",
+ resolution: "1080p",
+ targetDurationSeconds: 30,
+ });
+ project.timeline.tracks[0].clips.push({
+ id: "clip-treated",
+ assetId: "asset-a",
+ trackId: STORY_TRACK_ID,
+ startMs: 0,
+ durationMs: 2_000,
+ sourceStartMs: 0,
+ label: "Treated",
+ color: null,
+ status: "approved",
+ transform: { scale: 115, rotation: -4, positionX: 8, positionY: -3 },
+ look: { opacity: 72, temperature: 20 },
+ transition: { type: "wipe-left", durationMs: 600 },
+ });
+
+ expect(parseProjectConfig(project).timeline.tracks[0].clips[0]).toMatchObject({
+ transform: { scale: 115, rotation: -4, positionX: 8, positionY: -3 },
+ look: { opacity: 72, temperature: 20 },
+ transition: { type: "wipe-left", durationMs: 600 },
+ });
+ });
+
it("compiles the initial brief into exactly the three MiniMax H3 core fields", () => {
const project = createProjectConfig({
name: "Launch film",
diff --git a/src/lib/project.ts b/src/lib/project.ts
index df9ba1c..8a5d3b8 100644
--- a/src/lib/project.ts
+++ b/src/lib/project.ts
@@ -164,6 +164,27 @@ export const projectAssetSchema = z.object({
}
});
+export const clipTransformSchema = z.object({
+ /** Percentage of the fitted source size. */
+ scale: z.number().min(10).max(400),
+ /** Clockwise degrees around the centre of the picture. */
+ rotation: z.number().min(-180).max(180),
+ /** Percentage of the output frame in either direction. */
+ positionX: z.number().min(-100).max(100),
+ positionY: z.number().min(-100).max(100),
+});
+
+export const clipLookSchema = z.object({
+ opacity: z.number().min(0).max(100),
+ /** Cool at -100, neutral at 0, warm at +100. */
+ temperature: z.number().min(-100).max(100),
+});
+
+export const clipTransitionSchema = z.object({
+ type: z.enum(["cut", "fade", "wipe-left", "wipe-right"]),
+ durationMs: z.number().int().min(100).max(3000),
+});
+
export const timelineClipSchema = z.object({
id: idSchema,
assetId: idSchema,
@@ -175,6 +196,11 @@ export const timelineClipSchema = z.object({
// See the note on projectAssetSchema.durationMs — nullish, never bare optional.
color: z.string().nullish(),
status: z.enum(["draft", "generated", "approved"]).default("approved"),
+ /** Optional so projects written before clip styling keep their exact shape. */
+ transform: clipTransformSchema.nullish(),
+ look: clipLookSchema.nullish(),
+ /** Applied at this clip's head when the cut changes to it. */
+ transition: clipTransitionSchema.nullish(),
});
export const timelineTrackSchema = z.object({
@@ -455,6 +481,17 @@ export type Resolution = z.infer<typeof resolutionSchema>;
export type ProjectAsset = z.infer<typeof projectAssetSchema>;
export type TimelineClip = z.infer<typeof timelineClipSchema>;
export type TimelineTrack = z.infer<typeof timelineTrackSchema>;
+export type ClipTransform = z.infer<typeof clipTransformSchema>;
+export type ClipLook = z.infer<typeof clipLookSchema>;
+export type ClipTransition = z.infer<typeof clipTransitionSchema>;
+
+export const DEFAULT_CLIP_TRANSFORM: ClipTransform = { scale: 100, rotation: 0, positionX: 0, positionY: 0 };
+export const DEFAULT_CLIP_LOOK: ClipLook = { opacity: 100, temperature: 0 };
+export const DEFAULT_CLIP_TRANSITION: ClipTransition = { type: "cut", durationMs: 500 };
+
+export const clipTransform = (clip: TimelineClip): ClipTransform => clip.transform ?? DEFAULT_CLIP_TRANSFORM;
+export const clipLook = (clip: TimelineClip): ClipLook => clip.look ?? DEFAULT_CLIP_LOOK;
+export const clipTransition = (clip: TimelineClip): ClipTransition => clip.transition ?? DEFAULT_CLIP_TRANSITION;
export type GenerationBrief = z.infer<typeof generationBriefSchema>;
export type ProjectReference = z.infer<typeof projectReferenceSchema>;
export type GenerationJob = z.infer<typeof generationJobSchema>;
diff --git a/src/styles/timeline.css b/src/styles/timeline.css
index 5f7175a..c81369a 100644
--- a/src/styles/timeline.css
+++ b/src/styles/timeline.css
@@ -583,6 +583,8 @@
width: 100%;
height: 100%;
object-fit: contain;
+ transform-origin: center;
+ will-change: transform, opacity, filter, clip-path;
}
/* What the monitor has to say when there is no picture to show: a gap under the
@@ -696,7 +698,8 @@
line-height: var(--leading-snug);
}
-.inspector-section input:not([type="range"]) {
+.inspector-section input:not([type="range"]),
+.inspector-section select {
width: 100%;
height: var(--field-height);
padding: 0 var(--space-3);
@@ -708,10 +711,12 @@
font-size: var(--text-base);
}
-.inspector-section input:focus { border-color: var(--accent); }
+.inspector-section input:focus,
+.inspector-section select:focus { border-color: var(--accent); }
.inspector-section input[readonly] { color: var(--text-muted); }
.inspector-section--pending { opacity: 0.72; }
-.inspector-section input:disabled { color: var(--text-faint); cursor: not-allowed; }
+.inspector-section input:disabled,
+.inspector-section select:disabled { color: var(--text-faint); cursor: not-allowed; }
.field-pair {
display: grid;