Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Add timeline clip speed control

Bitti6ad85b792e62
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 14f1778..7a72a84 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -4,6 +4,7 @@ This alpha update adds LAN generation workers, Linux support, expanded color gra
## What's new
+- Timeline clips now offer Speed in Timing, from 25% to 400%. Speed changes preserve the source range, adjust duration and later clips on the same track, and apply to preview and export, including audio pitch.
- Open the agent from the Settings top bar in a resizable right-side panel to configure generators, weight paths and LoRAs, even without an open project.
- Scene Generation settings now offer Latent upscale: generate at half resolution and upscale to the selected output size, locally or on a LAN worker. Download its model or choose a local file in Other weights settings.
diff --git a/src-tauri/src/project/commands/clips.rs b/src-tauri/src/project/commands/clips.rs
index fe7a44c..04055bb 100644
--- a/src-tauri/src/project/commands/clips.rs
+++ b/src-tauri/src/project/commands/clips.rs
@@ -142,6 +142,7 @@ pub(super) fn apply(
start_ms,
duration_ms,
source_start_ms,
+ playback_rate: None,
label,
color: None,
status: status.into(),
@@ -209,7 +210,7 @@ pub(super) fn apply(
project,
&clip.asset_id,
clip.source_start_ms,
- clip.duration_ms,
+ (clip.duration_ms as f64 * clip.playback_rate.unwrap_or(1.0)).round() as u64,
)?;
clip.track_id = project.timeline.tracks[target_track_index].id.clone();
project.timeline.tracks[source_track_index]
diff --git a/src-tauri/src/project/commands/tests.rs b/src-tauri/src/project/commands/tests.rs
index 8de0145..c116e48 100644
--- a/src-tauri/src/project/commands/tests.rs
+++ b/src-tauri/src/project/commands/tests.rs
@@ -1,6 +1,18 @@
use super::batch::execute_commands;
use super::*;
+#[test]
+fn clip_speed_round_trips_and_validates() {
+ let mut config = fixture();
+ config.timeline.tracks[0].clips[0].playback_rate = Some(0.5);
+ let json = serde_json::to_value(&config).unwrap();
+ assert_eq!(json["timeline"]["tracks"][0]["clips"][0]["playbackRate"], 0.5);
+ let restored: ProjectConfig = serde_json::from_value(json).unwrap();
+ assert_eq!(validate_and_normalize_config(restored).unwrap().timeline.tracks[0].clips[0].playback_rate, Some(0.5));
+ config.timeline.tracks[0].clips[0].playback_rate = Some(0.0);
+ assert!(validate_and_normalize_config(config).unwrap_err().contains("playback rate"));
+}
+
#[test]
fn scene_creation_uses_injected_model_defaults() {
let current = fixture();
diff --git a/src-tauri/src/project/timeline.rs b/src-tauri/src/project/timeline.rs
index 93e2e31..e6ee1a6 100644
--- a/src-tauri/src/project/timeline.rs
+++ b/src-tauri/src/project/timeline.rs
@@ -27,6 +27,8 @@ pub(crate) struct TimelineClip {
pub(crate) duration_ms: u64,
#[serde(default)]
pub(crate) source_start_ms: u64,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub(crate) playback_rate: Option<f64>,
pub(crate) label: String,
// `.optional()` without `.nullable()` on the frontend — see ProjectAsset.
#[serde(default, skip_serializing_if = "Option::is_none")]
diff --git a/src-tauri/src/project/validation/timeline.rs b/src-tauri/src/project/validation/timeline.rs
index 04340ce..cd04302 100644
--- a/src-tauri/src/project/validation/timeline.rs
+++ b/src-tauri/src/project/validation/timeline.rs
@@ -8,6 +8,9 @@ pub(super) fn validate(config: &mut ProjectConfig) -> Result<(), String> {
return Err(format!("Track '{}' cannot have an empty name.", track.id));
}
for clip in &track.clips {
+ if clip.playback_rate.is_some_and(|rate| !rate.is_finite() || !(0.25..=4.0).contains(&rate)) {
+ return Err(format!("Clip '{}' has an invalid playback rate.", clip.id));
+ }
if clip.label.is_empty() {
return Err(format!("Clip '{}' cannot have an empty label.", clip.id));
}
diff --git a/src/components/workspace/ProgramLayer.tsx b/src/components/workspace/ProgramLayer.tsx
index aecb0a6..fa273fd 100644
--- a/src/components/workspace/ProgramLayer.tsx
+++ b/src/components/workspace/ProgramLayer.tsx
@@ -1,3 +1,4 @@
+import { clipPlaybackRate, clipSourceTimeMs } from "../../lib/clipTiming";
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState, type CSSProperties } from "react";
import { clipFrameStyle, clipVisualSettings, type ClipFrameStyle } from "../../lib/export";
import { PreviewSources } from "../../lib/exportPipeline";
@@ -77,13 +78,15 @@ export function ProgramLayer({ clip, asset, sources, playheadMs, playing, rate =
const run = state.playing && state.active;
if (!run) element.pause();
if (element.readyState < 1) return;
- const target = Math.max(0, (state.clip.sourceStartMs + state.playheadMs - state.clip.startMs) / 1000);
+ const target = Math.max(0, clipSourceTimeMs(state.clip, state.playheadMs) / 1000);
// Upcoming players seek while paused. At a normal cut they can start
// immediately without seeking again for a fraction of a display frame.
// Explicit jumps still seek precisely, including during playback.
const tolerance = run && !state.externalSeek ? 0.25 : 0.033;
if (!element.seeking && Math.abs(element.currentTime - target) > tolerance) element.currentTime = target;
- if (run && element.playbackRate !== rate) element.playbackRate = rate;
+ const mediaRate = rate * clipPlaybackRate(state.clip);
+ element.preservesPitch = false;
+ if (run && element.playbackRate !== mediaRate) element.playbackRate = mediaRate;
if (run && element.paused) void element.play().catch(() => undefined);
};
useLayoutEffect(sync, [playheadMs, playing, rate, active, externalSeek, clip, url]);
diff --git a/src/components/workspace/ProgramMonitor.test.tsx b/src/components/workspace/ProgramMonitor.test.tsx
index d2ac7a8..8aaf020 100644
--- a/src/components/workspace/ProgramMonitor.test.tsx
+++ b/src/components/workspace/ProgramMonitor.test.tsx
@@ -53,6 +53,27 @@ const monitor = (config: ProjectConfig, playheadMs: number) => render(
);
describe("the program monitor", () => {
+ it.each([0.5, 2])("seeks and plays video and audio at %sx clip speed", async (playbackRate) => {
+ vi.spyOn(PreviewSources.prototype, "url").mockImplementation(async (asset) => `blob:${asset.id}`);
+ vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue();
+ vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => undefined);
+ vi.spyOn(HTMLMediaElement.prototype, "readyState", "get").mockReturnValue(4);
+ vi.stubGlobal("requestAnimationFrame", () => 1);
+ vi.stubGlobal("cancelAnimationFrame", () => undefined);
+ const config = project([clip({ playbackRate })]);
+ config.assets.push({ ...config.assets[0], id: "sound", kind: "audio", sourcePath: "D:/sound.wav" });
+ config.timeline.tracks[1].clips = [clip({ id: "sound", assetId: "sound", trackId: config.timeline.tracks[1].id, playbackRate })];
+ const props = { config, folderPath: "C:/Film", onSeek: vi.fn(), onPlayingChange: vi.fn(), rate: 2 };
+ const view = render(<ProgramMonitor {...props} playheadMs={2000} playing={false} />);
+ await act(async () => undefined);
+ view.rerender(<ProgramMonitor {...props} playheadMs={3000} playing />);
+ for (const media of view.container.querySelectorAll<HTMLMediaElement>("video, audio")) {
+ expect(media.currentTime).toBe(8 + playbackRate);
+ expect(media.playbackRate).toBe(2 * playbackRate);
+ expect(media.preservesPitch).toBe(false);
+ }
+ expect(view.container.querySelectorAll("video, audio")).toHaveLength(2);
+ });
it.each([false, true])("prepares the next trim before a cut and promotes its decoder without seeking (same asset: %s)", async (sameAsset) => {
(window as unknown as Record<string, unknown>).__TAURI_INTERNALS__ = {};
const read = vi.spyOn(PreviewSources.prototype, "url").mockImplementation(async (asset) => `blob:${asset.id}`);
diff --git a/src/components/workspace/ProgramMonitor.tsx b/src/components/workspace/ProgramMonitor.tsx
index e805e80..790e81d 100644
--- a/src/components/workspace/ProgramMonitor.tsx
+++ b/src/components/workspace/ProgramMonitor.tsx
@@ -1,3 +1,4 @@
+import { clipPlaybackRate, clipSourceTimeMs } from "../../lib/clipTiming";
import { ProgramLayer } from "./ProgramLayer";
import { ProgramPicture } from "./ProgramPicture";
import { Film16 } from "../ui/icons";
@@ -28,7 +29,7 @@ const carriesAudio = (asset: ProjectAsset | undefined) => Boolean(
);
/** Where inside the FILE this moment of the timeline lives. */
-const sourceTimeMs = (clip: TimelineClip, timelineMs: number) => clip.sourceStartMs + (timelineMs - clip.startMs);
+const sourceTimeMs = clipSourceTimeMs;
type TransformGesture = {
kind: "move" | "scale" | "rotate";
@@ -244,7 +245,9 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, rate =
element.currentTime = Math.max(0, target);
}
if (mediaPlaying) {
- if (element.playbackRate !== rate) element.playbackRate = rate;
+ const mediaRate = rate * clipPlaybackRate(audioClip);
+ element.preservesPitch = false;
+ if (element.playbackRate !== mediaRate) element.playbackRate = mediaRate;
void element.play().catch(() => undefined);
} else element.pause();
}
diff --git a/src/components/workspace/TimelineClipThumbnails.tsx b/src/components/workspace/TimelineClipThumbnails.tsx
index 998500f..9addda0 100644
--- a/src/components/workspace/TimelineClipThumbnails.tsx
+++ b/src/components/workspace/TimelineClipThumbnails.tsx
@@ -1,3 +1,4 @@
+import { clipPlaybackRate } from "../../lib/clipTiming";
import { useEffect, useMemo, useState } from "react";
import { readProjectFileUrl } from "../../lib/persistence";
import { type GenerationJob, type TimelineClip } from "../../lib/project";
@@ -20,7 +21,8 @@ export function TimelineClipThumbnails({ folderPath, job, clip }: {
job: GenerationJob;
clip: TimelineClip;
}) {
- const sourceEndMs = clip.sourceStartMs + clip.durationMs;
+ const sourceDurationMs = clip.durationMs * clipPlaybackRate(clip);
+ const sourceEndMs = clip.sourceStartMs + sourceDurationMs;
const times = useMemo(() => {
const first = Math.floor(clip.sourceStartMs / TIMELINE_THUMBNAIL_INTERVAL_MS) * TIMELINE_THUMBNAIL_INTERVAL_MS;
const result: number[] = [];
@@ -72,8 +74,8 @@ export function TimelineClipThumbnails({ folderPath, job, clip }: {
alt=""
draggable={false}
style={{
- left: `${(still.timeMs - clip.sourceStartMs) / clip.durationMs * 100}%`,
- width: `${TIMELINE_THUMBNAIL_INTERVAL_MS / clip.durationMs * 100}%`,
+ left: `${(still.timeMs - clip.sourceStartMs) / sourceDurationMs * 100}%`,
+ width: `${TIMELINE_THUMBNAIL_INTERVAL_MS / sourceDurationMs * 100}%`,
}}
/>)}
</span>;
diff --git a/src/components/workspace/TimelineClipWaveform.tsx b/src/components/workspace/TimelineClipWaveform.tsx
index fe1a541..ada07a1 100644
--- a/src/components/workspace/TimelineClipWaveform.tsx
+++ b/src/components/workspace/TimelineClipWaveform.tsx
@@ -1,3 +1,4 @@
+import { clipPlaybackRate } from "../../lib/clipTiming";
import { useEffect, useMemo, useState } from "react";
import { audioWaveformSegment, loadAudioWaveform } from "../../lib/audioWaveform";
import { type ProjectAsset, type TimelineClip } from "../../lib/project";
@@ -23,12 +24,12 @@ export function TimelineClipWaveform({ folderPath, asset, clip, cacheVersion = "
let live = true;
setLevels(null);
void loadAudioWaveform(folderPath, asset, cacheVersion).then((overview) => {
- if (live) setLevels(audioWaveformSegment(overview, clip.sourceStartMs / 1_000, clip.durationMs / 1_000, DISPLAY_POINTS));
+ if (live) setLevels(audioWaveformSegment(overview, clip.sourceStartMs / 1_000, clip.durationMs * clipPlaybackRate(clip) / 1_000, DISPLAY_POINTS));
}).catch(() => {
if (live) setLevels([]);
});
return () => { live = false; };
- }, [asset, cacheVersion, clip.durationMs, clip.sourceStartMs, folderPath]);
+ }, [asset, cacheVersion, clip.durationMs, clip.playbackRate, clip.sourceStartMs, folderPath]);
const lines = useMemo(() => levels && levels.length > 0 ? sampleLines(levels) : null, [levels]);
diff --git a/src/components/workspace/TimelineView.test.tsx b/src/components/workspace/TimelineView.test.tsx
index 4a66890..808819a 100644
--- a/src/components/workspace/TimelineView.test.tsx
+++ b/src/components/workspace/TimelineView.test.tsx
@@ -220,6 +220,24 @@ const press = (container: HTMLElement, key: string, init: Partial<KeyboardEventI
const toolbarTime = (container: HTMLElement) => container.querySelector(".transport-time")!.textContent;
describe("the media and scene panels", () => {
+ it("changes clip speed in Timing and splits using the retimed source position", () => {
+ const config = withClip(measuredVideo(projectWithMedia(), 40000), { durationMs: 4000, sourceStartMs: 1000, playbackRate: 2 });
+ const { container, onChange } = render_(config);
+ const field = screen.getByRole("spinbutton", { name: "Clip speed" });
+ expect((field as HTMLInputElement).value).toBe("200");
+ fireEvent.change(field, { target: { value: "50" } });
+ fireEvent.blur(field);
+ const changed = wrote(onChange.mock.calls.at(-1)![0], config).timeline.tracks.flatMap((track) => track.clips)[0];
+ expect(changed.playbackRate).toBe(0.5);
+ expect(changed.durationMs).toBe(16000);
+ // This harness retains its input snapshot; split the original 2x clip.
+ onChange.mockClear();
+ press(container, "ArrowRight", { shiftKey: true });
+ press(container, "k", { ctrlKey: true });
+ const split = wrote(onChange.mock.calls.at(-1)![0], config).timeline.tracks.flatMap((track) => track.clips);
+ expect(split[1].sourceStartMs).toBe(Math.round(1000 + split[0].durationMs * 2));
+ expect(split[1].playbackRate).toBe(2);
+ });
it("switches the media panel between grid and list, and remembers which", () => {
const { container } = mediaPanel(projectWithMedia());
expect(container.querySelector(".media-grid--grid")).not.toBeNull();
diff --git a/src/components/workspace/TimelineView.tsx b/src/components/workspace/TimelineView.tsx
index 250b2a1..117f920 100644
--- a/src/components/workspace/TimelineView.tsx
+++ b/src/components/workspace/TimelineView.tsx
@@ -1,4 +1,5 @@
import { clipEffectCount, ClipEffects } from "./ClipEffects";
+import { clipPlaybackRate, clipSourceTimeMs, MIN_CLIP_SPEED, MAX_CLIP_SPEED } from "../../lib/clipTiming";
import { continuationPlaybackTracks, sceneMediaDurationMs, sceneMediaStartSeconds } from "../../lib/continuationMedia";
import { ProjectStatus } from "./ProjectStatus";
import { PreviewEngineStatus } from "./PreviewEngineStatus";
@@ -25,7 +26,7 @@ import {
} from "../../lib/project";
import {
adjacentCut, clipEndMs, findClip, formatTimecode, frameAt, frameStartMs, insertClip, moveClip, parseTimecode, pasteClip,
- removeClip, rulerLabel, rulerScale, scrollAfterZoom, snapTargets, sourceRoom, trimClip, withTimelineTracks,
+ removeClip, retimeClip, rulerLabel, rulerScale, scrollAfterZoom, snapTargets, sourceRoom, trimClip, withTimelineTracks,
type SnapOptions, type SourceRoom,
} from "../../lib/timeline";
import { useShortcut } from "../../lib/commands";
@@ -550,7 +551,7 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
const splitClip = (clip: TimelineClip | undefined) => {
if (!clip || !canSplit(clip)) return;
const leftDuration = playhead - clip.startMs;
- const right: TimelineClip = { ...clip, id: `${clip.id}-split-${Date.now()}`, startMs: playhead, durationMs: clip.durationMs - leftDuration, sourceStartMs: clip.sourceStartMs + leftDuration, label: `${clip.label} · B`, transition: undefined };
+ const right: TimelineClip = { ...clip, id: `${clip.id}-split-${Date.now()}`, startMs: playhead, durationMs: clip.durationMs - leftDuration, sourceStartMs: Math.round(clipSourceTimeMs(clip, playhead)), label: `${clip.label} · B`, transition: undefined };
updateTracks(tracks.map((track) => track.id === clip.trackId ? { ...track, clips: track.clips.flatMap((item) => item.id === clip.id ? [{ ...item, durationMs: leftDuration, label: `${item.label} · A` }, right] : [item]) } : track));
setSelectedId(right.id);
};
@@ -796,9 +797,15 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
is only bounded from below. */
const selectedAsset = selected ? assetById(selected.assetId) : undefined;
const sourceLimitMs = selected && selectedAsset?.kind !== "image" && selectedAsset?.durationMs
- ? Math.max(1, (sceneMediaDurationMs(selectedAsset) ?? selectedAsset.durationMs) - selected.sourceStartMs)
+ ? Math.max(1, ((sceneMediaDurationMs(selectedAsset) ?? selectedAsset.durationMs) - selected.sourceStartMs) / clipPlaybackRate(selected))
: null;
const minClipMs = Math.max(1, Math.round(1000 / fps));
+ const speedDisabled = !selected || tracks.find((track) => track.id === selected.trackId)?.locked || !selectedAsset || !["video", "generated", "audio"].includes(selectedAsset.kind);
+ const changeSpeed = (percent: number) => {
+ if (!selected || speedDisabled) return;
+ const next = retimeClip(tracks, selected.id, percent / 100);
+ if (next) { setDurationDraft(null); updateTracks(next, `speed:${selected.id}`); }
+ };
/** What the FILE has left at each end of a clip. A still image and a file
* nothing has measured have no source timeline to run out of, and get no
* limit rather than a guessed one. */
@@ -1462,6 +1469,13 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
><More16 aria-hidden="true" /></button>}
/>
<PropSection title="Timing" persistKey="timeline.clip.timing" summary={`${shortTimecode(selected.startMs, fps)} → ${shortTimecode(clipEndMs(selected), fps)}`}>
+ <PropRow label="Speed" htmlFor="clip-speed" value={clipPlaybackRate(selected)} defaultValue={1}
+ onReset={speedDisabled ? undefined : () => changeSpeed(100)}
+ tooltip="100% is normal speed. Changes duration while keeping the same source range and moving later clips on this track. Audio pitch changes with speed.">
+ <CommittedNumberInput id="clip-speed" className="text-field" aria-label="Clip speed" minimum={MIN_CLIP_SPEED * 100} maximum={MAX_CLIP_SPEED * 100} step={1}
+ value={Number((clipPlaybackRate(selected) * 100).toFixed(2))} disabled={speedDisabled} onCommit={changeSpeed} />
+ <span className="inspector-unit">%</span>
+ </PropRow>
<PropRow label="Starts at"><span className="inspector-value">{formatTimecode(selected.startMs, fps)}</span></PropRow>
<PropRow label="Duration" htmlFor="clip-duration">
<input
diff --git a/src/lib/clipTiming.test.ts b/src/lib/clipTiming.test.ts
new file mode 100644
index 0000000..b4e8d1c
--- /dev/null
+++ b/src/lib/clipTiming.test.ts
@@ -0,0 +1,54 @@
+import { expect, it } from "vitest";
+import { createProjectConfig, parseProjectConfig, type TimelineClip, type TimelineTrack } from "./project";
+import { clipPlaybackRate, clipSourceTimeMs } from "./clipTiming";
+import { retimeClip, sourceRoom, trimClip } from "./timeline";
+import { buildExportPlan, defaultExportSettings, sourceTimeMsForFrame } from "./export";
+
+const clip: TimelineClip = { id: "clip", assetId: "asset", trackId: "track", label: "Clip", status: "approved", startMs: 1000, durationMs: 4000, sourceStartMs: 2000 };
+const tracks: TimelineTrack[] = [{ id: "track", name: "Video", kind: "video", locked: false, muted: false,
+ clips: [clip, { ...clip, id: "next", startMs: 6000 }] }];
+
+it.each([0.25, 0.5, 2, 4])("retimes at %sx preserving the source range and following gaps", (rate) => {
+ const next = retimeClip(tracks, "clip", rate)!;
+ const changed = next[0].clips[0];
+ expect(changed.durationMs).toBe(4000 / rate);
+ expect(clipSourceTimeMs(changed, changed.startMs + changed.durationMs)).toBe(6000);
+ expect(next[0].clips[1].startMs - changed.startMs - changed.durationMs).toBe(1000);
+ expect(retimeClip(next, "clip", 1)![0].clips).toEqual([{ ...clip, playbackRate: 1 }, tracks[0].clips[1]]);
+ expect(tracks[0].clips[0]).toBe(clip);
+});
+
+it("rejects invalid rates and locked tracks", () => {
+ for (const rate of [0, -1, NaN, Infinity, 0.24, 4.1]) expect(retimeClip(tracks, "clip", rate)).toBeNull();
+ expect(retimeClip([{ ...tracks[0], locked: true }], "clip", 2)).toBeNull();
+ expect(clipPlaybackRate(clip)).toBe(1);
+});
+
+it("does not round retimed clips beyond the available source", () => {
+ const next = retimeClip([{ ...tracks[0], clips: [{ ...clip, durationMs: 4003 }] }], "clip", 4)![0].clips[0];
+ expect(clipSourceTimeMs(next, next.startMs + next.durationMs)).toBeLessThanOrEqual(6003);
+});
+
+it.each([0.5, 2])("trims in source time at %sx", (rate) => {
+ const changed = retimeClip(tracks, "clip", rate)!;
+ const room = sourceRoom(changed[0].clips[0], 8000);
+ expect(room).toEqual({ headMs: 2000 / rate, tailMs: 2000 / rate });
+ const trimmed = trimClip(changed, "clip", "start", 1500, { minClipMs: 33, room })![0].clips[0];
+ expect(trimmed.sourceStartMs).toBe(2000 + 500 * rate);
+ expect(clipSourceTimeMs(trimmed, trimmed.startMs + trimmed.durationMs)).toBe(6000);
+});
+
+it("persists rates and exports foreground, underlay and audio source times", () => {
+ const config = createProjectConfig({ name: "Speed", prompt: "", aspectRatio: "16:9", resolution: "768p", targetDurationSeconds: 10 });
+ config.assets = [{ id: "asset", name: "Video", kind: "video", relativePath: "video.mp4", mimeType: "video/mp4", hasAudio: true, durationMs: 20000, createdAt: config.createdAt }];
+ config.timeline.tracks = [{ ...tracks[0], clips: [{ ...clip, playbackRate: 2, look: { opacity: 50 } }] },
+ { ...tracks[0], id: "lower", clips: [{ ...clip, id: "lower", trackId: "lower", playbackRate: 0.5 }] }];
+ const loaded = parseProjectConfig(JSON.parse(JSON.stringify(config)));
+ const plan = buildExportPlan(loaded, { ...defaultExportSettings(loaded), frameRate: 30 });
+ const segment = plan.segments.find((item) => item.kind === "clip");
+ if (!segment || segment.kind !== "clip") throw new Error("Missing clip");
+ expect(sourceTimeMsForFrame(segment, 60, 30)).toBe(4000);
+ expect(sourceTimeMsForFrame(segment.underlays![0], 60, 30)).toBe(2500);
+ expect(plan.audio.map((item) => item.playbackRate)).toEqual([2, 0.5]);
+ expect(() => parseProjectConfig({ ...config, timeline: { tracks: [{ ...tracks[0], clips: [{ ...clip, playbackRate: 0 }] }] } })).toThrow();
+});
diff --git a/src/lib/clipTiming.ts b/src/lib/clipTiming.ts
new file mode 100644
index 0000000..839503f
--- /dev/null
+++ b/src/lib/clipTiming.ts
@@ -0,0 +1,7 @@
+import type { TimelineClip } from "./project";
+
+export const MIN_CLIP_SPEED = 0.25;
+export const MAX_CLIP_SPEED = 4;
+export const clipPlaybackRate = (clip: Pick<TimelineClip, "playbackRate">) => clip.playbackRate ?? 1;
+export const clipSourceTimeMs = (clip: Pick<TimelineClip, "sourceStartMs" | "startMs" | "playbackRate">, timelineMs: number) =>
+ clip.sourceStartMs + (timelineMs - clip.startMs) * clipPlaybackRate(clip);
diff --git a/src/lib/continuationMedia.ts b/src/lib/continuationMedia.ts
index 940c14a..829c54a 100644
--- a/src/lib/continuationMedia.ts
+++ b/src/lib/continuationMedia.ts
@@ -1,3 +1,4 @@
+import { clipPlaybackRate } from "./clipTiming";
import { GENERATION_FRAME_RATE, generationAssetId, sceneOutputFrames, type GenerationJob, type ProjectAsset, type ProjectConfig, type SceneMediaSegment, type TimelineClip } from "./project";
const milliseconds = (frames: number) => frames * 1000 / GENERATION_FRAME_RATE;
@@ -65,7 +66,7 @@ export function continuationPlaybackTracks(config: ProjectConfig) {
const matching = joined?.sceneSegments?.find((part) => part.sceneId === own.sceneId && part.latentRelativePath === own.latentRelativePath);
// A Start continuation contains only the opening overlap, so do
// not pretend that its source file contains the whole original scene.
- if (matching && clip.sourceStartMs + clip.durationMs <= milliseconds(matching.frameCount) + 1) {
+ if (matching && clip.sourceStartMs + clip.durationMs * clipPlaybackRate(clip) <= milliseconds(matching.frameCount) + 1) {
source = joined;
segment = matching;
break;
diff --git a/src/lib/export.ts b/src/lib/export.ts
index 67a07dd..c5d79db 100644
--- a/src/lib/export.ts
+++ b/src/lib/export.ts
@@ -11,6 +11,7 @@
has to come from, are the two questions an editor gets wrong silently.
========================================================================== */
+import { clipPlaybackRate, clipSourceTimeMs } from "./clipTiming";
import {
clipChromaKey,
clipLook,
@@ -286,6 +287,7 @@ export function estimatedBytes(bitrate: number, durationMs: number): number {
--------------------------------------------------------------------------- */
export interface ExportClipLayer {
+ playbackRate?: number;
clipId: string;
assetId: string;
label: string;
@@ -297,6 +299,7 @@ export interface ExportClipLayer {
export type ExportSegment =
| {
kind: "clip";
+ playbackRate?: number;
clipId: string;
assetId: string;
label: string;
@@ -347,6 +350,7 @@ const AUDIO_MIX_NOTE_BYTES = 200_000_000;
* to fit it, the audible part is worked out here and the remainder is reported
* as a note the user reads before pressing Export. */
export interface AudioSegment {
+ playbackRate?: number;
clipId: string;
assetId: string;
label: string;
@@ -395,7 +399,8 @@ export function audioSegments(config: ProjectConfig, durationMs: number): AudioS
startMs: start,
durationMs: end - start,
// A clip dragged to a negative start would begin further into its file.
- sourceStartMs: clip.sourceStartMs + (start - clip.startMs),
+ sourceStartMs: clipSourceTimeMs(clip, start),
+ ...(clipPlaybackRate(clip) !== 1 ? { playbackRate: clipPlaybackRate(clip) } : {}),
});
}
}
@@ -440,7 +445,7 @@ export function sourceTimeMsForFrame(
frameIndex: number,
frameRate: number,
): number {
- return segment.clipSourceStartMs + ((frameIndex * 1000) / frameRate - segment.clipStartMs);
+ return segment.clipSourceStartMs + ((frameIndex * 1000) / frameRate - segment.clipStartMs) * (segment.playbackRate ?? 1);
}
/** The clip a viewer sees at `timeMs`.
@@ -533,10 +538,12 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)
endFrame: frame + 1,
clipStartMs: clip.startMs,
clipSourceStartMs: clip.sourceStartMs,
+ ...(clipPlaybackRate(clip) !== 1 ? { playbackRate: clipPlaybackRate(clip) } : {}),
visual: clipVisualSettings(clip),
...(underlays.length ? { underlays: underlays.map((layer) => ({
clipId: layer.id, assetId: layer.assetId, label: layer.label,
clipStartMs: layer.startMs, clipSourceStartMs: layer.sourceStartMs,
+ ...(clipPlaybackRate(layer) !== 1 ? { playbackRate: clipPlaybackRate(layer) } : {}),
visual: clipVisualSettings(layer),
})) } : {}),
});
diff --git a/src/lib/exportPipeline.test.ts b/src/lib/exportPipeline.test.ts
index e6c7895..9bdc154 100644
--- a/src/lib/exportPipeline.test.ts
+++ b/src/lib/exportPipeline.test.ts
@@ -584,7 +584,7 @@ function installWebCodecs(options: HarnessOptions = {}): { restore: () => void;
readonly numberOfChannels: number;
readonly length: number;
readonly sampleRate: number;
- private readonly started: Array<{ buffer: HarnessAudioBuffer; when: number; offset: number; duration: number }> = [];
+ private readonly started: Array<{ buffer: HarnessAudioBuffer; when: number; offset: number; duration: number; rate: number }> = [];
constructor(init: { numberOfChannels: number; length: number; sampleRate: number }) {
this.numberOfChannels = init.numberOfChannels;
this.length = init.length;
@@ -605,13 +605,14 @@ function installWebCodecs(options: HarnessOptions = {}): { restore: () => void;
const context = this;
return {
buffer: null as HarnessAudioBuffer | null,
+ playbackRate: { value: 1 },
connected: false,
connect(destination: unknown) {
this.connected = destination === context.destination;
},
start(when: number, offset: number, duration: number) {
if (!this.buffer || !this.connected) return; // an unconnected source is silence
- context.started.push({ buffer: this.buffer, when, offset, duration });
+ context.started.push({ buffer: this.buffer, when, offset, duration, rate: this.playbackRate.value });
},
};
}
@@ -621,15 +622,15 @@ function installWebCodecs(options: HarnessOptions = {}): { restore: () => void;
const startFrame = Math.round(source.when * this.sampleRate);
const offsetFrame = Math.round(source.offset * this.sampleRate);
const count = Math.min(
- Math.round(source.duration * this.sampleRate),
- source.buffer.length - offsetFrame,
+ Math.round(source.duration / source.rate * this.sampleRate),
+ Math.floor((source.buffer.length - offsetFrame) / source.rate),
this.length - startFrame,
);
for (let channel = 0; channel < this.numberOfChannels; channel += 1) {
// Mono up-mixes onto both channels, as the browser's graph does.
const from = source.buffer.getChannelData(Math.min(channel, source.buffer.numberOfChannels - 1));
const into = out[channel];
- for (let index = 0; index < count; index += 1) into[startFrame + index] += from[offsetFrame + index];
+ for (let index = 0; index < count; index += 1) into[startFrame + index] += from[offsetFrame + Math.floor(index * source.rate)];
}
}
return new HarnessAudioBuffer(out, this.sampleRate);
@@ -922,6 +923,25 @@ async function audioTrackOf(bytes: Uint8Array) {
}
describe("mixing real sound into a real export", () => {
+ it.each([0.5, 2])("exports audio at %sx with the correct source samples and end time", async (playbackRate) => {
+ const harness = installWebCodecs({ decodes: true });
+ try {
+ const config = audioProject();
+ let keptAudio = false;
+ config.timeline.tracks = config.timeline.tracks.map((track) => ({ ...track, clips: track.clips.flatMap((clip) => {
+ if (config.assets.find((asset) => asset.id === clip.assetId)?.kind !== "audio") return [clip];
+ if (track.muted || keptAudio) return [];
+ keptAudio = true;
+ return [{ ...clip, startMs: 0, sourceStartMs: 500, durationMs: 1000, playbackRate }];
+ }) }));
+ serveFiles({ video: await sampleFile(120, 30), score: wavFile({ seconds: 3, sample: (frame) => frame / (48000 * 4) }) });
+ const result = await runAudioExport(harness, config);
+ expect(result.audioShortfalls).toEqual([]);
+ const channel = encoderChannels(harness.recorded)[0];
+ expect(channel[at(500)]).toBeCloseTo((0.5 + 0.5 * playbackRate) / 4, 3);
+ expect(channel[at(1100)]).toBe(0);
+ } finally { harness.restore(); }
+ });
it("reads each audio ASSET once, before the file is opened, and never a muted one", async () => {
const harness = installWebCodecs({ decodes: true });
try {
diff --git a/src/lib/exportPipeline.ts b/src/lib/exportPipeline.ts
index 5fb06c6..eaf10e7 100644
--- a/src/lib/exportPipeline.ts
+++ b/src/lib/exportPipeline.ts
@@ -710,7 +710,8 @@ async function mixAudio(options: {
right; what was missing was anyone saying so. Below a millisecond this is
rounding between a millisecond timeline and a 48 kHz buffer, not a hole. */
const wantedSeconds = segment.durationMs / 1000;
- const availableSeconds = buffer.duration - offsetSeconds;
+ const playbackRate = segment.playbackRate ?? 1;
+ const availableSeconds = (buffer.duration - offsetSeconds) / playbackRate;
const playSeconds = Math.min(wantedSeconds, availableSeconds);
if (wantedSeconds - availableSeconds > 0.001) {
shortfalls.push(
@@ -719,8 +720,9 @@ async function mixAudio(options: {
}
const source = context.createBufferSource();
source.buffer = buffer;
+ source.playbackRate.value = playbackRate;
source.connect(context.destination);
- source.start(segment.startMs / 1000, offsetSeconds, playSeconds);
+ source.start(segment.startMs / 1000, offsetSeconds, playSeconds * playbackRate);
used += 1;
}
diff --git a/src/lib/project.ts b/src/lib/project.ts
index 60c50ad..cef8482 100644
--- a/src/lib/project.ts
+++ b/src/lib/project.ts
@@ -279,6 +279,7 @@ export const timelineClipSchema = z.object({
startMs: z.number().int().nonnegative(),
durationMs: z.number().int().positive(),
sourceStartMs: z.number().int().nonnegative().default(0),
+ playbackRate: z.number().min(0.25).max(4).nullish(),
label: z.string().min(1),
// See the note on projectAssetSchema.durationMs — nullish, never bare optional.
color: z.string().nullish(),
diff --git a/src/lib/runtime.ts b/src/lib/runtime.ts
index 4f0bcb5..6ba5004 100644
--- a/src/lib/runtime.ts
+++ b/src/lib/runtime.ts
@@ -1,3 +1,4 @@
+import { clipPlaybackRate } from "./clipTiming";
import { invoke } from "@tauri-apps/api/core";
import { cancelAgentCaptures, listenForAgentCaptures } from "./agentCapture";
import type { SceneGenerateCommand } from "./agentGeneration";
@@ -345,7 +346,7 @@ function executeDemoCommands(config: ProjectConfig, commands: ProjectCommand[]):
const asset = next.assets.find((candidate) => candidate.id === clip.assetId);
if (!asset) throw new Error(`Asset '${clip.assetId}' was not found.`);
const durationMs = sceneMediaDurationMs(asset);
- if (asset.kind !== "image" && durationMs && clip.sourceStartMs + clip.durationMs > durationMs) {
+ if (asset.kind !== "image" && durationMs && clip.sourceStartMs + clip.durationMs * clipPlaybackRate(clip) > durationMs + 1) {
throw new Error(`Clip source range extends past asset '${asset.id}'.`);
}
};
diff --git a/src/lib/timeline.ts b/src/lib/timeline.ts
index b93ec0c..338614e 100644
--- a/src/lib/timeline.ts
+++ b/src/lib/timeline.ts
@@ -1,4 +1,5 @@
import type { ProjectConfig, TimelineClip, TimelineTrack } from "./project";
+import { clipPlaybackRate, clipSourceTimeMs, MIN_CLIP_SPEED, MAX_CLIP_SPEED } from "./clipTiming";
/** Grow to contain an edit, retaining the existing length when clips move left or are removed. */
export function withTimelineTracks(config: ProjectConfig, tracks: TimelineTrack[]): ProjectConfig {
@@ -30,6 +31,20 @@ export function withTimelineTracks(config: ProjectConfig, tracks: TimelineTrack[
* clip starting at 1000 sits flush against it without overlapping. */
export const clipEndMs = (clip: Pick<TimelineClip, "startMs" | "durationMs">) => clip.startMs + clip.durationMs;
+/** Keep the same source range and preserve the spacing of later clips on this track. */
+export function retimeClip(tracks: TimelineTrack[], clipId: string, playbackRate: number): TimelineTrack[] | null {
+ const found = findClip(tracks, clipId);
+ if (!found || found.track.locked || !Number.isFinite(playbackRate) || playbackRate < MIN_CLIP_SPEED || playbackRate > MAX_CLIP_SPEED) return null;
+ const { clip, track } = found;
+ if (playbackRate === clipPlaybackRate(clip)) return null;
+ // Round down so millisecond storage cannot extend past the source's end.
+ const durationMs = Math.max(1, Math.floor(clip.durationMs * clipPlaybackRate(clip) / playbackRate));
+ const delta = durationMs - clip.durationMs;
+ return withClips(tracks, track.id, track.clips.map((item) => item.id === clipId
+ ? { ...item, playbackRate, durationMs }
+ : item.startMs >= clipEndMs(clip) ? { ...item, startMs: item.startMs + delta } : item));
+}
+
const overlaps = (aStart: number, aEnd: number, bStart: number, bEnd: number) => aStart < bEnd && bStart < aEnd;
/** How close two edges have to be, and what they may line up with. Tolerance is
@@ -193,8 +208,8 @@ export function sourceRoom(clip: TimelineClip, sourceDurationMs: number | null):
return { headMs: null, tailMs: null };
}
return {
- headMs: Math.max(0, clip.sourceStartMs),
- tailMs: Math.max(0, sourceDurationMs - clip.sourceStartMs - clip.durationMs),
+ headMs: Math.floor(Math.max(0, clip.sourceStartMs / clipPlaybackRate(clip))),
+ tailMs: Math.floor(Math.max(0, (sourceDurationMs - clip.sourceStartMs) / clipPlaybackRate(clip) - clip.durationMs)),
};
}
@@ -237,14 +252,12 @@ export function trimClip(
const floor = Math.max(0, previousEnd, options.room.headMs === null ? 0 : clip.startMs - options.room.headMs);
const startMs = Math.min(Math.max(snapped, floor), end - minClipMs);
if (startMs === clip.startMs) return null;
- const shift = startMs - clip.startMs;
const next: TimelineClip = {
...clip,
startMs,
durationMs: end - startMs,
- // The head moved by `shift` on the ruler, so it moved by `shift` inside
- // the file too. Clamped at zero for a source with no timeline of its own.
- sourceStartMs: Math.max(0, clip.sourceStartMs + shift),
+ // Source time advances at the clip's playback rate.
+ sourceStartMs: Math.max(0, Math.round(clipSourceTimeMs(clip, startMs))),
};
return withClips(tracks, track.id, byStart([...others, next]));
}
diff --git a/src/lib/timelineCapture.test.ts b/src/lib/timelineCapture.test.ts
index 7c66425..4dde51b 100644
--- a/src/lib/timelineCapture.test.ts
+++ b/src/lib/timelineCapture.test.ts
@@ -19,6 +19,12 @@ function project() {
}
describe("timeline capture", () => {
+ it("captures each layer at its own playback rate", () => {
+ const config = project();
+ config.timeline.tracks[0].clips[0].playbackRate = 2;
+ config.timeline.tracks[1].clips[0].playbackRate = 0.5;
+ expect(timelineCapturePlan(config, 1.5).layers.map((layer) => layer.sourceTimeUs)).toEqual([500000, 1250000]);
+ });
beforeEach(() => vi.resetAllMocks());
afterEach(() => vi.unstubAllGlobals());
diff --git a/src/lib/timelineCapture.ts b/src/lib/timelineCapture.ts
index b89470e..e1c5c02 100644
--- a/src/lib/timelineCapture.ts
+++ b/src/lib/timelineCapture.ts
@@ -1,3 +1,4 @@
+import { clipSourceTimeMs } from "./clipTiming";
import { continuationPlaybackTracks } from "./continuationMedia";
import { clipFrameStyle, clipVisualSettings, outputDimensions, videoDurationMs, visibleClipsAt } from "./export";
import { createCaptureCompositor, openLayerFrameReader } from "./exportPipeline";
@@ -18,7 +19,7 @@ export function timelineCapturePlan(config: ProjectConfig, at: number) {
const asset = assets.get(clip.assetId);
if (!asset) throw new Error(`Clip "${clip.label}" points at missing media.`);
if (!asset.relativePath && !asset.sourcePath) throw new Error(`Clip "${clip.label}" has no generated or imported media to capture.`);
- return { asset, sourceTimeUs: Math.round((clip.sourceStartMs + timeMs - clip.startMs) * 1000),
+ return { asset, sourceTimeUs: Math.round((clipSourceTimeMs(clip, timeMs)) * 1000),
style: clipFrameStyle(clipVisualSettings(clip), timeMs - clip.startMs) };
});
const dimensions = outputDimensions(config.settings.resolution, config.settings.aspectRatio);