AI media harness for planning, generating, and editing locally
TypeScript
Add timeline clip speed control
Bitti6ad85b792e62
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.mdindex 14f1778..7a72a84 100644--- a/RELEASE_NOTES.md+++ b/RELEASE_NOTES.md@@ -4,6 +4,7 @@ This alpha update adds LAN generation workers, Linux support, expanded color gra## What's new+- Timeline clips now offer Speed in Timing, from 25% to 400%. Speed changes preserve the source range, adjust duration and later clips on the same track, and apply to preview and export, including audio pitch.- Open the agent from the Settings top bar in a resizable right-side panel to configure generators, weight paths and LoRAs, even without an open project.- Scene Generation settings now offer Latent upscale: generate at half resolution and upscale to the selected output size, locally or on a LAN worker. Download its model or choose a local file in Other weights settings.diff --git a/src-tauri/src/project/commands/clips.rs b/src-tauri/src/project/commands/clips.rsindex fe7a44c..04055bb 100644--- a/src-tauri/src/project/commands/clips.rs+++ b/src-tauri/src/project/commands/clips.rs@@ -142,6 +142,7 @@ pub(super) fn apply(start_ms,duration_ms,source_start_ms,+ playback_rate: None,label,color: None,status: status.into(),@@ -209,7 +210,7 @@ pub(super) fn apply(project,&clip.asset_id,clip.source_start_ms,- clip.duration_ms,+ (clip.duration_ms as f64 * clip.playback_rate.unwrap_or(1.0)).round() as u64,)?;clip.track_id = project.timeline.tracks[target_track_index].id.clone();project.timeline.tracks[source_track_index]diff --git a/src-tauri/src/project/commands/tests.rs b/src-tauri/src/project/commands/tests.rsindex 8de0145..c116e48 100644--- a/src-tauri/src/project/commands/tests.rs+++ b/src-tauri/src/project/commands/tests.rs@@ -1,6 +1,18 @@use super::batch::execute_commands;use super::*;+#[test]+fn clip_speed_round_trips_and_validates() {+ let mut config = fixture();+ config.timeline.tracks[0].clips[0].playback_rate = Some(0.5);+ let json = serde_json::to_value(&config).unwrap();+ assert_eq!(json["timeline"]["tracks"][0]["clips"][0]["playbackRate"], 0.5);+ let restored: ProjectConfig = serde_json::from_value(json).unwrap();+ assert_eq!(validate_and_normalize_config(restored).unwrap().timeline.tracks[0].clips[0].playback_rate, Some(0.5));+ config.timeline.tracks[0].clips[0].playback_rate = Some(0.0);+ assert!(validate_and_normalize_config(config).unwrap_err().contains("playback rate"));+}+#[test]fn scene_creation_uses_injected_model_defaults() {let current = fixture();diff --git a/src-tauri/src/project/timeline.rs b/src-tauri/src/project/timeline.rsindex 93e2e31..e6ee1a6 100644--- a/src-tauri/src/project/timeline.rs+++ b/src-tauri/src/project/timeline.rs@@ -27,6 +27,8 @@ pub(crate) struct TimelineClip {pub(crate) duration_ms: u64,#[serde(default)]pub(crate) source_start_ms: u64,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub(crate) playback_rate: Option<f64>,pub(crate) label: String,// `.optional()` without `.nullable()` on the frontend — see ProjectAsset.#[serde(default, skip_serializing_if = "Option::is_none")]diff --git a/src-tauri/src/project/validation/timeline.rs b/src-tauri/src/project/validation/timeline.rsindex 04340ce..cd04302 100644--- a/src-tauri/src/project/validation/timeline.rs+++ b/src-tauri/src/project/validation/timeline.rs@@ -8,6 +8,9 @@ pub(super) fn validate(config: &mut ProjectConfig) -> Result<(), String> {return Err(format!("Track '{}' cannot have an empty name.", track.id));}for clip in &track.clips {+ if clip.playback_rate.is_some_and(|rate| !rate.is_finite() || !(0.25..=4.0).contains(&rate)) {+ return Err(format!("Clip '{}' has an invalid playback rate.", clip.id));+ }if clip.label.is_empty() {return Err(format!("Clip '{}' cannot have an empty label.", clip.id));}diff --git a/src/components/workspace/ProgramLayer.tsx b/src/components/workspace/ProgramLayer.tsxindex aecb0a6..fa273fd 100644--- a/src/components/workspace/ProgramLayer.tsx+++ b/src/components/workspace/ProgramLayer.tsx@@ -1,3 +1,4 @@+import { clipPlaybackRate, clipSourceTimeMs } from "../../lib/clipTiming";import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState, type CSSProperties } from "react";import { clipFrameStyle, clipVisualSettings, type ClipFrameStyle } from "../../lib/export";import { PreviewSources } from "../../lib/exportPipeline";@@ -77,13 +78,15 @@ export function ProgramLayer({ clip, asset, sources, playheadMs, playing, rate =const run = state.playing && state.active;if (!run) element.pause();if (element.readyState < 1) return;- const target = Math.max(0, (state.clip.sourceStartMs + state.playheadMs - state.clip.startMs) / 1000);+ const target = Math.max(0, clipSourceTimeMs(state.clip, state.playheadMs) / 1000);// Upcoming players seek while paused. At a normal cut they can start// immediately without seeking again for a fraction of a display frame.// Explicit jumps still seek precisely, including during playback.const tolerance = run && !state.externalSeek ? 0.25 : 0.033;if (!element.seeking && Math.abs(element.currentTime - target) > tolerance) element.currentTime = target;- if (run && element.playbackRate !== rate) element.playbackRate = rate;+ const mediaRate = rate * clipPlaybackRate(state.clip);+ element.preservesPitch = false;+ if (run && element.playbackRate !== mediaRate) element.playbackRate = mediaRate;if (run && element.paused) void element.play().catch(() => undefined);};useLayoutEffect(sync, [playheadMs, playing, rate, active, externalSeek, clip, url]);diff --git a/src/components/workspace/ProgramMonitor.test.tsx b/src/components/workspace/ProgramMonitor.test.tsxindex d2ac7a8..8aaf020 100644--- a/src/components/workspace/ProgramMonitor.test.tsx+++ b/src/components/workspace/ProgramMonitor.test.tsx@@ -53,6 +53,27 @@ const monitor = (config: ProjectConfig, playheadMs: number) => render();describe("the program monitor", () => {+ it.each([0.5, 2])("seeks and plays video and audio at %sx clip speed", async (playbackRate) => {+ vi.spyOn(PreviewSources.prototype, "url").mockImplementation(async (asset) => `blob:${asset.id}`);+ vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue();+ vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => undefined);+ vi.spyOn(HTMLMediaElement.prototype, "readyState", "get").mockReturnValue(4);+ vi.stubGlobal("requestAnimationFrame", () => 1);+ vi.stubGlobal("cancelAnimationFrame", () => undefined);+ const config = project([clip({ playbackRate })]);+ config.assets.push({ ...config.assets[0], id: "sound", kind: "audio", sourcePath: "D:/sound.wav" });+ config.timeline.tracks[1].clips = [clip({ id: "sound", assetId: "sound", trackId: config.timeline.tracks[1].id, playbackRate })];+ const props = { config, folderPath: "C:/Film", onSeek: vi.fn(), onPlayingChange: vi.fn(), rate: 2 };+ const view = render(<ProgramMonitor {...props} playheadMs={2000} playing={false} />);+ await act(async () => undefined);+ view.rerender(<ProgramMonitor {...props} playheadMs={3000} playing />);+ for (const media of view.container.querySelectorAll<HTMLMediaElement>("video, audio")) {+ expect(media.currentTime).toBe(8 + playbackRate);+ expect(media.playbackRate).toBe(2 * playbackRate);+ expect(media.preservesPitch).toBe(false);+ }+ expect(view.container.querySelectorAll("video, audio")).toHaveLength(2);+ });it.each([false, true])("prepares the next trim before a cut and promotes its decoder without seeking (same asset: %s)", async (sameAsset) => {(window as unknown as Record<string, unknown>).__TAURI_INTERNALS__ = {};const read = vi.spyOn(PreviewSources.prototype, "url").mockImplementation(async (asset) => `blob:${asset.id}`);diff --git a/src/components/workspace/ProgramMonitor.tsx b/src/components/workspace/ProgramMonitor.tsxindex e805e80..790e81d 100644--- a/src/components/workspace/ProgramMonitor.tsx+++ b/src/components/workspace/ProgramMonitor.tsx@@ -1,3 +1,4 @@+import { clipPlaybackRate, clipSourceTimeMs } from "../../lib/clipTiming";import { ProgramLayer } from "./ProgramLayer";import { ProgramPicture } from "./ProgramPicture";import { Film16 } from "../ui/icons";@@ -28,7 +29,7 @@ const carriesAudio = (asset: ProjectAsset | undefined) => Boolean();/** Where inside the FILE this moment of the timeline lives. */-const sourceTimeMs = (clip: TimelineClip, timelineMs: number) => clip.sourceStartMs + (timelineMs - clip.startMs);+const sourceTimeMs = clipSourceTimeMs;type TransformGesture = {kind: "move" | "scale" | "rotate";@@ -244,7 +245,9 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, rate =element.currentTime = Math.max(0, target);}if (mediaPlaying) {- if (element.playbackRate !== rate) element.playbackRate = rate;+ const mediaRate = rate * clipPlaybackRate(audioClip);+ element.preservesPitch = false;+ if (element.playbackRate !== mediaRate) element.playbackRate = mediaRate;void element.play().catch(() => undefined);} else element.pause();}diff --git a/src/components/workspace/TimelineClipThumbnails.tsx b/src/components/workspace/TimelineClipThumbnails.tsxindex 998500f..9addda0 100644--- a/src/components/workspace/TimelineClipThumbnails.tsx+++ b/src/components/workspace/TimelineClipThumbnails.tsx@@ -1,3 +1,4 @@+import { clipPlaybackRate } from "../../lib/clipTiming";import { useEffect, useMemo, useState } from "react";import { readProjectFileUrl } from "../../lib/persistence";import { type GenerationJob, type TimelineClip } from "../../lib/project";@@ -20,7 +21,8 @@ export function TimelineClipThumbnails({ folderPath, job, clip }: {job: GenerationJob;clip: TimelineClip;}) {- const sourceEndMs = clip.sourceStartMs + clip.durationMs;+ const sourceDurationMs = clip.durationMs * clipPlaybackRate(clip);+ const sourceEndMs = clip.sourceStartMs + sourceDurationMs;const times = useMemo(() => {const first = Math.floor(clip.sourceStartMs / TIMELINE_THUMBNAIL_INTERVAL_MS) * TIMELINE_THUMBNAIL_INTERVAL_MS;const result: number[] = [];@@ -72,8 +74,8 @@ export function TimelineClipThumbnails({ folderPath, job, clip }: {alt=""draggable={false}style={{- left: `${(still.timeMs - clip.sourceStartMs) / clip.durationMs * 100}%`,- width: `${TIMELINE_THUMBNAIL_INTERVAL_MS / clip.durationMs * 100}%`,+ left: `${(still.timeMs - clip.sourceStartMs) / sourceDurationMs * 100}%`,+ width: `${TIMELINE_THUMBNAIL_INTERVAL_MS / sourceDurationMs * 100}%`,}}/>)}</span>;diff --git a/src/components/workspace/TimelineClipWaveform.tsx b/src/components/workspace/TimelineClipWaveform.tsxindex fe1a541..ada07a1 100644--- a/src/components/workspace/TimelineClipWaveform.tsx+++ b/src/components/workspace/TimelineClipWaveform.tsx@@ -1,3 +1,4 @@+import { clipPlaybackRate } from "../../lib/clipTiming";import { useEffect, useMemo, useState } from "react";import { audioWaveformSegment, loadAudioWaveform } from "../../lib/audioWaveform";import { type ProjectAsset, type TimelineClip } from "../../lib/project";@@ -23,12 +24,12 @@ export function TimelineClipWaveform({ folderPath, asset, clip, cacheVersion = "let live = true;setLevels(null);void loadAudioWaveform(folderPath, asset, cacheVersion).then((overview) => {- if (live) setLevels(audioWaveformSegment(overview, clip.sourceStartMs / 1_000, clip.durationMs / 1_000, DISPLAY_POINTS));+ if (live) setLevels(audioWaveformSegment(overview, clip.sourceStartMs / 1_000, clip.durationMs * clipPlaybackRate(clip) / 1_000, DISPLAY_POINTS));}).catch(() => {if (live) setLevels([]);});return () => { live = false; };- }, [asset, cacheVersion, clip.durationMs, clip.sourceStartMs, folderPath]);+ }, [asset, cacheVersion, clip.durationMs, clip.playbackRate, clip.sourceStartMs, folderPath]);const lines = useMemo(() => levels && levels.length > 0 ? sampleLines(levels) : null, [levels]);diff --git a/src/components/workspace/TimelineView.test.tsx b/src/components/workspace/TimelineView.test.tsxindex 4a66890..808819a 100644--- a/src/components/workspace/TimelineView.test.tsx+++ b/src/components/workspace/TimelineView.test.tsx@@ -220,6 +220,24 @@ const press = (container: HTMLElement, key: string, init: Partial<KeyboardEventIconst toolbarTime = (container: HTMLElement) => container.querySelector(".transport-time")!.textContent;describe("the media and scene panels", () => {+ it("changes clip speed in Timing and splits using the retimed source position", () => {+ const config = withClip(measuredVideo(projectWithMedia(), 40000), { durationMs: 4000, sourceStartMs: 1000, playbackRate: 2 });+ const { container, onChange } = render_(config);+ const field = screen.getByRole("spinbutton", { name: "Clip speed" });+ expect((field as HTMLInputElement).value).toBe("200");+ fireEvent.change(field, { target: { value: "50" } });+ fireEvent.blur(field);+ const changed = wrote(onChange.mock.calls.at(-1)![0], config).timeline.tracks.flatMap((track) => track.clips)[0];+ expect(changed.playbackRate).toBe(0.5);+ expect(changed.durationMs).toBe(16000);+ // This harness retains its input snapshot; split the original 2x clip.+ onChange.mockClear();+ press(container, "ArrowRight", { shiftKey: true });+ press(container, "k", { ctrlKey: true });+ const split = wrote(onChange.mock.calls.at(-1)![0], config).timeline.tracks.flatMap((track) => track.clips);+ expect(split[1].sourceStartMs).toBe(Math.round(1000 + split[0].durationMs * 2));+ expect(split[1].playbackRate).toBe(2);+ });it("switches the media panel between grid and list, and remembers which", () => {const { container } = mediaPanel(projectWithMedia());expect(container.querySelector(".media-grid--grid")).not.toBeNull();diff --git a/src/components/workspace/TimelineView.tsx b/src/components/workspace/TimelineView.tsxindex 250b2a1..117f920 100644--- a/src/components/workspace/TimelineView.tsx+++ b/src/components/workspace/TimelineView.tsx@@ -1,4 +1,5 @@import { clipEffectCount, ClipEffects } from "./ClipEffects";+import { clipPlaybackRate, clipSourceTimeMs, MIN_CLIP_SPEED, MAX_CLIP_SPEED } from "../../lib/clipTiming";import { continuationPlaybackTracks, sceneMediaDurationMs, sceneMediaStartSeconds } from "../../lib/continuationMedia";import { ProjectStatus } from "./ProjectStatus";import { PreviewEngineStatus } from "./PreviewEngineStatus";@@ -25,7 +26,7 @@ import {} from "../../lib/project";import {adjacentCut, clipEndMs, findClip, formatTimecode, frameAt, frameStartMs, insertClip, moveClip, parseTimecode, pasteClip,- removeClip, rulerLabel, rulerScale, scrollAfterZoom, snapTargets, sourceRoom, trimClip, withTimelineTracks,+ removeClip, retimeClip, rulerLabel, rulerScale, scrollAfterZoom, snapTargets, sourceRoom, trimClip, withTimelineTracks,type SnapOptions, type SourceRoom,} from "../../lib/timeline";import { useShortcut } from "../../lib/commands";@@ -550,7 +551,7 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {const splitClip = (clip: TimelineClip | undefined) => {if (!clip || !canSplit(clip)) return;const leftDuration = playhead - clip.startMs;- const right: TimelineClip = { ...clip, id: `${clip.id}-split-${Date.now()}`, startMs: playhead, durationMs: clip.durationMs - leftDuration, sourceStartMs: clip.sourceStartMs + leftDuration, label: `${clip.label} · B`, transition: undefined };+ const right: TimelineClip = { ...clip, id: `${clip.id}-split-${Date.now()}`, startMs: playhead, durationMs: clip.durationMs - leftDuration, sourceStartMs: Math.round(clipSourceTimeMs(clip, playhead)), label: `${clip.label} · B`, transition: undefined };updateTracks(tracks.map((track) => track.id === clip.trackId ? { ...track, clips: track.clips.flatMap((item) => item.id === clip.id ? [{ ...item, durationMs: leftDuration, label: `${item.label} · A` }, right] : [item]) } : track));setSelectedId(right.id);};@@ -796,9 +797,15 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {is only bounded from below. */const selectedAsset = selected ? assetById(selected.assetId) : undefined;const sourceLimitMs = selected && selectedAsset?.kind !== "image" && selectedAsset?.durationMs- ? Math.max(1, (sceneMediaDurationMs(selectedAsset) ?? selectedAsset.durationMs) - selected.sourceStartMs)+ ? Math.max(1, ((sceneMediaDurationMs(selectedAsset) ?? selectedAsset.durationMs) - selected.sourceStartMs) / clipPlaybackRate(selected)): null;const minClipMs = Math.max(1, Math.round(1000 / fps));+ const speedDisabled = !selected || tracks.find((track) => track.id === selected.trackId)?.locked || !selectedAsset || !["video", "generated", "audio"].includes(selectedAsset.kind);+ const changeSpeed = (percent: number) => {+ if (!selected || speedDisabled) return;+ const next = retimeClip(tracks, selected.id, percent / 100);+ if (next) { setDurationDraft(null); updateTracks(next, `speed:${selected.id}`); }+ };/** What the FILE has left at each end of a clip. A still image and a file* nothing has measured have no source timeline to run out of, and get no* limit rather than a guessed one. */@@ -1462,6 +1469,13 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {><More16 aria-hidden="true" /></button>}/><PropSection title="Timing" persistKey="timeline.clip.timing" summary={`${shortTimecode(selected.startMs, fps)} → ${shortTimecode(clipEndMs(selected), fps)}`}>+ <PropRow label="Speed" htmlFor="clip-speed" value={clipPlaybackRate(selected)} defaultValue={1}+ onReset={speedDisabled ? undefined : () => changeSpeed(100)}+ tooltip="100% is normal speed. Changes duration while keeping the same source range and moving later clips on this track. Audio pitch changes with speed.">+ <CommittedNumberInput id="clip-speed" className="text-field" aria-label="Clip speed" minimum={MIN_CLIP_SPEED * 100} maximum={MAX_CLIP_SPEED * 100} step={1}+ value={Number((clipPlaybackRate(selected) * 100).toFixed(2))} disabled={speedDisabled} onCommit={changeSpeed} />+ <span className="inspector-unit">%</span>+ </PropRow><PropRow label="Starts at"><span className="inspector-value">{formatTimecode(selected.startMs, fps)}</span></PropRow><PropRow label="Duration" htmlFor="clip-duration"><inputdiff --git a/src/lib/clipTiming.test.ts b/src/lib/clipTiming.test.tsnew file mode 100644index 0000000..b4e8d1c--- /dev/null+++ b/src/lib/clipTiming.test.ts@@ -0,0 +1,54 @@+import { expect, it } from "vitest";+import { createProjectConfig, parseProjectConfig, type TimelineClip, type TimelineTrack } from "./project";+import { clipPlaybackRate, clipSourceTimeMs } from "./clipTiming";+import { retimeClip, sourceRoom, trimClip } from "./timeline";+import { buildExportPlan, defaultExportSettings, sourceTimeMsForFrame } from "./export";++const clip: TimelineClip = { id: "clip", assetId: "asset", trackId: "track", label: "Clip", status: "approved", startMs: 1000, durationMs: 4000, sourceStartMs: 2000 };+const tracks: TimelineTrack[] = [{ id: "track", name: "Video", kind: "video", locked: false, muted: false,+ clips: [clip, { ...clip, id: "next", startMs: 6000 }] }];++it.each([0.25, 0.5, 2, 4])("retimes at %sx preserving the source range and following gaps", (rate) => {+ const next = retimeClip(tracks, "clip", rate)!;+ const changed = next[0].clips[0];+ expect(changed.durationMs).toBe(4000 / rate);+ expect(clipSourceTimeMs(changed, changed.startMs + changed.durationMs)).toBe(6000);+ expect(next[0].clips[1].startMs - changed.startMs - changed.durationMs).toBe(1000);+ expect(retimeClip(next, "clip", 1)![0].clips).toEqual([{ ...clip, playbackRate: 1 }, tracks[0].clips[1]]);+ expect(tracks[0].clips[0]).toBe(clip);+});++it("rejects invalid rates and locked tracks", () => {+ for (const rate of [0, -1, NaN, Infinity, 0.24, 4.1]) expect(retimeClip(tracks, "clip", rate)).toBeNull();+ expect(retimeClip([{ ...tracks[0], locked: true }], "clip", 2)).toBeNull();+ expect(clipPlaybackRate(clip)).toBe(1);+});++it("does not round retimed clips beyond the available source", () => {+ const next = retimeClip([{ ...tracks[0], clips: [{ ...clip, durationMs: 4003 }] }], "clip", 4)![0].clips[0];+ expect(clipSourceTimeMs(next, next.startMs + next.durationMs)).toBeLessThanOrEqual(6003);+});++it.each([0.5, 2])("trims in source time at %sx", (rate) => {+ const changed = retimeClip(tracks, "clip", rate)!;+ const room = sourceRoom(changed[0].clips[0], 8000);+ expect(room).toEqual({ headMs: 2000 / rate, tailMs: 2000 / rate });+ const trimmed = trimClip(changed, "clip", "start", 1500, { minClipMs: 33, room })![0].clips[0];+ expect(trimmed.sourceStartMs).toBe(2000 + 500 * rate);+ expect(clipSourceTimeMs(trimmed, trimmed.startMs + trimmed.durationMs)).toBe(6000);+});++it("persists rates and exports foreground, underlay and audio source times", () => {+ const config = createProjectConfig({ name: "Speed", prompt: "", aspectRatio: "16:9", resolution: "768p", targetDurationSeconds: 10 });+ config.assets = [{ id: "asset", name: "Video", kind: "video", relativePath: "video.mp4", mimeType: "video/mp4", hasAudio: true, durationMs: 20000, createdAt: config.createdAt }];+ config.timeline.tracks = [{ ...tracks[0], clips: [{ ...clip, playbackRate: 2, look: { opacity: 50 } }] },+ { ...tracks[0], id: "lower", clips: [{ ...clip, id: "lower", trackId: "lower", playbackRate: 0.5 }] }];+ const loaded = parseProjectConfig(JSON.parse(JSON.stringify(config)));+ const plan = buildExportPlan(loaded, { ...defaultExportSettings(loaded), frameRate: 30 });+ const segment = plan.segments.find((item) => item.kind === "clip");+ if (!segment || segment.kind !== "clip") throw new Error("Missing clip");+ expect(sourceTimeMsForFrame(segment, 60, 30)).toBe(4000);+ expect(sourceTimeMsForFrame(segment.underlays![0], 60, 30)).toBe(2500);+ expect(plan.audio.map((item) => item.playbackRate)).toEqual([2, 0.5]);+ expect(() => parseProjectConfig({ ...config, timeline: { tracks: [{ ...tracks[0], clips: [{ ...clip, playbackRate: 0 }] }] } })).toThrow();+});diff --git a/src/lib/clipTiming.ts b/src/lib/clipTiming.tsnew file mode 100644index 0000000..839503f--- /dev/null+++ b/src/lib/clipTiming.ts@@ -0,0 +1,7 @@+import type { TimelineClip } from "./project";++export const MIN_CLIP_SPEED = 0.25;+export const MAX_CLIP_SPEED = 4;+export const clipPlaybackRate = (clip: Pick<TimelineClip, "playbackRate">) => clip.playbackRate ?? 1;+export const clipSourceTimeMs = (clip: Pick<TimelineClip, "sourceStartMs" | "startMs" | "playbackRate">, timelineMs: number) =>+ clip.sourceStartMs + (timelineMs - clip.startMs) * clipPlaybackRate(clip);diff --git a/src/lib/continuationMedia.ts b/src/lib/continuationMedia.tsindex 940c14a..829c54a 100644--- a/src/lib/continuationMedia.ts+++ b/src/lib/continuationMedia.ts@@ -1,3 +1,4 @@+import { clipPlaybackRate } from "./clipTiming";import { GENERATION_FRAME_RATE, generationAssetId, sceneOutputFrames, type GenerationJob, type ProjectAsset, type ProjectConfig, type SceneMediaSegment, type TimelineClip } from "./project";const milliseconds = (frames: number) => frames * 1000 / GENERATION_FRAME_RATE;@@ -65,7 +66,7 @@ export function continuationPlaybackTracks(config: ProjectConfig) {const matching = joined?.sceneSegments?.find((part) => part.sceneId === own.sceneId && part.latentRelativePath === own.latentRelativePath);// A Start continuation contains only the opening overlap, so do// not pretend that its source file contains the whole original scene.- if (matching && clip.sourceStartMs + clip.durationMs <= milliseconds(matching.frameCount) + 1) {+ if (matching && clip.sourceStartMs + clip.durationMs * clipPlaybackRate(clip) <= milliseconds(matching.frameCount) + 1) {source = joined;segment = matching;break;diff --git a/src/lib/export.ts b/src/lib/export.tsindex 67a07dd..c5d79db 100644--- a/src/lib/export.ts+++ b/src/lib/export.ts@@ -11,6 +11,7 @@has to come from, are the two questions an editor gets wrong silently.========================================================================== */+import { clipPlaybackRate, clipSourceTimeMs } from "./clipTiming";import {clipChromaKey,clipLook,@@ -286,6 +287,7 @@ export function estimatedBytes(bitrate: number, durationMs: number): number {--------------------------------------------------------------------------- */export interface ExportClipLayer {+ playbackRate?: number;clipId: string;assetId: string;label: string;@@ -297,6 +299,7 @@ export interface ExportClipLayer {export type ExportSegment =| {kind: "clip";+ playbackRate?: number;clipId: string;assetId: string;label: string;@@ -347,6 +350,7 @@ const AUDIO_MIX_NOTE_BYTES = 200_000_000;* to fit it, the audible part is worked out here and the remainder is reported* as a note the user reads before pressing Export. */export interface AudioSegment {+ playbackRate?: number;clipId: string;assetId: string;label: string;@@ -395,7 +399,8 @@ export function audioSegments(config: ProjectConfig, durationMs: number): AudioSstartMs: start,durationMs: end - start,// A clip dragged to a negative start would begin further into its file.- sourceStartMs: clip.sourceStartMs + (start - clip.startMs),+ sourceStartMs: clipSourceTimeMs(clip, start),+ ...(clipPlaybackRate(clip) !== 1 ? { playbackRate: clipPlaybackRate(clip) } : {}),});}}@@ -440,7 +445,7 @@ export function sourceTimeMsForFrame(frameIndex: number,frameRate: number,): number {- return segment.clipSourceStartMs + ((frameIndex * 1000) / frameRate - segment.clipStartMs);+ return segment.clipSourceStartMs + ((frameIndex * 1000) / frameRate - segment.clipStartMs) * (segment.playbackRate ?? 1);}/** The clip a viewer sees at `timeMs`.@@ -533,10 +538,12 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)endFrame: frame + 1,clipStartMs: clip.startMs,clipSourceStartMs: clip.sourceStartMs,+ ...(clipPlaybackRate(clip) !== 1 ? { playbackRate: clipPlaybackRate(clip) } : {}),visual: clipVisualSettings(clip),...(underlays.length ? { underlays: underlays.map((layer) => ({clipId: layer.id, assetId: layer.assetId, label: layer.label,clipStartMs: layer.startMs, clipSourceStartMs: layer.sourceStartMs,+ ...(clipPlaybackRate(layer) !== 1 ? { playbackRate: clipPlaybackRate(layer) } : {}),visual: clipVisualSettings(layer),})) } : {}),});diff --git a/src/lib/exportPipeline.test.ts b/src/lib/exportPipeline.test.tsindex e6c7895..9bdc154 100644--- a/src/lib/exportPipeline.test.ts+++ b/src/lib/exportPipeline.test.ts@@ -584,7 +584,7 @@ function installWebCodecs(options: HarnessOptions = {}): { restore: () => void;readonly numberOfChannels: number;readonly length: number;readonly sampleRate: number;- private readonly started: Array<{ buffer: HarnessAudioBuffer; when: number; offset: number; duration: number }> = [];+ private readonly started: Array<{ buffer: HarnessAudioBuffer; when: number; offset: number; duration: number; rate: number }> = [];constructor(init: { numberOfChannels: number; length: number; sampleRate: number }) {this.numberOfChannels = init.numberOfChannels;this.length = init.length;@@ -605,13 +605,14 @@ function installWebCodecs(options: HarnessOptions = {}): { restore: () => void;const context = this;return {buffer: null as HarnessAudioBuffer | null,+ playbackRate: { value: 1 },connected: false,connect(destination: unknown) {this.connected = destination === context.destination;},start(when: number, offset: number, duration: number) {if (!this.buffer || !this.connected) return; // an unconnected source is silence- context.started.push({ buffer: this.buffer, when, offset, duration });+ context.started.push({ buffer: this.buffer, when, offset, duration, rate: this.playbackRate.value });},};}@@ -621,15 +622,15 @@ function installWebCodecs(options: HarnessOptions = {}): { restore: () => void;const startFrame = Math.round(source.when * this.sampleRate);const offsetFrame = Math.round(source.offset * this.sampleRate);const count = Math.min(- Math.round(source.duration * this.sampleRate),- source.buffer.length - offsetFrame,+ Math.round(source.duration / source.rate * this.sampleRate),+ Math.floor((source.buffer.length - offsetFrame) / source.rate),this.length - startFrame,);for (let channel = 0; channel < this.numberOfChannels; channel += 1) {// Mono up-mixes onto both channels, as the browser's graph does.const from = source.buffer.getChannelData(Math.min(channel, source.buffer.numberOfChannels - 1));const into = out[channel];- for (let index = 0; index < count; index += 1) into[startFrame + index] += from[offsetFrame + index];+ for (let index = 0; index < count; index += 1) into[startFrame + index] += from[offsetFrame + Math.floor(index * source.rate)];}}return new HarnessAudioBuffer(out, this.sampleRate);@@ -922,6 +923,25 @@ async function audioTrackOf(bytes: Uint8Array) {}describe("mixing real sound into a real export", () => {+ it.each([0.5, 2])("exports audio at %sx with the correct source samples and end time", async (playbackRate) => {+ const harness = installWebCodecs({ decodes: true });+ try {+ const config = audioProject();+ let keptAudio = false;+ config.timeline.tracks = config.timeline.tracks.map((track) => ({ ...track, clips: track.clips.flatMap((clip) => {+ if (config.assets.find((asset) => asset.id === clip.assetId)?.kind !== "audio") return [clip];+ if (track.muted || keptAudio) return [];+ keptAudio = true;+ return [{ ...clip, startMs: 0, sourceStartMs: 500, durationMs: 1000, playbackRate }];+ }) }));+ serveFiles({ video: await sampleFile(120, 30), score: wavFile({ seconds: 3, sample: (frame) => frame / (48000 * 4) }) });+ const result = await runAudioExport(harness, config);+ expect(result.audioShortfalls).toEqual([]);+ const channel = encoderChannels(harness.recorded)[0];+ expect(channel[at(500)]).toBeCloseTo((0.5 + 0.5 * playbackRate) / 4, 3);+ expect(channel[at(1100)]).toBe(0);+ } finally { harness.restore(); }+ });it("reads each audio ASSET once, before the file is opened, and never a muted one", async () => {const harness = installWebCodecs({ decodes: true });try {diff --git a/src/lib/exportPipeline.ts b/src/lib/exportPipeline.tsindex 5fb06c6..eaf10e7 100644--- a/src/lib/exportPipeline.ts+++ b/src/lib/exportPipeline.ts@@ -710,7 +710,8 @@ async function mixAudio(options: {right; what was missing was anyone saying so. Below a millisecond this isrounding between a millisecond timeline and a 48 kHz buffer, not a hole. */const wantedSeconds = segment.durationMs / 1000;- const availableSeconds = buffer.duration - offsetSeconds;+ const playbackRate = segment.playbackRate ?? 1;+ const availableSeconds = (buffer.duration - offsetSeconds) / playbackRate;const playSeconds = Math.min(wantedSeconds, availableSeconds);if (wantedSeconds - availableSeconds > 0.001) {shortfalls.push(@@ -719,8 +720,9 @@ async function mixAudio(options: {}const source = context.createBufferSource();source.buffer = buffer;+ source.playbackRate.value = playbackRate;source.connect(context.destination);- source.start(segment.startMs / 1000, offsetSeconds, playSeconds);+ source.start(segment.startMs / 1000, offsetSeconds, playSeconds * playbackRate);used += 1;}diff --git a/src/lib/project.ts b/src/lib/project.tsindex 60c50ad..cef8482 100644--- a/src/lib/project.ts+++ b/src/lib/project.ts@@ -279,6 +279,7 @@ export const timelineClipSchema = z.object({startMs: z.number().int().nonnegative(),durationMs: z.number().int().positive(),sourceStartMs: z.number().int().nonnegative().default(0),+ playbackRate: z.number().min(0.25).max(4).nullish(),label: z.string().min(1),// See the note on projectAssetSchema.durationMs — nullish, never bare optional.color: z.string().nullish(),diff --git a/src/lib/runtime.ts b/src/lib/runtime.tsindex 4f0bcb5..6ba5004 100644--- a/src/lib/runtime.ts+++ b/src/lib/runtime.ts@@ -1,3 +1,4 @@+import { clipPlaybackRate } from "./clipTiming";import { invoke } from "@tauri-apps/api/core";import { cancelAgentCaptures, listenForAgentCaptures } from "./agentCapture";import type { SceneGenerateCommand } from "./agentGeneration";@@ -345,7 +346,7 @@ function executeDemoCommands(config: ProjectConfig, commands: ProjectCommand[]):const asset = next.assets.find((candidate) => candidate.id === clip.assetId);if (!asset) throw new Error(`Asset '${clip.assetId}' was not found.`);const durationMs = sceneMediaDurationMs(asset);- if (asset.kind !== "image" && durationMs && clip.sourceStartMs + clip.durationMs > durationMs) {+ if (asset.kind !== "image" && durationMs && clip.sourceStartMs + clip.durationMs * clipPlaybackRate(clip) > durationMs + 1) {throw new Error(`Clip source range extends past asset '${asset.id}'.`);}};diff --git a/src/lib/timeline.ts b/src/lib/timeline.tsindex b93ec0c..338614e 100644--- a/src/lib/timeline.ts+++ b/src/lib/timeline.ts@@ -1,4 +1,5 @@import type { ProjectConfig, TimelineClip, TimelineTrack } from "./project";+import { clipPlaybackRate, clipSourceTimeMs, MIN_CLIP_SPEED, MAX_CLIP_SPEED } from "./clipTiming";/** Grow to contain an edit, retaining the existing length when clips move left or are removed. */export function withTimelineTracks(config: ProjectConfig, tracks: TimelineTrack[]): ProjectConfig {@@ -30,6 +31,20 @@ export function withTimelineTracks(config: ProjectConfig, tracks: TimelineTrack[* clip starting at 1000 sits flush against it without overlapping. */export const clipEndMs = (clip: Pick<TimelineClip, "startMs" | "durationMs">) => clip.startMs + clip.durationMs;+/** Keep the same source range and preserve the spacing of later clips on this track. */+export function retimeClip(tracks: TimelineTrack[], clipId: string, playbackRate: number): TimelineTrack[] | null {+ const found = findClip(tracks, clipId);+ if (!found || found.track.locked || !Number.isFinite(playbackRate) || playbackRate < MIN_CLIP_SPEED || playbackRate > MAX_CLIP_SPEED) return null;+ const { clip, track } = found;+ if (playbackRate === clipPlaybackRate(clip)) return null;+ // Round down so millisecond storage cannot extend past the source's end.+ const durationMs = Math.max(1, Math.floor(clip.durationMs * clipPlaybackRate(clip) / playbackRate));+ const delta = durationMs - clip.durationMs;+ return withClips(tracks, track.id, track.clips.map((item) => item.id === clipId+ ? { ...item, playbackRate, durationMs }+ : item.startMs >= clipEndMs(clip) ? { ...item, startMs: item.startMs + delta } : item));+}+const overlaps = (aStart: number, aEnd: number, bStart: number, bEnd: number) => aStart < bEnd && bStart < aEnd;/** How close two edges have to be, and what they may line up with. Tolerance is@@ -193,8 +208,8 @@ export function sourceRoom(clip: TimelineClip, sourceDurationMs: number | null):return { headMs: null, tailMs: null };}return {- headMs: Math.max(0, clip.sourceStartMs),- tailMs: Math.max(0, sourceDurationMs - clip.sourceStartMs - clip.durationMs),+ headMs: Math.floor(Math.max(0, clip.sourceStartMs / clipPlaybackRate(clip))),+ tailMs: Math.floor(Math.max(0, (sourceDurationMs - clip.sourceStartMs) / clipPlaybackRate(clip) - clip.durationMs)),};}@@ -237,14 +252,12 @@ export function trimClip(const floor = Math.max(0, previousEnd, options.room.headMs === null ? 0 : clip.startMs - options.room.headMs);const startMs = Math.min(Math.max(snapped, floor), end - minClipMs);if (startMs === clip.startMs) return null;- const shift = startMs - clip.startMs;const next: TimelineClip = {...clip,startMs,durationMs: end - startMs,- // The head moved by `shift` on the ruler, so it moved by `shift` inside- // the file too. Clamped at zero for a source with no timeline of its own.- sourceStartMs: Math.max(0, clip.sourceStartMs + shift),+ // Source time advances at the clip's playback rate.+ sourceStartMs: Math.max(0, Math.round(clipSourceTimeMs(clip, startMs))),};return withClips(tracks, track.id, byStart([...others, next]));}diff --git a/src/lib/timelineCapture.test.ts b/src/lib/timelineCapture.test.tsindex 7c66425..4dde51b 100644--- a/src/lib/timelineCapture.test.ts+++ b/src/lib/timelineCapture.test.ts@@ -19,6 +19,12 @@ function project() {}describe("timeline capture", () => {+ it("captures each layer at its own playback rate", () => {+ const config = project();+ config.timeline.tracks[0].clips[0].playbackRate = 2;+ config.timeline.tracks[1].clips[0].playbackRate = 0.5;+ expect(timelineCapturePlan(config, 1.5).layers.map((layer) => layer.sourceTimeUs)).toEqual([500000, 1250000]);+ });beforeEach(() => vi.resetAllMocks());afterEach(() => vi.unstubAllGlobals());diff --git a/src/lib/timelineCapture.ts b/src/lib/timelineCapture.tsindex b89470e..e1c5c02 100644--- a/src/lib/timelineCapture.ts+++ b/src/lib/timelineCapture.ts@@ -1,3 +1,4 @@+import { clipSourceTimeMs } from "./clipTiming";import { continuationPlaybackTracks } from "./continuationMedia";import { clipFrameStyle, clipVisualSettings, outputDimensions, videoDurationMs, visibleClipsAt } from "./export";import { createCaptureCompositor, openLayerFrameReader } from "./exportPipeline";@@ -18,7 +19,7 @@ export function timelineCapturePlan(config: ProjectConfig, at: number) {const asset = assets.get(clip.assetId);if (!asset) throw new Error(`Clip "${clip.label}" points at missing media.`);if (!asset.relativePath && !asset.sourcePath) throw new Error(`Clip "${clip.label}" has no generated or imported media to capture.`);- return { asset, sourceTimeUs: Math.round((clip.sourceStartMs + timeMs - clip.startMs) * 1000),+ return { asset, sourceTimeUs: Math.round((clipSourceTimeMs(clip, timeMs)) * 1000),style: clipFrameStyle(clipVisualSettings(clip), timeMs - clip.startMs) };});const dimensions = outputDimensions(config.settings.resolution, config.settings.aspectRatio);