AI media harness for planning, generating, and editing locally
TypeScript
Preload clips for smooth previews
Bittia408e6bf5fab
diff --git a/docs/architecture.md b/docs/architecture.mdindex b5d457b..2bd0f29 100644--- a/docs/architecture.md+++ b/docs/architecture.md@@ -61,6 +61,10 @@ Every queued generation saves a unique `latents/<generation-id>.safetensors` arcContinuation archives contain the full cumulative sequence. Slopus streams only the newly added video frames and the corresponding audio suffix to WebCodecs, so separate scenes do not repeat their predecessors on the timeline. The requested scene length denotes new frames, rounded up to a multiple of 17 by SlopFab. Continuations must use the source canvas size and compatible H3 weights/VAEs; SlopFab validates the archive and canvas during planning.+## Playback preview++Timeline and Export share `ProgramMonitor`. It keeps the current picture and the next two pictures mounted, loading and seeking upcoming clips before their cuts. Players are keyed by clip so separate trims of one file have separate decoder positions; a lower layer becoming the foreground retains its player. Upcoming players remain paused, hidden and muted. Effect renderers survive play/pause changes, and the timeline clock retains fractional milliseconds and advances continuously across cuts. Leaving the prepared window releases a clip's decoder; blob URLs remain cached until the monitor closes.+## Independently judged slices1. Foundation: desktop shell, library, portable project persistence, schema, and always-visible agent prompt.diff --git a/src/components/workspace/ChromaKeyPreview.tsx b/src/components/workspace/ChromaKeyPreview.tsxindex ec350c5..882468f 100644--- a/src/components/workspace/ChromaKeyPreview.tsx+++ b/src/components/workspace/ChromaKeyPreview.tsx@@ -12,6 +12,9 @@ export function ChromaKeyPreview({ source, sourceUrl, effect, playing, style, on}) {const canvas = useRef<HTMLCanvasElement>(null);const settings = useRef(effect);+ const running = useRef(playing);+ const updatePlayback = useRef<(() => void) | null>(null);+ running.current = playing;const errorHandler = useRef(onError);const redraw = useRef<(() => void) | null>(null);settings.current = effect;@@ -29,18 +32,25 @@ export function ChromaKeyPreview({ source, sourceUrl, effect, playing, style, ontry { renderer.draw(media, settings.current); }catch (reason) { failed = true; errorHandler.current(String(reason)); }};- const animate = () => { draw(); if (!failed) frame = requestAnimationFrame(animate); };+ const animate = () => { draw(); if (!failed && running.current) frame = requestAnimationFrame(animate); };+ updatePlayback.current = () => {+ cancelAnimationFrame(frame);+ draw();+ if (running.current) frame = requestAnimationFrame(animate);+ };redraw.current = draw;["loadeddata", "seeked", "load"].forEach((event) => media.addEventListener(event, draw));draw();- if (playing) frame = requestAnimationFrame(animate);+ if (running.current) frame = requestAnimationFrame(animate);return () => {cancelAnimationFrame(frame);+ updatePlayback.current = null;redraw.current = null;["loadeddata", "seeked", "load"].forEach((event) => media.removeEventListener(event, draw));renderer.dispose();};- }, [source, sourceUrl, playing]);+ }, [source, sourceUrl]);+ useEffect(() => updatePlayback.current?.(), [playing]);useEffect(() => redraw.current?.(), [effect]);return <canvas ref={canvas} className="program-keyed-picture" style={style} aria-label="Chroma Key preview" />;}diff --git a/src/components/workspace/ProgramLayer.tsx b/src/components/workspace/ProgramLayer.tsxindex 8b88e3c..cdc300f 100644--- a/src/components/workspace/ProgramLayer.tsx+++ b/src/components/workspace/ProgramLayer.tsx@@ -1,10 +1,11 @@-import { useEffect, useRef, useState, type CSSProperties } from "react";+import { useEffect, useLayoutEffect, useRef, useState, type CSSProperties } from "react";import { clipFrameStyle, clipVisualSettings, type ClipFrameStyle } from "../../lib/export";import { PreviewSources } from "../../lib/exportPipeline";import type { ProjectAsset, TimelineClip } from "../../lib/project";import { ChromaKeyPreview } from "./ChromaKeyPreview";import { VideoEffectsPreview } from "./VideoEffectsPreview";import { hasVideoEffects } from "../../lib/effectSettings";+import { isTauri } from "../../lib/persistence";export function previewMediaStyle(frame: ClipFrameStyle): CSSProperties {const { transform, look, opacity, revealStart, revealEnd } = frame;@@ -17,50 +18,82 @@ export function previewMediaStyle(frame: ClipFrameStyle): CSSProperties {};}-/** Lower picture tracks follow the monitor clock. Their audio is mixed by the- * monitor's existing audio elements, so these video elements stay muted. */-export function ProgramLayer({ clip, asset, sources, playheadMs, playing }: {+/** Prepared clips keep their decoder and effect renderer when made visible. */+export function ProgramLayer({ clip, asset, sources, playheadMs, playing, active, foreground, muted, externalSeek, depth }: {clip: TimelineClip;asset: ProjectAsset | undefined;sources: PreviewSources;playheadMs: number;playing: boolean;+ active: boolean;+ foreground: boolean;+ muted: boolean;+ externalSeek: boolean;+ depth: number;}) {const [url, setUrl] = useState<string | null>(null);const [error, setError] = useState<string | null>(null);+ const [ready, setReady] = useState(false);const video = useRef<HTMLVideoElement>(null);const image = useRef<HTMLImageElement>(null);- const position = useRef({ clip, playheadMs, playing });- position.current = { clip, playheadMs, playing };- const isImage = asset?.mimeType.startsWith("image/");+ const position = useRef({ clip, playheadMs, playing, active, externalSeek });+ position.current = { clip, playheadMs, playing, active, externalSeek };+ const isImage = asset?.kind === "image" || asset?.mimeType.startsWith("image/");useEffect(() => {let live = true;setUrl(null);setError(null);+ setReady(false);if (asset && (asset.relativePath || asset.sourcePath)) {void sources.url(asset).then((value) => { if (live) setUrl(value); }, (reason) => { if (live) setError(String(reason)); });} else setError(`Media for ${clip.label} is unavailable.`);return () => { live = false; };- }, [asset, sources, clip.label]);+ }, [asset, sources]);+ useEffect(() => {+ const element = video.current;+ return () => {+ if (!element) return;+ element.pause();+ element.removeAttribute("src");+ element.load();+ };+ }, [url]);const sync = () => {const element = video.current;- if (!element || element.readyState < 1) return;+ if (!element) return;const state = position.current;+ const run = state.playing && state.active;+ if (!run) element.pause();+ if (element.readyState < 1) return;const target = Math.max(0, (state.clip.sourceStartMs + state.playheadMs - state.clip.startMs) / 1000);- if (Math.abs(element.currentTime - target) > (state.playing ? 0.25 : 0.033)) element.currentTime = target;- if (state.playing) { if (element.paused) void element.play().catch(() => undefined); }- else element.pause();+ // Upcoming players seek while paused. At a normal cut they can start+ // immediately without seeking again for a fraction of a display frame.+ // Explicit jumps still seek precisely, including during playback.+ const tolerance = run && !state.externalSeek ? 0.25 : 0.033;+ if (!element.seeking && Math.abs(element.currentTime - target) > tolerance) element.currentTime = target;+ if (run && element.paused) void element.play().catch(() => undefined);+ };+ useLayoutEffect(sync, [playheadMs, playing, active, externalSeek, clip, url]);+ const loaded = () => {+ sync();+ if (video.current) setReady(video.current.readyState >= 2 && !video.current.seeking);};- useEffect(sync, [playheadMs, playing, clip, url]);const style = previewMediaStyle(clipFrameStyle(clipVisualSettings(clip), playheadMs - clip.startMs));const effects = hasVideoEffects(clip);const sourceStyle: CSSProperties = clip.chromaKey || effects ? { ...style, visibility: "hidden", position: "absolute" } : style;- return <div className="program-layer" aria-label={`Background clip: ${clip.label}`}>+ const showStatus = active && foreground && isTauri() && Boolean(asset?.relativePath || asset?.sourcePath);+ return <div className="program-layer" data-clip-id={clip.id} data-active={active} aria-hidden={!active}+ aria-label={`${foreground ? "Foreground" : "Background"} clip: ${clip.label}`}+ style={{ visibility: active ? "visible" : "hidden", zIndex: depth }}>{url && (isImage? <img ref={image} src={url} alt={clip.label} style={sourceStyle} onError={() => setError(`Could not decode ${clip.label}.`)} />- : <video ref={video} src={url} style={sourceStyle} muted playsInline preload="auto" onLoadedMetadata={sync} onError={() => setError(`Could not decode ${clip.label}.`)} />)}- {url && effects && <VideoEffectsPreview source={isImage ? image : video} sourceUrl={url} effects={clip} playing={playing && !isImage} style={style} onError={setError} />}- {url && !effects && clip.chromaKey && <ChromaKeyPreview source={isImage ? image : video} sourceUrl={url} effect={clip.chromaKey} playing={playing && !isImage} style={style} onError={setError} />}- {error && <div className="program-note program-note--error" role="alert">{error}</div>}+ : <video key={url} ref={video} src={url} style={sourceStyle} muted={!active || muted} playsInline preload="auto"+ onLoadedMetadata={loaded} onLoadedData={loaded} onCanPlay={loaded} onSeeked={loaded}+ onError={() => setError(`Could not decode ${clip.label}.`)} />)}+ {url && effects && <VideoEffectsPreview source={isImage ? image : video} sourceUrl={url} effects={clip} playing={playing && active && !isImage} style={style} onError={setError} />}+ {url && !effects && clip.chromaKey && <ChromaKeyPreview source={isImage ? image : video} sourceUrl={url} effect={clip.chromaKey} playing={playing && active && !isImage} style={style} onError={setError} />}+ {showStatus && (error+ ? <div className="program-note program-note--error" role="alert">{error}</div>+ : (!url || (!ready && !isImage)) && <div className="program-note"><span>Loading {clip.label}…</span></div>)}</div>;}diff --git a/src/components/workspace/ProgramMonitor.test.tsx b/src/components/workspace/ProgramMonitor.test.tsxindex 5a78111..6c80797 100644--- a/src/components/workspace/ProgramMonitor.test.tsx+++ b/src/components/workspace/ProgramMonitor.test.tsx@@ -2,12 +2,13 @@import { act, cleanup, fireEvent, render, screen } from "@testing-library/react";import { useState } from "react";-import { afterEach, describe, expect, it, vi } from "vitest";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import { createProjectConfig, parseProjectConfig, STORY_TRACK_ID, type ProjectConfig, type TimelineClip } from "../../lib/project";import { ProgramMonitor } from "./ProgramMonitor";import { PreviewSources } from "../../lib/exportPipeline";vi.mock("@tauri-apps/api/core", () => ({ invoke: vi.fn() }));+beforeEach(() => { vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => undefined); });afterEach(() => {cleanup();@@ -52,6 +53,89 @@ const monitor = (config: ProjectConfig, playheadMs: number) => render();describe("the program monitor", () => {+ it.each([false, true])("prepares the next trim before a cut and promotes its decoder without seeking (same asset: %s)", async (sameAsset) => {+ (window as unknown as Record<string, unknown>).__TAURI_INTERNALS__ = {};+ const read = vi.spyOn(PreviewSources.prototype, "url").mockImplementation(async (asset) => `blob:${asset.id}`);+ const playingElements = new WeakSet<HTMLMediaElement>();+ const play = vi.spyOn(HTMLMediaElement.prototype, "play").mockImplementation(function (this: HTMLMediaElement) {+ playingElements.add(this);+ return Promise.resolve();+ });+ vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(function (this: HTMLMediaElement) { playingElements.delete(this); });+ vi.spyOn(HTMLMediaElement.prototype, "paused", "get").mockImplementation(function (this: HTMLMediaElement) { return !playingElements.has(this); });+ vi.spyOn(HTMLMediaElement.prototype, "readyState", "get").mockReturnValue(4);+ const seek = vi.spyOn(HTMLMediaElement.prototype, "currentTime", "set");+ let now = 0, nextId = 0;+ const frames = new Map<number, FrameRequestCallback>();+ vi.spyOn(performance, "now").mockImplementation(() => now);+ vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => { frames.set(++nextId, callback); return nextId; });+ const cancel = vi.fn((id: number) => frames.delete(id));+ vi.stubGlobal("cancelAnimationFrame", cancel);+ const first = clip({ startMs: 0, durationMs: 1_000, sourceStartMs: 0 });+ const second = clip({ id: "clip-2", assetId: sameAsset ? first.assetId : "second-asset", startMs: 1_000, sourceStartMs: 8_000 });+ const config = project([first, second]);+ if (!sameAsset) config.assets.push({ ...config.assets[0], id: "second-asset", sourcePath: "D:/second.mp4" });+ const positions = vi.fn();+ function Harness() {+ const [position, setPosition] = useState(0);+ return <ProgramMonitor config={config} folderPath="C:\\Film" playheadMs={position} playing+ onSeek={(value) => { positions(value); setPosition(value); }} onPlayingChange={() => undefined} />;+ }+ const view = render(<Harness />);+ await act(async () => undefined);+ const incoming = view.container.querySelector<HTMLVideoElement>('[data-clip-id="clip-2"] video')!;+ expect(incoming.currentTime).toBe(8);+ expect(incoming.paused).toBe(true);+ expect(incoming.muted).toBe(true);+ expect(incoming.parentElement!.style.visibility).toBe("hidden");+ fireEvent.loadedData(incoming);+ const reads = read.mock.calls.length;+ seek.mockClear(); play.mockClear(); cancel.mockClear();+ // The display tick falls just past the cut. Keep that residual time and+ // start the frame already decoded at the trim, without another seek/load.+ await act(async () => {+ now = 1_016;+ const pending = [...frames.values()]; frames.clear(); pending.forEach((callback) => callback(now));+ });+ expect(positions).toHaveBeenLastCalledWith(1_016);+ expect(cancel).not.toHaveBeenCalled();+ expect(view.container.querySelector('[data-clip-id="clip-2"] video')).toBe(incoming);+ expect(incoming.parentElement!.style.visibility).toBe("visible");+ expect(incoming.paused).toBe(false);+ expect(incoming.muted).toBe(false);+ expect(seek).not.toHaveBeenCalled();+ expect(read).toHaveBeenCalledTimes(reads);+ expect(screen.queryByText(/Loading/)).toBeNull();+ expect(view.container.querySelectorAll("video")).toHaveLength(1);+ });++ it("keeps upcoming media silent through a gap and applies the latest scrub after a pending seek", async () => {+ vi.spyOn(PreviewSources.prototype, "url").mockImplementation(async (asset) => `blob:${asset.id}`);+ vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue();+ vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => undefined);+ vi.spyOn(HTMLMediaElement.prototype, "readyState", "get").mockReturnValue(4);+ let seeking = false;+ vi.spyOn(HTMLMediaElement.prototype, "seeking", "get").mockImplementation(() => seeking);+ const config = project([clip()]);+ const props = { config, folderPath: "C:\\Film", playing: false, onSeek: vi.fn(), onPlayingChange: vi.fn() };+ const view = render(<ProgramMonitor {...props} playheadMs={0} />);+ await act(async () => undefined);+ const video = view.container.querySelector("video")!;+ expect(video.currentTime).toBe(8);+ expect(video.muted).toBe(true);+ expect(video.parentElement!.style.visibility).toBe("hidden");+ view.rerender(<ProgramMonitor {...props} playheadMs={3_000} />);+ expect(video.currentTime).toBe(9);+ seeking = true;+ view.rerender(<ProgramMonitor {...props} playheadMs={4_000} />);+ view.rerender(<ProgramMonitor {...props} playheadMs={5_000} />);+ expect(video.currentTime).toBe(9);+ seeking = false;+ fireEvent.seeked(video);+ expect(video.currentTime).toBe(11);+ expect(view.container.querySelector("video")).toBe(video);+ });+it.each([0, 2_000])("continues the longer track at its timeline position after the foreground ends (source offset %i)", async (sourceStartMs) => {vi.spyOn(PreviewSources.prototype, "url").mockImplementation(async (asset) => `blob:${asset.id}`);vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue();@@ -93,7 +177,7 @@ describe("the program monitor", () => {}const { container } = render(<Harness />);await act(async () => undefined);- const video = container.querySelector<HTMLVideoElement>(".program-picture > video")!;+ const video = container.querySelector<HTMLVideoElement>('[data-clip-id="long-clip"] video')!;expect(container.querySelectorAll("video")).toHaveLength(2);for (const element of container.querySelectorAll("video")) {ready.add(element);@@ -104,14 +188,9 @@ describe("the program monitor", () => {await advance(4_000);expect(video.getAttribute("src")).toBe("blob:asset-long");expect(container.querySelectorAll("video")).toHaveLength(1);- // A browser resets the media position when src changes. Model a delayed- // load too: the correct seek must use where playback has reached by then.- ready.delete(video);- video.currentTime = 0;+ // Promoting the lower layer must preserve its existing decoder and src.+ expect(container.querySelector('[data-clip-id="long-clip"] video')).toBe(video);await advance(500);- ready.add(video);- fireEvent.loadedMetadata(video);- fireEvent.loadedData(video);expect(video.currentTime).toBe((4_500 + sourceStartMs) / 1000);expect(onPlayingChange).not.toHaveBeenCalled();@@ -204,4 +283,24 @@ describe("the program monitor", () => {act(() => { now = 32; frames.shift()?.(now); });expect(onSeek.mock.calls.map(([position]) => position)).toEqual([16, 32]);});++ it("preserves fractional display ticks instead of accumulating rounding drift", () => {+ let nextFrame: FrameRequestCallback | undefined;+ let now = 0;+ vi.spyOn(performance, "now").mockImplementation(() => now);+ vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => { nextFrame = callback; return 1; });+ vi.stubGlobal("cancelAnimationFrame", vi.fn());+ const config = project([clip({ startMs: 0, sourceStartMs: 0 })]);+ const onSeek = vi.fn();+ function Harness() {+ const [position, setPosition] = useState(0);+ return <ProgramMonitor config={config} folderPath="C:\\Film" playheadMs={position} playing+ onSeek={(value) => { onSeek(value); setPosition(value); }} onPlayingChange={() => undefined} />;+ }+ render(<Harness />);+ for (let frame = 1; frame <= 60; frame++) {+ act(() => { now = frame * 1000 / 60; nextFrame?.(now); });+ }+ expect(onSeek).toHaveBeenLastCalledWith(1000);+ });});diff --git a/src/components/workspace/ProgramMonitor.tsx b/src/components/workspace/ProgramMonitor.tsxindex 2af4de5..e536fa0 100644--- a/src/components/workspace/ProgramMonitor.tsx+++ b/src/components/workspace/ProgramMonitor.tsx@@ -1,37 +1,17 @@-import { ChromaKeyPreview } from "./ChromaKeyPreview";-import { VideoEffectsPreview } from "./VideoEffectsPreview";-import { hasVideoEffects } from "../../lib/effectSettings";-import { ProgramLayer, previewMediaStyle } from "./ProgramLayer";-import { Film, TriangleAlert } from "lucide-react";+import { ProgramLayer } from "./ProgramLayer";+import { Film } from "lucide-react";import { useCallback, useEffect, useMemo, useRef, useState } from "react";-import { clipFrameStyle, clipVisualSettings, visibleClipsAt } from "../../lib/export";+import { clipFrameStyle, clipVisualSettings } from "../../lib/export";import { isTauri } from "../../lib/persistence";import type { ClipTransform, ProjectAsset, ProjectConfig, TimelineClip } from "../../lib/project";import { clipEndMs } from "../../lib/timeline";import { PreviewSources } from "../../lib/exportPipeline";+import { preparedPreviewClips, previewSegmentIndex, previewSegments } from "../../lib/timelinePreview";-/* The picture, played back.- *- * Slopus has no player of its own and does not need one: the webview owns a- * hardware decoder, and a <video> pointed at the right file at the right offset- * IS the shot. What this component adds is the part a media element cannot do —- * being a TIMELINE rather than a file:- *- * * The clip under the playhead decides what is on screen, and crossing a cut- * re-points the same element at the next file.- * * Timeline time and source time are different numbers. A clip trimmed to- * start 8s into a rush shows second 8 of the file at second 0 of the cut.- * * Sound from every audiovisual track plays alongside, each additional clip in its own element,- * corrected back into step whenever it drifts.- *- * While playing, a monotonic animation clock advances the playhead and the- * media element plays alongside it. Explicit jumps and cuts seek the decoder;- * the same clock also carries playback through gaps and still images.- *- * The whole file is read into a blob, as everywhere else in Slopus (see- * MediaThumbnail): the project's media lives outside the webview's reach and- * comes back through Tauri. Blobs are cached per asset for the life of the- * view, so scrubbing back and forth across a cut re-reads nothing. */+/* The monitor owns a continuous timeline clock. Each visible or upcoming clip+ * owns a persistent media element: loading, decoding the first frame and seeking+ * a trim happen ahead of a cut. Promoting a prepared layer preserves its decoder+ * and effect renderer, including when a lower track becomes the foreground. *//** How far a sound may drift from the cut before it is pulled back. Below this* a correction is more audible than the drift it fixes. */@@ -39,7 +19,6 @@ const AUDIO_DRIFT_LIMIT_S = 0.25;/** Seeks below this are not worth the stutter — a paused element is already* showing that frame. Roughly one frame at 30fps. */const SEEK_EPSILON_S = 0.033;-const isImage = (asset: ProjectAsset) => asset.mimeType.startsWith("image/") || asset.kind === "image";const carriesAudio = (asset: ProjectAsset | undefined) => Boolean(asset && (asset.kind === "audio" || ((asset.kind === "video" || asset.kind === "generated") && asset.hasAudio === true)),);@@ -47,27 +26,6 @@ const carriesAudio = (asset: ProjectAsset | undefined) => Boolean(/** Where inside the FILE this moment of the timeline lives. */const sourceTimeMs = (clip: TimelineClip, timelineMs: number) => clip.sourceStartMs + (timelineMs - clip.startMs);-const describe = (reason: unknown) => (reason instanceof Error ? reason.message : String(reason));--/** Set an element's play position without fighting a seek already in flight.- * A scrub fires dozens of these a second; a media element that is asked to- * seek while seeking drops requests, so the latest target is remembered and- * applied when the current one lands. */-function seekTo(element: HTMLMediaElement, seconds: number, pending: { current: number | null }) {- if (!Number.isFinite(seconds)) return;- const target = Math.max(0, seconds);- if (element.readyState === 0) {- pending.current = target;- return;- }- if (element.seeking) {- pending.current = target;- return;- }- if (Math.abs(element.currentTime - target) < SEEK_EPSILON_S) return;- element.currentTime = target;-}-type TransformGesture = {kind: "move" | "scale" | "rotate";pointerId: number;@@ -105,7 +63,10 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek}) {const tracks = config.timeline.tracks;const assetsById = useMemo(() => new Map(config.assets.map((candidate) => [candidate.id, candidate])), [config.assets]);- const layers = useMemo(() => visibleClipsAt(tracks, playheadMs, assetsById), [tracks, playheadMs, assetsById]);+ const segments = useMemo(() => previewSegments(tracks, assetsById), [tracks, assetsById]);+ const segmentIndex = previewSegmentIndex(segments, playheadMs);+ const layers = segments[segmentIndex].clips;+ const prepared = useMemo(() => preparedPreviewClips(segments, segmentIndex), [segments, segmentIndex]);const clip = layers[0] ?? null;const asset = useMemo(() => (clip ? config.assets.find((candidate) => candidate.id === clip.assetId) : undefined),@@ -115,9 +76,6 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek() => clip ? clipFrameStyle(clipVisualSettings(clip), playheadMs - clip.startMs) : null,[clip, playheadMs],);- const mediaStyle = frameStyle ? previewMediaStyle(frameStyle) : undefined;- const effects = clip ? hasVideoEffects(clip) : false;- const sourceStyle: React.CSSProperties | undefined = clip?.chromaKey || effects ? { ...mediaStyle, visibility: "hidden", position: "absolute" } : mediaStyle;/* Sound that should be audible at this moment: audio-only files and embeddedsound from every active video, minus muted tracks. The visible video playsits own stream, so a second audio element is only needed for the rest. */@@ -127,11 +85,6 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek.filter((candidate) => playheadMs >= candidate.startMs && playheadMs < clipEndMs(candidate)).filter((candidate) => candidate.id !== clip?.id && carriesAudio(assetsById.get(candidate.assetId))),[tracks, playheadMs, clip?.id, assetsById]);- const videoTrackMuted = useMemo(- () => tracks.find((track) => track.id === clip?.trackId)?.muted ?? false,- [tracks, clip],- );-/** Where the picture ends. Playback stops there rather than running the clock* over a timeline that has nothing left on it. */const contentEndMs = useMemo(@@ -139,21 +92,15 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek[tracks],);- const videoRef = useRef<HTMLVideoElement>(null);- const imageRef = useRef<HTMLImageElement>(null);const pictureRef = useRef<HTMLDivElement>(null);const transformGesture = useRef<TransformGesture | null>(null);const audioRefs = useRef(new Map<string, HTMLAudioElement>());- const pendingVideoSeek = useRef<number | null>(null);/* The last playhead value THIS component produced. Anything else arriving inthe prop came from outside — the ruler, the scroll wheel, a scene card —and has to be seeked to even mid-playback. Without this the monitor cannottell its own echo from a scrub. */const advancedTo = useRef<number | null>(null);- const [url, setUrl] = useState<string | null>(null);const [audioUrls, setAudioUrls] = useState<Record<string, string>>({});- const [error, setError] = useState<string | null>(null);- const [ready, setReady] = useState(false);const [editingClipId, setEditingClipId] = useState<string | null>(null);const selectionFromPicture = useRef<string | null>(null);@@ -184,34 +131,21 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek/* The playhead as of this render, readable from inside the animation framewithout making the clock restart every time it ticks. */const positionRef = useRef(playheadMs);- positionRef.current = playheadMs;+ const externalSeek = advancedTo.current === null || Math.abs(advancedTo.current - playheadMs) > 1;+ if (externalSeek) positionRef.current = playheadMs;+ const callbacks = useRef({ onSeek, onPlayingChange });+ callbacks.current = { onSeek, onPlayingChange };const report = useCallback((ms: number) => {const rounded = Math.round(ms);/* Keep the local clock advancing even when React batches visual updateswhile pointer and hover events are busy. */- positionRef.current = rounded;+ // Keep fractional milliseconds internally; rounding every display tick+ // otherwise speeds the clock up and eventually forces corrective seeks.+ positionRef.current = ms;advancedTo.current = rounded;- onSeek(rounded);- }, [onSeek]);-- // The picture's file. Nothing is read for a clip the playhead is not inside.- useEffect(() => {- if (!asset || (!asset.relativePath && !asset.sourcePath)) {- setUrl(null);- setReady(false);- setError(null);- return;- }- let live = true;- setError(null);- setReady(false);- void sources.url(asset).then(- (value) => { if (live) setUrl(value); },- (reason) => { if (live) { setUrl(null); setError(describe(reason)); } },- );- return () => { live = false; };- }, [asset, sources]);+ callbacks.current.onSeek(rounded);+ }, []);// The sound files. Each audio clip under the playhead gets its own element,// so two tracks can be heard at once rather than one winning.@@ -233,59 +167,6 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek}}, [audioClips, config.assets, sources]);- /* Scrubbing, every other jump, and the first frame of every new shot.- While playing this deliberately does nothing for the playhead's own- advance — seeking the element to every animation tick would stutter. It- does have to fire on the two other cases:- a playhead that moved for some other reason (the ruler, the wheel, a scene- card), and a CUT. Crossing a cut re-points the element at another file, or- at another moment of the same file, and neither knows where in the footage- this clip begins until it is told. */- const playingClipId = useRef<string | null>(null);- useEffect(() => {- const video = videoRef.current;- if (!video || !clip || !url) return;- const crossedACut = playingClipId.current !== clip.id;- playingClipId.current = clip.id;- const external = advancedTo.current === null || Math.abs(advancedTo.current - playheadMs) > 1;- if (playing && !external && !crossedACut) return;- seekTo(video, sourceTimeMs(clip, playheadMs) / 1000, pendingVideoSeek);- }, [playheadMs, playing, clip, url]);-- // A seek that lands while another was queued: apply the newest target.- useEffect(() => {- const video = videoRef.current;- if (!video) return;- const flush = () => {- const target = pendingVideoSeek.current;- if (target === null) return;- pendingVideoSeek.current = null;- seekTo(video, target, pendingVideoSeek);- };- video.addEventListener("seeked", flush);- return () => {- video.removeEventListener("seeked", flush);- };- }, [url]);-- /* Changing src resets the decoder after the cut's seek may have already- reached the previous file. Once the new file loads, seek again using the- current playhead, including any time spent waiting for that file. */- const syncLoadedVideo = (video: HTMLVideoElement) => {- pendingVideoSeek.current = null;- if (clip) seekTo(video, sourceTimeMs(clip, positionRef.current) / 1000, pendingVideoSeek);- };-- /* Play and pause the picture. A media element that is asked to play before it- has data rejects, which is a promise nobody was awaiting — caught here so a- slow file does not throw into the console on every press. */- useEffect(() => {- const video = videoRef.current;- if (!video) return;- if (playing && url) void video.play().catch(() => undefined);- else video.pause();- }, [playing, url, clip?.id]);-/* The clock advances one animation frame at a time. It deliberately doesnot rely on the media element's coarser currentTime update cadence. */useEffect(() => {@@ -304,19 +185,17 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek// stops there instead of running the clock over an empty ruler.if (next >= contentEndMs) {report(contentEndMs);- onPlayingChange(false);+ callbacks.current.onPlayingChange(false);return;}- /* Past the end of THIS clip while its file still has footage left: the- cut is what ends the shot, not the file. Stepping just past the tail- hands the next clip — or the gap after it — to the next render. */- if (clip && next >= clipEndMs(clip)) report(clipEndMs(clip));- else report(next);+ // Preserve elapsed time across cuts; neither media loads nor React+ // commits restart the clock or discard the fraction past a boundary.+ report(next);frame = requestAnimationFrame(step);};frame = requestAnimationFrame(step);return () => cancelAnimationFrame(frame);- }, [playing, clip, url, contentEndMs, report, onPlayingChange]);+ }, [playing, contentEndMs, report]);/* Sound, kept in step with the cut. Each element is put where the playheadsays it should be, and corrected only once it has drifted audibly — a@@ -402,10 +281,6 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek}else if (!isTauri()) {overlay = <div className="program-note"><Film size={22} /><span>Playback needs the desktop app — the browser preview has no project folder to read the footage from.</span></div>;- } else if (error) {- overlay = <div className="program-note program-note--error" role="alert"><TriangleAlert size={22} /><span>{clip.label} could not be played: {error}</span></div>;- } else if (!url || (!ready && asset && !isImage(asset))) {- overlay = <div className="program-note"><span>Loading {clip.label}…</span></div>;}/* The frame is fitted inside the panel and the rest is the project's own@@ -423,23 +298,22 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeekonSelectClip(clip.id);}}>- {layers.slice(1).reverse().map((layer) => <ProgramLayer key={layer.id} clip={layer} asset={assetsById.get(layer.assetId)} sources={sources} playheadMs={playheadMs} playing={playing} />)}- {/* One element, re-pointed at whatever the playhead is over. Rebuilding it- per clip would drop the decoder and re-open the file on every cut. */}- {asset && !isImage(asset) && url && <video- ref={videoRef}- src={url}- style={sourceStyle}- muted={videoTrackMuted}- playsInline- preload="auto"- onLoadedMetadata={(event) => syncLoadedVideo(event.currentTarget)}- onLoadedData={(event) => { syncLoadedVideo(event.currentTarget); setReady(true); }}- onError={() => setError("This file could not be decoded.")}- />}- {asset && isImage(asset) && url && <img ref={imageRef} src={url} alt={clip?.label ?? ""} style={sourceStyle} />}- {asset && url && clip && effects && <VideoEffectsPreview source={isImage(asset) ? imageRef : videoRef} sourceUrl={url} effects={clip} playing={playing && !isImage(asset)} style={mediaStyle} onError={setError} />}- {asset && url && !effects && clip?.chromaKey && <ChromaKeyPreview source={isImage(asset) ? imageRef : videoRef} sourceUrl={url} effect={clip.chromaKey} playing={playing && !isImage(asset)} style={mediaStyle} onError={setError} />}+ <div className="program-media">{prepared.map(({ clip: layer, prepareAtMs }) => {+ const depth = layers.findIndex((visible) => visible.id === layer.id);+ return <ProgramLayer+ key={`${folderPath}:${layer.id}`}+ clip={layer}+ asset={assetsById.get(layer.assetId)}+ sources={sources}+ playheadMs={depth >= 0 ? playheadMs : prepareAtMs}+ playing={playing}+ active={depth >= 0}+ foreground={depth === 0}+ muted={depth !== 0 || (tracks.find((track) => track.id === layer.trackId)?.muted ?? false)}+ externalSeek={externalSeek}+ depth={depth >= 0 ? layers.length - depth : 0}+ />;+ })}</div>{clip && frameStyle && editingClipId === clip.id && onTransformChange && !transformEditingDisabled && <divclassName="program-transform"style={{ transform: `translate(${frameStyle.transform.positionX}%, ${frameStyle.transform.positionY}%) scale(${frameStyle.transform.scale / 100}) rotate(${frameStyle.transform.rotation}deg)` }}diff --git a/src/components/workspace/VideoEffectsPreview.test.tsx b/src/components/workspace/VideoEffectsPreview.test.tsxindex d004ee9..7ad97de 100644--- a/src/components/workspace/VideoEffectsPreview.test.tsx+++ b/src/components/workspace/VideoEffectsPreview.test.tsx@@ -8,12 +8,32 @@ import { VideoEffectsPreview } from "./VideoEffectsPreview";vi.mock("../../lib/videoEffectsGpu", () => ({ createVideoEffectsPreview: vi.fn() }));afterEach(() => { cleanup(); vi.resetAllMocks(); vi.unstubAllGlobals(); });-function Harness({ effects, onError = vi.fn() }: { effects: GpuEffects; onError?: (message: string) => void }) {+function Harness({ effects, playing = false, onError = vi.fn() }: { effects: GpuEffects; playing?: boolean; onError?: (message: string) => void }) {const image = useRef<HTMLImageElement>(null);- return <><img ref={image} src="blob:picture" alt="Source" /><VideoEffectsPreview source={image} sourceUrl="blob:picture" effects={effects} playing={false} onError={onError} /></>;+ return <><img ref={image} src="blob:picture" alt="Source" /><VideoEffectsPreview source={image} sourceUrl="blob:picture" effects={effects} playing={playing} onError={onError} /></>;}describe("GPU effects preview lifecycle", () => {+ it("reuses the prepared renderer when playback starts and stops", async () => {+ const renderer = { draw: vi.fn(), dispose: vi.fn() };+ vi.mocked(createVideoEffectsPreview).mockResolvedValue(renderer);+ let nextId = 0;+ const frames = new Map<number, FrameRequestCallback>();+ vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => { frames.set(++nextId, callback); return nextId; });+ vi.stubGlobal("cancelAnimationFrame", (id: number) => frames.delete(id));+ const view = render(<Harness effects={{ blur: { radius: 4 } }} />);+ await act(async () => undefined);+ expect(frames.size).toBe(0);+ view.rerender(<Harness effects={{ blur: { radius: 4 } }} playing />);+ expect(frames.size).toBe(1);+ view.rerender(<Harness effects={{ blur: { radius: 4 } }} />);+ expect(frames.size).toBe(0);+ expect(createVideoEffectsPreview).toHaveBeenCalledTimes(1);+ expect(renderer.dispose).not.toHaveBeenCalled();+ view.unmount();+ expect(renderer.dispose).toHaveBeenCalledTimes(1);+ });+it("redraws paused media after seeking and editing without recreating GPU resources", async () => {const renderer = { draw: vi.fn(), dispose: vi.fn() };vi.mocked(createVideoEffectsPreview).mockResolvedValue(renderer);diff --git a/src/components/workspace/VideoEffectsPreview.tsx b/src/components/workspace/VideoEffectsPreview.tsxindex c26b53c..a2293f7 100644--- a/src/components/workspace/VideoEffectsPreview.tsx+++ b/src/components/workspace/VideoEffectsPreview.tsx@@ -11,6 +11,9 @@ export function VideoEffectsPreview({ source, sourceUrl, effects, playing, style}) {const canvas = useRef<HTMLCanvasElement>(null);const settings = useRef(effects);+ const running = useRef(playing);+ const updatePlayback = useRef<(() => void) | null>(null);+ running.current = playing;const errorHandler = useRef(onError);const redraw = useRef<(() => void) | null>(null);settings.current = effects;@@ -27,10 +30,16 @@ export function VideoEffectsPreview({ source, sourceUrl, effects, playing, stylecatch (reason) { failed = true; errorHandler.current(String(reason)); }};const schedule = () => {- if (!live || failed || !playing) return;+ if (!live || failed || !running.current) return;if (video?.requestVideoFrameCallback) videoFrame = video.requestVideoFrameCallback(() => { draw(); schedule(); });else animation = requestAnimationFrame(() => { draw(); schedule(); });};+ const cancel = () => {+ cancelAnimationFrame(animation);+ if (videoFrame) video?.cancelVideoFrameCallback(videoFrame);+ animation = 0; videoFrame = 0;+ };+ updatePlayback.current = () => { cancel(); if (renderer) { draw(); schedule(); } };redraw.current = draw;const events = ["loadeddata", "seeked", "load"];events.forEach((event) => media.addEventListener(event, draw));@@ -41,13 +50,14 @@ export function VideoEffectsPreview({ source, sourceUrl, effects, playing, style}, (reason) => { if (live) errorHandler.current(String(reason)); });return () => {live = false;- cancelAnimationFrame(animation);- if (videoFrame) video?.cancelVideoFrameCallback(videoFrame);+ cancel();+ updatePlayback.current = null;redraw.current = null;events.forEach((event) => media.removeEventListener(event, draw));renderer?.dispose();};- }, [source, sourceUrl, playing]);+ }, [source, sourceUrl]);+ useEffect(() => updatePlayback.current?.(), [playing]);useEffect(() => redraw.current?.(), [effects]);return <canvas ref={canvas} className="program-keyed-picture" style={style} aria-label="Video effects preview" />;}diff --git a/src/lib/timelinePreview.test.ts b/src/lib/timelinePreview.test.tsnew file mode 100644index 0000000..1be30b7--- /dev/null+++ b/src/lib/timelinePreview.test.ts@@ -0,0 +1,43 @@+import { describe, expect, it } from "vitest";+import type { ProjectAsset, TimelineClip, TimelineTrack } from "./project";+import { preparedPreviewClips, previewSegmentIndex, previewSegments } from "./timelinePreview";++const asset = { id: "video", kind: "video", mimeType: "video/mp4" } as ProjectAsset;+const assets = new Map([[asset.id, asset]]);+const clip = (id: string, startMs: number, durationMs: number, extra: Partial<TimelineClip> = {}): TimelineClip => ({+ id, assetId: asset.id, trackId: "track", startMs, durationMs, sourceStartMs: 0, label: id, color: null, status: "approved", ...extra,+});+const track = (clips: TimelineClip[]): TimelineTrack => ({ id: "track", name: "Picture", kind: "video", muted: false, locked: false, clips });++describe("prepared timeline pictures", () => {+ it("prepares a covered lower track at the time it will be revealed", () => {+ const segments = previewSegments([track([clip("top", 0, 2000)]), track([clip("lower", 0, 6000, { sourceStartMs: 8000 })])], assets);+ expect(segments.map((segment) => [segment.startMs, segment.clips.map((clip) => clip.id)])).toEqual([+ [0, ["top"]], [2000, ["lower"]], [6000, []],+ ]);+ const prepared = preparedPreviewClips(segments, 0);+ expect(prepared.map((entry) => [entry.clip.id, entry.prepareAtMs])).toEqual([["top", 0], ["lower", 2000]]);+ expect(prepared[1].clip.sourceStartMs + prepared[1].prepareAtMs - prepared[1].clip.startMs).toBe(10000);+ });++ it("retains transition layers only while revealed and prepares beyond gaps", () => {+ const segments = previewSegments([+ track([clip("fade", 1000, 2000, { transition: { type: "fade", durationMs: 500 } }), clip("later", 5000, 1000)]),+ track([clip("lower", 1000, 2000)]),+ ], assets);+ expect(segments.map((segment) => [segment.startMs, segment.clips.map((clip) => clip.id)])).toEqual([+ [0, []], [1000, ["fade", "lower"]], [1500, ["fade"]], [3000, []], [5000, ["later"]], [6000, []],+ ]);+ expect(previewSegmentIndex(segments, 1499)).toBe(1);+ expect(previewSegmentIndex(segments, 1500)).toBe(2);+ expect(previewSegmentIndex(segments, 99999)).toBe(5);+ expect(preparedPreviewClips(segments, 2).map((entry) => entry.clip.id)).toEqual(["fade", "later"]);+ });++ it("bounds prepared players to the current and next two pictures, even for repeated assets", () => {+ const segments = previewSegments([track(Array.from({ length: 30 }, (_, index) => clip(String(index), index * 1000, 1000)))], assets);+ expect(preparedPreviewClips(segments, 0).map((entry) => entry.clip.id)).toEqual(["0", "1", "2"]);+ expect(preparedPreviewClips(segments, 1).map((entry) => entry.clip.id)).toEqual(["1", "2", "3"]);+ expect(preparedPreviewClips(segments, 30)).toEqual([]);+ });+});diff --git a/src/lib/timelinePreview.ts b/src/lib/timelinePreview.tsnew file mode 100644index 0000000..ca8ec5c--- /dev/null+++ b/src/lib/timelinePreview.ts@@ -0,0 +1,53 @@+import { visibleClipsAt } from "./export";+import type { ProjectAsset, TimelineClip, TimelineTrack } from "./project";++export type PreviewSegment = { startMs: number; clips: TimelineClip[] };++/** Resolve visibility once per edit, including a lower track revealed by a cut+ * or the end of a transition. Playback only needs a binary search afterwards. */+export function previewSegments(tracks: TimelineTrack[], assets: ReadonlyMap<string, ProjectAsset>): PreviewSegment[] {+ const boundaries = new Set([0]);+ for (const track of tracks) {+ if (track.kind !== "video") continue;+ for (const clip of track.clips) {+ boundaries.add(clip.startMs);+ boundaries.add(clip.startMs + clip.durationMs);+ if (clip.transition && clip.transition.type !== "cut") {+ boundaries.add(clip.startMs + Math.min(clip.durationMs, clip.transition.durationMs));+ }+ }+ }+ const segments: PreviewSegment[] = [];+ for (const startMs of [...boundaries].sort((a, b) => a - b)) {+ const clips = visibleClipsAt(tracks, startMs, assets);+ const previous = segments.at(-1)?.clips;+ if (previous && previous.length === clips.length && previous.every((clip, index) => clip.id === clips[index].id)) continue;+ segments.push({ startMs, clips });+ }+ return segments;+}++export function previewSegmentIndex(segments: PreviewSegment[], timeMs: number): number {+ let low = 0, high = segments.length - 1;+ while (low < high) {+ const middle = Math.ceil((low + high) / 2);+ if (segments[middle].startMs <= timeMs) low = middle;+ else high = middle - 1;+ }+ return low;+}++/** Keep the current picture and the next two pictures mounted. A prepared+ * player is keyed by clip, since two trims of one asset need separate seeks. */+export function preparedPreviewClips(segments: PreviewSegment[], index: number) {+ const prepared = new Map<string, { clip: TimelineClip; prepareAtMs: number }>();+ let pictures = 0;+ for (let next = index; next < segments.length && pictures < 3; next++) {+ const segment = segments[next];+ if (segment.clips.length) pictures++;+ for (const clip of segment.clips) {+ if (!prepared.has(clip.id)) prepared.set(clip.id, { clip, prepareAtMs: segment.startMs });+ }+ }+ return [...prepared.values()];+}diff --git a/src/styles/timeline.css b/src/styles/timeline.cssindex c49a772..b868c0a 100644--- a/src/styles/timeline.css+++ b/src/styles/timeline.css@@ -1544,5 +1544,6 @@.clip-effects__picker strong { font-size: var(--text-sm); font-weight: var(--weight-medium); }.clip-effects__picker span { font-size: var(--text-xs); color: var(--text-muted); }-.program-layer { position: absolute; inset: 0; pointer-events: none; }+.program-media { position: absolute; inset: 0; z-index: 0; pointer-events: none; }+.program-layer { position: absolute; inset: 0; display: flex; align-items: center; justify-content: center; pointer-events: none; }.program-picture > video, .program-picture > img, .program-keyed-picture { position: absolute; inset: 0; }