Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Use Timeline playback and larger thumbnail Video cards

Bitti181ec9dae065
diff --git a/src/components/workspace/ProgramMonitor.tsx b/src/components/workspace/ProgramMonitor.tsx
index f075640..ea911f1 100644
--- a/src/components/workspace/ProgramMonitor.tsx
+++ b/src/components/workspace/ProgramMonitor.tsx
@@ -52,7 +52,9 @@ const clamp = (value: number, minimum: number, maximum: number) => Math.max(mini
const angleAt = (x: number, y: number, centerX: number, centerY: number) => Math.atan2(y - centerY, x - centerX) * 180 / Math.PI;
const normalizedAngle = (value: number) => ((value + 180) % 360 + 360) % 360 - 180;
-export function ProgramMonitor({ config, folderPath, playheadMs, playing, rate = 1, onSeek, onPlayingChange, selectedClipId = null, transformEditingDisabled = false, onSelectClip, onTransformChange, selectedMotionNodeId, onSelectMotionNode, onMotionChange }: {
+export function ProgramMonitor({ config, folderPath, playheadMs, playing, rate = 1, onSeek, onPlayingChange, waitForInitialFrame = false, selectedClipId = null, transformEditingDisabled = false, onSelectClip, onTransformChange, selectedMotionNodeId, onSelectMotionNode, onMotionChange }: {
+ /** Card/popout autoplay waits for its opening picture before advancing the clock. */
+ waitForInitialFrame?: boolean;
selectedMotionNodeId?: string | null;
onSelectMotionNode?: (id: string) => void;
onMotionChange?: (motion: Motion) => void;
@@ -80,7 +82,7 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, rate =
const receiveMotionFrame = useCallback((id: string, nodes: MotionNode[]) => {
if (id === selectedClipId && !playing) setMotionFrame({ id, nodes });
}, [selectedClipId, playing]);
- const tracks = useMemo(() => continuationPlaybackTracks(config), [config.timeline.tracks, config.assets, config.generationJobs]);
+ const tracks = useMemo(() => continuationPlaybackTracks(config), [config.timeline.tracks, config.assets, config.generationJobs, config.videos]);
const assetsById = useMemo(() => new Map(config.assets.map((candidate) => [candidate.id, candidate])), [config.assets]);
const segments = useMemo(() => previewSegments(tracks, assetsById), [tracks, assetsById]);
const segmentIndex = previewSegmentIndex(segments, playheadMs);
@@ -172,6 +174,7 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, rate =
const callbacks = useRef({ onSeek, onPlayingChange });
callbacks.current = { onSeek, onPlayingChange };
+ const currentPicture = useRef({ layers, playerKeys }); currentPicture.current = { layers, playerKeys };
const report = useCallback((ms: number) => {
const rounded = Math.round(ms);
/* Keep the local clock advancing even when React batches visual updates
@@ -208,9 +211,19 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, rate =
useEffect(() => {
if (!playing) return;
let frame = 0;
+ let started = !waitForInitialFrame;
let lastTick = performance.now();
const step = () => {
const now = performance.now();
+ if (!started) {
+ started = currentPicture.current.layers.every((layer) => {
+ const element = media.get(currentPicture.current.playerKeys.get(layer.id) ?? layer.id);
+ return element instanceof HTMLVideoElement ? element.readyState >= 2 && !element.seeking
+ : element instanceof HTMLImageElement ? element.complete && element.naturalWidth > 0 : Boolean(element);
+ });
+ lastTick = now;
+ frame = requestAnimationFrame(step); return;
+ }
const elapsed = now - lastTick;
lastTick = now;
/* Media currentTime can be exposed in coarse jumps while Chromium is
@@ -237,7 +250,7 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, rate =
};
frame = requestAnimationFrame(step);
return () => cancelAnimationFrame(frame);
- }, [playing, rate, contentEndMs, report]);
+ }, [playing, rate, contentEndMs, report, waitForInitialFrame, media]);
/* The status bar's playback rate: counted by the picture on every tick,
published once a second, never through this component's state. */
useEffect(() => {
diff --git a/src/components/workspace/VideoLibraryView.test.tsx b/src/components/workspace/VideoLibraryView.test.tsx
index b30f5e9..1832caa 100644
--- a/src/components/workspace/VideoLibraryView.test.tsx
+++ b/src/components/workspace/VideoLibraryView.test.tsx
@@ -5,12 +5,13 @@ import { useState } from "react";
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { createDraftGenerationJob, createProjectConfig, generationAssetId, type ProjectConfig } from "../../lib/project";
import { newVideo } from "../../lib/videoLibrary";
-import { readMediaFileUrl } from "../../lib/persistence";
+import { PreviewSources } from "../../lib/exportPipeline";
+import { shotPoster } from "./ShotThumbnail";
import { VideoLibraryView } from "./VideoLibraryView";
import { VideoPlayback } from "./VideoPlayback";
import { choose } from "./comboTestUtils";
-vi.mock("../../lib/persistence", async (original) => ({ ...await original<typeof import("../../lib/persistence")>(), readMediaFileUrl: vi.fn(async (_, item) => `blob:${item.relativePath}`) }));
+vi.mock("./ShotThumbnail", async (original) => ({ ...await original<typeof import("./ShotThumbnail")>(), shotPoster: vi.fn(async () => ({ poster: "data:image/jpeg;base64,preview", failed: false })) }));
vi.mock("../../lib/nativeShell", async (original) => ({ ...await original<typeof import("../../lib/nativeShell")>(), askNative: vi.fn(async () => true) }));
vi.mock("./GeneratorView", () => ({ GeneratorView: ({ config, sceneIds, onChange }: { config: ProjectConfig; sceneIds: string[]; onChange: (next: ProjectConfig) => void }) => <div aria-label="Scenes editor">
<span>{sceneIds.join(",")}</span><span data-testid="all-scenes">{config.generationJobs.length}</span>
@@ -19,10 +20,11 @@ vi.mock("./GeneratorView", () => ({ GeneratorView: ({ config, sceneIds, onChange
beforeEach(() => {
localStorage.clear();
+ vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => undefined);
vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue(); vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => undefined);
vi.stubGlobal("URL", Object.assign(URL, { revokeObjectURL: vi.fn() }));
});
-afterEach(() => { cleanup(); vi.restoreAllMocks(); vi.clearAllMocks(); vi.unstubAllGlobals(); });
+afterEach(() => { cleanup(); vi.restoreAllMocks(); vi.clearAllMocks(); vi.unstubAllGlobals(); delete (window as unknown as Record<string, unknown>).__TAURI_INTERNALS__; });
const project = () => createProjectConfig({ name: "Library", prompt: "", resolution: "416p", aspectRatio: "16:9", targetDurationSeconds: 30 });
function mount(config = project(), openScene?: { id: string }) {
let latest = config;
@@ -37,7 +39,7 @@ it("creates nested folders and videos, moves cards to a folder, and opens only t
const video = view.config().videos!.find((video) => video.title === "New video")!;
expect(video.folderId).toBe(view.config().videoFolders![0].id);
fireEvent.change(screen.getByRole("textbox", { name: "Video title" }), { target: { value: "Track one" } });
- fireEvent.doubleClick(screen.getByRole("button", { name: "Select Track one" }));
+ fireEvent.click(screen.getByRole("button", { name: "Open Track one" }));
expect(screen.getByTestId("all-scenes")).toHaveTextContent("1"); fireEvent.click(screen.getByRole("button", { name: "Add scene" }));
expect(view.config().videos!.find((item) => item.id === video.id)?.sceneIds).toEqual(["added"]);
expect(view.config().generationJobs.map((scene) => scene.id)).toEqual(["legacy", "added"]);
@@ -47,7 +49,7 @@ it("creates nested folders and videos, moves cards to a folder, and opens only t
fireEvent.drop(screen.getByRole("button", { name: "Open Album" }).closest('[role="listitem"]')!, { dataTransfer });
expect(view.config().videos!.find((item) => item.id === "video-default")?.folderId).toBe(video.folderId);
fireEvent.click(screen.getByRole("button", { name: "List view" })); fireEvent.click(screen.getByRole("button", { name: "Open Album" }));
- expect(screen.getByRole("button", { name: "Select Track one" })).toBeInTheDocument(); expect(screen.getByRole("button", { name: "Select Library" })).toBeInTheDocument();
+ expect(screen.getByRole("button", { name: "Open Track one" })).toBeInTheDocument(); expect(screen.getByRole("button", { name: "Open Library" })).toBeInTheDocument();
});
it("creates a whole-song scene sequence and opens the owner for a timeline navigation", () => {
@@ -62,27 +64,50 @@ it("creates a whole-song scene sequence and opens the owner for a timeline navig
it("removes folders without deleting their videos and removes video scenes without removing timeline media", async () => {
const config = project(); config.videoFolders = [{ id: "folder", name: "Collection" }]; config.videos = [{ ...newVideo("folder"), title: "Film", sceneIds: ["a"] }]; config.generationJobs = [createDraftGenerationJob("A", { id: "a" })];
const view = mount(config); fireEvent.contextMenu(screen.getByRole("button", { name: "Open Collection" })); fireEvent.click(screen.getByRole("menuitem", { name: "Remove folder" }));
- expect(view.config().videoFolders).toHaveLength(0); expect(screen.getByRole("button", { name: "Select Film" })).toBeInTheDocument();
- fireEvent.contextMenu(screen.getByRole("button", { name: "Select Film" })); fireEvent.click(screen.getByRole("menuitem", { name: "Remove video" }));
+ expect(view.config().videoFolders).toHaveLength(0); expect(screen.getByRole("button", { name: "Open Film" })).toBeInTheDocument();
+ fireEvent.contextMenu(screen.getByRole("button", { name: "Open Film" })); fireEvent.click(screen.getByRole("menuitem", { name: "Remove video" }));
await waitFor(() => expect(view.config().videos).toHaveLength(0)); expect(view.config().generationJobs).toHaveLength(0); expect(view.config().timeline).toEqual(config.timeline);
});
-it("plays successive scene ranges, replays the full video, closes outside the popout and releases media", async () => {
+it("prepares later clips before playback, retains decoders across cuts, replays and closes the popout", async () => {
+ (window as unknown as Record<string, unknown>).__TAURI_INTERNALS__ = {};
+ vi.spyOn(PreviewSources.prototype, "url").mockImplementation(async (asset) => 'blob:' + asset.relativePath);
+ const ready = vi.spyOn(HTMLMediaElement.prototype, "readyState", "get").mockReturnValue(0);
+ let now = 0, nextId = 0;
+ const frames = new Map<number, FrameRequestCallback>();
+ vi.spyOn(performance, "now").mockImplementation(() => now);
+ vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => { frames.set(++nextId, callback); return nextId; });
+ vi.stubGlobal("cancelAnimationFrame", (id: number) => frames.delete(id));
+ const tick = async (ms: number) => { await act(async () => { now += ms; const callbacks = [...frames.values()]; frames.clear(); callbacks.forEach((callback) => callback(now)); }); };
const config = project(); config.generationJobs = ["a", "b"].map((id) => createDraftGenerationJob("Dance", { id, durationSeconds: 3 }));
- config.assets = config.generationJobs.map((job, index) => ({ id: generationAssetId(job.id), kind: "generated", name: job.id, relativePath: `media/${job.id}.mp4`, mimeType: "video/mp4", createdAt: config.createdAt, durationMs: 10000,
- sceneSegments: [{ sceneId: job.id, startFrame: index * 96, frameCount: 96, latentRelativePath: `latents/${job.id}.safetensors` }] }));
+ config.assets = config.generationJobs.map((job, index) => ({ id: generationAssetId(job.id), kind: "generated", name: job.id, relativePath: 'media/' + job.id + '.mp4', mimeType: "video/mp4", createdAt: config.createdAt, durationMs: 10000,
+ sceneSegments: [{ sceneId: job.id, startFrame: index * 96, frameCount: 96, latentRelativePath: 'latents/' + job.id + '.safetensors' }] }));
const video = { ...newVideo(null), title: "Film", sceneIds: ["a", "b"] }, close = vi.fn();
const view = render(<VideoPlayback video={video} config={config} folderPath="D:/Project" popout onClose={close} />);
- const player = view.container.querySelector("video")!;
- await waitFor(() => expect(player).toHaveAttribute("src", "blob:media/a.mp4"));
- await act(async () => { fireEvent.loadedMetadata(player); }); expect(player.currentTime).toBe(0);
- player.currentTime = 3; fireEvent.timeUpdate(player);
- await waitFor(() => expect(player).toHaveAttribute("src", "blob:media/b.mp4"));
- await act(async () => { fireEvent.loadedMetadata(player); }); expect(player.currentTime).toBe(4);
- fireEvent.ended(player); fireEvent.click(screen.getByRole("button", { name: "Replay Film" }));
- await waitFor(() => expect(player).toHaveAttribute("src", "blob:media/a.mp4"));
+ await waitFor(() => expect(view.container.querySelectorAll("video")).toHaveLength(2));
+ const incoming = view.container.querySelector<HTMLVideoElement>('[data-clip-id="video-preview-b"] video')!;
+ await tick(5000); expect(view.container.querySelector('.video-card__position')).toHaveTextContent('1 / 2');
+ ready.mockReturnValue(4); fireEvent.loadedData(incoming); expect(incoming.currentTime).toBe(4);
+ await tick(16); await tick(3001);
+ expect(view.container.querySelector('[data-clip-id="video-preview-b"] video')).toBe(incoming);
+ expect(view.container.querySelector('.video-card__position')).toHaveTextContent('2 / 2');
+ await tick(3100); fireEvent.click(screen.getByRole("button", { name: "Replay Film" }));
+ expect(view.container.querySelector('.video-card__position')).toHaveTextContent('1 / 2');
fireEvent.pointerDown(screen.getByRole("dialog")); expect(close).not.toHaveBeenCalled();
fireEvent.pointerDown(view.container.querySelector(".shot-popout")!); expect(close).toHaveBeenCalledTimes(1);
- view.unmount(); expect(URL.revokeObjectURL).toHaveBeenCalledWith("blob:media/a.mp4"); expect(URL.revokeObjectURL).toHaveBeenCalledWith("blob:media/b.mp4");
- expect(readMediaFileUrl).toHaveBeenCalledTimes(3);
+ view.unmount(); expect(incoming.pause).toHaveBeenCalled(); expect(incoming).not.toHaveAttribute('src');
+});
+
+it("uses the first playback frame as the poster and keeps playback buttons separate from opening scenes", async () => {
+ const config = project(), job = createDraftGenerationJob('Dance', {id: 'clip'});
+ config.generationJobs = [job]; config.videos = [{...newVideo(null), title: 'Film', sceneIds: [job.id]}];
+ config.assets = [{id:generationAssetId(job.id),kind:'generated',name:'Film',relativePath:'media/film.mp4',mimeType:'video/mp4',createdAt:config.createdAt,durationMs:10000,sceneSegments:[{sceneId:job.id,startFrame:96,frameCount:96,latentRelativePath:'latents/clip.safetensors'}]}];
+ const view = mount(config);
+ await screen.findByRole('img', {name: 'First frame of Film'});
+ expect(shotPoster).toHaveBeenCalledWith('D:/Project','media/film.mp4',4);
+ expect(screen.queryByRole('button',{name:'Open scenes'})).not.toBeInTheDocument();
+ fireEvent.click(screen.getByRole('button',{name:'Pop out Film'})); expect(screen.getByRole('dialog')).toBeInTheDocument(); expect(screen.queryByLabelText('Scenes editor')).not.toBeInTheDocument();
+ fireEvent.click(screen.getByRole('button',{name:'Close video preview'}));
+ fireEvent.click(screen.getByRole('button',{name:'Open Film'}).closest('[role="listitem"]')!); expect(screen.getByLabelText('Scenes editor')).toBeInTheDocument();
+ expect(view.config().timeline).toEqual(config.timeline);
});
diff --git a/src/components/workspace/VideoLibraryView.tsx b/src/components/workspace/VideoLibraryView.tsx
index 8822656..82c5f53 100644
--- a/src/components/workspace/VideoLibraryView.tsx
+++ b/src/components/workspace/VideoLibraryView.tsx
@@ -1,4 +1,4 @@
-import { useEffect, useRef, useState, type ComponentProps, type DragEvent } from "react";
+import { useEffect, useRef, useState, type ComponentProps, type CSSProperties, type DragEvent } from "react";
import { referenceTypeLabel, sceneDurationSeconds, type ProjectConfig, type ProjectVideo, type VideoSource } from "../../lib/project";
import { citableReferences } from "./SceneEditor";
import { referenceMediaTypes } from "../../lib/referenceSelection";
@@ -9,6 +9,7 @@ import { CommandBarButton, ComboBox, PropRow, PropSection, Splitter, useContextM
import { Add16, Delete16, Film32, FolderAdd16, FolderOpen16, GridView16, ListView16, OpenExternal16, Play16 } from "../ui/icons";
import { GeneratorView } from "./GeneratorView";
import { VideoPlayback, videoPlaylist } from "./VideoPlayback";
+import { VideoPoster } from "./VideoPoster";
import { PromptTextField } from "./PromptTextField";
const VIDEO_DRAG = "application/x-slopus-video";
@@ -105,15 +106,17 @@ export function VideoLibraryView(props: ComponentProps<typeof GeneratorView> & {
<button type="button" className="music-song__select" aria-label={`Open ${folder.name}`} onClick={() => browse(folder.id)}><span className="music-song__art"><FolderOpen16 /></span><span className="music-song__info"><strong>{folder.name}</strong></span></button>
</div>)}
{visible.map((video) => {
- const scenes = videoScenes(video, config), playable = videoPlaylist(video, config).length;
+ const scenes = videoScenes(video, config), playlist = videoPlaylist(video, config), playable = playlist.length, first = playlist[0];
+ const [width, height] = config.settings.aspectRatio.split(":").map(Number);
+ const aspect = first?.asset.width && first.asset.height ? first.asset.width / first.asset.height : width / height;
const busy = scenes.some((scene) => ["queued", "generating"].includes(scene.status));
- return <div key={video.id} role="listitem" className={`music-song video-card${selected?.id === video.id ? " music-song--selected" : ""}${playable ? "" : " music-song--draft"}`} draggable onDragStart={(event) => { event.dataTransfer.effectAllowed = "move"; event.dataTransfer.setData(VIDEO_DRAG, JSON.stringify({ projectId: config.id, videoId: video.id })); }}
- onContextMenu={(event) => menu.open(event, [{ label: "Open scenes", onSelect: () => openVideo(video) }, ...newItems(), { separator: true }, { label: "Remove video", icon: <Delete16 />, disabled: busy, danger: true, onSelect: () => { void remove(video); } }])}>
- <button type="button" className="music-song__select" aria-label={`Select ${video.title}`} aria-pressed={selected?.id === video.id} onClick={() => setSelectedId(video.id)} onDoubleClick={() => openVideo(video)}>
- <span className="music-song__art"><Film32 /></span><span className="music-song__info"><strong>{video.title}</strong><span>{scenes.length} scenes · {scenes.reduce((seconds, scene) => seconds + sceneDurationSeconds(scene), 0).toFixed(1)} s</span><span>{busy ? "Generating" : playable ? `${playable} / ${scenes.length} scenes ready` : "Draft"}</span></span>
+ return <div key={video.id} role="listitem" style={{ "--video-card-aspect": aspect } as CSSProperties} className={`music-song video-card${selected?.id === video.id ? " music-song--selected" : ""}${playable ? "" : " music-song--draft"}`} draggable onDragStart={(event) => { event.dataTransfer.effectAllowed = "move"; event.dataTransfer.setData(VIDEO_DRAG, JSON.stringify({ projectId: config.id, videoId: video.id })); }}
+ onClick={(event) => { if (!(event.target as Element).closest("button, .video-card__player")) openVideo(video); }}
+ onContextMenu={(event) => { setSelectedId(video.id); menu.open(event, [{ label: "Open scenes", onSelect: () => openVideo(video) }, { label: "Video settings", onSelect: () => setSelectedId(video.id) }, ...newItems(), { separator: true }, { label: "Remove video", icon: <Delete16 />, disabled: busy, danger: true, onSelect: () => { void remove(video); } }]); }}>
+ <button type="button" className="music-song__select" aria-label={`Open ${video.title}`} onFocus={() => setSelectedId(video.id)} onClick={() => openVideo(video)}>
+ <span className="music-song__art"><VideoPoster folderPath={folderPath} relativePath={first?.asset.relativePath} seconds={first?.start ?? 0} revision={props.generationCompletionTimes?.[first?.sceneId ?? ""] ?? first?.asset.createdAt} title={video.title} /></span><span className="music-song__info"><strong>{video.title}</strong><span>{scenes.length} scenes · {scenes.reduce((seconds, scene) => seconds + sceneDurationSeconds(scene), 0).toFixed(1)} s</span><span>{busy ? "Generating" : playable ? `${playable} / ${scenes.length} scenes ready` : "Draft"}</span></span>
</button>
{playable > 0 && <div className="video-card__controls"><button type="button" aria-label={`Play ${video.title}`} onClick={() => setPlayback({ id: video.id, popout: false })}><Play16 /></button><button type="button" aria-label={`Pop out ${video.title}`} onClick={() => setPlayback({ id: video.id, popout: true })}><OpenExternal16 /></button></div>}
- <button type="button" className="video-card__open secondary-button" onClick={() => openVideo(video)}>Open scenes</button>
{playing?.id === video.id && !playback?.popout && <VideoPlayback key={video.id} video={video} config={config} folderPath={folderPath} popout={false} onClose={() => setPlayback(undefined)} />}
</div>;
})}
@@ -137,7 +140,7 @@ export function VideoLibraryView(props: ComponentProps<typeof GeneratorView> & {
update((current) => ({ ...current, generationJobs: [...current.generationJobs, ...scenes], videos: current.videos!.map((video) => video.id === selected.id ? { ...video, sceneIds: scenes.map((scene) => scene.id) } : video) })); openVideo(selected);
}}>Create scenes for song</button>}
</PropSection>
- <PropSection title="Scenes"><button type="button" className="primary-button" onClick={() => openVideo(selected)}>Open scenes</button>
+ <PropSection title="Scenes">
<p className="video-reference-help">{videoScenes(selected, config).map((scene, index, all) => { const start = all.slice(0, index).reduce((sum, item) => sum + sceneDurationSeconds(item), 0); return <span className="video-reference-range" key={scene.id}>{scene.title}: {start.toFixed(1)}–{(start + sceneDurationSeconds(scene)).toFixed(1)} s</span>; })}</p>
</PropSection>
</> : <p className="music-inspector__empty">Select a video to edit its settings.</p>}</div></aside>
diff --git a/src/components/workspace/VideoPlayback.tsx b/src/components/workspace/VideoPlayback.tsx
index f6ff293..12c3529 100644
--- a/src/components/workspace/VideoPlayback.tsx
+++ b/src/components/workspace/VideoPlayback.tsx
@@ -1,62 +1,23 @@
-import { useEffect, useRef, useState, type CSSProperties } from "react";
-import { readMediaFileUrl } from "../../lib/persistence";
-import { sceneDurationSeconds, type ProjectConfig, type ProjectVideo } from "../../lib/project";
-import { scenePreviewAsset } from "../../lib/sceneHistory";
-import { sceneMediaStartSeconds, sceneMediaDurationMs } from "../../lib/continuationMedia";
-import { videoScenes } from "../../lib/videoLibrary";
+import { useEffect, useMemo, useRef, useState, type CSSProperties } from "react";
+import type { ProjectConfig, ProjectVideo } from "../../lib/project";
+import { outputDimensions } from "../../lib/export";
+import { videoPlaybackConfig, videoPlaylist } from "../../lib/videoPlayback";
+import { ProgramMonitor } from "./ProgramMonitor";
import { Dismiss16, Pause20, Play20Filled } from "../ui/icons";
+export { videoPlaylist } from "../../lib/videoPlayback";
-export function videoPlaylist(video: ProjectVideo, config: ProjectConfig) {
- return videoScenes(video, config).flatMap((scene) => {
- const asset = scenePreviewAsset(scene, config.assets);
- if (!asset?.relativePath) return [];
- const start = sceneMediaStartSeconds(asset);
- const duration = Math.min(sceneDurationSeconds(scene), (sceneMediaDurationMs(asset) ?? sceneDurationSeconds(scene) * 1000) / 1000);
- return [{ sceneId: scene.id, title: scene.title, asset, start, end: start + duration }];
- });
-}
-
-/** Plays each scene's own range, excluding retained continuation prefixes. */
+/** Timeline's continuous clock, prepared players and persistent compositor. */
export function VideoPlayback({ video: item, config, folderPath, popout, onClose }: {
video: ProjectVideo; config: ProjectConfig; folderPath: string; popout: boolean; onClose: () => void;
}) {
- const playlist = videoPlaylist(item, config);
- const [index, setIndex] = useState(0), [url, setUrl] = useState<string | null>(null);
- const [playing, setPlaying] = useState(false), [ended, setEnded] = useState(false), [error, setError] = useState<string | null>(null);
- const [aspect, setAspect] = useState(16 / 9);
- const media = useRef<HTMLVideoElement>(null), frame = useRef<HTMLDivElement>(null);
- const close = useRef(onClose); close.current = onClose;
- const timer = useRef<number>(), moving = useRef(false), epoch = useRef(0);
- const current = playlist[index];
- const advance = () => {
- if (moving.current) return;
- moving.current = true; clearTimeout(timer.current); media.current?.pause();
- if (index + 1 < playlist.length) setIndex(index + 1);
- else { setEnded(true); setPlaying(false); }
- };
- const play = async (restart = false) => {
- const player = media.current;
- if (!player || !current || !url) return;
- if (restart || player.currentTime < current.start || player.currentTime >= current.end - 0.02) player.currentTime = current.start;
- moving.current = false; setEnded(false); setError(null); clearTimeout(timer.current);
- const token = ++epoch.current;
- try {
- await player.play();
- if (token !== epoch.current) return;
- setPlaying(true);
- timer.current = window.setTimeout(advance, Math.max(0, (Math.min(current.end, player.duration || current.end) - player.currentTime) * 1000));
- } catch { if (token === epoch.current) { setPlaying(false); setError("Could not play this scene. Try replaying it."); } }
- };
- useEffect(() => {
- let disposed = false, owned: string | null = null;
- epoch.current++; moving.current = false; setUrl(null); setError(null); setPlaying(false);
- const player = media.current; player?.pause(); clearTimeout(timer.current);
- if (current) void readMediaFileUrl(folderPath, current.asset, "video/mp4").then((value) => {
- if (disposed) { if (value) URL.revokeObjectURL(value); return; }
- owned = value; setUrl(value); if (!value) setError("This scene's video is unavailable.");
- }).catch((reason) => { if (!disposed) setError(String(reason)); });
- return () => { disposed = true; epoch.current++; clearTimeout(timer.current); player?.pause(); if (owned) URL.revokeObjectURL(owned); };
- }, [folderPath, current?.sceneId, current?.asset.relativePath, current?.start, current?.end]);
+ const preview = useMemo(() => videoPlaybackConfig(item, config), [item, config]);
+ const playlist = useMemo(() => videoPlaylist(item, config), [item, config]);
+ const [playhead, setPlayhead] = useState(0), [playing, setPlaying] = useState(true);
+ const frame = useRef<HTMLDivElement>(null), close = useRef(onClose); close.current = onClose;
+ const duration = preview.timeline.tracks[0].clips.reduce((end, clip) => Math.max(end, clip.startMs + clip.durationMs), 0);
+ const index = Math.max(0, playlist.findIndex((clip) => playhead < clip.startMs + clip.durationMs));
+ const current = playlist[playhead >= duration ? playlist.length - 1 : index];
+ const dimensions = outputDimensions(config.settings.resolution, config.settings.aspectRatio);
useEffect(() => {
const focused = document.activeElement as HTMLElement | null;
const library = popout ? frame.current?.closest(".video-library-stage")?.querySelector<HTMLElement>(".music-library") : null;
@@ -74,17 +35,14 @@ export function VideoPlayback({ video: item, config, folderPath, popout, onClose
};
}, [popout]);
const player = <div className={popout ? "shot-popout__frame" : "video-card__player"} ref={frame} tabIndex={-1}
- role={popout ? "dialog" : "group"} aria-label={`${item.title} preview`} style={{ "--shot-aspect": aspect } as CSSProperties}>
- <video ref={media} src={url ?? undefined} playsInline onLoadedMetadata={(event) => { if (url) { setAspect(event.currentTarget.videoWidth / event.currentTarget.videoHeight || 16 / 9); void play(true); } }}
- onTimeUpdate={(event) => { if (playing && current && event.currentTarget.currentTime >= current.end - 0.02) advance(); }} onEnded={advance}
- onError={() => { if (url) { clearTimeout(timer.current); setPlaying(false); setError("This scene could not be played."); } }} />
+ role={popout ? "dialog" : "group"} aria-label={item.title + " preview"} style={{ "--shot-aspect": dimensions.width / dimensions.height } as CSSProperties}>
+ <ProgramMonitor config={preview} folderPath={folderPath} playheadMs={Math.min(playhead, Math.max(0, duration - 1))} playing={playing} waitForInitialFrame onSeek={setPlayhead} onPlayingChange={setPlaying} />
<button type="button" className="shot-popout__close" aria-label="Close video preview" onClick={onClose}><Dismiss16 /></button>
- <button type="button" className="shot-popout__play" aria-label={`${playing ? "Pause" : "Replay"} ${item.title}`} disabled={!url} onClick={() => {
- if (playing) { epoch.current++; clearTimeout(timer.current); media.current?.pause(); setPlaying(false); }
- else if (ended && index !== 0) { setEnded(false); setIndex(0); } else void play(ended);
+ <button type="button" className="shot-popout__play" aria-label={(playing ? "Pause " : "Replay ") + item.title} disabled={!duration} onClick={() => {
+ if (playing) setPlaying(false);
+ else { if (playhead >= duration) setPlayhead(0); setPlaying(true); }
}}>{playing ? <Pause20 /> : <Play20Filled />}</button>
- {error && <p className="shot-popout__message" role="alert">{error}</p>}
- <span className="video-card__position">{index + 1} / {playlist.length} · {current?.title}</span>
+ <span className="video-card__position">{playhead >= duration ? playlist.length : index + 1} / {playlist.length} · {current?.title}</span>
</div>;
return popout ? <div className="shot-popout">{player}</div> : player;
}
diff --git a/src/components/workspace/VideoPoster.tsx b/src/components/workspace/VideoPoster.tsx
new file mode 100644
index 0000000..508524a
--- /dev/null
+++ b/src/components/workspace/VideoPoster.tsx
@@ -0,0 +1,21 @@
+import { useEffect, useRef, useState } from "react";
+import { forgetShotPosters, shotPoster } from "./ShotThumbnail";
+import { Film32 } from "../ui/icons";
+
+export function VideoPoster({ folderPath, relativePath, seconds, revision, title }: {
+ folderPath: string; relativePath?: string | null; seconds: number; revision?: string | number; title: string;
+}) {
+ const key = `${folderPath}:${relativePath}:${seconds}:${revision}`;
+ const [result, setResult] = useState<{ key: string; poster: string | null }>();
+ const previous = useRef<{ path: string; revision?: string | number }>();
+ useEffect(() => {
+ if (!relativePath) return;
+ if (previous.current?.path === relativePath && previous.current.revision !== revision) forgetShotPosters(folderPath, relativePath);
+ previous.current = { path: relativePath, revision };
+ let live = true;
+ void shotPoster(folderPath, relativePath, seconds).then(({ poster }) => { if (live) setResult({ key, poster }); });
+ return () => { live = false; };
+ }, [key]);
+ return result?.key === key && result.poster
+ ? <img src={result.poster} alt={`First frame of ${title}`} draggable={false} /> : <Film32 aria-hidden="true" />;
+}
diff --git a/src/lib/videoLibrary.test.ts b/src/lib/videoLibrary.test.ts
index 4418364..a64510f 100644
--- a/src/lib/videoLibrary.test.ts
+++ b/src/lib/videoLibrary.test.ts
@@ -4,6 +4,9 @@ import { mergeVideoScenes, newVideo, resolveVideoSource, songScenes, videoGuided
import { sceneGenerationRequest, sendBlocker } from "./sceneGeneration";
import { videoPlaylist } from "../components/workspace/VideoPlayback";
import { longShotBlocker, longShotInputs } from "./longShot";
+import { videoPlaybackConfig } from "./videoPlayback";
+import { continuationPlaybackTracks } from "./continuationMedia";
+import { previewPlayerKeys } from "./timelinePreview";
const project = () => createProjectConfig({ name: "Videos", prompt: "", resolution: "416p", aspectRatio: "16:9", targetDurationSeconds: 30 });
const scene = (id: string, seconds = 10) => createDraftGenerationJob("A dancer moves", { id, sceneType: "prompt", durationSeconds: seconds });
@@ -91,3 +94,18 @@ it("plays scenes in video order, skips drafts and excludes joined continuation p
config.assets = [asset(generationAssetId("first"), "generated", 10250), { ...asset(generationAssetId("last"), "generated", 19000), sceneSegments: [{ sceneId: "last", startFrame: 240, frameCount: 216, latentRelativePath: "latents/last.safetensors" }] }];
expect(videoPlaylist(video, config).map(({ sceneId, start, end }) => ({ sceneId, start, end }))).toEqual([{ sceneId: "last", start: 10, end: 18 }, { sceneId: "first", start: 0, end: 10 }]);
});
+
+it("uses the latest joined decode for both the opening poster and uninterrupted playback across scene boundaries", () => {
+ const config = project();
+ const segments = ["a", "b"].map((sceneId, index) => ({ sceneId, startFrame: index * 120, frameCount: 120, latentRelativePath: `latents/${sceneId}.safetensors` }));
+ config.generationJobs = segments.map((part) => ({ ...scene(part.sceneId, 5), latentRelativePath: part.latentRelativePath }));
+ config.assets = segments.map((part, index) => ({ ...asset(generationAssetId(part.sceneId), "generated", (index + 1) * 5000), sceneSegments: segments.slice(0, index + 1) }));
+ const video = { ...newVideo(null), sceneIds: ["a", "b"] }, original = structuredClone(config);
+ const playlist = videoPlaylist(video, config);
+ expect(playlist.map((entry) => entry.asset.id)).toEqual([generationAssetId("b"), generationAssetId("b")]);
+ expect(playlist.map((entry) => [entry.start, entry.end])).toEqual([[0, 5], [5, 10]]);
+ const preview = videoPlaybackConfig(video, config), tracks = continuationPlaybackTracks(preview);
+ const keys = previewPlayerKeys(tracks, new Map(preview.assets.map((asset) => [asset.id, asset])));
+ expect(keys.get("video-preview-a")).toBe(keys.get("video-preview-b"));
+ expect(config).toEqual(original);
+});
diff --git a/src/lib/videoPlayback.ts b/src/lib/videoPlayback.ts
new file mode 100644
index 0000000..8fd0d2a
--- /dev/null
+++ b/src/lib/videoPlayback.ts
@@ -0,0 +1,38 @@
+import { generationAssetId, sceneDurationSeconds, type ProjectConfig, type ProjectVideo, type TimelineClip } from "./project";
+import { scenePreviewAsset, scenePreviewJob } from "./sceneHistory";
+import { continuationPlaybackTracks, sceneMediaDurationMs } from "./continuationMedia";
+import { videoScenes } from "./videoLibrary";
+
+/** A temporary timeline for the Video library. Never edits the project's timeline. */
+export function videoPlaybackConfig(video: ProjectVideo, config: ProjectConfig): ProjectConfig {
+ const members = videoScenes(video, config);
+ const assets = new Map(config.assets.map((asset) => [asset.id, asset]));
+ const clips: TimelineClip[] = [];
+ let startMs = 0;
+ for (const scene of members) {
+ const selected = scenePreviewAsset(scene, config.assets);
+ if (!selected?.relativePath) continue;
+ const asset = { ...selected, id: generationAssetId(scene.id) };
+ assets.set(asset.id, asset);
+ const durationMs = Math.round(Math.min(sceneDurationSeconds(scene) * 1000, sceneMediaDurationMs(asset) ?? Infinity));
+ if (durationMs <= 0) continue;
+ clips.push({ id: `video-preview-${scene.id}`, assetId: asset.id, trackId: "video-preview", startMs, durationMs,
+ sourceStartMs: 0, label: scene.title, status: "generated" });
+ startMs += durationMs;
+ }
+ const memberIds = new Set(video.sceneIds);
+ return { ...config, assets: [...assets.values()],
+ generationJobs: config.generationJobs.map((scene) => memberIds.has(scene.id) ? scenePreviewJob(scene, config.assets) : scene),
+ timeline: { tracks: [{ id: "video-preview", kind: "video", name: video.title, locked: false, muted: false, clips }] } };
+}
+
+/** Use exactly the resolved media/ranges that Timeline's monitor will display. */
+export function videoPlaylist(video: ProjectVideo, config: ProjectConfig) {
+ const preview = videoPlaybackConfig(video, config);
+ return continuationPlaybackTracks(preview)[0].clips.map((clip) => ({
+ sceneId: clip.id.slice("video-preview-".length), title: clip.label,
+ asset: preview.assets.find((asset) => asset.id === clip.assetId)!,
+ start: clip.sourceStartMs / 1000, end: (clip.sourceStartMs + clip.durationMs) / 1000,
+ startMs: clip.startMs, durationMs: clip.durationMs,
+ }));
+}
diff --git a/src/styles/video-library.css b/src/styles/video-library.css
index c08df35..c0ac37b 100644
--- a/src/styles/video-library.css
+++ b/src/styles/video-library.css
@@ -10,17 +10,19 @@
.video-scenes__path strong { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.video-scenes__path span, .video-reference-help { color: var(--text-secondary); font-size: var(--text-xs); }
.video-reference-range { display: block; margin-block: var(--space-1); }
-.video-card { overflow: hidden; }
-.video-card > .music-song__select { height: auto; padding-bottom: 44px; }
-.video-card .music-song__art { aspect-ratio: 16 / 9; }
+.video-library-stage .music-songs--grid { grid-template-columns: repeat(auto-fill, minmax(0, min(300px, 100%))); gap: var(--space-2); }
+.video-card { overflow: hidden; container-type: inline-size; border-radius: var(--radius-sm); }
+.video-card > .music-song__select { height: auto; padding: 6px; }
+.video-card .music-song__art { aspect-ratio: var(--video-card-aspect, 16 / 9); overflow: hidden; }
+.video-card .music-song__art img { display: block; width: 100%; height: 100%; object-fit: contain; }
.video-library-stage .music-folder .music-song__art { aspect-ratio: 16 / 9; }
-.video-card__open { position: absolute; bottom: var(--space-2); right: var(--space-2); }
-.video-card__controls { position: absolute; top: 30%; left: 50%; transform: translate(-50%, -50%); display: flex; gap: var(--space-2); }
+.video-card__controls { position: absolute; top: calc((100cqw - 12px) / var(--video-card-aspect, 1.7777778) / 2 + 6px); left: 50%; transform: translate(-50%, -50%); display: flex; gap: var(--space-2); }
.video-card__controls button { display: grid; place-items: center; width: 32px; height: 32px; border: 1px solid #ffffff80; border-radius: 50%; background: #15191fe6; color: #fff; box-shadow: 0 1px 6px #0008; }
.video-card__controls button:hover { background: #363d48; }
-.video-card__player { position: absolute; inset: 0; background: #000; z-index: 2; }
+.video-card__player { position: absolute; top: 6px; left: 6px; right: 6px; aspect-ratio: var(--video-card-aspect, 16 / 9); background: #000; z-index: 2; }
.video-card__player video { width: 100%; height: 100%; object-fit: contain; }
.video-card__position { position: absolute; left: var(--space-2); top: var(--space-2); color: #fff; background: #000b; padding: 2px 5px; font-size: var(--text-xs); pointer-events: none; max-width: 70%; }
-.music-songs--list .video-card > .music-song__select { padding-bottom: var(--space-3); padding-right: 190px; min-height: 70px; }
-.music-songs--list .video-card__controls { top: 50%; left: auto; right: 100px; transform: translateY(-50%); }
-.music-songs--list .video-card__open { bottom: 50%; transform: translateY(50%); }
+.music-songs--list .video-card > .music-song__select { padding-right: 100px; min-height: 82px; }
+.music-songs--list .video-card .music-song__art { width: 120px; height: 67.5px; }
+.music-songs--list .video-card__controls { top: 50%; left: auto; right: 12px; transform: translateY(-50%); }
+.music-songs--list .video-card__player { inset: 0; aspect-ratio: auto; }