AI media harness for planning, generating, and editing locally
TypeScript
Use Timeline playback and larger thumbnail Video cards
Bitti181ec9dae065
diff --git a/src/components/workspace/ProgramMonitor.tsx b/src/components/workspace/ProgramMonitor.tsxindex f075640..ea911f1 100644--- a/src/components/workspace/ProgramMonitor.tsx+++ b/src/components/workspace/ProgramMonitor.tsx@@ -52,7 +52,9 @@ const clamp = (value: number, minimum: number, maximum: number) => Math.max(miniconst angleAt = (x: number, y: number, centerX: number, centerY: number) => Math.atan2(y - centerY, x - centerX) * 180 / Math.PI;const normalizedAngle = (value: number) => ((value + 180) % 360 + 360) % 360 - 180;-export function ProgramMonitor({ config, folderPath, playheadMs, playing, rate = 1, onSeek, onPlayingChange, selectedClipId = null, transformEditingDisabled = false, onSelectClip, onTransformChange, selectedMotionNodeId, onSelectMotionNode, onMotionChange }: {+export function ProgramMonitor({ config, folderPath, playheadMs, playing, rate = 1, onSeek, onPlayingChange, waitForInitialFrame = false, selectedClipId = null, transformEditingDisabled = false, onSelectClip, onTransformChange, selectedMotionNodeId, onSelectMotionNode, onMotionChange }: {+ /** Card/popout autoplay waits for its opening picture before advancing the clock. */+ waitForInitialFrame?: boolean;selectedMotionNodeId?: string | null;onSelectMotionNode?: (id: string) => void;onMotionChange?: (motion: Motion) => void;@@ -80,7 +82,7 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, rate =const receiveMotionFrame = useCallback((id: string, nodes: MotionNode[]) => {if (id === selectedClipId && !playing) setMotionFrame({ id, nodes });}, [selectedClipId, playing]);- const tracks = useMemo(() => continuationPlaybackTracks(config), [config.timeline.tracks, config.assets, config.generationJobs]);+ const tracks = useMemo(() => continuationPlaybackTracks(config), [config.timeline.tracks, config.assets, config.generationJobs, config.videos]);const assetsById = useMemo(() => new Map(config.assets.map((candidate) => [candidate.id, candidate])), [config.assets]);const segments = useMemo(() => previewSegments(tracks, assetsById), [tracks, assetsById]);const segmentIndex = previewSegmentIndex(segments, playheadMs);@@ -172,6 +174,7 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, rate =const callbacks = useRef({ onSeek, onPlayingChange });callbacks.current = { onSeek, onPlayingChange };+ const currentPicture = useRef({ layers, playerKeys }); currentPicture.current = { layers, playerKeys };const report = useCallback((ms: number) => {const rounded = Math.round(ms);/* Keep the local clock advancing even when React batches visual updates@@ -208,9 +211,19 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, rate =useEffect(() => {if (!playing) return;let frame = 0;+ let started = !waitForInitialFrame;let lastTick = performance.now();const step = () => {const now = performance.now();+ if (!started) {+ started = currentPicture.current.layers.every((layer) => {+ const element = media.get(currentPicture.current.playerKeys.get(layer.id) ?? layer.id);+ return element instanceof HTMLVideoElement ? element.readyState >= 2 && !element.seeking+ : element instanceof HTMLImageElement ? element.complete && element.naturalWidth > 0 : Boolean(element);+ });+ lastTick = now;+ frame = requestAnimationFrame(step); return;+ }const elapsed = now - lastTick;lastTick = now;/* Media currentTime can be exposed in coarse jumps while Chromium is@@ -237,7 +250,7 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, rate =};frame = requestAnimationFrame(step);return () => cancelAnimationFrame(frame);- }, [playing, rate, contentEndMs, report]);+ }, [playing, rate, contentEndMs, report, waitForInitialFrame, media]);/* The status bar's playback rate: counted by the picture on every tick,published once a second, never through this component's state. */useEffect(() => {diff --git a/src/components/workspace/VideoLibraryView.test.tsx b/src/components/workspace/VideoLibraryView.test.tsxindex b30f5e9..1832caa 100644--- a/src/components/workspace/VideoLibraryView.test.tsx+++ b/src/components/workspace/VideoLibraryView.test.tsx@@ -5,12 +5,13 @@ import { useState } from "react";import { afterEach, beforeEach, expect, it, vi } from "vitest";import { createDraftGenerationJob, createProjectConfig, generationAssetId, type ProjectConfig } from "../../lib/project";import { newVideo } from "../../lib/videoLibrary";-import { readMediaFileUrl } from "../../lib/persistence";+import { PreviewSources } from "../../lib/exportPipeline";+import { shotPoster } from "./ShotThumbnail";import { VideoLibraryView } from "./VideoLibraryView";import { VideoPlayback } from "./VideoPlayback";import { choose } from "./comboTestUtils";-vi.mock("../../lib/persistence", async (original) => ({ ...await original<typeof import("../../lib/persistence")>(), readMediaFileUrl: vi.fn(async (_, item) => `blob:${item.relativePath}`) }));+vi.mock("./ShotThumbnail", async (original) => ({ ...await original<typeof import("./ShotThumbnail")>(), shotPoster: vi.fn(async () => ({ poster: "data:image/jpeg;base64,preview", failed: false })) }));vi.mock("../../lib/nativeShell", async (original) => ({ ...await original<typeof import("../../lib/nativeShell")>(), askNative: vi.fn(async () => true) }));vi.mock("./GeneratorView", () => ({ GeneratorView: ({ config, sceneIds, onChange }: { config: ProjectConfig; sceneIds: string[]; onChange: (next: ProjectConfig) => void }) => <div aria-label="Scenes editor"><span>{sceneIds.join(",")}</span><span data-testid="all-scenes">{config.generationJobs.length}</span>@@ -19,10 +20,11 @@ vi.mock("./GeneratorView", () => ({ GeneratorView: ({ config, sceneIds, onChangebeforeEach(() => {localStorage.clear();+ vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => undefined);vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue(); vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => undefined);vi.stubGlobal("URL", Object.assign(URL, { revokeObjectURL: vi.fn() }));});-afterEach(() => { cleanup(); vi.restoreAllMocks(); vi.clearAllMocks(); vi.unstubAllGlobals(); });+afterEach(() => { cleanup(); vi.restoreAllMocks(); vi.clearAllMocks(); vi.unstubAllGlobals(); delete (window as unknown as Record<string, unknown>).__TAURI_INTERNALS__; });const project = () => createProjectConfig({ name: "Library", prompt: "", resolution: "416p", aspectRatio: "16:9", targetDurationSeconds: 30 });function mount(config = project(), openScene?: { id: string }) {let latest = config;@@ -37,7 +39,7 @@ it("creates nested folders and videos, moves cards to a folder, and opens only tconst video = view.config().videos!.find((video) => video.title === "New video")!;expect(video.folderId).toBe(view.config().videoFolders![0].id);fireEvent.change(screen.getByRole("textbox", { name: "Video title" }), { target: { value: "Track one" } });- fireEvent.doubleClick(screen.getByRole("button", { name: "Select Track one" }));+ fireEvent.click(screen.getByRole("button", { name: "Open Track one" }));expect(screen.getByTestId("all-scenes")).toHaveTextContent("1"); fireEvent.click(screen.getByRole("button", { name: "Add scene" }));expect(view.config().videos!.find((item) => item.id === video.id)?.sceneIds).toEqual(["added"]);expect(view.config().generationJobs.map((scene) => scene.id)).toEqual(["legacy", "added"]);@@ -47,7 +49,7 @@ it("creates nested folders and videos, moves cards to a folder, and opens only tfireEvent.drop(screen.getByRole("button", { name: "Open Album" }).closest('[role="listitem"]')!, { dataTransfer });expect(view.config().videos!.find((item) => item.id === "video-default")?.folderId).toBe(video.folderId);fireEvent.click(screen.getByRole("button", { name: "List view" })); fireEvent.click(screen.getByRole("button", { name: "Open Album" }));- expect(screen.getByRole("button", { name: "Select Track one" })).toBeInTheDocument(); expect(screen.getByRole("button", { name: "Select Library" })).toBeInTheDocument();+ expect(screen.getByRole("button", { name: "Open Track one" })).toBeInTheDocument(); expect(screen.getByRole("button", { name: "Open Library" })).toBeInTheDocument();});it("creates a whole-song scene sequence and opens the owner for a timeline navigation", () => {@@ -62,27 +64,50 @@ it("creates a whole-song scene sequence and opens the owner for a timeline navigit("removes folders without deleting their videos and removes video scenes without removing timeline media", async () => {const config = project(); config.videoFolders = [{ id: "folder", name: "Collection" }]; config.videos = [{ ...newVideo("folder"), title: "Film", sceneIds: ["a"] }]; config.generationJobs = [createDraftGenerationJob("A", { id: "a" })];const view = mount(config); fireEvent.contextMenu(screen.getByRole("button", { name: "Open Collection" })); fireEvent.click(screen.getByRole("menuitem", { name: "Remove folder" }));- expect(view.config().videoFolders).toHaveLength(0); expect(screen.getByRole("button", { name: "Select Film" })).toBeInTheDocument();- fireEvent.contextMenu(screen.getByRole("button", { name: "Select Film" })); fireEvent.click(screen.getByRole("menuitem", { name: "Remove video" }));+ expect(view.config().videoFolders).toHaveLength(0); expect(screen.getByRole("button", { name: "Open Film" })).toBeInTheDocument();+ fireEvent.contextMenu(screen.getByRole("button", { name: "Open Film" })); fireEvent.click(screen.getByRole("menuitem", { name: "Remove video" }));await waitFor(() => expect(view.config().videos).toHaveLength(0)); expect(view.config().generationJobs).toHaveLength(0); expect(view.config().timeline).toEqual(config.timeline);});-it("plays successive scene ranges, replays the full video, closes outside the popout and releases media", async () => {+it("prepares later clips before playback, retains decoders across cuts, replays and closes the popout", async () => {+ (window as unknown as Record<string, unknown>).__TAURI_INTERNALS__ = {};+ vi.spyOn(PreviewSources.prototype, "url").mockImplementation(async (asset) => 'blob:' + asset.relativePath);+ const ready = vi.spyOn(HTMLMediaElement.prototype, "readyState", "get").mockReturnValue(0);+ let now = 0, nextId = 0;+ const frames = new Map<number, FrameRequestCallback>();+ vi.spyOn(performance, "now").mockImplementation(() => now);+ vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => { frames.set(++nextId, callback); return nextId; });+ vi.stubGlobal("cancelAnimationFrame", (id: number) => frames.delete(id));+ const tick = async (ms: number) => { await act(async () => { now += ms; const callbacks = [...frames.values()]; frames.clear(); callbacks.forEach((callback) => callback(now)); }); };const config = project(); config.generationJobs = ["a", "b"].map((id) => createDraftGenerationJob("Dance", { id, durationSeconds: 3 }));- config.assets = config.generationJobs.map((job, index) => ({ id: generationAssetId(job.id), kind: "generated", name: job.id, relativePath: `media/${job.id}.mp4`, mimeType: "video/mp4", createdAt: config.createdAt, durationMs: 10000,- sceneSegments: [{ sceneId: job.id, startFrame: index * 96, frameCount: 96, latentRelativePath: `latents/${job.id}.safetensors` }] }));+ config.assets = config.generationJobs.map((job, index) => ({ id: generationAssetId(job.id), kind: "generated", name: job.id, relativePath: 'media/' + job.id + '.mp4', mimeType: "video/mp4", createdAt: config.createdAt, durationMs: 10000,+ sceneSegments: [{ sceneId: job.id, startFrame: index * 96, frameCount: 96, latentRelativePath: 'latents/' + job.id + '.safetensors' }] }));const video = { ...newVideo(null), title: "Film", sceneIds: ["a", "b"] }, close = vi.fn();const view = render(<VideoPlayback video={video} config={config} folderPath="D:/Project" popout onClose={close} />);- const player = view.container.querySelector("video")!;- await waitFor(() => expect(player).toHaveAttribute("src", "blob:media/a.mp4"));- await act(async () => { fireEvent.loadedMetadata(player); }); expect(player.currentTime).toBe(0);- player.currentTime = 3; fireEvent.timeUpdate(player);- await waitFor(() => expect(player).toHaveAttribute("src", "blob:media/b.mp4"));- await act(async () => { fireEvent.loadedMetadata(player); }); expect(player.currentTime).toBe(4);- fireEvent.ended(player); fireEvent.click(screen.getByRole("button", { name: "Replay Film" }));- await waitFor(() => expect(player).toHaveAttribute("src", "blob:media/a.mp4"));+ await waitFor(() => expect(view.container.querySelectorAll("video")).toHaveLength(2));+ const incoming = view.container.querySelector<HTMLVideoElement>('[data-clip-id="video-preview-b"] video')!;+ await tick(5000); expect(view.container.querySelector('.video-card__position')).toHaveTextContent('1 / 2');+ ready.mockReturnValue(4); fireEvent.loadedData(incoming); expect(incoming.currentTime).toBe(4);+ await tick(16); await tick(3001);+ expect(view.container.querySelector('[data-clip-id="video-preview-b"] video')).toBe(incoming);+ expect(view.container.querySelector('.video-card__position')).toHaveTextContent('2 / 2');+ await tick(3100); fireEvent.click(screen.getByRole("button", { name: "Replay Film" }));+ expect(view.container.querySelector('.video-card__position')).toHaveTextContent('1 / 2');fireEvent.pointerDown(screen.getByRole("dialog")); expect(close).not.toHaveBeenCalled();fireEvent.pointerDown(view.container.querySelector(".shot-popout")!); expect(close).toHaveBeenCalledTimes(1);- view.unmount(); expect(URL.revokeObjectURL).toHaveBeenCalledWith("blob:media/a.mp4"); expect(URL.revokeObjectURL).toHaveBeenCalledWith("blob:media/b.mp4");- expect(readMediaFileUrl).toHaveBeenCalledTimes(3);+ view.unmount(); expect(incoming.pause).toHaveBeenCalled(); expect(incoming).not.toHaveAttribute('src');+});++it("uses the first playback frame as the poster and keeps playback buttons separate from opening scenes", async () => {+ const config = project(), job = createDraftGenerationJob('Dance', {id: 'clip'});+ config.generationJobs = [job]; config.videos = [{...newVideo(null), title: 'Film', sceneIds: [job.id]}];+ config.assets = [{id:generationAssetId(job.id),kind:'generated',name:'Film',relativePath:'media/film.mp4',mimeType:'video/mp4',createdAt:config.createdAt,durationMs:10000,sceneSegments:[{sceneId:job.id,startFrame:96,frameCount:96,latentRelativePath:'latents/clip.safetensors'}]}];+ const view = mount(config);+ await screen.findByRole('img', {name: 'First frame of Film'});+ expect(shotPoster).toHaveBeenCalledWith('D:/Project','media/film.mp4',4);+ expect(screen.queryByRole('button',{name:'Open scenes'})).not.toBeInTheDocument();+ fireEvent.click(screen.getByRole('button',{name:'Pop out Film'})); expect(screen.getByRole('dialog')).toBeInTheDocument(); expect(screen.queryByLabelText('Scenes editor')).not.toBeInTheDocument();+ fireEvent.click(screen.getByRole('button',{name:'Close video preview'}));+ fireEvent.click(screen.getByRole('button',{name:'Open Film'}).closest('[role="listitem"]')!); expect(screen.getByLabelText('Scenes editor')).toBeInTheDocument();+ expect(view.config().timeline).toEqual(config.timeline);});diff --git a/src/components/workspace/VideoLibraryView.tsx b/src/components/workspace/VideoLibraryView.tsxindex 8822656..82c5f53 100644--- a/src/components/workspace/VideoLibraryView.tsx+++ b/src/components/workspace/VideoLibraryView.tsx@@ -1,4 +1,4 @@-import { useEffect, useRef, useState, type ComponentProps, type DragEvent } from "react";+import { useEffect, useRef, useState, type ComponentProps, type CSSProperties, type DragEvent } from "react";import { referenceTypeLabel, sceneDurationSeconds, type ProjectConfig, type ProjectVideo, type VideoSource } from "../../lib/project";import { citableReferences } from "./SceneEditor";import { referenceMediaTypes } from "../../lib/referenceSelection";@@ -9,6 +9,7 @@ import { CommandBarButton, ComboBox, PropRow, PropSection, Splitter, useContextMimport { Add16, Delete16, Film32, FolderAdd16, FolderOpen16, GridView16, ListView16, OpenExternal16, Play16 } from "../ui/icons";import { GeneratorView } from "./GeneratorView";import { VideoPlayback, videoPlaylist } from "./VideoPlayback";+import { VideoPoster } from "./VideoPoster";import { PromptTextField } from "./PromptTextField";const VIDEO_DRAG = "application/x-slopus-video";@@ -105,15 +106,17 @@ export function VideoLibraryView(props: ComponentProps<typeof GeneratorView> & {<button type="button" className="music-song__select" aria-label={`Open ${folder.name}`} onClick={() => browse(folder.id)}><span className="music-song__art"><FolderOpen16 /></span><span className="music-song__info"><strong>{folder.name}</strong></span></button></div>)}{visible.map((video) => {- const scenes = videoScenes(video, config), playable = videoPlaylist(video, config).length;+ const scenes = videoScenes(video, config), playlist = videoPlaylist(video, config), playable = playlist.length, first = playlist[0];+ const [width, height] = config.settings.aspectRatio.split(":").map(Number);+ const aspect = first?.asset.width && first.asset.height ? first.asset.width / first.asset.height : width / height;const busy = scenes.some((scene) => ["queued", "generating"].includes(scene.status));- return <div key={video.id} role="listitem" className={`music-song video-card${selected?.id === video.id ? " music-song--selected" : ""}${playable ? "" : " music-song--draft"}`} draggable onDragStart={(event) => { event.dataTransfer.effectAllowed = "move"; event.dataTransfer.setData(VIDEO_DRAG, JSON.stringify({ projectId: config.id, videoId: video.id })); }}- onContextMenu={(event) => menu.open(event, [{ label: "Open scenes", onSelect: () => openVideo(video) }, ...newItems(), { separator: true }, { label: "Remove video", icon: <Delete16 />, disabled: busy, danger: true, onSelect: () => { void remove(video); } }])}>- <button type="button" className="music-song__select" aria-label={`Select ${video.title}`} aria-pressed={selected?.id === video.id} onClick={() => setSelectedId(video.id)} onDoubleClick={() => openVideo(video)}>- <span className="music-song__art"><Film32 /></span><span className="music-song__info"><strong>{video.title}</strong><span>{scenes.length} scenes · {scenes.reduce((seconds, scene) => seconds + sceneDurationSeconds(scene), 0).toFixed(1)} s</span><span>{busy ? "Generating" : playable ? `${playable} / ${scenes.length} scenes ready` : "Draft"}</span></span>+ return <div key={video.id} role="listitem" style={{ "--video-card-aspect": aspect } as CSSProperties} className={`music-song video-card${selected?.id === video.id ? " music-song--selected" : ""}${playable ? "" : " music-song--draft"}`} draggable onDragStart={(event) => { event.dataTransfer.effectAllowed = "move"; event.dataTransfer.setData(VIDEO_DRAG, JSON.stringify({ projectId: config.id, videoId: video.id })); }}+ onClick={(event) => { if (!(event.target as Element).closest("button, .video-card__player")) openVideo(video); }}+ onContextMenu={(event) => { setSelectedId(video.id); menu.open(event, [{ label: "Open scenes", onSelect: () => openVideo(video) }, { label: "Video settings", onSelect: () => setSelectedId(video.id) }, ...newItems(), { separator: true }, { label: "Remove video", icon: <Delete16 />, disabled: busy, danger: true, onSelect: () => { void remove(video); } }]); }}>+ <button type="button" className="music-song__select" aria-label={`Open ${video.title}`} onFocus={() => setSelectedId(video.id)} onClick={() => openVideo(video)}>+ <span className="music-song__art"><VideoPoster folderPath={folderPath} relativePath={first?.asset.relativePath} seconds={first?.start ?? 0} revision={props.generationCompletionTimes?.[first?.sceneId ?? ""] ?? first?.asset.createdAt} title={video.title} /></span><span className="music-song__info"><strong>{video.title}</strong><span>{scenes.length} scenes · {scenes.reduce((seconds, scene) => seconds + sceneDurationSeconds(scene), 0).toFixed(1)} s</span><span>{busy ? "Generating" : playable ? `${playable} / ${scenes.length} scenes ready` : "Draft"}</span></span></button>{playable > 0 && <div className="video-card__controls"><button type="button" aria-label={`Play ${video.title}`} onClick={() => setPlayback({ id: video.id, popout: false })}><Play16 /></button><button type="button" aria-label={`Pop out ${video.title}`} onClick={() => setPlayback({ id: video.id, popout: true })}><OpenExternal16 /></button></div>}- <button type="button" className="video-card__open secondary-button" onClick={() => openVideo(video)}>Open scenes</button>{playing?.id === video.id && !playback?.popout && <VideoPlayback key={video.id} video={video} config={config} folderPath={folderPath} popout={false} onClose={() => setPlayback(undefined)} />}</div>;})}@@ -137,7 +140,7 @@ export function VideoLibraryView(props: ComponentProps<typeof GeneratorView> & {update((current) => ({ ...current, generationJobs: [...current.generationJobs, ...scenes], videos: current.videos!.map((video) => video.id === selected.id ? { ...video, sceneIds: scenes.map((scene) => scene.id) } : video) })); openVideo(selected);}}>Create scenes for song</button>}</PropSection>- <PropSection title="Scenes"><button type="button" className="primary-button" onClick={() => openVideo(selected)}>Open scenes</button>+ <PropSection title="Scenes"><p className="video-reference-help">{videoScenes(selected, config).map((scene, index, all) => { const start = all.slice(0, index).reduce((sum, item) => sum + sceneDurationSeconds(item), 0); return <span className="video-reference-range" key={scene.id}>{scene.title}: {start.toFixed(1)}–{(start + sceneDurationSeconds(scene)).toFixed(1)} s</span>; })}</p></PropSection></> : <p className="music-inspector__empty">Select a video to edit its settings.</p>}</div></aside>diff --git a/src/components/workspace/VideoPlayback.tsx b/src/components/workspace/VideoPlayback.tsxindex f6ff293..12c3529 100644--- a/src/components/workspace/VideoPlayback.tsx+++ b/src/components/workspace/VideoPlayback.tsx@@ -1,62 +1,23 @@-import { useEffect, useRef, useState, type CSSProperties } from "react";-import { readMediaFileUrl } from "../../lib/persistence";-import { sceneDurationSeconds, type ProjectConfig, type ProjectVideo } from "../../lib/project";-import { scenePreviewAsset } from "../../lib/sceneHistory";-import { sceneMediaStartSeconds, sceneMediaDurationMs } from "../../lib/continuationMedia";-import { videoScenes } from "../../lib/videoLibrary";+import { useEffect, useMemo, useRef, useState, type CSSProperties } from "react";+import type { ProjectConfig, ProjectVideo } from "../../lib/project";+import { outputDimensions } from "../../lib/export";+import { videoPlaybackConfig, videoPlaylist } from "../../lib/videoPlayback";+import { ProgramMonitor } from "./ProgramMonitor";import { Dismiss16, Pause20, Play20Filled } from "../ui/icons";+export { videoPlaylist } from "../../lib/videoPlayback";-export function videoPlaylist(video: ProjectVideo, config: ProjectConfig) {- return videoScenes(video, config).flatMap((scene) => {- const asset = scenePreviewAsset(scene, config.assets);- if (!asset?.relativePath) return [];- const start = sceneMediaStartSeconds(asset);- const duration = Math.min(sceneDurationSeconds(scene), (sceneMediaDurationMs(asset) ?? sceneDurationSeconds(scene) * 1000) / 1000);- return [{ sceneId: scene.id, title: scene.title, asset, start, end: start + duration }];- });-}--/** Plays each scene's own range, excluding retained continuation prefixes. */+/** Timeline's continuous clock, prepared players and persistent compositor. */export function VideoPlayback({ video: item, config, folderPath, popout, onClose }: {video: ProjectVideo; config: ProjectConfig; folderPath: string; popout: boolean; onClose: () => void;}) {- const playlist = videoPlaylist(item, config);- const [index, setIndex] = useState(0), [url, setUrl] = useState<string | null>(null);- const [playing, setPlaying] = useState(false), [ended, setEnded] = useState(false), [error, setError] = useState<string | null>(null);- const [aspect, setAspect] = useState(16 / 9);- const media = useRef<HTMLVideoElement>(null), frame = useRef<HTMLDivElement>(null);- const close = useRef(onClose); close.current = onClose;- const timer = useRef<number>(), moving = useRef(false), epoch = useRef(0);- const current = playlist[index];- const advance = () => {- if (moving.current) return;- moving.current = true; clearTimeout(timer.current); media.current?.pause();- if (index + 1 < playlist.length) setIndex(index + 1);- else { setEnded(true); setPlaying(false); }- };- const play = async (restart = false) => {- const player = media.current;- if (!player || !current || !url) return;- if (restart || player.currentTime < current.start || player.currentTime >= current.end - 0.02) player.currentTime = current.start;- moving.current = false; setEnded(false); setError(null); clearTimeout(timer.current);- const token = ++epoch.current;- try {- await player.play();- if (token !== epoch.current) return;- setPlaying(true);- timer.current = window.setTimeout(advance, Math.max(0, (Math.min(current.end, player.duration || current.end) - player.currentTime) * 1000));- } catch { if (token === epoch.current) { setPlaying(false); setError("Could not play this scene. Try replaying it."); } }- };- useEffect(() => {- let disposed = false, owned: string | null = null;- epoch.current++; moving.current = false; setUrl(null); setError(null); setPlaying(false);- const player = media.current; player?.pause(); clearTimeout(timer.current);- if (current) void readMediaFileUrl(folderPath, current.asset, "video/mp4").then((value) => {- if (disposed) { if (value) URL.revokeObjectURL(value); return; }- owned = value; setUrl(value); if (!value) setError("This scene's video is unavailable.");- }).catch((reason) => { if (!disposed) setError(String(reason)); });- return () => { disposed = true; epoch.current++; clearTimeout(timer.current); player?.pause(); if (owned) URL.revokeObjectURL(owned); };- }, [folderPath, current?.sceneId, current?.asset.relativePath, current?.start, current?.end]);+ const preview = useMemo(() => videoPlaybackConfig(item, config), [item, config]);+ const playlist = useMemo(() => videoPlaylist(item, config), [item, config]);+ const [playhead, setPlayhead] = useState(0), [playing, setPlaying] = useState(true);+ const frame = useRef<HTMLDivElement>(null), close = useRef(onClose); close.current = onClose;+ const duration = preview.timeline.tracks[0].clips.reduce((end, clip) => Math.max(end, clip.startMs + clip.durationMs), 0);+ const index = Math.max(0, playlist.findIndex((clip) => playhead < clip.startMs + clip.durationMs));+ const current = playlist[playhead >= duration ? playlist.length - 1 : index];+ const dimensions = outputDimensions(config.settings.resolution, config.settings.aspectRatio);useEffect(() => {const focused = document.activeElement as HTMLElement | null;const library = popout ? frame.current?.closest(".video-library-stage")?.querySelector<HTMLElement>(".music-library") : null;@@ -74,17 +35,14 @@ export function VideoPlayback({ video: item, config, folderPath, popout, onClose};}, [popout]);const player = <div className={popout ? "shot-popout__frame" : "video-card__player"} ref={frame} tabIndex={-1}- role={popout ? "dialog" : "group"} aria-label={`${item.title} preview`} style={{ "--shot-aspect": aspect } as CSSProperties}>- <video ref={media} src={url ?? undefined} playsInline onLoadedMetadata={(event) => { if (url) { setAspect(event.currentTarget.videoWidth / event.currentTarget.videoHeight || 16 / 9); void play(true); } }}- onTimeUpdate={(event) => { if (playing && current && event.currentTarget.currentTime >= current.end - 0.02) advance(); }} onEnded={advance}- onError={() => { if (url) { clearTimeout(timer.current); setPlaying(false); setError("This scene could not be played."); } }} />+ role={popout ? "dialog" : "group"} aria-label={item.title + " preview"} style={{ "--shot-aspect": dimensions.width / dimensions.height } as CSSProperties}>+ <ProgramMonitor config={preview} folderPath={folderPath} playheadMs={Math.min(playhead, Math.max(0, duration - 1))} playing={playing} waitForInitialFrame onSeek={setPlayhead} onPlayingChange={setPlaying} /><button type="button" className="shot-popout__close" aria-label="Close video preview" onClick={onClose}><Dismiss16 /></button>- <button type="button" className="shot-popout__play" aria-label={`${playing ? "Pause" : "Replay"} ${item.title}`} disabled={!url} onClick={() => {- if (playing) { epoch.current++; clearTimeout(timer.current); media.current?.pause(); setPlaying(false); }- else if (ended && index !== 0) { setEnded(false); setIndex(0); } else void play(ended);+ <button type="button" className="shot-popout__play" aria-label={(playing ? "Pause " : "Replay ") + item.title} disabled={!duration} onClick={() => {+ if (playing) setPlaying(false);+ else { if (playhead >= duration) setPlayhead(0); setPlaying(true); }}}>{playing ? <Pause20 /> : <Play20Filled />}</button>- {error && <p className="shot-popout__message" role="alert">{error}</p>}- <span className="video-card__position">{index + 1} / {playlist.length} · {current?.title}</span>+ <span className="video-card__position">{playhead >= duration ? playlist.length : index + 1} / {playlist.length} · {current?.title}</span></div>;return popout ? <div className="shot-popout">{player}</div> : player;}diff --git a/src/components/workspace/VideoPoster.tsx b/src/components/workspace/VideoPoster.tsxnew file mode 100644index 0000000..508524a--- /dev/null+++ b/src/components/workspace/VideoPoster.tsx@@ -0,0 +1,21 @@+import { useEffect, useRef, useState } from "react";+import { forgetShotPosters, shotPoster } from "./ShotThumbnail";+import { Film32 } from "../ui/icons";++export function VideoPoster({ folderPath, relativePath, seconds, revision, title }: {+ folderPath: string; relativePath?: string | null; seconds: number; revision?: string | number; title: string;+}) {+ const key = `${folderPath}:${relativePath}:${seconds}:${revision}`;+ const [result, setResult] = useState<{ key: string; poster: string | null }>();+ const previous = useRef<{ path: string; revision?: string | number }>();+ useEffect(() => {+ if (!relativePath) return;+ if (previous.current?.path === relativePath && previous.current.revision !== revision) forgetShotPosters(folderPath, relativePath);+ previous.current = { path: relativePath, revision };+ let live = true;+ void shotPoster(folderPath, relativePath, seconds).then(({ poster }) => { if (live) setResult({ key, poster }); });+ return () => { live = false; };+ }, [key]);+ return result?.key === key && result.poster+ ? <img src={result.poster} alt={`First frame of ${title}`} draggable={false} /> : <Film32 aria-hidden="true" />;+}diff --git a/src/lib/videoLibrary.test.ts b/src/lib/videoLibrary.test.tsindex 4418364..a64510f 100644--- a/src/lib/videoLibrary.test.ts+++ b/src/lib/videoLibrary.test.ts@@ -4,6 +4,9 @@ import { mergeVideoScenes, newVideo, resolveVideoSource, songScenes, videoGuidedimport { sceneGenerationRequest, sendBlocker } from "./sceneGeneration";import { videoPlaylist } from "../components/workspace/VideoPlayback";import { longShotBlocker, longShotInputs } from "./longShot";+import { videoPlaybackConfig } from "./videoPlayback";+import { continuationPlaybackTracks } from "./continuationMedia";+import { previewPlayerKeys } from "./timelinePreview";const project = () => createProjectConfig({ name: "Videos", prompt: "", resolution: "416p", aspectRatio: "16:9", targetDurationSeconds: 30 });const scene = (id: string, seconds = 10) => createDraftGenerationJob("A dancer moves", { id, sceneType: "prompt", durationSeconds: seconds });@@ -91,3 +94,18 @@ it("plays scenes in video order, skips drafts and excludes joined continuation pconfig.assets = [asset(generationAssetId("first"), "generated", 10250), { ...asset(generationAssetId("last"), "generated", 19000), sceneSegments: [{ sceneId: "last", startFrame: 240, frameCount: 216, latentRelativePath: "latents/last.safetensors" }] }];expect(videoPlaylist(video, config).map(({ sceneId, start, end }) => ({ sceneId, start, end }))).toEqual([{ sceneId: "last", start: 10, end: 18 }, { sceneId: "first", start: 0, end: 10 }]);});++it("uses the latest joined decode for both the opening poster and uninterrupted playback across scene boundaries", () => {+ const config = project();+ const segments = ["a", "b"].map((sceneId, index) => ({ sceneId, startFrame: index * 120, frameCount: 120, latentRelativePath: `latents/${sceneId}.safetensors` }));+ config.generationJobs = segments.map((part) => ({ ...scene(part.sceneId, 5), latentRelativePath: part.latentRelativePath }));+ config.assets = segments.map((part, index) => ({ ...asset(generationAssetId(part.sceneId), "generated", (index + 1) * 5000), sceneSegments: segments.slice(0, index + 1) }));+ const video = { ...newVideo(null), sceneIds: ["a", "b"] }, original = structuredClone(config);+ const playlist = videoPlaylist(video, config);+ expect(playlist.map((entry) => entry.asset.id)).toEqual([generationAssetId("b"), generationAssetId("b")]);+ expect(playlist.map((entry) => [entry.start, entry.end])).toEqual([[0, 5], [5, 10]]);+ const preview = videoPlaybackConfig(video, config), tracks = continuationPlaybackTracks(preview);+ const keys = previewPlayerKeys(tracks, new Map(preview.assets.map((asset) => [asset.id, asset])));+ expect(keys.get("video-preview-a")).toBe(keys.get("video-preview-b"));+ expect(config).toEqual(original);+});diff --git a/src/lib/videoPlayback.ts b/src/lib/videoPlayback.tsnew file mode 100644index 0000000..8fd0d2a--- /dev/null+++ b/src/lib/videoPlayback.ts@@ -0,0 +1,38 @@+import { generationAssetId, sceneDurationSeconds, type ProjectConfig, type ProjectVideo, type TimelineClip } from "./project";+import { scenePreviewAsset, scenePreviewJob } from "./sceneHistory";+import { continuationPlaybackTracks, sceneMediaDurationMs } from "./continuationMedia";+import { videoScenes } from "./videoLibrary";++/** A temporary timeline for the Video library. Never edits the project's timeline. */+export function videoPlaybackConfig(video: ProjectVideo, config: ProjectConfig): ProjectConfig {+ const members = videoScenes(video, config);+ const assets = new Map(config.assets.map((asset) => [asset.id, asset]));+ const clips: TimelineClip[] = [];+ let startMs = 0;+ for (const scene of members) {+ const selected = scenePreviewAsset(scene, config.assets);+ if (!selected?.relativePath) continue;+ const asset = { ...selected, id: generationAssetId(scene.id) };+ assets.set(asset.id, asset);+ const durationMs = Math.round(Math.min(sceneDurationSeconds(scene) * 1000, sceneMediaDurationMs(asset) ?? Infinity));+ if (durationMs <= 0) continue;+ clips.push({ id: `video-preview-${scene.id}`, assetId: asset.id, trackId: "video-preview", startMs, durationMs,+ sourceStartMs: 0, label: scene.title, status: "generated" });+ startMs += durationMs;+ }+ const memberIds = new Set(video.sceneIds);+ return { ...config, assets: [...assets.values()],+ generationJobs: config.generationJobs.map((scene) => memberIds.has(scene.id) ? scenePreviewJob(scene, config.assets) : scene),+ timeline: { tracks: [{ id: "video-preview", kind: "video", name: video.title, locked: false, muted: false, clips }] } };+}++/** Use exactly the resolved media/ranges that Timeline's monitor will display. */+export function videoPlaylist(video: ProjectVideo, config: ProjectConfig) {+ const preview = videoPlaybackConfig(video, config);+ return continuationPlaybackTracks(preview)[0].clips.map((clip) => ({+ sceneId: clip.id.slice("video-preview-".length), title: clip.label,+ asset: preview.assets.find((asset) => asset.id === clip.assetId)!,+ start: clip.sourceStartMs / 1000, end: (clip.sourceStartMs + clip.durationMs) / 1000,+ startMs: clip.startMs, durationMs: clip.durationMs,+ }));+}diff --git a/src/styles/video-library.css b/src/styles/video-library.cssindex c08df35..c0ac37b 100644--- a/src/styles/video-library.css+++ b/src/styles/video-library.css@@ -10,17 +10,19 @@.video-scenes__path strong { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }.video-scenes__path span, .video-reference-help { color: var(--text-secondary); font-size: var(--text-xs); }.video-reference-range { display: block; margin-block: var(--space-1); }-.video-card { overflow: hidden; }-.video-card > .music-song__select { height: auto; padding-bottom: 44px; }-.video-card .music-song__art { aspect-ratio: 16 / 9; }+.video-library-stage .music-songs--grid { grid-template-columns: repeat(auto-fill, minmax(0, min(300px, 100%))); gap: var(--space-2); }+.video-card { overflow: hidden; container-type: inline-size; border-radius: var(--radius-sm); }+.video-card > .music-song__select { height: auto; padding: 6px; }+.video-card .music-song__art { aspect-ratio: var(--video-card-aspect, 16 / 9); overflow: hidden; }+.video-card .music-song__art img { display: block; width: 100%; height: 100%; object-fit: contain; }.video-library-stage .music-folder .music-song__art { aspect-ratio: 16 / 9; }-.video-card__open { position: absolute; bottom: var(--space-2); right: var(--space-2); }-.video-card__controls { position: absolute; top: 30%; left: 50%; transform: translate(-50%, -50%); display: flex; gap: var(--space-2); }+.video-card__controls { position: absolute; top: calc((100cqw - 12px) / var(--video-card-aspect, 1.7777778) / 2 + 6px); left: 50%; transform: translate(-50%, -50%); display: flex; gap: var(--space-2); }.video-card__controls button { display: grid; place-items: center; width: 32px; height: 32px; border: 1px solid #ffffff80; border-radius: 50%; background: #15191fe6; color: #fff; box-shadow: 0 1px 6px #0008; }.video-card__controls button:hover { background: #363d48; }-.video-card__player { position: absolute; inset: 0; background: #000; z-index: 2; }+.video-card__player { position: absolute; top: 6px; left: 6px; right: 6px; aspect-ratio: var(--video-card-aspect, 16 / 9); background: #000; z-index: 2; }.video-card__player video { width: 100%; height: 100%; object-fit: contain; }.video-card__position { position: absolute; left: var(--space-2); top: var(--space-2); color: #fff; background: #000b; padding: 2px 5px; font-size: var(--text-xs); pointer-events: none; max-width: 70%; }-.music-songs--list .video-card > .music-song__select { padding-bottom: var(--space-3); padding-right: 190px; min-height: 70px; }-.music-songs--list .video-card__controls { top: 50%; left: auto; right: 100px; transform: translateY(-50%); }-.music-songs--list .video-card__open { bottom: 50%; transform: translateY(50%); }+.music-songs--list .video-card > .music-song__select { padding-right: 100px; min-height: 82px; }+.music-songs--list .video-card .music-song__art { width: 120px; height: 67.5px; }+.music-songs--list .video-card__controls { top: 50%; left: auto; right: 12px; transform: translateY(-50%); }+.music-songs--list .video-card__player { inset: 0; aspect-ratio: auto; }