AI media harness for planning, generating, and editing locally
TypeScript
import { Clock18, Film18, OpenExternal16, Pause20, Play20Filled, Prohibited18, Spinner18, Spinner20, Warning18, Warning20 } from "../ui/icons";
import { useContext, useEffect, useRef, useState, type ReactNode } from "react";
import { GenerationPreviewContext } from "./GenerationPreviewSource";
import { createPortal } from "react-dom";
import { ShotPopout, SHOT_PLAY_EVENT } from "./ShotPopout";
import { readMediaFileUrl } from "../../lib/persistence";
import { sceneDurationSeconds, type GenerationJob } from "../../lib/project";
/* A real frame of ONE shot, taken from the scene's own video file.
*
* A scene is rendered as a single .mp4 in the project folder, and a shot is a
* span inside it — so the picture of shot 3 is the frame at the moment shot 3
* begins. Nothing here invents art: before a scene has been rendered there is
* no frame to show and this says which state it is in instead, the same rule
* the reference cards and the media thumbs follow.
*
* The file is opened ONCE per scene, not once per card. A four-shot scene asks
* for four frames of the same .mp4, and a component that read the file per
* thumbnail would pull a few hundred megabytes through the webview four times
* over. Requests are queued per file: the first one opens it, seeks to every
* time that is waiting — including ones that arrive while it is open — and
* hands each card a small JPEG data URL.
*
* As in MediaThumbnail, no FFmpeg and no WebCodecs: the <video> element already
* owns a hardware decoder and one still frame is all this needs. */
// Two physical pixels per typical 320px card keeps posters sharp on high-DPI
// displays without retaining full-resolution render frames in the cache.
const THUMB_WIDTH = 640;
/** How long a seek may take before the frame it was going to draw is given up
* on. A seek that neither lands nor errors settles nothing, and awaiting it
* would leave every card behind it in the queue spinning for the life of the
* panel. */
const SEEK_LIMIT_MS = 4_000;
/** Posters represent the cut itself: every shot card shows the exact frame at
* its own startSeconds. */
/** Small JPEGs, but not free. Bounded like MediaThumbnail's own cache. */
const CACHE_LIMIT = 96;
const PLAY_EVENT = SHOT_PLAY_EVENT;
/** Drawn frames, keyed by file and time. */
const POSTERS = new Map<string, string>();
/* Each poster's width / height, so a tile that remounts takes the frame's
shape at once instead of starting 16:9 and jumping. */
const POSTER_SHAPES = new Map<string, string>();
/** Files that opened and would not give up a picture, so the next card does not
* read the whole thing again for the same silence. */
const UNREADABLE = new Set<string>();
/** Files there is nothing to read — the browser preview has no project folder.
* Kept apart from UNREADABLE because that is not a failure to report. */
const ABSENT = new Set<string>();
interface Waiting {
seconds: number;
settle: (result: PosterResult) => void;
}
/** Requests still waiting on a file, and the files currently open. */
const QUEUES = new Map<string, Waiting[]>();
const OPEN = new Set<string>();
/** A drawn frame, or the reason there is none. `failed` separates "this file
* could not be previewed", which is worth a warning, from "there is no file to
* read here", which is not. */
export interface PosterResult {
poster: string | null;
failed: boolean;
}
const NOTHING: PosterResult = { poster: null, failed: false };
const BROKEN: PosterResult = { poster: null, failed: true };
const fileKey = (folderPath: string, relativePath: string) => `${folderPath}::${relativePath}`;
const posterKey = (file: string, seconds: number) => `${file}@${seconds.toFixed(2)}`;
function remember(key: string, poster: string) {
if (POSTERS.size >= CACHE_LIMIT) {
const oldest = POSTERS.keys().next().value;
if (oldest !== undefined) POSTERS.delete(oldest);
}
POSTERS.set(key, poster);
}
const settled = (element: HTMLMediaElement, event: string) =>
new Promise<void>((resolve, reject) => {
element.addEventListener(event, () => resolve(), { once: true });
element.addEventListener("error", () => reject(new Error("This file could not be decoded.")), { once: true });
});
const orGiveUp = (promise: Promise<void>, limitMs: number) =>
new Promise<void>((resolve) => {
const timer = setTimeout(resolve, limitMs);
const done = () => { clearTimeout(timer); resolve(); };
promise.then(done, done);
});
/* Drawing is the part that can fail on its own — a computer with no 2D canvas,
a frame the decoder will not hand over — and it must not take the rest of the
queue down with it. */
function tryDraw(video: HTMLVideoElement): string | null {
try {
const { videoWidth: width, videoHeight: height } = video;
if (!width || !height) return null;
const canvas = document.createElement("canvas");
const scale = Math.min(1, THUMB_WIDTH / width);
canvas.width = Math.max(1, Math.round(width * scale));
canvas.height = Math.max(1, Math.round(height * scale));
const context = canvas.getContext("2d");
if (!context) return null;
context.drawImage(video, 0, 0, canvas.width, canvas.height);
return canvas.toDataURL("image/jpeg", 0.92);
} catch {
return null;
}
}
/* Hand the file back: the decoder and the whole blob go with it. */
const release = (element: HTMLMediaElement) => {
try {
element.removeAttribute("src");
element.load();
} catch {
/* nothing to do about it */
}
};
/** The answer for a file nothing more can be read from, given to everyone still
* waiting on it. */
function abandon(file: string, result: PosterResult) {
const queue = QUEUES.get(file);
QUEUES.delete(file);
queue?.forEach((waiting) => waiting.settle(result));
}
/** loadeddata can precede a drawable frame in Chromium. Subscribe before loading
* or seeking so a paused video presents its exact opening/cut frame first. */
function presentedFrame(video: HTMLVideoElement) {
let finish = () => {};
const ready = new Promise<void>((resolve) => {
if (typeof video.requestVideoFrameCallback !== "function") { resolve(); return; }
let callback = 0;
const timer = window.setTimeout(() => finish(), SEEK_LIMIT_MS);
finish = () => { window.clearTimeout(timer); video.cancelVideoFrameCallback(callback); resolve(); };
callback = video.requestVideoFrameCallback(() => finish());
});
return { ready, cancel: () => finish() };
}
async function drain(folderPath: string, relativePath: string, file: string): Promise<void> {
const video = document.createElement("video");
let url: string | null = null;
let presentation: ReturnType<typeof presentedFrame> | undefined;
try {
url = await readMediaFileUrl(folderPath, { relativePath }, "video/mp4");
// Browser preview: there is no project folder, so there is nothing to show
// and nothing has gone wrong.
if (!url) {
ABSENT.add(file);
abandon(file, NOTHING);
return;
}
video.muted = true;
video.preload = "auto";
video.playsInline = true;
presentation = presentedFrame(video);
video.src = url;
await settled(video, "loadeddata");
await presentation.ready;
/* Drained rather than iterated: a card that mounts while the file is open
joins this same pass instead of opening it a second time. */
for (;;) {
const next = QUEUES.get(file)?.shift();
if (!next) break;
const key = posterKey(file, next.seconds);
const already = POSTERS.get(key);
if (already) {
next.settle({ poster: already, failed: false });
continue;
}
/* The last frame is not a valid seek target in every container, and a
shot that starts exactly at the end of the scene would ask for it. */
const end = Number.isFinite(video.duration) ? Math.max(0, video.duration - 0.05) : next.seconds;
const target = Math.min(Math.max(next.seconds, 0), end);
if (Math.abs(video.currentTime - target) > 0.01) {
const seek = settled(video, "seeked");
presentation = presentedFrame(video);
video.currentTime = target;
// A seek that fails costs this frame its offset and nothing else —
// whatever the decoder is holding is still a frame of this scene.
await orGiveUp(seek, SEEK_LIMIT_MS);
await presentation.ready;
}
const poster = tryDraw(video);
if (poster) {
remember(key, poster);
next.settle({ poster, failed: false });
} else {
next.settle(BROKEN);
}
}
} catch {
UNREADABLE.add(file);
abandon(file, BROKEN);
} finally {
presentation?.cancel();
release(video);
if (url) URL.revokeObjectURL(url);
OPEN.delete(file);
/* Anything that arrived while the file was being handed back. Re-opening is
the only way to serve it, and it cannot loop: the pass above empties the
queue before it ever reaches here. */
const left = QUEUES.get(file);
if (left && left.length > 0) {
if (UNREADABLE.has(file)) abandon(file, BROKEN);
else if (ABSENT.has(file)) abandon(file, NOTHING);
else {
OPEN.add(file);
void drain(folderPath, relativePath, file);
}
} else {
QUEUES.delete(file);
}
}
}
/** The frame at `seconds` into the scene's video file. Exported for tests. */
export function shotPoster(folderPath: string, relativePath: string, seconds: number): Promise<PosterResult> {
const file = fileKey(folderPath, relativePath);
const cached = POSTERS.get(posterKey(file, seconds));
if (cached) return Promise.resolve({ poster: cached, failed: false });
if (UNREADABLE.has(file)) return Promise.resolve(BROKEN);
if (ABSENT.has(file)) return Promise.resolve(NOTHING);
return new Promise<PosterResult>((settle) => {
const queue = QUEUES.get(file) ?? [];
queue.push({ seconds, settle });
QUEUES.set(file, queue);
if (!OPEN.has(file)) {
OPEN.add(file);
void drain(folderPath, relativePath, file);
}
});
}
/** What a card says when there is no frame. Every one of these is a state of
* the scene the user can see and act on, never a shrug. */
function placeholder(job: GenerationJob, failed: boolean, cancelling: boolean): { icon: ReactNode; word: string } {
if (cancelling && (job.status === "queued" || job.status === "generating")) {
return { icon: <Spinner18 className="spin" />, word: "Cancelling..." };
}
switch (job.status) {
case "generating": return { icon: <Spinner18 className="spin" />, word: `Rendering ${Math.round(job.progress * 100)}%` };
case "queued": return { icon: <Clock18 />, word: "Waiting to render" };
case "ready": return { icon: <Spinner18 className="spin" />, word: "Saving the file" };
case "failed": return { icon: <Warning18 />, word: "Didn’t finish" };
case "cancelled": return { icon: <Prohibited18 />, word: "Cancelled" };
case "completed": return job.outputRelativePath
? failed
? { icon: <Warning18 />, word: "No preview" }
: { icon: <Film18 />, word: "Opening the file…" }
: { icon: <Warning18 />, word: "No file saved" };
default: return { icon: <Film18 />, word: "Not rendered yet" };
}
}
/** Drops every in-memory still derived from one generated file. Regeneration
* overwrites that file in place, so keeping its old posters would make a fresh
* render look unchanged until the app restarted. */
export function forgetShotPosters(folderPath: string, relativePath: string): void {
const file = fileKey(folderPath, relativePath);
for (const key of POSTERS.keys()) {
if (key.startsWith(`${file}@`)) POSTERS.delete(key);
}
UNREADABLE.delete(file);
ABSENT.delete(file);
}
export function formatEstimatedTimeLeft(milliseconds: number): string {
const totalSeconds = Math.max(1, Math.ceil(milliseconds / 1_000));
if (totalSeconds < 60) return `${totalSeconds}s left`;
const hours = Math.floor(totalSeconds / 3_600);
const minutes = Math.floor((totalSeconds % 3_600) / 60);
const seconds = totalSeconds % 60;
if (hours > 0) return `${hours}h ${minutes}m left`;
return `${minutes}m ${seconds}s left`;
}
export function ShotThumbnail({ folderPath, job, seconds, endSeconds = sceneDurationSeconds(job), sourceOffsetSeconds = 0, shotNumber, estimatedCompletionAt = null, cancelling = false, onPlay }: {
folderPath: string;
/** The scene this shot belongs to: it owns the file and the status. */
job: GenerationJob;
/** Where the shot starts, in seconds from the head of the scene. */
seconds: number;
/** The next cut (or scene end). Playback never crosses this point. */
endSeconds?: number;
sourceOffsetSeconds?: number;
shotNumber: number;
/** Live rendering estimate derived from slopfab timing; never persisted. */
estimatedCompletionAt?: number | null;
/** Cancel was requested but the engine has not reported its terminal state. */
cancelling?: boolean;
/** Opens this shot in the editor when playback begins. */
onPlay?: () => void;
}) {
const live = useContext(GenerationPreviewContext);
const generation = !cancelling && ["queued", "generating", "ready"].includes(job.status) ? live : undefined;
const relativePath = job.status === "completed" ? job.outputRelativePath : null;
const at = generation ? Math.min(seconds, (generation.preview.frames.length - 1) / generation.preview.fps) : seconds + sourceOffsetSeconds;
const endAt = generation ? Math.min(endSeconds, generation.preview.frames.length / generation.preview.fps) : endSeconds + sourceOffsetSeconds;
const previewPosition = useRef({ paused: false, time: at });
const playbackId = `${job.id}@${seconds}`;
const [result, setResult] = useState<PosterResult>(() =>
relativePath ? { poster: POSTERS.get(posterKey(fileKey(folderPath, relativePath), at)) ?? null, failed: false } : NOTHING);
const [playbackUrl, setPlaybackUrl] = useState<string | null>(null);
const [loadingPlayback, setLoadingPlayback] = useState(false);
const [playing, setPlaying] = useState(false);
const [playFailed, setPlayFailed] = useState(false);
const [popoutHost, setPopoutHost] = useState<HTMLElement | null>(null);
const popoutButton = useRef<HTMLButtonElement>(null);
const [estimateNow, setEstimateNow] = useState(() => Date.now());
/* Bumped when a poster's shape is first measured, to re-render with it. */
const [, setShapeSeen] = useState(0);
const videoRef = useRef<HTMLVideoElement>(null);
const requestRef = useRef(0);
const cutTimerRef = useRef<number | null>(null);
const clearCutTimer = () => {
if (cutTimerRef.current !== null) window.clearTimeout(cutTimerRef.current);
cutTimerRef.current = null;
};
const finishPlayback = () => {
clearCutTimer();
if (generation?.url && videoRef.current && !previewPosition.current.paused) {
videoRef.current.currentTime = at;
void playToCut(videoRef.current);
return;
}
videoRef.current?.pause();
setPlaying(false);
setPlaybackUrl(null);
};
const playToCut = async (video: HTMLVideoElement) => {
clearCutTimer();
const request = requestRef.current;
try {
await video.play();
if (request !== requestRef.current || videoRef.current !== video) return;
setPlaying(true);
const rate = video.playbackRate > 0 ? video.playbackRate : 1;
cutTimerRef.current = window.setTimeout(finishPlayback, Math.max(0, (endAt - video.currentTime) * 1_000 / rate));
} catch {
setPlayFailed(true);
}
};
useEffect(() => {
setPopoutHost(null);
}, [folderPath, relativePath, at, endAt]);
useEffect(() => {
if (!relativePath) {
setResult(NOTHING);
return;
}
let live = true;
void shotPoster(folderPath, relativePath, at).then((value) => { if (live) setResult(value); });
return () => { live = false; };
}, [folderPath, relativePath, at]);
useEffect(() => {
const stopOtherShot = (event: Event) => {
if ((event as CustomEvent<string>).detail === playbackId) return;
requestRef.current += 1;
setLoadingPlayback(false);
previewPosition.current.paused = true;
finishPlayback();
};
window.addEventListener(PLAY_EVENT, stopOtherShot);
return () => window.removeEventListener(PLAY_EVENT, stopOtherShot);
}, [playbackId, generation?.url]);
useEffect(() => () => { requestRef.current += 1; clearCutTimer(); }, [generation?.url]);
useEffect(() => () => {
requestRef.current += 1;
clearCutTimer();
}, []);
useEffect(() => () => {
if (playbackUrl) URL.revokeObjectURL(playbackUrl);
}, [playbackUrl]);
useEffect(() => {
if (job.status !== "generating" || estimatedCompletionAt === null || cancelling) return;
setEstimateNow(Date.now());
const timer = window.setInterval(() => setEstimateNow(Date.now()), 1_000);
return () => window.clearInterval(timer);
}, [job.status, estimatedCompletionAt, cancelling]);
const stopAtCut = () => {
const video = videoRef.current;
if (video && generation?.url && video.currentTime < at) video.currentTime = at;
if (video && generation) previewPosition.current.time = video.currentTime;
if (!video || video.currentTime <= at || video.currentTime + 0.02 < endAt) return;
finishPlayback();
};
const togglePlayback = async (event: React.MouseEvent<HTMLButtonElement>) => {
event.stopPropagation();
if ((!relativePath && !generation?.url) || loadingPlayback) return;
const video = videoRef.current;
if (video) {
if (!video.paused) {
requestRef.current += 1;
previewPosition.current.paused = true;
clearCutTimer();
video.pause();
setPlaying(false);
} else {
previewPosition.current.paused = false;
onPlay?.();
window.dispatchEvent(new CustomEvent<string>(PLAY_EVENT, { detail: playbackId }));
if (video.currentTime < at || video.currentTime >= endAt) video.currentTime = at;
await playToCut(video);
}
return;
}
const request = ++requestRef.current;
onPlay?.();
window.dispatchEvent(new CustomEvent<string>(PLAY_EVENT, { detail: playbackId }));
setLoadingPlayback(true);
setPlayFailed(false);
try {
const url = await readMediaFileUrl(folderPath, { relativePath }, "video/mp4");
if (request !== requestRef.current) {
if (url) URL.revokeObjectURL(url);
return;
}
if (!url) throw new Error("The generated file is not available in this view.");
setPlaybackUrl(url);
} catch {
if (request === requestRef.current) setPlayFailed(true);
} finally {
if (request === requestRef.current) setLoadingPlayback(false);
}
};
const { icon, word } = placeholder(job, result.failed, cancelling);
const estimateLine = job.status === "generating" && !cancelling
? estimatedCompletionAt === null
? "Estimating time left…"
: formatEstimatedTimeLeft(estimatedCompletionAt - estimateNow)
: null;
const progress = job.status === "generating" && <span className="shot-thumb__progress" aria-hidden="true">
<i style={{ width: `${Math.max(0, Math.min(100, job.progress * 100))}%` }} />
</span>;
if (result.poster || generation) {
const poster = generation ? `data:image/jpeg;base64,${generation.preview.frames[Math.min(generation.preview.frames.length - 1, Math.floor(at * generation.preview.fps))]}` : result.poster!;
const shape = generation ? `${generation.preview.width} / ${generation.preview.height}` : POSTER_SHAPES.get(poster);
const action = playFailed ? "Playback unavailable" : loadingPlayback ? "Opening shot" : playing ? "Pause" : "Play";
return <div className="shot-thumb-frame" style={shape ? { aspectRatio: shape } : undefined}><button
type="button"
className="shot-thumb shot-thumb--poster shot-thumb--playable"
/* The tile takes the rendered frame's own shape, not a fixed 16:9. */
style={shape ? { aspectRatio: shape } : undefined}
aria-label={`${action} shot ${shotNumber} of ${job.title}`}
data-tooltip={`${action} this shot only (${seconds.toFixed(1)}s–${endSeconds.toFixed(1)}s)`}
onClick={(event) => void togglePlayback(event)}
>
{generation?.url || playbackUrl
? <video
ref={videoRef}
src={generation?.url ?? playbackUrl!}
playsInline
muted={!!generation}
autoPlay={!!generation && !previewPosition.current.paused}
loop={!!generation}
onLoadedMetadata={(event) => {
event.currentTarget.currentTime = Math.min(generation ? Math.max(at, Math.min(previewPosition.current.time, endAt - .01)) : at, Math.max(0, event.currentTarget.duration - 0.01));
if (!generation || !previewPosition.current.paused) void playToCut(event.currentTarget);
}}
onTimeUpdate={stopAtCut}
onEnded={stopAtCut}
/>
: <img src={poster} alt="" loading="lazy" onLoad={(event) => {
const { naturalWidth: width, naturalHeight: height } = event.currentTarget;
if (!width || !height || POSTER_SHAPES.get(poster) === `${width} / ${height}`) return;
POSTER_SHAPES.set(poster, `${width} / ${height}`);
setShapeSeen((count) => count + 1);
}} />}
{generation && <>
<span className="shot-thumb__status">
<em>{word}</em>
{estimateLine && <small className="shot-thumb__eta">{estimateLine}</small>}
</span>
{progress}
</>}
<span className={`shot-thumb__play ${loadingPlayback ? "shot-thumb__play--loading" : ""}`} aria-hidden="true">
{loadingPlayback ? <Spinner20 className="spin" /> : playing ? <Pause20 /> : playFailed ? <Warning20 /> : <Play20Filled />}
</span>
</button>
<button ref={popoutButton} type="button" className="shot-thumb__popout" aria-label={`Pop out shot ${shotNumber} of ${job.title}`}
data-tooltip="Pop out shot" onClick={event => {
event.stopPropagation();
const host = event.currentTarget.closest<HTMLElement>(".generator-board-viewport");
if (host) { onPlay?.(); setPopoutHost(host); }
}}><OpenExternal16 /></button>
{popoutHost && (relativePath || generation?.url) && createPortal(<ShotPopout folderPath={folderPath} relativePath={relativePath ?? undefined} sourceUrl={generation?.url} loop={!!generation} start={at} end={endAt}
poster={poster} title={`shot ${shotNumber} of ${job.title}`} onClose={() => { setPopoutHost(null); popoutButton.current?.focus(); }} />, popoutHost)}
</div>;
}
return <span className={`shot-thumb shot-thumb--${job.status}`} role="img" aria-label={`Shot ${shotNumber} — ${word}${estimateLine ? `. ${estimateLine}` : ""}`}>
{icon}
<em>{word}</em>
{estimateLine && <small className="shot-thumb__eta">{estimateLine}</small>}
{progress}
</span>;
}