Skip to content

AI media harness for planning, generating, and editing locally

TypeScript
7.8 KB
Raw Blame
import { MotionCanvas } from "./MotionCanvas";
import type { MotionNode } from "../../lib/motion";
import type { ComponentProps } from "react";
import { clipPlaybackRate, clipSourceTimeMs } from "../../lib/clipTiming";
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState, type CSSProperties } from "react";
import { clipFrameStyle, clipVisualSettings, type ClipFrameStyle } from "../../lib/export";
import { PreviewSources } from "../../lib/exportPipeline";
import { clipChromaKey, type ProjectAsset, type TimelineClip } from "../../lib/project";
import { ChromaKeyPreview } from "./ChromaKeyPreview";
import { VideoEffectsPreview } from "./VideoEffectsPreview";
import { activeVideoEffects, hasVideoEffects } from "../../lib/effectSettings";
import { usesBackdropKey } from "../../lib/chromaKey";
import type { GpuEffects } from "../../lib/videoEffectsGpu";
import { isTauri } from "../../lib/persistence";
import type { PreviewMedia } from "../../lib/previewPresentation";

export function previewMediaStyle(frame: ClipFrameStyle): CSSProperties {
  const { transform, opacity, revealStart, revealEnd } = frame;
  return {
    transform: `translate(${transform.positionX}%, ${transform.positionY}%) scale(${transform.scale / 100}) rotate(${transform.rotation}deg)`,
    opacity,
    clipPath: `inset(0 ${(1 - revealEnd) * 100}% 0 ${revealStart * 100}%)`,
  };
}

/** Prepared clips keep their decoder and effect renderer when made visible. */
function MediaProgramLayer({ clip, playerKey, asset, sources, playheadMs, playing, rate = 1, active, foreground, muted, externalSeek, depth, media, composited }: {
  clip: TimelineClip;
  playerKey: string;
  asset: ProjectAsset | undefined;
  sources: PreviewSources;
  playheadMs: number;
  playing: boolean;
  /** Forward playback speed (J/K/L shuttle). */
  rate?: number;
  active: boolean;
  foreground: boolean;
  muted: boolean;
  externalSeek: boolean;
  depth: number;
  media: PreviewMedia;
  composited: boolean;
}) {
  const [url, setUrl] = useState<string | null>(null);
  const [error, setError] = useState<string | null>(null);
  const [ready, setReady] = useState(false);
  const video = useRef<HTMLVideoElement | null>(null);
  const image = useRef<HTMLImageElement | null>(null);
  const registerVideo = useCallback((element: HTMLVideoElement | null) => {
    video.current = element;
    media.set(playerKey, element);
  }, [media, playerKey]);
  const registerImage = useCallback((element: HTMLImageElement | null) => {
    image.current = element;
    media.set(playerKey, element);
  }, [media, playerKey]);
  const position = useRef({ clip, playheadMs, playing, active, externalSeek });
  position.current = { clip, playheadMs, playing, active, externalSeek };
  const isImage = asset?.kind === "image" || asset?.mimeType.startsWith("image/");
  useEffect(() => {
    let live = true;
    setUrl(null);
    setError(null);
    setReady(false);
    if (asset && (asset.relativePath || asset.sourcePath)) {
      void sources.url(asset).then((value) => { if (live) setUrl(value); }, (reason) => { if (live) setError(String(reason)); });
    } else setError(`Media for ${clip.label} is unavailable.`);
    return () => { live = false; };
  }, [asset, sources]);
  useEffect(() => {
    const element = video.current;
    return () => {
      if (!element) return;
      element.pause();
      element.removeAttribute("src");
      element.load();
    };
  }, [url]);
  const sync = () => {
    const element = video.current;
    if (!element) return;
    const state = position.current;
    const run = state.playing && state.active;
    if (!run) element.pause();
    if (element.readyState < 1) return;
    const target = Math.max(0, clipSourceTimeMs(state.clip, state.playheadMs) / 1000);
    // Upcoming players seek while paused. At a normal cut they can start
    // immediately without seeking again for a fraction of a display frame.
    // Explicit jumps still seek precisely, including during playback.
    const tolerance = run && !state.externalSeek ? 0.25 : 0.033;
    if (!element.seeking && Math.abs(element.currentTime - target) > tolerance) element.currentTime = target;
    const mediaRate = rate * clipPlaybackRate(state.clip);
    element.preservesPitch = false;
    if (run && element.playbackRate !== mediaRate) element.playbackRate = mediaRate;
    if (run && element.paused) void element.play().catch(() => undefined);
  };
  useLayoutEffect(sync, [playheadMs, playing, rate, active, externalSeek, clip, url]);
  const loaded = () => {
    sync();
    if (video.current) setReady(video.current.readyState >= 2 && !video.current.seeking);
  };
  const style = previewMediaStyle(clipFrameStyle(clipVisualSettings(clip), playheadMs - clip.startMs));
  /* Bypassed effects are dropped here, the way the export drops them. Kept
     per clip object so the effect canvases only redraw when the clip does. */
  const gpuEffects = useMemo<GpuEffects>(() => ({ ...activeVideoEffects(clip), chromaKey: clipChromaKey(clip) }), [clip]);
  const chromaKey = gpuEffects.chromaKey;
  const effects = hasVideoEffects(gpuEffects) || usesBackdropKey(chromaKey);
  const sourceStyle: CSSProperties = composited || chromaKey || effects ? { ...style, visibility: "hidden", position: "absolute" } : style;
  const showStatus = active && foreground && isTauri() && Boolean(asset?.relativePath || asset?.sourcePath);
  return <div className="program-layer" data-clip-id={clip.id} data-active={active} aria-hidden={!active}
    aria-label={`${foreground ? "Foreground" : "Background"} clip: ${clip.label}`}
    style={{ visibility: active ? "visible" : "hidden", zIndex: depth }}>
    {url && (isImage
      ? <img ref={registerImage} src={url} alt={clip.label} style={sourceStyle} onError={() => setError(`Could not decode ${clip.label}.`)} />
      : <video key={url} ref={registerVideo} src={url} style={sourceStyle} muted={!active || muted} playsInline preload="auto"
        onLoadedMetadata={loaded} onLoadedData={loaded} onCanPlay={loaded} onSeeked={loaded}
        onError={() => setError(`Could not decode ${clip.label}.`)} />)}
    {!composited && url && effects && <VideoEffectsPreview source={isImage ? image : video} sourceUrl={url} effects={gpuEffects} playing={playing && active && !isImage} style={style} onError={setError} />}
    {!composited && url && !effects && chromaKey && <ChromaKeyPreview source={isImage ? image : video} sourceUrl={url} effect={chromaKey} playing={playing && active && !isImage} style={style} onError={setError} />}
    {showStatus && (error
      ? <div className="program-note program-note--error" role="alert">{error}</div>
      : (!url || (!ready && !isImage)) && <div className="program-note"><span>Loading {clip.label}…</span></div>)}
  </div>;
}

export function ProgramLayer(props: ComponentProps<typeof MediaProgramLayer> & { onMotionFrame?: (clipId: string, nodes: MotionNode[]) => void }) {
  const motionFrame = useCallback((nodes: MotionNode[]) => props.onMotionFrame?.(props.clip.id, nodes), [props.onMotionFrame, props.clip.id]);
  if (!props.asset?.motion) return <MediaProgramLayer {...props} />;
  const { clip, asset, active, depth, composited } = props;
  const needsGpu = hasVideoEffects(activeVideoEffects(clip)) || Boolean(clipChromaKey(clip));
  return <div className="program-layer" data-clip-id={clip.id} data-active={active} aria-hidden={!active}
    style={{ visibility: active ? "visible" : "hidden", zIndex: depth }}>
    <MotionCanvas motion={asset.motion!} time={clipSourceTimeMs(clip, props.playheadMs) / 1000}
      media={props.media} playerKey={props.playerKey} onFrame={motionFrame}
      style={{ ...previewMediaStyle(clipFrameStyle(clipVisualSettings(clip), props.playheadMs - clip.startMs)), ...(composited ? { visibility: "hidden" } : {}) }} />
    {active && !composited && needsGpu && <div className="program-note program-note--error" role="alert">Motion effects preview requires WebGPU.</div>}
  </div>;
}