AI media harness for planning, generating, and editing locally
TypeScript
Measure media on decode and let the inspector set a clip's length
Bitti754876fce56f
diff --git a/src/components/workspace/AgentDock.tsx b/src/components/workspace/AgentDock.tsxindex 3a923c8..b7ebde7 100644--- a/src/components/workspace/AgentDock.tsx+++ b/src/components/workspace/AgentDock.tsx@@ -33,8 +33,13 @@ export function AgentDock({ context, record, providers, onRecord }: {// The dock is one line tall, so on a narrow window the placeholder gets an// ellipsis (see .agent-dock input in workspace.css). Keep the full sentence// reachable on hover so nothing that matters is lost to the truncation.+ //+ // It used to offer to "generate a shot", which the generator flatly denies a+ // few inches above it: this bar changes a shot that already exists, it does+ // not start one. New shots are made in the generator. Two panels cannot+ // describe the same bar differently, so this one follows that rule.const placeholder = ready- ? `Ask Pol to refine ${context}, generate a shot, or make an edit…`+ ? `Ask Pol to refine ${context}, change a shot you already have, or make an edit…`: blockedDetail ?? "No agent provider is available";const send = async () => {diff --git a/src/components/workspace/MediaThumbnail.tsx b/src/components/workspace/MediaThumbnail.tsxindex b8da776..7bdb3d3 100644--- a/src/components/workspace/MediaThumbnail.tsx+++ b/src/components/workspace/MediaThumbnail.tsx@@ -1,9 +1,10 @@import { Image as ImageIcon, Music2, TriangleAlert, Video } from "lucide-react";-import { useEffect, useState } from "react";+import { useEffect, useRef, useState } from "react";import { readMediaFileUrl } from "../../lib/persistence";import type { ProjectAsset } from "../../lib/project";-/* A real picture of the media, not a striped placeholder.+/* A real picture of the media, not a striped placeholder — and, from the same+ * decode, the file's real length and size.** Project files live outside the webview's reach, so the bytes come back* through a Tauri command as a blob URL. That blob is the whole file — a@@ -11,6 +12,13 @@ import type { ProjectAsset } from "../../lib/project";* canvas, and revoked immediately. What survives is a ~320px JPEG data URL,* which is what the panel actually shows.*+ * The same <video> that gives up the frame already knows `duration`,+ * `videoWidth`, and `videoHeight`, so measuring the asset costs nothing beyond+ * the read this component was already doing. Nothing else in PolStudio has+ * decoded these files yet, so this is where an imported asset stops being+ * `durationMs: null` and becomes a number — and a clip dropped from it stops+ * being a stand-in five seconds long.+ ** No FFmpeg and no WebCodecs here on purpose: the <video> element already owns* a hardware decoder, and one still frame is all this needs. */@@ -23,6 +31,30 @@ const POSTER_TIME_SECONDS = 0.5;time would be felt. Bounded, because a data URL is small but not free. */const POSTER_CACHE = new Map<string, string>();const CACHE_LIMIT = 64;+/* Files the browser opened but could tell us nothing about — a stream with no+ declared duration, a container it half-understands. Without this the asset+ stays unmeasured, "unmeasured" reads as "try again", and every remount reads+ the whole file back for the same silence. */+const UNMEASURABLE = new Set<string>();++/** What the webview actually measured. A field is null when it was not+ * measured — never a plausible stand-in. Milliseconds and pixels. */+export interface MeasuredMedia {+ durationMs: number | null;+ width: number | null;+ height: number | null;+}++const NOTHING_MEASURED: MeasuredMedia = { durationMs: null, width: null, height: null };+const measuredSomething = (measured: MeasuredMedia) =>+ measured.durationMs !== null || measured.width !== null || measured.height !== null;++/* A media element reports `duration` as NaN before metadata arrives and+ Infinity for a live or unseekable stream. Neither is a length. */+const durationMsOf = (element: HTMLMediaElement): number | null =>+ Number.isFinite(element.duration) && element.duration > 0 ? Math.round(element.duration * 1000) : null;++const pixelsOf = (value: number): number | null => (Number.isFinite(value) && value > 0 ? value : null);function cachePoster(key: string, poster: string) {if (POSTER_CACHE.size >= CACHE_LIMIT) {@@ -44,19 +76,26 @@ function drawPoster(source: CanvasImageSource, width: number, height: number): sreturn canvas.toDataURL("image/jpeg", 0.72);}-const settled = (element: HTMLVideoElement, event: string) =>+const settled = (element: HTMLMediaElement, event: string) =>new Promise<void>((resolve, reject) => {element.addEventListener(event, () => resolve(), { once: true });element.addEventListener("error", () => reject(new Error("This file could not be decoded.")), { once: true });});-async function videoPoster(url: string): Promise<string> {+async function videoPoster(url: string): Promise<{ poster: string; measured: MeasuredMedia }> {const video = document.createElement("video");video.muted = true;video.preload = "auto";video.playsInline = true;video.src = url;await settled(video, "loadeddata");+ // Read the numbers before the seek: this is the one moment the whole file is+ // open, and a failed seek must not cost the measurement too.+ const measured: MeasuredMedia = {+ durationMs: durationMsOf(video),+ width: pixelsOf(video.videoWidth),+ height: pixelsOf(video.videoHeight),+ };const target = Number.isFinite(video.duration) ? Math.min(video.duration / 2, POSTER_TIME_SECONDS) : 0;// A seek that never lands would leave the thumbnail loading forever; frame// zero is a perfectly good fallback, so a failure here is not fatal.@@ -69,24 +108,63 @@ async function videoPoster(url: string): Promise<string> {// Detach the source so the decoder and the blob are released now.video.removeAttribute("src");video.load();- return poster;+ return { poster, measured };}-async function imagePoster(url: string): Promise<string> {+async function imagePoster(url: string): Promise<{ poster: string; measured: MeasuredMedia }> {const image = new Image();image.src = url;await image.decode();- return drawPoster(image, image.naturalWidth, image.naturalHeight);+ // A still has a size and no length. Leaving durationMs null is the whole+ // point: how long a still is on screen is the cut's decision, not the file's.+ return {+ poster: drawPoster(image, image.naturalWidth, image.naturalHeight),+ measured: { durationMs: null, width: pixelsOf(image.naturalWidth), height: pixelsOf(image.naturalHeight) },+ };+}++/* Sound has no picture to draw, but it has the length the timeline needs, and+ `preload="metadata"` is enough to read it. */+async function measureAudio(url: string): Promise<MeasuredMedia> {+ const audio = document.createElement("audio");+ audio.preload = "metadata";+ audio.src = url;+ await settled(audio, "loadedmetadata");+ const measured: MeasuredMedia = { durationMs: durationMsOf(audio), width: null, height: null };+ audio.removeAttribute("src");+ audio.load();+ return measured;}-export function MediaThumbnail({ folderPath, asset }: { folderPath: string; asset: ProjectAsset }) {+/** Whether this asset still has something worth measuring. A still never gets+ * a duration, so asking for one forever would re-read the file on every+ * mount; sound never gets a size. */+const wantsMeasuring = (asset: ProjectAsset) =>+ asset.kind === "image" ? !asset.width || !asset.height : !asset.durationMs;++export function MediaThumbnail({ folderPath, asset, onMeasured }: {+ folderPath: string;+ asset: ProjectAsset;+ /** Called once with whatever the decode could measure, so the project can+ * record the file's real length instead of carrying nulls. Omit it and this+ * component only draws. */+ onMeasured?: (measured: MeasuredMedia) => void;+}) {const cacheKey = `${folderPath}::${asset.sourcePath ?? asset.relativePath}`;const [poster, setPoster] = useState<string | null>(() => POSTER_CACHE.get(cacheKey) ?? null);const [failed, setFailed] = useState(false);+ /* Held in a ref because the parent passes a fresh closure on every render,+ and a callback in the dependency list would re-read the file each time. */+ const report = useRef(onMeasured);+ report.current = onMeasured;++ const needsPoster = asset.kind !== "audio" && !POSTER_CACHE.has(cacheKey);+ const needsMeasuring = Boolean(onMeasured) && !UNMEASURABLE.has(cacheKey) && wantsMeasuring(asset);useEffect(() => {- // Sound has no picture, and a cached poster is already the answer.- if (asset.kind === "audio" || POSTER_CACHE.has(cacheKey)) return;+ // Nothing to draw and nothing left to learn: sound has no picture, a cached+ // poster is already the answer, and a measured asset stays measured.+ if (!needsPoster && !needsMeasuring) return;let live = true;setFailed(false);void (async () => {@@ -96,17 +174,28 @@ export function MediaThumbnail({ folderPath, asset }: { folderPath: string; asse// Browser preview: there is no project folder, so there is nothing to// show and nothing has gone wrong. The kind icon stands in.if (!url) return;- const value = asset.kind === "image" ? await imagePoster(url) : await videoPoster(url);- cachePoster(cacheKey, value);- if (live) setPoster(value);+ let measured = NOTHING_MEASURED;+ if (asset.kind === "audio") {+ measured = await measureAudio(url);+ } else {+ const value = asset.kind === "image" ? await imagePoster(url) : await videoPoster(url);+ cachePoster(cacheKey, value.poster);+ measured = value.measured;+ if (live) setPoster(value.poster);+ }+ // Only ever hand up numbers something read off the file. When the file+ // gave none, remember that rather than reading it again next mount.+ if (measuredSomething(measured)) report.current?.(measured);+ else UNMEASURABLE.add(cacheKey);} catch {+ UNMEASURABLE.add(cacheKey);if (live) setFailed(true);} finally {if (url) URL.revokeObjectURL(url);}})();return () => { live = false; };- }, [cacheKey, folderPath, asset.relativePath, asset.sourcePath, asset.mimeType, asset.kind]);+ }, [cacheKey, folderPath, asset.relativePath, asset.sourcePath, asset.mimeType, asset.kind, needsPoster, needsMeasuring]);if (poster) {return <span className="media-thumb media-thumb--poster">diff --git a/src/components/workspace/TimelineView.tsx b/src/components/workspace/TimelineView.tsxindex d52d245..f8af17c 100644--- a/src/components/workspace/TimelineView.tsx+++ b/src/components/workspace/TimelineView.tsx@@ -7,14 +7,17 @@ import { useEffect, useMemo, useRef, useState } from "react";import { importMediaFiles, isTauri } from "../../lib/persistence";import { loadMediaLayout, saveMediaLayout, type MediaLayout } from "../../lib/settings";import type { ProjectAsset, ProjectConfig, TimelineClip } from "../../lib/project";-import { MediaThumbnail } from "./MediaThumbnail";+import { MediaThumbnail, type MeasuredMedia } from "./MediaThumbnail";const MIN_DURATION = 10_000;const NOT_YET = "Not available yet. This control doesn’t change your project.";-/* What a dropped media file is worth on the timeline until PolStudio can read- its real length. Imported assets carry no duration — nothing has decoded- them — so the clip starts at a stated default and the inspector's Lasts- field is editable so the user can set the truth. */+/* How long a dropped clip is when the file itself cannot say.+ A drop normally lasts as long as the footage: the media panel decodes each+ file to draw its thumbnail and records the duration it read there (see+ MediaThumbnail), so a 40-second rush drops as 40 seconds. This default+ covers the two cases where there is no such number — a still image, which+ has no length of its own, and a file the browser opened but could not+ measure. In both, the Lasts field in the inspector sets the truth. */const DROPPED_CLIP_MS = 5_000;/* The drag payload is the asset id. A custom type keeps files dragged in fromthe desktop, and text dragged from anywhere else, out of the drop handler. */@@ -46,6 +49,20 @@ const timecode = (ms: number) => {return `00:${String(Math.floor(seconds / 60)).padStart(2, "0")}:${String(seconds % 60).padStart(2, "0")}:${String(frames).padStart(2, "0")}`;};+/* The Lasts field read back. It prints MM:SS:FF, so that is what it takes;+ two parts are the minutes and seconds of it, and a bare number is seconds,+ because "8" and "8.5" are what people actually type into a length box.+ Null means this is not a length yet — half-typed, or not a number at all —+ and half-typed text must never be turned into a clip length. */+const parseDuration = (value: string, frameRate: number): number | null => {+ const parts = value.trim().split(":");+ if (parts.length > 3 || parts.some((part) => !/^\d+(\.\d+)?$/.test(part.trim()))) return null;+ const numbers = parts.map((part) => Number(part));+ const frames = parts.length === 3 ? numbers[2] * (1000 / frameRate) : 0;+ const seconds = parts.length === 1 ? numbers[0] : numbers[0] * 60 + numbers[1];+ return Math.round(seconds * 1000 + frames);+};+export function TimelineView({ config, folderPath, onChange, onOpenGenerator }: { config: ProjectConfig; folderPath: string; onChange: (next: ProjectConfig) => void; onOpenGenerator: (jobId?: string) => void }) {const firstClip = config.timeline.tracks.flatMap((track) => track.clips)[0];const [selectedId, setSelectedId] = useState(firstClip?.id ?? "");@@ -68,6 +85,11 @@ export function TimelineView({ config, folderPath, onChange, onOpenGenerator }:is being offered — it can only see the TYPE. Remembering what dragstartput there is the only way to know whether this lane can take it. */const [draggedAsset, setDraggedAsset] = useState<ProjectAsset | undefined>(undefined);+ /* What the user is typing into Lasts, while they are typing it. The field+ otherwise shows the clip's own length, and "00:0" on the way to "00:08"+ must not be snapped back to a formatted timecode mid-keystroke. Null means+ nothing is being typed and the clip speaks for itself. */+ const [durationDraft, setDurationDraft] = useState<string | null>(null);const tracks = config.timeline.tracks;const selected = useMemo(() => tracks.flatMap((track) => track.clips).find((clip) => clip.id === selectedId), [tracks, selectedId]);const selectedTrack = tracks.find((track) => track.id === selected?.trackId);@@ -88,6 +110,9 @@ export function TimelineView({ config, folderPath, onChange, onOpenGenerator }:useEffect(() => { if (clipCount === 0 && playing) setPlaying(false); }, [clipCount, playing]);+ // Another clip, another length: what was being typed belonged to the old one.+ useEffect(() => { setDurationDraft(null); }, [selectedId]);+/* Wheel over the timeline scrubs it. Registered by hand rather than withReact's onWheel because React attaches wheel listeners passively, and apassive listener cannot preventDefault — without that the project content@@ -124,6 +149,10 @@ export function TimelineView({ config, folderPath, onChange, onOpenGenerator }:};const duplicateSelected = () => {if (!selected || !selectedTrack || selectedTrack.locked) return;+ /* The spread carries sourceStartMs with everything else, so duplicating a+ clip that was split off the middle of a rush copies THAT range of the+ footage, not the head of the file. Only the id, the position on the+ ruler, and the name are new. */const copy = { ...selected, id: `${selected.id}-copy-${Date.now()}`, startMs: Math.min(duration - selected.durationMs, selected.startMs + selected.durationMs), label: `${selected.label} copy` };updateTracks(tracks.map((track) => track.id === selected.trackId ? { ...track, clips: [...track.clips, copy] } : track));setSelectedId(copy.id);@@ -198,10 +227,52 @@ export function TimelineView({ config, folderPath, onChange, onOpenGenerator }:const assetById = (id: string) => config.assets.find((asset) => asset.id === id);+ /* The media panel decodes each file to draw its thumbnail, and hands back+ what that decode measured. Write it onto the asset so the NEXT drop is as+ long as the footage, and so the file is only ever read once for it.+ Only fields nothing knew yet are filled — a measurement never overwrites a+ number already on the asset, and it never retimes a clip already cut into+ the timeline, which is the user's decision and not the file's. Bailing out+ when nothing changed is what stops this looping: the write re-renders the+ panel, which would otherwise measure again. */+ const recordMeasured = (assetId: string, measured: MeasuredMedia) => {+ const asset = assetById(assetId);+ if (!asset) return;+ const next: ProjectAsset = {+ ...asset,+ durationMs: asset.durationMs ?? measured.durationMs,+ width: asset.width ?? measured.width,+ height: asset.height ?? measured.height,+ };+ if (next.durationMs === asset.durationMs && next.width === asset.width && next.height === asset.height) return;+ onChange({ ...config, assets: config.assets.map((item) => (item.id === assetId ? next : item)) });+ };++ /* A trim, typed. The source is the ceiling when its length is known — a clip+ cannot play footage the file does not have — and one frame is the floor,+ because zod wants a positive whole number of milliseconds and a clip+ shorter than a frame cannot be shown. A still has no source length, so it+ is only bounded from below. */+ const selectedAsset = selected ? assetById(selected.assetId) : undefined;+ const sourceLimitMs = selected && selectedAsset?.kind !== "image" && selectedAsset?.durationMs+ ? Math.max(1, selectedAsset.durationMs - selected.sourceStartMs)+ : null;+ const minClipMs = Math.max(1, Math.round(1000 / config.settings.frameRate));+ const typeDuration = (value: string) => {+ setDurationDraft(value);+ const parsed = selected ? parseDuration(value, config.settings.frameRate) : null;+ if (parsed === null || !selected) return;+ const ceiling = sourceLimitMs ?? Number.MAX_SAFE_INTEGER;+ updateClip(selected.id, { durationMs: Math.min(Math.max(parsed, minClipMs), Math.max(ceiling, minClipMs)) });+ };+const dropAsset = (trackId: string, assetId: string, ratio: number) => {const track = tracks.find((item) => item.id === trackId);const asset = assetById(assetId);if (!track || !acceptsAsset(track, asset) || !asset) return;+ /* As long as the footage, when the footage has said how long it is. A still+ image and a file nothing could measure have no length to honour, so they+ get the stated default — see DROPPED_CLIP_MS. */const durationMs = asset.durationMs && asset.durationMs > 0 ? asset.durationMs : DROPPED_CLIP_MS;/* Clamp so a drop near the right edge still lands a whole clip on theruler, against THIS clip's length rather than the 5s default — a long@@ -230,9 +301,15 @@ export function TimelineView({ config, folderPath, onChange, onOpenGenerator }:const addScene = () => {// A new project already carries an unstarted draft made from the user's own// words, so open that rather than stacking a near-identical second shot.- // With none left, go to an empty composer: synthesising a shot from- // config.brief.prompt would describe the whole VIDEO as if it were a single- // shot and hand it back as words the user never wrote.+ // With none left, go to an empty composer.+ //+ // createProjectConfig does seed that FIRST draft from config.brief.prompt,+ // and this is not a disagreement with it. The brief describes the whole+ // video, which is a fair opening for the one shot a project starts with —+ // there is nothing else to go on, and it is there to be rewritten. Every+ // later shot has that same paragraph already spent on the shot before it,+ // so repeating it would describe the whole video a second time and call it+ // shot two.onOpenGenerator(config.generationJobs.find((job) => job.status === "draft")?.id);};@@ -295,7 +372,7 @@ export function TimelineView({ config, folderPath, onChange, onOpenGenerator }:onDragEnd={() => { setDraggedAsset(undefined); setDropTrackId(null); }}title={`${asset.name} — drag onto a ${asset.kind === "audio" ? "sound" : "video"} track below`}>- <MediaThumbnail folderPath={folderPath} asset={asset} />+ <MediaThumbnail folderPath={folderPath} asset={asset} onMeasured={(measured) => recordMeasured(asset.id, measured)} /><b>{asset.name}</b>{/* Duration is only claimed when something actually measured it. */}<small>{asset.kind}{asset.durationMs ? ` · ${(asset.durationMs / 1000).toFixed(1)}s` : ""}</small>@@ -343,7 +420,18 @@ export function TimelineView({ config, folderPath, onChange, onOpenGenerator }:<label><span>Name</span><input value={selected.label} onChange={(event) => updateClip(selected.id, { label: event.target.value || "Untitled clip" })} /></label><div className="field-pair"><label><span>Starts at</span><input value={timecode(selected.startMs).slice(3)} readOnly /></label>- <label><span>Lasts</span><input value={timecode(selected.durationMs).slice(3)} readOnly /></label>+ {/* The one field here that has to be editable: a clip is only+ as long as the cut says, and until this took typing there+ was no way at all to change a clip's length except to split+ it, which can only ever shorten. Typed as minutes, seconds,+ and frames, or as plain seconds. */}+ <label><span>Lasts</span><input+ value={durationDraft ?? timecode(selected.durationMs).slice(3)}+ onChange={(event) => typeDuration(event.target.value)}+ onBlur={() => setDurationDraft(null)}+ title={`How long this clip lasts — minutes:seconds:frames, or just seconds.${sourceLimitMs === null ? "" : ` This footage has ${(sourceLimitMs / 1000).toFixed(1)}s left from where the clip starts in it.`}`}+ inputMode="decimal"+ /></label></div></section><section className="inspector-section inspector-section--pending">