Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Measure media on decode and let the inspector set a clip's length

Bitti754876fce56f
diff --git a/src/components/workspace/AgentDock.tsx b/src/components/workspace/AgentDock.tsx
index 3a923c8..b7ebde7 100644
--- a/src/components/workspace/AgentDock.tsx
+++ b/src/components/workspace/AgentDock.tsx
@@ -33,8 +33,13 @@ export function AgentDock({ context, record, providers, onRecord }: {
// The dock is one line tall, so on a narrow window the placeholder gets an
// ellipsis (see .agent-dock input in workspace.css). Keep the full sentence
// reachable on hover so nothing that matters is lost to the truncation.
+ //
+ // It used to offer to "generate a shot", which the generator flatly denies a
+ // few inches above it: this bar changes a shot that already exists, it does
+ // not start one. New shots are made in the generator. Two panels cannot
+ // describe the same bar differently, so this one follows that rule.
const placeholder = ready
- ? `Ask Pol to refine ${context}, generate a shot, or make an edit…`
+ ? `Ask Pol to refine ${context}, change a shot you already have, or make an edit…`
: blockedDetail ?? "No agent provider is available";
const send = async () => {
diff --git a/src/components/workspace/MediaThumbnail.tsx b/src/components/workspace/MediaThumbnail.tsx
index b8da776..7bdb3d3 100644
--- a/src/components/workspace/MediaThumbnail.tsx
+++ b/src/components/workspace/MediaThumbnail.tsx
@@ -1,9 +1,10 @@
import { Image as ImageIcon, Music2, TriangleAlert, Video } from "lucide-react";
-import { useEffect, useState } from "react";
+import { useEffect, useRef, useState } from "react";
import { readMediaFileUrl } from "../../lib/persistence";
import type { ProjectAsset } from "../../lib/project";
-/* A real picture of the media, not a striped placeholder.
+/* A real picture of the media, not a striped placeholder — and, from the same
+ * decode, the file's real length and size.
*
* Project files live outside the webview's reach, so the bytes come back
* through a Tauri command as a blob URL. That blob is the whole file — a
@@ -11,6 +12,13 @@ import type { ProjectAsset } from "../../lib/project";
* canvas, and revoked immediately. What survives is a ~320px JPEG data URL,
* which is what the panel actually shows.
*
+ * The same <video> that gives up the frame already knows `duration`,
+ * `videoWidth`, and `videoHeight`, so measuring the asset costs nothing beyond
+ * the read this component was already doing. Nothing else in PolStudio has
+ * decoded these files yet, so this is where an imported asset stops being
+ * `durationMs: null` and becomes a number — and a clip dropped from it stops
+ * being a stand-in five seconds long.
+ *
* No FFmpeg and no WebCodecs here on purpose: the <video> element already owns
* a hardware decoder, and one still frame is all this needs. */
@@ -23,6 +31,30 @@ const POSTER_TIME_SECONDS = 0.5;
time would be felt. Bounded, because a data URL is small but not free. */
const POSTER_CACHE = new Map<string, string>();
const CACHE_LIMIT = 64;
+/* Files the browser opened but could tell us nothing about — a stream with no
+ declared duration, a container it half-understands. Without this the asset
+ stays unmeasured, "unmeasured" reads as "try again", and every remount reads
+ the whole file back for the same silence. */
+const UNMEASURABLE = new Set<string>();
+
+/** What the webview actually measured. A field is null when it was not
+ * measured — never a plausible stand-in. Milliseconds and pixels. */
+export interface MeasuredMedia {
+ durationMs: number | null;
+ width: number | null;
+ height: number | null;
+}
+
+const NOTHING_MEASURED: MeasuredMedia = { durationMs: null, width: null, height: null };
+const measuredSomething = (measured: MeasuredMedia) =>
+ measured.durationMs !== null || measured.width !== null || measured.height !== null;
+
+/* A media element reports `duration` as NaN before metadata arrives and
+ Infinity for a live or unseekable stream. Neither is a length. */
+const durationMsOf = (element: HTMLMediaElement): number | null =>
+ Number.isFinite(element.duration) && element.duration > 0 ? Math.round(element.duration * 1000) : null;
+
+const pixelsOf = (value: number): number | null => (Number.isFinite(value) && value > 0 ? value : null);
function cachePoster(key: string, poster: string) {
if (POSTER_CACHE.size >= CACHE_LIMIT) {
@@ -44,19 +76,26 @@ function drawPoster(source: CanvasImageSource, width: number, height: number): s
return canvas.toDataURL("image/jpeg", 0.72);
}
-const settled = (element: HTMLVideoElement, event: string) =>
+const settled = (element: HTMLMediaElement, event: string) =>
new Promise<void>((resolve, reject) => {
element.addEventListener(event, () => resolve(), { once: true });
element.addEventListener("error", () => reject(new Error("This file could not be decoded.")), { once: true });
});
-async function videoPoster(url: string): Promise<string> {
+async function videoPoster(url: string): Promise<{ poster: string; measured: MeasuredMedia }> {
const video = document.createElement("video");
video.muted = true;
video.preload = "auto";
video.playsInline = true;
video.src = url;
await settled(video, "loadeddata");
+ // Read the numbers before the seek: this is the one moment the whole file is
+ // open, and a failed seek must not cost the measurement too.
+ const measured: MeasuredMedia = {
+ durationMs: durationMsOf(video),
+ width: pixelsOf(video.videoWidth),
+ height: pixelsOf(video.videoHeight),
+ };
const target = Number.isFinite(video.duration) ? Math.min(video.duration / 2, POSTER_TIME_SECONDS) : 0;
// A seek that never lands would leave the thumbnail loading forever; frame
// zero is a perfectly good fallback, so a failure here is not fatal.
@@ -69,24 +108,63 @@ async function videoPoster(url: string): Promise<string> {
// Detach the source so the decoder and the blob are released now.
video.removeAttribute("src");
video.load();
- return poster;
+ return { poster, measured };
}
-async function imagePoster(url: string): Promise<string> {
+async function imagePoster(url: string): Promise<{ poster: string; measured: MeasuredMedia }> {
const image = new Image();
image.src = url;
await image.decode();
- return drawPoster(image, image.naturalWidth, image.naturalHeight);
+ // A still has a size and no length. Leaving durationMs null is the whole
+ // point: how long a still is on screen is the cut's decision, not the file's.
+ return {
+ poster: drawPoster(image, image.naturalWidth, image.naturalHeight),
+ measured: { durationMs: null, width: pixelsOf(image.naturalWidth), height: pixelsOf(image.naturalHeight) },
+ };
+}
+
+/* Sound has no picture to draw, but it has the length the timeline needs, and
+ `preload="metadata"` is enough to read it. */
+async function measureAudio(url: string): Promise<MeasuredMedia> {
+ const audio = document.createElement("audio");
+ audio.preload = "metadata";
+ audio.src = url;
+ await settled(audio, "loadedmetadata");
+ const measured: MeasuredMedia = { durationMs: durationMsOf(audio), width: null, height: null };
+ audio.removeAttribute("src");
+ audio.load();
+ return measured;
}
-export function MediaThumbnail({ folderPath, asset }: { folderPath: string; asset: ProjectAsset }) {
+/** Whether this asset still has something worth measuring. A still never gets
+ * a duration, so asking for one forever would re-read the file on every
+ * mount; sound never gets a size. */
+const wantsMeasuring = (asset: ProjectAsset) =>
+ asset.kind === "image" ? !asset.width || !asset.height : !asset.durationMs;
+
+export function MediaThumbnail({ folderPath, asset, onMeasured }: {
+ folderPath: string;
+ asset: ProjectAsset;
+ /** Called once with whatever the decode could measure, so the project can
+ * record the file's real length instead of carrying nulls. Omit it and this
+ * component only draws. */
+ onMeasured?: (measured: MeasuredMedia) => void;
+}) {
const cacheKey = `${folderPath}::${asset.sourcePath ?? asset.relativePath}`;
const [poster, setPoster] = useState<string | null>(() => POSTER_CACHE.get(cacheKey) ?? null);
const [failed, setFailed] = useState(false);
+ /* Held in a ref because the parent passes a fresh closure on every render,
+ and a callback in the dependency list would re-read the file each time. */
+ const report = useRef(onMeasured);
+ report.current = onMeasured;
+
+ const needsPoster = asset.kind !== "audio" && !POSTER_CACHE.has(cacheKey);
+ const needsMeasuring = Boolean(onMeasured) && !UNMEASURABLE.has(cacheKey) && wantsMeasuring(asset);
useEffect(() => {
- // Sound has no picture, and a cached poster is already the answer.
- if (asset.kind === "audio" || POSTER_CACHE.has(cacheKey)) return;
+ // Nothing to draw and nothing left to learn: sound has no picture, a cached
+ // poster is already the answer, and a measured asset stays measured.
+ if (!needsPoster && !needsMeasuring) return;
let live = true;
setFailed(false);
void (async () => {
@@ -96,17 +174,28 @@ export function MediaThumbnail({ folderPath, asset }: { folderPath: string; asse
// Browser preview: there is no project folder, so there is nothing to
// show and nothing has gone wrong. The kind icon stands in.
if (!url) return;
- const value = asset.kind === "image" ? await imagePoster(url) : await videoPoster(url);
- cachePoster(cacheKey, value);
- if (live) setPoster(value);
+ let measured = NOTHING_MEASURED;
+ if (asset.kind === "audio") {
+ measured = await measureAudio(url);
+ } else {
+ const value = asset.kind === "image" ? await imagePoster(url) : await videoPoster(url);
+ cachePoster(cacheKey, value.poster);
+ measured = value.measured;
+ if (live) setPoster(value.poster);
+ }
+ // Only ever hand up numbers something read off the file. When the file
+ // gave none, remember that rather than reading it again next mount.
+ if (measuredSomething(measured)) report.current?.(measured);
+ else UNMEASURABLE.add(cacheKey);
} catch {
+ UNMEASURABLE.add(cacheKey);
if (live) setFailed(true);
} finally {
if (url) URL.revokeObjectURL(url);
}
})();
return () => { live = false; };
- }, [cacheKey, folderPath, asset.relativePath, asset.sourcePath, asset.mimeType, asset.kind]);
+ }, [cacheKey, folderPath, asset.relativePath, asset.sourcePath, asset.mimeType, asset.kind, needsPoster, needsMeasuring]);
if (poster) {
return <span className="media-thumb media-thumb--poster">
diff --git a/src/components/workspace/TimelineView.tsx b/src/components/workspace/TimelineView.tsx
index d52d245..f8af17c 100644
--- a/src/components/workspace/TimelineView.tsx
+++ b/src/components/workspace/TimelineView.tsx
@@ -7,14 +7,17 @@ import { useEffect, useMemo, useRef, useState } from "react";
import { importMediaFiles, isTauri } from "../../lib/persistence";
import { loadMediaLayout, saveMediaLayout, type MediaLayout } from "../../lib/settings";
import type { ProjectAsset, ProjectConfig, TimelineClip } from "../../lib/project";
-import { MediaThumbnail } from "./MediaThumbnail";
+import { MediaThumbnail, type MeasuredMedia } from "./MediaThumbnail";
const MIN_DURATION = 10_000;
const NOT_YET = "Not available yet. This control doesn’t change your project.";
-/* What a dropped media file is worth on the timeline until PolStudio can read
- its real length. Imported assets carry no duration — nothing has decoded
- them — so the clip starts at a stated default and the inspector's Lasts
- field is editable so the user can set the truth. */
+/* How long a dropped clip is when the file itself cannot say.
+ A drop normally lasts as long as the footage: the media panel decodes each
+ file to draw its thumbnail and records the duration it read there (see
+ MediaThumbnail), so a 40-second rush drops as 40 seconds. This default
+ covers the two cases where there is no such number — a still image, which
+ has no length of its own, and a file the browser opened but could not
+ measure. In both, the Lasts field in the inspector sets the truth. */
const DROPPED_CLIP_MS = 5_000;
/* The drag payload is the asset id. A custom type keeps files dragged in from
the desktop, and text dragged from anywhere else, out of the drop handler. */
@@ -46,6 +49,20 @@ const timecode = (ms: number) => {
return `00:${String(Math.floor(seconds / 60)).padStart(2, "0")}:${String(seconds % 60).padStart(2, "0")}:${String(frames).padStart(2, "0")}`;
};
+/* The Lasts field read back. It prints MM:SS:FF, so that is what it takes;
+ two parts are the minutes and seconds of it, and a bare number is seconds,
+ because "8" and "8.5" are what people actually type into a length box.
+ Null means this is not a length yet — half-typed, or not a number at all —
+ and half-typed text must never be turned into a clip length. */
+const parseDuration = (value: string, frameRate: number): number | null => {
+ const parts = value.trim().split(":");
+ if (parts.length > 3 || parts.some((part) => !/^\d+(\.\d+)?$/.test(part.trim()))) return null;
+ const numbers = parts.map((part) => Number(part));
+ const frames = parts.length === 3 ? numbers[2] * (1000 / frameRate) : 0;
+ const seconds = parts.length === 1 ? numbers[0] : numbers[0] * 60 + numbers[1];
+ return Math.round(seconds * 1000 + frames);
+};
+
export function TimelineView({ config, folderPath, onChange, onOpenGenerator }: { config: ProjectConfig; folderPath: string; onChange: (next: ProjectConfig) => void; onOpenGenerator: (jobId?: string) => void }) {
const firstClip = config.timeline.tracks.flatMap((track) => track.clips)[0];
const [selectedId, setSelectedId] = useState(firstClip?.id ?? "");
@@ -68,6 +85,11 @@ export function TimelineView({ config, folderPath, onChange, onOpenGenerator }:
is being offered — it can only see the TYPE. Remembering what dragstart
put there is the only way to know whether this lane can take it. */
const [draggedAsset, setDraggedAsset] = useState<ProjectAsset | undefined>(undefined);
+ /* What the user is typing into Lasts, while they are typing it. The field
+ otherwise shows the clip's own length, and "00:0" on the way to "00:08"
+ must not be snapped back to a formatted timecode mid-keystroke. Null means
+ nothing is being typed and the clip speaks for itself. */
+ const [durationDraft, setDurationDraft] = useState<string | null>(null);
const tracks = config.timeline.tracks;
const selected = useMemo(() => tracks.flatMap((track) => track.clips).find((clip) => clip.id === selectedId), [tracks, selectedId]);
const selectedTrack = tracks.find((track) => track.id === selected?.trackId);
@@ -88,6 +110,9 @@ export function TimelineView({ config, folderPath, onChange, onOpenGenerator }:
useEffect(() => { if (clipCount === 0 && playing) setPlaying(false); }, [clipCount, playing]);
+ // Another clip, another length: what was being typed belonged to the old one.
+ useEffect(() => { setDurationDraft(null); }, [selectedId]);
+
/* Wheel over the timeline scrubs it. Registered by hand rather than with
React's onWheel because React attaches wheel listeners passively, and a
passive listener cannot preventDefault — without that the project content
@@ -124,6 +149,10 @@ export function TimelineView({ config, folderPath, onChange, onOpenGenerator }:
};
const duplicateSelected = () => {
if (!selected || !selectedTrack || selectedTrack.locked) return;
+ /* The spread carries sourceStartMs with everything else, so duplicating a
+ clip that was split off the middle of a rush copies THAT range of the
+ footage, not the head of the file. Only the id, the position on the
+ ruler, and the name are new. */
const copy = { ...selected, id: `${selected.id}-copy-${Date.now()}`, startMs: Math.min(duration - selected.durationMs, selected.startMs + selected.durationMs), label: `${selected.label} copy` };
updateTracks(tracks.map((track) => track.id === selected.trackId ? { ...track, clips: [...track.clips, copy] } : track));
setSelectedId(copy.id);
@@ -198,10 +227,52 @@ export function TimelineView({ config, folderPath, onChange, onOpenGenerator }:
const assetById = (id: string) => config.assets.find((asset) => asset.id === id);
+ /* The media panel decodes each file to draw its thumbnail, and hands back
+ what that decode measured. Write it onto the asset so the NEXT drop is as
+ long as the footage, and so the file is only ever read once for it.
+ Only fields nothing knew yet are filled — a measurement never overwrites a
+ number already on the asset, and it never retimes a clip already cut into
+ the timeline, which is the user's decision and not the file's. Bailing out
+ when nothing changed is what stops this looping: the write re-renders the
+ panel, which would otherwise measure again. */
+ const recordMeasured = (assetId: string, measured: MeasuredMedia) => {
+ const asset = assetById(assetId);
+ if (!asset) return;
+ const next: ProjectAsset = {
+ ...asset,
+ durationMs: asset.durationMs ?? measured.durationMs,
+ width: asset.width ?? measured.width,
+ height: asset.height ?? measured.height,
+ };
+ if (next.durationMs === asset.durationMs && next.width === asset.width && next.height === asset.height) return;
+ onChange({ ...config, assets: config.assets.map((item) => (item.id === assetId ? next : item)) });
+ };
+
+ /* A trim, typed. The source is the ceiling when its length is known — a clip
+ cannot play footage the file does not have — and one frame is the floor,
+ because zod wants a positive whole number of milliseconds and a clip
+ shorter than a frame cannot be shown. A still has no source length, so it
+ is only bounded from below. */
+ const selectedAsset = selected ? assetById(selected.assetId) : undefined;
+ const sourceLimitMs = selected && selectedAsset?.kind !== "image" && selectedAsset?.durationMs
+ ? Math.max(1, selectedAsset.durationMs - selected.sourceStartMs)
+ : null;
+ const minClipMs = Math.max(1, Math.round(1000 / config.settings.frameRate));
+ const typeDuration = (value: string) => {
+ setDurationDraft(value);
+ const parsed = selected ? parseDuration(value, config.settings.frameRate) : null;
+ if (parsed === null || !selected) return;
+ const ceiling = sourceLimitMs ?? Number.MAX_SAFE_INTEGER;
+ updateClip(selected.id, { durationMs: Math.min(Math.max(parsed, minClipMs), Math.max(ceiling, minClipMs)) });
+ };
+
const dropAsset = (trackId: string, assetId: string, ratio: number) => {
const track = tracks.find((item) => item.id === trackId);
const asset = assetById(assetId);
if (!track || !acceptsAsset(track, asset) || !asset) return;
+ /* As long as the footage, when the footage has said how long it is. A still
+ image and a file nothing could measure have no length to honour, so they
+ get the stated default — see DROPPED_CLIP_MS. */
const durationMs = asset.durationMs && asset.durationMs > 0 ? asset.durationMs : DROPPED_CLIP_MS;
/* Clamp so a drop near the right edge still lands a whole clip on the
ruler, against THIS clip's length rather than the 5s default — a long
@@ -230,9 +301,15 @@ export function TimelineView({ config, folderPath, onChange, onOpenGenerator }:
const addScene = () => {
// A new project already carries an unstarted draft made from the user's own
// words, so open that rather than stacking a near-identical second shot.
- // With none left, go to an empty composer: synthesising a shot from
- // config.brief.prompt would describe the whole VIDEO as if it were a single
- // shot and hand it back as words the user never wrote.
+ // With none left, go to an empty composer.
+ //
+ // createProjectConfig does seed that FIRST draft from config.brief.prompt,
+ // and this is not a disagreement with it. The brief describes the whole
+ // video, which is a fair opening for the one shot a project starts with —
+ // there is nothing else to go on, and it is there to be rewritten. Every
+ // later shot has that same paragraph already spent on the shot before it,
+ // so repeating it would describe the whole video a second time and call it
+ // shot two.
onOpenGenerator(config.generationJobs.find((job) => job.status === "draft")?.id);
};
@@ -295,7 +372,7 @@ export function TimelineView({ config, folderPath, onChange, onOpenGenerator }:
onDragEnd={() => { setDraggedAsset(undefined); setDropTrackId(null); }}
title={`${asset.name} — drag onto a ${asset.kind === "audio" ? "sound" : "video"} track below`}
>
- <MediaThumbnail folderPath={folderPath} asset={asset} />
+ <MediaThumbnail folderPath={folderPath} asset={asset} onMeasured={(measured) => recordMeasured(asset.id, measured)} />
<b>{asset.name}</b>
{/* Duration is only claimed when something actually measured it. */}
<small>{asset.kind}{asset.durationMs ? ` · ${(asset.durationMs / 1000).toFixed(1)}s` : ""}</small>
@@ -343,7 +420,18 @@ export function TimelineView({ config, folderPath, onChange, onOpenGenerator }:
<label><span>Name</span><input value={selected.label} onChange={(event) => updateClip(selected.id, { label: event.target.value || "Untitled clip" })} /></label>
<div className="field-pair">
<label><span>Starts at</span><input value={timecode(selected.startMs).slice(3)} readOnly /></label>
- <label><span>Lasts</span><input value={timecode(selected.durationMs).slice(3)} readOnly /></label>
+ {/* The one field here that has to be editable: a clip is only
+ as long as the cut says, and until this took typing there
+ was no way at all to change a clip's length except to split
+ it, which can only ever shorten. Typed as minutes, seconds,
+ and frames, or as plain seconds. */}
+ <label><span>Lasts</span><input
+ value={durationDraft ?? timecode(selected.durationMs).slice(3)}
+ onChange={(event) => typeDuration(event.target.value)}
+ onBlur={() => setDurationDraft(null)}
+ title={`How long this clip lasts — minutes:seconds:frames, or just seconds.${sourceLimitMs === null ? "" : ` This footage has ${(sourceLimitMs / 1000).toFixed(1)}s left from where the clip starts in it.`}`}
+ inputMode="decimal"
+ /></label>
</div>
</section>
<section className="inspector-section inspector-section--pending">