Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Test the measured drop, the editable Lasts field, and duplicate's source range

Bitti321b2eaba32a
diff --git a/src/components/ProjectWorkspace.test.ts b/src/components/ProjectWorkspace.test.ts
index 086ab4b..67815a9 100644
--- a/src/components/ProjectWorkspace.test.ts
+++ b/src/components/ProjectWorkspace.test.ts
@@ -1,10 +1,10 @@
// @vitest-environment jsdom
import { cleanup, createEvent, fireEvent, render, screen, waitFor, within } from "@testing-library/react";
-import { createElement } from "react";
+import { createElement, useEffect, useState } from "react";
import { afterEach, describe, expect, it, vi } from "vitest";
import completeFixture from "../../fixtures/project-v1-complete.json";
-import { compileMiniMaxH3Prompt, createProjectConfig, parseProjectConfig, STORY_TRACK_ID, type ProjectAsset, type ProjectConfig } from "../lib/project";
+import { compileMiniMaxH3Prompt, createProjectConfig, parseProjectConfig, STORY_TRACK_ID, type ProjectAsset, type ProjectConfig, type TimelineClip } from "../lib/project";
import { formatDurationTimecode } from "./ProjectWorkspace";
import { ProjectWorkspace } from "./ProjectWorkspace";
import { GeneratorView } from "./workspace/GeneratorView";
@@ -29,6 +29,58 @@ async function asDesktopApp(importedImage: { name: string; relativePath: string
}
}
+/** Runs `body` with a media stack that behaves like a browser's: the desktop
+ * shell serves the bytes, blob URLs exist, and a <video>/<audio> element
+ * reports the metadata of a file this long and this big. jsdom loads no media
+ * and never fires a media event, so the decode has to be stood in for — the
+ * numbers under test are the ones the real element would expose. */
+async function withFakeDecoder(file: { seconds: number; width: number; height: number }, body: () => Promise<void>) {
+ const { invoke } = await import("@tauri-apps/api/core");
+ (window as unknown as Record<string, unknown>).__TAURI_INTERNALS__ = {};
+ vi.mocked(invoke).mockResolvedValue(new ArrayBuffer(8));
+ // jsdom implements neither of these, so they are installed rather than
+ // replaced — and left installed, because restoring them to `undefined` turns
+ // any decode still finishing into an unhandled TypeError.
+ URL.createObjectURL = () => "blob:polstudio-test";
+ URL.revokeObjectURL = () => undefined;
+ /* jsdom answers `load` and `getContext` by raising a "not implemented" error
+ on its virtual console, which vitest counts as an error in the run. The
+ component already treats a canvas it cannot get as no thumbnail, so give it
+ the plain refusal rather than the environment's complaint. */
+ const media = HTMLMediaElement.prototype.load;
+ const canvas = HTMLCanvasElement.prototype.getContext;
+ HTMLMediaElement.prototype.load = () => undefined;
+ HTMLCanvasElement.prototype.getContext = () => null;
+ const create = document.createElement.bind(document);
+ const spy = vi.spyOn(document, "createElement").mockImplementation((tag: string, options?: ElementCreationOptions) => {
+ const element = create(tag, options);
+ if (tag === "video" || tag === "audio") {
+ for (const [name, value] of [["duration", file.seconds], ["videoWidth", file.width], ["videoHeight", file.height]] as const) {
+ Object.defineProperty(element, name, { value, configurable: true });
+ }
+ // jsdom's currentTime setter neither seeks nor complains, so the seek for
+ // a poster frame has to answer for itself too.
+ let currentTime = 0;
+ Object.defineProperty(element, "currentTime", {
+ configurable: true,
+ get: () => currentTime,
+ set: (value: number) => { currentTime = value; setTimeout(() => element.dispatchEvent(new Event("seeked")), 0); },
+ });
+ setTimeout(() => { element.dispatchEvent(new Event("loadeddata")); element.dispatchEvent(new Event("loadedmetadata")); }, 0);
+ }
+ return element;
+ });
+ try {
+ await body();
+ } finally {
+ spy.mockRestore();
+ HTMLMediaElement.prototype.load = media;
+ HTMLCanvasElement.prototype.getContext = canvas;
+ delete (window as unknown as Record<string, unknown>).__TAURI_INTERNALS__;
+ vi.mocked(invoke).mockReset();
+ }
+}
+
describe("project workspace timecode", () => {
it("formats project duration as valid hours, minutes, and seconds", () => {
expect(formatDurationTimecode(0)).toBe("00:00:00");
@@ -226,6 +278,130 @@ describe("project workspace timecode", () => {
expect(rejected()).toHaveLength(0);
});
+ /** The same project with its video asset already measured, as the media panel
+ * measures it the first time it decodes the file for a thumbnail. */
+ const measuredVideo = (config: ProjectConfig, durationMs: number) => parseProjectConfig({
+ ...config,
+ assets: config.assets.map((asset) => asset.kind === "video" ? { ...asset, durationMs, width: 1920, height: 1080 } : asset),
+ });
+
+ /** That project with one clip of the video already cut into the story track. */
+ const withClip = (config: ProjectConfig, clip: Partial<TimelineClip>) => parseProjectConfig({
+ ...config,
+ timeline: { tracks: config.timeline.tracks.map((track) => track.id === STORY_TRACK_ID ? { ...track, clips: [{
+ id: "clip-macro", assetId: config.assets[0].id, trackId: STORY_TRACK_ID,
+ startMs: 0, durationMs: 40_000, sourceStartMs: 0, label: "Macro footage", color: null, status: "approved", ...clip,
+ }] } : track) },
+ });
+
+ const droppedClip = (calls: { mock: { calls: unknown[][] } }) =>
+ ((calls.mock.calls[0][0] as ProjectConfig).timeline.tracks.flatMap((track) => track.clips))[0];
+
+ it("drops a clip as long as the footage, and falls back to the stated default only when nothing measured it", () => {
+ /* B3: every dropped clip used to be five seconds, forever — a 40-second
+ rush included, with no way anywhere in the app to change it. */
+ const known = measuredVideo(projectWithMedia(), 40_000);
+ const first = mediaPanel(known);
+ const transfer = fakeDataTransfer();
+ fireEvent.dragStart(cardFor(known.assets[0]), { dataTransfer: transfer });
+ fireEvent.drop(laneFor(first.container, known, "video"), { dataTransfer: transfer });
+ expect(droppedClip(first.onChange).durationMs).toBe(40_000);
+ expect(parseProjectConfig(first.onChange.mock.calls[0][0] as ProjectConfig)).toBeTruthy();
+ cleanup();
+
+ // Nothing has decoded this one, so there is no length to honour and the
+ // documented stand-in is what lands.
+ const unknown = projectWithMedia();
+ const second = mediaPanel(unknown);
+ const transferTwo = fakeDataTransfer();
+ fireEvent.dragStart(cardFor(unknown.assets[0]), { dataTransfer: transferTwo });
+ fireEvent.drop(laneFor(second.container, unknown, "video"), { dataTransfer: transferTwo });
+ expect(droppedClip(second.onChange).durationMs).toBe(5_000);
+ });
+
+ it("retimes the selected clip from the Lasts field, and will not run past the footage", () => {
+ const config = withClip(measuredVideo(projectWithMedia(), 40_000), {});
+ const onChange = vi.fn();
+ render(createElement(TimelineView, { config, folderPath: "C:\\Ceramic Lamp", onChange, onOpenGenerator: () => undefined }));
+ const lasts = screen.getByTitle(/^How long this clip lasts/) as HTMLInputElement;
+ // It was readOnly, which made the only stated way to fix a clip's length a
+ // field that could not be typed into.
+ expect(lasts.readOnly).toBe(false);
+ expect(lasts.value).toBe("00:40:00");
+ const lastDuration = () => (onChange.mock.calls.at(-1)![0] as ProjectConfig).timeline.tracks.flatMap((track) => track.clips)[0].durationMs;
+
+ // Plain seconds, and the minutes:seconds:frames the field itself prints.
+ fireEvent.change(lasts, { target: { value: "8" } });
+ expect(lastDuration()).toBe(8_000);
+ fireEvent.change(lasts, { target: { value: "00:12:15" } });
+ expect(lastDuration()).toBe(12_500);
+ expect(parseProjectConfig(onChange.mock.calls.at(-1)![0] as ProjectConfig)).toBeTruthy();
+
+ // Half-typed text is not a length, and must not be turned into one.
+ const before = onChange.mock.calls.length;
+ fireEvent.change(lasts, { target: { value: "00:" } });
+ expect(onChange.mock.calls.length).toBe(before);
+ expect(lasts.value).toBe("00:");
+
+ // Clamped at both ends: a clip cannot be nothing, and cannot play footage
+ // the file does not have.
+ fireEvent.change(lasts, { target: { value: "0" } });
+ expect(lastDuration()).toBe(33);
+ fireEvent.change(lasts, { target: { value: "90" } });
+ expect(lastDuration()).toBe(40_000);
+ expect(parseProjectConfig(onChange.mock.calls.at(-1)![0] as ProjectConfig)).toBeTruthy();
+
+ // Off the field, the clip's own length is what it shows again.
+ fireEvent.blur(lasts);
+ expect(lasts.value).toBe("00:40:00");
+ });
+
+ it("keeps a clip's place in the source when it is duplicated", () => {
+ // Otherwise every copy replays the head of the file, which is the same five
+ // seconds the original already showed.
+ const config = withClip(measuredVideo(projectWithMedia(), 40_000), { startMs: 0, durationMs: 6_000, sourceStartMs: 12_000 });
+ const onChange = vi.fn();
+ render(createElement(TimelineView, { config, folderPath: "C:\\Ceramic Lamp", onChange, onOpenGenerator: () => undefined }));
+ fireEvent.click(screen.getByRole("button", { name: "Duplicate" }));
+ const next = onChange.mock.calls[0][0] as ProjectConfig;
+ const clips = next.timeline.tracks.flatMap((track) => track.clips);
+ expect(clips).toHaveLength(2);
+ const copy = clips.find((clip) => clip.id !== "clip-macro")!;
+ expect(copy.sourceStartMs).toBe(12_000);
+ expect(copy.durationMs).toBe(6_000);
+ expect(parseProjectConfig(next)).toBeTruthy();
+ });
+
+ it("records the length the media panel actually measured onto the asset", async () => {
+ /* Held in state and fed back in, exactly as ProjectWorkspace holds it: two
+ files are measured here, and each measurement has to survive the other. */
+ const seen: ProjectConfig[] = [];
+ function Harness({ initial }: { initial: ProjectConfig }) {
+ const [config, setConfig] = useState(initial);
+ useEffect(() => { seen.push(config); }, [config]);
+ return createElement(TimelineView, { config, folderPath: "C:\\Measured Project", onChange: setConfig, onOpenGenerator: () => undefined });
+ }
+ await withFakeDecoder({ seconds: 40, width: 1920, height: 1080 }, async () => {
+ render(createElement(Harness, { initial: projectWithMedia() }));
+ fireEvent.click(screen.getByRole("button", { name: "Media" }));
+ await waitFor(() => expect(seen.at(-1)!.assets.every((asset) => asset.durationMs !== null)).toBe(true), { timeout: 8_000 });
+ });
+ const final = seen.at(-1)!;
+ const video = final.assets.find((asset) => asset.kind === "video")!;
+ expect(video.durationMs).toBe(40_000);
+ expect(video.width).toBe(1920);
+ expect(video.height).toBe(1080);
+ // Sound has no picture but it does have a length, and the timeline needs it.
+ expect(final.assets.find((asset) => asset.kind === "audio")!.durationMs).toBe(40_000);
+ expect(parseProjectConfig(final)).toBeTruthy();
+ // Measuring is not editing: once every file is known, it stops writing.
+ const settled = seen.length;
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ expect(seen.length).toBe(settled);
+ // Longer than the wait above: a decode that never finishes has to fail this
+ // test on its assertion, not by timing the test out before its cleanup runs.
+ }, 20_000);
+
it("keeps the transport on the timeline, with one timecode and one reason", () => {
const empty = createProjectConfig({ name: "Ceramic lamp", prompt: "A quiet product film", aspectRatio: "16:9", resolution: "1080p", targetDurationSeconds: 30 });
const { container, rerender } = render(createElement(TimelineView, { config: empty, folderPath: "C:\Ceramic Lamp", onChange: () => undefined, onOpenGenerator: () => undefined }));
diff --git a/src/components/workspace/MediaThumbnail.tsx b/src/components/workspace/MediaThumbnail.tsx
index 7bdb3d3..e82d2e0 100644
--- a/src/components/workspace/MediaThumbnail.tsx
+++ b/src/components/workspace/MediaThumbnail.tsx
@@ -82,7 +82,42 @@ const settled = (element: HTMLMediaElement, event: string) =>
element.addEventListener("error", () => reject(new Error("This file could not be decoded.")), { once: true });
});
-async function videoPoster(url: string): Promise<{ poster: string; measured: MeasuredMedia }> {
+/* A seek that neither lands nor errors settles nothing, and the await on it
+ would never return: no picture, no measurement, a spinner for the life of
+ the panel. Waiting is worth a few seconds and no more. */
+const SEEK_LIMIT_MS = 4_000;
+const orGiveUp = (promise: Promise<void>, limitMs: number) =>
+ new Promise<void>((resolve) => {
+ const timer = setTimeout(resolve, limitMs);
+ const done = () => { clearTimeout(timer); resolve(); };
+ promise.then(done, done);
+ });
+
+/* Drawing is the part that can fail on its own — a computer with no 2D canvas,
+ a frame the decoder will not hand over. The length and size were already read
+ by then, and throwing them away over a missing picture would leave the asset
+ unmeasured for a reason that has nothing to do with measuring it. */
+const tryDraw = (source: CanvasImageSource, width: number, height: number): string | null => {
+ try {
+ return drawPoster(source, width, height);
+ } catch {
+ return null;
+ }
+};
+
+/* Hand the file back: the decoder and the whole blob go with it. Not worth
+ losing a finished measurement over, so a shell that will not release stays
+ the browser's problem. */
+const release = (element: HTMLMediaElement) => {
+ try {
+ element.removeAttribute("src");
+ element.load();
+ } catch {
+ /* nothing to do about it */
+ }
+};
+
+async function videoPoster(url: string): Promise<{ poster: string | null; measured: MeasuredMedia }> {
const video = document.createElement("video");
video.muted = true;
video.preload = "auto";
@@ -97,28 +132,27 @@ async function videoPoster(url: string): Promise<{ poster: string; measured: Mea
height: pixelsOf(video.videoHeight),
};
const target = Number.isFinite(video.duration) ? Math.min(video.duration / 2, POSTER_TIME_SECONDS) : 0;
- // A seek that never lands would leave the thumbnail loading forever; frame
- // zero is a perfectly good fallback, so a failure here is not fatal.
+ // Frame zero is a perfectly good fallback, so a seek that fails — or one that
+ // simply never arrives — costs the half-second offset and nothing else.
if (target > 0) {
const seeked = settled(video, "seeked");
video.currentTime = target;
- await seeked.catch(() => undefined);
+ await orGiveUp(seeked, SEEK_LIMIT_MS);
}
- const poster = drawPoster(video, video.videoWidth, video.videoHeight);
+ const poster = tryDraw(video, video.videoWidth, video.videoHeight);
// Detach the source so the decoder and the blob are released now.
- video.removeAttribute("src");
- video.load();
+ release(video);
return { poster, measured };
}
-async function imagePoster(url: string): Promise<{ poster: string; measured: MeasuredMedia }> {
+async function imagePoster(url: string): Promise<{ poster: string | null; measured: MeasuredMedia }> {
const image = new Image();
image.src = url;
await image.decode();
// A still has a size and no length. Leaving durationMs null is the whole
// point: how long a still is on screen is the cut's decision, not the file's.
return {
- poster: drawPoster(image, image.naturalWidth, image.naturalHeight),
+ poster: tryDraw(image, image.naturalWidth, image.naturalHeight),
measured: { durationMs: null, width: pixelsOf(image.naturalWidth), height: pixelsOf(image.naturalHeight) },
};
}
@@ -131,8 +165,7 @@ async function measureAudio(url: string): Promise<MeasuredMedia> {
audio.src = url;
await settled(audio, "loadedmetadata");
const measured: MeasuredMedia = { durationMs: durationMsOf(audio), width: null, height: null };
- audio.removeAttribute("src");
- audio.load();
+ release(audio);
return measured;
}
@@ -160,11 +193,18 @@ export function MediaThumbnail({ folderPath, asset, onMeasured }: {
const needsPoster = asset.kind !== "audio" && !POSTER_CACHE.has(cacheKey);
const needsMeasuring = Boolean(onMeasured) && !UNMEASURABLE.has(cacheKey) && wantsMeasuring(asset);
+ /* One read of the file per mount, however often this effect re-runs.
+ Measuring the asset changes it, and that change re-runs the effect — so
+ without this, a file that gave up its numbers but no drawable frame would
+ be read from disk a second time for the same missing picture. */
+ const readOnce = useRef<string | null>(null);
useEffect(() => {
// Nothing to draw and nothing left to learn: sound has no picture, a cached
// poster is already the answer, and a measured asset stays measured.
if (!needsPoster && !needsMeasuring) return;
+ if (readOnce.current === cacheKey) return;
+ readOnce.current = cacheKey;
let live = true;
setFailed(false);
void (async () => {
@@ -179,9 +219,12 @@ export function MediaThumbnail({ folderPath, asset, onMeasured }: {
measured = await measureAudio(url);
} else {
const value = asset.kind === "image" ? await imagePoster(url) : await videoPoster(url);
- cachePoster(cacheKey, value.poster);
measured = value.measured;
- if (live) setPoster(value.poster);
+ // A file that opened but would not draw is still a file that could
+ // not be previewed, and the panel says so — after taking what it did
+ // learn from it.
+ if (value.poster) cachePoster(cacheKey, value.poster);
+ if (live) { if (value.poster) setPoster(value.poster); else setFailed(true); }
}
// Only ever hand up numbers something read off the file. When the file
// gave none, remember that rather than reading it again next mount.
diff --git a/src/components/workspace/TimelineView.tsx b/src/components/workspace/TimelineView.tsx
index f8af17c..c69582e 100644
--- a/src/components/workspace/TimelineView.tsx
+++ b/src/components/workspace/TimelineView.tsx
@@ -228,25 +228,46 @@ export function TimelineView({ config, folderPath, onChange, onOpenGenerator }:
const assetById = (id: string) => config.assets.find((asset) => asset.id === id);
/* The media panel decodes each file to draw its thumbnail, and hands back
- what that decode measured. Write it onto the asset so the NEXT drop is as
- long as the footage, and so the file is only ever read once for it.
- Only fields nothing knew yet are filled — a measurement never overwrites a
+ what that decode measured. Written onto the asset, that is what makes the
+ NEXT drop as long as the footage — and it means the file is read once for
+ it, not once per drop.
+ Only fields nothing knew yet are filled: a measurement never overwrites a
number already on the asset, and it never retimes a clip already cut into
- the timeline, which is the user's decision and not the file's. Bailing out
- when nothing changed is what stops this looping: the write re-renders the
- panel, which would otherwise measure again. */
- const recordMeasured = (assetId: string, measured: MeasuredMedia) => {
- const asset = assetById(assetId);
- if (!asset) return;
- const next: ProjectAsset = {
- ...asset,
- durationMs: asset.durationMs ?? measured.durationMs,
- width: asset.width ?? measured.width,
- height: asset.height ?? measured.height,
- };
- if (next.durationMs === asset.durationMs && next.width === asset.width && next.height === asset.height) return;
- onChange({ ...config, assets: config.assets.map((item) => (item.id === assetId ? next : item)) });
+ the timeline, which is the cut's decision and not the file's. */
+ const applyMeasured = (asset: ProjectAsset, value: MeasuredMedia): ProjectAsset => ({
+ ...asset,
+ durationMs: asset.durationMs ?? value.durationMs,
+ width: asset.width ?? value.width,
+ height: asset.height ?? value.height,
+ });
+ /* Measurements arrive from asynchronous decodes, and two of them can land
+ between one render and the next. Writing each straight into `config` would
+ write the second one over the config the first never reached, losing it.
+ So they queue here and are flushed from an effect, which only ever runs
+ with the config this component was last rendered with. */
+ const measurements = useRef(new Map<string, MeasuredMedia>());
+ const [measureTick, setMeasureTick] = useState(0);
+ const recordMeasured = (assetId: string, value: MeasuredMedia) => {
+ measurements.current.set(assetId, value);
+ setMeasureTick((tick) => tick + 1);
};
+ useEffect(() => {
+ if (measurements.current.size === 0) return;
+ const pending = measurements.current;
+ measurements.current = new Map();
+ let learned = false;
+ const assets = config.assets.map((asset) => {
+ const value = pending.get(asset.id);
+ if (!value) return asset;
+ const next = applyMeasured(asset, value);
+ if (next.durationMs === asset.durationMs && next.width === asset.width && next.height === asset.height) return asset;
+ learned = true;
+ return next;
+ });
+ // Nothing new is not a change: an onChange here would mark the project
+ // edited for having been looked at.
+ if (learned) onChange({ ...config, assets });
+ }, [measureTick, config, onChange]);
/* A trim, typed. The source is the ceiling when its length is known — a clip
cannot play footage the file does not have — and one frame is the floor,