AI media harness for planning, generating, and editing locally
TypeScript
Test the measured drop, the editable Lasts field, and duplicate's source range
Bitti321b2eaba32a
diff --git a/src/components/ProjectWorkspace.test.ts b/src/components/ProjectWorkspace.test.tsindex 086ab4b..67815a9 100644--- a/src/components/ProjectWorkspace.test.ts+++ b/src/components/ProjectWorkspace.test.ts@@ -1,10 +1,10 @@// @vitest-environment jsdomimport { cleanup, createEvent, fireEvent, render, screen, waitFor, within } from "@testing-library/react";-import { createElement } from "react";+import { createElement, useEffect, useState } from "react";import { afterEach, describe, expect, it, vi } from "vitest";import completeFixture from "../../fixtures/project-v1-complete.json";-import { compileMiniMaxH3Prompt, createProjectConfig, parseProjectConfig, STORY_TRACK_ID, type ProjectAsset, type ProjectConfig } from "../lib/project";+import { compileMiniMaxH3Prompt, createProjectConfig, parseProjectConfig, STORY_TRACK_ID, type ProjectAsset, type ProjectConfig, type TimelineClip } from "../lib/project";import { formatDurationTimecode } from "./ProjectWorkspace";import { ProjectWorkspace } from "./ProjectWorkspace";import { GeneratorView } from "./workspace/GeneratorView";@@ -29,6 +29,58 @@ async function asDesktopApp(importedImage: { name: string; relativePath: string}}+/** Runs `body` with a media stack that behaves like a browser's: the desktop+ * shell serves the bytes, blob URLs exist, and a <video>/<audio> element+ * reports the metadata of a file this long and this big. jsdom loads no media+ * and never fires a media event, so the decode has to be stood in for — the+ * numbers under test are the ones the real element would expose. */+async function withFakeDecoder(file: { seconds: number; width: number; height: number }, body: () => Promise<void>) {+ const { invoke } = await import("@tauri-apps/api/core");+ (window as unknown as Record<string, unknown>).__TAURI_INTERNALS__ = {};+ vi.mocked(invoke).mockResolvedValue(new ArrayBuffer(8));+ // jsdom implements neither of these, so they are installed rather than+ // replaced — and left installed, because restoring them to `undefined` turns+ // any decode still finishing into an unhandled TypeError.+ URL.createObjectURL = () => "blob:polstudio-test";+ URL.revokeObjectURL = () => undefined;+ /* jsdom answers `load` and `getContext` by raising a "not implemented" error+ on its virtual console, which vitest counts as an error in the run. The+ component already treats a canvas it cannot get as no thumbnail, so give it+ the plain refusal rather than the environment's complaint. */+ const media = HTMLMediaElement.prototype.load;+ const canvas = HTMLCanvasElement.prototype.getContext;+ HTMLMediaElement.prototype.load = () => undefined;+ HTMLCanvasElement.prototype.getContext = () => null;+ const create = document.createElement.bind(document);+ const spy = vi.spyOn(document, "createElement").mockImplementation((tag: string, options?: ElementCreationOptions) => {+ const element = create(tag, options);+ if (tag === "video" || tag === "audio") {+ for (const [name, value] of [["duration", file.seconds], ["videoWidth", file.width], ["videoHeight", file.height]] as const) {+ Object.defineProperty(element, name, { value, configurable: true });+ }+ // jsdom's currentTime setter neither seeks nor complains, so the seek for+ // a poster frame has to answer for itself too.+ let currentTime = 0;+ Object.defineProperty(element, "currentTime", {+ configurable: true,+ get: () => currentTime,+ set: (value: number) => { currentTime = value; setTimeout(() => element.dispatchEvent(new Event("seeked")), 0); },+ });+ setTimeout(() => { element.dispatchEvent(new Event("loadeddata")); element.dispatchEvent(new Event("loadedmetadata")); }, 0);+ }+ return element;+ });+ try {+ await body();+ } finally {+ spy.mockRestore();+ HTMLMediaElement.prototype.load = media;+ HTMLCanvasElement.prototype.getContext = canvas;+ delete (window as unknown as Record<string, unknown>).__TAURI_INTERNALS__;+ vi.mocked(invoke).mockReset();+ }+}+describe("project workspace timecode", () => {it("formats project duration as valid hours, minutes, and seconds", () => {expect(formatDurationTimecode(0)).toBe("00:00:00");@@ -226,6 +278,130 @@ describe("project workspace timecode", () => {expect(rejected()).toHaveLength(0);});+ /** The same project with its video asset already measured, as the media panel+ * measures it the first time it decodes the file for a thumbnail. */+ const measuredVideo = (config: ProjectConfig, durationMs: number) => parseProjectConfig({+ ...config,+ assets: config.assets.map((asset) => asset.kind === "video" ? { ...asset, durationMs, width: 1920, height: 1080 } : asset),+ });++ /** That project with one clip of the video already cut into the story track. */+ const withClip = (config: ProjectConfig, clip: Partial<TimelineClip>) => parseProjectConfig({+ ...config,+ timeline: { tracks: config.timeline.tracks.map((track) => track.id === STORY_TRACK_ID ? { ...track, clips: [{+ id: "clip-macro", assetId: config.assets[0].id, trackId: STORY_TRACK_ID,+ startMs: 0, durationMs: 40_000, sourceStartMs: 0, label: "Macro footage", color: null, status: "approved", ...clip,+ }] } : track) },+ });++ const droppedClip = (calls: { mock: { calls: unknown[][] } }) =>+ ((calls.mock.calls[0][0] as ProjectConfig).timeline.tracks.flatMap((track) => track.clips))[0];++ it("drops a clip as long as the footage, and falls back to the stated default only when nothing measured it", () => {+ /* B3: every dropped clip used to be five seconds, forever — a 40-second+ rush included, with no way anywhere in the app to change it. */+ const known = measuredVideo(projectWithMedia(), 40_000);+ const first = mediaPanel(known);+ const transfer = fakeDataTransfer();+ fireEvent.dragStart(cardFor(known.assets[0]), { dataTransfer: transfer });+ fireEvent.drop(laneFor(first.container, known, "video"), { dataTransfer: transfer });+ expect(droppedClip(first.onChange).durationMs).toBe(40_000);+ expect(parseProjectConfig(first.onChange.mock.calls[0][0] as ProjectConfig)).toBeTruthy();+ cleanup();++ // Nothing has decoded this one, so there is no length to honour and the+ // documented stand-in is what lands.+ const unknown = projectWithMedia();+ const second = mediaPanel(unknown);+ const transferTwo = fakeDataTransfer();+ fireEvent.dragStart(cardFor(unknown.assets[0]), { dataTransfer: transferTwo });+ fireEvent.drop(laneFor(second.container, unknown, "video"), { dataTransfer: transferTwo });+ expect(droppedClip(second.onChange).durationMs).toBe(5_000);+ });++ it("retimes the selected clip from the Lasts field, and will not run past the footage", () => {+ const config = withClip(measuredVideo(projectWithMedia(), 40_000), {});+ const onChange = vi.fn();+ render(createElement(TimelineView, { config, folderPath: "C:\\Ceramic Lamp", onChange, onOpenGenerator: () => undefined }));+ const lasts = screen.getByTitle(/^How long this clip lasts/) as HTMLInputElement;+ // It was readOnly, which made the only stated way to fix a clip's length a+ // field that could not be typed into.+ expect(lasts.readOnly).toBe(false);+ expect(lasts.value).toBe("00:40:00");+ const lastDuration = () => (onChange.mock.calls.at(-1)![0] as ProjectConfig).timeline.tracks.flatMap((track) => track.clips)[0].durationMs;++ // Plain seconds, and the minutes:seconds:frames the field itself prints.+ fireEvent.change(lasts, { target: { value: "8" } });+ expect(lastDuration()).toBe(8_000);+ fireEvent.change(lasts, { target: { value: "00:12:15" } });+ expect(lastDuration()).toBe(12_500);+ expect(parseProjectConfig(onChange.mock.calls.at(-1)![0] as ProjectConfig)).toBeTruthy();++ // Half-typed text is not a length, and must not be turned into one.+ const before = onChange.mock.calls.length;+ fireEvent.change(lasts, { target: { value: "00:" } });+ expect(onChange.mock.calls.length).toBe(before);+ expect(lasts.value).toBe("00:");++ // Clamped at both ends: a clip cannot be nothing, and cannot play footage+ // the file does not have.+ fireEvent.change(lasts, { target: { value: "0" } });+ expect(lastDuration()).toBe(33);+ fireEvent.change(lasts, { target: { value: "90" } });+ expect(lastDuration()).toBe(40_000);+ expect(parseProjectConfig(onChange.mock.calls.at(-1)![0] as ProjectConfig)).toBeTruthy();++ // Off the field, the clip's own length is what it shows again.+ fireEvent.blur(lasts);+ expect(lasts.value).toBe("00:40:00");+ });++ it("keeps a clip's place in the source when it is duplicated", () => {+ // Otherwise every copy replays the head of the file, which is the same five+ // seconds the original already showed.+ const config = withClip(measuredVideo(projectWithMedia(), 40_000), { startMs: 0, durationMs: 6_000, sourceStartMs: 12_000 });+ const onChange = vi.fn();+ render(createElement(TimelineView, { config, folderPath: "C:\\Ceramic Lamp", onChange, onOpenGenerator: () => undefined }));+ fireEvent.click(screen.getByRole("button", { name: "Duplicate" }));+ const next = onChange.mock.calls[0][0] as ProjectConfig;+ const clips = next.timeline.tracks.flatMap((track) => track.clips);+ expect(clips).toHaveLength(2);+ const copy = clips.find((clip) => clip.id !== "clip-macro")!;+ expect(copy.sourceStartMs).toBe(12_000);+ expect(copy.durationMs).toBe(6_000);+ expect(parseProjectConfig(next)).toBeTruthy();+ });++ it("records the length the media panel actually measured onto the asset", async () => {+ /* Held in state and fed back in, exactly as ProjectWorkspace holds it: two+ files are measured here, and each measurement has to survive the other. */+ const seen: ProjectConfig[] = [];+ function Harness({ initial }: { initial: ProjectConfig }) {+ const [config, setConfig] = useState(initial);+ useEffect(() => { seen.push(config); }, [config]);+ return createElement(TimelineView, { config, folderPath: "C:\\Measured Project", onChange: setConfig, onOpenGenerator: () => undefined });+ }+ await withFakeDecoder({ seconds: 40, width: 1920, height: 1080 }, async () => {+ render(createElement(Harness, { initial: projectWithMedia() }));+ fireEvent.click(screen.getByRole("button", { name: "Media" }));+ await waitFor(() => expect(seen.at(-1)!.assets.every((asset) => asset.durationMs !== null)).toBe(true), { timeout: 8_000 });+ });+ const final = seen.at(-1)!;+ const video = final.assets.find((asset) => asset.kind === "video")!;+ expect(video.durationMs).toBe(40_000);+ expect(video.width).toBe(1920);+ expect(video.height).toBe(1080);+ // Sound has no picture but it does have a length, and the timeline needs it.+ expect(final.assets.find((asset) => asset.kind === "audio")!.durationMs).toBe(40_000);+ expect(parseProjectConfig(final)).toBeTruthy();+ // Measuring is not editing: once every file is known, it stops writing.+ const settled = seen.length;+ await new Promise((resolve) => setTimeout(resolve, 50));+ expect(seen.length).toBe(settled);+ // Longer than the wait above: a decode that never finishes has to fail this+ // test on its assertion, not by timing the test out before its cleanup runs.+ }, 20_000);+it("keeps the transport on the timeline, with one timecode and one reason", () => {const empty = createProjectConfig({ name: "Ceramic lamp", prompt: "A quiet product film", aspectRatio: "16:9", resolution: "1080p", targetDurationSeconds: 30 });const { container, rerender } = render(createElement(TimelineView, { config: empty, folderPath: "C:\Ceramic Lamp", onChange: () => undefined, onOpenGenerator: () => undefined }));diff --git a/src/components/workspace/MediaThumbnail.tsx b/src/components/workspace/MediaThumbnail.tsxindex 7bdb3d3..e82d2e0 100644--- a/src/components/workspace/MediaThumbnail.tsx+++ b/src/components/workspace/MediaThumbnail.tsx@@ -82,7 +82,42 @@ const settled = (element: HTMLMediaElement, event: string) =>element.addEventListener("error", () => reject(new Error("This file could not be decoded.")), { once: true });});-async function videoPoster(url: string): Promise<{ poster: string; measured: MeasuredMedia }> {+/* A seek that neither lands nor errors settles nothing, and the await on it+ would never return: no picture, no measurement, a spinner for the life of+ the panel. Waiting is worth a few seconds and no more. */+const SEEK_LIMIT_MS = 4_000;+const orGiveUp = (promise: Promise<void>, limitMs: number) =>+ new Promise<void>((resolve) => {+ const timer = setTimeout(resolve, limitMs);+ const done = () => { clearTimeout(timer); resolve(); };+ promise.then(done, done);+ });++/* Drawing is the part that can fail on its own — a computer with no 2D canvas,+ a frame the decoder will not hand over. The length and size were already read+ by then, and throwing them away over a missing picture would leave the asset+ unmeasured for a reason that has nothing to do with measuring it. */+const tryDraw = (source: CanvasImageSource, width: number, height: number): string | null => {+ try {+ return drawPoster(source, width, height);+ } catch {+ return null;+ }+};++/* Hand the file back: the decoder and the whole blob go with it. Not worth+ losing a finished measurement over, so a shell that will not release stays+ the browser's problem. */+const release = (element: HTMLMediaElement) => {+ try {+ element.removeAttribute("src");+ element.load();+ } catch {+ /* nothing to do about it */+ }+};++async function videoPoster(url: string): Promise<{ poster: string | null; measured: MeasuredMedia }> {const video = document.createElement("video");video.muted = true;video.preload = "auto";@@ -97,28 +132,27 @@ async function videoPoster(url: string): Promise<{ poster: string; measured: Meaheight: pixelsOf(video.videoHeight),};const target = Number.isFinite(video.duration) ? Math.min(video.duration / 2, POSTER_TIME_SECONDS) : 0;- // A seek that never lands would leave the thumbnail loading forever; frame- // zero is a perfectly good fallback, so a failure here is not fatal.+ // Frame zero is a perfectly good fallback, so a seek that fails — or one that+ // simply never arrives — costs the half-second offset and nothing else.if (target > 0) {const seeked = settled(video, "seeked");video.currentTime = target;- await seeked.catch(() => undefined);+ await orGiveUp(seeked, SEEK_LIMIT_MS);}- const poster = drawPoster(video, video.videoWidth, video.videoHeight);+ const poster = tryDraw(video, video.videoWidth, video.videoHeight);// Detach the source so the decoder and the blob are released now.- video.removeAttribute("src");- video.load();+ release(video);return { poster, measured };}-async function imagePoster(url: string): Promise<{ poster: string; measured: MeasuredMedia }> {+async function imagePoster(url: string): Promise<{ poster: string | null; measured: MeasuredMedia }> {const image = new Image();image.src = url;await image.decode();// A still has a size and no length. Leaving durationMs null is the whole// point: how long a still is on screen is the cut's decision, not the file's.return {- poster: drawPoster(image, image.naturalWidth, image.naturalHeight),+ poster: tryDraw(image, image.naturalWidth, image.naturalHeight),measured: { durationMs: null, width: pixelsOf(image.naturalWidth), height: pixelsOf(image.naturalHeight) },};}@@ -131,8 +165,7 @@ async function measureAudio(url: string): Promise<MeasuredMedia> {audio.src = url;await settled(audio, "loadedmetadata");const measured: MeasuredMedia = { durationMs: durationMsOf(audio), width: null, height: null };- audio.removeAttribute("src");- audio.load();+ release(audio);return measured;}@@ -160,11 +193,18 @@ export function MediaThumbnail({ folderPath, asset, onMeasured }: {const needsPoster = asset.kind !== "audio" && !POSTER_CACHE.has(cacheKey);const needsMeasuring = Boolean(onMeasured) && !UNMEASURABLE.has(cacheKey) && wantsMeasuring(asset);+ /* One read of the file per mount, however often this effect re-runs.+ Measuring the asset changes it, and that change re-runs the effect — so+ without this, a file that gave up its numbers but no drawable frame would+ be read from disk a second time for the same missing picture. */+ const readOnce = useRef<string | null>(null);useEffect(() => {// Nothing to draw and nothing left to learn: sound has no picture, a cached// poster is already the answer, and a measured asset stays measured.if (!needsPoster && !needsMeasuring) return;+ if (readOnce.current === cacheKey) return;+ readOnce.current = cacheKey;let live = true;setFailed(false);void (async () => {@@ -179,9 +219,12 @@ export function MediaThumbnail({ folderPath, asset, onMeasured }: {measured = await measureAudio(url);} else {const value = asset.kind === "image" ? await imagePoster(url) : await videoPoster(url);- cachePoster(cacheKey, value.poster);measured = value.measured;- if (live) setPoster(value.poster);+ // A file that opened but would not draw is still a file that could+ // not be previewed, and the panel says so — after taking what it did+ // learn from it.+ if (value.poster) cachePoster(cacheKey, value.poster);+ if (live) { if (value.poster) setPoster(value.poster); else setFailed(true); }}// Only ever hand up numbers something read off the file. When the file// gave none, remember that rather than reading it again next mount.diff --git a/src/components/workspace/TimelineView.tsx b/src/components/workspace/TimelineView.tsxindex f8af17c..c69582e 100644--- a/src/components/workspace/TimelineView.tsx+++ b/src/components/workspace/TimelineView.tsx@@ -228,25 +228,46 @@ export function TimelineView({ config, folderPath, onChange, onOpenGenerator }:const assetById = (id: string) => config.assets.find((asset) => asset.id === id);/* The media panel decodes each file to draw its thumbnail, and hands back- what that decode measured. Write it onto the asset so the NEXT drop is as- long as the footage, and so the file is only ever read once for it.- Only fields nothing knew yet are filled — a measurement never overwrites a+ what that decode measured. Written onto the asset, that is what makes the+ NEXT drop as long as the footage — and it means the file is read once for+ it, not once per drop.+ Only fields nothing knew yet are filled: a measurement never overwrites anumber already on the asset, and it never retimes a clip already cut into- the timeline, which is the user's decision and not the file's. Bailing out- when nothing changed is what stops this looping: the write re-renders the- panel, which would otherwise measure again. */- const recordMeasured = (assetId: string, measured: MeasuredMedia) => {- const asset = assetById(assetId);- if (!asset) return;- const next: ProjectAsset = {- ...asset,- durationMs: asset.durationMs ?? measured.durationMs,- width: asset.width ?? measured.width,- height: asset.height ?? measured.height,- };- if (next.durationMs === asset.durationMs && next.width === asset.width && next.height === asset.height) return;- onChange({ ...config, assets: config.assets.map((item) => (item.id === assetId ? next : item)) });+ the timeline, which is the cut's decision and not the file's. */+ const applyMeasured = (asset: ProjectAsset, value: MeasuredMedia): ProjectAsset => ({+ ...asset,+ durationMs: asset.durationMs ?? value.durationMs,+ width: asset.width ?? value.width,+ height: asset.height ?? value.height,+ });+ /* Measurements arrive from asynchronous decodes, and two of them can land+ between one render and the next. Writing each straight into `config` would+ write the second one over the config the first never reached, losing it.+ So they queue here and are flushed from an effect, which only ever runs+ with the config this component was last rendered with. */+ const measurements = useRef(new Map<string, MeasuredMedia>());+ const [measureTick, setMeasureTick] = useState(0);+ const recordMeasured = (assetId: string, value: MeasuredMedia) => {+ measurements.current.set(assetId, value);+ setMeasureTick((tick) => tick + 1);};+ useEffect(() => {+ if (measurements.current.size === 0) return;+ const pending = measurements.current;+ measurements.current = new Map();+ let learned = false;+ const assets = config.assets.map((asset) => {+ const value = pending.get(asset.id);+ if (!value) return asset;+ const next = applyMeasured(asset, value);+ if (next.durationMs === asset.durationMs && next.width === asset.width && next.height === asset.height) return asset;+ learned = true;+ return next;+ });+ // Nothing new is not a change: an onChange here would mark the project+ // edited for having been looked at.+ if (learned) onChange({ ...config, assets });+ }, [measureTick, config, onChange]);/* A trim, typed. The source is the ceiling when its length is known — a clipcannot play footage the file does not have — and one frame is the floor,