Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Fix playback position after cuts

Bittic40419b1881a
diff --git a/src/components/workspace/ProgramMonitor.test.tsx b/src/components/workspace/ProgramMonitor.test.tsx
index 8a6d03e..5a78111 100644
--- a/src/components/workspace/ProgramMonitor.test.tsx
+++ b/src/components/workspace/ProgramMonitor.test.tsx
@@ -5,6 +5,7 @@ import { useState } from "react";
import { afterEach, describe, expect, it, vi } from "vitest";
import { createProjectConfig, parseProjectConfig, STORY_TRACK_ID, type ProjectConfig, type TimelineClip } from "../../lib/project";
import { ProgramMonitor } from "./ProgramMonitor";
+import { PreviewSources } from "../../lib/exportPipeline";
vi.mock("@tauri-apps/api/core", () => ({ invoke: vi.fn() }));
@@ -51,6 +52,77 @@ const monitor = (config: ProjectConfig, playheadMs: number) => render(
);
describe("the program monitor", () => {
+ it.each([0, 2_000])("continues the longer track at its timeline position after the foreground ends (source offset %i)", async (sourceStartMs) => {
+ vi.spyOn(PreviewSources.prototype, "url").mockImplementation(async (asset) => `blob:${asset.id}`);
+ vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue();
+ vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => undefined);
+ const ready = new WeakSet<HTMLMediaElement>();
+ vi.spyOn(HTMLMediaElement.prototype, "readyState", "get").mockImplementation(function (this: HTMLMediaElement) {
+ return ready.has(this) ? 4 : 0;
+ });
+ let now = 0;
+ let nextFrameId = 0;
+ const frames = new Map<number, FrameRequestCallback>();
+ vi.spyOn(performance, "now").mockImplementation(() => now);
+ vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => {
+ frames.set(++nextFrameId, callback);
+ return nextFrameId;
+ });
+ vi.stubGlobal("cancelAnimationFrame", (id: number) => frames.delete(id));
+ const advance = async (ms: number) => {
+ await act(async () => {
+ now += ms;
+ const callbacks = [...frames.values()];
+ frames.clear();
+ callbacks.forEach((callback) => callback(now));
+ });
+ };
+ const config = project([clip({ startMs: 0, sourceStartMs: 0, look: { opacity: 50, temperature: 0 } })]);
+ config.assets.push({ ...config.assets[0], id: "asset-long", sourcePath: "D:/rushes/long.mp4" });
+ config.timeline.tracks.push({
+ ...config.timeline.tracks.find((track) => track.id === STORY_TRACK_ID)!,
+ id: "long-track",
+ clips: [clip({ id: "long-clip", assetId: "asset-long", trackId: "long-track", startMs: 0, durationMs: 10_000, sourceStartMs })],
+ });
+ const onPlayingChange = vi.fn();
+ function Harness() {
+ const [playhead, setPlayhead] = useState(0);
+ const [playing, setPlaying] = useState(true);
+ return <ProgramMonitor config={config} folderPath="C:\\Film" playheadMs={playhead} playing={playing}
+ onSeek={setPlayhead} onPlayingChange={(value) => { onPlayingChange(value); setPlaying(value); }} />;
+ }
+ const { container } = render(<Harness />);
+ await act(async () => undefined);
+ const video = container.querySelector<HTMLVideoElement>(".program-picture > video")!;
+ expect(container.querySelectorAll("video")).toHaveLength(2);
+ for (const element of container.querySelectorAll("video")) {
+ ready.add(element);
+ fireEvent.loadedMetadata(element);
+ fireEvent.loadedData(element);
+ }
+
+ await advance(4_000);
+ expect(video.getAttribute("src")).toBe("blob:asset-long");
+ expect(container.querySelectorAll("video")).toHaveLength(1);
+ // A browser resets the media position when src changes. Model a delayed
+ // load too: the correct seek must use where playback has reached by then.
+ ready.delete(video);
+ video.currentTime = 0;
+ await advance(500);
+ ready.add(video);
+ fireEvent.loadedMetadata(video);
+ fireEvent.loadedData(video);
+ expect(video.currentTime).toBe((4_500 + sourceStartMs) / 1000);
+ expect(onPlayingChange).not.toHaveBeenCalled();
+
+ await advance(5_000);
+ expect(video.isConnected).toBe(true);
+ expect(onPlayingChange).not.toHaveBeenCalled();
+ await advance(500);
+ expect(onPlayingChange).toHaveBeenCalledWith(false);
+ expect(container.querySelector("video")).toBeNull();
+ });
+
it("shows move, scale, and rotate widgets after the timeline picture is selected", () => {
const config = project([clip()]);
function Harness() {
diff --git a/src/components/workspace/ProgramMonitor.tsx b/src/components/workspace/ProgramMonitor.tsx
index 4e374b6..c96adb9 100644
--- a/src/components/workspace/ProgramMonitor.tsx
+++ b/src/components/workspace/ProgramMonitor.tsx
@@ -260,13 +260,19 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek
seekTo(video, target, pendingVideoSeek);
};
video.addEventListener("seeked", flush);
- video.addEventListener("loadeddata", flush);
return () => {
video.removeEventListener("seeked", flush);
- video.removeEventListener("loadeddata", flush);
};
}, [url]);
+ /* Changing src resets the decoder after the cut's seek may have already
+ reached the previous file. Once the new file loads, seek again using the
+ current playhead, including any time spent waiting for that file. */
+ const syncLoadedVideo = (video: HTMLVideoElement) => {
+ pendingVideoSeek.current = null;
+ if (clip) seekTo(video, sourceTimeMs(clip, positionRef.current) / 1000, pendingVideoSeek);
+ };
+
/* Play and pause the picture. A media element that is asked to play before it
has data rejects, which is a promise nobody was awaiting — caught here so a
slow file does not throw into the console on every press. */
@@ -424,7 +430,8 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek
muted={videoTrackMuted}
playsInline
preload="auto"
- onLoadedData={() => setReady(true)}
+ onLoadedMetadata={(event) => syncLoadedVideo(event.currentTarget)}
+ onLoadedData={(event) => { syncLoadedVideo(event.currentTarget); setReady(true); }}
onError={() => setError("This file could not be decoded.")}
/>}
{asset && isImage(asset) && url && <img ref={imageRef} src={url} alt={clip?.label ?? ""} style={sourceStyle} />}