Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Add audiovisual timeline previews

Bitti877c82d3dc32
diff --git a/src/components/workspace/MediaThumbnail.tsx b/src/components/workspace/MediaThumbnail.tsx
index b5d367a..5ee1a4e 100644
--- a/src/components/workspace/MediaThumbnail.tsx
+++ b/src/components/workspace/MediaThumbnail.tsx
@@ -44,11 +44,28 @@ export interface MeasuredMedia {
durationMs: number | null;
width: number | null;
height: number | null;
+ hasAudio: boolean | null;
}
-const NOTHING_MEASURED: MeasuredMedia = { durationMs: null, width: null, height: null };
+const NOTHING_MEASURED: MeasuredMedia = { durationMs: null, width: null, height: null, hasAudio: null };
const measuredSomething = (measured: MeasuredMedia) =>
- measured.durationMs !== null || measured.width !== null || measured.height !== null;
+ measured.durationMs !== null || measured.width !== null || measured.height !== null || measured.hasAudio !== null;
+
+type AudioAwareVideo = HTMLVideoElement & {
+ mozHasAudio?: boolean;
+ webkitAudioDecodedByteCount?: number;
+ audioTracks?: { length: number };
+};
+
+/** Browser engines expose audio-stream presence through different optional
+ * properties. Unknown stays null; guessing false would hide real sound. */
+const videoHasAudio = (element: HTMLVideoElement): boolean | null => {
+ const video = element as AudioAwareVideo;
+ if (typeof video.mozHasAudio === "boolean") return video.mozHasAudio;
+ if (video.audioTracks) return video.audioTracks.length > 0;
+ if (typeof video.webkitAudioDecodedByteCount === "number" && video.webkitAudioDecodedByteCount > 0) return true;
+ return null;
+};
/* A media element reports `duration` as NaN before metadata arrives and
Infinity for a live or unseekable stream. Neither is a length. */
@@ -131,6 +148,7 @@ async function videoPoster(url: string, posterTimeSeconds: number, jpegQuality:
durationMs: durationMsOf(video),
width: pixelsOf(video.videoWidth),
height: pixelsOf(video.videoHeight),
+ hasAudio: videoHasAudio(video),
};
const target = Number.isFinite(video.duration)
? Math.min(Math.max(0, video.duration - 0.05), Math.max(0, posterTimeSeconds))
@@ -156,7 +174,7 @@ async function imagePoster(url: string, jpegQuality: number): Promise<{ poster:
// point: how long a still is on screen is the cut's decision, not the file's.
return {
poster: tryDraw(image, image.naturalWidth, image.naturalHeight, jpegQuality),
- measured: { durationMs: null, width: pixelsOf(image.naturalWidth), height: pixelsOf(image.naturalHeight) },
+ measured: { durationMs: null, width: pixelsOf(image.naturalWidth), height: pixelsOf(image.naturalHeight), hasAudio: false },
};
}
@@ -167,7 +185,7 @@ async function measureAudio(url: string): Promise<MeasuredMedia> {
audio.preload = "metadata";
audio.src = url;
await settled(audio, "loadedmetadata");
- const measured: MeasuredMedia = { durationMs: durationMsOf(audio), width: null, height: null };
+ const measured: MeasuredMedia = { durationMs: durationMsOf(audio), width: null, height: null, hasAudio: true };
release(audio);
return measured;
}
@@ -176,7 +194,9 @@ async function measureAudio(url: string): Promise<MeasuredMedia> {
* a duration, so asking for one forever would re-read the file on every
* mount; sound never gets a size. */
const wantsMeasuring = (asset: ProjectAsset) =>
- asset.kind === "image" ? !asset.width || !asset.height : !asset.durationMs;
+ asset.kind === "image"
+ ? !asset.width || !asset.height
+ : !asset.durationMs || (asset.kind === "video" && asset.hasAudio == null);
export function MediaThumbnail({ folderPath, asset, onMeasured, posterTimeSeconds = POSTER_TIME_SECONDS, jpegQuality = DEFAULT_JPEG_QUALITY }: {
folderPath: string;
diff --git a/src/components/workspace/ProgramMonitor.test.tsx b/src/components/workspace/ProgramMonitor.test.tsx
index 8ec87a6..a0de32e 100644
--- a/src/components/workspace/ProgramMonitor.test.tsx
+++ b/src/components/workspace/ProgramMonitor.test.tsx
@@ -1,6 +1,6 @@
// @vitest-environment jsdom
-import { cleanup, fireEvent, render, screen } from "@testing-library/react";
+import { act, cleanup, fireEvent, render, screen } from "@testing-library/react";
import { useState } from "react";
import { afterEach, describe, expect, it, vi } from "vitest";
import { createProjectConfig, parseProjectConfig, STORY_TRACK_ID, type ProjectConfig, type TimelineClip } from "../../lib/project";
@@ -10,6 +10,8 @@ vi.mock("@tauri-apps/api/core", () => ({ invoke: vi.fn() }));
afterEach(() => {
cleanup();
+ vi.restoreAllMocks();
+ vi.unstubAllGlobals();
delete (window as unknown as Record<string, unknown>).__TAURI_INTERNALS__;
});
@@ -106,4 +108,28 @@ describe("the program monitor", () => {
expect(container.querySelector(".program-note")).toBeNull();
expect(container.querySelector(".program-picture")).not.toBeNull();
});
+
+ it("advances from its local animation clock when renders are delayed", () => {
+ const frames: FrameRequestCallback[] = [];
+ let now = 0;
+ vi.spyOn(performance, "now").mockImplementation(() => now);
+ vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => {
+ frames.push(callback);
+ return frames.length;
+ });
+ vi.stubGlobal("cancelAnimationFrame", vi.fn());
+ const onSeek = vi.fn();
+ render(<ProgramMonitor
+ config={project([clip({ startMs: 0, sourceStartMs: 0 })])}
+ folderPath="C:\\Ceramic Lamp"
+ playheadMs={0}
+ playing
+ onSeek={onSeek}
+ onPlayingChange={() => undefined}
+ />);
+
+ act(() => { now = 16; frames.shift()?.(now); });
+ act(() => { now = 32; frames.shift()?.(now); });
+ expect(onSeek.mock.calls.map(([position]) => position)).toEqual([16, 32]);
+ });
});
diff --git a/src/components/workspace/ProgramMonitor.tsx b/src/components/workspace/ProgramMonitor.tsx
index 6cba3b7..783276d 100644
--- a/src/components/workspace/ProgramMonitor.tsx
+++ b/src/components/workspace/ProgramMonitor.tsx
@@ -17,14 +17,12 @@ import { PreviewSources } from "../../lib/exportPipeline";
* re-points the same element at the next file.
* * Timeline time and source time are different numbers. A clip trimmed to
* start 8s into a rush shows second 8 of the file at second 0 of the cut.
- * * Sound on the audio tracks plays alongside, each clip in its own element,
+ * * Sound from every audiovisual track plays alongside, each additional clip in its own element,
* corrected back into step whenever it drifts.
*
- * While playing, the VIDEO element is the clock: the playhead is derived from
- * where the decoder actually is, not from a timer running beside it, so the
- * picture and the playhead cannot disagree. Over a gap — or with nothing
- * readable — a wall clock takes over, because a black stretch still has to
- * play through.
+ * While playing, a monotonic animation clock advances the playhead and the
+ * media element plays alongside it. Explicit jumps and cuts seek the decoder;
+ * the same clock also carries playback through gaps and still images.
*
* The whole file is read into a blob, as everywhere else in PolStudio (see
* MediaThumbnail): the project's media lives outside the webview's reach and
@@ -37,12 +35,10 @@ const AUDIO_DRIFT_LIMIT_S = 0.25;
/** Seeks below this are not worth the stutter — a paused element is already
* showing that frame. Roughly one frame at 30fps. */
const SEEK_EPSILON_S = 0.033;
-/** How far outside a clip's own stretch of its file the decoder may read and
- * still be believed. A frame or two of slack, so ordinary rounding at a cut
- * does not hand the clock to the wall. */
-const CUT_SLACK_MS = 120;
-
const isImage = (asset: ProjectAsset) => asset.mimeType.startsWith("image/") || asset.kind === "image";
+const carriesAudio = (asset: ProjectAsset | undefined) => Boolean(
+ asset && (asset.kind === "audio" || ((asset.kind === "video" || asset.kind === "generated") && asset.hasAudio === true)),
+);
/** Where inside the FILE this moment of the timeline lives. */
const sourceTimeMs = (clip: TimelineClip, timelineMs: number) => clip.sourceStartMs + (timelineMs - clip.startMs);
@@ -92,8 +88,8 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek
folderPath: string;
playheadMs: number;
playing: boolean;
- /** Where playback has reached. The view owns the playhead; this only ever
- * reports where the decoder actually is. */
+ /** Where playback has reached. The view owns the playhead; this reports the
+ * monitor's monotonic playback clock. */
onSeek: (ms: number) => void;
onPlayingChange: (playing: boolean) => void;
/** Timeline-only direct manipulation. Export omits these props and therefore
@@ -104,7 +100,8 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek
onTransformChange?: (clipId: string, transform: ClipTransform) => void;
}) {
const tracks = config.timeline.tracks;
- const clip = useMemo(() => visibleClipAt(tracks, playheadMs), [tracks, playheadMs]);
+ const assetsById = useMemo(() => new Map(config.assets.map((candidate) => [candidate.id, candidate])), [config.assets]);
+ const clip = useMemo(() => visibleClipAt(tracks, playheadMs, assetsById), [tracks, playheadMs, assetsById]);
const asset = useMemo(
() => (clip ? config.assets.find((candidate) => candidate.id === clip.assetId) : undefined),
[clip, config.assets],
@@ -127,14 +124,15 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek
clipPath: `inset(0 ${(1 - revealEnd) * 100}% 0 ${revealStart * 100}%)`,
};
}, [frameStyle]);
- /* Sound that should be audible at this moment: every audio clip the playhead
- is inside, minus the tracks the user has muted. A muted track is dropped
- here rather than played silently, so nothing is decoded for it at all. */
+ /* Sound that should be audible at this moment: audio-only files and embedded
+ sound from every active video, minus muted tracks. The visible video plays
+ its own stream, so a second audio element is only needed for the rest. */
const audioClips = useMemo(() => tracks
- .filter((track) => track.kind === "audio" && !track.muted)
+ .filter((track) => !track.muted)
.flatMap((track) => track.clips)
- .filter((candidate) => playheadMs >= candidate.startMs && playheadMs < clipEndMs(candidate)),
- [tracks, playheadMs]);
+ .filter((candidate) => playheadMs >= candidate.startMs && playheadMs < clipEndMs(candidate))
+ .filter((candidate) => candidate.id !== clip?.id && carriesAudio(assetsById.get(candidate.assetId))),
+ [tracks, playheadMs, clip?.id, assetsById]);
const videoTrackMuted = useMemo(
() => tracks.find((track) => track.id === clip?.trackId)?.muted ?? false,
[tracks, clip],
@@ -195,6 +193,9 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek
const report = useCallback((ms: number) => {
const rounded = Math.round(ms);
+ /* Keep the local clock advancing even when React batches visual updates
+ while pointer and hover events are busy. */
+ positionRef.current = rounded;
advancedTo.current = rounded;
onSeek(rounded);
}, [onSeek]);
@@ -239,8 +240,8 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek
/* Scrubbing, every other jump, and the first frame of every new shot.
While playing this deliberately does nothing for the playhead's own
- advance — the element is the clock then, and seeking it to its own reading
- would stutter every frame. It does have to fire on the two other cases:
+ advance — seeking the element to every animation tick would stutter. It
+ does have to fire on the two other cases:
a playhead that moved for some other reason (the ruler, the wheel, a scene
card), and a CUT. Crossing a cut re-points the element at another file, or
at another moment of the same file, and neither knows where in the footage
@@ -284,10 +285,8 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek
else video.pause();
}, [playing, url, clip?.id]);
- /* The clock. One frame at a time: read where the decoder is, turn that back
- into a position on the ruler, and hand it up. With no picture to read —
- a gap, an unreadable file, a still image — the wall clock stands in, so a
- black stretch still plays through at the right speed. */
+ /* The clock advances one animation frame at a time. It deliberately does
+ not rely on the media element's coarser currentTime update cadence. */
useEffect(() => {
if (!playing) return;
let frame = 0;
@@ -296,22 +295,10 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek
const now = performance.now();
const elapsed = now - lastTick;
lastTick = now;
- const video = videoRef.current;
- /* The decoder is only the clock while it is actually inside THIS clip's
- stretch of the file. Just after a cut it is not: the element has been
- re-pointed and its currentTime still reads 0 (or the last shot's
- position) until the seek lands. Believing it there would put the
- playhead before the clip that is on screen, which hands the render back
- to the previous clip and bounces between the two. The wall clock covers
- those few frames instead. */
- const sourceNowMs = video ? video.currentTime * 1000 : 0;
- const insideThisClip = clip !== null
- && sourceNowMs >= clip.sourceStartMs - CUT_SLACK_MS
- && sourceNowMs <= clip.sourceStartMs + clip.durationMs + CUT_SLACK_MS;
- const decoded = clip && video && url && !video.paused && !video.seeking && Number.isFinite(video.currentTime) && insideThisClip
- ? clip.startMs + (sourceNowMs - clip.sourceStartMs)
- : null;
- const next = decoded ?? positionRef.current + elapsed;
+ /* Media currentTime can be exposed in coarse jumps while Chromium is
+ handling pointer/hover work. A monotonic local clock keeps the ruler
+ smooth; media is still sought at cuts and explicit user jumps. */
+ const next = positionRef.current + elapsed;
// Past the end of everything there is nothing left to show, so playback
// stops there instead of running the clock over an empty ruler.
if (next >= contentEndMs) {
diff --git a/src/components/workspace/TimelineClipThumbnails.test.tsx b/src/components/workspace/TimelineClipThumbnails.test.tsx
new file mode 100644
index 0000000..66c4c3a
--- /dev/null
+++ b/src/components/workspace/TimelineClipThumbnails.test.tsx
@@ -0,0 +1,57 @@
+// @vitest-environment jsdom
+
+import "@testing-library/jest-dom/vitest";
+import { cleanup, render, waitFor } from "@testing-library/react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { createDraftGenerationJob, type TimelineClip } from "../../lib/project";
+import { readProjectFileUrl } from "../../lib/persistence";
+import { TimelineClipThumbnails } from "./TimelineClipThumbnails";
+
+vi.mock("../../lib/persistence", () => ({
+ readProjectFileUrl: vi.fn(async (_folder: string, relativePath: string) => `blob:${relativePath}`),
+}));
+vi.mock("./ShotThumbnail", () => ({ shotPoster: vi.fn() }));
+
+beforeEach(() => {
+ Object.defineProperty(URL, "revokeObjectURL", { configurable: true, value: vi.fn() });
+});
+afterEach(cleanup);
+
+describe("timeline clip thumbnails", () => {
+ it("places cached source frames at their correct offsets in a trimmed clip", async () => {
+ const job = {
+ ...createDraftGenerationJob("A scene", { id: "scene-one", durationSeconds: 8 }),
+ status: "completed" as const,
+ stage: "completed" as const,
+ progress: 1,
+ outputRelativePath: "media/generated/scene-one.mp4",
+ };
+ const clip: TimelineClip = {
+ id: "clip-one",
+ assetId: "asset-scene-one",
+ trackId: "track-video-1",
+ startMs: 10_000,
+ durationMs: 5_000,
+ sourceStartMs: 1_000,
+ label: "A scene",
+ color: null,
+ status: "generated",
+ };
+ const view = render(<TimelineClipThumbnails folderPath="C:\Project" job={job} clip={clip} />);
+
+ await waitFor(() => expect(view.container.querySelectorAll("img")).toHaveLength(3));
+ expect(vi.mocked(readProjectFileUrl).mock.calls.map((call) => call[1])).toEqual([
+ "thumbnails/timeline/scene-one/00000000.jpg",
+ "thumbnails/timeline/scene-one/00002000.jpg",
+ "thumbnails/timeline/scene-one/00004000.jpg",
+ ]);
+ expect([...view.container.querySelectorAll("img")].map((image) => ({
+ left: image.style.left,
+ width: image.style.width,
+ }))).toEqual([
+ { left: "-20%", width: "40%" },
+ { left: "20%", width: "40%" },
+ { left: "60%", width: "40%" },
+ ]);
+ });
+});
diff --git a/src/components/workspace/TimelineClipThumbnails.tsx b/src/components/workspace/TimelineClipThumbnails.tsx
new file mode 100644
index 0000000..998500f
--- /dev/null
+++ b/src/components/workspace/TimelineClipThumbnails.tsx
@@ -0,0 +1,80 @@
+import { useEffect, useMemo, useState } from "react";
+import { readProjectFileUrl } from "../../lib/persistence";
+import { type GenerationJob, type TimelineClip } from "../../lib/project";
+import {
+ TIMELINE_THUMBNAIL_INTERVAL_MS,
+ timelineThumbnailRelativePath,
+ writeTimelineThumbnailDataUrl,
+} from "../../lib/timelineThumbnails";
+import { shotPoster } from "./ShotThumbnail";
+
+interface TimelineStill {
+ timeMs: number;
+ url: string;
+}
+
+/** Cached scene frames laid over the span of source footage a clip uses. The
+ * strip remains source-time accurate when a clip is trimmed, split or moved. */
+export function TimelineClipThumbnails({ folderPath, job, clip }: {
+ folderPath: string;
+ job: GenerationJob;
+ clip: TimelineClip;
+}) {
+ const sourceEndMs = clip.sourceStartMs + clip.durationMs;
+ const times = useMemo(() => {
+ const first = Math.floor(clip.sourceStartMs / TIMELINE_THUMBNAIL_INTERVAL_MS) * TIMELINE_THUMBNAIL_INTERVAL_MS;
+ const result: number[] = [];
+ for (let timeMs = Math.max(0, first); timeMs < sourceEndMs; timeMs += TIMELINE_THUMBNAIL_INTERVAL_MS) {
+ result.push(timeMs);
+ }
+ return result;
+ }, [clip.sourceStartMs, sourceEndMs]);
+ const [stills, setStills] = useState<TimelineStill[]>([]);
+
+ useEffect(() => {
+ if (job.status !== "completed" || !job.outputRelativePath || times.length === 0) {
+ setStills([]);
+ return;
+ }
+ const outputRelativePath = job.outputRelativePath;
+ let live = true;
+ const objectUrls: string[] = [];
+ void Promise.all(times.map(async (timeMs): Promise<TimelineStill | null> => {
+ try {
+ const cached = await readProjectFileUrl(folderPath, timelineThumbnailRelativePath(job.id, timeMs), "image/jpeg");
+ if (cached) {
+ objectUrls.push(cached);
+ return { timeMs, url: cached };
+ }
+ } catch {
+ // Older projects have no disk cache. Decode the same source frame once,
+ // show it now, and quietly seed the cache for the next visit.
+ }
+ const generated = await shotPoster(folderPath, outputRelativePath, timeMs / 1_000);
+ if (!generated.poster) return null;
+ void writeTimelineThumbnailDataUrl(folderPath, job.id, timeMs, generated.poster).catch(() => undefined);
+ return { timeMs, url: generated.poster };
+ })).then((loaded) => {
+ if (live) setStills(loaded.filter((still): still is TimelineStill => still !== null));
+ else objectUrls.forEach((url) => URL.revokeObjectURL(url));
+ });
+ return () => {
+ live = false;
+ objectUrls.forEach((url) => URL.revokeObjectURL(url));
+ };
+ }, [folderPath, job.id, job.outputRelativePath, job.status, times]);
+
+ if (stills.length === 0) return null;
+ return <span className="timeline-clip__thumbnails" aria-hidden="true">
+ {stills.map((still) => <img
+ key={still.timeMs}
+ src={still.url}
+ alt=""
+ draggable={false}
+ style={{
+ left: `${(still.timeMs - clip.sourceStartMs) / clip.durationMs * 100}%`,
+ width: `${TIMELINE_THUMBNAIL_INTERVAL_MS / clip.durationMs * 100}%`,
+ }}
+ />)}
+ </span>;
+}
diff --git a/src/components/workspace/TimelineClipWaveform.test.tsx b/src/components/workspace/TimelineClipWaveform.test.tsx
new file mode 100644
index 0000000..1506c0e
--- /dev/null
+++ b/src/components/workspace/TimelineClipWaveform.test.tsx
@@ -0,0 +1,32 @@
+// @vitest-environment jsdom
+
+import { cleanup, render, waitFor } from "@testing-library/react";
+import { afterEach, expect, it, vi } from "vitest";
+import { audioWaveformSegment, loadAudioWaveform } from "../../lib/audioWaveform";
+import { type ProjectAsset, type TimelineClip } from "../../lib/project";
+import { TimelineClipWaveform } from "./TimelineClipWaveform";
+
+vi.mock("../../lib/audioWaveform", () => ({
+ audioWaveformSegment: vi.fn(),
+ loadAudioWaveform: vi.fn(),
+}));
+
+afterEach(() => {
+ cleanup();
+ vi.clearAllMocks();
+});
+
+it("draws each waveform sample as a vertical stroke without an envelope outline", async () => {
+ vi.mocked(loadAudioWaveform).mockResolvedValue({ durationSeconds: 1, levels: new Float32Array([1]) });
+ vi.mocked(audioWaveformSegment).mockReturnValue([1, 0.5, 0]);
+ const { container } = render(<TimelineClipWaveform
+ folderPath="C:\\Project"
+ asset={{ id: "asset-audio", name: "Voice", kind: "audio" } as ProjectAsset}
+ clip={{ id: "clip-audio", assetId: "asset-audio", trackId: "track-a1", startMs: 0, durationMs: 1_000, sourceStartMs: 0 } as TimelineClip}
+ />);
+
+ await waitFor(() => expect(container.querySelector(".clip-audio-visualization__samples")).not.toBeNull());
+ const path = container.querySelector(".clip-audio-visualization__samples");
+ expect(container.querySelector("polyline")).toBeNull();
+ expect(path?.getAttribute("d")).toBe("M0.00,7.00V93.00 M500.00,28.50V71.50 M1000.00,50.00V50.00");
+});
diff --git a/src/components/workspace/TimelineClipWaveform.tsx b/src/components/workspace/TimelineClipWaveform.tsx
new file mode 100644
index 0000000..fe1a541
--- /dev/null
+++ b/src/components/workspace/TimelineClipWaveform.tsx
@@ -0,0 +1,40 @@
+import { useEffect, useMemo, useState } from "react";
+import { audioWaveformSegment, loadAudioWaveform } from "../../lib/audioWaveform";
+import { type ProjectAsset, type TimelineClip } from "../../lib/project";
+
+const DISPLAY_POINTS = 1_024;
+
+const sampleLines = (levels: number[]) => levels.map((level, index) => {
+ const x = index / Math.max(1, levels.length - 1) * 1000;
+ const amplitude = level * 43;
+ return `M${x.toFixed(2)},${(50 - amplitude).toFixed(2)}V${(50 + amplitude).toFixed(2)}`;
+}).join(" ");
+
+/** A source-time-accurate series of amplitude samples. Each sample is its own
+ * vertical stroke around the centre, avoiding a continuous envelope outline. */
+export function TimelineClipWaveform({ folderPath, asset, clip, cacheVersion = "" }: {
+ folderPath: string;
+ asset: ProjectAsset;
+ clip: TimelineClip;
+ cacheVersion?: string;
+}) {
+ const [levels, setLevels] = useState<number[] | null>(null);
+ useEffect(() => {
+ let live = true;
+ setLevels(null);
+ void loadAudioWaveform(folderPath, asset, cacheVersion).then((overview) => {
+ if (live) setLevels(audioWaveformSegment(overview, clip.sourceStartMs / 1_000, clip.durationMs / 1_000, DISPLAY_POINTS));
+ }).catch(() => {
+ if (live) setLevels([]);
+ });
+ return () => { live = false; };
+ }, [asset, cacheVersion, clip.durationMs, clip.sourceStartMs, folderPath]);
+
+ const lines = useMemo(() => levels && levels.length > 0 ? sampleLines(levels) : null, [levels]);
+
+ return <span className={`clip-audio-visualization ${levels === null ? "loading" : ""}`} aria-hidden="true">
+ <svg viewBox="0 0 1000 100" preserveAspectRatio="none" focusable="false">
+ {lines && <path className="clip-audio-visualization__samples" d={lines} />}
+ </svg>
+ </span>;
+}
diff --git a/src/components/workspace/TimelineView.tsx b/src/components/workspace/TimelineView.tsx
index 1ccf47f..472c9cf 100644
--- a/src/components/workspace/TimelineView.tsx
+++ b/src/components/workspace/TimelineView.tsx
@@ -1,6 +1,6 @@
import {
- Copy, Film, Layers3, LayoutGrid, List, Lock, LockOpen, Music2,
- Pause, Play, Plus, Scissors, SkipBack, SkipForward, Trash2, Upload, Video,
+ Copy, Film, Layers3, LayoutGrid, List, Lock, LockOpen,
+ Pause, Play, Plus, Scissors, SkipBack, SkipForward, Trash2, Upload,
Volume2, VolumeX, X,
} from "lucide-react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
@@ -29,8 +29,14 @@ import { ProgramMonitor } from "./ProgramMonitor";
import { sceneShape, STATUS_WORD } from "./sceneStatus";
import { ShotThumbnail } from "./ShotThumbnail";
import { CommittedNumberInput } from "./CommittedNumberInput";
+import { TimelineClipThumbnails } from "./TimelineClipThumbnails";
+import { TimelineClipWaveform } from "./TimelineClipWaveform";
const MIN_DURATION = 10_000;
+/* One minute fills the visible ruler. Longer films keep that readable scale
+ and extend sideways inside a scrollable canvas instead of squeezing every
+ cut into the same width. */
+const VISIBLE_TIMELINE_MS = 60_000;
/* How long a dropped clip is when the file itself cannot say.
A drop normally lasts as long as the footage: the media panel decodes each
file to draw its thumbnail and records the duration it read there (see
@@ -88,6 +94,7 @@ interface DragSession {
function trackIdAtPoint(x: number, y: number): string | undefined {
if (typeof document.elementsFromPoint !== "function") return undefined;
for (const element of document.elementsFromPoint(x, y)) {
+ if (!element) continue;
const lane = (element as HTMLElement).closest?.("[data-track-id]") as HTMLElement | null;
if (lane?.dataset.trackId) return lane.dataset.trackId;
}
@@ -189,17 +196,23 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
const tracks = config.timeline.tracks;
const selected = useMemo(() => tracks.flatMap((track) => track.clips).find((clip) => clip.id === selectedId), [tracks, selectedId]);
const selectedTrack = tracks.find((track) => track.id === selected?.trackId);
+ const selectedMediaAsset = config.assets.find((asset) => asset.id === selected?.assetId);
const selectedTransform = selected ? clipTransform(selected) : null;
const selectedLook = selected ? clipLook(selected) : null;
const selectedTransition = selected ? clipTransition(selected) : null;
- const visualControlsDisabled = selectedTrack?.kind !== "video" || selectedTrack.locked;
+ const visualControlsDisabled = selectedMediaAsset?.kind === "audio" || selectedTrack?.kind !== "video" || selectedTrack.locked;
const clipCount = tracks.reduce((total, track) => total + track.clips.length, 0);
/** Where the last clip ends — where playback stops, which is not the same as
* where the ruler stops (the canvas is at least as long as the film the user
* asked for, however little of it is cut yet). */
const contentEndMs = tracks.reduce((end, track) => track.clips.reduce((furthest, clip) => Math.max(furthest, clipEndMs(clip)), end), 0);
const duration = useMemo(() => canvasDuration(config), [config]);
+ const timelineWidth = `${Math.max(100, duration / VISIBLE_TIMELINE_MS * 100)}%`;
const mediaAssets = useMemo(() => config.assets.filter((asset) => asset.kind !== "generated"), [config.assets]);
+ const assetsById = useMemo(() => new Map(config.assets.map((asset) => [asset.id, asset])), [config.assets]);
+ const generatedJobsByAssetId = useMemo(() => new Map(
+ config.generationJobs.map((job) => [generationAssetId(job.id), job]),
+ ), [config.generationJobs]);
const ticks = useMemo(() => rulerTicks(duration), [duration]);
/* The lanes are ruled at the SAME interval as the ruler above them, so a line
under a clip is a line under a number. They used to be a fixed 14.7% of the
@@ -209,9 +222,8 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
useEffect(() => { if (clipCount === 0 && playing) setPlaying(false); }, [clipCount, playing]);
- /* The playhead is driven by the program monitor while playing — it reads
- where the decoder actually is, so the picture and the playhead cannot
- drift apart. These two are handed down rather than declared inline
+ /* The playhead is driven by the program monitor's monotonic animation clock
+ while playing. These two are handed down rather than declared inline
because the monitor's animation loop depends on their identity: a new
closure per render would tear its clock down sixty times a second. */
const seek = useCallback((ms: number) => setPlayhead(ms), []);
@@ -233,6 +245,9 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
const grid = timelineGrid.current;
if (!grid) return;
const onWheel = (event: WheelEvent) => {
+ // Preserve a trackpad's horizontal gesture for the long timeline's
+ // native scroll container. Vertical wheel movement remains scrubbing.
+ if (duration > VISIBLE_TIMELINE_MS && Math.abs(event.deltaX) > Math.abs(event.deltaY)) return;
const raw = Math.abs(event.deltaY) >= Math.abs(event.deltaX) ? event.deltaY : event.deltaX;
if (raw === 0) return;
event.preventDefault();
@@ -364,6 +379,7 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
durationMs: null,
width: null,
height: null,
+ hasAudio: file.kind === "audio" ? true : null,
createdAt: now,
}));
onChange({ ...config, assets: [...config.assets, ...assets] });
@@ -375,11 +391,11 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
}
};
- /* A video or image belongs on a video track and a sound on an audio track;
- dropping either on the other would make a clip nothing can ever play. */
+ /* Every lane is audiovisual, so video, stills, and sound can be arranged on
+ any unlocked track. */
const acceptsAsset = (track: ProjectConfig["timeline"]["tracks"][number], asset: ProjectAsset | undefined) => {
if (!asset || track.locked) return false;
- return track.kind === "audio" ? asset.kind === "audio" : asset.kind !== "audio";
+ return asset.kind !== "caption";
};
const acceptsScene = (track: ProjectConfig["timeline"]["tracks"][number] | undefined, scene: GenerationJob | undefined) =>
@@ -399,6 +415,7 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
durationMs: asset.durationMs ?? value.durationMs,
width: asset.width ?? value.width,
height: asset.height ?? value.height,
+ hasAudio: asset.hasAudio ?? value.hasAudio,
});
/* Measurements arrive from asynchronous decodes, and two of them can land
between one render and the next. They queue here so that two arriving in
@@ -433,7 +450,7 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
const value = pending.get(asset.id);
if (!value) return asset;
const next = applyMeasured(asset, value);
- if (next.durationMs === asset.durationMs && next.width === asset.width && next.height === asset.height) return asset;
+ if (next.durationMs === asset.durationMs && next.width === asset.width && next.height === asset.height && next.hasAudio === asset.hasAudio) return asset;
learned = true;
return next;
});
@@ -728,7 +745,7 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
onClick={() => void importMedia()}
disabled={importing || !isTauri()}
title={isTauri() ? "Add video, sound, or image files — video and sound stay where they are, images are copied in" : "Importing files is available in the desktop app"}
- ><Upload size={16} /> {importing ? "Importing…" : "Import"}</button>
+ ><Upload size={16} /> Import</button>
<div className="layout-toggle" role="group" aria-label="Media layout">
<button
className={mediaLayout === "grid" ? "active" : ""}
@@ -764,7 +781,7 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
setDraggedAsset(undefined);
}}
onDragEnd={() => { setDraggedScene(undefined); setDropTrackId(null); }}
- title={`${job.title} — drag onto a video track below`}
+ title={`${job.title} — drag onto a track below`}
>
<span className="scene-card__thumb">
{job.status !== "draft" && <ShotThumbnail folderPath={folderPath} job={job} seconds={0} shotNumber={1} estimatedCompletionAt={generationCompletionTimes[job.id] ?? null} />}
@@ -923,11 +940,12 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
<button onClick={removeSelected} disabled={deleteBlockedBy !== null} title={deleteBlockedBy ?? "Delete selected clip"}><Trash2 size={16} /> Delete</button>
</div>
</header>
- <div
- className="timeline-grid"
- ref={timelineGrid}
- style={{ "--lane-grid": `${(tickStepMs / duration) * 100}%` } as React.CSSProperties}
- title="Scroll to scrub, or drag along the ruler. Hold Shift to scrub one frame at a time. Clips snap to the cuts around them; Escape abandons a drag.">
+ <div className={`timeline-grid-scroll${duration > VISIBLE_TIMELINE_MS ? " timeline-grid-scroll--wide" : ""}`}>
+ <div
+ className="timeline-grid"
+ ref={timelineGrid}
+ style={{ "--lane-grid": `${(tickStepMs / duration) * 100}%`, width: timelineWidth } as React.CSSProperties}
+ title="Scroll to scrub, or drag along the ruler. Hold Shift to scrub one frame at a time. Clips snap to the cuts around them; Escape abandons a drag.">
<div className="track-corner"><span>Tracks</span></div>
{/* Press to jump, hold and drag to scrub. The monitor seeks to wherever
this lands, so dragging along the ruler runs the picture past under
@@ -952,15 +970,17 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
key={track.id}
track={track}
duration={duration}
+ folderPath={folderPath}
+ generatedJobsByAssetId={generatedJobsByAssetId}
+ assetsById={assetsById}
selectedId={selectedId}
draggingId={dragging ? dragRef.current?.clipId : undefined}
onClipPointerDown={beginDrag}
onDeleteClip={deleteClip}
dropActive={dropTrackId === track.id}
/* A lane that stays dark is indistinguishable from a lane the
- pointer simply is not over, so a sound held above a video track
- looked like it had not been picked up rather than like it was
- being refused. Marked for the whole drag, not just on hover. */
+ pointer simply is not over. Mark incompatible or locked lanes
+ for the whole drag, not just on hover. */
dropBlocked={(draggedAsset !== undefined && !acceptsAsset(track, draggedAsset))
|| (draggedScene !== undefined && !acceptsScene(track, draggedScene))}
onSelect={setSelectedId}
@@ -992,16 +1012,20 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {
dropAsset(track.id, assetId, ratio, rect.width);
}}
/>)}
- <div className="timeline-playhead" style={{ left: `calc(var(--track-column) + (100% - var(--track-column)) * ${playhead / duration})` }}><span /><i /></div>
+ <div className="timeline-playhead" style={{ left: `calc(var(--track-column) + (100% - var(--track-column)) * ${playhead / duration})` }}><span /><i /></div>
+ </div>
</div>
</section>
</div>
);
}
-function TrackRow({ track, duration, selectedId, draggingId, dropActive, dropBlocked, onSelect, onToggle, onRename, onClipPointerDown, onDeleteClip, onDragOverLane, onDragLeaveLane, onDropLane }: {
+function TrackRow({ track, duration, folderPath, generatedJobsByAssetId, assetsById, selectedId, draggingId, dropActive, dropBlocked, onSelect, onToggle, onRename, onClipPointerDown, onDeleteClip, onDragOverLane, onDragLeaveLane, onDropLane }: {
track: ProjectConfig["timeline"]["tracks"][number];
duration: number;
+ folderPath: string;
+ generatedJobsByAssetId: ReadonlyMap<string, GenerationJob>;
+ assetsById: ReadonlyMap<string, ProjectAsset>;
selectedId: string;
/** The clip currently being dragged, so it can be drawn as the thing under
* the pointer rather than as one more clip sitting on the lane. */
@@ -1017,14 +1041,10 @@ function TrackRow({ track, duration, selectedId, draggingId, dropActive, dropBlo
onDragLeaveLane: () => void;
onDropLane: (event: React.DragEvent<HTMLDivElement>) => void;
}) {
- /* The name no longer carries its kind ("Track 1, Video" is now "Track 1"),
- which the icon and the line under the field already say — but that leaves a
- video and an audio track both called "Track 1", and a button announced as
- "Mute Track 1" twice names neither. The controls say which one in full. */
- const named = `${track.kind === "audio" ? "audio" : "video"} ${track.name}`;
+ const named = track.name;
return <>
<div className="track-head">
- <span className={`track-kind track-kind--${track.kind}`}>{track.kind === "audio" ? <Music2 size={16} /> : <Video size={16} />}</span>
+ <span className="track-kind"><Film size={16} /></span>
{/* The name is an editable field, not a label: a project with three video
layers needs the user's own words on them, and an always-live input
needs no discovery. Blanking it falls back the way a clip name does,
@@ -1035,8 +1055,8 @@ function TrackRow({ track, duration, selectedId, draggingId, dropActive, dropBlo
aria-label={`Rename ${named}`}
title="Rename this track"
onChange={(event) => onRename(track.id, event.target.value || "Untitled track")}
- /><small>{track.kind === "audio" ? "Audio" : "Video"}</small></div>
- <button className={track.muted ? "active" : ""} onClick={() => onToggle(track.id, "muted")} aria-label={`${track.muted ? "Unmute" : "Mute"} ${named}`} title={`${track.muted ? "Unmute" : "Mute"} ${named}`}>{track.muted ? <VolumeX size={16} /> : <Volume2 size={16} />}</button>
+ /></div>
+ <button className={track.muted ? "active" : ""} onClick={() => onToggle(track.id, "muted")} aria-label={`${track.muted ? "Unmute" : "Mute"} audio on ${named}`} title={`${track.muted ? "Unmute" : "Mute"} audio on ${named}`}>{track.muted ? <VolumeX size={16} /> : <Volume2 size={16} />}</button>
<button className={track.locked ? "active" : ""} onClick={() => onToggle(track.id, "locked")} aria-label={`${track.locked ? "Unlock" : "Lock"} ${named}`} title={`${track.locked ? "Unlock" : "Lock"} ${named}`}>{track.locked ? <Lock size={16} /> : <LockOpen size={16} />}</button>
</div>
<div
@@ -1045,12 +1065,25 @@ function TrackRow({ track, duration, selectedId, draggingId, dropActive, dropBlo
lanes is hit-tested against the document, and the id is how the answer
gets back to the model. */
data-track-id={track.id}
- title={dropBlocked ? (track.locked ? `${track.name} is locked` : `${track.name} only takes ${track.kind === "audio" ? "sound" : "video and pictures"}`) : undefined}
+ title={dropBlocked ? (track.locked ? `${track.name} is locked` : `${track.name} cannot take this item`) : undefined}
onDragOver={onDragOverLane}
onDragLeave={onDragLeaveLane}
onDrop={onDropLane}
>
- {track.clips.map((clip) => <div
+ {track.clips.map((clip) => {
+ const asset = assetsById.get(clip.assetId);
+ const audioOnly = asset?.kind === "audio";
+ const generation = generatedJobsByAssetId.get(clip.assetId);
+ /* Current renders record this exactly. Completed files from projects
+ saved before that metadata existed use the successful saved output
+ as the compatibility signal; an explicit false always wins. */
+ const hasAudio = audioOnly || asset?.hasAudio === true || Boolean(
+ asset?.kind === "generated"
+ && asset.hasAudio == null
+ && generation?.outputRelativePath
+ && !generation.error?.startsWith("Saved without sound"),
+ );
+ return <div
key={clip.id}
className={`clip-slot ${selectedId === clip.id ? "selected" : ""} ${draggingId === clip.id ? "clip-slot--dragging" : ""}`}
style={{ left: `${clip.startMs / duration * 100}%`, width: `${clip.durationMs / duration * 100}%`, "--clip-color": clip.color } as React.CSSProperties}
@@ -1061,7 +1094,7 @@ function TrackRow({ track, duration, selectedId, draggingId, dropActive, dropBlo
can make sense of. Everything a button gave it is still here —
focus, Enter, the pressed state, a name. */}
<div
- className={`timeline-clip timeline-clip--${track.kind} ${selectedId === clip.id ? "selected" : ""} ${track.locked ? "timeline-clip--locked" : ""}`}
+ className={`timeline-clip ${audioOnly ? "timeline-clip--audio-only" : ""} ${hasAudio ? "timeline-clip--has-audio" : ""} ${selectedId === clip.id ? "selected" : ""} ${track.locked ? "timeline-clip--locked" : ""}`}
role="button"
tabIndex={0}
aria-pressed={selectedId === clip.id}
@@ -1072,7 +1105,19 @@ function TrackRow({ track, duration, selectedId, draggingId, dropActive, dropBlo
onClick={() => onSelect(clip.id)}
onKeyDown={(event) => { if (event.key === "Enter") onSelect(clip.id); }}
onPointerDown={(event) => onClipPointerDown(event, clip, "move")}
- ><span className="clip-text"><b>{clip.label}</b><small>{track.kind === "audio" ? "▂▅▃▆▂▃▇▅▂▆▃▅▂" : `${(clip.durationMs / 1000).toFixed(1)}s · ${clip.status}`}</small></span></div>
+ >
+ {!audioOnly && generation && <TimelineClipThumbnails
+ folderPath={folderPath}
+ job={generation}
+ clip={clip}
+ />}
+ {hasAudio && asset && <TimelineClipWaveform
+ folderPath={folderPath}
+ asset={asset}
+ clip={clip}
+ cacheVersion={generation?.updatedAt}
+ />}
+ </div>
{/* The two ends, which cut rather than move. They stop the press from
reaching the clip body, or every trim would also be a move. A locked
track gets none of them: nothing on it can be changed. */}
@@ -1095,7 +1140,8 @@ function TrackRow({ track, duration, selectedId, draggingId, dropActive, dropBlo
onClick={(event) => { event.stopPropagation(); onDeleteClip(clip.id); }}
><X size={12} /></button>
</>}
- </div>)}
+ </div>;
+ })}
</div>
</>;
}
diff --git a/src/lib/audioWaveform.test.ts b/src/lib/audioWaveform.test.ts
new file mode 100644
index 0000000..fcc990a
--- /dev/null
+++ b/src/lib/audioWaveform.test.ts
@@ -0,0 +1,30 @@
+import { describe, expect, it } from "vitest";
+import { audioWaveformSegment, createAudioWaveformOverview } from "./audioWaveform";
+
+describe("audio waveform", () => {
+ it("keeps quiet passages visible on a logarithmic envelope", () => {
+ const samples = new Float32Array(400);
+ samples.fill(1, 0, 100);
+ samples.fill(0.01, 100, 200);
+ samples.fill(0, 200, 300);
+ samples.fill(0.1, 300, 400);
+ const overview = createAudioWaveformOverview({
+ duration: 4,
+ length: samples.length,
+ numberOfChannels: 1,
+ getChannelData: () => samples,
+ }, 4);
+
+ const levels = audioWaveformSegment(overview, 0, 4, 4);
+ expect(levels[0]).toBeCloseTo(1);
+ expect(levels[1]).toBeGreaterThan(0.3);
+ expect(levels[1]).toBeLessThan(levels[3]);
+ expect(levels[2]).toBe(0);
+ expect(levels[3]).toBeGreaterThan(0.6);
+ });
+
+ it("selects only the source range represented by a trimmed clip", () => {
+ const overview = { durationSeconds: 4, levels: new Float32Array([1, 0, 0.5, 0]) };
+ expect(audioWaveformSegment(overview, 2, 1, 2)).toEqual([1, 1]);
+ });
+});
diff --git a/src/lib/audioWaveform.ts b/src/lib/audioWaveform.ts
new file mode 100644
index 0000000..c149965
--- /dev/null
+++ b/src/lib/audioWaveform.ts
@@ -0,0 +1,103 @@
+import { readMediaFileBytes } from "./persistence";
+import { type ProjectAsset } from "./project";
+
+export interface AudioWaveformOverview {
+ durationSeconds: number;
+ levels: Float32Array;
+}
+
+type WaveformAudioBuffer = Pick<AudioBuffer, "duration" | "length" | "numberOfChannels" | "getChannelData">;
+
+const MIN_OVERVIEW_POINTS = 1_024;
+const MAX_OVERVIEW_POINTS = 16_384;
+const OVERVIEW_POINTS_PER_SECOND = 64;
+const MAX_READS_PER_BIN = 128;
+const DISPLAY_FLOOR_DB = -60;
+const CACHE_LIMIT = 24;
+const OVERVIEWS = new Map<string, Promise<AudioWaveformOverview>>();
+
+/** Reduces decoded PCM to a reusable overview. RMS keeps sustained quiet
+ * passages legible while the peak term retains short transients. */
+export function createAudioWaveformOverview(
+ buffer: WaveformAudioBuffer,
+ requestedPoints?: number,
+): AudioWaveformOverview {
+ const desiredPoints = requestedPoints ?? Math.max(MIN_OVERVIEW_POINTS, Math.ceil(buffer.duration * OVERVIEW_POINTS_PER_SECOND));
+ const pointCount = Math.max(1, Math.min(buffer.length, Math.min(MAX_OVERVIEW_POINTS, desiredPoints)));
+ const levels = new Float32Array(pointCount);
+ const channels = Array.from({ length: buffer.numberOfChannels }, (_, channel) => buffer.getChannelData(channel));
+
+ for (let point = 0; point < pointCount; point += 1) {
+ const start = Math.floor(point * buffer.length / pointCount);
+ const end = Math.max(start + 1, Math.floor((point + 1) * buffer.length / pointCount));
+ const stride = Math.max(1, Math.ceil((end - start) / MAX_READS_PER_BIN));
+ let squares = 0;
+ let peak = 0;
+ let reads = 0;
+ for (const channel of channels) {
+ for (let sample = start; sample < end; sample += stride) {
+ const amplitude = Math.abs(channel[sample] ?? 0);
+ squares += amplitude * amplitude;
+ peak = Math.max(peak, amplitude);
+ reads += 1;
+ }
+ }
+ const rms = reads > 0 ? Math.sqrt(squares / reads) : 0;
+ levels[point] = peak > 0 ? Math.sqrt(rms * peak) : 0;
+ }
+ return { durationSeconds: buffer.duration, levels };
+}
+
+/** Selects the part used by a timeline clip and maps it on a logarithmic
+ * -60 dB..0 dB scale. This prevents quiet speech or room tone from
+ * disappearing against the centre line. */
+export function audioWaveformSegment(
+ overview: AudioWaveformOverview,
+ sourceStartSeconds: number,
+ durationSeconds: number,
+ pointCount = 512,
+): number[] {
+ const result = Array.from({ length: pointCount }, () => 0);
+ if (overview.durationSeconds <= 0 || overview.levels.length === 0 || durationSeconds <= 0) return result;
+ const first = Math.max(0, sourceStartSeconds / overview.durationSeconds * overview.levels.length);
+ const last = Math.min(overview.levels.length, (sourceStartSeconds + durationSeconds) / overview.durationSeconds * overview.levels.length);
+ for (let point = 0; point < pointCount; point += 1) {
+ const start = Math.floor(first + (last - first) * point / pointCount);
+ const end = Math.max(start + 1, Math.ceil(first + (last - first) * (point + 1) / pointCount));
+ for (let index = start; index < Math.min(end, overview.levels.length); index += 1) {
+ result[point] = Math.max(result[point], overview.levels[index]);
+ }
+ }
+ const peak = Math.max(...result);
+ if (peak <= 0) return result;
+ return result.map((level) => {
+ if (level <= 0) return 0;
+ const decibels = 20 * Math.log10(level / peak);
+ return Math.max(0, Math.min(1, (decibels - DISPLAY_FLOOR_DB) / -DISPLAY_FLOOR_DB));
+ });
+}
+
+/** Reads and decodes each source once. A generation timestamp belongs in the
+ * cache key so regenerating the same scene path cannot reuse its old sound. */
+export function loadAudioWaveform(
+ folderPath: string,
+ asset: ProjectAsset,
+ cacheVersion = "",
+): Promise<AudioWaveformOverview> {
+ const location = asset.sourcePath ?? asset.relativePath;
+ if (!location) return Promise.reject(new Error(`${asset.name} has no readable media path.`));
+ const key = `${folderPath}::${location}::${cacheVersion}`;
+ const cached = OVERVIEWS.get(key);
+ if (cached) return cached;
+ const pending = readMediaFileBytes(folderPath, asset).then(async (bytes) => {
+ const context = new OfflineAudioContext(1, 1, 44_100);
+ const decoded = await context.decodeAudioData(bytes.slice(0));
+ return createAudioWaveformOverview(decoded);
+ }).catch((error) => {
+ OVERVIEWS.delete(key);
+ throw error;
+ });
+ OVERVIEWS.set(key, pending);
+ while (OVERVIEWS.size > CACHE_LIMIT) OVERVIEWS.delete(OVERVIEWS.keys().next().value!);
+ return pending;
+}
diff --git a/src/lib/export.test.ts b/src/lib/export.test.ts
index c8f1386..58c2511 100644
--- a/src/lib/export.test.ts
+++ b/src/lib/export.test.ts
@@ -47,7 +47,7 @@ const clip = (id: string, trackId: string, startMs: number, durationMs: number,
...overrides,
});
-/** A project with the app's own four tracks, `clips` distributed by trackId.
+/** A project with the app's three audiovisual tracks, `clips` distributed by trackId.
* `track-story` is the video "Track 1" — the FIRST video track, and the one
* that composites over the video "Track 2" (`track-v2`), because the timeline draws
* tracks in array order from the top. */
@@ -287,7 +287,7 @@ describe("the plan", () => {
it("mixes the audio that lands inside the picture, and cuts it where the picture ends", () => {
const config = project(
- [clip("a", "track-story", 0, 1_000), clip("score", "track-a2", 0, 5_000)],
+ [clip("a", "track-story", 0, 1_000), clip("score", "track-v3", 0, 5_000)],
[asset("asset-a"), asset("asset-score", { kind: "audio", mimeType: "audio/wav" })],
);
const plan = buildExportPlan(config, settings());
@@ -306,7 +306,7 @@ describe("the plan", () => {
const config = project(
[
clip("a", "track-story", 0, 8_000),
- clip("vo", "track-a1", 2_000, 3_000, { sourceStartMs: 4_500 }),
+ clip("vo", "track-v2", 2_000, 3_000, { sourceStartMs: 4_500 }),
],
[asset("asset-a"), asset("asset-vo", { kind: "audio", mimeType: "audio/wav" })],
);
@@ -317,7 +317,7 @@ describe("the plan", () => {
it("leaves out audio that starts after the last frame, and says how much", () => {
const config = project(
- [clip("a", "track-story", 0, 1_000), clip("late", "track-a1", 4_000, 2_000)],
+ [clip("a", "track-story", 0, 1_000), clip("late", "track-v2", 4_000, 2_000)],
[asset("asset-a"), asset("asset-late", { kind: "audio", mimeType: "audio/wav" })],
);
const plan = buildExportPlan(config, settings());
@@ -327,13 +327,13 @@ describe("the plan", () => {
it("does not mix a muted track, and says that too", () => {
const config = project(
- [clip("a", "track-story", 0, 4_000), clip("score", "track-a2", 0, 4_000)],
+ [clip("a", "track-story", 0, 4_000), clip("score", "track-v3", 0, 4_000)],
[asset("asset-a"), asset("asset-score", { kind: "audio", mimeType: "audio/wav" })],
);
const muted = {
...config,
timeline: {
- tracks: config.timeline.tracks.map((track) => (track.id === "track-a2" ? { ...track, muted: true } : track)),
+ tracks: config.timeline.tracks.map((track) => (track.id === "track-v3" ? { ...track, muted: true } : track)),
},
};
const plan = buildExportPlan(muted, settings());
@@ -345,8 +345,8 @@ describe("the plan", () => {
const config = project(
[
clip("a", "track-story", 0, 6_000),
- clip("vo", "track-a1", 0, 4_000),
- clip("score", "track-a2", 2_000, 4_000),
+ clip("vo", "track-v2", 0, 4_000),
+ clip("score", "track-v3", 2_000, 4_000),
],
[
asset("asset-a"),
@@ -359,6 +359,20 @@ describe("the plan", () => {
expect(plan.notes.join(" ")).toContain("overlaps are summed");
});
+ it("mixes embedded video audio and removes it when that audiovisual track is muted", () => {
+ const config = project(
+ [clip("a", "track-story", 0, 2_000), clip("under", "track-v2", 0, 2_000)],
+ [asset("asset-a", { hasAudio: true }), asset("asset-under", { hasAudio: true })],
+ );
+ expect(buildExportPlan(config, settings()).audio.map((segment) => segment.clipId)).toEqual(["a", "under"]);
+
+ const muted = {
+ ...config,
+ timeline: { tracks: config.timeline.tracks.map((track) => track.id === "track-v2" ? { ...track, muted: true } : track) },
+ };
+ expect(buildExportPlan(muted, settings()).audio.map((segment) => segment.clipId)).toEqual(["a"]);
+ });
+
it("blocks on an empty timeline, on missing media and on a container it cannot demux", () => {
expect(buildExportPlan(project([], []), settings()).blockers[0]).toContain("no video clips");
diff --git a/src/lib/export.ts b/src/lib/export.ts
index 05fb4fa..959fd77 100644
--- a/src/lib/export.ts
+++ b/src/lib/export.ts
@@ -19,6 +19,7 @@ import {
type ClipLook,
type ClipTransform,
type ClipTransition,
+ type ProjectAsset,
type ProjectConfig,
type Resolution,
type TimelineClip,
@@ -291,6 +292,21 @@ export interface AudioSegment {
sourceStartMs: number;
}
+const assetCarriesAudio = (asset: ProjectAsset | undefined): boolean => Boolean(
+ asset && (asset.kind === "audio" || ((asset.kind === "video" || asset.kind === "generated") && asset.hasAudio === true)),
+);
+
+const assetCarriesPicture = (asset: ProjectAsset | undefined): boolean =>
+ asset?.kind !== "audio" && asset?.kind !== "caption";
+
+const audioTimelineClips = (config: ProjectConfig, muted: boolean) => {
+ const assets = new Map(config.assets.map((asset) => [asset.id, asset]));
+ return config.timeline.tracks
+ .filter((track) => track.muted === muted)
+ .flatMap((track) => track.clips)
+ .filter((clip) => assetCarriesAudio(assets.get(clip.assetId)));
+};
+
/** Every audio clip's audible part, in timeline order.
*
* Overlaps are NOT resolved here: two clips over the same moment are two
@@ -298,9 +314,11 @@ export interface AudioSegment {
* not a segment — a gap is simply a stretch no segment covers. */
export function audioSegments(config: ProjectConfig, durationMs: number): AudioSegment[] {
const segments: AudioSegment[] = [];
+ const assets = new Map(config.assets.map((asset) => [asset.id, asset]));
for (const track of config.timeline.tracks) {
- if (track.kind !== "audio" || track.muted) continue;
+ if (track.muted) continue;
for (const clip of track.clips) {
+ if (!assetCarriesAudio(assets.get(clip.assetId))) continue;
const start = Math.max(0, clip.startMs);
const end = Math.min(durationMs, clip.startMs + clip.durationMs);
if (end <= start) continue;
@@ -330,7 +348,7 @@ export interface ExportPlan {
clipCount: number;
/** Frames with nothing over them; they get the project's background colour. */
gapFrames: number;
- /** Clips on audio tracks, muted tracks included. */
+ /** Clips carrying audio streams, muted tracks included. */
audioClipCount: number;
/** The audible part of each one. Empty means the file has no sound, and the
* notes say why. */
@@ -365,11 +383,16 @@ export function sourceTimeMsForFrame(
* has something at this moment wins — an overlay covers the story beneath it.
* Within one track two clips can overlap after a drag; the later one is the
* more recent edit, so it is the one on top. */
-export function visibleClipAt(tracks: TimelineTrack[], timeMs: number): TimelineClip | null {
+export function visibleClipAt(
+ tracks: TimelineTrack[],
+ timeMs: number,
+ assetsById?: ReadonlyMap<string, ProjectAsset>,
+): TimelineClip | null {
for (const track of tracks) {
if (track.kind !== "video") continue;
let best: TimelineClip | null = null;
for (const clip of track.clips) {
+ if (assetsById && !assetCarriesPicture(assetsById.get(clip.assetId))) continue;
if (timeMs < clip.startMs || timeMs >= clip.startMs + clip.durationMs) continue;
if (!best || clip.startMs >= best.startMs) best = clip;
}
@@ -382,9 +405,11 @@ export function visibleClipAt(tracks: TimelineTrack[], timeMs: number): Timeline
* past the last shot cannot extend the file — claiming otherwise would promise
* frames that do not exist. */
export function videoDurationMs(config: ProjectConfig): number {
+ const assets = new Map(config.assets.map((asset) => [asset.id, asset]));
return config.timeline.tracks
.filter((track) => track.kind === "video")
.flatMap((track) => track.clips)
+ .filter((clip) => assetCarriesPicture(assets.get(clip.assetId)))
.reduce((end, clip) => Math.max(end, clip.startMs + clip.durationMs), 0);
}
@@ -393,6 +418,7 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)
const frameRate = settings.frameRate;
const tracks = config.timeline.tracks;
const videoTracks = tracks.filter((track) => track.kind === "video");
+ const assetsById = new Map(config.assets.map((asset) => [asset.id, asset]));
const durationMs = videoDurationMs(config);
const frameCount = Math.round((durationMs * frameRate) / 1000);
@@ -402,7 +428,7 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)
const segments: ExportSegment[] = [];
let gapFrames = 0;
for (let frame = 0; frame < frameCount; frame += 1) {
- const clip = visibleClipAt(videoTracks, (frame * 1000) / frameRate);
+ const clip = visibleClipAt(videoTracks, (frame * 1000) / frameRate, assetsById);
const last = segments[segments.length - 1];
if (!clip) {
gapFrames += 1;
@@ -428,9 +454,7 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)
}
const usedClipIds = new Set(segments.flatMap((segment) => (segment.kind === "clip" ? [segment.clipId] : [])));
- const audioClipCount = tracks
- .filter((track) => track.kind === "audio")
- .reduce((total, track) => total + track.clips.length, 0);
+ const audioClipCount = audioTimelineClips(config, false).length + audioTimelineClips(config, true).length;
const blockers: string[] = [];
if (frameCount === 0) {
@@ -497,9 +521,7 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)
/* Audio the file cannot hold, counted rather than dropped in silence. The
picture decides the length; sound past the last frame has nowhere to go. */
const audible = new Set(audio.map((segment) => segment.clipId));
- const beyondPicture = tracks
- .filter((track) => track.kind === "audio" && !track.muted)
- .flatMap((track) => track.clips)
+ const beyondPicture = audioTimelineClips(config, false)
.filter((clip) => !audible.has(clip.id)).length;
if (beyondPicture > 0) {
notes.push(
@@ -512,9 +534,7 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)
`The sound stops with the picture at ${formatDuration(durationMs)}, where ${trimmed === 1 ? "one clip is" : `${trimmed} clips are`} cut off.`,
);
}
- const mutedClips = tracks
- .filter((track) => track.kind === "audio" && track.muted)
- .reduce((total, track) => total + track.clips.length, 0);
+ const mutedClips = audioTimelineClips(config, true).length;
if (mutedClips > 0) {
notes.push(`${mutedClips} audio ${mutedClips === 1 ? "clip is" : "clips are"} on a muted track and will not be heard.`);
}
@@ -525,6 +545,7 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)
}
const unusedClips = videoTracks
.flatMap((track) => track.clips)
+ .filter((clip) => assetCarriesPicture(assetsById.get(clip.assetId)))
.filter((clip) => !usedClipIds.has(clip.id));
if (unusedClips.length > 0) {
notes.push(
diff --git a/src/styles/timeline.css b/src/styles/timeline.css
index eb527e0..747da11 100644
--- a/src/styles/timeline.css
+++ b/src/styles/timeline.css
@@ -987,6 +987,17 @@
/* --- Track grid ------------------------------------------------------------ */
+.timeline-grid-scroll {
+ min-width: 0;
+ min-height: 0;
+ overflow-x: hidden;
+ overflow-y: hidden;
+}
+
+.timeline-grid-scroll--wide {
+ overflow-x: auto;
+}
+
.timeline-grid {
/* Wide enough for a track name to be read in full beside the kind icon and
the two toggles. It was cut to fit the seeded names when those still ended
@@ -996,10 +1007,17 @@
display: grid;
grid-template-columns: var(--track-column) minmax(0, 1fr);
grid-template-rows: 36px;
- grid-auto-rows: 72px;
- height: 100%;
- overflow-x: hidden;
- overflow-y: auto;
+ grid-auto-rows: 96px;
+ min-width: 100%;
+ min-height: 100%;
+}
+
+/* Track controls remain available while the ruler and clips move sideways. */
+.timeline-grid-scroll--wide .track-corner,
+.timeline-grid-scroll--wide .track-head {
+ position: sticky;
+ left: 0;
+ z-index: 10;
}
.track-corner {
@@ -1061,11 +1079,6 @@
background: var(--accent-soft-strong);
}
-.track-kind--audio {
- color: var(--audio);
- background: var(--audio-soft);
-}
-
.track-head .track-name,
.track-head small {
display: block;
@@ -1143,8 +1156,8 @@
background-size: var(--lane-grid, 14.7%) 100%;
}
-.track-lane.muted {
- opacity: 0.46;
+.track-lane.muted .clip-audio-visualization {
+ opacity: 0.24;
}
/* Lit while a compatible media file is held over this lane. A drop target the
@@ -1180,7 +1193,7 @@
.clip-slot {
position: absolute;
top: var(--space-2);
- height: 56px;
+ height: 80px;
min-width: 22px;
}
@@ -1216,11 +1229,34 @@
.timeline-clip::before {
content: "";
position: absolute;
+ z-index: 1;
inset: 0;
background: repeating-linear-gradient(90deg, var(--clip-stripe) 0 18px, transparent 18px 21px);
+ pointer-events: none;
}
-.timeline-clip--audio {
+.timeline-clip__thumbnails {
+ position: absolute;
+ z-index: 0;
+ inset: 0;
+ overflow: hidden;
+ pointer-events: none;
+}
+
+.timeline-clip--has-audio:not(.timeline-clip--audio-only) .timeline-clip__thumbnails {
+ bottom: 24px;
+}
+
+.timeline-clip__thumbnails img {
+ position: absolute;
+ top: 0;
+ height: 100%;
+ max-width: none;
+ object-fit: cover;
+ user-select: none;
+}
+
+.timeline-clip--audio-only {
background: linear-gradient(
180deg,
color-mix(in srgb, var(--clip-color, var(--clip-audio-default)) 62%, var(--clip-audio-shade-top)),
@@ -1228,6 +1264,46 @@
);
}
+.clip-audio-visualization {
+ position: absolute;
+ z-index: 2;
+ right: 0;
+ bottom: 0;
+ left: 0;
+ height: 24px;
+ padding: 3px 4px;
+ overflow: hidden;
+ border-top: 1px solid color-mix(in srgb, var(--line-strong) 70%, transparent);
+ background: color-mix(in srgb, var(--panel) 34%, transparent);
+ transition: opacity 120ms ease;
+ pointer-events: none;
+}
+
+.timeline-clip--audio-only .clip-audio-visualization {
+ top: 0;
+ height: auto;
+ background: transparent;
+}
+
+.clip-audio-visualization svg {
+ display: block;
+ width: 100%;
+ height: 100%;
+ overflow: visible;
+}
+
+.clip-audio-visualization__samples {
+ fill: none;
+ stroke: color-mix(in srgb, var(--text-primary) 82%, var(--clip-color, var(--audio)));
+ stroke-width: 1px;
+ stroke-linecap: round;
+ vector-effect: non-scaling-stroke;
+}
+
+.clip-audio-visualization.loading {
+ opacity: 0.48;
+}
+
.timeline-clip:hover {
filter: brightness(1.14);
}
@@ -1327,35 +1403,6 @@
cursor: grabbing;
}
-.clip-text {
- position: relative;
- z-index: 2;
- display: block;
- padding: var(--space-2);
-}
-
-.clip-text b,
-.clip-text small {
- display: block;
- overflow: hidden;
- white-space: nowrap;
- text-overflow: ellipsis;
-}
-
-.clip-text b {
- color: var(--text-primary);
- font-size: var(--text-sm);
- font-weight: var(--weight-semibold);
- line-height: var(--leading-snug);
-}
-
-.clip-text small {
- margin-top: var(--space-1);
- color: var(--text-primary);
- opacity: 0.78;
- font: var(--weight-medium) var(--text-2xs) / 1.1 var(--font-mono);
-}
-
.timeline-playhead {
position: absolute;
z-index: 9;