AI media harness for planning, generating, and editing locally
TypeScript
Add audiovisual timeline previews
Bitti877c82d3dc32
diff --git a/src/components/workspace/MediaThumbnail.tsx b/src/components/workspace/MediaThumbnail.tsxindex b5d367a..5ee1a4e 100644--- a/src/components/workspace/MediaThumbnail.tsx+++ b/src/components/workspace/MediaThumbnail.tsx@@ -44,11 +44,28 @@ export interface MeasuredMedia {durationMs: number | null;width: number | null;height: number | null;+ hasAudio: boolean | null;}-const NOTHING_MEASURED: MeasuredMedia = { durationMs: null, width: null, height: null };+const NOTHING_MEASURED: MeasuredMedia = { durationMs: null, width: null, height: null, hasAudio: null };const measuredSomething = (measured: MeasuredMedia) =>- measured.durationMs !== null || measured.width !== null || measured.height !== null;+ measured.durationMs !== null || measured.width !== null || measured.height !== null || measured.hasAudio !== null;++type AudioAwareVideo = HTMLVideoElement & {+ mozHasAudio?: boolean;+ webkitAudioDecodedByteCount?: number;+ audioTracks?: { length: number };+};++/** Browser engines expose audio-stream presence through different optional+ * properties. Unknown stays null; guessing false would hide real sound. */+const videoHasAudio = (element: HTMLVideoElement): boolean | null => {+ const video = element as AudioAwareVideo;+ if (typeof video.mozHasAudio === "boolean") return video.mozHasAudio;+ if (video.audioTracks) return video.audioTracks.length > 0;+ if (typeof video.webkitAudioDecodedByteCount === "number" && video.webkitAudioDecodedByteCount > 0) return true;+ return null;+};/* A media element reports `duration` as NaN before metadata arrives andInfinity for a live or unseekable stream. Neither is a length. */@@ -131,6 +148,7 @@ async function videoPoster(url: string, posterTimeSeconds: number, jpegQuality:durationMs: durationMsOf(video),width: pixelsOf(video.videoWidth),height: pixelsOf(video.videoHeight),+ hasAudio: videoHasAudio(video),};const target = Number.isFinite(video.duration)? Math.min(Math.max(0, video.duration - 0.05), Math.max(0, posterTimeSeconds))@@ -156,7 +174,7 @@ async function imagePoster(url: string, jpegQuality: number): Promise<{ poster:// point: how long a still is on screen is the cut's decision, not the file's.return {poster: tryDraw(image, image.naturalWidth, image.naturalHeight, jpegQuality),- measured: { durationMs: null, width: pixelsOf(image.naturalWidth), height: pixelsOf(image.naturalHeight) },+ measured: { durationMs: null, width: pixelsOf(image.naturalWidth), height: pixelsOf(image.naturalHeight), hasAudio: false },};}@@ -167,7 +185,7 @@ async function measureAudio(url: string): Promise<MeasuredMedia> {audio.preload = "metadata";audio.src = url;await settled(audio, "loadedmetadata");- const measured: MeasuredMedia = { durationMs: durationMsOf(audio), width: null, height: null };+ const measured: MeasuredMedia = { durationMs: durationMsOf(audio), width: null, height: null, hasAudio: true };release(audio);return measured;}@@ -176,7 +194,9 @@ async function measureAudio(url: string): Promise<MeasuredMedia> {* a duration, so asking for one forever would re-read the file on every* mount; sound never gets a size. */const wantsMeasuring = (asset: ProjectAsset) =>- asset.kind === "image" ? !asset.width || !asset.height : !asset.durationMs;+ asset.kind === "image"+ ? !asset.width || !asset.height+ : !asset.durationMs || (asset.kind === "video" && asset.hasAudio == null);export function MediaThumbnail({ folderPath, asset, onMeasured, posterTimeSeconds = POSTER_TIME_SECONDS, jpegQuality = DEFAULT_JPEG_QUALITY }: {folderPath: string;diff --git a/src/components/workspace/ProgramMonitor.test.tsx b/src/components/workspace/ProgramMonitor.test.tsxindex 8ec87a6..a0de32e 100644--- a/src/components/workspace/ProgramMonitor.test.tsx+++ b/src/components/workspace/ProgramMonitor.test.tsx@@ -1,6 +1,6 @@// @vitest-environment jsdom-import { cleanup, fireEvent, render, screen } from "@testing-library/react";+import { act, cleanup, fireEvent, render, screen } from "@testing-library/react";import { useState } from "react";import { afterEach, describe, expect, it, vi } from "vitest";import { createProjectConfig, parseProjectConfig, STORY_TRACK_ID, type ProjectConfig, type TimelineClip } from "../../lib/project";@@ -10,6 +10,8 @@ vi.mock("@tauri-apps/api/core", () => ({ invoke: vi.fn() }));afterEach(() => {cleanup();+ vi.restoreAllMocks();+ vi.unstubAllGlobals();delete (window as unknown as Record<string, unknown>).__TAURI_INTERNALS__;});@@ -106,4 +108,28 @@ describe("the program monitor", () => {expect(container.querySelector(".program-note")).toBeNull();expect(container.querySelector(".program-picture")).not.toBeNull();});++ it("advances from its local animation clock when renders are delayed", () => {+ const frames: FrameRequestCallback[] = [];+ let now = 0;+ vi.spyOn(performance, "now").mockImplementation(() => now);+ vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => {+ frames.push(callback);+ return frames.length;+ });+ vi.stubGlobal("cancelAnimationFrame", vi.fn());+ const onSeek = vi.fn();+ render(<ProgramMonitor+ config={project([clip({ startMs: 0, sourceStartMs: 0 })])}+ folderPath="C:\\Ceramic Lamp"+ playheadMs={0}+ playing+ onSeek={onSeek}+ onPlayingChange={() => undefined}+ />);++ act(() => { now = 16; frames.shift()?.(now); });+ act(() => { now = 32; frames.shift()?.(now); });+ expect(onSeek.mock.calls.map(([position]) => position)).toEqual([16, 32]);+ });});diff --git a/src/components/workspace/ProgramMonitor.tsx b/src/components/workspace/ProgramMonitor.tsxindex 6cba3b7..783276d 100644--- a/src/components/workspace/ProgramMonitor.tsx+++ b/src/components/workspace/ProgramMonitor.tsx@@ -17,14 +17,12 @@ import { PreviewSources } from "../../lib/exportPipeline";* re-points the same element at the next file.* * Timeline time and source time are different numbers. A clip trimmed to* start 8s into a rush shows second 8 of the file at second 0 of the cut.- * * Sound on the audio tracks plays alongside, each clip in its own element,+ * * Sound from every audiovisual track plays alongside, each additional clip in its own element,* corrected back into step whenever it drifts.*- * While playing, the VIDEO element is the clock: the playhead is derived from- * where the decoder actually is, not from a timer running beside it, so the- * picture and the playhead cannot disagree. Over a gap — or with nothing- * readable — a wall clock takes over, because a black stretch still has to- * play through.+ * While playing, a monotonic animation clock advances the playhead and the+ * media element plays alongside it. Explicit jumps and cuts seek the decoder;+ * the same clock also carries playback through gaps and still images.** The whole file is read into a blob, as everywhere else in PolStudio (see* MediaThumbnail): the project's media lives outside the webview's reach and@@ -37,12 +35,10 @@ const AUDIO_DRIFT_LIMIT_S = 0.25;/** Seeks below this are not worth the stutter — a paused element is already* showing that frame. Roughly one frame at 30fps. */const SEEK_EPSILON_S = 0.033;-/** How far outside a clip's own stretch of its file the decoder may read and- * still be believed. A frame or two of slack, so ordinary rounding at a cut- * does not hand the clock to the wall. */-const CUT_SLACK_MS = 120;-const isImage = (asset: ProjectAsset) => asset.mimeType.startsWith("image/") || asset.kind === "image";+const carriesAudio = (asset: ProjectAsset | undefined) => Boolean(+ asset && (asset.kind === "audio" || ((asset.kind === "video" || asset.kind === "generated") && asset.hasAudio === true)),+);/** Where inside the FILE this moment of the timeline lives. */const sourceTimeMs = (clip: TimelineClip, timelineMs: number) => clip.sourceStartMs + (timelineMs - clip.startMs);@@ -92,8 +88,8 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeekfolderPath: string;playheadMs: number;playing: boolean;- /** Where playback has reached. The view owns the playhead; this only ever- * reports where the decoder actually is. */+ /** Where playback has reached. The view owns the playhead; this reports the+ * monitor's monotonic playback clock. */onSeek: (ms: number) => void;onPlayingChange: (playing: boolean) => void;/** Timeline-only direct manipulation. Export omits these props and therefore@@ -104,7 +100,8 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeekonTransformChange?: (clipId: string, transform: ClipTransform) => void;}) {const tracks = config.timeline.tracks;- const clip = useMemo(() => visibleClipAt(tracks, playheadMs), [tracks, playheadMs]);+ const assetsById = useMemo(() => new Map(config.assets.map((candidate) => [candidate.id, candidate])), [config.assets]);+ const clip = useMemo(() => visibleClipAt(tracks, playheadMs, assetsById), [tracks, playheadMs, assetsById]);const asset = useMemo(() => (clip ? config.assets.find((candidate) => candidate.id === clip.assetId) : undefined),[clip, config.assets],@@ -127,14 +124,15 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeekclipPath: `inset(0 ${(1 - revealEnd) * 100}% 0 ${revealStart * 100}%)`,};}, [frameStyle]);- /* Sound that should be audible at this moment: every audio clip the playhead- is inside, minus the tracks the user has muted. A muted track is dropped- here rather than played silently, so nothing is decoded for it at all. */+ /* Sound that should be audible at this moment: audio-only files and embedded+ sound from every active video, minus muted tracks. The visible video plays+ its own stream, so a second audio element is only needed for the rest. */const audioClips = useMemo(() => tracks- .filter((track) => track.kind === "audio" && !track.muted)+ .filter((track) => !track.muted).flatMap((track) => track.clips)- .filter((candidate) => playheadMs >= candidate.startMs && playheadMs < clipEndMs(candidate)),- [tracks, playheadMs]);+ .filter((candidate) => playheadMs >= candidate.startMs && playheadMs < clipEndMs(candidate))+ .filter((candidate) => candidate.id !== clip?.id && carriesAudio(assetsById.get(candidate.assetId))),+ [tracks, playheadMs, clip?.id, assetsById]);const videoTrackMuted = useMemo(() => tracks.find((track) => track.id === clip?.trackId)?.muted ?? false,[tracks, clip],@@ -195,6 +193,9 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeekconst report = useCallback((ms: number) => {const rounded = Math.round(ms);+ /* Keep the local clock advancing even when React batches visual updates+ while pointer and hover events are busy. */+ positionRef.current = rounded;advancedTo.current = rounded;onSeek(rounded);}, [onSeek]);@@ -239,8 +240,8 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeek/* Scrubbing, every other jump, and the first frame of every new shot.While playing this deliberately does nothing for the playhead's own- advance — the element is the clock then, and seeking it to its own reading- would stutter every frame. It does have to fire on the two other cases:+ advance — seeking the element to every animation tick would stutter. It+ does have to fire on the two other cases:a playhead that moved for some other reason (the ruler, the wheel, a scenecard), and a CUT. Crossing a cut re-points the element at another file, orat another moment of the same file, and neither knows where in the footage@@ -284,10 +285,8 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeekelse video.pause();}, [playing, url, clip?.id]);- /* The clock. One frame at a time: read where the decoder is, turn that back- into a position on the ruler, and hand it up. With no picture to read —- a gap, an unreadable file, a still image — the wall clock stands in, so a- black stretch still plays through at the right speed. */+ /* The clock advances one animation frame at a time. It deliberately does+ not rely on the media element's coarser currentTime update cadence. */useEffect(() => {if (!playing) return;let frame = 0;@@ -296,22 +295,10 @@ export function ProgramMonitor({ config, folderPath, playheadMs, playing, onSeekconst now = performance.now();const elapsed = now - lastTick;lastTick = now;- const video = videoRef.current;- /* The decoder is only the clock while it is actually inside THIS clip's- stretch of the file. Just after a cut it is not: the element has been- re-pointed and its currentTime still reads 0 (or the last shot's- position) until the seek lands. Believing it there would put the- playhead before the clip that is on screen, which hands the render back- to the previous clip and bounces between the two. The wall clock covers- those few frames instead. */- const sourceNowMs = video ? video.currentTime * 1000 : 0;- const insideThisClip = clip !== null- && sourceNowMs >= clip.sourceStartMs - CUT_SLACK_MS- && sourceNowMs <= clip.sourceStartMs + clip.durationMs + CUT_SLACK_MS;- const decoded = clip && video && url && !video.paused && !video.seeking && Number.isFinite(video.currentTime) && insideThisClip- ? clip.startMs + (sourceNowMs - clip.sourceStartMs)- : null;- const next = decoded ?? positionRef.current + elapsed;+ /* Media currentTime can be exposed in coarse jumps while Chromium is+ handling pointer/hover work. A monotonic local clock keeps the ruler+ smooth; media is still sought at cuts and explicit user jumps. */+ const next = positionRef.current + elapsed;// Past the end of everything there is nothing left to show, so playback// stops there instead of running the clock over an empty ruler.if (next >= contentEndMs) {diff --git a/src/components/workspace/TimelineClipThumbnails.test.tsx b/src/components/workspace/TimelineClipThumbnails.test.tsxnew file mode 100644index 0000000..66c4c3a--- /dev/null+++ b/src/components/workspace/TimelineClipThumbnails.test.tsx@@ -0,0 +1,57 @@+// @vitest-environment jsdom++import "@testing-library/jest-dom/vitest";+import { cleanup, render, waitFor } from "@testing-library/react";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { createDraftGenerationJob, type TimelineClip } from "../../lib/project";+import { readProjectFileUrl } from "../../lib/persistence";+import { TimelineClipThumbnails } from "./TimelineClipThumbnails";++vi.mock("../../lib/persistence", () => ({+ readProjectFileUrl: vi.fn(async (_folder: string, relativePath: string) => `blob:${relativePath}`),+}));+vi.mock("./ShotThumbnail", () => ({ shotPoster: vi.fn() }));++beforeEach(() => {+ Object.defineProperty(URL, "revokeObjectURL", { configurable: true, value: vi.fn() });+});+afterEach(cleanup);++describe("timeline clip thumbnails", () => {+ it("places cached source frames at their correct offsets in a trimmed clip", async () => {+ const job = {+ ...createDraftGenerationJob("A scene", { id: "scene-one", durationSeconds: 8 }),+ status: "completed" as const,+ stage: "completed" as const,+ progress: 1,+ outputRelativePath: "media/generated/scene-one.mp4",+ };+ const clip: TimelineClip = {+ id: "clip-one",+ assetId: "asset-scene-one",+ trackId: "track-video-1",+ startMs: 10_000,+ durationMs: 5_000,+ sourceStartMs: 1_000,+ label: "A scene",+ color: null,+ status: "generated",+ };+ const view = render(<TimelineClipThumbnails folderPath="C:\Project" job={job} clip={clip} />);++ await waitFor(() => expect(view.container.querySelectorAll("img")).toHaveLength(3));+ expect(vi.mocked(readProjectFileUrl).mock.calls.map((call) => call[1])).toEqual([+ "thumbnails/timeline/scene-one/00000000.jpg",+ "thumbnails/timeline/scene-one/00002000.jpg",+ "thumbnails/timeline/scene-one/00004000.jpg",+ ]);+ expect([...view.container.querySelectorAll("img")].map((image) => ({+ left: image.style.left,+ width: image.style.width,+ }))).toEqual([+ { left: "-20%", width: "40%" },+ { left: "20%", width: "40%" },+ { left: "60%", width: "40%" },+ ]);+ });+});diff --git a/src/components/workspace/TimelineClipThumbnails.tsx b/src/components/workspace/TimelineClipThumbnails.tsxnew file mode 100644index 0000000..998500f--- /dev/null+++ b/src/components/workspace/TimelineClipThumbnails.tsx@@ -0,0 +1,80 @@+import { useEffect, useMemo, useState } from "react";+import { readProjectFileUrl } from "../../lib/persistence";+import { type GenerationJob, type TimelineClip } from "../../lib/project";+import {+ TIMELINE_THUMBNAIL_INTERVAL_MS,+ timelineThumbnailRelativePath,+ writeTimelineThumbnailDataUrl,+} from "../../lib/timelineThumbnails";+import { shotPoster } from "./ShotThumbnail";++interface TimelineStill {+ timeMs: number;+ url: string;+}++/** Cached scene frames laid over the span of source footage a clip uses. The+ * strip remains source-time accurate when a clip is trimmed, split or moved. */+export function TimelineClipThumbnails({ folderPath, job, clip }: {+ folderPath: string;+ job: GenerationJob;+ clip: TimelineClip;+}) {+ const sourceEndMs = clip.sourceStartMs + clip.durationMs;+ const times = useMemo(() => {+ const first = Math.floor(clip.sourceStartMs / TIMELINE_THUMBNAIL_INTERVAL_MS) * TIMELINE_THUMBNAIL_INTERVAL_MS;+ const result: number[] = [];+ for (let timeMs = Math.max(0, first); timeMs < sourceEndMs; timeMs += TIMELINE_THUMBNAIL_INTERVAL_MS) {+ result.push(timeMs);+ }+ return result;+ }, [clip.sourceStartMs, sourceEndMs]);+ const [stills, setStills] = useState<TimelineStill[]>([]);++ useEffect(() => {+ if (job.status !== "completed" || !job.outputRelativePath || times.length === 0) {+ setStills([]);+ return;+ }+ const outputRelativePath = job.outputRelativePath;+ let live = true;+ const objectUrls: string[] = [];+ void Promise.all(times.map(async (timeMs): Promise<TimelineStill | null> => {+ try {+ const cached = await readProjectFileUrl(folderPath, timelineThumbnailRelativePath(job.id, timeMs), "image/jpeg");+ if (cached) {+ objectUrls.push(cached);+ return { timeMs, url: cached };+ }+ } catch {+ // Older projects have no disk cache. Decode the same source frame once,+ // show it now, and quietly seed the cache for the next visit.+ }+ const generated = await shotPoster(folderPath, outputRelativePath, timeMs / 1_000);+ if (!generated.poster) return null;+ void writeTimelineThumbnailDataUrl(folderPath, job.id, timeMs, generated.poster).catch(() => undefined);+ return { timeMs, url: generated.poster };+ })).then((loaded) => {+ if (live) setStills(loaded.filter((still): still is TimelineStill => still !== null));+ else objectUrls.forEach((url) => URL.revokeObjectURL(url));+ });+ return () => {+ live = false;+ objectUrls.forEach((url) => URL.revokeObjectURL(url));+ };+ }, [folderPath, job.id, job.outputRelativePath, job.status, times]);++ if (stills.length === 0) return null;+ return <span className="timeline-clip__thumbnails" aria-hidden="true">+ {stills.map((still) => <img+ key={still.timeMs}+ src={still.url}+ alt=""+ draggable={false}+ style={{+ left: `${(still.timeMs - clip.sourceStartMs) / clip.durationMs * 100}%`,+ width: `${TIMELINE_THUMBNAIL_INTERVAL_MS / clip.durationMs * 100}%`,+ }}+ />)}+ </span>;+}diff --git a/src/components/workspace/TimelineClipWaveform.test.tsx b/src/components/workspace/TimelineClipWaveform.test.tsxnew file mode 100644index 0000000..1506c0e--- /dev/null+++ b/src/components/workspace/TimelineClipWaveform.test.tsx@@ -0,0 +1,32 @@+// @vitest-environment jsdom++import { cleanup, render, waitFor } from "@testing-library/react";+import { afterEach, expect, it, vi } from "vitest";+import { audioWaveformSegment, loadAudioWaveform } from "../../lib/audioWaveform";+import { type ProjectAsset, type TimelineClip } from "../../lib/project";+import { TimelineClipWaveform } from "./TimelineClipWaveform";++vi.mock("../../lib/audioWaveform", () => ({+ audioWaveformSegment: vi.fn(),+ loadAudioWaveform: vi.fn(),+}));++afterEach(() => {+ cleanup();+ vi.clearAllMocks();+});++it("draws each waveform sample as a vertical stroke without an envelope outline", async () => {+ vi.mocked(loadAudioWaveform).mockResolvedValue({ durationSeconds: 1, levels: new Float32Array([1]) });+ vi.mocked(audioWaveformSegment).mockReturnValue([1, 0.5, 0]);+ const { container } = render(<TimelineClipWaveform+ folderPath="C:\\Project"+ asset={{ id: "asset-audio", name: "Voice", kind: "audio" } as ProjectAsset}+ clip={{ id: "clip-audio", assetId: "asset-audio", trackId: "track-a1", startMs: 0, durationMs: 1_000, sourceStartMs: 0 } as TimelineClip}+ />);++ await waitFor(() => expect(container.querySelector(".clip-audio-visualization__samples")).not.toBeNull());+ const path = container.querySelector(".clip-audio-visualization__samples");+ expect(container.querySelector("polyline")).toBeNull();+ expect(path?.getAttribute("d")).toBe("M0.00,7.00V93.00 M500.00,28.50V71.50 M1000.00,50.00V50.00");+});diff --git a/src/components/workspace/TimelineClipWaveform.tsx b/src/components/workspace/TimelineClipWaveform.tsxnew file mode 100644index 0000000..fe1a541--- /dev/null+++ b/src/components/workspace/TimelineClipWaveform.tsx@@ -0,0 +1,40 @@+import { useEffect, useMemo, useState } from "react";+import { audioWaveformSegment, loadAudioWaveform } from "../../lib/audioWaveform";+import { type ProjectAsset, type TimelineClip } from "../../lib/project";++const DISPLAY_POINTS = 1_024;++const sampleLines = (levels: number[]) => levels.map((level, index) => {+ const x = index / Math.max(1, levels.length - 1) * 1000;+ const amplitude = level * 43;+ return `M${x.toFixed(2)},${(50 - amplitude).toFixed(2)}V${(50 + amplitude).toFixed(2)}`;+}).join(" ");++/** A source-time-accurate series of amplitude samples. Each sample is its own+ * vertical stroke around the centre, avoiding a continuous envelope outline. */+export function TimelineClipWaveform({ folderPath, asset, clip, cacheVersion = "" }: {+ folderPath: string;+ asset: ProjectAsset;+ clip: TimelineClip;+ cacheVersion?: string;+}) {+ const [levels, setLevels] = useState<number[] | null>(null);+ useEffect(() => {+ let live = true;+ setLevels(null);+ void loadAudioWaveform(folderPath, asset, cacheVersion).then((overview) => {+ if (live) setLevels(audioWaveformSegment(overview, clip.sourceStartMs / 1_000, clip.durationMs / 1_000, DISPLAY_POINTS));+ }).catch(() => {+ if (live) setLevels([]);+ });+ return () => { live = false; };+ }, [asset, cacheVersion, clip.durationMs, clip.sourceStartMs, folderPath]);++ const lines = useMemo(() => levels && levels.length > 0 ? sampleLines(levels) : null, [levels]);++ return <span className={`clip-audio-visualization ${levels === null ? "loading" : ""}`} aria-hidden="true">+ <svg viewBox="0 0 1000 100" preserveAspectRatio="none" focusable="false">+ {lines && <path className="clip-audio-visualization__samples" d={lines} />}+ </svg>+ </span>;+}diff --git a/src/components/workspace/TimelineView.tsx b/src/components/workspace/TimelineView.tsxindex 1ccf47f..472c9cf 100644--- a/src/components/workspace/TimelineView.tsx+++ b/src/components/workspace/TimelineView.tsx@@ -1,6 +1,6 @@import {- Copy, Film, Layers3, LayoutGrid, List, Lock, LockOpen, Music2,- Pause, Play, Plus, Scissors, SkipBack, SkipForward, Trash2, Upload, Video,+ Copy, Film, Layers3, LayoutGrid, List, Lock, LockOpen,+ Pause, Play, Plus, Scissors, SkipBack, SkipForward, Trash2, Upload,Volume2, VolumeX, X,} from "lucide-react";import { useCallback, useEffect, useMemo, useRef, useState } from "react";@@ -29,8 +29,14 @@ import { ProgramMonitor } from "./ProgramMonitor";import { sceneShape, STATUS_WORD } from "./sceneStatus";import { ShotThumbnail } from "./ShotThumbnail";import { CommittedNumberInput } from "./CommittedNumberInput";+import { TimelineClipThumbnails } from "./TimelineClipThumbnails";+import { TimelineClipWaveform } from "./TimelineClipWaveform";const MIN_DURATION = 10_000;+/* One minute fills the visible ruler. Longer films keep that readable scale+ and extend sideways inside a scrollable canvas instead of squeezing every+ cut into the same width. */+const VISIBLE_TIMELINE_MS = 60_000;/* How long a dropped clip is when the file itself cannot say.A drop normally lasts as long as the footage: the media panel decodes eachfile to draw its thumbnail and records the duration it read there (see@@ -88,6 +94,7 @@ interface DragSession {function trackIdAtPoint(x: number, y: number): string | undefined {if (typeof document.elementsFromPoint !== "function") return undefined;for (const element of document.elementsFromPoint(x, y)) {+ if (!element) continue;const lane = (element as HTMLElement).closest?.("[data-track-id]") as HTMLElement | null;if (lane?.dataset.trackId) return lane.dataset.trackId;}@@ -189,17 +196,23 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {const tracks = config.timeline.tracks;const selected = useMemo(() => tracks.flatMap((track) => track.clips).find((clip) => clip.id === selectedId), [tracks, selectedId]);const selectedTrack = tracks.find((track) => track.id === selected?.trackId);+ const selectedMediaAsset = config.assets.find((asset) => asset.id === selected?.assetId);const selectedTransform = selected ? clipTransform(selected) : null;const selectedLook = selected ? clipLook(selected) : null;const selectedTransition = selected ? clipTransition(selected) : null;- const visualControlsDisabled = selectedTrack?.kind !== "video" || selectedTrack.locked;+ const visualControlsDisabled = selectedMediaAsset?.kind === "audio" || selectedTrack?.kind !== "video" || selectedTrack.locked;const clipCount = tracks.reduce((total, track) => total + track.clips.length, 0);/** Where the last clip ends — where playback stops, which is not the same as* where the ruler stops (the canvas is at least as long as the film the user* asked for, however little of it is cut yet). */const contentEndMs = tracks.reduce((end, track) => track.clips.reduce((furthest, clip) => Math.max(furthest, clipEndMs(clip)), end), 0);const duration = useMemo(() => canvasDuration(config), [config]);+ const timelineWidth = `${Math.max(100, duration / VISIBLE_TIMELINE_MS * 100)}%`;const mediaAssets = useMemo(() => config.assets.filter((asset) => asset.kind !== "generated"), [config.assets]);+ const assetsById = useMemo(() => new Map(config.assets.map((asset) => [asset.id, asset])), [config.assets]);+ const generatedJobsByAssetId = useMemo(() => new Map(+ config.generationJobs.map((job) => [generationAssetId(job.id), job]),+ ), [config.generationJobs]);const ticks = useMemo(() => rulerTicks(duration), [duration]);/* The lanes are ruled at the SAME interval as the ruler above them, so a lineunder a clip is a line under a number. They used to be a fixed 14.7% of the@@ -209,9 +222,8 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {useEffect(() => { if (clipCount === 0 && playing) setPlaying(false); }, [clipCount, playing]);- /* The playhead is driven by the program monitor while playing — it reads- where the decoder actually is, so the picture and the playhead cannot- drift apart. These two are handed down rather than declared inline+ /* The playhead is driven by the program monitor's monotonic animation clock+ while playing. These two are handed down rather than declared inlinebecause the monitor's animation loop depends on their identity: a newclosure per render would tear its clock down sixty times a second. */const seek = useCallback((ms: number) => setPlayhead(ms), []);@@ -233,6 +245,9 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {const grid = timelineGrid.current;if (!grid) return;const onWheel = (event: WheelEvent) => {+ // Preserve a trackpad's horizontal gesture for the long timeline's+ // native scroll container. Vertical wheel movement remains scrubbing.+ if (duration > VISIBLE_TIMELINE_MS && Math.abs(event.deltaX) > Math.abs(event.deltaY)) return;const raw = Math.abs(event.deltaY) >= Math.abs(event.deltaX) ? event.deltaY : event.deltaX;if (raw === 0) return;event.preventDefault();@@ -364,6 +379,7 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {durationMs: null,width: null,height: null,+ hasAudio: file.kind === "audio" ? true : null,createdAt: now,}));onChange({ ...config, assets: [...config.assets, ...assets] });@@ -375,11 +391,11 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {}};- /* A video or image belongs on a video track and a sound on an audio track;- dropping either on the other would make a clip nothing can ever play. */+ /* Every lane is audiovisual, so video, stills, and sound can be arranged on+ any unlocked track. */const acceptsAsset = (track: ProjectConfig["timeline"]["tracks"][number], asset: ProjectAsset | undefined) => {if (!asset || track.locked) return false;- return track.kind === "audio" ? asset.kind === "audio" : asset.kind !== "audio";+ return asset.kind !== "caption";};const acceptsScene = (track: ProjectConfig["timeline"]["tracks"][number] | undefined, scene: GenerationJob | undefined) =>@@ -399,6 +415,7 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {durationMs: asset.durationMs ?? value.durationMs,width: asset.width ?? value.width,height: asset.height ?? value.height,+ hasAudio: asset.hasAudio ?? value.hasAudio,});/* Measurements arrive from asynchronous decodes, and two of them can landbetween one render and the next. They queue here so that two arriving in@@ -433,7 +450,7 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {const value = pending.get(asset.id);if (!value) return asset;const next = applyMeasured(asset, value);- if (next.durationMs === asset.durationMs && next.width === asset.width && next.height === asset.height) return asset;+ if (next.durationMs === asset.durationMs && next.width === asset.width && next.height === asset.height && next.hasAudio === asset.hasAudio) return asset;learned = true;return next;});@@ -728,7 +745,7 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {onClick={() => void importMedia()}disabled={importing || !isTauri()}title={isTauri() ? "Add video, sound, or image files — video and sound stay where they are, images are copied in" : "Importing files is available in the desktop app"}- ><Upload size={16} /> {importing ? "Importing…" : "Import"}</button>+ ><Upload size={16} /> Import</button><div className="layout-toggle" role="group" aria-label="Media layout"><buttonclassName={mediaLayout === "grid" ? "active" : ""}@@ -764,7 +781,7 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {setDraggedAsset(undefined);}}onDragEnd={() => { setDraggedScene(undefined); setDropTrackId(null); }}- title={`${job.title} — drag onto a video track below`}+ title={`${job.title} — drag onto a track below`}><span className="scene-card__thumb">{job.status !== "draft" && <ShotThumbnail folderPath={folderPath} job={job} seconds={0} shotNumber={1} estimatedCompletionAt={generationCompletionTimes[job.id] ?? null} />}@@ -923,11 +940,12 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {<button onClick={removeSelected} disabled={deleteBlockedBy !== null} title={deleteBlockedBy ?? "Delete selected clip"}><Trash2 size={16} /> Delete</button></div></header>- <div- className="timeline-grid"- ref={timelineGrid}- style={{ "--lane-grid": `${(tickStepMs / duration) * 100}%` } as React.CSSProperties}- title="Scroll to scrub, or drag along the ruler. Hold Shift to scrub one frame at a time. Clips snap to the cuts around them; Escape abandons a drag.">+ <div className={`timeline-grid-scroll${duration > VISIBLE_TIMELINE_MS ? " timeline-grid-scroll--wide" : ""}`}>+ <div+ className="timeline-grid"+ ref={timelineGrid}+ style={{ "--lane-grid": `${(tickStepMs / duration) * 100}%`, width: timelineWidth } as React.CSSProperties}+ title="Scroll to scrub, or drag along the ruler. Hold Shift to scrub one frame at a time. Clips snap to the cuts around them; Escape abandons a drag."><div className="track-corner"><span>Tracks</span></div>{/* Press to jump, hold and drag to scrub. The monitor seeks to whereverthis lands, so dragging along the ruler runs the picture past under@@ -952,15 +970,17 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {key={track.id}track={track}duration={duration}+ folderPath={folderPath}+ generatedJobsByAssetId={generatedJobsByAssetId}+ assetsById={assetsById}selectedId={selectedId}draggingId={dragging ? dragRef.current?.clipId : undefined}onClipPointerDown={beginDrag}onDeleteClip={deleteClip}dropActive={dropTrackId === track.id}/* A lane that stays dark is indistinguishable from a lane the- pointer simply is not over, so a sound held above a video track- looked like it had not been picked up rather than like it was- being refused. Marked for the whole drag, not just on hover. */+ pointer simply is not over. Mark incompatible or locked lanes+ for the whole drag, not just on hover. */dropBlocked={(draggedAsset !== undefined && !acceptsAsset(track, draggedAsset))|| (draggedScene !== undefined && !acceptsScene(track, draggedScene))}onSelect={setSelectedId}@@ -992,16 +1012,20 @@ export function TimelineView({ config, folderPath, generationCompletionTimes = {dropAsset(track.id, assetId, ratio, rect.width);}}/>)}- <div className="timeline-playhead" style={{ left: `calc(var(--track-column) + (100% - var(--track-column)) * ${playhead / duration})` }}><span /><i /></div>+ <div className="timeline-playhead" style={{ left: `calc(var(--track-column) + (100% - var(--track-column)) * ${playhead / duration})` }}><span /><i /></div>+ </div></div></section></div>);}-function TrackRow({ track, duration, selectedId, draggingId, dropActive, dropBlocked, onSelect, onToggle, onRename, onClipPointerDown, onDeleteClip, onDragOverLane, onDragLeaveLane, onDropLane }: {+function TrackRow({ track, duration, folderPath, generatedJobsByAssetId, assetsById, selectedId, draggingId, dropActive, dropBlocked, onSelect, onToggle, onRename, onClipPointerDown, onDeleteClip, onDragOverLane, onDragLeaveLane, onDropLane }: {track: ProjectConfig["timeline"]["tracks"][number];duration: number;+ folderPath: string;+ generatedJobsByAssetId: ReadonlyMap<string, GenerationJob>;+ assetsById: ReadonlyMap<string, ProjectAsset>;selectedId: string;/** The clip currently being dragged, so it can be drawn as the thing under* the pointer rather than as one more clip sitting on the lane. */@@ -1017,14 +1041,10 @@ function TrackRow({ track, duration, selectedId, draggingId, dropActive, dropBloonDragLeaveLane: () => void;onDropLane: (event: React.DragEvent<HTMLDivElement>) => void;}) {- /* The name no longer carries its kind ("Track 1, Video" is now "Track 1"),- which the icon and the line under the field already say — but that leaves a- video and an audio track both called "Track 1", and a button announced as- "Mute Track 1" twice names neither. The controls say which one in full. */- const named = `${track.kind === "audio" ? "audio" : "video"} ${track.name}`;+ const named = track.name;return <><div className="track-head">- <span className={`track-kind track-kind--${track.kind}`}>{track.kind === "audio" ? <Music2 size={16} /> : <Video size={16} />}</span>+ <span className="track-kind"><Film size={16} /></span>{/* The name is an editable field, not a label: a project with three videolayers needs the user's own words on them, and an always-live inputneeds no discovery. Blanking it falls back the way a clip name does,@@ -1035,8 +1055,8 @@ function TrackRow({ track, duration, selectedId, draggingId, dropActive, dropBloaria-label={`Rename ${named}`}title="Rename this track"onChange={(event) => onRename(track.id, event.target.value || "Untitled track")}- /><small>{track.kind === "audio" ? "Audio" : "Video"}</small></div>- <button className={track.muted ? "active" : ""} onClick={() => onToggle(track.id, "muted")} aria-label={`${track.muted ? "Unmute" : "Mute"} ${named}`} title={`${track.muted ? "Unmute" : "Mute"} ${named}`}>{track.muted ? <VolumeX size={16} /> : <Volume2 size={16} />}</button>+ /></div>+ <button className={track.muted ? "active" : ""} onClick={() => onToggle(track.id, "muted")} aria-label={`${track.muted ? "Unmute" : "Mute"} audio on ${named}`} title={`${track.muted ? "Unmute" : "Mute"} audio on ${named}`}>{track.muted ? <VolumeX size={16} /> : <Volume2 size={16} />}</button><button className={track.locked ? "active" : ""} onClick={() => onToggle(track.id, "locked")} aria-label={`${track.locked ? "Unlock" : "Lock"} ${named}`} title={`${track.locked ? "Unlock" : "Lock"} ${named}`}>{track.locked ? <Lock size={16} /> : <LockOpen size={16} />}</button></div><div@@ -1045,12 +1065,25 @@ function TrackRow({ track, duration, selectedId, draggingId, dropActive, dropBlolanes is hit-tested against the document, and the id is how the answergets back to the model. */data-track-id={track.id}- title={dropBlocked ? (track.locked ? `${track.name} is locked` : `${track.name} only takes ${track.kind === "audio" ? "sound" : "video and pictures"}`) : undefined}+ title={dropBlocked ? (track.locked ? `${track.name} is locked` : `${track.name} cannot take this item`) : undefined}onDragOver={onDragOverLane}onDragLeave={onDragLeaveLane}onDrop={onDropLane}>- {track.clips.map((clip) => <div+ {track.clips.map((clip) => {+ const asset = assetsById.get(clip.assetId);+ const audioOnly = asset?.kind === "audio";+ const generation = generatedJobsByAssetId.get(clip.assetId);+ /* Current renders record this exactly. Completed files from projects+ saved before that metadata existed use the successful saved output+ as the compatibility signal; an explicit false always wins. */+ const hasAudio = audioOnly || asset?.hasAudio === true || Boolean(+ asset?.kind === "generated"+ && asset.hasAudio == null+ && generation?.outputRelativePath+ && !generation.error?.startsWith("Saved without sound"),+ );+ return <divkey={clip.id}className={`clip-slot ${selectedId === clip.id ? "selected" : ""} ${draggingId === clip.id ? "clip-slot--dragging" : ""}`}style={{ left: `${clip.startMs / duration * 100}%`, width: `${clip.durationMs / duration * 100}%`, "--clip-color": clip.color } as React.CSSProperties}@@ -1061,7 +1094,7 @@ function TrackRow({ track, duration, selectedId, draggingId, dropActive, dropBlocan make sense of. Everything a button gave it is still here —focus, Enter, the pressed state, a name. */}<div- className={`timeline-clip timeline-clip--${track.kind} ${selectedId === clip.id ? "selected" : ""} ${track.locked ? "timeline-clip--locked" : ""}`}+ className={`timeline-clip ${audioOnly ? "timeline-clip--audio-only" : ""} ${hasAudio ? "timeline-clip--has-audio" : ""} ${selectedId === clip.id ? "selected" : ""} ${track.locked ? "timeline-clip--locked" : ""}`}role="button"tabIndex={0}aria-pressed={selectedId === clip.id}@@ -1072,7 +1105,19 @@ function TrackRow({ track, duration, selectedId, draggingId, dropActive, dropBloonClick={() => onSelect(clip.id)}onKeyDown={(event) => { if (event.key === "Enter") onSelect(clip.id); }}onPointerDown={(event) => onClipPointerDown(event, clip, "move")}- ><span className="clip-text"><b>{clip.label}</b><small>{track.kind === "audio" ? "▂▅▃▆▂▃▇▅▂▆▃▅▂" : `${(clip.durationMs / 1000).toFixed(1)}s · ${clip.status}`}</small></span></div>+ >+ {!audioOnly && generation && <TimelineClipThumbnails+ folderPath={folderPath}+ job={generation}+ clip={clip}+ />}+ {hasAudio && asset && <TimelineClipWaveform+ folderPath={folderPath}+ asset={asset}+ clip={clip}+ cacheVersion={generation?.updatedAt}+ />}+ </div>{/* The two ends, which cut rather than move. They stop the press fromreaching the clip body, or every trim would also be a move. A lockedtrack gets none of them: nothing on it can be changed. */}@@ -1095,7 +1140,8 @@ function TrackRow({ track, duration, selectedId, draggingId, dropActive, dropBloonClick={(event) => { event.stopPropagation(); onDeleteClip(clip.id); }}><X size={12} /></button></>}- </div>)}+ </div>;+ })}</div></>;}diff --git a/src/lib/audioWaveform.test.ts b/src/lib/audioWaveform.test.tsnew file mode 100644index 0000000..fcc990a--- /dev/null+++ b/src/lib/audioWaveform.test.ts@@ -0,0 +1,30 @@+import { describe, expect, it } from "vitest";+import { audioWaveformSegment, createAudioWaveformOverview } from "./audioWaveform";++describe("audio waveform", () => {+ it("keeps quiet passages visible on a logarithmic envelope", () => {+ const samples = new Float32Array(400);+ samples.fill(1, 0, 100);+ samples.fill(0.01, 100, 200);+ samples.fill(0, 200, 300);+ samples.fill(0.1, 300, 400);+ const overview = createAudioWaveformOverview({+ duration: 4,+ length: samples.length,+ numberOfChannels: 1,+ getChannelData: () => samples,+ }, 4);++ const levels = audioWaveformSegment(overview, 0, 4, 4);+ expect(levels[0]).toBeCloseTo(1);+ expect(levels[1]).toBeGreaterThan(0.3);+ expect(levels[1]).toBeLessThan(levels[3]);+ expect(levels[2]).toBe(0);+ expect(levels[3]).toBeGreaterThan(0.6);+ });++ it("selects only the source range represented by a trimmed clip", () => {+ const overview = { durationSeconds: 4, levels: new Float32Array([1, 0, 0.5, 0]) };+ expect(audioWaveformSegment(overview, 2, 1, 2)).toEqual([1, 1]);+ });+});diff --git a/src/lib/audioWaveform.ts b/src/lib/audioWaveform.tsnew file mode 100644index 0000000..c149965--- /dev/null+++ b/src/lib/audioWaveform.ts@@ -0,0 +1,103 @@+import { readMediaFileBytes } from "./persistence";+import { type ProjectAsset } from "./project";++export interface AudioWaveformOverview {+ durationSeconds: number;+ levels: Float32Array;+}++type WaveformAudioBuffer = Pick<AudioBuffer, "duration" | "length" | "numberOfChannels" | "getChannelData">;++const MIN_OVERVIEW_POINTS = 1_024;+const MAX_OVERVIEW_POINTS = 16_384;+const OVERVIEW_POINTS_PER_SECOND = 64;+const MAX_READS_PER_BIN = 128;+const DISPLAY_FLOOR_DB = -60;+const CACHE_LIMIT = 24;+const OVERVIEWS = new Map<string, Promise<AudioWaveformOverview>>();++/** Reduces decoded PCM to a reusable overview. RMS keeps sustained quiet+ * passages legible while the peak term retains short transients. */+export function createAudioWaveformOverview(+ buffer: WaveformAudioBuffer,+ requestedPoints?: number,+): AudioWaveformOverview {+ const desiredPoints = requestedPoints ?? Math.max(MIN_OVERVIEW_POINTS, Math.ceil(buffer.duration * OVERVIEW_POINTS_PER_SECOND));+ const pointCount = Math.max(1, Math.min(buffer.length, Math.min(MAX_OVERVIEW_POINTS, desiredPoints)));+ const levels = new Float32Array(pointCount);+ const channels = Array.from({ length: buffer.numberOfChannels }, (_, channel) => buffer.getChannelData(channel));++ for (let point = 0; point < pointCount; point += 1) {+ const start = Math.floor(point * buffer.length / pointCount);+ const end = Math.max(start + 1, Math.floor((point + 1) * buffer.length / pointCount));+ const stride = Math.max(1, Math.ceil((end - start) / MAX_READS_PER_BIN));+ let squares = 0;+ let peak = 0;+ let reads = 0;+ for (const channel of channels) {+ for (let sample = start; sample < end; sample += stride) {+ const amplitude = Math.abs(channel[sample] ?? 0);+ squares += amplitude * amplitude;+ peak = Math.max(peak, amplitude);+ reads += 1;+ }+ }+ const rms = reads > 0 ? Math.sqrt(squares / reads) : 0;+ levels[point] = peak > 0 ? Math.sqrt(rms * peak) : 0;+ }+ return { durationSeconds: buffer.duration, levels };+}++/** Selects the part used by a timeline clip and maps it on a logarithmic+ * -60 dB..0 dB scale. This prevents quiet speech or room tone from+ * disappearing against the centre line. */+export function audioWaveformSegment(+ overview: AudioWaveformOverview,+ sourceStartSeconds: number,+ durationSeconds: number,+ pointCount = 512,+): number[] {+ const result = Array.from({ length: pointCount }, () => 0);+ if (overview.durationSeconds <= 0 || overview.levels.length === 0 || durationSeconds <= 0) return result;+ const first = Math.max(0, sourceStartSeconds / overview.durationSeconds * overview.levels.length);+ const last = Math.min(overview.levels.length, (sourceStartSeconds + durationSeconds) / overview.durationSeconds * overview.levels.length);+ for (let point = 0; point < pointCount; point += 1) {+ const start = Math.floor(first + (last - first) * point / pointCount);+ const end = Math.max(start + 1, Math.ceil(first + (last - first) * (point + 1) / pointCount));+ for (let index = start; index < Math.min(end, overview.levels.length); index += 1) {+ result[point] = Math.max(result[point], overview.levels[index]);+ }+ }+ const peak = Math.max(...result);+ if (peak <= 0) return result;+ return result.map((level) => {+ if (level <= 0) return 0;+ const decibels = 20 * Math.log10(level / peak);+ return Math.max(0, Math.min(1, (decibels - DISPLAY_FLOOR_DB) / -DISPLAY_FLOOR_DB));+ });+}++/** Reads and decodes each source once. A generation timestamp belongs in the+ * cache key so regenerating the same scene path cannot reuse its old sound. */+export function loadAudioWaveform(+ folderPath: string,+ asset: ProjectAsset,+ cacheVersion = "",+): Promise<AudioWaveformOverview> {+ const location = asset.sourcePath ?? asset.relativePath;+ if (!location) return Promise.reject(new Error(`${asset.name} has no readable media path.`));+ const key = `${folderPath}::${location}::${cacheVersion}`;+ const cached = OVERVIEWS.get(key);+ if (cached) return cached;+ const pending = readMediaFileBytes(folderPath, asset).then(async (bytes) => {+ const context = new OfflineAudioContext(1, 1, 44_100);+ const decoded = await context.decodeAudioData(bytes.slice(0));+ return createAudioWaveformOverview(decoded);+ }).catch((error) => {+ OVERVIEWS.delete(key);+ throw error;+ });+ OVERVIEWS.set(key, pending);+ while (OVERVIEWS.size > CACHE_LIMIT) OVERVIEWS.delete(OVERVIEWS.keys().next().value!);+ return pending;+}diff --git a/src/lib/export.test.ts b/src/lib/export.test.tsindex c8f1386..58c2511 100644--- a/src/lib/export.test.ts+++ b/src/lib/export.test.ts@@ -47,7 +47,7 @@ const clip = (id: string, trackId: string, startMs: number, durationMs: number,...overrides,});-/** A project with the app's own four tracks, `clips` distributed by trackId.+/** A project with the app's three audiovisual tracks, `clips` distributed by trackId.* `track-story` is the video "Track 1" — the FIRST video track, and the one* that composites over the video "Track 2" (`track-v2`), because the timeline draws* tracks in array order from the top. */@@ -287,7 +287,7 @@ describe("the plan", () => {it("mixes the audio that lands inside the picture, and cuts it where the picture ends", () => {const config = project(- [clip("a", "track-story", 0, 1_000), clip("score", "track-a2", 0, 5_000)],+ [clip("a", "track-story", 0, 1_000), clip("score", "track-v3", 0, 5_000)],[asset("asset-a"), asset("asset-score", { kind: "audio", mimeType: "audio/wav" })],);const plan = buildExportPlan(config, settings());@@ -306,7 +306,7 @@ describe("the plan", () => {const config = project([clip("a", "track-story", 0, 8_000),- clip("vo", "track-a1", 2_000, 3_000, { sourceStartMs: 4_500 }),+ clip("vo", "track-v2", 2_000, 3_000, { sourceStartMs: 4_500 }),],[asset("asset-a"), asset("asset-vo", { kind: "audio", mimeType: "audio/wav" })],);@@ -317,7 +317,7 @@ describe("the plan", () => {it("leaves out audio that starts after the last frame, and says how much", () => {const config = project(- [clip("a", "track-story", 0, 1_000), clip("late", "track-a1", 4_000, 2_000)],+ [clip("a", "track-story", 0, 1_000), clip("late", "track-v2", 4_000, 2_000)],[asset("asset-a"), asset("asset-late", { kind: "audio", mimeType: "audio/wav" })],);const plan = buildExportPlan(config, settings());@@ -327,13 +327,13 @@ describe("the plan", () => {it("does not mix a muted track, and says that too", () => {const config = project(- [clip("a", "track-story", 0, 4_000), clip("score", "track-a2", 0, 4_000)],+ [clip("a", "track-story", 0, 4_000), clip("score", "track-v3", 0, 4_000)],[asset("asset-a"), asset("asset-score", { kind: "audio", mimeType: "audio/wav" })],);const muted = {...config,timeline: {- tracks: config.timeline.tracks.map((track) => (track.id === "track-a2" ? { ...track, muted: true } : track)),+ tracks: config.timeline.tracks.map((track) => (track.id === "track-v3" ? { ...track, muted: true } : track)),},};const plan = buildExportPlan(muted, settings());@@ -345,8 +345,8 @@ describe("the plan", () => {const config = project([clip("a", "track-story", 0, 6_000),- clip("vo", "track-a1", 0, 4_000),- clip("score", "track-a2", 2_000, 4_000),+ clip("vo", "track-v2", 0, 4_000),+ clip("score", "track-v3", 2_000, 4_000),],[asset("asset-a"),@@ -359,6 +359,20 @@ describe("the plan", () => {expect(plan.notes.join(" ")).toContain("overlaps are summed");});+ it("mixes embedded video audio and removes it when that audiovisual track is muted", () => {+ const config = project(+ [clip("a", "track-story", 0, 2_000), clip("under", "track-v2", 0, 2_000)],+ [asset("asset-a", { hasAudio: true }), asset("asset-under", { hasAudio: true })],+ );+ expect(buildExportPlan(config, settings()).audio.map((segment) => segment.clipId)).toEqual(["a", "under"]);++ const muted = {+ ...config,+ timeline: { tracks: config.timeline.tracks.map((track) => track.id === "track-v2" ? { ...track, muted: true } : track) },+ };+ expect(buildExportPlan(muted, settings()).audio.map((segment) => segment.clipId)).toEqual(["a"]);+ });+it("blocks on an empty timeline, on missing media and on a container it cannot demux", () => {expect(buildExportPlan(project([], []), settings()).blockers[0]).toContain("no video clips");diff --git a/src/lib/export.ts b/src/lib/export.tsindex 05fb4fa..959fd77 100644--- a/src/lib/export.ts+++ b/src/lib/export.ts@@ -19,6 +19,7 @@ import {type ClipLook,type ClipTransform,type ClipTransition,+ type ProjectAsset,type ProjectConfig,type Resolution,type TimelineClip,@@ -291,6 +292,21 @@ export interface AudioSegment {sourceStartMs: number;}+const assetCarriesAudio = (asset: ProjectAsset | undefined): boolean => Boolean(+ asset && (asset.kind === "audio" || ((asset.kind === "video" || asset.kind === "generated") && asset.hasAudio === true)),+);++const assetCarriesPicture = (asset: ProjectAsset | undefined): boolean =>+ asset?.kind !== "audio" && asset?.kind !== "caption";++const audioTimelineClips = (config: ProjectConfig, muted: boolean) => {+ const assets = new Map(config.assets.map((asset) => [asset.id, asset]));+ return config.timeline.tracks+ .filter((track) => track.muted === muted)+ .flatMap((track) => track.clips)+ .filter((clip) => assetCarriesAudio(assets.get(clip.assetId)));+};+/** Every audio clip's audible part, in timeline order.** Overlaps are NOT resolved here: two clips over the same moment are two@@ -298,9 +314,11 @@ export interface AudioSegment {* not a segment — a gap is simply a stretch no segment covers. */export function audioSegments(config: ProjectConfig, durationMs: number): AudioSegment[] {const segments: AudioSegment[] = [];+ const assets = new Map(config.assets.map((asset) => [asset.id, asset]));for (const track of config.timeline.tracks) {- if (track.kind !== "audio" || track.muted) continue;+ if (track.muted) continue;for (const clip of track.clips) {+ if (!assetCarriesAudio(assets.get(clip.assetId))) continue;const start = Math.max(0, clip.startMs);const end = Math.min(durationMs, clip.startMs + clip.durationMs);if (end <= start) continue;@@ -330,7 +348,7 @@ export interface ExportPlan {clipCount: number;/** Frames with nothing over them; they get the project's background colour. */gapFrames: number;- /** Clips on audio tracks, muted tracks included. */+ /** Clips carrying audio streams, muted tracks included. */audioClipCount: number;/** The audible part of each one. Empty means the file has no sound, and the* notes say why. */@@ -365,11 +383,16 @@ export function sourceTimeMsForFrame(* has something at this moment wins — an overlay covers the story beneath it.* Within one track two clips can overlap after a drag; the later one is the* more recent edit, so it is the one on top. */-export function visibleClipAt(tracks: TimelineTrack[], timeMs: number): TimelineClip | null {+export function visibleClipAt(+ tracks: TimelineTrack[],+ timeMs: number,+ assetsById?: ReadonlyMap<string, ProjectAsset>,+): TimelineClip | null {for (const track of tracks) {if (track.kind !== "video") continue;let best: TimelineClip | null = null;for (const clip of track.clips) {+ if (assetsById && !assetCarriesPicture(assetsById.get(clip.assetId))) continue;if (timeMs < clip.startMs || timeMs >= clip.startMs + clip.durationMs) continue;if (!best || clip.startMs >= best.startMs) best = clip;}@@ -382,9 +405,11 @@ export function visibleClipAt(tracks: TimelineTrack[], timeMs: number): Timeline* past the last shot cannot extend the file — claiming otherwise would promise* frames that do not exist. */export function videoDurationMs(config: ProjectConfig): number {+ const assets = new Map(config.assets.map((asset) => [asset.id, asset]));return config.timeline.tracks.filter((track) => track.kind === "video").flatMap((track) => track.clips)+ .filter((clip) => assetCarriesPicture(assets.get(clip.assetId))).reduce((end, clip) => Math.max(end, clip.startMs + clip.durationMs), 0);}@@ -393,6 +418,7 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)const frameRate = settings.frameRate;const tracks = config.timeline.tracks;const videoTracks = tracks.filter((track) => track.kind === "video");+ const assetsById = new Map(config.assets.map((asset) => [asset.id, asset]));const durationMs = videoDurationMs(config);const frameCount = Math.round((durationMs * frameRate) / 1000);@@ -402,7 +428,7 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)const segments: ExportSegment[] = [];let gapFrames = 0;for (let frame = 0; frame < frameCount; frame += 1) {- const clip = visibleClipAt(videoTracks, (frame * 1000) / frameRate);+ const clip = visibleClipAt(videoTracks, (frame * 1000) / frameRate, assetsById);const last = segments[segments.length - 1];if (!clip) {gapFrames += 1;@@ -428,9 +454,7 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)}const usedClipIds = new Set(segments.flatMap((segment) => (segment.kind === "clip" ? [segment.clipId] : [])));- const audioClipCount = tracks- .filter((track) => track.kind === "audio")- .reduce((total, track) => total + track.clips.length, 0);+ const audioClipCount = audioTimelineClips(config, false).length + audioTimelineClips(config, true).length;const blockers: string[] = [];if (frameCount === 0) {@@ -497,9 +521,7 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)/* Audio the file cannot hold, counted rather than dropped in silence. Thepicture decides the length; sound past the last frame has nowhere to go. */const audible = new Set(audio.map((segment) => segment.clipId));- const beyondPicture = tracks- .filter((track) => track.kind === "audio" && !track.muted)- .flatMap((track) => track.clips)+ const beyondPicture = audioTimelineClips(config, false).filter((clip) => !audible.has(clip.id)).length;if (beyondPicture > 0) {notes.push(@@ -512,9 +534,7 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)`The sound stops with the picture at ${formatDuration(durationMs)}, where ${trimmed === 1 ? "one clip is" : `${trimmed} clips are`} cut off.`,);}- const mutedClips = tracks- .filter((track) => track.kind === "audio" && track.muted)- .reduce((total, track) => total + track.clips.length, 0);+ const mutedClips = audioTimelineClips(config, true).length;if (mutedClips > 0) {notes.push(`${mutedClips} audio ${mutedClips === 1 ? "clip is" : "clips are"} on a muted track and will not be heard.`);}@@ -525,6 +545,7 @@ export function buildExportPlan(config: ProjectConfig, settings: ExportSettings)}const unusedClips = videoTracks.flatMap((track) => track.clips)+ .filter((clip) => assetCarriesPicture(assetsById.get(clip.assetId))).filter((clip) => !usedClipIds.has(clip.id));if (unusedClips.length > 0) {notes.push(diff --git a/src/styles/timeline.css b/src/styles/timeline.cssindex eb527e0..747da11 100644--- a/src/styles/timeline.css+++ b/src/styles/timeline.css@@ -987,6 +987,17 @@/* --- Track grid ------------------------------------------------------------ */+.timeline-grid-scroll {+ min-width: 0;+ min-height: 0;+ overflow-x: hidden;+ overflow-y: hidden;+}++.timeline-grid-scroll--wide {+ overflow-x: auto;+}+.timeline-grid {/* Wide enough for a track name to be read in full beside the kind icon andthe two toggles. It was cut to fit the seeded names when those still ended@@ -996,10 +1007,17 @@display: grid;grid-template-columns: var(--track-column) minmax(0, 1fr);grid-template-rows: 36px;- grid-auto-rows: 72px;- height: 100%;- overflow-x: hidden;- overflow-y: auto;+ grid-auto-rows: 96px;+ min-width: 100%;+ min-height: 100%;+}++/* Track controls remain available while the ruler and clips move sideways. */+.timeline-grid-scroll--wide .track-corner,+.timeline-grid-scroll--wide .track-head {+ position: sticky;+ left: 0;+ z-index: 10;}.track-corner {@@ -1061,11 +1079,6 @@background: var(--accent-soft-strong);}-.track-kind--audio {- color: var(--audio);- background: var(--audio-soft);-}-.track-head .track-name,.track-head small {display: block;@@ -1143,8 +1156,8 @@background-size: var(--lane-grid, 14.7%) 100%;}-.track-lane.muted {- opacity: 0.46;+.track-lane.muted .clip-audio-visualization {+ opacity: 0.24;}/* Lit while a compatible media file is held over this lane. A drop target the@@ -1180,7 +1193,7 @@.clip-slot {position: absolute;top: var(--space-2);- height: 56px;+ height: 80px;min-width: 22px;}@@ -1216,11 +1229,34 @@.timeline-clip::before {content: "";position: absolute;+ z-index: 1;inset: 0;background: repeating-linear-gradient(90deg, var(--clip-stripe) 0 18px, transparent 18px 21px);+ pointer-events: none;}-.timeline-clip--audio {+.timeline-clip__thumbnails {+ position: absolute;+ z-index: 0;+ inset: 0;+ overflow: hidden;+ pointer-events: none;+}++.timeline-clip--has-audio:not(.timeline-clip--audio-only) .timeline-clip__thumbnails {+ bottom: 24px;+}++.timeline-clip__thumbnails img {+ position: absolute;+ top: 0;+ height: 100%;+ max-width: none;+ object-fit: cover;+ user-select: none;+}++.timeline-clip--audio-only {background: linear-gradient(180deg,color-mix(in srgb, var(--clip-color, var(--clip-audio-default)) 62%, var(--clip-audio-shade-top)),@@ -1228,6 +1264,46 @@);}+.clip-audio-visualization {+ position: absolute;+ z-index: 2;+ right: 0;+ bottom: 0;+ left: 0;+ height: 24px;+ padding: 3px 4px;+ overflow: hidden;+ border-top: 1px solid color-mix(in srgb, var(--line-strong) 70%, transparent);+ background: color-mix(in srgb, var(--panel) 34%, transparent);+ transition: opacity 120ms ease;+ pointer-events: none;+}++.timeline-clip--audio-only .clip-audio-visualization {+ top: 0;+ height: auto;+ background: transparent;+}++.clip-audio-visualization svg {+ display: block;+ width: 100%;+ height: 100%;+ overflow: visible;+}++.clip-audio-visualization__samples {+ fill: none;+ stroke: color-mix(in srgb, var(--text-primary) 82%, var(--clip-color, var(--audio)));+ stroke-width: 1px;+ stroke-linecap: round;+ vector-effect: non-scaling-stroke;+}++.clip-audio-visualization.loading {+ opacity: 0.48;+}+.timeline-clip:hover {filter: brightness(1.14);}@@ -1327,35 +1403,6 @@cursor: grabbing;}-.clip-text {- position: relative;- z-index: 2;- display: block;- padding: var(--space-2);-}--.clip-text b,-.clip-text small {- display: block;- overflow: hidden;- white-space: nowrap;- text-overflow: ellipsis;-}--.clip-text b {- color: var(--text-primary);- font-size: var(--text-sm);- font-weight: var(--weight-semibold);- line-height: var(--leading-snug);-}--.clip-text small {- margin-top: var(--space-1);- color: var(--text-primary);- opacity: 0.78;- font: var(--weight-medium) var(--text-2xs) / 1.1 var(--font-mono);-}-.timeline-playhead {position: absolute;z-index: 9;