The open source AI video agent: turns any coding agent into a video agent, with an editor to change anything it makes.
Studio: fix Final Cut Pro audio connections and roles (#90)
taavie-z91b4e90c13be
diff --git a/.changeset/fcp-footage-audio.md b/.changeset/fcp-footage-audio.mdnew file mode 100644index 0000000..f0dde1a--- /dev/null+++ b/.changeset/fcp-footage-audio.md@@ -0,0 +1,10 @@+---+"openfilm": patch+"@openfilm/desktop": patch+---++Keep footage audio connected to its own picture in Final Cut Pro exports, preserving source trims, volume and disabled audio. Export separate sound as audio-only clips.++Match exported audio roles to Studio’s displayed row numbers, including overlapping clips and empty rows, and assign dialogue, effects and music roles to mixed stems. Keep Voice and Footage in separate dialogue subroles so they can be soloed and exported independently.++Write audio volume keyframes without unsupported interpolation attributes so Final Cut Pro imports faded audio correctly.diff --git a/packages/openfilm/studio/server/exports.mjs b/packages/openfilm/studio/server/exports.mjsindex 007ccb2..60cd520 100644--- a/packages/openfilm/studio/server/exports.mjs+++ b/packages/openfilm/studio/server/exports.mjs@@ -59,6 +59,7 @@ import { fileURLToPath } from 'node:url';import { FilmPage, launch } from '../../src/host.mjs';import { filmSound, kill as killProcess, mix, open } from '../../src/shared.mjs';import { FILM_FILE, clipFade, filmHtml, fittedFade, kindOf, pictureRect, readFilmFile } from '../../src/film-doc.mjs';+import { layerBlocks, rowBadges, trackKind } from './track-layout.mjs';import { fcpxmlDocument } from './fcpxml.mjs';import { NLE_FPS, nleFps, planNleTimeline, probeSource, soundStem, xmemlDocument } from './nle.mjs';import { draftsRoot, jianyingDraft } from './jianying.mjs';@@ -1108,6 +1109,17 @@ export function createExports({ onChange = () => {}, captions = subtitlesForExpoif (signal.aborted) throw new Error('cancelled');const byId = new Map(loaded.map((c) => [c.id, c]));+ const rows = ask.format === 'fcpxml' ? doc.tracks.flatMap((track) => {+ const blocks = track.clips.flatMap((clip) => {+ const c = byId.get(clip.id);+ return c ? [{ id: clip.id, startMs: Math.round(c.at * 1000), endMs: Math.round(c.end * 1000) }] : [];+ });+ const layers = layerBlocks(blocks);+ return (layers.length ? layers : [[]]).map((clips) => ({ kind: trackKind(track), clips }));+ }) : [];+ const badges = rowBadges(rows.map((row) => row.kind));+ const sourceRows = new Map(rows.flatMap((row, i) => row.clips.map((clip) => [clip.id, badges[i]])));+/* every source's facts, once *//** @type {Map<string, import('./nle.mjs').SourceFacts>} */const facts = new Map();@@ -1145,7 +1157,7 @@ export function createExports({ onChange = () => {}, captions = subtitlesForExpoconst f = await factsOf(path);if (f.hasAudio) {sounds.push({ id: row.id, src: row.src, path, at: c.at, from: c.from, to: c.to, speed: c.speed, volume,- enabled: heard, role: soundStem(row.src), track: ti, facts: f, ...(kind === 'video' ? { picture: row.id } : {}), ...(fade ? { fade } : {}) });+ enabled: heard, role: soundStem(row.src), track: ti, sourceRow: sourceRows.get(row.id), facts: f, ...(kind === 'video' ? { picture: row.id } : {}), ...(fade ? { fade } : {}) });}}}diff --git a/packages/openfilm/studio/server/exports.test.mjs b/packages/openfilm/studio/server/exports.test.mjsindex d8b9328..bcd0bad 100644--- a/packages/openfilm/studio/server/exports.test.mjs+++ b/packages/openfilm/studio/server/exports.test.mjs@@ -12,6 +12,7 @@ import { pptxParts } from './slides.mjs';import { filmSubtitleSrt, parseFilmSubtitleStyle } from './film-subtitle.mjs';import { zipFiles } from './zip.mjs';import { launch } from '../../src/host.mjs';+import { layoutTracks } from '../ui/src/lib/timeline-layout.ts';import { filmHtml } from '../../src/film-doc.mjs';const hasFfmpeg = (() => { try { execFileSync('ffmpeg', ['-version'], { stdio: 'ignore' }); return true; } catch { return false; } })();@@ -445,21 +446,129 @@ test('Final Cut Pro keeps source audio rows when an effect overlaps adjacent nar]);assert.deepEqual(timeline.audio.map((t) => t.sourceTrack), [1, 2, 3]);const xml = fcpxmlDocument(timeline);- assert.match(xml, /name="story-01-1" lane="-1"[^>]*audioRole="OpenFilm A1"/);- assert.match(xml, /name="story-01-2" lane="-1"[^>]*audioRole="OpenFilm A1"/);- assert.match(xml, /name="game-sfx-1" lane="-2"[^>]*audioRole="OpenFilm A2"/);- assert.match(xml, /name="adventure-music" lane="-3"[^>]*audioRole="OpenFilm A3"/);+ assert.match(xml, /name="story-01-1" lane="-1"[^>]*role="OpenFilm A1"/);+ assert.match(xml, /name="story-01-2" lane="-1"[^>]*role="OpenFilm A1"/);+ assert.match(xml, /name="game-sfx-1" lane="-2"[^>]*role="OpenFilm A2"/);+ assert.match(xml, /name="adventure-music" lane="-3"[^>]*role="OpenFilm A3"/);assert.notDeepEqual(planNleTimeline(film, opts).audio.map((t) => t.clips.map((c) => c.name)), timeline.audio.map((t) => t.clips.map((c) => c.name)), 'other editors keep their existing role grouping');const overlapping = planNleTimeline({ ...film, sounds: [...film.sounds, sound('overlapping-voice', 1, 1, 2)] }, { ...opts, audioTracks: 'source' });assert.equal(overlapping.audio.length, 4, 'only an overlap within a source row needs another lane');assert.deepEqual(overlapping.audio.map((t) => t.sourceTrack), [1, 1, 2, 3]);- assert.match(fcpxmlDocument(overlapping), /name="overlapping-voice"[^>]*audioRole="OpenFilm A1"/);+ assert.match(fcpxmlDocument(overlapping), /name="overlapping-voice"[^>]*role="OpenFilm A1"/);const reversed = planNleTimeline({ ...film, sounds: [...film.sounds].reverse() }, { ...opts, audioTracks: 'source' });assert.deepEqual(reversed.audio.map((t) => t.clips.map((c) => c.name)), timeline.audio.map((t) => t.clips.map((c) => c.name)));const { track, ...untracked } = film.sounds[0];assert.throws(() => planNleTimeline({ ...film, sounds: [untracked] }, { ...opts, audioTracks: 'source' }), /story-01-1 has no source track/);});+test('Final Cut Pro footage audio follows its own picture, including reused sources and processed audio', async () => {+ const facts = { w: 320, h: 180, duration: 20, fps: 25, hasVideo: true, hasAudio: true, channels: 2, sampleRate: 48000 };+ const picture = (id, track, at, end, from) => ({ id, track, at, end, from, kind: 'video',+ src: 'source.mov', path: join(scratch, 'source.mov'), speed: 1, hidden: false, facts });+ const pictures = [picture('main', 1, 0, 3, 0), picture('next', 1, 3, 8, 3),+ picture('overlay', 0, 1, 4, 2), picture('reused overlay', 0, 5, 7, 10)];+ const sounds = pictures.map((p) => ({ ...p, picture: p.id, role: 'footage', to: p.from + p.end - p.at,+ enabled: p.id !== 'main', volume: 0.5, ...(p.id === 'reused overlay' ? { fade: [0.2, 0.3] } : {}) }));+ sounds.push({ ...sounds[2], id: 'independent', picture: undefined, track: 2, enabled: true });+ const plans = [25, 29.97].map((fps) => planNleTimeline({+ stage: { w: 320, h: 180 }, duration: 8, pictures, sounds,+ }, { title: 'Linked footage', fps, audio: 'clips', media: 'copy', mediaDir: scratch, audioTracks: 'source' }));+ const parsed = await parseXml(plans.map((plan) => fcpxmlDocument(plan)), (doc) => {+ const seconds = (value) => {+ const [n, d = '1'] = (value ?? '0s').replace(/s$/, '').split('/');+ return Number(n) / Number(d);+ };+ const at = (el) => el.parentElement.tagName === 'spine' ? seconds(el.getAttribute('offset'))+ : at(el.parentElement) + seconds(el.getAttribute('offset')) - seconds(el.parentElement.getAttribute('start'));+ const clips = [...doc.querySelectorAll('audio, asset-clip[srcEnable="audio"]')].map((el) => ({+ name: el.getAttribute('name'), parent: el.parentElement.getAttribute('name'), parentTag: el.parentElement.tagName,+ owner: el.parentElement.parentElement.getAttribute('name'), at: at(el),+ from: seconds(el.getAttribute('start')), enabled: el.getAttribute('enabled'),+ volume: Boolean(el.querySelector('adjust-volume')), keyframes: el.querySelectorAll('keyframe').length,+ }));+ const overlay = doc.querySelector('clip[name="overlay"]');+ overlay.setAttribute('offset', `${seconds(overlay.getAttribute('offset')) + 1}s`);+ return { clips, moved: overlay.querySelector('audio') ? at(overlay.querySelector('audio')) : null, ambiguous: doc.querySelectorAll('asset-clip').length };+ });+ for (const [i, { error, found }] of parsed.entries()) {+ assert.equal(error, null);+ const plan = plans[i];+ assert.equal(found.clips.length, sounds.length, 'each sound appears exactly once as audio');+ const fps = plan.rate.fps;+ for (const original of sounds) {+ const clip = plan.audio.flatMap((track) => track.clips).find((clip) => clip.name === original.id);+ const actual = found.clips.find((clip) => clip.name === original.id);+ assert.ok(actual, original.id);+ assert.equal(actual.parent, original.picture ?? 'main', original.id);+ assert.equal(actual.parentTag, original.picture ? 'video' : 'clip', original.id);+ if (original.picture) assert.equal(actual.owner, original.picture, original.id);+ assert.equal(Math.round(actual.at * fps), clip.start, original.id);+ assert.equal(Math.round(actual.from * fps), clip.in, original.id);+ assert.equal(actual.enabled, original.enabled ? '1' : '0', original.id);+ assert.ok(actual.volume, original.id);+ }+ assert.equal(found.ambiguous, 0, 'audio nodes cannot enable video from an A/V asset');+ assert.ok(found.clips.find((clip) => clip.name === 'reused overlay').keyframes > 0);+ assert.ok(Math.abs(found.moved - 2) < 1 / fps, 'moving an overlay moves its sound by the same amount');+ }+});++test('Final Cut Pro keeps original audio paired when its picture is rendered', async () => {+ const facts = { w: 320, h: 180, duration: 8, hasVideo: true, hasAudio: true, channels: 2, sampleRate: 48000 };+ const documents = [1, 2].map((speed) => {+ const picture = { id: 'rendered', kind: 'video', src: 'source.mov', path: join(scratch, 'source.mov'),+ at: 1, end: 3, from: 2, speed, track: 0, hidden: false, styled: true, facts };+ const plan = planNleTimeline({ stage: { w: 320, h: 180 }, duration: 4, pictures: [picture],+ sounds: [{ ...picture, picture: picture.id, to: 2 + 2 * speed, volume: 0.5, enabled: true, role: 'footage' }],+ }, { title: 'Rendered footage', fps: 25, audio: 'clips', media: 'copy', mediaDir: scratch, audioTracks: 'source' });+ return fcpxmlDocument(plan);+ });+ for (const { error, found } of await parseXml(documents, (doc) => {+ const picture = doc.querySelector('clip[name="rendered"]');+ const audio = picture.querySelector('video > audio');+ return { linked: Boolean(audio), offset: audio?.getAttribute('offset'), count: doc.querySelectorAll('audio').length };+ })) {+ assert.equal(error, null);+ assert.ok(found.linked, 'rendering the picture must not detach its original sound');+ assert.equal(found.offset, '0/25s');+ assert.equal(found.count, 1);+ }+});++test('Final Cut Pro exports Studio row names for empty tracks, overlaps and footage sound', ffmpeg, async () => {+ const folder = join(scratch, 'Source rows');+ mkdirSync(folder);+ execFileSync('ffmpeg', ['-v', 'error', '-f', 'lavfi', '-i', 'color=size=160x90:rate=25:duration=1',+ '-f', 'lavfi', '-i', 'sine=frequency=440:duration=1', '-c:v', 'libx264', '-pix_fmt', 'yuv420p',+ '-c:a', 'aac', '-shortest', join(folder, 'picture.mp4')]);+ for (const name of ['voice.wav', 'music.wav']) writeFileSync(join(folder, name), readFileSync(join(root, 'assets/tone.wav')));+ const definitions = [+ { clips: [{ id: 'picture', src: 'picture.mp4', at: 0, time: [0, 1] }] },+ { clips: [] },+ { clips: [{ id: 'voice', src: 'voice.wav', at: 0, time: [0, 1] },+ { id: 'overlap', src: 'voice.wav', at: 0.5, time: [0, 0.5] }] },+ { clips: [{ id: 'music', src: 'music.wav', at: 0, time: [0, 1] }] },+ ];+ const doc = { stage: { w: 160, h: 90 }, tracks: definitions };+ writeFileSync(join(folder, 'film.html'), filmHtml(doc));+ const evaluated = definitions.flatMap((track, index) => track.clips.map((clip) => ({+ clipId: clip.id, key: clip.id, src: clip.src, loc: `film.html#${index}.${track.clips.indexOf(clip)}`,+ startMs: clip.at * 1000, durMs: (clip.time[1] - clip.time[0]) * 1000,+ track: { index, kind: index === 0 ? 'video' : 'audio', name: index === 0 ? 'video' : 'audio' },+ })));+ const studio = layoutTracks({ doc, scenes: [], captions: [],+ videos: evaluated.filter((clip) => clip.track.kind === 'video'),+ sounds: evaluated.filter((clip) => clip.track.kind === 'audio').map((clip) => ({ ...clip, kind: 'voice' })),+ });+ assert.deepEqual(studio.map((row) => row.badge), ['V2', 'V1', 'A1', 'A2', 'A3']);+ const expected = Object.fromEntries(studio.flatMap((row) => row.blocks.map((clip) => [clip.clipId, `OpenFilm ${row.badge}`])));+ const done = await exported({ kind: 'nle', format: 'fcpxml', fps: 25, audio: 'clips', subtitles: false }, 'Source rows', folder);+ const [{ error, found }] = await parseXml([readFileSync(done.out, 'utf8')], (doc) =>+ Object.fromEntries([...doc.querySelectorAll('audio')].map((audio) => [audio.getAttribute('name'), audio.getAttribute('role')])));+ assert.equal(error, null);+ assert.deepEqual(found, expected);+});+test('Final Cut Pro reports unreadable subtitles before writing an incomplete export', async () => {const exports = createExports({ captions: async () => { throw new Error('subtitle storage is unreadable'); } });const name = 'Unreadable captions';@@ -505,11 +614,11 @@ test('Final Cut Pro connections retain their parent and sequence positions acrosconst clips = [];const visit = (el, parentAt, parentIn) => {const at = parentAt + attr(el, 'offset') - parentIn;- if (el.tagName === 'asset-clip' || el.tagName === 'title') clips.push({+ if (['clip', 'audio', 'title'].includes(el.tagName)) clips.push({name: el.getAttribute('name'), parent: el.parentElement?.getAttribute('name'),at, end: at + attr(el, 'duration'), from: attr(el, 'start'),});- for (const child of el.children) if (['asset-clip', 'gap', 'title'].includes(child.tagName)) visit(child, at, attr(el, 'start'));+ for (const child of el.children) if (['clip', 'video', 'audio', 'gap', 'title'].includes(child.tagName)) visit(child, at, attr(el, 'start'));};const spine = doc.querySelector('sequence > spine');for (const child of spine.children) visit(child, 0, 0);@@ -544,8 +653,8 @@ test('Final Cut Pro export packages FCPXML, rendered pages, audio and optional sassert.equal(done.outputs.some((f) => f.endsWith('.srt')), audio === 'clips');const [{ error, found }] = await parseXml([readFileSync(done.out, 'utf8')], (doc) => ({version: doc.documentElement.getAttribute('version'),- primary: [...doc.querySelectorAll('sequence > spine > asset-clip')].map((c) => c.getAttribute('name')),- clips: [...doc.querySelectorAll('asset-clip')].map((c) => ({ name: c.getAttribute('name'), role: c.getAttribute('audioRole'), lane: Number(c.getAttribute('lane')), enabled: c.getAttribute('enabled'), kind: c.getAttribute('srcEnable') })),+ primary: [...doc.querySelectorAll('sequence > spine > clip')].map((c) => c.getAttribute('name')),+ clips: [...doc.querySelectorAll('clip, audio')].map((c) => ({ name: c.getAttribute('name'), role: c.getAttribute('role'), lane: Number(c.getAttribute('lane')), enabled: c.getAttribute('enabled'), kind: c.tagName === 'clip' ? 'video' : 'audio' })),media: [...doc.querySelectorAll('media-rep')].map((m) => m.getAttribute('src')),formats: [...doc.querySelectorAll('format')].map((f) => f.getAttribute('frameDuration')),subtitles: [...doc.querySelectorAll('title > text')].map((c) => c.textContent),@@ -559,8 +668,13 @@ test('Final Cut Pro export packages FCPXML, rendered pages, audio and optional sassert.equal(found.clips.filter((c) => c.kind === 'video').length, 2);assert.ok(found.clips.some((c) => c.kind === 'audio' && c.lane < 0));if (audio === 'clips') {- for (const name of ['tone', 'click']) assert.equal(found.clips.find((c) => c.name === name)?.role, 'OpenFilm A1');- assert.equal(found.clips.find((c) => c.name === 'bed')?.role, 'OpenFilm A2');+ assert.equal(found.clips.find((c) => c.name === 'tone')?.role, 'OpenFilm A1');+ assert.equal(found.clips.find((c) => c.name === 'click')?.role, 'OpenFilm A2');+ assert.equal(found.clips.find((c) => c.name === 'bed')?.role, 'OpenFilm A3');+ } else {+ assert.equal(found.clips.find((c) => c.name === 'Voice stem')?.role, 'dialogue.Voice');+ assert.equal(found.clips.find((c) => c.name === 'SFX stem')?.role, 'effects');+ assert.equal(found.clips.find((c) => c.name === 'Music stem')?.role, 'music');}assert.ok(found.formats.every((f) => f === '1/25s'));for (const url of found.media) assert.ok(existsSync(fileURLToPath(url)), url);diff --git a/packages/openfilm/studio/server/fcpxml.mjs b/packages/openfilm/studio/server/fcpxml.mjsindex 952980d..c65e273 100644--- a/packages/openfilm/studio/server/fcpxml.mjs+++ b/packages/openfilm/studio/server/fcpxml.mjs@@ -3,15 +3,15 @@ import { pathToFileURL } from 'node:url';import { frameSpan, nleRate, xmlEscape } from './nle.mjs';import { filmSubtitleCuesInRange, filmSubtitleSequence, filmSubtitleText } from './film-subtitle.mjs';+const STEM_ROLES = { voice: 'dialogue.Voice', footage: 'dialogue.Footage', sfx: 'effects', music: 'music' };// FCP's bundled template identifier, resolved by FCP on import rather than by the exporting machine.const BASIC_TITLE = '.../Titles.localized/Bumper:Opener.localized/Basic Title.localized/Basic Title.moti';/*** @typedef {import('./nle.mjs').NleClip} NleClip- * @typedef {import('./nle.mjs').NleTrack} NleTrack* @typedef {{ start: number, end: number, in: number, clip?: NleClip, children: string[] }} PrimarySegment* @typedef {{ enabled: boolean, lane: number, offset: number, kind: 'video' | 'audio',- * children?: string[], audioRole?: string }} ClipPlacement+ * children?: string[], audio?: string[], audioRole?: string }} ClipPlacement*//** XML attributes must preserve whitespace in names; text nodes keep literal line breaks. @param {string} value */@@ -71,16 +71,18 @@ export function fcpxmlDocument(timeline, { cues = [] } = {}) {);}/** @param {NleClip} clip @param {ClipPlacement} placement */- const clipXml = (clip, { enabled, lane, offset, kind, children = [], audioRole }) => {+ const clipXml = (clip, { enabled, lane, offset, kind, children = [], audio = [], audioRole }) => {+ const tag = kind === 'video' ? 'clip' : 'audio';const pad = lane === 0 ? ' ' : ' ';const attrs = [- `ref="${clip.fileId}"`, `name="${attribute(clip.name)}"`, `lane="${lane}"`,+ ...(kind === 'audio' ? [`ref="${clip.fileId}"`] : ['format="sequence-format"']), `name="${attribute(clip.name)}"`, `lane="${lane}"`,`offset="${time(offset)}"`, `start="${time(clip.in)}"`, `duration="${time(clip.end - clip.start)}"`,- `enabled="${enabled && clip.enabled ? 1 : 0}"`, `srcEnable="${kind}"`,+ `enabled="${enabled && clip.enabled ? 1 : 0}"`,];- if (audioRole) attrs.push(`audioRole="${attribute(audioRole)}"`);- const lines = [`${pad}<asset-clip ${attrs.join(' ')}>`];+ if (audioRole) attrs.push(`role="${attribute(audioRole)}"`);+ const lines = [`${pad}<${tag} ${attrs.join(' ')}>`];if (kind === 'video') {+ lines.push(`${pad} <video ref="${clip.fileId}" name="${attribute(clip.name)}" offset="${time(clip.in)}" start="${time(clip.in)}" duration="${time(clip.end - clip.start)}">`);// Disable automatic fitting: the planner's scale is relative to the source's own pixels.lines.push(`${pad} <adjust-conform type="none"/>`);if (clip.motion) {@@ -88,30 +90,50 @@ export function fcpxmlDocument(timeline, { cues = [] } = {}) {const position = `${decimal(center.h * width / height * 100)} ${decimal(-center.v * 100)}`;lines.push(`${pad} <adjust-transform position="${position}" scale="${decimal(scale / 100)} ${decimal(scale / 100)}"/>`);}+ // FCP keeps footage audio connected when it is nested inside the video component.+ lines.push(...audio, `${pad} </video>`);} else if (clip.levels?.length) {lines.push(`${pad} <adjust-volume>`, `${pad} <param name="amount">`, `${pad} <keyframeAnimation>`);for (const key of clip.levels) {- lines.push(`${pad} <keyframe time="${time(clip.in + key.when)}" value="${decibels(key.value)}" interp="linear" curve="linear"/>`);+ lines.push(`${pad} <keyframe time="${time(clip.in + key.when)}" value="${decibels(key.value)}"/>`);}lines.push(`${pad} </keyframeAnimation>`, `${pad} </param>`, `${pad} </adjust-volume>`);} else if (clip.gain !== undefined) lines.push(`${pad} <adjust-volume amount="${decibels(clip.gain)}"/>`);- return [...lines, ...children, `${pad}</asset-clip>`];+ return [...lines, ...children, `${pad}</${tag}>`];};const main = timeline.video[0];const primary = primaryStoryline(main?.clips ?? [], timeline.durationFrames);const sourceTracks = [...new Set(timeline.audio.flatMap((track) => track.sourceTrack === undefined ? [] : [track.sourceTrack]))].sort((a, b) => a - b);const audioRoles = new Map(sourceTracks.map((track, i) => [track, `OpenFilm A${i + 1}`]));- /** @param {NleTrack} track @param {number} lane @param {'video' | 'audio'} kind */- const connect = (track, lane, kind) => {- const audioRole = kind === 'audio' && track.sourceTrack !== undefined ? audioRoles.get(track.sourceTrack) : undefined;+ /** Paired footage sound follows its own picture; unrelated sound stays connected to the primary storyline.+ * @type {Map<string, { start: number, in: number, children: string[] }>}+ */+ const linked = new Map();+ for (const segment of primary) {+ if (!segment.clip?.linkGroup) continue;+ const children = /** @type {string[]} */ ([]);+ linked.set(segment.clip.linkGroup, { start: segment.start, in: segment.in, children });+ }+ const overlays = timeline.video.slice(1).flatMap((track, i) => track.clips.map((clip) => {+ const { parent, offset } = connectionAt(primary, clip.start);+ const audio = /** @type {string[]} */ ([]);+ if (clip.linkGroup) linked.set(clip.linkGroup, { start: clip.start, in: clip.in, children: audio });+ return { clip, parent, offset, lane: i + 1, enabled: track.enabled, audio };+ }));+ timeline.audio.forEach((track, i) => {+ const role = track.role ? STEM_ROLES[track.role] : track.sourceTrack === undefined ? undefined : audioRoles.get(track.sourceTrack);for (const clip of track.clips) {- const { parent, offset } = connectionAt(primary, clip.start);- parent.children.push(...clipXml(clip, { enabled: track.enabled, lane, offset, kind, audioRole }));+ const audioRole = clip.sourceRow ? `OpenFilm ${clip.sourceRow}` : role;+ const picture = clip.linkGroup ? linked.get(clip.linkGroup) : undefined;+ const parent = picture ?? connectionAt(primary, clip.start).parent;+ const offset = parent.in + clip.start - parent.start;+ parent.children.push(...clipXml(clip, { enabled: track.enabled, lane: -i - 1, offset, kind: 'audio', audioRole }));}- };- timeline.video.slice(1).forEach((track, i) => connect(track, i + 1, 'video'));- timeline.audio.forEach((track, i) => connect(track, -i - 1, 'audio'));+ });+ for (const { clip, parent, ...placement } of overlays) {+ parent.children.push(...clipXml(clip, { ...placement, kind: 'video' }));+ }const subtitleCues = filmSubtitleCuesInRange(cues, 0, timeline.durationFrames / rate.fps * 1000);const fontSize = decimal(Math.min(width, height) * 48 / 1080);let titleCount = 0;@@ -138,7 +160,7 @@ export function fcpxmlDocument(timeline, { cues = [] } = {}) {}const spine = primary.flatMap((segment) => {if (segment.clip) return clipXml(segment.clip, {- enabled: main?.enabled ?? true, lane: 0, offset: segment.start, kind: 'video', children: segment.children,+ enabled: main?.enabled ?? true, lane: 0, offset: segment.start, kind: 'video', children: segment.children, audio: segment.clip.linkGroup ? linked.get(segment.clip.linkGroup)?.children : undefined,});return [` <gap name="OpenFilm" offset="${time(segment.start)}" start="0s" duration="${time(segment.end - segment.start)}">`,diff --git a/packages/openfilm/studio/server/fcpxml.test.mjs b/packages/openfilm/studio/server/fcpxml.test.mjsindex f4fdaaa..e3e654e 100644--- a/packages/openfilm/studio/server/fcpxml.test.mjs+++ b/packages/openfilm/studio/server/fcpxml.test.mjs@@ -36,7 +36,7 @@ test('continuous main-track pictures occupy the primary storyline instead of con] },{ enabled: true, clips: [{ name: 'overlay', fileId: 'file-3', start: 45, end: 75, in: 5, out: 35, enabled: true }] },], audio: [{ enabled: true, clips: [{ name: 'music', fileId: 'file-2', start: 0, end: 90, in: 0, out: 90, enabled: true }] }] });- assert.match(xml, /<spine>\s*<asset-clip ref="file-1" name="first" lane="0"/);+ assert.match(xml, /<spine>\s*<clip format="sequence-format" name="first" lane="0"/);assert.match(xml, /name="second" lane="0" offset="30030\/30000s" start="15015\/30000s"/);assert.doesNotMatch(xml, /<gap /);// A connection's offset uses its parent's source timeline, not the sequence timeline.@@ -55,14 +55,17 @@ test('FCPXML keeps exact NTSC times, source trims, layering, transforms and sepaassert.ok(sourceUrl);assert.equal(fileURLToPath(sourceUrl), timeline.files[0].path);assert.match(sourceUrl, /%E7%94%B5%E5%BD%B1\/shot%20&%20one.mov$/);- assert.match(xml, /lane="0" offset="15015\/30000s" start="30030\/30000s" duration="45045\/30000s" enabled="1" srcEnable="video"/);+ assert.match(xml, /lane="0" offset="15015\/30000s" start="30030\/30000s" duration="45045\/30000s" enabled="1"/);assert.match(xml, /name="overlay" lane="1"/);+ assert.match(xml, /<video ref="file-1" name="footage" offset="30030\/30000s" start="30030\/30000s" duration="45045\/30000s">/);+ assert.doesNotMatch(xml, /<asset-clip |srcEnable=/);assert.match(xml, /<adjust-conform type="none"\/>/);assert.match(xml, /position="44.444444 25" scale="2 2"/);- assert.match(xml, /name="sound" lane="-1"[^>]*srcEnable="audio"/);+ assert.match(xml, /<audio ref="file-1" name="sound" lane="-1"/);assert.match(xml, /amount="-6.0206dB"/);assert.match(xml, /name="muted" lane="-2"[^>]*enabled="0"/);assert.match(xml, /<keyframe time="0\/30000s" value="-120dB"/);+ assert.doesNotMatch(xml, /<keyframe[^>]*(?:interp|curve)=/, 'FCP audio volume parameters do not accept interpolation attributes');assert.doesNotMatch(xml, /Infinity|NaN|undefined/);});@@ -73,14 +76,24 @@ test('only real empty intervals become primary gaps, with overlays anchored in sassert.equal((xml.match(/<gap /g) ?? []).length, 2);assert.match(xml, /name="footage" lane="0" offset="15015\/30000s" start="30030\/30000s"/);assert.match(xml, /name="overlay" lane="1" offset="45045\/30000s" start="0\/30000s"/);- assert.doesNotMatch(xml, /duration="90090\/30000s">\s*<asset-clip ref="file-1"/);+ assert.doesNotMatch(xml, /duration="90090\/30000s">\s*<clip /);});test('FCPXML also writes integer-rate, audio-only timelines with a full-length primary storyline', () => {const xml = fcpxmlDocument({ ...timeline, rate: nleRate(25), video: [], files: [timeline.files[1]], audio: [timeline.audio[1]] });assert.match(xml, /frameDuration="1\/25s"/);assert.match(xml, /<gap name="OpenFilm" offset="0\/25s" start="0s" duration="90\/25s">/);- assert.doesNotMatch(xml, /srcEnable="video"/);+ assert.doesNotMatch(xml, /<video /);+});++test('mixed stems keep Voice and Footage in separate dialogue subroles', () => {+ const roles = ['voice', 'footage', 'sfx', 'music'];+ const xml = fcpxmlDocument({ ...timeline, video: [], files: [timeline.files[1]], audio: roles.map((role) => ({+ enabled: true, role, clips: [{ name: `${role} stem`, fileId: 'file-2', start: 0, end: 90, in: 0, out: 90, enabled: true, gain: 0.5 }],+ })) });+ assert.deepEqual([...xml.matchAll(/<audio [^>]*role="([^"]+)"/g)].map((match) => match[1]),+ ['dialogue.Voice', 'dialogue.Footage', 'effects', 'music']);+ assert.equal((xml.match(/<adjust-volume amount="-6.0206dB"\/>/g) ?? []).length, 4);});test('FCPXML includes editable subtitle titles above pictures, clipped to the sequence, with escaped multiline text', () => {diff --git a/packages/openfilm/studio/server/nle.mjs b/packages/openfilm/studio/server/nle.mjsindex 022a30d..52ef6b0 100644--- a/packages/openfilm/studio/server/nle.mjs+++ b/packages/openfilm/studio/server/nle.mjs@@ -37,7 +37,7 @@ import { clipKind, pictureRect, soundRole } from '../../src/film-doc.mjs';* @typedef {'voice' | 'music' | 'sfx' | 'footage'} SoundRole* A sound's stem (see soundStem)* @typedef {{ id: string, src: string, path: string, at: number, from: number, to: number, speed: number, volume: number,- * enabled: boolean, role: SoundRole, track?: number, picture?: string, facts: SourceFacts | null, fade?: [number, number] }} NleSound+ * enabled: boolean, role: SoundRole, track?: number, sourceRow?: string, picture?: string, facts: SourceFacts | null, fade?: [number, number] }} NleSound* A sound: the source seconds [from, to) at `speed`, placed at film second `at`; `picture`: the footage clip it is* the sound of; `fade`: its fades in and out, film seconds (fitted to its length)* @typedef {{ stage: { w: number, h: number }, duration: number, pictures: NlePicture[], sounds: NleSound[] }} NleFilm@@ -56,10 +56,10 @@ import { clipKind, pictureRect, soundRole } from '../../src/film-doc.mjs';* video?: { w: number, h: number, alpha: boolean, still: boolean, fps?: number }, audio?: { channels: number, sampleRate: number } }} NleFile* @typedef {{ id: string, name: string, fileId: string, start: number, end: number, in: number, out: number, enabled: boolean,* motion?: { scale: number, center: { h: number, v: number } }, gain?: number,- * levels?: { when: number, value: number }[], linkGroup?: string }} NleClip+ * levels?: { when: number, value: number }[], linkGroup?: string, sourceRow?: string }} NleClip* Sequence frames [start, end), file frames [in, out); `gain` linear; `levels` its gain's keyframes (its fades),* frames from the clip's first- * @typedef {{ name: string, enabled: boolean, clips: NleClip[], sourceTrack?: number }} NleTrack+ * @typedef {{ name: string, enabled: boolean, clips: NleClip[], sourceTrack?: number, role?: SoundRole }} NleTrack* @typedef {{ name: string, rate: NleRate, width: number, height: number, durationFrames: number, video: NleTrack[],* audio: NleTrack[], files: NleFile[] }} NleTimeline*/@@ -168,6 +168,7 @@ function sourceAudioGroups(sounds) {*/export function planNleTimeline(film, opts) {const rate = nleRate(opts.fps);+ const sourceTracks = opts.audioTracks === 'source';const { stage } = film;/** @type {NleFile[]} */const files = [];@@ -241,9 +242,10 @@ export function planNleTimeline(film, opts) {const span = frameSpan(p.at, p.end, rate);const len = span.end - span.start;if (len <= 0) continue;+ const link = sourceTracks && p.kind === 'video' ? { linkGroup: `footage-${p.id}` } : {};const bake = () => {const file = addFile({ ...mediaPath(p.id, 'mov'), job: { kind: 'render', clip: p.id, frames: len }, durationFrames: len, video: { w: stage.w, h: stage.h, alpha: true, still: false } });- visual.push({ track: p.track, ...span, clip: { id: clipId(), name: p.id, fileId: file.id, ...span, in: 0, out: len, enabled: !p.hidden } });+ visual.push({ track: p.track, ...span, clip: { id: clipId(), name: p.id, fileId: file.id, ...span, in: 0, out: len, enabled: !p.hidden, ...link } });};/* a clip with a look of its own (the film's CSS: a crop, round corners) is drawn as the film draws it */if (p.kind === 'page' || p.styled || p.faded || !p.facts || (p.kind === 'video' && Math.abs(p.speed - 1) > 1e-6)) { bake(); continue; }@@ -256,6 +258,7 @@ export function planNleTimeline(film, opts) {/** @type {NleClip} */const clip = {id: clipId(), name: p.id, fileId: file.id, start: span.start, end: span.start + out - inFrame, in: inFrame, out, enabled: !p.hidden,+ ...link,...(placement.kind === 'motion' ? { motion: placement.motion } : {}),};if (p.kind === 'video') footage.set(p.id, { clip, file, picture: p });@@ -280,10 +283,9 @@ export function planNleTimeline(film, opts) {for (const role of ROLE_ORDER) {if (!opts.stems?.includes(role)) continue;const file = addFile({ ...mediaPath(`${opts.title || 'Film'} - ${role}`, 'wav'), job: { kind: 'stem', role }, durationFrames: total, audio: { channels: 2, sampleRate: 48_000 } });- audio.push({ name: ROLE_NAME[role], enabled: true, clips: [{ id: clipId(), name: `${ROLE_NAME[role]} stem`, fileId: file.id, start: 0, end: total, in: 0, out: total, enabled: true }] });+ audio.push({ name: ROLE_NAME[role], enabled: true, role, clips: [{ id: clipId(), name: `${ROLE_NAME[role]} stem`, fileId: file.id, start: 0, end: total, in: 0, out: total, enabled: true }] });}} else {- const sourceTracks = opts.audioTracks === 'source';/** @type {{ key: string, name: string, sourceTrack?: number }[]} */const groups = sourceTracks? sourceAudioGroups(film.sounds)@@ -325,13 +327,15 @@ export function planNleTimeline(film, opts) {const clip = {id: clipId(), name: s.id, fileId: file.id, start: span.start, end: span.start + out - inFrame, in: inFrame, out,enabled: s.enabled,+ ...(sourceTracks && s.sourceRow ? { sourceRow: s.sourceRow } : {}),...(Math.abs(s.volume - 1) > 1e-6 ? { gain: round6(Math.min(3.98109, s.volume)) } : {}),};if (faded && s.fade) clip.levels = fadeKeyframes(s.fade, out - inFrame, Math.min(3.98109, s.volume), rate);- if (own && linked) {+ if (own && linked && !sourceTracks) {linked.clip.linkGroup = `link-${linked.clip.id}`;clip.linkGroup = linked.clip.linkGroup;}+ if (sourceTracks && s.picture) clip.linkGroup = `footage-${s.picture}`;const group = sourceTracks ? `track-${s.track}` : s.role;const list = byGroup.get(group) ?? [];list.push({ start: clip.start, end: clip.end, clip });diff --git a/packages/openfilm/studio/server/track-layout.mjs b/packages/openfilm/studio/server/track-layout.mjsnew file mode 100644index 0000000..0ee7e1d--- /dev/null+++ b/packages/openfilm/studio/server/track-layout.mjs@@ -0,0 +1,43 @@+// @ts-check+import { kindOf } from '../../src/film-doc.mjs';++/** Studio's row kind, including audio-only uses of video files.+ * @param {{ clips: readonly { src: string, sound?: boolean }[] }} track+ * @returns {'mg' | 'video' | 'audio'}+ */+export function trackKind(track) {+ const kinds = new Set(track.clips.map((clip) => {+ const kind = kindOf(clip);+ return kind === 'page' ? 'mg' : kind === 'sound' ? 'audio' : 'video';+ }));+ return kinds.size === 1 ? /** @type {'mg' | 'video' | 'audio'} */ ([...kinds][0]) : 'video';+}++/** Picture rows count bottom up, sound rows top down; each prefix counts separately.+ * @param {readonly string[]} kinds+ */+export function rowBadges(kinds) {+ const prefixes = kinds.map((kind) => kind === 'mg' ? 'P' : kind === 'video' ? 'V' : 'A');+ const total = new Map();+ for (const prefix of prefixes) total.set(prefix, (total.get(prefix) ?? 0) + 1);+ const seen = new Map();+ return prefixes.map((prefix) => {+ const nth = (seen.get(prefix) ?? 0) + 1;+ seen.set(prefix, nth);+ return `${prefix}${prefix === 'A' ? nth : total.get(prefix) - nth + 1}`;+ });+}++/** Greedy rows in milliseconds; one millisecond of overlap is rounding at a shared edge.+ * @template {{ startMs: number, endMs: number }} T+ * @param {readonly T[]} blocks @returns {T[][]}+ */+export function layerBlocks(blocks) {+ /** @type {T[][]} */+ const rows = [];+ for (const block of [...blocks].sort((a, b) => a.startMs - b.startMs || a.endMs - b.endMs)) {+ const row = rows.find((r) => (r.at(-1)?.endMs ?? 0) <= block.startMs + 1);+ if (row) row.push(block); else rows.push([block]);+ }+ return rows;+}diff --git a/packages/openfilm/studio/ui/src/lib/film.ts b/packages/openfilm/studio/ui/src/lib/film.tsindex 43ecc2a..8d18b71 100644--- a/packages/openfilm/studio/ui/src/lib/film.ts+++ b/packages/openfilm/studio/ui/src/lib/film.ts@@ -2,6 +2,7 @@* film.html as the editor sees it: the format's own helpers (src/film-doc.mjs, shared with the CLI) and the few the* editor adds — Studio's track kinds and badges, clip locations, volume in dB.*/+import { rowBadges, trackKind } from '../../../server/track-layout.mjs';import { DEFAULT_STAGE, SPEED_MAX, SPEED_MIN, clipKind, isPage, isStill, kindOf, soundRole } from '../../../../src/film-doc.mjs';import type { FilmBox, FilmClip, FilmClipKind as FilmSrcKind, FilmDoc, FilmOverride, FilmTrack } from '../../../../src/film-doc.d.mts';@@ -28,11 +29,7 @@ export function filmClipKind(of: string | { src: string; sound?: boolean }): Fil}/** How Studio shows a track, from what is on it: all sound → audio, all pages → mg, anything else → video. */-export function filmTrackKind(track: { clips: readonly { src: string; sound?: boolean }[] }): FilmClipKind {- const kinds = new Set(track.clips.map((clip) => filmClipKind(clip)));- if (kinds.size === 1) return [...kinds][0]!;- return 'video';-}+export const filmTrackKind = trackKind;/** A file's name without its folder or extension (`assets/intro.html` → `intro`). */export function fileStem(path: string): string {@@ -52,15 +49,7 @@ export function filmTrackPrefix(kind: string): string {* the position itself.*/export function filmTrackBadges(tracks: readonly ({ kind: string } | { clips: readonly { src: string }[] })[]): string[] {- const prefixes = tracks.map((track) => filmTrackPrefix('kind' in track ? track.kind : filmTrackKind(track)));- const total = new Map<string, number>();- for (const p of prefixes) total.set(p, (total.get(p) ?? 0) + 1);- const seen = new Map<string, number>();- return prefixes.map((prefix) => {- const nth = (seen.get(prefix) ?? 0) + 1;- seen.set(prefix, nth);- return `${prefix}${prefix === 'A' ? nth : total.get(prefix)! - nth + 1}`;- });+ return rowBadges(tracks.map((track) => 'kind' in track ? track.kind : filmTrackKind(track)));}/** A sound's role in the mix, from where it lives (generated sounds are filed by kind; anything else is a voice). */diff --git a/packages/openfilm/studio/ui/src/lib/timeline-layout.ts b/packages/openfilm/studio/ui/src/lib/timeline-layout.tsindex 3d45fa6..c46ad6c 100644--- a/packages/openfilm/studio/ui/src/lib/timeline-layout.ts+++ b/packages/openfilm/studio/ui/src/lib/timeline-layout.ts@@ -5,6 +5,7 @@* clip turns a division into Infinity, a wrong ruler step only looks "a bit dense", a wrong track index shows the* wrong name on a row.*/+import { layerBlocks } from '../../../server/track-layout.mjs';import { clipSpan } from '../../../../src/film-doc.mjs';import { fileStem, filmTrackBadges, filmTrackKind, filmTrackPrefix, type FilmBox } from './film.ts';@@ -572,25 +573,8 @@ export function blockTitleOf(block: { title: string; atRef?: string | undefined;return notes.length ? `${block.title} · ${notes.join(' · ')}` : block.title;}-/**- * Clips that overlap by no more than this touch (SPEC): times are kept to the millisecond, a clip's start and its- * length each rounded alone, so two that touch can overlap by one.- */-const TOUCH_MS = 1;--/**- * Lay clips into rows that do not overlap. Two clips overlapping on one row cannot be drawn honestly, so an overlap- * opens another row. Greedy (the first row it fits), so the rows stay stable: drawing the same film again must not- * move the clip just clicked.- */-function layered(blocks: Block[]): Block[][] {- const rows: Block[][] = [];- for (const block of [...blocks].sort((a, b) => a.startMs - b.startMs || a.endMs - b.endMs)) {- const row = rows.find((r) => (r.at(-1)?.endMs ?? 0) <= block.startMs + TOUCH_MS);- if (row) row.push(block); else rows.push([block]);- }- return rows;-}+/** The same row allocation is used when exporting Studio's row names. */+const layered = layerBlocks;/*** Each clip where film.html puts it now.