Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

41.1 KB
Raw Blame
import { assertMusicVideoMediaSelections, musicVideoMediaMode } from '../../lib/musicVideoMediaPolicy.js';
import { isEndFrameImageEffect, sceneImageEffect } from '../../lib/musicVideoImageEffects.js';
import { musicVideoGradeFilter } from '../../lib/musicVideoGrade.js';
import { filmLookRuntimeSource, isFilmLookNeutral } from '../../lib/filmLook.js';
import { titleCardVisible, titleCardWindows } from '../../lib/musicVideoTitleCard.js';
import { titleCardRuntime } from './titleCardFonts.js';
/**
 * Music Video — render a project's composition document over the song.
 *
 * `composition.mode === 'document'` renders the project-owned HTML composition
 * (compositionDocument.js) through the music-video owner of the sandboxed
 * composition browser. Per job the document folder is copied into a private
 * scratch directory (htmlComposition `stageMusicVideoComposition`), and PortOS
 * adds the data the document draws from, before the snapshot freezes:
 *
 *   portos-mv.js  `window.PORTOS_MV = { project, render, song, lyrics, lyricMarkers,
 *                  scenes, textCues, composition }` — read with a plain <script>,
 *                  (scenes carry textZone/lyricRole for the shared lyricType.js),
 *                  no fetch (the sandbox refuses network).
 *   song.json     the same song block the code-rendered mode reads.
 *   media/        each scene's SELECTED take (video preferred, else its still)
 *                 as `media/scene-<sceneId>.<ext>`, named by `scenes[].media.src`.
 *
 * The document stays song-timed: a full render seeks 0…durationSec, an excerpt
 * seeks the SAME song times (`startSec + n / fps`), so a draft frame matches the
 * full render at that song time. The picture is captured silent; the master
 * song (plus the optional sound-design bed) is muxed after, trimmed to the
 * window. The page declares its own size; the project's aspect ratio picks the
 * frame, and a different aspect must be listed in `portosComposition.formats`
 * (the page's optional `layout({ width, height })` hook reframes it).
 */

import { constants as fsConstants } from 'fs';
import { copyFile, mkdir, rm, writeFile } from 'fs/promises';
import { existsSync } from 'fs';
import { join } from 'path';
import { ServerError } from '../../lib/errorHandler.js';
import { PATHS } from '../../lib/fileUtils.js';
import { htmlCompositionContractSchemaFor } from '../../lib/validation.js';
import { selectedPerformanceInstruction } from '../../lib/musicVideoShotTiming.js';
import { clipSpeedRate, clipOffsetSec, documentSceneVisualLayer, MUSIC_VIDEO_LYRIC_ROLES, MUSIC_VIDEO_LYRIC_SIZES, MUSIC_VIDEO_TEXT_ZONES } from '../../lib/musicVideoLayers.js';
import { musicVideoAspect } from '../../lib/musicVideoAspect.js';
import { documentDirectoryForRender } from './compositionDocument.js';
import { musicVideoSongDocument } from './compositionRender.js';
import { buildSongDocument } from './codeTimeline.js';
import { resolveNarrativeEvents, narrativeFrameState } from '../../lib/musicVideoNarrativeEvents.js';

export const DOCUMENT_RENDER_FPS = 24;
// The frame each project aspect renders at (the composition browser's sizes).
export const DOCUMENT_FRAME_SIZES = Object.freeze({
  '16:9': Object.freeze({ width: 1920, height: 1080 }),
  '9:16': Object.freeze({ width: 1080, height: 1920 }),
  '1:1': Object.freeze({ width: 1080, height: 1080 }),
});
const SONG_CAP_SEC = 900;

const round3 = (n) => Math.round(n * 1000) / 1000;
const finite = (n) => (typeof n === 'number' && Number.isFinite(n) ? n : null);
const scriptJson = (value) => JSON.stringify(value).replace(/</g, '\\u003c');
const safeSegment = (id) => String(id || 'scene').replace(/[^A-Za-z0-9_-]+/g, '-').slice(0, 80) || 'scene';
const documentImageEffect = (scene) => {
  const effect = sceneImageEffect(scene, { layered: true });
  return effect && !isEndFrameImageEffect(effect.effect) ? effect : null;
};

/** The project's aspect ratio (brief), defaulting to 16:9 (lib/musicVideoAspect.js). */
export const documentAspect = musicVideoAspect;

/**
 * The browser init scripts a document render opens its page with: the film
 * look runtime (lib/filmLook.js) when the project has a non-neutral look, and
 * the opening title card (lib/musicVideoTitleCard.js) when it has one to show,
 * so every captured frame, probe sample and worker browser draws what the live
 * preview shows. Pure.
 */
export function documentRenderInitScripts(project) {
  return [
    ...(isFilmLookNeutral(project?.filmLook) ? [] : [filmLookRuntimeSource(project.filmLook)]),
    ...(titleCardVisible(project?.titleCard) ? [titleCardRuntime(project.titleCard)] : []),
  ];
}

const aspectOf = (width, height) => {
  const r = width / height;
  if (Math.abs(r - 16 / 9) < 0.01) return '16:9';
  if (Math.abs(r - 9 / 16) < 0.01) return '9:16';
  if (Math.abs(r - 1) < 0.01) return '1:1';
  return `${width}x${height}`;
};

/**
 * The frame to render a validated page contract at for this project: the
 * page's own size when its aspect matches, else a size it declares in
 * `formats`. Throws COMPOSITION_DOCUMENT_FORMAT otherwise.
 */
function documentTargetFrame(contract, project) {
  const aspect = documentAspect(project);
  if (aspectOf(contract.width, contract.height) === aspect) return { ...contract };
  const { width, height } = DOCUMENT_FRAME_SIZES[aspect];
  const size = `${width}x${height}`;
  if (!contract.formats?.includes(size)) {
    throw new ServerError(`The composition document is ${contract.width}x${contract.height} and does not declare ${size} in portosComposition.formats (this project is ${aspect})`, {
      status: 422, code: 'COMPOSITION_DOCUMENT_FORMAT',
    });
  }
  return { ...contract, width, height };
}

/** The song length the document is timed against (analysis first). */
export function documentSongDuration(project) {
  const duration = finite(project?.audioAnalysis?.durationSec);
  return duration && duration > 0 ? Math.min(duration, SONG_CAP_SEC) : null;
}

/** Whole frames that fit inside the song at `fps` (the contract ceiling). */
export function documentRenderClock(songDurationSec, fps = DOCUMENT_RENDER_FPS) {
  const frames = Math.max(0, Math.floor(songDurationSec * fps + 1e-6));
  return { fps, frames, durationSec: frames / fps };
}

/**
 * Each scene's selected take as a file to stage: `Map<sceneId, { kind, path,
 * ext, inSec, outSec, fps, width, height }>`. `retimed` requests output-time
 * in/out points and staging instructions; raw-asset previews keep source time.
 * Video is preferred over the still;
 * a scene whose selection is missing on disk gets no entry (media: null).
 * `history` is the video history list; `probe(path)` measures a clip the
 * history did not (hosted renders record no geometry).
 */
export async function resolveSceneMedia(project, { history = [], probe = async () => null, strictLayers = false, retimed = false } = {}) {
  assertMusicVideoMediaSelections(project);
  const byId = new Map((Array.isArray(history) ? history : []).map((entry) => [entry.id, entry]));
  const out = new Map();
  const { safeUnder } = await import('../../lib/ffmpeg.js');
  for (const scene of Array.isArray(project?.scenes) ? project.scenes : []) {
    if (!scene?.sceneId) continue;
    const layer = documentSceneVisualLayer(project, scene, { generated: strictLayers });
    // A code shot (#10297) is drawn by the document itself; it is handed no media.
    if (layer === 'code' || (strictLayers && layer === 'card')) continue;
    const entry = scene.videoHistoryId ? byId.get(scene.videoHistoryId) : null;
    const videoPath = entry?.filename ? safeUnder(PATHS.videos, entry.filename) : null;
    if ((!strictLayers || layer === 'footage') && videoPath && existsSync(videoPath)) {
      const measured = entry.numFrames && entry.fps && entry.width && entry.height ? entry : { ...entry, ...((await probe(videoPath)) || {}) };
      const duration = measured.numFrames && measured.fps ? measured.numFrames / measured.fps : null;
      const edit = selectedPerformanceInstruction(scene)?.edit ?? null;
      const ext = (/\.([a-z0-9]{2,5})$/i.exec(entry.filename)?.[1] || 'mp4').toLowerCase();
      const rate = clipSpeedRate(scene);
      const originalIn = edit ? Math.min(edit.inSec, duration ?? edit.inSec) : 0;
      const originalOut = edit ? Math.min(edit.outSec, duration ?? edit.outSec) : duration;
      const start = originalIn + clipOffsetSec(scene);
      const sourceIn = Math.min(duration ?? Infinity, Math.max(0, start));
      // Offset holds are song/output seconds, independent of playback rate:
      // clipTimeAt uses sourceIn + max(0, local + min(0, start)) * rate.
      const leadSec = Math.max(0, -start);
      const retime = rate !== 1 || leadSec > 0;
      // A raw-asset preview plays the original bytes, so it keeps source-time
      // in/out and carries the same timing for the page's seek to apply.
      if (!retimed) {
        out.set(scene.sceneId, {
          kind: 'video',
          path: videoPath,
          ext,
          inSec: round3(originalIn),
          outSec: originalOut == null ? null : round3(originalOut),
          ...(retime || sourceIn !== originalIn ? { timing: { rate, leadSec: round3(leadSec), startSec: round3(sourceIn) } } : {}),
          fps: finite(measured.fps),
          width: finite(measured.width),
          height: finite(measured.height),
        });
        continue;
      }
      out.set(scene.sceneId, {
        kind: 'video',
        path: videoPath,
        ext: retime ? 'mp4' : ext,
        inSec: round3(sourceIn / rate),
        outSec: originalOut == null ? null : round3(leadSec + Math.max(sourceIn, originalOut) / rate),
        ...(retime ? { rate, leadSec } : {}),
        fps: retime ? DOCUMENT_RENDER_FPS : finite(measured.fps),
        width: finite(measured.width),
        height: finite(measured.height),
      });
      continue;
    }
    const imagePath = (!strictLayers || layer === 'still') && scene.referenceImageId ? safeUnder(PATHS.images, scene.referenceImageId) : null;
    if (imagePath && existsSync(imagePath)) {
      const ext = (/\.([a-z0-9]{2,5})$/i.exec(scene.referenceImageId)?.[1] || 'png').toLowerCase();
      out.set(scene.sceneId, { kind: 'image', path: imagePath, ext: ext === 'jpeg' ? 'jpg' : ext, inSec: null, outSec: null, fps: null, width: null, height: null });
    }
  }
  return out;
}

const mediaSrc = (sceneId, media) => `media/scene-${safeSegment(sceneId)}.${media.ext}`;

function sceneDirection(project, sceneId) {
  const direction = (project?.treatment?.shotDirections || []).find((d) => d?.sceneId === sceneId);
  if (!direction) return null;
  const { mode, route, focalSubject, framing, typographyRole, emphasis, transitionIn, transitionOut } = direction;
  return { mode, route, focalSubject, framing, typographyRole, emphasis, transitionIn, transitionOut };
}

/**
 * The `window.PORTOS_MV` payload (pure). `media` is resolveSceneMedia's map;
 * `frame` is `{ width, height }`; `clock` is documentRenderClock's result.
 */
export function buildDocumentData(project, { media = new Map(), frame, clock, songDurationSec, generated = false }) {
  const song = musicVideoSongDocument(project);
  const analysis = project?.audioAnalysis || {};
  const scenes = (Array.isArray(project?.scenes) ? project.scenes : [])
    .filter((scene) => scene?.sceneId)
    .slice()
    .sort((a, b) => (a.order ?? 0) - (b.order ?? 0))
    .map((scene) => {
      const m = media.get(scene.sceneId) || null;
      const start = finite(scene.startSec);
      const end = finite(scene.endSec);
      return {
        sceneId: scene.sceneId,
        label: typeof scene.label === 'string' ? scene.label : '',
        sectionLabel: typeof scene.sectionLabel === 'string' ? scene.sectionLabel : null,
        startSec: start,
        endSec: start != null && end != null && end > start ? end : null,
        shotMode: scene.shotMode === 'performance' ? 'performance' : 'cutaway',
        visualLayer: documentSceneVisualLayer(project, scene, { generated }),
        // A document has no end-frame media yet, so a start-to-end effect plays its still.
        imageEffect: documentImageEffect(scene),
        stillMove: typeof scene.stillMove === 'string' ? scene.stillMove : null,
        // #10589: the planned camera move the template's camera rig follows (only when planned).
        ...(typeof scene.camera?.move === 'string' ? { camera: structuredClone(scene.camera) } : {}),
        cardText: typeof scene.cardText === 'string' ? scene.cardText : null,
        cardColor: typeof scene.cardColor === 'string' ? scene.cardColor : null,
        textZone: MUSIC_VIDEO_TEXT_ZONES.includes(scene.textZone) ? scene.textZone : null,
        lyricRole: MUSIC_VIDEO_LYRIC_ROLES.includes(scene.lyricRole) ? scene.lyricRole : null,
        lyricSize: MUSIC_VIDEO_LYRIC_SIZES.includes(scene.lyricSize) ? scene.lyricSize : null,
        lyricText: typeof scene.lyricText === 'string' ? scene.lyricText : null,
        visualIntent: typeof scene.visualIntent === 'string' ? scene.visualIntent : null,
        direction: sceneDirection(project, scene.sceneId),
        media: m ? {
          kind: m.kind, src: mediaSrc(scene.sceneId, m),
          inSec: m.inSec, outSec: m.outSec, fps: m.fps, width: m.width, height: m.height,
          // Raw-asset previews only: seek source time = startSec + max(0, local - leadSec) * rate.
          ...(m.timing ? { timing: m.timing } : {}),
        } : null,
      };
    });
  const songDocument = buildSongDocument(project);
  const lyrics = songDocument.lyrics;
  const narrative = resolveNarrativeEvents(project, songDocument.sections, clock.fps);
  if (narrative.unresolved.length) throw new ServerError(`Rebind narrative events: ${narrative.unresolved.join(', ')}`, { status: 422, code: 'NARRATIVE_EVENT_UNRESOLVED' });
  if ((project.composition?.reactiveSections || []).some((entry) => !songDocument.sections.some((section) => section.id === entry.sectionId))) {
    throw new ServerError('Rebind reactive gain caps to current composition sections', { status: 422, code: 'NARRATIVE_SECTION_UNRESOLVED' });
  }
  const composition = project?.composition || {};
  return {
    version: 1,
    project: { id: project?.id ?? null, name: project?.name ?? '', aspect: documentAspect(project) },
    render: { width: frame.width, height: frame.height, fps: clock.fps, frames: clock.frames, durationSec: clock.durationSec },
    song: {
      durationSec: songDurationSec,
      bpm: finite(analysis.bpm) ?? finite(analysis.tempo),
      beats: song.beats,
      downbeats: song.downbeats,
      sections: song.sections,
      words: song.words || [],
      narrativeEvents: narrative.events,
      narrativeSections: songDocument.sections,
      reactiveSections: project.composition?.reactiveSections || [],
      features: analysis.features || null,
    },
    lyrics,
    lyricMarkers: Array.isArray(project?.lyricMarkers) ? project.lyricMarkers : [],
    scenes,
    // Where and when the title card shows, so the lyric type keeps its default zones off it.
    titleCard: titleCardVisible(project?.titleCard)
      ? { position: project.titleCard.position || 'lower-left', windows: titleCardWindows(project.titleCard) }
      : null,
    textCues: (composition.textCues || []).filter((cue) => cue?.text && cue.startSec != null && cue.endSec != null),
    composition: {
      mode: composition.mode || 'concat',
      style: composition.style || null,
      posterSec: composition.posterSec ?? null,
      overlay: composition.overlay || null,
    },
  };
}

/**
 * Everything a document render needs before the page opens. Throws
 * COMPOSITION_DOCUMENT_MISSING (409) or NO_TIMELINE (422).
 */
export async function prepareDocumentRender(project) {
  const directory = await documentDirectoryForRender(project);
  const songDurationSec = documentSongDuration(project);
  if (!songDurationSec) {
    throw new ServerError('A composition document is timed against the song — analyze the track first', { status: 422, code: 'NO_TIMELINE' });
  }
  const clock = documentRenderClock(songDurationSec);
  if (clock.frames < 1) throw new ServerError('The song is shorter than one frame', { status: 422, code: 'NO_TIMELINE' });
  const frame = DOCUMENT_FRAME_SIZES[documentAspect(project)];
  return { directory, songDurationSec, clock, frame, width: frame.width, height: frame.height, fps: clock.fps, durationSec: clock.durationSec };
}

/** Write portos-mv.js and copy the selected takes into a staged document folder. */
async function stageDocumentData(compositionDir, data, media, signal) {
  await mkdir(join(compositionDir, 'media'), { recursive: true });
  for (const scene of data.scenes) {
    const m = scene.media && media.get(scene.sceneId);
    if (!m) continue;
    const target = join(compositionDir, scene.media.src);
    signal?.throwIfAborted();
    if (m.kind === 'video' && m.rate != null) {
      // Documents seek output seconds themselves. Retime the staged copy so
      // existing/custom documents need no new playback contract.
      if (existsSync(target)) throw new ServerError(`The composition document ships ${scene.media.src}, a name PortOS writes at render time`, { status: 422, code: 'COMPOSITION_DOCUMENT_INVALID' });
      const { findFfmpeg, runFfmpegProcess, H264_ENCODE_ARGS } = await import('../../lib/ffmpeg.js');
      const bin = await findFfmpeg();
      if (!bin) throw new Error('ffmpeg not found on PATH');
      const filters = [
        `setpts=(PTS-STARTPTS)/${Number(m.rate.toFixed(6))}`,
        ...(m.leadSec > 0 ? [`tpad=start_mode=clone:start_duration=${round3(m.leadSec)}`] : []),
        `fps=${DOCUMENT_RENDER_FPS}`,
      ];
      const result = await runFfmpegProcess({ bin, signal, args: [
        '-hide_banner', '-loglevel', 'error', '-i', m.path,
        '-map', '0:v:0', '-vf', filters.join(','), ...H264_ENCODE_ARGS,
        '-an', '-movflags', '+faststart', '-n', target,
      ] });
      signal?.throwIfAborted();
      if (!result.ok) throw new Error(`Shot timing failed: ${result.reason}`);
      continue;
    }
    // Clone where the file system can (APFS), refuse to overwrite a file the
    // document shipped under PortOS's reserved name.
    await copyFile(m.path, target, fsConstants.COPYFILE_EXCL | fsConstants.COPYFILE_FICLONE).catch((error) => {
      if (error.code === 'EEXIST') throw new ServerError(`The composition document ships ${scene.media.src}, a name PortOS writes at render time`, { status: 422, code: 'COMPOSITION_DOCUMENT_INVALID' });
      throw error;
    });
  }
  await writeFile(join(compositionDir, 'portos-mv.js'), `window.PORTOS_MV = ${scriptJson(data)};\nwindow.PORTOS_MV_EVENT_STATE = ${narrativeFrameState.toString()};\n`);
}

/**
 * Stage a document the way a render does and open it in the OS-jailed preview
 * browser. The caller closes `page` and removes `scratchRoot`. Private media
 * stays in the staged copy; it is not posted into the user's browser.
 */
export async function openIsolatedDocumentPreview({ project, jobId, signal, onEvent }) {
  const plan = await prepareDocumentRender(project);
  const { probeVideoGeometry } = await import('../../lib/ffmpeg.js');
  const { stageMusicVideoComposition } = await import('../htmlComposition/index.js');
  const { openComposition } = await import('../htmlComposition/browser.js');
  const { loadHistory } = await import('../videoGen/history.js');
  const generated = project.composition?.document?.source?.kind === 'generated';
  const history = (project.scenes || []).some((scene) => scene?.videoHistoryId) ? await loadHistory() : [];
  const media = await resolveSceneMedia(project, { history, probe: probeVideoGeometry, retimed: true, strictLayers: generated });
  const data = buildDocumentData(project, {
    media, frame: plan.frame, clock: plan.clock, songDurationSec: plan.songDurationSec, generated,
  });
  let staged;
  let page;
  let playbackAdapter;
  let authoredDeclaration;
  try {
    const { adaptStagedDocumentPreview } = await import('./documentPreviewPlayback.js');
    const { readAuthoredPreviewDeclaration, bindAuthoredPreviewPlayback } = await import('./authoredPreviewPlayback.js');
    staged = await stageMusicVideoComposition(plan.directory, jobId, musicVideoSongDocument(project), {
      prepare: async (dir) => {
        authoredDeclaration = await readAuthoredPreviewDeclaration(dir, { scenes: data.scenes });
        await stageDocumentData(dir, data, media, signal);
        signal?.throwIfAborted();
        // A declared producer owns its implementation. Never rewrite it through
        // the canonical recognizer, including an unknown future declaration.
        playbackAdapter = authoredDeclaration.declared
          ? { adapted: false, producer: 'authored-v1', ...authoredDeclaration }
          : await adaptStagedDocumentPreview(dir, { scenes: data.scenes });
      },
    });
    signal?.throwIfAborted();
    page = await openComposition(staged.directory, {
      signal, streamMedia: true, mediaMode: musicVideoMediaMode(project), ownedBrowser: true, isolation: 'required',
      initScripts: [filmLookRuntimeSource(project?.filmLook), titleCardRuntime(project?.titleCard)],
      previewBridge: true, refuseUnknown: 'empty', lockLoopback: true, onEvent,
    });
    signal?.throwIfAborted();
    await page.send('Emulation.setDeviceMetricsOverride', {
      width: plan.width, height: plan.height, deviceScaleFactor: 1, mobile: false,
    });
    signal?.throwIfAborted();
    let playbackAvailable = false;
    if (playbackAdapter?.adapted || authoredDeclaration.supported) {
      let captureProtocolSupported = false;
      try {
        const version = await page.send('Browser.getVersion', {}, null);
        const match = typeof version?.product === 'string' && /^(?:HeadlessChrome|Chrome)\/(\d+)\.\d+\.\d+\.\d+$/.exec(version.product);
        const major = match ? Number(match[1]) : NaN;
        // 155 is our verified cap=1 capture-lifetime baseline, not the first
        // Chromium release to declare these optional screencast parameters.
        captureProtocolSupported = Number.isSafeInteger(major) && major >= 155;
      } catch { /* Unknown protocol support keeps the existing exact path. */ }
      signal?.throwIfAborted();
      const runtimeSupported = captureProtocolSupported && await page.evaluate(
        `typeof HTMLVideoElement.prototype.requestVideoFrameCallback === 'function' && typeof HTMLVideoElement.prototype.cancelVideoFrameCallback === 'function'`,
      );
      signal?.throwIfAborted();
      playbackAvailable = captureProtocolSupported && runtimeSupported;
      playbackAdapter = { ...playbackAdapter, runtimeSupported, captureProtocolSupported,
        ...(!captureProtocolSupported ? { reason: 'unsupported-capture-protocol' }
          : !runtimeSupported ? { reason: 'unsupported-frame-callbacks' } : {}) };
      if (playbackAvailable && authoredDeclaration.supported) {
        const bound = await bindAuthoredPreviewPlayback(page);
        signal?.throwIfAborted();
        playbackAvailable = bound.supported;
        playbackAdapter = { ...playbackAdapter, ...bound };
      }
    }
    return {
      page, scratchRoot: staged.scratchRoot, playbackAdapter,
      ...(playbackAvailable ? { playback: { ...playbackAdapter, version: 1 } } : {}),
      width: plan.width, height: plan.height, fps: plan.fps, durationSec: plan.durationSec,
    };
  } catch (error) {
    if (page) await page.close().catch(() => {});
    if (staged?.scratchRoot) await rm(staged.scratchRoot, { recursive: true, force: true }).catch(() => {});
    throw error;
  }
}

/**
 * The window a render seeks, on the document's own frame grid: the full
 * contract, or `[startSec, endSec)` snapped down to a frame boundary.
 */
function documentRenderWindow(contract, { windowStart = null, windowEnd = null } = {}) {
  if (windowStart == null && windowEnd == null) return { startSec: 0, durationSec: contract.durationSec, frames: Math.round(contract.durationSec * contract.fps) };
  const fps = contract.fps;
  if (!(windowStart >= 0) || !(windowEnd > windowStart) || windowStart >= contract.durationSec - 1e-6) {
    throw new ServerError('The excerpt range must fall within the composition document', {
      status: 422, code: 'INVALID_EXCERPT_RANGE', context: { totalDuration: contract.durationSec },
    });
  }
  const firstFrame = Math.floor(windowStart * fps + 1e-6);
  const lastFrame = Math.min(Math.round(contract.durationSec * fps), Math.ceil(Math.min(windowEnd, contract.durationSec) * fps - 1e-6));
  const frames = Math.max(1, lastFrame - firstFrame);
  return { startSec: firstFrame / fps, durationSec: frames / fps, frames };
}

/** Scene boundaries inside the window, relative to it (contact-sheet sample times). */
function documentBoundaryTimes(data, window) {
  const end = window.startSec + window.durationSec;
  const times = new Set([0]);
  for (const scene of data.scenes) {
    for (const t of [scene.startSec, scene.endSec]) {
      if (t == null || t < window.startSec - 1e-6 || t >= end - 1e-3) continue;
      times.add(round3(Math.max(0, t - window.startSec)));
    }
  }
  return [...times].sort((a, b) => a - b);
}

// Mux the silent picture with the master song (and the optional bed), both
// cut to the window on SONG time.
function documentMuxArgs(videoPath, audioPath, outputPath, { startSec, durationSec, songDurationSec, soundBed = null, buildBed, audioNorm, fade = '' }) {
  if (!soundBed?.path) {
    const args = ['-hide_banner', '-loglevel', 'error', '-i', videoPath];
    if (startSec > 0) args.push('-ss', String(startSec));
    args.push('-i', audioPath, '-map', '0:v:0', '-map', '1:a:0', '-t', String(durationSec), '-c:v', 'copy',
      '-af', `atrim=duration=${durationSec},apad=whole_dur=${durationSec},asetpts=PTS-STARTPTS${fade}`,
      '-c:a', 'aac', '-b:a', '192k', '-movflags', '+faststart', '-y', outputPath);
    return args;
  }
  const bed = buildBed({
    beds: [{ assetPath: soundBed.path, offsetSec: 0, durationSec: songDurationSec, volume: soundBed.volume }],
    firstInputIdx: 2, mainLabel: '[master]', outLabel: '[mixa]', limitPeak: true,
  });
  const filters = [`[1:a]${audioNorm}[master]`, ...bed.filters,
    `[mixa]atrim=start=${startSec}:end=${startSec + durationSec},asetpts=PTS-STARTPTS,apad=whole_dur=${durationSec}${fade}[outa]`];
  return ['-hide_banner', '-loglevel', 'error', '-i', videoPath, '-i', audioPath, ...bed.inputs,
    '-filter_complex', filters.join(';'), '-map', '0:v:0', '-map', '[outa]', '-t', String(durationSec), '-c:v', 'copy',
    '-c:a', 'aac', '-b:a', '192k', '-movflags', '+faststart', '-y', outputPath];
}

/**
 * Stage, open, capture and mux one document render (full song or a window).
 * Resolves `{ width, height, fps, durationSec, startSec, boundaryTimes }`.
 */
export async function encodeDocumentComposition({
  project, plan, jobId, audioPath, soundBed = null, outputPath, signal, onProgress, windowStart = null, windowEnd = null, fade = false, collectFootageVisibility = false,
}) {
  const { findFfmpeg, runFfmpegProcess, probeVideoGeometry, edgeFadeFilter } = await import('../../lib/ffmpeg.js');
  const { stageMusicVideoComposition } = await import('../htmlComposition/index.js');
  const { openComposition } = await import('../htmlComposition/browser.js');
  const { encodeComposition, encodeCompositionSegments } = await import('../htmlComposition/encode.js');
  const { loadHistory } = await import('../videoGen/history.js');
  const { AUDIO_NORM, buildAudioBedMix } = await import('../videoTimeline/audioBedMix.js');
  const ffmpeg = await findFfmpeg();
  if (!ffmpeg) throw new ServerError('ffmpeg not found on PATH', { status: 500, code: 'FFMPEG_MISSING' });
  const history = (project.scenes || []).some((s) => s?.videoHistoryId) ? await loadHistory() : [];
  const media = await resolveSceneMedia(project, { history, probe: probeVideoGeometry, retimed: true, strictLayers: project.composition?.document?.source?.kind === 'generated' });
  const data = buildDocumentData(project, { media, frame: plan.frame, clock: plan.clock, songDurationSec: plan.songDurationSec,
    generated: project.composition?.document?.source?.kind === 'generated' });
  const silent = `${outputPath}.silent.mp4`;
  let staged;
  let page;
  try {
    staged = await stageMusicVideoComposition(plan.directory, jobId, musicVideoSongDocument(project), {
      prepare: (dir) => stageDocumentData(dir, data, media, signal),
    });
    signal?.throwIfAborted();
    const openPage = (pageSignal) => openComposition(staged.directory, { signal: pageSignal, streamMedia: true, mediaMode: musicVideoMediaMode(project), ownedBrowser: true,
      initScripts: documentRenderInitScripts(project) });
    page = await openPage(signal);
    const metadata = await page.evaluate(`(() => {
      const c = globalThis.portosComposition;
      if (!c || typeof c.seek !== 'function') throw new Error('portosComposition.seek is required');
      return { durationSec: c.durationSec, fps: c.fps, width: c.width, height: c.height, motionBlur: c.motionBlur, formats: c.formats, layout: typeof c.layout === 'function' };
    })()`);
    const frames = metadata.durationSec * metadata.fps;
    if (Number.isFinite(frames) && metadata.fps > 0 && Math.abs(frames - Math.round(frames)) > 1e-8) {
      const lowerFrames = Math.floor(frames);
      throw new ServerError(`The composition document declares ${metadata.durationSec}s at ${metadata.fps} fps (${Number(frames.toFixed(8))} frames). Edit portosComposition.durationSec to a whole-frame duration, for example ${lowerFrames}/${metadata.fps} (${Number((lowerFrames / metadata.fps).toFixed(8))}s), and re-import the document. Review the song ending before choosing a shorter duration; PortOS has not rounded or trimmed the document.`, {
        status: 422, code: 'COMPOSITION_DOCUMENT_FRAME_ALIGNMENT',
        context: { durationSec: metadata.durationSec, fps: metadata.fps, frames },
      });
    }
    // The managed browser is Google Chrome by default; a Chromium build without
    // proprietary codecs cannot decode the H.264 takes. Say so up front instead
    // of failing on the first scene's <video> error.
    if ([...media.values()].some((m) => m.ext === 'mp4' || m.ext === 'mov')) {
      const h264 = await page.evaluate(`document.createElement('video').canPlayType('video/mp4; codecs="avc1.640028"')`);
      if (!h264) {
        throw new ServerError('The managed browser cannot decode H.264 video (a Chromium build without proprietary codecs) — point Settings › Browser at Google Chrome', {
          status: 422, code: 'COMPOSITION_BROWSER_CODEC',
        });
      }
    }
    const parsed = htmlCompositionContractSchemaFor(Math.max(1, plan.songDurationSec)).safeParse(metadata);
    if (!parsed.success) {
      throw new ServerError(`The composition document's contract is invalid — ${parsed.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`).join('; ')}`, {
        status: 422, code: 'COMPOSITION_DOCUMENT_CONTRACT',
      });
    }
    const target = documentTargetFrame(parsed.data, project);
    const window = documentRenderWindow(target, { windowStart, windowEnd });
    // Sample three moments per footage shot against its actual transformed source.
    // Bound cost for long edits; unsampled/missing hooks remain explicit unknowns.
    if (target.width !== parsed.data.width || target.height !== parsed.data.height) {
      await page.evaluate(`globalThis.portosComposition.layout({ width: ${target.width}, height: ${target.height} })`);
    }
    const footageVisibility = [];
    for (const scene of data.scenes.filter(s => collectFootageVisibility && s.visualLayer === 'footage' && s.startSec < window.startSec + window.durationSec && s.endSec > window.startSec)) {
      const start = Math.max(scene.startSec, window.startSec);
      const end = Math.min(scene.endSec, window.startSec + window.durationSec);
      for (const fraction of [0.2, 0.5, 0.8]) {
        signal?.throwIfAborted();
        const at = Math.floor((start + (end - start) * fraction) * target.fps) / target.fps;
        let sample = null;
        if (footageVisibility.length < 288) sample = await page.evaluate(`(async () => {
          const c = globalThis.portosComposition;
          await c.seek(${at}, { reviewFootage: true });
          return c.footageVisibility || null;
        })()`).catch(() => null);
        const measured = sample?.sceneId === scene.sceneId
          && ['visibleFraction', 'hiddenFraction', 'measuredFraction'].every(key => Number.isFinite(sample[key]) && sample[key] >= 0 && sample[key] <= 1)
          && Math.abs(sample.visibleFraction + sample.hiddenFraction - sample.measuredFraction) < 0.001;
        footageVisibility.push({ sceneId: scene.sceneId, atSec: round3(at - window.startSec),
          ...(measured ? { status: 'measured', visibleFraction: sample.visibleFraction, hiddenFraction: sample.hiddenFraction, measuredFraction: sample.measuredFraction } : { status: 'unverified' }) });
      }
    }
    // Long windows split across several browsers (encodeCompositionSegments).
    // A stalled worker browser restarts and redraws its frame (#10839).
    const encoded = await encodeCompositionSegments(page, { ...target, durationSec: window.durationSec }, silent, {
      openPage, encode: encodeComposition,
      videoFilterAt: (offsetSec) => musicVideoGradeFilter(project.composition?.grade, data.scenes, { fps: target.fps, offsetSec }),
      signal, offsetSec: window.startSec, onProgress: (fraction) => onProgress?.(fraction * 0.95),
      onRetry: ({ frame, attempt, segment, error }) => console.warn(`⚠️ Music-video composition render [${String(jobId).slice(0, 8)}] ${segment == null ? '' : `worker ${segment + 1} `}restarting its browser at song frame ${frame} (retry ${attempt}): ${error.message}`),
    });
    // A replaced first page was already closed after its failure.
    if (!encoded?.pageReplaced) {
      page.check();
      await page.close({ verify: true });
    }
    page = null;
    signal?.throwIfAborted();
    const mux = await runFfmpegProcess({
      bin: ffmpeg, signal,
      args: documentMuxArgs(silent, audioPath, outputPath, {
        startSec: window.startSec, durationSec: window.durationSec, songDurationSec: plan.songDurationSec,
        soundBed, buildBed: buildAudioBedMix, audioNorm: AUDIO_NORM, fade: fade ? edgeFadeFilter(window.durationSec) : '',
      }),
    });
    if (!mux.ok) {
      if (signal?.aborted || /cancelled/.test(mux.reason || '')) {
        const canceled = new Error('Render cancelled');
        canceled.code = 'CANCELED';
        throw canceled;
      }
      throw new Error(mux.reason || 'audio mux failed');
    }
    onProgress?.(1);
    return {
      width: target.width, height: target.height, fps: target.fps,
      durationSec: window.durationSec, startSec: window.startSec,
      boundaryTimes: documentBoundaryTimes(data, window), ...(collectFootageVisibility ? { footageVisibility } : {}),
    };
  } finally {
    if (page) await page.close().catch(() => {});
    if (staged?.scratchRoot) await rm(staged.scratchRoot, { recursive: true, force: true }).catch(() => {});
    await rm(silent, { force: true }).catch(() => {});
  }
}

const PROBE_SCREENSHOT = Object.freeze({ format: 'png', optimizeForSpeed: true, fromSurface: true, captureBeyondViewport: false });
// Backdrop luminance is measured on a half-size frame (plenty for a contrast percentile).
const PROBE_BACKDROP_SCALE = 0.5;
// Two animation frames after the seek and the text-hiding style: the first
// commits what seek() painted, the second runs once that frame was produced, so
// the backdrop screenshot cannot catch the page as it was before the seek (a
// caption on a plate would read as pale on the bare page). Bounded for a page
// that schedules no frames.
const PROBE_PAINTED = `new Promise((resolve) => {
  const timer = setTimeout(() => resolve(false), 1000);
  requestAnimationFrame(() => requestAnimationFrame(() => { clearTimeout(timer); resolve(true); }));
})`;

/**
 * Open a project's composition document the way a render does (same staged
 * data and selected takes, same frame) with the overlay text probe installed
 * (overlayTextProbe.js), and visit each song time `sampleTimes(data, frame)`
 * returns. At each one `onSample({ index, atSec, records, backdrop })` gets the
 * text the page drew and, when there was any, the frame with that text hidden
 * as linear luminance (`{ width, height, scale, values: Float32Array }`). Times
 * `onSample` returns are visited after the planned ones. Resolves
 * `{ frame, scenes, samples }` (the PORTOS_MV scenes, for naming shots).
 */
export async function probeDocumentText({ project, jobId, signal, sampleTimes, onSample, onProgress }) {
  const plan = await prepareDocumentRender(project);
  const { probeVideoGeometry } = await import('../../lib/ffmpeg.js');
  const { stageMusicVideoComposition } = await import('../htmlComposition/index.js');
  const { openComposition } = await import('../htmlComposition/browser.js');
  const { loadHistory } = await import('../videoGen/history.js');
  const { OVERLAY_TEXT_PROBE_SCRIPT } = await import('./overlayTextProbe.js');
  const { SRGB_TO_LINEAR } = await import('./overlayText.js');
  const { default: sharp } = await import('sharp');
  const generated = project.composition?.document?.source?.kind === 'generated';
  const history = (project.scenes || []).some((s) => s?.videoHistoryId) ? await loadHistory() : [];
  const media = await resolveSceneMedia(project, { history, probe: probeVideoGeometry, retimed: true, strictLayers: generated });
  const data = buildDocumentData(project, { media, frame: plan.frame, clock: plan.clock, songDurationSec: plan.songDurationSec, generated });
  let staged;
  let page;
  try {
    staged = await stageMusicVideoComposition(plan.directory, jobId, musicVideoSongDocument(project), {
      prepare: (dir) => stageDocumentData(dir, data, media, signal),
    });
    signal?.throwIfAborted();
    page = await openComposition(staged.directory, { signal, streamMedia: true, mediaMode: musicVideoMediaMode(project), ownedBrowser: true,
      initScripts: [...documentRenderInitScripts(project), OVERLAY_TEXT_PROBE_SCRIPT] });
    const metadata = await page.evaluate(`(() => {
      const c = globalThis.portosComposition;
      if (!c || typeof c.seek !== 'function') throw new Error('portosComposition.seek is required');
      if (!globalThis.__portosTextProbe) throw new Error('The overlay text probe did not load');
      return { durationSec: c.durationSec, fps: c.fps, width: c.width, height: c.height, motionBlur: c.motionBlur, formats: c.formats, layout: typeof c.layout === 'function' };
    })()`);
    const parsed = htmlCompositionContractSchemaFor(Math.max(1, plan.songDurationSec)).safeParse(metadata);
    if (!parsed.success) {
      throw new ServerError(`The composition document's contract is invalid — ${parsed.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`).join('; ')}`, {
        status: 422, code: 'COMPOSITION_DOCUMENT_CONTRACT',
      });
    }
    const target = documentTargetFrame(parsed.data, project);
    await page.send('Emulation.setDeviceMetricsOverride', { width: target.width, height: target.height, deviceScaleFactor: 1, mobile: false });
    if (target.width !== parsed.data.width || target.height !== parsed.data.height) {
      await page.evaluate(`globalThis.portosComposition.layout({ width: ${target.width}, height: ${target.height} })`);
    }
    const frame = { width: target.width, height: target.height };
    // onSample may return extra times to probe (a problem traced frame by frame); they
    // run after the planned ones, in the order returned, each with its own index.
    const times = [...sampleTimes(data, { ...frame, fps: target.fps, durationSec: target.durationSec })];
    for (let index = 0; index < times.length; index++) {
      const atSec = times[index];
      signal?.throwIfAborted();
      page.check();
      await page.evaluate('globalThis.__portosTextProbe.begin()');
      await page.evaluate(`globalThis.portosComposition.seek(${atSec})`).catch((error) => {
        throw new Error(`Composition seek(${atSec}) failed: ${error.message}`);
      });
      const records = await page.evaluate('globalThis.__portosTextProbe.end()');
      let backdrop = null;
      if (Array.isArray(records) && records.length) {
        await page.evaluate(PROBE_PAINTED);
        page.check();
        const { data: png } = await page.send('Page.captureScreenshot', PROBE_SCREENSHOT);
        const width = Math.max(1, Math.round(target.width * PROBE_BACKDROP_SCALE));
        const { data: rgb, info } = await sharp(Buffer.from(png, 'base64')).removeAlpha().resize({ width }).raw().toBuffer({ resolveWithObject: true });
        const values = new Float32Array(info.width * info.height);
        for (let p = 0, o = 0; p < values.length; p++, o += info.channels) {
          values[p] = 0.2126 * SRGB_TO_LINEAR[rgb[o]] + 0.7152 * SRGB_TO_LINEAR[rgb[o + 1]] + 0.0722 * SRGB_TO_LINEAR[rgb[o + 2]];
        }
        backdrop = { width: info.width, height: info.height, scale: info.width / target.width, values };
      }
      const extra = onSample({ index, atSec, frame, records: Array.isArray(records) ? records : [], backdrop });
      for (const t of Array.isArray(extra) ? extra : []) times.push(t);
      onProgress?.((index + 1) / times.length);
    }
    page.check();
    await page.close({ verify: true });
    page = null;
    return { frame, scenes: data.scenes, samples: times.length };
  } finally {
    if (page) await page.close().catch(() => {});
    if (staged?.scratchRoot) await rm(staged.scratchRoot, { recursive: true, force: true }).catch(() => {});
  }
}

/** Boot sweep: no document render survives a restart, so its scratch is stale. */
export async function sweepDocumentScratch() {
  const { MUSIC_VIDEO_SCRATCH_DIR } = await import('../htmlComposition/index.js');
  await rm(join(PATHS.data, MUSIC_VIDEO_SCRATCH_DIR), { recursive: true, force: true });
}