Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

7.0 KB
Raw Blame
/**
 * Music Video scene i2v-clip attach hook (issue #1760, Phase 1).
 *
 * Subscribes to mediaJobEvents and, for each completed VIDEO job that carries
 * `params.musicVideo`, appends the resulting history id to that project scene's
 * immutable takes (#8965) — server-side, independent of any mounted client. The
 * take becomes the scene's `videoHistoryId` only while that slot is still
 * unselected; it never replaces a clip the director already selected. This is the
 * i2v counterpart to the Phase 1b reference-frame hook
 * (`musicVideoSceneImageHook`): a scene's video is generated from its chosen
 * reference frame via the video route's `image` (i2v) mode, and a long local/
 * Codex render that completes after the director navigated away, refreshed, or
 * moved on still lands on the scene (otherwise the clip reaches the video history
 * but the scene link is lost).
 *
 * Video jobs always ride the mediaJobQueue this hook listens to (there is no
 * synchronous video lane), so — unlike the image hook — there's no inline-attach
 * fallback; every scene clip lands through here.
 *
 * The video history record's id IS the job id (videoGen/local.js: meta.id =
 * jobId), and the `videoGenEvents` 'completed' payload the queue stores as
 * `job.result` carries it as `generationId`. `extractResult` reads
 * `job.result.generationId` and falls back to `job.id` so a runtime that doesn't
 * echo the field still attaches the right clip.
 *
 * The shared completion-hook scaffold (tag-decode, per-project serialization,
 * best-effort error handling, idempotent init/reset) lives in `createMediaJobImageHook` (#1791) — generalized to the video
 * `kind` in #1760 Phase 1. This file is just the music-video-video config,
 * structurally identical to its scene-image sibling. Mounted once at server boot
 * from server/index.js (after the media job queue is running).
 */

import { createMediaJobImageHook } from './mediaJobImageHook.js';
import { basename } from 'path';
import { appendSceneTakes } from './musicVideo/projects.js';
import { musicVideoEvents } from './musicVideo/events.js';
import { recordSceneLastFailure } from './musicVideo/sceneFailure.js';
import { alignPerformanceInstruction } from './musicVideo/takeAudioAlignment.js';
import { getHistoryItem } from './videoGen/history.js';
import { PATHS } from '../lib/fileUtils.js';
import { safeUnder } from '../lib/ffmpeg.js';

// #11210: a provider that pads the clip's start (fal OmniHuman) would sing
// late at the planned in-point; the clip's own audio says where the song is.
async function alignedShotInstruction(instruction, videoHistoryId) {
  if (instruction?.shotMode !== 'performance') return instruction ?? null;
  const entry = await getHistoryItem(videoHistoryId).catch(() => null);
  const clipPath = entry?.filename ? safeUnder(PATHS.videos, entry.filename) : null;
  const aligned = await alignPerformanceInstruction(instruction, clipPath).catch((err) => {
    console.warn(`⚠️ music-video take ${videoHistoryId.slice(0, 8)}: could not align its audio to the song: ${err.message}`);
    return instruction;
  });
  if (aligned.alignment?.offsetSec) console.log(`🎤 music-video take ${videoHistoryId.slice(0, 8)} aligned to the song: speech starts ${aligned.alignment.offsetSec.toFixed(3)}s into the clip (r=${aligned.alignment.correlation})`);
  return aligned;
}

const hook = createMediaJobImageHook({
  label: 'music-video scene-video',
  initLog: '🎬 Music Video scene-video hook initialized',
  kind: 'video',
  tagKey: 'musicVideo',
  // History id = the completed video job's id. The videoGenEvents 'completed'
  // payload (stored as job.result) echoes it as generationId; fall back to
  // job.id so a runtime that omits the field still attaches correctly.
  extractResult: (job) => {
    const videoHistoryId = (typeof job.result?.generationId === 'string' && job.result.generationId)
      || (typeof job.id === 'string' ? job.id : null);
    return videoHistoryId ? { videoHistoryId } : null;
  },
  // Require both ids; the tag is otherwise ambiguous about which scene to file.
  identify: (tag) => (tag?.projectId && tag.sceneId
    ? { projectId: tag.projectId, sceneId: tag.sceneId }
    : null),
  // Serialize per PROJECT: two scene clips for the same project completing close
  // together would otherwise both load→modify→save the one project record (file
  // backend) and the later write would clobber the earlier scene's
  // `videoHistoryId`. Different projects still attach concurrently.
  serializeKey: ({ projectId }) => projectId,
  // No newest-render-wins guard (#8965): out-of-order clips are all kept as
  // takes, and none of them can displace a selection.
  describe: ({ projectId, sceneId }) => `${projectId}/${sceneId}`,
  // A deleted project/scene 404s here, so a late completion can't resurrect it.
  attach: async ({ projectId, sceneId, videoHistoryId, job }) => appendSceneTakes(projectId, sceneId, [{
    kind: 'video',
    assetId: videoHistoryId,
    source: 'generated',
    provider: 'portos',
    jobId: typeof job.id === 'string' ? job.id : null,
    prompt: typeof job.params?.prompt === 'string' ? job.params.prompt : null,
    // The frame this clip was generated from — a basename only; job params
    // carry the server's absolute path, which never belongs on a synced record.
    sourceImageId: typeof job.params?.sourceImagePath === 'string' ? basename(job.params.sourceImagePath) : null,
    // #8977: a performance take keeps the immutable record of the song slice,
    // edit in/out points and capability it was generated against — the
    // renderer places the take by it, so a later retime can't misalign lips.
    shotInstruction: await alignedShotInstruction(job.params?.shotInstruction, videoHistoryId),
    dependencies: job.params?.musicVideoDependencies ?? null,
    // #10157: the estimate this clip was priced at; a revision-tagged job is
    // an auto-review generation, the rest are manual takes.
    costUsd: job.params?.musicVideoCostUsd ?? null,
    costModelId: job.params?.musicVideoCostModelId ?? null,
    spendKind: job.params?.musicVideo?.revisionId ? 'autoReview' : 'manual',
  }]),
  onAttached: ({ projectId, sceneId, videoHistoryId }, { scene, appended }) => {
    musicVideoEvents.emit('scene-video', {
      projectId,
      sceneId,
      videoHistoryId: scene.videoHistoryId ?? null,
      takes: scene.takes,
      // A landed clip retires this lane's recorded failure (takes.js).
      lastFailure: scene.lastFailure ?? null,
      takeId: appended[0]?.takeId ?? null,
    });
    console.log(`🎬 music-video scene clip take ${projectId.slice(0, 8)}/${sceneId} ← ${videoHistoryId.slice(0, 8)}`);
  },
  // A render that FAILED (a cancel is not a failure) is recorded on the scene
  // so the board can say which scene failed and why, including after a reload.
  onTerminal: (ctx, status, job) => recordSceneLastFailure(ctx, status, job, 'video'),
});

export function initMusicVideoSceneVideoHook() {
  hook.init();
}

// Test-only reset so suites that re-init can do so cleanly.
export const __testing = hook.__testing;