Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

18.1 KB
Raw Blame
import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';
import { discardUnpublishedVideo } from './generateVideoHelpers.js';
/** Lossless hand stitching and trim-aware chained-video assembly. */

import { existsSync } from 'fs';
import { join, basename } from 'path';
import { tmpdir } from 'os';
import { randomUUID } from 'crypto';
import { spawn } from '../../lib/childProcess.js';
import { PATHS, writeFileGuarded, unlinkGuarded } from '../../lib/fileUtils.js';
import { ServerError } from '../../lib/errorHandler.js';
import {
  findFfmpeg, safeUnder, generateThumbnail, optimizeForStreaming,
  hasAudioStream, buildTrimConcatArgs, bt709TagFilter,
} from '../../lib/ffmpeg.js';
import { safeChildProcessOptions } from '../../lib/processEnv.js';
import { killWithEscalation } from '../../lib/killWithEscalation.js';
import { renderTimingFields } from '../../lib/renderTiming.js';
import { loadHistory, mutateVideoHistory } from './history.js';

// Wall-clock render timing (#5878) for a CHAINED render's stitched entry. A
// chain writes its chunk rows `hidden: true`, so this is the only row the user
// ever sees. Chunks render sequentially, so the honest number is the span from
// the first chunk's start to now — it includes the inter-chunk trimming and
// this concat, which summing the chunks' own `renderMs` would drop. A
// hand-stitched clip is assembled from renders that already existed and gets
// nothing; so does a chain missing any chunk's start instant, rather than a
// span measured from a later chunk.
const chainRenderTiming = (videos, historyKey) => {
  if (historyKey !== 'chainedFrom') return {};
  const starts = videos.map((v) => Date.parse(v.renderStartedAt));
  return starts.every(Number.isFinite) ? renderTimingFields(Math.min(...starts)) : {};
};

// Fold the per-chunk DRAFTDECODE: reports into one claim, or none. Unanimity is
// on the `applied` verdict rather than deep equality: two chunks that both fell
// back agree about the clip's fidelity even if their failure reasons differ.
const unanimousDraftDecodeOutcome = (videos) => {
  const first = videos[0]?.draftDecodeApplied;
  if (first === undefined) return {};
  return videos.every((v) => v?.draftDecodeApplied?.applied === first.applied)
    ? { draftDecodeApplied: first }
    : {};
};

// Fold the per-chunk STREAMPOLICY: reports into one claim, or none (#6499).
// Unanimity is on the `active` verdict — every chunk streamed, or every chunk
// stayed resident — rather than deep equality, so a peakMb reading that
// differs slightly between chunks doesn't defeat a chain that otherwise
// agrees on WHETHER it streamed.
const unanimousStreamingPolicyOutcome = (videos) => {
  const first = videos[0]?.streamingPolicyApplied;
  if (first === undefined) return {};
  return videos.every((v) => v?.streamingPolicyApplied?.active === first.active)
    ? { streamingPolicyApplied: first }
    : {};
};

// Fold the per-chunk LORASTRATEGY: reports into one claim, or none (#12262).
// Unanimity is on every field the stitched record would claim for each adapter
// (role, name, strength, stages, strategy) — only the per-chunk target counts
// may differ — so a chain whose chunks applied the same adapters the same way
// keeps the claim and any other chain makes none.
const loraStrategySignature = (applied) => JSON.stringify(
  (applied?.adapters || []).map((a) => [a?.role, a?.name, a?.strength, a?.stages, a?.strategy]),
);
const unanimousLoraStrategyOutcome = (videos) => {
  const first = videos[0]?.loraStrategyApplied;
  if (first === undefined) return {};
  const signature = loraStrategySignature(first);
  return videos.every((v) => v?.loraStrategyApplied && loraStrategySignature(v.loraStrategyApplied) === signature)
    ? { loraStrategyApplied: first }
    : {};
};

export async function stitchVideos(videoIds, opts = {}) {
  const {
    id = randomUUID(),
    filenamePrefix = 'stitched',
    historyKey = 'stitchedFrom',
    promptOverride = null,
    // Per-chunk prompt beats to record on the stitched entry (#3695) — chained
    // renders only; a hand-stitched clip has no beats.
    chunkPrompts = null,
    // Optional `Map<videoId, { startFrame, frames }>` — leading frames to drop
    // from an input, and the frame count it contributes once they're gone.
    //
    // A chained render's windowed chunks open with an echo of the tail window
    // they were conditioned on, which the timeline already holds. Cutting that
    // echo in this concat's filter graph costs nothing beyond the timeline
    // encode; pre-trimming each chunk file instead would add one full encode
    // per chunk and re-grade the trimmed chunks relative to their siblings.
    //
    // Both numbers are MEASUREMENTS of the file, which is why they have to
    // come from the caller: a history entry's own `numFrames` is the count the
    // clip was RENDERED at — a render parameter Remix reuses — and an extend
    // render's output is `source + extension`, so its file is materially
    // longer than that. `startFrame: 0` is therefore meaningful on its own:
    // "no cut, but here is the real length." Only a non-zero offset routes the
    // concat through the filter graph.
    trims = null,
    // Optional AbortSignal (the stitch media job's cancel): kills the ffmpeg
    // child and removes the partial output so a cancelled encode publishes no row.
    signal = null,
  } = opts;
  if (!Array.isArray(videoIds) || videoIds.length < 2) {
    throw new ServerError('Need at least 2 videos to stitch', { status: 400, code: 'VALIDATION_ERROR' });
  }
  const ffmpeg = await findFfmpeg();
  if (!ffmpeg) throw new ServerError('ffmpeg not found on PATH', { status: 500, code: 'FFMPEG_MISSING' });

  const history = await loadHistory();
  const videos = videoIds.map((vid) => history.find((h) => h.id === vid)).filter(Boolean);
  if (videos.length !== videoIds.length) throw new ServerError('Some videos not found', { status: 400, code: 'VALIDATION_ERROR' });

  // Validate every history-supplied filename through safeUnder before
  // letting it reach ffmpeg's concat manifest. Tampered history entries
  // could otherwise smuggle `..` segments into ffmpeg input.
  const videoPaths = videos.map((v) => safeUnder(PATHS.videos, v.filename));
  if (videoPaths.some((p) => !p)) {
    throw new ServerError('One or more video filenames failed validation', { status: 400, code: 'VALIDATION_ERROR' });
  }
  for (const p of videoPaths) {
    if (!existsSync(p)) throw new ServerError(`Missing: ${basename(p)}`, { status: 404, code: 'NOT_FOUND' });
  }

  // Measured cut + contribution per input, in `videos` order. A zero offset
  // means the input joins whole (the entry is then purely a length
  // measurement); any non-zero offset routes the whole concat through the
  // filter graph, since only that path can express a cut.
  const trimPlan = videos.map((v) => {
    const entry = trims?.get?.(v.id);
    if (!entry) return null;
    const frames = Number(entry.frames);
    return {
      startFrame: Math.max(0, Math.floor(Number(entry.startFrame) || 0)),
      frames: Number.isFinite(frames) && frames >= 0 ? frames : null,
    };
  });

  const listFile = join(tmpdir(), `concat-${id}.txt`);
  // Set before the write, not after: `writeFile` creates and truncates the
  // file before it writes, so a failure partway through still leaves one on
  // disk to clean up. Unlinking a file that was never created is a no-op here.
  let listFileWritten = false;
  const writeConcatList = async () => {
    // ffmpeg concat-demuxer escape: per its docs, single quotes in filenames
    // must be replaced with `'\''`. Inside quoted strings ffmpeg also treats
    // backslash as an escape character — on Windows where paths are
    // `C:\foo\bar.mp4`, that corrupts the path. Normalize to forward slashes
    // (which ffmpeg accepts on Windows just fine) before quoting.
    const escapeForConcat = (p) => p.replace(/\\/g, '/').replace(/'/g, "'\\''");
    listFileWritten = true;
    await writeFileGuarded(listFile, videoPaths.map((p) => `file '${escapeForConcat(p)}'`).join('\n'));
  };

  const outFilename = `${filenamePrefix}-${id}.mp4`;
  const outPath = join(PATHS.videos, outFilename);

  // `captureStderr` keeps the last line of ffmpeg's own diagnostics on the
  // error, so a failure reads as something other than a bare "Stitch failed" —
  // which can't tell a filter-graph parse error from a full disk. Split on \r
  // as well as \n: ffmpeg separates its progress lines with a bare carriage
  // return, so a failure mid-encode would otherwise trail a run of them behind
  // the line that matters.
  const runFfmpeg = (args, { captureStderr = false } = {}) => new Promise((resolve, reject) => {
    if (signal?.aborted) {
      reject(new ServerError('Stitch canceled', { status: 499, code: 'STITCH_CANCELED' }));
      return;
    }
    const proc = spawn(ffmpeg, args, safeChildProcessOptions({ stdio: captureStderr ? ['ignore', 'ignore', 'pipe'] : 'ignore' }));
    const onAbort = () => killWithEscalation(proc, { label: 'stitch ffmpeg child', stillRunning: () => true });
    signal?.addEventListener('abort', onAbort, { once: true });
    proc.on('close', () => signal?.removeEventListener('abort', onAbort));
    let tail = '';
    proc.stderr?.on('data', (d) => { tail = `${tail}${d}`.slice(-400); });
    proc.on('close', (code) => {
      if (signal?.aborted) return reject(new ServerError('Stitch canceled', { status: 499, code: 'STITCH_CANCELED' }));
      if (code === 0) return resolve();
      const detail = tail ? `: ${tail.split(/[\r\n]+/).filter(Boolean).pop()?.trim() || ''}` : '';
      reject(new ServerError(`Stitch failed${detail}`, { status: 500, code: 'FFMPEG_FAILED' }));
    });
    proc.on('error', (err) => reject(new ServerError(`ffmpeg failed to spawn: ${err.message}`, { status: 500, code: 'FFMPEG_FAILED' })));
  });

  // Inputs with a real cut, as opposed to the measurement-only entries. Zero
  // means the demuxer can do the job.
  const cutCount = trimPlan.filter((t) => t?.startFrame > 0).length;
  // Tracks whether the cuts actually made it into the output, so `numFrames`
  // below reports the timeline that exists rather than the one we asked for.
  let trimsApplied = cutCount > 0;
  // Use a try/finally so the concat list temp file is cleaned up even when
  // ffmpeg rejects — otherwise it leaks one file per failed stitch.
  try {
    if (trimsApplied) {
      const args = buildTrimConcatArgs({
        inputs: videoPaths.map((path, i) => ({ path, startFrame: trimPlan[i]?.startFrame || 0 })),
        outPath,
        // Canonical geometry/rate for the graph's normalization filters. Taken
        // from the first input the same way modelId/seed are below — every
        // caller that reaches here stitches one model's own output.
        width: videos[0].width,
        height: videos[0].height,
        fps: videos[0].fps,
        // `concat=a=1` needs an audio leg from EVERY input; one silent clip in
        // the set makes the whole graph video-only.
        withAudio: (await Promise.all(videoPaths.map((p) => hasAudioStream(p)))).every(Boolean),
        // Pin BT.709 on the stitched output — this is the only re-encode a
        // chained render's timeline gets, so an untagged result here is what a
        // player would have to guess at. `null` on an ffmpeg without the
        // filter; the container flags ride along inside the builder regardless.
        colorTagFilter: await bt709TagFilter(),
      });
      const failure = args
        ? await runFfmpeg(args, { captureStderr: true }).then(() => null, (err) => err)
        : new Error('could not build the concat filter graph');
      if (failure) {
        // Degrade rather than throw away a whole chained render: the untrimmed
        // inputs were never re-encoded, so they're still in codec lockstep and
        // the stream-copy concat below can salvage the clip. The cost is the
        // echoed context replaying at each trimmed seam.
        console.log(`⚠️ Trimmed concat failed (${failure.message}) — falling back to a stream copy; ${cutCount} seam(s) will repeat their context`);
        trimsApplied = false;
      }
    }
    if (!trimsApplied) {
      await writeConcatList();
      // This one is fatal — its rejection is what the caller surfaces — so it
      // needs the cause even more than the survivable run above does.
      await runFfmpeg(['-f', 'concat', '-safe', '0', '-i', listFile, '-c', 'copy', '-y', outPath], { captureStderr: true });
    }
    await optimizeForStreaming(outPath);
    if (signal?.aborted) throw new ServerError('Stitch canceled', { status: 499, code: 'STITCH_CANCELED' });
  } catch (err) {
    // A canceled encode leaves a partial file that no history row will ever own.
    if (signal?.aborted) await unlinkGuarded(outPath).catch(() => {});
    throw err;
  } finally {
    if (listFileWritten) await unlinkGuarded(listFile).catch(() => {});
  }

  return withBackupAssetPublication(async () => {
    let committed = false;
    try {
      const thumb = await generateThumbnail(outPath, id);
      const stitchedMeta = {
        id,
        prompt: promptOverride != null
          ? promptOverride
          : `Stitched: ${videos.map((v) => v.prompt).join(' + ')}`,
        modelId: videos[0].modelId,
        seed: videos[0].seed ?? 0,
        width: videos[0].width,
        height: videos[0].height,
        // What each input actually contributes. A measured plan wins over the
        // entry's own `numFrames` either way — but when the cuts didn't make it
        // into the output, the input contributes its WHOLE measured length
        // (`startFrame + frames`), not the trimmed length we asked for.
        numFrames: videos.reduce((sum, v, i) => {
          const plan = trimPlan[i];
          const measured = plan?.frames == null
            ? null
            : (trimsApplied ? plan.frames : plan.startFrame + plan.frames);
          return sum + (measured ?? v.numFrames ?? 0);
        }, 0),
        fps: videos[0].fps,
        filename: outFilename,
        thumbnail: thumb,
        createdAt: new Date().toISOString(),
        ...chainRenderTiming(videos, historyKey),
        [historyKey]: videoIds,
        ...(Array.isArray(chunkPrompts) ? { chunkPrompts } : {}),
        // The stitched clip is the chain's visible history row, so it must carry
        // the same render controls and provenance as the hidden chunks. Preserve
        // meaningful falsey values (guidance 0, audio enabled, empty conditioning)
        // while keeping legacy/partial entries free of explicit undefined fields.
        ...Object.fromEntries([
          'steps',
          'guidanceScale',
          'tiling',
          'disableAudio',
          'mode',
          'textEncoderId',
          'imageStrength',
          'i2vReferenceMode',
          'conditioning',
          'renderInputsVersion',
          // Speed profile (#4875) — the REQUESTED schedule and what the runner
          // actually applied. Inherited for a stronger reason than the dials above:
          // a chain's chunk entries are written `hidden: true`, so this stitched
          // record is the ONLY one the user ever sees. Without it the lightbox's
          // "Speed profile" row never renders for a chained render — including a
          // chain whose TeaCache or adapter was unavailable, which is exactly the
          // silent speed claim the feature exists to prevent — and a Remix quietly
          // reverts to Quality. A chain applies its profile to every chunk or to
          // none (resolveVideoSpeedProfileForModes), so videos[0] speaks for all.
          'speedProfileId',
          'speedProfileApplied',
          // Draft decode REQUEST (#5423) — chain-wide by construction: every chunk
          // is submitted with the same `draftDecode`, and the gate that resolves it
          // is mode-independent, so videos[0] speaks for all. Inherited for the same
          // reason as the speed profile above — the chunks are hidden, so without it
          // the stitched record could never say the clip was decoded at preview
          // fidelity, and a Remix would quietly revert to Full.
          'draftDecode',
          // Block-streaming REQUEST (#6499) — same reasoning as draftDecode above:
          // every chunk is submitted with the same streamingMode, and the chunks
          // are hidden, so without inheriting it the stitched record could never
          // show which mode the chain was rendered with.
          'streamingMode',
        ].flatMap((key) => videos[0][key] === undefined ? [] : [[key, videos[0][key]]])),
        // The draft-decode OUTCOME is decided per child process — the runner falls
        // back to the full decoder on any load failure — so unlike the request
        // above, videos[0] does NOT speak for the chain. Inherited only when every
        // chunk agrees; on a mixed chain the field is omitted, which the lightbox
        // already renders as "outcome not reported". Asserting chunk 0's verdict
        // over a clip whose later chunks decoded differently would be exactly the
        // false fidelity claim this field exists to prevent.
        ...unanimousDraftDecodeOutcome(videos),
        ...unanimousStreamingPolicyOutcome(videos),
        ...unanimousLoraStrategyOutcome(videos),
        // Inherit applied LoRAs from the first constituent clip (a chunk chain
        // shares one LoRA set across all chunks), so the visible stitched entry
        // round-trips LoRAs on Remix the same way a single render does — mirrors
        // how modelId/seed/width above are taken from videos[0].
        ...(Array.isArray(videos[0].loraFilenames) && videos[0].loraFilenames.length ? {
          loraFilenames: videos[0].loraFilenames,
          loraScales: videos[0].loraScales,
        } : {}),
      };
      // Serialized append against the shared history tail (re-reads the freshest
      // list inside the mutator) so a concurrent download/render write can't drop
      // this stitched entry.
      await mutateVideoHistory((history) => { history.unshift(stitchedMeta); return history; });
      committed = true;
      console.log(`🎬 Stitched ${videos.length} videos → ${outFilename}`);
      return stitchedMeta;
    } catch (error) {
      if (!committed) await discardUnpublishedVideo({ jobId: id, outputPath: outPath });
      throw error;
    }
  });
}