Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

6.8 KB
Raw Blame
/**
 * Shared yt-dlp FULL-VIDEO download core.
 *
 * The mirror image of ytdlpAudioImport.js, extracted for the same reason: the
 * YouTube brain ingest (#3565) wants the exact video-download behavior the Dev
 * Tools downloader (#1946) already had — the format fallback chain, the merge/
 * remux flags, the `--progress-template` machine-readable markers, the
 * two-`--match-filters` duration bound, and the "find the produced file, because
 * the extension isn't knowable up front" logic — but as ONE step inside a larger
 * job rather than as its own SSE-owning service. The spawn/marker-parsing/exit
 * classification middle both importers share lives in ytdlpRun.js.
 *
 * Like the audio core, this is deliberately SSE-agnostic: it takes `onProgress`
 * and `registerProcess` callbacks and RETURNS an outcome. The caller owns the
 * job map, SSE broadcasting, terminal frames, and what happens to the produced
 * file. URL validation is the caller's job too, so the core never sees an
 * unvetted URL.
 */

import { readdir, unlink } from 'fs/promises';
import { join, dirname } from 'path';
import { ensureDir } from '../lib/fileUtils.js';
import { runYtDlp, ytdlpMarkerArgs, describeYtDlpFailure } from './ytdlpRun.js';

/**
 * Locate the final produced file for a download prefix. yt-dlp writes
 * intermediate streams as `<prefix>.f<code>.<ext>` and a `.part`/`.ytdl` in
 * progress, then merges/remuxes to a single `<prefix>.<ext>` and deletes the
 * rest. We do NOT hardcode `.mp4`: the format fallback chain can land a
 * single-file webm/mkv (VP9/AV1) that `--merge-output-format`/`--remux-video
 * mp4` only converts on a best-effort basis, so assuming `.mp4` would miss a
 * perfectly-good download and then delete it as a "failure". Prefer an exact
 * `.mp4`, else the lone remaining non-intermediate file.
 *
 * @returns {Promise<string|null>} basename, or null when nothing was produced.
 */
export async function findProducedFile(filePrefix, dir) {
  const prefix = `${filePrefix}.`;
  const entries = await readdir(dir).catch(() => []);
  const candidates = entries.filter((n) =>
    n.startsWith(prefix)
    && !n.endsWith('.part')
    && !n.endsWith('.ytdl')
    && !/\.f\d+\.[^.]+$/.test(n), // format-fragment intermediates (.f137.mp4)
  );
  return candidates.find((n) => n === `${prefix}mp4`) || candidates[0] || null;
}

/**
 * A cancelled/failed run can leave yt-dlp's pre-merge fragment files behind
 * (`<prefix>.f137.mp4`, `.part`, etc.), so cleanup globs every file this
 * prefix touched rather than unlinking one assumed path.
 */
export async function cleanupProducedFiles(filePrefix, dir) {
  const entries = await readdir(dir).catch(() => []);
  await Promise.all(
    entries.filter((name) => name.startsWith(filePrefix)).map((name) => unlink(join(dir, name)).catch(() => {})),
  );
}

/**
 * Download a full video (best mp4 video+audio, merged) into `outDir`.
 *
 * @param {object}   opts
 * @param {string}   opts.url             Already-validated source URL.
 * @param {string}   opts.ytDlp           yt-dlp binary path.
 * @param {string}   opts.ffmpeg          ffmpeg binary path (for merge/remux).
 * @param {string}   opts.outDir          Destination directory (created if missing).
 * @param {string}   opts.filePrefix      Unique per-job basename prefix (no extension).
 * @param {number}   opts.maxBytes        `--max-filesize` cap.
 * @param {number}   opts.maxDurationSec  `--match-filters duration<=` cap.
 * @param {function} opts.onProgress      ({ percent, stage }) => void — SSE-agnostic.
 * @param {function} opts.registerProcess (proc|null) => void — lets the caller wire cancel.
 * @returns {Promise<{ outcome:'complete'|'canceled'|'failed', filename?:string, title?:string, reason?:string }>}
 *
 * Never throws for a yt-dlp runtime failure (returns `failed` + `reason`); the
 * caller decides how to surface it. Cleans partial files on `canceled`/`failed`;
 * on `complete` the caller owns the produced file.
 */
export async function downloadVideoToDir({
  url, ytDlp, ffmpeg, outDir, filePrefix, maxBytes, maxDurationSec, onProgress, registerProcess,
}) {
  // On a fresh install the destination may not exist yet (setup-data.js doesn't
  // create data/videos, and no prior render/import may have); without this
  // yt-dlp points at a missing directory and fails before producing a file.
  await ensureDir(outDir);

  const args = [
    // Prefer an mp4 (h264/aac) video+audio pair that merges cleanly for broad
    // browser playback; fall back to best single-file mp4, then best.
    '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best',
    '--merge-output-format', 'mp4',
    // Best-effort remux a single-file/non-mp4 result into an mp4 container so
    // browser playback is broad; when the codec can't be remuxed losslessly
    // yt-dlp leaves the native container, which findProducedFile handles.
    '--remux-video', 'mp4',
    '--no-playlist',
    '--ffmpeg-location', dirname(ffmpeg),
    ...ytdlpMarkerArgs('merging'),
    '--max-filesize', String(maxBytes),
    // Two --match-filters are OR'd by yt-dlp: bound KNOWN-duration videos to the
    // cap, but let a post whose duration can't be resolved pre-download (common
    // on x.com/Twitter) through rather than silently rejecting it — the byte cap
    // above still bounds it. A known video longer than the cap matches neither
    // filter and is skipped.
    '--match-filters', `duration <= ${maxDurationSec}`,
    '--match-filters', '!duration',
    '-o', join(outDir, `${filePrefix}.%(ext)s`),
    url,
  ];

  const exit = await runYtDlp({ ytDlp, args, url, onProgress, registerProcess });

  if (exit.canceled) {
    await cleanupProducedFiles(filePrefix, outDir);
    return { outcome: 'canceled' };
  }

  const filename = await findProducedFile(filePrefix, outDir);
  if (exit.code !== 0 || !filename) {
    // A --match-filters/--max-filesize rejection exits 0 with no output file
    // (yt-dlp treats a filtered-out video as "nothing to do") — and --print
    // suppresses the specific reason, so name the known bounds. x.com/Twitter
    // downloads also fail here on login-walled/rate-limited content. Only the
    // bounds are ours; describeYtDlpFailure owns the prose and appends whatever
    // yt-dlp actually printed, which outranks the guess.
    const reason = exit.code === 0
      ? describeYtDlpFailure(exit.code, exit.output, {
        url,
        fallback: `no video was produced — it may be longer than ${Math.round(maxDurationSec / 60)} minutes or larger than ${Math.round(maxBytes / 1024 / 1024 / 1024)}GB, or (for x.com) login-walled, rate-limited, or otherwise unavailable`,
      })
      : exit.reason; // always populated for a non-zero exit (see runYtDlp)
    await cleanupProducedFiles(filePrefix, outDir);
    return { outcome: 'failed', reason };
  }

  return { outcome: 'complete', filename, title: exit.title };
}