Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

12.1 KB
Raw Blame
/**
 * Video downloader (#1946) — follow-up to #1945. Where trackYoutubeImport.js
 * extracts just the AUDIO track, this downloads the FULL video from a YouTube
 * or x.com/Twitter URL via yt-dlp, lands it under PATHS.videos as a
 * distinguishable `downloaded-<uuid>.mp4`, generates a thumbnail, and writes a
 * `source: 'download'` entry into the shared video-history store so it shows up
 * in the existing media library/gallery and gets picked up by the mediaAssetIndex
 * `videoGenEvents 'completed'` hook UNMODIFIED (issue design option (b) —
 * "write a lightweight video-history-like entry"; the `source` marker + filename
 * prefix keep the "derived, not a generation" framing honest without a new row
 * shape or a new index event).
 *
 * Mirrors trackYoutubeImport's job pattern exactly: kickoff returns a jobId
 * immediately, the download runs detached, progress streams over the shared SSE
 * helpers, and the client attaches with useSseProgress. A single yt-dlp
 * invocation does the whole job (best mp4 video+audio merged via ffmpeg).
 *
 * The yt-dlp invocation itself lives in ytdlpVideoImport.js — extracted so the
 * YouTube brain ingest can run the same download as one step of a bigger job
 * (the same split ytdlpAudioImport.js already made for the audio path). This
 * module keeps the download-specific concerns: the host allowlist, the SSE job
 * map, and the `source: 'download'` video-history entry.
 */

import { join } from 'path';
import { randomUUID } from 'crypto';
import { ServerError } from '../lib/errorHandler.js';
import { shortId, PATHS, unlinkGuarded } from '../lib/fileUtils.js';
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import { findFfmpeg, generateThumbnail, probeVideoDuration } from '../lib/ffmpeg.js';
import { findYtDlp } from '../lib/ytdlp.js';
import { killWithEscalation } from '../lib/killWithEscalation.js';
import { broadcastSse, attachSseClient as attachSse, closeJobAfterDelay } from '../lib/sseUtils.js';
import { downloadVideoToDir, cleanupProducedFiles } from './ytdlpVideoImport.js';
import { loadHistory, mutateVideoHistory } from './videoGen/history.js';
import { deleteHistoryItem } from './videoGen/local.js';
import { videoGenEvents } from './videoGen/events.js';

// Host allowlist: YouTube (watch/shorts/youtu.be) + x.com/Twitter status URLs.
// Same reasoning as #1945's YOUTUBE_URL_RE — constrains what a shelled-out
// yt-dlp will touch even though args are passed as an array (no shell
// interpolation), and gives the user a clear "unsupported host" error up front
// instead of a cryptic yt-dlp failure.
export const SUPPORTED_VIDEO_URL_RE =
  /^https?:\/\/(www\.|m\.|mobile\.)?(youtube\.com\/(watch\?[^\s#]*\bv=[\w-]{6,}|shorts\/[\w-]{6,})|youtu\.be\/[\w-]{6,}|(x|twitter)\.com\/[^\s/]+\/status\/\d+)/i;

export function assertSupportedVideoUrl(url) {
  if (typeof url !== 'string' || !SUPPORTED_VIDEO_URL_RE.test(url)) {
    throw new ServerError(
      'Unsupported video URL (expected a YouTube video/shorts, youtu.be, or x.com/twitter.com status URL)',
      { status: 400, code: 'VIDEO_URL_INVALID' },
    );
  }
}

// Bound resource use the same way #1945 does for audio: without these, a long
// video, livestream, or archive URL downloads unbounded, eating disk in
// data/videos and CPU with no cap. Generous but finite for full-video pulls.
export const VIDEO_DOWNLOAD_MAX_BYTES = 2 * 1024 * 1024 * 1024; // 2 GB
export const VIDEO_DOWNLOAD_MAX_DURATION_SEC = 60 * 60; // 1 hour

const FILENAME_PREFIX = 'downloaded-';

// jobId -> { id, status, clients, lastPayload, process }
const downloadJobs = new Map();

export const attachDownloadSseClient = (jobId, res) => attachSse(downloadJobs, jobId, res);

// The job map is module-private, which left cancelVideoDownload untestable — and
// an untested cancel path is exactly how a refactor shipped it referencing an
// unimported killWithEscalation while the whole suite stayed green. Exposing the
// map lets a test register a fake job and actually run the cancel.
export const __testing = { downloadJobs };

/** Cancel an in-flight download. Returns false if the job is unknown or already finished. */
export function cancelVideoDownload(jobId) {
  const job = downloadJobs.get(jobId);
  if (!job || !job.process) return false;
  const proc = job.process;
  killWithEscalation(proc, { label: 'yt-dlp download', stillRunning: () => job.process === proc });
  return true;
}

/** List downloaded videos — the `source: 'download'` slice of video-history, newest first. */
export async function listDownloads() {
  // No `.catch(() => [])`: that re-swallowed exactly what `loadHistory`'s strict
  // read exists to surface (#4115), turning an unreadable history into a
  // confident "you have no downloads". Let it bubble to the error middleware.
  const history = await loadHistory();
  return (Array.isArray(history) ? history : []).filter((h) => h?.source === 'download');
}

/**
 * Delete a downloaded video. Verifies the id is actually a download (not a
 * generated render) before delegating to the shared deleteHistoryItem, which
 * removes the file, thumbnail, history entry, and mediaAssetIndex row
 * (identical lifecycle to a deleted generation).
 */
export async function deleteDownload(id) {
  // Same reason as listDownloads: an unreadable history must not answer
  // "not found" (a 404 the user would read as "already gone").
  const history = await loadHistory();
  const item = (Array.isArray(history) ? history : []).find((h) => h?.id === id);
  if (!item || item.source !== 'download') {
    throw new ServerError('Downloaded video not found', { status: 404, code: 'NOT_FOUND' });
  }
  return deleteHistoryItem(id);
}

// A cancelled/failed run can leave yt-dlp's pre-merge fragment files behind
// (`downloaded-<uuid>.f137.mp4`, `.part`, etc.), so cleanup globs every file
// this job's prefix touched in PATHS.videos rather than unlinking one path.
const cleanupDownloadFiles = (jobId) => cleanupProducedFiles(`${FILENAME_PREFIX}${jobId}`, PATHS.videos);

// Build the `source: 'download'` video-history entry. Pure + exported so the
// load-bearing shape (the fields normalizeVideo, mediaAssetIndex videoToRow, and
// deleteHistoryItem all depend on) is pinned by a unit test rather than only
// implicitly exercised by the spawn path. `id === jobId` so the live media-index
// `completed` hook loads it by generationId AND deleteHistoryItem's `${id}.jpg`
// thumbnail + `${filename}` cleanup both resolve.
export function buildDownloadHistoryEntry({ jobId, filename, thumbnail, durationSec, title, sourceUrl }) {
  return {
    id: jobId,
    filename,
    thumbnail,
    createdAt: new Date().toISOString(),
    source: 'download',
    sourceUrl,
    title: title || 'Downloaded video',
    ...(durationSec != null ? { durationSec } : {}),
  };
}

/**
 * Download a video and land it in the shared media library.
 *
 * This — not the bare yt-dlp spawn — is the unit a second caller wants: it owns
 * the `downloaded-<id>` filename prefix, the thumbnail + duration probe, the
 * `source: 'download'` video-history entry, the media-index `completed` event,
 * and cleanup of partial files when any of that fails. Extracted so the YouTube
 * brain ingest lands a video EXACTLY the way the Dev Tools downloader does;
 * duplicating the tail is how the two silently drift.
 *
 * `outcome: 'canceled' | 'failed'` are returned rather than thrown (matching
 * `downloadVideoToDir`) so the caller decides how to surface them; anything that
 * fails AFTER a successful download throws, with the partial files already
 * cleaned up.
 *
 * `id` is the library id: the filename prefix, the thumbnail name and the
 * history-entry id all derive from it, so a caller that already has a job id
 * passes it to keep the two aligned (the SSE downloader does — its terminal
 * frame's `id` has always been the video's id).
 *
 * @returns {Promise<{ outcome:'complete'|'canceled'|'failed', entry?:object, reason?:string }>}
 */
export async function downloadVideoIntoLibrary({
  url, ytDlp, ffmpeg, id, maxBytes = VIDEO_DOWNLOAD_MAX_BYTES,
  maxDurationSec = VIDEO_DOWNLOAD_MAX_DURATION_SEC, onProgress, registerProcess,
}) {
  const jobId = id || randomUUID();
  let result;
  try {
    result = await downloadVideoToDir({
      url,
      ytDlp,
      ffmpeg,
      outDir: PATHS.videos,
      filePrefix: `${FILENAME_PREFIX}${jobId}`,
      maxBytes,
      maxDurationSec,
      onProgress,
      registerProcess,
    });
  } catch (err) {
    await cleanupDownloadFiles(jobId);
    throw err;
  }
  if (result.outcome !== 'complete') return result;
  // The long producer writes fresh, unreferenced bytes outside admission.
  // Its poster and the history commit (including rollback) share one lease.
  const entry = await withBackupAssetPublication(async () => {
    try {
      const { filename, title } = result;
      const outPath = join(PATHS.videos, filename);
      onProgress?.({ percent: 100, stage: 'finalizing' });
      const [thumbnail, durationSec] = await Promise.all([
        generateThumbnail(outPath, jobId),
        probeVideoDuration(outPath).catch(() => null),
      ]);
      if (!thumbnail) await unlinkGuarded(join(PATHS.videoThumbnails, `${jobId}.jpg`)).catch(() => {});

      // Derived, not a generation: a `source: 'download'` video-history entry so
      // the existing videoToRow / onVideoCompleted media-index path and the
      // gallery pick it up unmodified. Serialized read-modify-write so two
      // near-simultaneous downloads can't clobber each other's entry.
      const entry = buildDownloadHistoryEntry({ jobId, filename, thumbnail, durationSec, title, sourceUrl: url });
      await mutateVideoHistory((history) => { history.unshift(entry); return history; });

      return entry;
    } catch (err) {
      // A throw between the download and the history write would otherwise orphan
      // a multi-GB file in data/videos with nothing pointing at it.
      await cleanupDownloadFiles(jobId);
      await unlinkGuarded(join(PATHS.videoThumbnails, `${jobId}.jpg`)).catch(() => {});
      throw err;
    }
  });
  // Notify only after publication; a listener failure cannot remove committed bytes.
  videoGenEvents.emit('completed', { generationId: jobId, filename: entry.filename, path: `/data/videos/${entry.filename}`, thumbnail: entry.thumbnail });
  return { outcome: 'complete', entry };
}

/**
 * Kick off a full-video download. Returns `{ jobId }` immediately; the download
 * runs detached and streams progress over SSE. Terminal frames:
 * `{ type: 'complete', id, video }`, `{ type: 'error', error }`, or
 * `{ type: 'canceled' }`.
 */
export async function startVideoDownload(url) {
  assertSupportedVideoUrl(url);

  const [ytDlp, ffmpeg] = await Promise.all([findYtDlp(), findFfmpeg()]);
  if (!ytDlp) throw new ServerError('yt-dlp not found on PATH', { status: 500, code: 'YTDLP_MISSING' });
  if (!ffmpeg) throw new ServerError('ffmpeg not found on PATH', { status: 500, code: 'FFMPEG_MISSING' });

  const jobId = randomUUID();
  const job = { id: jobId, status: 'running', clients: [], process: null };
  downloadJobs.set(jobId, job);
  console.log(`📥 Video download ${shortId(jobId)} — ${url}`);

  (async () => {
    try {
      const result = await downloadVideoIntoLibrary({
        url,
        ytDlp,
        ffmpeg,
        id: jobId, // keep the library id == the SSE job id, as it has always been
        onProgress: ({ percent, stage }) => broadcastSse(job, { type: 'progress', percent, ...(stage ? { stage } : {}) }),
        registerProcess: (proc) => { job.process = proc; },
      });

      if (result.outcome === 'canceled') {
        console.log(`🛑 Video download ${shortId(jobId)} cancelled`);
        broadcastSse(job, { type: 'canceled' });
        return;
      }
      if (result.outcome === 'failed') throw new Error(result.reason);

      const { entry } = result;
      console.log(`📥 Video download ${shortId(jobId)} complete — "${entry.title}" (${entry.filename})`);
      broadcastSse(job, { type: 'complete', id: jobId, video: entry });
    } catch (err) {
      console.error(`❌ Video download ${shortId(jobId)} failed: ${err?.message || err}`);
      broadcastSse(job, { type: 'error', error: err?.message || String(err) });
    } finally {
      closeJobAfterDelay(downloadJobs, jobId);
    }
  })();

  return { jobId };
}