Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Shared yt-dlp FULL-VIDEO download core.
*
* The mirror image of ytdlpAudioImport.js, extracted for the same reason: the
* YouTube brain ingest (#3565) wants the exact video-download behavior the Dev
* Tools downloader (#1946) already had — the format fallback chain, the merge/
* remux flags, the `--progress-template` machine-readable markers, the
* two-`--match-filters` duration bound, and the "find the produced file, because
* the extension isn't knowable up front" logic — but as ONE step inside a larger
* job rather than as its own SSE-owning service. The spawn/marker-parsing/exit
* classification middle both importers share lives in ytdlpRun.js.
*
* Like the audio core, this is deliberately SSE-agnostic: it takes `onProgress`
* and `registerProcess` callbacks and RETURNS an outcome. The caller owns the
* job map, SSE broadcasting, terminal frames, and what happens to the produced
* file. URL validation is the caller's job too, so the core never sees an
* unvetted URL.
*/
import { readdir, unlink } from 'fs/promises';
import { join, dirname } from 'path';
import { ensureDir } from '../lib/fileUtils.js';
import { runYtDlp, ytdlpMarkerArgs, describeYtDlpFailure } from './ytdlpRun.js';
/**
* Locate the final produced file for a download prefix. yt-dlp writes
* intermediate streams as `<prefix>.f<code>.<ext>` and a `.part`/`.ytdl` in
* progress, then merges/remuxes to a single `<prefix>.<ext>` and deletes the
* rest. We do NOT hardcode `.mp4`: the format fallback chain can land a
* single-file webm/mkv (VP9/AV1) that `--merge-output-format`/`--remux-video
* mp4` only converts on a best-effort basis, so assuming `.mp4` would miss a
* perfectly-good download and then delete it as a "failure". Prefer an exact
* `.mp4`, else the lone remaining non-intermediate file.
*
* @returns {Promise<string|null>} basename, or null when nothing was produced.
*/
export async function findProducedFile(filePrefix, dir) {
const prefix = `${filePrefix}.`;
const entries = await readdir(dir).catch(() => []);
const candidates = entries.filter((n) =>
n.startsWith(prefix)
&& !n.endsWith('.part')
&& !n.endsWith('.ytdl')
&& !/\.f\d+\.[^.]+$/.test(n), // format-fragment intermediates (.f137.mp4)
);
return candidates.find((n) => n === `${prefix}mp4`) || candidates[0] || null;
}
/**
* A cancelled/failed run can leave yt-dlp's pre-merge fragment files behind
* (`<prefix>.f137.mp4`, `.part`, etc.), so cleanup globs every file this
* prefix touched rather than unlinking one assumed path.
*/
export async function cleanupProducedFiles(filePrefix, dir) {
const entries = await readdir(dir).catch(() => []);
await Promise.all(
entries.filter((name) => name.startsWith(filePrefix)).map((name) => unlink(join(dir, name)).catch(() => {})),
);
}
/**
* Download a full video (best mp4 video+audio, merged) into `outDir`.
*
* @param {object} opts
* @param {string} opts.url Already-validated source URL.
* @param {string} opts.ytDlp yt-dlp binary path.
* @param {string} opts.ffmpeg ffmpeg binary path (for merge/remux).
* @param {string} opts.outDir Destination directory (created if missing).
* @param {string} opts.filePrefix Unique per-job basename prefix (no extension).
* @param {number} opts.maxBytes `--max-filesize` cap.
* @param {number} opts.maxDurationSec `--match-filters duration<=` cap.
* @param {function} opts.onProgress ({ percent, stage }) => void — SSE-agnostic.
* @param {function} opts.registerProcess (proc|null) => void — lets the caller wire cancel.
* @returns {Promise<{ outcome:'complete'|'canceled'|'failed', filename?:string, title?:string, reason?:string }>}
*
* Never throws for a yt-dlp runtime failure (returns `failed` + `reason`); the
* caller decides how to surface it. Cleans partial files on `canceled`/`failed`;
* on `complete` the caller owns the produced file.
*/
export async function downloadVideoToDir({
url, ytDlp, ffmpeg, outDir, filePrefix, maxBytes, maxDurationSec, onProgress, registerProcess,
}) {
// On a fresh install the destination may not exist yet (setup-data.js doesn't
// create data/videos, and no prior render/import may have); without this
// yt-dlp points at a missing directory and fails before producing a file.
await ensureDir(outDir);
const args = [
// Prefer an mp4 (h264/aac) video+audio pair that merges cleanly for broad
// browser playback; fall back to best single-file mp4, then best.
'-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best',
'--merge-output-format', 'mp4',
// Best-effort remux a single-file/non-mp4 result into an mp4 container so
// browser playback is broad; when the codec can't be remuxed losslessly
// yt-dlp leaves the native container, which findProducedFile handles.
'--remux-video', 'mp4',
'--no-playlist',
'--ffmpeg-location', dirname(ffmpeg),
...ytdlpMarkerArgs('merging'),
'--max-filesize', String(maxBytes),
// Two --match-filters are OR'd by yt-dlp: bound KNOWN-duration videos to the
// cap, but let a post whose duration can't be resolved pre-download (common
// on x.com/Twitter) through rather than silently rejecting it — the byte cap
// above still bounds it. A known video longer than the cap matches neither
// filter and is skipped.
'--match-filters', `duration <= ${maxDurationSec}`,
'--match-filters', '!duration',
'-o', join(outDir, `${filePrefix}.%(ext)s`),
url,
];
const exit = await runYtDlp({ ytDlp, args, url, onProgress, registerProcess });
if (exit.canceled) {
await cleanupProducedFiles(filePrefix, outDir);
return { outcome: 'canceled' };
}
const filename = await findProducedFile(filePrefix, outDir);
if (exit.code !== 0 || !filename) {
// A --match-filters/--max-filesize rejection exits 0 with no output file
// (yt-dlp treats a filtered-out video as "nothing to do") — and --print
// suppresses the specific reason, so name the known bounds. x.com/Twitter
// downloads also fail here on login-walled/rate-limited content. Only the
// bounds are ours; describeYtDlpFailure owns the prose and appends whatever
// yt-dlp actually printed, which outranks the guess.
const reason = exit.code === 0
? describeYtDlpFailure(exit.code, exit.output, {
url,
fallback: `no video was produced — it may be longer than ${Math.round(maxDurationSec / 60)} minutes or larger than ${Math.round(maxBytes / 1024 / 1024 / 1024)}GB, or (for x.com) login-walled, rate-limited, or otherwise unavailable`,
})
: exit.reason; // always populated for a non-zero exit (see runYtDlp)
await cleanupProducedFiles(filePrefix, outDir);
return { outcome: 'failed', reason };
}
return { outcome: 'complete', filename, title: exit.title };
}