Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { execFile } from '../../lib/childProcess.js';
import { promisify } from 'util';
import { join, basename } from 'path';
import { tmpdir } from 'os';
import { unlinkGuarded } from '../../lib/fileUtils.js';
import { ServerError } from '../../lib/errorHandler.js';
import { maintenance } from '../../lib/maintenanceAdmission.js';
import { findFfmpeg, findFfprobe } from '../../lib/ffmpeg.js';
import { safeChildProcessOptions } from '../../lib/processEnv.js';
import { isIcLoraMode, icLoraSpecForMode } from '../../lib/icLoraWeights.js';
import { modelAnchorsLastFrame, routesToWindowsHelper } from './runtimes.js';
import { IC_STILL_REFERENCE_FRAMES } from './renderArgs.js';
const execFileAsync = promisify(execFile);
// LTX-2.x was trained on video frames, so Lightricks' pipelines re-compress a
// conditioning still through H.264 before encoding it (CRF 33 for LTX-2.0 to
// 2.3, `DEFAULT_IMAGE_CRF` in ltx_pipelines). A pristine still is out of
// distribution: the model holds it for the first frame and then cuts away to
// a different scene. The dgrauet/MrMofer runtimes do this round trip in their
// own loader; the notapalindrome `mlx_video.generate_av` CLI does not, so
// PortOS does it here for that runtime.
const CONDITIONING_CRF_BY_RUNTIME = Object.freeze({ mlx_video: 33 });
/** Resolve local conditioning media and return its existing cleanup capability.
* Derived files are always owned here; upload/audio cleanup stays caller-selected.
*/
export async function prepareVideoConditioningMedia({
model, mode, sourceImagePath, lastImagePath, keyframes, icReferencePaths,
w, h, parsedFps, jobId, uploadedTempPath, uploadedTempPaths, audioFilePath,
}) {
// Resize conditioning images to match the model resolution. mlx_video and
// ltx2 both require exact dimensions (they don't auto-pad), and pixie-forge
// learned the hard way that letting the model upscale a portrait reference
// makes garbled output.
//
// Skip the last-image resize when buildArgs / the Python child won't
// actually consume it:
// - A last-frame-anchored runtime (see LAST_FRAME_ANCHORED_RUNTIMES) really
// consumes both frames — ltx2 via --image/--last-image, MiniMax H3 via
// --image/--anchor pairs — so resize the last frame even when a source
// image is also present.
// - On macOS/mlx_video the FFLF fallback only consumes the last image when
// no source image is also provided (single conditioning frame only).
// Anything else is a no-op, so resizing is wasted ffmpeg work.
// - A model that routes to generate_win.py takes --last-image only so the
// script can log status; the LTX-Video 0.9.5 pipeline reads --image
// alone, so it never opens the last-frame file. That gate keys on the
// RUNNER, not on the platform (see routesToWindowsHelper): both Windows
// and Linux have BYOV runtimes whose helper genuinely anchors the last
// frame, and a bare platform check would hand them an unresized frame.
const lastImageWillBeUsed = !!lastImagePath && !routesToWindowsHelper(model) && mode === 'fflf'
&& (modelAnchorsLastFrame(model) || !sourceImagePath);
// A non-null `keyframes` that ISN'T a length-≥2 array is malformed —
// fail fast instead of silently dropping it (which would produce an
// unexpected text/i2v render with the user's anchors ignored). The
// route guarantees the array shape, but non-route callers (tests,
// persisted queue replays) could pass a stray scalar/empty array.
if (keyframes != null && !(Array.isArray(keyframes) && keyframes.length >= 2)) {
throw new ServerError(
`keyframes must be null OR an array of length >= 2; got ${Array.isArray(keyframes) ? `array(length=${keyframes.length})` : typeof keyframes}`,
{ status: 400, code: 'KEYFRAME_INVALID_SHAPE' },
);
}
const hasMultiKeyframes = Array.isArray(keyframes) && keyframes.length >= 2;
if (hasMultiKeyframes) {
// The route validates shape, but a non-route caller (test, persisted
// queue replay, future internal API) could pass malformed entries.
// Validate before any resize subprocess starts so a rejection cannot
// strand temp files created for the other conditioning inputs.
keyframes.forEach((kf, i) => {
if (!kf || typeof kf !== 'object') {
throw new ServerError(`keyframes[${i}] must be an object: got ${typeof kf}`, { status: 400, code: 'KEYFRAME_INVALID_SHAPE' });
}
if (typeof kf.path !== 'string' || !kf.path) {
throw new ServerError(`keyframes[${i}].path must be a non-empty string`, { status: 400, code: 'KEYFRAME_INVALID_SHAPE' });
}
// The Python helper enforces `index` is an int; a float or numeric
// string here would crash mid-render. Coerce + verify integerness
// up-front so non-route callers (tests, persisted queue replays)
// get a clear 400 instead of a Python traceback.
const n = Number(kf.index);
if (!Number.isInteger(n)) {
throw new ServerError(`keyframes[${i}].index must be an integer: got ${kf.index}`, { status: 400, code: 'KEYFRAME_INVALID_SHAPE' });
}
});
}
const ffmpeg = (sourceImagePath || lastImageWillBeUsed || hasMultiKeyframes) ? await findFfmpeg() : null;
const ffprobe = model.runtime === 'minimax_h3_ref2va' ? await findFfprobe() : null;
// Keyed on the runner, not just the runtime name: a model that routes to
// generate_win.py also reads as mlx_video, but that pipeline is LTX-Video 0.9.
const conditioningCrf = routesToWindowsHelper(model) ? null : (CONDITIONING_CRF_BY_RUNTIME[model.runtime || 'mlx_video'] ?? null);
const resizeImage = async (srcPath, tag) => {
if (!srcPath || !ffmpeg) return { resolved: srcPath, tempPath: null };
const resizedPath = join(tmpdir(), `resized-${tag}-${jobId}.png`);
const resizeResult = await execFileAsync(ffmpeg, [
'-i', srcPath,
'-vf', `scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`,
'-update', '1', '-frames:v', '1',
'-y', resizedPath,
], safeChildProcessOptions({ timeout: 10000 })).catch((err) => ({ error: err }));
if (resizeResult.error) {
console.log(`⚠️ Failed to resize ${tag} image, using original: ${resizeResult.error.message}`);
return { resolved: srcPath, tempPath: null };
}
if (conditioningCrf != null) await recompressAsVideoFrame(ffmpeg, resizedPath, conditioningCrf, `${tag}-${jobId}`);
return { resolved: resizedPath, tempPath: resizedPath };
};
// Two independent ffmpeg spawns — fan out for the same reason the keyframe
// loop below does, so a true-FFLF render doesn't pay them back to back.
const [
{ resolved: resolvedSourceImage, tempPath: resizedSrcTempPath },
{ resolved: resolvedLastImage, tempPath: resizedLastTempPath },
] = await Promise.all([
resizeImage(sourceImagePath, 'src'),
lastImageWillBeUsed
? resizeImage(lastImagePath, 'last')
: { resolved: lastImagePath, tempPath: null },
]);
// Resize each multi-keyframe image to the target resolution (the helper
// requires exact W×H, same as i2v). Indices pass through unchanged.
// Each ffmpeg subprocess is independent — fan out so 8 keyframes don't
// serialize behind 7 unrelated ffmpeg startups.
const resizedKeyframeTempPaths = [];
let resolvedKeyframes = null;
if (hasMultiKeyframes) {
const results = await Promise.all(keyframes.map((kf, i) => resizeImage(kf.path, `kf${i}`)));
resolvedKeyframes = results.map((r, i) => {
if (r.tempPath) resizedKeyframeTempPaths.push(r.tempPath);
// Normalize index to a real Number so the JSON we hand to the
// Python helper is unambiguous (no '5' string sneaking through
// from a multipart form).
return { path: r.resolved, index: Number(keyframes[i].index) };
});
}
// An `image`-kind IC weight (Ingredients) takes STILLS, but the pipeline's
// reference channel is a video encoder end-to-end: iclora_utils probes the
// reference with ffprobe and feeds it to the VAE, which requires a (1 + 8k)
// frame count. A bare PNG has neither a probeable frame count nor 9 frames, so
// materialize each still into a tiny 9-frame constant clip at the render
// resolution first. 9 = the smallest legal (1 + 8k) count, so this is the
// cheapest possible encode and every frame is identical — the reference is a
// still either way.
//
// Done here rather than in the route because the target resolution is only
// known after the resolution flooring in the renderer, and it mirrors resizeImage's contract:
// temp paths are tracked for the same cleanup sites.
const icReferenceTempPaths = [];
// Both throw sites in this block land BEFORE the renderer's buildArgs try/catch
// (whose catch is what normally unlinks resizedSrcTempPath/
// resizedLastTempPath/resizedKeyframeTempPaths/icReferenceTempPaths) — a
// throw here escapes uncaught by that cleanup, and the caller's outer catch
// only unlinks the route-level upload/audio temp files, not these
// internally-created resize/still-clip temp files. Clean them up explicitly
// before either throw so a missing ffmpeg or a failed still-encode doesn't
// leak every resized temp file for the request into os.tmpdir().
const cleanupTempFiles = ({ includeUploads = false, includeUntrackedAudio = false } = {}) => {
const paths = [
resizedSrcTempPath,
resizedLastTempPath,
...resizedKeyframeTempPaths,
...icReferenceTempPaths,
...(includeUploads ? [uploadedTempPath, ...uploadedTempPaths] : []),
...(includeUntrackedAudio && audioFilePath && !uploadedTempPaths.includes(audioFilePath)
? [audioFilePath]
: []),
];
return Promise.all(paths.filter(Boolean).map((path) => unlinkGuarded(path).catch(error => {
if (error.code !== 'ENOENT') {
maintenance.markCurrentUnsettled();
maintenance.markResourceUnsettled('media', jobId);
}
})));
};
const resolvedIcReferencePaths = await materializeIngredientsReferences({
mode, icReferencePaths, ffmpeg, w, h, parsedFps, jobId,
icReferenceTempPaths, cleanupTempFiles,
});
return {
resolvedSourceImage, resolvedLastImage, resolvedKeyframes, resolvedIcReferencePaths,
hasMultiKeyframes, ffmpeg, ffprobe, cleanupTempFiles,
};
}
/** Materialize Ingredients stills, settling every encode before failure cleanup. */
async function materializeIngredientsReferences({
mode, icReferencePaths, ffmpeg, w, h, parsedFps, jobId,
icReferenceTempPaths, cleanupTempFiles,
}) {
if (isIcLoraMode(mode) && icLoraSpecForMode(mode)?.referenceKind === 'image'
&& Array.isArray(icReferencePaths) && icReferencePaths.length) {
const stillFfmpeg = ffmpeg || await findFfmpeg();
if (!stillFfmpeg) {
await cleanupTempFiles();
throw new ServerError(
'ffmpeg is required to prepare still references for Ingredients mode — install it (brew install ffmpeg) and retry.',
{ status: 400, code: 'IC_LORA_STILL_NEEDS_FFMPEG' },
);
}
// Register EVERY target path up-front, before any encode starts, and settle
// all of them before deciding. `Promise.all` + push-on-success would reject
// at the first failure while sibling encodes were still in flight, so their
// files would land after cleanup already ran and leak. Registering the paths
// eagerly also means the outer error/close handlers can clean up regardless
// of which encodes finished.
const clipPaths = icReferencePaths.map((_, i) => join(tmpdir(), `ic-still-${i}-${jobId}.mp4`));
icReferenceTempPaths.push(...clipPaths);
const encodes = await Promise.all(icReferencePaths.map((stillPath, i) => execFileAsync(stillFfmpeg, [
'-loop', '1', '-i', stillPath,
'-vf', `scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`,
'-frames:v', String(IC_STILL_REFERENCE_FRAMES),
'-r', String(parsedFps),
'-pix_fmt', 'yuv420p', '-an',
'-y', clipPaths[i],
], safeChildProcessOptions({ timeout: 30000 })).catch((err) => ({ error: err }))));
const failedAt = encodes.findIndex((r) => r?.error);
if (failedAt !== -1) {
// Unlike the resizeImage fallback (which degrades to the original), there is
// no usable degradation here — a still handed straight to the pipeline fails
// deep inside the VAE reshape. Fail loudly with the ffmpeg reason.
// cleanupTempFiles() also unlinks clipPaths (already pushed into
// icReferenceTempPaths above), plus any earlier resize temp files.
await cleanupTempFiles();
throw new ServerError(
`Failed to prepare Ingredients reference ${basename(icReferencePaths[failedAt])}: ${encodes[failedAt].error.message}`,
{ status: 400, code: 'IC_LORA_STILL_PREP_FAILED' },
);
}
return clipPaths;
}
return icReferencePaths;
}
/** Round-trip a still through one H.264 frame at `crf`, in place (see CONDITIONING_CRF_BY_RUNTIME). */
async function recompressAsVideoFrame(ffmpeg, pngPath, crf, tag) {
const framePath = join(tmpdir(), `crf-${tag}.mp4`);
const options = safeChildProcessOptions({ timeout: 10000 });
const failed = await execFileAsync(ffmpeg, [
'-i', pngPath, '-frames:v', '1', '-c:v', 'libx264', '-preset', 'veryfast', '-crf', String(crf),
'-pix_fmt', 'yuv420p', '-y', framePath,
], options)
.then(() => execFileAsync(ffmpeg, ['-i', framePath, '-frames:v', '1', '-update', '1', '-y', pngPath], options))
.then(() => null, (err) => err);
await unlinkGuarded(framePath).catch(() => {});
// A still that keeps its pristine pixels is still a valid conditioning, so
// a failed round trip degrades to the old behavior rather than the render.
if (failed) console.warn(`⚠️ Could not re-compress the ${tag} conditioning frame: ${failed.message}`);
}