Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';
/**
* Generative video upscale — the media-job lifecycle (#6511).
*
* The Lanczos pass is an inline ffmpeg filter, so `upscaleVideo.js` runs it
* during the request. A generative pass is a multi-minute GPU render, so it
* goes through `mediaJobQueue` instead: `enqueueLtxUpscale` validates and
* queues, `runVideoUpscale` is what the queue worker invokes, and `cancel` is
* the hook `cancelJob` reaches through `getGenModuleForJob`.
*
* Two contracts govern everything here:
*
* - **The source is never touched.** Cancellation, a probe failure, a render
* failure and a mux failure all leave the source clip and its history row
* byte-identical and remove every partial output. That mirrors the Lanczos
* path's copy-then-transform shape: the deliverable is a NEW file and a NEW
* history row, or it does not exist at all.
* - **Local only.** `LTX_UPSCALE_JOB_KIND` is deliberately absent from the
* federation layer's kind maps, so the source video cannot be offered to a
* peer. See the privacy rules in AGENTS.md.
*/
import { existsSync } from 'fs';
import { randomUUID, randomInt } from 'crypto';
import { join } from 'path';
import { tmpdir } from 'os';
import { PATHS, unlinkGuarded } from '../../lib/fileUtils.js';
import { ServerError } from '../../lib/errorHandler.js';
import { maintenance } from '../../lib/maintenanceAdmission.js';
import {
safeUnder, generateThumbnail, probeVideoStreamInfo, probeVideoDuration,
} from '../../lib/ffmpeg.js';
import { createLineReader } from '../../lib/streamLines.js';
import { spawnDetached } from '../../lib/detachedSpawn.js';
import { killWithEscalation } from '../../lib/killWithEscalation.js';
import { safeChildProcessEnv } from '../../lib/processEnv.js';
import { PYTHON_NOISE_RE } from '../../lib/sseUtils.js';
import { omitRenderTiming, renderTimingFields } from '../../lib/renderTiming.js';
import { resolveIcLoraWeightByKey } from '../../lib/icLoraWeights.js';
import { isHardwareCompatible, hardwareUnavailableReason } from '../../lib/systemCapabilities.js';
import { hfChildEnv } from '../hfToken.js';
import { enqueueJob } from '../mediaJobQueue/index.js';
import { videoGenEvents } from './events.js';
import {
RUNTIME_LINE_PREFIX, parseRuntimeFingerprintLine, formatRuntimeFingerprint,
LORA_STRATEGY_LINE_PREFIX, parseLoraStrategyLine,
} from './generateVideoHelpers.js';
import { getHistoryItem, loadHistory, mutateVideoHistory } from './history.js';
import { buildArgs } from './renderArgs.js';
import { runtimeIsCacheOnly, runtimeNeedsProcessGroupKill } from './runtimes.js';
import { padSourceForUpscale, finalizeUpscaleOutput } from './upscaleFfmpeg.js';
import { planUpscaleHistoryItem } from './upscaleVideo.js';
import {
LTX_UPSCALE_JOB_KIND, LTX_UPSCALE_WEIGHT_KEY, UPSCALE_SCALE,
ltxAlignmentBlocker, ltxAlignmentIsNoop,
} from './upscalePlan.js';
const MAX_SEED = 2 ** 32 - 1;
const STAGE_RE = /^STAGE:\s*(\S+)\s*(.*)$/;
// Both runners emit these once, before the pipeline loads: the runtime
// fingerprint (parsed by the same helper the render path uses) and the
// adapter's `reference_downscale_factor` read off the weight. They are
// provenance, so they are captured here and written onto the history row —
// a render can then be tied to the exact ltx/mlx/torch + chip stack and the
// factor it enforced, the way a plain render's row carries `runtime`.
const REFERENCE_DOWNSCALE_RE = /^UPSCALE_REFERENCE_DOWNSCALE:\s*(\d+)\s*$/;
const STDERR_TAIL_LINES = 20;
// jobId → the in-flight run's cancel handles. Module-local rather than the
// shared `videoJobState`, because an upscale child is not a render child: the
// GPU lane already serializes the two, and mixing them would let a plain
// `videoGen.cancel()` kill an upscale (and vice versa) with no job id to
// discriminate on.
const activeUpscales = new Map();
// Same suffix convention the Lanczos pass uses, so the gallery reads
// "<original prompt> (2×)" whichever method produced the row.
const upscaledPrompt = (prompt) => (prompt ? `${prompt} (2×)` : '(upscaled 2×)');
/**
* Validate an upscale request and queue it. Everything that can be known
* before a GPU is committed is checked HERE — an unsupported/uninstalled
* runtime, a missing adapter, a missing base checkpoint, and a source the model
* grid cannot take without losing duration — so a refusal is an immediate
* 4xx/501 rather than a job that fails minutes later.
*/
export async function enqueueLtxUpscale(historyId) {
const plan = await planUpscaleHistoryItem(historyId, { method: 'ltx' });
if (plan.alreadyUpscaled) {
throw new ServerError('Cannot upscale an already-upscaled video', { status: 400, code: 'ALREADY_UPSCALED' });
}
if (!plan.runtime.supported || !plan.runtime.installed) {
throw new ServerError(
`Generative upscale is not available on this install: ${plan.runtime.reason}`,
{ status: 501, code: 'UNSUPPORTED_RUNTIME' },
);
}
// The host gate (#6537), BEFORE the cache checks below: a machine that cannot
// run the pack must not be told to download 72 GB of it. `generateVideo.js`
// already refuses a plain render here; an upscale renders ABOVE a plain
// render's peak, so anything it refuses this must refuse too — same helper on
// the same annotation, so the two gates cannot drift. `isHardwareCompatible`
// refuses only an explicit `unavailable` verdict, so a requirement this host
// could not MEASURE stays allowed rather than becoming a refusal invented from
// a missing number, and a plan from an older server (no annotation at all)
// queues exactly what it queues today.
//
// Deliberately NOT the `MODEL_HARDWARE_UNAVAILABLE`/400 those render sites
// raise: there, the user CHOSE an incompatible model, so the request is at
// fault. The upscale's checkpoint is pinned and not part of the request
// (#6511), so nothing the caller sent is wrong — the host simply cannot carry
// the method, which is what `UNSUPPORTED_RUNTIME`/501 already means here.
if (!isHardwareCompatible(plan.baseModel?.hardwareCompatibility)) {
throw new ServerError(
`Generative upscale is not available on this install: ${
hardwareUnavailableReason(plan.baseModel?.name || 'The LTX-2.5 pack', plan.baseModel?.hardwareCompatibility)}`,
{ status: 501, code: 'UNSUPPORTED_RUNTIME' },
);
}
// Cache-only resolve (#6508): a `requiresPreDownload` weight never falls back
// to a repo id, so an un-cached adapter is a refusal here rather than a
// silent multi-hundred-MB pull inside the render.
const adapter = await resolveIcLoraWeightByKey(LTX_UPSCALE_WEIGHT_KEY);
if (!adapter?.path) {
throw new ServerError(
`${plan.adapter.label} is not downloaded — download it from the model panel before upscaling.`,
{ status: 400, code: 'IC_LORA_WEIGHT_UNRESOLVED' },
);
}
// The runner is never allowed to resolve its own checkpoint: every LTX loader
// falls back to `snapshot_download` for a path it cannot stat, which for this
// pack is an unannounced ~68 GB pull inside a render. Resolved here, cache-only,
// so a missing pack is a refusal before the GPU is committed.
if (!plan.baseModel?.cached || !plan.baseModel.path) {
throw new ServerError(
plan.baseModel?.reason || 'The LTX-2.5 model pack for this backend is not downloaded.',
{ status: 400, code: 'UPSCALE_BASE_MODEL_UNRESOLVED' },
);
}
const blocker = ltxAlignmentBlocker(plan.alignment);
if (blocker) {
throw new ServerError(blocker, { status: 400, code: 'UPSCALE_SOURCE_UNALIGNABLE' });
}
// The frame rate is not part of the spatial grid, but the plan cannot be
// APPLIED without it: it sets the duration of the padded tail and is a
// required runner argument. An unmeasured rate is the same refusal.
if (!(Number(plan.source.fps) > 0)) {
throw new ServerError(
'The source frame rate could not be measured, so the padded tail and the render frame rate cannot be set.',
{ status: 400, code: 'UPSCALE_SOURCE_UNALIGNABLE' },
);
}
// Read back for the two fields the plan projection deliberately omits (it is
// a geometry/capability disclosure, not a row dump).
const item = await getHistoryItem(plan.id);
if (!item) throw new ServerError('Video not found', { status: 404, code: 'NOT_FOUND' });
return enqueueJob({
kind: LTX_UPSCALE_JOB_KIND,
params: {
historyId: plan.id,
// The FILENAME, not an absolute path: job params are persisted and
// replayed, and `runVideoUpscale` re-resolves them through `safeUnder`
// so a hand-edited media-jobs.json cannot point the render (or the
// audio re-mux) at a file outside data/videos.
sourceFilename: item.filename,
method: 'ltx',
runtime: plan.runtime.id,
adapterKey: plan.adapter.key,
adapterPath: adapter.path,
// The resolved snapshot directory, not the repo id — see the refusal above.
baseModelPath: plan.baseModel.path,
// Explicit, from the weight registry — the Python helper is never allowed
// to carry its own copy of these bounds (see buildLtxUpscaleArgs).
icMinReferences: adapter.spec.minReferences,
icMaxReferences: adapter.spec.maxReferences,
seed: randomInt(0, MAX_SEED + 1),
source: plan.source,
alignment: plan.alignment,
target: plan.target,
// Projected by sanitizeJob, so the Render Queue names what is being
// upscaled instead of showing an unlabeled row.
prompt: upscaledPrompt(item.prompt),
width: plan.target.width,
height: plan.target.height,
numFrames: plan.target.frameCount,
fps: plan.source.fps,
},
});
}
// Cancel the running upscale for `jobId`. Called by mediaJobQueue's cancelJob
// and by its idle watchdog; both then read the resulting child death as the
// job's terminal state.
export function cancel(jobId) {
const entry = activeUpscales.get(jobId);
if (!entry) return false;
entry.canceled = true;
entry.abort.abort();
if (entry.proc) {
killWithEscalation(entry.proc, {
label: 'video upscale child',
stillRunning: () => activeUpscales.get(jobId)?.proc === entry.proc,
});
}
return true;
}
// Build the child env for the upscale runner, mirroring the render path: no
// inherited PYTHONPATH (the venv owns its site-packages), unbuffered stdio so
// STAGE lines reach the watchdog as they happen, and no ambient HF credential
// for a cache-only runtime that must never reach the network.
const upscaleChildEnv = async (runtime) => {
const cacheOnly = runtimeIsCacheOnly(runtime);
const env = cacheOnly ? safeChildProcessEnv() : await hfChildEnv();
delete env.PYTHONPATH;
env.PYTHONUNBUFFERED = '1';
if (cacheOnly) {
delete env.HF_TOKEN;
delete env.HUGGING_FACE_HUB_TOKEN;
env.HF_HUB_DISABLE_IMPLICIT_TOKEN = '1';
env.HF_HUB_OFFLINE = '1';
env.TRANSFORMERS_OFFLINE = '1';
}
return env;
};
/**
* Spawn the upscale runner and settle when it exits.
*
* Resolves `{ ok }` rather than rejecting so every terminal path in
* `runVideoUpscale` converges on the same cleanup. Non-noise child output is
* forwarded as `activity` so the queue's idle watchdog measures real silence,
* and `STAGE:` markers become status frames the UI can name.
*/
const runUpscaleChild = async ({ jobId, bin, args, runtime, entry }) => {
// A cancel that landed while the alignment pass was still running has no
// child to kill yet, so it is honored HERE rather than letting the runner
// start and burn a GPU on work nobody is waiting for.
if (entry.canceled) return { ok: false, reason: 'canceled before the runner started' };
const proc = await spawnDetached(bin, args, {
env: await upscaleChildEnv(runtime),
controlDir: join(PATHS.videos, '.detached', jobId),
cleanup: true,
killProcessGroup: runtimeNeedsProcessGroupKill(runtime),
});
entry.proc = proc;
// …and a cancel that landed during the spawn itself: the handle exists now,
// so signal it immediately instead of waiting for a render nobody wants.
if (entry.canceled) cancel(jobId);
const stderrTail = [];
const provenance = { runtime: null, referenceDownscale: null, loraStrategy: null };
const onLine = (line, isStderr) => {
const text = String(line).trim();
if (!text || PYTHON_NOISE_RE.test(text)) return;
if (isStderr) {
stderrTail.push(text);
if (stderrTail.length > STDERR_TAIL_LINES) stderrTail.shift();
}
videoGenEvents.emit('activity', { generationId: jobId });
if (text.startsWith(RUNTIME_LINE_PREFIX)) {
// A malformed payload stays in the stderr tail rather than becoming a
// half-parsed fingerprint on the row.
const fp = parseRuntimeFingerprintLine(text);
if (!fp) return;
provenance.runtime = fp;
console.log(`🏷️ runtime [${jobId.slice(0, 8)}] ${formatRuntimeFingerprint(fp) || '?'}`);
return;
}
if (text.startsWith(LORA_STRATEGY_LINE_PREFIX)) {
const report = parseLoraStrategyLine(text);
if (report) provenance.loraStrategy = report;
return;
}
const downscale = REFERENCE_DOWNSCALE_RE.exec(text);
if (downscale) {
provenance.referenceDownscale = Number(downscale[1]);
return;
}
const stage = STAGE_RE.exec(text);
if (stage) {
videoGenEvents.emit('status', { generationId: jobId, phase: stage[1], message: stage[2] || stage[1] });
}
};
const outReader = createLineReader((l) => onLine(l, false));
const errReader = createLineReader((l) => onLine(l, true));
proc.stdout.on('data', (c) => outReader.push(c));
proc.stderr.on('data', (c) => errReader.push(c));
return new Promise((resolve) => {
let settled = false;
let processError;
const settle = (result) => {
if (settled) return;
settled = true;
outReader.flush();
errReader.flush();
resolve(result);
};
proc.on('error', (err) => {
processError = err;
if (!proc.pid) settle({ ok: false, reason: `spawn failed: ${err.message}` });
});
proc.on('close', (code, signal) => {
if (processError) { settle({ ok: false, reason: `upscale process failed: ${processError.message}` }); return; }
if (code === 0) { settle({ ok: true, ...provenance }); return; }
const tail = stderrTail.slice(-4).join(' | ');
const how = signal ? `killed (${signal})` : `exit ${code}`;
settle({ ok: false, reason: tail ? `upscale runner ${how}: ${tail}` : `upscale runner ${how}` });
});
});
};
/**
* The queue worker's entry point. Never throws — every outcome is reported on
* `videoGenEvents` as `completed` or `failed`, which is the terminal signal
* `mediaJobQueue`'s dispatcher awaits. A `failed` on a job whose cancel was
* requested is mapped to `canceled` by the queue, so this does not need to
* distinguish them on the wire.
*/
export async function runVideoUpscale({
jobId, historyId, sourceFilename, runtime, adapterPath, adapterKey, baseModelPath,
icMinReferences, icMaxReferences, seed, source, alignment, target,
}) {
const entry = { canceled: false, proc: null, abort: new AbortController() };
activeUpscales.set(jobId, entry);
const renderStartedAtMs = Date.now();
// Every path this run might create, so a failure at ANY step removes all of
// them. The source clip is never in this list.
const scratch = [];
let deliverablePath = null;
let thumbnailPath = null;
let result = null;
let failureReason;
const removeOwnedFile = async path => {
if (!path) return;
await unlinkGuarded(path).catch(error => {
if (error.code === 'ENOENT') return;
maintenance.markCurrentUnsettled();
maintenance.markResourceUnsettled('media', jobId);
console.error(`❌ Video upscale cleanup failed [${jobId.slice(0, 8)}]: ${error.message}`);
});
};
try {
// Re-read the row rather than trusting the enqueue-time snapshot: a queued
// job can wait behind a long render, and the user may have deleted or
// already upscaled the source in the meantime.
const history = await loadHistory();
const item = history.find((h) => h.id === historyId);
if (!item) throw new ServerError('Video not found', { status: 404, code: 'NOT_FOUND' });
if (item.upscaledFrom) {
throw new ServerError('Cannot upscale an already-upscaled video', { status: 400, code: 'ALREADY_UPSCALED' });
}
// Re-resolve the queued filename through safeUnder rather than trusting the
// persisted string: media-jobs.json is replayed on boot and hand-editable,
// and this path is handed to both the render and the audio re-mux.
if (item.filename !== sourceFilename) {
throw new ServerError('The source clip was replaced while this upscale was queued.', { status: 409, code: 'UPSCALE_SOURCE_CHANGED' });
}
const sourcePath = safeUnder(PATHS.videos, sourceFilename);
if (!sourcePath || !existsSync(sourcePath)) {
throw new ServerError('Video file not found on disk', { status: 404, code: 'NOT_FOUND' });
}
videoGenEvents.emit('started', {
generationId: jobId,
id: jobId,
upscaledFrom: historyId,
upscaleMethod: 'ltx',
upscaleRuntime: runtime,
width: target.width,
height: target.height,
numFrames: target.frameCount,
fps: source.fps,
seed,
etaMs: null,
});
console.log(`🔍 Upscaling video [${historyId.slice(0, 8)}] on ${runtime}: ${source.width}×${source.height} → ${target.width}×${target.height}`);
// Apply exactly #6509's plan: pad to the grid, never crop, never trim. A
// conforming source skips the pass entirely and is READ in place.
let referencePath = sourcePath;
if (!ltxAlignmentIsNoop(alignment)) {
referencePath = join(tmpdir(), `portos-upscale-src-${jobId}.mp4`);
scratch.push(referencePath);
const padded = await padSourceForUpscale(sourcePath, referencePath, {
width: alignment.paddedSource.width,
height: alignment.paddedSource.height,
padFrames: alignment.padFrames,
fps: source.fps,
signal: entry.abort.signal,
});
if (!padded.ok) {
throw new ServerError(`Failed to align the source clip: ${padded.reason}`, { status: 500, code: 'UPSCALE_ALIGN_FAILED' });
}
}
const renderPath = join(tmpdir(), `portos-upscale-out-${jobId}.mp4`);
scratch.push(renderPath);
const { bin, args } = buildArgs({
upscale: {
runtime,
sourceVideoPath: referencePath,
baseModelPath,
icLoraWeightPath: adapterPath,
icMinReferences,
icMaxReferences,
width: target.width,
height: target.height,
numFrames: target.frameCount,
fps: source.fps,
seed,
},
outputPath: renderPath,
});
const rendered = await runUpscaleChild({ jobId, bin, args, runtime, entry });
if (!rendered.ok) {
throw new ServerError(rendered.reason, { status: 500, code: 'UPSCALE_RENDER_FAILED' });
}
if (entry.canceled) {
throw new ServerError('Upscale canceled', { status: 499, code: 'UPSCALE_CANCELED' });
}
if (!existsSync(renderPath)) {
throw new ServerError('The upscale runner exited cleanly but wrote no video.', { status: 500, code: 'UPSCALE_RENDER_FAILED' });
}
// Crop the alignment padding back off, trim the cloned tail frames, and
// re-mux the ORIGINAL clip's audio at offset zero. A source with no audio
// track yields a silent deliverable, not a failure.
const newId = randomUUID();
const newFilename = `${newId}.mp4`;
deliverablePath = join(PATHS.videos, newFilename);
const finalized = await finalizeUpscaleOutput(renderPath, deliverablePath, {
width: source.width * UPSCALE_SCALE,
height: source.height * UPSCALE_SCALE,
frameCount: source.frameCount,
audioSourcePath: sourcePath,
signal: entry.abort.signal,
});
if (!finalized.ok) {
throw new ServerError(`Failed to finish the upscaled clip: ${finalized.reason}`, { status: 500, code: 'UPSCALE_MUX_FAILED' });
}
await withBackupAssetPublication(async () => {
try {
// Provenance is MEASURED off the deliverable, not assumed from the plan —
// a generative render is the one upscale method whose output can legally
// differ from 2× the source, so a recorded guess would be a lie.
const [measured, duration] = await Promise.all([
probeVideoStreamInfo(deliverablePath),
probeVideoDuration(deliverablePath),
]);
const thumbnail = await generateThumbnail(deliverablePath, newId);
// Tracked so a failure between here and the history write cannot leave a
// poster for a clip that does not exist.
thumbnailPath = thumbnail ? safeUnder(PATHS.videoThumbnails, thumbnail) : null;
const newEntry = {
...omitRenderTiming(item),
id: newId,
filename: newFilename,
thumbnail,
createdAt: new Date().toISOString(),
width: measured.width ?? source.width * UPSCALE_SCALE,
height: measured.height ?? source.height * UPSCALE_SCALE,
fps: measured.fps ?? source.fps,
numFrames: measured.frameCount ?? source.frameCount,
duration,
seed,
upscaledFrom: item.id,
upscaleMethod: 'ltx',
upscaleRuntime: runtime,
upscaleAdapter: adapterKey,
// Measured by the runner off the weight it fused (the rule it enforced),
// and — in the same shape a plain render's row carries — the exact
// ltx/mlx/torch + chip + OS stack this clip rendered on. Both are absent
// sentinels when the runner did not report them, never a guessed value.
...(rendered.referenceDownscale !== null ? { upscaleReferenceDownscale: rendered.referenceDownscale } : {}),
...(rendered.runtime ? { runtime: rendered.runtime } : {}),
...(rendered.loraStrategy ? { loraStrategyApplied: rendered.loraStrategy } : {}),
// What the grid actually required of this source, kept beside the result
// so a reader can tell a padded render from a conforming one.
upscaleAlignment: {
padWidth: alignment.padWidth,
padHeight: alignment.padHeight,
padFrames: alignment.padFrames,
trimFrames: alignment.trimFrames,
},
prompt: upscaledPrompt(item.prompt),
hidden: false,
...renderTimingFields(renderStartedAtMs),
};
await mutateVideoHistory((h) => { h.unshift(newEntry); return h; });
result = newEntry;
console.log(`✅ Upscaled [${newId.slice(0, 8)}]: ${newFilename} (${newEntry.width}×${newEntry.height}, ${Math.round((newEntry.renderMs ?? 0) / 1000)}s)`);
} catch (error) {
if (!result) for (const path of [deliverablePath, join(PATHS.videoThumbnails, `${newId}.jpg`)]) await removeOwnedFile(path);
throw error;
}
});
} catch (err) {
// The history row is written LAST, so nothing here can orphan one — only a
// partial file can exist, and it is removed before the failure is reported.
for (const path of result ? [] : [deliverablePath, thumbnailPath]) {
await removeOwnedFile(path);
}
const reason = entry.canceled ? 'Canceled while running' : (err.message || 'Upscale failed');
console.error(`❌ Video upscale ${entry.canceled ? 'canceled' : 'failed'} [${jobId.slice(0, 8)}]: ${reason}`);
failureReason = reason;
} finally {
for (const path of scratch) await removeOwnedFile(path);
activeUpscales.delete(jobId);
}
if (failureReason) videoGenEvents.emit('failed', { generationId: jobId, error: failureReason });
else videoGenEvents.emit('completed', { generationId: jobId, ...result });
return result;
}