Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { channel } from 'node:diagnostics_channel';
import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';
import { maintenance } from '../../lib/maintenanceAdmission.js';
import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
import { cp, lstat, mkdtemp, open, realpath, rm, stat, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { createReadStream } from 'node:fs';
import { pipeline } from 'node:stream/promises';
import { PATHS, ensureDir, unlinkGuarded } from '../../lib/fileUtils.js';
import { LAUNCH_VIDEO_FORMATS, LAUNCH_VIDEO_FORMAT_SIZES, htmlCompositionContractSchema, htmlCompositionContractSchemaFor, htmlCompositionRenderSchema, validateRequest } from '../../lib/validation.js';
import { generateThumbnail } from '../../lib/ffmpeg.js';
import { videoGenEvents } from '../videoGen/events.js';
import { mutateVideoHistory } from '../videoGen/history.js';
import { resolveMusicTrackPath } from '../pipeline/audioMux.js';
import { openComposition } from './browser.js';
import { encodeComposition, encodeContactSheet, proofTimes, synthesizeCompositionMusic } from './encode.js';
import { validateLaunchVideoAssets } from '../../lib/launchVideoValidation.js';
import { makePathResolver } from '../../lib/pathSafety.js';
import { UPLOAD_AUDIO_EXTENSIONS } from '../../lib/mimeTypes.js';
const active = new Map();
const resolveAudioUpload = makePathResolver(() => PATHS.uploads, { extensions: UPLOAD_AUDIO_EXTENSIONS });
const lifecycle = channel('portos.composition-render.lifecycle');
// A music-video owner may run as long as its song, and no longer than this.
const MUSIC_VIDEO_DURATION_CAP_SEC = 900;
const MUSIC_VIDEO_OWNER = 'music-video';
function insideData(root, path) {
const rel = relative(root, path);
return Boolean(rel) && rel !== '..' && !rel.startsWith(`..${sep}`) && !rel.startsWith('../') && !isAbsolute(rel);
}
// A missing feature block is null. An array (including []) is not a feature
// track — #9073's block is an object, and "not analyzed" must not look like
// a measured empty track.
function normalizeSongDocument(song) {
const source = song && typeof song === 'object' && !Array.isArray(song) ? song : {};
const features = source.features;
return {
beats: Array.isArray(source.beats) ? source.beats : [],
downbeats: Array.isArray(source.downbeats) ? source.downbeats : [],
sections: Array.isArray(source.sections) ? source.sections : [],
features: features !== null && typeof features === 'object' && !Array.isArray(features) ? features : null,
words: Array.isArray(source.words) ? source.words : null,
};
}
async function resolveMusicVideoOwner({ owner, audio, maxDurationSec }) {
if (owner == null) return null;
if (owner !== MUSIC_VIDEO_OWNER) throw new Error('Unknown composition owner');
if (!audio || typeof audio.path !== 'string' || !audio.path || audio.path.includes('\0')) {
throw new Error('Music-video renders require an audio master path');
}
const startSec = audio.startSec ?? 0;
if (!Number.isFinite(startSec) || startSec < 0) throw new Error('audio.startSec must be a non-negative number');
if (!Number.isFinite(maxDurationSec) || maxDurationSec < 1) throw new Error('maxDurationSec must be the song duration');
let info;
try { info = await stat(audio.path); } catch (error) {
if (error.code === 'ENOENT') throw new Error('Music-video audio master is missing');
throw error;
}
if (!info.isFile()) throw new Error('Music-video audio master must be a regular file');
return {
audio: { path: audio.path, startSec },
maxDurationSec: Math.min(MUSIC_VIDEO_DURATION_CAP_SEC, maxDurationSec),
};
}
export const MUSIC_VIDEO_SCRATCH_DIR = 'music-video-song-renders';
// Copy the composition into this job's scratch directory and write song.json
// before openComposition freezes the snapshot. The page fetches it by relative
// URL, so the network-refusing sandbox stays unchanged. `prepare(dir)` lets a
// caller add job files (a music-video document's portos-mv.js and scene media)
// to the private copy before the snapshot; the caller removes `scratchRoot`.
export async function stageMusicVideoComposition(sourceDirectory, jobId, song, { prepare } = {}) {
if (typeof jobId !== 'string' || !/^[A-Za-z0-9_-]+$/.test(jobId)) throw new Error('Invalid composition job id');
const root = await realpath(PATHS.data);
const source = await realpath(resolve(root, sourceDirectory));
if (!insideData(root, source)) throw new Error('directory must be inside data');
const scratchRoot = join(root, MUSIC_VIDEO_SCRATCH_DIR, jobId);
try {
await rm(scratchRoot, { recursive: true, force: true });
const compositionDir = join(scratchRoot, 'composition');
await ensureDir(scratchRoot);
await cp(source, compositionDir, { recursive: true, verbatimSymlinks: true });
await writeFile(join(compositionDir, 'song.json'), `${JSON.stringify(normalizeSongDocument(song))}\n`);
if (prepare) await prepare(compositionDir);
return { directory: `${MUSIC_VIDEO_SCRATCH_DIR}/${jobId}/composition`, scratchRoot };
} catch (error) {
await rm(scratchRoot, { recursive: true, force: true }).catch(cleanupError => {
maintenance.markCurrentUnsettled();
throw cleanupError;
});
throw error;
}
}
export function cancel(jobId) {
const job = active.get(jobId);
if (!job || job.committing) return false;
job.controller.abort(new Error('Render canceled'));
return true;
}
// The frame fields a render publishes; `formats`/`layout` describe the source only.
// Render-time motion-blur presets (#9080), in the contract's own motionBlur shape.
const MOTION_BLUR_PRESETS = { off: 1, light: { shutter: 0.25, samples: 'auto', tolerance: 2 }, film: { shutter: 0.5, samples: 'auto', tolerance: 2 } };
const frameOf = ({ durationSec, fps, width, height, motionBlur }) => ({ durationSec, fps, width, height, motionBlur });
/**
* The frames one job renders from a single timeline (#8960). Without requested
* formats it is the composition's own size under the job id, exactly as before.
* A named format other than the composition's own size must be declared in
* portosComposition.formats, so a fixed-layout scene is never squeezed into a
* frame it was not written for. Duration, fps and timing are shared by all.
*/
function renderTargets(contract, jobId, formats) {
if (!formats) return [{ id: jobId, format: null, contract }];
const own = `${contract.width}x${contract.height}`;
return LAUNCH_VIDEO_FORMATS.filter(format => formats.includes(format)).map(format => {
const size = LAUNCH_VIDEO_FORMAT_SIZES[format];
if (size !== own && !contract.formats?.includes(size)) {
throw new Error(`portosComposition.formats must include ${size} to render ${format}`);
}
const [width, height] = size.split('x').map(Number);
return { id: `${jobId}-${format}`, format, contract: { ...contract, width, height } };
});
}
// Every name deliver() will exclusively create for this run, single- and
// multi-format alike, in the same write order as the delivery loop below.
function deliveryNames(targets) {
const names = ['plan.md', 'storyboard.json', 'caption.txt'];
for (const target of targets) {
const suffix = target.format ? `-${target.format}` : '';
names.push(`video${suffix}.mp4`, `poster${suffix}.jpg`);
}
return names;
}
export async function renderComposition(options) {
return maintenance.run('composition', options.jobId, () => renderCompositionAdmitted(options), { continuation: true });
}
async function renderCompositionAdmitted({ jobId, owner, audio, maxDurationSec, song, signal: parentSignal, ...input }) {
const job = { controller: new AbortController(), committing: false };
active.set(jobId, job);
// Relay cancellation through the same commit guard as cancel(jobId). The
// returned promise still waits for browser/encoder cleanup before settling.
const abort = () => { if (!job.committing) job.controller.abort(parentSignal.reason); };
parentSignal?.addEventListener('abort', abort, { once: true });
if (parentSignal?.aborted) abort();
const { signal } = job.controller;
const began = performance.now();
const report = (phase, { encoderPid, frame, frames, failed } = {}) => lifecycle.publish({
jobId, phase, elapsedMs: Math.round(performance.now() - began),
...(encoderPid !== undefined ? { encoderPid, frame, frames } : {}),
...(failed !== undefined ? { failed } : {}),
});
let page;
let audioDirectory;
let scratchRoot;
let success = false;
let result;
let failure;
// Every file this job creates, recorded before its first byte, so a failure
// or cancellation removes all formats' partial output.
const ownedPaths = [];
const deliver = async (target, write) => {
const handle = await open(target, 'wx');
// Record ownership after exclusive creation, before any fallible I/O.
ownedPaths.push(target);
try { await write(handle); } finally { await handle.close(); }
};
try {
report('preflight');
signal.throwIfAborted();
const musicVideo = await resolveMusicVideoOwner({ owner, audio, maxDurationSec });
const parsedInput = validateRequest(htmlCompositionRenderSchema, input);
let { directory } = parsedInput;
const sourceDirectory = directory;
const { musicTrack, audioUpload, launchVideo, synthesizeMusic, proof, formats, motionBlur: blurChoice, masterLoudness } = parsedInput;
// A music-video render may still ask for the extra frames its contract
// declares in portosComposition.formats (renderTargets enforces that).
if (musicVideo && (launchVideo || synthesizeMusic || musicTrack || audioUpload || proof)) {
throw new Error('A music-video composition render cannot use launch-video, proof, or soundtrack options');
}
let musicPath = musicTrack ? await resolveMusicTrackPath(musicTrack) : null;
if (musicTrack && !musicPath) throw new Error('musicTrack is missing from the Music library');
if (audioUpload) {
musicPath = resolveAudioUpload(audioUpload);
if (!musicPath) throw new Error('audioUpload is missing or is not a supported audio file');
}
signal.throwIfAborted();
let launchPlan;
let launchAssets;
let deliveryRoot;
if (launchVideo?.appId) {
const expected = `launch-videos/${launchVideo.appId}/${launchVideo.runId}/composition`;
if (directory !== expected) throw new Error('Launch-video directory does not match its app and run');
const dataRoot = await realpath(PATHS.data);
if (await realpath(join(dataRoot, directory)) !== resolve(dataRoot, directory)) throw new Error('Launch-video directory must not be a symlink');
deliveryRoot = join(dataRoot, 'launch-videos', launchVideo.appId, launchVideo.runId);
}
const needsLaunchGate = launchVideo || directory.split('/')[0] === 'launch-videos';
// One page and one frozen snapshot per job: the privacy and storyboard
// gates run once however many formats render from it.
if (musicVideo) {
const staged = await stageMusicVideoComposition(directory, jobId, song);
directory = staged.directory;
scratchRoot = staged.scratchRoot;
signal.throwIfAborted();
}
report('browser-open');
page = await openComposition(directory, { signal, validateAssets: needsLaunchGate
? assets => { launchPlan = validateLaunchVideoAssets(assets, launchVideo); launchAssets = assets; }
: undefined });
report('contract');
const metadata = await page.evaluate(`(() => {
const c = globalThis.portosComposition;
if (!c || typeof c.seek !== 'function') throw new Error('portosComposition.seek is required');
return { durationSec: c.durationSec, fps: c.fps, width: c.width, height: c.height, motionBlur: c.motionBlur,
formats: c.formats, layout: typeof c.layout === 'function' };
})()`);
const contractSchema = musicVideo
? htmlCompositionContractSchemaFor(musicVideo.maxDurationSec)
: htmlCompositionContractSchema;
const parsed = contractSchema.safeParse(metadata);
if (!parsed.success) throw new Error(parsed.error.issues.map(issue => `${issue.path.join('.')}: ${issue.message}`).join('; '));
const contract = blurChoice ? { ...parsed.data, motionBlur: MOTION_BLUR_PRESETS[blurChoice] } : parsed.data;
if (launchPlan && Math.abs(contract.durationSec - launchPlan.durationSec) > 1e-8) {
throw new Error('Composition durationSec must match storyboard.json');
}
if (proof) {
const [target] = renderTargets(contract, jobId, proof.format && [proof.format]);
// Launch runs keep their proofs beside the run so the iteration history
// stays with the take; other compositions share one proofs directory.
// The result names the sheet relative to the data directory, never an
// absolute host path: job results are visible to API and SSE clients.
const proofName = `contact-${jobId}.png`;
const proofFile = deliveryRoot ? `launch-videos/${launchVideo.appId}/${launchVideo.runId}/proofs/${proofName}` : `composition-proofs/${proofName}`;
const proofPath = deliveryRoot ? join(deliveryRoot, 'proofs', proofName) : join(PATHS.data, proofFile);
await ensureDir(dirname(proofPath));
const times = proofTimes(contract.durationSec, proof.everySec);
ownedPaths.push(proofPath);
const { columns } = await encodeContactSheet(page, target.contract, proofPath, { times, signal });
page.check();
report('browser-close');
await page.close({ verify: true });
page = null;
signal.throwIfAborted();
success = true;
result = { generationId: jobId, id: jobId, proof: { file: proofFile, url: `/data/${proofFile}`, times, columns,
...(target.format ? { format: target.format } : {}), ...frameOf(target.contract) } };
} else {
const targets = renderTargets(contract, jobId, formats);
if (deliveryRoot) {
// Refuse before spending a capture. deliver()'s exclusive open('wx')
// below is still the race-safe authority; this only saves the retry
// a full render before it learns what that open would have told it.
// lstat also refuses a symlinked delivery name.
for (const name of deliveryNames(targets)) {
const target = join(deliveryRoot, name);
let exists = true;
try { await lstat(target); } catch (error) { if (error.code !== 'ENOENT') throw error; exists = false; }
if (exists) {
const error = new Error(`EEXIST: file already exists, open '${target}'`);
error.code = 'EEXIST';
throw error;
}
}
}
if (synthesizeMusic) {
// One score for every format: they share the timeline and duration.
const wav = await synthesizeCompositionMusic(page, contract.durationSec);
page.check();
audioDirectory = await mkdtemp(join(tmpdir(), 'portos-composition-audio-'));
musicPath = join(audioDirectory, 'score.wav');
await writeFile(musicPath, wav, { flag: 'wx' });
}
await ensureDir(PATHS.videos);
const frameCount = (item) => Math.round(item.durationSec * item.fps);
const framesTotal = targets.reduce((sum, target) => sum + frameCount(target.contract), 0);
let framesDone = 0;
const startedAt = Date.now();
const rendered = [];
for (const target of targets) {
const filename = `composition-${target.id}.mp4`;
const outputPath = join(PATHS.videos, filename);
const targetFrames = frameCount(target.contract);
ownedPaths.push(outputPath, join(PATHS.videoThumbnails, `${target.id}.jpg`));
const { sampleHistogram, loudness } = await encodeComposition(page, target.contract, outputPath, {
musicPath: musicVideo ? null : musicPath,
master: masterLoudness !== false,
audio: musicVideo?.audio,
signal,
onLifecycle: detail => report(detail.phase, detail),
onProgress: (fraction, detail) => {
const done = framesDone + (detail?.frame ?? Math.round(fraction * targetFrames));
const elapsed = Date.now() - startedAt;
const etaMs = done > 0 && done < framesTotal && elapsed > 0
? Math.max(1, Math.round(elapsed * (framesTotal - done) / done))
: undefined;
videoGenEvents.emit('progress', {
generationId: jobId,
progress: framesTotal ? (done / framesTotal) * 0.95 : 0,
step: done,
totalSteps: framesTotal,
...(etaMs != null ? { etaMs } : {}),
message: `Rendering frame ${done}/${framesTotal}`,
});
},
});
framesDone += targetFrames;
page.check();
rendered.push({ ...target, filename, outputPath, sampleHistogram, loudness });
}
// End script execution before post-processing and publishing the result.
report('browser-close');
await page.close({ verify: true });
page = null;
signal.throwIfAborted();
report('publication-admission');
await withBackupAssetPublication(async () => {
try {
for (const video of rendered) {
report('thumbnail');
video.thumbnail = await generateThumbnail(video.outputPath, video.id, launchPlan ? { atSec: launchPlan.posterSec } : undefined);
if (!video.thumbnail) throw new Error('Composition thumbnail generation failed');
signal.throwIfAborted();
}
let launchMetadata;
if (deliveryRoot) {
// Exclusive creation refuses pre-existing files/symlinks. Publish only the
// validated snapshot, never re-read source prose after rendering.
for (const name of ['plan.md', 'storyboard.json', 'caption.txt']) {
await deliver(join(deliveryRoot, name), handle => handle.writeFile(launchAssets.get(`/${name}`)));
}
// A multi-format run names each file by its frame; a single-format
// render keeps the original video.mp4/poster.jpg names.
for (const video of rendered) {
const suffix = video.format ? `-${video.format}` : '';
for (const [source, name] of [[video.outputPath, `video${suffix}.mp4`], [join(PATHS.videoThumbnails, video.thumbnail), `poster${suffix}.jpg`]]) {
await deliver(join(deliveryRoot, name), handle => pipeline(createReadStream(source), handle.createWriteStream()));
}
}
launchMetadata = { appId: launchVideo.appId, runId: launchVideo.runId,
...(launchVideo.sourceVideoId ? { sourceVideoId: launchVideo.sourceVideoId } : {}),
musicTrack: musicTrack ?? null, synthesizeMusic: Boolean(synthesizeMusic),
caption: launchAssets.get('/caption.txt').toString('utf8').trim(), posterSec: launchPlan.posterSec };
}
signal.throwIfAborted();
// Once the shared history write starts, cancellation must be refused.
job.committing = true;
const createdAt = new Date().toISOString();
// One Media History entry per format, linked by launchVideo.runId and
// written in a single mutation so a run registers all formats or none.
const metas = rendered.map(video => ({
id: video.id, prompt: `HTML composition: ${sourceDirectory}`, modelId: 'html-composition', seed: 0,
...frameOf(video.contract), numFrames: Math.round(contract.durationSec * contract.fps),
...(launchMetadata ? { launchVideo: launchMetadata, appId: launchMetadata.appId, posterSec: launchPlan.posterSec } : {}),
...(video.sampleHistogram ? { sampleHistogram: video.sampleHistogram } : {}),
...(video.loudness ? { loudness: video.loudness } : {}),
filename: video.filename, thumbnail: video.thumbnail, createdAt,
}));
report('history');
await mutateVideoHistory(history => { history.unshift(...metas); return history; });
success = true;
// `id`/`filename` name a real history entry (the first format); `generationId`
// is the job. A single-format render keeps id === generationId as before.
// A shutter-blur render reports how many output frames took each
// sub-frame count, so the user can see where the render time went.
const summary = ({ id, filename, thumbnail, sampleHistogram, loudness }) => ({ id, filename, thumbnail, path: `/data/videos/${filename}`,
...(sampleHistogram ? { sampleHistogram } : {}), ...(loudness ? { loudness } : {}) });
const [first] = rendered;
result = { ...(launchMetadata ? { appId: launchMetadata.appId } : {}), generationId: jobId, ...summary(first),
...(formats ? { videos: rendered.map(video => ({ format: video.format, ...summary(video) })) } : {}) };
} catch (error) {
if (!success) for (const path of ownedPaths) await unlinkGuarded(path).catch(error => { if (error.code !== 'ENOENT') maintenance.markCurrentUnsettled(); });
throw error;
}
});
}
} catch (error) {
if (job.committing) maintenance.markCurrentUnsettled();
failure = error;
} finally {
report('cleanup');
if (page) await page.close().catch(error => {
maintenance.markCurrentUnsettled();
failure ||= error;
});
if (audioDirectory) await rm(audioDirectory, { recursive: true, force: true }).catch(() => {
maintenance.markCurrentUnsettled();
console.warn('⚠️ Could not remove temporary composition audio');
});
if (scratchRoot) await rm(scratchRoot, { recursive: true, force: true }).catch(() => {
maintenance.markCurrentUnsettled();
console.warn('⚠️ Could not remove temporary music-video composition');
});
if (!success) {
for (const path of ownedPaths) await unlinkGuarded(path).catch(error => { if (error.code !== 'ENOENT') maintenance.markCurrentUnsettled(); });
}
parentSignal?.removeEventListener('abort', abort);
active.delete(jobId);
report('settled', { failed: Boolean(failure) });
}
// Cleanup precedes the terminal event, so the queue cannot advance early.
if (failure) throw failure;
videoGenEvents.emit('completed', result);
return result;
}