Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Video Gen — xAI Grok Build CLI provider (image-first flow).
*
* Grok has no text-to-video tool: its `image_to_video` animates a source
* image (durations 6/10/15 s). So the flow, following grok's bundled
* `game-animation-frames` skill, is image-first — one headless agent turn
* that (1) generates a base image with `image_gen` when the caller didn't
* supply one, then (2) animates it with `image_to_video`, saving the MP4 to
* a directed scratch path PortOS harvests. When the caller supplies a
* source image (i2v from the gallery or an upload), step 1 is skipped and
* `image_to_video` runs on it directly.
*
* Mirrors the imageGen/grok.js provider's lifecycle (scratch cleanup,
* signature sniff, tree-kill on cancel — see that module's authority note)
* and hands the harvested file to the shared
* `finalizeGeneratedVideo` helper so streaming optimization, thumbnailing,
* the history entry, and the completed events all match local renders.
*
* The user gate is the same `imageGen.grok.enabled` toggle as image
* generation — one Grok backend, both media kinds. The route rejects
* disabled/missing-binary up front; this module assumes it's enabled.
*/
import { createMediaJobPreparation } from '../../lib/mediaJobPreparation.js';
import { spawn } from '../../lib/childProcess.js';
import { mkdir, open, stat } from 'fs/promises';
import { isAbsolute, join, resolve as pathResolve, sep } from 'path';
import { tmpdir } from 'os';
import { randomUUID } from 'crypto';
import { ensureDir, PATHS, copyFileGuarded, unlinkGuarded, rmGuarded } from '../../lib/fileUtils.js';
import { ServerError } from '../../lib/errorHandler.js';
import { attachSseClient as attachSse, closeJobAfterDelay, createJobFailureFinalizer } from '../../lib/sseUtils.js';
import { killWithEscalation } from '../../lib/killWithEscalation.js';
import { killProcessTree, prepareCliSpawn } from '../../lib/bufferedSpawn.js';
import { ensureGrokHeadlessArgs, prepareGrokPromptFile } from '../../lib/grok.js';
import { videoGenEvents } from './events.js';
import { discardUnpublishedVideo, finalizeGeneratedVideo, emitCloudRenderStatus, CLOUD_RENDER_PHASE } from './generateVideoHelpers.js';
import { mutateVideoHistory } from './history.js';
import { noImageReason, deriveAspectRatio, GROK_ASPECT_RATIOS } from '../imageGen/grok.js';
import { grokVideoBillableUnits, resolveGrokDuration } from '../../lib/grokVideoClip.js';
import { withSpawnCwdEnv } from '../../lib/spawnCwd.js';
import { probeVideoDuration } from '../../lib/ffmpeg.js';
import { maintenance } from '../../lib/maintenanceAdmission.js';
import { agentInstructionsAddition, carriedPromptAdditions, sentPromptRecord } from '../../lib/modelInput.js';
// 30 minutes — an image-first video turn is two sequential tool calls
// (image_gen then image_to_video render + download), so it runs meaningfully
// longer than a single image. Keep this ABOVE the mediaJobQueue cloud-lane
// idle watchdog (20 min) — the provider emits `activity` on every stdout
// chunk (grok narrates while it works), so the idle watchdog only trips on
// a truly silent hang while this wall-clock cap bounds the whole run.
const GROK_VIDEO_TIMEOUT_MS = (() => {
const n = Number(process.env.GROK_VIDEO_TIMEOUT_MS);
return Number.isFinite(n) && n > 0 ? n : 30 * 60 * 1000;
})();
const DEFAULT_BIN = 'grok';
const DEFAULT_HARVEST_TIMEOUT_MS = 10000;
let harvestTimeoutMs = DEFAULT_HARVEST_TIMEOUT_MS;
// The clip lengths this module gates on live in lib/grokVideoClip.js — the one
// list the Zod schemas and the client picker also derive from (#3022).
// Per-job state — keyed by jobId (cloud lane allows parallel renders). Same
// client shape as videoGen/local.js so attachSseClient/broadcastSse work.
const jobs = new Map();
const activeProcs = new Map();
const activeJobs = new Map();
export const getActiveJob = () => {
const entries = [...activeJobs.values()];
return entries.length ? entries[entries.length - 1] : null;
};
export const attachSseClient = (jobId, res) => attachSse(jobs, jobId, res);
const sigtermWithEscalation = (id, proc) => {
if (process.platform === 'win32') {
// prepareCliSpawn wraps a .cmd shim in cmd.exe on Windows — taskkill /T
// the whole tree so the real grok child dies too (mirrors imageGen/grok.js).
killProcessTree(proc);
return;
}
killWithEscalation(proc, { label: 'grok video child', delayMs: 5000, stillRunning: () => activeProcs.get(id) === proc });
};
const preparations = createMediaJobPreparation();
export const cancel = (jobId) => {
if (!jobId) {
throw new Error("videoGen/grok.cancel requires a jobId — use cancelAll() to terminate every in-flight render");
}
const pending = preparations.cancel(jobId);
if (pending) return pending;
const proc = activeProcs.get(jobId);
if (!proc) return false;
sigtermWithEscalation(jobId, proc);
return true;
};
export const cancelAll = () => {
const ids = [...preparations.ids(), ...activeProcs.keys()];
if (ids.length === 0) return false;
for (const id of ids) cancel(id);
return true;
};
// MP4/QuickTime signature: a top-level `ftyp` box at byte offset 4. Grok is
// directed to save MP4; anything else (webm, text error, truncated file) is
// rejected rather than shipped into the gallery mislabeled.
const isMp4Header = (buf) =>
Buffer.isBuffer(buf) && buf.length >= 8
&& buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70;
// Build the single-turn agent prompt for the image-first flow. Explicit about
// the tools, the save path, and staying out of everything else. Scratch cwd
// organizes output, not host authority; HTTP generation is operator-gated.
export function buildGrokVideoPrompt({ prompt, negativePrompt, aspectRatio, duration, stagingPath, sourceImagePath, scratchDir }) {
const avoid = negativePrompt?.trim() ? `\nAvoid: ${negativePrompt.trim()}` : '';
const ratio = aspectRatio ? `\nUse aspect_ratio "${aspectRatio}" for the base image.` : '';
const steps = sourceImagePath
? `Use your built-in image_to_video tool to animate the source image at ${sourceImagePath} for ${duration} seconds.\nMotion/direction prompt: ${prompt.trim()}${avoid}`
: `First, use your built-in image_gen tool to generate one base image for this prompt and save it as ${join(scratchDir, 'base.png')}:\n${prompt.trim()}${avoid}${ratio}\nThen use your built-in image_to_video tool to animate that base image for ${duration} seconds, using the same prompt as the motion description.`;
return `${steps}\nSave the resulting video as an MP4 file at exactly this path: ${stagingPath}\nDo not create any other files, do not modify any code, and do not run any other tools beyond what is needed to generate the image/video and write the MP4 to that path. When the file is written, you are done.`;
}
/**
* The agent prompt a grok video render sends and its record (#11317): shared by
* the render and the Video Gen form's preview so both show the same text.
*/
export function grokVideoSentPrompt({
prompt = '', negativePrompt, aspectRatio, width, height, duration, sourceImagePath = null, scratchDir, promptAdditions = [],
}) {
const effectiveDuration = resolveGrokDuration(duration);
const configured = GROK_ASPECT_RATIOS.includes(aspectRatio) ? aspectRatio : null;
const effectiveRatio = deriveAspectRatio(width, height) || configured;
const buildWith = (text) => buildGrokVideoPrompt({
prompt: text, negativePrompt, aspectRatio: effectiveRatio, duration: effectiveDuration,
stagingPath: join(scratchDir, 'output.mp4'), sourceImagePath, scratchDir,
});
const fullPrompt = buildWith(prompt);
return {
fullPrompt, effectiveRatio, effectiveDuration,
record: sentPromptRecord({ sentPrompt: fullPrompt, carried: carriedPromptAdditions({ promptAdditions }),
added: [agentInstructionsAddition('Grok video tool instructions', buildWith)] }),
};
}
export const generateVideo = (params) => preparations.run(params, generateVideoPrepared);
async function generateVideoPrepared({
grokPath, aspectRatio, prompt = '', negativePrompt, width, height,
sourceImagePath = null, duration,
uploadedTempPath = null,
jobId: providedJobId = null,
promptAdditions = [],
}) {
await ensureDir(PATHS.videos);
// Wall-clock render timing (#5878). finalizeGeneratedVideo reads this off the
// job; without it a grok render lands with no `renderMs` at all — the cloud
// lane is exactly the one whose cost the user can't guess from hardware.
// Measured from here (queue ingestion) rather than from the spawn, which is
// what videoGen/spawnWatch.js does for local renders.
const renderStartedAtMs = Date.now();
if (!prompt?.trim()) {
throw new ServerError('Prompt is required', { status: 400, code: 'VALIDATION_ERROR' });
}
const jobId = providedJobId || randomUUID();
const filename = `${jobId}.mp4`;
const outputPath = join(PATHS.videos, filename);
// Per-job scratch dir doubles as the child's cwd (containment) and holds
// both the intermediate base image and the staged MP4. Removed on every
// terminal path.
const scratchDir = join(tmpdir(), `portos-grok-video-${jobId}`);
const stagingPath = join(scratchDir, 'output.mp4');
await mkdir(scratchDir, { recursive: true });
const { fullPrompt, effectiveRatio, effectiveDuration, record: sentRecord } = grokVideoSentPrompt({
prompt, negativePrompt, aspectRatio, width, height, duration, sourceImagePath, scratchDir, promptAdditions,
});
const bin = grokPath || DEFAULT_BIN;
const baseArgs = ensureGrokHeadlessArgs([], null);
const { args, useStdin, cleanup: cleanupPromptFile } = prepareGrokPromptFile(baseArgs, fullPrompt);
// History/meta shape mirrors local.js's fields that the Media History
// consumers (normalize.js, the lightbox) actually read; grok has no
// seed/steps/fps knobs so those stay absent. `modelId: 'grok'` tags the
// record's engine for the grid badge and keeps Remix from offering
// local-only dials.
const meta = {
id: jobId,
prompt: prompt.trim(),
negativePrompt: negativePrompt || '',
modelId: 'grok',
duration: effectiveDuration,
...(effectiveRatio ? { aspectRatio: effectiveRatio } : {}),
filename,
createdAt: new Date().toISOString(),
mode: sourceImagePath ? 'image' : 'text',
...sentRecord,
};
const job = { ...meta, clients: [], status: 'running', renderStartedAtMs };
jobs.set(jobId, job);
console.log(`🎬 Generating video [${jobId.slice(0, 8)}] grok`);
videoGenEvents.emit('started', { generationId: jobId, totalSteps: 1, ...meta });
activeJobs.set(jobId, { ...meta, generationId: jobId, totalSteps: 1, step: 0, progress: 0 });
emitCloudRenderStatus(job, jobId, CLOUD_RENDER_PHASE.SUBMIT, 'Spawning grok…');
runGrokVideo(job, jobId, bin, args, {
useStdin, fullPrompt, cleanupPromptFile, scratchDir, stagingPath, outputPath, filename, meta, uploadedTempPath,
}).catch((err) => {
console.error(`❌ grok video run failed [${jobId.slice(0, 8)}]: ${err?.message}`);
});
return {
jobId, filename, path: `/data/videos/${filename}`, generationId: jobId,
mode: 'grok',
status: 'running',
};
}
async function runGrokVideo(job, jobId, bin, args, {
useStdin, fullPrompt, cleanupPromptFile, scratchDir, stagingPath, outputPath, filename, meta, uploadedTempPath,
}) {
const retainOwnership = (error) => {
maintenance.markCurrentUnsettled();
maintenance.markResourceUnsettled('media', jobId);
console.error(`❌ Grok video settlement failed [${jobId.slice(0, 8)}]: ${error?.message || error}`);
};
// The route stages a multipart source image into data/uploads and hands us
// its path — the provider owns unlinking it on every terminal path (same
// contract as videoGen/local.js; the queue only cleans up when the
// provider throws before spawning, or on boot-restore of a dead job).
let cleanupPromise;
const cleanup = () => cleanupPromise ||= Promise.allSettled([
Promise.resolve().then(() => cleanupPromptFile({ throwOnError: true })),
Promise.resolve().then(() => rmGuarded(scratchDir, { recursive: true, force: true })),
Promise.resolve().then(() => uploadedTempPath && unlinkGuarded(uploadedTempPath)),
]).then(results => {
for (const result of results) {
if (result.status === 'rejected' && result.reason?.code !== 'ENOENT') retainOwnership(result.reason);
}
});
if (preparations.take(jobId)) {
await cleanup();
return finalizeJobFailure.canceled(job, jobId);
}
// Resolve a path-shaped grokPath against the PortOS cwd before the child's
// cwd moves to the scratch dir; bare names stay bare for PATH lookup.
const resolvedBin = (!isAbsolute(bin) && (bin.includes('/') || bin.includes(sep))) ? pathResolve(bin) : bin;
const { command: spawnBin, args: spawnArgs } = prepareCliSpawn(resolvedBin, args);
// Pin PWD to the spawn cwd — see withSpawnCwdEnv (#3193). grok reads
// process.cwd(), so this is defensive rather than a live fix; it keeps every
// scratch-dir spawn telling the child one consistent story about where it is.
const proc = spawn(spawnBin, spawnArgs, { cwd: scratchDir, env: withSpawnCwdEnv(process.env, scratchDir), shell: false, stdio: [useStdin ? 'pipe' : 'ignore', 'pipe', 'pipe'] });
activeProcs.set(jobId, proc);
if (useStdin) {
proc.stdin.on('error', () => {});
proc.stdin.write(fullPrompt);
proc.stdin.end();
}
let stdoutTail = '';
const STDOUT_TAIL_BYTES = 8 * 1024;
let stderrTail = '';
const STDERR_TAIL_BYTES = 32 * 1024;
const timeoutTimer = setTimeout(() => {
if (activeProcs.get(jobId) === proc) {
console.log(`⏱️ grok video timed out after ${GROK_VIDEO_TIMEOUT_MS}ms [${jobId.slice(0, 8)}]`);
sigtermWithEscalation(jobId, proc);
}
}, GROK_VIDEO_TIMEOUT_MS);
let terminalStarted = false;
const publication = {};
let processError;
proc.on('error', (err) => {
if (terminalStarted) return;
processError = err;
// A failed kill/send can report error while the child is still alive.
// Keep its cancel handle (including SIGKILL escalation) until close.
if (proc.pid) return;
terminalStarted = true;
clearTimeout(timeoutTimer);
cleanup().then(() => {
finalizeJobFailure(job, jobId, proc, `Failed to spawn ${bin}: ${err.message}`);
}).catch(retainOwnership);
});
proc.stdout.on('data', (chunk) => {
stdoutTail += chunk.toString();
if (stdoutTail.length > STDOUT_TAIL_BYTES) stdoutTail = stdoutTail.slice(-STDOUT_TAIL_BYTES);
emitCloudRenderStatus(job, jobId, CLOUD_RENDER_PHASE.RENDER, 'Running…');
// Feed the mediaJobQueue's idle watchdog — grok narrates while working,
// so a genuinely wedged child stops emitting and still trips the 20-min
// idle cap, while a long-but-active image_to_video render doesn't.
videoGenEvents.emit('activity', { generationId: jobId });
});
proc.stderr.on('data', (chunk) => {
const text = chunk.toString();
stderrTail += text;
if (stderrTail.length > STDERR_TAIL_BYTES) stderrTail = stderrTail.slice(-STDERR_TAIL_BYTES);
videoGenEvents.emit('activity', { generationId: jobId });
});
proc.on('close', async (code, signal) => {
if (terminalStarted) return;
terminalStarted = true;
clearTimeout(timeoutTimer);
try {
if (code !== 0 || processError) {
const reason = processError?.message || (signal ? `Killed by signal ${signal}` : `Exit code ${code}`);
const tail = stderrTail.trim().split('\n').slice(-6).join('\n');
await cleanup();
return finalizeJobFailure(job, jobId, proc, `Grok video generation failed: ${reason}\n${tail}`);
}
const harvested = await harvestStagedVideo(stagingPath, harvestTimeoutMs);
if (!harvested.found) {
await cleanup();
const prefix = harvested.invalid ? 'Grok wrote a non-MP4 file at the directed path. ' : '';
return finalizeJobFailure(job, jobId, proc, `${prefix}${noVideoReason(stdoutTail)}`);
}
await copyFileGuarded(stagingPath, outputPath);
// grok's agent picks its own image_to_video length (see grokVideoClip.js),
// so record what it actually delivered and flag a clip that came back
// short: that is how a 15 s request is confirmed through the CLI.
const deliveredSec = await probeVideoDuration(outputPath).catch(() => null);
if (Number.isFinite(deliveredSec)) {
meta.deliveredSec = Math.round(deliveredSec * 100) / 100;
if (deliveredSec < meta.duration - 1) console.warn(`⚠️ grok delivered ${meta.deliveredSec}s for a ${meta.duration}s request [${jobId.slice(0, 8)}]`);
}
try {
const units = grokVideoBillableUnits({
requestedSeconds: meta.duration,
deliveredSeconds: Number.isFinite(deliveredSec) ? deliveredSec : null,
hasSourceImage: meta.mode === 'image',
});
const { recordGrokVideoUsage } = await import('../grokMediaUsage.js');
await recordGrokVideoUsage({ jobId, createdAt: meta.createdAt, ...units });
} catch (usageErr) {
console.error(`❌ Failed to record Grok video usage [${jobId.slice(0, 8)}]: ${usageErr.message}`);
}
// Removing the scratch directory also removes the harvested staging file.
await cleanup();
if (activeProcs.get(jobId) === proc) activeProcs.delete(jobId);
activeJobs.delete(jobId);
// Shared finalizer: faststart optimization, thumbnail, history entry,
// SSE complete + videoGenEvents 'completed' — identical to local
// renders so every downstream consumer (history grid, media index,
// completion hooks) sees the same contract.
await finalizeGeneratedVideo({ job, jobId, outputPath, filename, meta, actualSeed: null, mutateHistory: mutateVideoHistory, publication });
closeJobAfterDelay(jobs, jobId);
} catch (err) {
await cleanup();
if (!publication.committed) await discardUnpublishedVideo({ jobId, outputPath }).catch(retainOwnership);
retainOwnership(err);
// A dispatch failure can follow a durable commit; preserve its bytes and
// still surface a terminal failure past the completion idempotence guard.
finalizeJobFailure(job, jobId, proc, `Grok video post-exit handler failed: ${err?.message || err}`, { force: true });
}
});
}
// Reuse the image provider's narration-tail formatting but with video wording.
function noVideoReason(stdoutTail = '') {
const reason = noImageReason(stdoutTail);
return reason
.replace('Grok returned no image', 'Grok returned no video')
.replace('Grok did not produce an image at the directed path', 'Grok did not produce a video at the directed path')
.replace('the image_gen tool may be unavailable', 'the image_to_video tool may be unavailable');
}
const finalizeJobFailure = createJobFailureFinalizer({
jobs,
activeJobs,
activeSlots: activeProcs,
label: 'grok video generation',
events: videoGenEvents,
});
// Poll for the directed MP4 until it exists non-empty with an `ftyp` header,
// or timeoutMs elapses. { found, invalid } mirrors the image harvest —
// `invalid` means a non-empty file appeared that never matched MP4.
async function harvestStagedVideo(stagingPath, timeoutMs) {
const deadline = Date.now() + timeoutMs;
let sawInvalid = false;
while (Date.now() < deadline) {
const s = await stat(stagingPath).catch(() => null);
if (s && s.size > 0) {
const head = Buffer.alloc(12);
const fh = await open(stagingPath, 'r').catch(() => null);
if (fh) {
const { bytesRead } = await fh.read(head, 0, 12, 0).catch(() => ({ bytesRead: 0 }));
await fh.close().catch(() => {});
if (isMp4Header(head.subarray(0, bytesRead))) return { found: true, invalid: false };
sawInvalid = true;
}
}
const remainingMs = Math.max(1, deadline - Date.now());
await new Promise((r) => setTimeout(r, Math.min(250, remainingMs)));
}
return { found: false, invalid: sawInvalid };
}
// Test-only handles (mirrors imageGen/grok.js's carve-out).
export const _internals = {
harvestStagedVideo,
buildGrokVideoPrompt,
isMp4Header,
setHarvestTimeoutForTests: (timeoutMs = DEFAULT_HARVEST_TIMEOUT_MS) => {
harvestTimeoutMs = Number.isFinite(timeoutMs) && timeoutMs > 0
? timeoutMs
: DEFAULT_HARVEST_TIMEOUT_MS;
},
};