Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Keep renders alive through restarts: updates wait for every render, and the final render resumes (#11587)

[._.]/30e9674f9387
diff --git a/client/src/components/dashboard/ActiveProcessingWidget.jsx b/client/src/components/dashboard/ActiveProcessingWidget.jsx
index b2b3c09ba0..f25ed93891 100644
--- a/client/src/components/dashboard/ActiveProcessingWidget.jsx
+++ b/client/src/components/dashboard/ActiveProcessingWidget.jsx
@@ -106,6 +106,8 @@ function ActiveProcessingWidget() {
const now = useTimeTick(1000);
const ollama = telemetry?.ollama ?? data?.ollama;
const appOperations = data?.appOperations || [];
+ // Renders outside the media queue (music video, video timeline, Code Animation).
+ const renders = data?.renders || [];
const activeAgents = data?.agents?.active || 0;
// Derived server-side (server/lib/systemIdle.js) and rendered as sent: the
// unattended auto-updater refuses to restart the install on this same verdict,
@@ -115,7 +117,7 @@ function ActiveProcessingWidget() {
// screen, but the idle headline is only the server's confirmed verdict.
const confirmed = status === 'ready';
const idle = confirmed && activity?.idle === true;
- const runningJobs = jobs.filter((job) => job.status !== 'queued').length;
+ const runningJobs = jobs.filter((job) => job.status !== 'queued').length + renders.length;
return (
// @container on the widget's OWN root: a dashboard cell is ~250px wide on a
// 2560px screen, so a viewport breakpoint here would pin the four-up metric
@@ -129,6 +131,7 @@ function ActiveProcessingWidget() {
{!idle ? <div className="mb-3 grid grid-cols-2 gap-1.5 text-center @xs:grid-cols-5"><Metric icon={Bot} value={activeAgents} label="agents" /><Metric icon={Layers3} value={runningJobs} label="rendering" /><Metric icon={Cpu} value={llm?.trusted === false ? '?' : formatCount(llm?.active || 0)} label="LLM" /><Metric icon={Brain} value={mind?.thinking ? 'yes' : 'no'} label="thinking" /><Metric icon={Cpu} value={gpu?.laneBusy ? 'busy' : 'ready'} label="GPU" /></div> : null}
{idle ? <Link to="/system-resources/overview" className="flex items-center justify-between rounded-lg border border-port-border bg-port-bg px-3 py-3 text-xs text-gray-400 transition-colors hover:border-port-accent/50 hover:text-gray-200"><span>Nothing is running</span><span>GPU {gpu?.status === 'available' ? 'ready' : gpu?.status || 'unknown'} →</span></Link> : null}
<div className="space-y-1.5">{jobs.map((job) => <JobRow key={job.id} job={job} onCancel={cancel} />)}</div>
+ {renders.map((render, index) => <LaneRow key={`render-${render.kind}-${index}`} to={render.to || '/system-resources/overview'} icon={Film} label={render.noun ? render.noun.charAt(0).toUpperCase() + render.noun.slice(1) : 'Render'} detail={elapsed(render.startedAt, now)} />)}
{(llm?.runs || []).map((run) => <LaneRow key={`llm-${run.runId}`} to={`/cos/runs?run=${encodeURIComponent(run.runId)}`} icon={Cpu} label={`${run.providerId || 'LLM'} · ${run.model || 'default model'} · ${run.source || 'run'}`} detail={elapsed(run.startedAt, now)} />)}
{llm?.active > (llm?.runs?.length || 0) ? <LaneRow to="/cos/runs" icon={Cpu} label="Additional LLM runs" detail={formatCount(llm.active - (llm?.runs?.length || 0))} /> : null}
{(ollama?.models || []).map((model) => <LaneRow key={`ollama-${model.id}`} to="/models/llms" icon={Cpu} label={`Ollama loaded · ${model.name || model.id}`} detail={[model.size == null ? 'size unknown' : formatBytes(model.size), model.sizeVram == null ? 'processor unknown' : model.sizeVram === 0 ? 'CPU' : model.size != null && model.sizeVram >= model.size ? 'GPU' : 'GPU/CPU'].join(' · ')} />)}
diff --git a/client/src/components/dashboard/ActiveProcessingWidget.test.jsx b/client/src/components/dashboard/ActiveProcessingWidget.test.jsx
index 6e196ffcb9..cdd555b31e 100644
--- a/client/src/components/dashboard/ActiveProcessingWidget.test.jsx
+++ b/client/src/components/dashboard/ActiveProcessingWidget.test.jsx
@@ -106,6 +106,18 @@ describe('ActiveProcessingWidget', () => {
expect(await screen.findByRole('link', { name: /Example App · update/ })).toHaveAttribute('href', '/apps');
});
+ it('shows a render running outside the media queue, linked to its project', async () => {
+ mockGetSystemActivity.mockResolvedValue({
+ jobs: [],
+ extras: {},
+ agents: {},
+ renders: [{ kind: 'music-video-final', noun: 'music video render', to: '/music-video/mv-example', startedAt: new Date().toISOString() }],
+ activity: { idle: false, activeCount: 1, queuedCount: 0, blockers: [] },
+ });
+ renderWidget();
+ expect(await screen.findByRole('link', { name: /Music video render/ })).toHaveAttribute('href', '/music-video/mv-example');
+ });
+
it('shows LLM model/source, elapsed time, a run deep link and Ollama residency', async () => {
mockGetSystemActivity.mockResolvedValue({
...idle,
diff --git a/client/src/components/musicVideo/ShotFinalRender.jsx b/client/src/components/musicVideo/ShotFinalRender.jsx
index ba520ece7e..0e0757fe23 100644
--- a/client/src/components/musicVideo/ShotFinalRender.jsx
+++ b/client/src/components/musicVideo/ShotFinalRender.jsx
@@ -47,7 +47,7 @@ export default function ShotFinalRender({ project, scene, onChange, fieldId }) {
else if (value !== 'own') onChange({ filmLook: filmLookPreset(value) });
};
- const state = status.error ? null : rendering ? `Rendering ${Math.round((live?.progress || 0) * 100)}%` : rendered ? 'Rendered' : shot ? 'Not rendered' : 'Not in the cut';
+ const state = status.error ? null : rendering ? (live ? `Rendering ${Math.round((live.progress || 0) * 100)}%` : 'Rendering') : rendered ? 'Rendered' : shot ? 'Not rendered' : 'Not in the cut';
return (
<section aria-label="Final render" className="min-w-0 space-y-2 rounded border border-port-border p-2">
<div className="flex flex-wrap items-center gap-2">
diff --git a/client/src/hooks/useSystemActivity.js b/client/src/hooks/useSystemActivity.js
index 04f010d625..3e9a8fd897 100644
--- a/client/src/hooks/useSystemActivity.js
+++ b/client/src/hooks/useSystemActivity.js
@@ -42,6 +42,7 @@ export const sameProcessingSnapshot = (a, b) => {
&& a.activity?.activeCount === b.activity?.activeCount
&& a.activity?.queuedCount === b.activity?.queuedCount
&& (a.appOperations || []).length === (b.appOperations || []).length
+ && JSON.stringify(a.renders || []) === JSON.stringify(b.renders || [])
&& a.jobs?.length === b.jobs?.length
&& (a.extras?.imageTo3d || []).length === (b.extras?.imageTo3d || []).length
&& (a.jobs || []).every((job, index) => job?.id === b.jobs[index]?.id
diff --git a/server/lib/systemIdle.js b/server/lib/systemIdle.js
index ed104f5e03..eca5a2f206 100644
--- a/server/lib/systemIdle.js
+++ b/server/lib/systemIdle.js
@@ -119,6 +119,17 @@ export function summarizeSystemActivity(snapshot) {
add('app-operations', `${pluralize(appOperations.length, 'app operation')} running`, appOperations.length);
}
+ // Renders outside the media queue (a music-video final render runs for an
+ // hour or more); a restart through one loses it (services/renderActivity.js).
+ const renders = Array.isArray(snapshot?.renders) ? snapshot.renders : [];
+ const renderKinds = new Map();
+ for (const render of renders) {
+ const kind = typeof render?.kind === 'string' ? render.kind : 'render';
+ const noun = typeof render?.noun === 'string' && render.noun ? render.noun : 'render';
+ renderKinds.set(kind, { noun, count: (renderKinds.get(kind)?.count || 0) + 1 });
+ }
+ for (const [kind, { noun, count }] of renderKinds) add(`render:${kind}`, `${pluralize(count, noun)} running`, count);
+
if (snapshot?.update?.inProgress) add('update-in-progress', 'An update is already running', 1);
// Milder than the others: a killed backup wastes a snapshot rather than
@@ -128,7 +139,7 @@ export function summarizeSystemActivity(snapshot) {
const activeCount = jobs.filter((job) => job?.status === 'running').length
+ imageTo3d + activeAgents + activeLlmRuns + (mind?.thinking ? 1 : 0) + appOperations.length
- + backupActive;
+ + backupActive + renders.length;
const queuedCount = jobs.filter((job) => job?.status === 'queued').length
+ queuedAgents + (Number(mind?.queued) || 0);
diff --git a/server/lib/systemIdle.test.js b/server/lib/systemIdle.test.js
index 22634f440d..dc5df4024c 100644
--- a/server/lib/systemIdle.test.js
+++ b/server/lib/systemIdle.test.js
@@ -107,6 +107,22 @@ describe('system idle verdict', () => {
// Milder than the others (a wasted snapshot, not lost work), but a running
// backup still gets killed mid-rsync by an unattended restart.
+ // Renders outside the media queue; restarting through one lost a multi-hour
+ // music-video render to the unattended updater.
+ it('counts each kind of running render as activity', () => {
+ const verdict = summarizeSystemActivity(snapshot({ renders: [
+ { kind: 'music-video-final', noun: 'music video render' },
+ { kind: 'video-timeline', noun: 'video timeline render' },
+ { kind: 'video-timeline', noun: 'video timeline render' },
+ ] }));
+ expect(verdict.idle).toBe(false);
+ expect(verdict.activeCount).toBe(3);
+ expect(verdict.blockers).toEqual([
+ { kind: 'render:music-video-final', label: '1 music video render running', count: 1 },
+ { kind: 'render:video-timeline', label: '2 video timeline renders running', count: 2 },
+ ]);
+ });
+
it('counts a running backup snapshot as activity', () => {
const verdict = summarizeSystemActivity(snapshot({ backup: { inProgress: true } }));
expect(verdict.idle).toBe(false);
diff --git a/server/routes/musicVideo.js b/server/routes/musicVideo.js
index 87d70b063b..9a9ba48300 100644
--- a/server/routes/musicVideo.js
+++ b/server/routes/musicVideo.js
@@ -708,8 +708,9 @@ router.get('/:id/sharing-copy/download', asyncHandler(async (req, res) => {
// Assemble the scenes' i2v clips into one MP4 over the track as the master audio
// bed. Kickoff returns { jobId }; progress streams over SSE (mirrors
// videoTimeline). Per-project mutex returns 409 with the live jobId for re-attach.
+// A shot-by-shot render started here picks up again after a server restart.
router.post('/:id/render', asyncHandler(async (req, res) => {
- res.json(await renderMusicVideo(req.params.id));
+ res.json(await renderMusicVideo(req.params.id, { resumable: true }));
}));
// Read-only: the live final-render job for a project (null when none runs on
diff --git a/server/routes/musicVideo.test.js b/server/routes/musicVideo.test.js
index 688c3a6160..5e335f8648 100644
--- a/server/routes/musicVideo.test.js
+++ b/server/routes/musicVideo.test.js
@@ -840,7 +840,7 @@ describe('musicVideo routes', () => {
it('POST /:id/render kicks off the render and returns the jobId', async () => {
const r = await request(app).post('/api/music-video/mv-1/render').send({});
expect(r.status).toBe(200);
- expect(renderSvc.renderMusicVideo).toHaveBeenCalledWith('mv-1');
+ expect(renderSvc.renderMusicVideo).toHaveBeenCalledWith('mv-1', { resumable: true });
expect(r.body).toEqual({ jobId: 'job-1' });
});
diff --git a/server/services/activeProcessing.js b/server/services/activeProcessing.js
index f572e54d0f..33c1e14461 100644
--- a/server/services/activeProcessing.js
+++ b/server/services/activeProcessing.js
@@ -8,6 +8,7 @@ import { getLoadedModels, getLastLoadedModelsError } from './ollamaManager.js';
import { listActiveAppOperations } from './appOperations.js';
import { readPersistentMindStateForSafetyCheck } from './cosState.js';
import { isUpdateInProgress } from './updateChecker.js';
+import { listActiveRenders } from './renderActivity.js';
import * as cos from './cos.js';
// Lazy — NOT a static top-level import. `services/runner.js` and
@@ -170,6 +171,9 @@ export async function getSystemActivity({ includeOllama = false } = {}) {
// deciding whether it may restart the install.
appOperations: listActiveAppOperations(),
update: { inProgress: isUpdateInProgress() },
+ // Final renders, draft excerpts and shot renders run their own ffmpeg and
+ // browsers outside the media queue; an in-process count, no I/O.
+ renders: listActiveRenders(),
// No trusted contract needed: an in-process flag with no I/O once loaded.
backup: { inProgress: backupInProgress },
};
diff --git a/server/services/codeAnimation/stages.js b/server/services/codeAnimation/stages.js
index e1b2edfc1b..6d0688a4ee 100644
--- a/server/services/codeAnimation/stages.js
+++ b/server/services/codeAnimation/stages.js
@@ -29,6 +29,7 @@ import { checkSoundConsent, produceSoundtrack, muxSoundtrack } from './sound.js'
import { renderViaMediaQueue, sampleFilm } from './stageRender.js';
import { resolveBlenderExecution } from './execution.js';
import { renderBlenderSequence, publishBlenderVideo } from './blenderRender.js';
+import { trackedRenderMap } from '../renderActivity.js';
export const STAGE_RUN_KIND = 'production-stages';
const STYLE_POINTS = [0.1, 0.5, 0.9];
@@ -38,7 +39,7 @@ const RESUMABLE = new Set(['interrupted', 'canceled', 'exhausted', 'failed']);
const BUDGET_CODE = 'CODE_ANIMATION_BUDGET_EXHAUSTED';
const CANCEL_CODE = 'CODE_ANIMATION_RUN_CANCELED';
-const activeRuns = new Map();
+const activeRuns = trackedRenderMap('code-animation', { noun: 'Code Animation production run', link: () => '/code-animation/production' });
export const activeStageRunIds = () => [...activeRuns.keys()];
const exhausted = dimension => new ServerError(`Production budget exhausted: ${dimension}`, { status: 409, code: BUDGET_CODE, context: { dimension } });
diff --git a/server/services/musicVideo/excerptRender.js b/server/services/musicVideo/excerptRender.js
index 720a00243c..f56f31146a 100644
--- a/server/services/musicVideo/excerptRender.js
+++ b/server/services/musicVideo/excerptRender.js
@@ -49,9 +49,10 @@ import { musicVideoEvents } from './events.js';
import { pilotDependencies, productionPilotRenderProject } from './productionPilot.js';
import { musicVideoDependencyChanges } from '../../lib/musicVideoDependencies.js';
import { excerptRangeFits } from '../../lib/musicVideoExcerptRange.js';
+import { trackedRenderMap } from '../renderActivity.js';
const jobs = new Map();
-const projectExcerptRenders = new Map();
+const projectExcerptRenders = trackedRenderMap('music-video-excerpt', { noun: 'music video draft render', link: (key) => `/music-video/${encodeURIComponent(key)}` });
const PENDING = Symbol('mv-excerpt-render-pending');
export const attachExcerptRenderSseClient = (jobId, res) => attachSse(jobs, jobId, res);
diff --git a/server/services/musicVideo/projectsLogic.js b/server/services/musicVideo/projectsLogic.js
index 047cbdf602..6b49d0ff59 100644
--- a/server/services/musicVideo/projectsLogic.js
+++ b/server/services/musicVideo/projectsLogic.js
@@ -308,6 +308,7 @@ export function cloneProjectRecord(source, {
// #9010: the source's in-flight render mark is not the clone's.
renderingOn: null,
renderPartialFilename: null,
+ renderResume: null,
deleted: false,
deletedAt: null,
};
diff --git a/server/services/musicVideo/publishKit.js b/server/services/musicVideo/publishKit.js
index 332fb4c03f..6e07f31638 100644
--- a/server/services/musicVideo/publishKit.js
+++ b/server/services/musicVideo/publishKit.js
@@ -27,9 +27,10 @@ import { musicVideoAspect } from '../../lib/musicVideoAspect.js';
import { musicVideoDependencyChanges } from '../../lib/musicVideoDependencies.js';
import { buildChapters, buildSrt, buildPublishCopyPrompt, parsePublishCopy, normalizeCopyOptions, PUBLISH_PLATFORMS } from './publishKitText.js';
import { canonicalizeSunoUrl, isSunoShareLink } from '../sunoShareLink.js';
+import { trackedRenderMap } from '../renderActivity.js';
const jobs = new Map();
-const projectBuilds = new Map();
+const projectBuilds = trackedRenderMap('music-video-publish-kit', { noun: 'publishing kit build', link: (projectId) => `/music-video/${encodeURIComponent(projectId)}` });
const MAX_THUMBNAILS = 6;
// X caps a Premium upload's bitrate well under a 1080p master's; ~12 Mbps
// keeps the analog grain without the upload being re-crushed.
diff --git a/server/services/musicVideo/render.js b/server/services/musicVideo/render.js
index 88a5a59750..c8ff932d9a 100644
--- a/server/services/musicVideo/render.js
+++ b/server/services/musicVideo/render.js
@@ -64,11 +64,12 @@ import { cutEffectCuts, plannedTransitions, sceneCutEffect } from '../../lib/mus
import { imageEffectGlitchFrames, mergeGlitchFrames, musicVideoTransitionFilter, sceneTransitions, shotGlitchFrames, transitionGlitchFrames } from '../../lib/musicVideoTransitions.js';
import { stillMoveForCamera } from '../../lib/cameraMovements.js';
import { isEndFrameImageEffect, musicVideoImageEffectFilter, sceneImageEffect } from '../../lib/musicVideoImageEffects.js';
+import { trackedRenderMap } from '../renderActivity.js';
// Per-project render mutex (keyed by projectId so two projects can render in
// parallel; same-project re-entry returns 409 with the live jobId for re-attach).
const jobs = new Map();
-const projectRenders = new Map();
+const projectRenders = trackedRenderMap('music-video-final', { noun: 'music video render', link: (key) => `/music-video/${encodeURIComponent(key)}` });
// Reserved synchronously the instant a render is accepted, BEFORE the async
// prep (ffmpeg/audio/clip resolution) — so a double-click or second client
// can't pass the 409 check during that await window and spawn a duplicate.
@@ -97,7 +98,7 @@ export const isLocalRenderMark = (renderingOn, instanceId) => !renderingOn || re
// Every terminal project write clears the in-flight mark's stamp and the
// partial-output pointer alongside the status.
-const settledRender = (status, extra = {}) => ({ status, ...extra, renderingOn: null, renderPartialFilename: null });
+const settledRender = (status, extra = {}) => ({ status, ...extra, renderingOn: null, renderPartialFilename: null, renderResume: null });
// Tell listeners (the autonomous run waiting on its final render) that a render
// job reached a terminal state. Emitted after the project record's terminal
@@ -1039,7 +1040,10 @@ async function renderFromShotSegments(projectId, project, handOff, options) {
maintenance.markCurrentUnsettled();
console.error(`❌ Music-video render [${jobId.slice(0, 8)}] project ${projectId.slice(0, 8)} status→${status} write failed: ${err.message}`);
});
- await writeProject({ status: 'rendering', renderingOn, renderPartialFilename: filename, renderError: null }, 'rendering');
+ // The mark boot recovery reads to start this render again after a restart:
+ // the shots it finished are cached, so the rerun picks up where it stopped.
+ const renderResume = options.resumable ? { attempt: options.resumeAttempt || 0 } : null;
+ await writeProject({ status: 'rendering', renderingOn, renderPartialFilename: filename, renderError: null, renderResume }, 'rendering');
const all = [...plan.segments, ...(plan.outro ? [plan.outro] : [])];
const cached = all.filter((segment) => existsSync(segment.path)).length;
console.log(`🎬 Rendering music video [${jobId.slice(0, 8)}]: project=${projectId.slice(0, 8)} segments=${all.length} cached=${cached} duration=${plan.songSec.toFixed(2)}s`);
@@ -1448,6 +1452,7 @@ export async function recoverStuckMusicVideoRenders() {
// its (finished) file — that file is the gallery's, not a partial.
let finished = null;
let recovered = 0;
+ const resumes = [];
for (const project of stuck) {
// Re-checked against the FRESHEST record under the write lock: a peer sync
// landing between the list and this write may have handed the mark to
@@ -1457,13 +1462,14 @@ export async function recoverStuckMusicVideoRenders() {
&& isLocalRenderMark(current.renderingOn, instanceId);
if (!stillStuck) return { project: current, demoted: false };
const targetStatus = current.renderHistoryId ? 'complete' : 'ready';
- return { project: applyProjectPatch(current, settledRender(targetStatus)), demoted: true, partial: current.renderPartialFilename };
+ return { project: applyProjectPatch(current, settledRender(targetStatus)), demoted: true, partial: current.renderPartialFilename, resume: current.renderResume };
}).catch((err) => {
console.error(`❌ Music Video recovery: project ${project.id.slice(0, 8)} demotion write failed: ${err.message}`);
return null;
});
if (!outcome?.demoted) continue;
recovered++;
+ if (outcome.resume && (Number(outcome.resume.attempt) || 0) < MAX_RENDER_RESUMES) resumes.push({ projectId: project.id, attempt: (Number(outcome.resume.attempt) || 0) + 1 });
// Only once the record no longer points at it. The filename came off a
// (possibly peer-synced) record, so it is only resolved inside videos/.
const { partial } = outcome;
@@ -1477,4 +1483,28 @@ export async function recoverStuckMusicVideoRenders() {
if (partialPath) await unlink(partialPath).catch(() => {});
}
if (stuck.length > 0) console.log(`🎬 Music Video boot recovery: demoted ${recovered}/${stuck.length} stuck render(s)`);
+ if (resumes.length > 0) scheduleRenderResumes(resumes);
+}
+
+// A shot-by-shot final render the user started is started again after a
+// restart: every shot it finished is cached, so the rerun only renders the
+// shots left and stitches. Bounded, so a render that itself takes the server
+// down is not restarted forever.
+const MAX_RENDER_RESUMES = 2;
+const RENDER_RESUME_DELAY_MS = 15_000;
+
+function scheduleRenderResumes(resumes) {
+ // After boot settles (maintenance admission, the media lane): the render
+ // admits like any other.
+ setTimeout(async () => {
+ for (const { projectId, attempt } of resumes) {
+ try {
+ const { jobId } = await renderMusicVideo(projectId, { resumable: true, resumeAttempt: attempt });
+ console.log(`🎬 Music Video boot recovery: resumed the final render of ${projectId.slice(0, 8)} [${jobId.slice(0, 8)}] from its cached shots`);
+ } catch (err) {
+ console.error(`❌ Music Video boot recovery: could not resume the final render of ${projectId.slice(0, 8)}: ${err.message}`);
+ await updateProject(projectId, { renderError: `Interrupted by a server restart and could not restart: ${err.message}`.slice(0, 2000) }).catch(() => {});
+ }
+ }
+ }, RENDER_RESUME_DELAY_MS).unref?.();
}
diff --git a/server/services/musicVideo/render.terminal.test.js b/server/services/musicVideo/render.terminal.test.js
index 84b803f262..baa80805ed 100644
--- a/server/services/musicVideo/render.terminal.test.js
+++ b/server/services/musicVideo/render.terminal.test.js
@@ -113,7 +113,7 @@ import { maintenance } from '../../lib/maintenanceAdmission.js';
const tick = () => new Promise((r) => setTimeout(r, 0));
const lastProc = () => h.procs[h.procs.length - 1];
// A terminal project write: the status plus the cleared in-flight mark (#9010).
-const settled = (status, extra = {}) => ({ status, ...extra, renderingOn: null, renderPartialFilename: null });
+const settled = (status, extra = {}) => ({ status, ...extra, renderingOn: null, renderPartialFilename: null, renderResume: null });
const prime = (projectId) => {
getProject.mockResolvedValue({
@@ -421,6 +421,35 @@ describe('recoverStuckMusicVideoRenders (#8430)', () => {
errorSpy.mockRestore();
logSpy.mockRestore();
});
+
+ it('starts an interrupted shot-by-shot render again from its cached shots, a bounded number of times', async () => {
+ const { recoverStuckMusicVideoRenders } = await import('./render.js');
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
+ vi.useFakeTimers({ toFake: ['setTimeout'] });
+ prime('resume-1');
+ getProject.mockResolvedValue({
+ id: 'resume-1', name: 'P', trackId: 't1', status: 'ready',
+ scenes: [{ sceneId: 's1', order: 0, videoHistoryId: 'h1' }],
+ composition: { version: 1, mode: 'composed', textCues: [], style: { color: '#ffffff', font: 'sans' }, posterSec: null },
+ });
+ const store = seedStore([
+ { id: 'resume-1', status: 'rendering', renderResume: { attempt: 0 } },
+ { id: 'spent-1', status: 'rendering', renderResume: { attempt: 2 } },
+ { id: 'plain-2', status: 'rendering', renderResume: null },
+ ]);
+
+ await recoverStuckMusicVideoRenders();
+ expect(statusOf(store)).toEqual({ 'resume-1': 'ready', 'spent-1': 'ready', 'plain-2': 'ready' });
+ expect(store.get('resume-1').renderResume).toBeNull();
+ expect(seg.plans).toHaveLength(0);
+
+ await vi.runOnlyPendingTimersAsync();
+ vi.useRealTimers();
+ await tick();
+ expect(seg.plans.map((plan) => plan.project.id)).toEqual(['resume-1']);
+ expect(updateProject).toHaveBeenCalledWith('resume-1', expect.objectContaining({ status: 'rendering', renderResume: { attempt: 1 } }));
+ logSpy.mockRestore();
+ });
});
it('chooses the loudest section midpoint within the rendered duration', async () => {
@@ -525,6 +554,13 @@ describe('composed render lifecycle (#8984)', () => {
expect(lastProc().args).not.toContain('-itsoffset');
});
+ it('marks a render the user started as resumable after a restart', async () => {
+ primeComposed('mark-1', { mode: 'composed' });
+ await renderMusicVideo('mark-1', { resumable: true });
+ await tick();
+ expect(updateProject).toHaveBeenCalledWith('mark-1', expect.objectContaining({ status: 'rendering', renderResume: { attempt: 0 } }));
+ });
+
it('cancels while shots render: no stitch, prior status restored, slot released', async () => {
primeComposed('cancel-1', { mode: 'composed', textCues: [cue] });
const { jobId } = await renderMusicVideo('cancel-1');
diff --git a/server/services/musicVideo/shotSegments.js b/server/services/musicVideo/shotSegments.js
index 464dab09f7..40bbce1f2c 100644
--- a/server/services/musicVideo/shotSegments.js
+++ b/server/services/musicVideo/shotSegments.js
@@ -22,7 +22,7 @@
import { createHash, randomUUID } from 'crypto';
import { existsSync } from 'fs';
import { copyFile, mkdir, readdir, rename, rm, stat, writeFile } from 'fs/promises';
-import { availableParallelism, tmpdir } from 'os';
+import { availableParallelism, tmpdir, totalmem } from 'os';
import { join } from 'path';
import { PATHS } from '../../lib/fileUtils.js';
import { ServerError } from '../../lib/errorHandler.js';
@@ -41,6 +41,7 @@ import { nameShots } from './productionReview.js';
import { getProject } from './projects.js';
import { musicVideoEvents } from './events.js';
import { maintenance } from '../../lib/maintenanceAdmission.js';
+import { trackedRenderMap } from '../renderActivity.js';
export const SHOT_SEGMENT_DIR = 'music-video-segments';
// Bump when a segment's encode changes in a way its key cannot see.
@@ -278,8 +279,17 @@ async function renderShotBase(plan, segment, outputPath, { jobId, signal, onProg
}
}
+// Each segment in flight may hold a film-look browser (about one core, since
+// the filter rasterizes serially) and an encoder. Two thirds of the cores keep
+// the server and the user's own apps responsive; the memory term keeps a small
+// machine from running out of RAM with a browser per shot.
+const SEGMENT_MAX_CONCURRENCY = 12;
+const SEGMENT_BYTES_EACH = 3 * 1024 ** 3;
+
/** How many segments render at once: each may hold a browser and an encoder. */
-export const segmentConcurrency = ({ cores = availableParallelism() } = {}) => Math.max(1, Math.min(3, Math.floor(cores / 4)));
+export const segmentConcurrency = ({ cores = availableParallelism(), memoryBytes = totalmem() } = {}) => Math.max(1, Math.min(
+ SEGMENT_MAX_CONCURRENCY, Math.floor((cores * 2) / 3), Math.floor(memoryBytes / SEGMENT_BYTES_EACH),
+));
/**
* Render every missing segment of `plan` (`concurrency` at a time). `onProgress`
@@ -300,6 +310,8 @@ export async function renderMissingSegments(plan, { signal, onProgress, concurre
await renderSegment(plan, segment, { signal, onProgress: (f) => { done.set(segment, f); report(); } });
done.set(segment, 1);
report();
+ // An open shot card flips to Rendered (and can play it) mid-render.
+ if (segment.sceneId) emitShotRender({ projectId: plan.project?.id, sceneId: segment.sceneId, state: 'rendered', progress: 1 });
} catch (err) {
if (signal?.aborted || err.code === 'CANCELED') throw err;
failures.push(err);
@@ -411,7 +423,10 @@ function segmentStatus(plan) {
}
// One shot render per shot at a time: `${projectId}:${sceneId}` → { controller }.
-const shotJobs = new Map();
+const shotJobs = trackedRenderMap('music-video-shot', {
+ noun: 'music video shot render',
+ link: (key) => { const [projectId, sceneId] = key.split(':'); return `/music-video/${encodeURIComponent(projectId)}/scene/${encodeURIComponent(sceneId)}`; },
+});
const shotJobKey = (projectId, sceneId) => `${projectId}:${sceneId}`;
const emitShotRender = (payload) => musicVideoEvents.emit('shot-render', payload);
diff --git a/server/services/musicVideo/shotSegments.test.js b/server/services/musicVideo/shotSegments.test.js
index 46511173e8..995087971a 100644
--- a/server/services/musicVideo/shotSegments.test.js
+++ b/server/services/musicVideo/shotSegments.test.js
@@ -27,6 +27,7 @@ vi.mock('./render.js', async (importOriginal) => {
const { planShotSegments, renderMissingSegments, stitchSegments } = await import('./shotSegments.js');
const { buildMusicVideoFfmpegArgs } = await import('./render.js');
+const { musicVideoEvents } = await import('./events.js');
afterAll(cleanupTempDataRoots);
@@ -66,7 +67,13 @@ describe.skipIf(!hasFfmpeg)('shot segments', () => {
it('stitch to the picture the whole-film graph draws, with the song under it', async () => {
const plan = await planShotSegments(project());
expect(plan.segments.map((s) => [s.sceneId, s.frames])).toEqual([['a', 24], ['b', 24], ['c', 24]]);
+ // Each shot a final render finishes is announced, so an open shot card can play it mid-render.
+ const announced = [];
+ const onShot = (event) => announced.push(`${event.projectId}:${event.sceneId}:${event.state}`);
+ musicVideoEvents.on('shot-render', onShot);
expect(await renderMissingSegments(plan, { concurrency: 2 })).toBe(0);
+ musicVideoEvents.off('shot-render', onShot);
+ expect(announced.sort()).toEqual(['a', 'b', 'c'].map((id) => `mv-segments-test:${id}:rendered`));
const out = join(ROOT(), 'stitched.mp4');
await stitchSegments(plan, out);
expect(probe(out, 'stream=codec_type').sort()).toEqual(['audio', 'video']);
diff --git a/server/services/renderActivity.js b/server/services/renderActivity.js
new file mode 100644
index 0000000000..f5c5a4bbb0
--- /dev/null
+++ b/server/services/renderActivity.js
@@ -0,0 +1,65 @@
+/**
+ * Renders this process is running outside the media job queue: music-video
+ * final renders, draft excerpts and single-shot renders, video timeline
+ * renders, publishing-kit builds and Code Animation production runs. Each runs
+ * its own ffmpeg, browsers or Blender, so the system idle verdict
+ * (lib/systemIdle.js), the update preflight and Live activity read them here;
+ * without this, the unattended updater restarted the server through a final
+ * render.
+ *
+ * A leaf on purpose: each render module keeps its in-flight slots in a map made
+ * here, so reading the list never imports the render code (a module that never
+ * loaded has nothing running), and a slot opening or closing invalidates the
+ * shared activity snapshot like any other lifecycle edge.
+ */
+
+import { noteSystemActivity } from './systemActivityNotify.js';
+
+const kinds = new Map();
+
+/**
+ * A Map of one kind's in-flight render slots that counts toward system activity
+ * and notes each slot opening and closing. `noun` names one slot in blocker
+ * lines ("music video render"); `link(key, value)` is the page that owns a slot.
+ */
+export function trackedRenderMap(kind, { noun = 'render', link = () => null } = {}) {
+ const map = new Map();
+ const started = new Map();
+ const set = map.set.bind(map);
+ const remove = map.delete.bind(map);
+ map.set = (key, value) => {
+ const opened = !map.has(key);
+ set(key, value);
+ if (opened) {
+ started.set(key, new Date().toISOString());
+ noteSystemActivity('render', 'start');
+ }
+ return map;
+ };
+ map.delete = (key) => {
+ const closed = remove(key);
+ if (closed) {
+ started.delete(key);
+ noteSystemActivity('render', countActiveRenders() ? 'completion' : 'drained');
+ }
+ return closed;
+ };
+ kinds.set(kind, { map, started, noun, link });
+ return map;
+}
+
+/** Every tracked render in flight: `{ kind, noun, to, startedAt }`. */
+export function listActiveRenders() {
+ const renders = [];
+ for (const [kind, { map, started, noun, link }] of kinds) {
+ for (const [key, value] of map) renders.push({ kind, noun, to: link(key, value) || null, startedAt: started.get(key) || null });
+ }
+ return renders;
+}
+
+/** Total tracked renders in flight in this process. */
+function countActiveRenders() {
+ let total = 0;
+ for (const { map } of kinds.values()) total += map.size;
+ return total;
+}
diff --git a/server/services/systemActivityNotify.js b/server/services/systemActivityNotify.js
index 5ddcb3c1d3..7b1d070027 100644
--- a/server/services/systemActivityNotify.js
+++ b/server/services/systemActivityNotify.js
@@ -14,7 +14,7 @@
import { isTestRunner } from '../lib/runtimeEnv.js';
const ACTIVITY_SOURCES = new Set([
- 'agents', 'media', 'imageTo3d', 'llm', 'mind', 'appOperations', 'backup', 'update',
+ 'agents', 'media', 'imageTo3d', 'llm', 'mind', 'appOperations', 'backup', 'update', 'render',
]);
export const SYSTEM_ACTIVITY_COALESCE_MS = 50;
diff --git a/server/services/updatePreflight.js b/server/services/updatePreflight.js
index dbf3499143..b3343f6dd5 100644
--- a/server/services/updatePreflight.js
+++ b/server/services/updatePreflight.js
@@ -4,6 +4,8 @@ import { getActiveAgentIds, spawningTasks } from './agentState.js';
import { readPersistentMindStateForSafetyCheck } from './cosState.js';
import { filterLiveAgentIds } from './cosAgentLifecycle.js';
import * as updateChecker from './updateChecker.js';
+import { listActiveRenders } from './renderActivity.js';
+import { summarizeSystemActivity } from '../lib/systemIdle.js';
// Shared refusal logic for the PortOS self-update path, called from both
// POST /api/update/execute (server/routes/update.js) and the app:update
@@ -72,6 +74,19 @@ export async function getPersistentMindImageWorkGuard() {
return persistentMindImageWorkGuard(snapshot.persistentMind);
}
+/**
+ * The update restarts the server, which ends every in-flight render outside the
+ * media queue. `renders` is a `listActiveRenders()` list; the line names each
+ * kind the way Live activity and the idle verdict do.
+ */
+export const rendersActiveError = (renders) => {
+ const running = summarizeSystemActivity({ renders }).blockers.map((blocker) => blocker.label).join(', ');
+ return new ServerError(
+ `${running}. Let ${renders.length === 1 ? 'it' : 'them'} finish, or stop ${renders.length === 1 ? 'it' : 'them'}, before updating: the update restarts PortOS and ${renders.length === 1 ? 'the render' : 'the renders'} would be lost.`,
+ { status: 409, code: 'RENDERS_ACTIVE', context: { renders: renders.length } },
+ );
+};
+
export const persistentMindStateUntrustedError = () => new ServerError(
'Persistent Mind state could not be validated. Restore data/cos/state.json from backup before updating.',
{ status: 409, code: 'PERSISTENT_MIND_STATE_UNTRUSTED' },
@@ -104,6 +119,8 @@ export async function checkPortosUpdatePreflight({
} = {}) {
const activeCosAgents = await countActiveCosAgents();
if (activeCosAgents > 0) throw agentsActiveError(activeCosAgents);
+ const renders = listActiveRenders();
+ if (renders.length > 0) throw rendersActiveError(renders);
const imageCheck = await getPersistentMindImageWorkGuard();
if (!imageCheck.trusted) throw persistentMindStateUntrustedError();
diff --git a/server/services/updatePreflightParity.test.js b/server/services/updatePreflightParity.test.js
index 9e32510038..6ea49262c8 100644
--- a/server/services/updatePreflightParity.test.js
+++ b/server/services/updatePreflightParity.test.js
@@ -125,6 +125,27 @@ describe('PortOS update preflight parity — route vs. socket', () => {
expect(appUpdaterUpdateApp).not.toHaveBeenCalled();
});
+ it('refuses identically while a render is running', async () => {
+ const { trackedRenderMap } = await import('./renderActivity.js');
+ const renders = trackedRenderMap('parity-test', { noun: 'music video render' });
+ renders.set('project-example', 'job-example');
+ try {
+ const routeRes = await request(makeRouteApp()).post('/api/update/execute').send({});
+ expect(routeRes.status).toBe(409);
+ expect(routeRes.body.code).toBe('RENDERS_ACTIVE');
+ expect(routeRes.body.error).toMatch(/^1 music video render running\. Let it finish/);
+
+ const { fireUpdate, emitted } = makeSocketHarness();
+ await fireUpdate({ appId: PORTOS_APP_ID });
+ const errorEvent = emitted.find((e) => e.event === 'app:update:error');
+ expect(errorEvent.payload.code).toBe('RENDERS_ACTIVE');
+ expect(errorEvent.payload.message).toBe(routeRes.body.error);
+ expect(appUpdaterUpdateApp).not.toHaveBeenCalled();
+ } finally {
+ renders.delete('project-example');
+ }
+ });
+
it('refuses identically when Persistent Mind has unacknowledged queued image work', async () => {
mockCosState.persistentMind = {
queuedMessages: [{ id: 'message-example', images: [{ attachmentId: 'attachment-example' }] }],
diff --git a/server/services/videoTimeline/local.js b/server/services/videoTimeline/local.js
index 01c6ea4a22..296f47f888 100644
--- a/server/services/videoTimeline/local.js
+++ b/server/services/videoTimeline/local.js
@@ -48,6 +48,7 @@ import {
} from './segments.js';
import { AUDIO_NORM, buildAudioBedMix, fmtSec } from './audioBedMix.js';
import { historySourceDuration, resolveClipSpan, resolveStillFades, resolveAudioSpan } from '../../lib/videoTimelineSourceSpans.js';
+import { trackedRenderMap } from '../renderActivity.js';
const PROJECTS_FILE = join(PATHS.data, 'video-projects.json');
@@ -61,7 +62,7 @@ const DEFAULT_CANVAS = { width: 1280, height: 720, fps: 24 };
// can render in parallel; same project re-render returns 409 with the
// existing jobId so the UI can attach SSE instead of getting a stale failure.
const jobs = new Map();
-const projectRenders = new Map(); // projectId → jobId
+const projectRenders = trackedRenderMap('video-timeline', { noun: 'video timeline render', link: (projectId) => `/media/timeline/${encodeURIComponent(projectId)}` }); // projectId → jobId
export const attachSseClient = (jobId, res) => attachSse(jobs, jobId, res);