Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Keep renders alive through restarts: updates wait for every render, and the final render resumes (#11587)
[._.]/30e9674f9387
diff --git a/client/src/components/dashboard/ActiveProcessingWidget.jsx b/client/src/components/dashboard/ActiveProcessingWidget.jsxindex b2b3c09ba0..f25ed93891 100644--- a/client/src/components/dashboard/ActiveProcessingWidget.jsx+++ b/client/src/components/dashboard/ActiveProcessingWidget.jsx@@ -106,6 +106,8 @@ function ActiveProcessingWidget() {const now = useTimeTick(1000);const ollama = telemetry?.ollama ?? data?.ollama;const appOperations = data?.appOperations || [];+ // Renders outside the media queue (music video, video timeline, Code Animation).+ const renders = data?.renders || [];const activeAgents = data?.agents?.active || 0;// Derived server-side (server/lib/systemIdle.js) and rendered as sent: the// unattended auto-updater refuses to restart the install on this same verdict,@@ -115,7 +117,7 @@ function ActiveProcessingWidget() {// screen, but the idle headline is only the server's confirmed verdict.const confirmed = status === 'ready';const idle = confirmed && activity?.idle === true;- const runningJobs = jobs.filter((job) => job.status !== 'queued').length;+ const runningJobs = jobs.filter((job) => job.status !== 'queued').length + renders.length;return (// @container on the widget's OWN root: a dashboard cell is ~250px wide on a// 2560px screen, so a viewport breakpoint here would pin the four-up metric@@ -129,6 +131,7 @@ function ActiveProcessingWidget() {{!idle ? <div className="mb-3 grid grid-cols-2 gap-1.5 text-center @xs:grid-cols-5"><Metric icon={Bot} value={activeAgents} label="agents" /><Metric icon={Layers3} value={runningJobs} label="rendering" /><Metric icon={Cpu} value={llm?.trusted === false ? '?' : formatCount(llm?.active || 0)} label="LLM" /><Metric icon={Brain} value={mind?.thinking ? 'yes' : 'no'} label="thinking" /><Metric icon={Cpu} value={gpu?.laneBusy ? 'busy' : 'ready'} label="GPU" /></div> : null}{idle ? <Link to="/system-resources/overview" className="flex items-center justify-between rounded-lg border border-port-border bg-port-bg px-3 py-3 text-xs text-gray-400 transition-colors hover:border-port-accent/50 hover:text-gray-200"><span>Nothing is running</span><span>GPU {gpu?.status === 'available' ? 'ready' : gpu?.status || 'unknown'} →</span></Link> : null}<div className="space-y-1.5">{jobs.map((job) => <JobRow key={job.id} job={job} onCancel={cancel} />)}</div>+ {renders.map((render, index) => <LaneRow key={`render-${render.kind}-${index}`} to={render.to || '/system-resources/overview'} icon={Film} label={render.noun ? render.noun.charAt(0).toUpperCase() + render.noun.slice(1) : 'Render'} detail={elapsed(render.startedAt, now)} />)}{(llm?.runs || []).map((run) => <LaneRow key={`llm-${run.runId}`} to={`/cos/runs?run=${encodeURIComponent(run.runId)}`} icon={Cpu} label={`${run.providerId || 'LLM'} · ${run.model || 'default model'} · ${run.source || 'run'}`} detail={elapsed(run.startedAt, now)} />)}{llm?.active > (llm?.runs?.length || 0) ? <LaneRow to="/cos/runs" icon={Cpu} label="Additional LLM runs" detail={formatCount(llm.active - (llm?.runs?.length || 0))} /> : null}{(ollama?.models || []).map((model) => <LaneRow key={`ollama-${model.id}`} to="/models/llms" icon={Cpu} label={`Ollama loaded · ${model.name || model.id}`} detail={[model.size == null ? 'size unknown' : formatBytes(model.size), model.sizeVram == null ? 'processor unknown' : model.sizeVram === 0 ? 'CPU' : model.size != null && model.sizeVram >= model.size ? 'GPU' : 'GPU/CPU'].join(' · ')} />)}diff --git a/client/src/components/dashboard/ActiveProcessingWidget.test.jsx b/client/src/components/dashboard/ActiveProcessingWidget.test.jsxindex 6e196ffcb9..cdd555b31e 100644--- a/client/src/components/dashboard/ActiveProcessingWidget.test.jsx+++ b/client/src/components/dashboard/ActiveProcessingWidget.test.jsx@@ -106,6 +106,18 @@ describe('ActiveProcessingWidget', () => {expect(await screen.findByRole('link', { name: /Example App · update/ })).toHaveAttribute('href', '/apps');});+ it('shows a render running outside the media queue, linked to its project', async () => {+ mockGetSystemActivity.mockResolvedValue({+ jobs: [],+ extras: {},+ agents: {},+ renders: [{ kind: 'music-video-final', noun: 'music video render', to: '/music-video/mv-example', startedAt: new Date().toISOString() }],+ activity: { idle: false, activeCount: 1, queuedCount: 0, blockers: [] },+ });+ renderWidget();+ expect(await screen.findByRole('link', { name: /Music video render/ })).toHaveAttribute('href', '/music-video/mv-example');+ });+it('shows LLM model/source, elapsed time, a run deep link and Ollama residency', async () => {mockGetSystemActivity.mockResolvedValue({...idle,diff --git a/client/src/components/musicVideo/ShotFinalRender.jsx b/client/src/components/musicVideo/ShotFinalRender.jsxindex ba520ece7e..0e0757fe23 100644--- a/client/src/components/musicVideo/ShotFinalRender.jsx+++ b/client/src/components/musicVideo/ShotFinalRender.jsx@@ -47,7 +47,7 @@ export default function ShotFinalRender({ project, scene, onChange, fieldId }) {else if (value !== 'own') onChange({ filmLook: filmLookPreset(value) });};- const state = status.error ? null : rendering ? `Rendering ${Math.round((live?.progress || 0) * 100)}%` : rendered ? 'Rendered' : shot ? 'Not rendered' : 'Not in the cut';+ const state = status.error ? null : rendering ? (live ? `Rendering ${Math.round((live.progress || 0) * 100)}%` : 'Rendering') : rendered ? 'Rendered' : shot ? 'Not rendered' : 'Not in the cut';return (<section aria-label="Final render" className="min-w-0 space-y-2 rounded border border-port-border p-2"><div className="flex flex-wrap items-center gap-2">diff --git a/client/src/hooks/useSystemActivity.js b/client/src/hooks/useSystemActivity.jsindex 04f010d625..3e9a8fd897 100644--- a/client/src/hooks/useSystemActivity.js+++ b/client/src/hooks/useSystemActivity.js@@ -42,6 +42,7 @@ export const sameProcessingSnapshot = (a, b) => {&& a.activity?.activeCount === b.activity?.activeCount&& a.activity?.queuedCount === b.activity?.queuedCount&& (a.appOperations || []).length === (b.appOperations || []).length+ && JSON.stringify(a.renders || []) === JSON.stringify(b.renders || [])&& a.jobs?.length === b.jobs?.length&& (a.extras?.imageTo3d || []).length === (b.extras?.imageTo3d || []).length&& (a.jobs || []).every((job, index) => job?.id === b.jobs[index]?.iddiff --git a/server/lib/systemIdle.js b/server/lib/systemIdle.jsindex ed104f5e03..eca5a2f206 100644--- a/server/lib/systemIdle.js+++ b/server/lib/systemIdle.js@@ -119,6 +119,17 @@ export function summarizeSystemActivity(snapshot) {add('app-operations', `${pluralize(appOperations.length, 'app operation')} running`, appOperations.length);}+ // Renders outside the media queue (a music-video final render runs for an+ // hour or more); a restart through one loses it (services/renderActivity.js).+ const renders = Array.isArray(snapshot?.renders) ? snapshot.renders : [];+ const renderKinds = new Map();+ for (const render of renders) {+ const kind = typeof render?.kind === 'string' ? render.kind : 'render';+ const noun = typeof render?.noun === 'string' && render.noun ? render.noun : 'render';+ renderKinds.set(kind, { noun, count: (renderKinds.get(kind)?.count || 0) + 1 });+ }+ for (const [kind, { noun, count }] of renderKinds) add(`render:${kind}`, `${pluralize(count, noun)} running`, count);+if (snapshot?.update?.inProgress) add('update-in-progress', 'An update is already running', 1);// Milder than the others: a killed backup wastes a snapshot rather than@@ -128,7 +139,7 @@ export function summarizeSystemActivity(snapshot) {const activeCount = jobs.filter((job) => job?.status === 'running').length+ imageTo3d + activeAgents + activeLlmRuns + (mind?.thinking ? 1 : 0) + appOperations.length- + backupActive;+ + backupActive + renders.length;const queuedCount = jobs.filter((job) => job?.status === 'queued').length+ queuedAgents + (Number(mind?.queued) || 0);diff --git a/server/lib/systemIdle.test.js b/server/lib/systemIdle.test.jsindex 22634f440d..dc5df4024c 100644--- a/server/lib/systemIdle.test.js+++ b/server/lib/systemIdle.test.js@@ -107,6 +107,22 @@ describe('system idle verdict', () => {// Milder than the others (a wasted snapshot, not lost work), but a running// backup still gets killed mid-rsync by an unattended restart.+ // Renders outside the media queue; restarting through one lost a multi-hour+ // music-video render to the unattended updater.+ it('counts each kind of running render as activity', () => {+ const verdict = summarizeSystemActivity(snapshot({ renders: [+ { kind: 'music-video-final', noun: 'music video render' },+ { kind: 'video-timeline', noun: 'video timeline render' },+ { kind: 'video-timeline', noun: 'video timeline render' },+ ] }));+ expect(verdict.idle).toBe(false);+ expect(verdict.activeCount).toBe(3);+ expect(verdict.blockers).toEqual([+ { kind: 'render:music-video-final', label: '1 music video render running', count: 1 },+ { kind: 'render:video-timeline', label: '2 video timeline renders running', count: 2 },+ ]);+ });+it('counts a running backup snapshot as activity', () => {const verdict = summarizeSystemActivity(snapshot({ backup: { inProgress: true } }));expect(verdict.idle).toBe(false);diff --git a/server/routes/musicVideo.js b/server/routes/musicVideo.jsindex 87d70b063b..9a9ba48300 100644--- a/server/routes/musicVideo.js+++ b/server/routes/musicVideo.js@@ -708,8 +708,9 @@ router.get('/:id/sharing-copy/download', asyncHandler(async (req, res) => {// Assemble the scenes' i2v clips into one MP4 over the track as the master audio// bed. Kickoff returns { jobId }; progress streams over SSE (mirrors// videoTimeline). Per-project mutex returns 409 with the live jobId for re-attach.+// A shot-by-shot render started here picks up again after a server restart.router.post('/:id/render', asyncHandler(async (req, res) => {- res.json(await renderMusicVideo(req.params.id));+ res.json(await renderMusicVideo(req.params.id, { resumable: true }));}));// Read-only: the live final-render job for a project (null when none runs ondiff --git a/server/routes/musicVideo.test.js b/server/routes/musicVideo.test.jsindex 688c3a6160..5e335f8648 100644--- a/server/routes/musicVideo.test.js+++ b/server/routes/musicVideo.test.js@@ -840,7 +840,7 @@ describe('musicVideo routes', () => {it('POST /:id/render kicks off the render and returns the jobId', async () => {const r = await request(app).post('/api/music-video/mv-1/render').send({});expect(r.status).toBe(200);- expect(renderSvc.renderMusicVideo).toHaveBeenCalledWith('mv-1');+ expect(renderSvc.renderMusicVideo).toHaveBeenCalledWith('mv-1', { resumable: true });expect(r.body).toEqual({ jobId: 'job-1' });});diff --git a/server/services/activeProcessing.js b/server/services/activeProcessing.jsindex f572e54d0f..33c1e14461 100644--- a/server/services/activeProcessing.js+++ b/server/services/activeProcessing.js@@ -8,6 +8,7 @@ import { getLoadedModels, getLastLoadedModelsError } from './ollamaManager.js';import { listActiveAppOperations } from './appOperations.js';import { readPersistentMindStateForSafetyCheck } from './cosState.js';import { isUpdateInProgress } from './updateChecker.js';+import { listActiveRenders } from './renderActivity.js';import * as cos from './cos.js';// Lazy — NOT a static top-level import. `services/runner.js` and@@ -170,6 +171,9 @@ export async function getSystemActivity({ includeOllama = false } = {}) {// deciding whether it may restart the install.appOperations: listActiveAppOperations(),update: { inProgress: isUpdateInProgress() },+ // Final renders, draft excerpts and shot renders run their own ffmpeg and+ // browsers outside the media queue; an in-process count, no I/O.+ renders: listActiveRenders(),// No trusted contract needed: an in-process flag with no I/O once loaded.backup: { inProgress: backupInProgress },};diff --git a/server/services/codeAnimation/stages.js b/server/services/codeAnimation/stages.jsindex e1b2edfc1b..6d0688a4ee 100644--- a/server/services/codeAnimation/stages.js+++ b/server/services/codeAnimation/stages.js@@ -29,6 +29,7 @@ import { checkSoundConsent, produceSoundtrack, muxSoundtrack } from './sound.js'import { renderViaMediaQueue, sampleFilm } from './stageRender.js';import { resolveBlenderExecution } from './execution.js';import { renderBlenderSequence, publishBlenderVideo } from './blenderRender.js';+import { trackedRenderMap } from '../renderActivity.js';export const STAGE_RUN_KIND = 'production-stages';const STYLE_POINTS = [0.1, 0.5, 0.9];@@ -38,7 +39,7 @@ const RESUMABLE = new Set(['interrupted', 'canceled', 'exhausted', 'failed']);const BUDGET_CODE = 'CODE_ANIMATION_BUDGET_EXHAUSTED';const CANCEL_CODE = 'CODE_ANIMATION_RUN_CANCELED';-const activeRuns = new Map();+const activeRuns = trackedRenderMap('code-animation', { noun: 'Code Animation production run', link: () => '/code-animation/production' });export const activeStageRunIds = () => [...activeRuns.keys()];const exhausted = dimension => new ServerError(`Production budget exhausted: ${dimension}`, { status: 409, code: BUDGET_CODE, context: { dimension } });diff --git a/server/services/musicVideo/excerptRender.js b/server/services/musicVideo/excerptRender.jsindex 720a00243c..f56f31146a 100644--- a/server/services/musicVideo/excerptRender.js+++ b/server/services/musicVideo/excerptRender.js@@ -49,9 +49,10 @@ import { musicVideoEvents } from './events.js';import { pilotDependencies, productionPilotRenderProject } from './productionPilot.js';import { musicVideoDependencyChanges } from '../../lib/musicVideoDependencies.js';import { excerptRangeFits } from '../../lib/musicVideoExcerptRange.js';+import { trackedRenderMap } from '../renderActivity.js';const jobs = new Map();-const projectExcerptRenders = new Map();+const projectExcerptRenders = trackedRenderMap('music-video-excerpt', { noun: 'music video draft render', link: (key) => `/music-video/${encodeURIComponent(key)}` });const PENDING = Symbol('mv-excerpt-render-pending');export const attachExcerptRenderSseClient = (jobId, res) => attachSse(jobs, jobId, res);diff --git a/server/services/musicVideo/projectsLogic.js b/server/services/musicVideo/projectsLogic.jsindex 047cbdf602..6b49d0ff59 100644--- a/server/services/musicVideo/projectsLogic.js+++ b/server/services/musicVideo/projectsLogic.js@@ -308,6 +308,7 @@ export function cloneProjectRecord(source, {// #9010: the source's in-flight render mark is not the clone's.renderingOn: null,renderPartialFilename: null,+ renderResume: null,deleted: false,deletedAt: null,};diff --git a/server/services/musicVideo/publishKit.js b/server/services/musicVideo/publishKit.jsindex 332fb4c03f..6e07f31638 100644--- a/server/services/musicVideo/publishKit.js+++ b/server/services/musicVideo/publishKit.js@@ -27,9 +27,10 @@ import { musicVideoAspect } from '../../lib/musicVideoAspect.js';import { musicVideoDependencyChanges } from '../../lib/musicVideoDependencies.js';import { buildChapters, buildSrt, buildPublishCopyPrompt, parsePublishCopy, normalizeCopyOptions, PUBLISH_PLATFORMS } from './publishKitText.js';import { canonicalizeSunoUrl, isSunoShareLink } from '../sunoShareLink.js';+import { trackedRenderMap } from '../renderActivity.js';const jobs = new Map();-const projectBuilds = new Map();+const projectBuilds = trackedRenderMap('music-video-publish-kit', { noun: 'publishing kit build', link: (projectId) => `/music-video/${encodeURIComponent(projectId)}` });const MAX_THUMBNAILS = 6;// X caps a Premium upload's bitrate well under a 1080p master's; ~12 Mbps// keeps the analog grain without the upload being re-crushed.diff --git a/server/services/musicVideo/render.js b/server/services/musicVideo/render.jsindex 88a5a59750..c8ff932d9a 100644--- a/server/services/musicVideo/render.js+++ b/server/services/musicVideo/render.js@@ -64,11 +64,12 @@ import { cutEffectCuts, plannedTransitions, sceneCutEffect } from '../../lib/musimport { imageEffectGlitchFrames, mergeGlitchFrames, musicVideoTransitionFilter, sceneTransitions, shotGlitchFrames, transitionGlitchFrames } from '../../lib/musicVideoTransitions.js';import { stillMoveForCamera } from '../../lib/cameraMovements.js';import { isEndFrameImageEffect, musicVideoImageEffectFilter, sceneImageEffect } from '../../lib/musicVideoImageEffects.js';+import { trackedRenderMap } from '../renderActivity.js';// Per-project render mutex (keyed by projectId so two projects can render in// parallel; same-project re-entry returns 409 with the live jobId for re-attach).const jobs = new Map();-const projectRenders = new Map();+const projectRenders = trackedRenderMap('music-video-final', { noun: 'music video render', link: (key) => `/music-video/${encodeURIComponent(key)}` });// Reserved synchronously the instant a render is accepted, BEFORE the async// prep (ffmpeg/audio/clip resolution) — so a double-click or second client// can't pass the 409 check during that await window and spawn a duplicate.@@ -97,7 +98,7 @@ export const isLocalRenderMark = (renderingOn, instanceId) => !renderingOn || re// Every terminal project write clears the in-flight mark's stamp and the// partial-output pointer alongside the status.-const settledRender = (status, extra = {}) => ({ status, ...extra, renderingOn: null, renderPartialFilename: null });+const settledRender = (status, extra = {}) => ({ status, ...extra, renderingOn: null, renderPartialFilename: null, renderResume: null });// Tell listeners (the autonomous run waiting on its final render) that a render// job reached a terminal state. Emitted after the project record's terminal@@ -1039,7 +1040,10 @@ async function renderFromShotSegments(projectId, project, handOff, options) {maintenance.markCurrentUnsettled();console.error(`❌ Music-video render [${jobId.slice(0, 8)}] project ${projectId.slice(0, 8)} status→${status} write failed: ${err.message}`);});- await writeProject({ status: 'rendering', renderingOn, renderPartialFilename: filename, renderError: null }, 'rendering');+ // The mark boot recovery reads to start this render again after a restart:+ // the shots it finished are cached, so the rerun picks up where it stopped.+ const renderResume = options.resumable ? { attempt: options.resumeAttempt || 0 } : null;+ await writeProject({ status: 'rendering', renderingOn, renderPartialFilename: filename, renderError: null, renderResume }, 'rendering');const all = [...plan.segments, ...(plan.outro ? [plan.outro] : [])];const cached = all.filter((segment) => existsSync(segment.path)).length;console.log(`🎬 Rendering music video [${jobId.slice(0, 8)}]: project=${projectId.slice(0, 8)} segments=${all.length} cached=${cached} duration=${plan.songSec.toFixed(2)}s`);@@ -1448,6 +1452,7 @@ export async function recoverStuckMusicVideoRenders() {// its (finished) file — that file is the gallery's, not a partial.let finished = null;let recovered = 0;+ const resumes = [];for (const project of stuck) {// Re-checked against the FRESHEST record under the write lock: a peer sync// landing between the list and this write may have handed the mark to@@ -1457,13 +1462,14 @@ export async function recoverStuckMusicVideoRenders() {&& isLocalRenderMark(current.renderingOn, instanceId);if (!stillStuck) return { project: current, demoted: false };const targetStatus = current.renderHistoryId ? 'complete' : 'ready';- return { project: applyProjectPatch(current, settledRender(targetStatus)), demoted: true, partial: current.renderPartialFilename };+ return { project: applyProjectPatch(current, settledRender(targetStatus)), demoted: true, partial: current.renderPartialFilename, resume: current.renderResume };}).catch((err) => {console.error(`❌ Music Video recovery: project ${project.id.slice(0, 8)} demotion write failed: ${err.message}`);return null;});if (!outcome?.demoted) continue;recovered++;+ if (outcome.resume && (Number(outcome.resume.attempt) || 0) < MAX_RENDER_RESUMES) resumes.push({ projectId: project.id, attempt: (Number(outcome.resume.attempt) || 0) + 1 });// Only once the record no longer points at it. The filename came off a// (possibly peer-synced) record, so it is only resolved inside videos/.const { partial } = outcome;@@ -1477,4 +1483,28 @@ export async function recoverStuckMusicVideoRenders() {if (partialPath) await unlink(partialPath).catch(() => {});}if (stuck.length > 0) console.log(`🎬 Music Video boot recovery: demoted ${recovered}/${stuck.length} stuck render(s)`);+ if (resumes.length > 0) scheduleRenderResumes(resumes);+}++// A shot-by-shot final render the user started is started again after a+// restart: every shot it finished is cached, so the rerun only renders the+// shots left and stitches. Bounded, so a render that itself takes the server+// down is not restarted forever.+const MAX_RENDER_RESUMES = 2;+const RENDER_RESUME_DELAY_MS = 15_000;++function scheduleRenderResumes(resumes) {+ // After boot settles (maintenance admission, the media lane): the render+ // admits like any other.+ setTimeout(async () => {+ for (const { projectId, attempt } of resumes) {+ try {+ const { jobId } = await renderMusicVideo(projectId, { resumable: true, resumeAttempt: attempt });+ console.log(`🎬 Music Video boot recovery: resumed the final render of ${projectId.slice(0, 8)} [${jobId.slice(0, 8)}] from its cached shots`);+ } catch (err) {+ console.error(`❌ Music Video boot recovery: could not resume the final render of ${projectId.slice(0, 8)}: ${err.message}`);+ await updateProject(projectId, { renderError: `Interrupted by a server restart and could not restart: ${err.message}`.slice(0, 2000) }).catch(() => {});+ }+ }+ }, RENDER_RESUME_DELAY_MS).unref?.();}diff --git a/server/services/musicVideo/render.terminal.test.js b/server/services/musicVideo/render.terminal.test.jsindex 84b803f262..baa80805ed 100644--- a/server/services/musicVideo/render.terminal.test.js+++ b/server/services/musicVideo/render.terminal.test.js@@ -113,7 +113,7 @@ import { maintenance } from '../../lib/maintenanceAdmission.js';const tick = () => new Promise((r) => setTimeout(r, 0));const lastProc = () => h.procs[h.procs.length - 1];// A terminal project write: the status plus the cleared in-flight mark (#9010).-const settled = (status, extra = {}) => ({ status, ...extra, renderingOn: null, renderPartialFilename: null });+const settled = (status, extra = {}) => ({ status, ...extra, renderingOn: null, renderPartialFilename: null, renderResume: null });const prime = (projectId) => {getProject.mockResolvedValue({@@ -421,6 +421,35 @@ describe('recoverStuckMusicVideoRenders (#8430)', () => {errorSpy.mockRestore();logSpy.mockRestore();});++ it('starts an interrupted shot-by-shot render again from its cached shots, a bounded number of times', async () => {+ const { recoverStuckMusicVideoRenders } = await import('./render.js');+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});+ vi.useFakeTimers({ toFake: ['setTimeout'] });+ prime('resume-1');+ getProject.mockResolvedValue({+ id: 'resume-1', name: 'P', trackId: 't1', status: 'ready',+ scenes: [{ sceneId: 's1', order: 0, videoHistoryId: 'h1' }],+ composition: { version: 1, mode: 'composed', textCues: [], style: { color: '#ffffff', font: 'sans' }, posterSec: null },+ });+ const store = seedStore([+ { id: 'resume-1', status: 'rendering', renderResume: { attempt: 0 } },+ { id: 'spent-1', status: 'rendering', renderResume: { attempt: 2 } },+ { id: 'plain-2', status: 'rendering', renderResume: null },+ ]);++ await recoverStuckMusicVideoRenders();+ expect(statusOf(store)).toEqual({ 'resume-1': 'ready', 'spent-1': 'ready', 'plain-2': 'ready' });+ expect(store.get('resume-1').renderResume).toBeNull();+ expect(seg.plans).toHaveLength(0);++ await vi.runOnlyPendingTimersAsync();+ vi.useRealTimers();+ await tick();+ expect(seg.plans.map((plan) => plan.project.id)).toEqual(['resume-1']);+ expect(updateProject).toHaveBeenCalledWith('resume-1', expect.objectContaining({ status: 'rendering', renderResume: { attempt: 1 } }));+ logSpy.mockRestore();+ });});it('chooses the loudest section midpoint within the rendered duration', async () => {@@ -525,6 +554,13 @@ describe('composed render lifecycle (#8984)', () => {expect(lastProc().args).not.toContain('-itsoffset');});+ it('marks a render the user started as resumable after a restart', async () => {+ primeComposed('mark-1', { mode: 'composed' });+ await renderMusicVideo('mark-1', { resumable: true });+ await tick();+ expect(updateProject).toHaveBeenCalledWith('mark-1', expect.objectContaining({ status: 'rendering', renderResume: { attempt: 0 } }));+ });+it('cancels while shots render: no stitch, prior status restored, slot released', async () => {primeComposed('cancel-1', { mode: 'composed', textCues: [cue] });const { jobId } = await renderMusicVideo('cancel-1');diff --git a/server/services/musicVideo/shotSegments.js b/server/services/musicVideo/shotSegments.jsindex 464dab09f7..40bbce1f2c 100644--- a/server/services/musicVideo/shotSegments.js+++ b/server/services/musicVideo/shotSegments.js@@ -22,7 +22,7 @@import { createHash, randomUUID } from 'crypto';import { existsSync } from 'fs';import { copyFile, mkdir, readdir, rename, rm, stat, writeFile } from 'fs/promises';-import { availableParallelism, tmpdir } from 'os';+import { availableParallelism, tmpdir, totalmem } from 'os';import { join } from 'path';import { PATHS } from '../../lib/fileUtils.js';import { ServerError } from '../../lib/errorHandler.js';@@ -41,6 +41,7 @@ import { nameShots } from './productionReview.js';import { getProject } from './projects.js';import { musicVideoEvents } from './events.js';import { maintenance } from '../../lib/maintenanceAdmission.js';+import { trackedRenderMap } from '../renderActivity.js';export const SHOT_SEGMENT_DIR = 'music-video-segments';// Bump when a segment's encode changes in a way its key cannot see.@@ -278,8 +279,17 @@ async function renderShotBase(plan, segment, outputPath, { jobId, signal, onProg}}+// Each segment in flight may hold a film-look browser (about one core, since+// the filter rasterizes serially) and an encoder. Two thirds of the cores keep+// the server and the user's own apps responsive; the memory term keeps a small+// machine from running out of RAM with a browser per shot.+const SEGMENT_MAX_CONCURRENCY = 12;+const SEGMENT_BYTES_EACH = 3 * 1024 ** 3;+/** How many segments render at once: each may hold a browser and an encoder. */-export const segmentConcurrency = ({ cores = availableParallelism() } = {}) => Math.max(1, Math.min(3, Math.floor(cores / 4)));+export const segmentConcurrency = ({ cores = availableParallelism(), memoryBytes = totalmem() } = {}) => Math.max(1, Math.min(+ SEGMENT_MAX_CONCURRENCY, Math.floor((cores * 2) / 3), Math.floor(memoryBytes / SEGMENT_BYTES_EACH),+));/*** Render every missing segment of `plan` (`concurrency` at a time). `onProgress`@@ -300,6 +310,8 @@ export async function renderMissingSegments(plan, { signal, onProgress, concurreawait renderSegment(plan, segment, { signal, onProgress: (f) => { done.set(segment, f); report(); } });done.set(segment, 1);report();+ // An open shot card flips to Rendered (and can play it) mid-render.+ if (segment.sceneId) emitShotRender({ projectId: plan.project?.id, sceneId: segment.sceneId, state: 'rendered', progress: 1 });} catch (err) {if (signal?.aborted || err.code === 'CANCELED') throw err;failures.push(err);@@ -411,7 +423,10 @@ function segmentStatus(plan) {}// One shot render per shot at a time: `${projectId}:${sceneId}` → { controller }.-const shotJobs = new Map();+const shotJobs = trackedRenderMap('music-video-shot', {+ noun: 'music video shot render',+ link: (key) => { const [projectId, sceneId] = key.split(':'); return `/music-video/${encodeURIComponent(projectId)}/scene/${encodeURIComponent(sceneId)}`; },+});const shotJobKey = (projectId, sceneId) => `${projectId}:${sceneId}`;const emitShotRender = (payload) => musicVideoEvents.emit('shot-render', payload);diff --git a/server/services/musicVideo/shotSegments.test.js b/server/services/musicVideo/shotSegments.test.jsindex 46511173e8..995087971a 100644--- a/server/services/musicVideo/shotSegments.test.js+++ b/server/services/musicVideo/shotSegments.test.js@@ -27,6 +27,7 @@ vi.mock('./render.js', async (importOriginal) => {const { planShotSegments, renderMissingSegments, stitchSegments } = await import('./shotSegments.js');const { buildMusicVideoFfmpegArgs } = await import('./render.js');+const { musicVideoEvents } = await import('./events.js');afterAll(cleanupTempDataRoots);@@ -66,7 +67,13 @@ describe.skipIf(!hasFfmpeg)('shot segments', () => {it('stitch to the picture the whole-film graph draws, with the song under it', async () => {const plan = await planShotSegments(project());expect(plan.segments.map((s) => [s.sceneId, s.frames])).toEqual([['a', 24], ['b', 24], ['c', 24]]);+ // Each shot a final render finishes is announced, so an open shot card can play it mid-render.+ const announced = [];+ const onShot = (event) => announced.push(`${event.projectId}:${event.sceneId}:${event.state}`);+ musicVideoEvents.on('shot-render', onShot);expect(await renderMissingSegments(plan, { concurrency: 2 })).toBe(0);+ musicVideoEvents.off('shot-render', onShot);+ expect(announced.sort()).toEqual(['a', 'b', 'c'].map((id) => `mv-segments-test:${id}:rendered`));const out = join(ROOT(), 'stitched.mp4');await stitchSegments(plan, out);expect(probe(out, 'stream=codec_type').sort()).toEqual(['audio', 'video']);diff --git a/server/services/renderActivity.js b/server/services/renderActivity.jsnew file mode 100644index 0000000000..f5c5a4bbb0--- /dev/null+++ b/server/services/renderActivity.js@@ -0,0 +1,65 @@+/**+ * Renders this process is running outside the media job queue: music-video+ * final renders, draft excerpts and single-shot renders, video timeline+ * renders, publishing-kit builds and Code Animation production runs. Each runs+ * its own ffmpeg, browsers or Blender, so the system idle verdict+ * (lib/systemIdle.js), the update preflight and Live activity read them here;+ * without this, the unattended updater restarted the server through a final+ * render.+ *+ * A leaf on purpose: each render module keeps its in-flight slots in a map made+ * here, so reading the list never imports the render code (a module that never+ * loaded has nothing running), and a slot opening or closing invalidates the+ * shared activity snapshot like any other lifecycle edge.+ */++import { noteSystemActivity } from './systemActivityNotify.js';++const kinds = new Map();++/**+ * A Map of one kind's in-flight render slots that counts toward system activity+ * and notes each slot opening and closing. `noun` names one slot in blocker+ * lines ("music video render"); `link(key, value)` is the page that owns a slot.+ */+export function trackedRenderMap(kind, { noun = 'render', link = () => null } = {}) {+ const map = new Map();+ const started = new Map();+ const set = map.set.bind(map);+ const remove = map.delete.bind(map);+ map.set = (key, value) => {+ const opened = !map.has(key);+ set(key, value);+ if (opened) {+ started.set(key, new Date().toISOString());+ noteSystemActivity('render', 'start');+ }+ return map;+ };+ map.delete = (key) => {+ const closed = remove(key);+ if (closed) {+ started.delete(key);+ noteSystemActivity('render', countActiveRenders() ? 'completion' : 'drained');+ }+ return closed;+ };+ kinds.set(kind, { map, started, noun, link });+ return map;+}++/** Every tracked render in flight: `{ kind, noun, to, startedAt }`. */+export function listActiveRenders() {+ const renders = [];+ for (const [kind, { map, started, noun, link }] of kinds) {+ for (const [key, value] of map) renders.push({ kind, noun, to: link(key, value) || null, startedAt: started.get(key) || null });+ }+ return renders;+}++/** Total tracked renders in flight in this process. */+function countActiveRenders() {+ let total = 0;+ for (const { map } of kinds.values()) total += map.size;+ return total;+}diff --git a/server/services/systemActivityNotify.js b/server/services/systemActivityNotify.jsindex 5ddcb3c1d3..7b1d070027 100644--- a/server/services/systemActivityNotify.js+++ b/server/services/systemActivityNotify.js@@ -14,7 +14,7 @@import { isTestRunner } from '../lib/runtimeEnv.js';const ACTIVITY_SOURCES = new Set([- 'agents', 'media', 'imageTo3d', 'llm', 'mind', 'appOperations', 'backup', 'update',+ 'agents', 'media', 'imageTo3d', 'llm', 'mind', 'appOperations', 'backup', 'update', 'render',]);export const SYSTEM_ACTIVITY_COALESCE_MS = 50;diff --git a/server/services/updatePreflight.js b/server/services/updatePreflight.jsindex dbf3499143..b3343f6dd5 100644--- a/server/services/updatePreflight.js+++ b/server/services/updatePreflight.js@@ -4,6 +4,8 @@ import { getActiveAgentIds, spawningTasks } from './agentState.js';import { readPersistentMindStateForSafetyCheck } from './cosState.js';import { filterLiveAgentIds } from './cosAgentLifecycle.js';import * as updateChecker from './updateChecker.js';+import { listActiveRenders } from './renderActivity.js';+import { summarizeSystemActivity } from '../lib/systemIdle.js';// Shared refusal logic for the PortOS self-update path, called from both// POST /api/update/execute (server/routes/update.js) and the app:update@@ -72,6 +74,19 @@ export async function getPersistentMindImageWorkGuard() {return persistentMindImageWorkGuard(snapshot.persistentMind);}+/**+ * The update restarts the server, which ends every in-flight render outside the+ * media queue. `renders` is a `listActiveRenders()` list; the line names each+ * kind the way Live activity and the idle verdict do.+ */+export const rendersActiveError = (renders) => {+ const running = summarizeSystemActivity({ renders }).blockers.map((blocker) => blocker.label).join(', ');+ return new ServerError(+ `${running}. Let ${renders.length === 1 ? 'it' : 'them'} finish, or stop ${renders.length === 1 ? 'it' : 'them'}, before updating: the update restarts PortOS and ${renders.length === 1 ? 'the render' : 'the renders'} would be lost.`,+ { status: 409, code: 'RENDERS_ACTIVE', context: { renders: renders.length } },+ );+};+export const persistentMindStateUntrustedError = () => new ServerError('Persistent Mind state could not be validated. Restore data/cos/state.json from backup before updating.',{ status: 409, code: 'PERSISTENT_MIND_STATE_UNTRUSTED' },@@ -104,6 +119,8 @@ export async function checkPortosUpdatePreflight({} = {}) {const activeCosAgents = await countActiveCosAgents();if (activeCosAgents > 0) throw agentsActiveError(activeCosAgents);+ const renders = listActiveRenders();+ if (renders.length > 0) throw rendersActiveError(renders);const imageCheck = await getPersistentMindImageWorkGuard();if (!imageCheck.trusted) throw persistentMindStateUntrustedError();diff --git a/server/services/updatePreflightParity.test.js b/server/services/updatePreflightParity.test.jsindex 9e32510038..6ea49262c8 100644--- a/server/services/updatePreflightParity.test.js+++ b/server/services/updatePreflightParity.test.js@@ -125,6 +125,27 @@ describe('PortOS update preflight parity — route vs. socket', () => {expect(appUpdaterUpdateApp).not.toHaveBeenCalled();});+ it('refuses identically while a render is running', async () => {+ const { trackedRenderMap } = await import('./renderActivity.js');+ const renders = trackedRenderMap('parity-test', { noun: 'music video render' });+ renders.set('project-example', 'job-example');+ try {+ const routeRes = await request(makeRouteApp()).post('/api/update/execute').send({});+ expect(routeRes.status).toBe(409);+ expect(routeRes.body.code).toBe('RENDERS_ACTIVE');+ expect(routeRes.body.error).toMatch(/^1 music video render running\. Let it finish/);++ const { fireUpdate, emitted } = makeSocketHarness();+ await fireUpdate({ appId: PORTOS_APP_ID });+ const errorEvent = emitted.find((e) => e.event === 'app:update:error');+ expect(errorEvent.payload.code).toBe('RENDERS_ACTIVE');+ expect(errorEvent.payload.message).toBe(routeRes.body.error);+ expect(appUpdaterUpdateApp).not.toHaveBeenCalled();+ } finally {+ renders.delete('project-example');+ }+ });+it('refuses identically when Persistent Mind has unacknowledged queued image work', async () => {mockCosState.persistentMind = {queuedMessages: [{ id: 'message-example', images: [{ attachmentId: 'attachment-example' }] }],diff --git a/server/services/videoTimeline/local.js b/server/services/videoTimeline/local.jsindex 01c6ea4a22..296f47f888 100644--- a/server/services/videoTimeline/local.js+++ b/server/services/videoTimeline/local.js@@ -48,6 +48,7 @@ import {} from './segments.js';import { AUDIO_NORM, buildAudioBedMix, fmtSec } from './audioBedMix.js';import { historySourceDuration, resolveClipSpan, resolveStillFades, resolveAudioSpan } from '../../lib/videoTimelineSourceSpans.js';+import { trackedRenderMap } from '../renderActivity.js';const PROJECTS_FILE = join(PATHS.data, 'video-projects.json');@@ -61,7 +62,7 @@ const DEFAULT_CANVAS = { width: 1280, height: 720, fps: 24 };// can render in parallel; same project re-render returns 409 with the// existing jobId so the UI can attach SSE instead of getting a stale failure.const jobs = new Map();-const projectRenders = new Map(); // projectId → jobId+const projectRenders = trackedRenderMap('video-timeline', { noun: 'video timeline render', link: (projectId) => `/media/timeline/${encodeURIComponent(projectId)}` }); // projectId → jobIdexport const attachSseClient = (jobId, res) => attachSse(jobs, jobId, res);