Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Save rendered clips through a backup cut and retry infrastructure failures in autopilot (#11183)
[._.]/7272e09284de
diff --git a/server/lib/backupSnapshotBoundary.js b/server/lib/backupSnapshotBoundary.jsindex 717139a81f..bb65387396 100644--- a/server/lib/backupSnapshotBoundary.js+++ b/server/lib/backupSnapshotBoundary.js@@ -32,13 +32,32 @@ function releasePublications() {for (const resolve of waiters) resolve();}-/** Hold one admission across an entire file-plus-row workflow. Nested calls reuse it. */-export async function withBackupAssetPublication(work, { timeoutMs = DEFAULT_DRAIN_TIMEOUT_MS } = {}) {- let started = false;- return maintenance.continueSettlement(() => withAdmittedBackupAssetPublication(() => {- started = true;- return work();- }, timeoutMs), { hasStarted: () => started });+const DEFAULT_WAIT_NOTICE_MS = 60_000;++/**+ * Hold one admission across an entire file-plus-row workflow. Nested calls reuse it.+ *+ * A caller publishing output that already exists (a finished render) passes+ * `onWait` with a `timeoutMs` long enough to outlast a whole backup cut, which+ * spans rsync and the database dump. `onWait` runs every `waitNoticeMs` while+ * admission is still closed, so the caller can keep its job watchdog alive and+ * say why it is waiting; only a wait that started no writes is retried.+ */+export async function withBackupAssetPublication(work, { timeoutMs = DEFAULT_DRAIN_TIMEOUT_MS, onWait = null, waitNoticeMs = DEFAULT_WAIT_NOTICE_MS } = {}) {+ const deadline = Date.now() + timeoutMs;+ for (;;) {+ let started = false;+ const attemptMs = onWait ? Math.max(0, Math.min(waitNoticeMs, deadline - Date.now())) : timeoutMs;+ try {+ return await maintenance.continueSettlement(() => withAdmittedBackupAssetPublication(() => {+ started = true;+ return work();+ }, attemptMs), { hasStarted: () => started });+ } catch (error) {+ if (!onWait || started || error?.code !== 'BACKUP_SNAPSHOT_BUSY' || Date.now() >= deadline) throw error;+ onWait(error);+ }+ }}function publicationTimeoutError() {diff --git a/server/lib/backupSnapshotBoundary.test.js b/server/lib/backupSnapshotBoundary.test.jsindex 025a03d6b9..c5272a9ab5 100644--- a/server/lib/backupSnapshotBoundary.test.js+++ b/server/lib/backupSnapshotBoundary.test.js@@ -119,6 +119,21 @@ describe('backup snapshot publication admission', () => {await expect(withBackupAssetPublication(() => 'recovered')).resolves.toBe('recovered');});+ // Uniquely catches a finished output failing because a backup cut outlasted one admission wait.+ it('keeps a patient publication waiting through a long cut, noting each wait, then runs it once', async () => {+ vi.useFakeTimers();+ const release = await acquireBackupSnapshotCut();+ const work = vi.fn(() => 'saved');+ const onWait = vi.fn();+ const pending = withBackupAssetPublication(work, { timeoutMs: 1_000, waitNoticeMs: 100, onWait });+ await vi.advanceTimersByTimeAsync(350);+ expect(onWait).toHaveBeenCalledTimes(3);+ expect(work).not.toHaveBeenCalled();+ release();+ await expect(pending).resolves.toBe('saved');+ expect(work).toHaveBeenCalledOnce();+ });+it('lets a listener spawned by an admitted workflow join it only while that workflow holds its lease', async () => {const listenerWrite = deferred();let listener;diff --git a/server/services/mediaJobQueue/index.js b/server/services/mediaJobQueue/index.jsindex 19e967ea78..19b0a95661 100644--- a/server/services/mediaJobQueue/index.js+++ b/server/services/mediaJobQueue/index.js@@ -482,7 +482,12 @@ const pendingTransitions = new Map();// from staging the terminal row through the hook fan-out, never while rendering// or waiting in the queue; failures and cancellations stay unadmitted so a// backup never delays them.-const admitCompletion = (state, commit) => (state === 'completed' ? withBackupAssetPublication(commit) : commit());+// The output is already rendered, so the commit waits out a whole backup cut+// rather than stranding a finished job half-terminated.+const COMPLETION_ADMISSION_MS = 60 * 60 * 1000;+const admitCompletion = (state, commit) => (state === 'completed'+ ? withBackupAssetPublication(commit, { timeoutMs: COMPLETION_ADMISSION_MS, onWait: () => console.warn('⏸️ media-job completion waiting for the backup to finish') })+ : commit());function stageTerminalTransition(job, apply, acknowledge) {job.terminating = true;const target = { ...job };diff --git a/server/services/musicVideo/autonomousService.js b/server/services/musicVideo/autonomousService.jsindex 76200e4cfb..da329e7379 100644--- a/server/services/musicVideo/autonomousService.js+++ b/server/services/musicVideo/autonomousService.js@@ -71,7 +71,7 @@ import { getMusicVideoCharacterStyle } from '../../lib/musicVideoCharacterStylesimport { RENDER_TARGET } from '../../lib/renderTargets.js';import { IMAGE_GEN_MODE } from '../../lib/generationModes.js';import { assertFootageVideoModelsCapable, loadPoolEnv } from './productionPool.js';-import { PRODUCTION_RESUMABLE_STATUSES, acceptProductionPilot, productionTargets } from './production.js';+import { PRODUCTION_RESUMABLE_STATUSES, PRODUCTION_TRANSIENT_RETRIES, acceptProductionPilot, productionTargets, transientProductionRetry } from './production.js';import { currentPilotPass, pilotInputBasis, pilotRepair } from './productionPilot.js';import {AUTONOMOUS_DEFAULT_LIMITS,@@ -1782,6 +1782,8 @@ async function resumeDelegatedProduction(projectId, run, { swapModels = false, l// Only approving the stills goes on to the clips; any other resume reviews them (again).settleStills(projectId, run, adopt.id).catch((err) => console.error(`❌ Autonomous music video ${short(run.id)} stills review: ${err.message}`));} else {+ // A footage run started before shots were composed by default renders its drafts from them.+ if (!rendersFromCode(project, run)) await ensureFootageComposition(projectId);// Resuming a "running" run is safe and re-pins one left by a previous server process.const productionLimits = limits ? Object.fromEntries(Object.entries(limits).filter(([, v]) => v != null)) : undefined;const failure = await deps.resumeProduction(projectId, adopt.id, {@@ -1945,9 +1947,9 @@ async function approvePilotProof(projectId, run, production) {}/**- * Production stopped (`blocked`) on something a run set to approve can settle- * itself. Returns true when it settled it and production resumed; false parks- * the run for the director as before.+ * Production stopped (`blocked`, or `stopped` by its draft review) on something+ * the run can settle itself. Returns true when it settled it and production+ * resumed; false parks the run for the director as before.*/async function settleBlockedProduction(projectId, run, productionRunId) {if (settlingProduction.has(productionRunId)) return true;@@ -1958,8 +1960,17 @@ async function settleBlockedProduction(projectId, run, productionRunId) {if (!production) return false;const orchestrated = isOrchestratedRun(run);const readiness = productionReadiness(project);+ const retry = production.status === 'blocked' && orchestrated ? transientProductionRetry(production) : null;let settled = false;- if (!readiness.storyboard.approved && (orchestrated || autoApproves(run, 'storyboard'))) {+ if (production.status === 'stopped') {+ // Only the draft review's own refusal, never a director's stop: a footage+ // project made before it composed its shots has no document to render.+ settled = /no composition document/i.test(production.stopReason || '') && await ensureFootageComposition(projectId);+ } else if (retry) {+ // The machine failed, not the shot: generate it again, within the slot's retries.+ console.log(`🔁 Autonomous music video ${short(run.id)} retrying ${retry.map((s) => `${s.kind} ${s.sceneId}`).join(', ')} after a transient failure (up to ${PRODUCTION_TRANSIENT_RETRIES} retries)`);+ settled = true;+ } else if (!readiness.storyboard.approved && (orchestrated || autoApproves(run, 'storyboard'))) {project = await settleProductionStage(project, run, 'art');project = await settleProductionStage(project, run, 'storyboard');settled = productionReadiness(project).storyboard.approved;@@ -2012,7 +2023,7 @@ async function onProductionEvent({ projectId, runId, run: production }) {if (run.status === 'running') await settleStills(projectId, run, runId);} else if (production.status === 'limit-reached' && run.status === 'running' && await raiseSizedLimit(projectId, runId, run, project, production)) {// The default generation limit grew to the planned shots and production goes on.- } else if (production.status === 'blocked' && run.status === 'running' && await settleBlockedProduction(projectId, run, runId)) {+ } else if (['blocked', 'stopped'].includes(production.status) && run.status === 'running' && await settleBlockedProduction(projectId, run, runId)) {// A run set to approve settled the stop itself and production goes on.} else if (PRODUCTION_PARKED.has(production.status) && run.status === 'running') {await park(projectId, 'needs-human', { error: reason || `Production is ${production.status}`, errorCode: 'PRODUCTION_PARKED' });@@ -2152,10 +2163,11 @@ async function ensureFootageComposition(projectId) {const project = await getProject(projectId);const composition = project?.composition;if (composition?.mode !== 'document' || composition.document || composition.documentDraft- || normalizeMusicVideoProductionPolicy(project.productionPolicy).strategy === 'code-first') return;+ || normalizeMusicVideoProductionPolicy(project.productionPolicy).strategy === 'code-first') return false;// The manifest is replaced whole, so its cues, cutting and grade ride along.await deps.updateProject(projectId, { composition: { ...composition, mode: 'composed' } });console.log(`🎬 Autonomous music video set ${short(projectId)} to compose its footage (it had no composition document)`);+ return true;}/** Render the film (again) and wait for the render's own event. */diff --git a/server/services/musicVideo/autonomousService.test.js b/server/services/musicVideo/autonomousService.test.jsindex 9d835cd680..bbfdc73243 100644--- a/server/services/musicVideo/autonomousService.test.js+++ b/server/services/musicVideo/autonomousService.test.js@@ -1604,6 +1604,38 @@ describe('orchestrated mode (brief.orchestrator)', () => {expect(reviews()).toContain('proof:approve');expect(doubles.resumeProduction).toHaveBeenCalledWith('mv-auto', 'mvpr-1', { acceptBasis: true });});++ // Uniquely catches an orchestrated run parking on a clip the machine failed to save, or retrying one forever.+ it('retries a clip that failed on infrastructure, then parks once the slot spent its retries', async () => {+ await pilotFixture();+ const busy = 'Timed out waiting for backup publication admission at <data>/backup-admission/cut. No publication writes started.';+ const failedClip = (n) => ({ key: `clip:mvs-a:base:${n}`, kind: 'clip', sceneId: 'mvs-a', revisionId: null, status: 'failed', error: busy });+ const production = store.get('mv-auto').productionRuns[0];+ production.steps = [failedClip(1)];+ await blocked();+ expect(doubles.resumeProduction).toHaveBeenLastCalledWith('mv-auto', 'mvpr-1', { acceptBasis: true });+ expect(runOf().status).toBe('running');++ // The slot's retries are spent: the third failure parks.+ doubles.resumeProduction.mockClear();+ production.retryAcknowledged = ['clip:mvs-a:base:1', 'clip:mvs-a:base:2'];+ production.steps = [failedClip(1), failedClip(2), failedClip(3)];+ await blocked();+ expect(doubles.resumeProduction).not.toHaveBeenCalled();+ expect(runOf()).toMatchObject({ status: 'needs-human', errorCode: 'PRODUCTION_PARKED' });+ });++ // Uniquely catches a pre-composed-default footage run parking on its draft render instead of composing its shots.+ it('composes a legacy footage project whose draft review stopped for want of a document', async () => {+ await pilotFixture();+ store.get('mv-auto').composition = { mode: 'document', textCues: [{ id: 'cue' }] };+ const stopped = { status: 'stopped', stopReason: 'The review paused: The draft could not render: This project has no composition document on this machine' };+ Object.assign(store.get('mv-auto').productionRuns[0], stopped);+ await service.__testing.onProductionEvent({ projectId: 'mv-auto', runId: 'mvpr-1', run: stopped });+ expect(store.get('mv-auto').composition).toEqual({ mode: 'composed', textCues: [{ id: 'cue' }] });+ expect(doubles.resumeProduction).toHaveBeenLastCalledWith('mv-auto', 'mvpr-1', { acceptBasis: true });+ expect(runOf().status).toBe('running');+ });});// Uniquely catches an unviewable still being recorded as an approved review while the run still waits for the director.diff --git a/server/services/musicVideo/production.js b/server/services/musicVideo/production.jsindex e98b84b625..6db8d0fd84 100644--- a/server/services/musicVideo/production.js+++ b/server/services/musicVideo/production.js@@ -395,6 +395,25 @@ function liveSlotJob(jobs, projectId, sceneId, stepKind) {const slotSteps = (run, sceneId, stepKind, revisionId = null) => run.steps.filter((s) => s.sceneId === sceneId && s.kind === stepKind && (s.revisionId || null) === revisionId);+// Failures of the machine around a generation (a backup holding publication+// admission, a lost queue record, an idle watchdog), not of its inputs.+const TRANSIENT_FAILURE = /backup publication admission|BACKUP_SNAPSHOT_BUSY|no longer in the queue history|watchdog timeout|awaiting storage recovery/i;+/** Retries per slot a run takes on its own for transient failures before it parks. */+export const PRODUCTION_TRANSIENT_RETRIES = 2;++/**+ * The failed frame/clip steps a run may retry without the director: every+ * unacknowledged failure is transient and no slot spent its retries. Null+ * otherwise, so an input-caused or repeated failure still halts.+ */+export function transientProductionRetry(run) {+ const transient = (step) => step.status === 'failed' && TRANSIENT_FAILURE.test(step.error || '');+ const failed = (run?.steps || []).filter((s) => s.status === 'failed' && !(run.retryAcknowledged || []).includes(s.key));+ if (!failed.length || !failed.every((s) => ['frame', 'clip'].includes(s.kind) && transient(s))) return null;+ const spent = failed.some((s) => slotSteps(run, s.sceneId, s.kind, s.revisionId || null).filter(transient).length > PRODUCTION_TRANSIENT_RETRIES);+ return spent ? null : failed;+}+/*** The attached full-song review's verdict on the run, or null when no review* is attached. A passing review still needs its dependencies unchanged; adiff --git a/server/services/videoGen/generateVideoHelpers.js b/server/services/videoGen/generateVideoHelpers.jsindex 733f11c04c..1b03abcbf3 100644--- a/server/services/videoGen/generateVideoHelpers.js+++ b/server/services/videoGen/generateVideoHelpers.js@@ -742,6 +742,9 @@ export function bufferChildExit(proc, { bufferOutput = false } = {}) {};}+/** How long a finished output waits for a backup cut before giving up (a cut spans rsync and the database dump). */+export const FINISHED_OUTPUT_ADMISSION_MS = 60 * 60 * 1000;+// Only fresh, unreferenced outputs belong here. Callers retain the publication// latch so a later notification failure never sends committed bytes to cleanup.export async function discardUnpublishedVideo({ jobId, outputPath }) {@@ -774,6 +777,15 @@ export async function discardUnpublishedVideo({ jobId, outputPath }) {* spawned. Defaults to `job.renderStartedAtMs`, which generateVideo stamps.*/export async function finalizeGeneratedVideo({ job, jobId, outputPath, filename, meta, actualSeed, mutateHistory, startedAtMs = job?.renderStartedAtMs, terminal = true, publication = {} }) {+ // The render already exists, so a backup cut holding admission is a pause,+ // not a failure: wait it out, heartbeating so the queue watchdog never+ // reaps a finished clip as idle.+ const waitForBackup = () => {+ const message = 'Waiting for the backup to finish before saving the video';+ videoGenEvents.emit('activity', { generationId: jobId });+ broadcastSse(job, { type: 'status', message });+ videoGenEvents.emit('status', { generationId: jobId, message });+ };const thumbnail = await withBackupAssetPublication(async () => {try {await optimizeForStreaming(outputPath);@@ -834,7 +846,7 @@ export async function finalizeGeneratedVideo({ job, jobId, outputPath, filename,await discardUnpublishedVideo({ jobId, outputPath });throw error;}- });+ }, { timeoutMs: FINISHED_OUTPUT_ADMISSION_MS, onWait: waitForBackup });if (terminal) job.status = 'complete';console.log(`✅ Video generated [${jobId.slice(0, 8)}]: ${filename}`);if (terminal) {