Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Save rendered clips through a backup cut and retry infrastructure failures in autopilot (#11183)

[._.]/7272e09284de
diff --git a/server/lib/backupSnapshotBoundary.js b/server/lib/backupSnapshotBoundary.js
index 717139a81f..bb65387396 100644
--- a/server/lib/backupSnapshotBoundary.js
+++ b/server/lib/backupSnapshotBoundary.js
@@ -32,13 +32,32 @@ function releasePublications() {
for (const resolve of waiters) resolve();
}
-/** Hold one admission across an entire file-plus-row workflow. Nested calls reuse it. */
-export async function withBackupAssetPublication(work, { timeoutMs = DEFAULT_DRAIN_TIMEOUT_MS } = {}) {
- let started = false;
- return maintenance.continueSettlement(() => withAdmittedBackupAssetPublication(() => {
- started = true;
- return work();
- }, timeoutMs), { hasStarted: () => started });
+const DEFAULT_WAIT_NOTICE_MS = 60_000;
+
+/**
+ * Hold one admission across an entire file-plus-row workflow. Nested calls reuse it.
+ *
+ * A caller publishing output that already exists (a finished render) passes
+ * `onWait` with a `timeoutMs` long enough to outlast a whole backup cut, which
+ * spans rsync and the database dump. `onWait` runs every `waitNoticeMs` while
+ * admission is still closed, so the caller can keep its job watchdog alive and
+ * say why it is waiting; only a wait that started no writes is retried.
+ */
+export async function withBackupAssetPublication(work, { timeoutMs = DEFAULT_DRAIN_TIMEOUT_MS, onWait = null, waitNoticeMs = DEFAULT_WAIT_NOTICE_MS } = {}) {
+ const deadline = Date.now() + timeoutMs;
+ for (;;) {
+ let started = false;
+ const attemptMs = onWait ? Math.max(0, Math.min(waitNoticeMs, deadline - Date.now())) : timeoutMs;
+ try {
+ return await maintenance.continueSettlement(() => withAdmittedBackupAssetPublication(() => {
+ started = true;
+ return work();
+ }, attemptMs), { hasStarted: () => started });
+ } catch (error) {
+ if (!onWait || started || error?.code !== 'BACKUP_SNAPSHOT_BUSY' || Date.now() >= deadline) throw error;
+ onWait(error);
+ }
+ }
}
function publicationTimeoutError() {
diff --git a/server/lib/backupSnapshotBoundary.test.js b/server/lib/backupSnapshotBoundary.test.js
index 025a03d6b9..c5272a9ab5 100644
--- a/server/lib/backupSnapshotBoundary.test.js
+++ b/server/lib/backupSnapshotBoundary.test.js
@@ -119,6 +119,21 @@ describe('backup snapshot publication admission', () => {
await expect(withBackupAssetPublication(() => 'recovered')).resolves.toBe('recovered');
});
+ // Uniquely catches a finished output failing because a backup cut outlasted one admission wait.
+ it('keeps a patient publication waiting through a long cut, noting each wait, then runs it once', async () => {
+ vi.useFakeTimers();
+ const release = await acquireBackupSnapshotCut();
+ const work = vi.fn(() => 'saved');
+ const onWait = vi.fn();
+ const pending = withBackupAssetPublication(work, { timeoutMs: 1_000, waitNoticeMs: 100, onWait });
+ await vi.advanceTimersByTimeAsync(350);
+ expect(onWait).toHaveBeenCalledTimes(3);
+ expect(work).not.toHaveBeenCalled();
+ release();
+ await expect(pending).resolves.toBe('saved');
+ expect(work).toHaveBeenCalledOnce();
+ });
+
it('lets a listener spawned by an admitted workflow join it only while that workflow holds its lease', async () => {
const listenerWrite = deferred();
let listener;
diff --git a/server/services/mediaJobQueue/index.js b/server/services/mediaJobQueue/index.js
index 19e967ea78..19b0a95661 100644
--- a/server/services/mediaJobQueue/index.js
+++ b/server/services/mediaJobQueue/index.js
@@ -482,7 +482,12 @@ const pendingTransitions = new Map();
// from staging the terminal row through the hook fan-out, never while rendering
// or waiting in the queue; failures and cancellations stay unadmitted so a
// backup never delays them.
-const admitCompletion = (state, commit) => (state === 'completed' ? withBackupAssetPublication(commit) : commit());
+// The output is already rendered, so the commit waits out a whole backup cut
+// rather than stranding a finished job half-terminated.
+const COMPLETION_ADMISSION_MS = 60 * 60 * 1000;
+const admitCompletion = (state, commit) => (state === 'completed'
+ ? withBackupAssetPublication(commit, { timeoutMs: COMPLETION_ADMISSION_MS, onWait: () => console.warn('⏸️ media-job completion waiting for the backup to finish') })
+ : commit());
function stageTerminalTransition(job, apply, acknowledge) {
job.terminating = true;
const target = { ...job };
diff --git a/server/services/musicVideo/autonomousService.js b/server/services/musicVideo/autonomousService.js
index 76200e4cfb..da329e7379 100644
--- a/server/services/musicVideo/autonomousService.js
+++ b/server/services/musicVideo/autonomousService.js
@@ -71,7 +71,7 @@ import { getMusicVideoCharacterStyle } from '../../lib/musicVideoCharacterStyles
import { RENDER_TARGET } from '../../lib/renderTargets.js';
import { IMAGE_GEN_MODE } from '../../lib/generationModes.js';
import { assertFootageVideoModelsCapable, loadPoolEnv } from './productionPool.js';
-import { PRODUCTION_RESUMABLE_STATUSES, acceptProductionPilot, productionTargets } from './production.js';
+import { PRODUCTION_RESUMABLE_STATUSES, PRODUCTION_TRANSIENT_RETRIES, acceptProductionPilot, productionTargets, transientProductionRetry } from './production.js';
import { currentPilotPass, pilotInputBasis, pilotRepair } from './productionPilot.js';
import {
AUTONOMOUS_DEFAULT_LIMITS,
@@ -1782,6 +1782,8 @@ async function resumeDelegatedProduction(projectId, run, { swapModels = false, l
// Only approving the stills goes on to the clips; any other resume reviews them (again).
settleStills(projectId, run, adopt.id).catch((err) => console.error(`❌ Autonomous music video ${short(run.id)} stills review: ${err.message}`));
} else {
+ // A footage run started before shots were composed by default renders its drafts from them.
+ if (!rendersFromCode(project, run)) await ensureFootageComposition(projectId);
// Resuming a "running" run is safe and re-pins one left by a previous server process.
const productionLimits = limits ? Object.fromEntries(Object.entries(limits).filter(([, v]) => v != null)) : undefined;
const failure = await deps.resumeProduction(projectId, adopt.id, {
@@ -1945,9 +1947,9 @@ async function approvePilotProof(projectId, run, production) {
}
/**
- * Production stopped (`blocked`) on something a run set to approve can settle
- * itself. Returns true when it settled it and production resumed; false parks
- * the run for the director as before.
+ * Production stopped (`blocked`, or `stopped` by its draft review) on something
+ * the run can settle itself. Returns true when it settled it and production
+ * resumed; false parks the run for the director as before.
*/
async function settleBlockedProduction(projectId, run, productionRunId) {
if (settlingProduction.has(productionRunId)) return true;
@@ -1958,8 +1960,17 @@ async function settleBlockedProduction(projectId, run, productionRunId) {
if (!production) return false;
const orchestrated = isOrchestratedRun(run);
const readiness = productionReadiness(project);
+ const retry = production.status === 'blocked' && orchestrated ? transientProductionRetry(production) : null;
let settled = false;
- if (!readiness.storyboard.approved && (orchestrated || autoApproves(run, 'storyboard'))) {
+ if (production.status === 'stopped') {
+ // Only the draft review's own refusal, never a director's stop: a footage
+ // project made before it composed its shots has no document to render.
+ settled = /no composition document/i.test(production.stopReason || '') && await ensureFootageComposition(projectId);
+ } else if (retry) {
+ // The machine failed, not the shot: generate it again, within the slot's retries.
+ console.log(`🔁 Autonomous music video ${short(run.id)} retrying ${retry.map((s) => `${s.kind} ${s.sceneId}`).join(', ')} after a transient failure (up to ${PRODUCTION_TRANSIENT_RETRIES} retries)`);
+ settled = true;
+ } else if (!readiness.storyboard.approved && (orchestrated || autoApproves(run, 'storyboard'))) {
project = await settleProductionStage(project, run, 'art');
project = await settleProductionStage(project, run, 'storyboard');
settled = productionReadiness(project).storyboard.approved;
@@ -2012,7 +2023,7 @@ async function onProductionEvent({ projectId, runId, run: production }) {
if (run.status === 'running') await settleStills(projectId, run, runId);
} else if (production.status === 'limit-reached' && run.status === 'running' && await raiseSizedLimit(projectId, runId, run, project, production)) {
// The default generation limit grew to the planned shots and production goes on.
- } else if (production.status === 'blocked' && run.status === 'running' && await settleBlockedProduction(projectId, run, runId)) {
+ } else if (['blocked', 'stopped'].includes(production.status) && run.status === 'running' && await settleBlockedProduction(projectId, run, runId)) {
// A run set to approve settled the stop itself and production goes on.
} else if (PRODUCTION_PARKED.has(production.status) && run.status === 'running') {
await park(projectId, 'needs-human', { error: reason || `Production is ${production.status}`, errorCode: 'PRODUCTION_PARKED' });
@@ -2152,10 +2163,11 @@ async function ensureFootageComposition(projectId) {
const project = await getProject(projectId);
const composition = project?.composition;
if (composition?.mode !== 'document' || composition.document || composition.documentDraft
- || normalizeMusicVideoProductionPolicy(project.productionPolicy).strategy === 'code-first') return;
+ || normalizeMusicVideoProductionPolicy(project.productionPolicy).strategy === 'code-first') return false;
// The manifest is replaced whole, so its cues, cutting and grade ride along.
await deps.updateProject(projectId, { composition: { ...composition, mode: 'composed' } });
console.log(`🎬 Autonomous music video set ${short(projectId)} to compose its footage (it had no composition document)`);
+ return true;
}
/** Render the film (again) and wait for the render's own event. */
diff --git a/server/services/musicVideo/autonomousService.test.js b/server/services/musicVideo/autonomousService.test.js
index 9d835cd680..bbfdc73243 100644
--- a/server/services/musicVideo/autonomousService.test.js
+++ b/server/services/musicVideo/autonomousService.test.js
@@ -1604,6 +1604,38 @@ describe('orchestrated mode (brief.orchestrator)', () => {
expect(reviews()).toContain('proof:approve');
expect(doubles.resumeProduction).toHaveBeenCalledWith('mv-auto', 'mvpr-1', { acceptBasis: true });
});
+
+ // Uniquely catches an orchestrated run parking on a clip the machine failed to save, or retrying one forever.
+ it('retries a clip that failed on infrastructure, then parks once the slot spent its retries', async () => {
+ await pilotFixture();
+ const busy = 'Timed out waiting for backup publication admission at <data>/backup-admission/cut. No publication writes started.';
+ const failedClip = (n) => ({ key: `clip:mvs-a:base:${n}`, kind: 'clip', sceneId: 'mvs-a', revisionId: null, status: 'failed', error: busy });
+ const production = store.get('mv-auto').productionRuns[0];
+ production.steps = [failedClip(1)];
+ await blocked();
+ expect(doubles.resumeProduction).toHaveBeenLastCalledWith('mv-auto', 'mvpr-1', { acceptBasis: true });
+ expect(runOf().status).toBe('running');
+
+ // The slot's retries are spent: the third failure parks.
+ doubles.resumeProduction.mockClear();
+ production.retryAcknowledged = ['clip:mvs-a:base:1', 'clip:mvs-a:base:2'];
+ production.steps = [failedClip(1), failedClip(2), failedClip(3)];
+ await blocked();
+ expect(doubles.resumeProduction).not.toHaveBeenCalled();
+ expect(runOf()).toMatchObject({ status: 'needs-human', errorCode: 'PRODUCTION_PARKED' });
+ });
+
+ // Uniquely catches a pre-composed-default footage run parking on its draft render instead of composing its shots.
+ it('composes a legacy footage project whose draft review stopped for want of a document', async () => {
+ await pilotFixture();
+ store.get('mv-auto').composition = { mode: 'document', textCues: [{ id: 'cue' }] };
+ const stopped = { status: 'stopped', stopReason: 'The review paused: The draft could not render: This project has no composition document on this machine' };
+ Object.assign(store.get('mv-auto').productionRuns[0], stopped);
+ await service.__testing.onProductionEvent({ projectId: 'mv-auto', runId: 'mvpr-1', run: stopped });
+ expect(store.get('mv-auto').composition).toEqual({ mode: 'composed', textCues: [{ id: 'cue' }] });
+ expect(doubles.resumeProduction).toHaveBeenLastCalledWith('mv-auto', 'mvpr-1', { acceptBasis: true });
+ expect(runOf().status).toBe('running');
+ });
});
// Uniquely catches an unviewable still being recorded as an approved review while the run still waits for the director.
diff --git a/server/services/musicVideo/production.js b/server/services/musicVideo/production.js
index e98b84b625..6db8d0fd84 100644
--- a/server/services/musicVideo/production.js
+++ b/server/services/musicVideo/production.js
@@ -395,6 +395,25 @@ function liveSlotJob(jobs, projectId, sceneId, stepKind) {
const slotSteps = (run, sceneId, stepKind, revisionId = null) => run.steps
.filter((s) => s.sceneId === sceneId && s.kind === stepKind && (s.revisionId || null) === revisionId);
+// Failures of the machine around a generation (a backup holding publication
+// admission, a lost queue record, an idle watchdog), not of its inputs.
+const TRANSIENT_FAILURE = /backup publication admission|BACKUP_SNAPSHOT_BUSY|no longer in the queue history|watchdog timeout|awaiting storage recovery/i;
+/** Retries per slot a run takes on its own for transient failures before it parks. */
+export const PRODUCTION_TRANSIENT_RETRIES = 2;
+
+/**
+ * The failed frame/clip steps a run may retry without the director: every
+ * unacknowledged failure is transient and no slot spent its retries. Null
+ * otherwise, so an input-caused or repeated failure still halts.
+ */
+export function transientProductionRetry(run) {
+ const transient = (step) => step.status === 'failed' && TRANSIENT_FAILURE.test(step.error || '');
+ const failed = (run?.steps || []).filter((s) => s.status === 'failed' && !(run.retryAcknowledged || []).includes(s.key));
+ if (!failed.length || !failed.every((s) => ['frame', 'clip'].includes(s.kind) && transient(s))) return null;
+ const spent = failed.some((s) => slotSteps(run, s.sceneId, s.kind, s.revisionId || null).filter(transient).length > PRODUCTION_TRANSIENT_RETRIES);
+ return spent ? null : failed;
+}
+
/**
* The attached full-song review's verdict on the run, or null when no review
* is attached. A passing review still needs its dependencies unchanged; a
diff --git a/server/services/videoGen/generateVideoHelpers.js b/server/services/videoGen/generateVideoHelpers.js
index 733f11c04c..1b03abcbf3 100644
--- a/server/services/videoGen/generateVideoHelpers.js
+++ b/server/services/videoGen/generateVideoHelpers.js
@@ -742,6 +742,9 @@ export function bufferChildExit(proc, { bufferOutput = false } = {}) {
};
}
+/** How long a finished output waits for a backup cut before giving up (a cut spans rsync and the database dump). */
+export const FINISHED_OUTPUT_ADMISSION_MS = 60 * 60 * 1000;
+
// Only fresh, unreferenced outputs belong here. Callers retain the publication
// latch so a later notification failure never sends committed bytes to cleanup.
export async function discardUnpublishedVideo({ jobId, outputPath }) {
@@ -774,6 +777,15 @@ export async function discardUnpublishedVideo({ jobId, outputPath }) {
* spawned. Defaults to `job.renderStartedAtMs`, which generateVideo stamps.
*/
export async function finalizeGeneratedVideo({ job, jobId, outputPath, filename, meta, actualSeed, mutateHistory, startedAtMs = job?.renderStartedAtMs, terminal = true, publication = {} }) {
+ // The render already exists, so a backup cut holding admission is a pause,
+ // not a failure: wait it out, heartbeating so the queue watchdog never
+ // reaps a finished clip as idle.
+ const waitForBackup = () => {
+ const message = 'Waiting for the backup to finish before saving the video';
+ videoGenEvents.emit('activity', { generationId: jobId });
+ broadcastSse(job, { type: 'status', message });
+ videoGenEvents.emit('status', { generationId: jobId, message });
+ };
const thumbnail = await withBackupAssetPublication(async () => {
try {
await optimizeForStreaming(outputPath);
@@ -834,7 +846,7 @@ export async function finalizeGeneratedVideo({ job, jobId, outputPath, filename,
await discardUnpublishedVideo({ jobId, outputPath });
throw error;
}
- });
+ }, { timeoutMs: FINISHED_OUTPUT_ADMISSION_MS, onWait: waitForBackup });
if (terminal) job.status = 'complete';
console.log(`✅ Video generated [${jobId.slice(0, 8)}]: ${filename}`);
if (terminal) {