Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

refactor: return named recovery evidence from media job lifecycle (#10735)

Adam Eivy2ec2142cb116
diff --git a/server/services/README.md b/server/services/README.md
index 856be1a669..2f9e67be21 100644
--- a/server/services/README.md
+++ b/server/services/README.md
@@ -12,7 +12,7 @@ image-generation paths below are useful entry points when tracing render jobs.
| `databaseMaintenanceQuiescence.js` | Internal coordinator stage after producer shutdown: terminate verified running detached-writer groups, prove every registry record quiescent from process-table evidence, archive it with the operation, and refuse pending, legacy, orphaned, ambiguous or unregistered writers. |
| `imageGen/prepareParams.js` | Shared image request preparation and the canonical local image model selector. Use `selectLocalImageModelFromSettings()` for model identity and `resolveLocalImageModel()` when the caller must also enforce local runtime, hardware, and edit-image requirements. |
| `musicVideo/sharingCopy.js` | Source-hashed private sharing MP4s: queued 720p H.264/AAC export, strict decimal 100 MB cap, supervised cancellation, verified download and owned-file cleanup. |
-| `mediaJobQueue/index.js` | Owns admission, lanes, durability and terminal effects; `videoHolds.js` returns local-video cohort preparation results and owns circuit breakers, while `videoGen/modelSelection.js` owns model-selection policy. |
+| `mediaJobQueue/index.js` | Owns admission, lanes, durability and terminal effects; `videoHolds.js` returns local-video cohort preparation results and owns circuit breakers, `recoverySettlement.js` names the transient recovery evidence and the maintenance-permit release decision, while `videoGen/modelSelection.js` owns model-selection policy. |
| `imageGen/index.js` | Mode-aware image backend dispatcher and shared image-generation helpers. Direct queue consumers still resolve local model identity before enqueueing. |
| `superColliderRuntime.js` | Managed SuperCollider runtime I/O (#9412): Docker probe, recipe-hash-stamped image build, `runSuperColliderContainer` (contained `sclang` run with forced container removal), the synthetic render probe and its cached evidence, `getSuperColliderStatus` and idempotent `setupSuperColliderRuntime`. Never runs at boot. |
| `superColliderRender.js` | Contained SuperCollider renders (#9413): media-queue `supercollider` jobs that snapshot source into a job-private dir, run the trusted NRT wrapper via `runSuperColliderContainer` (output quota, timeout, cancel → container force-removed), validate the decoded WAV and publish a 24 h preview + provenance sidecar (`readSuperColliderPreview`). Sweeps orphaned containers/scratch before each render. |
diff --git a/server/services/mediaJobQueue/index.js b/server/services/mediaJobQueue/index.js
index 71cb92d9ea..4542716c02 100644
--- a/server/services/mediaJobQueue/index.js
+++ b/server/services/mediaJobQueue/index.js
@@ -55,6 +55,10 @@ import {
canReceiveProgress, canRequestCancellation, claimTerminalOutcome,
snapshotCancellationIntent, applyCancellationIntent, restoreCancellationIntent,
} from './jobLifecycle.js';
+import {
+ createRecoveryEvidence, observeProviderTerminal, recordTerminalDurable, recordOwnedInputCleanup,
+ decideOwnershipRelease, OWNERSHIP_RELEASE,
+} from './recoverySettlement.js';
import { routedJobParams } from '../federatedMedia/routedJobParams.js';
import {
MAX_PENDING_MEDIA_JOBS, MEDIA_QUEUE_PERSIST_FAILED, mediaQueueFullError,
@@ -856,7 +860,8 @@ function startLaneJob(job, { lane }) {
return archiveWrite;
}
let markRunning;
- let recoverySettled = false;
+ // Null when the lifecycle threw before returning its evidence.
+ let lifecycleEvidence = null;
trackOperation(dispatchOperations, new Promise((resolve) => { markRunning = resolve; }));
permit.run(async () => {
try {
@@ -868,7 +873,7 @@ function startLaneJob(job, { lane }) {
console.log(`▶️ media-job [${job.id.slice(0, 8)}] ${label} started`);
const lifecycle = runJob(job, permit);
markRunning();
- recoverySettled = await lifecycle;
+ lifecycleEvidence = await lifecycle;
} catch (err) {
markRunning();
// runJob threw before its own terminal handlers ran (e.g. PYTHON
@@ -893,8 +898,12 @@ function startLaneJob(job, { lane }) {
await finalizeLane();
const settled = await flushMediaJobQueue();
recoveringJobs.delete(job.id);
- if (!settled.ok) permit.markUnsettled();
- else if (recoverySettled && permit.completeRecovery) await permit.completeRecovery();
+ // Lifecycle evidence certifies recovery only together with this final flush.
+ const release = decideOwnershipRelease({
+ evidence: lifecycleEvidence, flushed: settled.ok, canCompleteRecovery: Boolean(permit.completeRecovery),
+ });
+ if (release === OWNERSHIP_RELEASE.UNSETTLED) permit.markUnsettled();
+ else if (release === OWNERSHIP_RELEASE.COMPLETE_RECOVERY) await permit.completeRecovery();
else await permit.finish();
}).catch((error) => console.error(`❌ media-job [${job.id.slice(0, 8)}] storage settlement pending: ${error.message}`));
return true;
@@ -1180,11 +1189,9 @@ async function runJobLifecycle(job, markDispatched, permit) {
let resolveProviderSettlement;
const providerSettlement = new Promise(resolve => { resolveProviderSettlement = resolve; });
let watchdogNeedsProviderSettlement = false;
- let providerInputsDisposable = false;
- let providerRecoveryConfirmed = false;
- let terminalPublished = false;
+ // Transient: returned to the lane finalizer, never persisted or projected.
+ const recovery = createRecoveryEvidence();
let inputCleanupAttempted = false;
- let inputCleanupResult = { ok: false };
let progressPersistDirty = false;
let progressPersistTimer = null;
let progressPersisting = null;
@@ -1230,17 +1237,17 @@ async function runJobLifecycle(job, markDispatched, permit) {
// (excluded from persistImpl's serialized field set).
let watchdogTimer;
async function releaseRemoteAudioInputs(disposable) {
- if (!disposable || !isRemoteMediaJob(job)) return { ok: false };
- if (disposable && isRemoteMediaJob(job) && job.params?.remotePeerReservation === true) {
+ if (!disposable || !isRemoteMediaJob(job)) return;
+ if (job.params?.remotePeerReservation === true) {
job.params = { ...job.params, remotePeerReservation: false };
try { await persist(); }
catch (_error) {
job.params = { ...job.params, remotePeerReservation: true };
maintenance.markResourceUnsettled('media', job.id);
- return { ok: false };
+ return;
}
}
- if (inputCleanupAttempted) return inputCleanupResult;
+ if (inputCleanupAttempted) return;
inputCleanupAttempted = true;
try {
// withResource deliberately skips unsettled operations. Cleanup must
@@ -1259,14 +1266,13 @@ async function runJobLifecycle(job, markDispatched, permit) {
}
// This recovery certifies the feature's owned WAV windows. Other
// staged multipart inputs have no equivalent cleanup receipt here.
- inputCleanupResult = { ok: [...owned, job.params?.uploadedTempPath, job.params?.audioFilePath]
- .filter(Boolean).every(path => cleaned.has(path)) };
+ recordOwnedInputCleanup(recovery, [...owned, job.params?.uploadedTempPath, job.params?.audioFilePath]
+ .filter(Boolean).every(path => cleaned.has(path)));
});
} catch (error) {
maintenance.markResourceUnsettled('media', job.id);
console.error(`❌ Remote video input cleanup needs recovery for ${job.id}: ${error.message}`);
}
- return inputCleanupResult;
}
async function terminate(state, apply, remoteInputsDisposable = false) {
if (!claimTerminalOutcome(job)) return;
@@ -1296,9 +1302,9 @@ async function runJobLifecycle(job, markDispatched, permit) {
// so the residual values are not displayed for terminal jobs.
job.completedAt = new Date().toISOString();
}, async () => {
- terminalPublished = true;
// The terminal snapshot is durable now; a restart no longer rebuilds
// this request. Absence of the executor verdict retains all inputs.
+ recordTerminalDurable(recovery);
// A shared persist can acknowledge another job: restore THAT job's
// maintenance context, and never let cleanup failure lose this callback.
await releaseRemoteAudioInputs(remoteInputsDisposable);
@@ -1421,12 +1427,7 @@ async function runJobLifecycle(job, markDispatched, permit) {
// cancellation has reached physical exit and provider cleanup. Only a real
// provider terminal event can release that additional wait.
const observeProviderSettlement = payload => {
- providerInputsDisposable = payload.remoteInputsDisposable === true;
- const proof = payload.remoteRecoverySettlement;
- providerRecoveryConfirmed = isRemoteMediaJob(job) && job.params?.remoteMedia?.reconcile === true
- && proof?.jobId === job.id && proof?.peerId === job.params.remoteMedia.peerId
- && typeof proof?.remoteJobId === 'string' && proof.remoteJobId.length > 0
- && ['completed', 'failed', 'canceled'].includes(proof.status) && proof.executorSettled === true;
+ observeProviderTerminal(recovery, job, payload);
resolveProviderSettlement();
};
const dispatcher = makeGenDispatcher(emitter, job, {
@@ -1598,9 +1599,10 @@ async function runJobLifecycle(job, markDispatched, permit) {
// A watchdog can win the terminal row before the real provider responds.
// Keep its later trusted verdict through physical settlement, then release
// inputs under the still-owned lane after that row is durable.
- await releaseRemoteAudioInputs(providerInputsDisposable);
+ await releaseRemoteAudioInputs(recovery.inputsDisposable);
dispatcher.detach();
- return providerRecoveryConfirmed && providerInputsDisposable && terminalPublished && inputCleanupResult.ok;
+ // The lane finalizer combines this with its final flush before releasing.
+ return recovery;
}
/**
diff --git a/server/services/mediaJobQueue/recoverySettlement.js b/server/services/mediaJobQueue/recoverySettlement.js
new file mode 100644
index 0000000000..58aa01e604
--- /dev/null
+++ b/server/services/mediaJobQueue/recoverySettlement.js
@@ -0,0 +1,80 @@
+/**
+ * Recovery evidence for one media job lifecycle, and the one decision that
+ * turns it into a maintenance-permit release.
+ *
+ * A recovered remote job may retire its maintenance ownership only when four
+ * independent facts hold, each recorded by a different asynchronous actor:
+ *
+ * - `executorTerminated` — the provider's terminal event carried an executor
+ * teardown proof bound to THIS job and peer. Public terminal status alone does
+ * not prove physical teardown.
+ * - `inputsDisposable` — the provider allows disposing the staged inputs. This
+ * can also mean "never submitted", so it never certifies termination of a
+ * previously admitted peer job.
+ * - `terminalDurable` — the terminal snapshot was persisted. A watchdog can win
+ * this row before the provider's later verdict arrives.
+ * - `ownedInputsCleaned` — every owned source-audio input was discarded under
+ * the captured permit, with a verified receipt.
+ *
+ * The record is transient: it lives in the lifecycle closure and the lane
+ * finalizer only, and never enters persisted job state, route projections or
+ * federated responses. Every operation here is synchronous; persistence,
+ * cleanup, provider waits and permit effects stay at their call sites.
+ */
+
+import { isRemoteMediaJob } from './remoteMediaJob.js';
+
+const TERMINAL_STATUSES = ['completed', 'failed', 'canceled'];
+
+export const createRecoveryEvidence = () => ({
+ executorTerminated: false,
+ inputsDisposable: false,
+ terminalDurable: false,
+ ownedInputsCleaned: false,
+});
+
+// The proof is created by the executor only after teardown (see
+// federatedMedia/remoteExecutor.js); it must name this job, its peer and a
+// terminal remote job, and only a reconciling recovery can be certified by it.
+const certifiesExecutorTermination = (job, proof) => isRemoteMediaJob(job)
+ && job.params?.remoteMedia?.reconcile === true
+ && proof?.jobId === job.id && proof?.peerId === job.params.remoteMedia.peerId
+ && typeof proof?.remoteJobId === 'string' && proof.remoteJobId.length > 0
+ && TERMINAL_STATUSES.includes(proof.status) && proof.executorSettled === true;
+
+// Each provider terminal event replaces the previous verdict: the latest real
+// provider event is the one that describes physical settlement.
+export function observeProviderTerminal(evidence, job, payload) {
+ evidence.inputsDisposable = payload.remoteInputsDisposable === true;
+ evidence.executorTerminated = certifiesExecutorTermination(job, payload.remoteRecoverySettlement);
+}
+
+export function recordTerminalDurable(evidence) {
+ evidence.terminalDurable = true;
+}
+
+export function recordOwnedInputCleanup(evidence, cleaned) {
+ evidence.ownedInputsCleaned = cleaned === true;
+}
+
+// Lifecycle evidence alone is sufficient, not final: the archive/queue flush
+// that follows the lifecycle is still required before recovery completes.
+const lifecycleCertifiesRecovery = (evidence) => evidence?.executorTerminated === true
+ && evidence.inputsDisposable === true && evidence.terminalDurable === true
+ && evidence.ownedInputsCleaned === true;
+
+export const OWNERSHIP_RELEASE = Object.freeze({
+ // Final persistence failed: keep ownership and mark it uncertain.
+ UNSETTLED: 'unsettled',
+ // Verified recovery: retire the exact observed uncertain operation.
+ COMPLETE_RECOVERY: 'complete-recovery',
+ // Ordinary finish, which deliberately preserves uncertain operations.
+ FINISH: 'finish',
+});
+
+// `evidence` is null when the lifecycle threw before returning any.
+export function decideOwnershipRelease({ evidence, flushed, canCompleteRecovery }) {
+ if (!flushed) return OWNERSHIP_RELEASE.UNSETTLED;
+ if (canCompleteRecovery && lifecycleCertifiesRecovery(evidence)) return OWNERSHIP_RELEASE.COMPLETE_RECOVERY;
+ return OWNERSHIP_RELEASE.FINISH;
+}