Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
refactor: return named recovery evidence from media job lifecycle (#10735)
Adam Eivy2ec2142cb116
diff --git a/server/services/README.md b/server/services/README.mdindex 856be1a669..2f9e67be21 100644--- a/server/services/README.md+++ b/server/services/README.md@@ -12,7 +12,7 @@ image-generation paths below are useful entry points when tracing render jobs.| `databaseMaintenanceQuiescence.js` | Internal coordinator stage after producer shutdown: terminate verified running detached-writer groups, prove every registry record quiescent from process-table evidence, archive it with the operation, and refuse pending, legacy, orphaned, ambiguous or unregistered writers. || `imageGen/prepareParams.js` | Shared image request preparation and the canonical local image model selector. Use `selectLocalImageModelFromSettings()` for model identity and `resolveLocalImageModel()` when the caller must also enforce local runtime, hardware, and edit-image requirements. || `musicVideo/sharingCopy.js` | Source-hashed private sharing MP4s: queued 720p H.264/AAC export, strict decimal 100 MB cap, supervised cancellation, verified download and owned-file cleanup. |-| `mediaJobQueue/index.js` | Owns admission, lanes, durability and terminal effects; `videoHolds.js` returns local-video cohort preparation results and owns circuit breakers, while `videoGen/modelSelection.js` owns model-selection policy. |+| `mediaJobQueue/index.js` | Owns admission, lanes, durability and terminal effects; `videoHolds.js` returns local-video cohort preparation results and owns circuit breakers, `recoverySettlement.js` names the transient recovery evidence and the maintenance-permit release decision, while `videoGen/modelSelection.js` owns model-selection policy. || `imageGen/index.js` | Mode-aware image backend dispatcher and shared image-generation helpers. Direct queue consumers still resolve local model identity before enqueueing. || `superColliderRuntime.js` | Managed SuperCollider runtime I/O (#9412): Docker probe, recipe-hash-stamped image build, `runSuperColliderContainer` (contained `sclang` run with forced container removal), the synthetic render probe and its cached evidence, `getSuperColliderStatus` and idempotent `setupSuperColliderRuntime`. Never runs at boot. || `superColliderRender.js` | Contained SuperCollider renders (#9413): media-queue `supercollider` jobs that snapshot source into a job-private dir, run the trusted NRT wrapper via `runSuperColliderContainer` (output quota, timeout, cancel → container force-removed), validate the decoded WAV and publish a 24 h preview + provenance sidecar (`readSuperColliderPreview`). Sweeps orphaned containers/scratch before each render. |diff --git a/server/services/mediaJobQueue/index.js b/server/services/mediaJobQueue/index.jsindex 71cb92d9ea..4542716c02 100644--- a/server/services/mediaJobQueue/index.js+++ b/server/services/mediaJobQueue/index.js@@ -55,6 +55,10 @@ import {canReceiveProgress, canRequestCancellation, claimTerminalOutcome,snapshotCancellationIntent, applyCancellationIntent, restoreCancellationIntent,} from './jobLifecycle.js';+import {+ createRecoveryEvidence, observeProviderTerminal, recordTerminalDurable, recordOwnedInputCleanup,+ decideOwnershipRelease, OWNERSHIP_RELEASE,+} from './recoverySettlement.js';import { routedJobParams } from '../federatedMedia/routedJobParams.js';import {MAX_PENDING_MEDIA_JOBS, MEDIA_QUEUE_PERSIST_FAILED, mediaQueueFullError,@@ -856,7 +860,8 @@ function startLaneJob(job, { lane }) {return archiveWrite;}let markRunning;- let recoverySettled = false;+ // Null when the lifecycle threw before returning its evidence.+ let lifecycleEvidence = null;trackOperation(dispatchOperations, new Promise((resolve) => { markRunning = resolve; }));permit.run(async () => {try {@@ -868,7 +873,7 @@ function startLaneJob(job, { lane }) {console.log(`▶️ media-job [${job.id.slice(0, 8)}] ${label} started`);const lifecycle = runJob(job, permit);markRunning();- recoverySettled = await lifecycle;+ lifecycleEvidence = await lifecycle;} catch (err) {markRunning();// runJob threw before its own terminal handlers ran (e.g. PYTHON@@ -893,8 +898,12 @@ function startLaneJob(job, { lane }) {await finalizeLane();const settled = await flushMediaJobQueue();recoveringJobs.delete(job.id);- if (!settled.ok) permit.markUnsettled();- else if (recoverySettled && permit.completeRecovery) await permit.completeRecovery();+ // Lifecycle evidence certifies recovery only together with this final flush.+ const release = decideOwnershipRelease({+ evidence: lifecycleEvidence, flushed: settled.ok, canCompleteRecovery: Boolean(permit.completeRecovery),+ });+ if (release === OWNERSHIP_RELEASE.UNSETTLED) permit.markUnsettled();+ else if (release === OWNERSHIP_RELEASE.COMPLETE_RECOVERY) await permit.completeRecovery();else await permit.finish();}).catch((error) => console.error(`❌ media-job [${job.id.slice(0, 8)}] storage settlement pending: ${error.message}`));return true;@@ -1180,11 +1189,9 @@ async function runJobLifecycle(job, markDispatched, permit) {let resolveProviderSettlement;const providerSettlement = new Promise(resolve => { resolveProviderSettlement = resolve; });let watchdogNeedsProviderSettlement = false;- let providerInputsDisposable = false;- let providerRecoveryConfirmed = false;- let terminalPublished = false;+ // Transient: returned to the lane finalizer, never persisted or projected.+ const recovery = createRecoveryEvidence();let inputCleanupAttempted = false;- let inputCleanupResult = { ok: false };let progressPersistDirty = false;let progressPersistTimer = null;let progressPersisting = null;@@ -1230,17 +1237,17 @@ async function runJobLifecycle(job, markDispatched, permit) {// (excluded from persistImpl's serialized field set).let watchdogTimer;async function releaseRemoteAudioInputs(disposable) {- if (!disposable || !isRemoteMediaJob(job)) return { ok: false };- if (disposable && isRemoteMediaJob(job) && job.params?.remotePeerReservation === true) {+ if (!disposable || !isRemoteMediaJob(job)) return;+ if (job.params?.remotePeerReservation === true) {job.params = { ...job.params, remotePeerReservation: false };try { await persist(); }catch (_error) {job.params = { ...job.params, remotePeerReservation: true };maintenance.markResourceUnsettled('media', job.id);- return { ok: false };+ return;}}- if (inputCleanupAttempted) return inputCleanupResult;+ if (inputCleanupAttempted) return;inputCleanupAttempted = true;try {// withResource deliberately skips unsettled operations. Cleanup must@@ -1259,14 +1266,13 @@ async function runJobLifecycle(job, markDispatched, permit) {}// This recovery certifies the feature's owned WAV windows. Other// staged multipart inputs have no equivalent cleanup receipt here.- inputCleanupResult = { ok: [...owned, job.params?.uploadedTempPath, job.params?.audioFilePath]- .filter(Boolean).every(path => cleaned.has(path)) };+ recordOwnedInputCleanup(recovery, [...owned, job.params?.uploadedTempPath, job.params?.audioFilePath]+ .filter(Boolean).every(path => cleaned.has(path)));});} catch (error) {maintenance.markResourceUnsettled('media', job.id);console.error(`❌ Remote video input cleanup needs recovery for ${job.id}: ${error.message}`);}- return inputCleanupResult;}async function terminate(state, apply, remoteInputsDisposable = false) {if (!claimTerminalOutcome(job)) return;@@ -1296,9 +1302,9 @@ async function runJobLifecycle(job, markDispatched, permit) {// so the residual values are not displayed for terminal jobs.job.completedAt = new Date().toISOString();}, async () => {- terminalPublished = true;// The terminal snapshot is durable now; a restart no longer rebuilds// this request. Absence of the executor verdict retains all inputs.+ recordTerminalDurable(recovery);// A shared persist can acknowledge another job: restore THAT job's// maintenance context, and never let cleanup failure lose this callback.await releaseRemoteAudioInputs(remoteInputsDisposable);@@ -1421,12 +1427,7 @@ async function runJobLifecycle(job, markDispatched, permit) {// cancellation has reached physical exit and provider cleanup. Only a real// provider terminal event can release that additional wait.const observeProviderSettlement = payload => {- providerInputsDisposable = payload.remoteInputsDisposable === true;- const proof = payload.remoteRecoverySettlement;- providerRecoveryConfirmed = isRemoteMediaJob(job) && job.params?.remoteMedia?.reconcile === true- && proof?.jobId === job.id && proof?.peerId === job.params.remoteMedia.peerId- && typeof proof?.remoteJobId === 'string' && proof.remoteJobId.length > 0- && ['completed', 'failed', 'canceled'].includes(proof.status) && proof.executorSettled === true;+ observeProviderTerminal(recovery, job, payload);resolveProviderSettlement();};const dispatcher = makeGenDispatcher(emitter, job, {@@ -1598,9 +1599,10 @@ async function runJobLifecycle(job, markDispatched, permit) {// A watchdog can win the terminal row before the real provider responds.// Keep its later trusted verdict through physical settlement, then release// inputs under the still-owned lane after that row is durable.- await releaseRemoteAudioInputs(providerInputsDisposable);+ await releaseRemoteAudioInputs(recovery.inputsDisposable);dispatcher.detach();- return providerRecoveryConfirmed && providerInputsDisposable && terminalPublished && inputCleanupResult.ok;+ // The lane finalizer combines this with its final flush before releasing.+ return recovery;}/**diff --git a/server/services/mediaJobQueue/recoverySettlement.js b/server/services/mediaJobQueue/recoverySettlement.jsnew file mode 100644index 0000000000..58aa01e604--- /dev/null+++ b/server/services/mediaJobQueue/recoverySettlement.js@@ -0,0 +1,80 @@+/**+ * Recovery evidence for one media job lifecycle, and the one decision that+ * turns it into a maintenance-permit release.+ *+ * A recovered remote job may retire its maintenance ownership only when four+ * independent facts hold, each recorded by a different asynchronous actor:+ *+ * - `executorTerminated` — the provider's terminal event carried an executor+ * teardown proof bound to THIS job and peer. Public terminal status alone does+ * not prove physical teardown.+ * - `inputsDisposable` — the provider allows disposing the staged inputs. This+ * can also mean "never submitted", so it never certifies termination of a+ * previously admitted peer job.+ * - `terminalDurable` — the terminal snapshot was persisted. A watchdog can win+ * this row before the provider's later verdict arrives.+ * - `ownedInputsCleaned` — every owned source-audio input was discarded under+ * the captured permit, with a verified receipt.+ *+ * The record is transient: it lives in the lifecycle closure and the lane+ * finalizer only, and never enters persisted job state, route projections or+ * federated responses. Every operation here is synchronous; persistence,+ * cleanup, provider waits and permit effects stay at their call sites.+ */++import { isRemoteMediaJob } from './remoteMediaJob.js';++const TERMINAL_STATUSES = ['completed', 'failed', 'canceled'];++export const createRecoveryEvidence = () => ({+ executorTerminated: false,+ inputsDisposable: false,+ terminalDurable: false,+ ownedInputsCleaned: false,+});++// The proof is created by the executor only after teardown (see+// federatedMedia/remoteExecutor.js); it must name this job, its peer and a+// terminal remote job, and only a reconciling recovery can be certified by it.+const certifiesExecutorTermination = (job, proof) => isRemoteMediaJob(job)+ && job.params?.remoteMedia?.reconcile === true+ && proof?.jobId === job.id && proof?.peerId === job.params.remoteMedia.peerId+ && typeof proof?.remoteJobId === 'string' && proof.remoteJobId.length > 0+ && TERMINAL_STATUSES.includes(proof.status) && proof.executorSettled === true;++// Each provider terminal event replaces the previous verdict: the latest real+// provider event is the one that describes physical settlement.+export function observeProviderTerminal(evidence, job, payload) {+ evidence.inputsDisposable = payload.remoteInputsDisposable === true;+ evidence.executorTerminated = certifiesExecutorTermination(job, payload.remoteRecoverySettlement);+}++export function recordTerminalDurable(evidence) {+ evidence.terminalDurable = true;+}++export function recordOwnedInputCleanup(evidence, cleaned) {+ evidence.ownedInputsCleaned = cleaned === true;+}++// Lifecycle evidence alone is sufficient, not final: the archive/queue flush+// that follows the lifecycle is still required before recovery completes.+const lifecycleCertifiesRecovery = (evidence) => evidence?.executorTerminated === true+ && evidence.inputsDisposable === true && evidence.terminalDurable === true+ && evidence.ownedInputsCleaned === true;++export const OWNERSHIP_RELEASE = Object.freeze({+ // Final persistence failed: keep ownership and mark it uncertain.+ UNSETTLED: 'unsettled',+ // Verified recovery: retire the exact observed uncertain operation.+ COMPLETE_RECOVERY: 'complete-recovery',+ // Ordinary finish, which deliberately preserves uncertain operations.+ FINISH: 'finish',+});++// `evidence` is null when the lifecycle threw before returning any.+export function decideOwnershipRelease({ evidence, flushed, canCompleteRecovery }) {+ if (!flushed) return OWNERSHIP_RELEASE.UNSETTLED;+ if (canCompleteRecovery && lifecycleCertifiesRecovery(evidence)) return OWNERSHIP_RELEASE.COMPLETE_RECOVERY;+ return OWNERSHIP_RELEASE.FINISH;+}