Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: reconcile a retained remote peer reservation in-process and explain the hold (#12055)
[._.]/ Adam Eivyc56ddbc61f91
diff --git a/server/services/mediaJobQueue/index.js b/server/services/mediaJobQueue/index.jsindex ba2739ecc8..0a3a6a1b2c 100644--- a/server/services/mediaJobQueue/index.js+++ b/server/services/mediaJobQueue/index.js@@ -105,6 +105,10 @@ const restoredParams = (job) => (isRemoteMediaJob(job)? routedJobParams({ params: job.params }): job.params);+// A reservation row whose own reconcile attempt failed again is retried when a+// later job to the same peer is blocked behind it, but no faster than this.+const RESERVATION_RECONCILE_RETRY_MS = 30_000;+const JOBS_FILE = join(PATHS.data, 'media-jobs.json');const COMPLETED_TTL_MS = 24 * 60 * 60 * 1000;const MAX_PERSISTED_ARCHIVE = 500;@@ -1050,6 +1054,38 @@ function startWorker() {}).finally(() => { workerStarted = false; });}+// Transient (never persisted): why a queued remote job is not dispatching.+// Logged once per blocked job so a silent per-tick skip is still diagnosable.+function holdForPeer(job, blockingJobId) {+ if (job.peerHold?.blockingJobId === blockingJobId) return;+ job.peerHold = { reason: 'peer-settling', blockingJobId };+ console.log(`⏳ media-job [${job.id.slice(0, 8)}] waiting for peer to settle earlier job [${blockingJobId.slice(0, 8)}]`);+}++// A failed/canceled remote job that kept its peer reservation (no terminal peer+// report) is reconciled in-process exactly as boot does, so the peer is not+// blocked until the next restart. Returns whether the row was re-enqueued.+function requeueRetainedReservation(entry) {+ const idx = archive.indexOf(entry);+ if (idx < 0 || entry.durabilityPending || !isRemoteMediaJob(entry)+ || entry.params?.remotePeerReservation !== true || !['failed', 'canceled'].includes(entry.status)) return false;+ const marker = entry.params?.remoteMedia;+ archive.splice(idx, 1);+ // Rebuild from persisted fields only, as boot does: the in-memory row carries+ // transient terminal state (terminating, providerInvoked, ...) that would+ // make the re-run look already settled.+ const { id, kind, owner, queuedAt, startedAt, completedAt, result, error, progress, statusMsg, etaMs } = entry;+ queue.push({+ id, kind, owner, queuedAt, startedAt, completedAt, result, error, progress, statusMsg, etaMs,+ status: 'queued',+ cancelRequested: marker?.cancelRequested === true,+ params: restoredParams({ ...entry, params: { ...entry.params, remoteMedia: { ...marker, reconcile: true } } }),+ });+ recomputeQueuePositions();+ console.log(`🔁 media-job [${entry.id.slice(0, 8)}] remote ${entry.kind} ended without a terminal peer report — re-enqueued for reconciliation`);+ return true;+}+// Every lane uses fire-and-forget so the poll loop is never blocked by a// running job in another lane.function startLaneJob(job, { lane }) {@@ -1066,8 +1102,21 @@ function startLaneJob(job, { lane }) {return false;}if (lane === 'remote' && remoteRunning.some((active) => active.params?.remoteMedia?.peerId === job.params?.remoteMedia?.peerId)) return false;- if (lane === 'remote' && archive.some((entry) => entry.id !== job.id && entry.params?.remotePeerReservation === true- && entry.params?.remoteMedia?.peerId === job.params?.remoteMedia?.peerId)) return false;+ if (lane === 'remote') {+ const blocker = archive.find((entry) => entry.id !== job.id && entry.params?.remotePeerReservation === true+ && entry.params?.remoteMedia?.peerId === job.params?.remoteMedia?.peerId);+ if (blocker) {+ holdForPeer(job, blocker.id);+ // The blocker settled failed/canceled without a terminal peer report;+ // only a reconcile can release it, so do not wait for a restart.+ // A prior reconcile run that failed again is retried at a bounded rate.+ const retryDue = blocker.params.remoteMedia?.reconcile !== true+ || Date.now() - new Date(blocker.completedAt || 0).getTime() >= RESERVATION_RECONCILE_RETRY_MS;+ if (retryDue) requeueRetainedReservation(blocker);+ return false;+ }+ }+ delete job.peerHold;const permit = recoveringJobs.has(job.id)? maintenance.recoverOwned('media', job.id): maintenance.tryAdmit('media', job.id, { reconnect: isRemoteMediaJob(job) && job.params?.remoteMedia?.reconcile === true });@@ -1262,7 +1311,10 @@ async function drainLoop() {if (slots[lane] <= 0) continue;// One GPU job per peer initially. Different peers and this Mac remain// independent. Reconciliation occupies the same slot until settled.- if (lane === 'remote' && !job.params?.remoteMedia?.reconcile && candidates.some((pending) => pending.status === 'queued' && pending.params?.remoteMedia?.reconcile && pending.params.remoteMedia.peerId === job.params?.remoteMedia?.peerId)) continue;+ if (lane === 'remote' && !job.params?.remoteMedia?.reconcile) {+ const reconciling = candidates.find((pending) => pending.status === 'queued' && pending.params?.remoteMedia?.reconcile && pending.params.remoteMedia.peerId === job.params?.remoteMedia?.peerId);+ if (reconciling) { holdForPeer(job, reconciling.id); continue; }+ }if (!startLaneJob(job, { lane })) continue;slots[lane] -= 1;if (Object.values(slots).every((remaining) => remaining <= 0)) break;diff --git a/server/services/mediaJobQueue/index.test.js b/server/services/mediaJobQueue/index.test.jsindex 43deeed2c6..48a88da97a 100644--- a/server/services/mediaJobQueue/index.test.js+++ b/server/services/mediaJobQueue/index.test.js@@ -3062,6 +3062,28 @@ describe('durable remote source-audio settlement', () => {}});+ it('reconciles a failed reservation in-process, explains the hold, then dispatches the next job to that peer', async () => {+ const first = await startAudioJob();+ stubs.generateVideoRemote.mockClear();+ videoGenEvents.emit('failed', { generationId: first.jobId, error: 'Peer disabled', remoteInputsDisposable: false });+ await waitFor(() => mediaJobQueue.getJob(first.jobId).status === 'failed');+ const next = await mediaJobQueue.enqueueJob({ kind: 'video', params: { remoteMedia: remoteVideoMediaParams() } });+ // No restart: the blocked peer re-enqueues the retained reservation as a reconcile run.+ await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === first.jobId));+ expect(stubs.generateVideoRemote.mock.calls.find(([params]) => params.jobId === first.jobId)[0].remoteMedia.reconcile).toBe(true);+ await waitFor(() => mediaJobQueue.getJob(next.jobId)?.peerHold);+ expect(mediaJobQueue.getJob(next.jobId)).toMatchObject({+ status: 'queued',+ peerHold: { reason: 'peer-settling', blockingJobId: first.jobId },+ });+ expect(stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === next.jobId)).toBe(false);+ videoGenEvents.emit('completed', { generationId: first.jobId, remoteInputsDisposable: true,+ remoteRecoverySettlement: remoteRecoverySettlement(first.jobId) });+ await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === next.jobId));+ expect(mediaJobQueue.getJob(next.jobId)).not.toHaveProperty('peerHold');+ videoGenEvents.emit('completed', { generationId: next.jobId, remoteInputsDisposable: true });+ });+it('preserves recovered uncertainty when owned cleanup creates a new mark', async () => {const { first, maintenance } = await restoreUnsettledAudioJob(true);const journal = () => JSON.parse(readFileSync(join(maintenance.directory, 'state.json'), 'utf8'));diff --git a/server/services/mediaJobQueue/sanitizeJob.js b/server/services/mediaJobQueue/sanitizeJob.jsindex edf54f75e3..bf01d5a34e 100644--- a/server/services/mediaJobQueue/sanitizeJob.js+++ b/server/services/mediaJobQueue/sanitizeJob.js@@ -134,7 +134,10 @@ export function sanitizeJob(job) {completedAt: job.completedAt,position: job.position,progress: job.progress,- statusMsg: job.statusMsg,+ // A queued remote job held behind an unsettled earlier job to its peer.+ ...(job.status === 'queued' && job.peerHold+ ? { peerHold: { reason: job.peerHold.reason, blockingJobId: job.peerHold.blockingJobId } } : {}),+ statusMsg: job.status === 'queued' && job.peerHold ? 'Waiting for peer to settle an earlier job' : job.statusMsg,etaMs: job.etaMs,error: job.error,result: safeResult,diff --git a/server/services/mediaJobQueue/sanitizeJob.test.js b/server/services/mediaJobQueue/sanitizeJob.test.jsindex 56143ee5db..24f6539089 100644--- a/server/services/mediaJobQueue/sanitizeJob.test.js+++ b/server/services/mediaJobQueue/sanitizeJob.test.js@@ -261,4 +261,14 @@ describe('sanitizeJob', () => {expect(sanitized.renderer).toBe('local');expect(sanitized.executionLane).toBe('cloud');});++ it('explains a queued job held behind an unsettled earlier peer job, and only while queued', () => {+ const peerHold = { reason: 'peer-settling', blockingJobId: 'job-earlier-example', internal: 'x' };+ const held = sanitizeJob({ id: 'job-held-example', kind: 'video', status: 'queued', statusMsg: 'Queued', peerHold });+ expect(held.peerHold).toEqual({ reason: 'peer-settling', blockingJobId: 'job-earlier-example' });+ expect(held.statusMsg).toBe('Waiting for peer to settle an earlier job');+ const running = sanitizeJob({ id: 'job-held-example', kind: 'video', status: 'running', statusMsg: 'Starting', peerHold });+ expect(running).not.toHaveProperty('peerHold');+ expect(running.statusMsg).toBe('Starting');+ });});