Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: hold every later same-peer job while a retained reservation reconciles (#12055)

[._.]/ Adam Eivy145219d398ad
diff --git a/server/services/mediaJobQueue/index.js b/server/services/mediaJobQueue/index.js
index 0a3a6a1b2c..c493671ed8 100644
--- a/server/services/mediaJobQueue/index.js
+++ b/server/services/mediaJobQueue/index.js
@@ -1103,6 +1103,13 @@ function startLaneJob(job, { lane }) {
}
if (lane === 'remote' && remoteRunning.some((active) => active.params?.remoteMedia?.peerId === job.params?.remoteMedia?.peerId)) return false;
if (lane === 'remote') {
+ // Reconciliation occupies the peer slot while it waits for admission. Read
+ // the live queue: an in-process requeue can land mid-pass, after the
+ // drain loop built its candidate list.
+ const reconciling = job.params?.remoteMedia?.reconcile ? null : queue.find((pending) => pending !== job
+ && pending.status === 'queued' && pending.params?.remoteMedia?.reconcile
+ && pending.params.remoteMedia.peerId === job.params?.remoteMedia?.peerId);
+ if (reconciling) { holdForPeer(job, reconciling.id); return false; }
const blocker = archive.find((entry) => entry.id !== job.id && entry.params?.remotePeerReservation === true
&& entry.params?.remoteMedia?.peerId === job.params?.remoteMedia?.peerId);
if (blocker) {
@@ -1311,10 +1318,6 @@ async function drainLoop() {
if (slots[lane] <= 0) continue;
// One GPU job per peer initially. Different peers and this Mac remain
// independent. Reconciliation occupies the same slot until settled.
- if (lane === 'remote' && !job.params?.remoteMedia?.reconcile) {
- const reconciling = candidates.find((pending) => pending.status === 'queued' && pending.params?.remoteMedia?.reconcile && pending.params.remoteMedia.peerId === job.params?.remoteMedia?.peerId);
- if (reconciling) { holdForPeer(job, reconciling.id); continue; }
- }
if (!startLaneJob(job, { lane })) continue;
slots[lane] -= 1;
if (Object.values(slots).every((remaining) => remaining <= 0)) break;
diff --git a/server/services/mediaJobQueue/index.test.js b/server/services/mediaJobQueue/index.test.js
index 48a88da97a..741290989b 100644
--- a/server/services/mediaJobQueue/index.test.js
+++ b/server/services/mediaJobQueue/index.test.js
@@ -3084,6 +3084,24 @@ describe('durable remote source-audio settlement', () => {
videoGenEvents.emit('completed', { generationId: next.jobId, remoteInputsDisposable: true });
});
+ it('keeps every later job to the peer waiting while its reservation reconciles', async () => {
+ const first = await startAudioJob();
+ videoGenEvents.emit('failed', { generationId: first.jobId, error: 'Peer disabled', remoteInputsDisposable: false });
+ await waitFor(() => mediaJobQueue.getJob(first.jobId).status === 'failed');
+ stubs.generateVideoRemote.mockClear();
+ const later = [];
+ for (let i = 0; i < 2; i += 1) later.push(await mediaJobQueue.enqueueJob({ kind: 'video', params: { remoteMedia: remoteVideoMediaParams() } }));
+ await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === first.jobId));
+ await flush();
+ expect(stubs.generateVideoRemote.mock.calls.map(([params]) => params.jobId)).toEqual([first.jobId]);
+ videoGenEvents.emit('completed', { generationId: first.jobId, remoteInputsDisposable: true,
+ remoteRecoverySettlement: remoteRecoverySettlement(first.jobId) });
+ await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === later[0].jobId));
+ videoGenEvents.emit('completed', { generationId: later[0].jobId, remoteInputsDisposable: true });
+ await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === later[1].jobId));
+ videoGenEvents.emit('completed', { generationId: later[1].jobId, remoteInputsDisposable: true });
+ });
+
it('preserves recovered uncertainty when owned cleanup creates a new mark', async () => {
const { first, maintenance } = await restoreUnsettledAudioJob(true);
const journal = () => JSON.parse(readFileSync(join(maintenance.directory, 'state.json'), 'utf8'));