Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: share one relaunch budget between the Metal buffer and prompt-encode retries (#11172)

Adam Eivy1258c4190d3d
diff --git a/server/services/videoGen/local.test.js b/server/services/videoGen/local.test.js
index c5b803e262..0a5d0c1def 100644
--- a/server/services/videoGen/local.test.js
+++ b/server/services/videoGen/local.test.js
@@ -5258,6 +5258,24 @@ describe('generateVideo — Gemma prompt-encode watchdog relaunch', () => {
expect(failures[0].error).toMatch(/SIGABRT/);
});
+ it('spends one relaunch per job — a buffer retry is not followed by a prompt-encode retry', async () => {
+ const first = makeDrivenProc(112);
+ const second = makeDrivenProc(113);
+ const spawnMock = await startRender('pe-buffer-then-encode', [first, second]);
+
+ first.stderr('STAGE:encode-prompt-done');
+ first.stderr(TIMEOUT_ABORT);
+ await first.abort();
+ expect(ltx2Calls(spawnMock)).toHaveLength(2);
+
+ second.stderr('STAGE:encode-prompt');
+ second.stderr(TIMEOUT_ABORT);
+ await second.abort();
+
+ expect(ltx2Calls(spawnMock)).toHaveLength(2);
+ expect(failures).toHaveLength(1);
+ });
+
it('does not use smaller command buffers off Apple silicon', async () => {
restoreArch();
const { pinArch } = await import('../../lib/testHelper.js');
diff --git a/server/services/videoGen/spawnWatch.js b/server/services/videoGen/spawnWatch.js
index b2fa8d54c4..8e3ca3ca24 100644
--- a/server/services/videoGen/spawnWatch.js
+++ b/server/services/videoGen/spawnWatch.js
@@ -654,19 +654,20 @@ export async function spawnAndWatchVideo({
// would be answered by relaunching the render the user just stopped.
if (childKilled) return false;
const stderrText = typeof stderr === 'string' ? stderr : '';
+ // At most ONE relaunch per job across both mitigations: a render that already
+ // spent either retry does not get a second GPU-grinding attempt.
+ const relaunchesUsed = promptEncodingRetriesUsed + metalBufferRetriesUsed;
const encodePlan = planPromptEncodingRetry({
signal,
stderr,
promptEncodePhase: job.promptEncodePhase,
- retriesUsed: promptEncodingRetriesUsed,
+ retriesUsed: relaunchesUsed,
platform: process.platform,
});
- // At most ONE relaunch per job across both mitigations: a render that already
- // spent its prompt-encode retry does not get a second GPU-grinding attempt.
- const bufferPlan = encodePlan || promptEncodingRetriesUsed > 0 ? null : planMetalBufferRetry({
+ const bufferPlan = encodePlan || relaunchesUsed > 0 ? null : planMetalBufferRetry({
signal,
stderr: stderrText,
- retriesUsed: metalBufferRetriesUsed,
+ retriesUsed: 0,
mlxRuntime: runtimeUsesMlx(model.runtime),
});
if (!encodePlan && !bufferPlan) return false;