Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
feat: relaunch a Metal-watchdog-killed MLX render once with smaller command buffers (#11172)
Adam Eivy28b6e364ba94
diff --git a/server/services/videoGen/generateVideoHelpers.js b/server/services/videoGen/generateVideoHelpers.jsindex 733f11c04c..c9551adc2f 100644--- a/server/services/videoGen/generateVideoHelpers.js+++ b/server/services/videoGen/generateVideoHelpers.js@@ -538,6 +538,35 @@ export function planPromptEncodingRetry({ signal = null, stderr = '', promptEncoreturn { gemmaMaxLength: RETRY_GEMMA_MAX_LENGTH };}+/**+ * Per-command-buffer caps MLX reads at startup (`MLX_MAX_OPS_PER_BUFFER` /+ * `MLX_MAX_MB_PER_BUFFER`). Smaller values only move command-buffer boundaries —+ * same kernels, same output, slightly slower — so each buffer finishes inside the+ * macOS GPU watchdog window. Applied reactively on a retry, never by default.+ */+export const METAL_BUFFER_RETRY_ENV = Object.freeze({+ MLX_MAX_OPS_PER_BUFFER: '4',+ MLX_MAX_MB_PER_BUFFER: '16',+});++/**+ * Decide whether a render may be relaunched once with tiny MLX command buffers+ * after a Metal watchdog abort ANYWHERE in the render (the prompt-encode case has+ * its own plan, planPromptEncodingRetry). Pure — returns the plan, never acts.+ * Apple-silicon MLX runtimes only; OutOfMemory / InnocentVictim aborts never+ * qualify (smaller buffers do not fix them). One retry per job.+ *+ * @returns {{ env: Record<string, string> }|null}+ */+export function planMetalBufferRetry({ signal = null, stderr = '', retriesUsed = 0, platform = process.platform, arch = process.arch, mlxRuntime = false } = {}) {+ if (platform !== 'darwin' || arch !== 'arm64' || !mlxRuntime) return null;+ if (!Number.isInteger(retriesUsed) || retriesUsed > 0) return null;+ if (signal !== 'SIGABRT') return null;+ const text = typeof stderr === 'string' ? stderr : '';+ if (METAL_WATCHDOG_VETO_RE.test(text) || !METAL_WATCHDOG_RE.test(text)) return null;+ return { env: { ...METAL_BUFFER_RETRY_ENV } };+}+/*** Whether a PortOS-fired completion-watchdog SIGKILL should be treated as* success rather than a failure. The watchdog is armed after a completiondiff --git a/server/services/videoGen/local.test.js b/server/services/videoGen/local.test.jsindex 988bec7e88..c5b803e262 100644--- a/server/services/videoGen/local.test.js+++ b/server/services/videoGen/local.test.js@@ -5152,6 +5152,7 @@ describe('generateVideo — Gemma prompt-encode watchdog relaunch', () => {};let restorePlatform = () => {};+ let restoreArch = () => {};let failures;let onFailed;@@ -5161,8 +5162,9 @@ describe('generateVideo — Gemma prompt-encode watchdog relaunch', () => {// describe would silently assert "never relaunches" on Windows CI.// Pinned inside the hook, never at module scope: local.js is already// imported by then.- const { pinPlatform } = await import('../../lib/testHelper.js');+ const { pinPlatform, pinArch } = await import('../../lib/testHelper.js');restorePlatform = pinPlatform('darwin');+ restoreArch = pinArch('arm64');failures = [];onFailed = (payload) => failures.push(payload);videoGenEvents.on('failed', onFailed);@@ -5171,6 +5173,7 @@ describe('generateVideo — Gemma prompt-encode watchdog relaunch', () => {afterEach(() => {videoGenEvents.off('failed', onFailed);restorePlatform();+ restoreArch();});const startRender = async (jobId, children) => {@@ -5227,22 +5230,61 @@ describe('generateVideo — Gemma prompt-encode watchdog relaunch', () => {await second.finish();});- // The bypass probe for the phase gate: identical abort, identical signal, one- // extra marker line. If the gate were dropped this case would relaunch too.- it('does not relaunch once the prompt encode has finished', async () => {+ // After the encode finished, a shorter prompt cannot help — but smaller MLX+ // command buffers can. Same abort, one extra marker line: the relaunch carries+ // the buffer caps in its env and NOT the Gemma flag.+ it('relaunches once with smaller MLX command buffers when the abort lands after the prompt encode', async () => {const first = makeDrivenProc(103);- const spawnMock = await startRender('pe-after-encode', [first]);+ const second = makeDrivenProc(109);+ const spawnMock = await startRender('pe-after-encode', [first, second]);first.stderr('STAGE:encode-prompt');first.stderr('STAGE:encode-prompt-done');first.stderr(TIMEOUT_ABORT);await first.abort();- expect(ltx2Calls(spawnMock)).toHaveLength(1);+ const calls = ltx2Calls(spawnMock);+ expect(calls).toHaveLength(2);+ expect(calls[1][1]).not.toContain('--gemma-max-length');+ expect(calls[0][2].env.MLX_MAX_OPS_PER_BUFFER).toBeUndefined();+ expect(calls[1][2].env).toMatchObject({ MLX_MAX_OPS_PER_BUFFER: '4', MLX_MAX_MB_PER_BUFFER: '16' });+ expect(failures).toHaveLength(0);++ // A second abort at the short setting surfaces the normal error.+ second.stderr(TIMEOUT_ABORT);+ await second.abort();+ expect(ltx2Calls(spawnMock)).toHaveLength(2);expect(failures).toHaveLength(1);expect(failures[0].error).toMatch(/SIGABRT/);});+ it('does not use smaller command buffers off Apple silicon', async () => {+ restoreArch();+ const { pinArch } = await import('../../lib/testHelper.js');+ restoreArch = pinArch('x64');+ const first = makeDrivenProc(110);+ const spawnMock = await startRender('pe-x64', [first]);++ first.stderr('STAGE:encode-prompt-done');+ first.stderr(TIMEOUT_ABORT);+ await first.abort();++ expect(ltx2Calls(spawnMock)).toHaveLength(1);+ expect(failures).toHaveLength(1);+ });++ it('does not use smaller command buffers on an out-of-memory abort after the encode', async () => {+ const first = makeDrivenProc(111);+ const spawnMock = await startRender('pe-oom-after-encode', [first]);++ first.stderr('STAGE:encode-prompt-done');+ first.stderr(OOM_ABORT);+ await first.abort();++ expect(ltx2Calls(spawnMock)).toHaveLength(1);+ expect(failures).toHaveLength(1);+ });+// An OOM abort is not a watchdog timeout: a shorter prompt does not fix it,// and relaunching burns another model load on a machine already out of room.it('does not relaunch on an out-of-memory abort inside the encoder', async () => {diff --git a/server/services/videoGen/spawnWatch.js b/server/services/videoGen/spawnWatch.jsindex bb8f475d7a..b2fa8d54c4 100644--- a/server/services/videoGen/spawnWatch.js+++ b/server/services/videoGen/spawnWatch.js@@ -27,6 +27,7 @@ import {verifyPostCompletionOutputs,describeSignalDeath,planPromptEncodingRetry,+ planMetalBufferRetry,bufferChildExit,} from './generateVideoHelpers.js';import {@@ -99,6 +100,9 @@ export async function spawnAndWatchVideo({// a second watchdog abort at the reduced budget is a real failure the user has// to see, not something to keep grinding the GPU over.let promptEncodingRetriesUsed = 0;+ // Same single-shot rule for the small-command-buffer relaunch (any-phase Metal+ // watchdog abort on an MLX runtime).+ let metalBufferRetriesUsed = 0;// Hoisted out of the try so a relaunch can respawn with the SAME child env// plus a lowered Gemma budget, instead of rebuilding it from scratch.let childEnv;@@ -649,14 +653,23 @@ export async function spawnAndWatchVideo({// finally landed on. Without this a cancel that raced an in-flight abort// would be answered by relaunching the render the user just stopped.if (childKilled) return false;- const plan = planPromptEncodingRetry({+ const stderrText = typeof stderr === 'string' ? stderr : '';+ const encodePlan = planPromptEncodingRetry({signal,stderr,promptEncodePhase: job.promptEncodePhase,retriesUsed: promptEncodingRetriesUsed,platform: process.platform,});- if (!plan) return false;+ // At most ONE relaunch per job across both mitigations: a render that already+ // spent its prompt-encode retry does not get a second GPU-grinding attempt.+ const bufferPlan = encodePlan || promptEncodingRetriesUsed > 0 ? null : planMetalBufferRetry({+ signal,+ stderr: stderrText,+ retriesUsed: metalBufferRetriesUsed,+ mlxRuntime: runtimeUsesMlx(model.runtime),+ });+ if (!encodePlan && !bufferPlan) return false;// Runs from a child 'close' handler, outside the Express request lifecycle —// an uncaught throw here would crash the process, and a failed relaunch must// degrade into the normal failure report rather than strand the job.@@ -668,14 +681,23 @@ export async function spawnAndWatchVideo({let retryProc = null;let retryWired = false;try {- promptEncodingRetriesUsed += 1;- const retryArgs = [...args, '--gemma-max-length', String(plan.gemmaMaxLength)];- const message = `Metal watchdog aborted the prompt encoder — retrying once at ${plan.gemmaMaxLength} Gemma tokens`;+ let retryArgs = args;+ let retryEnv = childEnv;+ let message;+ if (encodePlan) {+ promptEncodingRetriesUsed += 1;+ retryArgs = [...args, '--gemma-max-length', String(encodePlan.gemmaMaxLength)];+ message = `Metal watchdog aborted the prompt encoder — retrying once at ${encodePlan.gemmaMaxLength} Gemma tokens`;+ } else {+ metalBufferRetriesUsed += 1;+ retryEnv = { ...childEnv, ...bufferPlan.env };+ message = 'Metal watchdog stopped the GPU — retrying once with smaller command buffers';+ }console.log(`♻️ ${message} [${jobId.slice(0, 8)}]`);broadcastSse(job, { type: 'status', message });videoGenEvents.emit('status', { generationId: jobId, message });retryProc = await spawnDetached(bin, retryArgs, {- env: childEnv,+ env: retryEnv,controlDir: join(PATHS.videos, '.detached', `${jobId}-retry`),cleanup: true,killProcessGroup: runtimeNeedsProcessGroupKill(model.runtime),