Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

feat: relaunch a Metal-watchdog-killed MLX render once with smaller command buffers (#11172)

Adam Eivy28b6e364ba94
diff --git a/server/services/videoGen/generateVideoHelpers.js b/server/services/videoGen/generateVideoHelpers.js
index 733f11c04c..c9551adc2f 100644
--- a/server/services/videoGen/generateVideoHelpers.js
+++ b/server/services/videoGen/generateVideoHelpers.js
@@ -538,6 +538,35 @@ export function planPromptEncodingRetry({ signal = null, stderr = '', promptEnco
return { gemmaMaxLength: RETRY_GEMMA_MAX_LENGTH };
}
+/**
+ * Per-command-buffer caps MLX reads at startup (`MLX_MAX_OPS_PER_BUFFER` /
+ * `MLX_MAX_MB_PER_BUFFER`). Smaller values only move command-buffer boundaries —
+ * same kernels, same output, slightly slower — so each buffer finishes inside the
+ * macOS GPU watchdog window. Applied reactively on a retry, never by default.
+ */
+export const METAL_BUFFER_RETRY_ENV = Object.freeze({
+ MLX_MAX_OPS_PER_BUFFER: '4',
+ MLX_MAX_MB_PER_BUFFER: '16',
+});
+
+/**
+ * Decide whether a render may be relaunched once with tiny MLX command buffers
+ * after a Metal watchdog abort ANYWHERE in the render (the prompt-encode case has
+ * its own plan, planPromptEncodingRetry). Pure — returns the plan, never acts.
+ * Apple-silicon MLX runtimes only; OutOfMemory / InnocentVictim aborts never
+ * qualify (smaller buffers do not fix them). One retry per job.
+ *
+ * @returns {{ env: Record<string, string> }|null}
+ */
+export function planMetalBufferRetry({ signal = null, stderr = '', retriesUsed = 0, platform = process.platform, arch = process.arch, mlxRuntime = false } = {}) {
+ if (platform !== 'darwin' || arch !== 'arm64' || !mlxRuntime) return null;
+ if (!Number.isInteger(retriesUsed) || retriesUsed > 0) return null;
+ if (signal !== 'SIGABRT') return null;
+ const text = typeof stderr === 'string' ? stderr : '';
+ if (METAL_WATCHDOG_VETO_RE.test(text) || !METAL_WATCHDOG_RE.test(text)) return null;
+ return { env: { ...METAL_BUFFER_RETRY_ENV } };
+}
+
/**
* Whether a PortOS-fired completion-watchdog SIGKILL should be treated as
* success rather than a failure. The watchdog is armed after a completion
diff --git a/server/services/videoGen/local.test.js b/server/services/videoGen/local.test.js
index 988bec7e88..c5b803e262 100644
--- a/server/services/videoGen/local.test.js
+++ b/server/services/videoGen/local.test.js
@@ -5152,6 +5152,7 @@ describe('generateVideo — Gemma prompt-encode watchdog relaunch', () => {
};
let restorePlatform = () => {};
+ let restoreArch = () => {};
let failures;
let onFailed;
@@ -5161,8 +5162,9 @@ describe('generateVideo — Gemma prompt-encode watchdog relaunch', () => {
// describe would silently assert "never relaunches" on Windows CI.
// Pinned inside the hook, never at module scope: local.js is already
// imported by then.
- const { pinPlatform } = await import('../../lib/testHelper.js');
+ const { pinPlatform, pinArch } = await import('../../lib/testHelper.js');
restorePlatform = pinPlatform('darwin');
+ restoreArch = pinArch('arm64');
failures = [];
onFailed = (payload) => failures.push(payload);
videoGenEvents.on('failed', onFailed);
@@ -5171,6 +5173,7 @@ describe('generateVideo — Gemma prompt-encode watchdog relaunch', () => {
afterEach(() => {
videoGenEvents.off('failed', onFailed);
restorePlatform();
+ restoreArch();
});
const startRender = async (jobId, children) => {
@@ -5227,22 +5230,61 @@ describe('generateVideo — Gemma prompt-encode watchdog relaunch', () => {
await second.finish();
});
- // The bypass probe for the phase gate: identical abort, identical signal, one
- // extra marker line. If the gate were dropped this case would relaunch too.
- it('does not relaunch once the prompt encode has finished', async () => {
+ // After the encode finished, a shorter prompt cannot help — but smaller MLX
+ // command buffers can. Same abort, one extra marker line: the relaunch carries
+ // the buffer caps in its env and NOT the Gemma flag.
+ it('relaunches once with smaller MLX command buffers when the abort lands after the prompt encode', async () => {
const first = makeDrivenProc(103);
- const spawnMock = await startRender('pe-after-encode', [first]);
+ const second = makeDrivenProc(109);
+ const spawnMock = await startRender('pe-after-encode', [first, second]);
first.stderr('STAGE:encode-prompt');
first.stderr('STAGE:encode-prompt-done');
first.stderr(TIMEOUT_ABORT);
await first.abort();
- expect(ltx2Calls(spawnMock)).toHaveLength(1);
+ const calls = ltx2Calls(spawnMock);
+ expect(calls).toHaveLength(2);
+ expect(calls[1][1]).not.toContain('--gemma-max-length');
+ expect(calls[0][2].env.MLX_MAX_OPS_PER_BUFFER).toBeUndefined();
+ expect(calls[1][2].env).toMatchObject({ MLX_MAX_OPS_PER_BUFFER: '4', MLX_MAX_MB_PER_BUFFER: '16' });
+ expect(failures).toHaveLength(0);
+
+ // A second abort at the short setting surfaces the normal error.
+ second.stderr(TIMEOUT_ABORT);
+ await second.abort();
+ expect(ltx2Calls(spawnMock)).toHaveLength(2);
expect(failures).toHaveLength(1);
expect(failures[0].error).toMatch(/SIGABRT/);
});
+ it('does not use smaller command buffers off Apple silicon', async () => {
+ restoreArch();
+ const { pinArch } = await import('../../lib/testHelper.js');
+ restoreArch = pinArch('x64');
+ const first = makeDrivenProc(110);
+ const spawnMock = await startRender('pe-x64', [first]);
+
+ first.stderr('STAGE:encode-prompt-done');
+ first.stderr(TIMEOUT_ABORT);
+ await first.abort();
+
+ expect(ltx2Calls(spawnMock)).toHaveLength(1);
+ expect(failures).toHaveLength(1);
+ });
+
+ it('does not use smaller command buffers on an out-of-memory abort after the encode', async () => {
+ const first = makeDrivenProc(111);
+ const spawnMock = await startRender('pe-oom-after-encode', [first]);
+
+ first.stderr('STAGE:encode-prompt-done');
+ first.stderr(OOM_ABORT);
+ await first.abort();
+
+ expect(ltx2Calls(spawnMock)).toHaveLength(1);
+ expect(failures).toHaveLength(1);
+ });
+
// An OOM abort is not a watchdog timeout: a shorter prompt does not fix it,
// and relaunching burns another model load on a machine already out of room.
it('does not relaunch on an out-of-memory abort inside the encoder', async () => {
diff --git a/server/services/videoGen/spawnWatch.js b/server/services/videoGen/spawnWatch.js
index bb8f475d7a..b2fa8d54c4 100644
--- a/server/services/videoGen/spawnWatch.js
+++ b/server/services/videoGen/spawnWatch.js
@@ -27,6 +27,7 @@ import {
verifyPostCompletionOutputs,
describeSignalDeath,
planPromptEncodingRetry,
+ planMetalBufferRetry,
bufferChildExit,
} from './generateVideoHelpers.js';
import {
@@ -99,6 +100,9 @@ export async function spawnAndWatchVideo({
// a second watchdog abort at the reduced budget is a real failure the user has
// to see, not something to keep grinding the GPU over.
let promptEncodingRetriesUsed = 0;
+ // Same single-shot rule for the small-command-buffer relaunch (any-phase Metal
+ // watchdog abort on an MLX runtime).
+ let metalBufferRetriesUsed = 0;
// Hoisted out of the try so a relaunch can respawn with the SAME child env
// plus a lowered Gemma budget, instead of rebuilding it from scratch.
let childEnv;
@@ -649,14 +653,23 @@ export async function spawnAndWatchVideo({
// finally landed on. Without this a cancel that raced an in-flight abort
// would be answered by relaunching the render the user just stopped.
if (childKilled) return false;
- const plan = planPromptEncodingRetry({
+ const stderrText = typeof stderr === 'string' ? stderr : '';
+ const encodePlan = planPromptEncodingRetry({
signal,
stderr,
promptEncodePhase: job.promptEncodePhase,
retriesUsed: promptEncodingRetriesUsed,
platform: process.platform,
});
- if (!plan) return false;
+ // At most ONE relaunch per job across both mitigations: a render that already
+ // spent its prompt-encode retry does not get a second GPU-grinding attempt.
+ const bufferPlan = encodePlan || promptEncodingRetriesUsed > 0 ? null : planMetalBufferRetry({
+ signal,
+ stderr: stderrText,
+ retriesUsed: metalBufferRetriesUsed,
+ mlxRuntime: runtimeUsesMlx(model.runtime),
+ });
+ if (!encodePlan && !bufferPlan) return false;
// Runs from a child 'close' handler, outside the Express request lifecycle —
// an uncaught throw here would crash the process, and a failed relaunch must
// degrade into the normal failure report rather than strand the job.
@@ -668,14 +681,23 @@ export async function spawnAndWatchVideo({
let retryProc = null;
let retryWired = false;
try {
- promptEncodingRetriesUsed += 1;
- const retryArgs = [...args, '--gemma-max-length', String(plan.gemmaMaxLength)];
- const message = `Metal watchdog aborted the prompt encoder — retrying once at ${plan.gemmaMaxLength} Gemma tokens`;
+ let retryArgs = args;
+ let retryEnv = childEnv;
+ let message;
+ if (encodePlan) {
+ promptEncodingRetriesUsed += 1;
+ retryArgs = [...args, '--gemma-max-length', String(encodePlan.gemmaMaxLength)];
+ message = `Metal watchdog aborted the prompt encoder — retrying once at ${encodePlan.gemmaMaxLength} Gemma tokens`;
+ } else {
+ metalBufferRetriesUsed += 1;
+ retryEnv = { ...childEnv, ...bufferPlan.env };
+ message = 'Metal watchdog stopped the GPU — retrying once with smaller command buffers';
+ }
console.log(`♻️ ${message} [${jobId.slice(0, 8)}]`);
broadcastSse(job, { type: 'status', message });
videoGenEvents.emit('status', { generationId: jobId, message });
retryProc = await spawnDetached(bin, retryArgs, {
- env: childEnv,
+ env: retryEnv,
controlDir: join(PATHS.videos, '.detached', `${jobId}-retry`),
cleanup: true,
killProcessGroup: runtimeNeedsProcessGroupKill(model.runtime),