Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: classify SIGABRT render aborts from captured stderr instead of assuming a Metal watchdog (#12263)

[._.]/ Adam Eivy0aaed908557e
diff --git a/server/services/videoGen/generateVideoHelpers.js b/server/services/videoGen/generateVideoHelpers.js
index 08c6e515a3..818c2c3b5f 100644
--- a/server/services/videoGen/generateVideoHelpers.js
+++ b/server/services/videoGen/generateVideoHelpers.js
@@ -400,14 +400,15 @@ export function formatRuntimeFingerprint(fp) {
// Signal → actionable cause for a render child that died on a signal.
//
-// Apple Silicon reality drives this map. The dominant real-world render abort is
+// Apple Silicon reality drives this map. A common real-world render abort is
// the macOS Metal command-buffer watchdog killing an over-long GPU command
// buffer (`kIOGPUCommandBufferCallbackErrorImpactingInteractivity`), which the
// MLX/Metal layer surfaces as SIGABRT (exit -6) — NOT as a Python-level
// assertion. Reporting that as a bare `Killed by signal SIGABRT` (or as a
// "C-level assertion failed" hint, the mistake upstream made) sends users
// hunting a model/PortOS bug when the fix is to shrink the work per command
-// buffer. SIGBUS/SIGSEGV are the other native-crash arrivals from the same
+// buffer — but only captured stderr can prove it (see abortCause); a bare SIGABRT
+// is reported as an unexplained native abort. SIGBUS/SIGSEGV are the other native-crash arrivals from the same
// layer. SIGKILL stays the OOM message it has always been.
//
// Keys are the signal NAMES Node reports on `close` (spawnDetached decodes the
@@ -424,28 +425,52 @@ const nativeCrashCause = (signal) => `Render crashed (${signal}) — a native fa
const SIGNAL_DEATH_CAUSES = Object.freeze({
SIGKILL: 'Process killed (likely out of memory — try a smaller model or resolution)',
- SIGABRT: 'Render aborted (SIGABRT) — on Apple Silicon this is almost always the macOS Metal command-buffer watchdog killing an over-long GPU command buffer (kIOGPUCommandBufferCallbackErrorImpactingInteractivity), not a bug in the model or PortOS. Lower the resolution and/or frame count so each command buffer does less work, and quit other GPU-heavy apps so the render is not competing for the GPU with WindowServer compositing (driving this machine headless over SSH avoids that contention entirely).',
+ // SIGABRT is evidence-dependent — see abortCause.
SIGBUS: nativeCrashCause('SIGBUS'),
SIGSEGV: nativeCrashCause('SIGSEGV'),
});
+const WATCHDOG_ABORT_CAUSE = 'Render aborted (SIGABRT) — the macOS Metal command-buffer watchdog killed an over-long GPU command buffer (kIOGPUCommandBufferCallbackErrorImpactingInteractivity), not a bug in the model or PortOS. Lower the resolution and/or frame count so each command buffer does less work, and quit other GPU-heavy apps so the render is not competing for the GPU with WindowServer compositing (driving this machine headless over SSH avoids that contention entirely).';
+const OOM_ABORT_CAUSE = 'Render aborted (SIGABRT) — Metal reported the GPU ran out of memory. Try a smaller model, resolution or frame count.';
+const VICTIM_ABORT_CAUSE = 'Render aborted (SIGABRT) — Metal reported this render was an innocent victim of another process wedging the GPU. Retry once other GPU-heavy apps are closed.';
+const ASSERTION_ABORT_CAUSE = 'Render aborted (SIGABRT) — the native layer hit a failed assertion. See the runtime log for the assertion text.';
+const UNEXPLAINED_ABORT_CAUSE = 'Render aborted (SIGABRT) — a native abort with no diagnostic line captured, so the cause is unknown. Check the runtime log; retrying at a lower resolution/frame count may help.';
+const NATIVE_ASSERTION_RE = /\bassertion\b.*\bfailed\b|\bfailed\b.*\bassertion\b|__assert_rtn|Assertion`/i;
+
+// Classify from the child's ORIGINAL stderr only, before any explanatory text
+// exists, so an enum name quoted in our own advice can never act as proof. The
+// veto and signature regexes are the same ones that gate retry eligibility.
+function abortCause(stderr, { platform, arch, mlxRuntime }) {
+ const text = typeof stderr === 'string' ? stderr : '';
+ if (/kIOGPUCommandBufferCallbackErrorOutOfMemory/i.test(text)) return OOM_ABORT_CAUSE;
+ if (/kIOGPUCommandBufferCallbackErrorInnocentVictim/i.test(text)) return VICTIM_ABORT_CAUSE;
+ if (METAL_WATCHDOG_RE.test(text) && platform === 'darwin' && arch === 'arm64' && mlxRuntime) return WATCHDOG_ABORT_CAUSE;
+ if (NATIVE_ASSERTION_RE.test(text)) return ASSERTION_ABORT_CAUSE;
+ return UNEXPLAINED_ABORT_CAUSE;
+}
+
/**
* Failure reason for a render child that exited on a signal. Pure.
*
* Unmapped signals keep the historical `Killed by signal <SIG>` wording so a
- * novel signal is still reported verbatim rather than mis-attributed. The
- * resolved runtime fingerprint is appended when known, so the report names the
- * mlx / mlx-metal / chip / macOS stack that died without the user having to go
- * look it up (matching what RuntimeFingerprint.jsx surfaces on the page).
+ * novel signal is still reported verbatim rather than mis-attributed. SIGABRT is
+ * explained only from captured evidence: the Metal watchdog is named solely for a
+ * recognised timeout/interactivity signature on an Apple-silicon MLX runtime;
+ * otherwise OOM, innocent-victim, assertion or a generic unexplained abort. The
+ * resolved runtime fingerprint is appended when known.
*
* @param {string|null} signal - signal name from `proc.on('close', (code, signal))`
* @param {object} [opts]
* @param {object|null} [opts.fingerprint] - runtime fingerprint (see `pickDeathFingerprint` in runtimes.js)
+ * @param {string} [opts.stderr] - captured stderr tail of the dead child (original evidence)
+ * @param {boolean} [opts.mlxRuntime] - whether the render ran on an MLX runtime
* @returns {string|null} reason, or null when the child did not die on a signal
*/
-export function describeSignalDeath(signal, { fingerprint = null } = {}) {
+export function describeSignalDeath(signal, { fingerprint = null, stderr = '', mlxRuntime = false, platform = process.platform, arch = process.arch } = {}) {
if (!signal) return null;
- const cause = SIGNAL_DEATH_CAUSES[signal] || `Killed by signal ${signal}`;
+ const cause = signal === 'SIGABRT'
+ ? abortCause(stderr, { platform, arch, mlxRuntime })
+ : SIGNAL_DEATH_CAUSES[signal] || `Killed by signal ${signal}`;
const fp = formatRuntimeFingerprint(fingerprint);
return fp ? `${cause} [runtime: ${fp}]` : cause;
}
diff --git a/server/services/videoGen/generateVideoHelpers.test.js b/server/services/videoGen/generateVideoHelpers.test.js
index beac85357c..491451c720 100644
--- a/server/services/videoGen/generateVideoHelpers.test.js
+++ b/server/services/videoGen/generateVideoHelpers.test.js
@@ -449,16 +449,31 @@ describe('describeSignalDeath (#3101 signal → actionable cause)', () => {
expect(describeSignalDeath('')).toBeNull();
});
- it('names the macOS Metal command-buffer watchdog for SIGABRT and points at resolution/frame count', () => {
- const reason = describeSignalDeath('SIGABRT');
- expect(reason).toMatch(/SIGABRT/);
- expect(reason).toMatch(/Metal command-buffer watchdog/i);
- expect(reason).toMatch(/kIOGPUCommandBufferCallbackErrorImpactingInteractivity/);
- expect(reason).toMatch(/resolution/i);
- expect(reason).toMatch(/frame count/i);
- // Must NOT send the user hunting a model/PortOS bug (the upstream mistake).
- expect(reason).toMatch(/not a bug in the model or PortOS/i);
- expect(reason).not.toMatch(/assertion/i);
+ const MLX = { mlxRuntime: true, platform: 'darwin', arch: 'arm64' };
+
+ it('names the Metal watchdog only for a captured timeout signature on an Apple-silicon MLX runtime', () => {
+ for (const stderr of ['libc++abi: kIOGPUCommandBufferCallbackErrorImpactingInteractivity', 'Caused GPU Timeout Error', 'kIOGPUCommandBufferCallbackErrorTimeout']) {
+ const reason = describeSignalDeath('SIGABRT', { stderr, ...MLX });
+ expect(reason).toMatch(/Metal command-buffer watchdog/i);
+ expect(reason).toMatch(/frame count/i);
+ }
+ // Same evidence on a CUDA/Windows host or a non-MLX runtime is not a Metal watchdog.
+ expect(describeSignalDeath('SIGABRT', { stderr: 'Caused GPU Timeout Error', ...MLX, platform: 'win32' })).toMatch(/unknown/);
+ expect(describeSignalDeath('SIGABRT', { stderr: 'Caused GPU Timeout Error', ...MLX, mlxRuntime: false })).toMatch(/unknown/);
+ });
+
+ it('reports an unexplained native abort when no diagnostic was captured, never the watchdog', () => {
+ for (const opts of [undefined, { stderr: '', ...MLX }, { stderr: 'Loading weights...', ...MLX }]) {
+ const reason = describeSignalDeath('SIGABRT', opts);
+ expect(reason).toMatch(/native abort with no diagnostic/);
+ expect(reason).not.toMatch(/watchdog/i);
+ }
+ });
+
+ it('keeps OOM, innocent-victim and assertion aborts distinct from the watchdog, even beside a timeout line', () => {
+ expect(describeSignalDeath('SIGABRT', { stderr: 'kIOGPUCommandBufferCallbackErrorOutOfMemory', ...MLX })).toMatch(/ran out of memory/);
+ expect(describeSignalDeath('SIGABRT', { stderr: 'kIOGPUCommandBufferCallbackErrorInnocentVictim\nCaused GPU Timeout Error', ...MLX })).toMatch(/innocent victim/);
+ expect(describeSignalDeath('SIGABRT', { stderr: 'Assertion failed: (x > 0), function f, file a.cpp', ...MLX })).toMatch(/failed assertion/);
});
it('names a native MLX/Metal crash for SIGBUS and SIGSEGV', () => {
diff --git a/server/services/videoGen/local.test.js b/server/services/videoGen/local.test.js
index 75ef580947..3c4a5729e7 100644
--- a/server/services/videoGen/local.test.js
+++ b/server/services/videoGen/local.test.js
@@ -4065,13 +4065,10 @@ describe('generateVideo — signal-death diagnosis (#3101)', () => {
expect((await failed).failure).toMatchObject({ classification: 'attributeerror', signature: expect.stringMatching(/^[a-f0-9]{64}$/) });
});
- it('SIGABRT names the macOS Metal command-buffer watchdog with a resolution/frame-count next step', async () => {
+ it('a bare SIGABRT with no native line is an unexplained native abort, not a Metal watchdog', async () => {
const evt = await failWithSignal('signal-sigabrt', 'SIGABRT');
- expect(evt.error).toMatch(/Metal command-buffer watchdog/i);
- expect(evt.error).toMatch(/kIOGPUCommandBufferCallbackErrorImpactingInteractivity/);
- expect(evt.error).toMatch(/resolution/i);
- expect(evt.error).toMatch(/frame count/i);
- // The old bare wording must be gone.
+ expect(evt.error).toMatch(/native abort with no diagnostic/);
+ expect(evt.error).not.toMatch(/watchdog/i);
expect(evt.error).not.toMatch(/Killed by signal SIGABRT/);
});
diff --git a/server/services/videoGen/spawnWatch.js b/server/services/videoGen/spawnWatch.js
index e60a611336..da6e10460d 100644
--- a/server/services/videoGen/spawnWatch.js
+++ b/server/services/videoGen/spawnWatch.js
@@ -509,11 +509,13 @@ export async function spawnAndWatchVideo({
}
} else if (signal) {
// Signal → actionable cause (SIGABRT = the macOS Metal command-buffer
- // watchdog, SIGBUS/SIGSEGV = a native MLX/Metal crash, SIGKILL = OOM),
+ // watchdog only when stderr proves it, SIGBUS/SIGSEGV = a native MLX/Metal crash, SIGKILL = OOM),
// stamped with the runtime fingerprint that died so the report is
// self-documenting. See describeSignalDeath in generateVideoHelpers.js.
reason = describeSignalDeath(signal, {
fingerprint: await pickDeathFingerprint({ emitted: job.runtime, runtimeId: model.runtime }),
+ stderr: stderrTail.join('\n'),
+ mlxRuntime: runtimeUsesMlx(model.runtime),
});
} else {
const diagnostic = failure?.summary || failure?.cause;
diff --git a/server/services/videoGen/spawnWatch.test.js b/server/services/videoGen/spawnWatch.test.js
index 42036453c6..44e8652b2d 100644
--- a/server/services/videoGen/spawnWatch.test.js
+++ b/server/services/videoGen/spawnWatch.test.js
@@ -14,6 +14,7 @@ const mocks = vi.hoisted(() => ({
splice: vi.fn(),
thumbnail: vi.fn(),
history: [],
+ mlx: false,
}));
vi.mock('../../lib/heavyJobClaim.js', () => ({ claimHeavyLocalJob: mocks.claim }));
@@ -43,7 +44,7 @@ vi.mock('./runtimes.js', () => ({
BYOV_RUNTIME_INFO: {},
runtimeIsCacheOnly: () => false,
runtimeNeedsProcessGroupKill: () => false,
- runtimeUsesMlx: () => false,
+ runtimeUsesMlx: () => mocks.mlx,
invalidateByovReadyCache: vi.fn(),
pickDeathFingerprint: async () => null,
}));
@@ -76,6 +77,7 @@ describe('spawnAndWatchVideo claim lifecycle', () => {
vi.useFakeTimers();
vi.resetAllMocks();
mocks.history = [];
+ mocks.mlx = false;
mocks.release.mockResolvedValue();
mocks.handoff.mockResolvedValue();
mocks.claim.mockResolvedValue({ ok: true, release: mocks.release, handoffTo: mocks.handoff });
@@ -129,6 +131,45 @@ describe('spawnAndWatchVideo claim lifecycle', () => {
await vi.advanceTimersByTimeAsync(0);
};
+ describe('SIGABRT cause classification', () => {
+ const abortWith = async (stderrText) => {
+ child.stderr.emit('data', Buffer.from(stderrText));
+ child.signalCode = 'SIGABRT';
+ child.emit('close', null, 'SIGABRT');
+ await vi.advanceTimersByTimeAsync(0);
+ expect(failed).toHaveBeenCalledTimes(1);
+ return failed.mock.calls[0][0].error;
+ };
+ const asAppleSilicon = () => {
+ const orig = [Object.getOwnPropertyDescriptor(process, 'platform'), Object.getOwnPropertyDescriptor(process, 'arch')];
+ Object.defineProperty(process, 'platform', { value: 'darwin' });
+ Object.defineProperty(process, 'arch', { value: 'arm64' });
+ return () => { Object.defineProperty(process, 'platform', orig[0]); Object.defineProperty(process, 'arch', orig[1]); };
+ };
+
+ it('classifies from a final unterminated stderr line and arms no relaunch', async () => {
+ const restore = asAppleSilicon();
+ mocks.mlx = true;
+ try {
+ await spawnAndWatchVideo(params);
+ const error = await abortWith('Loading\nlibc++abi: kIOGPUCommandBufferCallbackErrorOutOfMemory');
+ expect(error).toMatch(/ran out of memory/);
+ expect(mocks.spawn).toHaveBeenCalledTimes(1);
+ } finally { restore(); }
+ });
+
+ it('reports an unexplained native abort when stderr carries no native line', async () => {
+ const restore = asAppleSilicon();
+ mocks.mlx = true;
+ try {
+ await spawnAndWatchVideo(params);
+ const error = await abortWith('Loading weights\n');
+ expect(error).toMatch(/native abort with no diagnostic/);
+ expect(error).not.toMatch(/watchdog/i);
+ } finally { restore(); }
+ });
+ });
+
const expectCleanup = () => {
expect(videoJobState.activeProcess).toBeNull();
expect(mocks.release).toHaveBeenCalledTimes(1);