Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: classify SIGABRT render aborts from captured stderr instead of assuming a Metal watchdog (#12263)
[._.]/ Adam Eivy0aaed908557e
diff --git a/server/services/videoGen/generateVideoHelpers.js b/server/services/videoGen/generateVideoHelpers.jsindex 08c6e515a3..818c2c3b5f 100644--- a/server/services/videoGen/generateVideoHelpers.js+++ b/server/services/videoGen/generateVideoHelpers.js@@ -400,14 +400,15 @@ export function formatRuntimeFingerprint(fp) {// Signal → actionable cause for a render child that died on a signal.//-// Apple Silicon reality drives this map. The dominant real-world render abort is+// Apple Silicon reality drives this map. A common real-world render abort is// the macOS Metal command-buffer watchdog killing an over-long GPU command// buffer (`kIOGPUCommandBufferCallbackErrorImpactingInteractivity`), which the// MLX/Metal layer surfaces as SIGABRT (exit -6) — NOT as a Python-level// assertion. Reporting that as a bare `Killed by signal SIGABRT` (or as a// "C-level assertion failed" hint, the mistake upstream made) sends users// hunting a model/PortOS bug when the fix is to shrink the work per command-// buffer. SIGBUS/SIGSEGV are the other native-crash arrivals from the same+// buffer — but only captured stderr can prove it (see abortCause); a bare SIGABRT+// is reported as an unexplained native abort. SIGBUS/SIGSEGV are the other native-crash arrivals from the same// layer. SIGKILL stays the OOM message it has always been.//// Keys are the signal NAMES Node reports on `close` (spawnDetached decodes the@@ -424,28 +425,52 @@ const nativeCrashCause = (signal) => `Render crashed (${signal}) — a native faconst SIGNAL_DEATH_CAUSES = Object.freeze({SIGKILL: 'Process killed (likely out of memory — try a smaller model or resolution)',- SIGABRT: 'Render aborted (SIGABRT) — on Apple Silicon this is almost always the macOS Metal command-buffer watchdog killing an over-long GPU command buffer (kIOGPUCommandBufferCallbackErrorImpactingInteractivity), not a bug in the model or PortOS. Lower the resolution and/or frame count so each command buffer does less work, and quit other GPU-heavy apps so the render is not competing for the GPU with WindowServer compositing (driving this machine headless over SSH avoids that contention entirely).',+ // SIGABRT is evidence-dependent — see abortCause.SIGBUS: nativeCrashCause('SIGBUS'),SIGSEGV: nativeCrashCause('SIGSEGV'),});+const WATCHDOG_ABORT_CAUSE = 'Render aborted (SIGABRT) — the macOS Metal command-buffer watchdog killed an over-long GPU command buffer (kIOGPUCommandBufferCallbackErrorImpactingInteractivity), not a bug in the model or PortOS. Lower the resolution and/or frame count so each command buffer does less work, and quit other GPU-heavy apps so the render is not competing for the GPU with WindowServer compositing (driving this machine headless over SSH avoids that contention entirely).';+const OOM_ABORT_CAUSE = 'Render aborted (SIGABRT) — Metal reported the GPU ran out of memory. Try a smaller model, resolution or frame count.';+const VICTIM_ABORT_CAUSE = 'Render aborted (SIGABRT) — Metal reported this render was an innocent victim of another process wedging the GPU. Retry once other GPU-heavy apps are closed.';+const ASSERTION_ABORT_CAUSE = 'Render aborted (SIGABRT) — the native layer hit a failed assertion. See the runtime log for the assertion text.';+const UNEXPLAINED_ABORT_CAUSE = 'Render aborted (SIGABRT) — a native abort with no diagnostic line captured, so the cause is unknown. Check the runtime log; retrying at a lower resolution/frame count may help.';+const NATIVE_ASSERTION_RE = /\bassertion\b.*\bfailed\b|\bfailed\b.*\bassertion\b|__assert_rtn|Assertion`/i;++// Classify from the child's ORIGINAL stderr only, before any explanatory text+// exists, so an enum name quoted in our own advice can never act as proof. The+// veto and signature regexes are the same ones that gate retry eligibility.+function abortCause(stderr, { platform, arch, mlxRuntime }) {+ const text = typeof stderr === 'string' ? stderr : '';+ if (/kIOGPUCommandBufferCallbackErrorOutOfMemory/i.test(text)) return OOM_ABORT_CAUSE;+ if (/kIOGPUCommandBufferCallbackErrorInnocentVictim/i.test(text)) return VICTIM_ABORT_CAUSE;+ if (METAL_WATCHDOG_RE.test(text) && platform === 'darwin' && arch === 'arm64' && mlxRuntime) return WATCHDOG_ABORT_CAUSE;+ if (NATIVE_ASSERTION_RE.test(text)) return ASSERTION_ABORT_CAUSE;+ return UNEXPLAINED_ABORT_CAUSE;+}+/*** Failure reason for a render child that exited on a signal. Pure.** Unmapped signals keep the historical `Killed by signal <SIG>` wording so a- * novel signal is still reported verbatim rather than mis-attributed. The- * resolved runtime fingerprint is appended when known, so the report names the- * mlx / mlx-metal / chip / macOS stack that died without the user having to go- * look it up (matching what RuntimeFingerprint.jsx surfaces on the page).+ * novel signal is still reported verbatim rather than mis-attributed. SIGABRT is+ * explained only from captured evidence: the Metal watchdog is named solely for a+ * recognised timeout/interactivity signature on an Apple-silicon MLX runtime;+ * otherwise OOM, innocent-victim, assertion or a generic unexplained abort. The+ * resolved runtime fingerprint is appended when known.** @param {string|null} signal - signal name from `proc.on('close', (code, signal))`* @param {object} [opts]* @param {object|null} [opts.fingerprint] - runtime fingerprint (see `pickDeathFingerprint` in runtimes.js)+ * @param {string} [opts.stderr] - captured stderr tail of the dead child (original evidence)+ * @param {boolean} [opts.mlxRuntime] - whether the render ran on an MLX runtime* @returns {string|null} reason, or null when the child did not die on a signal*/-export function describeSignalDeath(signal, { fingerprint = null } = {}) {+export function describeSignalDeath(signal, { fingerprint = null, stderr = '', mlxRuntime = false, platform = process.platform, arch = process.arch } = {}) {if (!signal) return null;- const cause = SIGNAL_DEATH_CAUSES[signal] || `Killed by signal ${signal}`;+ const cause = signal === 'SIGABRT'+ ? abortCause(stderr, { platform, arch, mlxRuntime })+ : SIGNAL_DEATH_CAUSES[signal] || `Killed by signal ${signal}`;const fp = formatRuntimeFingerprint(fingerprint);return fp ? `${cause} [runtime: ${fp}]` : cause;}diff --git a/server/services/videoGen/generateVideoHelpers.test.js b/server/services/videoGen/generateVideoHelpers.test.jsindex beac85357c..491451c720 100644--- a/server/services/videoGen/generateVideoHelpers.test.js+++ b/server/services/videoGen/generateVideoHelpers.test.js@@ -449,16 +449,31 @@ describe('describeSignalDeath (#3101 signal → actionable cause)', () => {expect(describeSignalDeath('')).toBeNull();});- it('names the macOS Metal command-buffer watchdog for SIGABRT and points at resolution/frame count', () => {- const reason = describeSignalDeath('SIGABRT');- expect(reason).toMatch(/SIGABRT/);- expect(reason).toMatch(/Metal command-buffer watchdog/i);- expect(reason).toMatch(/kIOGPUCommandBufferCallbackErrorImpactingInteractivity/);- expect(reason).toMatch(/resolution/i);- expect(reason).toMatch(/frame count/i);- // Must NOT send the user hunting a model/PortOS bug (the upstream mistake).- expect(reason).toMatch(/not a bug in the model or PortOS/i);- expect(reason).not.toMatch(/assertion/i);+ const MLX = { mlxRuntime: true, platform: 'darwin', arch: 'arm64' };++ it('names the Metal watchdog only for a captured timeout signature on an Apple-silicon MLX runtime', () => {+ for (const stderr of ['libc++abi: kIOGPUCommandBufferCallbackErrorImpactingInteractivity', 'Caused GPU Timeout Error', 'kIOGPUCommandBufferCallbackErrorTimeout']) {+ const reason = describeSignalDeath('SIGABRT', { stderr, ...MLX });+ expect(reason).toMatch(/Metal command-buffer watchdog/i);+ expect(reason).toMatch(/frame count/i);+ }+ // Same evidence on a CUDA/Windows host or a non-MLX runtime is not a Metal watchdog.+ expect(describeSignalDeath('SIGABRT', { stderr: 'Caused GPU Timeout Error', ...MLX, platform: 'win32' })).toMatch(/unknown/);+ expect(describeSignalDeath('SIGABRT', { stderr: 'Caused GPU Timeout Error', ...MLX, mlxRuntime: false })).toMatch(/unknown/);+ });++ it('reports an unexplained native abort when no diagnostic was captured, never the watchdog', () => {+ for (const opts of [undefined, { stderr: '', ...MLX }, { stderr: 'Loading weights...', ...MLX }]) {+ const reason = describeSignalDeath('SIGABRT', opts);+ expect(reason).toMatch(/native abort with no diagnostic/);+ expect(reason).not.toMatch(/watchdog/i);+ }+ });++ it('keeps OOM, innocent-victim and assertion aborts distinct from the watchdog, even beside a timeout line', () => {+ expect(describeSignalDeath('SIGABRT', { stderr: 'kIOGPUCommandBufferCallbackErrorOutOfMemory', ...MLX })).toMatch(/ran out of memory/);+ expect(describeSignalDeath('SIGABRT', { stderr: 'kIOGPUCommandBufferCallbackErrorInnocentVictim\nCaused GPU Timeout Error', ...MLX })).toMatch(/innocent victim/);+ expect(describeSignalDeath('SIGABRT', { stderr: 'Assertion failed: (x > 0), function f, file a.cpp', ...MLX })).toMatch(/failed assertion/);});it('names a native MLX/Metal crash for SIGBUS and SIGSEGV', () => {diff --git a/server/services/videoGen/local.test.js b/server/services/videoGen/local.test.jsindex 75ef580947..3c4a5729e7 100644--- a/server/services/videoGen/local.test.js+++ b/server/services/videoGen/local.test.js@@ -4065,13 +4065,10 @@ describe('generateVideo — signal-death diagnosis (#3101)', () => {expect((await failed).failure).toMatchObject({ classification: 'attributeerror', signature: expect.stringMatching(/^[a-f0-9]{64}$/) });});- it('SIGABRT names the macOS Metal command-buffer watchdog with a resolution/frame-count next step', async () => {+ it('a bare SIGABRT with no native line is an unexplained native abort, not a Metal watchdog', async () => {const evt = await failWithSignal('signal-sigabrt', 'SIGABRT');- expect(evt.error).toMatch(/Metal command-buffer watchdog/i);- expect(evt.error).toMatch(/kIOGPUCommandBufferCallbackErrorImpactingInteractivity/);- expect(evt.error).toMatch(/resolution/i);- expect(evt.error).toMatch(/frame count/i);- // The old bare wording must be gone.+ expect(evt.error).toMatch(/native abort with no diagnostic/);+ expect(evt.error).not.toMatch(/watchdog/i);expect(evt.error).not.toMatch(/Killed by signal SIGABRT/);});diff --git a/server/services/videoGen/spawnWatch.js b/server/services/videoGen/spawnWatch.jsindex e60a611336..da6e10460d 100644--- a/server/services/videoGen/spawnWatch.js+++ b/server/services/videoGen/spawnWatch.js@@ -509,11 +509,13 @@ export async function spawnAndWatchVideo({}} else if (signal) {// Signal → actionable cause (SIGABRT = the macOS Metal command-buffer- // watchdog, SIGBUS/SIGSEGV = a native MLX/Metal crash, SIGKILL = OOM),+ // watchdog only when stderr proves it, SIGBUS/SIGSEGV = a native MLX/Metal crash, SIGKILL = OOM),// stamped with the runtime fingerprint that died so the report is// self-documenting. See describeSignalDeath in generateVideoHelpers.js.reason = describeSignalDeath(signal, {fingerprint: await pickDeathFingerprint({ emitted: job.runtime, runtimeId: model.runtime }),+ stderr: stderrTail.join('\n'),+ mlxRuntime: runtimeUsesMlx(model.runtime),});} else {const diagnostic = failure?.summary || failure?.cause;diff --git a/server/services/videoGen/spawnWatch.test.js b/server/services/videoGen/spawnWatch.test.jsindex 42036453c6..44e8652b2d 100644--- a/server/services/videoGen/spawnWatch.test.js+++ b/server/services/videoGen/spawnWatch.test.js@@ -14,6 +14,7 @@ const mocks = vi.hoisted(() => ({splice: vi.fn(),thumbnail: vi.fn(),history: [],+ mlx: false,}));vi.mock('../../lib/heavyJobClaim.js', () => ({ claimHeavyLocalJob: mocks.claim }));@@ -43,7 +44,7 @@ vi.mock('./runtimes.js', () => ({BYOV_RUNTIME_INFO: {},runtimeIsCacheOnly: () => false,runtimeNeedsProcessGroupKill: () => false,- runtimeUsesMlx: () => false,+ runtimeUsesMlx: () => mocks.mlx,invalidateByovReadyCache: vi.fn(),pickDeathFingerprint: async () => null,}));@@ -76,6 +77,7 @@ describe('spawnAndWatchVideo claim lifecycle', () => {vi.useFakeTimers();vi.resetAllMocks();mocks.history = [];+ mocks.mlx = false;mocks.release.mockResolvedValue();mocks.handoff.mockResolvedValue();mocks.claim.mockResolvedValue({ ok: true, release: mocks.release, handoffTo: mocks.handoff });@@ -129,6 +131,45 @@ describe('spawnAndWatchVideo claim lifecycle', () => {await vi.advanceTimersByTimeAsync(0);};+ describe('SIGABRT cause classification', () => {+ const abortWith = async (stderrText) => {+ child.stderr.emit('data', Buffer.from(stderrText));+ child.signalCode = 'SIGABRT';+ child.emit('close', null, 'SIGABRT');+ await vi.advanceTimersByTimeAsync(0);+ expect(failed).toHaveBeenCalledTimes(1);+ return failed.mock.calls[0][0].error;+ };+ const asAppleSilicon = () => {+ const orig = [Object.getOwnPropertyDescriptor(process, 'platform'), Object.getOwnPropertyDescriptor(process, 'arch')];+ Object.defineProperty(process, 'platform', { value: 'darwin' });+ Object.defineProperty(process, 'arch', { value: 'arm64' });+ return () => { Object.defineProperty(process, 'platform', orig[0]); Object.defineProperty(process, 'arch', orig[1]); };+ };++ it('classifies from a final unterminated stderr line and arms no relaunch', async () => {+ const restore = asAppleSilicon();+ mocks.mlx = true;+ try {+ await spawnAndWatchVideo(params);+ const error = await abortWith('Loading\nlibc++abi: kIOGPUCommandBufferCallbackErrorOutOfMemory');+ expect(error).toMatch(/ran out of memory/);+ expect(mocks.spawn).toHaveBeenCalledTimes(1);+ } finally { restore(); }+ });++ it('reports an unexplained native abort when stderr carries no native line', async () => {+ const restore = asAppleSilicon();+ mocks.mlx = true;+ try {+ await spawnAndWatchVideo(params);+ const error = await abortWith('Loading weights\n');+ expect(error).toMatch(/native abort with no diagnostic/);+ expect(error).not.toMatch(/watchdog/i);+ } finally { restore(); }+ });+ });+const expectCleanup = () => {expect(videoJobState.activeProcess).toBeNull();expect(mocks.release).toHaveBeenCalledTimes(1);