Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: keep LTX prompt tokens intact during Metal watchdog recovery (#12265)
[._.]/ Adam Eivy3bb413656ae3
diff --git a/scripts/generate_fastvideo.py b/scripts/generate_fastvideo.pyindex fdf299bdd6..fb576b046d 100755--- a/scripts/generate_fastvideo.py+++ b/scripts/generate_fastvideo.py@@ -60,11 +60,9 @@ _PERCENT_PATTERN = re.compile(r'\d+%')# from the key order of the label table below.# NOT "encode-prompt": that exact marker is generate_ltx2.py's Gemma# prompt-encode BEGIN sentinel (PROMPT_ENCODE_BEGIN_MARKER in-# generateVideoHelpers.js), and the server arms an ltx2-specific relaunch on it.+# generateVideoHelpers.js), and the server names the prompt-encode phase from it.# Emitting it here would open a phase this runner never closes, so a FastVideo-# render killed by the Metal watchdog would be "retried" with ltx2's-# --gemma-max-length flag, which this script's argparse rejects — replacing the-# real watchdog error with an exit-2 after a wasted full model re-spawn.+# watchdog abort would be reported as an encoder abort._PHASE_ORDER = ("load-pipeline", "conditioning", "sampling", "mux")# Phase id -> the sentence the user reads, emitted once per transition. Ids arediff --git a/scripts/generate_ltx2.py b/scripts/generate_ltx2.pyindex 91fe1503e3..be8f836a91 100644--- a/scripts/generate_ltx2.py+++ b/scripts/generate_ltx2.py@@ -569,24 +569,34 @@ PROMPT_ENCODE_BEGIN_MARKER = "STAGE:encode-prompt"PROMPT_ENCODE_END_MARKER = "STAGE:encode-prompt-done"-def configure_gemma_max_length(max_length: int | None) -> None:- """Pin the Gemma prompt-encode sequence length for this run.+def gemma_max_length() -> int:+ """The sequence budget ltx-2-mlx will apply (``LTX2_GEMMA_MAX_LENGTH``, default 1024)."""+ return int(os.environ.get("LTX2_GEMMA_MAX_LENGTH", "1024"))- ltx-2-mlx reads ``LTX2_GEMMA_MAX_LENGTH`` at encode time- (``ltx_pipelines_mlx.utils.blocks.PromptEncoder.encode``, and again in- ``_base`` for Prompt Relay token ranges), defaulting to 1024. PortOS passes a- lowered value on its one-shot relaunch after the macOS Metal command-buffer- watchdog aborts the encoder, so this ASSIGNS rather than ``setdefault``s: an- explicit flag has to beat whatever the parent environment already carried.- No-op when the flag is absent, which leaves upstream's own default in force.+def enforce_prompt_token_limit(encoder, prompt: str) -> None:+ """Refuse a prompt the Gemma encoder would silently truncate.++ ltx-2-mlx's ``GemmaEncoder.tokenize`` keeps only the LAST ``max_length``+ tokens of an over-long input, so the start of an accepted prompt (usually the+ subject) would vanish while history still shows the full text. This counts+ with the encoder's own tokenizer on the exact string it will tokenize+ (``text.strip()``) and exits before any sampling work. PortOS never lowers+ the budget to recover from a Metal watchdog abort for the same reason.++ Silently skipped when the loaded encoder exposes no tokenizer, so a pin with a+ different layout keeps its upstream behavior."""- if max_length is None:+ tokenizer = getattr(getattr(encoder, "_text_encoder", None), "_tokenizer", None)+ if not callable(getattr(tokenizer, "encode", None)):return- if max_length < 1:- raise SystemExit(f"--gemma-max-length must be a positive integer; got {max_length}.")- os.environ["LTX2_GEMMA_MAX_LENGTH"] = str(max_length)- emit_status(f"Gemma prompt-encode capped at {max_length} tokens")+ count = len(tokenizer.encode(prompt.strip()))+ limit = gemma_max_length()+ if count > limit:+ raise SystemExit(+ f"Prompt is {count} Gemma tokens but the text encoder accepts at most {limit}; "+ "shorten the prompt (it would otherwise lose its beginning)."+ )def install_prompt_encode_markers() -> None:@@ -618,10 +628,15 @@ def install_prompt_encode_markers() -> None:return@functools.wraps(original)- def encode(self, *args, **kwargs):+ def encode(self, prompt, *args, **kwargs):print(PROMPT_ENCODE_BEGIN_MARKER, file=sys.stderr, flush=True)try:- return original(self, *args, **kwargs)+ # The tokenizer only exists after load(); upstream's encode() calls it+ # first and it is idempotent, so loading here costs nothing extra.+ if callable(getattr(self, "load", None)):+ self.load()+ enforce_prompt_token_limit(self, prompt)+ return original(self, prompt, *args, **kwargs)finally:print(PROMPT_ENCODE_END_MARKER, file=sys.stderr, flush=True)@@ -762,12 +777,6 @@ def parse_args() -> argparse.Namespace:"measured with. Reported as degraded (never fatal) when the model ""pack does not carry it, so a slower render is never presented as ""the profile's validated speed.")- p.add_argument("--gemma-max-length", type=int, default=None,- help="Gemma prompt-encode sequence length (LTX2_GEMMA_MAX_LENGTH; "- "upstream default 1024). PortOS lowers this on its one-shot "- "relaunch after the macOS Metal command-buffer watchdog aborts "- "the prompt encoder. An explicit value always wins over the "- "ambient environment.")p.add_argument("--mlx-cache-limit-mb", default=None,help="Cap the MLX allocator cache at this many MB for the whole run, ""reasserted before every render. Omit to derive a conservative "@@ -2136,7 +2145,7 @@ def physical_memory_bytes() -> int | None:def resolve_mlx_cache_policy(physical_bytes, override_raw=None, env_raw=None) -> dict:"""Effective cache policy as ``{"limitMb": int | None, "source": str}``.- Precedence mirrors --gemma-max-length: an explicit flag beats the ambient+ Precedence: an explicit flag beats the ambientenvironment, which beats the ceiling derived from physical memory. A machinewhose memory could not be read gets ``limitMb: None`` and MLX's own defaultstands — a blind floor would throttle a large box exactly as readily as it@@ -2423,7 +2432,6 @@ def main() -> NoReturn:# before any model load). Each run_* sets pipe._pending_loras from this.args.user_lora_specs = parse_user_loras(args.user_loras)configure_negative_prompt(args.negative_prompt)- configure_gemma_max_length(args.gemma_max_length)install_prompt_encode_markers()configure_mlx_cache(args)# Frame-one anchor contract (#5422). Runs before any pipeline is built so adiff --git a/scripts/generate_ltx2.test.js b/scripts/generate_ltx2.test.jsindex 3babcdb65f..1b27991b96 100644--- a/scripts/generate_ltx2.test.js+++ b/scripts/generate_ltx2.test.js@@ -345,7 +345,7 @@ describe.skipIf(!pyBin)('generate_ltx2.py', () => {// ── Gemma prompt-encode budget + boundary markers (#4589) ──────────────────// The markers are what lets PortOS tell "the Metal watchdog killed the prompt- // encoder" (worth one relaunch at a smaller budget) from "it killed the+ // encoder" (worth one relaunch with smaller command buffers) from "it killed the// denoise loop" (not worth anything), so their exact text and their emission// on BOTH the success and the failure path are load-bearing.const stubPromptEncoderModule = (body) => [@@ -424,50 +424,61 @@ describe.skipIf(!pyBin)('generate_ltx2.py', () => {expect(output.trim()).toBe('ok');});- // ltx-2-mlx reads LTX2_GEMMA_MAX_LENGTH at encode time and defaults it to- // 1024, so the flag has to ASSIGN — a setdefault would be swallowed by the- // ambient value the parent already exported and the relaunch would re-run at- // the budget that just aborted.- it.each([- ['None', 'None', null, '1024'],- ['a lowered budget', '512', null, '512'],- ['a lowered budget over an ambient value', '512', '1024', '512'],- ])('configure_gemma_max_length(%s) resolves LTX2_GEMMA_MAX_LENGTH to the expected value', (_label, flag, ambient, expected) => {+ // The upstream tokenizer keeps only the LAST max_length tokens, so an over-long+ // prompt silently loses its beginning. The stub reproduces that truncation so+ // the assertion is on the token sequence the encoder actually receives, not on+ // the unchanged Python string.+ const truncatingEncoder = [+ ' _text_encoder = type("E", (), {"_tokenizer": type("T", (), {"encode": lambda self, t: t.split()})()})()',+ ' def load(self): pass',+ ' def encode(self, prompt):',+ ' import os',+ ' limit = int(os.environ.get("LTX2_GEMMA_MAX_LENGTH", "1024"))',+ ' tokens = prompt.strip().split()',+ ' if len(tokens) > limit: tokens = tokens[-limit:]',+ ' return (tokens[0], tokens[-1])',+ ];++ it.each([['at the 1024 limit', 1024], ['inside 513-1024', 800]])('hands the encoder the whole prompt %s (first and last token intact)', (_label, count) => {const output = runPython(`${importRunner}\n${[- 'import os',- ambient === null- ? 'os.environ.pop("LTX2_GEMMA_MAX_LENGTH", None)'- : `os.environ["LTX2_GEMMA_MAX_LENGTH"] = ${JSON.stringify(ambient)}`,- `runner.configure_gemma_max_length(${flag})`,- // Read back through the same default upstream applies, so "unset" and- // "explicitly 1024" are both expressed as the value the encoder would use.- 'print(os.environ.get("LTX2_GEMMA_MAX_LENGTH", "1024"))',+ ...stubPromptEncoderModule(truncatingEncoder),+ 'import contextlib, os',+ 'os.environ.pop("LTX2_GEMMA_MAX_LENGTH", None)',+ 'runner.install_prompt_encode_markers()',+ `prompt = " ".join("t%d" % i for i in range(${count}))`,+ 'with contextlib.redirect_stderr(sys.stdout):',+ ' print(PromptEncoder().encode(prompt))',].join('\n')}`);- expect(output.trim()).toBe(expected);+ expect(output.trim().split(/\r?\n/).pop()).toBe(`('t0', 't${count - 1}')`);});- it.each([['0'], ['-1']])('rejects a non-positive --gemma-max-length (%s)', (value) => {+ it('refuses an over-limit prompt with the measured count and limit before encoding', () => {const output = runPython(`${importRunner}\n${[- 'try:',- ` runner.configure_gemma_max_length(${value})`,- 'except SystemExit as exc:',- ' print(str(exc))',- 'else:',- ' raise SystemExit("a non-positive gemma budget was accepted")',+ ...stubPromptEncoderModule(truncatingEncoder.map((l) => (l.includes('return (tokens[0]') ? ' raise RuntimeError("encoder reached")' : l))),+ 'import contextlib, os',+ 'os.environ.pop("LTX2_GEMMA_MAX_LENGTH", None)',+ 'runner.install_prompt_encode_markers()',+ 'prompt = " ".join("t%d" % i for i in range(1025))',+ 'with contextlib.redirect_stderr(sys.stdout):',+ ' try:',+ ' PromptEncoder().encode(prompt)',+ ' except SystemExit as exc:',+ ' print("REFUSED:" + str(exc))',].join('\n')}`);- expect(output).toMatch(/must be a positive integer/);+ expect(output).toMatch(/REFUSED:Prompt is 1025 Gemma tokens but the text encoder accepts at most 1024/);+ expect(output).not.toMatch(/encoder reached/);});- it.each([- ['omitted', [], 'None'],- ['given', ['--gemma-max-length', '512'], '512'],- ])('parses --gemma-max-length when %s', (_label, extra, expected) => {+ it('no longer exposes a flag that lowers the encoder sequence budget', () => {const output = runPython(`${importRunner}\n${['import sys',- `sys.argv = ["generate_ltx2.py", "--mode", "text", "--prompt", "p", "--output", "/tmp/o.mp4", "--model", "m"] + ${JSON.stringify(extra)}`,- 'print(runner.parse_args().gemma_max_length)',+ 'sys.argv = ["generate_ltx2.py", "--mode", "text", "--prompt", "p", "--output", "/tmp/o.mp4", "--model", "m", "--gemma-max-length", "512"]',+ 'try:',+ ' runner.parse_args()',+ 'except SystemExit:',+ ' print("REJECTED")',].join('\n')}`);- expect(output.trim()).toBe(expected);+ expect(output).toMatch(/REJECTED/);});// The reference-mode promise (#4874). "inspire" is only expressible where thediff --git a/server/services/videoGen/generateVideoHelpers.js b/server/services/videoGen/generateVideoHelpers.jsindex 08c6e515a3..25707f645c 100644--- a/server/services/videoGen/generateVideoHelpers.js+++ b/server/services/videoGen/generateVideoHelpers.js@@ -233,7 +233,7 @@ export function makeVideoGenLineHandler({ job, jobId, pythonNoiseRe }) {// 'active' = encoding right now, 'done' = finished — so a later SIGABRT can// be attributed to the encode without conflating "never reported" with// "finished". Compared exactly: the '-done' marker is a prefix extension of- // the begin marker. See planPromptEncodingRetry.+ // the begin marker. Used only to name the phase in the recovery status.if (line === PROMPT_ENCODE_BEGIN_MARKER) job.promptEncodePhase = 'active';else if (line === PROMPT_ENCODE_END_MARKER) job.promptEncodePhase = 'done';const parts = line.split(':');@@ -450,19 +450,6 @@ export function describeSignalDeath(signal, { fingerprint = null } = {}) {return fp ? `${cause} [runtime: ${fp}]` : cause;}-/**- * Gemma prompt-encode sequence length. `LTX2_GEMMA_MAX_LENGTH` is read by- * ltx-2-mlx at encode time (`utils/blocks.py#PromptEncoder.encode`) and defaults- * to 1024 there; the retry halves it so the encoder's attention matrices are a- * quarter the size and each Metal command buffer finishes well inside the- * watchdog window. Halving (rather than a deeper cut) keeps a long prompt's- * tail intact — the tokenizer truncates from the LEFT, so a smaller budget- * silently drops the START of the prompt, which is where the subject usually- * is.- */-export const DEFAULT_GEMMA_MAX_LENGTH = 1024;-export const RETRY_GEMMA_MAX_LENGTH = 512;-// The macOS Metal command-buffer watchdog reports WHY it killed a buffer via a// `kIOGPUCommandBufferCallbackError*` enum name in the abort text. Two of those// mean "this buffer ran too long for the compositor to stay responsive", which@@ -512,33 +499,6 @@ export function isPromptEncodingMetalWatchdog({ signal = null, stderr = '', promreturn METAL_WATCHDOG_RE.test(text);}-/**- * Decide whether one render may be relaunched after a prompt-encode watchdog- * abort, and with what Gemma budget. Pure — returns the plan, never acts.- *- * Returns `null` for "do not retry" and `{ gemmaMaxLength }` for "relaunch this- * same job once with that budget". Exactly one retry is ever allowed: a second- * watchdog abort at the reduced budget is a real failure the user must see, not- * something to keep grinding the GPU over.- *- * @param {object} [opts]- * @param {string|null} [opts.signal]- * @param {string} [opts.stderr]- * @param {'active'|'done'|null} [opts.promptEncodePhase]- * @param {number} [opts.retriesUsed] - retries already spent on this job- * @param {string} [opts.platform] - `process.platform` of the host- * @returns {{ gemmaMaxLength: number }|null}- */-export function planPromptEncodingRetry({ signal = null, stderr = '', promptEncodePhase = null, retriesUsed = 0, platform = process.platform } = {}) {- // The command-buffer watchdog is a macOS construct. A Windows/CUDA render- // that somehow produced a matching string must not be relaunched with an- // Apple-specific mitigation.- if (platform !== 'darwin') return null;- if (!Number.isInteger(retriesUsed) || retriesUsed > 0) return null;- if (!isPromptEncodingMetalWatchdog({ signal, stderr, promptEncodePhase })) return null;- return { gemmaMaxLength: RETRY_GEMMA_MAX_LENGTH };-}-/*** Per-command-buffer caps MLX reads at startup (`MLX_MAX_OPS_PER_BUFFER` /* `MLX_MAX_MB_PER_BUFFER`). Smaller values only move command-buffer boundaries —@@ -552,8 +512,10 @@ export const METAL_BUFFER_RETRY_ENV = Object.freeze({/*** Decide whether a render may be relaunched once with tiny MLX command buffers- * after a Metal watchdog abort ANYWHERE in the render (the prompt-encode case has- * its own plan, planPromptEncodingRetry). Pure — returns the plan, never acts.+ * after a Metal watchdog abort ANYWHERE in the render, including inside the Gemma+ * prompt encoder. The prompt token sequence is never shortened to recover: the+ * encoder keeps only the LAST max_length tokens, so a lowered sequence budget+ * would silently drop the start of an accepted prompt. Pure — returns the plan, never acts.* Apple-silicon MLX runtimes only; OutOfMemory / InnocentVictim aborts never* qualify (smaller buffers do not fix them). One retry per job.*diff --git a/server/services/videoGen/generateVideoHelpers.test.js b/server/services/videoGen/generateVideoHelpers.test.jsindex beac85357c..a49e2628b1 100644--- a/server/services/videoGen/generateVideoHelpers.test.js+++ b/server/services/videoGen/generateVideoHelpers.test.js@@ -4,7 +4,7 @@ import { tmpdir } from 'os';import { join } from 'path';import { EventEmitter } from 'events';import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../../lib/mockPathsDataRoot.js';-import { makeVideoGenLineHandler, isWatchdogSuccess, finalizeGeneratedVideo, parseByteProgress, formatBytes, formatDownloadMessage, describeSignalDeath, formatRuntimeFingerprint, describeRenderConditioning, isPromptEncodingMetalWatchdog, planPromptEncodingRetry, DEFAULT_GEMMA_MAX_LENGTH, RETRY_GEMMA_MAX_LENGTH, bufferChildExit, isNativeTeardownAbort, verifyPostCompletionOutputs, POST_COMPLETION_ABORT_RUNTIME, RENDER_INPUTS_VERSION, emitCloudRenderStatus, CLOUD_RENDER_PHASE } from './generateVideoHelpers.js';+import { makeVideoGenLineHandler, isWatchdogSuccess, finalizeGeneratedVideo, parseByteProgress, formatBytes, formatDownloadMessage, describeSignalDeath, formatRuntimeFingerprint, describeRenderConditioning, isPromptEncodingMetalWatchdog, bufferChildExit, isNativeTeardownAbort, verifyPostCompletionOutputs, POST_COMPLETION_ABORT_RUNTIME, RENDER_INPUTS_VERSION, emitCloudRenderStatus, CLOUD_RENDER_PHASE } from './generateVideoHelpers.js';describe('parseByteProgress', () => {it('parses single byte value (e.g., "2.5G")', () => {@@ -133,7 +133,7 @@ describe('makeVideoGenLineHandler', () => {const eventsOfType = (t) => emitted.filter((e) => e.type === t).map((e) => e.payload);// Prompt-encode phase (#4589) — the signal that decides whether a later- // SIGABRT is worth one relaunch at a smaller Gemma budget.+ // SIGABRT is worth one relaunch with smaller command buffers.it('tracks the Gemma prompt-encode phase as three distinct states', () => {// Never reported: every runtime other than the LTX-2 MLX helper stays here,// and an unstamped field must not read as "encoding right now".@@ -594,41 +594,6 @@ describe('isPromptEncodingMetalWatchdog', () => {});});-describe('planPromptEncodingRetry', () => {- const WATCHDOG_ABORT = '[METAL] Command buffer execution failed: (00000004:kIOGPUCommandBufferCallbackErrorImpactingInteractivity)';- const qualifying = {- signal: 'SIGABRT',- stderr: WATCHDOG_ABORT,- promptEncodePhase: 'active',- retriesUsed: 0,- platform: 'darwin',- };-- it('plans one relaunch at the reduced Gemma budget', () => {- expect(planPromptEncodingRetry(qualifying)).toEqual({ gemmaMaxLength: RETRY_GEMMA_MAX_LENGTH });- // The reduced budget is a real cut from what upstream would otherwise use —- // a plan that handed back the default would relaunch into the same abort.- expect(RETRY_GEMMA_MAX_LENGTH).toBeLessThan(DEFAULT_GEMMA_MAX_LENGTH);- });-- it('allows exactly one — a second abort at the reduced budget is a real failure', () => {- expect(planPromptEncodingRetry({ ...qualifying, retriesUsed: 1 })).toBeNull();- expect(planPromptEncodingRetry({ ...qualifying, retriesUsed: 2 })).toBeNull();- });-- // The command-buffer watchdog is a macOS construct; a Windows/CUDA render that- // somehow produced a matching string must not be relaunched with an- // Apple-specific mitigation.- it.each([['win32'], ['linux']])('never fires on %s', (platform) => {- expect(planPromptEncodingRetry({ ...qualifying, platform })).toBeNull();- });-- it('passes the classifier verdict straight through', () => {- expect(planPromptEncodingRetry({ ...qualifying, promptEncodePhase: 'done' })).toBeNull();- expect(planPromptEncodingRetry({ ...qualifying, signal: 'SIGKILL' })).toBeNull();- });-});-// The window between "spawn resolved" and "real listeners attached" is an await// on real file I/O (the accelerator claim handoff). A child that dies in it// emits unsubscribed — and for 'error' that is not merely lost but fatal, sincediff --git a/server/services/videoGen/local.test.js b/server/services/videoGen/local.test.jsindex 75ef580947..278dbf2168 100644--- a/server/services/videoGen/local.test.js+++ b/server/services/videoGen/local.test.js@@ -5297,7 +5297,7 @@ describe('generateVideo — Gemma prompt-encode watchdog relaunch', () => {it.each([['the classic timeout signature', TIMEOUT_ABORT],['the impacting-interactivity signature newer macOS reports', INTERACTIVITY_ABORT],- ])('relaunches once at a reduced Gemma budget on %s', async (_label, abort) => {+ ])('relaunches once with smaller command buffers, prompt untouched, on %s', async (_label, abort) => {const first = makeDrivenProc(101);const second = makeDrivenProc(102);const spawnMock = await startRender(`pe-${abort.length}`, [first, second]);@@ -5309,11 +5309,13 @@ describe('generateVideo — Gemma prompt-encode watchdog relaunch', () => {const calls = ltx2Calls(spawnMock);expect(calls).toHaveLength(2);const [firstArgs, retryArgs] = calls.map(([, args]) => args);- // The original render never carries the flag — the reduced budget exists- // only as the mitigation, not as a new default.- expect(firstArgs).not.toContain('--gemma-max-length');- expect(flagValue(retryArgs, '--gemma-max-length')).toBe('512');- // Same render, smaller prompt budget: seed and output must survive verbatim+ // Recovery never shortens the encoder sequence (the encoder would drop the+ // START of the prompt): the args are byte-identical and only the env differs.+ expect(retryArgs).toEqual(firstArgs);+ expect(retryArgs).not.toContain('--gemma-max-length');+ expect(calls[0][2].env.MLX_MAX_OPS_PER_BUFFER).toBeUndefined();+ expect(calls[1][2].env).toMatchObject({ MLX_MAX_OPS_PER_BUFFER: '4', MLX_MAX_MB_PER_BUFFER: '16' });+ // Same render, smaller command buffers: seed and output must survive verbatim// or the relaunch silently produces a different clip than the user asked for.expect(flagValue(retryArgs, '--seed')).toBe(flagValue(firstArgs, '--seed'));expect(flagValue(retryArgs, '--seed')).toBe('987654');@@ -5413,7 +5415,7 @@ describe('generateVideo — Gemma prompt-encode watchdog relaunch', () => {expect(failures).toHaveLength(1);});- it('relaunches at most once — a second abort at the reduced budget fails the job', async () => {+ it('relaunches at most once — a second abort at the smaller setting fails the job', async () => {const first = makeDrivenProc(105);const second = makeDrivenProc(106);const spawnMock = await startRender('pe-twice', [first, second]);diff --git a/server/services/videoGen/spawnWatch.js b/server/services/videoGen/spawnWatch.jsindex e60a611336..b0e9908f46 100644--- a/server/services/videoGen/spawnWatch.js+++ b/server/services/videoGen/spawnWatch.js@@ -26,7 +26,7 @@ import {isNativeTeardownAbort,verifyPostCompletionOutputs,describeSignalDeath,- planPromptEncodingRetry,+ isPromptEncodingMetalWatchdog,planMetalBufferRetry,bufferChildExit,} from './generateVideoHelpers.js';@@ -101,15 +101,12 @@ export async function spawnAndWatchVideo({// Reads (and detaches) whatever terminal event the first child emitted before// its real listeners were attached. Set the instant the spawn resolves.let takeEarlyExit = null;- // Prompt-encode relaunches already spent on this job. Exactly one is allowed:- // a second watchdog abort at the reduced budget is a real failure the user has- // to see, not something to keep grinding the GPU over.- let promptEncodingRetriesUsed = 0;- // Same single-shot rule for the small-command-buffer relaunch (any-phase Metal- // watchdog abort on an MLX runtime).+ // The one allowed relaunch per job: a second watchdog abort at the smaller+ // command-buffer setting is a real failure the user has to see, not something+ // to keep grinding the GPU over. Covers every phase, prompt encode included.let metalBufferRetriesUsed = 0;// Hoisted out of the try so a relaunch can respawn with the SAME child env- // plus a lowered Gemma budget, instead of rebuilding it from scratch.+ // plus smaller MLX command buffers, instead of rebuilding it from scratch.let childEnv;// Runners publish one atomically-replaced `preview.png`. Keep the reader@@ -192,8 +189,8 @@ export async function spawnAndWatchVideo({// without minting a new one. Everything job-level (jobId, args, seed, meta,// history entry, heavy claim, staged temp files) is closed over and survives// the relaunch. The only thing that relaunches a render is a Metal- // command-buffer watchdog abort inside the Gemma prompt encoder; see- // maybeRelaunchForPromptEncoding below.+ // command-buffer watchdog abort on an MLX runtime; see+ // maybeRelaunchForMetalWatchdog below.const wireRenderChild = (proc) => {// The prompt-encode phase belongs to ONE child, but `job` outlives the// relaunch — reset it explicitly so a phase left open by the child that just@@ -427,12 +424,11 @@ export async function spawnAndWatchVideo({// guard survives anyone later putting an await between the two.const cancelEpochAtClose = videoJobState.cancelEpoch;videoJobState.activeProcess = null;- // One-shot relaunch for a Metal command-buffer watchdog abort that landed- // INSIDE the Gemma prompt encoder (issue #4589). Decided here, ahead of+ // One-shot relaunch for a Metal command-buffer watchdog abort (issue #4589). Decided here, ahead of// the claim release and the temp-file cleanup below, because a relaunch// has to keep both: it renders the SAME job off the SAME staged inputs and// must not surrender the GPU lane between the two children.- if (await maybeRelaunchForPromptEncoding({ code, signal, childKilled: proc.killed, cancelEpochAtClose, stderr: stderrTail.join('\n') })) return;+ if (await maybeRelaunchForMetalWatchdog({ code, signal, childKilled: proc.killed, cancelEpochAtClose, stderr: stderrTail.join('\n') })) return;// The child has exited, so its accelerator allocation is gone. Release// before emitting the terminal completion event: an extend chain starts its// next child from that event and must be able to acquire the machine claim.@@ -640,19 +636,19 @@ export async function spawnAndWatchVideo({};// Relaunch this render once when the macOS Metal command-buffer watchdog- // aborted the child while the Gemma prompt encoder was running, with a lowered- // prompt-encode budget so each encoder command buffer finishes inside the- // watchdog window. Returns true when a replacement child owns the job (the+ // aborted the child, with smaller MLX command buffers so each buffer finishes+ // inside the watchdog window. Never shortens the prompt: the args are reused+ // verbatim, so history's prompt is exactly what the replacement encodes. Returns true when a replacement child owns the job (the// caller must then leave the failure path alone), false when the render should// fail normally.//// Everything the render is defined by — jobId, seed, output path, history// metadata, staged source images/audio — is closed over and reused verbatim, so- // the relaunch is the same render at a smaller prompt budget, not a new job.+ // the relaunch is the same render with smaller command buffers, not a new job.// Deliberately NOT gated on a runtime allowlist: the phase markers the decision// reads are emitted by scripts/generate_ltx2.py alone, so every other runtime is// excluded by construction rather than by a list that could drift.- const maybeRelaunchForPromptEncoding = async ({ code, signal, childKilled, cancelEpochAtClose, stderr }) => {+ const maybeRelaunchForMetalWatchdog = async ({ code, signal, childKilled, cancelEpochAtClose, stderr }) => {if (code === 0) return false;// A batch member already in history must never be rendered again over its// committed filename by a whole-batch prompt-encode retry.@@ -663,23 +659,14 @@ export async function spawnAndWatchVideo({// would be answered by relaunching the render the user just stopped.if (childKilled) return false;const stderrText = typeof stderr === 'string' ? stderr : '';- // At most ONE relaunch per job across both mitigations: a render that already- // spent either retry does not get a second GPU-grinding attempt.- const relaunchesUsed = promptEncodingRetriesUsed + metalBufferRetriesUsed;- const encodePlan = planPromptEncodingRetry({- signal,- stderr,- promptEncodePhase: job.promptEncodePhase,- retriesUsed: relaunchesUsed,- platform: process.platform,- });- const bufferPlan = encodePlan || relaunchesUsed > 0 ? null : planMetalBufferRetry({+ const bufferPlan = planMetalBufferRetry({signal,stderr: stderrText,- retriesUsed: 0,+ retriesUsed: metalBufferRetriesUsed,mlxRuntime: runtimeUsesMlx(model.runtime),});- if (!encodePlan && !bufferPlan) return false;+ if (!bufferPlan) return false;+ const inEncoder = isPromptEncodingMetalWatchdog({ signal, stderr: stderrText, promptEncodePhase: job.promptEncodePhase });// Runs from a child 'close' handler, outside the Express request lifecycle —// an uncaught throw here would crash the process, and a failed relaunch must// degrade into the normal failure report rather than strand the job.@@ -691,18 +678,12 @@ export async function spawnAndWatchVideo({let retryProc = null;let retryWired = false;try {- let retryArgs = args;- let retryEnv = childEnv;- let message;- if (encodePlan) {- promptEncodingRetriesUsed += 1;- retryArgs = [...args, '--gemma-max-length', String(encodePlan.gemmaMaxLength)];- message = `Metal watchdog aborted the prompt encoder — retrying once at ${encodePlan.gemmaMaxLength} Gemma tokens`;- } else {- metalBufferRetriesUsed += 1;- retryEnv = { ...childEnv, ...bufferPlan.env };- message = 'Metal watchdog stopped the GPU — retrying once with smaller command buffers';- }+ metalBufferRetriesUsed += 1;+ const retryArgs = args;+ const retryEnv = { ...childEnv, ...bufferPlan.env };+ const message = inEncoder+ ? 'Metal watchdog aborted the prompt encoder — retrying once with smaller command buffers (prompt unchanged)'+ : 'Metal watchdog stopped the GPU — retrying once with smaller command buffers';console.log(`♻️ ${message} [${jobId.slice(0, 8)}]`);broadcastSse(job, { type: 'status', message });videoGenEvents.emit('status', { generationId: jobId, message });