Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: keep LTX prompt tokens intact during Metal watchdog recovery (#12265)

[._.]/ Adam Eivy3bb413656ae3
diff --git a/scripts/generate_fastvideo.py b/scripts/generate_fastvideo.py
index fdf299bdd6..fb576b046d 100755
--- a/scripts/generate_fastvideo.py
+++ b/scripts/generate_fastvideo.py
@@ -60,11 +60,9 @@ _PERCENT_PATTERN = re.compile(r'\d+%')
# from the key order of the label table below.
# NOT "encode-prompt": that exact marker is generate_ltx2.py's Gemma
# prompt-encode BEGIN sentinel (PROMPT_ENCODE_BEGIN_MARKER in
-# generateVideoHelpers.js), and the server arms an ltx2-specific relaunch on it.
+# generateVideoHelpers.js), and the server names the prompt-encode phase from it.
# Emitting it here would open a phase this runner never closes, so a FastVideo
-# render killed by the Metal watchdog would be "retried" with ltx2's
-# --gemma-max-length flag, which this script's argparse rejects — replacing the
-# real watchdog error with an exit-2 after a wasted full model re-spawn.
+# watchdog abort would be reported as an encoder abort.
_PHASE_ORDER = ("load-pipeline", "conditioning", "sampling", "mux")
# Phase id -> the sentence the user reads, emitted once per transition. Ids are
diff --git a/scripts/generate_ltx2.py b/scripts/generate_ltx2.py
index 91fe1503e3..be8f836a91 100644
--- a/scripts/generate_ltx2.py
+++ b/scripts/generate_ltx2.py
@@ -569,24 +569,34 @@ PROMPT_ENCODE_BEGIN_MARKER = "STAGE:encode-prompt"
PROMPT_ENCODE_END_MARKER = "STAGE:encode-prompt-done"
-def configure_gemma_max_length(max_length: int | None) -> None:
- """Pin the Gemma prompt-encode sequence length for this run.
+def gemma_max_length() -> int:
+ """The sequence budget ltx-2-mlx will apply (``LTX2_GEMMA_MAX_LENGTH``, default 1024)."""
+ return int(os.environ.get("LTX2_GEMMA_MAX_LENGTH", "1024"))
- ltx-2-mlx reads ``LTX2_GEMMA_MAX_LENGTH`` at encode time
- (``ltx_pipelines_mlx.utils.blocks.PromptEncoder.encode``, and again in
- ``_base`` for Prompt Relay token ranges), defaulting to 1024. PortOS passes a
- lowered value on its one-shot relaunch after the macOS Metal command-buffer
- watchdog aborts the encoder, so this ASSIGNS rather than ``setdefault``s: an
- explicit flag has to beat whatever the parent environment already carried.
- No-op when the flag is absent, which leaves upstream's own default in force.
+def enforce_prompt_token_limit(encoder, prompt: str) -> None:
+ """Refuse a prompt the Gemma encoder would silently truncate.
+
+ ltx-2-mlx's ``GemmaEncoder.tokenize`` keeps only the LAST ``max_length``
+ tokens of an over-long input, so the start of an accepted prompt (usually the
+ subject) would vanish while history still shows the full text. This counts
+ with the encoder's own tokenizer on the exact string it will tokenize
+ (``text.strip()``) and exits before any sampling work. PortOS never lowers
+ the budget to recover from a Metal watchdog abort for the same reason.
+
+ Silently skipped when the loaded encoder exposes no tokenizer, so a pin with a
+ different layout keeps its upstream behavior.
"""
- if max_length is None:
+ tokenizer = getattr(getattr(encoder, "_text_encoder", None), "_tokenizer", None)
+ if not callable(getattr(tokenizer, "encode", None)):
return
- if max_length < 1:
- raise SystemExit(f"--gemma-max-length must be a positive integer; got {max_length}.")
- os.environ["LTX2_GEMMA_MAX_LENGTH"] = str(max_length)
- emit_status(f"Gemma prompt-encode capped at {max_length} tokens")
+ count = len(tokenizer.encode(prompt.strip()))
+ limit = gemma_max_length()
+ if count > limit:
+ raise SystemExit(
+ f"Prompt is {count} Gemma tokens but the text encoder accepts at most {limit}; "
+ "shorten the prompt (it would otherwise lose its beginning)."
+ )
def install_prompt_encode_markers() -> None:
@@ -618,10 +628,15 @@ def install_prompt_encode_markers() -> None:
return
@functools.wraps(original)
- def encode(self, *args, **kwargs):
+ def encode(self, prompt, *args, **kwargs):
print(PROMPT_ENCODE_BEGIN_MARKER, file=sys.stderr, flush=True)
try:
- return original(self, *args, **kwargs)
+ # The tokenizer only exists after load(); upstream's encode() calls it
+ # first and it is idempotent, so loading here costs nothing extra.
+ if callable(getattr(self, "load", None)):
+ self.load()
+ enforce_prompt_token_limit(self, prompt)
+ return original(self, prompt, *args, **kwargs)
finally:
print(PROMPT_ENCODE_END_MARKER, file=sys.stderr, flush=True)
@@ -762,12 +777,6 @@ def parse_args() -> argparse.Namespace:
"measured with. Reported as degraded (never fatal) when the model "
"pack does not carry it, so a slower render is never presented as "
"the profile's validated speed.")
- p.add_argument("--gemma-max-length", type=int, default=None,
- help="Gemma prompt-encode sequence length (LTX2_GEMMA_MAX_LENGTH; "
- "upstream default 1024). PortOS lowers this on its one-shot "
- "relaunch after the macOS Metal command-buffer watchdog aborts "
- "the prompt encoder. An explicit value always wins over the "
- "ambient environment.")
p.add_argument("--mlx-cache-limit-mb", default=None,
help="Cap the MLX allocator cache at this many MB for the whole run, "
"reasserted before every render. Omit to derive a conservative "
@@ -2136,7 +2145,7 @@ def physical_memory_bytes() -> int | None:
def resolve_mlx_cache_policy(physical_bytes, override_raw=None, env_raw=None) -> dict:
"""Effective cache policy as ``{"limitMb": int | None, "source": str}``.
- Precedence mirrors --gemma-max-length: an explicit flag beats the ambient
+ Precedence: an explicit flag beats the ambient
environment, which beats the ceiling derived from physical memory. A machine
whose memory could not be read gets ``limitMb: None`` and MLX's own default
stands — a blind floor would throttle a large box exactly as readily as it
@@ -2423,7 +2432,6 @@ def main() -> NoReturn:
# before any model load). Each run_* sets pipe._pending_loras from this.
args.user_lora_specs = parse_user_loras(args.user_loras)
configure_negative_prompt(args.negative_prompt)
- configure_gemma_max_length(args.gemma_max_length)
install_prompt_encode_markers()
configure_mlx_cache(args)
# Frame-one anchor contract (#5422). Runs before any pipeline is built so a
diff --git a/scripts/generate_ltx2.test.js b/scripts/generate_ltx2.test.js
index 3babcdb65f..1b27991b96 100644
--- a/scripts/generate_ltx2.test.js
+++ b/scripts/generate_ltx2.test.js
@@ -345,7 +345,7 @@ describe.skipIf(!pyBin)('generate_ltx2.py', () => {
// ── Gemma prompt-encode budget + boundary markers (#4589) ──────────────────
// The markers are what lets PortOS tell "the Metal watchdog killed the prompt
- // encoder" (worth one relaunch at a smaller budget) from "it killed the
+ // encoder" (worth one relaunch with smaller command buffers) from "it killed the
// denoise loop" (not worth anything), so their exact text and their emission
// on BOTH the success and the failure path are load-bearing.
const stubPromptEncoderModule = (body) => [
@@ -424,50 +424,61 @@ describe.skipIf(!pyBin)('generate_ltx2.py', () => {
expect(output.trim()).toBe('ok');
});
- // ltx-2-mlx reads LTX2_GEMMA_MAX_LENGTH at encode time and defaults it to
- // 1024, so the flag has to ASSIGN — a setdefault would be swallowed by the
- // ambient value the parent already exported and the relaunch would re-run at
- // the budget that just aborted.
- it.each([
- ['None', 'None', null, '1024'],
- ['a lowered budget', '512', null, '512'],
- ['a lowered budget over an ambient value', '512', '1024', '512'],
- ])('configure_gemma_max_length(%s) resolves LTX2_GEMMA_MAX_LENGTH to the expected value', (_label, flag, ambient, expected) => {
+ // The upstream tokenizer keeps only the LAST max_length tokens, so an over-long
+ // prompt silently loses its beginning. The stub reproduces that truncation so
+ // the assertion is on the token sequence the encoder actually receives, not on
+ // the unchanged Python string.
+ const truncatingEncoder = [
+ ' _text_encoder = type("E", (), {"_tokenizer": type("T", (), {"encode": lambda self, t: t.split()})()})()',
+ ' def load(self): pass',
+ ' def encode(self, prompt):',
+ ' import os',
+ ' limit = int(os.environ.get("LTX2_GEMMA_MAX_LENGTH", "1024"))',
+ ' tokens = prompt.strip().split()',
+ ' if len(tokens) > limit: tokens = tokens[-limit:]',
+ ' return (tokens[0], tokens[-1])',
+ ];
+
+ it.each([['at the 1024 limit', 1024], ['inside 513-1024', 800]])('hands the encoder the whole prompt %s (first and last token intact)', (_label, count) => {
const output = runPython(`${importRunner}\n${[
- 'import os',
- ambient === null
- ? 'os.environ.pop("LTX2_GEMMA_MAX_LENGTH", None)'
- : `os.environ["LTX2_GEMMA_MAX_LENGTH"] = ${JSON.stringify(ambient)}`,
- `runner.configure_gemma_max_length(${flag})`,
- // Read back through the same default upstream applies, so "unset" and
- // "explicitly 1024" are both expressed as the value the encoder would use.
- 'print(os.environ.get("LTX2_GEMMA_MAX_LENGTH", "1024"))',
+ ...stubPromptEncoderModule(truncatingEncoder),
+ 'import contextlib, os',
+ 'os.environ.pop("LTX2_GEMMA_MAX_LENGTH", None)',
+ 'runner.install_prompt_encode_markers()',
+ `prompt = " ".join("t%d" % i for i in range(${count}))`,
+ 'with contextlib.redirect_stderr(sys.stdout):',
+ ' print(PromptEncoder().encode(prompt))',
].join('\n')}`);
- expect(output.trim()).toBe(expected);
+ expect(output.trim().split(/\r?\n/).pop()).toBe(`('t0', 't${count - 1}')`);
});
- it.each([['0'], ['-1']])('rejects a non-positive --gemma-max-length (%s)', (value) => {
+ it('refuses an over-limit prompt with the measured count and limit before encoding', () => {
const output = runPython(`${importRunner}\n${[
- 'try:',
- ` runner.configure_gemma_max_length(${value})`,
- 'except SystemExit as exc:',
- ' print(str(exc))',
- 'else:',
- ' raise SystemExit("a non-positive gemma budget was accepted")',
+ ...stubPromptEncoderModule(truncatingEncoder.map((l) => (l.includes('return (tokens[0]') ? ' raise RuntimeError("encoder reached")' : l))),
+ 'import contextlib, os',
+ 'os.environ.pop("LTX2_GEMMA_MAX_LENGTH", None)',
+ 'runner.install_prompt_encode_markers()',
+ 'prompt = " ".join("t%d" % i for i in range(1025))',
+ 'with contextlib.redirect_stderr(sys.stdout):',
+ ' try:',
+ ' PromptEncoder().encode(prompt)',
+ ' except SystemExit as exc:',
+ ' print("REFUSED:" + str(exc))',
].join('\n')}`);
- expect(output).toMatch(/must be a positive integer/);
+ expect(output).toMatch(/REFUSED:Prompt is 1025 Gemma tokens but the text encoder accepts at most 1024/);
+ expect(output).not.toMatch(/encoder reached/);
});
- it.each([
- ['omitted', [], 'None'],
- ['given', ['--gemma-max-length', '512'], '512'],
- ])('parses --gemma-max-length when %s', (_label, extra, expected) => {
+ it('no longer exposes a flag that lowers the encoder sequence budget', () => {
const output = runPython(`${importRunner}\n${[
'import sys',
- `sys.argv = ["generate_ltx2.py", "--mode", "text", "--prompt", "p", "--output", "/tmp/o.mp4", "--model", "m"] + ${JSON.stringify(extra)}`,
- 'print(runner.parse_args().gemma_max_length)',
+ 'sys.argv = ["generate_ltx2.py", "--mode", "text", "--prompt", "p", "--output", "/tmp/o.mp4", "--model", "m", "--gemma-max-length", "512"]',
+ 'try:',
+ ' runner.parse_args()',
+ 'except SystemExit:',
+ ' print("REJECTED")',
].join('\n')}`);
- expect(output.trim()).toBe(expected);
+ expect(output).toMatch(/REJECTED/);
});
// The reference-mode promise (#4874). "inspire" is only expressible where the
diff --git a/server/services/videoGen/generateVideoHelpers.js b/server/services/videoGen/generateVideoHelpers.js
index 08c6e515a3..25707f645c 100644
--- a/server/services/videoGen/generateVideoHelpers.js
+++ b/server/services/videoGen/generateVideoHelpers.js
@@ -233,7 +233,7 @@ export function makeVideoGenLineHandler({ job, jobId, pythonNoiseRe }) {
// 'active' = encoding right now, 'done' = finished — so a later SIGABRT can
// be attributed to the encode without conflating "never reported" with
// "finished". Compared exactly: the '-done' marker is a prefix extension of
- // the begin marker. See planPromptEncodingRetry.
+ // the begin marker. Used only to name the phase in the recovery status.
if (line === PROMPT_ENCODE_BEGIN_MARKER) job.promptEncodePhase = 'active';
else if (line === PROMPT_ENCODE_END_MARKER) job.promptEncodePhase = 'done';
const parts = line.split(':');
@@ -450,19 +450,6 @@ export function describeSignalDeath(signal, { fingerprint = null } = {}) {
return fp ? `${cause} [runtime: ${fp}]` : cause;
}
-/**
- * Gemma prompt-encode sequence length. `LTX2_GEMMA_MAX_LENGTH` is read by
- * ltx-2-mlx at encode time (`utils/blocks.py#PromptEncoder.encode`) and defaults
- * to 1024 there; the retry halves it so the encoder's attention matrices are a
- * quarter the size and each Metal command buffer finishes well inside the
- * watchdog window. Halving (rather than a deeper cut) keeps a long prompt's
- * tail intact — the tokenizer truncates from the LEFT, so a smaller budget
- * silently drops the START of the prompt, which is where the subject usually
- * is.
- */
-export const DEFAULT_GEMMA_MAX_LENGTH = 1024;
-export const RETRY_GEMMA_MAX_LENGTH = 512;
-
// The macOS Metal command-buffer watchdog reports WHY it killed a buffer via a
// `kIOGPUCommandBufferCallbackError*` enum name in the abort text. Two of those
// mean "this buffer ran too long for the compositor to stay responsive", which
@@ -512,33 +499,6 @@ export function isPromptEncodingMetalWatchdog({ signal = null, stderr = '', prom
return METAL_WATCHDOG_RE.test(text);
}
-/**
- * Decide whether one render may be relaunched after a prompt-encode watchdog
- * abort, and with what Gemma budget. Pure — returns the plan, never acts.
- *
- * Returns `null` for "do not retry" and `{ gemmaMaxLength }` for "relaunch this
- * same job once with that budget". Exactly one retry is ever allowed: a second
- * watchdog abort at the reduced budget is a real failure the user must see, not
- * something to keep grinding the GPU over.
- *
- * @param {object} [opts]
- * @param {string|null} [opts.signal]
- * @param {string} [opts.stderr]
- * @param {'active'|'done'|null} [opts.promptEncodePhase]
- * @param {number} [opts.retriesUsed] - retries already spent on this job
- * @param {string} [opts.platform] - `process.platform` of the host
- * @returns {{ gemmaMaxLength: number }|null}
- */
-export function planPromptEncodingRetry({ signal = null, stderr = '', promptEncodePhase = null, retriesUsed = 0, platform = process.platform } = {}) {
- // The command-buffer watchdog is a macOS construct. A Windows/CUDA render
- // that somehow produced a matching string must not be relaunched with an
- // Apple-specific mitigation.
- if (platform !== 'darwin') return null;
- if (!Number.isInteger(retriesUsed) || retriesUsed > 0) return null;
- if (!isPromptEncodingMetalWatchdog({ signal, stderr, promptEncodePhase })) return null;
- return { gemmaMaxLength: RETRY_GEMMA_MAX_LENGTH };
-}
-
/**
* Per-command-buffer caps MLX reads at startup (`MLX_MAX_OPS_PER_BUFFER` /
* `MLX_MAX_MB_PER_BUFFER`). Smaller values only move command-buffer boundaries —
@@ -552,8 +512,10 @@ export const METAL_BUFFER_RETRY_ENV = Object.freeze({
/**
* Decide whether a render may be relaunched once with tiny MLX command buffers
- * after a Metal watchdog abort ANYWHERE in the render (the prompt-encode case has
- * its own plan, planPromptEncodingRetry). Pure — returns the plan, never acts.
+ * after a Metal watchdog abort ANYWHERE in the render, including inside the Gemma
+ * prompt encoder. The prompt token sequence is never shortened to recover: the
+ * encoder keeps only the LAST max_length tokens, so a lowered sequence budget
+ * would silently drop the start of an accepted prompt. Pure — returns the plan, never acts.
* Apple-silicon MLX runtimes only; OutOfMemory / InnocentVictim aborts never
* qualify (smaller buffers do not fix them). One retry per job.
*
diff --git a/server/services/videoGen/generateVideoHelpers.test.js b/server/services/videoGen/generateVideoHelpers.test.js
index beac85357c..a49e2628b1 100644
--- a/server/services/videoGen/generateVideoHelpers.test.js
+++ b/server/services/videoGen/generateVideoHelpers.test.js
@@ -4,7 +4,7 @@ import { tmpdir } from 'os';
import { join } from 'path';
import { EventEmitter } from 'events';
import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../../lib/mockPathsDataRoot.js';
-import { makeVideoGenLineHandler, isWatchdogSuccess, finalizeGeneratedVideo, parseByteProgress, formatBytes, formatDownloadMessage, describeSignalDeath, formatRuntimeFingerprint, describeRenderConditioning, isPromptEncodingMetalWatchdog, planPromptEncodingRetry, DEFAULT_GEMMA_MAX_LENGTH, RETRY_GEMMA_MAX_LENGTH, bufferChildExit, isNativeTeardownAbort, verifyPostCompletionOutputs, POST_COMPLETION_ABORT_RUNTIME, RENDER_INPUTS_VERSION, emitCloudRenderStatus, CLOUD_RENDER_PHASE } from './generateVideoHelpers.js';
+import { makeVideoGenLineHandler, isWatchdogSuccess, finalizeGeneratedVideo, parseByteProgress, formatBytes, formatDownloadMessage, describeSignalDeath, formatRuntimeFingerprint, describeRenderConditioning, isPromptEncodingMetalWatchdog, bufferChildExit, isNativeTeardownAbort, verifyPostCompletionOutputs, POST_COMPLETION_ABORT_RUNTIME, RENDER_INPUTS_VERSION, emitCloudRenderStatus, CLOUD_RENDER_PHASE } from './generateVideoHelpers.js';
describe('parseByteProgress', () => {
it('parses single byte value (e.g., "2.5G")', () => {
@@ -133,7 +133,7 @@ describe('makeVideoGenLineHandler', () => {
const eventsOfType = (t) => emitted.filter((e) => e.type === t).map((e) => e.payload);
// Prompt-encode phase (#4589) — the signal that decides whether a later
- // SIGABRT is worth one relaunch at a smaller Gemma budget.
+ // SIGABRT is worth one relaunch with smaller command buffers.
it('tracks the Gemma prompt-encode phase as three distinct states', () => {
// Never reported: every runtime other than the LTX-2 MLX helper stays here,
// and an unstamped field must not read as "encoding right now".
@@ -594,41 +594,6 @@ describe('isPromptEncodingMetalWatchdog', () => {
});
});
-describe('planPromptEncodingRetry', () => {
- const WATCHDOG_ABORT = '[METAL] Command buffer execution failed: (00000004:kIOGPUCommandBufferCallbackErrorImpactingInteractivity)';
- const qualifying = {
- signal: 'SIGABRT',
- stderr: WATCHDOG_ABORT,
- promptEncodePhase: 'active',
- retriesUsed: 0,
- platform: 'darwin',
- };
-
- it('plans one relaunch at the reduced Gemma budget', () => {
- expect(planPromptEncodingRetry(qualifying)).toEqual({ gemmaMaxLength: RETRY_GEMMA_MAX_LENGTH });
- // The reduced budget is a real cut from what upstream would otherwise use —
- // a plan that handed back the default would relaunch into the same abort.
- expect(RETRY_GEMMA_MAX_LENGTH).toBeLessThan(DEFAULT_GEMMA_MAX_LENGTH);
- });
-
- it('allows exactly one — a second abort at the reduced budget is a real failure', () => {
- expect(planPromptEncodingRetry({ ...qualifying, retriesUsed: 1 })).toBeNull();
- expect(planPromptEncodingRetry({ ...qualifying, retriesUsed: 2 })).toBeNull();
- });
-
- // The command-buffer watchdog is a macOS construct; a Windows/CUDA render that
- // somehow produced a matching string must not be relaunched with an
- // Apple-specific mitigation.
- it.each([['win32'], ['linux']])('never fires on %s', (platform) => {
- expect(planPromptEncodingRetry({ ...qualifying, platform })).toBeNull();
- });
-
- it('passes the classifier verdict straight through', () => {
- expect(planPromptEncodingRetry({ ...qualifying, promptEncodePhase: 'done' })).toBeNull();
- expect(planPromptEncodingRetry({ ...qualifying, signal: 'SIGKILL' })).toBeNull();
- });
-});
-
// The window between "spawn resolved" and "real listeners attached" is an await
// on real file I/O (the accelerator claim handoff). A child that dies in it
// emits unsubscribed — and for 'error' that is not merely lost but fatal, since
diff --git a/server/services/videoGen/local.test.js b/server/services/videoGen/local.test.js
index 75ef580947..278dbf2168 100644
--- a/server/services/videoGen/local.test.js
+++ b/server/services/videoGen/local.test.js
@@ -5297,7 +5297,7 @@ describe('generateVideo — Gemma prompt-encode watchdog relaunch', () => {
it.each([
['the classic timeout signature', TIMEOUT_ABORT],
['the impacting-interactivity signature newer macOS reports', INTERACTIVITY_ABORT],
- ])('relaunches once at a reduced Gemma budget on %s', async (_label, abort) => {
+ ])('relaunches once with smaller command buffers, prompt untouched, on %s', async (_label, abort) => {
const first = makeDrivenProc(101);
const second = makeDrivenProc(102);
const spawnMock = await startRender(`pe-${abort.length}`, [first, second]);
@@ -5309,11 +5309,13 @@ describe('generateVideo — Gemma prompt-encode watchdog relaunch', () => {
const calls = ltx2Calls(spawnMock);
expect(calls).toHaveLength(2);
const [firstArgs, retryArgs] = calls.map(([, args]) => args);
- // The original render never carries the flag — the reduced budget exists
- // only as the mitigation, not as a new default.
- expect(firstArgs).not.toContain('--gemma-max-length');
- expect(flagValue(retryArgs, '--gemma-max-length')).toBe('512');
- // Same render, smaller prompt budget: seed and output must survive verbatim
+ // Recovery never shortens the encoder sequence (the encoder would drop the
+ // START of the prompt): the args are byte-identical and only the env differs.
+ expect(retryArgs).toEqual(firstArgs);
+ expect(retryArgs).not.toContain('--gemma-max-length');
+ expect(calls[0][2].env.MLX_MAX_OPS_PER_BUFFER).toBeUndefined();
+ expect(calls[1][2].env).toMatchObject({ MLX_MAX_OPS_PER_BUFFER: '4', MLX_MAX_MB_PER_BUFFER: '16' });
+ // Same render, smaller command buffers: seed and output must survive verbatim
// or the relaunch silently produces a different clip than the user asked for.
expect(flagValue(retryArgs, '--seed')).toBe(flagValue(firstArgs, '--seed'));
expect(flagValue(retryArgs, '--seed')).toBe('987654');
@@ -5413,7 +5415,7 @@ describe('generateVideo — Gemma prompt-encode watchdog relaunch', () => {
expect(failures).toHaveLength(1);
});
- it('relaunches at most once — a second abort at the reduced budget fails the job', async () => {
+ it('relaunches at most once — a second abort at the smaller setting fails the job', async () => {
const first = makeDrivenProc(105);
const second = makeDrivenProc(106);
const spawnMock = await startRender('pe-twice', [first, second]);
diff --git a/server/services/videoGen/spawnWatch.js b/server/services/videoGen/spawnWatch.js
index e60a611336..b0e9908f46 100644
--- a/server/services/videoGen/spawnWatch.js
+++ b/server/services/videoGen/spawnWatch.js
@@ -26,7 +26,7 @@ import {
isNativeTeardownAbort,
verifyPostCompletionOutputs,
describeSignalDeath,
- planPromptEncodingRetry,
+ isPromptEncodingMetalWatchdog,
planMetalBufferRetry,
bufferChildExit,
} from './generateVideoHelpers.js';
@@ -101,15 +101,12 @@ export async function spawnAndWatchVideo({
// Reads (and detaches) whatever terminal event the first child emitted before
// its real listeners were attached. Set the instant the spawn resolves.
let takeEarlyExit = null;
- // Prompt-encode relaunches already spent on this job. Exactly one is allowed:
- // a second watchdog abort at the reduced budget is a real failure the user has
- // to see, not something to keep grinding the GPU over.
- let promptEncodingRetriesUsed = 0;
- // Same single-shot rule for the small-command-buffer relaunch (any-phase Metal
- // watchdog abort on an MLX runtime).
+ // The one allowed relaunch per job: a second watchdog abort at the smaller
+ // command-buffer setting is a real failure the user has to see, not something
+ // to keep grinding the GPU over. Covers every phase, prompt encode included.
let metalBufferRetriesUsed = 0;
// Hoisted out of the try so a relaunch can respawn with the SAME child env
- // plus a lowered Gemma budget, instead of rebuilding it from scratch.
+ // plus smaller MLX command buffers, instead of rebuilding it from scratch.
let childEnv;
// Runners publish one atomically-replaced `preview.png`. Keep the reader
@@ -192,8 +189,8 @@ export async function spawnAndWatchVideo({
// without minting a new one. Everything job-level (jobId, args, seed, meta,
// history entry, heavy claim, staged temp files) is closed over and survives
// the relaunch. The only thing that relaunches a render is a Metal
- // command-buffer watchdog abort inside the Gemma prompt encoder; see
- // maybeRelaunchForPromptEncoding below.
+ // command-buffer watchdog abort on an MLX runtime; see
+ // maybeRelaunchForMetalWatchdog below.
const wireRenderChild = (proc) => {
// The prompt-encode phase belongs to ONE child, but `job` outlives the
// relaunch — reset it explicitly so a phase left open by the child that just
@@ -427,12 +424,11 @@ export async function spawnAndWatchVideo({
// guard survives anyone later putting an await between the two.
const cancelEpochAtClose = videoJobState.cancelEpoch;
videoJobState.activeProcess = null;
- // One-shot relaunch for a Metal command-buffer watchdog abort that landed
- // INSIDE the Gemma prompt encoder (issue #4589). Decided here, ahead of
+ // One-shot relaunch for a Metal command-buffer watchdog abort (issue #4589). Decided here, ahead of
// the claim release and the temp-file cleanup below, because a relaunch
// has to keep both: it renders the SAME job off the SAME staged inputs and
// must not surrender the GPU lane between the two children.
- if (await maybeRelaunchForPromptEncoding({ code, signal, childKilled: proc.killed, cancelEpochAtClose, stderr: stderrTail.join('\n') })) return;
+ if (await maybeRelaunchForMetalWatchdog({ code, signal, childKilled: proc.killed, cancelEpochAtClose, stderr: stderrTail.join('\n') })) return;
// The child has exited, so its accelerator allocation is gone. Release
// before emitting the terminal completion event: an extend chain starts its
// next child from that event and must be able to acquire the machine claim.
@@ -640,19 +636,19 @@ export async function spawnAndWatchVideo({
};
// Relaunch this render once when the macOS Metal command-buffer watchdog
- // aborted the child while the Gemma prompt encoder was running, with a lowered
- // prompt-encode budget so each encoder command buffer finishes inside the
- // watchdog window. Returns true when a replacement child owns the job (the
+ // aborted the child, with smaller MLX command buffers so each buffer finishes
+ // inside the watchdog window. Never shortens the prompt: the args are reused
+ // verbatim, so history's prompt is exactly what the replacement encodes. Returns true when a replacement child owns the job (the
// caller must then leave the failure path alone), false when the render should
// fail normally.
//
// Everything the render is defined by — jobId, seed, output path, history
// metadata, staged source images/audio — is closed over and reused verbatim, so
- // the relaunch is the same render at a smaller prompt budget, not a new job.
+ // the relaunch is the same render with smaller command buffers, not a new job.
// Deliberately NOT gated on a runtime allowlist: the phase markers the decision
// reads are emitted by scripts/generate_ltx2.py alone, so every other runtime is
// excluded by construction rather than by a list that could drift.
- const maybeRelaunchForPromptEncoding = async ({ code, signal, childKilled, cancelEpochAtClose, stderr }) => {
+ const maybeRelaunchForMetalWatchdog = async ({ code, signal, childKilled, cancelEpochAtClose, stderr }) => {
if (code === 0) return false;
// A batch member already in history must never be rendered again over its
// committed filename by a whole-batch prompt-encode retry.
@@ -663,23 +659,14 @@ export async function spawnAndWatchVideo({
// would be answered by relaunching the render the user just stopped.
if (childKilled) return false;
const stderrText = typeof stderr === 'string' ? stderr : '';
- // At most ONE relaunch per job across both mitigations: a render that already
- // spent either retry does not get a second GPU-grinding attempt.
- const relaunchesUsed = promptEncodingRetriesUsed + metalBufferRetriesUsed;
- const encodePlan = planPromptEncodingRetry({
- signal,
- stderr,
- promptEncodePhase: job.promptEncodePhase,
- retriesUsed: relaunchesUsed,
- platform: process.platform,
- });
- const bufferPlan = encodePlan || relaunchesUsed > 0 ? null : planMetalBufferRetry({
+ const bufferPlan = planMetalBufferRetry({
signal,
stderr: stderrText,
- retriesUsed: 0,
+ retriesUsed: metalBufferRetriesUsed,
mlxRuntime: runtimeUsesMlx(model.runtime),
});
- if (!encodePlan && !bufferPlan) return false;
+ if (!bufferPlan) return false;
+ const inEncoder = isPromptEncodingMetalWatchdog({ signal, stderr: stderrText, promptEncodePhase: job.promptEncodePhase });
// Runs from a child 'close' handler, outside the Express request lifecycle —
// an uncaught throw here would crash the process, and a failed relaunch must
// degrade into the normal failure report rather than strand the job.
@@ -691,18 +678,12 @@ export async function spawnAndWatchVideo({
let retryProc = null;
let retryWired = false;
try {
- let retryArgs = args;
- let retryEnv = childEnv;
- let message;
- if (encodePlan) {
- promptEncodingRetriesUsed += 1;
- retryArgs = [...args, '--gemma-max-length', String(encodePlan.gemmaMaxLength)];
- message = `Metal watchdog aborted the prompt encoder — retrying once at ${encodePlan.gemmaMaxLength} Gemma tokens`;
- } else {
- metalBufferRetriesUsed += 1;
- retryEnv = { ...childEnv, ...bufferPlan.env };
- message = 'Metal watchdog stopped the GPU — retrying once with smaller command buffers';
- }
+ metalBufferRetriesUsed += 1;
+ const retryArgs = args;
+ const retryEnv = { ...childEnv, ...bufferPlan.env };
+ const message = inEncoder
+ ? 'Metal watchdog aborted the prompt encoder — retrying once with smaller command buffers (prompt unchanged)'
+ : 'Metal watchdog stopped the GPU — retrying once with smaller command buffers';
console.log(`♻️ ${message} [${jobId.slice(0, 8)}]`);
broadcastSse(job, { type: 'status', message });
videoGenEvents.emit('status', { generationId: jobId, message });