Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

64.1 KB
Raw Blame
import { execFileSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';
import { describe, expect, it } from 'vitest';
import { resolveTestPython } from '../server/lib/testHelper.js';

const script = join(dirname(fileURLToPath(import.meta.url)), 'generate_minimax_h3.py');

// Probe for an interpreter that actually RUNS rather than assuming a name.
// On Windows a machine with no Store-installed Python still has `python` on
// PATH as an alias STUB: it exists, exits non-zero, and prints "Python was
// not found". A name-only choice passes there and then every case dies with
// an opaque "Command failed" — and PortOS provisions its own interpreters
// (~/.portos venvs), so a box can be set up for image/video gen while the
// bare name is still a stub. Null when there is genuinely none, so the suite
// skips instead of failing.
const pyBin = resolveTestPython();
const runPython = (source) => execFileSync(pyBin, ['-c', source, script], {
  encoding: 'utf8',
});

// The final-norm synthesis is the one helper here that actually needs mlx (it
// writes a real safetensors). CI's bare python3 is Linux and has no mlx wheel,
// so probe once and skip just that case rather than making the whole file
// macOS-only — every other helper is import-free by design.
const hasMlx = (() => {
  try {
    execFileSync(pyBin, ['-c', 'import mlx.core'], { stdio: 'ignore' });
    return true;
  } catch {
    return false;
  }
})();

// The served-embedding cases below round-trip real tensors AND describe them
// through numpy, so an interpreter carrying mlx but not numpy would fail them
// rather than skip. Probed together for that reason — hasMlx alone gates the
// final-norm case above, which genuinely needs only mlx.
const hasEmbeddingStack = (() => {
  try {
    execFileSync(pyBin, ['-c', 'import mlx.core, numpy'], { stdio: 'ignore' });
    return true;
  } catch {
    return false;
  }
})();

// Route the runner's warnings into stdout so a case can assert on them, with
// the same reconfigure the runnable script tests use: Windows CI's python
// writes cp1252, and PortOS's ⚠️-prefixed log lines are unencodable there — the
// print would abort the case before its assertion rather than fail it.
const captureWarnings = [
  'import sys',
  'if hasattr(sys.stdout, "reconfigure"):',
  '    sys.stdout.reconfigure(encoding="utf-8", errors="replace")',
  'sys.stderr = sys.stdout',
].join('\n');

// A stand-in for the decoded PIL keyframe `encode` receives: the cache reads
// only the three attributes the digest is taken from, so the suite can exercise
// keying without Pillow (which CI's bare python3 does not have either).
const stubImage = [
  'class StubImage:',
  '    def __init__(self, mode, size, data):',
  '        self.mode, self.size, self._data = mode, size, data',
  '    def tobytes(self): return self._data',
].join('\n');

// The pin facts these corrections guard against moved to `_minimax_h3_mlx_pins.py`
// so the install-time probe and these render-time guards read one copy. Reach the
// module the RUNNER imported rather than loading a second one by spec: a private
// copy would let a rebinding here pass while the code under test still read the
// shipped digest.
const importPins = 'pins = sys.modules["_minimax_h3_mlx_pins"]';

const importRunner = [
  'import importlib.util, sys',
  'from pathlib import Path',
  'script = Path(sys.argv[1])',
  'spec = importlib.util.spec_from_file_location("generate_minimax_h3", script)',
  'runner = importlib.util.module_from_spec(spec)',
  'spec.loader.exec_module(runner)',
].join('\n');

// validate_args reads the namespace argparse produces, so every field it touches
// has to be present here — a hand-built literal per test drifts the moment a flag
// is added (and fails as an AttributeError rather than an assertion). One builder
// carries the parser's own defaults; each test overrides only what it exercises.
const VALIDATE_ARGS_DEFAULTS = {
  fps: 24,
  width: 512,
  height: 320,
  num_frames: 124,
  steps: 9,
  image: [],
  anchor: [],
  lora: [],
  lora_scale: [],
  text_encoder_id: null,
  text_encoder_file: [],
  text_encoder_shim_root: null,
  text_encoder_key_prefix: [],
  text_encoder_final_norm_key: null,
  draft_decoder_id: null,
  draft_decoder_file: [],
  draft_decoder_shim_root: null,
  // Not read by validate_args — prompt_embedding_identity (#5443) reads these
  // off the same Namespace, and one builder keeps both from drifting.
  runtime_revision: "runtime-revision",
  checkpoint_repo: "example/checkpoint",
  checkpoint_revision: "checkpoint-revision",
};
const argsExpr = (overrides = {}) => {
  const fields = Object.entries({ ...VALIDATE_ARGS_DEFAULTS, ...overrides })
    .map(([key, value]) => `${key}=${value === null ? 'None' : JSON.stringify(value)}`);
  return `args = SimpleNamespace(${fields.join(', ')})`;
};

// Stand-in for the pinned runtime: the installers below reach for
// `minimax_h3_mlx.text_encoder.MiniMaxH3TextEncoder`, so every case that
// exercises one has to put a class there first. `preamble` runs before the
// class body, for recorders the stub's methods close over.
const stubPin = (classBody, preamble = []) => [
  'import sys, types',
  ...preamble,
  'class MiniMaxH3TextEncoder:',
  ...classBody,
  'module = types.ModuleType("minimax_h3_mlx.text_encoder")',
  'module.MiniMaxH3TextEncoder = MiniMaxH3TextEncoder',
  'sys.modules["minimax_h3_mlx"] = types.ModuleType("minimax_h3_mlx")',
  'sys.modules["minimax_h3_mlx.text_encoder"] = module',
].join('\n');

// The merge correction guards itself with `inspect.getsource`, which needs a
// real file — so that stand-in is written out and imported rather than declared
// inline. Each `encodeBody` entry is a PYTHON expression evaluated against the
// already-imported runner, so the line under test comes from the runner's own
// constant instead of a copy that could drift from it. The result is carried as
// a COMMENT: the stub only has to look like the pin to `getsource`, not run.
const filePin = (encodeBody) => [
  'import atexit, importlib.util, shutil, sys, tempfile, types',
  'temp = tempfile.mkdtemp()',
  'atexit.register(shutil.rmtree, temp, True)',
  'pin = Path(temp) / "pinned_text_encoder.py"',
  'pin.write_text("\\n".join([',
  '    "class MiniMaxH3TextEncoder:",',
  '    "    def encode(self, prompt, images=None):",',
  ...encodeBody.map((expr) => `    "        # " + ${expr},`),
  '    "        return (\'pinned\', prompt, images)",',
  ']))',
  'spec = importlib.util.spec_from_file_location("minimax_h3_mlx.text_encoder", pin)',
  'module = importlib.util.module_from_spec(spec)',
  'sys.modules["minimax_h3_mlx"] = types.ModuleType("minimax_h3_mlx")',
  'sys.modules["minimax_h3_mlx.text_encoder"] = module',
  'spec.loader.exec_module(module)',
  'MiniMaxH3TextEncoder = module.MiniMaxH3TextEncoder',
].join('\n');

describe.skipIf(!pyBin)('generate_minimax_h3.py', () => {
  it('defaults the MLX runner to nine sigma points for eight forwards', () => {
    const output = runPython(`${importRunner}\n${[
      'import sys',
      'sys.argv = ["generate_minimax_h3.py", "--model-repo", "example/model", "--model-revision", "revision",',
      '    "--runtime-dir", "/tmp/runtime", "--runtime-revision", "revision",',
      '    "--checkpoint-repo", "example/checkpoint", "--checkpoint-revision", "revision",',
      '    "--prompt", "a test prompt", "--width", "1344", "--height", "768",',
      '    "--num-frames", "124", "--output", "test.mp4"]',
      'print(runner.parse_args().steps)',
    ].join('\n')}`);
    expect(output.trim()).toBe('9');
  });

  it('parses the optional bounded preview directory', () => {
    const output = runPython(`${importRunner}\n${[
      'import sys',
      'sys.argv = ["generate_minimax_h3.py", "--model-repo", "example/model", "--model-revision", "revision",',
      '    "--runtime-dir", "/tmp/runtime", "--runtime-revision", "revision",',
      '    "--checkpoint-repo", "example/checkpoint", "--checkpoint-revision", "revision",',
      '    "--prompt", "a test prompt", "--width", "1344", "--height", "768", "--num-frames", "124",',
      '    "--output", "test.mp4", "--preview-dir", "/tmp/previews"]',
      'print(runner.parse_args().preview_dir)',
    ].join('\n')}`);
    expect(output.trim()).toBe('/tmp/previews');
  });

  it('decodes only a bounded midpoint window of generated rows for the stepwise preview', () => {
    const output = runPython(`${importRunner}\n${[
      'import json, sys, types',
      'packing = types.ModuleType("minimax_h3_mlx.packing")',
      'packing.align_num_frames = lambda value: value',
      'packing.video_latent_num_frames = lambda value: 37',
      'sys.modules["minimax_h3_mlx"] = types.ModuleType("minimax_h3_mlx")',
      'sys.modules["minimax_h3_mlx.packing"] = packing',
      'class Config:',
      '    spatial_compression_ratio = 2',
      '    token_drop = 3',
      'class DitConfig:',
      '    patch_size = (1, 2, 2)',
      'class VideoVae:',
      '    config = Config()',
      '    tokens_chunk_size = 5',
      'class Dit:',
      '    config = DitConfig()',
      'class Rows:',
      '    def __init__(self, shape, start=0): self.shape, self.start = shape, start',
      '    def __getitem__(self, key):',
      '        if key == 0: return Rows((152, 4))',
      '        if isinstance(key, slice):',
      '            start, stop, _ = key.indices(self.shape[0])',
      '            return Rows((stop - start, 4), self.start + start)',
      '        raise AssertionError(f"unexpected row key: {key!r}")',
      'class Frame:',
      '    shape = (4, 4, 3)',
      'class Frames:',
      '    def __len__(self): return 2',
      '    def __getitem__(self, key): return Frame()',
      'class Pipe:',
      '    video_vae = VideoVae()',
      '    dit = Dit()',
      '    def _decode_video(self, rows, *shape):',
      '        print(json.dumps({"rows": list(rows.shape), "start": rows.start, "shape": list(shape)}))',
      '        return Frames()',
      'seen = []',
      'runner.write_stepwise_preview = lambda directory, frame: seen.append((directory, list(frame.shape))) or True',
      'preview = runner._H3StepwisePreview(Pipe(), "/tmp/previews", 124, 8, 8)',
      'rows = Rows((1, 152, 4))',
      'proxy = runner._PreviewingDiT(lambda *args: "ok", preview)',
      'proxy(rows)',
      'preview.publish(1, 2)',
      'print(json.dumps({"seen": seen, "saved": preview.saved}))',
    ].join('\n')}`);
    const lines = output.trim().split('\n').map((line) => JSON.parse(line));
    expect(lines[0]).toMatchObject({ rows: [28, 4], start: 64, shape: [7, 4, 4] });
    expect(lines[1]).toEqual({ seen: [['/tmp/previews', [4, 4, 3]]], saved: 1 });
  });

  // Only this runner shells out to ffmpeg (the CUDA sibling muxes in-process
  // via PyAV), so the preflight lives here rather than in the shared module.
  // Tens of GB of weights load before the mux; discovering a missing ffmpeg
  // after that wastes the whole render.
  it('fails the ffmpeg preflight before any model load can begin', () => {
    const output = runPython(`${importRunner}\n${[
      'runner.shutil.which = lambda name: None',
      'try:',
      '    runner.require_ffmpeg()',
      'except RuntimeError as exc:',
      '    print(exc)',
    ].join('\n')}`);
    expect(output).toContain('ffmpeg is required');
  });

  it('preflights transformer config, quant config, index, and every indexed shard', () => {
    const output = runPython(`${importRunner}\n${[
      'import json, tempfile',
      'with tempfile.TemporaryDirectory() as temp:',
      '    root = Path(temp)',
      '    (root / "model.safetensors.index.json").write_text(json.dumps({"weight_map": {"a": "model-1.safetensors", "b": "model-2.safetensors"}}))',
      '    calls = []',
      '    def fake_resolve(repo, revision, files):',
      '        calls.append(files)',
      '        return root',
      '    runner.resolve_cached_snapshot = fake_resolve',
      '    resolved = runner.resolve_transformer_snapshot("example/model", "revision")',
      '    print(json.dumps({"calls": calls, "resolved": str(resolved)}))',
    ].join('\n')}`);
    const result = JSON.parse(output);
    expect(result.calls).toEqual([
      ['config.json', 'quant_config.json', 'model.safetensors.index.json'],
      ['model-1.safetensors', 'model-2.safetensors'],
    ]);
    expect(result.resolved).toBeTruthy();
  });

  // The duration window is the one output fact that is per-runner: the MLX port
  // takes H3's full documented 4-15s range, where the diffusers CUDA runner is
  // narrower. Everything else on the grid (fps, the 32px canvas, the 17n+5 step,
  // anchoring) is shared, and asserted once in _minimax_h3_common.test.js.
  it.each([107, 124, 362])('accepts the %i-frame grid point inside the MLX window', (frames) => {
    const output = runPython(`${importRunner}\n${[
      'from types import SimpleNamespace',
      argsExpr({ width: 1536, height: 672, num_frames: frames }),
      'runner.validate_args(args)',
      'print("ok")',
    ].join('\n')}`);
    expect(output.trim()).toBe('ok');
  });

  it('rejects a frame count outside the MLX window', () => {
    const output = runPython(`${importRunner}\n${[
      'from types import SimpleNamespace',
      argsExpr({ width: 1536, height: 672, num_frames: 90 }),
      'try:',
      '    runner.validate_args(args)',
      'except SystemExit as exc:',
      '    print(str(exc))',
      'else:',
      '    raise SystemExit("an out-of-window frame count was accepted")',
    ].join('\n')}`);
    expect(output).toMatch(/approximately 4-15 seconds \(107-362 aligned frames\)/i);
  });

  // An unpaired --lora/--lora-scale would apply the wrong strength (or none) to
  // an adapter, and a missing file would only surface after the ~83 GB load.
  it.each([
    [['a.safetensors'], [], /one --lora-scale per --lora/i],
    [['a.safetensors', 'b.safetensors'], [0.8], /one --lora-scale per --lora/i],
    [['/nonexistent/a.safetensors'], [0.8], /LoRA file is missing/i],
  ])('rejects mismatched or missing LoRA arguments (%j / %j)', (loras, scales, pattern) => {
    const output = runPython(`${importRunner}\n${[
      'from types import SimpleNamespace',
      argsExpr({ lora: loras, lora_scale: scales }),
      'try:',
      '    runner.validate_args(args)',
      'except SystemExit as exc:',
      '    print(str(exc))',
      'else:',
      '    raise SystemExit("bad LoRA args were accepted")',
    ].join('\n')}`);
    expect(output).toMatch(pattern);
  });

  // A partial flag set would silently fall back to the stock conditioner and
  // hand back a render the user has no way to tell apart from the one they asked
  // for, so the boundary rejects it instead of choosing for them.
  it.each([
    [{ text_encoder_id: 'heretic-bf16' }, /must be given together/i],
    [{ text_encoder_file: ['/tmp/e.safetensors'] }, /must be given together/i],
    [{ text_encoder_id: 'heretic-bf16', text_encoder_file: ['/tmp/e.safetensors'] }, /must be given together/i],
    // A traversal-shaped id would put the shim tree outside the root PortOS
    // chose (and `rmtree` it from there).
    [
      {
        text_encoder_id: '../escape',
        text_encoder_file: ['/tmp/e.safetensors'],
        text_encoder_shim_root: '/tmp/shims',
      },
      /bare directory-safe name/i,
    ],
    // Loader mechanics with nothing to apply them to: accepting these would
    // rewrite the STOCK conditioner's keys and fail deep inside the load.
    [{ text_encoder_key_prefix: ['model.=model.language_model.'] }, /need --text-encoder-file/i],
    [{ text_encoder_final_norm_key: 'model.norm.weight' }, /need --text-encoder-file/i],
  ])('rejects an incoherent text-encoder argument set (%j)', (overrides, pattern) => {
    const output = runPython(`${importRunner}\n${[
      'from types import SimpleNamespace',
      argsExpr(overrides),
      'try:',
      '    runner.validate_args(args)',
      'except SystemExit as exc:',
      '    print(str(exc))',
      'else:',
      '    raise SystemExit("incoherent text-encoder args were accepted")',
    ].join('\n')}`);
    expect(output).toMatch(pattern);
  });

  // Draft decode (#5423). Same all-or-nothing rule as the conditioner above, and
  // for the same reason: a partial set would decode on the FULL decoder while
  // PortOS's history record claimed a draft one.
  it.each([
    [{ draft_decoder_id: 'draft' }, /must be given together/i],
    [{ draft_decoder_file: ['/tmp/d.safetensors'] }, /must be given together/i],
    [{ draft_decoder_shim_root: '/tmp/decoder-shims' }, /must be given together/i],
    // The id names a directory under the shim root, so a traversing value is a
    // real escape rather than a cosmetic problem.
    [
      {
        draft_decoder_id: '../escape',
        draft_decoder_file: ['/tmp/d.safetensors'],
        draft_decoder_shim_root: '/tmp/decoder-shims',
      },
      /bare directory-safe name/i,
    ],
  ])('rejects an incoherent draft-decoder argument set (%j)', (overrides, pattern) => {
    const output = runPython(`${importRunner}\n${[
      'from types import SimpleNamespace',
      argsExpr(overrides),
      'try:',
      '    runner.validate_args(args)',
      'except SystemExit as exc:',
      '    print(str(exc))',
      'else:',
      '    raise SystemExit("incoherent draft-decoder args were accepted")',
    ].join('\n')}`);
    expect(output).toMatch(pattern);
  });

  it('accepts a complete draft-decoder argument set', () => {
    const output = runPython(`${importRunner}\n${[
      'from types import SimpleNamespace',
      argsExpr({
        draft_decoder_id: 'draft',
        draft_decoder_file: ['/tmp/d.safetensors'],
        draft_decoder_shim_root: '/tmp/decoder-shims',
      }),
      'runner.validate_args(args)',
      'print("ok")',
    ].join('\n')}`);
    expect(output.trim()).toBe('ok');
  });

  it('accepts a complete text-encoder argument set', () => {
    const output = runPython(`${importRunner}\n${[
      'from types import SimpleNamespace',
      argsExpr({
        text_encoder_id: 'heretic-bf16',
        text_encoder_file: ['/tmp/e.safetensors'],
        text_encoder_shim_root: '/tmp/shims',
        text_encoder_key_prefix: ['model.=model.language_model.', 'visual.=model.visual.'],
        text_encoder_final_norm_key: 'model.norm.weight',
      }),
      'runner.validate_args(args)',
      'print("ok")',
    ].join('\n')}`);
    expect(output.trim()).toBe('ok');
  });

  // Sorted longest-source-first so a broader rule can never shadow a narrower
  // one that also matches, whatever order PortOS declares them in.
  it('parses key-prefix rules longest-source-first', () => {
    const output = runPython(`${importRunner}\n${[
      'import json',
      'print(json.dumps(runner.parse_key_prefixes(["model.=A.", "model.layers.=B.", "visual.=C."])))',
    ].join('\n')}`);
    expect(JSON.parse(output)).toEqual([
      ['model.layers.', 'B.'],
      ['visual.', 'C.'],
      ['model.', 'A.'],
    ]);
  });

  it.each(['model.', 'model.=', '=model.'])('rejects a malformed key-prefix rule (%j)', (rule) => {
    const output = runPython(`${importRunner}\n${[
      'try:',
      `    print(runner.parse_key_prefixes(${JSON.stringify([rule])}))`,
      'except SystemExit as exc:',
      '    print("REJECTED", str(exc))',
    ].join('\n')}`);
    // `model.=` is a legal rule (strip the prefix), so only the two without a
    // usable source are rejected — assert the split rather than blanket-failing.
    if (rule === 'model.=') expect(output).toMatch(/\[\('model\.', ''\)\]/);
    else expect(output).toMatch(/REJECTED/);
  });

  // The remap has to translate the SUBSTITUTE's namespace onto the one the
  // pinned loader matches, then delegate every real decision (which layers are
  // past the conditioning depth, what lm_head maps to) back to it — so the
  // adapter can't drift from the port's own contract.
  it('remaps checkpoint keys through the pinned loader rather than replacing it', () => {
    const output = runPython(`${importRunner}\n${[
      'import json, sys, types',
      // Stand in for the pinned module: `_wanted` records what it was handed.
      'seen = []',
      'class FakeEncoder:',
      '    def _wanted(self, key):',
      '        seen.append(key)',
      '        return ("language", key) if key.startswith("model.language_model.") else None',
      'module = types.ModuleType("minimax_h3_mlx.text_encoder")',
      'module.MiniMaxH3TextEncoder = FakeEncoder',
      'package = types.ModuleType("minimax_h3_mlx")',
      'sys.modules["minimax_h3_mlx"] = package',
      'sys.modules["minimax_h3_mlx.text_encoder"] = module',
      'runner.install_key_prefix_map(runner.parse_key_prefixes(["model.=model.language_model.", "visual.=model.visual."]))',
      'encoder = FakeEncoder()',
      'results = [encoder._wanted(k) for k in ["model.layers.0.mlp.up_proj.weight", "visual.blocks.0.attn.qkv.weight", "unrelated.weight"]]',
      'print(json.dumps({"seen": seen, "results": results}))',
    ].join('\n')}`);
    const { seen, results } = JSON.parse(output);
    expect(seen).toEqual([
      'model.language_model.layers.0.mlp.up_proj.weight',
      'model.visual.blocks.0.attn.qkv.weight',
      'unrelated.weight',
    ]);
    // The delegate's verdict is returned untouched — including its `None` for a
    // key it doesn't want, which is what keeps unmapped tensors skipped.
    expect(results).toEqual([
      ['language', 'model.language_model.layers.0.mlp.up_proj.weight'],
      null,
      null,
    ]);
  });

  // A pin whose loader no longer exposes `_wanted` must say so, not swap in a
  // conditioner whose keys nothing will map.
  it('refuses to swap a conditioner onto a pin with no _wanted hook', () => {
    const output = runPython(`${importRunner}\n${[
      'import sys, types',
      'class Bare: pass',
      'module = types.ModuleType("minimax_h3_mlx.text_encoder")',
      'module.MiniMaxH3TextEncoder = Bare',
      'sys.modules["minimax_h3_mlx"] = types.ModuleType("minimax_h3_mlx")',
      'sys.modules["minimax_h3_mlx.text_encoder"] = module',
      'try:',
      '    runner.install_key_prefix_map([("model.", "model.language_model.")])',
      'except RuntimeError as exc:',
      '    print(str(exc))',
      'else:',
      '    raise SystemExit("a pin with no _wanted hook was accepted")',
    ].join('\n')}`);
    expect(output).toMatch(/no longer exposes MiniMaxH3TextEncoder\._wanted/);
  });

  // The shim is what lets the PINNED `from_pretrained` load a swapped
  // conditioner unmodified: everything but `text_encoder/` is linked straight
  // through, and the tokenizer/processor the encoder reads from its parent
  // directory have to come along or the load fails after the weights are read.
  it('composes a checkpoint root with only text_encoder swapped', () => {
    const output = runPython(`${importRunner}\n${[
      'import json, tempfile',
      'with tempfile.TemporaryDirectory() as temp:',
      '    root = Path(temp)',
      '    checkpoint = root / "FL2VA"',
      '    for name in ("video_vae", "audio_vae", "tokenizer", "processor", "text_encoder"):',
      '        (checkpoint / name).mkdir(parents=True)',
      '    (checkpoint / "model_index.json").write_text("{}")',
      '    (checkpoint / "text_encoder" / "config.json").write_text(json.dumps({"text_config": {"hidden_size": 8}}))',
      '    (checkpoint / "text_encoder" / "model-00001-of-00002.safetensors").write_text("stock")',
      '    substitute = root / "substitute.safetensors"',
      '    substitute.write_text("swapped")',
      '    shim = runner.build_encoder_shim(checkpoint, root / "shims", "heretic-bf16", [substitute], None)',
      '    print(json.dumps({',
      '        "name": shim.name,',
      '        "root": sorted(p.name for p in shim.iterdir()),',
      '        "encoder": sorted(p.name for p in (shim / "text_encoder").iterdir()),',
      '        "tokenizer_links": (shim / "tokenizer").resolve() == (checkpoint / "tokenizer").resolve(),',
      '    }))',
    ].join('\n')}`);
    const result = JSON.parse(output);
    expect(result.name).toBe('heretic-bf16');
    expect(result.root).toEqual(['audio_vae', 'model_index.json', 'processor', 'text_encoder', 'tokenizer', 'video_vae']);
    // The stock shards are NOT linked in — the loader globs *.safetensors, so
    // leaving them would load both conditioners' weights over each other.
    expect(result.encoder).toEqual(['config.json', 'substitute.safetensors']);
    // The encoder resolves its tokenizer/processor from `_model_dir.parent`.
    expect(result.tokenizer_links).toBe(true);
  });

  // H3 reads the hidden state BEFORE the final norm, so a conditioner published
  // for it correctly omits that tensor — but the pinned loader builds the whole
  // module tree and refuses to load with any parameter missing.
  it.skipIf(!hasMlx)('synthesizes the missing final norm in the substitute key namespace', () => {
    const output = runPython(`${importRunner}\n${[
      'import json, tempfile',
      'with tempfile.TemporaryDirectory() as temp:',
      '    root = Path(temp)',
      '    checkpoint = root / "FL2VA"',
      '    (checkpoint / "text_encoder").mkdir(parents=True)',
      '    (checkpoint / "text_encoder" / "config.json").write_text(json.dumps({"text_config": {"hidden_size": 8}}))',
      '    substitute = root / "substitute.safetensors"',
      '    substitute.write_text("swapped")',
      '    shim = runner.build_encoder_shim(checkpoint, root / "shims", "heretic-bf16", [substitute], "model.norm.weight")',
      '    import mlx.core as mx',
      '    loaded = mx.load(str(shim / "text_encoder" / "_portos_final_norm.safetensors"))',
      '    key = next(iter(loaded))',
      '    print(json.dumps({"key": key, "shape": list(loaded[key].shape), "values": loaded[key].tolist()}))',
    ].join('\n')}`);
    const result = JSON.parse(output);
    // Written under the SUBSTITUTE's namespace so the prefix map rewrites it
    // like every other key — writing the post-map name would leave it unmapped
    // and the parameter still missing.
    expect(result.key).toBe('model.norm.weight');
    expect(result.shape).toEqual([8]);
    expect(result.values).toEqual(Array(8).fill(1));
  });

  it('rebuilds the shim from scratch so a stale link cannot survive', () => {
    const output = runPython(`${importRunner}\n${[
      'import json, tempfile',
      'with tempfile.TemporaryDirectory() as temp:',
      '    root = Path(temp)',
      '    checkpoint = root / "FL2VA"',
      '    (checkpoint / "text_encoder").mkdir(parents=True)',
      '    (checkpoint / "text_encoder" / "config.json").write_text(json.dumps({"text_config": {"hidden_size": 8}}))',
      '    stale = root / "shims" / "heretic-bf16" / "text_encoder"',
      '    stale.mkdir(parents=True)',
      '    (stale / "old.safetensors").write_text("stale")',
      '    substitute = root / "substitute.safetensors"',
      '    substitute.write_text("swapped")',
      '    shim = runner.build_encoder_shim(checkpoint, root / "shims", "heretic-bf16", [substitute], None)',
      '    print(json.dumps(sorted(p.name for p in (shim / "text_encoder").iterdir())))',
    ].join('\n')}`);
    expect(JSON.parse(output)).toEqual(['config.json', 'substitute.safetensors']);
  });

  it('fails a missing substitute before anything is composed', () => {
    const output = runPython(`${importRunner}\n${[
      'import json, tempfile',
      'with tempfile.TemporaryDirectory() as temp:',
      '    root = Path(temp)',
      '    checkpoint = root / "FL2VA"',
      '    (checkpoint / "text_encoder").mkdir(parents=True)',
      '    try:',
      '        runner.build_encoder_shim(checkpoint, root / "shims", "heretic-bf16", [root / "absent.safetensors"], None)',
      '    except RuntimeError as exc:',
      '        print(json.dumps({"error": str(exc), "built": (root / "shims").exists()}))',
      '    else:',
      '        raise SystemExit("a missing substitute was accepted")',
    ].join('\n')}`);
    const result = JSON.parse(output);
    expect(result.error).toMatch(/Substituted text encoder is missing/);
    expect(result.built).toBe(false);
  });

  // An upstream (rather than repackaged) conditioner arrives as several shards.
  // The loader globs *.safetensors in the shim, so every shard has to be linked
  // in — a partial link set loads a module tree with missing parameters.
  it('links every shard of a multi-shard substitute into one text_encoder', () => {
    const output = runPython(`${importRunner}\n${[
      'import json, tempfile',
      'with tempfile.TemporaryDirectory() as temp:',
      '    root = Path(temp)',
      '    checkpoint = root / "FL2VA"',
      '    (checkpoint / "text_encoder").mkdir(parents=True)',
      '    (checkpoint / "text_encoder" / "config.json").write_text(json.dumps({"text_config": {"hidden_size": 8}}))',
      '    shards = []',
      '    for name in ("model-00001-of-00014.safetensors", "model-00002-of-00014.safetensors", "model-00014-of-00014.safetensors"):',
      '        shard = root / name',
      '        shard.write_text(name)',
      '        shards.append(shard)',
      '    shim = runner.build_encoder_shim(checkpoint, root / "shims", "huihui-abliterated", shards, None)',
      '    linked = sorted(p.name for p in (shim / "text_encoder").iterdir())',
      '    print(json.dumps({"linked": linked, "resolves": [(shim / "text_encoder" / s.name).read_text() for s in shards]}))',
    ].join('\n')}`);
    const result = JSON.parse(output);
    expect(result.linked).toEqual([
      'config.json',
      'model-00001-of-00014.safetensors',
      'model-00002-of-00014.safetensors',
      'model-00014-of-00014.safetensors',
    ]);
    expect(result.resolves).toEqual([
      'model-00001-of-00014.safetensors',
      'model-00002-of-00014.safetensors',
      'model-00014-of-00014.safetensors',
    ]);
  });

  // Two shards sharing a basename would collide on the single symlink name, and
  // whichever lost would be silently absent from the glob.
  it('refuses a substitute whose shards share a basename', () => {
    const output = runPython(`${importRunner}\n${[
      'import json, tempfile',
      'with tempfile.TemporaryDirectory() as temp:',
      '    root = Path(temp)',
      '    checkpoint = root / "FL2VA"',
      '    (checkpoint / "text_encoder").mkdir(parents=True)',
      '    shards = []',
      '    for parent in ("a", "b"):',
      '        (root / parent).mkdir()',
      '        shard = root / parent / "model-00001-of-00002.safetensors"',
      '        shard.write_text(parent)',
      '        shards.append(shard)',
      '    try:',
      '        runner.build_encoder_shim(checkpoint, root / "shims", "dupe", shards, None)',
      '    except RuntimeError as exc:',
      '        print(json.dumps({"error": str(exc), "built": (root / "shims").exists()}))',
      '    else:',
      '        raise SystemExit("colliding shard names were accepted")',
    ].join('\n')}`);
    const result = JSON.parse(output);
    expect(result.error).toMatch(/duplicate shard names/);
    expect(result.built).toBe(false);
  });

  it('loads only the pinned namespace and cannot import a root-level shadow module', () => {
    const output = runPython(`${importRunner}\n${[
      'import importlib, tempfile',
      'with tempfile.TemporaryDirectory() as temp:',
      '    root = Path(temp) / "runtime"',
      '    package = root / "minimax_h3_mlx"',
      '    trusted = Path(temp) / "venv"',
      '    package.mkdir(parents=True)',
      '    trusted.mkdir()',
      '    (root / "mlx_vlm.py").write_text("SOURCE = \'runtime-shadow\'")',
      '    (trusted / "mlx_vlm.py").write_text("SOURCE = \'locked-venv\'")',
      '    (package / "pipeline.py").write_text("from mlx_vlm import SOURCE")',
      '    sys.path.insert(0, str(trusted))',
      '    runner.register_source_namespace("minimax_h3_mlx", package)',
      '    pipeline = importlib.import_module("minimax_h3_mlx.pipeline")',
      '    print(pipeline.SOURCE)',
      '    print(str(root) in sys.path)',
    ].join('\n')}`);
    expect(output.trim().split(/\r?\n/)).toEqual(['locked-venv', 'False']);
  });

  it('bounds the git checkout verification probe', () => {
    const output = runPython(`${importRunner}\n${[
      'from types import SimpleNamespace',
      'seen = {}',
      'def fake_run(*_args, **kwargs):',
      '    seen.update(kwargs)',
      '    return SimpleNamespace(stdout="# branch.oid expected\\n# branch.head (detached)\\n")',
      'runner.subprocess.run = fake_run',
      'runner.verify_runtime_checkout(Path("/tmp/runtime"), "expected")',
      'print(seen["timeout"])',
    ].join('\n')}`);
    expect(output.trim()).toBe('10');
  });
  // Keyframe conditioning is the one path with no torch under it: the MLX lock
  // ships neither torch nor torchvision, and transformers 5 routes every auto
  // image-processor load through them. Text-only renders never touch it, which
  // is why image-to-video was the only mode that failed.
  it('reads the runtime for the torch stack the auto processor needs', () => {
    const output = runPython(`${importRunner}\n${[
      'import json',
      'present = {"torch": object(), "torchvision": object(), "numpy": object()}',
      'runner.importlib.util.find_spec = present.get',
      'both = runner.torch_image_stack_available()',
      'present.pop("torchvision")',
      'vision_only = runner.torch_image_stack_available()',
      'present.pop("torch")',
      'neither = runner.torch_image_stack_available()',
      'print(json.dumps([both, vision_only, neither]))',
    ].join('\n')}`);
    // torch alone is not enough: it is the torchvision-backed video processor
    // that `AutoProcessor.from_pretrained` builds and dies on.
    expect(JSON.parse(output)).toEqual([true, false, false]);
  });

  it('loads the PIL twin of the image processor the checkpoint declares', () => {
    const output = runPython(`${importRunner}\n${[
      'import json, sys, tempfile, types',
      'loaded = {}',
      'class FakePil:',
      '    @classmethod',
      '    def from_pretrained(cls, path):',
      '        loaded["path"] = path',
      '        return "pil-image-processor"',
      'transformers = types.ModuleType("transformers")',
      'transformers.__version__ = "5.14.1"',
      'transformers.Qwen2VLImageProcessorPil = FakePil',
      'sys.modules["transformers"] = transformers',
      'with tempfile.TemporaryDirectory() as temp:',
      '    processor = Path(temp) / "processor"',
      '    processor.mkdir()',
      // The checkpoint names the torchvision class; the twin is derived off the
      // `Fast`-stripped base name rather than hardcoded to one checkpoint.
      '    (processor / "preprocessor_config.json").write_text(json.dumps({"image_processor_type": "Qwen2VLImageProcessorFast"}))',
      '    result = runner.load_pil_image_processor(processor)',
      '    print(json.dumps({"result": result, "path": loaded["path"] == str(processor)}))',
    ].join('\n')}`);
    expect(JSON.parse(output)).toEqual({ result: 'pil-image-processor', path: true });
  });

  it.each([
    ['{}', /names no image_processor_type/],
    ['{"image_processor_type": "InventedImageProcessor"}', /exposes no InventedImageProcessorPil/],
  ])('refuses a processor config it cannot resolve a PIL twin from (%s)', (config, expected) => {
    const output = runPython(`${importRunner}\n${[
      'import sys, tempfile, types',
      'transformers = types.ModuleType("transformers")',
      'transformers.__version__ = "5.14.1"',
      'sys.modules["transformers"] = transformers',
      'with tempfile.TemporaryDirectory() as temp:',
      '    processor = Path(temp) / "processor"',
      '    processor.mkdir()',
      `    (processor / "preprocessor_config.json").write_text(${JSON.stringify(config)})`,
      '    try:',
      '        runner.load_pil_image_processor(processor)',
      '    except RuntimeError as exc:',
      '        print(str(exc))',
      '    else:',
      '        raise SystemExit("an unresolvable processor config was accepted")',
    ].join('\n')}`);
    expect(output).toMatch(expected);
  });

  // The encoder reads exactly one thing off its processor, so the twin is bound
  // to that one attribute: building the rest is what pulls in the video
  // processor nothing here uses.
  it('binds the PIL twin to the only sub-processor the pinned encoder reads', () => {
    const output = runPython(`${importRunner}\n${stubPin([
      '    def __init__(self, model_dir):',
      '        self._model_dir = Path(model_dir)',
      '        self._processor = None',
      '    @property',
      '    def processor(self):',
      '        raise AssertionError("the pinned auto-processor property must not run")',
    ])}\n${[
      'import json',
      'seen = []',
      'runner.load_pil_image_processor = lambda directory: seen.append(directory) or "pil-image-processor"',
      'runner.install_pil_image_processor()',
      'encoder = MiniMaxH3TextEncoder("/checkpoint/text_encoder")',
      'first = encoder.processor',
      'print(json.dumps({',
      '    "image_processor": first.image_processor,',
      '    "cached": encoder.processor is first,',
      '    "loads": [str(p) for p in seen],',
      '}))',
    ].join('\n')}`);
    const result = JSON.parse(output);
    expect(result.image_processor).toBe('pil-image-processor');
    // One load per encoder — the pinned property memoizes on `_processor` and
    // the twin has to keep doing that or every keyframe re-reads the config.
    expect(result.cached).toBe(true);
    // Resolved off the encoder's own model dir, so a composed text-encoder shim
    // root keeps pointing at the processor it linked through.
    expect(result.loads).toEqual([join('/checkpoint', 'processor')]);
  });

  // mlx-vlm's loader sanitizes `patch_embed.proj.weight` into MLX's conv layout;
  // the pinned port loads the safetensors itself and skips it, so the tower gets
  // torch's `(C_out, C_in, kD, kH, kW)` and conv3d rejects the first keyframe.
  it('sanitizes the vision tower after the pinned load, not instead of it', () => {
    const output = runPython(`${importRunner}\n${stubPin([
      '    def _load_weights(self, model_dir, dtype, verbose):',
      '        order.append(("pinned load", str(model_dir), str(dtype), verbose))',
      '        self.vision = "vision-tower"',
    ], ['order = []'])}\n${[
      'import json',
      'runner.sanitize_vision_weights = lambda vision: order.append(("sanitize", vision))',
      'runner.install_vision_weight_sanitizer()',
      'MiniMaxH3TextEncoder()._load_weights(Path("/checkpoint/text_encoder"), "bfloat16", False)',
      'print(json.dumps(order))',
    ].join('\n')}`);
    // The pinned load runs first and unchanged — the correction is applied to
    // what it produced, so a pin that starts loading a sanitized weight keeps
    // working (mlx-vlm's own shape check makes the second pass a no-op).
    expect(JSON.parse(output)).toEqual([
      ['pinned load', join('/checkpoint', 'text_encoder'), 'bfloat16', false],
      ['sanitize', 'vision-tower'],
    ]);
  });

  it('leaves the vision tower alone when the encoder built none', () => {
    const output = runPython(`${importRunner}\n${[
      'runner.sanitize_vision_weights(None)',
      'print("ok")',
    ].join('\n')}`);
    // Reached on a text-only encoder (`load_vision=False`); importing mlx to
    // sanitize nothing would cost a load the run does not need.
    expect(output.trim()).toBe('ok');
  });

  // Each adapter patches a different method, and each is only correct against
  // the implementation it was written for — so a pin that moved has to be told
  // apart from one that still fits, before a render spends minutes loading.
  it.each([
    ['install_pil_image_processor', 'processor'],
    ['install_vision_weight_sanitizer', '_load_weights'],
    ['install_vision_embed_merge', 'encode'],
  ])('refuses to install %s onto a pin with no %s hook', (installer, hook) => {
    const output = runPython(`${importRunner}\n${stubPin(['    pass'])}\n${[
      'try:',
      `    runner.${installer}()`,
      'except RuntimeError as exc:',
      '    print(str(exc))',
      'else:',
      `    raise SystemExit("a pin with no ${hook} hook was accepted")`,
    ].join('\n')}`);
    expect(output).toMatch(new RegExp(`no longer exposes MiniMaxH3TextEncoder\\.${hook}`));
  });

  // A `processor` that is no longer a property is as much a pin change as an
  // absent one: the pinned caller would start calling it, and the replacement
  // is a property.
  it('refuses to bind the twin onto a processor that is no longer a property', () => {
    const output = runPython(`${importRunner}\n${stubPin(['    processor = "not-a-property"'])}\n${[
      'try:',
      '    runner.install_pil_image_processor()',
      'except RuntimeError as exc:',
      '    print(str(exc))',
      'else:',
      '    raise SystemExit("a non-property processor was accepted")',
    ].join('\n')}`);
    expect(output).toMatch(/no longer exposes MiniMaxH3TextEncoder\.processor/);
  });

  // The pinned encode merges the keyframe's vision rows with a BROADCAST, which
  // only lines up when the request is nothing but image tokens — every real
  // prompt + keyframe pair dies in `[broadcast_shapes]`. The correction re-runs
  // the pinned body with that one line swapped, so it is guarded twice: by the
  // line it replaces, and by a digest over the whole body it copied.
  it('corrects a pin that still merges by broadcast, and hands its text-only path back', () => {
    const output = runPython(`${importRunner}\n${importPins}\n${filePin(["pins.PINNED_BROADCAST_MERGE"])}\n${[
      'import hashlib, inspect',
      'pins.PINNED_ENCODE_DIGEST = hashlib.sha256(',
      '    inspect.getsource(MiniMaxH3TextEncoder.encode).encode("utf-8")',
      ').hexdigest()',
      'runner.install_vision_embed_merge()',
      'print(MiniMaxH3TextEncoder().encode("a prompt"))',
    ].join('\n')}`);
    // A text-only request has no merge to correct, so it must come back from the
    // pinned implementation untouched.
    expect(output.trim()).toBe("('pinned', 'a prompt', None)");
  });

  it.each([
    // Upstream fixed the merge: the correction has to retire, not shadow it.
    [['pins.PINNED_BROADCAST_MERGE.replace("mx.where", "masked_scatter")'], 'digest-of-this-pin', /no longer merges keyframe embeddings/],
    // The merge is untouched but something else in the body moved — the copy is
    // stale in a way matching one line could never see.
    [['pins.PINNED_BROADCAST_MERGE'], `"${'0'.repeat(64)}"`, /changed MiniMaxH3TextEncoder\.encode outside the merge/],
  ])('refuses a pin the copied encode no longer matches (%j)', (body, digest, expected) => {
    const output = runPython(`${importRunner}\n${importPins}\n${filePin(body)}\n${[
      'import hashlib, inspect',
      `pins.PINNED_ENCODE_DIGEST = ${digest === 'digest-of-this-pin'
        ? 'hashlib.sha256(inspect.getsource(MiniMaxH3TextEncoder.encode).encode("utf-8")).hexdigest()'
        : digest}`,
      'try:',
      '    runner.install_vision_embed_merge()',
      'except RuntimeError as exc:',
      '    print(str(exc))',
      'else:',
      '    raise SystemExit("a drifted pin was accepted")',
    ].join('\n')}`);
    expect(output).toMatch(expected);
  });

  // --- Prompt-embedding cache (#5443) ---------------------------------------

  it('parses the optional prompt-embedding cache directory', () => {
    const output = runPython(`${importRunner}\n${[
      'import sys',
      'sys.argv = ["generate_minimax_h3.py", "--model-repo", "example/model", "--model-revision", "revision",',
      '    "--runtime-dir", "/tmp/runtime", "--runtime-revision", "revision",',
      '    "--checkpoint-repo", "example/checkpoint", "--checkpoint-revision", "revision",',
      '    "--prompt", "a test prompt", "--width", "1344", "--height", "768", "--num-frames", "124",',
      '    "--output", "test.mp4", "--prompt-embedding-cache-dir", "/tmp/embeddings"]',
      'print(runner.parse_args().prompt_embedding_cache_dir)',
    ].join('\n')}`);
    expect(output.trim()).toBe('/tmp/embeddings');
  });

  // The whole point of the key: an embedding is only reusable against the exact
  // request that produced it. Two renders sharing a prompt but conditioned on
  // different keyframes must not share an entry — and because the keyframes are
  // identified by their PIXELS, the same frame decoded again from another
  // job-scoped scratch path still hits.
  it('keys an embedding on the prompt, the conditioner and each keyframe content digest', () => {
    const output = runPython(`${importRunner}\n${stubImage}\n${[
      'import json',
      'first = StubImage("RGB", (2, 2), b"IMAGE-A")',
      '# A second decode of the same source frame — what a re-render actually',
      '# hands `encode` once the scratch copy has moved.',
      'replayed = StubImage("RGB", (2, 2), b"IMAGE-A")',
      '# Different pixels at the SAME byte length, which a size-based identity',
      '# could not tell apart from the pair above.',
      'other = StubImage("RGB", (2, 2), b"IMAGE-B")',
      '# Same buffer, different geometry: the vision tower sees another request.',
      'reshaped = StubImage("RGB", (4, 1), b"IMAGE-A")',
      'identity = {"runtime_revision": "rev", "checkpoint": "example/checkpoint@rev",',
      '            "text_encoder_id": "", "text_encoder_files": [],',
      '            "text_encoder_key_prefix": [], "text_encoder_final_norm_key": ""}',
      'digest = runner.hash_conditioning_image',
      'key = runner.prompt_embedding_key',
      'print(json.dumps({',
      '    "textOnly": key(identity, "a prompt", []),',
      '    "imageA": key(identity, "a prompt", [digest(first)]),',
      '    "imageAReplayed": key(identity, "a prompt", [digest(replayed)]),',
      '    "imageB": key(identity, "a prompt", [digest(other)]),',
      '    "imageAReshaped": key(identity, "a prompt", [digest(reshaped)]),',
      '    "bothFrames": key(identity, "a prompt", [digest(first), digest(other)]),',
      '    "bothFramesSwapped": key(identity, "a prompt", [digest(other), digest(first)]),',
      '    "otherPrompt": key(identity, "another prompt", [digest(first)]),',
      '    "substitutedEncoder": key({**identity, "text_encoder_id": "example-abliterated"},',
      '                              "a prompt", [digest(first)]),',
      '}))',
    ].join('\n')}`);
    const keys = JSON.parse(output);
    // The same frame decoded again is the SAME request.
    expect(keys.imageAReplayed).toBe(keys.imageA);
    // Everything else names a different one, text-to-video included.
    const distinct = Object.entries(keys).filter(([name]) => name !== 'imageAReplayed');
    expect(new Set(distinct.map(([, value]) => value)).size).toBe(distinct.length);
    for (const value of Object.values(keys)) expect(value).toMatch(/^[0-9a-f]{64}$/);
  });

  // The identity is what makes an entry unreadable by a DIFFERENT stack: a pin
  // bump, another checkpoint or a swapped conditioner all produce different
  // embeddings for one prompt, and serving the previous one would be a silently
  // wrong render rather than an error.
  it('folds the pinned runtime, the checkpoint and any substituted conditioner into the identity', () => {
    const output = runPython(`${importRunner}\n${[
      'import json',
      'from types import SimpleNamespace',
      argsExpr({
        runtime_revision: 'runtime-revision',
        checkpoint_repo: 'example/checkpoint',
        checkpoint_revision: 'checkpoint-revision',
        text_encoder_id: 'example-abliterated',
        text_encoder_file: [
          '/hf/models--example--encoder/snapshots/encoder-revision/shard-2.safetensors',
          '/hf/models--example--encoder/snapshots/encoder-revision/shard-1.safetensors',
        ],
        text_encoder_key_prefix: ['visual.=model.visual.', 'model.layers.=model.language_model.layers.'],
        text_encoder_final_norm_key: 'model.norm.weight',
      }),
      'print(json.dumps(runner.prompt_embedding_identity(args)))',
    ].join('\n')}`);
    expect(JSON.parse(output)).toEqual({
      runtime_revision: 'runtime-revision',
      checkpoint: 'example/checkpoint@checkpoint-revision',
      text_encoder_id: 'example-abliterated',
      // The snapshot segment, not just the basename: a release that repins an
      // existing catalog id onto new weights keeps the id and the shard names,
      // so dropping the revision would serve the old weights' embeddings.
      // Sorted, so the order PortOS happened to pass the shards in cannot split
      // one conditioner across two entries.
      text_encoder_files: ['encoder-revision/shard-1.safetensors', 'encoder-revision/shard-2.safetensors'],
      text_encoder_key_prefix: ['model.layers.=model.language_model.layers.', 'visual.=model.visual.'],
      text_encoder_final_norm_key: 'model.norm.weight',
    });
  });

  // Retention: a render session that keeps changing prompt would otherwise grow
  // the directory without bound, and each entry is tens of megabytes.
  it('evicts least-recently-used entries, and everything below the first that does not fit', () => {
    const output = runPython(`${importRunner}\n${[
      'import json, os, tempfile',
      'with tempfile.TemporaryDirectory() as temp:',
      '    root = Path(temp)',
      '    for name, size, mtime in (("a" * 64, 300, 3000), ("b" * 64, 500, 2000), ("c" * 64, 100, 1000)):',
      '        entry = root / f"{name}.safetensors"',
      '        entry.write_bytes(b"x" * size)',
      '        os.utime(entry, (mtime, mtime))',
      '    evicted = runner.prune_prompt_embedding_cache(root, 700)',
      '    print(json.dumps({"evicted": [name[0] for name in evicted],',
      '                      "left": sorted(path.name[0] for path in root.iterdir())}))',
    ].join('\n')}`);
    // "a" fits (300), "b" does not (800 > 700) — and "c" goes with it even
    // though 400 bytes were free, because retention ranks on use, not on size.
    expect(JSON.parse(output)).toEqual({ evicted: ['b', 'c'], left: ['a'] });
  });

  // mtime has coarse enough resolution that two entries written in one render
  // can share a tick; without the name tiebreak the survivor would be whichever
  // one the filesystem happened to list first.
  it('breaks an eviction tie on the key digest rather than on directory order', () => {
    const output = runPython(`${importRunner}\n${[
      'import json, os, tempfile',
      'with tempfile.TemporaryDirectory() as temp:',
      '    root = Path(temp)',
      '    for name in ("a" * 64, "b" * 64, "c" * 64):',
      '        entry = root / f"{name}.safetensors"',
      '        entry.write_bytes(b"x" * 300)',
      '        os.utime(entry, (2000, 2000))',
      '    print(json.dumps([name[0] for name in runner.prune_prompt_embedding_cache(root, 700)]))',
    ].join('\n')}`);
    expect(JSON.parse(output)).toEqual(['a']);
  });

  // A temp only outlives its own render when that render was killed between the
  // save and the rename. It is never a readable entry, so it is swept — but on a
  // TTL, so a sibling render's in-flight write is not pulled out from under it.
  it('sweeps a stranded partial write past its TTL and leaves a fresh one alone', () => {
    const output = runPython(`${importRunner}\n${[
      'import json, os, tempfile, time',
      'with tempfile.TemporaryDirectory() as temp:',
      '    root = Path(temp)',
      '    stale = root / f\'{"a" * 64}.partial-101.safetensors\'',
      '    stale.write_bytes(b"x" * 4000)',
      '    old = time.time() - runner.PROMPT_EMBEDDING_PARTIAL_TTL_SECONDS - 60',
      '    os.utime(stale, (old, old))',
      '    fresh = root / f\'{"b" * 64}.partial-102.safetensors\'',
      '    fresh.write_bytes(b"x" * 4000)',
      '    entry = root / f\'{"c" * 64}.safetensors\'',
      '    entry.write_bytes(b"x" * 100)',
      '    evicted = runner.prune_prompt_embedding_cache(root, 200)',
      '    print(json.dumps({"evicted": evicted, "left": sorted(path.name for path in root.iterdir())}))',
    ].join('\n')}`);
    const result = JSON.parse(output);
    // Neither partial counts against the ceiling — 8000 bytes of them would
    // otherwise have evicted the one real 100-byte entry.
    expect(result.evicted).toEqual([]);
    expect(result.left).toEqual([`${'b'.repeat(64)}.partial-102.safetensors`, `${'c'.repeat(64)}.safetensors`]);
  });

  // An unusable cache directory is not a render failure: the runner recomputes,
  // which is exactly what it did before the cache existed.
  it('disables the cache when its directory cannot be created', () => {
    const output = runPython(`${importRunner}\n${[
      'import sys, tempfile',
      captureWarnings,
      'with tempfile.TemporaryDirectory() as temp:',
      '    blocker = Path(temp) / "embeddings"',
      '    blocker.write_bytes(b"not a directory")',
      '    print(repr(runner.prepare_prompt_embedding_cache(blocker)))',
    ].join('\n')}`);
    expect(output).toMatch(/prompt-embedding cache disabled, recomputing every embedding/);
    expect(output.trim().split(/\r?\n/).pop()).toBe('None');
  });

  // The directory already exists and only the WRITE fails — a read-only mount,
  // a sandbox, a full disk. `os.access` reports the permission bits rather than
  // what the filesystem will do, so readiness is proven by an actual write.
  // POSIX-only: `os.chmod` on Windows toggles a file's read-only bit and cannot
  // make a directory unwritable.
  // Root ignores the 0o500 mode, so the write succeeds and nothing is refused.
  it.skipIf(process.platform === 'win32' || process.getuid?.() === 0)('disables the cache when its existing directory refuses a write', () => {
    const output = runPython(`${importRunner}\n${[
      'import os, sys, tempfile',
      captureWarnings,
      'with tempfile.TemporaryDirectory() as temp:',
      '    readonly = Path(temp) / "embeddings"',
      '    readonly.mkdir()',
      '    os.chmod(readonly, 0o500)',
      '    try:',
      '        print(repr(runner.prepare_prompt_embedding_cache(readonly)))',
      '    finally:',
      '        os.chmod(readonly, 0o700)',
    ].join('\n')}`);
    expect(output).toMatch(/prompt-embedding cache disabled, recomputing every embedding/);
    expect(output.trim().split(/\r?\n/).pop()).toBe('None');
  });

  // Everything above is key and retention arithmetic, which needs neither mlx
  // nor numpy. These last cases round-trip real tensors and describe them
  // through numpy, so they run only where both import (CI's bare python3 is
  // Linux, which has no mlx wheel).
  describe.skipIf(!hasEmbeddingStack)('served embeddings', () => {
    // The stand-in encoder returns the same shape the pinned `encode` does —
    // `(mx.array, np.ndarray[int64])` — and records every call, which is how a
    // hit is told from a recompute.
    const stubEncoder = stubPin([
      '    calls = []',
      '    def encode(self, prompt, images=None):',
      '        MiniMaxH3TextEncoder.calls.append((prompt, len(images or [])))',
      '        return (mx.arange(12).reshape(1, 3, 4).astype(mx.bfloat16),',
      '                np.array([0, 1, 0], dtype=np.int64))',
    ], ['import mlx.core as mx', 'import numpy as np']);

    // Reported as a float32 view: bfloat16 widens to it losslessly, so equal
    // float32 bytes mean bit-identical bfloat16 — and numpy has no bfloat16 of
    // its own to hash directly.
    const describeResult = [
      'def describe(pair):',
      '    hidden, tags = pair',
      '    return {"dtype": str(hidden.dtype), "shape": list(hidden.shape),',
      '            "bytes": hashlib.sha256(np.asarray(hidden.astype(mx.float32)).tobytes()).hexdigest(),',
      '            "tagsDtype": str(tags.dtype), "tags": tags.tolist()}',
    ];

    it('serves a second render of the same request byte-identically, without re-encoding', () => {
      const output = runPython(`${importRunner}\n${stubEncoder}\n${stubImage}\n${[
        'import hashlib, json, mlx.core as mx, numpy as np, tempfile',
        captureWarnings,
        ...describeResult,
        'with tempfile.TemporaryDirectory() as temp:',
        '    cache = runner.prepare_prompt_embedding_cache(Path(temp) / "embeddings")',
        '    runner.install_prompt_embedding_cache(cache, {"runtime_revision": "rev"})',
        '    encoder = MiniMaxH3TextEncoder()',
        '    keyframe = StubImage("RGB", (2, 2), b"IMAGE-A")',
        '    cold = describe(encoder.encode("a prompt", [keyframe]))',
        '    # A second decode of the same frame, as a re-render would hand it over.',
        '    hit = describe(encoder.encode("a prompt", [StubImage("RGB", (2, 2), b"IMAGE-A")]))',
        '    print(json.dumps({"cold": cold, "hit": hit, "calls": MiniMaxH3TextEncoder.calls,',
        '                      "entries": [path.name for path in cache.iterdir()]}))',
      ].join('\n')}`);
      const result = JSON.parse(output.trim().split(/\r?\n/).pop());
      expect(result.hit).toEqual(result.cold);
      expect(result.hit.dtype).toBe('mlx.core.bfloat16');
      // The DiT reads `token_tags` as a numpy ndarray, so a hit has to hand back
      // one rather than the mx.array the entry stores.
      expect(result.hit.tagsDtype).toBe('int64');
      // One encode for two renders.
      expect(result.calls).toEqual([['a prompt', 1]]);
      expect(result.entries).toEqual([expect.stringMatching(/^[0-9a-f]{64}\.safetensors$/)]);
    });

    // The same prompt against a different keyframe is a different request, and
    // serving the first one's embedding would be a silently wrong render.
    it('re-encodes the same prompt when the keyframe changes', () => {
      const output = runPython(`${importRunner}\n${stubEncoder}\n${stubImage}\n${[
        'import json, mlx.core as mx, numpy as np, tempfile',
        captureWarnings,
        'with tempfile.TemporaryDirectory() as temp:',
        '    cache = runner.prepare_prompt_embedding_cache(Path(temp) / "embeddings")',
        '    runner.install_prompt_embedding_cache(cache, {"runtime_revision": "rev"})',
        '    encoder = MiniMaxH3TextEncoder()',
        '    encoder.encode("a prompt", [StubImage("RGB", (2, 2), b"IMAGE-A")])',
        '    encoder.encode("a prompt", [StubImage("RGB", (2, 2), b"IMAGE-B")])',
        '    # And text-to-video, which shares neither entry.',
        '    encoder.encode("a prompt")',
        '    print(json.dumps({"calls": len(MiniMaxH3TextEncoder.calls),',
        '                      "entries": len(list(cache.iterdir()))}))',
      ].join('\n')}`);
      expect(JSON.parse(output.trim().split(/\r?\n/).pop())).toEqual({ calls: 3, entries: 3 });
    });

    it('discards a truncated entry and recomputes instead of failing the render', () => {
      const output = runPython(`${importRunner}\n${stubEncoder}\n${[
        'import hashlib, json, mlx.core as mx, numpy as np, tempfile',
        captureWarnings,
        ...describeResult,
        'with tempfile.TemporaryDirectory() as temp:',
        '    cache = runner.prepare_prompt_embedding_cache(Path(temp) / "embeddings")',
        '    runner.install_prompt_embedding_cache(cache, {"runtime_revision": "rev"})',
        '    encoder = MiniMaxH3TextEncoder()',
        '    cold = describe(encoder.encode("a prompt"))',
        '    entry = next(cache.iterdir())',
        '    entry.write_bytes(entry.read_bytes()[:16])',
        '    recovered = describe(encoder.encode("a prompt"))',
        '    print(json.dumps({"cold": cold, "recovered": recovered,',
        '                      "calls": MiniMaxH3TextEncoder.calls,',
        '                      "rewritten": entry.stat().st_size > 16}))',
      ].join('\n')}`);
      expect(output).toMatch(/Discarding an unreadable MiniMax H3 prompt-embedding cache entry/);
      const result = JSON.parse(output.trim().split(/\r?\n/).pop());
      expect(result.recovered).toEqual(result.cold);
      expect(result.calls).toHaveLength(2);
      // The bad entry is replaced, not merely skipped, so the next render hits.
      expect(result.rewritten).toBe(true);
    });

    // A cache that stops being writable mid-render still must not cost the
    // render — the embedding is already computed by the time the store fails.
    it.skipIf(process.platform === 'win32')('returns the freshly-encoded embedding when the entry cannot be stored', () => {
      const output = runPython(`${importRunner}\n${stubEncoder}\n${[
        'import hashlib, json, mlx.core as mx, numpy as np, os, tempfile',
        captureWarnings,
        ...describeResult,
        'with tempfile.TemporaryDirectory() as temp:',
        '    cache = runner.prepare_prompt_embedding_cache(Path(temp) / "embeddings")',
        '    runner.install_prompt_embedding_cache(cache, {"runtime_revision": "rev"})',
        '    os.chmod(cache, 0o500)',
        '    try:',
        '        result = describe(MiniMaxH3TextEncoder().encode("a prompt"))',
        '    finally:',
        '        os.chmod(cache, 0o700)',
        '    print(json.dumps({"result": result, "entries": [path.name for path in cache.iterdir()]}))',
      ].join('\n')}`);
      expect(output).toMatch(/prompt embedding not cached/);
      const result = JSON.parse(output.trim().split(/\r?\n/).pop());
      expect(result.result.tags).toEqual([0, 1, 0]);
      expect(result.entries).toEqual([]);
    });
  });
});

it.skipIf(!pyBin)('renders a batch on one pipeline with one encode and preserves earlier outputs on failure', () => {
  const output = runPython(`${importRunner}
import contextlib, io, tempfile
from types import SimpleNamespace as NS
runner.heartbeat = lambda *_: contextlib.nullcontext()
runner._install_h3_stepwise_preview = lambda *_: None
calls, encodes, saves = [], [], []
encoder = NS(encode=lambda *a, **k: encodes.append(a) or ('embeddings', 'tags'))
def render(prompt, **kwargs):
    encoded = encoder.encode(prompt, kwargs['images'])
    calls.append((kwargs['seed'], encoded))
    if kwargs['seed'] == 2:
        raise RuntimeError('render failed')
    return NS(video=NS(shape=[124]), fps=24, audio=None, sample_rate=32000)
class Pipe:
    text_encoder = encoder
    def __call__(self, *a, **kw): return render(*a, **kw)
def save(path, *args):
    saves.append(path.name)
    path.write_bytes(b'example-video')
with tempfile.TemporaryDirectory() as tmp:
    args = NS(output=str(Path(tmp) / 'clip.mp4'), seed=0, prompt='Example shot', num_frames=124, steps=8, anchor=[], height=768, width=1344)
    frames = io.StringIO()
    with contextlib.redirect_stdout(frames):
        try:
            runner.render_outputs(Pipe(), args, [], save, [0, 1, 2, 3])
        except RuntimeError as exc:
            assert str(exc) == 'render failed'
    results = [runner.json.loads(line) for line in frames.getvalue().splitlines()]
    assert [r['seed'] for r in results] == [0, 1]
    assert [r['batch_index'] for r in results] == [0, 1]
    assert saves == ['clip.mp4', 'clip-2.mp4']
    assert len(encodes) == 1
    assert [seed for seed, _ in calls] == [0, 1, 2]
    assert all(embeds is calls[0][1] for _, embeds in calls)
print('OK')
`);
  expect(output.trim()).toBe('OK');
});

// #8867: the FL2VA reference feeds its vision tower the keyframes it already put
// onto the canvas. The pinned port prepares them only for the VAE, so without
// this the text encoder would read the raw upload at a different size and token
// grid than the conditioning rows it is paired with.
it.skipIf(!pyBin)('conditions the vision encoder on canvas-prepared keyframes, as the reference does', () => {
  const output = runPython(`${importRunner}
import contextlib, io, sys, tempfile, types
from types import SimpleNamespace as NS
runner.heartbeat = lambda *_: contextlib.nullcontext()
runner._install_h3_stepwise_preview = lambda *_: None
prepared = []
def prepare_keyframe_image(image, height, width, stretch):
    prepared.append((image.size, stretch))
    return NS(size=(width, height), source=image.size)
packing = types.ModuleType('minimax_h3_mlx.packing')
packing.prepare_keyframe_image = prepare_keyframe_image
sys.modules['minimax_h3_mlx'] = types.ModuleType('minimax_h3_mlx')
sys.modules['minimax_h3_mlx.packing'] = packing
seen = {}
def encode(prompt, images):
    seen['encoder'] = [(image.size, image.source) for image in images]
    return ('embeddings', 'tags')
class Pipe:
    text_encoder = NS(encode=encode)
    def __call__(self, prompt, **kwargs):
        # The pinned pipeline hands its images to the text encoder unchanged.
        self.text_encoder.encode(prompt, kwargs['images'])
        return NS(video=NS(shape=[124]), fps=24, audio=None, sample_rate=32000)
with tempfile.TemporaryDirectory() as tmp:
    args = NS(output=str(Path(tmp) / 'clip.mp4'), seed=0, prompt='Example shot', num_frames=124, steps=8,
              anchor=['first', 'last'], height=1344, width=768)
    with contextlib.redirect_stdout(io.StringIO()):
        runner.render_outputs(Pipe(), args, [NS(size=(3000, 4000)), NS(size=(1920, 1080))],
                              lambda path, *_: path.write_bytes(b'example-video'))
assert prepared == [((3000, 4000), True), ((1920, 1080), False)], prepared
assert seen['encoder'] == [((768, 1344), (3000, 4000)), ((768, 1344), (1920, 1080))], seen
print('OK')
`);
  expect(output.trim()).toBe('OK');
});