Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: keep LoRA effects on quantized LTX renders by applying adapters at inference (#12262)

[._.]/ Adam Eivy01ac87b6834e
diff --git a/scripts/generate_ltx2.py b/scripts/generate_ltx2.py
index be8f836a91..96b34b4d1a 100644
--- a/scripts/generate_ltx2.py
+++ b/scripts/generate_ltx2.py
@@ -697,9 +697,9 @@ def parse_args() -> argparse.Namespace:
help="Distilled-LoRA fusion strength (default 1.0, matches dgrauet's CLI).")
p.add_argument("--user-loras", default=None,
help="JSON list of {path, strength} dicts — user LoRAs (e.g. a "
- "fal LTX-2.3 LoRA) fused into the transformer via the pipeline's "
- "_pending_loras hook, the same mechanism upstream's "
- "`ltx-2-mlx generate --lora <path> <strength>` uses. Works across "
+ "fal LTX-2.3 LoRA) applied on every transformer load: added at "
+ "inference beside quantized projections, fused into float ones "
+ "(see ltx_runtime_lora.py). Works across "
"all modes (text/image/fflf/extend/a2v) because every pipeline "
"loads its DiT through _load_transformer_with_optional_streaming.")
p.add_argument("--image", default=None, help="Source/start frame (image, fflf modes)")
@@ -978,23 +978,48 @@ def install_ltx25_encoder_override(shim_dir: Path) -> None:
PromptEncoder._text_encoder_source = _text_encoder_source
-def _apply_user_loras(pipe, specs: list[tuple[str, float]]) -> None:
- """Set the pipeline's _pending_loras hook so user-LoRA deltas fuse into the
- transformer at load time.
-
- The DiT loads lazily inside generate_and_save (via
- _load_transformer_with_optional_streaming, which reads _pending_loras and
- fuses the deltas before quantization), so setting it on the constructed
- pipe — before the generate call — is sufficient. This is the exact
- mechanism the upstream `ltx-2-mlx generate --lora` CLI uses, and it works
- across every mode because all pipelines route DiT construction through that
- one method. No-op when no user LoRAs were requested.
+# What each adapter actually received (#12262), emitted as LORASTRATEGY: after
+# the render so PortOS stamps it on the history record. Plain data only — it
+# never holds a pipeline or transformer.
+_LORA_REPORT = None
+
+
+def _lora_report():
+ global _LORA_REPORT
+ if _LORA_REPORT is None:
+ from ltx_runtime_lora import AdapterReport
+ _LORA_REPORT = AdapterReport()
+ return _LORA_REPORT
+
+
+def _install_adapters(pipe, specs: list[tuple[str, float]], *, ic: bool) -> None:
+ """Apply user (and, for IC, reference) adapters at inference time (#12262).
+
+ Every pipeline builds its DiT through _load_transformer_with_optional_streaming,
+ which the runtime uses to merge `_pending_loras` into the weights and
+ re-quantize — lossy on a q4/q8 pack. ltx_runtime_lora replaces that hook so
+ each quantized projection keeps its packed weights and adds the delta at
+ inference, decided per projection from the loaded module type. A user
+ adapter still attaches on every load, so stage reloads keep it; IC adapters
+ attach once, before stage 1 (see install_pipeline_adapters).
"""
- if not specs:
+ if not specs and not ic:
return
- pipe._pending_loras = list(specs)
- names = ", ".join(f"{Path(p).name}@{s:g}" for p, s in specs)
- emit_status(f"Fusing {len(specs)} user LoRA(s): {names}")
+ from ltx_runtime_lora import install_pipeline_adapters
+ if specs:
+ names = ", ".join(f"{Path(p).name}@{s:g}" for p, s in specs)
+ emit_status(f"Applying {len(specs)} user LoRA(s): {names}")
+ install_pipeline_adapters(pipe, list(specs or []), report=_lora_report(), status=emit_status, ic=ic)
+
+
+def _apply_user_loras(pipe, specs: list[tuple[str, float]]) -> None:
+ """User LoRAs for every non-IC mode. No-op when none were requested."""
+ _install_adapters(pipe, specs, ic=False)
+
+
+def report_lora_strategy() -> None:
+ if _LORA_REPORT is not None:
+ _LORA_REPORT.emit()
def bind_output_fps(pipe, fps: float) -> None:
@@ -1955,9 +1980,11 @@ def run_ic_lora(args: argparse.Namespace) -> str:
inside the pipeline; we surface the resolution rule in the error so the
user knows to nudge the dimensions rather than the reference.
- User LoRAs stack ON TOP of the IC-LoRA rather than replacing it: the
- IC-LoRA rides `lora_paths` (fused by _fuse_loras before Stage 1) while
- user LoRAs ride the separate `_pending_loras` hook, so an
- Ingredients x Character stack composes.
+ IC-LoRA rides `lora_paths` (attached before Stage 1 only) while user
+ LoRAs attach on every transformer load, so an Ingredients x Character
+ stack composes and the clean Stage-2 reload keeps the user adapters
+ but not the IC one. Both apply at inference time on a quantized pack
+ rather than being merged and re-quantized (#12262).
- Ingredients is MULTI-reference (2-8) and conditions on stills, but the
reference channel is a video encoder end-to-end: iclora_utils probes each
reference with ffprobe and feeds it to the video VAE, whose reshape needs
@@ -2012,9 +2039,9 @@ def run_ic_lora(args: argparse.Namespace) -> str:
**optional_kwargs,
**_gemma_kwargs(args),
)
- # User LoRAs go through _pending_loras (fused at DiT load), NOT lora_paths —
- # so they stack with the IC-LoRA above instead of displacing it.
- _apply_user_loras(pipe, args.user_lora_specs)
+ # `lora_paths` stays populated: the constructor reads the reference
+ # downscale factor from it, and the installed _fuse_loras attaches it.
+ _install_adapters(pipe, args.user_lora_specs, ic=True)
bind_output_fps(pipe, args.fps)
emit_stage(1, 1, 1, "Loaded")
ref_names = ", ".join(Path(r).name for r in references)
@@ -2034,7 +2061,11 @@ def run_ic_lora(args: argparse.Namespace) -> str:
)
if args.ic_attention_strength is not None:
kwargs["conditioning_attention_strength"] = args.ic_attention_strength
- return _run_with_ltx_stepwise_preview(pipe, args, lambda: pipe.generate_and_save(**kwargs))
+ from ltx_runtime_lora import release_pipeline_adapters
+ try:
+ return _run_with_ltx_stepwise_preview(pipe, args, lambda: pipe.generate_and_save(**kwargs))
+ finally:
+ release_pipeline_adapters(pipe)
def maybe_strip_audio(output_path: str) -> None:
@@ -2429,7 +2460,7 @@ def main() -> NoReturn:
parse_text_encoder_config_overrides(args.text_encoder_config_json),
))
# Parse user LoRAs once up-front (strict validation surfaces bad input
- # before any model load). Each run_* sets pipe._pending_loras from this.
+ # before any model load). Each run_* installs them on its pipeline.
args.user_lora_specs = parse_user_loras(args.user_loras)
configure_negative_prompt(args.negative_prompt)
install_prompt_encode_markers()
@@ -2461,6 +2492,7 @@ def main() -> NoReturn:
# pipeline construction. No-op for a process whose mode never resolved a
# streaming policy.
report_streaming_policy()
+ report_lora_strategy()
if args.teacache and not _SPEED_PROFILE_REPORT.get("teacache") \
and "teacache" not in _SPEED_PROFILE_REPORT.get("degraded", []):
speed_profile_degrade(
diff --git a/scripts/generate_ltx2_ic_lora.test.js b/scripts/generate_ltx2_ic_lora.test.js
new file mode 100644
index 0000000000..6d9ccf4e95
--- /dev/null
+++ b/scripts/generate_ltx2_ic_lora.test.js
@@ -0,0 +1,19 @@
+import { execFileSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { describe, expect, it } from 'vitest';
+import { PY_SUBPROCESS_TIMEOUT_MS, PY_TEST_TIMEOUT_MS, resolveTestPython } from '../server/lib/testHelper.js';
+
+// Stdlib-only: the runtime is faked, so this runs on any interpreter.
+const python = resolveTestPython();
+const testScript = fileURLToPath(new URL('./generate_ltx2_ic_lora_test.py', import.meta.url));
+
+describe.skipIf(!python)('generate_ltx2.py IC-LoRA runner', () => {
+ it('enforces the reference contract and routes adapters around merge-and-requantize', () => {
+ expect(() => execFileSync(python, [testScript], {
+ encoding: 'utf8',
+ timeout: PY_SUBPROCESS_TIMEOUT_MS,
+ env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
+ stdio: 'pipe',
+ })).not.toThrow();
+ }, PY_TEST_TIMEOUT_MS);
+});
diff --git a/scripts/generate_ltx2_ic_lora_test.py b/scripts/generate_ltx2_ic_lora_test.py
index 17ff9a5fe5..4ba0dad72f 100644
--- a/scripts/generate_ltx2_ic_lora_test.py
+++ b/scripts/generate_ltx2_ic_lora_test.py
@@ -11,7 +11,7 @@ table in Python. These tests cover the Python layer: that it honors whatever
bounds it is handed (so a direct/script caller is still guarded), and that it
rejects nonsense bounds instead of trusting them.
-Run: python3 scripts/generate_ltx2_ic_lora_test.py
+Run: python3 scripts/generate_ltx2_ic_lora_test.py (CI: generate_ltx2_ic_lora.test.js)
"""
from __future__ import annotations
@@ -63,8 +63,20 @@ class RunIcLoraBoundsTest(unittest.TestCase):
recorded = self.recorded = {}
class FakeICLoraPipeline:
+ # The surface ltx_runtime_lora.install_pipeline_adapters replaces:
+ # the per-load hook, the pre-Stage-1 _fuse_loras, and the streaming
+ # flag that decides between inference-time and merged adapters.
+ low_ram_streaming = False
+
def __init__(self, **kwargs):
recorded["init"] = kwargs
+ self._lora_paths = list(kwargs.get("lora_paths") or [])
+
+ def _load_transformer_with_optional_streaming(self, path):
+ return path
+
+ def _fuse_loras(self):
+ raise AssertionError("upstream merge-and-requantize _fuse_loras must be replaced")
# `bind_output_fps` rebinds this to inject the output frame rate, so
# it has to exist even though nothing decodes here.
@@ -77,6 +89,8 @@ class RunIcLoraBoundsTest(unittest.TestCase):
# reads _pending_loras when it lazily loads the DiT inside this
# call, so that is when both channels must be populated.
recorded["pending_loras"] = getattr(self, "_pending_loras", None)
+ recorded["fuse_replaced"] = "_fuse_loras" in vars(self)
+ recorded["load_hooked"] = "_load_transformer_with_optional_streaming" in vars(self)
return kwargs.get("output_path")
ic_lora = types.ModuleType("ltx_pipelines_mlx.ic_lora")
@@ -216,19 +230,20 @@ class RunIcLoraBoundsTest(unittest.TestCase):
self.assertIn("does not exist", str(ctx.exception))
def test_ic_weight_rides_lora_paths_so_user_loras_stack(self):
- # The payoff of the Phase 1 channel split: the IC weight is fused via the
- # constructor's `lora_paths` (pre-Stage-1 `_fuse_loras`) while user LoRAs
- # go through `_pending_loras`, so an Ingredients x Character stack
- # COMPOSES rather than one displacing the other.
+ # The IC weight still rides the constructor's `lora_paths` (it carries
+ # the reference downscale factor) and attaches once, before Stage 1;
+ # user LoRAs attach on every transformer load, so an Ingredients x
+ # Character stack COMPOSES. Neither goes through the runtime's
+ # merge-and-requantize channels (#12262): `_pending_loras` stays empty
+ # and the pipeline's own _fuse_loras is replaced. The attach itself is
+ # proven on real quantized modules in ltx_runtime_lora_test.py.
user = [("/fake/character.safetensors", 0.9)]
args = self._args(references=self.refs[:2], user_loras=user)
self.helper.run_ic_lora(args)
- # IC weight → constructor `lora_paths` (fused pre-Stage-1 at strength 1.0:
- # the weight IS the mode, not a stylistic dial).
self.assertEqual(self.recorded["init"]["lora_paths"], [(self.weight, 1.0)])
- # User LoRAs → the SEPARATE `_pending_loras` hook. Both channels populated
- # at once is the assertion that matters; one list would mean replacement.
- self.assertEqual(self.recorded["pending_loras"], user)
+ self.assertIsNone(self.recorded["pending_loras"])
+ self.assertTrue(self.recorded["fuse_replaced"])
+ self.assertTrue(self.recorded["load_hooked"])
def test_every_reference_carries_the_strength_dial(self):
args = self._args(references=self.refs[:3])
diff --git a/scripts/ltx_runtime_lora.py b/scripts/ltx_runtime_lora.py
new file mode 100644
index 0000000000..d9b8163084
--- /dev/null
+++ b/scripts/ltx_runtime_lora.py
@@ -0,0 +1,346 @@
+#!/usr/bin/env python3
+"""Apply LTX LoRA adapters at inference time without re-quantizing packed weights (#12262).
+
+The pinned ltx-2-mlx / ltx-2.5-mlx runtimes fuse an adapter by dequantizing
+each q4/q8 projection, adding ``strength * B @ A`` and quantizing again. A
+rank-r delta is small next to the base weight, so the second quantization
+rounds most of it away and an Ingredients / IC adapter on a low-bit pack can
+render as the plain base model. This module keeps each quantized projection
+exactly as loaded and adds the delta in activation space instead:
+
+ quantized_linear(x) + (x @ A.T) @ (strength * B).T
+
+Float (unquantized) projections keep fusion: adding the delta to a float
+weight retains it, and that is the runtime's own high-precision path.
+
+Why a class swap instead of a wrapper module (the MiniMax H3 precedent in
+``minimax_h3_lora.py``): the LTX pipelines flatten ``dit.parameters()`` AFTER
+loading — the two-stage pipelines fuse their distilled LoRA that way, and block
+streaming rebinds weights with ``load_weights(strict=True)``. A wrapper moves
+``to_q.weight`` to ``to_q.base.weight``, which those paths would then silently
+miss. Swapping the instance to a cached subclass whose ``__call__`` adds the
+delta, with the adapter arrays held outside the parameter tree, keeps every
+parameter name and every packed array untouched.
+
+The math contract mirrors the runtime's ``apply_loras`` exactly — the same
+renamed keys, ``delta = strength * B @ A`` and no alpha rescale — so a q4 render
+and a bf16 render at one strength mean the same thing. Validation is stricter:
+``apply_loras`` silently skips a key it cannot place, and that silent no-op is
+the failure this module exists to remove, so an unplaceable tensor is refused.
+
+MLX is imported lazily, so the module (and the runners that import it) stay
+importable by a stdlib-only interpreter.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import sys
+from pathlib import Path
+from typing import Any, Callable, Iterable
+
+STRATEGY_RUNTIME = "runtime-delta"
+STRATEGY_FUSED = "fused-float"
+STRATEGY_MIXED = "runtime-delta+fused-float"
+# Block streaming rebinds one shared, compiled block per index, so an
+# activation-space delta has nowhere stable to live; those renders keep the
+# runtime's own per-block merge-and-requantize and say so.
+STRATEGY_REQUANTIZED = "merged-requantized"
+
+_A_SUFFIX = ".lora_A.weight"
+_B_SUFFIX = ".lora_B.weight"
+# The runtime's apply_loras never reads alpha; ignoring it here keeps strength
+# semantics identical across the quantized and float paths.
+_IGNORED_SUFFIXES = (".alpha",)
+
+_ADAPTERS_ATTR = "_portos_runtime_lora"
+_BASE_CLASS_ATTR = "_portos_runtime_lora_base"
+_RUNTIME_CLASSES: dict[type, type] = {}
+
+
+def _mlx():
+ import mlx.core as mx
+ import mlx.nn as nn
+
+ return mx, nn
+
+
+def _runtime_class(base_cls: type) -> type:
+ cls = _RUNTIME_CLASSES.get(base_cls)
+ if cls is not None:
+ return cls
+
+ def __call__(self, x):
+ output = base_cls.__call__(self, x)
+ for down, up in self.__dict__[_ADAPTERS_ATTR]:
+ output = output + ((x.astype(down.dtype) @ down.T) @ up.T).astype(output.dtype)
+ return output
+
+ cls = type(f"RuntimeLora{base_cls.__name__}", (base_cls,), {
+ "__call__": __call__,
+ _BASE_CLASS_ATTR: base_cls,
+ })
+ _RUNTIME_CLASSES[base_cls] = cls
+ return cls
+
+
+def _module_at(model, target: str):
+ node: Any = model
+ for part in target.split("."):
+ if not part:
+ return None
+ try:
+ node = node[int(part)] if part.isdigit() else getattr(node, part)
+ except (AttributeError, IndexError, KeyError, TypeError):
+ return None
+ return node
+
+
+def _logical_shape(module, quantized: bool) -> tuple[int, int]:
+ output_dims, stored_input_dims = (int(v) for v in module.weight.shape)
+ if not quantized:
+ return output_dims, stored_input_dims
+ bits = int(getattr(module, "bits", 0) or 0)
+ if bits <= 0 or 32 % bits:
+ raise ValueError(f"{type(module).__name__} reports unusable quantization bits {bits!r}")
+ return output_dims, stored_input_dims * 32 // bits
+
+
+def _group_tensors(label: str, tensors: dict) -> dict[str, dict[str, Any]]:
+ groups: dict[str, dict[str, Any]] = {}
+ unsupported: list[str] = []
+ for key, value in tensors.items():
+ if key.endswith(_A_SUFFIX):
+ groups.setdefault(key[: -len(_A_SUFFIX)], {})["A"] = value
+ elif key.endswith(_B_SUFFIX):
+ groups.setdefault(key[: -len(_B_SUFFIX)], {})["B"] = value
+ elif not key.endswith(_IGNORED_SUFFIXES):
+ unsupported.append(key)
+ if unsupported:
+ shown = ", ".join(sorted(unsupported)[:3])
+ more = f" (+{len(unsupported) - 3} more)" if len(unsupported) > 3 else ""
+ raise ValueError(
+ f"LoRA {label} carries tensors the LTX adapter contract cannot apply: {shown}{more}"
+ )
+ if not groups:
+ raise ValueError(f"LoRA {label} contains no lora_A/lora_B adapter tensors")
+ return groups
+
+
+def _plan(model, label: str, tensors: dict, strength: float) -> list[tuple]:
+ """Validate one adapter against the loaded model; mutate nothing."""
+ _mx_core, nn = _mlx()
+ if not math.isfinite(float(strength)):
+ raise ValueError(f"LoRA {label} strength must be finite, got {strength!r}")
+ planned = []
+ for target, pair in sorted(_group_tensors(label, tensors).items()):
+ if "A" not in pair or "B" not in pair:
+ missing = "lora_B" if "A" in pair else "lora_A"
+ raise ValueError(f"LoRA {label} target {target} has no {missing} tensor")
+ module = _module_at(model, target)
+ if module is None:
+ raise ValueError(
+ f"LoRA {label} targets {target}, which this transformer does not have — "
+ "the adapter was trained for a different model"
+ )
+ if isinstance(module, nn.QuantizedLinear):
+ quantized = True
+ elif isinstance(module, nn.Linear):
+ quantized = False
+ else:
+ raise ValueError(
+ f"LoRA {label} target {target} is a {type(module).__name__}, not a linear projection"
+ )
+ down, up = pair["A"], pair["B"]
+ if down.ndim != 2 or up.ndim != 2:
+ raise ValueError(f"LoRA {label} target {target} tensors must be rank 2")
+ output_dims, input_dims = _logical_shape(module, quantized)
+ rank = int(down.shape[0])
+ if rank <= 0 or tuple(down.shape) != (rank, input_dims) or tuple(up.shape) != (output_dims, rank):
+ raise ValueError(
+ f"LoRA {label} target {target} has lora_A {tuple(down.shape)} / lora_B "
+ f"{tuple(up.shape)}, which do not fit a ({output_dims}, {input_dims}) projection"
+ )
+ planned.append((target, module, quantized, down, up, float(strength)))
+ return planned
+
+
+def _apply(module, quantized: bool, down, up, strength: float) -> None:
+ mx, _nn = _mlx()
+ if quantized:
+ # The quantized kernel's scales carry the activation dtype the base
+ # projection computes in; hold the delta in it so nothing promotes.
+ dtype = module.scales.dtype
+ entry = (down.astype(dtype), (up.astype(mx.float32) * strength).astype(dtype))
+ if _ADAPTERS_ATTR not in module.__dict__:
+ module.__class__ = _runtime_class(type(module))
+ object.__setattr__(module, _ADAPTERS_ATTR, [])
+ module.__dict__[_ADAPTERS_ATTR].append(entry)
+ return
+ delta = (up.astype(mx.float32) * strength) @ down.astype(mx.float32)
+ module.weight = (module.weight.astype(mx.float32) + delta).astype(module.weight.dtype)
+
+
+def attach_adapters(model, adapters: Iterable[tuple[str, dict, float]]) -> list[dict]:
+ """Attach ``(label, renamed_tensors, strength)`` adapters to ``model`` in place.
+
+ Every adapter is validated before any module changes, so a malformed file
+ later in the list cannot leave an earlier one half-applied. Returns one
+ summary per adapter with the strategy its targets actually received.
+ """
+ plans = [(label, strength, _plan(model, label, tensors, strength)) for label, tensors, strength in adapters]
+ summaries = []
+ for label, strength, planned in plans:
+ quantized_targets = 0
+ for _target, module, quantized, down, up, scale in planned:
+ _apply(module, quantized, down, up, scale)
+ quantized_targets += int(quantized)
+ fused_targets = len(planned) - quantized_targets
+ strategy = (STRATEGY_MIXED if quantized_targets and fused_targets
+ else STRATEGY_RUNTIME if quantized_targets else STRATEGY_FUSED)
+ summaries.append({
+ "name": Path(label).name,
+ "strength": float(strength),
+ "strategy": strategy,
+ "quantizedTargets": quantized_targets,
+ "fusedTargets": fused_targets,
+ })
+ return summaries
+
+
+def runtime_adapter_targets(model) -> list[str]:
+ """Names of the projections currently carrying an activation-space delta."""
+ return [name for name, module in model.named_modules() if _ADAPTERS_ATTR in module.__dict__]
+
+
+def detach_runtime_adapters(model) -> int:
+ """Restore every swapped projection to its original class. Returns the count."""
+ count = 0
+ for _name, module in model.named_modules():
+ if _ADAPTERS_ATTR in module.__dict__:
+ module.__class__ = getattr(type(module), _BASE_CLASS_ATTR)
+ del module.__dict__[_ADAPTERS_ATTR]
+ count += 1
+ return count
+
+
+def load_ltx_adapter(path: str) -> dict:
+ """Read an adapter with the runtime's own key renaming, so targets match the DiT."""
+ from ltx_core_mlx.loader import LTXV_LORA_COMFY_RENAMING_MAP, SafetensorsStateDictLoader
+
+ return SafetensorsStateDictLoader().load(path, sd_ops=LTXV_LORA_COMFY_RENAMING_MAP).sd
+
+
+class AdapterReport:
+ """What each adapter actually received, for the render record (LORASTRATEGY:)."""
+
+ def __init__(self):
+ self.entries: list[dict] = []
+
+ def record(self, role: str, stage: str, summary: dict) -> bool:
+ """Returns True the first time this adapter is recorded."""
+ for entry in self.entries:
+ if entry["role"] == role and entry["name"] == summary["name"]:
+ if stage not in entry["stages"]:
+ entry["stages"].append(stage)
+ return False
+ self.entries.append({"role": role, "stages": [stage], **summary})
+ return True
+
+ def emit(self, stream=None) -> None:
+ if not self.entries:
+ return
+ print(f"LORASTRATEGY:{json.dumps({'adapters': self.entries})}", file=stream or sys.stderr, flush=True)
+
+
+def install_pipeline_adapters(
+ pipe,
+ user_specs: list[tuple[str, float]],
+ *,
+ report: AdapterReport,
+ status: Callable[[str], None],
+ ic: bool = False,
+ load_adapter: Callable[[str], dict] = load_ltx_adapter,
+) -> None:
+ """Route a pipeline's adapters through inference-time application.
+
+ User adapters attach on EVERY transformer load (the runtime's
+ ``_pending_loras`` contract), so a stage that reloads keeps them. With
+ ``ic=True`` the pipeline's ``_fuse_loras`` — called once, right before
+ stage 1 — attaches the IC adapter(s) to the loaded DiT instead; the clean
+ stage-2 reload goes back through the load hook and so carries the user
+ adapters but never the IC one.
+
+ Paths must already be local files: `parse_user_loras` requires that, and
+ the IC constructor resolves its repo ids before this runs.
+ """
+ if getattr(pipe, "low_ram_streaming", False):
+ if user_specs:
+ pipe._pending_loras = list(user_specs)
+ merged = [("user", "all", spec) for spec in user_specs]
+ if ic:
+ merged += [("ic", "stage-1", spec) for spec in getattr(pipe, "_lora_paths", [])]
+ for role, stage, (path, strength) in merged:
+ report.record(role, stage, _requantized_summary(path, strength))
+ if merged:
+ names = ", ".join(Path(path).name for _role, _stage, (path, _s) in merged)
+ status(
+ "Block streaming merges LoRA deltas into the quantized weights per block; a low-bit "
+ f"pack can lose part of their effect ({names})"
+ )
+ return
+
+ pipe._pending_loras = None
+ load_transformer = pipe._load_transformer_with_optional_streaming
+
+ def attach(dit, role: str, stage: str, specs) -> None:
+ loaded = [(path, load_adapter(path), strength) for path, strength in specs]
+ for summary in attach_adapters(dit, loaded):
+ if report.record(role, stage, summary):
+ status(
+ f"{role} LoRA {summary['name']}: {summary['strategy']} "
+ f"({summary['quantizedTargets']} quantized, {summary['fusedTargets']} float projection(s))"
+ )
+
+ def load_with_user_adapters(transformer_path):
+ dit = load_transformer(transformer_path)
+ if user_specs:
+ attach(dit, "user", "all", user_specs)
+ return dit
+
+ pipe._load_transformer_with_optional_streaming = load_with_user_adapters
+
+ if ic:
+ def attach_ic_adapters() -> None:
+ effective = getattr(pipe, "_effective_lora_paths", None)
+ specs = effective() if callable(effective) else list(pipe._lora_paths)
+ if specs:
+ attach(pipe.dit, "ic", "stage-1", specs)
+
+ pipe._fuse_loras = attach_ic_adapters
+
+
+def release_pipeline_adapters(pipe) -> None:
+ """Drop the transformer and every adapter this module attached to it.
+
+ Called when a render ends either way: the DiT is the one large object the
+ adapters hang off, and a failed or cancelled render must not keep it (or a
+ swapped class pointing at it) alive.
+ """
+ dit = getattr(pipe, "dit", None)
+ if dit is not None:
+ detach_runtime_adapters(dit)
+ pipe.dit = None
+ if hasattr(pipe, "_loaded"):
+ pipe._loaded = False
+
+
+def _requantized_summary(path: str, strength: float) -> dict:
+ return {
+ "name": Path(path).name,
+ "strength": float(strength),
+ "strategy": STRATEGY_REQUANTIZED,
+ "quantizedTargets": None,
+ "fusedTargets": None,
+ }
diff --git a/scripts/ltx_runtime_lora.test.js b/scripts/ltx_runtime_lora.test.js
new file mode 100644
index 0000000000..c189da4b47
--- /dev/null
+++ b/scripts/ltx_runtime_lora.test.js
@@ -0,0 +1,36 @@
+import { execFileSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { describe, expect, it } from 'vitest';
+import { PY_SUBPROCESS_TIMEOUT_MS, PY_TEST_TIMEOUT_MS, resolveTestPython } from '../server/lib/testHelper.js';
+
+// The contracts need real MLX quantized modules, so prefer the LTX runtimes'
+// own venvs and skip on a host with no MLX (CI's Linux/Windows runners).
+const candidates = [
+ join(homedir(), '.portos', 'ltx-2-mlx', '.venv', 'bin', 'python'),
+ join(homedir(), '.portos', 'ltx-2.5-mlx', '.venv', 'bin', 'python'),
+ resolveTestPython(),
+].filter((candidate, index, all) => candidate && all.indexOf(candidate) === index
+ && (!candidate.includes('/') || existsSync(candidate)));
+const python = candidates.find((candidate) => {
+ try {
+ execFileSync(candidate, ['-c', 'import mlx.core, mlx.nn'], { stdio: 'ignore' });
+ return true;
+ } catch {
+ return false;
+ }
+}) || null;
+const testScript = fileURLToPath(new URL('./ltx_runtime_lora_test.py', import.meta.url));
+
+describe.skipIf(!python)('ltx_runtime_lora.py', () => {
+ it('keeps quantized adapter deltas a merge loses and scopes IC adapters to stage 1', () => {
+ expect(() => execFileSync(python, [testScript], {
+ encoding: 'utf8',
+ timeout: PY_SUBPROCESS_TIMEOUT_MS,
+ env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
+ stdio: 'pipe',
+ })).not.toThrow();
+ }, PY_TEST_TIMEOUT_MS);
+});
diff --git a/scripts/ltx_runtime_lora_test.py b/scripts/ltx_runtime_lora_test.py
new file mode 100644
index 0000000000..62fc2097f8
--- /dev/null
+++ b/scripts/ltx_runtime_lora_test.py
@@ -0,0 +1,303 @@
+#!/usr/bin/env python3
+"""Contracts for ltx_runtime_lora (#12262). Needs MLX; run by ltx_runtime_lora.test.js.
+
+The fixtures are tiny real MLX modules rather than mocks: the defect is a
+numeric one (a re-quantization rounding a rank-r delta away), so only real
+packed weights can show that the runtime path keeps a delta the merge loses.
+
+Run: <mlx python> scripts/ltx_runtime_lora_test.py
+"""
+from __future__ import annotations
+
+import sys
+import unittest
+from pathlib import Path
+
+import mlx.core as mx
+import mlx.nn as nn
+import mlx.utils
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import ltx_runtime_lora as runtime # noqa: E402
+
+IN, OUT, RANK, GROUP, BITS = 256, 64, 4, 64, 4
+TARGET = "transformer_blocks.0.attn1.to_q"
+
+
+class Attention(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.to_q = nn.Linear(IN, OUT, bias=True)
+
+
+class Block(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.attn1 = Attention()
+
+
+class TinyDiT(nn.Module):
+ """The LTX key shape (`transformer_blocks.N.attn1.to_q`) at toy size."""
+
+ def __init__(self):
+ super().__init__()
+ self.transformer_blocks = [Block()]
+
+ def __call__(self, x):
+ return self.transformer_blocks[0].attn1.to_q(x)
+
+
+def fixture_model(*, quantize=True):
+ mx.random.seed(7)
+ model = TinyDiT()
+ model.update(mlx.utils.tree_map(lambda v: mx.random.normal(v.shape) * 0.02, model.parameters()))
+ if quantize:
+ nn.quantize(model, group_size=GROUP, bits=BITS, class_predicate=lambda p, m: p.endswith("to_q"))
+ mx.eval(model.parameters())
+ return model
+
+
+def adapter(target=TARGET, scale=1e-3, rank=RANK, out=OUT, inp=IN):
+ mx.random.seed(11)
+ return {
+ f"{target}.lora_A.weight": mx.random.normal((rank, inp)) * scale ** 0.5,
+ f"{target}.lora_B.weight": mx.random.normal((out, rank)) * scale ** 0.5,
+ }
+
+
+def delta_of(tensors, target=TARGET, strength=1.0):
+ return (tensors[f"{target}.lora_B.weight"] * strength) @ tensors[f"{target}.lora_A.weight"]
+
+
+def retention(observed, base, expected_delta):
+ """Fraction of the expected output delta present in `observed - base`."""
+ shift = (observed - base).flatten()
+ want = expected_delta.flatten()
+ return float((shift * want).sum() / (want * want).sum())
+
+
+def residual(observed, base, expected_delta):
+ """How far `observed` lands from `base + expected_delta`, relative to the delta."""
+ return float(mx.linalg.norm(observed - base - expected_delta) / mx.linalg.norm(expected_delta))
+
+
+class RetentionTest(unittest.TestCase):
+ def test_quantized_projection_keeps_a_delta_that_merge_and_requantize_loses(self):
+ model = fixture_model()
+ q = model.transformer_blocks[0].attn1.to_q
+ packed = (q.weight, q.scales, q.biases)
+ x = mx.random.normal((8, IN))
+ base = model(x)
+ # Per-element delta ~1e-4 against weights with std 0.02: a realistic
+ # adapter-to-weight ratio, and well under one q4 rounding step.
+ tensors = adapter(scale=1e-4)
+ expected = x @ delta_of(tensors).T
+
+ # The control: what the runtime's apply_loras does to a q4 weight.
+ dense = mx.dequantize(q.weight, q.scales, q.biases, group_size=GROUP, bits=BITS)
+ merged = mx.quantize(dense + delta_of(tensors), group_size=GROUP, bits=BITS)
+ control = mx.quantized_matmul(x, *merged, transpose=True, group_size=GROUP, bits=BITS) + q.bias
+
+ [summary] = runtime.attach_adapters(model, [("ingredients.safetensors", tensors, 1.0)])
+ applied = model(x)
+ mx.eval(base, applied, control, expected)
+
+ self.assertGreater(retention(applied, base, expected), 0.98)
+ self.assertLess(residual(applied, base, expected), 0.02)
+ # The merge keeps a small fraction of the delta and buries it in
+ # re-quantization noise larger than the delta itself.
+ self.assertLess(retention(control, base, expected), 0.25)
+ self.assertGreater(residual(control, base, expected), 1.0)
+ self.assertEqual(summary["strategy"], runtime.STRATEGY_RUNTIME)
+ self.assertEqual((summary["quantizedTargets"], summary["fusedTargets"]), (1, 0))
+ # The packed base and its metadata are the very arrays that were loaded.
+ for before, after in zip(packed, (q.weight, q.scales, q.biases)):
+ self.assertIs(before, after)
+ self.assertEqual((q.group_size, q.bits), (GROUP, BITS))
+
+ def test_parameter_tree_is_unchanged_so_later_weight_rebinds_keep_the_delta(self):
+ model = fixture_model()
+ names = sorted(k for k, _ in mlx.utils.tree_flatten(model.parameters()))
+ tensors = adapter()
+ runtime.attach_adapters(model, [("user.safetensors", tensors, 0.5)])
+ self.assertEqual(sorted(k for k, _ in mlx.utils.tree_flatten(model.parameters())), names)
+
+ # The two-stage pipelines fuse their distilled LoRA by flattening the
+ # parameters and calling load_weights(strict) — the adapter must ride
+ # through that rebind rather than being silently dropped by it.
+ x = mx.random.normal((4, IN))
+ model.load_weights(mlx.utils.tree_flatten(model.parameters()), strict=True)
+ q = model.transformer_blocks[0].attn1.to_q
+ plain = mx.quantized_matmul(x, q.weight, q.scales, q.biases, transpose=True,
+ group_size=GROUP, bits=BITS) + q.bias
+ observed = model(x)
+ mx.eval(plain, observed)
+ self.assertGreater(retention(observed, plain, x @ delta_of(tensors, strength=0.5).T), 0.98)
+ self.assertEqual(runtime.runtime_adapter_targets(model), [TARGET])
+
+ def test_float_projection_keeps_fusion(self):
+ model = fixture_model(quantize=False)
+ x = mx.random.normal((4, IN))
+ base = model(x)
+ tensors = adapter()
+ [summary] = runtime.attach_adapters(model, [("user.safetensors", tensors, 1.0)])
+ observed = model(x)
+ mx.eval(base, observed)
+ self.assertEqual(summary["strategy"], runtime.STRATEGY_FUSED)
+ self.assertEqual(runtime.runtime_adapter_targets(model), [])
+ self.assertGreater(retention(observed, base, x @ delta_of(tensors).T), 0.98)
+
+ def test_detach_restores_the_plain_quantized_projection(self):
+ model = fixture_model()
+ x = mx.random.normal((4, IN))
+ base = model(x)
+ runtime.attach_adapters(model, [("user.safetensors", adapter(), 1.0)])
+ self.assertEqual(runtime.detach_runtime_adapters(model), 1)
+ restored = model(x)
+ mx.eval(base, restored)
+ self.assertIs(type(model.transformer_blocks[0].attn1.to_q), nn.QuantizedLinear)
+ self.assertTrue(mx.array_equal(base, restored).item())
+
+
+class ValidationTest(unittest.TestCase):
+ def assert_refused(self, tensors, pattern):
+ model = fixture_model()
+ good = ("first.safetensors", adapter(), 1.0)
+ with self.assertRaisesRegex(ValueError, pattern):
+ runtime.attach_adapters(model, [good, ("bad.safetensors", tensors, 1.0)])
+ # Validation precedes mutation: the valid adapter listed first was not attached.
+ self.assertEqual(runtime.runtime_adapter_targets(model), [])
+
+ def test_refuses_a_target_the_transformer_does_not_have(self):
+ self.assert_refused(adapter("transformer_blocks.9.attn1.to_q"), "does not have")
+
+ def test_refuses_a_shape_that_does_not_fit_the_projection(self):
+ self.assert_refused(adapter(inp=IN // 2), "do not fit")
+
+ def test_refuses_half_of_a_pair(self):
+ tensors = adapter()
+ tensors.pop(f"{TARGET}.lora_B.weight")
+ self.assert_refused(tensors, "no lora_B")
+
+ def test_refuses_tensors_outside_the_adapter_contract(self):
+ tensors = adapter()
+ tensors[f"{TARGET}.diff_b"] = mx.zeros((OUT,))
+ self.assert_refused(tensors, "cannot apply")
+
+ def test_refuses_a_non_linear_target(self):
+ tensors = adapter("transformer_blocks.0.attn1")
+ self.assert_refused(tensors, "not a linear projection")
+
+ def test_ignores_alpha_like_the_runtime_merge(self):
+ model = fixture_model()
+ tensors = {**adapter(), f"{TARGET}.alpha": mx.array(8.0)}
+ [summary] = runtime.attach_adapters(model, [("user.safetensors", tensors, 1.0)])
+ self.assertEqual(summary["quantizedTargets"], 1)
+
+
+class FakeICLoraPipeline:
+ """The upstream ICLoraPipeline surface install_pipeline_adapters drives.
+
+ Mirrors the pinned flow: generate() loads the DiT, calls _fuse_loras()
+ once, denoises stage 1, then (distilled, non-dev) reloads a clean
+ transformer for stage 2 through the same load method.
+ """
+
+ def __init__(self, lora_paths, *, low_ram_streaming=False, fail_in_stage_2=False):
+ self._lora_paths = list(lora_paths)
+ self.low_ram_streaming = low_ram_streaming
+ self.fail_in_stage_2 = fail_in_stage_2
+ self.dit = None
+ self._loaded = False
+ self.stage_targets = {}
+ self.stage_outputs = {}
+
+ def _load_transformer_with_optional_streaming(self, _path):
+ if getattr(self, "_pending_loras", None):
+ raise AssertionError("runtime path must not hand LoRAs to the merge-and-requantize loader")
+ return fixture_model()
+
+ def _fuse_loras(self):
+ raise AssertionError("upstream merge-and-requantize _fuse_loras must not run")
+
+ def _reload_clean_transformer(self):
+ self.dit = None
+ self.dit = self._load_transformer_with_optional_streaming("transformer.safetensors")
+
+ def generate(self, x):
+ self.dit = self._load_transformer_with_optional_streaming("transformer.safetensors")
+ self._loaded = True
+ self._fuse_loras()
+ self.stage_targets[1] = runtime.runtime_adapter_targets(self.dit)
+ self.stage_outputs[1] = self.dit(x)
+ self._reload_clean_transformer()
+ if self.fail_in_stage_2:
+ raise RuntimeError("cancelled during stage 2")
+ self.stage_targets[2] = runtime.runtime_adapter_targets(self.dit)
+ self.stage_outputs[2] = self.dit(x)
+
+
+class TwoStageContractTest(unittest.TestCase):
+ def setUp(self):
+ self.ic_tensors = adapter(scale=1e-3)
+ self.user_tensors = adapter("transformer_blocks.0.attn1.to_q", scale=4e-3)
+ self.files = {"ingredients.safetensors": self.ic_tensors, "character.safetensors": self.user_tensors}
+ self.report = runtime.AdapterReport()
+ self.statuses = []
+
+ def install(self, pipe):
+ runtime.install_pipeline_adapters(
+ pipe, [("character.safetensors", 0.8)], report=self.report, status=self.statuses.append,
+ ic=True, load_adapter=self.files.__getitem__,
+ )
+
+ def test_ingredients_conditions_stage_1_and_never_leaks_into_the_clean_refine(self):
+ pipe = FakeICLoraPipeline([("ingredients.safetensors", 1.0)])
+ self.install(pipe)
+ x = mx.random.normal((4, IN))
+ pipe.generate(x)
+
+ clean = fixture_model()(x)
+ user_only = x @ delta_of(self.user_tensors, strength=0.8).T
+ both = user_only + x @ delta_of(self.ic_tensors).T
+ mx.eval(clean, user_only, both, *pipe.stage_outputs.values())
+ # Same projection carries both adapters in stage 1; stage 2 is the clean
+ # reload plus the user adapter only.
+ self.assertEqual(pipe.stage_targets, {1: [TARGET], 2: [TARGET]})
+ self.assertGreater(retention(pipe.stage_outputs[1], clean, both), 0.98)
+ self.assertGreater(retention(pipe.stage_outputs[2], clean, user_only), 0.98)
+ stage2_shift = (pipe.stage_outputs[2] - clean - user_only).flatten()
+ self.assertLess(float(mx.abs(stage2_shift).max()), 1e-3)
+
+ by_role = {entry["role"]: entry for entry in self.report.entries}
+ self.assertEqual(by_role["ic"]["stages"], ["stage-1"])
+ self.assertEqual(by_role["user"]["stages"], ["all"])
+ self.assertEqual(by_role["ic"]["strategy"], runtime.STRATEGY_RUNTIME)
+ self.assertEqual(by_role["user"]["strength"], 0.8)
+
+ def test_failure_mid_render_releases_the_transformer_and_its_adapters(self):
+ pipe = FakeICLoraPipeline([("ingredients.safetensors", 1.0)], fail_in_stage_2=True)
+ self.install(pipe)
+ with self.assertRaises(RuntimeError):
+ try:
+ pipe.generate(mx.random.normal((4, IN)))
+ finally:
+ held = pipe.dit
+ runtime.release_pipeline_adapters(pipe)
+ self.assertIsNone(pipe.dit)
+ self.assertFalse(pipe._loaded)
+ self.assertEqual(runtime.runtime_adapter_targets(held), [])
+ self.assertIs(type(held.transformer_blocks[0].attn1.to_q), nn.QuantizedLinear)
+
+ def test_block_streaming_keeps_the_runtime_merge_and_says_so(self):
+ pipe = FakeICLoraPipeline([("ingredients.safetensors", 1.0)], low_ram_streaming=True)
+ original_fuse = pipe._fuse_loras
+ self.install(pipe)
+ self.assertEqual(pipe._pending_loras, [("character.safetensors", 0.8)])
+ self.assertEqual(pipe._fuse_loras, original_fuse)
+ self.assertEqual({e["strategy"] for e in self.report.entries}, {runtime.STRATEGY_REQUANTIZED})
+ self.assertTrue(any("Block streaming" in s for s in self.statuses))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/scripts/upscale_ltx25.py b/scripts/upscale_ltx25.py
index 49ee05ec08..ed85e8cc4e 100755
--- a/scripts/upscale_ltx25.py
+++ b/scripts/upscale_ltx25.py
@@ -29,12 +29,16 @@ gated model card:
- The schedule is the fixed distilled one: `DISTILLED_SIGMAS` is 8 steps.
Passing no step count selects exactly that, which is why this runner
exposes no steps flag.
- - Quantization-safe fusion is the runtime's own contract: `apply_loras()`
- dequantizes an int4/int8 weight, adds the delta, and re-quantizes. What it
- does NOT do is complain when an adapter addresses keys the transformer does
- not have — every delta is simply missing, the fusion is a silent no-op, and
- the render is the plain base model. `assert_adapter_fuses` is the guard for
- that: it is the "loads but produces garbage" case #6512 names.
+ - The runtime's `apply_loras()` dequantizes an int4/int8 weight, adds the
+ delta and re-quantizes, which rounds most of a rank-r delta away on a
+ low-bit pack (#12262). This runner instead applies the adapters at
+ inference through `ltx_runtime_lora`: quantized projections keep their
+ packed weights and add the delta in activation space, float ones fuse.
+ `apply_loras()` also never complains when an adapter addresses keys the
+ transformer does not have — the delta simply vanishes and the render is the
+ plain base model, the "loads but produces garbage" case #6512 names.
+ `assert_adapter_targets_resolve` refuses that from the headers alone, and
+ the runtime attach re-checks every target and shape on the loaded model.
- `reference_downscale_factor` is read from the adapter's safetensors
`__metadata__` (`iclora_utils.read_lora_reference_downscale_factor`) and
enforced against the STAGE-1 dims, which are half the output. It is
@@ -71,6 +75,12 @@ from _upscale_contract import ( # noqa: E402
reference_downscale_factor,
validate_args,
)
+# Stdlib-only at import; MLX loads only when an adapter is attached.
+from ltx_runtime_lora import ( # noqa: E402
+ AdapterReport,
+ install_pipeline_adapters,
+ release_pipeline_adapters,
+)
# The 2.5 pack's own prompt conditioner. Its absence is a hard refusal rather
# than a fallback — see the module docstring.
@@ -198,6 +208,10 @@ def make_pipeline(pipeline_cls, model_dir: Path, transformer_path: Path, lora_pa
runs (`skip_stage_2`), reading a ~1 GB weight into memory that then sits
beside the DiT for the whole render. Pre-seeding the slot makes `load()`
skip it; nothing under `skip_stage_2` dereferences the upsampler.
+
+ The adapters are applied at inference rather than merged and re-quantized
+ (#12262): `install_pipeline_adapters` replaces this instance's `_fuse_loras`,
+ and what each adapter received lands in `LORA_REPORT`.
"""
class PackLayoutPipeline(pipeline_cls):
def load(self):
@@ -207,11 +221,16 @@ def make_pipeline(pipeline_cls, model_dir: Path, transformer_path: Path, lora_pa
self.upsampler = _UNUSED_UPSAMPLER
super().load()
- return PackLayoutPipeline(model_dir=str(model_dir), lora_paths=list(lora_paths))
+ pipe = PackLayoutPipeline(model_dir=str(model_dir), lora_paths=list(lora_paths))
+ install_pipeline_adapters(pipe, [], report=LORA_REPORT, status=lambda message: log(f"STATUS:{message}"), ic=True)
+ return pipe
_UNUSED_UPSAMPLER = object()
+# What each adapter received, emitted as LORASTRATEGY: after the render.
+LORA_REPORT = AdapterReport()
+
def transformer_weight_keys(transformer_path: Path) -> "set[str]":
"""The transformer's fusable parameter names, read from its header alone.
@@ -235,17 +254,18 @@ def transformer_weight_keys(transformer_path: Path) -> "set[str]":
}
-def assert_adapter_fuses(adapter_path: str, model_keys, rename) -> int:
- """Refuse an adapter whose tensors address none of the transformer's weights.
+def assert_adapter_targets_resolve(adapter_path: str, model_keys, rename) -> int:
+ """Refuse an adapter with any tensor pair that addresses no transformer weight.
This is the failure #6512 calls out: `apply_loras` reports nothing when a
- key does not match — every delta is simply absent, the fusion is a silent
- no-op, and the render is the un-adapted base model dressed as an upscale.
- Only headers are read, so the check costs no tensor I/O and runs BEFORE the
+ key does not match — the delta is simply absent, and an adapter that misses
+ every key renders the un-adapted base model dressed as an upscale. The
+ runtime attach (#12262) refuses ANY unplaceable target, so this header-only
+ pre-check holds the same line and fails in milliseconds, BEFORE the
pipeline loads anything.
- Returns the number of weights the adapter will actually fuse into, so the
- render records real coverage instead of "it loaded".
+ Returns the number of weights the adapter applies to, so the render records
+ real coverage instead of "it loaded".
"""
header = read_safetensors_header(adapter_path)
if not header:
@@ -258,14 +278,28 @@ def assert_adapter_fuses(adapter_path: str, model_keys, rename) -> int:
"Could not read the LTX-2.5 transformer's weight names, so there is no way to tell whether "
"the upscale adapter would fuse into anything. Repair the model in Video Gen."
)
- renamed = [rename(name) for name in header if name != "__metadata__"]
- matched = lora_target_keys(name for name in renamed if name) & set(model_keys)
+ renamed = [name for name in (rename(raw) for raw in header if raw != "__metadata__") if name]
+ targets = lora_target_keys(renamed)
+ matched = targets & set(model_keys)
if not matched:
raise SystemExit(
- "The upscale adapter's tensors do not address any weight in this transformer, so fusing it "
+ "The upscale adapter's tensors do not address any weight in this transformer, so applying it "
"would be a no-op and the 'upscale' would be an un-adapted render. The adapter and the "
"LTX-2.5 pack are mismatched — repair both in Video Gen."
)
+ half_pairs = {
+ name.rsplit(".", 2)[0] + ".weight"
+ for name in renamed
+ if name.endswith((".lora_A.weight", ".lora_B.weight"))
+ } - targets
+ unplaced = sorted((targets - matched) | half_pairs)
+ if unplaced:
+ shown = ", ".join(unplaced[:3]) + (f" (+{len(unplaced) - 3} more)" if len(unplaced) > 3 else "")
+ raise SystemExit(
+ f"The upscale adapter addresses {len(unplaced)} weight(s) this transformer cannot take ({shown}), "
+ "so part of it would silently not apply. The adapter and the LTX-2.5 pack are mismatched — "
+ "repair both in Video Gen."
+ )
return len(matched)
@@ -296,12 +330,12 @@ def main() -> None:
transformer_path, pack_loras = resolve_transformer_layout(model_dir)
layout = " + ".join([transformer_path.name, *(Path(path).name for path, _ in pack_loras)])
log(f"STATUS:Transformer layout: {layout}")
- fused = assert_adapter_fuses(
+ targets = assert_adapter_targets_resolve(
args.ic_lora_path,
transformer_weight_keys(transformer_path),
LTXV_LORA_COMFY_RENAMING_MAP.apply_to_key,
)
- log(f"STATUS:Adapter fuses into {fused} transformer weights")
+ log(f"STATUS:Adapter applies to {targets} transformer weights")
log("STAGE:load-pipeline")
log(f"STATUS:Loading LTX-2.5 MLX upscale pipeline ({args.width}x{args.height}, {args.num_frames} frames)")
@@ -316,17 +350,21 @@ def main() -> None:
# output (see `conditioned_stage_request`). No stage-1 step count:
# `DISTILLED_SIGMAS` IS the distilled schedule (8 steps), and passing a
# count would truncate it.
- pipe.generate_and_save(
- prompt=args.prompt,
- output_path=str(output),
- video_conditioning=[(reference, REFERENCE_STRENGTH) for reference in args.ic_reference],
- height=request_height,
- width=request_width,
- num_frames=args.num_frames,
- frame_rate=args.fps,
- seed=args.seed,
- skip_stage_2=True,
- )
+ try:
+ pipe.generate_and_save(
+ prompt=args.prompt,
+ output_path=str(output),
+ video_conditioning=[(reference, REFERENCE_STRENGTH) for reference in args.ic_reference],
+ height=request_height,
+ width=request_width,
+ num_frames=args.num_frames,
+ frame_rate=args.fps,
+ seed=args.seed,
+ skip_stage_2=True,
+ )
+ finally:
+ release_pipeline_adapters(pipe)
+ LORA_REPORT.emit()
if not output.is_file():
raise SystemExit(f"The LTX-2.5 upscale completed but did not write {output}.")
log(f"STATUS:Upscaled {output.name} ({args.width}x{args.height}, {args.num_frames} frames)")
diff --git a/scripts/upscale_ltx25.test.js b/scripts/upscale_ltx25.test.js
index 162a4ef11c..149db04957 100644
--- a/scripts/upscale_ltx25.test.js
+++ b/scripts/upscale_ltx25.test.js
@@ -258,7 +258,7 @@ describe.skipIf(!pyBin)('upscale_ltx25.py — adapter metadata (#6512)', () => {
const modelKeys = JSON.stringify(['transformer_blocks.0.attn1.to_q.weight']);
// `rename` is the runtime's own SDOps map in production; identity here keeps
// the test off the MLX wheel while exercising the same intersection.
- expect(call(`runner.assert_adapter_fuses(${JSON.stringify(foreign)}, ${modelKeys}, lambda name: name)`))
+ expect(call(`runner.assert_adapter_targets_resolve(${JSON.stringify(foreign)}, ${modelKeys}, lambda name: name)`))
.toMatch(/^REJECTED:.*do not address any weight/);
const matching = join(scratch, 'matching.safetensors');
@@ -266,7 +266,32 @@ describe.skipIf(!pyBin)('upscale_ltx25.py — adapter metadata (#6512)', () => {
'transformer_blocks.0.attn1.to_q.lora_A.weight': {},
'transformer_blocks.0.attn1.to_q.lora_B.weight': {},
});
- expect(call(`runner.assert_adapter_fuses(${JSON.stringify(matching)}, ${modelKeys}, lambda name: name)`)).toBe('1');
+ expect(call(`runner.assert_adapter_targets_resolve(${JSON.stringify(matching)}, ${modelKeys}, lambda name: name)`)).toBe('1');
+ }, PY_TEST_TIMEOUT_MS);
+
+ // #12262: the runtime attach refuses ANY unplaceable target, because a
+ // partially-applied adapter renders plausibly while silently missing part
+ // of its effect. The header pre-check holds the same line before a load.
+ it('refuses an adapter that only partly addresses the transformer, or carries a half pair', () => {
+ const modelKeys = JSON.stringify(['transformer_blocks.0.attn1.to_q.weight']);
+ const partial = join(scratch, 'partial.safetensors');
+ writeSafetensors(partial, {
+ 'transformer_blocks.0.attn1.to_q.lora_A.weight': {},
+ 'transformer_blocks.0.attn1.to_q.lora_B.weight': {},
+ 'transformer_blocks.7.attn1.to_q.lora_A.weight': {},
+ 'transformer_blocks.7.attn1.to_q.lora_B.weight': {},
+ });
+ expect(call(`runner.assert_adapter_targets_resolve(${JSON.stringify(partial)}, ${modelKeys}, lambda name: name)`))
+ .toMatch(/^REJECTED:.*1 weight\(s\).*transformer_blocks\.7\.attn1\.to_q\.weight/);
+
+ const halfPair = join(scratch, 'half-pair.safetensors');
+ writeSafetensors(halfPair, {
+ 'transformer_blocks.0.attn1.to_q.lora_A.weight': {},
+ 'transformer_blocks.0.attn1.to_q.lora_B.weight': {},
+ 'transformer_blocks.0.attn1.to_k.lora_A.weight': {},
+ });
+ expect(call(`runner.assert_adapter_targets_resolve(${JSON.stringify(halfPair)}, ${modelKeys}, lambda name: name)`))
+ .toMatch(/^REJECTED:.*to_k\.weight/);
}, PY_TEST_TIMEOUT_MS);
// An unreadable transformer means the guard cannot answer its own question.
@@ -274,7 +299,7 @@ describe.skipIf(!pyBin)('upscale_ltx25.py — adapter metadata (#6512)', () => {
// would defeat the guard — so it refuses on its own blindness instead.
it('refuses when the transformer weight names could not be read at all', () => {
const matching = join(scratch, 'matching.safetensors');
- expect(call(`runner.assert_adapter_fuses(${JSON.stringify(matching)}, set(), lambda name: name)`))
+ expect(call(`runner.assert_adapter_targets_resolve(${JSON.stringify(matching)}, set(), lambda name: name)`))
.toMatch(/^REJECTED:.*Could not read the LTX-2\.5 transformer/);
}, PY_TEST_TIMEOUT_MS);
});
diff --git a/server/services/videoGen/generateVideoHelpers.js b/server/services/videoGen/generateVideoHelpers.js
index f11f3b6c8b..a3e424ca40 100644
--- a/server/services/videoGen/generateVideoHelpers.js
+++ b/server/services/videoGen/generateVideoHelpers.js
@@ -217,6 +217,14 @@ export function makeVideoGenLineHandler({ job, jobId, pythonNoiseRe }) {
return false;
}
}
+ if (line.startsWith(LORA_STRATEGY_LINE_PREFIX)) {
+ const report = parseLoraStrategyLine(line);
+ // Malformed payload — fall through to raw-logging (same as RUNTIME: above).
+ if (!report) return false;
+ job.loraStrategy = report;
+ console.log(`🧩 LoRA strategy [${jobId.slice(0, 8)}] ${report.adapters.map((a) => `${a?.name || '?'}=${a?.strategy || '?'}`).join(', ')}`);
+ return true;
+ }
// Heartbeat for the queue's idle watchdog (see imageGen/local.js).
videoGenEvents.emit('activity', { generationId: jobId });
if (line.startsWith('STATUS:')) {
@@ -380,6 +388,28 @@ export function parseRuntimeFingerprintLine(line) {
}
}
+// How each LoRA adapter was applied (#12262): `scripts/ltx_runtime_lora.py
+// AdapterReport.emit`. A quantized pack can apply a delta at inference or merge
+// and re-quantize it (which can erase most of it), so the record keeps the
+// outcome beside the request. Shared by the render path and the upscale job.
+export const LORA_STRATEGY_LINE_PREFIX = 'LORASTRATEGY:';
+
+/**
+ * The `{ adapters: [...] }` payload of a `LORASTRATEGY:<json>` line, or null
+ * when the line is not one or its payload is malformed.
+ * @param {string} line - one trimmed child-output line
+ * @returns {{ adapters: object[] }|null}
+ */
+export function parseLoraStrategyLine(line) {
+ if (typeof line !== 'string' || !line.startsWith(LORA_STRATEGY_LINE_PREFIX)) return null;
+ try {
+ const report = JSON.parse(line.slice(LORA_STRATEGY_LINE_PREFIX.length));
+ return Array.isArray(report?.adapters) ? report : null;
+ } catch {
+ return null;
+ }
+}
+
/**
* Render a runtime fingerprint (either the `RUNTIME:` payload a helper script
* emits — `{ runtime, versions, chip, os, python }` — or the Node-side
@@ -855,6 +885,10 @@ export async function finalizeGeneratedVideo({ job, jobId, outputPath, filename,
// decision, and (when active) the render's peak MLX memory. Absent on
// a resident-by-default render and on runners that don't report one.
...(job.streamingPolicy ? { streamingPolicyApplied: job.streamingPolicy } : {}),
+ // How each LoRA adapter was actually applied (#12262) — at inference
+ // beside the packed weights, fused into float ones, or merged and
+ // re-quantized under block streaming. Absent on adapter-free renders.
+ ...(job.loraStrategy ? { loraStrategyApplied: job.loraStrategy } : {}),
});
return history;
});
diff --git a/server/services/videoGen/generateVideoHelpers.test.js b/server/services/videoGen/generateVideoHelpers.test.js
index 64e1b974ac..ceffe83db1 100644
--- a/server/services/videoGen/generateVideoHelpers.test.js
+++ b/server/services/videoGen/generateVideoHelpers.test.js
@@ -281,6 +281,17 @@ describe('makeVideoGenLineHandler', () => {
expect(handle('SPEEDPROFILE:{not json')).toBe(false);
expect(job.speedProfile).toBeUndefined();
});
+
+ // #12262 — how each LoRA adapter was actually applied on this render.
+ it('LORASTRATEGY:<json> → stamps job.loraStrategy; a malformed payload falls through', () => {
+ const report = { adapters: [{ role: 'ic', name: 'ingredients.safetensors', strategy: 'runtime-delta', stages: ['stage-1'], strength: 1, quantizedTargets: 480, fusedTargets: 0 }] };
+ expect(handle('LORASTRATEGY:{not json')).toBe(false);
+ expect(handle('LORASTRATEGY:{"adapters":"nope"}')).toBe(false);
+ expect(job.loraStrategy).toBeUndefined();
+ expect(handle(`LORASTRATEGY:${JSON.stringify(report)}`)).toBe(true);
+ expect(job.loraStrategy).toEqual(report);
+ expect(sse).not.toHaveBeenCalled();
+ });
});
describe('finalizeGeneratedVideo runtime persistence', () => {
@@ -365,6 +376,16 @@ describe('finalizeGeneratedVideo runtime persistence', () => {
expect(saved[0].speedProfileApplied).toEqual(applied);
});
+ it('persists the LoRA strategy the runner reported, and omits it when none was (#12262)', async () => {
+ const report = { adapters: [{ role: 'user', name: 'character.safetensors', strategy: 'runtime-delta', stages: ['all'], strength: 0.8, quantizedTargets: 480, fusedTargets: 0 }] };
+ let saved = null;
+ const mutateHistory = async (fn) => { saved = await fn([]); return saved; };
+ await finalizeGeneratedVideo({ ...baseCtx({ id: 'job-abcdef12', clients: [], loraStrategy: report }), mutateHistory });
+ expect(saved[0].loraStrategyApplied).toEqual(report);
+ await finalizeGeneratedVideo({ ...baseCtx({ id: 'job-abcdef12', clients: [] }), mutateHistory });
+ expect('loraStrategyApplied' in saved[0]).toBe(false);
+ });
+
it('omits speedProfileApplied when the runner reported none (absent sentinel)', async () => {
const job = { id: 'job-abcdef12', clients: [] };
let saved = null;
diff --git a/server/services/videoGen/local.test.js b/server/services/videoGen/local.test.js
index 8514a9dc57..0fb01ff993 100644
--- a/server/services/videoGen/local.test.js
+++ b/server/services/videoGen/local.test.js
@@ -787,6 +787,29 @@ describe('stitchVideos — history provenance', () => {
expect(stitched).not.toHaveProperty('streamingPolicyApplied');
});
});
+
+ // LoRA application strategy (#12262). Each chunk's child decides whether an
+ // adapter applied at inference or was merged and re-quantized, so a stitched
+ // clip may claim a strategy only when every chunk agrees on it.
+ describe('LoRA strategy outcome', () => {
+ const applied = (strategy, quantizedTargets = 480) => ({
+ loraStrategyApplied: {
+ adapters: [{ role: 'user', name: 'character.safetensors', strategy, stages: ['all'], strength: 0.8, quantizedTargets, fusedTargets: 0 }],
+ },
+ });
+
+ it('inherits a unanimous outcome even when target counts differ', async () => {
+ const stitched = await stitchHistory(applied('runtime-delta'), applied('runtime-delta', 479));
+
+ expect(stitched.loraStrategyApplied).toEqual(applied('runtime-delta').loraStrategyApplied);
+ });
+
+ it('omits the outcome when the chunks applied the adapter differently', async () => {
+ const stitched = await stitchHistory(applied('runtime-delta'), applied('merged-requantized'));
+
+ expect(stitched).not.toHaveProperty('loraStrategyApplied');
+ });
+ });
});
describe('extractLastFrame — anchor selection', () => {
diff --git a/server/services/videoGen/stitchVideos.js b/server/services/videoGen/stitchVideos.js
index 0e4bf5a89a..d851237fe8 100644
--- a/server/services/videoGen/stitchVideos.js
+++ b/server/services/videoGen/stitchVideos.js
@@ -55,6 +55,22 @@ const unanimousStreamingPolicyOutcome = (videos) => {
: {};
};
+// Fold the per-chunk LORASTRATEGY: reports into one claim, or none (#12262).
+// Unanimity is on each adapter's name + strategy, so a chain whose chunks all
+// applied the same adapters the same way keeps the claim and a chain that
+// mixed strategies makes none.
+const loraStrategySignature = (applied) => JSON.stringify(
+ (applied?.adapters || []).map((a) => [a?.role, a?.name, a?.strategy]),
+);
+const unanimousLoraStrategyOutcome = (videos) => {
+ const first = videos[0]?.loraStrategyApplied;
+ if (first === undefined) return {};
+ const signature = loraStrategySignature(first);
+ return videos.every((v) => v?.loraStrategyApplied && loraStrategySignature(v.loraStrategyApplied) === signature)
+ ? { loraStrategyApplied: first }
+ : {};
+};
+
export async function stitchVideos(videoIds, opts = {}) {
const {
id = randomUUID(),
@@ -282,6 +298,7 @@ export async function stitchVideos(videoIds, opts = {}) {
// false fidelity claim this field exists to prevent.
...unanimousDraftDecodeOutcome(videos),
...unanimousStreamingPolicyOutcome(videos),
+ ...unanimousLoraStrategyOutcome(videos),
// Inherit applied LoRAs from the first constituent clip (a chunk chain
// shares one LoRA set across all chunks), so the visible stitched entry
// round-trips LoRAs on Remix the same way a single render does — mirrors
diff --git a/server/services/videoGen/upscaleJob.js b/server/services/videoGen/upscaleJob.js
index ebdc401518..e60852135c 100644
--- a/server/services/videoGen/upscaleJob.js
+++ b/server/services/videoGen/upscaleJob.js
@@ -41,7 +41,10 @@ import { isHardwareCompatible, hardwareUnavailableReason } from '../../lib/syste
import { hfChildEnv } from '../hfToken.js';
import { enqueueJob } from '../mediaJobQueue/index.js';
import { videoGenEvents } from './events.js';
-import { RUNTIME_LINE_PREFIX, parseRuntimeFingerprintLine, formatRuntimeFingerprint } from './generateVideoHelpers.js';
+import {
+ RUNTIME_LINE_PREFIX, parseRuntimeFingerprintLine, formatRuntimeFingerprint,
+ LORA_STRATEGY_LINE_PREFIX, parseLoraStrategyLine,
+} from './generateVideoHelpers.js';
import { getHistoryItem, loadHistory, mutateVideoHistory } from './history.js';
import { buildArgs } from './renderArgs.js';
import { runtimeIsCacheOnly, runtimeNeedsProcessGroupKill } from './runtimes.js';
@@ -245,7 +248,7 @@ const runUpscaleChild = async ({ jobId, bin, args, runtime, entry }) => {
// so signal it immediately instead of waiting for a render nobody wants.
if (entry.canceled) cancel(jobId);
const stderrTail = [];
- const provenance = { runtime: null, referenceDownscale: null };
+ const provenance = { runtime: null, referenceDownscale: null, loraStrategy: null };
const onLine = (line, isStderr) => {
const text = String(line).trim();
if (!text || PYTHON_NOISE_RE.test(text)) return;
@@ -263,6 +266,11 @@ const runUpscaleChild = async ({ jobId, bin, args, runtime, entry }) => {
console.log(`🏷️ runtime [${jobId.slice(0, 8)}] ${formatRuntimeFingerprint(fp) || '?'}`);
return;
}
+ if (text.startsWith(LORA_STRATEGY_LINE_PREFIX)) {
+ const report = parseLoraStrategyLine(text);
+ if (report) provenance.loraStrategy = report;
+ return;
+ }
const downscale = REFERENCE_DOWNSCALE_RE.exec(text);
if (downscale) {
provenance.referenceDownscale = Number(downscale[1]);
@@ -466,6 +474,7 @@ export async function runVideoUpscale({
// sentinels when the runner did not report them, never a guessed value.
...(rendered.referenceDownscale !== null ? { upscaleReferenceDownscale: rendered.referenceDownscale } : {}),
...(rendered.runtime ? { runtime: rendered.runtime } : {}),
+ ...(rendered.loraStrategy ? { loraStrategyApplied: rendered.loraStrategy } : {}),
// What the grid actually required of this source, kept beside the result
// so a reader can tell a padded render from a conforming one.
upscaleAlignment: {
diff --git a/server/services/videoGen/upscaleJob.test.js b/server/services/videoGen/upscaleJob.test.js
index 055f11269d..f715cc3e55 100644
--- a/server/services/videoGen/upscaleJob.test.js
+++ b/server/services/videoGen/upscaleJob.test.js
@@ -375,16 +375,21 @@ describe('runVideoUpscale — success', () => {
// factor they read off the adapter before the pipeline loads. Those are
// provenance: a clip must be traceable to the exact runtime and the rule it
// enforced, the way a plain render's row carries `runtime` (#6512).
- it('records the runtime fingerprint and measured reference factor the runner reports', async () => {
+ it('records the runtime fingerprint, measured reference factor and adapter strategy the runner reports', async () => {
const fingerprint = { runtime: 'ltx25', versions: { mlx: '0.32.0' }, chip: 'Apple Mx' };
+ // #12262: whether the adapter applied at inference or was merged and
+ // re-quantized decides whether a low-bit upscale kept its effect.
+ const loraStrategy = { adapters: [{ role: 'ic', name: 'upscaler.safetensors', strategy: 'runtime-delta', stages: ['stage-1'], strength: 1, quantizedTargets: 480, fusedTargets: 0 }] };
const entry = await runWith(() => {
state.child.stderr.emit('data', 'UPSCALE_REFERENCE_DOWNSCALE:2\n');
state.child.stderr.emit('data', `RUNTIME:${JSON.stringify(fingerprint)}\n`);
state.child.stderr.emit('data', 'STAGE:inference\n');
+ state.child.stderr.emit('data', `LORASTRATEGY:${JSON.stringify(loraStrategy)}\n`);
finishChild(0);
});
expect(entry.upscaleReferenceDownscale).toBe(2);
expect(entry.runtime).toEqual(fingerprint);
+ expect(entry.loraStrategyApplied).toEqual(loraStrategy);
});
it('keeps a malformed fingerprint line off the row instead of half-parsing it', async () => {
@@ -422,6 +427,7 @@ describe('runVideoUpscale — success', () => {
// guessed factor or a half-parsed fingerprint.
expect(entry.upscaleReferenceDownscale).toBeUndefined();
expect(entry.runtime).toBeUndefined();
+ expect(entry).not.toHaveProperty('loraStrategyApplied');
// The source render's timing must not ride along on a row that only paid
// for the upscale.
expect(entry.renderStartedAt).not.toBe('2026-01-01T00:00:00.000Z');