Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
#!/usr/bin/env python3
"""Apply LTX LoRA adapters at inference time without re-quantizing packed weights (#12262).
The pinned ltx-2-mlx / ltx-2.5-mlx runtimes fuse an adapter by dequantizing
each q4/q8 projection, adding ``strength * B @ A`` and quantizing again. A
rank-r delta is small next to the base weight, so the second quantization
rounds most of it away and an Ingredients / IC adapter on a low-bit pack can
render as the plain base model. This module keeps each quantized projection
exactly as loaded and adds the delta in activation space instead:
quantized_linear(x) + (x @ A.T) @ (strength * B).T
Float (unquantized) projections keep fusion: adding the delta to a float
weight retains it, and that is the runtime's own high-precision path.
Why a class swap instead of a wrapper module (the MiniMax H3 precedent in
``minimax_h3_lora.py``): the LTX pipelines flatten ``dit.parameters()`` AFTER
loading — the two-stage pipelines fuse their distilled LoRA that way, and block
streaming rebinds weights with ``load_weights(strict=True)``. A wrapper moves
``to_q.weight`` to ``to_q.base.weight``, which those paths would then silently
miss. Swapping the instance to a cached subclass whose ``__call__`` adds the
delta, with the adapter arrays held outside the parameter tree, keeps every
parameter name and every packed array untouched.
Block streaming (#12327) rebinds ONE shared block per index from the mmap'd
pack and runs it through ``mx.compile(shared, inputs=shared)``, so a delta held
outside that block's parameter tree would be traced as a constant and applied
to every index. Under streaming the model's class is swapped too: its block
provider binds the packed weights exactly as the runtime does (runtime-owned
``_lora_sources`` such as a custom-strength distilled LoRA still merge there),
then points each quantized projection's adapter slots at block ``idx``'s
arrays and fuses float targets into the freshly bound float weight. The block
is recompiled with those slots as declared inputs, so every index sees only
its own delta and the compiled graph stays valid.
The math contract mirrors the runtime's ``apply_loras`` exactly — the same
renamed keys, ``delta = strength * B @ A`` and no alpha rescale — so a q4 render
and a bf16 render at one strength mean the same thing. Validation is stricter:
``apply_loras`` silently skips a key it cannot place, and that silent no-op is
the failure this module exists to remove, so an unplaceable tensor is refused.
MLX is imported lazily, so the module (and the runners that import it) stay
importable by a stdlib-only interpreter.
"""
from __future__ import annotations
import json
import math
import sys
from pathlib import Path
from typing import Any, Callable, Iterable
STRATEGY_RUNTIME = "runtime-delta"
STRATEGY_FUSED = "fused-float"
STRATEGY_MIXED = "runtime-delta+fused-float"
_A_SUFFIX = ".lora_A.weight"
_B_SUFFIX = ".lora_B.weight"
# The runtime's apply_loras never reads alpha; ignoring it here keeps strength
# semantics identical across the quantized and float paths.
_IGNORED_SUFFIXES = (".alpha",)
_ADAPTERS_ATTR = "_portos_runtime_lora"
_BASE_CLASS_ATTR = "_portos_runtime_lora_base"
_RUNTIME_CLASSES: dict[type, type] = {}
_STREAMING_ATTR = "_portos_streaming_lora"
_STREAMING_BASE_ATTR = "_portos_streaming_lora_base"
_STREAMING_CLASSES: dict[type, type] = {}
# The runtime's renamed adapter keys address blocks under this prefix
# (ltx_core_mlx.loader.sd_ops.LTXV_LORA_BLOCK_PREFIX).
_BLOCK_PREFIX = "transformer_blocks."
# A float projection outside the streamed blocks is fused once; its pre-fusion
# weight is kept so dropping one adapter can re-fuse the rest.
_FUSED_BASE_ATTR = "_portos_fused_base"
def _mlx():
import mlx.core as mx
import mlx.nn as nn
return mx, nn
def _runtime_class(base_cls: type) -> type:
cls = _RUNTIME_CLASSES.get(base_cls)
if cls is not None:
return cls
def __call__(self, x):
output = base_cls.__call__(self, x)
for down, up in self.__dict__[_ADAPTERS_ATTR]:
output = output + ((x.astype(down.dtype) @ down.T) @ up.T).astype(output.dtype)
return output
cls = type(f"RuntimeLora{base_cls.__name__}", (base_cls,), {
"__call__": __call__,
_BASE_CLASS_ATTR: base_cls,
})
_RUNTIME_CLASSES[base_cls] = cls
return cls
def _module_at(model, target: str):
node: Any = model
for part in target.split("."):
if not part:
return None
try:
node = node[int(part)] if part.isdigit() else getattr(node, part)
except (AttributeError, IndexError, KeyError, TypeError):
return None
return node
def is_streaming_model(model) -> bool:
"""True for the runtime's StreamingLTXModel: one shared block rebound per index."""
attrs = getattr(model, "__dict__", {})
return "_streamer" in attrs and "_shared_block" in attrs
def _locate(model, target: str):
"""``(module, block)`` for a renamed target; ``block`` is ``(index, path)`` when streamed."""
if not is_streaming_model(model):
return _module_at(model, target), None
attrs = model.__dict__
if target.startswith(_BLOCK_PREFIX):
index, _, path = target[len(_BLOCK_PREFIX):].partition(".")
if index.isdigit() and path:
if int(index) >= attrs["_streamer"].block_count:
return None, None
return _module_at(attrs["_shared_block"], path), (int(index), path)
return _module_at(model.inner, target), None
def _logical_shape(module, quantized: bool) -> tuple[int, int]:
output_dims, stored_input_dims = (int(v) for v in module.weight.shape)
if not quantized:
return output_dims, stored_input_dims
bits = int(getattr(module, "bits", 0) or 0)
if bits <= 0 or 32 % bits:
raise ValueError(f"{type(module).__name__} reports unusable quantization bits {bits!r}")
return output_dims, stored_input_dims * 32 // bits
def _group_tensors(label: str, tensors: dict) -> dict[str, dict[str, Any]]:
groups: dict[str, dict[str, Any]] = {}
unsupported: list[str] = []
for key, value in tensors.items():
if key.endswith(_A_SUFFIX):
groups.setdefault(key[: -len(_A_SUFFIX)], {})["A"] = value
elif key.endswith(_B_SUFFIX):
groups.setdefault(key[: -len(_B_SUFFIX)], {})["B"] = value
elif not key.endswith(_IGNORED_SUFFIXES):
unsupported.append(key)
if unsupported:
shown = ", ".join(sorted(unsupported)[:3])
more = f" (+{len(unsupported) - 3} more)" if len(unsupported) > 3 else ""
raise ValueError(
f"LoRA {label} carries tensors the LTX adapter contract cannot apply: {shown}{more}"
)
if not groups:
raise ValueError(f"LoRA {label} contains no lora_A/lora_B adapter tensors")
return groups
def _plan(model, label: str, tensors: dict, strength: float) -> list[tuple]:
"""Validate one adapter against the loaded model; mutate nothing."""
_mx_core, nn = _mlx()
if not math.isfinite(float(strength)):
raise ValueError(f"LoRA {label} strength must be finite, got {strength!r}")
planned = []
for target, pair in sorted(_group_tensors(label, tensors).items()):
if "A" not in pair or "B" not in pair:
missing = "lora_B" if "A" in pair else "lora_A"
raise ValueError(f"LoRA {label} target {target} has no {missing} tensor")
module, block = _locate(model, target)
if module is None:
raise ValueError(
f"LoRA {label} targets {target}, which this transformer does not have — "
"the adapter was trained for a different model"
)
if isinstance(module, nn.QuantizedLinear):
quantized = True
elif isinstance(module, nn.Linear):
quantized = False
else:
raise ValueError(
f"LoRA {label} target {target} is a {type(module).__name__}, not a linear projection"
)
down, up = pair["A"], pair["B"]
if down.ndim != 2 or up.ndim != 2:
raise ValueError(f"LoRA {label} target {target} tensors must be rank 2")
output_dims, input_dims = _logical_shape(module, quantized)
rank = int(down.shape[0])
if rank <= 0 or tuple(down.shape) != (rank, input_dims) or tuple(up.shape) != (output_dims, rank):
raise ValueError(
f"LoRA {label} target {target} has lora_A {tuple(down.shape)} / lora_B "
f"{tuple(up.shape)}, which do not fit a ({output_dims}, {input_dims}) projection"
)
planned.append((target, module, quantized, down, up, float(strength), block))
return planned
def _runtime_pair(down, up, strength: float, dtype) -> tuple:
"""``(A, strength * B)`` in the quantized kernel's activation dtype."""
mx, _nn = _mlx()
return down.astype(dtype), (up.astype(mx.float32) * strength).astype(dtype)
def _fused_weight(weight, pairs) -> Any:
"""``weight + sum(strength * B @ A)``, accumulated in float32."""
mx, _nn = _mlx()
fused = weight.astype(mx.float32)
for down, up, strength in pairs:
fused = fused + (up.astype(mx.float32) * strength) @ down.astype(mx.float32)
return fused.astype(weight.dtype)
def _apply(module, quantized: bool, down, up, strength: float) -> None:
if quantized:
# The quantized kernel's scales carry the activation dtype the base
# projection computes in; hold the delta in it so nothing promotes.
entry = _runtime_pair(down, up, strength, module.scales.dtype)
if _ADAPTERS_ATTR not in module.__dict__:
module.__class__ = _runtime_class(type(module))
object.__setattr__(module, _ADAPTERS_ATTR, [])
module.__dict__[_ADAPTERS_ATTR].append(entry)
return
module.weight = _fused_weight(module.weight, [(down, up, strength)])
def attach_adapters(model, adapters: Iterable[tuple[str, dict, float]], *, tag: str | None = None) -> list[dict]:
"""Attach ``(label, renamed_tensors, strength)`` adapters to ``model`` in place.
Every adapter is validated before any module changes, so a malformed file
later in the list cannot leave an earlier one half-applied. Returns one
summary per adapter with the strategy its targets actually received.
``tag`` lets ``drop_streaming_adapters`` remove these later from a streamed
model, which the runtime never reloads between stages.
"""
plans = [(label, strength, _plan(model, label, tensors, strength)) for label, tensors, strength in adapters]
streaming = is_streaming_model(model)
summaries = []
for label, strength, planned in plans:
if streaming:
_attach_streaming(model, planned, tag)
else:
for _target, module, quantized, down, up, scale, _block in planned:
_apply(module, quantized, down, up, scale)
quantized_targets = sum(int(quantized) for _t, _m, quantized, *_rest in planned)
fused_targets = len(planned) - quantized_targets
strategy = (STRATEGY_MIXED if quantized_targets and fused_targets
else STRATEGY_RUNTIME if quantized_targets else STRATEGY_FUSED)
summaries.append({
"name": Path(label).name,
"strength": float(strength),
"strategy": strategy,
"quantizedTargets": quantized_targets,
"fusedTargets": fused_targets,
})
return summaries
def _streaming_class(base_cls: type) -> type:
cls = _STREAMING_CLASSES.get(base_cls)
if cls is not None:
return cls
def __call__(self, *args, **kwargs):
if kwargs.get("block_provider") is None:
kwargs["block_provider"] = _block_provider(self, use_compiled=kwargs.get("perturbations") is None)
return base_cls.__call__(self, *args, **kwargs)
cls = type(f"RuntimeLora{base_cls.__name__}", (base_cls,), {
"__call__": __call__,
_STREAMING_BASE_ATTR: base_cls,
})
_STREAMING_CLASSES[base_cls] = cls
return cls
def _block_provider(model, *, use_compiled: bool):
"""The runtime's per-forward block provider, plus the per-index adapter bind.
Mirrors ``StreamingLTXModel.__call__`` at the pinned runtimes. ``_streamer``
and ``_lora_sources`` are read per forward because the two-stage pipelines
swap the streamer and append a distilled-LoRA source between stages; those
runtime-owned sources still merge inside ``bind``. Guidance perturbations
cannot be traced by mx.compile, so they run the eager block, as upstream.
"""
attrs = model.__dict__
streamer, shared, state = attrs["_streamer"], attrs["_shared_block"], attrs[_STREAMING_ATTR]
runtime_sources = attrs.get("_lora_sources") or None
block = state["compiled"] if use_compiled else shared
previous: list[int | None] = [None]
def provider(index: int):
streamer.bind(shared, index, evict_previous=previous[0], lora_sources=runtime_sources)
previous[0] = index
_bind_streamed_block(state, shared, index)
return block
return provider
def _bind_streamed_block(state: dict, shared, index: int) -> None:
"""Point the shared block's adapters at block ``index`` right after its weights bind.
The casts are built here, per bind, so only the bound block's copies are
ever alive; a source tensor is whatever the adapter loader returned.
"""
for path, slots in state["slots"].items():
for slot, source in zip(slots, state["slot_sources"][path]):
pair = source["blocks"].get(index)
slot[0], slot[1] = (source["zero"] if pair is None
else _runtime_pair(*pair, source["strength"], source["dtype"]))
for path, sources in state["fused"].items():
pairs = [(*source["blocks"][index], source["strength"]) for source in sources if index in source["blocks"]]
if pairs:
module = _module_at(shared, path)
module.weight = _fused_weight(module.weight, pairs)
def _streaming_state(model) -> dict:
state = model.__dict__.get(_STREAMING_ATTR)
if state is None:
# slots: path -> [[down, up], ...], the SAME lists the quantized
# projection iterates, declared as compile inputs. fused: float block
# projections, fused into each freshly bound weight. outside: targets
# beyond the streamed blocks, which stay resident.
state = {"slots": {}, "slot_sources": {}, "fused": {}, "outside": [], "compiled": None}
model.__class__ = _streaming_class(type(model))
object.__setattr__(model, _STREAMING_ATTR, state)
return state
def _recompile(model) -> None:
"""Compile the shared block with its adapter slots as declared inputs.
A fresh compiled function per slot layout: mx.compile retraces when input
shapes change, but a layout change with equal shapes would reuse a trace
that reads the wrong slots.
"""
mx, _nn = _mlx()
state = model.__dict__[_STREAMING_ATTR]
shared = model.__dict__["_shared_block"]
state["compiled"] = mx.compile(shared, inputs=[shared, state["slots"]])
def _attach_streaming(model, planned: list[tuple], tag: str | None) -> None:
mx, _nn = _mlx()
state = _streaming_state(model)
sources: dict[str, dict] = {}
for _target, module, quantized, down, up, strength, block in planned:
if block is None:
state["outside"].append(_attach_outside_blocks(module, quantized, down, up, strength, tag))
continue
index, path = block
source = sources.get(path)
if source is None:
source = sources[path] = {"tag": tag, "strength": strength, "blocks": {}}
if quantized:
dtype = module.scales.dtype
source["dtype"] = dtype
# A block this adapter does not address keeps the slot layout
# (and so the compiled trace) and adds nothing.
source["zero"] = (mx.zeros(down.shape, dtype=dtype), mx.zeros(up.shape, dtype=dtype))
slots = state["slots"].get(path)
if slots is None:
slots = state["slots"][path] = []
state["slot_sources"][path] = []
module.__class__ = _runtime_class(type(module))
object.__setattr__(module, _ADAPTERS_ATTR, slots)
slots.append(list(source["zero"]))
state["slot_sources"][path].append(source)
else:
state["fused"].setdefault(path, []).append(source)
source["blocks"][index] = (down, up)
_recompile(model)
def _attach_outside_blocks(module, quantized: bool, down, up, strength: float, tag: str | None) -> dict:
"""Apply a non-block target as ``_apply`` does, recording what a drop must undo."""
if not quantized:
module.__dict__.setdefault(_FUSED_BASE_ATTR, module.weight)
_apply(module, quantized, down, up, strength)
record = {"module": module, "quantized": quantized, "tag": tag, "delta": (down, up, strength)}
if quantized:
record["entry"] = module.__dict__[_ADAPTERS_ATTR][-1]
return record
def drop_streaming_adapters(model, tag: str | None = None) -> int:
"""Remove adapters attached with ``tag`` (every one when None) from a streamed model.
Under streaming the runtime's clean stage-2 refine keeps the same model and
only clears its own ``_lora_sources``, so an IC adapter is dropped here or
it would condition the refine. Returns the number of projections restored.
"""
state = model.__dict__.get(_STREAMING_ATTR)
if state is None:
return 0
shared = model.__dict__["_shared_block"]
def dropped(item: dict) -> bool:
return tag is None or item["tag"] == tag
restored = 0
for path in list(state["slots"]):
keep = [i for i, source in enumerate(state["slot_sources"][path]) if not dropped(source)]
slots = state["slots"][path]
slots[:] = [slots[i] for i in keep]
state["slot_sources"][path] = [state["slot_sources"][path][i] for i in keep]
if not slots:
_restore_class(_module_at(shared, path))
del state["slots"][path], state["slot_sources"][path]
restored += 1
for path in list(state["fused"]):
state["fused"][path] = [source for source in state["fused"][path] if not dropped(source)]
if not state["fused"][path]:
del state["fused"][path]
removed = [record for record in state["outside"] if dropped(record)]
state["outside"] = [record for record in state["outside"] if not dropped(record)]
refuse: dict[int, Any] = {}
for record in removed:
module = record["module"]
if not record["quantized"]:
refuse[id(module)] = module
continue
entries = module.__dict__[_ADAPTERS_ATTR]
entries[:] = [entry for entry in entries if entry is not record["entry"]]
if not entries:
_restore_class(module)
restored += 1
for module in refuse.values():
remaining = [record["delta"] for record in state["outside"] if record["module"] is module]
module.weight = _fused_weight(module.__dict__[_FUSED_BASE_ATTR], remaining)
if not remaining:
del module.__dict__[_FUSED_BASE_ATTR]
restored += 1
if tag is None:
model.__class__ = getattr(type(model), _STREAMING_BASE_ATTR)
del model.__dict__[_STREAMING_ATTR]
else:
_recompile(model)
return restored
def _restore_class(module) -> None:
module.__class__ = getattr(type(module), _BASE_CLASS_ATTR)
del module.__dict__[_ADAPTERS_ATTR]
def runtime_adapter_targets(model) -> list[str]:
"""Names of the projections currently carrying an activation-space delta."""
return [name for name, module in model.named_modules() if _ADAPTERS_ATTR in module.__dict__]
def detach_runtime_adapters(model) -> int:
"""Restore every swapped projection to its original class. Returns the count."""
count = drop_streaming_adapters(model)
for _name, module in model.named_modules():
if _ADAPTERS_ATTR in module.__dict__:
_restore_class(module)
count += 1
return count
def load_ltx_adapter(path: str) -> dict:
"""Read an adapter with the runtime's own key renaming, so targets match the DiT."""
from ltx_core_mlx.loader import LTXV_LORA_COMFY_RENAMING_MAP, SafetensorsStateDictLoader
return SafetensorsStateDictLoader().load(path, sd_ops=LTXV_LORA_COMFY_RENAMING_MAP).sd
class AdapterReport:
"""What each adapter actually received, for the render record (LORASTRATEGY:)."""
def __init__(self):
self.entries: list[dict] = []
# Identity is the full source path + strength, never the basename: two
# files sharing a name, or one file at two strengths, are distinct
# applications. The path stays out of the emitted record (history is
# user-facing and machine paths are private).
self._index: dict[tuple, dict] = {}
def record(self, role: str, stage: str, source: str, summary: dict) -> bool:
"""Returns True the first time this application is recorded."""
key = (role, str(source), summary["strength"])
entry = self._index.get(key)
if entry is not None:
if stage not in entry["stages"]:
entry["stages"].append(stage)
return False
entry = {"role": role, "stages": [stage], **summary}
self._index[key] = entry
self.entries.append(entry)
return True
def emit(self, stream=None) -> None:
if not self.entries:
return
print(f"LORASTRATEGY:{json.dumps({'adapters': self.entries})}", file=stream or sys.stderr, flush=True)
def install_pipeline_adapters(
pipe,
user_specs: list[tuple[str, float]],
*,
report: AdapterReport,
status: Callable[[str], None],
ic: bool = False,
load_adapter: Callable[[str], dict] = load_ltx_adapter,
) -> None:
"""Route a pipeline's adapters through inference-time application.
User adapters attach on EVERY transformer load (the runtime's
``_pending_loras`` contract), so a stage that reloads keeps them. With
``ic=True`` the pipeline's ``_fuse_loras`` — called once, right before
stage 1 — attaches the IC adapter(s) to the loaded DiT instead; the clean
stage-2 reload goes back through the load hook and so carries the user
adapters but never the IC one. Block streaming takes the same route: the
load hook returns a StreamingLTXModel and ``attach_adapters`` binds each
block's delta per index (#12327); its clean refine keeps the model, so the
reload wrapper drops the IC adapters from it instead.
Paths must already be local files: `parse_user_loras` requires that, and
the IC constructor resolves its repo ids before this runs.
"""
pipe._pending_loras = None
load_transformer = pipe._load_transformer_with_optional_streaming
def attach(dit, role: str, stage: str, specs) -> None:
loaded = [(path, load_adapter(path), strength) for path, strength in specs]
for (path, _tensors, _strength), summary in zip(loaded, attach_adapters(dit, loaded, tag=role)):
if report.record(role, stage, path, summary):
status(
f"{role} LoRA {summary['name']}: {summary['strategy']} "
f"({summary['quantizedTargets']} quantized, {summary['fusedTargets']} float projection(s))"
)
def load_with_user_adapters(transformer_path):
dit = load_transformer(transformer_path)
if user_specs:
attach(dit, "user", "all", user_specs)
return dit
pipe._load_transformer_with_optional_streaming = load_with_user_adapters
if ic:
def attach_ic_adapters() -> None:
effective = getattr(pipe, "_effective_lora_paths", None)
specs = effective() if callable(effective) else list(pipe._lora_paths)
if specs:
attach(pipe.dit, "ic", "stage-1", specs)
pipe._fuse_loras = attach_ic_adapters
reload_clean = getattr(pipe, "_reload_clean_transformer", None)
if callable(reload_clean):
def reload_clean_without_ic() -> None:
reload_clean()
if pipe.dit is not None and is_streaming_model(pipe.dit):
drop_streaming_adapters(pipe.dit, "ic")
pipe._reload_clean_transformer = reload_clean_without_ic
def release_pipeline_adapters(pipe) -> None:
"""Drop the transformer and every adapter this module attached to it.
Called when a render ends either way: the DiT is the one large object the
adapters hang off, and a failed or cancelled render must not keep it (or a
swapped class pointing at it) alive.
"""
dit = getattr(pipe, "dit", None)
if dit is not None:
detach_runtime_adapters(dit)
pipe.dit = None
if hasattr(pipe, "_loaded"):
pipe._loaded = False