Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""HiDream Llama encoder loads the byte-identical unsloth mirror since upstream is gated; the
unsloth org is inside the non-GGUF trust gate; output_hidden_states=True, as the pipeline
consumes hidden states."""
from __future__ import annotations
from typing import Any, Optional
from loggers import get_logger
logger = get_logger(__name__)
HIDREAM_FAMILY_NAME = "hidream-i1"
# Open mirror of the gated meta-llama/Meta-Llama-3.1-8B-Instruct the pipeline expects.
HIDREAM_LLAMA_REPO = "unsloth/Meta-Llama-3.1-8B-Instruct"
HIDREAM_LLAMA_BF16_BYTES = 16_060_556_376
def hidream_te4_kwargs(
dtype: Any,
hf_token: Optional[str] = None,
*,
fam: Any = None,
te_quant_mode: Optional[str] = None,
target: Any = None,
local_files_only: bool = False,
) -> dict[str, Any]:
"""Returns {text_encoder_4, tokenizer_4} kwargs; TE4 is fp8-cast only under layerwise fp8; local_files_only raises instead of downloading."""
import torch # noqa: F401 -- dtype values are torch dtypes; import keeps parity with callers
from transformers import AutoTokenizer, LlamaForCausalLM
# Live hub root: an unpinned lookup after a cache-folder change misses the cached encoder.
from utils.hf_cache_settings import active_hf_hub_cache
cache_dir = active_hf_hub_cache()
tokenizer_4 = AutoTokenizer.from_pretrained(
HIDREAM_LLAMA_REPO,
token = hf_token,
local_files_only = local_files_only,
cache_dir = cache_dir,
)
fp8_engages = False
hosted_int8 = False
if target is not None:
try:
from . import diffusion_precision as precision
from .diffusion_precision import (
TE_QUANT_FP8,
normalize_te_quant,
te_quant_supported,
)
from .diffusion_te_prequant import int8_convrot_te_runs_on
mode = normalize_te_quant(te_quant_mode)
denied = getattr(precision, "_te_family_denied", None)
allowed = not (callable(denied) and denied(getattr(fam, "name", None), mode))
# fp8 (the hosted fp8 file, else a dense load and cast) needs fp8 support; int8 takes the hosted int8
# ConvRot file, which Apple Silicon runs as plain tensors where fp8 does not.
fp8_engages = (
mode in (TE_QUANT_FP8, "int8")
and te_quant_supported(target, TE_QUANT_FP8)
and allowed
)
hosted_int8 = mode == "int8" and allowed and int8_convrot_te_runs_on(target)
except Exception: # noqa: BLE001 -- quant probe failure keeps the dense bf16 path
fp8_engages = hosted_int8 = False
if (fp8_engages or hosted_int8) and fam is not None:
from .diffusion_te_prequant import (
load_prequant_text_encoder,
te_prequant_sources_for_base,
)
source = te_prequant_sources_for_base(
fam,
HIDREAM_LLAMA_REPO,
te_quant_mode = te_quant_mode,
target = target,
components = ("text_encoder_4",),
standalone_component_bases = {"text_encoder_4": HIDREAM_LLAMA_REPO},
).get("text_encoder_4")
if source is not None:
scheme = "int8" if normalize_te_quant(te_quant_mode) == "int8" else "fp8"
encoder = load_prequant_text_encoder(
HIDREAM_LLAMA_REPO,
"text_encoder_4",
source,
dtype = dtype,
hf_token = hf_token,
scheme = scheme,
logger = logger,
# The int8 file carries no lm_head; the pipeline reads hidden states only.
trim_lm_head = scheme == "int8",
# The Llama TE4 lives in its own standalone repo (config at the root), and the pipeline needs hidden
# states/attentions from its forward.
config_subfolder = "",
config_overrides = {
"output_hidden_states": True,
"output_attentions": True,
},
local_files_only = local_files_only,
)
if encoder is not None:
return {"text_encoder_4": encoder, "tokenizer_4": tokenizer_4}
if hosted_int8 and not fp8_engages:
from .diffusion_precision import te_quant_picked_automatically
if not te_quant_picked_automatically(te_quant_mode):
# Apple Silicon: the plan priced the hosted int8 TE4 and nothing else runs here, so the 16 GB dense
# Llama is not loaded in its place (as _te_prequant_pipe_kwargs refuses for the other encoders).
raise RuntimeError(
f"The pre-quantized text_encoder_4 for {HIDREAM_LLAMA_REPO} could not be loaded "
f"({'not available' if source is None else 'see the server log'}). The memory check sized "
"this load for it, so the larger dense encoder is not loaded in its place. Set Text encoder "
"precision to Dense (bf16) to load the dense encoder instead."
)
logger.info("diffusion.hidream: loading Llama TE4 from %s", HIDREAM_LLAMA_REPO)
from .safetensors_pread import pread_safetensors
# loaded before the pipeline's own scope: a 16 GB bf16 Llama converted on Windows reads through pread too
with pread_safetensors(convert_to = dtype):
text_encoder_4 = LlamaForCausalLM.from_pretrained(
HIDREAM_LLAMA_REPO,
output_hidden_states = True,
output_attentions = True,
torch_dtype = dtype,
token = hf_token,
local_files_only = local_files_only,
cache_dir = cache_dir,
)
if fp8_engages:
try:
from .diffusion_precision import _cast_fp8
class _Target:
pass
cast_target = _Target()
cast_target.dtype = dtype
_cast_fp8(text_encoder_4, cast_target)
from .diffusion_precision import TE_RUNTIME_SCHEME_ATTR
setattr(text_encoder_4, TE_RUNTIME_SCHEME_ATTR, "fp8")
logger.info("diffusion.hidream: TE4 layerwise fp8 cast engaged")
except Exception as exc: # noqa: BLE001 -- best-effort like the generic TE pass
# A half-cast encoder cannot run dense, so rebuild it fresh.
logger.warning("diffusion.hidream: TE4 fp8 cast failed, reloading dense: %s", exc)
text_encoder_4 = LlamaForCausalLM.from_pretrained(
HIDREAM_LLAMA_REPO,
output_hidden_states = True,
output_attentions = True,
torch_dtype = dtype,
local_files_only = local_files_only,
token = hf_token,
cache_dir = cache_dir,
)
return {"text_encoder_4": text_encoder_4, "tokenizer_4": tokenizer_4}