Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: pool SDXL textual inversion prompts at EOS (#14846)
Daniel Han2b052f09f2d7
diff --git a/studio/backend/core/inference/diffusion_embeddings.py b/studio/backend/core/inference/diffusion_embeddings.pyindex f9a00dd9e88..c2a0fde9267 100644--- a/studio/backend/core/inference/diffusion_embeddings.py+++ b/studio/backend/core/inference/diffusion_embeddings.py@@ -25,7 +25,7 @@ import copyimport hashlibimport osimport re-from dataclasses import dataclass+from dataclasses import dataclass, fieldfrom pathlib import Pathfrom typing import Any, Iterable, Optional@@ -285,6 +285,8 @@ class AppliedEmbeddings:skipped: list[str]pattern: Optional[re.Pattern]marker_owners: list[Any]+ # (pooling module, original eos_token_id)+ pool_eos: list[tuple[Any, Any]] = field(default_factory = list)def rewrite(self, call_kwargs: dict) -> dict:"""``call_kwargs`` with every trigger replaced by its internal token (a new dict; the input is kept)."""@@ -304,6 +306,12 @@ class AppliedEmbeddings:return outdef restore(self) -> None:+ for text_model, original in reversed(self.pool_eos):+ try:+ text_model.eos_token_id = original+ except Exception: # noqa: BLE001+ pass+ self.pool_eos.clear()for encoder, original in reversed(self.encoders):try:encoder.set_input_embeddings(original)@@ -324,6 +332,31 @@ class AppliedEmbeddings:self.tokenizers.clear()+def _clip_pooling_module(encoder: Any) -> Any:+ """CLIP module owning ``eos_token_id``: ``text_model`` (transformers 4.x) or the model itself (5.x)."""+ module = encoder+ for _ in range(3):+ if module is None:+ return None+ if "eos_token_id" in getattr(module, "__dict__", {}):+ return module+ module = getattr(module, "text_model", None)+ return None+++def _pool_at_eos(applied: AppliedEmbeddings, encoder: Any, tokenizer: Any) -> None:+ """Legacy ``eos_token_id == 2`` configs (pre transformers #24773, both SDXL encoders) pool at+ ``input_ids.argmax(-1)``, which an added token outranks; pool at the real EOS instead."""+ text_model = _clip_pooling_module(encoder)+ eos = getattr(tokenizer, "eos_token_id", None)+ if text_model is None or getattr(text_model, "eos_token_id", None) != 2:+ return+ if not isinstance(eos, int) or isinstance(eos, bool) or eos == 2:+ return+ applied.pool_eos.append((text_model, text_model.eos_token_id))+ text_model.eos_token_id = eos++def _slots_for(pipe: Any, kind: str) -> list[tuple[str, Any, Any, str]]:slots = []for tok_attr, enc_attr, key in _ENCODER_SLOTS:@@ -423,6 +456,7 @@ def apply(table = table.to(device = weight.device, dtype = weight.dtype)applied.encoders.append((encoder, original_embedding))encoder.set_input_embeddings(_make_extended_embedding(original_embedding, table))+ _pool_at_eos(applied, encoder, tok_copy)applied.tokenizers.append((tok_attr, tokenizer))setattr(pipe, tok_attr, tok_copy)except BaseException:diff --git a/studio/backend/tests/test_diffusion_embeddings.py b/studio/backend/tests/test_diffusion_embeddings.pyindex 6efa0a11348..67ac73ba26d 100644--- a/studio/backend/tests/test_diffusion_embeddings.py+++ b/studio/backend/tests/test_diffusion_embeddings.py@@ -217,3 +217,102 @@ def test_families_gate_textual_inversion():from core.inference.diffusion_families import _FAMILIESassert {f.name for f in _FAMILIES if f.textual_inversion} == {"sdxl"}assert next(f for f in _FAMILIES if f.name == "sdxl").textual_inversion == de.KIND_SDXL+++class _AddedTokenizer:+ eos_token_id = 49407++ def __init__(self):+ self.added = {}++ def add_tokens(self, names):+ for name in names:+ self.added.setdefault(name, 49408 + len(self.added))++ def convert_tokens_to_ids(self, name):+ return self.added[name]+++def _random_sdxl_encoders(eos_token_id):+ transformers = pytest.importorskip("transformers")+ from types import SimpleNamespace++ def config(width):+ return transformers.CLIPTextConfig(+ vocab_size = 49408,+ hidden_size = width,+ intermediate_size = width * 2,+ num_hidden_layers = 1,+ num_attention_heads = 2,+ max_position_embeddings = 8,+ projection_dim = width,+ bos_token_id = 49406,+ eos_token_id = eos_token_id,+ pad_token_id = 1,+ )++ torch.manual_seed(0)+ return SimpleNamespace(+ tokenizer = _AddedTokenizer(),+ tokenizer_2 = _AddedTokenizer(),+ text_encoder = transformers.CLIPTextModel(config(8)).eval(),+ text_encoder_2 = transformers.CLIPTextModelWithProjection(config(12)).eval(),+ )+++def _pool_owner(encoder):+ return getattr(encoder, "text_model", encoder)+++def _pooled(pipe, ids):+ ids = torch.tensor(ids)+ with torch.no_grad():+ return pipe.text_encoder(ids).pooler_output, pipe.text_encoder_2(ids).text_embeds+++def test_sdxl_legacy_config_pools_at_eos_not_added_token(monkeypatch):+ pipe = _random_sdxl_encoders(eos_token_id = 2)+ fixture = {"clip_l": torch.randn(3, 8), "clip_g": torch.randn(3, 12)}+ monkeypatch.setattr(de, "_read_tensors", lambda path: fixture)+ plain = [[49406, 4, 5, 49407, 49407, 49407], [49406, 4, 5, 49407, 0, 0]]+ base = _pooled(pipe, plain)++ applied = de.apply(+ pipe,+ de.KIND_SDXL,+ {"prompt": "EasyNegXL x"},+ entries = [_entry("EasyNegXL", kind = de.KIND_SDXL)],+ )+ assert applied.used == ["EasyNegXL"]+ assert all(torch.equal(a, b) for a, b in zip(_pooled(pipe, plain), base))++ ids = [[49406, 49408, 49409, 49410, 5, 49407, 49407, 49407]]+ other_suffix = [[49406, 49408, 49409, 49410, 6, 49407, 49407, 49407]]+ with torch.no_grad():+ t = torch.tensor(ids)+ hidden_1 = pipe.text_encoder(t).last_hidden_state+ hidden_2 = pipe.text_encoder_2(t).last_hidden_state+ expect_2 = pipe.text_encoder_2.text_projection(hidden_2[:, 5])+ pooled_1, pooled_2 = _pooled(pipe, ids)+ assert torch.equal(pooled_1, hidden_1[:, 5])+ assert torch.equal(pooled_2, expect_2)+ assert not torch.equal(pooled_2, _pooled(pipe, other_suffix)[1])++ applied.restore()+ assert _pool_owner(pipe.text_encoder).eos_token_id == 2+ assert _pool_owner(pipe.text_encoder_2).eos_token_id == 2+ assert all(torch.equal(a, b) for a, b in zip(_pooled(pipe, plain), base))+++def test_current_clip_config_pooling_left_alone(monkeypatch):+ pipe = _random_sdxl_encoders(eos_token_id = 49407)+ fixture = {"clip_l": torch.randn(2, 8), "clip_g": torch.randn(2, 12)}+ monkeypatch.setattr(de, "_read_tensors", lambda path: fixture)+ applied = de.apply(+ pipe,+ de.KIND_SDXL,+ {"prompt": "EasyNegXL"},+ entries = [_entry("EasyNegXL", kind = de.KIND_SDXL)],+ )+ assert _pool_owner(pipe.text_encoder_2).eos_token_id == 49407+ applied.restore()