The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support SenseNova U1.5 (CORE-411) (#15922)
T8star-Aix3216c62e9962
diff --git a/comfy/ldm/sensenova/__init__.py b/comfy/ldm/sensenova/__init__.pynew file mode 100644index 000000000..a8bf993b9--- /dev/null+++ b/comfy/ldm/sensenova/__init__.py@@ -0,0 +1 @@+"""SenseNova U1.5 model implementation."""diff --git a/comfy/ldm/sensenova/conditioning.py b/comfy/ldm/sensenova/conditioning.pynew file mode 100644index 000000000..9023bb076--- /dev/null+++ b/comfy/ldm/sensenova/conditioning.py@@ -0,0 +1,135 @@+import torch+++IMAGE_CONTEXT_ID = 151669+IMAGE_START_ID = 151670+IMAGE_END_ID = 151671+IM_START_ID = 151644+IM_END_ID = 151645+USER_ID = 872+ASSISTANT_ID = 77091+NEWLINE_ID = 198+IMAGE_LABEL_ID = 1906+HYPHEN_ID = 12+DIGIT_ZERO_ID = 15+COLON_ID = 25+++def preprocess_reference(image):+ if image.ndim == 3:+ image = image.unsqueeze(0)+ image = image[:, :, :, :3].movedim(-1, 1).float()+ if image.shape[1] == 0:+ image = image.new_zeros((image.shape[0], 3, *image.shape[-2:]))+ elif image.shape[1] < 3:+ repeats = (3 + image.shape[1] - 1) // image.shape[1]+ image = image.repeat(1, repeats, 1, 1)[:, :3]+ mean = image.new_tensor((0.485, 0.456, 0.406)).view(1, 3, 1, 1)+ std = image.new_tensor((0.229, 0.224, 0.225)).view(1, 3, 1, 1)+ return (image - mean) / std+++def split_reference_batches(images):+ references = []+ for image in images:+ if image.ndim == 3:+ image = image.unsqueeze(0)+ references.extend(image[index : index + 1] for index in range(image.shape[0]))+ return references+++def preprocess_references(images):+ return [preprocess_reference(image) for image in split_reference_batches(images)]+++def _image_tokens(token_height, token_width):+ return (+ [IMAGE_START_ID]+ + [IMAGE_CONTEXT_ID] * (token_height * token_width)+ + [IMAGE_END_ID]+ )+++def _image_label_tokens(index):+ digits = (DIGIT_ZERO_ID + int(digit) for digit in str(index + 1))+ return (IMAGE_LABEL_ID, HYPHEN_ID, *digits, COLON_ID)+++def conditioned_input_length(input_length, reference_grids, image_only=False):+ image_token_count = sum(height * width for height, width in reference_grids)+ if image_only:+ return image_token_count + 9 + 2 * len(reference_grids)+ label_count = (+ sum(len(_image_label_tokens(index)) for index in range(len(reference_grids)))+ if len(reference_grids) > 1+ else 0+ )+ return input_length + image_token_count + 3 * len(reference_grids) + label_count+++def condition_input_ids(input_ids, reference_grids, image_only=False):+ image_blocks = [_image_tokens(height, width) for height, width in reference_grids]+ if image_only:+ values = (+ [IM_START_ID, USER_ID, NEWLINE_ID]+ + [token for block in image_blocks for token in block]+ + [+ IM_END_ID,+ NEWLINE_ID,+ IM_START_ID,+ ASSISTANT_ID,+ NEWLINE_ID,+ IMAGE_START_ID,+ ]+ )+ return torch.tensor([values], dtype=torch.long, device=input_ids.device)++ values = input_ids[0].tolist()+ starts = [index for index, value in enumerate(values) if value == IM_START_ID]+ insert_at = starts[1] + 3 if len(starts) > 1 else len(values)+ inserted = []+ for index, block in enumerate(image_blocks):+ if len(image_blocks) > 1:+ inserted.extend(_image_label_tokens(index))+ inserted.extend(block)+ inserted.append(NEWLINE_ID)+ values[insert_at:insert_at] = inserted+ return torch.tensor([values], dtype=torch.long, device=input_ids.device)+++def thw_indexes(input_ids, reference_grids):+ values = input_ids[0]+ image_start_shift = torch.cat(+ (+ torch.zeros(1, dtype=torch.long, device=values.device),+ (values == IMAGE_START_ID).long(),+ )+ )[:-1]+ not_image = (values != IMAGE_CONTEXT_ID).long()+ time_indexes = (image_start_shift + not_image).cumsum(0) - 1+ height_indexes = torch.zeros_like(time_indexes)+ width_indexes = torch.zeros_like(time_indexes)+ selected = values == IMAGE_CONTEXT_ID+ height_positions = []+ width_positions = []+ for token_height, token_width in reference_grids:+ positions = torch.arange(+ token_height * token_width, dtype=torch.long, device=values.device+ )+ height_positions.append(positions // token_width)+ width_positions.append(positions % token_width)+ if height_positions:+ height_indexes[selected] = torch.cat(height_positions)+ width_indexes[selected] = torch.cat(width_positions)+ return torch.stack((time_indexes, height_indexes, width_indexes)).unsqueeze(0)+++def block_causal_mask(time_indexes, dtype=torch.float32):+ values = time_indexes[0, 0]+ length = values.shape[0]+ same_block = values[:, None] == values[None, :]+ positions = torch.arange(length, device=values.device)+ causal = positions[None, :] <= positions[:, None]+ allowed = same_block | causal+ mask = torch.zeros((1, 1, length, length), dtype=dtype, device=values.device)+ return mask.masked_fill_(~allowed[None, None], float("-inf"))diff --git a/comfy/ldm/sensenova/model.py b/comfy/ldm/sensenova/model.pynew file mode 100644index 000000000..9dffff301--- /dev/null+++ b/comfy/ldm/sensenova/model.py@@ -0,0 +1,622 @@+import torch+import torch.nn as nn+import torch.nn.functional as F++import comfy.patcher_extension+import comfy.utils+from comfy.ldm.common_dit import pad_to_patch_size+from comfy.ldm.flux.math import apply_rope1+from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.diffusionmodules.mmdit import TimestepEmbedder++from .sampling import resolution_noise_scale+++HIDDEN_SIZE = 4096+INTERMEDIATE_SIZE = 12288+NUM_LAYERS = 42+NUM_HEADS = 32+NUM_KV_HEADS = 8+HEAD_DIM = 128+MERGED_PATCH_SIZE = 32+VOCAB_SIZE = 151936+++def _pad_to_merged_patch_size(value):+ height, width = value.shape[-2:]+ height_pad = max(16 - height, 0)+ width_pad = max(16 - width, 0)+ if height_pad or width_pad:+ value = F.pad(+ value,+ (0, width_pad, 0, height_pad),+ mode="replicate" if height > 0 and width > 0 else "constant",+ )+ return pad_to_patch_size(value, (MERGED_PATCH_SIZE, MERGED_PATCH_SIZE))+++def _generation_batch_size(total_batch, prefix_batch):+ if prefix_batch < 1 or total_batch < 1 or total_batch % prefix_batch != 0:+ raise ValueError(+ "SenseNova generation batch must be a positive multiple of the prefix batch "+ f"(generation={total_batch}, prefix={prefix_batch})"+ )+ return total_batch // prefix_batch+++def _match_prefix_batch(total_batch, text_input_ids, prefix_indexes, prefix_mask):+ prefix_batch = text_input_ids.shape[0]+ if prefix_batch > 0 and total_batch % prefix_batch:+ text_input_ids = comfy.utils.resize_to_batch_size(text_input_ids, total_batch)+ if prefix_indexes is not None:+ prefix_indexes = comfy.utils.resize_to_batch_size(+ prefix_indexes, total_batch+ )+ if prefix_mask is not None:+ prefix_mask = comfy.utils.resize_to_batch_size(prefix_mask, total_batch)+ return text_input_ids, prefix_indexes, prefix_mask+++def _expand_prefix_batch(value, generation_batch):+ """Repeat each guidance branch's prefix KV for its generated variants."""+ if generation_batch == 1:+ return value+ prefix_batch = value.shape[0]+ return (+ value.unsqueeze(1)+ .expand(prefix_batch, generation_batch, *value.shape[1:])+ .reshape(prefix_batch * generation_batch, *value.shape[1:])+ )+++def _prepare_llm_rope(positions, dim, theta, device, dtype):+ frequencies = theta ** (+ -torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim+ )+ positions = positions.to(device=device, dtype=torch.float32)+ if positions.ndim == 1:+ positions = positions.unsqueeze(0)+ angles = positions.unsqueeze(-1) * frequencies+ embedding = torch.cat((angles, angles), dim=-1).unsqueeze(1)+ return embedding.cos().to(dtype), embedding.sin().to(dtype)+++def _prepare_mrope(indexes, device, dtype):+ return (+ _prepare_llm_rope(indexes[0], HEAD_DIM // 2, 5000000.0, device, dtype),+ _prepare_llm_rope(indexes[1], HEAD_DIM // 4, 10000.0, device, dtype),+ _prepare_llm_rope(indexes[2], HEAD_DIM // 4, 10000.0, device, dtype),+ )+++def _apply_llm_rope(query, key, rope):+ cosine, sine = rope++ def rotate_half(value):+ first, second = value.chunk(2, dim=-1)+ return torch.cat((-second, first), dim=-1)++ # Keep this split-half RoPE on the reference PyTorch formula. The+ # comfy-kitchen CUDA kernel is selected automatically on CUDA 13 builds;+ # on Blackwell it can return finite but numerically incorrect values, which+ # corrupts the generated image without raising an execution error.+ return (+ query * cosine + rotate_half(query) * sine,+ key * cosine + rotate_half(key) * sine,+ )+++def _apply_interleaved_rope(value, positions, theta):+ dim = value.shape[-1]+ frequencies = theta ** (+ -torch.arange(0, dim, 2, dtype=torch.float32, device=value.device) / dim+ )+ angles = (+ positions.to(device=value.device, dtype=torch.float32).unsqueeze(-1)+ * frequencies+ )+ cosine = angles.cos()+ sine = angles.sin()+ # comfy-kitchen acceleration backends use the canonical four-dimensional+ # input and six-dimensional rotation layout. SenseNova's vision patches+ # have no head axis, so add a singleton one instead of relying on the eager+ # backend's more permissive rank handling.+ rotation = torch.stack((cosine, -sine, sine, cosine), dim=-1).reshape(+ 1, 1, *angles.shape, 2, 2+ )+ return apply_rope1(value.float().unsqueeze(1), rotation).squeeze(1)+++class VisionEmbeddings(nn.Module):+ def __init__(self, device=None, dtype=None, operations=None):+ super().__init__()+ self.patch_embedding = operations.Conv2d(+ 3, 1024, kernel_size=16, stride=16, device=device, dtype=dtype+ )+ self.dense_embedding = operations.Conv2d(+ 1024, HIDDEN_SIZE, kernel_size=2, stride=2, device=device, dtype=dtype+ )+ self.gelu = nn.GELU()++ def forward(self, image):+ patches = self.gelu(self.patch_embedding(image))+ batch, channels, height, width = patches.shape+ patches = patches.flatten(2).transpose(1, 2)+ indexes = torch.arange(height * width, device=patches.device)+ x_positions = indexes % width+ y_positions = indexes // width+ first = _apply_interleaved_rope(+ patches[..., : channels // 2], x_positions, 10000.0+ )+ second = _apply_interleaved_rope(+ patches[..., channels // 2 :], y_positions, 10000.0+ )+ patches = torch.cat((first, second), dim=-1).to(image.dtype)+ patches = patches.transpose(1, 2).reshape(batch, channels, height, width)+ patches = self.dense_embedding(patches)+ return patches.flatten(2).transpose(1, 2)+++class VisionModel(nn.Module):+ def __init__(self, device=None, dtype=None, operations=None):+ super().__init__()+ self.embeddings = VisionEmbeddings(+ device=device, dtype=dtype, operations=operations+ )++ def forward(self, image):+ return self.embeddings(image)+++class MLP(nn.Module):+ def __init__(self, device=None, dtype=None, operations=None):+ super().__init__()+ self.gate_proj = operations.Linear(+ HIDDEN_SIZE, INTERMEDIATE_SIZE, bias=False, device=device, dtype=dtype+ )+ self.up_proj = operations.Linear(+ HIDDEN_SIZE, INTERMEDIATE_SIZE, bias=False, device=device, dtype=dtype+ )+ self.down_proj = operations.Linear(+ INTERMEDIATE_SIZE, HIDDEN_SIZE, bias=False, device=device, dtype=dtype+ )++ def forward(self, hidden_states):+ return self.down_proj(+ F.silu(self.gate_proj(hidden_states)) * self.up_proj(hidden_states)+ )+++class Attention(nn.Module):+ def __init__(self, device=None, dtype=None, operations=None):+ super().__init__()+ self.q_proj = operations.Linear(+ HIDDEN_SIZE, NUM_HEADS * HEAD_DIM, bias=False, device=device, dtype=dtype+ )+ self.q_proj_mot_gen = operations.Linear(+ HIDDEN_SIZE, NUM_HEADS * HEAD_DIM, bias=False, device=device, dtype=dtype+ )+ self.k_proj = operations.Linear(+ HIDDEN_SIZE, NUM_KV_HEADS * HEAD_DIM, bias=False, device=device, dtype=dtype+ )+ self.k_proj_mot_gen = operations.Linear(+ HIDDEN_SIZE, NUM_KV_HEADS * HEAD_DIM, bias=False, device=device, dtype=dtype+ )+ self.v_proj = operations.Linear(+ HIDDEN_SIZE, NUM_KV_HEADS * HEAD_DIM, bias=False, device=device, dtype=dtype+ )+ self.v_proj_mot_gen = operations.Linear(+ HIDDEN_SIZE, NUM_KV_HEADS * HEAD_DIM, bias=False, device=device, dtype=dtype+ )+ self.o_proj = operations.Linear(+ NUM_HEADS * HEAD_DIM, HIDDEN_SIZE, bias=False, device=device, dtype=dtype+ )+ self.o_proj_mot_gen = operations.Linear(+ NUM_HEADS * HEAD_DIM, HIDDEN_SIZE, bias=False, device=device, dtype=dtype+ )++ self.q_norm = operations.RMSNorm(+ HEAD_DIM // 2, eps=1e-6, device=device, dtype=dtype+ )+ self.q_norm_mot_gen = operations.RMSNorm(+ HEAD_DIM // 2, eps=1e-6, device=device, dtype=dtype+ )+ self.q_norm_hw = operations.RMSNorm(+ HEAD_DIM // 2, eps=1e-6, device=device, dtype=dtype+ )+ self.q_norm_hw_mot_gen = operations.RMSNorm(+ HEAD_DIM // 2, eps=1e-6, device=device, dtype=dtype+ )+ self.k_norm = operations.RMSNorm(+ HEAD_DIM // 2, eps=1e-6, device=device, dtype=dtype+ )+ self.k_norm_mot_gen = operations.RMSNorm(+ HEAD_DIM // 2, eps=1e-6, device=device, dtype=dtype+ )+ self.k_norm_hw = operations.RMSNorm(+ HEAD_DIM // 2, eps=1e-6, device=device, dtype=dtype+ )+ self.k_norm_hw_mot_gen = operations.RMSNorm(+ HEAD_DIM // 2, eps=1e-6, device=device, dtype=dtype+ )++ def _project(self, hidden_states, rope, generation):+ batch, length, _ = hidden_states.shape+ if generation:+ query = self.q_proj_mot_gen(hidden_states).view(+ batch, length, NUM_HEADS, HEAD_DIM+ )+ key = self.k_proj_mot_gen(hidden_states).view(+ batch, length, NUM_KV_HEADS, HEAD_DIM+ )+ value = (+ self.v_proj_mot_gen(hidden_states)+ .view(batch, length, NUM_KV_HEADS, HEAD_DIM)+ .transpose(1, 2)+ )+ query_t, query_hw = query.chunk(2, dim=-1)+ key_t, key_hw = key.chunk(2, dim=-1)+ query_t = self.q_norm_mot_gen(query_t).transpose(1, 2)+ query_hw = self.q_norm_hw_mot_gen(query_hw).transpose(1, 2)+ key_t = self.k_norm_mot_gen(key_t).transpose(1, 2)+ key_hw = self.k_norm_hw_mot_gen(key_hw).transpose(1, 2)+ else:+ query = self.q_proj(hidden_states).view(batch, length, NUM_HEADS, HEAD_DIM)+ key = self.k_proj(hidden_states).view(batch, length, NUM_KV_HEADS, HEAD_DIM)+ value = (+ self.v_proj(hidden_states)+ .view(batch, length, NUM_KV_HEADS, HEAD_DIM)+ .transpose(1, 2)+ )+ query_t, query_hw = query.chunk(2, dim=-1)+ key_t, key_hw = key.chunk(2, dim=-1)+ query_t = self.q_norm(query_t).transpose(1, 2)+ query_hw = self.q_norm_hw(query_hw).transpose(1, 2)+ key_t = self.k_norm(key_t).transpose(1, 2)+ key_hw = self.k_norm_hw(key_hw).transpose(1, 2)++ query_h, query_w = query_hw.chunk(2, dim=-1)+ key_h, key_w = key_hw.chunk(2, dim=-1)+ query_t, key_t = _apply_llm_rope(query_t, key_t, rope[0])+ query_h, key_h = _apply_llm_rope(query_h, key_h, rope[1])+ query_w, key_w = _apply_llm_rope(query_w, key_w, rope[2])+ query = torch.cat((query_t, query_h, query_w), dim=-1)+ key = torch.cat((key_t, key_h, key_w), dim=-1)+ return query, key, value++ def forward_prefix(+ self, hidden_states, rope, attention_mask, transformer_options+ ):+ query, key, value = self._project(hidden_states, rope, False)+ output = optimized_attention(+ query,+ key,+ value,+ NUM_HEADS,+ mask=attention_mask,+ skip_reshape=True,+ transformer_options=transformer_options,+ enable_gqa=True,+ )+ return self.o_proj(output), key, value++ def forward_generation(+ self, hidden_states, rope, prefix_key, prefix_value, transformer_options+ ):+ query, key, value = self._project(hidden_states, rope, True)+ key = torch.cat((prefix_key, key), dim=2)+ value = torch.cat((prefix_value, value), dim=2)+ output = optimized_attention(+ query,+ key,+ value,+ NUM_HEADS,+ mask=None,+ skip_reshape=True,+ transformer_options=transformer_options,+ enable_gqa=True,+ )+ return self.o_proj_mot_gen(output)+++class DecoderLayer(nn.Module):+ def __init__(self, device=None, dtype=None, operations=None):+ super().__init__()+ self.self_attn = Attention(device=device, dtype=dtype, operations=operations)+ self.mlp = MLP(device=device, dtype=dtype, operations=operations)+ self.mlp_mot_gen = MLP(device=device, dtype=dtype, operations=operations)+ self.input_layernorm = operations.RMSNorm(+ HIDDEN_SIZE, eps=1e-6, device=device, dtype=dtype+ )+ self.input_layernorm_mot_gen = operations.RMSNorm(+ HIDDEN_SIZE, eps=1e-6, device=device, dtype=dtype+ )+ self.post_attention_layernorm = operations.RMSNorm(+ HIDDEN_SIZE, eps=1e-6, device=device, dtype=dtype+ )+ self.post_attention_layernorm_mot_gen = operations.RMSNorm(+ HIDDEN_SIZE, eps=1e-6, device=device, dtype=dtype+ )++ def forward_prefix(self, prefix, prefix_rope, prefix_mask, transformer_options):+ prefix_attention, prefix_key, prefix_value = self.self_attn.forward_prefix(+ self.input_layernorm(prefix),+ prefix_rope,+ prefix_mask,+ transformer_options,+ )+ prefix = prefix + prefix_attention+ prefix = prefix + self.mlp(self.post_attention_layernorm(prefix))+ return prefix, prefix_key, prefix_value++ def forward_generation(+ self, image, image_rope, prefix_key, prefix_value, transformer_options+ ):+ image_attention = self.self_attn.forward_generation(+ self.input_layernorm_mot_gen(image),+ image_rope,+ prefix_key,+ prefix_value,+ transformer_options,+ )+ image = image + image_attention+ image = image + self.mlp_mot_gen(self.post_attention_layernorm_mot_gen(image))+ return image+++class LanguageBackbone(nn.Module):+ def __init__(self, device=None, dtype=None, operations=None):+ super().__init__()+ self.embed_tokens = operations.Embedding(+ VOCAB_SIZE, HIDDEN_SIZE, padding_idx=151643, device=device, dtype=dtype+ )+ self.layers = nn.ModuleList(+ DecoderLayer(device=device, dtype=dtype, operations=operations)+ for _ in range(NUM_LAYERS)+ )+ self.norm = operations.RMSNorm(+ HIDDEN_SIZE, eps=1e-6, device=device, dtype=dtype+ )+ self.norm_mot_gen = operations.RMSNorm(+ HIDDEN_SIZE, eps=1e-6, device=device, dtype=dtype+ )+++class LanguageModel(nn.Module):+ def __init__(self, device=None, dtype=None, operations=None):+ super().__init__()+ self.model = LanguageBackbone(device=device, dtype=dtype, operations=operations)+++class ConvDecoder(nn.Module):+ def __init__(self, device=None, dtype=None, operations=None):+ super().__init__()+ self.ps1 = nn.PixelShuffle(2)+ self.conv1 = operations.Conv2d(+ 1024, 1024, kernel_size=3, padding=1, device=device, dtype=dtype+ )+ self.act1 = nn.GELU()+ self.ps2 = nn.PixelShuffle(2)+ self.conv2 = operations.Conv2d(+ 256, 192, kernel_size=3, padding=1, device=device, dtype=dtype+ )+ self.ps3 = nn.PixelShuffle(8)++ def forward(self, hidden_states):+ hidden_states = self.act1(self.conv1(self.ps1(hidden_states)))+ return self.ps3(self.conv2(self.ps2(hidden_states)))+++class SenseNovaU15(nn.Module):+ def __init__(+ self, image_model=None, dtype=None, device=None, operations=None, **kwargs+ ):+ super().__init__()+ self.dtype = dtype+ self.vision_model = VisionModel(+ device=device, dtype=dtype, operations=operations+ )+ self.language_model = LanguageModel(+ device=device, dtype=dtype, operations=operations+ )+ self.fm_modules = nn.ModuleDict(+ {+ "vision_model_mot_gen": VisionModel(+ device=device, dtype=dtype, operations=operations+ ),+ "timestep_embedder": TimestepEmbedder(+ HIDDEN_SIZE, device=device, dtype=dtype, operations=operations+ ),+ "fm_head": ConvDecoder(+ device=device, dtype=dtype, operations=operations+ ),+ "noise_scale_embedder": TimestepEmbedder(+ HIDDEN_SIZE, device=device, dtype=dtype, operations=operations+ ),+ }+ )++ def forward(self, x, timesteps, context=None, transformer_options={}, **kwargs):+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(+ self._forward,+ self,+ comfy.patcher_extension.get_all_wrappers(+ comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options+ ),+ ).execute(x, timesteps, context, transformer_options, **kwargs)++ def _prepare_prefix(+ self, text_input_ids, reference_images, prefix_indexes, prefix_mask+ ):+ prefix = self.language_model.model.embed_tokens(text_input_ids)+ if reference_images:+ reference_embeds = [+ self.vision_model(_pad_to_merged_patch_size(reference))+ for reference in reference_images+ ]+ selected = text_input_ids == 151669+ prefix = prefix.clone()+ prefix[selected] = torch.cat(reference_embeds, dim=1).reshape(+ -1, HIDDEN_SIZE+ )++ prefix_length = text_input_ids.shape[1]+ if prefix_indexes is None:+ prefix_positions = torch.arange(+ prefix_length, dtype=torch.long, device=prefix.device+ )+ zeros = torch.zeros_like(prefix_positions)+ prefix_indexes = torch.stack((prefix_positions, zeros, zeros))+ prefix_mask = torch.full(+ (prefix_length, prefix_length),+ float("-inf"),+ dtype=prefix.dtype,+ device=prefix.device,+ ).triu(1)+ prefix_time = torch.full(+ (prefix.shape[0],),+ prefix_length,+ dtype=torch.long,+ device=prefix.device,+ )+ else:+ prefix_indexes = prefix_indexes.transpose(0, 1)+ prefix_time = prefix_indexes[0].amax(dim=-1) + 1++ return prefix, prefix_indexes, prefix_mask, prefix_time++ def preprocess_prefix(+ self,+ text_input_ids,+ reference_images=None,+ prefix_indexes=None,+ prefix_mask=None,+ ):+ prefix, prefix_indexes, prefix_mask, prefix_time = self._prepare_prefix(+ text_input_ids, reference_images, prefix_indexes, prefix_mask+ )+ prefix_keys = []+ prefix_values = []+ prefix_rope = _prepare_mrope(prefix_indexes, prefix.device, prefix.dtype)+ transformer_options = {}+ for layer_index, layer in enumerate(self.language_model.model.layers):+ transformer_options["block_index"] = layer_index+ prefix, prefix_key, prefix_value = layer.forward_prefix(+ prefix,+ prefix_rope,+ prefix_mask,+ transformer_options,+ )+ prefix_keys.append(prefix_key)+ prefix_values.append(prefix_value)+ return prefix_keys, prefix_values, prefix_time++ def _forward(+ self,+ x,+ timesteps,+ context=None,+ transformer_options={},+ text_input_ids=None,+ reference_images=None,+ prefix_indexes=None,+ prefix_mask=None,+ prefix_keys=None,+ prefix_values=None,+ prefix_time=None,+ **kwargs,+ ):+ if text_input_ids is None and prefix_keys is None:+ raise ValueError("SenseNova-U1.5 requires text conditioning")++ original_height, original_width = x.shape[-2:]+ x = _pad_to_merged_patch_size(x)+ batch, _, height, width = x.shape+ if prefix_keys is None:+ text_input_ids, prefix_indexes, prefix_mask = _match_prefix_batch(+ batch, text_input_ids, prefix_indexes, prefix_mask+ )+ prefix_batch = text_input_ids.shape[0]+ if reference_images:+ reference_images = [+ comfy.utils.resize_to_batch_size(reference, prefix_batch)+ for reference in reference_images+ ]+ else:+ reference_images = None+ else:+ prefix_batch = prefix_keys[0].shape[0]+ if prefix_batch > 0 and batch % prefix_batch:+ prefix_keys = [+ comfy.utils.resize_to_batch_size(value, batch)+ for value in prefix_keys+ ]+ prefix_values = [+ comfy.utils.resize_to_batch_size(value, batch)+ for value in prefix_values+ ]+ prefix_time = comfy.utils.resize_to_batch_size(prefix_time, batch)+ prefix_batch = batch+ generation_batch = _generation_batch_size(batch, prefix_batch)+ token_height = height // MERGED_PATCH_SIZE+ token_width = width // MERGED_PATCH_SIZE+ image_length = token_height * token_width++ image = self.fm_modules["vision_model_mot_gen"](x)+ time_embedding = self.fm_modules["timestep_embedder"](timesteps, image.dtype)+ noise_scale = resolution_noise_scale(height, width) / 16.0+ scale_timesteps = torch.full_like(timesteps, noise_scale)+ time_embedding = time_embedding + self.fm_modules["noise_scale_embedder"](+ scale_timesteps, image.dtype+ )+ image = image + time_embedding[:, None, :]++ if prefix_keys is None:+ prefix, prefix_indexes, prefix_mask, prefix_time = self._prepare_prefix(+ text_input_ids, reference_images, prefix_indexes, prefix_mask+ )+ prefix_rope = _prepare_mrope(prefix_indexes, prefix.device, prefix.dtype)+ image_time = prefix_time.repeat_interleave(generation_batch)++ image_positions = torch.arange(image_length, dtype=torch.long, device=x.device)+ image_indexes = torch.stack(+ (+ image_time[:, None].expand(batch, image_length),+ (image_positions // token_width)[None].expand(batch, image_length),+ (image_positions % token_width)[None].expand(batch, image_length),+ )+ )+ image_rope = _prepare_mrope(image_indexes, image.device, image.dtype)++ for layer_index, layer in enumerate(self.language_model.model.layers):+ transformer_options["block_index"] = layer_index+ if prefix_keys is None:+ prefix, prefix_key, prefix_value = layer.forward_prefix(+ prefix,+ prefix_rope,+ prefix_mask,+ transformer_options,+ )+ else:+ prefix_key = prefix_keys[layer_index]+ prefix_value = prefix_values[layer_index]+ generation_prefix_key = _expand_prefix_batch(prefix_key, generation_batch)+ generation_prefix_value = _expand_prefix_batch(+ prefix_value, generation_batch+ )+ image = layer.forward_generation(+ image,+ image_rope,+ generation_prefix_key,+ generation_prefix_value,+ transformer_options,+ )++ image = self.language_model.model.norm_mot_gen(image)+ image = image.view(batch, token_height, token_width, HIDDEN_SIZE).permute(+ 0, 3, 1, 2+ )+ predicted = self.fm_modules["fm_head"](image)+ denominator = (1.0 - timesteps).clamp_min(0.02).view(batch, 1, 1, 1)+ velocity = (x - predicted) / denominator+ return velocity[..., :original_height, :original_width]diff --git a/comfy/ldm/sensenova/sampling.py b/comfy/ldm/sensenova/sampling.pynew file mode 100644index 000000000..c6043038d--- /dev/null+++ b/comfy/ldm/sensenova/sampling.py@@ -0,0 +1,69 @@+import math++import torch++import comfy.model_sampling+++def time_snr_shift(shift, value):+ if shift == 1.0:+ return value+ return shift * value / (1.0 + (shift - 1.0) * value)+++def inverse_time_snr_shift(shift, value):+ if shift == 1.0:+ return value+ return value / (shift - (shift - 1.0) * value)+++def upstream_timesteps(steps, shift, device=None):+ base = torch.linspace(0.0, 1.0, steps + 1, device=device)+ return 1.0 - time_snr_shift(shift, 1.0 - base)+++def upstream_sigmas(steps, shift, device=None):+ return 1.0 - upstream_timesteps(steps, shift, device=device)+++def resolution_noise_scale(+ height, width, base_seq_len=64, noise_scale=1.0, maximum=16.0+):+ token_height = math.ceil(height / 32)+ token_width = math.ceil(width / 32)+ scale = math.sqrt(token_height * token_width / base_seq_len) * noise_scale+ return min(scale, maximum)+++class SenseNovaModelSampling(+ comfy.model_sampling.ModelSamplingDiscreteFlow, comfy.model_sampling.CONST+):+ def set_parameters(self, shift=1.0, timesteps=1000, multiplier=1000):+ self.shift = shift+ self.multiplier = multiplier+ base_timesteps = torch.linspace(multiplier, 0.0, timesteps + 1)+ self.register_buffer("sigmas", self.sigma(base_timesteps))++ def timestep(self, sigma):+ base_sigma = inverse_time_snr_shift(self.shift, sigma)+ return (1.0 - base_sigma) * self.multiplier++ def sigma(self, timestep):+ base_sigma = 1.0 - timestep / self.multiplier+ return time_snr_shift(self.shift, base_sigma)++ def percent_to_sigma(self, percent):+ if percent <= 0.0:+ return 1.0+ if percent >= 1.0:+ return 0.0+ return float(time_snr_shift(self.shift, 1.0 - percent))++ def noise_scaling(self, sigma, noise, latent_image, max_denoise=False):+ sigma = comfy.model_sampling.reshape_sigma(sigma, noise.ndim)+ scale = resolution_noise_scale(+ latent_image.shape[-2],+ latent_image.shape[-1],+ noise_scale=self.noise_scale,+ )+ return sigma * (scale * noise) + (1.0 - sigma) * latent_imagediff --git a/comfy/model_base.py b/comfy/model_base.pyindex ba59d2705..c6fa1d68c 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -76,6 +76,9 @@ import comfy.ldm.ernie.modelimport comfy.ldm.sam3.detectorimport comfy.ldm.hidream_o1.modelfrom comfy.ldm.hidream_o1.conditioning import build_extra_conds+import comfy.ldm.sensenova.conditioning+import comfy.ldm.sensenova.model+from comfy.ldm.sensenova.sampling import SenseNovaModelSampling, time_snr_shiftimport comfy.ldm.depth_anything_3.modelimport comfy.model_management@@ -2342,6 +2345,134 @@ class HiDreamO1(BaseModel):out[k] = cls(v)return out+class SenseNovaSharedRegular(comfy.conds.CONDRegular):+ """Keep the shared text/reference prefix at one copy per guidance branch."""++ def process_cond(self, batch_size, **kwargs):+ return self._copy_with(self.cond)++class SenseNovaSharedList(comfy.conds.CONDList):+ def process_cond(self, batch_size, **kwargs):+ return self._copy_with(self.cond)++class SenseNovaU15(BaseModel):+ PATCH_SIZE = 32++ def __init__(self, model_config, model_type=ModelType.FLOW, device=None):+ super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.sensenova.model.SenseNovaU15)+ self.model_sampling = SenseNovaModelSampling(model_config)+ self.memory_usage_factor_conds = ("reference_images",)++ def process_timestep(self, timestep, **kwargs):+ base_timestep = timestep / self.model_sampling.multiplier+ return 1.0 - time_snr_shift(self.model_sampling.shift, 1.0 - base_timestep)++ def extra_conds(self, **kwargs):+ out = super().extra_conds(**kwargs)+ text_input_ids = kwargs.get("text_input_ids")+ if text_input_ids is not None:+ device = kwargs["device"]+ reference_images = kwargs.get("reference_latents")+ if reference_images is not None:+ reference_images = comfy.ldm.sensenova.conditioning.preprocess_references(reference_images)+ image_only = kwargs.get("prompt_type") == "negative"+ indexes = None+ prefix_mask = None+ if reference_images:+ reference_grids = [+ (+ max(1, math.ceil(image.shape[-2] / self.PATCH_SIZE)),+ max(1, math.ceil(image.shape[-1] / self.PATCH_SIZE)),+ )+ for image in reference_images+ ]+ text_input_ids = comfy.ldm.sensenova.conditioning.condition_input_ids(+ text_input_ids,+ reference_grids,+ image_only=image_only,+ )+ indexes = comfy.ldm.sensenova.conditioning.thw_indexes(text_input_ids, reference_grids)+ prefix_mask = comfy.ldm.sensenova.conditioning.block_causal_mask(+ indexes, dtype=self.get_dtype_inference()+ )++ if kwargs.get("hooks") is None:+ dtype = self.get_dtype_inference()+ prefix_keys, prefix_values, prefix_time = (+ self.diffusion_model.preprocess_prefix(+ text_input_ids.to(device=device),+ [+ image.to(device=device, dtype=dtype)+ for image in reference_images+ ]+ if reference_images+ else None,+ indexes.to(device=device) if indexes is not None else None,+ prefix_mask.to(device=device)+ if prefix_mask is not None+ else None,+ )+ )+ out["prefix_keys"] = SenseNovaSharedList(prefix_keys)+ out["prefix_values"] = SenseNovaSharedList(prefix_values)+ out["prefix_time"] = SenseNovaSharedRegular(prefix_time)+ else:+ if reference_images:+ out["prefix_indexes"] = SenseNovaSharedRegular(indexes)+ out["prefix_mask"] = SenseNovaSharedRegular(prefix_mask)+ out["reference_images"] = SenseNovaSharedList(reference_images)+ out["text_input_ids"] = SenseNovaSharedRegular(text_input_ids)+ return out++ def extra_conds_shapes(self, **kwargs):+ images = kwargs.get("reference_latents")+ images = comfy.ldm.sensenova.conditioning.split_reference_batches(images) if images is not None else []+ reference_grids = [+ (+ max(1, math.ceil(image.shape[-3] / self.PATCH_SIZE)),+ max(1, math.ceil(image.shape[-2] / self.PATCH_SIZE)),+ )+ for image in images+ ]+ reference_pixels = sum(+ height * width * self.PATCH_SIZE**2+ for height, width in reference_grids+ )+ out = {}+ if reference_pixels:+ out["reference_images"] = [1, 3, reference_pixels]+ text_input_ids = kwargs.get("text_input_ids")+ if text_input_ids is not None:+ if reference_grids:+ length = comfy.ldm.sensenova.conditioning.conditioned_input_length(+ text_input_ids.shape[1],+ reference_grids,+ image_only=kwargs.get("prompt_type") == "negative",+ )+ else:+ length = text_input_ids.shape[1]+ out["prefix_mask"] = [1, 1, length, length]+ if kwargs.get("hooks") is None:+ prefix_shape = [+ 1,+ comfy.ldm.sensenova.model.NUM_KV_HEADS,+ comfy.ldm.sensenova.model.NUM_LAYERS+ * length+ * comfy.ldm.sensenova.model.HEAD_DIM,+ ]+ out["prefix_keys"] = prefix_shape+ out["prefix_values"] = prefix_shape+ return out++ def memory_required(self, input_shape, cond_shapes={}):+ memory = super().memory_required(input_shape, cond_shapes)+ dtype_size = comfy.model_management.dtype_size(self.get_dtype_inference())+ return memory + sum(+ math.prod(shape) * dtype_size+ for key in ("prefix_mask", "prefix_keys", "prefix_values")+ for shape in cond_shapes.get(key, ())+ )+class Chroma(Flux):def __init__(self, model_config, model_type=ModelType.FLUX, device=None, unet_model=comfy.ldm.chroma.model.Chroma):super().__init__(model_config, model_type, device=device, unet_model=unet_model)diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex 3bfc20a36..098e6a088 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -815,6 +815,16 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):if '{}t_embedder1.mlp.0.weight'.format(key_prefix) in state_dict_keys and '{}x_embedder.proj1.weight'.format(key_prefix) in state_dict_keys: # HiDream-O1return {"image_model": "hidream_o1"}+ vision_key = f"{key_prefix}fm_modules.vision_model_mot_gen.embeddings.patch_embedding.weight"+ query_key = f"{key_prefix}language_model.model.layers.0.self_attn.q_proj_mot_gen.weight"+ if (+ vision_key in state_dict+ and query_key in state_dict+ and state_dict[vision_key].shape[0] == 1024+ and state_dict[query_key].shape[0] == 4096+ ): # SenseNova U1.5+ return {"image_model": "sensenova_u15"}+if '{}caption_projection.0.linear.weight'.format(key_prefix) in state_dict_keys: # HiDreamdit_config = {}dit_config["image_model"] = "hidream"@@ -1301,6 +1311,13 @@ def unet_prefix_from_state_dict(state_dict):if any(k.startswith("detector.") for k in state_dict) and any(k.startswith("tracker.") for k in state_dict):return ""+ # SenseNova checkpoints store the diffusion and language backbones at top level.+ if (+ "fm_modules.vision_model_mot_gen.embeddings.patch_embedding.weight" in state_dict+ and "language_model.model.layers.0.self_attn.q_proj_mot_gen.weight" in state_dict+ ):+ return ""+candidates = ["model.diffusion_model.", #ldm/sgm models"model.model.", #audio models"net.", #cosmosdiff --git a/comfy/supported_models.py b/comfy/supported_models.pyindex d2b4e6af2..ae2d37dad 100644--- a/comfy/supported_models.py+++ b/comfy/supported_models.py@@ -37,6 +37,7 @@ import comfy.text_encoders.longcat_imageimport comfy.text_encoders.ernieimport comfy.text_encoders.cogvideoimport comfy.text_encoders.hidream_o1+import comfy.text_encoders.sensenovaimport comfy.text_encoders.pixelditfrom . import supported_models_base@@ -1720,6 +1721,44 @@ class HiDreamO1(supported_models_base.BASE):comfy.text_encoders.hidream_o1.HiDreamO1TE,)+class SenseNovaU15(supported_models_base.BASE):+ unet_config = {+ "image_model": "sensenova_u15",+ }++ sampling_settings = {+ "shift": 3.0,+ "noise_scale": 1.0,+ }++ latent_format = latent_formats.HiDreamO1Pixel+ memory_usage_factor = 0.033+ supported_inference_dtypes = [torch.bfloat16, torch.float32]++ vae_key_prefix = ["vae."]+ text_encoder_key_prefix = ["text_encoders."]++ optimizations = {"fp8": False}++ def get_model(self, state_dict, prefix="", device=None):+ return model_base.SenseNovaU15(self, device=device)++ def process_unet_state_dict(self, state_dict):+ state_dict.pop("language_model.lm_head.weight", None)+ return state_dict++ def process_vae_state_dict(self, state_dict):+ return {"pixel_space_vae": torch.tensor(1.0)}++ def process_clip_state_dict(self, state_dict):+ return {"_sensenova_te_sentinel": torch.zeros(1)}++ def clip_target(self, state_dict={}):+ return supported_models_base.ClipTarget(+ comfy.text_encoders.sensenova.SenseNovaTokenizer,+ comfy.text_encoders.sensenova.SenseNovaTextEncoder,+ )+class Chroma(supported_models_base.BASE):unet_config = {"image_model": "chroma",@@ -2537,6 +2576,7 @@ models = [TripoSplat,HiDream,HiDreamO1,+ SenseNovaU15,Chroma,SeedVR2,ChromaRadiance,diff --git a/comfy/text_encoders/sensenova.py b/comfy/text_encoders/sensenova.pynew file mode 100644index 000000000..598a771f0--- /dev/null+++ b/comfy/text_encoders/sensenova.py@@ -0,0 +1,149 @@+"""Tokenizer-only conditioning for SenseNova U1.5.++The language model is part of the diffusion checkpoint, so CLIP only needs to+produce token ids. SenseNova extends the Qwen vocabulary with image-control+tokens; their order is significant because the checkpoint embeds them by id.+"""++import os++import torch+from transformers import Qwen2Tokenizer++from comfy import sd1_clip+++SYSTEM_MESSAGE = (+ "You are an image generation and editing assistant that accurately understands and executes "+ "user intent.\n\nYou support two modes:\n\n1. Think Mode:\nIf the task requires reasoning, you "+ "MUST start with a <think></think> block. Put all reasoning inside the block using plain text. "+ "DO NOT include any image tags. Keep it reasonable and directly useful for producing the final "+ "image.\n\n2. Non-Think Mode:\nIf no reasoning is needed, directly produce the final image.\n\n"+ "Task Types:\n\nA. Text-to-Image Generation:\n"+ "- Generate a high-quality image based on the user's description.\n"+ "- Ensure visual clarity, semantic consistency, and completeness.\n"+ "- DO NOT introduce elements that contradict or override the user's intent.\n\n"+ "B. Image Editing:\n"+ "- Use the provided image(s) as input or reference for modification or transformation.\n"+ "- The result can be an edited image or a new image based on the reference(s).\n"+ "- Preserve all unspecified attributes unless explicitly changed.\n\n"+ "General Rules:\n"+ "- For any visible text in the image, follow the language specified for the rendered text in "+ "the user's description, not the language of the prompt. If no language is specified, use the "+ "user's input language."+)+++def build_generation_prompt(text):+ return (+ f"<|im_start|>system\n{SYSTEM_MESSAGE}<|im_end|>\n"+ f"<|im_start|>user\n{text}<|im_end|>\n"+ "<|im_start|>assistant\n<think>\n\n</think>\n\n<img>"+ )+++def build_unconditional_prompt():+ return "<|im_start|>user\n<|im_end|>\n<|im_start|>assistant\n<img>"+++class SenseNovaQwen2Tokenizer:+ @classmethod+ def from_pretrained(cls, *args, **kwargs):+ tokenizer = Qwen2Tokenizer.from_pretrained(*args, **kwargs)+ existing_special_tokens = [+ token+ for _, token in sorted(tokenizer.added_tokens_decoder.items())+ if token.special+ ]+ extra_tokens = [+ "<IMG_CONTEXT>",+ "<img>",+ "</img>",+ "<quad>",+ "</quad>",+ "<ref>",+ "</ref>",+ "<box>",+ "</box>",+ "<|action_start|>",+ "<|action_end|>",+ "<|plugin|>",+ "<|interpreter|>",+ ]+ extra_tokens.extend(f"<FAKE_PAD_{index}>" for index in range(254))+ tokenizer.add_special_tokens(+ {"additional_special_tokens": existing_special_tokens + extra_tokens}+ )+ return tokenizer+++class SenseNovaQwenTokenizer(sd1_clip.SDTokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ tokenizer_path = os.path.join(+ os.path.dirname(os.path.realpath(__file__)), "qwen25_tokenizer"+ )+ super().__init__(+ tokenizer_path,+ pad_with_end=False,+ embedding_size=4096,+ embedding_key="sensenova_u15",+ tokenizer_class=SenseNovaQwen2Tokenizer,+ has_start_token=False,+ has_end_token=False,+ pad_to_max_length=False,+ max_length=99999999,+ min_length=1,+ pad_token=151643,+ tokenizer_data=tokenizer_data,+ )+++class SenseNovaTokenizer(sd1_clip.SD1Tokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ super().__init__(+ embedding_directory=embedding_directory,+ tokenizer_data=tokenizer_data,+ name="sensenova_u15",+ tokenizer=SenseNovaQwenTokenizer,+ )++ def tokenize_with_weights(self, text, return_word_ids=False, **kwargs):+ prompt = build_generation_prompt(text) if text else build_unconditional_prompt()+ tokens = super().tokenize_with_weights(+ prompt,+ return_word_ids=return_word_ids,+ disable_weights=True,+ **kwargs,+ )+ values = tokens["sensenova_u15"][0]+ values = [value for value in values if int(value[0]) != 151643]+ return {"sensenova_u15": [values]}+++class SenseNovaTextEncoder(torch.nn.Module):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ super().__init__()+ self.dtypes = {torch.float32}+ self.disable_offload = True+ self.device = torch.device("cpu") if device is None else torch.device(device)++ def encode_token_weights(self, token_weight_pairs):+ pairs = token_weight_pairs["sensenova_u15"][0]+ input_ids = torch.tensor([[int(value[0]) for value in pairs]], dtype=torch.long)+ return (+ input_ids.unsqueeze(-1).to(torch.float32),+ None,+ {"text_input_ids": input_ids},+ )++ def load_sd(self, sd):+ return []++ def get_sd(self):+ return {}++ def reset_clip_options(self):+ pass++ def set_clip_options(self, options):+ passdiff --git a/comfy_extras/nodes_hidream_o1.py b/comfy_extras/nodes_hidream_o1.pyindex 85693fce6..caa980b6e 100644--- a/comfy_extras/nodes_hidream_o1.py+++ b/comfy_extras/nodes_hidream_o1.py@@ -8,6 +8,9 @@ import node_helpersfrom comfy_api.latest import ComfyExtension, io+REFERENCE_IMAGE_INPUT_SLOTS = 100++class EmptyHiDreamO1LatentImage(io.ComfyNode):@classmethoddef define_schema(cls) -> io.Schema:@@ -40,8 +43,6 @@ class EmptyHiDreamO1LatentImage(io.ComfyNode):class HiDreamO1ReferenceImages(io.ComfyNode):- """Attach reference images to both positive and negative conditioning."""-@classmethoddef define_schema(cls) -> io.Schema:return io.Schema(@@ -49,9 +50,9 @@ class HiDreamO1ReferenceImages(io.ComfyNode):display_name="HiDream-O1 Reference Images",category="model/conditioning/hidream",description=(- "Attach 1-10 reference images to conditioning, one for edit instruction"- "or multiple for subject-driven personalization."+ "Attach ordered reference images to positive and negative conditioning."),+ search_aliases=["sensenova reference images"],inputs=[io.Conditioning.Input(id="positive"),io.Conditioning.Input(id="negative"),@@ -59,11 +60,14 @@ class HiDreamO1ReferenceImages(io.ComfyNode):"images",template=io.Autogrow.TemplateNames(io.Image.Input("image"),- names=[f"image_{i}" for i in range(1, 11)],- min=1,- ),- tooltip=("Reference images. 1 image = instruction edit; 2-10 images = multi reference."+ names=[+ f"image_{index}"+ for index in range(1, REFERENCE_IMAGE_INPUT_SLOTS + 1)+ ],+ min=0,),+ optional=True,+ tooltip="Reference images are used in numeric socket order.",),],outputs=[@@ -73,10 +77,31 @@ class HiDreamO1ReferenceImages(io.ComfyNode):)@classmethod- def execute(cls, *, positive, negative, images: io.Autogrow.Type) -> io.NodeOutput:- refs = [images[f"image_{i}"] for i in range(1, 11) if f"image_{i}" in images]- positive = node_helpers.conditioning_set_values(positive, {"reference_latents": refs}, append=True)- negative = node_helpers.conditioning_set_values(negative, {"reference_latents": refs}, append=True)+ def execute(+ cls, *, positive, negative, images: io.Autogrow.Type = None+ ) -> io.NodeOutput:+ images = images or {}+ ordered_names = [+ f"image_{index}"+ for index in range(1, REFERENCE_IMAGE_INPUT_SLOTS + 1)+ if f"image_{index}" in images+ ]+ known_names = set(ordered_names)+ refs = [images[name] for name in ordered_names]+ refs.extend(+ image for name, image in images.items() if name not in known_names+ )+ if not refs:+ return io.NodeOutput(positive, negative)+ positive = node_helpers.conditioning_set_values(+ positive, {"reference_latents": refs}, append=True+ )+ negative = node_helpers.conditioning_set_values(+ negative, {"prompt_type": "negative"}+ )+ negative = node_helpers.conditioning_set_values(+ negative, {"reference_latents": refs}, append=True+ )return io.NodeOutput(positive, negative)diff --git a/comfy_extras/nodes_sensenova.py b/comfy_extras/nodes_sensenova.pynew file mode 100644index 000000000..5ed13af5a--- /dev/null+++ b/comfy_extras/nodes_sensenova.py@@ -0,0 +1,40 @@+from typing_extensions import override++from comfy.ldm.sensenova.sampling import SenseNovaModelSampling+from comfy_api.latest import ComfyExtension, io+++class SenseNovaSamplingOptions(io.ComfyNode):+ @classmethod+ def define_schema(cls) -> io.Schema:+ return io.Schema(+ node_id="SenseNovaSamplingOptions",+ display_name="SenseNova Sampling Options",+ category="model/patch/sensenova",+ description="Set the SenseNova flow shift.",+ inputs=[+ io.Model.Input(id="model"),+ io.Float.Input(id="shift", default=3.0, step=0.01),+ ],+ outputs=[io.Model.Output()],+ )++ @classmethod+ def execute(cls, *, model, shift: float) -> io.NodeOutput:+ patched = model.clone()+ model_sampling = SenseNovaModelSampling(patched.model.model_config)+ model_sampling.set_parameters(shift=shift)+ patched.add_object_patch("model_sampling", model_sampling)+ return io.NodeOutput(patched)+++class SenseNovaExtension(ComfyExtension):+ @override+ async def get_node_list(self) -> list[type[io.ComfyNode]]:+ return [+ SenseNovaSamplingOptions,+ ]+++async def comfy_entrypoint() -> SenseNovaExtension:+ return SenseNovaExtension()diff --git a/nodes.py b/nodes.pyindex e20721442..b27a21463 100644--- a/nodes.py+++ b/nodes.py@@ -2521,6 +2521,7 @@ async def init_builtin_extra_nodes():"nodes_void.py","nodes_wandancer.py","nodes_hidream_o1.py",+ "nodes_sensenova.py","nodes_save_3d.py","nodes_mesh_io.py","nodes_moge.py",diff --git a/tests-unit/comfy_test/test_sensenova.py b/tests-unit/comfy_test/test_sensenova.pynew file mode 100644index 000000000..4e2398e41--- /dev/null+++ b/tests-unit/comfy_test/test_sensenova.py@@ -0,0 +1,657 @@+from types import SimpleNamespace++import torch++from comfy.cli_args import args++args.cpu = True++from comfy import model_base, model_detection+import comfy.latent_formats+import comfy.sample+import nodes+from comfy.ldm.sensenova import model as sensenova_model+from comfy.ldm.sensenova.conditioning import (+ block_causal_mask,+ condition_input_ids,+ conditioned_input_length,+ preprocess_references,+ thw_indexes,+)+from comfy.ldm.sensenova.model import _match_prefix_batch, _pad_to_merged_patch_size+from comfy.ldm.sensenova.sampling import (+ SenseNovaModelSampling,+ resolution_noise_scale,+ upstream_sigmas,+)+from comfy.text_encoders.sensenova import SenseNovaTokenizer+from comfy_extras.nodes_hidream_o1 import HiDreamO1ReferenceImages+from comfy_extras.nodes_sensenova import SenseNovaSamplingOptions+++def _minimal_state_dict():+ return {+ "fm_modules.vision_model_mot_gen.embeddings.patch_embedding.weight": torch.empty(+ 1024, 3, 16, 16, device="meta"+ ),+ "language_model.model.layers.0.self_attn.q_proj_mot_gen.weight": torch.empty(+ 4096, 4096, device="meta"+ ),+ }+++def _generation_input_ids():+ return torch.tensor(+ [+ [+ 151644,+ 8948,+ 198,+ 1,+ 151645,+ 198,+ 151644,+ 872,+ 198,+ 2,+ 151645,+ 198,+ 151644,+ 77091,+ 198,+ 151670,+ ]+ ],+ dtype=torch.long,+ )+++def _tokenize_generation_prompt(text):+ tokenizer = SenseNovaTokenizer()+ values = tokenizer.tokenize_with_weights(text)["sensenova_u15"][0]+ return torch.tensor([[int(value[0]) for value in values]], dtype=torch.long)+++def test_sensenova_top_level_checkpoint_detection():+ state_dict = _minimal_state_dict()++ assert model_detection.unet_prefix_from_state_dict(state_dict) == ""+ assert model_detection.detect_unet_config(state_dict, "") == {+ "image_model": "sensenova_u15"+ }+ assert (+ type(model_detection.model_config_from_unet(state_dict, "")).__name__+ == "SenseNovaU15"+ )+++def test_sensenova_detection_rejects_incompatible_dimensions():+ state_dict = _minimal_state_dict()+ state_dict["language_model.model.layers.0.self_attn.q_proj_mot_gen.weight"] = (+ torch.empty(2048, 2048, device="meta")+ )++ assert model_detection.detect_unet_config(state_dict, "") is None+++def test_sensenova_model_config_builds_pixel_space_outputs():+ model_config = model_detection.model_config_from_unet(_minimal_state_dict(), "")+ state_dict = {+ "language_model.lm_head.weight": torch.empty(1),+ "kept": torch.empty(1),+ }++ processed = model_config.process_unet_state_dict(state_dict)+ assert set(processed) == {"kept"}+ assert torch.equal(processed["kept"], state_dict["kept"])+ assert "pixel_space_vae" in model_config.process_vae_state_dict({})+ assert "_sensenova_te_sentinel" in model_config.process_clip_state_dict({})+++def test_sensenova_sampling_matches_upstream_schedule_and_resolution_scale():+ config = SimpleNamespace(sampling_settings={"shift": 3.0, "noise_scale": 1.0})+ sampling = SenseNovaModelSampling(config)++ expected = upstream_sigmas(50, 3.0)+ actual = sampling.sigma(torch.linspace(0.0, 1000.0, 51))+ assert torch.allclose(actual, expected)+ assert sampling.percent_to_sigma(0.0) == 1.0+ assert sampling.percent_to_sigma(1.0) == 0.0+ assert resolution_noise_scale(2048, 2048) == 8.0+ assert resolution_noise_scale(4096, 4096) == 16.0++ scaled_sampling = SenseNovaModelSampling(+ SimpleNamespace(sampling_settings={"shift": 3.0, "noise_scale": 0.5})+ )+ noise = torch.ones(1, 3, 256, 256)+ latent = torch.zeros_like(noise)+ scaled = scaled_sampling.noise_scaling(torch.ones(1), noise, latent)+ assert torch.allclose(scaled, torch.full_like(noise, 0.5))+++def test_shared_reference_images_append_when_chained():+ conditioning = [[torch.empty(1), {}]]+ first_image = torch.empty(1, 8, 8, 3)+ second_image = torch.empty(1, 8, 8, 3)++ first = HiDreamO1ReferenceImages.execute(+ positive=conditioning,+ negative=conditioning,+ images={"image_1": first_image},+ )+ second = HiDreamO1ReferenceImages.execute(+ positive=first[0],+ negative=first[1],+ images={"image_1": second_image},+ )++ references = second[0][0][1]["reference_latents"]+ assert len(references) == 2+ assert references[0] is first_image+ assert references[1] is second_image+ assert second[1][0][1]["reference_latents"] == references+ assert second[1][0][1]["prompt_type"] == "negative"+++def test_shared_reference_images_use_numeric_socket_order():+ conditioning = [[torch.empty(1), {}]]+ first_image = torch.empty(1, 8, 8, 3)+ second_image = torch.empty(1, 8, 8, 3)+ extra_image = torch.empty(1, 8, 8, 3)++ output = HiDreamO1ReferenceImages.execute(+ positive=conditioning,+ negative=conditioning,+ images={+ "image_2": second_image,+ "extra_image": extra_image,+ "image_1": first_image,+ },+ )++ references = output[0][0][1]["reference_latents"]+ assert references[0] is first_image+ assert references[1] is second_image+ assert references[2] is extra_image+++def test_shared_reference_images_allow_empty_inputs_and_image_batches():+ conditioning = [[torch.empty(1), {}]]+ empty = HiDreamO1ReferenceImages.execute(+ positive=conditioning,+ negative=conditioning,+ images={},+ )+ assert empty[0] is conditioning+ assert empty[1] is conditioning++ image_batch = torch.empty(2, 8, 8, 3)+ attached = HiDreamO1ReferenceImages.execute(+ positive=conditioning,+ negative=conditioning,+ images={"image_1": image_batch},+ )+ assert attached[0][0][1]["reference_latents"] == [image_batch]+++def test_hidream_o1_ignores_shared_negative_marker(monkeypatch):+ calls = []++ def build_extra_conds(text_input_ids, noise, ref_images, target_patch_size):+ calls.append((text_input_ids, noise, ref_images, target_patch_size))+ return {+ "input_ids": text_input_ids,+ "ar_len": text_input_ids.shape[1] - 1,+ }++ monkeypatch.setattr(model_base, "build_extra_conds", build_extra_conds)+ model = object.__new__(model_base.HiDreamO1)+ torch.nn.Module.__init__(model)+ model.concat_keys = ()+ input_ids = torch.tensor([[1, 2, 3]], dtype=torch.long)+ noise = torch.empty(1, 3, 64, 64)+ references = [torch.empty(1, 32, 32, 3)]++ positive = model.extra_conds(+ text_input_ids=input_ids,+ noise=noise,+ reference_latents=references,+ )+ negative = model.extra_conds(+ text_input_ids=input_ids,+ noise=noise,+ reference_latents=references,+ prompt_type="negative",+ )++ assert len(calls) == 2+ for call_input_ids, call_noise, call_references, call_patch_size in calls:+ assert call_input_ids is input_ids+ assert call_noise is noise+ assert call_references is references+ assert call_patch_size == 32+ assert positive.keys() == negative.keys()+ assert torch.equal(positive["input_ids"].cond, negative["input_ids"].cond)+ assert positive["ar_len"].cond == negative["ar_len"].cond+++def test_sensenova_reference_preprocessing_preserves_size_and_splits_batches():+ references = preprocess_references(+ [torch.rand(2, 9, 13, 1), torch.empty(1, 0, 0, 0)]+ )++ assert len(references) == 3+ assert all(reference.shape == (1, 3, 9, 13) for reference in references[:2])+ assert references[2].shape == (1, 3, 0, 0)+ assert _pad_to_merged_patch_size(references[0]).shape == (1, 3, 32, 32)+ assert _pad_to_merged_patch_size(references[2]).shape == (1, 3, 32, 32)+++def test_sensenova_reference_shape_estimate_uses_padded_image_sizes():+ model = object.__new__(model_base.SenseNovaU15)+ input_ids = _generation_input_ids()++ shapes = model.extra_conds_shapes(+ reference_latents=[torch.empty(2, 33, 65, 3)],+ text_input_ids=input_ids,+ )++ grids = [(2, 3), (2, 3)]+ length = conditioned_input_length(input_ids.shape[1], grids)+ assert shapes["reference_images"] == [1, 3, 12288]+ assert shapes["prefix_mask"] == [1, 1, length, length]+ prefix_shape = [+ 1,+ sensenova_model.NUM_KV_HEADS,+ sensenova_model.NUM_LAYERS * length * sensenova_model.HEAD_DIM,+ ]+ assert shapes["prefix_keys"] == prefix_shape+ assert shapes["prefix_values"] == prefix_shape++ negative_shapes = model.extra_conds_shapes(+ reference_latents=[torch.empty(2, 33, 65, 3)],+ text_input_ids=input_ids,+ prompt_type="negative",+ )+ negative_length = conditioned_input_length(+ input_ids.shape[1], grids, image_only=True+ )+ assert negative_shapes["prefix_mask"] == [+ 1,+ 1,+ negative_length,+ negative_length,+ ]+++def test_standard_empty_latent_adapts_to_sensenova_pixel_format():+ latent = nodes.EmptyLatentImage().generate(width=64, height=96, batch_size=2)[0]+ model = SimpleNamespace(+ get_model_object=lambda name: comfy.latent_formats.HiDreamO1Pixel()+ )++ samples = comfy.sample.fix_empty_latent_channels(+ model,+ latent["samples"],+ latent["downscale_ratio_spacial"],+ )++ assert samples.shape == (2, 3, 96, 64)+++def test_sensenova_prefix_conditioning_adapts_to_mismatched_batches():+ input_ids = torch.tensor([[1], [2]])+ indexes = torch.arange(6).reshape(2, 3, 1)+ mask = torch.zeros(2, 1, 1, 1)++ input_ids, indexes, mask = _match_prefix_batch(3, input_ids, indexes, mask)++ assert input_ids[:, 0].tolist() == [1, 2, 2]+ assert indexes.shape == (3, 3, 1)+ assert mask.shape == (3, 1, 1, 1)+++def test_reference_node_and_sensenova_sampling_do_not_add_quality_limits():+ sampling_inputs = {+ input.id: input for input in SenseNovaSamplingOptions.define_schema().inputs+ }+ assert sampling_inputs["shift"].min is None+ assert sampling_inputs["shift"].max is None++ reference_inputs = {+ input.id: input for input in HiDreamO1ReferenceImages.define_schema().inputs+ }+ images = reference_inputs["images"]+ assert images.optional+ assert images.template.min == 0+ assert len(images.template.names) == 100+++def test_sensenova_prefix_preprocessing_runs_each_prefix_layer_once(monkeypatch):+ calls = []+ rope_calls = []++ prepare_mrope = sensenova_model._prepare_mrope++ def tracked_prepare_mrope(indexes, device, dtype):+ rope_calls.append((indexes.shape, device, dtype))+ return prepare_mrope(indexes, device, dtype)++ monkeypatch.setattr(sensenova_model, "_prepare_mrope", tracked_prepare_mrope)++ class Layer:+ def forward_prefix(self, prefix, prefix_rope, prefix_mask, transformer_options):+ calls.append(+ (+ transformer_options["block_index"],+ tuple(axis[0].shape for axis in prefix_rope),+ prefix_mask.dtype,+ )+ )+ key = prefix[..., :1].unsqueeze(1)+ value = key + 1+ return prefix + 1, key, value++ input_ids = torch.tensor([[1, 2, 3]])+ model = SimpleNamespace(+ language_model=SimpleNamespace(+ model=SimpleNamespace(+ embed_tokens=lambda values: torch.zeros(+ *values.shape, sensenova_model.HIDDEN_SIZE+ ),+ layers=[Layer(), Layer()],+ )+ )+ )+ model._prepare_prefix = lambda *args: sensenova_model.SenseNovaU15._prepare_prefix(+ model, *args+ )++ prefix_keys, prefix_values, prefix_time = (+ sensenova_model.SenseNovaU15.preprocess_prefix(model, input_ids)+ )++ assert calls == [+ (+ 0,+ (+ torch.Size([1, 1, 3, 64]),+ torch.Size([1, 1, 3, 32]),+ torch.Size([1, 1, 3, 32]),+ ),+ torch.float32,+ ),+ (+ 1,+ (+ torch.Size([1, 1, 3, 64]),+ torch.Size([1, 1, 3, 32]),+ torch.Size([1, 1, 3, 32]),+ ),+ torch.float32,+ ),+ ]+ assert rope_calls == [(torch.Size([3, 3]), torch.device("cpu"), torch.float32)]+ assert len(prefix_keys) == 2+ assert len(prefix_values) == 2+ assert prefix_keys[0].shape == (1, 1, 3, 1)+ assert prefix_time.tolist() == [3]+++def test_sensenova_model_base_preprocesses_prefix_conditioning():+ calls = []++ def preprocess_prefix(input_ids, references, indexes, prefix_mask):+ calls.append((input_ids, references, indexes, prefix_mask))+ return (+ [torch.zeros(1, 1, 3, 1, dtype=torch.bfloat16)],+ [torch.ones(1, 1, 3, 1, dtype=torch.bfloat16)],+ torch.tensor([3]),+ )++ model = object.__new__(model_base.SenseNovaU15)+ torch.nn.Module.__init__(model)+ model.concat_keys = ()+ model.manual_cast_dtype = None+ model.diffusion_model = SimpleNamespace(+ dtype=torch.bfloat16,+ preprocess_prefix=preprocess_prefix,+ )+ input_ids = torch.tensor([[1, 2, 3]])++ conds = model.extra_conds(+ text_input_ids=input_ids,+ device=torch.device("cpu"),+ )++ assert len(calls) == 1+ assert calls[0][0] is input_ids+ assert calls[0][1:] == (None, None, None)+ assert "text_input_ids" not in conds+ assert conds["prefix_keys"].cond[0].dtype == torch.bfloat16+ assert conds["prefix_values"].cond[0].dtype == torch.bfloat16+ assert conds["prefix_time"].cond.tolist() == [3]+++def test_sensenova_uses_prompt_type_for_negative_reference_conditioning():+ calls = []++ def preprocess_prefix(input_ids, references, indexes, prefix_mask):+ calls.append((input_ids, references, indexes, prefix_mask))+ return (+ [torch.zeros(1, 1, 1, 1, dtype=torch.bfloat16)],+ [torch.ones(1, 1, 1, 1, dtype=torch.bfloat16)],+ torch.tensor([1]),+ )++ model = object.__new__(model_base.SenseNovaU15)+ torch.nn.Module.__init__(model)+ model.concat_keys = ()+ model.manual_cast_dtype = None+ model.diffusion_model = SimpleNamespace(+ dtype=torch.bfloat16,+ preprocess_prefix=preprocess_prefix,+ )+ input_ids = _generation_input_ids()+ reference = torch.rand(1, 32, 32, 3)++ model.extra_conds(+ text_input_ids=input_ids,+ reference_latents=[reference],+ prompt_type="negative",+ device=torch.device("cpu"),+ )++ expected_ids = condition_input_ids(input_ids, [(1, 1)], image_only=True)+ assert torch.equal(calls[0][0], expected_ids)+ assert len(calls[0][1]) == 1+ assert calls[0][1][0].shape == (1, 3, 32, 32)+ assert calls[0][2].shape == (1, 3, expected_ids.shape[1])+ assert calls[0][3].shape == (1, 1, expected_ids.shape[1], expected_ids.shape[1])+++def test_sensenova_preprocessed_prefix_matches_raw_forward():+ class Layer:+ def forward_prefix(self, prefix, prefix_rope, prefix_mask, transformer_options):+ key = prefix[..., :1].unsqueeze(1)+ return prefix + 1, key, key + 1++ def forward_generation(+ self, image, image_rope, prefix_key, prefix_value, transformer_options+ ):+ offset = (prefix_key + prefix_value).mean(dim=(1, 2, 3))+ return image + offset[:, None, None]++ class VisionModel:+ def __call__(self, image):+ batch, _, height, width = image.shape+ length = (height // sensenova_model.MERGED_PATCH_SIZE) * (+ width // sensenova_model.MERGED_PATCH_SIZE+ )+ return image.new_zeros(batch, length, sensenova_model.HIDDEN_SIZE)++ class TimestepEmbedder:+ def __init__(self):+ self.shapes = []++ def __call__(self, timesteps, dtype):+ self.shapes.append(timesteps.shape)+ return torch.zeros(+ timesteps.shape[0], sensenova_model.HIDDEN_SIZE, dtype=dtype+ )++ class Head:+ def __call__(self, image):+ return (+ image[:, :3]+ .repeat_interleave(sensenova_model.MERGED_PATCH_SIZE, dim=2)+ .repeat_interleave(sensenova_model.MERGED_PATCH_SIZE, dim=3)+ )++ timestep_embedder = TimestepEmbedder()+ noise_scale_embedder = TimestepEmbedder()+ backbone = SimpleNamespace(+ embed_tokens=lambda values: torch.zeros(+ *values.shape, sensenova_model.HIDDEN_SIZE+ ),+ layers=[Layer(), Layer()],+ norm_mot_gen=lambda image: image,+ )+ model = SimpleNamespace(+ language_model=SimpleNamespace(model=backbone),+ fm_modules={+ "vision_model_mot_gen": VisionModel(),+ "timestep_embedder": timestep_embedder,+ "noise_scale_embedder": noise_scale_embedder,+ "fm_head": Head(),+ },+ )+ model._prepare_prefix = lambda *args: sensenova_model.SenseNovaU15._prepare_prefix(+ model, *args+ )+ input_ids = torch.tensor([[1, 2, 3]])+ image = torch.zeros(1, 3, 64, 64)+ timesteps = torch.tensor([0.5])++ raw = sensenova_model.SenseNovaU15._forward(+ model,+ image,+ timesteps,+ text_input_ids=input_ids,+ transformer_options={},+ )+ prefix_keys, prefix_values, prefix_time = (+ sensenova_model.SenseNovaU15.preprocess_prefix(model, input_ids)+ )+ preprocessed = sensenova_model.SenseNovaU15._forward(+ model,+ image,+ timesteps,+ prefix_keys=prefix_keys,+ prefix_values=prefix_values,+ prefix_time=prefix_time,+ transformer_options={},+ )++ assert torch.equal(raw, preprocessed)+ assert timestep_embedder.shapes == [torch.Size([1]), torch.Size([1])]+ assert noise_scale_embedder.shapes == [torch.Size([1]), torch.Size([1])]+++def test_sensenova_reference_tokens_and_indexes():+ input_ids = _generation_input_ids()+ grids = [(2, 3), (1, 2)]++ conditioned = condition_input_ids(input_ids, grids)+ indexes = thw_indexes(conditioned, grids)++ assert conditioned.shape[1] == conditioned_input_length(input_ids.shape[1], grids)+ assert torch.count_nonzero(conditioned == 151669) == 8+ assert indexes.shape == (1, 3, conditioned.shape[1])+++def test_sensenova_prefix_mask_matches_attention_dtype(monkeypatch):+ query = torch.empty(1, 32, 3, 128, dtype=torch.bfloat16)+ key = torch.empty(1, 8, 3, 128, dtype=torch.bfloat16)+ value = torch.empty_like(key)+ captured = {}++ def optimized_attention(query, key, value, heads, **kwargs):+ captured.update(query=query, key=key, value=value, heads=heads, kwargs=kwargs)+ return torch.empty(1, 3, 4096, dtype=torch.bfloat16)++ monkeypatch.setattr(sensenova_model, "optimized_attention", optimized_attention)+ attention = SimpleNamespace(+ _project=lambda hidden_states, rope, generation: (query, key, value),+ o_proj=lambda output: output,+ )++ mask = torch.zeros(1, 1, 3, 3, dtype=torch.bfloat16)+ output, _, _ = sensenova_model.Attention.forward_prefix(+ attention,+ torch.empty(1, 3, 4096, dtype=torch.bfloat16),+ torch.empty(3, 1, 3),+ mask,+ {},+ )++ assert output.shape == (1, 3, 4096)+ assert captured["kwargs"]["mask"] is mask+++def test_sensenova_prefix_mask_is_created_in_the_model_dtype():+ indexes = torch.tensor([[[0, 1, 1], [0, 0, 0], [0, 0, 0]]])++ mask = block_causal_mask(indexes, dtype=torch.bfloat16)++ assert mask.dtype == torch.bfloat16+ assert torch.equal(+ mask[0, 0],+ torch.tensor(+ [+ [0.0, float("-inf"), float("-inf")],+ [0.0, 0.0, 0.0],+ [0.0, 0.0, 0.0],+ ],+ dtype=torch.bfloat16,+ ),+ )+++def test_sensenova_reference_tokens_allow_more_than_ten_images():+ input_ids = _tokenize_generation_prompt("test")+ grids = [(1, 1)] * 12++ conditioned = condition_input_ids(input_ids, grids)+ indexes = thw_indexes(conditioned, grids)+ expected_text = (+ "".join(+ f"Image-{index}:<img><IMG_CONTEXT></img>\n"+ for index in range(1, 13)+ )+ + "test"+ )++ assert torch.equal(conditioned, _tokenize_generation_prompt(expected_text))+ assert conditioned.shape[1] == conditioned_input_length(input_ids.shape[1], grids)+ assert torch.count_nonzero(conditioned == 151669) == 12+ assert indexes.shape == (1, 3, conditioned.shape[1])+++def test_sensenova_reference_tokens_tolerate_nonstandard_prompt_templates():+ conditioned = condition_input_ids(torch.tensor([[1, 2]]), [(1, 1)])++ assert torch.count_nonzero(conditioned == 151669) == 1+++def test_sensenova_tokenizer_control_token_ids():+ tokenizer = SenseNovaTokenizer()+ backend = tokenizer.sensenova_u15.tokenizer++ assert len(backend) == 151936+ assert backend.convert_tokens_to_ids(+ ["<IMG_CONTEXT>", "<img>", "</img>", "<FAKE_PAD_253>"]+ ) == [151669, 151670, 151671, 151935]+ assert "<|im_start|>" in backend.all_special_tokens+ assert "<|vision_pad|>" in backend.all_special_tokens+ assert tokenizer.tokenize_with_weights("")["sensenova_u15"][0][-1][0] == 151670