Skip to content

Transform your 3D texturing workflow with the power of generative AI, directly within Blender!

add support for flux ipadapter

sakalond3be1811f5695
diff --git a/stablegen/generator.py b/stablegen/generator.py
index e2de867..b80a274 100644
--- a/stablegen/generator.py
+++ b/stablegen/generator.py
@@ -574,8 +574,7 @@ class ComfyUIGenerate(bpy.types.Operator):
if context.scene.steps != 0:
# Generate image without ControlNet if needed
if camera_id == 0 and (context.scene.generation_method == 'sequential' or context.scene.generation_method == 'separate' or context.scene.generation_method == 'refine')\
- and context.scene.sequential_ipadapter and context.scene.sequential_ipadapter_regenerate and not context.scene.use_ipadapter and context.scene.sequential_ipadapter_mode == 'first'\
- and context.scene.model_architecture == 'sdxl':
+ and context.scene.sequential_ipadapter and context.scene.sequential_ipadapter_regenerate and not context.scene.use_ipadapter and context.scene.sequential_ipadapter_mode == 'first':
self._stage = "Generating Reference Image"
# Don't use ControlNet for the first image if sequential_ipadapter_regenerate_wo_controlnet is enabled
if context.scene.sequential_ipadapter_regenerate_wo_controlnet:
@@ -645,8 +644,7 @@ class ComfyUIGenerate(bpy.types.Operator):
# Use hack to re-generate the image using IPAdapter to match IPAdapter style
if camera_id == 0 and (context.scene.generation_method == 'sequential' or context.scene.generation_method == 'separate' or context.scene.generation_method == 'refine')\
- and context.scene.sequential_ipadapter and context.scene.sequential_ipadapter_regenerate and not context.scene.use_ipadapter and context.scene.sequential_ipadapter_mode == 'first'\
- and context.scene.model_architecture == 'sdxl':
+ and context.scene.sequential_ipadapter and context.scene.sequential_ipadapter_regenerate and not context.scene.use_ipadapter and context.scene.sequential_ipadapter_mode == 'first':
# Restore original strengths
if context.scene.sequential_ipadapter_regenerate_wo_controlnet:
@@ -1860,7 +1858,42 @@ class ComfyUIGenerate(bpy.types.Operator):
# Flux does not use negative prompt or LoRA.
return prompt, NODES
+
+ def configure_ipadapter_flux(self, prompt, context, NODES):
+ # Configure IPAdapter if enabled
+ from .util.helpers import ipadapter_flux
+ ipadapter_dict = json.loads(ipadapter_flux)
+ prompt.update(ipadapter_dict)
+
+ # Label nodes
+ NODES['ipadapter_loader'] = "242" # IPAdapterFluxLoader
+ NODES['ipadapter'] = "243" # ApplyIPAdapterFlux
+ NODES['ipadapter_image'] = "244" # LoadImage for IPAdapter input
+
+ # Connect IPAdapter output to guider and scheduler
+ prompt[NODES['guider']]["inputs"]["model"] = [NODES['ipadapter'], 0]
+ prompt[NODES['scheduler']]["inputs"]["model"] = [NODES['ipadapter'], 0]
+
+ # Set IPAdapter image source based on ipadapter_image
+ if context.scene.use_ipadapter:
+ image_path = bpy.path.abspath(context.scene.ipadapter_image)
+ prompt[NODES['ipadapter_image']]["inputs"]["image"] = image_path
+ elif context.scene.sequential_ipadapter:
+ if context.scene.sequential_ipadapter_mode == 'first':
+ prompt[NODES['ipadapter_image']]["inputs"]["image"] = get_file_path(context, "generated", camera_id=0, material_id=self._material_id)
+ else:
+ prompt[NODES['ipadapter_image']]["inputs"]["image"] = get_file_path(context, "generated", camera_id=self._current_image - 1, material_id=self._material_id)
+ # Connect ipadapter image to the input
+ prompt[NODES['ipadapter']]["inputs"]["image"] = [NODES['ipadapter_image'], 0]
+
+ # Configure IPAdapter settings
+ prompt[NODES['ipadapter']]["inputs"]["weight"] = context.scene.ipadapter_strength
+ prompt[NODES['ipadapter']]["inputs"]["start_percent"] = context.scene.ipadapter_start
+ prompt[NODES['ipadapter']]["inputs"]["end_percent"] = context.scene.ipadapter_end
+
+ # There is no weight type for Flux IPAdapter
+
def generate_flux(self, context, depth_path=None, canny_path=None, normal_path=None):
"""Generates an image using Flux 1.
Similar in structure to generate() but uses Flux nodes, skips negative prompt and LoRA.
@@ -1874,6 +1907,10 @@ class ComfyUIGenerate(bpy.types.Operator):
prompt, NODES = self.create_base_prompt_flux(context)
self._configure_resolution(prompt, context, NODES)
+
+ # Configure IPAdapter for Flux if enabled
+ if context.scene.use_ipadapter or (context.scene.generation_method == 'separate' and context.scene.sequential_ipadapter and self._current_image > 0):
+ self.configure_ipadapter_flux(prompt, context, NODES)
prompt, final_node = self._build_controlnet_chain_extended(
context, prompt, NODES['pos_prompt'], NODES['pos_prompt'], NODES['vae_loader'],
@@ -2024,6 +2061,10 @@ class ComfyUIGenerate(bpy.types.Operator):
# Initialize the img2img prompt template for Flux
prompt, NODES = self._create_img2img_base_prompt_flux(context)
+ # Configure IPAdapter for Flux if enabled
+ if context.scene.use_ipadapter or (context.scene.generation_method == 'separate' and context.scene.sequential_ipadapter and self._current_image > 0):
+ self.configure_ipadapter_flux(prompt, context, NODES)
+
# Configure based on generation method
self._configure_refinement_mode_flux(prompt, context, render_path, mask_path, NODES)
diff --git a/stablegen/project.py b/stablegen/project.py
index 454902f..6c6af89 100644
--- a/stablegen/project.py
+++ b/stablegen/project.py
@@ -301,7 +301,7 @@ def project_image(context, to_project, mat_id, stop_index=1000000):
if not context.scene.bake_texture:
image = bpy.data.images.load(get_file_path(context, "generated", camera_id=stop_index, material_id=mat_id))
if (context.scene.generation_method == 'sequential' or context.scene.generation_method == 'separate') and context.scene.sequential_ipadapter and context.scene.sequential_ipadapter_regenerate \
- and not context.scene.use_ipadapter and stop_index == 0 and context.scene.sequential_ipadapter_mode == 'first' and context.scene.model_architecture == 'sdxl':
+ and not context.scene.use_ipadapter and stop_index == 0 and context.scene.sequential_ipadapter_mode == 'first':
image = bpy.data.images.load(get_file_path(context, "generated", camera_id=stop_index, material_id=mat_id).replace(".png", "_ipadapter.png"))
else:
# Use baked texture
@@ -356,7 +356,7 @@ def project_image(context, to_project, mat_id, stop_index=1000000):
if not context.scene.bake_texture:
image = bpy.data.images.load(get_file_path(context, "generated", camera_id=i, material_id=mat_id))
if (context.scene.generation_method == 'sequential' or context.scene.generation_method == 'separate') and context.scene.sequential_ipadapter and context.scene.sequential_ipadapter_regenerate \
- and not context.scene.use_ipadapter and i == 0 and context.scene.sequential_ipadapter_mode == 'first' and context.scene.model_architecture == 'sdxl':
+ and not context.scene.use_ipadapter and i == 0 and context.scene.sequential_ipadapter_mode == 'first':
image = bpy.data.images.load(get_file_path(context, "generated", camera_id=i, material_id=mat_id).replace(".png", "_ipadapter.png"))
else:
# Use baked texture
@@ -534,7 +534,7 @@ def simple_project_bake(context, camera_id, obj, mat_id):
file_path = get_file_path(context, "generated", camera_id=camera_id, material_id=mat_id)
if (context.scene.generation_method == 'sequential' or context.scene.generation_method == 'separate') and context.scene.sequential_ipadapter and context.scene.sequential_ipadapter_regenerate \
- and not context.scene.use_ipadapter and camera_id == 0 and context.scene.sequential_ipadapter_mode == 'first' and context.scene.model_architecture == 'sdxl':
+ and not context.scene.use_ipadapter and camera_id == 0 and context.scene.sequential_ipadapter_mode == 'first':
file_path = get_file_path(context, "generated", camera_id=camera_id, material_id=mat_id).replace(".png", "_ipadapter.png")
tex_image.image = bpy.data.images.load(file_path)
diff --git a/stablegen/stablegen.py b/stablegen/stablegen.py
index 47356d2..0516f8c 100755
--- a/stablegen/stablegen.py
+++ b/stablegen/stablegen.py
@@ -498,7 +498,7 @@ class StableGenPanel(bpy.types.Panel):
content_box = draw_collapsible_section(advanced_params_box, "show_image_guidance_settings", "Image Guidance (IPAdapter & ControlNet)", icon="MODIFIER")
if content_box:
# IPAdapter Parameters
- if scene.model_architecture == 'sdxl' and not scene.generation_method == 'uv_inpaint':
+ if not scene.generation_method == 'uv_inpaint':
ipadapter_main_box = content_box.box() # Group IPAdapter settings together
row = ipadapter_main_box.row()
row.prop(scene, "use_ipadapter", text="Use IPAdapter (External image)", toggle=True, icon="MOD_MULTIRES")
@@ -515,8 +515,9 @@ class StableGenPanel(bpy.types.Panel):
row = sub_ip_box.row()
row.prop(scene, "ipadapter_end", text="End")
split = sub_ip_box.split(factor=0.5)
- split.label(text="Weight Type:")
- split.prop(scene, "ipadapter_weight_type", text="")
+ if context.scene.model_architecture == 'sdxl':
+ split.label(text="Weight Type:")
+ split.prop(scene, "ipadapter_weight_type", text="")
content_box.separator() # Separator between IPAdapter and ControlNet if both are shown
# ControlNet Parameters
@@ -637,7 +638,7 @@ class StableGenPanel(bpy.types.Panel):
split.prop(scene, "refine_upscale_method", text="")
# Separate Mode Parameters
- elif scene.generation_method == 'separate' and scene.model_architecture == 'sdxl':
+ elif scene.generation_method == 'separate':
row = mode_specific_outer_box.row()
row.alignment = 'CENTER'
row.label(text="Separate Mode Parameters", icon='FORCE_FORCE')
@@ -652,8 +653,9 @@ class StableGenPanel(bpy.types.Panel):
split.prop(scene, "sequential_ipadapter_mode", text="")
split = sub_ip_box_separate.split(factor=0.5)
- split.label(text="Weight Type:")
- split.prop(scene, "ipadapter_weight_type", text="")
+ if context.scene.model_architecture == 'sdxl':
+ split.label(text="Weight Type:")
+ split.prop(scene, "ipadapter_weight_type", text="")
row = sub_ip_box_separate.row()
row.prop(scene, "ipadapter_strength", text="Strength")
@@ -690,8 +692,9 @@ class StableGenPanel(bpy.types.Panel):
split.prop(scene, "sequential_ipadapter_mode", text="")
split = sub_ip_box_separate.split(factor=0.5)
- split.label(text="Weight Type:")
- split.prop(scene, "ipadapter_weight_type", text="")
+ if context.scene.model_architecture == 'sdxl':
+ split.label(text="Weight Type:")
+ split.prop(scene, "ipadapter_weight_type", text="")
row = sub_ip_box_separate.row()
row.prop(scene, "ipadapter_strength", text="Strength")
@@ -746,35 +749,35 @@ class StableGenPanel(bpy.types.Panel):
row = mode_specific_outer_box.row()
row.prop(scene, "sequential_factor_smooth_2", text="Smooth Visibility White Point")
- if scene.model_architecture == 'sdxl':
- row = mode_specific_outer_box.row()
- row.prop(scene, "sequential_ipadapter", text="Use IPAdapter for Sequential Mode", toggle=True, icon="MODIFIER")
- if scene.sequential_ipadapter:
- sub_ip_seq_box = mode_specific_outer_box.box()
-
- split = sub_ip_seq_box.split(factor=0.5)
- split.label(text="Mode:")
- split.prop(scene, "sequential_ipadapter_mode", text="")
+ row = mode_specific_outer_box.row()
+ row.prop(scene, "sequential_ipadapter", text="Use IPAdapter for Sequential Mode", toggle=True, icon="MODIFIER")
+ if scene.sequential_ipadapter:
+ sub_ip_seq_box = mode_specific_outer_box.box()
+
+ split = sub_ip_seq_box.split(factor=0.5)
+ split.label(text="Mode:")
+ split.prop(scene, "sequential_ipadapter_mode", text="")
- split = sub_ip_seq_box.split(factor=0.5)
+ split = sub_ip_seq_box.split(factor=0.5)
+ if context.scene.model_architecture == 'sdxl':
split.label(text="Weight Type:")
- split.prop(scene, "ipadapter_weight_type", text="")
+ split.prop(scene, "ipadapter_weight_type", text="")
+ row = sub_ip_seq_box.row()
+ row.prop(scene, "ipadapter_strength", text="Strength")
+ if width_mode == 'narrow':
row = sub_ip_seq_box.row()
- row.prop(scene, "ipadapter_strength", text="Strength")
- if width_mode == 'narrow':
- row = sub_ip_seq_box.row()
- row.prop(scene, "ipadapter_start", text="Start")
- if width_mode == 'narrow':
- row = sub_ip_seq_box.row()
- row.prop(scene, "ipadapter_end", text="End")
-
- if context.scene.sequential_ipadapter_mode == 'first':
+ row.prop(scene, "ipadapter_start", text="Start")
+ if width_mode == 'narrow':
+ row = sub_ip_seq_box.row()
+ row.prop(scene, "ipadapter_end", text="End")
+
+ if context.scene.sequential_ipadapter_mode == 'first':
+ row = sub_ip_seq_box.row()
+ row.prop(scene, "sequential_ipadapter_regenerate", text="Regenerate First Image", toggle=True, icon="FILE_REFRESH")
+ if context.scene.sequential_ipadapter_regenerate:
row = sub_ip_seq_box.row()
- row.prop(scene, "sequential_ipadapter_regenerate", text="Regenerate First Image", toggle=True, icon="FILE_REFRESH")
- if context.scene.sequential_ipadapter_regenerate:
- row = sub_ip_seq_box.row()
- row.prop(scene, "sequential_ipadapter_regenerate_wo_controlnet", text="Generate reference without ControlNet", toggle=True, icon="HIDE_OFF")
+ row.prop(scene, "sequential_ipadapter_regenerate_wo_controlnet", text="Generate reference without ControlNet", toggle=True, icon="HIDE_OFF")
# --- Tools ---
layout.separator()
diff --git a/stablegen/util/helpers.py b/stablegen/util/helpers.py
index 6685156..01b7c15 100755
--- a/stablegen/util/helpers.py
+++ b/stablegen/util/helpers.py
@@ -1167,3 +1167,51 @@ prompt_text_img2img_flux = """ {
}
}
"""
+
+ipadapter_flux = """
+{
+ "242": {
+ "inputs": {
+ "ipadapter": "ip-adapter.bin",
+ "clip_vision": "google/siglip-so400m-patch14-384",
+ "provider": "cuda"
+ },
+ "class_type": "IPAdapterFluxLoader",
+ "_meta": {
+ "title": "Load IPAdapter Flux Model"
+ }
+ },
+ "243": {
+ "inputs": {
+ "weight": 1,
+ "start_percent": 0,
+ "end_percent": 1,
+ "model": [
+ "12",
+ 0
+ ],
+ "ipadapter_flux": [
+ "242",
+ 0
+ ],
+ "image": [
+ "244",
+ 0
+ ]
+ },
+ "class_type": "ApplyIPAdapterFlux",
+ "_meta": {
+ "title": "Apply IPAdapter Flux Model"
+ }
+ },
+ "244": {
+ "inputs": {
+ "image": "00010-1613580828.jpeg"
+ },
+ "class_type": "LoadImage",
+ "_meta": {
+ "title": "Load Image"
+ }
+ }
+}
+"""