Skip to content

Transform your 3D texturing workflow with the power of generative AI, directly within Blender!

add support for flux.1-depth-dev lora

sakalondb3cc6ebc10c1
diff --git a/stablegen/__init__.py b/stablegen/__init__.py
index f172c67..7b43a9a 100755
--- a/stablegen/__init__.py
+++ b/stablegen/__init__.py
@@ -1236,6 +1236,13 @@ def register():
update=update_parameters
)
+ bpy.types.Scene.use_flux_lora = bpy.props.BoolProperty(
+ name="Use FLUX Depth LoRA",
+ description="Use FLUX.1-Depth-dev LoRA for depth conditioning instead of ControlNet. This disables all ControlNet units.",
+ default=True,
+ update=update_parameters
+ )
+
# IPADAPTER parameters
bpy.types.Scene.controlnet_units = bpy.props.CollectionProperty(type=ControlNetUnit)
@@ -1249,6 +1256,7 @@ def unregister():
Unregisters the addon.
:return: None
"""
+ del bpy.types.Scene.use_flux_lora
del bpy.types.Scene.comfyui_prompt
del bpy.types.Scene.comfyui_negative_prompt
del bpy.types.Scene.model_name
diff --git a/stablegen/generator.py b/stablegen/generator.py
index c54f5d4..53dcfd7 100644
--- a/stablegen/generator.py
+++ b/stablegen/generator.py
@@ -274,7 +274,7 @@ class ComfyUIGenerate(bpy.types.Operator):
# Check if there is at least one ControlNet unit
controlnet_units = getattr(context.scene, "controlnet_units", [])
- if not controlnet_units:
+ if not controlnet_units and not (context.scene.use_flux_lora and context.scene.model_architecture == 'flux1'):
self.report({'ERROR'}, "At least one ControlNet unit is required to run the operator.")
context.scene.generation_status = 'idle'
ComfyUIGenerate._is_running = False
@@ -367,7 +367,7 @@ class ComfyUIGenerate(bpy.types.Operator):
if not context.scene.project_only:
# If there is depth controlnet unit
- if any(unit["unit_type"] == "depth" for unit in controlnet_units):
+ if any(unit["unit_type"] == "depth" for unit in controlnet_units) or (context.scene.use_flux_lora and context.scene.model_architecture == 'flux1'):
if context.scene.generation_method != 'uv_inpaint':
# Export depth maps for each camera
for i, camera in enumerate(self._cameras):
@@ -1912,10 +1912,44 @@ class ComfyUIGenerate(bpy.types.Operator):
if context.scene.use_ipadapter or (context.scene.generation_method == 'separate' and context.scene.sequential_ipadapter and self._current_image > 0):
self.configure_ipadapter_flux(prompt, context, NODES)
- prompt, final_node = self._build_controlnet_chain_extended(
- context, prompt, NODES['pos_prompt'], NODES['pos_prompt'], NODES['vae_loader'],
- {"depth": depth_path, "canny": canny_path, "normal": normal_path}
- )
+ # Build ControlNet chain if not using Depth LoRA
+ if not context.scene.use_flux_lora:
+ prompt, final_node = self._build_controlnet_chain_extended(
+ context, prompt, NODES['pos_prompt'], NODES['pos_prompt'], NODES['vae_loader'],
+ {"depth": depth_path, "canny": canny_path, "normal": normal_path}
+ )
+ else: # If using Depth LoRA instead of ControlNet, we do not build a ControlNet chain
+ final_node = NODES['pos_prompt'] # Use positive prompt directly if not using ControlNet
+ # Add Required nodes for the FLUX.1-Depth-dev LoRA
+ from .util.helpers import depth_lora_flux
+ depth_lora_dict = json.loads(depth_lora_flux)
+ prompt.update(depth_lora_dict)
+
+ # Label nodes
+ NODES['flux_lora_image'] = "245" # LoadImage
+ NODES['instruct_pix'] = "246" # InstructPixToPixConditioning
+ NODES['flux_lora'] = "247" # LoraLoaderModelOnly
+
+ # Connect nodes
+ final_node = NODES['instruct_pix'] # To be connected to flux_guidance
+ prompt[NODES['sampler']]["inputs"]["latent_image"] = [NODES['instruct_pix'], 2]
+
+ # If using ipadapter, set the apply_ipadapter_flux node to use the flux_lora_image
+ if context.scene.use_ipadapter or (context.scene.generation_method == 'separate' and context.scene.sequential_ipadapter and self._current_image > 0):
+ prompt[NODES['ipadapter']]["inputs"]["model"] = [NODES['flux_lora'], 0]
+ else:
+ prompt[NODES['guider']]["inputs"]["model"] = [NODES['flux_lora'], 0]
+ prompt[NODES['scheduler']]["inputs"]["model"] = [NODES['flux_lora'], 0]
+
+ # Delete unnecessary nodes
+ if "239" in prompt:
+ del prompt["239"] # SetUnionControlNetType
+ if "30" in prompt:
+ del prompt["30"] # EmptyLatentImage
+
+ # Set the image for the Flux LoRA
+ prompt[NODES['flux_lora_image']]["inputs"]["image"] = depth_path
+
# Connect final node to FluxGuidance
prompt[NODES['flux_guidance']]["inputs"]["conditioning"] = [final_node, 0]
# Note: No negative prompt is connected.
@@ -2062,7 +2096,7 @@ class ComfyUIGenerate(bpy.types.Operator):
prompt, NODES = self._create_img2img_base_prompt_flux(context)
# Configure IPAdapter for Flux if enabled
- if context.scene.use_ipadapter or (context.scene.generation_method == 'separate' and context.scene.sequential_ipadapter and self._current_image > 0):
+ if (context.scene.use_ipadapter or (context.scene.sequential_ipadapter and self._current_image > 0)) and context.scene.generation_method != 'uv_inpaint':
self.configure_ipadapter_flux(prompt, context, NODES)
# Configure based on generation method
@@ -2071,9 +2105,51 @@ class ComfyUIGenerate(bpy.types.Operator):
# Set up image inputs for different controlnet types
self._refine_configure_images_flux(prompt, depth_path, canny_path, normal_path, render_path, NODES)
- # Build controlnet chain for refinement if needed
+ # Build ControlNet chain if not using Depth LoRA
if not context.scene.generation_method == 'uv_inpaint':
- prompt = self._refine_build_controlnet_chain_flux(prompt, context, depth_path, canny_path, normal_path, NODES)
+ if not context.scene.use_flux_lora:
+ prompt= self._refine_build_controlnet_chain_flux(
+ context, prompt, NODES['pos_prompt'], NODES['pos_prompt'], NODES['vae_loader'],
+ {"depth": depth_path, "canny": canny_path, "normal": normal_path}
+ )
+ else: # If using Depth LoRA instead of ControlNet, we do not build a ControlNet chain
+ final_node = NODES['pos_prompt'] # Use positive prompt directly if not using ControlNet
+ # Add Required nodes for the FLUX.1-Depth-dev LoRA
+ from .util.helpers import depth_lora_flux
+ depth_lora_dict = json.loads(depth_lora_flux)
+ prompt.update(depth_lora_dict)
+
+ # Label nodes
+ NODES['flux_lora_image'] = "245" # LoadImage
+ NODES['instruct_pix'] = "246" # InstructPixToPixConditioning
+ NODES['flux_lora'] = "247" # LoraLoaderModelOnly
+
+ # Configure InstructPixToPixConditioning inputs to InpaintModelConditioning if using differential diffusion
+ if context.scene.differential_diffusion:
+ prompt[NODES['instruct_pix']]["inputs"]["positive"] = [NODES['inpaint_conditioning'], 0]
+ prompt[NODES['instruct_pix']]["inputs"]["negative"] = [NODES['inpaint_conditioning'], 1]
+
+ # Connect nodes
+ prompt[NODES['flux_guidance']]["inputs"]["conditioning"] = [NODES['instruct_pix'], 0]
+ # prompt[NODES['sampler']]["inputs"]["latent_image"] = [NODES['instruct_pix'], 2] # Not doing since we need to respect the mask
+
+ # If using ipadapter, set the apply_ipadapter_flux node to use the flux_lora_image
+ if (context.scene.use_ipadapter or (context.scene.sequential_ipadapter and self._current_image > 0)) and context.scene.generation_method != 'uv_inpaint':
+ prompt[NODES['ipadapter']]["inputs"]["model"] = [NODES['flux_lora'], 0]
+ prompt[NODES['differential_diffusion']]["inputs"]["model"] = [NODES['ipadapter'], 0]
+ else:
+ prompt[NODES['guider']]["inputs"]["model"] = [NODES['flux_lora'], 0]
+ prompt[NODES['scheduler']]["inputs"]["model"] = [NODES['flux_lora'], 0]
+ prompt[NODES['differential_diffusion']]["inputs"]["model"] = [NODES['flux_lora'], 0]
+
+ # Delete unnecessary nodes
+ if "239" in prompt:
+ del prompt["239"] # SetUnionControlNetType
+ if "30" in prompt:
+ del prompt["30"] # EmptyLatentImage
+
+ # Set the image for the Flux LoRA
+ prompt[NODES['flux_lora_image']]["inputs"]["image"] = depth_path
# Save prompt for debugging
with open(os.path.join(output_dir, "prompt_flux_img2img.json"), 'w') as f:
diff --git a/stablegen/stablegen.py b/stablegen/stablegen.py
index 0516f8c..d41ae55 100755
--- a/stablegen/stablegen.py
+++ b/stablegen/stablegen.py
@@ -500,6 +500,9 @@ class StableGenPanel(bpy.types.Panel):
# IPAdapter Parameters
if not scene.generation_method == 'uv_inpaint':
ipadapter_main_box = content_box.box() # Group IPAdapter settings together
+ if scene.model_architecture == 'flux1':
+ row = ipadapter_main_box.row()
+ row.prop(scene, "use_flux_lora", text="Use Flux Depth LoRA", toggle=True, icon="MODIFIER")
row = ipadapter_main_box.row()
row.prop(scene, "use_ipadapter", text="Use IPAdapter (External image)", toggle=True, icon="MOD_MULTIRES")
if scene.use_ipadapter:
@@ -521,49 +524,50 @@ class StableGenPanel(bpy.types.Panel):
content_box.separator() # Separator between IPAdapter and ControlNet if both are shown
# ControlNet Parameters
- ctrl_box_group = content_box.box() # Group ControlNet settings together
- row = ctrl_box_group.row()
- row.alignment = 'CENTER'
- row.label(text="ControlNet Units", icon="NODETREE")
- for i, unit in enumerate(scene.controlnet_units):
- sub_unit_box = ctrl_box_group.box() # Each unit gets its own box
- row = sub_unit_box.row()
- row.label(text=f"Unit: {unit.unit_type.replace('_', ' ').title()}", icon="DOT")
- row.alignment = 'LEFT'
-
- if width_mode == 'narrow':
- split = sub_unit_box.split(factor=0.35, align=True)
- else:
- split = sub_unit_box.split(factor=0.2, align=True)
- split.label(text="Model:")
- split.prop(unit, "model_name", text="")
-
- row = sub_unit_box.row()
- row.prop(unit, "strength", text="Strength")
- if width_mode == 'narrow':
- row = sub_unit_box.row()
- row.prop(unit, "start_percent", text="Start")
- if width_mode == 'narrow':
+ if not (scene.model_architecture == 'flux1' and scene.use_flux_lora):
+ ctrl_box_group = content_box.box() # Group ControlNet settings together
+ row = ctrl_box_group.row()
+ row.alignment = 'CENTER'
+ row.label(text="ControlNet Units", icon="NODETREE")
+ for i, unit in enumerate(scene.controlnet_units):
+ sub_unit_box = ctrl_box_group.box() # Each unit gets its own box
row = sub_unit_box.row()
- row.prop(unit, "end_percent", text="End")
-
- if unit.unit_type == 'canny':
+ row.label(text=f"Unit: {unit.unit_type.replace('_', ' ').title()}", icon="DOT")
+ row.alignment = 'LEFT'
+
+ if width_mode == 'narrow':
+ split = sub_unit_box.split(factor=0.35, align=True)
+ else:
+ split = sub_unit_box.split(factor=0.2, align=True)
+ split.label(text="Model:")
+ split.prop(unit, "model_name", text="")
+
row = sub_unit_box.row()
- row.prop(scene, "canny_threshold_low", text="Canny Low")
+ row.prop(unit, "strength", text="Strength")
if width_mode == 'narrow':
row = sub_unit_box.row()
- row.prop(scene, "canny_threshold_high", text="Canny High")
- if hasattr(unit, 'is_union') and unit.is_union:
- row = sub_unit_box.row()
- row.prop(unit, "use_union_type", text="Set Union Type", toggle=True, icon="MOD_BOOLEAN")
-
- btn_row = ctrl_box_group.row(align=True)
- if width_mode == 'wide':
- btn_row.operator("stablegen.add_controlnet_unit", text="Add Unit", icon="ADD")
- btn_row.operator("stablegen.remove_controlnet_unit", text="Remove Unit", icon="REMOVE")
- else:
- ctrl_box_group.operator("stablegen.add_controlnet_unit", text="Add ControlNet Unit", icon="ADD")
- ctrl_box_group.operator("stablegen.remove_controlnet_unit", text="Remove Last ControlNet Unit", icon="REMOVE")
+ row.prop(unit, "start_percent", text="Start")
+ if width_mode == 'narrow':
+ row = sub_unit_box.row()
+ row.prop(unit, "end_percent", text="End")
+
+ if unit.unit_type == 'canny':
+ row = sub_unit_box.row()
+ row.prop(scene, "canny_threshold_low", text="Canny Low")
+ if width_mode == 'narrow':
+ row = sub_unit_box.row()
+ row.prop(scene, "canny_threshold_high", text="Canny High")
+ if hasattr(unit, 'is_union') and unit.is_union:
+ row = sub_unit_box.row()
+ row.prop(unit, "use_union_type", text="Set Union Type", toggle=True, icon="MOD_BOOLEAN")
+
+ btn_row = ctrl_box_group.row(align=True)
+ if width_mode == 'wide':
+ btn_row.operator("stablegen.add_controlnet_unit", text="Add Unit", icon="ADD")
+ btn_row.operator("stablegen.remove_controlnet_unit", text="Remove Unit", icon="REMOVE")
+ else:
+ ctrl_box_group.operator("stablegen.add_controlnet_unit", text="Add ControlNet Unit", icon="ADD")
+ ctrl_box_group.operator("stablegen.remove_controlnet_unit", text="Remove Last ControlNet Unit", icon="REMOVE")
# --- Inpainting Options (Conditional) ---
diff --git a/stablegen/util/helpers.py b/stablegen/util/helpers.py
index 01b7c15..08e9e9b 100755
--- a/stablegen/util/helpers.py
+++ b/stablegen/util/helpers.py
@@ -1206,7 +1206,7 @@ ipadapter_flux = """
},
"244": {
"inputs": {
- "image": "00010-1613580828.jpeg"
+ "image": ""
},
"class_type": "LoadImage",
"_meta": {
@@ -1215,3 +1215,56 @@ ipadapter_flux = """
}
}
"""
+
+depth_lora_flux = """
+{
+ "245": {
+ "inputs": {
+ "image": ""
+ },
+ "class_type": "LoadImage",
+ "_meta": {
+ "title": "Load Image"
+ }
+ },
+ "246": {
+ "inputs": {
+ "positive": [
+ "6",
+ 0
+ ],
+ "negative": [
+ "6",
+ 0
+ ],
+ "vae": [
+ "10",
+ 0
+ ],
+ "pixels": [
+ "245",
+ 0
+ ]
+ },
+ "class_type": "InstructPixToPixConditioning",
+ "_meta": {
+ "title": "InstructPixToPixConditioning"
+ }
+ },
+ "247": {
+ "inputs": {
+ "lora_name": "flux1-depth-dev-lora.safetensors",
+ "strength_model": 1,
+ "model": [
+ "12",
+ 0
+ ]
+ },
+ "class_type": "LoraLoaderModelOnly",
+ "_meta": {
+ "title": "LoraLoaderModelOnly"
+ }
+ }
+}
+"""
+