Skip to content

Transform your 3D texturing workflow with the power of generative AI, directly within Blender!

harden unattended texture batch recovery

sakalond47d0fef03394
diff --git a/stablegen/dae_import/batch_texture.py b/stablegen/dae_import/batch_texture.py
index bd4be31..0e82b1e 100644
--- a/stablegen/dae_import/batch_texture.py
+++ b/stablegen/dae_import/batch_texture.py
@@ -6,7 +6,9 @@ user intervention. Follows the same timer-based state-machine pattern
as :pymod:`mesh_gen.batch`.
"""
+import json
import os
+from datetime import datetime
import bpy # pylint: disable=import-error
from ..utils import sg_modal_active
@@ -31,11 +33,16 @@ _batch_state = {
'out_dir': '',
'phase': 'idle', # idle | import | cameras | texturing | bake | export
'settle_count': 0,
+ 'retry_count': 0,
+ 'bake_retry_count': 0,
+ 'failures': [],
# Operator settings (copied from the invoking operator)
'settings': {},
}
_SETTLE_TICKS = 12
+_BAKE_SETTLE_TICKS = 4
+_BAKE_RETRIES = 2
_TICK_INTERVAL = 0.5
@@ -61,6 +68,126 @@ def _sync_wm():
pass
+def _failure_report_path():
+ """Return the persistent report path for skipped batch entries."""
+ return os.path.join(_batch_state['out_dir'], 'batch_failures.json')
+
+
+def _load_failure_report(out_dir):
+ """Load failures from an earlier/resumed run, if present."""
+ path = os.path.join(out_dir, 'batch_failures.json')
+ try:
+ with open(path, 'r', encoding='utf-8') as handle:
+ data = json.load(handle)
+ failures = data.get('failures', [])
+ return failures if isinstance(failures, list) else []
+ except FileNotFoundError:
+ return []
+ except Exception as exc: # noqa: BLE001
+ print(f"[BatchTex] Could not read existing failure report: {exc}")
+ return []
+
+
+def _write_failure_report():
+ """Atomically persist failures so an unattended run is auditable."""
+ state = _batch_state
+ path = _failure_report_path()
+ tmp_path = path + '.tmp'
+ payload = {
+ 'schema': 1,
+ 'updated_at': datetime.now().astimezone().isoformat(timespec='seconds'),
+ 'total_models': state['total'],
+ 'failure_count': len(state['failures']),
+ 'failures': state['failures'],
+ }
+ try:
+ os.makedirs(state['out_dir'], exist_ok=True)
+ with open(tmp_path, 'w', encoding='utf-8') as handle:
+ json.dump(payload, handle, indent=2, ensure_ascii=False)
+ handle.write('\n')
+ os.replace(tmp_path, path)
+ except Exception as exc: # noqa: BLE001
+ print(f"[BatchTex] Could not write failure report: {exc}")
+
+
+def _record_failure(stage, error, attempts=None):
+ """Record the current model's terminal failure once."""
+ state = _batch_state
+ entry = state['files'][state['index']]
+ record = {
+ 'index': state['index'] + 1,
+ 'relative_path': entry['relative_path'],
+ 'model_path': entry['model_path'],
+ 'reference_path': entry.get('reference_path', ''),
+ 'output_relative_path': entry['output_relative_path'],
+ 'stage': stage,
+ 'error': str(error),
+ 'attempts': (
+ int(attempts)
+ if attempts is not None else state['retry_count'] + 1
+ ),
+ 'failed_at': datetime.now().astimezone().isoformat(timespec='seconds'),
+ }
+ # Replace an older record for the same output when a resumed run fails
+ # again, instead of accumulating indistinguishable duplicates.
+ state['failures'] = [
+ item for item in state['failures']
+ if item.get('output_relative_path') != entry['output_relative_path']
+ ]
+ state['failures'].append(record)
+ _write_failure_report()
+
+
+def _clear_generation_error():
+ """Reset per-attempt generation state before retrying a clean import."""
+ try:
+ scene = bpy.context.scene
+ scene.sg_last_gen_error = False
+ scene.sg_last_gen_error_message = ''
+ scene.generation_status = 'idle'
+ except Exception: # noqa: BLE001
+ pass
+
+
+def _retry_or_skip(stage, error):
+ """Retry the current model, or log and skip it after retries expire."""
+ state = _batch_state
+ max_retries = state['settings'].get('generation_retries', 2)
+ current_attempt = state['retry_count'] + 1
+ total_attempts = max_retries + 1
+ rel = state['files'][state['index']]['relative_path']
+
+ if state['retry_count'] < max_retries:
+ state['retry_count'] += 1
+ print(f"[BatchTex] {stage} failed for {rel}: {error}")
+ print(f"[BatchTex] Retrying current model "
+ f"(attempt {state['retry_count'] + 1}/{total_attempts})")
+ _clear_generation_error()
+ state['phase'] = 'import'
+ state['settle_count'] = 0
+ _sync_wm()
+ return _TICK_INTERVAL
+
+ _record_failure(stage, error)
+ print(f"[BatchTex] {stage} failed for {rel} after "
+ f"{current_attempt}/{total_attempts} attempt(s): {error}")
+
+ if state['settings'].get('continue_on_error', True):
+ print("[BatchTex] Skipping failed model and continuing batch")
+ _clear_generation_error()
+ state['phase'] = 'idle'
+ state['settle_count'] = 0
+ state['retry_count'] = 0
+ _sync_wm()
+ return _TICK_INTERVAL
+
+ print("[BatchTex] Stop-on-error is enabled — stopping batch")
+ state['active'] = False
+ state['phase'] = 'idle'
+ _sync_wm()
+ return None
+
+
# ── Reference image resolution ──────────────────────────────────────────────
def _find_ref_image(model_path, top_dir, ref_dir='', explicit_ref=''):
@@ -386,16 +513,18 @@ def _do_start_texturing():
from .. import utils as _sg_utils
_sg_utils._sg_bypass_modal_check = True
try:
- bpy.ops.object.test_stable('INVOKE_DEFAULT')
+ result = bpy.ops.object.test_stable('INVOKE_DEFAULT')
finally:
_sg_utils._sg_bypass_modal_check = False
+ if not ({'RUNNING_MODAL', 'FINISHED'} & set(result)):
+ raise RuntimeError(f"Texture generation operator returned {set(result)}")
print("[BatchTex] Texture generation started")
def _do_bake():
"""Bake textures for all mesh objects using the utility functions directly."""
from ..texturing.rendering import prepare_baking, bake_texture
- from ..utils import get_dir_path
+ from ..utils import get_dir_path, get_file_path
settings = _batch_state['settings']
tex_res = settings.get('bake_resolution', 2048)
@@ -403,11 +532,17 @@ def _do_bake():
context = bpy.context
original_engine = context.scene.render.engine
- prepare_baking(context)
+ baked_objects = []
+ try:
+ prepare_baking(context)
- mesh_objects = [obj for obj in context.scene.objects if obj.type == 'MESH']
- for obj in mesh_objects:
- try:
+ mesh_objects = [
+ obj for obj in context.scene.objects if obj.type == 'MESH'
+ ]
+ if not mesh_objects:
+ raise RuntimeError("No mesh objects available for baking")
+
+ for obj in mesh_objects:
# ── Create a fresh BakeUV and unwrap directly ──
# Remove all non-projection UV maps (DAE originals are per-material
# SketchUp UVs, not suitable as a bake atlas)
@@ -457,22 +592,149 @@ def _do_bake():
bpy.ops.uv.smart_project(correct_aspect=False)
bpy.ops.object.mode_set(mode='OBJECT')
- bake_texture(context, obj, texture_resolution=tex_res,
- output_dir=get_dir_path(context, "baked"))
- # Apply baked texture as a simple Principled BSDF material so
- # GLB export sees a standard material instead of the complex
- # StableGen projection node tree.
+ baked = bake_texture(
+ context,
+ obj,
+ texture_resolution=tex_res,
+ output_dir=get_dir_path(context, "baked"),
+ )
+ if not baked:
+ raise RuntimeError(f"Bake did not run for {obj.name}")
+
+ baked_path = get_file_path(
+ context, "baked", object_name=obj.name)
+ if not _baked_file_has_signal(baked_path):
+ raise RuntimeError(
+ f"Bake output for {obj.name} is uniformly black; "
+ "keeping the projection material and retrying"
+ )
+ baked_objects.append(obj)
+
+ # Only replace projection materials after every object has a verified
+ # bake. A failed attempt remains recoverable and can be retried.
+ for obj in baked_objects:
_apply_baked_material(context, obj)
- except Exception as exc: # noqa: BLE001
- print(f"[BatchTex] Bake failed for {obj.name}: {exc}")
+ print("[BatchTex] Bake complete")
+ return True
+ finally:
+ if context.scene.render.engine != original_engine:
+ context.scene.render.engine = original_engine
+
- if context.scene.render.engine != original_engine:
- context.scene.render.engine = original_engine
- print("[BatchTex] Bake complete")
+def _baked_file_has_signal(filepath):
+ """Return whether a saved batch bake contains any non-black RGB value."""
+ if not filepath or not os.path.isfile(filepath):
+ return False
+ try:
+ from PIL import Image
+
+ with Image.open(filepath) as image:
+ extrema = image.convert('RGB').getextrema()
+ return any(high > 0 for _low, high in extrema)
+ except Exception as exc: # noqa: BLE001
+ print(f"[BatchTex] Could not validate baked image {filepath}: {exc}")
+ return False
+
+
+def _refresh_projection_for_bake(context):
+ """Reload projection inputs and force dependency-graph evaluation."""
+ seen_images = set()
+ for obj in context.scene.objects:
+ if obj.type != 'MESH':
+ continue
+ obj.data.update()
+ for material in obj.data.materials:
+ if not material or not material.use_nodes:
+ continue
+ material.node_tree.update_tag()
+ for node in material.node_tree.nodes:
+ image = getattr(node, 'image', None)
+ if image is None or image.as_pointer() in seen_images:
+ continue
+ seen_images.add(image.as_pointer())
+ filepath = bpy.path.abspath(image.filepath)
+ if filepath and os.path.isfile(filepath):
+ image.reload()
+ image.update()
+ context.view_layer.update()
+
+
+def _write_camera_recovery_state(context):
+ """Persist exact camera transforms needed to reproject saved views."""
+ from ..utils import get_dir_path, get_file_path
+
+ state = _batch_state
+ entry = state['files'][state['index']]
+ cameras = sorted(
+ (obj for obj in context.scene.objects if obj.type == 'CAMERA'),
+ key=lambda camera: camera.name,
+ )
+ payload = {
+ 'schema': 1,
+ 'relative_path': entry['relative_path'],
+ 'model_path': entry['model_path'],
+ 'output_relative_path': entry['output_relative_path'],
+ 'output_timestamp': context.scene.output_timestamp,
+ 'cameras': [],
+ }
+ for index, camera in enumerate(cameras):
+ data = camera.data
+ payload['cameras'].append({
+ 'index': index,
+ 'name': camera.name,
+ 'matrix_world': [
+ [float(value) for value in row]
+ for row in camera.matrix_world
+ ],
+ 'type': data.type,
+ 'lens': float(data.lens),
+ 'sensor_width': float(data.sensor_width),
+ 'sensor_height': float(data.sensor_height),
+ 'shift_x': float(data.shift_x),
+ 'shift_y': float(data.shift_y),
+ 'resolution_x': int(camera.get(
+ 'sg_res_x', context.scene.render.resolution_x)),
+ 'resolution_y': int(camera.get(
+ 'sg_res_y', context.scene.render.resolution_y)),
+ 'generated_image': get_file_path(
+ context, 'generated', camera_id=index, material_id=0),
+ })
- if context.scene.render.engine != original_engine:
- context.scene.render.engine = original_engine
- print("[BatchTex] Bake complete")
+ revision_dir = get_dir_path(context, 'revision')
+ path = os.path.join(revision_dir, 'batch_camera_state.json')
+ tmp_path = path + '.tmp'
+ try:
+ with open(tmp_path, 'w', encoding='utf-8') as handle:
+ json.dump(payload, handle, indent=2, ensure_ascii=False)
+ handle.write('\n')
+ os.replace(tmp_path, path)
+ print(f"[BatchTex] Saved camera recovery state: {path}")
+ except Exception as exc: # noqa: BLE001
+ print(f"[BatchTex] Could not save camera recovery state: {exc}")
+
+
+def _finish_terminal_bake_failure(error):
+ """Log a verified bake failure and either skip or stop the batch."""
+ state = _batch_state
+ attempts = state['bake_retry_count'] + 1
+ _record_failure('bake', error, attempts=attempts)
+ rel = state['files'][state['index']]['relative_path']
+ print(
+ f"[BatchTex] Bake failed for {rel} after {attempts} attempt(s): "
+ f"{error}"
+ )
+ state['bake_retry_count'] = 0
+ state['settle_count'] = 0
+ if state['settings'].get('continue_on_error', True):
+ print("[BatchTex] Skipping failed model and continuing batch")
+ state['phase'] = 'idle'
+ _sync_wm()
+ return _TICK_INTERVAL
+ print("[BatchTex] Stop-on-error is enabled — stopping batch")
+ state['active'] = False
+ state['phase'] = 'idle'
+ _sync_wm()
+ return None
def _apply_baked_material(context, obj):
@@ -589,10 +851,16 @@ def _batch_tex_tick():
state['active'] = False
state['phase'] = 'idle'
_sync_wm()
- print(f"[BatchTex] Complete! {state['total']} file(s) processed.")
+ failed = len(state['failures'])
+ print(f"[BatchTex] Complete! {state['total']} file(s) processed, "
+ f"{failed} failed/skipped.")
+ if failed:
+ print(f"[BatchTex] Failure report: {_failure_report_path()}")
return None
state['index'] = next_idx
+ state['retry_count'] = 0
+ state['bake_retry_count'] = 0
state['phase'] = 'import'
_sync_wm()
rel = state['files'][next_idx]['relative_path']
@@ -603,9 +871,7 @@ def _batch_tex_tick():
if phase == 'import':
ok = _do_import()
if not ok:
- print(f"[BatchTex] Skipping file {state['index'] + 1} (import failed)")
- state['phase'] = 'idle'
- return _TICK_INTERVAL
+ return _retry_or_skip('import', 'Model import failed')
state['phase'] = 'cameras'
return _TICK_INTERVAL
@@ -615,9 +881,7 @@ def _batch_tex_tick():
try:
_do_cameras()
except Exception as exc: # noqa: BLE001
- print(f"[BatchTex] Camera placement failed: {exc}")
- state['phase'] = 'idle'
- return _TICK_INTERVAL
+ return _retry_or_skip('camera placement', exc)
# Set reference image before texturing. Manifest batches are strict:
# validation guarantees a reference for every row, so a lookup failure
@@ -628,6 +892,7 @@ def _batch_tex_tick():
state['settings'].get('require_reference', False)):
rel = state['files'][state['index']]['relative_path']
print(f"[BatchTex] Stopping: required reference missing for {rel}")
+ _record_failure('reference', 'Required manifest reference missing')
state['cancelled'] = True
return _TICK_INTERVAL
@@ -643,9 +908,7 @@ def _batch_tex_tick():
try:
_do_start_texturing()
except Exception as exc: # noqa: BLE001
- print(f"[BatchTex] Texturing failed to start: {exc}")
- state['phase'] = 'export'
- return _TICK_INTERVAL
+ return _retry_or_skip('texturing startup', exc)
state['settle_count'] = 1
return _TICK_INTERVAL
@@ -665,28 +928,61 @@ def _batch_tex_tick():
except Exception: # noqa: BLE001
had_error = False
gen_status = getattr(bpy.context.scene, 'generation_status', 'idle')
- if had_error or gen_status == 'waiting':
- print("[BatchTex] Generation was cancelled/failed — stopping batch")
+ if gen_status == 'waiting':
+ print("[BatchTex] Generation was cancelled by user — stopping batch")
state['active'] = False
state['phase'] = 'idle'
_sync_wm()
return None
+ if had_error:
+ error = getattr(
+ bpy.context.scene,
+ 'sg_last_gen_error_message',
+ '',
+ ) or 'Texture generation failed'
+ return _retry_or_skip('texturing', error)
+
+ # Preserve the exact camera rig before baking/export. Generated views
+ # can then be reprojected without rerunning the image model.
+ try:
+ _write_camera_recovery_state(bpy.context)
+ except Exception as exc: # noqa: BLE001
+ print(f"[BatchTex] Could not save camera recovery state: {exc}")
# Texturing done — move to bake
if _batch_state['settings'].get('bake', True):
state['phase'] = 'bake'
state['settle_count'] = 0
+ state['bake_retry_count'] = 0
else:
state['phase'] = 'export'
return _TICK_INTERVAL
# ── BAKE (synchronous — no operator popup) ──────────────────────────
if phase == 'bake':
+ # Generation completion and Cycles/image dependency evaluation are
+ # separate Blender events. Give the graph time to settle before each
+ # attempt, particularly in long unattended runs.
+ if state['settle_count'] < _BAKE_SETTLE_TICKS:
+ state['settle_count'] += 1
+ return _TICK_INTERVAL
try:
+ _refresh_projection_for_bake(bpy.context)
_do_bake()
except Exception as exc: # noqa: BLE001
- print(f"[BatchTex] Bake failed: {exc}")
+ if state['bake_retry_count'] < _BAKE_RETRIES:
+ state['bake_retry_count'] += 1
+ state['settle_count'] = 0
+ print(
+ f"[BatchTex] Bake attempt "
+ f"{state['bake_retry_count']}/{_BAKE_RETRIES + 1} "
+ f"failed: {exc}"
+ )
+ print("[BatchTex] Retrying bake without regenerating images")
+ return _TICK_INTERVAL
+ return _finish_terminal_bake_failure(exc)
+ state['bake_retry_count'] = 0
state['phase'] = 'export'
return _TICK_INTERVAL
@@ -770,6 +1066,26 @@ class BatchTextureDAE(bpy.types.Operator):
max=8192,
) # type: ignore
+ generation_retries: bpy.props.IntProperty(
+ name="Generation Retries",
+ description=(
+ "Number of clean re-import and texturing retries before a model "
+ "is skipped"
+ ),
+ default=2,
+ min=0,
+ max=10,
+ ) # type: ignore
+
+ continue_on_error: bpy.props.BoolProperty(
+ name="Continue After Failed Model",
+ description=(
+ "After retries are exhausted, log and skip the failed model "
+ "instead of stopping the entire batch"
+ ),
+ default=True,
+ ) # type: ignore
+
# ── Import / cleanup settings (same as BatchImportDAE) ──
merge_threshold: bpy.props.FloatProperty(
@@ -900,6 +1216,8 @@ class BatchTextureDAE(bpy.types.Operator):
box = layout.box()
box.label(text="Uses current scene generation settings")
box.label(text="Camera placement uses TRELLIS.2 camera settings")
+ box.prop(self, "generation_retries")
+ box.prop(self, "continue_on_error")
def execute(self, context):
manifest_path = bpy.path.abspath(self.manifest_path).strip()
@@ -937,6 +1255,7 @@ class BatchTextureDAE(bpy.types.Operator):
configured_output = bpy.path.abspath(self.output_dir).strip()
out_dir = configured_output or os.path.join(top_dir, "textured")
os.makedirs(out_dir, exist_ok=True)
+ existing_failures = _load_failure_report(out_dir)
scene = context.scene
_batch_state.update({
@@ -949,6 +1268,9 @@ class BatchTextureDAE(bpy.types.Operator):
'out_dir': out_dir,
'phase': 'idle',
'settle_count': 0,
+ 'retry_count': 0,
+ 'bake_retry_count': 0,
+ 'failures': existing_failures,
'settings': {
# Import settings
'merge_threshold': self.merge_threshold,
@@ -966,6 +1288,8 @@ class BatchTextureDAE(bpy.types.Operator):
# Bake settings
'bake': self.bake,
'bake_resolution': self.bake_resolution,
+ 'generation_retries': self.generation_retries,
+ 'continue_on_error': self.continue_on_error,
# Reference image directory
'ref_image_dir': self.ref_image_dir.rstrip('/\\') if self.ref_image_dir else '',
'require_reference': bool(manifest_path),
diff --git a/stablegen/texturing/generator.py b/stablegen/texturing/generator.py
index ff3b882..bffae11 100644
--- a/stablegen/texturing/generator.py
+++ b/stablegen/texturing/generator.py
@@ -785,6 +785,7 @@ class ComfyUIGenerate(_PBRMixin, bpy.types.Operator):
context.scene.generation_status = 'running'
context.scene.sg_last_gen_error = False
+ context.scene.sg_last_gen_error_message = ""
ComfyUIGenerate._is_running = True
print("[StableGen] Executing ComfyUI Generation")
@@ -1141,7 +1142,12 @@ class ComfyUIGenerate(_PBRMixin, bpy.types.Operator):
# Bake the texture if it doesn't exist
self._stage = f"Baking UV Textures ({obj.name})"
prepare_baking(context)
- unwrap(obj, method='pack', overlap_only=True)
+ unwrap(
+ obj,
+ method='pack',
+ overlap_only=True,
+ texture_resolution=2048,
+ )
bake_texture(context, obj, texture_resolution=2048, output_dir=get_dir_path(context, "baked"))
# Check if the material is compatible (uses projection shader)
@@ -1286,8 +1292,10 @@ class ComfyUIGenerate(_PBRMixin, bpy.types.Operator):
finally:
self._restore_texturing_pose(context)
if self._error:
- if self._error == "'25'" or self._error == "'111'" or self._error == "'5'":
- # Probably canceled by user, quietly return
+ context.scene.sg_last_gen_error_message = str(self._error)
+ if context.scene.generation_status == 'waiting':
+ # Explicit cancellation is tracked by generation_status;
+ # missing output-node keys are real generation failures.
context.scene.generation_status = 'idle'
context.scene.sg_last_gen_error = True
self.report({'WARNING'}, "Generation cancelled.")
diff --git a/stablegen/texturing/rendering.py b/stablegen/texturing/rendering.py
index 9a18626..26e0beb 100644
--- a/stablegen/texturing/rendering.py
+++ b/stablegen/texturing/rendering.py
@@ -1979,6 +1979,252 @@ def _set_bake_uv_active(obj):
return _ensure_bake_uv(obj)
+def _projection_bake_face_mask(obj, threshold=1.0e-8):
+ """Return faces that can receive color from a projection material.
+
+ Projection materials read ``_SG_VisWeight_*`` POINT attributes and
+ interpolate them across each face. A face whose vertices are zero in
+ every such attribute cannot receive any projected texture, so giving it a
+ full BakeUV island only wastes atlas resolution.
+
+ ``None`` means the object has neither baked visibility nor projection UV
+ data and should retain the normal all-face unwrap behavior.
+ """
+ if obj is None or obj.type != 'MESH' or not obj.data:
+ return None
+
+ mesh = obj.data
+ visibility_attributes = [
+ attribute for attribute in mesh.attributes
+ if (
+ attribute.name.startswith("_SG_VisWeight_")
+ and attribute.domain == 'POINT'
+ and attribute.data_type == 'FLOAT'
+ )
+ ]
+ if not mesh.polygons:
+ return np.zeros(0, dtype=bool)
+
+ loop_vertices = np.empty(len(mesh.loops), dtype=np.int32)
+ mesh.loops.foreach_get("vertex_index", loop_vertices)
+ face_starts = np.empty(len(mesh.polygons), dtype=np.int32)
+ mesh.polygons.foreach_get("loop_start", face_starts)
+
+ if visibility_attributes:
+ vertex_coverage = np.zeros(len(mesh.vertices), dtype=np.float32)
+ values = np.empty(len(mesh.vertices), dtype=np.float32)
+ for attribute in visibility_attributes:
+ attribute.data.foreach_get("value", values)
+ np.maximum(vertex_coverage, values, out=vertex_coverage)
+ loop_coverage = vertex_coverage[loop_vertices]
+ return (
+ np.maximum.reduceat(loop_coverage, face_starts)
+ > float(threshold)
+ )
+
+ # Compatibility fallback for older projection materials that predate
+ # _SG_VisWeight attributes. Projection image nodes use CLIP extension, so
+ # a face wholly outside every camera's [0, 1] UV rectangle cannot
+ # contribute. Bounding-box intersection is intentionally conservative:
+ # it can retain extra faces, but cannot discard a triangle crossing a view
+ # merely because all three corners lie outside it.
+ projection_attributes = [
+ attribute for attribute in mesh.attributes
+ if (
+ attribute.name.startswith("ProjectionUV_")
+ and attribute.domain == 'CORNER'
+ and attribute.data_type == 'FLOAT_VECTOR'
+ )
+ ]
+ if not projection_attributes:
+ return None
+
+ covered_faces = np.zeros(len(mesh.polygons), dtype=bool)
+ vectors = np.empty(len(mesh.loops) * 3, dtype=np.float32)
+ for attribute in projection_attributes:
+ attribute.data.foreach_get("vector", vectors)
+ uv = vectors.reshape((-1, 3))[:, :2]
+ min_u = np.minimum.reduceat(uv[:, 0], face_starts)
+ max_u = np.maximum.reduceat(uv[:, 0], face_starts)
+ min_v = np.minimum.reduceat(uv[:, 1], face_starts)
+ max_v = np.maximum.reduceat(uv[:, 1], face_starts)
+ covered_faces |= (
+ (max_u >= 0.0)
+ & (min_u <= 1.0)
+ & (max_v >= 0.0)
+ & (min_v <= 1.0)
+ )
+ return covered_faces
+
+
+def _ensure_projection_visibility_for_bake(obj):
+ """Backfill visibility weights for legacy projection materials."""
+ mesh = getattr(obj, "data", None)
+ if mesh is None or any(
+ attribute.name.startswith("_SG_VisWeight_")
+ and attribute.domain == 'POINT'
+ for attribute in mesh.attributes
+ ):
+ return
+
+ projection_pairs = []
+ for attribute in mesh.attributes:
+ if not attribute.name.startswith("ProjectionUV_"):
+ continue
+ parts = attribute.name.rsplit('_', 2)
+ if len(parts) != 3:
+ continue
+ try:
+ projection_pairs.append((int(parts[1]), int(parts[2])))
+ except ValueError:
+ continue
+ if not projection_pairs:
+ return
+
+ from .view_plan import order_cameras
+
+ max_camera_index = max(camera_index for camera_index, _ in projection_pairs)
+ cameras = order_cameras(
+ scene_obj
+ for scene_obj in bpy.context.scene.objects
+ if scene_obj.type == 'CAMERA'
+ )
+ if len(cameras) <= max_camera_index:
+ print(
+ f"[StableGen] Warning: cannot rebuild projection visibility for "
+ f"{obj.name}; expected at least {max_camera_index + 1} cameras, "
+ f"found {len(cameras)}. Using conservative camera-UV coverage."
+ )
+ return
+
+ # Visibility is geometry/camera dependent, not material-ID dependent. One
+ # representative material ID supplies the union needed for UV allocation.
+ material_id = min(material_id for _, material_id in projection_pairs)
+ angle_thresholds = []
+ for material in mesh.materials:
+ if not material or not material.use_nodes:
+ continue
+ for node in material.node_tree.nodes:
+ if (
+ getattr(node, "label", "").startswith("AngleThreshold-")
+ and len(node.inputs) > 1
+ ):
+ try:
+ angle_thresholds.append(float(node.inputs[1].default_value))
+ except (AttributeError, TypeError, ValueError):
+ pass
+ angle_threshold = max(
+ angle_thresholds
+ or [
+ float(getattr(bpy.context.scene, "discard_factor", 90.0)),
+ float(getattr(
+ bpy.context.scene,
+ "sg_material_discard_factor",
+ 90.0,
+ )),
+ ]
+ )
+ try:
+ from .projection import _bake_visibility_weights
+
+ scene = bpy.context.scene
+ original_silhouette_margin = getattr(
+ scene,
+ "sg_silhouette_margin",
+ None,
+ )
+ if original_silhouette_margin is not None:
+ # Silhouette erosion only removes uncertain edge pixels. Atlas
+ # coverage should remain conservative and needs just one ray per
+ # candidate vertex, not the extra 4/8 offset rays.
+ scene.sg_silhouette_margin = 0
+ try:
+ print(
+ f"[StableGen] Rebuilding legacy projection visibility for "
+ f"BakeUV on {obj.name}"
+ )
+ _bake_visibility_weights(
+ bpy.context,
+ [obj],
+ cameras[:max_camera_index + 1],
+ material_id,
+ stop_index=max_camera_index,
+ angle_threshold=angle_threshold,
+ )
+ finally:
+ if original_silhouette_margin is not None:
+ scene.sg_silhouette_margin = original_silhouette_margin
+ except Exception as exc:
+ print(
+ f"[StableGen] Warning: could not rebuild projection visibility "
+ f"for {obj.name}: {exc}. Using conservative camera-UV coverage."
+ )
+
+
+def _compact_uncovered_projection_uvs(
+ obj, covered_faces, texture_resolution):
+ """Fill the tile with covered UVs and collapse uncovered faces."""
+ bm = bmesh.from_edit_mesh(obj.data)
+ bm.faces.ensure_lookup_table()
+ bm.faces.index_update()
+ uv_layer = bm.loops.layers.uv.active
+ if uv_layer is None:
+ return
+
+ resolution = max(1, int(texture_resolution or 2048))
+ bake_margin = max(
+ 0,
+ int(getattr(bpy.context.scene.render.bake, "margin", 16)),
+ )
+ guard = min(
+ 0.2,
+ max(2.0 / resolution, (bake_margin + 2.0) / resolution),
+ )
+
+ covered_loops = [
+ loop
+ for face in bm.faces
+ if (
+ face.index < len(covered_faces)
+ and bool(covered_faces[face.index])
+ )
+ for loop in face.loops
+ ]
+ if covered_loops:
+ min_u = min(loop[uv_layer].uv.x for loop in covered_loops)
+ max_u = max(loop[uv_layer].uv.x for loop in covered_loops)
+ min_v = min(loop[uv_layer].uv.y for loop in covered_loops)
+ max_v = max(loop[uv_layer].uv.y for loop in covered_loops)
+ span_u = max(max_u - min_u, 1.0e-12)
+ span_v = max(max_v - min_v, 1.0e-12)
+ target_span = max(0.01, 1.0 - 2.0 * guard)
+ scale_u = target_span / span_u
+ scale_v = target_span / span_v
+ else:
+ min_u = min_v = 0.0
+ scale_u = scale_v = 1.0
+
+ for face in bm.faces:
+ is_covered = (
+ face.index < len(covered_faces)
+ and bool(covered_faces[face.index])
+ )
+ for loop in face.loops:
+ if is_covered:
+ uv = loop[uv_layer].uv
+ # The UV map and its baked image are consumed together, so a
+ # global per-axis reframe is lossless. It avoids keeping a
+ # wide, short island footprint centered in a square texture.
+ uv.x = guard + (uv.x - min_u) * scale_u
+ uv.y = guard + (uv.y - min_v) * scale_v
+ else:
+ # Degenerate UV faces occupy no atlas area. The guard around
+ # (0, 0) prevents their bake margin touching useful islands.
+ loop[uv_layer].uv = (0.0, 0.0)
+
+ bmesh.update_edit_mesh(obj.data, loop_triangles=False, destructive=False)
+
+
def _pin_unlinked_image_textures_to_uv(nodes, links, uv_name, exclude_nodes=()):
if not uv_name:
return None
@@ -2178,7 +2424,7 @@ def _activate_mesh_for_object_ops(obj):
}
-def unwrap(obj, method, overlap_only):
+def unwrap(obj, method, overlap_only, texture_resolution=2048):
"""
Unwraps the UVs of the given object using the selected method.
:param obj: Blender object.
@@ -2188,6 +2434,13 @@ def unwrap(obj, method, overlap_only):
return
op_context = _activate_mesh_for_object_ops(obj)
+ _ensure_projection_visibility_for_bake(obj)
+ covered_faces = _projection_bake_face_mask(obj)
+ projection_coverage_pack = covered_faces is not None
+ selective_projection_pack = (
+ projection_coverage_pack
+ and not bool(np.all(covered_faces))
+ )
_ensure_bake_uv(obj)
@@ -2197,7 +2450,7 @@ def unwrap(obj, method, overlap_only):
# Ensure UV selection sync is OFF
bpy.context.scene.tool_settings.use_uv_select_sync = False
- if overlap_only:
+ if overlap_only and not selective_projection_pack:
# Deselect
bpy.ops.uv.select_all(action='DESELECT')
# Check if the object has overlapping UVs, if not, skip unwrapping
@@ -2232,19 +2485,52 @@ def unwrap(obj, method, overlap_only):
return
bpy.context.scene.tool_settings.use_uv_select_sync = True
- # Select all faces
- bpy.ops.mesh.select_all(action='SELECT')
+ if selective_projection_pack:
+ bpy.ops.mesh.select_all(action='DESELECT')
+ bm = bmesh.from_edit_mesh(obj.data)
+ bm.faces.ensure_lookup_table()
+ bm.faces.index_update()
+ for face in bm.faces:
+ face.select_set(
+ face.index < len(covered_faces)
+ and bool(covered_faces[face.index])
+ )
+ bmesh.update_edit_mesh(
+ obj.data, loop_triangles=False, destructive=False)
+ else:
+ bpy.ops.mesh.select_all(action='SELECT')
- if method == 'basic':
- bpy.ops.uv.unwrap(method='ANGLE_BASED', margin=0.001)
- elif method == 'smart':
- bpy.ops.uv.smart_project()
- elif method == 'cube':
- bpy.ops.uv.cube_project(cube_size=1.0)
- elif method == 'lightmap':
- bpy.ops.uv.lightmap_pack()
- elif method == 'pack':
- bpy.ops.uv.pack_islands()
+ has_selected_faces = (
+ not selective_projection_pack
+ or bool(np.any(covered_faces))
+ )
+ if has_selected_faces:
+ if method == 'basic':
+ bpy.ops.uv.unwrap(method='ANGLE_BASED', margin=0.001)
+ elif method == 'smart':
+ bpy.ops.uv.smart_project(
+ rotate_method='AXIS_ALIGNED',
+ scale_to_bounds=True,
+ )
+ elif method == 'cube':
+ bpy.ops.uv.cube_project(cube_size=1.0)
+ elif method == 'lightmap':
+ bpy.ops.uv.lightmap_pack()
+ elif method == 'pack':
+ bpy.ops.uv.pack_islands()
+
+ if projection_coverage_pack:
+ _compact_uncovered_projection_uvs(
+ obj,
+ covered_faces,
+ texture_resolution,
+ )
+ covered_count = int(np.count_nonzero(covered_faces))
+ print(
+ f"[StableGen] Filled BakeUV with {covered_count}/"
+ f"{len(covered_faces)} projection-covered faces on "
+ f"{obj.name}; collapsed fully uncovered faces"
+ )
bpy.ops.uv.select_all(action='DESELECT')
bpy.ops.object.mode_set(mode='OBJECT')
@@ -2260,107 +2546,130 @@ def bake_texture(context, obj, texture_resolution, suffix = "", view_transform =
if _is_gltf_not_exported_helper(obj) or _is_bone_custom_shape_object(obj):
print(f"[StableGen] Skipping helper mesh during bake: {obj.name}")
return False
- bpy.context.scene.display_settings.display_device = 'sRGB'
- # Backup original view transform
- original_view_transform = bpy.context.scene.view_settings.view_transform
- # Backup whether the object was enabled for rendering
- original_render = obj.hide_render
-
- bpy.context.scene.view_settings.view_transform = view_transform
- # Set the object to be rendered
- obj.hide_render = False
-
- source_uv_name = _first_source_uv_name(obj)
- _set_bake_uv_active(obj)
-
- # Create a new image for baking
- image_name = f"{obj.name}_baked{suffix}"
- image = bpy.data.images.new(name=image_name, width=texture_resolution, height=texture_resolution)
-
if not obj.data.materials:
print("[StableGen] No materials found")
return False
- original_materials, original_active_index, copied_materials = _copy_material_slots_for_bake(obj)
- if not copied_materials:
- _restore_material_slots_after_bake(
- obj, original_materials, original_active_index, copied_materials)
- print("[StableGen] No material slots found")
- return False
+ scene = bpy.context.scene
+ scene.display_settings.display_device = 'sRGB'
+ original_view_transform = scene.view_settings.view_transform
+ original_render = obj.hide_render
+ original_materials = []
+ original_active_index = obj.active_material_index
+ copied_materials = []
- op_context = _activate_mesh_for_object_ops(obj)
- _set_bake_uv_active(obj)
+ try:
+ scene.view_settings.view_transform = view_transform
+ obj.hide_render = False
+
+ source_uv_name = _first_source_uv_name(obj)
+ bake_uv = _set_bake_uv_active(obj)
+
+ image_name = f"{obj.name}_baked{suffix}"
+ old_image = bpy.data.images.get(image_name)
+ if old_image is not None and old_image.users == 0:
+ bpy.data.images.remove(old_image)
+ image = bpy.data.images.new(
+ name=image_name,
+ width=texture_resolution,
+ height=texture_resolution,
+ )
- for mat in copied_materials:
- mat.use_nodes = True
- nodes = mat.node_tree.nodes
- links = mat.node_tree.links
+ (original_materials,
+ original_active_index,
+ copied_materials) = _copy_material_slots_for_bake(obj)
+ if not copied_materials:
+ print("[StableGen] No material slots found")
+ return False
- tex_image = nodes.new("ShaderNodeTexImage")
- tex_image.image = image
- tex_image.location = (0, 0)
- nodes.active = tex_image
+ op_context = _activate_mesh_for_object_ops(obj)
+ bake_uv = _set_bake_uv_active(obj)
- _pin_unlinked_image_textures_to_uv(
- nodes, links, source_uv_name, exclude_nodes={tex_image})
-
- # Check if there is a BSDF node before the output node
- output = None
- for node in nodes:
- if node.type == 'OUTPUT_MATERIAL':
- output = node
- break
+ for mat in copied_materials:
+ mat.use_nodes = True
+ nodes = mat.node_tree.nodes
+ links = mat.node_tree.links
- before_output = output.inputs[0].links[0].from_node if output and output.inputs[0].is_linked else None
- if before_output and before_output.type != 'BSDF_PRINCIPLED':
- bsdf_node = nodes.new("ShaderNodeBsdfPrincipled")
- bsdf_node.location = (before_output.location[0] + 200, before_output.location[1])
- # Connect the before output node to the BSDF node
- links.new(before_output.outputs[0], bsdf_node.inputs[0])
- # Connect the BSDF node to the output node
- links.new(bsdf_node.outputs[0], output.inputs[0])
-
- # Disconnect Metallic so the DIFFUSE color-only pass returns the
- # true Base Color. Principled BSDF multiplies diffuse by
- # (1 - Metallic), which would dim metallic areas incorrectly.
- for node in nodes:
- if node.type == 'BSDF_PRINCIPLED':
- met = node.inputs.get("Metallic")
- if met:
- for link in list(met.links):
- links.remove(link)
- met.default_value = 0.0
- break
-
- # Bake the texture
- # Ensure OBJECT mode — bake fails silently in EDIT mode
- with bpy.context.temp_override(**op_context):
- if bpy.context.mode != 'OBJECT':
- bpy.ops.object.mode_set(mode='OBJECT')
- bpy.ops.object.bake(type='DIFFUSE', width=texture_resolution, height=texture_resolution)
-
- # Save the image if required
- if not os.path.exists(output_dir):
- os.makedirs(output_dir)
- if suffix == "":
- image.filepath_raw = os.path.join(output_dir, f"{obj.name}.png")
- else:
- image.filepath_raw = os.path.join(output_dir, f"{obj.name}_baked{suffix}.png")
- image.file_format = 'PNG'
- image.save()
+ tex_image = nodes.new("ShaderNodeTexImage")
+ tex_image.image = image
+ tex_image.location = (0, 0)
+ nodes.active = tex_image
+
+ _pin_unlinked_image_textures_to_uv(
+ nodes, links, source_uv_name, exclude_nodes={tex_image})
+
+ output = next(
+ (node for node in nodes
+ if node.type == 'OUTPUT_MATERIAL'),
+ None,
+ )
+ before_output = (
+ output.inputs[0].links[0].from_node
+ if output and output.inputs[0].is_linked else None
+ )
+ if before_output and before_output.type != 'BSDF_PRINCIPLED':
+ bsdf_node = nodes.new("ShaderNodeBsdfPrincipled")
+ bsdf_node.location = (
+ before_output.location[0] + 200,
+ before_output.location[1],
+ )
+ links.new(before_output.outputs[0], bsdf_node.inputs[0])
+ links.new(bsdf_node.outputs[0], output.inputs[0])
+
+ # DIFFUSE color-only is multiplied by (1 - Metallic).
+ # Disconnect Metallic so the pass returns true Base Color.
+ for node in nodes:
+ if node.type == 'BSDF_PRINCIPLED':
+ metallic = node.inputs.get("Metallic")
+ if metallic:
+ for link in list(metallic.links):
+ links.remove(link)
+ metallic.default_value = 0.0
+ break
+ mat.node_tree.update_tag()
- # Remove tex_image node
- nodes.remove(tex_image)
+ obj.data.update()
+ context.view_layer.update()
- # Restore original view transform
- bpy.context.scene.view_settings.view_transform = original_view_transform
- # Restore original render settings
- obj.hide_render = original_render
+ # Operators can return CANCELLED without raising. Treat anything
+ # other than FINISHED as a real failure; otherwise an untouched
+ # newly-created (black) image would be saved and exported.
+ with bpy.context.temp_override(**op_context):
+ if bpy.context.mode != 'OBJECT':
+ bpy.ops.object.mode_set(mode='OBJECT')
+ bake_result = bpy.ops.object.bake(
+ 'EXEC_DEFAULT',
+ type='DIFFUSE',
+ width=texture_resolution,
+ height=texture_resolution,
+ uv_layer=bake_uv.name if bake_uv else '',
+ )
+ if 'FINISHED' not in bake_result:
+ raise RuntimeError(
+ f"Cycles bake returned {set(bake_result)} for {obj.name}"
+ )
- _restore_material_slots_after_bake(
- obj, original_materials, original_active_index, copied_materials)
-
- return True
+ if output_dir is None:
+ output_dir = get_dir_path(context, "baked")
+ os.makedirs(output_dir, exist_ok=True)
+ if suffix == "":
+ image.filepath_raw = os.path.join(output_dir, f"{obj.name}.png")
+ else:
+ image.filepath_raw = os.path.join(
+ output_dir, f"{obj.name}_baked{suffix}.png")
+ image.file_format = 'PNG'
+ image.save()
+ return True
+ finally:
+ scene.view_settings.view_transform = original_view_transform
+ obj.hide_render = original_render
+ if original_materials or copied_materials:
+ _restore_material_slots_after_bake(
+ obj,
+ original_materials,
+ original_active_index,
+ copied_materials,
+ )
def _has_non_projection_uv(obj):
"""Return True if the object has at least one UV map that can be used
@@ -3023,14 +3332,29 @@ class BakeTextures(bpy.types.Operator):
if self.try_unwrap != 'none':
if not _has_non_projection_uv(obj):
# No usable UV exists — must unwrap
- unwrap(obj, self.try_unwrap, self.overlap_only)
+ unwrap(
+ obj,
+ self.try_unwrap,
+ self.overlap_only,
+ self.texture_resolution,
+ )
elif self.overlap_only:
# Only re-unwrap if existing UVs overlap
if _uvs_likely_overlap(obj):
- unwrap(obj, self.try_unwrap, self.overlap_only)
+ unwrap(
+ obj,
+ self.try_unwrap,
+ self.overlap_only,
+ self.texture_resolution,
+ )
else:
# Always re-unwrap when overlap_only is disabled
- unwrap(obj, self.try_unwrap, self.overlap_only)
+ unwrap(
+ obj,
+ self.try_unwrap,
+ self.overlap_only,
+ self.texture_resolution,
+ )
self._progress = 100
elif self._phase == 'bake':
diff --git a/stablegen/workflows.py b/stablegen/workflows.py
index d2d3f9e..31e4039 100644
--- a/stablegen/workflows.py
+++ b/stablegen/workflows.py
@@ -4,6 +4,7 @@ import json
import uuid
import websocket
import socket
+import time
import urllib.request
import urllib.parse
import email.utils
@@ -243,6 +244,78 @@ class _WorkflowBase:
print(f"[StableGen] HTTP image fallback failed: {e}")
return []
+ @staticmethod
+ def _queue_contains_prompt(entries, prompt_id):
+ """Return True when a ComfyUI queue payload contains *prompt_id*."""
+ for entry in entries or []:
+ if isinstance(entry, (list, tuple)):
+ if len(entry) > 1 and str(entry[1]) == str(prompt_id):
+ return True
+ elif isinstance(entry, dict):
+ queued_id = entry.get('prompt_id') or entry.get('id')
+ if queued_id is not None and str(queued_id) == str(prompt_id):
+ return True
+ return False
+
+ @staticmethod
+ def _history_error_message(history_entry):
+ """Extract the most useful execution error from a history entry."""
+ status = history_entry.get('status', {})
+ for message in reversed(status.get('messages', [])):
+ if not isinstance(message, (list, tuple)) or len(message) < 2:
+ continue
+ if message[0] != 'execution_error' or not isinstance(message[1], dict):
+ continue
+ data = message[1]
+ node_id = data.get('node_id', '?')
+ detail = (data.get('exception_message') or data.get('error')
+ or 'Unknown ComfyUI execution error')
+ return f"node {node_id}: {detail}"
+ return status.get('status_str') or 'Unknown ComfyUI execution error'
+
+ def _get_prompt_runtime_state(self, server_address, prompt_id):
+ """Return ``(state, detail)`` for a queued ComfyUI prompt.
+
+ ``state`` is one of ``completed``, ``error``, ``running``, ``queued``,
+ ``unknown`` (server responded but the prompt is between states), or
+ ``unavailable`` (both status requests failed).
+ """
+ history_error = None
+ try:
+ history_url = (
+ f"http://{server_address}/history/"
+ f"{urllib.parse.quote(str(prompt_id))}"
+ )
+ history = json.loads(urllib.request.urlopen(
+ history_url, timeout=get_timeout('api')).read())
+ entry = history.get(prompt_id) or history.get(str(prompt_id))
+ if entry:
+ status = entry.get('status', {})
+ status_str = str(status.get('status_str', '')).lower()
+ if status_str in {'error', 'failed'}:
+ return 'error', self._history_error_message(entry)
+ if status.get('completed'):
+ return 'completed', status_str or 'completed'
+ return 'running', status_str or 'history entry is active'
+ except Exception as exc: # noqa: BLE001
+ history_error = exc
+
+ queue_error = None
+ try:
+ queue_url = f"http://{server_address}/queue"
+ queue = json.loads(urllib.request.urlopen(
+ queue_url, timeout=get_timeout('api')).read())
+ if self._queue_contains_prompt(queue.get('queue_running'), prompt_id):
+ return 'running', 'present in ComfyUI running queue'
+ if self._queue_contains_prompt(queue.get('queue_pending'), prompt_id):
+ return 'queued', 'present in ComfyUI pending queue'
+ return 'unknown', 'not yet visible in history or queue'
+ except Exception as exc: # noqa: BLE001
+ queue_error = exc
+
+ detail = f"history: {history_error}; queue: {queue_error}"
+ return 'unavailable', detail
+
def _connect_to_websocket(self, server_address, client_id):
"""Establishes WebSocket connection to ComfyUI server."""
try:
@@ -250,10 +323,11 @@ class _WorkflowBase:
# Use a short timeout for the initial TCP+WS handshake.
ws.settimeout(get_timeout('api'))
ws.connect(f"ws://{server_address}/ws?clientId={client_id}")
- # Switch to a longer timeout for recv() during generation —
- # ComfyUI may take a while between progress messages (model
- # loading, TRELLIS.2 processing, etc.).
- ws.settimeout(get_timeout('transfer'))
+ # This is a heartbeat interval, not a generation deadline. Slow
+ # nodes can legitimately emit no WebSocket events for minutes;
+ # timeout handling below checks /history and /queue and continues
+ # waiting while the prompt is still active.
+ ws.settimeout(get_timeout('generation_heartbeat'))
return ws
except ConnectionRefusedError:
self._error = f"Connection to ComfyUI WebSocket was refused at {server_address}. Is ComfyUI running and accessible?"
@@ -317,18 +391,69 @@ class _WorkflowBase:
# Send the prompt to the queue
prompt_id = self._queue_prompt(prompt, client_id, server_address)
+
+ # Make every texturing workflow cancellable while blocked in recv().
+ # cancel_generate() closes this socket after interrupting ComfyUI.
+ if hasattr(self.operator, '_active_ws'):
+ self.operator._active_ws = ws
+ type(self.operator)._active_ws = ws
# Process the WebSocket messages and collect images
output_images = {}
current_node = ""
+ unavailable_heartbeats = 0
+ unknown_heartbeats = 0
while True:
try:
out = ws.recv()
+ unavailable_heartbeats = 0
+ unknown_heartbeats = 0
except websocket.WebSocketTimeoutException:
- print(f"[StableGen] WebSocket recv timed out after "
- f"{get_timeout('transfer')}s. The server may be "
- f"slow or the connection was lost.")
+ heartbeat = get_timeout('generation_heartbeat')
+ state, detail = self._get_prompt_runtime_state(
+ server_address, prompt_id)
+ if state == 'completed':
+ print(f"[StableGen] No WebSocket event for {heartbeat}s, "
+ "but ComfyUI history reports completion. "
+ "Recovering the result via HTTP.")
+ break
+ if state == 'error':
+ self.operator._error = f"ComfyUI execution error: {detail}"
+ print(f"[StableGen] {self.operator._error}")
+ break
+ if state in {'running', 'queued'}:
+ unavailable_heartbeats = 0
+ unknown_heartbeats = 0
+ print(f"[StableGen] No WebSocket event for {heartbeat}s; "
+ f"prompt is still {state} ({detail}). Continuing to wait.")
+ continue
+ if state == 'unknown':
+ unavailable_heartbeats = 0
+ unknown_heartbeats += 1
+ print(f"[StableGen] No WebSocket event for {heartbeat}s; "
+ f"prompt state is temporarily unknown "
+ f"({unknown_heartbeats}/5). Continuing to wait.")
+ if unknown_heartbeats < 5:
+ continue
+ self.operator._error = (
+ f"ComfyUI stopped reporting prompt {prompt_id} in both "
+ "history and queue after five heartbeat checks."
+ )
+ print(f"[StableGen] {self.operator._error}")
+ break
+
+ unavailable_heartbeats += 1
+ print(f"[StableGen] No WebSocket event for {heartbeat}s and "
+ f"ComfyUI status is unavailable "
+ f"({unavailable_heartbeats}/3): {detail}")
+ if unavailable_heartbeats < 3:
+ continue
+ self.operator._error = (
+ f"Lost contact with ComfyUI while waiting for prompt "
+ f"{prompt_id} after three heartbeat checks."
+ )
+ print(f"[StableGen] {self.operator._error}")
break
except (ConnectionError, OSError, Exception) as ws_err:
print(f"[StableGen] WebSocket error during generation: {ws_err}")
@@ -392,31 +517,69 @@ class _WorkflowBase:
images_output.append(out[8:]) # Skip the first 8 bytes (header)
output_images[current_node] = images_output
- # --- Validate received images; fall back to HTTP if corrupt ----------
- save_node = NODES.get('save_image')
- if save_node and save_node in output_images:
- needs_http = False
- for img_bytes in output_images[save_node]:
- if not self._validate_image_bytes(img_bytes):
- needs_http = True
- break
+ if hasattr(self.operator, '_active_ws'):
+ if getattr(self.operator, '_active_ws', None) is ws:
+ self.operator._active_ws = None
+ if getattr(type(self.operator), '_active_ws', None) is ws:
+ type(self.operator)._active_ws = None
- if needs_http and fallback_node_id:
- print("[StableGen] WebSocket image data is corrupt "
- "(likely packet loss on remote network). "
- "Falling back to HTTP /view download...")
- http_images = self._get_images_via_http(
+ # --- Validate received images; fall back to HTTP if missing/corrupt --
+ save_node = NODES.get('save_image')
+ ws_images = output_images.get(save_node, []) if save_node else []
+ needs_http = (
+ bool(save_node)
+ and (not ws_images or any(
+ not self._validate_image_bytes(img_bytes)
+ for img_bytes in ws_images
+ ))
+ )
+
+ if needs_http and fallback_node_id:
+ reason = "missing" if not ws_images else "corrupt"
+ print(f"[StableGen] WebSocket image data is {reason}. "
+ "Falling back to HTTP /history + /view download...")
+
+ # ComfyUI can announce execution completion just before the
+ # history entry or SaveImage file becomes visible. Retry briefly
+ # so a harmless publication race does not fail a long generation.
+ http_images = []
+ retry_delays = (0.0, 0.25, 0.5, 1.0, 2.0)
+ for attempt, delay in enumerate(retry_delays, start=1):
+ if delay:
+ time.sleep(delay)
+ candidates = self._get_images_via_http(
server_address, prompt_id, fallback_node_id)
- if http_images:
- output_images[save_node] = http_images
- print(f"[StableGen] Successfully retrieved "
- f"{len(http_images)} image(s) via HTTP fallback.")
- else:
- print("[StableGen] HTTP fallback also failed. "
- "The image data may be unrecoverable.")
- elif needs_http:
- print("[StableGen] WebSocket image data is corrupt "
- "and no HTTP fallback node was injected.")
+ if candidates and all(
+ self._validate_image_bytes(data)
+ for data in candidates):
+ http_images = candidates
+ break
+ if attempt < len(retry_delays):
+ print(f"[StableGen] HTTP image fallback not ready "
+ f"(attempt {attempt}/{len(retry_delays)}); retrying...")
+
+ if http_images:
+ output_images[save_node] = http_images
+ print(f"[StableGen] Successfully retrieved "
+ f"{len(http_images)} image(s) via HTTP fallback.")
+ else:
+ print("[StableGen] HTTP fallback also failed. "
+ "The image data may be unrecoverable.")
+ elif needs_http:
+ print("[StableGen] WebSocket image data is missing or corrupt "
+ "and no HTTP fallback node was injected.")
+
+ final_images = output_images.get(save_node, []) if save_node else []
+ if (not save_node or not final_images or any(
+ not self._validate_image_bytes(data)
+ for data in final_images)):
+ if not self.operator._error:
+ self.operator._error = (
+ f"ComfyUI completed but StableGen received no valid image "
+ f"from output node {save_node!r}."
+ )
+ print(f"[StableGen] {self.operator._error}")
+ return {"error": "missing_output"}
return output_images