Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

36.5 KB
Raw Blame
"""
Shared helpers for the PortOS local image and music-generation Python runners.

The image runners present the same CLI surface to `server/services/imageGen/local.js`
— STAGE: markers, USER_ERROR: lines, stepwise PNG previews, sidecar JSON — and
the bits that own that contract live here. Keeping these in one module means a
fix to the HF error mapping or the stepwise decode lands in both runners at
once, instead of drifting.

Music sidecars reuse the liveness portion of that contract: a long model load or
inference call must emit periodic STAGE heartbeat markers so the media queue can
distinguish a slow render from a wedged child.

The runner-specific bits each script still owns:
  - argparse + which pipeline class / weight repo to load
  - `load_pipeline_*` (sdnq / int8 / auto-pipeline / ERNIE)
  - the txt2img-vs-i2i pipe swap (`to_i2i_pipeline` in z-image)

Anything generic — device picking, RNG seeding, memory-saving knobs, sidecar
writes, stepwise preview decode, USER_ERROR markers, and the cause-chain
walker that turns a buried `GatedRepoError` into a friendly link — belongs here.
"""

import json
import importlib.util
import logging
import os
import platform
import subprocess
import sys
import threading
from contextlib import contextmanager
from functools import wraps
from pathlib import Path

# Lazy heavy-import note: `torch` and `PIL` are deferred into the functions
# that actually need them (pick_device, make_generator, make_stepwise_callback).
# Lightweight helpers — heartbeat, install_hf_error_handler, write_sidecar —
# stay usable from venvs that haven't pip-installed torch yet during a partial
# runtime bootstrap.


def register_source_namespace(package_name: str, package_dir: "str | Path"):
    """Register one source-only namespace package without exposing its parent.

    Adding a third-party checkout root to ``sys.path`` lets unrelated files at
    that root (for example an untracked ``numpy.py``) shadow packages from the
    locked venv. A namespace package needs only its own directory in
    ``__path__``; relative imports keep working while sibling checkout files are
    never considered as top-level modules.
    """
    source_dir = Path(package_dir).absolute()
    if not source_dir.is_dir():
        raise RuntimeError(f"Source package directory is missing: {source_dir}")
    spec = importlib.util.spec_from_loader(package_name, loader=None, is_package=True)
    if spec is None:
        raise RuntimeError(f"Could not register source package: {package_name}")
    spec.submodule_search_locations = [str(source_dir)]
    package = importlib.util.module_from_spec(spec)
    sys.modules[package_name] = package
    return package


def decode_with_finite_retry(decode, retry_decode, isfinite):
    """Decode once, retry once after non-finite output, then fail closed."""
    decoded = decode()
    if bool(isfinite(decoded).all().item()):
        return decoded, False
    decoded = retry_decode()
    if bool(isfinite(decoded).all().item()):
        return decoded, True
    raise FloatingPointError("decode produced non-finite values after retry")


@contextmanager
def heartbeat(stage: "str | Callable[[], str]", interval: float = 20.0):
    """Emit a periodic STAGE:<stage>:heartbeat:Ns marker so the JS idle
    watchdog doesn't kill a silent but active model load or inference call.

    Diffusers' from_pretrained on a fully-cached 10-25 GB model, and music
    inference calls, can be silent for several minutes (mmap + weight
    assignment, no tqdm), which trips the JS-side idle watchdog.
    `handleLine()` in imageGen/local.js and the audio sidecar bridge see the
    heartbeat line and reset lastActivityAt. True hangs (GIL-pinned C
    extension, no I/O) still trip the watchdog because the heartbeat thread
    can't print either.

    `stage` may be a callable resolved per beat, for a runner that wraps a
    whole child process rather than one load step: generate_fastvideo.py scrapes
    the phase out of the child's own log lines, so the phase the heartbeat is
    reporting changes underneath it. The server stamps that phase onto the
    status frame, which is what lets the UI name the step during the many
    minutes a large checkpoint spends streaming in silence.
    """
    stop = threading.Event()
    resolve_stage = stage if callable(stage) else (lambda: stage)

    def beat():
        elapsed = 0
        while not stop.wait(interval):
            elapsed += int(interval)
            # One write, not print()'s two — a caller that also writes to stderr
            # from the main thread (generate_fastvideo.py streams its child's
            # output there) would otherwise see the beat land between a line's
            # text and its newline, gluing two protocol lines together.
            sys.stderr.write(f"STAGE:{resolve_stage()}:heartbeat:{elapsed}s\n")
            sys.stderr.flush()

    t = threading.Thread(target=beat, daemon=True)
    t.start()
    try:
        yield
    finally:
        stop.set()
        t.join(timeout=interval + 1)


# The LTX-2.5 MLX q8 pack's dev-layout files. The distilled model ships either
# pre-fused (`transformer-distilled.safetensors`) or as the dev transformer
# plus this 450-step distilled LoRA; both the render bridge
# (`generate_ltx2.py`) and the upscale runner (`upscale_ltx25.py`) resolve the
# same pair, so a re-pin of the adapter is one edit rather than two.
LTX25_DEV_TRANSFORMER_FILENAME = "transformer-dev.safetensors"
LTX25_DISTILLED_LORA_FILENAME = "ltx-2.5-22b-distilled-lora-450.safetensors"


def parse_user_loras(raw: "str | None") -> "list[tuple[str, float]]":
    """Parse a `--user-loras` JSON string into a list of (path, strength) tuples.

    Shared by the video LoRA runners (scripts/generate_ltx2.py for the dgrauet
    ltx2 runtime, scripts/generate_av_lora.py for the mlx_video runtime) so the
    strict-validation contract that protects against a Node-side bug lives in one
    place. Each entry must be {path: str (existing file), strength: number}.
    Returns [] when the arg is unset. Raises SystemExit with a precise message on
    any malformed input so the failure surfaces before any model load / GPU work.
    """
    if not raw:
        return []
    try:
        data = json.loads(raw)
    except json.JSONDecodeError as e:
        raise SystemExit(f"--user-loras is not valid JSON: {e}")
    if not isinstance(data, list):
        raise SystemExit("--user-loras must be a JSON list of {path, strength}")
    specs: "list[tuple[str, float]]" = []
    for i, item in enumerate(data):
        if not isinstance(item, dict) or "path" not in item:
            raise SystemExit(f"user-loras[{i}] must be an object with 'path'")
        path = item["path"]
        if not isinstance(path, str) or not path:
            raise SystemExit(f"user-loras[{i}].path must be a non-empty string")
        if not Path(path).exists():
            raise SystemExit(f"user-loras[{i}].path does not exist: {path}")
        strength = item.get("strength", 1.0)
        try:
            strength = float(strength)
        except (TypeError, ValueError):
            raise SystemExit(f"user-loras[{i}].strength must be a number")
        specs.append((path, strength))
    return specs


def _mac_chip() -> str:
    """Human chip name on macOS ('Apple M5 Max'). Falls back to
    platform.processor()/machine() so the field is never empty."""
    try:
        out = subprocess.run(
            ["sysctl", "-n", "machdep.cpu.brand_string"],
            capture_output=True, text=True, timeout=2,
        )
        if out.returncode == 0 and out.stdout.strip():
            return out.stdout.strip()
    except (OSError, subprocess.SubprocessError):
        pass
    return platform.processor() or platform.machine() or "unknown"


def _resolve_pkg_version(pkg: str) -> "str | None":
    """importlib.metadata.version for `pkg`, tolerant of the
    underscore/dash skew between import names and distribution names
    (callers pass whichever they know). Returns None when not installed."""
    from importlib.metadata import PackageNotFoundError, version as _version
    for candidate in (pkg, pkg.replace("_", "-"), pkg.replace("-", "_")):
        try:
            return _version(candidate)
        except PackageNotFoundError:
            continue
        except Exception:
            # An odd/transient metadata error on one spelling shouldn't skip the
            # remaining variants — keep trying, give up (None) only after all do.
            continue
    return None


def build_runtime_fingerprint(runtime_id: str, packages, extra_versions=None) -> dict:
    """Build a runtime-fingerprint dict — which runtime + resolved package
    versions + chip/os/python — so a render (or a bug report) self-documents the
    exact numerical stack it ran on. Garbled/"mosaic" video output is often a
    per-chip/per-runtime numerical bug rather than a crash, and the version trio
    is what makes those reports actionable (see issue #1325).

    Pure (no printing) so the on-demand /status probe (scripts/runtime_fingerprint.py)
    and the inline render-time emit share one definition. Best-effort: a package
    that isn't installed is simply omitted from `versions` rather than raising.
    """
    versions = {}
    for pkg in packages or []:
        v = _resolve_pkg_version(pkg)
        if v is not None:
            versions[pkg] = v
    # Caller-supplied versions that aren't pip distributions (e.g. the CUDA
    # toolkit version from torch.version.cuda) — merge into `versions` so they
    # render in the same list the log + /status UI already show, instead of a
    # top-level field nothing displays.
    for name, val in (extra_versions or {}).items():
        if val is not None:
            versions[name] = val
    fp = {
        "runtime": runtime_id,
        "versions": versions,
        "chip": _mac_chip() if sys.platform == "darwin"
        else (platform.processor() or platform.machine() or "unknown"),
        "os": platform.platform(),
        "python": platform.python_version(),
    }
    return fp


def emit_runtime_fingerprint(runtime_id: str, packages, extra_versions=None) -> dict:
    """Print a single `RUNTIME:<json>` line to stderr — the channel PortOS's
    videoGen line handler (makeVideoGenLineHandler) parses to stamp the
    fingerprint onto the render job + history record. Best-effort: never raises
    (a fingerprint failure must not abort a render). Returns the dict it emitted
    (or {} on failure) so callers can also log it locally if they want."""
    try:
        fp = build_runtime_fingerprint(runtime_id, packages, extra_versions)
        print(f"RUNTIME:{json.dumps(fp)}", file=sys.stderr, flush=True)
        return fp
    except Exception as err:  # pragma: no cover - defensive
        print(f"⚠️ runtime fingerprint failed: {err}", file=sys.stderr, flush=True)
        return {}


def build_image_execution_marker(
    runtime_id: str,
    requested_device: str,
    effective_device: str,
    placement: str,
    packages,
    extra_versions=None,
) -> dict:
    """Return the bounded execution evidence emitted by local image runners.

    A successful PNG is not evidence that an accelerator actually ran it.  Keep
    this marker deliberately small and portable: it records the caller's
    request, the resolved device/placement, and only the runtime's allowlisted
    package versions.  Host paths, prompts, environment, and stderr never
    belong in durable gallery metadata.
    """
    fingerprint = build_runtime_fingerprint(runtime_id, packages, extra_versions)
    cpu_fallback = effective_device == "cpu"
    return {
        "version": 1,
        "state": "degraded" if cpu_fallback else "confirmed",
        "requestedDevice": requested_device,
        "effectiveDevice": effective_device,
        "placement": placement,
        "cpuFallback": cpu_fallback,
        "runtime": {
            "runtime": fingerprint["runtime"],
            "versions": fingerprint["versions"],
        },
    }


def emit_image_execution_marker(
    runtime_id: str,
    requested_device: str,
    effective_device: str,
    placement: str,
    packages,
    extra_versions=None,
) -> dict:
    """Print one parseable image-execution marker after placement resolves."""
    try:
        marker = build_image_execution_marker(
            runtime_id,
            requested_device,
            effective_device,
            placement,
            packages,
            extra_versions,
        )
        print(f"IMAGE_EXECUTION:{json.dumps(marker)}", file=sys.stderr, flush=True)
        return marker
    except Exception as err:  # pragma: no cover - defensive
        print(f"⚠️ image execution marker failed: {err}", file=sys.stderr, flush=True)
        return {}


class _ClipTruncationFilter(logging.Filter):
    _MARKER = "input was truncated because CLIP can only handle"

    def filter(self, record: logging.LogRecord) -> bool:
        return self._MARKER not in record.getMessage()


def suppress_cosmetic_clip_truncation() -> None:
    """Hide the "CLIP can only handle N tokens" warning on multi-encoder
    pipelines (FLUX.2, HiDream, Qwen-Image, Z-Image, ERNIE).

    Every pipeline in this runner stack runs CLIP alongside a high-context
    encoder (T5 / Llama-3.1-8B / Qwen2-VL); CLIP only contributes pooled
    style/color vectors derived from the first ~77 tokens, while the long
    encoder reads the full prompt. The diffusers warning fires anyway and
    bleeds red into PortOS server logs as if the prompt were being damaged.
    Filter just that one message — every other diffusers warning still flows.

    Logger-level filters don't apply to propagated child records, so the
    filter has to be on the handlers. Idempotent (skips handlers it's
    already attached to) so callers don't need to track first-run state.
    """
    f = _ClipTruncationFilter()
    for name in ("diffusers", "transformers"):
        for handler in logging.getLogger(name).handlers:
            if not any(isinstance(existing, _ClipTruncationFilter) for existing in handler.filters):
                handler.addFilter(f)


def establish_process_group() -> None:
    """Make this helper the leader of its own process group, where supported.

    PortOS cancels a render by signalling the whole group, so any child the
    runner spawns (an ffmpeg mux, a git pin probe) is torn down with it rather
    than surviving to finish work the user just cancelled. Call it before the
    first child is spawned.

    A no-op on Windows, which has no POSIX process groups — the Node side kills
    by PID there. `setpgid` can also fail with EPERM when the process is already
    a group leader, which is harmless and equally not worth failing a render
    over, so both cases fall through quietly.
    """
    if not hasattr(os, "setpgid"):
        return
    try:
        os.setpgid(0, 0)
    except OSError:
        pass


def pick_device(requested: str) -> str:
    """Resolve `auto`/`mps`/`cuda`/`cpu` against what torch actually has.
    Falls back to CPU with a warning when the requested accelerator isn't
    available — never silently downgrades without telling the user."""
    import torch
    if requested == "auto":
        if torch.backends.mps.is_available():
            return "mps"
        if torch.cuda.is_available():
            return "cuda"
        return "cpu"
    if requested == "mps" and not torch.backends.mps.is_available():
        print("⚠️ MPS requested but unavailable — falling back to CPU", file=sys.stderr)
        return "cpu"
    if requested == "cuda" and not torch.cuda.is_available():
        print("⚠️ CUDA requested but unavailable — falling back to CPU", file=sys.stderr)
        return "cpu"
    return requested


def empty_device_cache(device: str, *, synchronize: bool = False) -> None:
    """Return cached-but-free blocks to the driver for the RESOLVED device.

    The caching allocator otherwise holds them, which starves a later large
    contiguous allocation (moving a multi-GB transformer onto the card) even
    though the memory is nominally free.

    Keyed on the device string `pick_device` resolved — NOT on
    `torch.cuda.is_available()`. A capability probe is the wrong predicate: on a
    box that *has* a CUDA card but is running `--device cpu`, probing would clear
    a cache this process never filled.

    `synchronize=True` blocks on the device afterwards, for callers that want the
    reclaim to have actually completed before they measure or allocate."""
    import torch
    if device == "mps":
        torch.mps.empty_cache()
        if synchronize:
            torch.mps.synchronize()
    elif device == "cuda":
        torch.cuda.empty_cache()
        if synchronize:
            torch.cuda.synchronize()


def make_generator(device: str, seed: int) -> "torch.Generator":
    """Seed a torch Generator on the right device. Accelerator generators
    must be initialised with the device string; the CPU fallback uses the
    no-arg form."""
    import torch
    if device in ("cuda", "mps"):
        return torch.Generator(device).manual_seed(int(seed))
    return torch.Generator().manual_seed(int(seed))


# VAE *tiling* is a memory optimization for very large decodes: it splits a
# single image's latent into overlapping spatial tiles, decodes each, and
# blends the overlaps. That blend leaves visible seams / dark bands on the
# small-to-mid resolutions PortOS actually renders — the same reason both
# runners already disable tiling on their i2i paths ("tiled encode of a small
# image produces seams"). At these sizes tiling buys no memory (the VAE decode
# is a rounding error next to the ~20 GB transformer), so only turn it on above
# an area floor no standard/experimental preset reaches. VAE-decode peak memory
# scales with output *area*, so the floor is in total pixels, not edge length:
# the largest preset that routes through these native runners is Qwen's
# experimental 1328×2048 (~2.7 MP), and the image-gen route caps a single edge
# at 3840 (so a 3840² custom render is ~14.7 MP and genuinely wants tiling to
# avoid an OOM on decode). ~6.5 MP sits between the two. VAE *slicing* (batch-dim
# split, no-op at batch 1) and attention slicing never touch spatial layout, so
# they stay unconditional.
_VAE_TILING_MIN_PIXELS = 2560 * 2560  # ~6.5 MP


def set_vae_tiling(pipe, enabled: bool) -> None:
    """Toggle VAE tiling on a loaded pipeline, preferring the pipeline-level
    switch and falling back to the VAE's own. Best-effort (gated on hasattr) so
    pipelines that expose neither still work. Shared by the txt2img size gate in
    `apply_memory_optimizations` and the runners' i2i paths, which force tiling
    off unconditionally (tiled encode of a small init image seams regardless of
    output size)."""
    pipe_attr = "enable_vae_tiling" if enabled else "disable_vae_tiling"
    vae_attr = "enable_tiling" if enabled else "disable_tiling"
    if hasattr(pipe, pipe_attr):
        getattr(pipe, pipe_attr)()
        return
    vae = getattr(pipe, "vae", None)
    if vae is not None and hasattr(vae, vae_attr):
        getattr(vae, vae_attr)()


def apply_memory_optimizations(pipe, width=None, height=None) -> None:
    """Enable memory-saving knobs on the loaded pipeline. Best-effort: each
    call is gated on hasattr so older / smaller pipelines that don't ship a
    given slice / tile path still work.

    Attention slicing and VAE slicing are always safe and always applied. VAE
    *tiling* is applied only when the target render is large enough to need it
    (total pixels > `_VAE_TILING_MIN_PIXELS`); below that it is explicitly
    disabled to avoid tile-seam / black-band artifacts (issue: z-image-turbo
    576×1024 renders showed a black strip at a content-dependent point near the
    bottom). When `width`/`height` are omitted the old always-tile behavior is
    preserved for back-compat."""
    if hasattr(pipe, "enable_attention_slicing"):
        pipe.enable_attention_slicing()
    if hasattr(pipe, "enable_vae_slicing"):
        pipe.enable_vae_slicing()

    # A diffusers default or a reused pipeline could leave tiling enabled, so
    # drive it explicitly in both directions rather than only turning it on.
    want_tiling = True
    if width and height:
        want_tiling = (int(width) * int(height)) > _VAE_TILING_MIN_PIXELS
    set_vae_tiling(pipe, want_tiling)


# Headroom reserved on top of the resident weights for activations + allocator
# fragmentation when deciding whether a pipeline fits in *currently free* VRAM.
# Catches the case where another process is hogging the card even for a
# mid-size model.
_ACTIVATION_RESERVE_BYTES = 3 * 1024 ** 3

# Second, resolution-independent offload trigger: when the weights alone exceed
# this fraction of the card's *total* VRAM, force offload regardless of how
# much is momentarily free. A model that fills most of the card leaves no
# bounded margin for high-res activations or the per-step stepwise VAE decode,
# and on Windows the NVIDIA driver answers the overflow with silent sysmem
# fallback (~50x slowdown) rather than OOM — so a fixed activation reserve
# can't protect against it. Offload is cheap for these (the transformer stays
# resident across the whole denoising loop), so erring toward it costs a few
# seconds of one-time component paging and removes the thrash cliff entirely.
# 0.5 keeps SDXL (~7 GB) / SD-1.5 (~2 GB) fully resident on a 24 GB card while
# routing the ~19-20 GB Z-Image / ERNIE / Qwen bf16 pipelines to offload.
_OFFLOAD_VRAM_FRACTION = 0.5


def _pipeline_weight_bytes(pipe) -> int:
    """Total bytes of the pipeline's parameters + buffers, de-duplicated by
    storage pointer so tied / shared weights aren't double-counted."""
    components = getattr(pipe, "components", None) or {}
    modules = [m for m in components.values() if hasattr(m, "parameters")]
    seen = set()
    total = 0
    for module in modules:
        for tensor in list(module.parameters()) + list(module.buffers()):
            key = tensor.data_ptr()
            if key in seen or key == 0:
                continue
            seen.add(key)
            total += tensor.numel() * tensor.element_size()
    return total


def choose_cuda_pipeline_placement(
    pipe,
    torch,
    *,
    override_env: str,
    offload_vram_fraction=None,
    log_label: str,
) -> dict:
    """Choose full CUDA residency or CPU offload from live VRAM state.

    The model weights plus a fixed activation reserve must fit in *currently
    free* VRAM. Callers may additionally provide a total-card fraction to avoid
    Windows sysmem fallback for pipelines that leave too little proportional
    headroom. ``override_env`` is deliberately caller-owned so one pipeline's
    override cannot change another pipeline family.
    """
    override = os.environ.get(override_env, "").strip().lower()
    if override in ("0", "false", "never"):
        return {"use_offload": False, "reason": f"{override_env}=0"}
    if override in ("1", "true", "force", "always"):
        return {"use_offload": True, "reason": f"{override_env}=1"}

    weight_bytes = _pipeline_weight_bytes(pipe)
    free_bytes, total_bytes = torch.cuda.mem_get_info()
    fits_now = (weight_bytes + _ACTIVATION_RESERVE_BYTES) <= free_bytes
    fills_card = (
        offload_vram_fraction is not None
        and weight_bytes > offload_vram_fraction * total_bytes
    )
    use_offload = fills_card or not fits_now
    reason = "fills card" if fills_card else ("low free VRAM" if not fits_now else "fits")
    gb = 1024 ** 3
    print(
        f"🧮 {log_label} VRAM: weights ~{weight_bytes / gb:.1f}GB, "
        f"free ~{free_bytes / gb:.1f}GB, total ~{total_bytes / gb:.1f}GB ({reason}) → "
        f"{'CPU offload' if use_offload else 'full GPU residency'}",
        file=sys.stderr, flush=True,
    )
    return {
        "use_offload": use_offload,
        "reason": reason,
        "weight_bytes": weight_bytes,
        "free_bytes": free_bytes,
        "total_bytes": total_bytes,
    }


def place_pipeline(pipe, device: str) -> str:
    """Move a loaded diffusers pipeline onto the compute device, choosing
    between full-GPU residency and model CPU offload based on free VRAM.

    On CUDA, a pipeline whose weights nearly fill the card (Z-Image-Turbo /
    ERNIE / Qwen bf16 are ~20 GB on a 24 GB GPU) leaves no room for the
    forward pass's activations. On Windows the NVIDIA driver then silently
    spills the overflow into system RAM ("sysmem fallback") instead of
    OOM-ing, and every denoising step thrashes across PCIe — ~50x slower than
    the GPU is capable of. When the weights plus an activation reserve won't
    fit in free VRAM, fall back to `enable_model_cpu_offload()`: the
    transformer stays resident across the loop while the text encoder and VAE
    are paged in only for their one-shot forward, so peak VRAM drops well
    under the card's limit and the steps run at full GPU speed.

    MPS / CPU keep the plain `.to(device)` path — offload is a CUDA+accelerate
    feature and the sysmem-fallback failure mode it guards against is
    CUDA-specific. Override the heuristic with `PORTOS_IMAGE_OFFLOAD=1`
    (force offload) or `PORTOS_IMAGE_OFFLOAD=0` (force full residency).
    Returns the effective placement string for logging.
    """
    if device != "cuda":
        pipe.to(device)
        return device

    import torch

    can_offload = hasattr(pipe, "enable_model_cpu_offload")
    placement = choose_cuda_pipeline_placement(
        pipe,
        torch,
        override_env="PORTOS_IMAGE_OFFLOAD",
        offload_vram_fraction=_OFFLOAD_VRAM_FRACTION,
        log_label="image-gen",
    )
    force_offload = placement["use_offload"]

    if force_offload and can_offload:
        pipe.enable_model_cpu_offload()
        return "cuda+offload"
    pipe.to(device)
    return device


def write_sidecar(output: str, payload: dict) -> None:
    """Write `<output>.metadata.json` next to the generated image. The
    server's gallery scanner picks this up to surface prompt/seed/model
    in the lightbox without re-parsing the PNG."""
    sidecar = Path(output).with_suffix(".metadata.json")
    sidecar.write_text(json.dumps(payload, indent=2))


def write_stepwise_preview(stepwise_dir: str, frame) -> bool:
    """Atomically publish one bounded preview frame for the active render.

    The server watches a job-scoped directory and reads ``preview.png``. A
    fixed filename keeps the directory bounded even when a runner emits a
    progress callback for every denoise step; replacing it atomically also
    prevents the watcher from reading a partially-written PNG.
    """
    if not stepwise_dir:
        return False
    from PIL import Image

    out = Path(stepwise_dir)
    out.mkdir(parents=True, exist_ok=True)
    image = frame if isinstance(frame, Image.Image) else Image.fromarray(frame)
    # Pillow < 9 exposes the resampling constants directly on Image rather
    # than through Image.Resampling. The runner environments are user-managed,
    # so keep the bounded preview contract compatible with both shapes.
    resampling = getattr(Image, "Resampling", Image).LANCZOS
    image.thumbnail((512, 512), resampling)
    # Keep the staging file out of directory scans that look for visible PNGs;
    # the explicit format argument still makes Pillow encode it as PNG.
    temporary = out / f".preview-{os.getpid()}.tmp"
    image.save(temporary, "PNG", optimize=False)
    os.replace(temporary, out / "preview.png")
    return True


def make_stepwise_callback(
    stepwise_dir: str,
    pipe,
    height: int,
    width: int,
    *,
    unpack_latents=None,
    preview_decoder=None,
):
    """Return a `callback_on_step_end` that decodes the running latent into
    a small preview PNG. `local.js#processLatestFrame` watches this dir and
    streams the freshest frame to the SSE client.

    The only per-runner difference is how packed latents are projected back
    to `(B, C, H_lat, W_lat)` before the VAE decode:

      - mflux / z-image: latents already arrive in `(B, C, H_lat, W_lat)`
        layout. Pass `unpack_latents=None`; the callback simply skips the
        step when latents are not 4-D (rather than guessing the shape).
      - flux2: transformer-packed latents come back as
        `(B, num_patches, C*p*p)`. Pass a callable that takes
        `(latents, height, width)` and returns the unpacked tensor.
      - ernie: latents stay 4-D but are 2x2 patch-packed (e.g. `(B, 128, H/2, W/2)`
        for a 32-ch VAE) AND need pipeline-specific BN-stats unnormalization
        before `vae.decode`, not the standard `latents/scaling + shift`.
        Pass `preview_decoder=fn(pipe, latents) -> image_tensor` to fully
        override the per-step decode path.
    """
    if not stepwise_dir:
        return None
    import torch
    out = Path(stepwise_dir)
    out.mkdir(parents=True, exist_ok=True)
    vae = pipe.vae
    scaling = getattr(vae.config, "scaling_factor", 1.0) or 1.0
    shift = getattr(vae.config, "shift_factor", 0.0) or 0.0
    # Capture the VAE's weight dtype + device so the callback can align the
    # scaled latents before decode. Some pipelines (ERNIE, Flux2) keep latents
    # in float32 even when the pipeline was loaded with torch_dtype=bfloat16,
    # so `vae.decode(latents / scaling + shift)` would feed float32 into a
    # bfloat16 VAE and error with "Input type (float) and bias type
    # (c10::BFloat16) should be the same".
    try:
        vae_param = next(vae.parameters())
        vae_dtype, vae_device = vae_param.dtype, vae_param.device
    except StopIteration:
        vae_dtype, vae_device = None, None
    # Under `enable_model_cpu_offload()` (see place_pipeline) the VAE's params
    # rest on CPU but its forward runs on the accelerate hook's execution
    # device (CUDA) — and the hook moves the *weights* there without moving the
    # latents we hand it, so aligning the preview latents to the resting
    # (CPU) device decodes with "weight on cuda:0, other tensors on cpu". Target
    # the execution device instead, so the latents land where the VAE will run.
    _vae_hook = getattr(vae, "_hf_hook", None)
    _exec_device = getattr(_vae_hook, "execution_device", None)
    if _exec_device is not None:
        vae_device = _exec_device

    fired = {"count": 0, "saved": 0}

    @torch.no_grad()
    def cb(pipe, step_index, _timestep, callback_kwargs):
        fired["count"] += 1
        latents = callback_kwargs.get("latents")
        if latents is None:
            if step_index == 0:
                print("⚠️ stepwise: latents missing from callback_kwargs", file=sys.stderr)
            return callback_kwargs
        if step_index == 0:
            print(f"🖼️  stepwise: callback live, latents.shape={tuple(latents.shape)}", file=sys.stderr)
        # Best-effort decode. Errors here must not abort generation — the
        # final image is still produced after the last step.
        try:
            if latents.dim() == 3:
                if unpack_latents is None:
                    # No unpack helper provided; skip preview rather than
                    # guess the shape. The final image still saves.
                    return callback_kwargs
                latents = unpack_latents(latents, height, width)
            elif latents.dim() != 4:
                return callback_kwargs
            if preview_decoder is not None:
                aligned = latents
                if vae_dtype is not None and (aligned.dtype != vae_dtype or aligned.device != vae_device):
                    aligned = aligned.to(device=vae_device, dtype=vae_dtype)
                decoded = preview_decoder(pipe, aligned)
            else:
                scaled = latents / scaling + shift
                if vae_dtype is not None and (scaled.dtype != vae_dtype or scaled.device != vae_device):
                    scaled = scaled.to(device=vae_device, dtype=vae_dtype)
                decoded = vae.decode(scaled, return_dict=False)[0]
            decoded = (decoded.clamp(-1, 1) + 1) / 2
            arr = (decoded[0].float().cpu().permute(1, 2, 0).numpy() * 255).astype("uint8")
            if write_stepwise_preview(str(out), arr):
                fired["saved"] += 1
        except Exception as err:
            print(f"⚠️ stepwise preview failed at step {step_index}: {type(err).__name__}: {err}", file=sys.stderr)
        return callback_kwargs

    cb._stats = fired
    return cb


def _emit_user_error(kind: str, message: str, repo: str = "") -> None:
    """Emit a structured single-line error the server's stderr parser picks up
    for the SSE error event. `kind` is a stable identifier the UI maps to a
    friendly heading; `message` is the human prose; `repo` (optional) is the
    HF repo to deep-link the user to so they can request access / check token."""
    line = f"USER_ERROR:{kind}"
    if repo:
        line += f":{repo}"
    print(line, file=sys.stderr, flush=True)
    print(f"❌ {message}", file=sys.stderr, flush=True)


def _repo_from_hf_error(hf_err) -> str:
    """huggingface_hub doesn't always populate `repo_id` on the error.
    Fall back to parsing the failing URL — `/<owner>/<repo>/resolve/...`
    for file downloads or `/api/models/<owner>/<repo>` for API hits."""
    repo = getattr(hf_err, "repo_id", None) or ""
    if repo:
        return repo
    url = (
        getattr(getattr(hf_err, "response", None), "url", None)
        or getattr(getattr(hf_err, "request", None), "url", None)
    )
    if url is None:
        return ""
    path = str(url).split("huggingface.co", 1)[-1].lstrip("/")
    parts = path.split("/")
    if parts[:1] == ["api"] and len(parts) >= 4 and parts[1] in {"models", "datasets", "spaces"}:
        return f"{parts[2]}/{parts[3]}"
    if len(parts) >= 2 and parts[0] not in {"api", "settings", "join"}:
        return f"{parts[0]}/{parts[1]}"
    return ""


def install_hf_error_handler(main_fn):
    """Decorator that wraps a runner's `main()` in the canonical HuggingFace
    error-to-USER_ERROR-line translation.

    Walks the exception cause chain so a buried `GatedRepoError` /
    `RepositoryNotFoundError` / 401 `HfHubHTTPError` produces a friendly
    `USER_ERROR:<kind>:<repo>` line + `❌ <message>` on stderr — even when
    diffusers wraps the underlying HF error in OSError. Unknown failures
    emit a generic `USER_ERROR:unknown` marker and re-raise so the original
    traceback still surfaces.

    Usage:

        @install_hf_error_handler
        def main():
            ...

        if __name__ == "__main__":
            main()
    """

    @wraps(main_fn)
    def wrapped(*args, **kwargs):
        try:
            return main_fn(*args, **kwargs)
        except KeyboardInterrupt:
            sys.exit(130)
        except SystemExit:
            raise
        except Exception as err:
            # Lazy-import so the module loads on systems without
            # huggingface_hub installed (e.g. test environments).
            from huggingface_hub.errors import (
                GatedRepoError,
                HfHubHTTPError,
                RepositoryNotFoundError,
            )
            # Walk the cause chain — diffusers wraps HF errors in OSError, so
            # the innermost exception is what tells us the real story.
            chain = []
            cur = err
            while cur is not None:
                chain.append(cur)
                cur = cur.__cause__ or cur.__context__
            gated = next((e for e in chain if isinstance(e, GatedRepoError)), None)
            notfound = next(
                (e for e in chain if isinstance(e, RepositoryNotFoundError)), None
            )
            http = next((e for e in chain if isinstance(e, HfHubHTTPError)), None)
            if gated is not None:
                repo = _repo_from_hf_error(gated)
                url = f"https://huggingface.co/{repo}" if repo else "https://huggingface.co/"
                _emit_user_error(
                    "gated_repo",
                    f"Access to {repo or 'the model repo'} is restricted. Visit {url} "
                    f"to request access, then make sure your HF token is set in PortOS.",
                    repo,
                )
                sys.exit(2)
            status = (
                getattr(getattr(http, "response", None), "status_code", None)
                if http
                else None
            )
            if status == 401:
                _emit_user_error(
                    "hf_unauthorized",
                    "HuggingFace rejected the token (401). Check that the token is valid "
                    "and has read access, then re-paste it in PortOS.",
                )
                sys.exit(2)
            if notfound is not None:
                repo = _repo_from_hf_error(notfound)
                _emit_user_error(
                    "repo_not_found", f"HF repo not found: {repo or '(unknown)'}", repo
                )
                sys.exit(2)
            # Unknown failure — emit the original traceback (raised below) plus
            # a generic structured marker so the UI shows something useful.
            _emit_user_error("unknown", f"{type(err).__name__}: {err}")
            raise

    return wrapped