Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

9.4 KB
Raw Blame
"""
Windows image generation backend using diffusers + local Flux model files.
Accepts the same CLI args as mflux-generate so imagine.js needs no change.
"""
import argparse
import json
import os
import sys
import warnings

warnings.filterwarnings('ignore')
os.environ['XFORMERS_DISABLED'] = '1'
# Avoid protobuf descriptor crash when converting T5 slow tokenizer
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python'

import torch
from diffusers import AutoencoderKL, FluxPipeline, FluxTransformer2DModel
from diffusers.schedulers import FlowMatchEulerDiscreteScheduler
from safetensors.torch import load_file
from transformers import (
    CLIPTextConfig, CLIPTextModel, CLIPTokenizer,
    T5Config, T5EncoderModel, T5TokenizerFast,
)

from _runner_common import emit_image_execution_marker

# ---------------------------------------------------------------------------
# Paths
# ---------------------------------------------------------------------------
DOWNLOADS = os.path.join(os.path.expanduser('~'), 'Downloads')
HF_FLUX_CACHE = None  # resolved at runtime

FLUX_FILES = {
    'schnell': os.path.join(DOWNLOADS, 'flux1-schnell-fp8.safetensors'),
    'dev':     os.path.join(DOWNLOADS, 'flux1-dev-fp8.safetensors'),
}
VAE_FILE   = os.path.join(DOWNLOADS, 'ae.safetensors')
CLIP_FILE  = os.path.join(DOWNLOADS, 'clip_l.safetensors')
T5_FILE    = os.path.join(DOWNLOADS, 't5xxl_fp16.safetensors')

MODEL_ID_MAP = {
    'schnell':        'schnell',
    'flux2-klein-4b': 'schnell',
    'flux2-klein-9b': 'dev',
    'dev':            'dev',
}

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

def log(msg):
    print(msg, file=sys.stderr, flush=True)


def find_flux_hf_cache():
    """Return the local snapshot dir for FLUX.1-schnell configs."""
    hub = os.path.join(os.path.expanduser('~'), '.cache', 'huggingface', 'hub')
    snap_root = os.path.join(hub, 'models--black-forest-labs--FLUX.1-schnell', 'snapshots')
    if not os.path.isdir(snap_root):
        return None
    snaps = sorted(os.listdir(snap_root))
    return os.path.join(snap_root, snaps[-1]) if snaps else None


def progress_callback(pipe, step, timestep, callback_kwargs):
    total = pipe._num_timesteps
    pct = int((step + 1) / total * 100)
    bar = '█' * (pct // 5) + '▒' * (20 - pct // 5)
    log(f'{pct}%|{bar}| {step + 1}/{total}')
    return callback_kwargs


def build_pipeline(flux_key, hf_cache, dtype):
    log(f'STATUS:Loading tokenizers...')
    tokenizer = CLIPTokenizer.from_pretrained(hf_cache, subfolder='tokenizer')
    # Load T5 fast tokenizer from pre-built tokenizer.json to avoid protobuf/spiece issues
    t2_tok_file = os.path.join(hf_cache, 'tokenizer_2', 'tokenizer.json')
    tokenizer_2 = T5TokenizerFast(
        tokenizer_file=t2_tok_file,
        model_max_length=512,
        legacy=False,
    )
    scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(hf_cache, subfolder='scheduler')

    log('STATUS:Loading CLIP text encoder...')
    clip_cfg     = CLIPTextConfig.from_pretrained(os.path.join(hf_cache, 'text_encoder'))
    text_encoder = CLIPTextModel(clip_cfg)
    text_encoder.load_state_dict(load_file(CLIP_FILE), strict=False)
    # CLIP is small (~0.5 GB) — keep on CUDA so the pipeline's _execution_device resolves to CUDA
    text_encoder = text_encoder.to(dtype).to('cuda')

    log('STATUS:Loading T5 text encoder (CPU)...')
    t5_cfg          = T5Config.from_pretrained(os.path.join(hf_cache, 'text_encoder_2'))
    text_encoder_2  = T5EncoderModel(t5_cfg)
    text_encoder_2.load_state_dict(load_file(T5_FILE), strict=False)
    # T5 stays on CPU to save ~4 GB of VRAM for the transformer.
    # Register a pre-hook that moves any CUDA inputs back to CPU so the
    # pipeline's .to(execution_device) call doesn't cause a device mismatch.
    text_encoder_2  = text_encoder_2.to(dtype)
    def _cpu_input_hook(module, args, kwargs):
        args = tuple(x.cpu() if isinstance(x, torch.Tensor) else x for x in args)
        kwargs = {k: v.cpu() if isinstance(v, torch.Tensor) else v for k, v in kwargs.items()}
        return args, kwargs
    text_encoder_2.register_forward_pre_hook(_cpu_input_hook, with_kwargs=True)

    log('STATUS:Loading VAE...')
    vae = AutoencoderKL.from_single_file(
        VAE_FILE, config=hf_cache, subfolder='vae', torch_dtype=dtype
    ).to('cuda')

    log(f'STATUS:Loading Flux transformer ({flux_key})...')
    transformer = FluxTransformer2DModel.from_single_file(
        FLUX_FILES[flux_key], torch_dtype=dtype
    )
    # The flux1-*-fp8 checkpoints are fp8 on disk, but from_single_file upcasts
    # them to the bf16 compute dtype (~23GB). On a 24GB card that overflows once
    # the desktop/browser baseline (~5GB) plus activations are accounted for, so
    # the driver spills to shared system RAM and each diffusion step takes
    # minutes — long enough to trip the media-job idle watchdog (300s). Layerwise
    # casting keeps the weights stored as fp8 (~15GB resident — fp8 weight bytes
    # plus the norm/embedding modules that stay bf16) and upcasts each layer to
    # bf16 only during its forward pass, so the model fits in VRAM while compute
    # stays bf16. (Ampere/3090 has no fp8 tensor cores, so we must NOT compute in
    # fp8 — storage-only casting is the correct strategy here.) Set
    # IMAGINE_WIN_FP8=0 (or false/never) to fall back to the old full-bf16 load.
    if os.environ.get('IMAGINE_WIN_FP8', '1').strip().lower() not in ('0', 'false', 'never'):
        transformer.enable_layerwise_casting(
            storage_dtype=torch.float8_e4m3fn, compute_dtype=dtype
        )
        log('STATUS:Transformer weights stored as fp8 (bf16 compute)...')
    transformer = transformer.to('cuda')

    log('STATUS:Assembling pipeline...')
    pipe = FluxPipeline(
        scheduler=scheduler,
        text_encoder=text_encoder,
        tokenizer=tokenizer,
        text_encoder_2=text_encoder_2,
        tokenizer_2=tokenizer_2,
        vae=vae,
        transformer=transformer,
    )
    # Reduces activation peak memory so the 22GB transformer has breathing room
    pipe.enable_attention_slicing()
    return pipe


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

def main():
    p = argparse.ArgumentParser()
    p.add_argument('--model',          default='schnell')
    p.add_argument('--prompt',         required=True)
    p.add_argument('--negative-prompt', default='')
    p.add_argument('--height',         type=int, default=1024)
    p.add_argument('--width',          type=int, default=1024)
    p.add_argument('--steps',          type=int, default=4)
    p.add_argument('--seed',           type=int, default=None)
    p.add_argument('--guidance',       type=float, default=0.0)
    p.add_argument('--quantize',       type=int, default=8)   # ignored on Windows
    p.add_argument('--output',         required=True)
    p.add_argument('--metadata',       action='store_true')
    p.add_argument('--lora-paths',     nargs='*', default=[])
    p.add_argument('--lora-scales',    nargs='*', type=float, default=[])
    args = p.parse_args()

    flux_key = MODEL_ID_MAP.get(args.model, 'schnell')
    dtype    = torch.bfloat16

    hf_cache = find_flux_hf_cache()
    if not hf_cache:
        log('ERROR: Flux config cache not found. Run the app Settings page first.')
        sys.exit(1)

    for label, path in [('Flux model', FLUX_FILES[flux_key]), ('VAE', VAE_FILE),
                         ('CLIP', CLIP_FILE), ('T5', T5_FILE)]:
        if not os.path.exists(path):
            log(f'ERROR: {label} file not found: {path}')
            sys.exit(1)

    generator = torch.Generator('cuda').manual_seed(args.seed) if args.seed is not None else None

    pipe = build_pipeline(flux_key, hf_cache, dtype)

    # The transformer and CLIP run on CUDA while T5 is intentionally kept on
    # CPU to fit the Windows pipeline in available VRAM.  Emit this evidence
    # only after placement is complete so the gallery can distinguish the
    # split CUDA path from an unmarked legacy runner or a CPU fallback.
    emit_image_execution_marker(
        'diffusers-image',
        'cuda',
        'cuda',
        'cuda+offload',
        ['torch', 'diffusers', 'transformers', 'accelerate'],
    )

    log(f'STATUS:Generating image ({args.width}x{args.height}, {args.steps} steps)...')

    result = pipe(
        prompt=args.prompt,
        negative_prompt=args.negative_prompt or None,
        height=args.height,
        width=args.width,
        num_inference_steps=args.steps,
        guidance_scale=args.guidance if args.guidance > 0 else 0.0,
        generator=generator,
        callback_on_step_end=progress_callback,
        callback_on_step_end_tensor_inputs=['latents'],
    )

    image = result.images[0]
    os.makedirs(os.path.dirname(args.output), exist_ok=True)
    image.save(args.output)
    log(f'STATUS:Saved to {args.output}')

    if args.metadata:
        meta = {
            'model': args.model,
            'prompt': args.prompt,
            'negative_prompt': args.negative_prompt,
            'height': args.height,
            'width': args.width,
            'steps': args.steps,
            'seed': args.seed,
            'guidance': args.guidance,
        }
        meta_path = args.output.replace('.png', '.metadata.json')
        with open(meta_path, 'w') as f:
            json.dump(meta, f, indent=2)

    log('STATUS:Done')


if __name__ == '__main__':
    main()