Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

9.6 KB
Raw Blame
#!/usr/bin/env python3
"""
PortOS AudioLDM2 runner — local OSS long-form background-music generation.
Spawned by `server/services/pipeline/musicGen.js` as a sibling backend to
`scripts/generate_musicgen.py`.

Pipeline Audio Phase 4c.2 shipped local MusicGen (MLX) as the first generator
behind the audio stage's `source: 'gen'` library entry. AudioLDM2 is the second
backend: a latent-diffusion text-to-audio model (CVSSP/audioldm2) that runs via
HuggingFace `diffusers`. Its draw over MusicGen is *long-form* output — MusicGen
degrades past ~30s (trained on 30s windows), whereas AudioLDM2 takes an explicit
`audio_length_in_s` and renders coherent clips well past that, with no API key
and no network at inference time (after the one-time weight download).

This script deliberately MIRRORS generate_musicgen.py's CLI/contract so the JS
side can spawn either behind the same `generateMusic` path:
  --model <hf-repo>   --text <prompt>   --output <wav>   --duration <seconds>
  [--runtime-dir <dir>]  [--seed <int>]  [--steps <int>]  [--guidance <float>]

Progress protocol (mirrors the image/video/musicgen sidecars): STAGE:<name>
[:detail] lines on stderr drive the JS-side phase tracker; a final
`RESULT:<json>` line on stdout reports the saved path + actual duration.

Output: a 16 kHz mono 16-bit PCM WAV at --output. AudioLDM2 generates at 16 kHz;
we write the WAV with the stdlib `wave` module (+ numpy for the float→int16
conversion) rather than depending on scipy/soundfile, matching the musicgen
sidecar so the runtime's dependency surface stays small.
"""

import argparse
import json
import os
import sys
import wave

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from _runner_common import heartbeat  # noqa: E402


# AudioLDM2 decodes audio at 16 kHz (fixed by the model's VAE / vocoder).
SAMPLE_RATE = 16000


def log_stage(name, detail=""):
    """Emit a STAGE: line the JS sidecar tails for phase/progress display."""
    line = f"STAGE:{name}" + (f":{detail}" if detail else "")
    print(line, file=sys.stderr, flush=True)


def _import_pipeline(runtime_dir):
    """Resolve the diffusers AudioLDM2Pipeline class.

    AudioLDM2 ships in HuggingFace `diffusers` (a pip package), so unlike the
    MLX MusicGen example we normally just import it. `runtime_dir` is honored
    for parity with the musicgen sidecar — if a caller vendored a diffusers
    build into a clone dir, prepend it to sys.path first. Raising a clear
    ImportError lets the JS side surface "run the installer" rather than a bare
    traceback.
    """
    if runtime_dir and os.path.isdir(runtime_dir) and runtime_dir not in sys.path:
        sys.path.insert(0, runtime_dir)
    try:
        from diffusers import AudioLDM2Pipeline  # type: ignore
        return AudioLDM2Pipeline
    except ImportError as exc:
        raise ImportError(
            "Could not import AudioLDM2Pipeline from diffusers. Run "
            "`INSTALL_AUDIOLDM2=1 bash scripts/setup-image-video.sh` to build "
            f"the venv (torch + diffusers + transformers; looked in: {runtime_dir})."
        ) from exc


def _pick_device():
    """Prefer Apple-Silicon MPS, then CUDA, else CPU.

    Returns the torch device string. AudioLDM2 runs on all three; MPS is the
    common case on PortOS's target hardware (Apple Silicon), CUDA covers Linux
    GPU boxes, CPU is the slow fallback so generation still completes.
    """
    import torch

    if torch.backends.mps.is_available():
        return "mps"
    if torch.cuda.is_available():
        return "cuda"
    return "cpu"


def _patch_language_model_generation(pipe):
    """Re-bind GenerationMixin generation helpers onto AudioLDM2's language model.

    AudioLDM2's `language_model` is a bare `GPT2Model` (no LM head — the pipeline
    reads its hidden states). diffusers' `AudioLDM2Pipeline.generate_language_model`
    drives it by hand and calls `_update_model_kwargs_for_generation` (and, on
    transformers >=4.52.1, `_get_initial_cache_position`). Those used to live on
    every `PreTrainedModel`, but transformers >=4.50 decoupled `GenerationMixin`
    so non-generative base models like `GPT2Model` no longer carry them — leaving
    diffusers calling a method that no longer exists:
        AttributeError: 'GPT2Model' object has no attribute
        '_update_model_kwargs_for_generation'

    We restore the missing helpers from `GenerationMixin` onto the model's class
    (their signatures already match diffusers' call sites). This keeps the venv on
    a current transformers instead of pinning an EOL release. No-op on older
    transformers where the methods are already present.
    """
    lm = getattr(pipe, "language_model", None)
    if lm is None:
        return
    try:
        from transformers.generation.utils import GenerationMixin
    except Exception:
        return
    cls = type(lm)
    for name in ("_update_model_kwargs_for_generation", "_get_initial_cache_position"):
        if not hasattr(lm, name) and hasattr(GenerationMixin, name):
            setattr(cls, name, getattr(GenerationMixin, name))


def _to_int16_pcm(audio):
    """Flatten a numpy audio array to a 1-D int16 numpy buffer.

    AudioLDM2 returns a float array (shape [batch, samples]); we take the first
    waveform, flatten to mono, clip to [-1, 1] so a hot sample can't wrap on the
    int16 cast, and scale to full-scale PCM. Mirrors the musicgen sidecar's
    conversion so both backends emit byte-identical WAV framing.
    """
    import numpy as np

    arr = np.array(audio, dtype=np.float32)
    if arr.ndim > 1:
        arr = arr[0]
    arr = arr.reshape(-1)
    np.clip(arr, -1.0, 1.0, out=arr)
    return (arr * 32767.0).astype(np.int16)


def _write_wav(path, pcm_int16):
    with wave.open(path, "wb") as wav:
        wav.setnchannels(1)
        wav.setsampwidth(2)  # 16-bit
        wav.setframerate(SAMPLE_RATE)
        wav.writeframes(pcm_int16.tobytes())


def main():
    parser = argparse.ArgumentParser(description="PortOS AudioLDM2 runner (diffusers)")
    parser.add_argument("--model", default="cvssp/audioldm2",
                        help="HF repo id of the AudioLDM2 weights")
    parser.add_argument("--text", required=True, help="Text prompt")
    parser.add_argument("--output", required=True, help="Output WAV path")
    parser.add_argument("--duration", type=float, default=20.0,
                        help="Target clip length in seconds (long-form supported)")
    parser.add_argument("--runtime-dir", default=os.environ.get("PORTOS_AUDIOLDM2_RUNTIME_DIR", ""),
                        help="Optional dir prepended to sys.path before importing diffusers")
    parser.add_argument("--seed", type=int, default=0,
                        help="RNG seed for reproducible generation")
    parser.add_argument("--steps", type=int, default=200,
                        help="Number of diffusion inference steps")
    parser.add_argument("--guidance", type=float, default=3.5,
                        help="Classifier-free guidance scale")
    args = parser.parse_args()

    text = args.text or ""
    if not text.strip():
        print("ERROR: --text is required", file=sys.stderr, flush=True)
        return 2

    # AudioLDM2 supports long-form, but a hard floor keeps the VAE from
    # producing a zero-length latent; the ceiling is a sanity guard, not a model
    # limit (the JS side clamps to MAX_DURATION_SEC for its engine before here).
    duration = max(1.0, min(float(args.duration or 20.0), 300.0))
    steps = max(1, int(args.steps or 200))

    log_stage("import-runtime")
    AudioLDM2Pipeline = _import_pipeline(args.runtime_dir)

    import torch

    device = _pick_device()
    # float16 on accelerators halves memory + speeds inference; CPU stays float32
    # because half-precision matmul is unsupported / slow there.
    dtype = torch.float32 if device == "cpu" else torch.float16

    log_stage("load-model", args.model)
    with heartbeat("load-model"):
        pipe = AudioLDM2Pipeline.from_pretrained(args.model, torch_dtype=dtype)
        pipe = pipe.to(device)
    # transformers >=4.50 compatibility: restore the generation helpers the
    # pipeline's language-model loop relies on (see helper docstring).
    _patch_language_model_generation(pipe)

    # PyTorch's MPS backend does not support torch.Generator(device='mps') — it
    # raises before inference on Apple Silicon (our primary target). Seed on CPU
    # there; diffusers accepts a CPU generator for an MPS pipeline. CUDA/CPU use
    # their native device generator. https://github.com/pytorch/pytorch/issues/84288
    generator_device = "cpu" if device == "mps" else device
    generator = torch.Generator(device=generator_device).manual_seed(int(args.seed or 0))

    log_stage("generate", f"{duration:.1f}s/{steps}steps")
    with heartbeat("generate"):
        out = pipe(
            text,
            num_inference_steps=steps,
            audio_length_in_s=duration,
            guidance_scale=float(args.guidance or 3.5),
            generator=generator,
        )

    log_stage("encode-wav")
    pcm = _to_int16_pcm(out.audios)
    out_dir = os.path.dirname(os.path.abspath(args.output))
    if out_dir:
        os.makedirs(out_dir, exist_ok=True)
    _write_wav(args.output, pcm)

    # The rendered length can differ slightly from the request (the model
    # quantizes to its latent frame rate). Report what we actually wrote so the
    # JS side persists truth — identical contract to the musicgen sidecar.
    actual_seconds = len(pcm) / float(SAMPLE_RATE)
    result = {
        "output": args.output,
        "model": args.model,
        "durationSec": round(actual_seconds, 3),
        "sampleRate": SAMPLE_RATE,
    }
    print("RESULT:" + json.dumps(result), flush=True)
    log_stage("done")
    return 0


if __name__ == "__main__":
    sys.exit(main())