Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
#!/usr/bin/env python3
"""Qwen3-TTS Runner for PortOS.
Provides CLI entry points for:
- Environment & hardware probe (--probe)
- Voice design inference (--mode design)
- Consented instant cloning (--mode clone)
- Verified fine-tuned checkpoint synthesis (--mode fine-tuned)
- Official single-speaker fine-tuning on CUDA (--mode fine-tune)
"""
from __future__ import annotations
import argparse
import contextlib
import hashlib
import importlib.util
import math
import os
import re
import json
import platform
import shutil
import sys
import tempfile
from pathlib import Path
HUB_ENDPOINT = "https://huggingface.co"
SUPPORTED_MODELS = (
"Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign",
"Qwen/Qwen3-TTS-12Hz-1.7B-Base",
"Qwen/Qwen3-TTS-12Hz-0.6B-Base",
)
# Both the language model and speech codec are required. A directory, old
# metadata.json, or a lone language-model weight is not an installed snapshot.
REQUIRED_FILES = (
"config.json", "generation_config.json", "merges.txt", "model.safetensors",
"preprocessor_config.json", "tokenizer_config.json", "vocab.json",
"speech_tokenizer/config.json", "speech_tokenizer/configuration.json",
"speech_tokenizer/model.safetensors", "speech_tokenizer/preprocessor_config.json",
)
# Official single-speaker SFT recipe for the 12 Hz Base models, ported from
# QwenLM/Qwen3-TTS finetuning/ (sft_12hz.py + dataset.py, Apache-2.0) at commit
# 022e286b98fbec7e1e916cb940cdf532cd9f488e. The recipe trains in bf16 on CUDA.
TRAINING_ADAPTER = "qwen-tts-sft-12hz"
TRAINABLE_MODELS = tuple(model for model in SUPPORTED_MODELS if model.endswith("-Base"))
CHECKPOINT_MANIFEST = "portos-checkpoint.json"
CHECKPOINT_WEIGHTS = "model.safetensors"
# The recipe writes the target speaker embedding into this codec-embedding row
# and registers the speaker name against it in the checkpoint config.
SPEAKER_TOKEN_ID = 3000
SPEAKER_NAME_RE = re.compile(r"[a-z][a-z0-9_]{0,39}")
AUDITION_TEXT = "This is a short audition of the fine-tuned voice."
TRAINING_BATCH_SIZE = 2
TRAINING_GRADIENT_ACCUMULATION = 4
TRAINING_LEARNING_RATE = 2e-5
MAX_TRAINING_SAMPLES = 500
MAX_TRAINING_TEXT = 2000
def publish_json(path: Path, data: dict) -> None:
descriptor, temporary = tempfile.mkstemp(prefix=".verified-", dir=path.parent)
try:
with os.fdopen(descriptor, "w") as output:
json.dump(data, output)
output.flush()
os.fsync(output.fileno())
os.replace(temporary, path)
finally:
Path(temporary).unlink(missing_ok=True)
def file_fingerprint(path: Path) -> list[int]:
stat = path.stat()
return [stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns, stat.st_ino, stat.st_dev]
def verify_file(path: Path, metadata: dict) -> list[int]:
"""Verify bytes and reject files changed while hashing."""
before = file_fingerprint(path)
size, algorithm, digest = metadata["size"], metadata["algorithm"], metadata["digest"]
if not isinstance(size, int) or size <= 0 or path.stat().st_size != size:
raise ValueError(f"Incomplete model file: {path.name}")
if algorithm not in ("sha1", "sha256") or not isinstance(digest, str):
raise ValueError("Invalid model verification metadata")
checksum = hashlib.new(algorithm)
if algorithm == "sha1":
checksum.update(f"blob {size}\0".encode())
with path.open("rb") as source:
for chunk in iter(lambda: source.read(1024 * 1024), b""):
checksum.update(chunk)
if checksum.hexdigest() != digest:
raise ValueError(f"Model file checksum mismatch: {path.name}")
if before != file_fingerprint(path):
raise ValueError(f"Model file changed during verification: {path.name}")
return before
def installed_snapshot(model_dir: Path, model_id: str) -> Path | None:
"""Read only a completely verified revision; never perform network access."""
try:
manifest = json.loads((model_dir / "verified.json").read_text())
revision = manifest["revision"]
if manifest["model_id"] != model_id or not re.fullmatch(r"[0-9a-f]{40}", revision):
return None
snapshot = model_dir / revision
cache_path = snapshot / ".verification.json"
try:
cache = json.loads(cache_path.read_text())
if not isinstance(cache, dict):
cache = {}
except (OSError, ValueError):
cache = {}
refreshed = {}
changed = False
for filename in REQUIRED_FILES:
path = snapshot / filename
metadata = dict(manifest["files"][filename])
cached = cache.get(filename)
if isinstance(cached, dict) and all(cached.get(key) == metadata[key] for key in ("size", "algorithm", "digest")):
metadata = cached
# The verified bytes stay valid while their filesystem identity and
# change timestamps match. Rehash changed files, including equal-size
# replacements, without rereading multi-GB weights on every status.
# Windows Python reports creation time as ctime, not change time;
# it cannot prove unchanged bytes when mtime has been restored.
if sys.platform == "win32" or file_fingerprint(path) != metadata.get("fingerprint"):
metadata["fingerprint"] = verify_file(path, metadata)
changed = True
refreshed[filename] = metadata
if changed:
# Per-revision cache cannot overwrite the readiness pointer when a
# concurrent download publishes a newer revision.
publish_json(cache_path, refreshed)
return snapshot
except (OSError, ValueError, KeyError, TypeError):
return None
def download_model(model_id: str, models_dir: Path) -> dict:
"""Fetch an immutable Hub revision and verify every required file's digest.
Hub blob IDs are Git SHA-1 for ordinary files and SHA-256 for LFS weights.
Only publish the readiness marker after verification, leaving a prior
installed revision usable if an update fails or the process is interrupted.
"""
if model_id not in SUPPORTED_MODELS:
raise ValueError("Unsupported Qwen3-TTS model")
from huggingface_hub import HfApi, hf_hub_download
info = HfApi(endpoint=HUB_ENDPOINT).model_info(model_id, files_metadata=True)
revision = info.sha
if not isinstance(revision, str) or not re.fullmatch(r"[0-9a-f]{40}", revision):
raise ValueError("Model repository did not return an immutable revision")
files = {entry.rfilename: entry for entry in info.siblings}
if not all(filename in files for filename in REQUIRED_FILES):
raise ValueError("Model repository is missing required Qwen3-TTS files")
model_dir = models_dir / model_id.replace("/", "--")
snapshot = model_dir / revision
verified_files = {}
for filename in REQUIRED_FILES:
entry = files[filename]
if not isinstance(entry.size, int) or entry.size <= 0:
raise ValueError(f"Missing file size: {filename}")
lfs = entry.lfs
digest = lfs.sha256 if lfs else entry.blob_id
algorithm = "sha256" if lfs else "sha1"
if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}" if lfs else r"[0-9a-f]{40}", digest):
raise ValueError(f"Missing file digest: {filename}")
# The explicit download operation is the only path allowed to fetch.
hf_hub_download(repo_id=model_id, filename=filename, revision=revision,
local_dir=snapshot, endpoint=HUB_ENDPOINT)
metadata = {"size": entry.size, "algorithm": algorithm, "digest": digest}
metadata["fingerprint"] = verify_file(snapshot / filename, metadata)
verified_files[filename] = metadata
publish_json(model_dir / "verified.json", {
"model_id": model_id, "revision": revision, "files": verified_files,
})
return {"ok": True, "modelId": model_id, "revision": revision, "path": str(snapshot)}
def is_apple_silicon() -> bool:
return platform.system() == "Darwin" and platform.machine() == "arm64"
def training_runtime_available() -> bool:
"""True only where the official recipe can run: bf16 CUDA plus its imports."""
if is_apple_silicon():
return False
try:
import librosa # noqa: F401
import soundfile # noqa: F401
import torch
from qwen_tts import Qwen3TTSModel # noqa: F401
from qwen_tts.core.models.modeling_qwen3_tts import mel_spectrogram # noqa: F401
from safetensors.torch import save_file # noqa: F401
except ImportError:
return False
return bool(torch.cuda.is_available() and torch.cuda.is_bf16_supported())
def probe_runtime(models_dir: Path | None = None) -> dict:
"""Probe hardware, PyTorch, Transformers, and cached model weights."""
result = {
"ok": False,
"error": "Qwen3-TTS inference requires qwen-tts, torch and soundfile on CPU/CUDA",
"torch_installed": False,
"transformers_installed": False,
"device": "cpu",
"cuda_available": False,
"mps_available": False,
"vram_gb": None,
"models": {},
# Inference readiness never implies training support; PortOS refuses
# fine-tuning unless the probe names the adapter that will train.
"training_adapter": None,
}
try:
import torch
result["torch_installed"] = True
result["torch_version"] = torch.__version__
if torch.cuda.is_available():
result["cuda_available"] = True
result["device"] = "cuda"
try:
result["vram_gb"] = round(torch.cuda.get_device_properties(0).total_memory / (1024**3), 2)
except Exception:
pass
elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
result["mps_available"] = True
result["device"] = "mps"
except ImportError:
pass
try:
import transformers
result["transformers_installed"] = True
result["transformers_version"] = transformers.__version__
except ImportError:
pass
# Probe the actual backend without loading a model or acquiring weights.
if is_apple_silicon():
result["error"] = "Qwen3-TTS inference requires mlx-audio and soundfile on Apple Silicon"
try:
import mlx.core as mx
from mlx_audio.tts.models.qwen3_tts import Model
from mlx_audio.tts.utils import load_model
import soundfile
if mx.metal.is_available():
result["ok"] = True
result["error"] = None
result["device"] = "mlx"
except ImportError:
pass
elif result["torch_installed"] and result["transformers_installed"]:
try:
from qwen_tts import Qwen3TTSModel
import soundfile
result["ok"] = True
result["error"] = None
except ImportError:
pass
if result["ok"] and training_runtime_available():
result["training_adapter"] = TRAINING_ADAPTER
for model_id in SUPPORTED_MODELS:
model_path = models_dir / model_id.replace("/", "--") if models_dir else None
snapshot = installed_snapshot(model_path, model_id) if model_path else None
result["models"][model_id] = {
"downloaded": snapshot is not None,
"path": str(snapshot) if snapshot else None,
}
return result
def unavailable(operation: str) -> int:
"""Refuse unsupported operations without producing audio or checkpoints."""
print(json.dumps({
"ok": False,
"code": "QWEN3_RUNTIME_UNAVAILABLE",
"error": f"Qwen3-TTS {operation} is unavailable",
}), file=sys.stderr)
return 1
def use_offline_hub() -> None:
# qwen-tts loads its processor/codec independently. Offline flags cover those
# nested loads too; status, inference and training cannot acquire weights.
os.environ["HF_HUB_OFFLINE"] = "1"
os.environ["TRANSFORMERS_OFFLINE"] = "1"
def load_torch_model(path: Path, seed: int):
"""Load a verified local snapshot or checkpoint with the official package."""
import torch
from qwen_tts import Qwen3TTSModel
device = "cuda:0" if torch.cuda.is_available() else "cpu"
dtype = (torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16) if device != "cpu" else torch.float32
torch.manual_seed(seed)
model = Qwen3TTSModel.from_pretrained(
str(path), device_map=device, dtype=dtype,
attn_implementation="eager", local_files_only=True,
)
return torch, model
def write_model_audio(wavs, sample_rate, output_path) -> None:
"""Publish one finite, non-silent PCM WAV; never write rejected audio."""
import numpy as np
import soundfile as sf
if len(wavs) != 1 or not isinstance(sample_rate, int) or sample_rate <= 0:
raise ValueError("Invalid model audio result")
audio = np.asarray(wavs[0])
if audio.ndim != 1 or not audio.size or not np.isfinite(audio).all() or not np.any(audio):
raise ValueError("Model returned empty, silent or invalid audio")
# Publish only the completed PCM WAV; the JS transport buffers it.
sf.write(output_path, audio, sample_rate, format="WAV", subtype="PCM_16")
def synthesis_failed(output_wav) -> int:
# Provider exceptions may contain private reference paths/transcripts.
# Return a bounded error instead of publishing their raw exception text.
Path(output_wav).unlink(missing_ok=True)
print(json.dumps({"ok": False, "code": "QWEN3_SYNTHESIS_FAILED",
"error": "Qwen3-TTS model inference failed; no audio was published"}), file=sys.stderr)
return 1
def run_synthesis(args: argparse.Namespace) -> int:
if args.mode == "fine-tuned":
return run_checkpoint_synthesis(args)
# Resolve the verified immutable snapshot ourselves. Never let a model ID,
# arbitrary checkpoint, or missing local file trigger an implicit download.
if args.checkpoint_path or args.model_path:
return unavailable("custom checkpoints/model paths")
if args.rate != 1.0:
return unavailable("speech-rate control")
use_mlx = is_apple_silicon()
if args.model_id not in SUPPORTED_MODELS or not args.models_dir:
return unavailable("inference without a supported verified local model")
model_dir = Path(args.models_dir) / args.model_id.replace("/", "--")
snapshot = installed_snapshot(model_dir, args.model_id)
if snapshot is None:
return unavailable("inference without downloaded and verified weights")
if not args.text or not args.text.strip():
return unavailable("inference without text")
design = args.model_id.endswith("-VoiceDesign")
if (args.mode == "clone" and design) or (args.mode == "design" and not design):
return unavailable("the requested mode for this model variant")
if not design and (not args.reference_audio or not Path(args.reference_audio).is_file()):
return unavailable("cloning without a local reference recording")
if not design and args.instructions:
return unavailable("instruction-controlled reference cloning")
# MLX Base uses in-context cloning. Without both a transcript and an
# encoder it falls back to unconditioned speech, which is not cloning.
if use_mlx and not design and not (args.reference_transcript or "").strip():
return unavailable("MLX reference cloning without a transcript")
use_offline_hub()
try:
with contextlib.redirect_stdout(sys.stderr):
import numpy # noqa: F401 - both backends need the audio writer deps
import soundfile # noqa: F401
if use_mlx:
import mlx.core as mx
from mlx_audio.tts.utils import load_model
mx.random.seed(args.seed)
model = load_model(snapshot, strict=True)
if model.tokenizer is None or model.speech_tokenizer is None:
raise ValueError("MLX model tokenizer is unavailable")
if design:
results = list(model.generate_voice_design(
text=args.text, language="auto",
instruct=args.instructions or "A clear natural speaking voice.",
stream=False, verbose=False,
))
else:
if not model.speech_tokenizer.has_encoder:
raise ValueError("MLX reference encoder is unavailable")
results = list(model.generate(
text=args.text, lang_code="auto",
ref_audio=str(Path(args.reference_audio).resolve()),
ref_text=args.reference_transcript,
split_pattern=None, stream=False, verbose=False,
))
if len(results) != 1:
raise ValueError("Invalid MLX model audio result")
# Materialize lazy MLX output before publishing any audio.
mx.eval(results[0].audio)
wavs, sample_rate = [results[0].audio], results[0].sample_rate
else:
torch, model = load_torch_model(snapshot, args.seed)
with torch.inference_mode():
if design:
wavs, sample_rate = model.generate_voice_design(
text=args.text, language="Auto",
instruct=args.instructions or "A clear natural speaking voice.",
)
else:
wavs, sample_rate = model.generate_voice_clone(
text=args.text, language="Auto",
ref_audio=str(Path(args.reference_audio).resolve()),
ref_text=args.reference_transcript or None,
x_vector_only_mode=not bool(args.reference_transcript),
)
write_model_audio(wavs, sample_rate, args.output_wav)
print(json.dumps({
"ok": True, "modelRevision": f"{args.model_id}@{snapshot.name}",
"effectiveControls": {"rate": 1.0, "seed": args.seed,
"instructions": (args.instructions or "A clear natural speaking voice.") if design else None,
"mode": "design" if design else "clone"},
}))
return 0
except ImportError:
return unavailable("inference dependencies (install mlx-audio on Apple Silicon or qwen-tts on CPU/CUDA)")
except Exception:
return synthesis_failed(args.output_wav)
def run_checkpoint_synthesis(args: argparse.Namespace) -> int:
"""Speak with a checkpoint this adapter produced, verified before loading."""
if is_apple_silicon():
return unavailable("fine-tuned checkpoints on Apple Silicon")
if args.rate != 1.0:
return unavailable("speech-rate control")
if args.instructions:
return unavailable("instruction control for fine-tuned checkpoints")
if not args.text or not args.text.strip():
return unavailable("inference without text")
checkpoint = Path(args.checkpoint_path) if args.checkpoint_path else None
manifest = verified_checkpoint(checkpoint) if checkpoint else None
if manifest is None:
return unavailable("an unverified fine-tuned checkpoint")
use_offline_hub()
try:
with contextlib.redirect_stdout(sys.stderr):
torch, model = load_torch_model(checkpoint, args.seed)
with torch.inference_mode():
wavs, sample_rate = model.generate_custom_voice(
text=args.text, speaker=manifest["speaker"], language="Auto",
)
write_model_audio(wavs, sample_rate, args.output_wav)
print(json.dumps({
"ok": True, "modelRevision": checkpoint_revision(manifest),
"effectiveControls": {"rate": 1.0, "seed": args.seed, "instructions": None, "mode": "fine-tuned"},
}))
return 0
except ImportError:
return unavailable("fine-tuned inference dependencies (install qwen-tts on CPU/CUDA)")
except Exception:
return synthesis_failed(args.output_wav)
# --- Fine-tuning -----------------------------------------------------------
def emit_event(stream, event: dict) -> None:
print(json.dumps(event), file=stream, flush=True)
def training_failed(code: str, error: str) -> int:
# Fixed messages only: dataset paths and transcripts are private.
print(json.dumps({"ok": False, "code": code, "error": error}), file=sys.stderr)
return 1
def load_training_dataset(path: Path) -> dict:
"""Validate the dataset manifest PortOS writes for one training job."""
dataset = json.loads(path.read_text())
speaker = dataset["speaker"]
if not isinstance(speaker, str) or not SPEAKER_NAME_RE.fullmatch(speaker):
raise ValueError("Invalid speaker name")
reference = Path(dataset["reference_audio"])
samples = dataset["samples"]
if not reference.is_file() or not isinstance(samples, list) or not 0 < len(samples) <= MAX_TRAINING_SAMPLES:
raise ValueError("Invalid training dataset")
cleaned = []
for sample in samples:
audio, text = Path(sample["audio"]), sample["text"]
if not audio.is_file() or not isinstance(text, str) or not text.strip() or len(text) > MAX_TRAINING_TEXT:
raise ValueError("Invalid training sample")
cleaned.append({"audio": str(audio), "text": text.strip()})
return {"speaker": speaker, "reference_audio": str(reference), "samples": cleaned}
def checkpoint_revision(manifest: dict) -> str:
"""Bind a fine-tuned voice to its base revision and exact trained weights."""
digest = manifest["files"][CHECKPOINT_WEIGHTS]["digest"]
return f"{manifest['base_model_id']}@{manifest['base_revision']}+sha256.{digest}"
def write_checkpoint_files(snapshot: Path, target: Path, speaker: str) -> None:
"""Copy the verified base files a checkpoint loads, as the recipe does, and
register the trained speaker. The recipe's trainer writes the weights."""
for filename in REQUIRED_FILES:
if filename == CHECKPOINT_WEIGHTS:
continue
destination = target / filename
destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(snapshot / filename, destination)
config = json.loads((snapshot / "config.json").read_text(encoding="utf-8"))
config["tts_model_type"] = "custom_voice"
talker = config.get("talker_config", {})
talker["spk_id"] = {speaker: SPEAKER_TOKEN_ID}
talker["spk_is_dialect"] = {speaker: False}
config["talker_config"] = talker
(target / "config.json").write_text(json.dumps(config, indent=2, ensure_ascii=False), encoding="utf-8")
def seal_checkpoint(target: Path, *, base_model_id: str, base_revision: str, speaker: str, step: int, loss) -> dict:
"""Record digests of every file the checkpoint loads. Synthesis refuses a
checkpoint whose bytes no longer match."""
files = {}
for filename in REQUIRED_FILES:
path = target / filename
fingerprint = file_fingerprint(path)
checksum = hashlib.sha256()
with path.open("rb") as source:
for chunk in iter(lambda: source.read(1024 * 1024), b""):
checksum.update(chunk)
if file_fingerprint(path) != fingerprint:
raise ValueError(f"Checkpoint file changed while sealing: {filename}")
files[filename] = {"size": fingerprint[0], "algorithm": "sha256",
"digest": checksum.hexdigest(), "fingerprint": fingerprint}
manifest = {
"adapter": TRAINING_ADAPTER, "base_model_id": base_model_id, "base_revision": base_revision,
"speaker": speaker, "step": step, "loss": loss, "files": files,
}
publish_json(target / CHECKPOINT_MANIFEST, manifest)
return manifest
def verified_checkpoint(checkpoint: Path) -> dict | None:
"""Read a sealed checkpoint produced by this adapter; never fetch or trust
unsealed, foreign or modified files."""
try:
manifest = json.loads((checkpoint / CHECKPOINT_MANIFEST).read_text())
if (manifest["adapter"] != TRAINING_ADAPTER or manifest["base_model_id"] not in TRAINABLE_MODELS
or not re.fullmatch(r"[0-9a-f]{40}", manifest["base_revision"])
or not SPEAKER_NAME_RE.fullmatch(manifest["speaker"])):
return None
for filename in REQUIRED_FILES:
metadata = manifest["files"][filename]
if metadata["algorithm"] != "sha256":
return None
path = checkpoint / filename
if sys.platform == "win32" or file_fingerprint(path) != metadata.get("fingerprint"):
verify_file(path, metadata)
return manifest
except (OSError, ValueError, KeyError, TypeError):
return None
def train_checkpoints(snapshot: Path, dataset: dict, args: argparse.Namespace, staging: Path, emit) -> list[dict]:
"""Run the official single-speaker SFT recipe and stage weight snapshots.
This mirrors sft_12hz.py/dataset.py: codec-0 language-model loss plus 0.3x
sub-talker loss, AdamW (lr 2e-5, weight decay 0.01), batch 2 with 4-step
gradient accumulation and norm-1.0 clipping, bf16 weights, and the reference
speaker embedding written into codec row 3000. Staged directories are not
promotable until `run_fine_tuning` reloads and auditions them.
"""
import librosa
import torch
from qwen_tts import Qwen3TTSModel
from qwen_tts.core.models.modeling_qwen3_tts import mel_spectrogram
from safetensors.torch import save_file
torch.manual_seed(args.seed)
attention = "flash_attention_2" if importlib.util.find_spec("flash_attn") else "eager"
qwen = Qwen3TTSModel.from_pretrained(
str(snapshot), device_map="cuda:0", dtype=torch.bfloat16,
attn_implementation=attention, local_files_only=True,
)
model, processor, config = qwen.model, qwen.processor, qwen.model.config
talker_config = config.talker_config
def load_24k(path):
audio, _ = librosa.load(path, sr=24000, mono=True)
return audio.astype("float32")
with torch.no_grad():
reference = torch.from_numpy(load_24k(dataset["reference_audio"])).unsqueeze(0)
reference_mel = mel_spectrogram(
reference, n_fft=1024, num_mels=128, sampling_rate=24000,
hop_size=256, win_size=1024, fmin=0, fmax=12000,
).transpose(1, 2)
speaker_embedding = model.speaker_encoder(reference_mel.to(model.device).to(model.dtype)).detach()
items = []
for sample in dataset["samples"]:
codes = model.speech_tokenizer.encode(load_24k(sample["audio"]), sr=24000).audio_codes[0]
prompt = f"<|im_start|>assistant\n{sample['text']}<|im_end|>\n<|im_start|>assistant\n"
text_ids = processor(text=prompt, return_tensors="pt", padding=True)["input_ids"]
text_ids = text_ids.unsqueeze(0) if text_ids.dim() == 1 else text_ids
# Codec encode runs in inference mode; rebuild plain tensors so the
# embedding lookups can be saved for backward.
items.append({"text_ids": text_ids[:, :-5], "codes": torch.tensor(codes.cpu().tolist(), dtype=torch.long)})
def collate(batch):
length = max(item["text_ids"].shape[1] + item["codes"].shape[0] for item in batch) + 8
size = len(batch)
input_ids = torch.zeros((size, length, 2), dtype=torch.long)
codec_ids = torch.zeros((size, length, 16), dtype=torch.long)
text_mask = torch.zeros((size, length), dtype=torch.bool)
codec_embedding_mask = torch.zeros((size, length), dtype=torch.bool)
codec_mask = torch.zeros((size, length), dtype=torch.bool)
attention_mask = torch.zeros((size, length), dtype=torch.long)
labels = torch.full((size, length), -100, dtype=torch.long)
for i, item in enumerate(batch):
text_ids, codes = item["text_ids"], item["codes"]
text_len, codec_len = text_ids.shape[1], codes.shape[0]
end = 8 + text_len + codec_len
input_ids[i, :3, 0] = text_ids[0, :3]
input_ids[i, 3:7, 0] = config.tts_pad_token_id
input_ids[i, 7, 0] = config.tts_bos_token_id
input_ids[i, 8:8 + text_len - 3, 0] = text_ids[0, 3:]
input_ids[i, 8 + text_len - 3, 0] = config.tts_eos_token_id
input_ids[i, 8 + text_len - 2:end, 0] = config.tts_pad_token_id
text_mask[i, :end] = True
input_ids[i, 3:8, 1] = torch.tensor([
talker_config.codec_nothink_id, talker_config.codec_think_bos_id,
talker_config.codec_think_eos_id, 0, talker_config.codec_pad_id,
])
input_ids[i, 8:8 + text_len - 2, 1] = talker_config.codec_pad_id
input_ids[i, 8 + text_len - 2, 1] = talker_config.codec_bos_id
audio_start, audio_end = 8 + text_len - 1, 8 + text_len - 1 + codec_len
input_ids[i, audio_start:audio_end, 1] = codes[:, 0]
input_ids[i, audio_end, 1] = talker_config.codec_eos_token_id
labels[i, audio_start:audio_end] = codes[:, 0]
labels[i, audio_end] = talker_config.codec_eos_token_id
codec_ids[i, audio_start:audio_end, :] = codes
codec_embedding_mask[i, 3:end] = True
codec_embedding_mask[i, 6] = False # speaker embedding slot
codec_mask[i, audio_start:audio_end] = True
attention_mask[i, :end] = 1
device = model.device
return {
"input_ids": input_ids.to(device), "codec_ids": codec_ids.to(device),
"text_mask": text_mask.unsqueeze(-1).to(device),
"codec_embedding_mask": codec_embedding_mask.unsqueeze(-1).to(device),
"codec_mask": codec_mask.to(device), "attention_mask": attention_mask.to(device),
"labels": labels.to(device),
}
def batch_loss(batch):
input_ids, codec_ids, codec_mask = batch["input_ids"], batch["codec_ids"], batch["codec_mask"]
text_embedding = model.talker.model.text_embedding(input_ids[:, :, 0]) * batch["text_mask"]
codec_embedding = model.talker.model.codec_embedding(input_ids[:, :, 1]) * batch["codec_embedding_mask"]
codec_embedding[:, 6, :] = speaker_embedding
embeddings = text_embedding + codec_embedding
for i in range(1, 16):
layer = model.talker.code_predictor.get_input_embeddings()[i - 1]
embeddings = embeddings + layer(codec_ids[:, :, i]) * codec_mask.unsqueeze(-1)
outputs = model.talker(
inputs_embeds=embeddings[:, :-1, :], attention_mask=batch["attention_mask"][:, :-1],
labels=batch["labels"][:, 1:], output_hidden_states=True,
)
# The talker returns hidden_states as (per-layer states, codec ids);
# [0][-1] is the final layer, batch dimension intact.
hidden = outputs.hidden_states[0][-1][codec_mask[:, :-1]]
_, sub_talker_loss = model.talker.forward_sub_talker_finetune(codec_ids[codec_mask], hidden)
return outputs.loss + 0.3 * sub_talker_loss
def stage(step, loss):
target = staging / f"step-{step}"
shutil.rmtree(target, ignore_errors=True)
write_checkpoint_files(snapshot, target, dataset["speaker"])
state = {key: value.detach().to("cpu") for key, value in model.state_dict().items()
if not key.startswith("speaker_encoder")}
row = "talker.model.codec_embedding.weight"
state[row] = state[row].clone()
state[row][SPEAKER_TOKEN_ID] = speaker_embedding[0].detach().to("cpu", state[row].dtype)
save_file(state, str(target / CHECKPOINT_WEIGHTS))
return {"path": target, "step": step, "loss": loss}
optimizer = torch.optim.AdamW(model.parameters(), lr=TRAINING_LEARNING_RATE, weight_decay=0.01)
batches_per_epoch = math.ceil(len(items) / TRAINING_BATCH_SIZE)
steps_per_epoch = math.ceil(batches_per_epoch / TRAINING_GRADIENT_ACCUMULATION)
total_steps = steps_per_epoch * args.epochs
generator = torch.Generator().manual_seed(args.seed)
staged, step = [], 0
model.train()
for _epoch in range(args.epochs):
order = torch.randperm(len(items), generator=generator).tolist()
batches = [order[i:i + TRAINING_BATCH_SIZE] for i in range(0, len(order), TRAINING_BATCH_SIZE)]
for start in range(0, len(batches), TRAINING_GRADIENT_ACCUMULATION):
group = batches[start:start + TRAINING_GRADIENT_ACCUMULATION]
total = 0.0
for indices in group:
loss = batch_loss(collate([items[i] for i in indices]))
(loss / len(group)).backward()
total += loss.item() / len(group)
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
optimizer.zero_grad(set_to_none=True)
step += 1
if not math.isfinite(total):
raise ValueError("Training loss diverged")
loss_value = round(total, 6)
emit({"stage": "training", "step": step, "total_steps": total_steps, "loss": loss_value,
"progress": round(100 * step / total_steps, 2)})
if step % args.checkpoint_interval == 0 or step == total_steps:
staged.append(stage(step, loss_value))
# Release the trainer before auditions load each checkpoint on the device.
del optimizer, model, qwen
torch.cuda.empty_cache()
return staged
def run_fine_tuning(args: argparse.Namespace) -> int:
"""Train, then publish only checkpoints that reload and speak."""
if args.model_id not in TRAINABLE_MODELS or not args.models_dir:
return unavailable("fine-tuning without a supported verified Base model")
if args.epochs < 1 or args.checkpoint_interval < 1:
return training_failed("QWEN3_TRAINING_INVALID_REQUEST", "Invalid fine-tuning schedule")
if not training_runtime_available():
return unavailable("fine-tuning on this hardware/runtime (requires CUDA with bf16, qwen-tts and librosa)")
snapshot = installed_snapshot(Path(args.models_dir) / args.model_id.replace("/", "--"), args.model_id)
if snapshot is None:
return unavailable("fine-tuning without downloaded and verified weights")
try:
dataset = load_training_dataset(Path(args.dataset_manifest))
except (OSError, ValueError, KeyError, TypeError):
return training_failed("QWEN3_TRAINING_INVALID_DATASET", "Fine-tuning dataset is missing or invalid")
output_dir = Path(args.output_dir)
staging = output_dir / ".staging"
events = sys.stdout
emit = lambda event: emit_event(events, event) # noqa: E731
use_offline_hub()
try:
with contextlib.redirect_stdout(sys.stderr):
staged = train_checkpoints(snapshot, dataset, args, staging, emit)
published = 0
for item in staged:
sample = item["path"] / "audition.wav"
torch = model = None
try:
# Seal first, then pass the same gate and loader synthesis
# uses: only sealed bytes that reload and speak are promotable.
manifest = seal_checkpoint(
item["path"], base_model_id=args.model_id, base_revision=snapshot.name,
speaker=dataset["speaker"], step=item["step"], loss=item["loss"],
)
if verified_checkpoint(item["path"]) != manifest:
raise ValueError("Checkpoint seal did not verify")
torch, model = load_torch_model(item["path"], args.seed)
with torch.inference_mode():
wavs, sample_rate = model.generate_custom_voice(
text=AUDITION_TEXT, speaker=manifest["speaker"], language="Auto",
)
write_model_audio(wavs, sample_rate, sample)
except Exception:
print(f"❌ Checkpoint at step {item['step']} failed its reload audition", file=sys.stderr)
shutil.rmtree(item["path"], ignore_errors=True)
continue
finally:
# Release each audition model, passed or failed, before the
# next checkpoint loads onto the same device.
model = None
if torch is not None:
torch.cuda.empty_cache()
target = output_dir / f"checkpoint-step-{item['step']}"
os.replace(item["path"], target)
published += 1
emit({"stage": "checkpoint", "checkpoint": target.name, "step": item["step"],
"checkpoint_path": str(target), "sample_wav": str(target / "audition.wav"),
"loss": item["loss"], "model_revision": checkpoint_revision(manifest)})
if not published:
return training_failed("QWEN3_TRAINING_FAILED", "No fine-tuned checkpoint passed its reload audition")
emit({"stage": "completed", "checkpoints": published})
return 0
except ImportError:
return unavailable("fine-tuning dependencies")
except Exception:
return training_failed("QWEN3_TRAINING_FAILED", "Qwen3-TTS fine-tuning failed; unverified checkpoints were not published")
finally:
# Unpublished snapshots are full model copies; never leave them behind.
shutil.rmtree(staging, ignore_errors=True)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--download", action="store_true", help="Explicitly download and verify a model snapshot")
parser.add_argument("--probe", action="store_true", help="Probe runtime and available models")
parser.add_argument("--models-dir", type=str, help="Directory containing downloaded model weights")
parser.add_argument("--mode", choices=["design", "clone", "synthesize", "fine-tuned", "fine-tune"], default="synthesize")
parser.add_argument("--text", type=str, help="Input text for synthesis")
parser.add_argument("--instructions", type=str, help="Delivery / voice design instructions")
parser.add_argument("--seed", type=int, default=42, help="RNG seed")
parser.add_argument("--rate", type=float, default=1.0, help="Speech rate (0.25 - 4.0)")
parser.add_argument("--reference-audio", type=str, help="Path to reference audio file for cloning")
parser.add_argument("--reference-transcript", type=str, help="Transcript of reference audio")
parser.add_argument("--checkpoint-path", type=str, help="Path to fine-tuned model checkpoint")
parser.add_argument("--model-id", type=str, help="HuggingFace model ID")
parser.add_argument("--model-path", type=str, help="Local directory containing model snapshot")
parser.add_argument("--output-wav", type=str, help="Target path for synthesized WAV")
parser.add_argument("--dataset-manifest", type=str, help="JSON manifest of training audio, transcripts and reference")
parser.add_argument("--output-dir", type=str, help="Output directory for training checkpoints")
parser.add_argument("--epochs", type=int, default=5, help="Number of training epochs")
parser.add_argument("--checkpoint-interval", type=int, default=50, help="Optimizer steps between checkpoints")
args = parser.parse_args()
if args.download:
if not args.model_id or not args.models_dir:
parser.error("--download requires --model-id and --models-dir")
try:
print(json.dumps(download_model(args.model_id, Path(args.models_dir))))
return 0
except Exception as error:
code = "QWEN3_DOWNLOAD_UNAVAILABLE" if isinstance(error, ImportError) else "QWEN3_DOWNLOAD_FAILED"
print(json.dumps({"ok": False, "code": code, "error": "Qwen3 model download failed"}), file=sys.stderr)
return 1
if args.probe:
models_dir = Path(args.models_dir) if args.models_dir else None
with contextlib.redirect_stdout(sys.stderr):
status = probe_runtime(models_dir)
print(json.dumps(status))
return 0
if args.mode == "fine-tune":
if not args.dataset_manifest or not args.output_dir:
sys.stderr.write("Error: --dataset-manifest and --output-dir are required for fine-tune mode\n")
return 1
return run_fine_tuning(args)
if not args.output_wav:
sys.stderr.write("Error: --output-wav is required for synthesis\n")
return 1
return run_synthesis(args)
if __name__ == "__main__":
raise SystemExit(main())