Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Checks for Edit speech requests (DotTTS markup or FireRedAudio word diffs, verified against both
transcripts) and the source clip an edit keeps in history, so malformed or injected values never
reach the runtime."""
from __future__ import annotations
import re
import time
from collections import Counter
from pathlib import Path
from typing import Any, Optional, Sequence
from loggers import get_logger
logger = get_logger(__name__)
# Refused, never cut: a cut recording no longer matches its transcript.
EDIT_SOURCE_MAX_SECONDS = 30.0
TOO_LONG = "Edit works on recordings up to 30 s. Record or upload a shorter take."
NO_CHANGE = "Change at least one word."
EMPTY_TARGET = "Keep at least one word."
MISMATCH = "The changes do not match the transcript. Check ① and ② again."
UNKNOWN_INSTRUCTION = "That change is not one FireRedAudio understands."
NEEDS_DELIVERY_MODEL = "Delivery changes need FireRedAudio."
NO_DELIVERY = "Pick a speed or a pitch change."
_MARKUP_TAG_RE = re.compile(
r'<sub targ="([^"<>]*)">([^<>]*)</sub>|<del>([^<>]*)</del>|<ins>([^<>]*)</ins>'
)
INSTRUCTION_RE = re.compile(
r"^(?:Replace '(.+)' with '(.+)'|Delete '(.+)'|Insert '(.+)' before '(.+)')\.$"
)
def _collapse(text: Optional[str]) -> str:
return " ".join(str(text or "").split())
def markup_tags(markup: str) -> int:
return len(_MARKUP_TAG_RE.findall(markup or ""))
def markup_sides(markup: str) -> Optional[tuple[str, str]]:
"""``(source, target)`` text a DotTTS markup describes, whitespace-collapsed; None when it holds
anything but the three tags."""
source: list[str] = []
target: list[str] = []
position = 0
for match in _MARKUP_TAG_RE.finditer(markup or ""):
between = markup[position : match.start()]
if "<" in between or ">" in between:
return None
source.append(between)
target.append(between)
targ, sub, deleted, inserted = match.groups()
if sub is not None:
source.append(sub)
target.append(targ)
elif deleted is not None:
source.append(deleted)
else:
target.append(inserted)
position = match.end()
rest = (markup or "")[position:]
if "<" in rest or ">" in rest:
return None
source.append(rest)
target.append(rest)
return _collapse("".join(source)), _collapse("".join(target))
def check_markup(markup: str, original: str, edited: str) -> Optional[str]:
"""Why ``markup`` does not turn ``original`` into ``edited``; None when it does."""
sides = markup_sides(markup)
if sides is None:
return MISMATCH
if not markup_tags(markup):
return NO_CHANGE
if sides != (_collapse(original), _collapse(edited)):
return MISMATCH
return None
def check_instructions(
instructions: Sequence[str],
original: str,
edited: str,
max_changes: Optional[int],
label: str = "FireRedAudio",
) -> Optional[str]:
"""Why FireRedAudio ``instructions`` are refused; None when each is a known form whose old words
(or anchor) are in ``original`` and new words in ``edited``, and the words they remove and add
turn the one transcript into the other."""
items = list(instructions or ())
if not items:
return NO_CHANGE
if max_changes is not None and len(items) > max_changes:
return (
f"{label} applies at most {max_changes} changes. "
"Make fewer changes, or use DotTTS Edit."
)
before, after = _collapse(original), _collapse(edited)
# Instruction and edited text must match as word bags; the runtime picks which repeated word is meant
words = Counter(before.split())
for instruction in items:
match = INSTRUCTION_RE.match(str(instruction))
if match is None:
return UNKNOWN_INSTRUCTION
old, new, deleted, inserted, anchor = match.groups()
if old is not None:
olds, news, removed = [old], [new], [old]
elif deleted is not None:
olds, news, removed = [deleted], [], [deleted]
else:
olds, news, removed = [anchor], [inserted], []
if not all(_collapse(o) and _collapse(o) in before for o in olds):
return MISMATCH
if not all(_collapse(n) and _collapse(n) in after for n in news):
return MISMATCH
# Running bag, not the transcript: a word an earlier instruction removed is gone.
if not all(words[token] > 0 for o in olds for token in o.split()):
return MISMATCH
for phrase in removed:
words.subtract(phrase.split())
if any(count < 0 for count in words.values()):
return MISMATCH
for phrase in news:
words.update(phrase.split())
if +words != Counter(after.split()):
return MISMATCH
return None
def delivery_instructions(speed: Optional[float], pitch_steps: Optional[int]) -> list[str]:
"""FireRedAudio acoustic_edit instructions, one per call; pitch only rises."""
out: list[str] = []
if speed is not None and float(speed) != 1.0:
out.append(f"adjust the speed to {float(speed):g}x")
if pitch_steps:
out.append(f"shift the pitch by {int(pitch_steps)} steps")
return out
def change_count(edit: dict[str, Any]) -> int:
if edit.get("mode") == "delivery":
return len(delivery_instructions(edit.get("speed"), edit.get("pitch_steps")))
if edit.get("markup"):
return markup_tags(edit["markup"])
if edit.get("instructions"):
return len(edit["instructions"])
return 1
def request_problem(
rules: Optional[dict[str, Any]],
edit: Optional[dict[str, Any]],
text: str,
reference_text: Optional[str],
label: str,
) -> Optional[str]:
"""Why the loaded model's ``audio_edit`` rules refuse this edit; else None."""
edit = edit or {}
rules = rules or {}
style = rules.get("style")
original = reference_text or ""
if edit.get("mode") == "delivery":
if not rules.get("delivery"):
return NEEDS_DELIVERY_MODEL
if not delivery_instructions(edit.get("speed"), edit.get("pitch_steps")):
return NO_DELIVERY
return None
# Before the per-style checks: a no-op such as <sub targ="x">x</sub> passes them.
if style in ("markup", "instructions", "sentence"):
if not _collapse(text):
return EMPTY_TARGET
if _collapse(text) == _collapse(original):
return NO_CHANGE
if style == "markup":
markup = edit.get("markup")
if not markup:
return f"{label} needs the marked-up changes."
return check_markup(markup, original, text)
if style == "instructions":
return check_instructions(
edit.get("instructions") or (), original, text, rules.get("max_changes"), label
)
if style == "sentence":
return None
return "Load a model that can edit speech."
def wav_seconds(path: Path) -> Optional[float]:
"""Length of a WAV from its header; None when unreadable."""
import wave
try:
with wave.open(str(path)) as w:
return w.getnframes() / w.getframerate()
except Exception: # noqa: BLE001 - the caller measures the prepared copy instead
return None
def save_source_clip(
source,
prepared_path: Path,
reference_text: Optional[str],
created: Optional[list[str]] = None,
) -> Optional[dict[str, Any]]:
"""Keeps an upload in history beside its edit, so A/B outlives it; reuses a same-audio source
clip; None for a history clip; saved ids appended to created."""
import wave
from core.inference import audio_gallery, audio_inputs
if getattr(source, "kind", None) != "input":
return None
sidecar = audio_inputs._read_sidecar(audio_inputs._sidecar(source.path.parent, source.id))
sha = (sidecar or {}).get("sha256")
if not sha:
return None
directory = audio_gallery.gallery_dir()
try:
sidecars = list(directory.glob("*.json"))
except OSError:
sidecars = []
for path in sidecars:
meta = audio_gallery._read_meta(path)
if (
meta is not None
and meta.get("role") == "source"
and meta.get("workflow") == "edit"
and meta.get("source_sha256") == sha
and audio_gallery.owned_audio_path(path.stem, directory) is not None
):
return audio_gallery._record(path.stem, meta, directory = directory)
with wave.open(str(prepared_path)) as w:
sample_rate, duration_s = w.getframerate(), round(w.getnframes() / w.getframerate(), 3)
meta: dict[str, Any] = {
"prompt": " ".join(str(reference_text or "").split()) or source.name,
"model": "Recording",
"audio_type": "recording",
"workflow": "edit",
"role": "source",
"source_sha256": sha,
"reference_name": source.name,
"sample_rate": sample_rate,
"duration_s": duration_s,
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
}
record = audio_gallery.save(Path(prepared_path).read_bytes(), meta)
if record is not None and created is not None:
created.append(record["id"])
return record