Skip to content

Open-Source Frontier Voice AI

Python
13.5 KB
Raw Blame
"""The streaming ASR sequence format, apart from the server that drives it.

The model was trained on a single interleaved sequence

    Prompt + [<sp_start> AudioWindow <sp_end> Text <|text_chunk_end|>] * N

so a client's job is to cut audio into the windows training used and rebuild
that sequence chunk by chunk. This module does that, plus the inverse: turning
the per-chunk text back into readable segments. Three processes need one or
both -- the vLLM engine, the streaming server, and the browser demo -- and a
copy of the chunk arithmetic in each would be free to drift.

Nothing here imports vLLM or torch at module level, so the demo can reach it by
bare import (``vllm_plugin/`` on sys.path, ``import asr_streaming``) without
executing ``vllm_plugin/__init__.py`` and its model registration.
"""
import json
import os
import re
import tempfile
from dataclasses import dataclass
from typing import Dict, List, Optional

import numpy as np

# The model signals end-of-chunk with this token rather than EOS. It is the
# first free slot after Qwen2.5's vocabulary and was trained, not synthesised.
# The tokenizer reads it from the checkpoint rather than defining it, so a
# checkpoint whose files disagree with this literal resolves to something else
# -- or to nothing, in which case every chunk runs to max_tokens.
TEXT_CHUNK_END_ID = 151665

# Plain text, no chat template, no BOS. Not the processor's SYSTEM_PROMPT +
# <|im_start|> template, which serves the non-streaming single-shot path only.
_PROMPT_HEAD = (
    "You are a helpful assistant that transcribes audio input into text output. "
    "Please transcribe the following audios streamingly with these keys: "
    "speaker, content"
)


@dataclass(frozen=True)
class ChunkGeometry:
    """How a checkpoint cuts audio: read from it, never guessed.

    A window carries ``chunk_frames`` of new audio plus ``lookahead_frames``
    that the next window repeats. The released checkpoints differ here -- 15
    frames (2.0s) and 22 frames (2.9333s), 4 frames of lookahead either way --
    and serving one at the other's geometry does not raise, it just transcribes
    worse. So it is read from ``preprocessor_config.json``, which every
    streaming checkpoint records it in.
    """

    sample_rate: int
    frame_samples: int
    chunk_frames: int
    lookahead_frames: int

    @property
    def window_frames(self) -> int:
        return self.chunk_frames + self.lookahead_frames

    @property
    def chunk_samples(self) -> int:
        return self.chunk_frames * self.frame_samples

    @property
    def window_samples(self) -> int:
        return self.window_frames * self.frame_samples

    @property
    def chunk_seconds(self) -> float:
        """The advance, which segment times and the demo's capture use.

        Chunk k is fed audio ``[k*advance, k*advance + window)`` and the text it
        returns covers ``[k*advance, (k+1)*advance)`` -- the trailing lookahead
        is audio whose transcript arrives with the next chunk.
        """
        return self.chunk_samples / self.sample_rate

    @classmethod
    def from_pretrained(cls, model_path: str) -> "ChunkGeometry":
        """Load the geometry from a local checkpoint directory or a HF repo id."""
        name = "preprocessor_config.json"
        if os.path.isdir(model_path):
            config_path = os.path.join(model_path, name)
        else:
            from huggingface_hub import hf_hub_download

            config_path = hf_hub_download(model_path, name)

        with open(config_path) as f:
            cfg = json.load(f)

        missing = [k for k in ("chunk_frames", "lookahead_frames") if k not in cfg]
        if missing:
            raise ValueError(
                f"{config_path} has no {', '.join(missing)}, so {model_path} is "
                "not a streaming checkpoint. Serve it with start_server.py instead."
            )
        return cls(
            sample_rate=cfg["target_sample_rate"],
            frame_samples=cfg["speech_tok_compress_ratio"],
            chunk_frames=cfg["chunk_frames"],
            lookahead_frames=cfg["lookahead_frames"],
        )

    def describe(self) -> str:
        return (f"chunk {self.chunk_frames} frames ({self.chunk_seconds:.4f}s), "
                f"lookahead {self.lookahead_frames} frames, "
                f"window {self.window_samples} samples @ {self.sample_rate} Hz")


def build_prompt(context_info: Optional[str] = None) -> str:
    """Training-format prompt, with the optional hotword/context suffix."""
    if context_info:
        return f"{_PROMPT_HEAD} and extra info: {context_info}\n"
    return f"{_PROMPT_HEAD}\n"


def split_windows(audio, geometry: ChunkGeometry, pad_last: bool = True) -> List[np.ndarray]:
    """Cut audio into overlapping windows, matching ``split_then_encode``.

    ``pad_last`` zero-pads the final window to a full window. That is not
    cosmetic: training split already-encoded features, so its last chunk had
    ``floor(remainder/frame)`` frames, while encoding a short segment rounds
    *up*. Without the pad the final window would carry one extra frame and no
    lookahead -- off-distribution exactly where the sequence ends.
    """
    audio = np.asarray(audio, dtype=np.float32)
    total = audio.shape[0]
    window_samples = geometry.window_samples

    windows = []
    start = 0
    while start < total:
        seg = audio[start:min(start + window_samples, total)]
        if seg.shape[0]:
            if pad_last and seg.shape[0] < window_samples:
                seg = np.pad(seg, (0, window_samples - seg.shape[0]))
            windows.append(seg)
        # Advance by the text boundary, not by the window: a window that already
        # reaches the end of the audio still only transcribes its first chunk,
        # so the trailing lookahead needs a window of its own.
        start = min(start + geometry.chunk_samples, total)
    return windows


def load_audio_bytes(audio_bytes: bytes, target_sr: int) -> np.ndarray:
    """Decode in-memory audio to the mono float32 the windows are cut from.

    The suffix is a placeholder: ffmpeg sniffs the container from the bytes, so
    an mp3 or a webm opus blob written to a ``.wav`` name decodes fine.

    Decodes at the native rate and resamples with librosa rather than letting
    ffmpeg do both, which is what the streaming checkpoints were evaluated with.
    The two resamplers differ by ~1% RMS, and near-tied greedy argmaxes turn
    that into different words.
    """
    from vibevoice.processor.audio_utils import load_audio_use_ffmpeg

    with tempfile.NamedTemporaryFile(suffix=".wav", delete=True) as tmp:
        tmp.write(audio_bytes)
        tmp.flush()
        audio, sr = load_audio_use_ffmpeg(tmp.name, resample=False)

    if sr != target_sr:
        import librosa

        audio = librosa.resample(audio, orig_sr=sr, target_sr=target_sr)
    return audio


# --- Chunks back into readable segments -------------------------------------
# The model emits one block per chunk, each re-stating the speaker, each free to
# end mid-sentence. Shown verbatim that is thirty fragments a minute. The job
# below merges them into paragraphs that end where sentences do, identically for
# every caller -- a rule that differs between the demo and the server is a bug
# nobody notices until the two are compared side by side.

_CHUNK_SPEAKER_RE = re.compile(r"Speaker\s+(\d+)\s*:")

# Chunks merge until the segment is at least this long, then close at the next
# sentence-ending punctuation. The cap stops a stretch with no punctuation from
# growing one card until it swallows the whole clip; kept at twice the minimum,
# so punctuation and not the cap is what usually ends a card.
SEGMENT_MIN_S = 10.0
SEGMENT_MAX_S = 20.0

# Two dialects, because the rule differs. A CJK stop mark is unambiguous. An
# ASCII period is not -- requiring whitespace or end-of-text after it keeps
# "3.5" and "Mr. Smith" from splitting a segment in half.
_SENTENCE_END_RE = re.compile(
    r"[。!?…]+[”’」』)】》]*"
    r"|[.!?]+[\"'’”)\]]*(?=\s|$)"
)

# Backreference, so only a repeat of the *same* tag collapses: [Noise][Noise]
# folds, [Noise][Laughter] does not. Tags become adjacent only when blocks are
# merged into one card, which is why this runs here and not per block.
_REPEAT_TAG_RE = re.compile(r"(\[[^\[\]]*\])(?:\s*\1)+")

# A card is one paragraph, so the line breaks the model emits inside a block
# fold to single spaces -- except between two CJK characters, where a space is
# not a word separator and just leaves a visible gap.
_WS_RUN_RE = re.compile(r"\s+")
_CJK = r" -〿぀-ヿ一-鿿＀-・"
_CJK_SPACE_RE = re.compile(rf"(?<=[{_CJK}]) (?=[{_CJK}])")


def tidy_body(text: str) -> str:
    """Fold a merged card's text back into one paragraph."""
    return _CJK_SPACE_RE.sub("", _WS_RUN_RE.sub(" ", text)).strip()


def chunk_segments(chunk_texts: List[str],
                   geometry: ChunkGeometry,
                   duration: Optional[float] = None) -> List[Dict]:
    """Merge the per-chunk stream into readable, sentence-aligned segments.

    Times stay arithmetic rather than inferred: chunk k *is* the k-th window, so
    every boundary is a multiple of ``geometry.chunk_seconds``. Two things split
    a chunk internally -- a speaker change, and a sentence end past the
    SEGMENT_MIN_S mark. In both cases the words go to the right card while the
    timestamp stays on the chunk edge.

    The speaker carries forward because the model only re-states it when it
    changes, so a chunk with no label belongs to the previous speaker.

    ``duration`` only ever clamps the tail. While a clip is still streaming its
    real duration is unknown, and the chunk count is what defines the timeline.
    """
    advance = geometry.chunk_seconds
    segments: List[Dict] = []
    cur_start = 0.0
    cur_text = ""
    speaker = None

    def flush(end: float) -> None:
        nonlocal cur_start, cur_text
        body = tidy_body(_REPEAT_TAG_RE.sub(r"\1", cur_text))
        # Silence really does come back empty; those chunks fold into whichever
        # segment is open instead of becoming a card of their own.
        if body:
            if end > cur_start:
                segments.append({
                    "Start": round(cur_start, 2),
                    "End": round(end, 2),
                    "Speaker": speaker if speaker is not None else "?",
                    "Content": body,
                })
            elif segments:
                # No timeline left to give this text -- the remainder after a
                # sentence split in the final chunk. Dropping it is how words go
                # missing off the end of a transcript, so it joins the card it
                # was split from instead.
                prev = segments[-1]
                prev["Content"] = tidy_body(
                    _REPEAT_TAG_RE.sub(r"\1", prev["Content"] + " " + body))
        cur_start = end
        cur_text = ""

    for k, text in enumerate(chunk_texts):
        start = k * advance
        end = start + advance
        if duration and duration > start:
            end = min(end, duration)

        # split() on a single-group pattern yields [pre, id, text, id, text...],
        # which is what lets a mid-chunk speaker change break the segment.
        pieces = _CHUNK_SPEAKER_RE.split(text)
        added = pieces[0]
        # Two people can easily both talk inside one chunk. Closing the card at
        # ``start`` would put the boundary where the previous card already
        # ended, and a zero-width card carries no text -- so the cut is placed
        # proportionally through the chunk, by how much of its text is consumed.
        chunk_chars = max(1, sum(len(p) for p in pieces))
        consumed = len(pieces[0])
        for i in range(1, len(pieces), 2):
            if speaker is not None and pieces[i] != speaker:
                cur_text += added
                flush(max(start + (end - start) * (consumed / chunk_chars), cur_start))
                added = ""
            speaker = pieces[i]
            added += pieces[i + 1]
            consumed += len(pieces[i]) + len(pieces[i + 1])
        cur_text += added

        span = end - cur_start
        if span >= SEGMENT_MIN_S:
            # Only a stop mark inside *this* chunk is usable: one further back
            # would strand the text between it and here in the wrong card.
            last = None
            for last in _SENTENCE_END_RE.finditer(added):
                pass
            if last is not None:
                carry = added[last.end():]
                cur_text = cur_text[:len(cur_text) - len(carry)] if carry else cur_text
                flush(end)
                cur_text = carry
                continue
        if span >= SEGMENT_MAX_S:
            flush(end)

    total = len(chunk_texts) * advance
    flush(min(total, duration) if duration and duration > cur_start else total)
    return segments


def segments_to_text(segments: List[Dict]) -> str:
    """Render segments as the plain transcript, one line per segment."""
    return "\n".join(
        f"Speaker {seg.get('Speaker', '?')}: {seg.get('Content', '')}"
        for seg in segments
    )


def _srt_time(seconds: float) -> str:
    ms = int(round(seconds * 1000))
    h, ms = divmod(ms, 3_600_000)
    m, ms = divmod(ms, 60_000)
    s, ms = divmod(ms, 1000)
    return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}"


def segments_to_srt(segments: List[Dict]) -> str:
    """SRT, driven by the same segments as everything else so they agree."""
    return "\n".join(
        f"{i}\n{_srt_time(seg['Start'])} --> {_srt_time(seg['End'])}\n"
        f"Speaker {seg.get('Speaker', '?')}: {seg.get('Content', '')}\n"
        for i, seg in enumerate(segments, 1)
    )