Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

10.8 KB
Raw Blame
"""Where someone speaks in a sound file, so generated music can sit lower under dialogue.

Silero VAD (snakers4/silero-vad, MIT; the onnx-community export) runs on the CPU in torch: the 16 kHz
weights are read straight from the ONNX file, so no ONNX runtime is needed. The file is downloaded with the
music model. One pass over a file gives speech spans in seconds of that file; they are cached next to the
editor's other derived data, and the preview and the export read the same spans.
"""
from __future__ import annotations

import hashlib
import json
import os
from pathlib import Path
import threading

import numpy as np

MODEL_FILE = 'music/silero-vad/model.onnx'
VERSION = 1             # change when the spans for the same file would change
RATE = 16000
WINDOW, CONTEXT = 512, 64
ENTER, LEAVE = .5, .35  # Silero's own hysteresis
MIN_SPEECH, MIN_SILENCE, PAD = .25, .1, .03
# Music under dialogue stays down across short pauses between sentences instead of pumping.
JOIN_GAP = .6
LIMIT_SECONDS = 4 * 3600


class SpeechError(Exception):
    def __init__(self, reason):
        super().__init__(reason)
        self.reason = reason


# --- a small protobuf reader: only what an ONNX file's tensors need -----------------------------------------
def _varint(data, index):
    shift = value = 0
    while True:
        byte = data[index]
        index += 1
        value |= (byte & 0x7f) << shift
        if byte < 0x80:
            return value, index
        shift += 7


def _fields(data):
    index = 0
    while index < len(data):
        key, index = _varint(data, index)
        number, wire = key >> 3, key & 7
        if wire == 0:
            value, index = _varint(data, index)
        elif wire == 1:
            value, index = data[index:index + 8], index + 8
        elif wire == 2:
            size, index = _varint(data, index)
            value, index = data[index:index + size], index + size
        elif wire == 5:
            value, index = data[index:index + 4], index + 4
        else:
            raise SpeechError('model')
        yield number, wire, value


def _tensor(data):
    dims, kind, name, raw = [], None, '', None
    for number, wire, value in _fields(data):
        if number == 1:
            if wire == 2:
                index = 0
                while index < len(value):
                    dim, index = _varint(value, index)
                    dims.append(dim)
            else:
                dims.append(value)
        elif number == 2:
            kind = value
        elif number == 8:
            name = bytes(value).decode()
        elif number == 9:
            raw = value
    if kind != 1 or raw is None or len(raw) != 4 * int(np.prod(dims or [1])):
        return name, None
    return name, np.frombuffer(bytes(raw), '<f4').reshape(dims).copy()


def _tensors(graph, found):
    for number, _, value in _fields(graph):
        if number == 5:
            name, array = _tensor(value)
            if array is not None:
                found[name] = array
        elif number == 1:
            for field, _, attribute in _fields(value):
                if field == 5:
                    for part, _, inner in _fields(attribute):
                        if part == 6:
                            _tensors(inner, found)
    return found


def onnx_weights(path):
    """The 16 kHz branch of the onnx-community Silero export, as numpy arrays named for _Model."""
    data = memoryview(Path(path).read_bytes())
    found = {}
    for number, _, value in _fields(data):
        if number == 7:
            _tensors(value, found)
    prefix = 'If_0_then_branch__Inline_0__'
    lstm = [array for name, array in found.items()
            if name.startswith(prefix) and name.endswith('_subg_96_sub_graph2') and array.ndim >= 2]
    try:
        weights = {'basis': found[prefix + 'stft.forward_basis_buffer']}
        for layer in range(4):
            weights[f'conv{layer}.weight'] = found[f'{prefix}encoder.{layer}.reparam_conv.weight']
            weights[f'conv{layer}.bias'] = found[f'{prefix}encoder.{layer}.reparam_conv.bias']
        weights['out.weight'] = found[prefix + 'decoder.decoder.2.weight']
        weights['out.bias'] = found[prefix + 'decoder.decoder.2.bias']
        w, r = [array[0] for array in lstm if array.shape == (1, 512, 128)]
        (b,) = [array[0] for array in lstm if array.shape == (1, 1024)]
    except (KeyError, ValueError):
        raise SpeechError('model') from None
    # ONNX gates are input, output, forget, cell; torch's are input, forget, cell, output.
    order = lambda x: np.concatenate([x[0:128], x[256:384], x[384:512], x[128:256]])
    weights.update({'lstm.weight_ih_l0': order(w), 'lstm.weight_hh_l0': order(r),
                    'lstm.bias_ih_l0': order(b[:512]), 'lstm.bias_hh_l0': order(b[512:])})
    if weights['basis'].shape != (258, 1, 256) or weights['conv0.weight'].shape != (128, 129, 3):
        raise SpeechError('model')
    return weights


def _model(weights):
    import torch
    from torch import nn

    class Model(nn.Module):
        def __init__(self):
            super().__init__()
            self.register_buffer('basis', torch.zeros(258, 1, 256))
            self.conv0 = nn.Conv1d(129, 128, 3, padding=1)
            self.conv1 = nn.Conv1d(128, 64, 3, stride=2, padding=1)
            self.conv2 = nn.Conv1d(64, 64, 3, stride=2, padding=1)
            self.conv3 = nn.Conv1d(64, 128, 3, padding=1)
            self.lstm = nn.LSTM(128, 128, batch_first=True)
            self.out = nn.Conv1d(128, 1, 1)

        def forward(self, windows):
            """[n, 576] windows (64 context + 512 new samples) in order → [n] speech probabilities."""
            x = nn.functional.pad(windows, (0, 64), mode='reflect').unsqueeze(1)
            x = nn.functional.conv1d(x, self.basis, stride=128)
            x = torch.sqrt(x[:, :129] ** 2 + x[:, 129:] ** 2)
            for conv in (self.conv0, self.conv1, self.conv2, self.conv3):
                x = torch.relu(conv(x))
            x, _ = self.lstm(x.squeeze(-1).unsqueeze(0))
            x = torch.sigmoid(self.out(torch.relu(x[0]).unsqueeze(-1)))
            return x[:, 0, 0]

    model = Model()
    model.load_state_dict({name: torch.from_numpy(array) for name, array in weights.items()})
    return model.eval()


_LOADED = {}
_LOCK = threading.Lock()


def probabilities(samples, model_path):
    """float32 mono samples at 16 kHz → one speech probability per 512 samples (32 ms)."""
    import torch
    with _LOCK:
        key = str(model_path)
        if key not in _LOADED:
            _LOADED.clear()
            _LOADED[key] = _model(onnx_weights(model_path))
        model = _LOADED[key]
    samples = np.asarray(samples, np.float32)
    count = -(-len(samples) // WINDOW)
    if count == 0:
        return np.zeros(0, np.float32)
    padded = np.zeros(CONTEXT + count * WINDOW, np.float32)
    padded[CONTEXT:CONTEXT + len(samples)] = samples
    windows = np.lib.stride_tricks.sliding_window_view(padded, CONTEXT + WINDOW)[::WINDOW][:count]
    threads = torch.get_num_threads()
    try:
        torch.set_num_threads(min(threads, 4))
        with torch.inference_mode():
            return model(torch.from_numpy(np.ascontiguousarray(windows))).numpy()
    finally:
        torch.set_num_threads(threads)


def spans(probs, duration):
    """Speech probabilities → merged [start, end] seconds, with Silero's hysteresis and padding."""
    step = WINDOW / RATE
    found, start, quiet = [], None, None
    for index, value in enumerate(probs):
        at = index * step
        if start is None:
            if value >= ENTER:
                start, quiet = at, None
        elif value < LEAVE:
            quiet = at if quiet is None else quiet
            if at + step - quiet >= MIN_SILENCE:
                found.append([start, quiet])
                start, quiet = None, None
        else:
            quiet = None
    if start is not None:
        found.append([start, len(probs) * step])
    joined = []
    for begin, end in found:
        if end - begin < MIN_SPEECH:
            continue
        begin, end = max(0., begin - PAD), min(duration, end + PAD)
        if joined and begin - joined[-1][1] <= JOIN_GAP:
            joined[-1][1] = max(joined[-1][1], end)
        else:
            joined.append([begin, end])
    return [[round(begin, 3), round(end, 3)] for begin, end in joined]


def decode(path, cancelled=None):
    """The first audio stream of a media file as float32 mono at 16 kHz."""
    import av
    parts, total = [], 0
    try:
        with av.open(str(path)) as container:
            if not container.streams.audio:
                return np.zeros(0, np.float32)
            stream = container.streams.audio[0]
            resampler = av.AudioResampler(format='flt', layout='mono', rate=RATE)
            for frame in container.decode(stream):
                for out in resampler.resample(frame):
                    part = out.to_ndarray().reshape(-1)
                    parts.append(part)
                    total += len(part)
                if total > LIMIT_SECONDS * RATE:
                    raise SpeechError('too_long')
                if cancelled is not None and cancelled():
                    raise SpeechError('cancelled')
            for out in resampler.resample(None):
                parts.append(out.to_ndarray().reshape(-1))
    except av.error.FFmpegError:
        raise SpeechError('unreadable') from None
    return np.concatenate(parts).astype(np.float32) if parts else np.zeros(0, np.float32)


def _identity(path, model_path):
    stat, model = Path(path).stat(), Path(model_path).stat()
    raw = json.dumps([str(Path(path).resolve()), stat.st_size, stat.st_mtime_ns, model.st_size, VERSION])
    return hashlib.sha256(raw.encode()).hexdigest()[:32]


def cached(path, model_path, cache):
    """The cached spans for this exact file and model, or None."""
    try:
        value = json.loads((Path(cache) / (_identity(path, model_path) + '.json')).read_text(encoding='utf-8'))
        return value['spans'] if isinstance(value, dict) and isinstance(value.get('spans'), list) else None
    except (OSError, ValueError, KeyError):
        return None


def speech_spans(path, model_path, cache, cancelled=None):
    """Speech spans in seconds of `path`; computed once per file and model, then read from `cache`."""
    found = cached(path, model_path, cache)
    if found is not None:
        return found
    if not Path(model_path).is_file():
        raise SpeechError('model_missing')
    samples = decode(path, cancelled)
    found = spans(probabilities(samples, model_path), len(samples) / RATE)
    folder = Path(cache)
    folder.mkdir(parents=True, exist_ok=True)
    target = folder / (_identity(path, model_path) + '.json')
    temporary = target.with_name(f'{target.name}.{os.getpid()}.{threading.get_ident()}.tmp')
    temporary.write_text(json.dumps({'spans': found}), encoding='utf-8')
    os.replace(temporary, target)
    return found