Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

11.6 KB
Raw Blame
"""Keep the voice of a long video's parts as loud, and as full, as in the first part.

Each continued part takes its sound context from the part before it, and over
several parts the voice can drift: louder, and fuller in the low range. When the
join is told which parts have spoken lines, it measures the voice in each of them
and, only where a part has drifted more than THRESHOLD_DB from the first one,
brings it back by at most MOST_DB with one gain and one low-range shelf. A part's
whole sound is changed together, so music and background keep their place
against the voice; parts without spoken lines are left exactly as they are. Each
part is filtered on its own, so nothing spills across a seam, and a change eases
in over RAMP_SECONDS on the filtered side of the seam.
"""
import math
import re

RATE = 32000
THRESHOLD_DB = 2.
MOST_DB = 6.
RAMP_SECONDS = .5
# The voice's body (fundamental and first harmonics) against its clear middle range.
LOW, MIDDLE = (60, 300), (300, 3400)
SHELF_HZ = 300.
# A part is never made louder than this peak.
HEADROOM = .95
FRAME, HOP = 4096, 2048
_GATE_DB, _FLOOR_DB, _LEAST = 15., -55., 20

# "A sign on the door says": the writing itself says the words, in the same clause.
_WRITING_SAYS = re.compile(
    r'\b(?:signs?|banners?|captions?|posters?|screens?|labels?|boards?|neon|text|placards?|graffiti|headlines?|'
    r'subtitles?|lettering)\b[^,;:.!?\n]*\b(?:says|said|reads|shows|displays)\s*[:,]?\s*$', re.I)


def voiced(text):
    """Whether a part's description has a spoken line; when unsure, no (the part is then left as it is)."""
    if not isinstance(text, str):
        return False
    from .prompt_vlm.rules import QUOTED, SPEAKER, plain, spoken_lines
    if any(plain(match.group(2)) for match in SPEAKER.finditer(text)):
        return True
    lines = {plain(line) for line in spoken_lines(text)}
    for match in QUOTED.finditer(text):
        words = plain(next(group for group in match.groups() if group is not None))
        if words in lines and not _WRITING_SAYS.search(text[:match.start()]):
            return True
    return False




def _shelf(frequencies):
    return 1. / (1. + (frequencies / SHELF_HZ) ** 2)


def _native(path, cancelled=None):
    import av
    import numpy as np
    chunks = []
    with av.open(str(path)) as container:
        if not container.streams.audio:
            return None
        stream = container.streams.audio[0]
        resampler = av.AudioResampler(format='fltp', layout='stereo', rate=RATE)
        for packet in container.demux(stream):
            if cancelled is not None and cancelled():
                return None
            for decoded in packet.decode():
                chunks.extend(frame.to_ndarray() for frame in resampler.resample(decoded))
        chunks.extend(frame.to_ndarray() for frame in resampler.resample(None))
    return np.concatenate(chunks, axis=1) if chunks else None


def measure(path, cancelled=None):
    """The part's average spectrum over its speaking moments and its peak, or None."""
    import numpy as np
    sound = _native(path, cancelled)
    if sound is None or sound.shape[1] < FRAME * 2:
        return None
    window = np.hanning(FRAME)
    count = 1 + (sound.shape[1] - FRAME) // HOP
    frequencies = np.fft.rfftfreq(FRAME, 1 / RATE)
    middle, low = [((frequencies >= a) & (frequencies < b)) for a, b in (MIDDLE, LOW)]
    power = np.empty((count, len(frequencies)))
    for start in range(0, count, 256):
        rows = np.arange(start, min(count, start + 256))
        # Power per channel, then averaged: opposite channels must not cancel out.
        frames = sound[:, rows[:, None] * HOP + np.arange(FRAME)] * window
        power[rows] = (np.abs(np.fft.rfft(frames, axis=2)) ** 2).mean(axis=0)
    power /= (FRAME * window.sum() / 2)
    speech = power[:, middle].sum(axis=1)
    level = 10 * np.log10(speech + 1e-20)
    active = level >= max(np.percentile(level, 95) - _GATE_DB, _FLOOR_DB)
    if active.sum() < _LEAST:
        return None
    spectrum = power[active].mean(axis=0)
    return dict(spectrum=spectrum, frequencies=frequencies, peak=float(np.abs(sound).max()))


def _band(entry, band, shelf_db=0.):
    import numpy as np
    frequencies = entry['frequencies']
    weight = 10 ** (shelf_db * _shelf(frequencies) / 10)
    chosen = (frequencies >= band[0]) & (frequencies < band[1])
    return 10 * math.log10(float((entry['spectrum'] * weight)[chosen].sum()) + 1e-20)


def _pull(difference):
    if not math.isfinite(difference) or abs(difference) <= THRESHOLD_DB:
        return 0.
    return max(-MOST_DB, min(MOST_DB, difference))


def _fit(gain, shelf, room):
    """Lower a boost (gain, then a raised shelf) to at most `room` dB."""
    excess = gain + max(shelf, 0.) - max(room, 0.)
    if excess > 0:
        cut = min(excess, max(gain, 0.))
        gain, shelf = gain - cut, shelf - min(excess - cut, max(shelf, 0.))
    return gain, shelf


def _room(entry):
    """How much the part may be raised before its peak passes HEADROOM, in dB."""
    return 20 * math.log10(HEADROOM / entry['peak']) if entry['peak'] > 0 else MOST_DB


def plan(sources, speaking, cancelled=None):
    """Per part (gain dB, low shelf dB, room dB), None for a part left as it is; or None when no part needs a change.

    The room is how far the part may be raised, also while it eases in from the part before."""
    if speaking is None or len(speaking) != len(sources) or not any(speaking):
        return None
    reference, settings = None, []
    for source, spoken in zip(sources, speaking):
        entry = measure(source, cancelled) if spoken else None
        if entry is None:
            settings.append(None)
            continue
        tilt = _band(entry, LOW) - _band(entry, MIDDLE)
        if reference is None:
            reference = entry, tilt
            settings.append((0., 0., _room(entry)))
            continue
        shelf = _pull(reference[1] - tilt)
        voice = (LOW[0], MIDDLE[1])
        gain = _pull(_band(reference[0], voice) - _band(entry, voice, shelf))
        settings.append(_fit(gain, shelf, _room(entry)) + (_room(entry),))
    return settings if any(setting and any(setting[:2]) for setting in settings) else None


class Shaper:
    """Streaming short-time filter, one part at a time: samples in, the same samples out a little later.

    A part that needs no change (no setting, and no change easing in from the part
    before) is given back exactly as it came. A filtered part sees only its own
    samples; a change eases in over its first RAMP_SECONDS and, when the next part
    is left as it is, back out over its last RAMP_SECONDS.
    """
    def __init__(self, settings, rate, channels):
        import numpy as np
        self.settings, self.rate, self.channels = list(settings), rate, channels
        self.ramp = max(1, round(RAMP_SECONDS * rate))
        self.window = np.sqrt(np.hanning(FRAME + 1)[:FRAME])
        self.shape = _shelf(np.fft.rfftfreq(FRAME, 1 / rate))
        self.index, self.pushed, self.emitted, self.ready = -1, 0, 0, []
        self.filtered = False

    def _setting(self, index):
        return self.settings[index] if 0 <= index < len(self.settings) else None

    def _filters(self, index):
        own, before = self._setting(index), self._setting(index - 1)
        return own is not None and (any(own[:2]) or before is not None and any(before[:2]))

    def mark(self):
        """The next pushed sample starts the next part."""
        self._close(ended=True)
        self.index += 1
        self.filtered = self._filters(self.index)
        if self.filtered:
            import numpy as np
            own, before = self._setting(self.index), self._setting(self.index - 1)
            self.own = tuple(own[:2])
            # Easing in from the part before must not raise this part past its own room either.
            self.start = tuple(before[:2]) if before is not None else (0., 0.)
            if len(own) > 2:
                self.start = _fit(*self.start, own[2])
            self.end = None if self._filters(self.index + 1) else (0., 0.)
            # Half a frame of silence ahead gives the first samples two overlapping frames, as all others have.
            self.buffer = np.zeros((self.channels, FRAME // 2), dtype=np.float64)
            self.offset = -(FRAME // 2)  # the part's position of buffer[:, 0]
            self.added = np.zeros((self.channels, FRAME - HOP), dtype=np.float64)
            self.skip = FRAME // 2
            self.length = self.out = 0  # samples of this part pushed, and given out

    @property
    def pending(self):
        return self.pushed - self.emitted

    def _at(self, position, length):
        """(gain, shelf) at a position in the part; `length` is the part's length once known."""
        weight = min(1., max(0., position / self.ramp))
        gain = self.start[0] + (self.own[0] - self.start[0]) * weight
        shelf = self.start[1] + (self.own[1] - self.start[1]) * weight
        if length is not None and self.end is not None:
            weight = min(1., max(0., (length - position) / self.ramp))
            gain, shelf = self.end[0] + (gain - self.end[0]) * weight, self.end[1] + (shelf - self.end[1]) * weight
        return gain, shelf

    def _frames(self, length=None):
        import numpy as np
        out = []
        # Until the part's end is known, its last RAMP_SECONDS may still ease out: keep them back.
        while self.buffer.shape[1] >= FRAME and (length is not None or self.end is None
                                                 or self.offset + FRAME // 2 < self.length - self.ramp):
            gain, shelf = self._at(self.offset + FRAME // 2, length)
            response = 10 ** ((gain + shelf * self.shape) / 20)
            spectrum = np.fft.rfft(self.buffer[:, :FRAME] * self.window, axis=1) * response
            frame = np.fft.irfft(spectrum, FRAME, axis=1) * self.window
            frame[:, :FRAME - HOP] += self.added
            out.append(frame[:, :HOP])
            self.added = frame[:, HOP:]
            self.buffer = self.buffer[:, HOP:]
            self.offset += HOP
        if not out:
            return
        data = np.concatenate(out, axis=1)
        skip, self.skip = min(self.skip, data.shape[1]), self.skip - min(self.skip, data.shape[1])
        # Less the leading silence, and only the part's own samples: what rang past its end is dropped.
        data = data[:, skip:][:, :self.length - self.out]
        self.out += data.shape[1]
        self.ready.append(np.clip(data, -1., 1.).astype('float32'))

    def _close(self, ended):
        """Finish the current part: `ended` when another part follows it."""
        import numpy as np
        if not self.filtered:
            return
        self.filtered = False
        if not ended:
            self.end = None
        self.buffer = np.concatenate((self.buffer, np.zeros((self.channels, FRAME), dtype=np.float64)), axis=1)
        self._frames(self.length)

    def push(self, data):
        """Add [channels, n] samples; return the finished ones (or None)."""
        import numpy as np
        self.pushed += data.shape[1]
        if not self.filtered:
            self.ready.append(np.asarray(data))
        else:
            self.buffer = np.concatenate((self.buffer, np.asarray(data, dtype=np.float64)), axis=1)
            self.length += data.shape[1]
            self._frames()
        return self._take()

    def flush(self):
        self._close(ended=False)
        return self._take()

    def _take(self):
        import numpy as np
        if not self.ready:
            return None
        data = np.concatenate(self.ready, axis=1) if len(self.ready) > 1 else self.ready[0]
        self.ready = []
        if not data.shape[1]:
            return None
        self.emitted += data.shape[1]
        return data