Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

20.6 KB
Raw Blame
"""Integer clip clocks and bounded CPU speed/reverse readers and previews."""
import asyncio
import math
import os
import threading
import uuid
from collections import OrderedDict
from concurrent.futures import ThreadPoolExecutor
from contextlib import ExitStack
from fractions import Fraction
from decimal import Decimal
from pathlib import Path
from urllib.parse import urlencode

FPS = 24
RATE = 48000
SAMPLES_PER_FRAME = RATE // FPS
VIDEO_CACHE_BYTES = 384 * 1024**2
VIDEO_CACHE_FRAMES = 128
AUDIO_BLOCK_SAMPLES = 65536


def percent(clip):
    return round(clip.get('speed', 1) * 100)


def changed(clip):
    return clip is not None and (clip.get('speed', 1) != 1 or clip.get('reverse', False))


def audio_changed(clip):
    return changed(clip) or bool(clip.get('speed_phase', 0))


def source_span(clip):
    return ((int(clip['frames']) - 1) * percent(clip) + clip.get('speed_phase', 0)) // 100 + 1


def validate_clip(clip, *, error=ValueError):
    """Call after ordinary range checks; return only explicitly saved fields."""
    speed, reverse = clip.get('speed', 1), clip.get('reverse', False)
    if (type(speed) not in (int, float) or not math.isfinite(speed) or not .25 <= speed <= 4
            or Decimal(str(speed)) * 100 != round(speed * 100)):
        raise error('Invalid clip speed')
    if type(reverse) is not bool:
        raise error('Invalid clip reverse')
    phase = clip.get('speed_phase', 0)
    if type(phase) is not int or not 0 <= phase <= 99:
        raise error('Invalid speed phase')
    if 'speed_source_frames' in clip and (type(clip['speed_source_frames']) is not int
                                        or clip['speed_source_frames'] <= 0):
        raise error('Invalid speed source frames')
    if clip.get('speed_audio', 'follow') not in ('follow', 'mute'):
        raise error('Invalid speed audio')
    if int(clip['in']) + source_span(clip) > clip['length']:
        raise error('Invalid clip source span')
    return {key: clip[key] for key in ('speed', 'reverse', 'speed_phase', 'speed_source_frames', 'speed_audio')
            if key in clip}


def source_frame(clip, local):
    k = clip['frames'] - 1 - local if clip.get('reverse', False) else local
    return clip['in'] + (k * percent(clip) + clip.get('speed_phase', 0)) // 100


def source_sample_start(clip):
    return clip['in'] * SAMPLES_PER_FRAME + clip.get('speed_phase', 0) * 20


def source_sample_count(clip):
    # At 48 kHz / 24 fps every integer percentage is an exact sample count.
    return clip['frames'] * 20 * percent(clip)


def tempo_factors(speed):
    """Use the older FFmpeg atempo range, even on builds accepting larger rates."""
    rate, factors = Fraction(speed, 100), []
    while rate < Fraction(1, 2):
        factors.append(Fraction(1, 2))
        rate *= 2
    while rate > 2:
        factors.append(Fraction(2))
        rate /= 2
    if rate != 1:
        factors.append(rate)
    return factors


def independent_join(left, right):
    return changed(left) or changed(right)


def _file_identity(path):
    info = path.stat()
    return info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns


class VideoReader:
    """Forward decode once; reverse bounded windows from a backward keyframe seek.

    A long GOP is decoded in several passes. Only the current window's RGB
    frames are retained, independently of the GOP or clip length.
    """
    def __init__(self, path, clip, width, height, cancelled, *, reader_factory=None):
        from .editor_export import _VideoReader
        self.factory = reader_factory or _VideoReader
        self.path, self.clip = path, clip
        self.width, self.height, self.cancelled = width, height, cancelled
        self.capacity = max(1, min(VIDEO_CACHE_FRAMES, VIDEO_CACHE_BYTES // (width * height * 3)))
        self.cache = {}
        self.peak_cached_frames = self.decode_passes = 0
        self.reader = None
        if not clip.get('reverse', False):
            self.reader = self.factory(path, clip['in'], width, height, cancelled, clip)
            self.decode_passes = 1

    def pixels(self, number):
        from .editor_export import _check
        mapped = source_frame(self.clip, number - self.clip['in'])
        if self.reader is not None:
            return self.reader.pixels(mapped)
        if mapped not in self.cache:
            self.cache.clear()
            low = max(self.clip['in'], mapped - self.capacity + 1)
            reader = self.factory(self.path, low, self.width, self.height, self.cancelled, self.clip)
            try:
                self.decode_passes += 1
                for index in range(low, mapped + 1):
                    _check(self.cancelled)
                    # Copy also detaches a cover crop from its larger backing image.
                    self.cache[index] = reader.pixels(index).copy()
                    self.peak_cached_frames = max(self.peak_cached_frames, len(self.cache))
            finally:
                reader.close()
        return self.cache[mapped]

    def close(self):
        if self.reader is not None:
            self.reader.close()
            self.reader = None
        self.cache.clear()


class AudioReader:
    """Stream atempo, feeding reverse audio in bounded blocks from the tail."""
    def __init__(self, path, clip, cancelled):
        from .editor_export import _audio_start
        self.path, self.clip, self.cancelled = path, clip, cancelled
        # Keep the caller's old samples() clock; changed clips ignore splice.
        self.call_origin = _audio_start(clip)
        self.chunks = iter(self._chunks())
        self.chunk = None
        try:
            self.chunk = next(self.chunks, None)
        except BaseException:
            self.close()
            raise

    def _input(self):
        from .editor_export import _AudioReader, _check
        first = source_sample_start(self.clip)
        remaining = source_sample_count(self.clip)
        reader = None
        try:
            if not self.clip.get('reverse', False):
                reader = _AudioReader(self.path, first, self.cancelled)
                position = first
                while remaining:
                    _check(self.cancelled)
                    count = min(8192, remaining)
                    yield reader.samples(position, count)
                    remaining -= count
                    position += count
            else:
                while remaining:
                    _check(self.cancelled)
                    count = min(AUDIO_BLOCK_SAMPLES, remaining)
                    position = first + remaining - count
                    reader = _AudioReader(self.path, position, self.cancelled)
                    data = reader.samples(position, count)
                    reader.close()
                    reader = None
                    yield data[:, ::-1].copy()
                    remaining -= count
        finally:
            if reader is not None:
                reader.close()

    def _chunks(self):
        import av
        import numpy as np
        from .editor_export import _check
        graph = None
        factors = tempo_factors(percent(self.clip))
        if factors:
            graph = av.filter.Graph()
            graph.threads = 1
            node = graph.add('abuffer', sample_rate=str(RATE), sample_fmt='fltp',
                             channel_layout='stereo', time_base='1/48000')
            for factor in factors:
                following = graph.add('atempo', str(float(factor)))
                node.link_to(following)
                node = following
            # atempo may negotiate packed float; readers consume planar stereo.
            planar = graph.add('aformat', sample_fmts='fltp', sample_rates=str(RATE), channel_layouts='stereo')
            node.link_to(planar)
            sink = graph.add('abuffersink')
            planar.link_to(sink)
            graph.configure()
        position, native, accepted = 0, 0, 0
        tail = np.empty((2, 0), dtype=np.float32)
        total = self.clip['frames'] * SAMPLES_PER_FRAME

        def emit(output):
            nonlocal position, accepted, tail
            kept = min(output.shape[1], total - accepted)
            if kept:
                accepted += kept
                data = np.concatenate((tail, output[:, :kept]), axis=1)
                count = max(0, data.shape[1] - RATE // 200)
                tail = data[:, count:].copy()
                if count:
                    yield position, data[:, :count]
                    position += count

        def pull():
            while True:
                _check(self.cancelled)
                try:
                    yield graph.pull().to_ndarray()
                except (av.error.BlockingIOError, av.error.EOFError):
                    return

        inputs = self._input()
        try:
            for data in inputs:
                _check(self.cancelled)
                if graph is None:
                    outputs = (data,)
                else:
                    frame = av.AudioFrame.from_ndarray(data, format='fltp', layout='stereo')
                    frame.sample_rate, frame.pts, frame.time_base = RATE, native, Fraction(1, RATE)
                    native += data.shape[1]
                    graph.push(frame)
                    outputs = pull()
                for output in outputs:
                    yield from emit(output)
            if graph is not None:
                # A short slow clip may not fill atempo's analysis window.
                # Supply bounded zero context, without reading beyond its range.
                frame = av.AudioFrame.from_ndarray(np.zeros((2, 8192), dtype=np.float32),
                                                  format='fltp', layout='stereo')
                frame.sample_rate, frame.pts, frame.time_base = RATE, native, Fraction(1, RATE)
                graph.push(frame)
                for output in pull():
                    yield from emit(output)
                graph.push(None)
                for output in pull():
                    yield from emit(output)
            # atempo's overlap rounding need not produce the exact timeline count.
            # Fade a short filter tail before padding, avoiding an early hard cut.
            # The renderer still applies the exact timeline's ordinary 5 ms fades.
            if tail.shape[1]:
                if accepted < total:
                    tail *= np.linspace(1, 0, tail.shape[1], dtype=np.float32)
                yield position, tail
                position += tail.shape[1]
            while position < total:
                _check(self.cancelled)
                count = min(8192, total - position)
                yield position, np.zeros((2, count), dtype=np.float32)
                position += count
        finally:
            inputs.close()

    def samples(self, start, count):
        from .editor_export import _AudioReader
        return _AudioReader.samples(self, start - self.call_origin, count)

    def close(self):
        if self.chunks is not None:
            self.chunks.close()
        self.chunk = self.chunks = None


def reverse_copy(path, target, cancelled=None):
    """Reverse the source's 24 fps display grid; encode a browser-playable copy."""
    import av
    from .editor_export import _check, _audio_frame, _checked_target
    from .editor_frames import last_frame_end
    from .media_encoding import clip_seconds
    _check(cancelled)
    _checked_target(path)
    with av.open(str(path)) as source:
        vin, ain = next(iter(source.streams.video), None), next(iter(source.streams.audio), None)
        stream = vin or ain
        if stream is None:
            raise ValueError('Unplayable reverse source')
        seconds = clip_seconds(source, stream)
        if vin is None and (seconds is None or not math.isfinite(seconds) or seconds <= 0):
            raise ValueError('Reverse duration unavailable')
        length = math.ceil(seconds * FPS - 1e-6) if seconds is not None and seconds > 0 else 0
        width = height = 0
        if vin is not None:
            if min(vin.width, vin.height) < 2 or vin.width * vin.height > 64_000_000:
                raise ValueError('Invalid reverse dimensions')
            scale = min(1., 720 / min(vin.width, vin.height), 1920 / max(vin.width, vin.height))
            width, height = (max(2, int(side * scale) // 2 * 2) for side in (vin.width, vin.height))
            # Some containers report only the last PTS, excluding its duration.
            # Count the actual display grid with one decoded frame, including VFR.
            vin.thread_count = 1
            origin = Fraction(vin.start_time or 0) * vin.time_base
            end = None
            rate = vin.average_rate or vin.guessed_rate or FPS
            for index, frame in enumerate(source.decode(vin)):
                _check(cancelled)
                stamp = (Fraction(frame.pts) * frame.time_base - origin if frame.pts is not None
                         else Fraction(index, 1) / rate)
                end = last_frame_end(stamp, frame, rate)
            if end is None or end <= 0:
                raise ValueError('Empty reverse video')
            length = math.ceil(end * FPS)
    clip = dict(frames=length, length=length, speed=1, reverse=True, **{'in': 0})
    partial = target.with_name('.reverse-' + uuid.uuid4().hex + '.mp4')
    published = complete = False
    try:
        _checked_target(target)
        _checked_target(partial)
        target.parent.mkdir(parents=True, exist_ok=True)
        _checked_target(partial)
        with ExitStack() as readers, av.open(str(partial), 'w', format='mp4',
                options={'movflags': 'faststart', 'movie_timescale': str(RATE)}) as output:
            video = audio = pictures = sound = None
            if width:
                pictures = VideoReader(path, clip, width, height, cancelled)
                readers.callback(pictures.close)
                video = output.add_stream('libx264', rate=FPS)
                video.width, video.height, video.pix_fmt = width, height, 'yuv420p'
                video.options = {'crf': '18', 'preset': 'fast'}
                video.codec_context.thread_count = 2
            if ain is not None:
                sound = AudioReader(path, clip, cancelled)
                readers.callback(sound.close)
                audio = output.add_stream('aac', rate=RATE)
                audio.layout = 'stereo'
                audio.codec_context.time_base = Fraction(1, RATE)
                audio.codec_context.thread_count = 1
            for number in range(length):
                _check(cancelled)
                if video is not None:
                    frame = av.VideoFrame.from_ndarray(pictures.pixels(number), format='rgb24')
                    frame.pts, frame.time_base = number, Fraction(1, FPS)
                    for packet in video.encode(frame):
                        output.mux(packet)
                if audio is not None:
                    frame = _audio_frame(sound.samples(number * SAMPLES_PER_FRAME, SAMPLES_PER_FRAME),
                                         format='fltp', layout='stereo')
                    frame.sample_rate = RATE
                    frame.pts, frame.time_base = number * SAMPLES_PER_FRAME, Fraction(1, RATE)
                    for packet in audio.encode(frame):
                        output.mux(packet)
            for stream in (video, audio):
                if stream is not None:
                    for packet in stream.encode(None):
                        _check(cancelled)
                        output.mux(packet)
        _check(cancelled)
        _checked_target(target)
        _checked_target(partial)
        os.replace(partial, target)
        published = True
        _check(cancelled)
        complete = True
        return dict(frames=length, fps=FPS, width=width, height=height,
                    has_video=video is not None, has_audio=audio is not None)
    finally:
        partial.unlink(missing_ok=True)
        if published and not complete:
            target.unlink(missing_ok=True)


class ReverseCache:
    """One worker per server, deduplicated by _cache's source file identity."""
    def __init__(self, root):
        self.root = root
        self.lock = threading.RLock()
        self.executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='editor-reverse')
        self.pending, self.failed = {}, OrderedDict()
        self.closed = False

    def _location(self, key):
        from .comfy_editor import _cache, _import_path, _key, _safe, media_path
        path = media_path(self.root, key)
        source = _key(key)
        if source['kind'] == 'import':
            # Exports decode the original, rather than the lossy import preview.
            original = _import_path(self.root, source['id'], 'source.bin')
            if original.is_file():
                path = original
        if not path.is_file():
            raise FileNotFoundError('Reverse source unavailable')
        stamp = '24fps-v2:' + ':'.join(map(str, _file_identity(path)))
        target = _cache(self.root, 'reverse', key, path, stamp).with_suffix('.mp4')
        return path, _safe(self.root, target.relative_to(Path(self.root).resolve()).as_posix())

    def status(self, key):
        from .comfy_editor import probe
        path, target = self._location(key)
        with self.lock:
            if self.closed:
                return dict(state='cancelled', reason='cancelled')
            if target in self.pending:
                return dict(state='preparing', poll_after_ms=250)
            if target.is_file():
                info = probe(target)
                address = '/freevideo/editor/reverse?' + urlencode(dict(key=key, file='preview', cache=target.stem))
                return dict(state='ready', **info, video=address if info['has_video'] else None,
                            audio=address if info['has_audio'] else None)
            if target in self.failed:
                return dict(state='failed', reason=self.failed[target])
            if len(self.pending) >= 64:
                return dict(state='busy', poll_after_ms=1000)
            stop = threading.Event()
            self.pending[target] = stop
            self.executor.submit(self._make, key, path, target, stop)
            return dict(state='preparing', poll_after_ms=250)

    def _make(self, key, path, target, stop):
        from .editor_export import _check, _reason
        try:
            identity = _file_identity(path)
            if (path, target) != self._location(key):
                raise ValueError('Reverse source changed')
            reverse_copy(path, target, stop.is_set)
            _check(stop.is_set)
            if identity != _file_identity(path) or (path, target) != self._location(key):
                raise ValueError('Reverse source changed')
        except BaseException as error:
            target.unlink(missing_ok=True)
            with self.lock:
                self.failed[target] = 'cancelled' if stop.is_set() else _reason(error)
                while len(self.failed) > 128:
                    self.failed.popitem(last=False)
        finally:
            with self.lock:
                self.pending.pop(target, None)

    def media(self, key, token=None):
        _, target = self._location(key)
        with self.lock:
            if target in self.pending or not target.is_file() or token is not None and token != target.stem:
                raise FileNotFoundError('Reverse preview unavailable')
            return target

    def close(self):
        with self.lock:
            self.closed = True
            for stop in self.pending.values():
                stop.set()
        # Let queued workers observe cancellation and clean their owned files.
        self.executor.shutdown(wait=True)


def register_reverse(endpoint, root, server):
    from aiohttp import web
    from .comfy_editor import probe
    cache = ReverseCache(root())
    server._freevideo_editor_reverse = cache

    @endpoint('get', '/freevideo/editor/reverse')
    async def reverse(request):
        key = request.query.get('key')
        if 'file' in request.query:
            if request.query['file'] != 'preview':
                raise ValueError('Invalid reverse media')
            path = await asyncio.to_thread(cache.media, key, request.query.get('cache'))
            info = await asyncio.to_thread(probe, path)
            content_type = 'video/mp4' if info['has_video'] else 'audio/mp4'
            return web.FileResponse(path, headers={'Content-Type': content_type, 'Cache-Control': 'no-store'})
        value = await asyncio.to_thread(cache.status, key)
        status = 202 if value['state'] == 'preparing' else 503 if value['state'] == 'busy' else 200
        return web.json_response(value, status=status, headers={'Cache-Control': 'no-store'})

    async def shutdown(app):
        await asyncio.to_thread(cache.close)

    if getattr(server, 'app', None) is not None:
        server.app.on_shutdown.append(shutdown)