Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

15.9 KB
Raw Blame
"""Supervise speculative encoder placement after a saved interactive request.

This controller imports no tensor runtime. The existing resident worker owns
CUDA, and a new foreground request cancels this lower-priority work first.
"""
import argparse
from contextlib import contextmanager
import json
import os
import signal
from pathlib import Path
import subprocess
import sys
import threading
import time

from . import processes
from .monitoring import save


def gpu_preload_budget(previous, limits, *, free, total, reserved):
    """Reconsider idle placement from live bytes, preserving explicit limits.

    Only our idle allocator's storage earns credit. Another application's
    allocations and the CUDA context are absent from ``reserved``. No compute
    profile, quality setting, or foreground placement changes here.
    """
    from .hardware import GiB
    limits = limits or {}
    capacity = total
    if limits.get('vram_gib') is not None:
        capacity = min(capacity, int(limits['vram_gib'] * GiB))
    available = max(0, min(capacity, free + reserved))
    reserve = limits.get('gpu_reserve_gib')
    if reserve is None:
        reserve = 1. if limits.get('system') == 'Windows' else max(.5, min(1., available / GiB * .025))
    reserve = max(.5, reserve) * GiB
    budget = max(0, available - reserve)
    if limits.get('automatic') is not True:
        budget = min(budget, previous)
    if limits.get('allocator_limit_bytes') is not None:
        budget = min(budget, limits['allocator_limit_bytes'])
    return dict(budget_bytes=int(budget), live_free_bytes=free, owned_reserved_bytes=reserved,
                capacity_bytes=capacity, system_reserve_bytes=int(reserve), previous_budget_bytes=previous,
                reason='Refresh idle GPU allowance from live free bytes plus owned cache; keep explicit caps and system headroom')


class PreloadCancelled(Exception):
    """A completed video stays successful when speculative work is cancelled."""


# How long cancelled placement may take to hand the worker back. Past this the
# worker is stopped, and with it every model it holds: the next request then
# pays a new process and loads everything again, which costs far more than
# these seconds. One native transfer in flight cannot be interrupted, and on a
# slow disk a single large weight takes longer than a second.
YIELD_SECONDS = 6.


@contextmanager
def interruptible_load(patcher, check):
    """Yield between native module transfers, with the original loader intact.

    CPU cleanup must remain callable after cancellation. Speculative loading
    also avoids pinning all offloaded weights merely to warm a few GPU layers.
    No forward, tokenization, patched arithmetic or warm-up inference runs here.
    """
    saved = []
    def replace(obj, name, value):
        saved.append((obj, name, name in vars(obj), vars(obj).get(name)))
        setattr(obj, name, value)
    try:
        for module in patcher.model.modules():
            original = module.to
            def move(*args, _original=original, **kwargs):
                device = kwargs.get('device', args[0] if args else None)
                if str(device).startswith('cuda'):
                    check()
                return _original(*args, **kwargs)
            replace(module, 'to', move)
        original_patch = patcher.patch_weight_to_device
        def patch(*args, **kwargs):
            device = kwargs.get('device_to', args[1] if len(args) > 1 else None)
            if str(device).startswith('cuda'):
                check()
            return original_patch(*args, **kwargs)
        replace(patcher, 'patch_weight_to_device', patch)
        replace(patcher, 'pin_weight_to_device', lambda *args, **kwargs: check())
        # The native loader answers a failed placement by moving every weight
        # it already placed back to host memory. Here that is a full
        # GPU-to-CPU copy, as much new host memory as was placed, before the
        # worker can yield; under RAM pressure it is the wrong direction too.
        # The caller discards a model whose placement did not finish, so its
        # tensors stay where they are. None is unpatch_model's own "do not
        # move" mode.
        native_detach = getattr(patcher, 'detach', None)
        def detach(*args, **kwargs):
            native_unpatch = patcher.unpatch_model
            existed, previous = 'unpatch_model' in vars(patcher), vars(patcher).get('unpatch_model')
            patcher.unpatch_model = lambda device_to=None, unpatch_weights=True: native_unpatch(
                device_to=None, unpatch_weights=unpatch_weights)
            try:
                return native_detach(*args, **kwargs)
            finally:
                if existed:
                    patcher.unpatch_model = previous
                else:
                    del patcher.unpatch_model
        if callable(native_detach):
            replace(patcher, 'detach', detach)
        yield
    finally:
        for obj, name, existed, value in reversed(saved):
            if existed:
                setattr(obj, name, value)
            else:
                delattr(obj, name)


class IdleEncoder:
    def __init__(self):
        self.thread = None
        self.cancelled = threading.Event()
        self.process = None
        self.cancel_path = None
        self.report = None

    def stop(self):
        self.cancelled.set()
        if self.cancel_path is not None:
            try:
                self.cancel_path.touch(exist_ok=True)
            except OSError:
                if self.process is not None and self.process.poll() is None:
                    processes.stop(self.process, grace=1.)
        if self.thread is not None:
            self.thread.join(timeout=YIELD_SECONDS+2.)
            if self.thread.is_alive():
                # A blocked disk read/native copy is not indefinitely allowed
                # to delay the next prompt. The controller stops the real GPU
                # worker on termination; SessionOwner confirms it is idle/dead.
                if self.process is not None:
                    processes.stop(self.process, grace=1.)
                self.thread.join(timeout=2.)
            return not self.thread.is_alive()
        return True

    def schedule(self, endpoint, output, python, environment, *, busy=None, notify=None):
        if not self.stop():
            return False
        self.cancelled = threading.Event()
        cancel = self.cancelled
        report = Path(output).with_suffix('.prewarm.json')
        self.report = report
        self.cancel_path = Path(output).with_suffix('.prewarm.cancel')
        request_path = Path(output).with_suffix('.prewarm-request.json')
        request = dict(endpoint=str(endpoint), previous=str(Path(output).with_suffix('.request.json')),
                       cancel=str(self.cancel_path), metrics=str(report))
        save(request_path, request)
        env = dict(environment)
        env.pop('FREEVIDEO_RUNTIME_LOCK_FD', None)
        env.pop(processes.HANDLE_ENV, None)
        self.process = None

        def work():
            try:
                # Let Comfy publish the video before optional background work.
                if cancel.wait(1.):
                    save(report, dict(state='cancelled', reason='A queued request takes priority'))
                    return
                # Comfy may still be saving/displaying this video's downstream
                # nodes. Wait for the graph to finish instead of permanently
                # skipping preloading because it was busy at one instant.
                while busy and busy():
                    if cancel.wait(.1):
                        save(report, dict(state='cancelled', reason='A queued request takes priority'))
                        return
                with report.with_suffix('.log').open('wb') as log:
                    self.process = processes.popen([str(python), '-B', '-m',
                        'freevideo_engine.idle_encoder', '--request', str(request_path)],
                        env=env, cwd=Path(__file__).resolve().parents[1], stdin=subprocess.DEVNULL,
                        stdout=log, stderr=subprocess.STDOUT, start_new_session=True, supervise=True)
                    previous = None
                    while self.process.poll() is None:
                        if cancel.is_set() or busy and busy():
                            cancel.set()
                            self.cancel_path.touch(exist_ok=True)
                        try:
                            current = json.loads(report.read_text(encoding='utf-8'))
                            if current != previous and notify:
                                try:
                                    notify(current)
                                except Exception:
                                    pass  # A disconnected UI does not orphan the controller.
                            previous = current
                        except (OSError, ValueError):
                            pass
                        try:
                            self.process.wait(timeout=.1)
                        except subprocess.TimeoutExpired:
                            pass
                if notify and report.is_file():
                    notify(json.loads(report.read_text(encoding='utf-8')))
            except Exception as error:
                if self.process is not None and self.process.poll() is None:
                    processes.stop(self.process, grace=1.)
                save(report, dict(state='failed', reason=str(error), error_type=type(error).__name__))

        self.thread = threading.Thread(target=work, name='FreeVideo idle encoder', daemon=True)
        self.thread.start()
        return True


def run(request):
    from .adaptive import classify_failure
    from .locking import runtime_lock
    from .resource_history import ResourceHistory
    from .resident_process import ENV, RemoteProcess
    from .ram import ProcessMemory
    from .system import inference_headroom, inference_memory_sample, windows

    result_path = Path(request['metrics'])
    cancelled = Path(request['cancel'])
    worker = None
    attempt = None
    history = None
    result = dict(state='skipped', scope='Idle GPU encoder weights; no prompt encoding')
    def interrupted(signum, frame):
        cancelled.touch(exist_ok=True)
        raise KeyboardInterrupt('Foreground work takes priority over encoder preloading')
    previous_handler = processes.termination_handler(interrupted)
    try:
        if cancelled.exists():
            raise PreloadCancelled('A new request takes priority')
        previous = json.loads(Path(request['previous']).read_text(encoding='utf-8'))
        attempts = previous.get('resource_attempts', [])
        if not previous.get('success') or not attempts or not previous.get('encoding'):
            result['reason'] = 'No completed interactive encoder request to prepare again'
            return result
        history = ResourceHistory(previous['resource_history'])
        measured = history.attempt(attempts[-1]['id'])
        if measured is None or measured['state'] != 'success':
            raise ValueError('Previous request has no settled success in local history')
        original = json.loads((Path(previous['artifacts'])/'encode.json').read_text(encoding='utf-8'))
        # Deliberately do not forward text, images, seed or other next-request
        # inputs. Preloading uses only the already validated encoder recipe.
        encoding = {k: original[k] for k in ('encoder', 'comfy_root', 'model_paths') if k in original}
        profile = previous['profile']
        budget = profile['gpu_budget_gb']*1e9
        if profile.get('benchmark_allocator_limit_bytes') is not None:
            budget = min(budget, profile['benchmark_allocator_limit_bytes'])
        encoding.update(idle_preload=True, cancel=str(cancelled), metrics=str(result_path),
                        idle_resources=previous.get('idle_resources'),
                        gpu_budget_gb=budget/1e9,
                        ram_budget_bytes=round(profile['inference_ram_budget_gb']*1e9))
        worker_request = result_path.with_name(result_path.stem+'-worker.json')
        save(worker_request, encoding)
        environment = dict(os.environ)
        environment[ENV] = request['endpoint']
        allocator = profile.get('allocator_config', '')
        environment.update(PYTORCH_ALLOC_CONF=allocator, PYTORCH_CUDA_ALLOC_CONF=allocator)
        memory = ProcessMemory()
        with runtime_lock(inherit=False) as descriptor:
            if cancelled.exists():
                raise PreloadCancelled('A new request takes priority')
            attempt = history.begin(measured['identity'],
                dict(encoder=encoding['encoder'], operation='idle-preload'),
                measured['geometry'], purpose='encoding',
                evidence=dict(reason='Saved video returned; speculative weights only', previous=measured['id']))
            worker = RemoteProcess(request['endpoint'],
                [sys.executable, '-B', '-m', 'freevideo_engine.encode_worker', '--request', str(worker_request)],
                environment, descriptor, result_path.with_name(result_path.stem+'-worker.log'),
                required_capabilities=('encoder-weight-preload',))
            history.attach_worker(attempt, worker.pid, phase='idle-encoder-load')
            cancel_started = None
            while worker.poll() is None:
                sample = memory.sample(worker.pid)
                used = inference_memory_sample(sample)
                if used is None or used > encoding['ram_budget_bytes'] or inference_headroom(sample) < (2 if windows() else 1)*2**30:
                    cancelled.touch(exist_ok=True)
                    result.update(state='released', reason='Live RAM pressure interrupted speculative loading')
                if cancelled.exists():
                    cancel_started = cancel_started or time.monotonic()
                    if time.monotonic()-cancel_started > YIELD_SECONDS:
                        raise PreloadCancelled('Preload did not yield promptly; stop the worker before foreground work')
                try:
                    worker.wait(timeout=.1)
                except subprocess.TimeoutExpired:
                    pass
            loaded = json.loads(result_path.read_text(encoding='utf-8')) if result_path.is_file() else {}
            if worker.returncode or not loaded.get('success'):
                from .adaptive import WorkerExit
                raise WorkerExit(worker.returncode, result_path.with_name(result_path.stem+'-worker.log'))
            result.update(loaded, resources=memory.result())
            history.finish(attempt, 'cancelled' if result.get('state') in ('cancelled', 'released') else 'success', details=result)
            attempt = None
    except BaseException as error:
        if worker is not None:
            try:
                worker.stop_request(grace=.5)
            except BaseException as cleanup:
                error.worker_stop_confirmed = False
                error.worker_stop_error = repr(cleanup)
        failure = classify_failure(KeyboardInterrupt(str(error)) if isinstance(error, PreloadCancelled) else error)
        if getattr(error, 'worker_stop_confirmed', None) is False:
            failure = classify_failure(error)
        result.update(state='cancelled' if failure['outcome'] == 'cancelled' else 'skipped' if isinstance(error, BlockingIOError) else 'failed',
                      reason=str(error), failure=failure, error_type=type(error).__name__)
        if attempt is not None:
            history.finish(attempt, failure['outcome'], details=result)
    finally:
        for signum, handler in previous_handler.items():
            signal.signal(signum, handler)
        result['completed_epoch'] = time.time()
        save(result_path, result)
    return result


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--request', type=Path, required=True)
    args = parser.parse_args()
    run(json.loads(args.request.read_text(encoding='utf-8')))


if __name__ == '__main__':
    main()