Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

23.2 KB
Raw Blame
"""Optional, file-backed depth references. Torch lives only in a disposable worker.

Small's vendored Apache-2.0 network and preprocessing follow Quinn's fixed
DAV2 experiment. Cache identity deliberately excludes prompt, seed and K.
"""
from contextlib import contextmanager
from fractions import Fraction
import hashlib
import json
import os
from pathlib import Path
import subprocess
import tempfile
import time
import uuid

from . import processes
from .paths import data_root

PREPROCESSING_VERSION = 'dav2-small-pyav-torch-bgr-imagenet-lower-bound-14-clay-relief-v1'
CODE_VERSION = 'freevideo-depth-reference-v1'
STAGE = 'depth_reference'


def catalog():
    return json.loads(Path(__file__).with_name('depth_reference_model.json').read_text(encoding='utf-8'))


def canonical_json(value):
    return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False, allow_nan=False)


def digest(path, check=lambda: None):
    value = hashlib.sha256()
    with Path(path).open('rb') as stream:
        while block := stream.read(1024 * 1024):
            check()
            value.update(block)
    return value.hexdigest()


def cache_directory(root=None):
    # download_cache is OS page-cache advice, not an application cache folder.
    return Path(root or data_root()) / 'input-cache' / 'depth-cache'


@contextmanager
def _lock(path, check, *, inherit=False):
    from .locking import runtime_lock
    while True:
        check()
        try:
            lease = runtime_lock(path, inherit=inherit)
            descriptor = lease.__enter__()
            break
        except BlockingIOError:
            time.sleep(.1)
    try:
        yield descriptor
    finally:
        lease.__exit__(None, None, None)


def ensure_weights(root, emit, check):
    """The same optional-model catalog/transfer mechanism as FreeToken and ASR."""
    from . import network
    from .adaln_assets import _download_plan
    from .prompt_vlm.assets import remote
    spec = catalog()
    row = spec['files'][0]
    folder = Path(root) / 'models' / 'depth-anything-v2-small'
    path = folder / row['file']
    with _lock(folder / 'download.lock', check):
        if path.is_file() and path.stat().st_size == row['bytes'] and digest(path, check) == row['sha256']:
            return path
        if path.exists():
            network.retain_partial(path, 'rejected')
        plan = _download_plan(Path(root))
        sources = plan['sources'] = dict(plan.get('sources', {}))
        if not sources.get('image-models'):
            sources['image-models'] = [dict(row) for row in sources.get('models', [])
                                       if row['id'] in ('official', 'hf-mirror')]
            sources['image-models'].append({'id': 'modelscope'})
        plan.update(quiet=True, resource_check=check)
        # remote() describes HF pins and a hash-locked ModelScope fallback;
        # network retains its measured routes and the user's connection mode.
        def moved(current, total, speed, **_):
            emit(phase='dependencies', stage='depth_reference_download', label='Downloading depth reference model',
                 done=current, total=row['bytes'], unit='bytes', bytes_per_second=speed)
        check()
        try:
            network.download(network.model_urls(plan, remote(spec, row)), path, row['sha256'], moved,
                             network=plan, size=row['bytes'], category='image-models', keep_partial=True,
                             stall_seconds=30, slow_seconds=15, low_speed_limit=64 * 1024)
        except Exception as error:
            from .sampling_assets import _cancel_or_disk
            if _cancel_or_disk(error):
                raise
            check()
            raise RuntimeError('Depth reference weights are unavailable. Retry with a connection, or put the '
                               'official depth_anything_v2_vits.pth (99,218,434 bytes, SHA-256 '
                               + row['sha256'] + ') in ' + str(folder)) from error
        if path.stat().st_size != row['bytes'] or digest(path, check) != row['sha256']:
            raise ValueError('Depth reference weight size/SHA-256 verification failed: ' + str(path))
        return path


def _rotation(frame):
    # PyAV exposes the display matrix as counterclockwise degrees, as in agent.media.
    try:
        return round(frame.rotation or 0) % 360
    except (AttributeError, ValueError):
        return 0


def _upright_bgr(frame):
    import numpy as np
    return np.ascontiguousarray(np.rot90(frame.to_ndarray(format='bgr24'), round(_rotation(frame) / 90) % 4))


def inspect_video(path, check=lambda: None, *, depth=False):
    """Decode every frame, rather than trusting container frame-count headers."""
    import av
    pts, dimensions, rotation = [], None, None
    with av.open(str(path)) as container:
        if not container.streams.video:
            raise ValueError('Depth reference needs a decodable video')
        if depth and container.streams.audio:
            raise ValueError('Depth reference cache unexpectedly contains audio')
        stream = container.streams.video[0]
        rate = stream.average_rate
        if rate is None or rate <= 0:
            raise ValueError('Depth reference video has no valid frame rate')
        time_base = stream.time_base
        for frame in container.decode(stream):
            check()
            current_rotation = _rotation(frame)
            if rotation is not None and current_rotation != rotation:
                raise ValueError('Depth reference video changes display rotation')
            rotation = current_rotation
            size = (frame.width, frame.height)
            if dimensions is not None and dimensions != size:
                raise ValueError('Depth reference video changes dimensions')
            dimensions = size
            pts.append(frame.pts)
    if not pts:
        raise ValueError('Depth reference video has no decodable frames')
    if depth and rotation:
        raise ValueError('Depth reference cache unexpectedly contains display rotation')
    if round(rotation / 90) % 2:
        dimensions = dimensions[::-1]
    return dict(width=dimensions[0], height=dimensions[1], fps_num=rate.numerator, fps_den=rate.denominator,
                frames=len(pts), pts=pts, time_base=str(time_base), rotation=rotation)


def identity(source, video, device, check=lambda: None):
    spec = catalog()
    output = {key: video[key] for key in ('width', 'height', 'fps_num', 'fps_den', 'frames')}
    output.update(fit='source', audio=False)
    pix_fmt = 'yuv420p' if video['width'] % 2 == video['height'] % 2 == 0 else 'yuv444p'
    return dict(source_sha256=digest(source, check), source_size=Path(source).stat().st_size,
                model_id=spec['repo'], weights_sha256=spec['files'][0]['sha256'], source_commit=spec['source_commit'],
                code_version=CODE_VERSION, preprocessing_version=PREPROCESSING_VERSION,
                selection_spec=dict(source_start_pts=video['pts'][0], source_end_pts=video['pts'][-1],
                                    source_time_base=video['time_base'], crop_rect=None,
                                    rotation=video['rotation'], tail_policy='none'),
                output_spec=output,
                inference_spec=dict(input_size=518, precision='fp16' if device == 'cuda' else 'fp32',
                    normalization='whole_clip_p1_p99_from_240x136_near_white_v1',
                    render='clay-relief-v1',
                    encoder='rgb-to-'+pix_fmt+'/libx264/crf16/threads2', runtime_backend='torch-'+device))


def cache_key(config):
    return hashlib.sha256(canonical_json(config).encode('utf-8')).hexdigest()


def _validate(path, expected, check):
    observed = inspect_video(path, check, depth=True)
    if any(observed[key] != expected[key] for key in ('width', 'height', 'fps_num', 'fps_den', 'frames')):
        raise ValueError('Depth reference output dimensions, frame rate or frame count differ from the input')
    return observed


def _cached(folder, config, check):
    try:
        manifest = json.loads((folder / 'manifest.json').read_text(encoding='utf-8'))
        output = folder / 'depth.mp4'
        if (manifest['config'] != config or manifest['output_sha256'] != digest(output, check)
                or manifest['output_bytes'] != output.stat().st_size):
            return None
        _validate(output, config['output_spec'], check)
        return output
    except (OSError, ValueError, KeyError, TypeError) as error:
        if isinstance(error, InterruptedError):
            raise
        return None


def _load_model(weights, device):
    import torch
    from .depth_anything_v2.dpt import DepthAnythingV2
    model = DepthAnythingV2()
    model.load_state_dict(torch.load(weights, map_location='cpu', weights_only=True), strict=True)
    return model.to(device).eval()


def _clay_relief(gray):
    """Render quantized, near-bright depth with Quinn's fixed shade.py recipe.

    SciPy's default Gaussian uses sigma=3, radius=12 and half-sample symmetric
    reflection (including the edge pixel). Torch's reflect pad excludes that
    pixel, so use reflected indices; this also works on frames below 13 pixels.
    """
    import numpy as np
    import torch
    from torch.nn.functional import conv2d

    d = torch.from_numpy(gray.astype(np.float32) / 255)[None, None]
    offsets = torch.arange(-12, 13, dtype=torch.float64)
    kernel = torch.exp(-0.5 * offsets.square() / 3.0**2)
    kernel = (kernel / kernel.sum()).float()
    # Match gaussian_filter's axis order and float32 intermediate images.
    for axis in (2, 3):
        size = d.shape[axis]
        indices = torch.arange(-12, size + 12) % (2 * size)
        indices = torch.where(indices < size, indices, 2 * size - 1 - indices)
        weight = kernel.reshape(1, 1, 25, 1) if axis == 2 else kernel.reshape(1, 1, 1, 25)
        d = conv2d(d.index_select(axis, indices), weight)
    d = d[0, 0].numpy()
    height, width = d.shape
    # numpy.gradient uses central differences inside and one-sided edges.
    gy = np.gradient(d, axis=0) if height > 1 else np.zeros_like(d)
    gx = np.gradient(d, axis=1) if width > 1 else np.zeros_like(d)
    k = 5.0
    nx, ny, nz = -gx*k*width/64, -gy*k*height/64, np.ones_like(d)
    n = np.sqrt(nx**2 + ny**2 + nz**2)
    nx, ny, nz = nx/n, ny/n, nz/n
    light = np.array([-0.5, -0.6, 0.62])
    light /= np.linalg.norm(light)
    lam = np.clip(nx*light[0] + ny*light[1] + nz*light[2], 0, 1)
    shade = (0.25 + 0.75*lam) * (0.75 + 0.25*d)
    base = np.array([0.82, 0.80, 0.77])
    return (np.clip(shade, 0, 1)[..., None] * base * 255).astype(np.uint8)


def _infer(source, stage, video, weights, device, emit, check):
    import av
    import numpy as np
    import psutil
    import torch
    # Reserve system RAM and account for live free memory before loading.
    if psutil.virtual_memory().available < 2 * 2**30:
        raise MemoryError('Depth reference needs 1 GiB working RAM plus 1 GiB system reserve; free memory and retry')
    if device == 'cuda':
        if not torch.cuda.is_available():
            raise RuntimeError('CUDA is unavailable for depth reference; select the CPU backend to use FP32')
        free, total = torch.cuda.mem_get_info()
        reserve = max(512 * 2**20, total // 10)
        if free < reserve + 512 * 2**20:
            raise MemoryError('Not enough free GPU memory for depth reference and the GPU reserve')
    torch.set_num_threads(min(8, os.cpu_count() or 1))
    shape = (video['frames'], video['height'], video['width'])
    from .disk_space import free_bytes
    if free_bytes(stage) < (shape[0] * (shape[1] * shape[2] + 240 * 136) * 4 + 256 * 2**20):
        raise OSError('Not enough disk space to stage depth frames; free space and retry')
    model = _load_model(weights, device)
    raw_path, small_path = stage / 'raw.f32', stage / 'small.f32'
    depths = np.memmap(raw_path, mode='w+', dtype=np.float32, shape=shape)
    small = np.memmap(small_path, mode='w+', dtype=np.float32, shape=(video['frames'], 136, 240))
    try:
        with av.open(str(source)) as container:
            count = 0
            for frame in container.decode(video=0):
                check()
                bgr = _upright_bgr(frame)
                if (count >= video['frames'] or bgr.shape[:2] != shape[1:]
                        or _rotation(frame) != video['rotation']):
                    raise ValueError('Source video changed during depth conversion')
                if psutil.virtual_memory().available < 2**30:
                    raise MemoryError('Depth reference stopped to preserve 1 GiB of free system RAM')
                with torch.inference_mode(), torch.autocast('cuda', dtype=torch.float16, enabled=device == 'cuda'):
                    depth = np.asarray(model.infer_image(bgr, 518), dtype=np.float32)
                if depth.shape != shape[1:] or not np.isfinite(depth).all():
                    raise ValueError('Depth model returned invalid/nonfinite pixels')
                depths[count] = depth
                small[count] = torch.nn.functional.interpolate(torch.from_numpy(depth)[None, None],
                                                               (136, 240), mode='area')[0, 0].numpy()
                count += 1
                emit(done=count, total=video['frames'], unit='frames', phase='depth_reference',
                     detail='%d/%d frames' % (count, video['frames']))
            if count != video['frames']:
                raise ValueError('Source video frame count changed during depth conversion')
        # Whole-clip p1/p99, in-place selection on disk-backed samples, bounded RAM.
        small.flush()
        lo, hi = np.percentile(small.reshape(-1), [1, 99], overwrite_input=True)
        span = max(float(hi - lo), 1e-6)
        output = stage / 'depth.mp4'
        emit(phase='depth_reference', detail='Encoding depth video')
        with av.open(str(output), 'w') as container:
            stream = container.add_stream('libx264', rate=Fraction(video['fps_num'], video['fps_den']))
            stream.width, stream.height = video['width'], video['height']
            stream.pix_fmt = 'yuv420p' if stream.width % 2 == stream.height % 2 == 0 else 'yuv444p'
            stream.options = {'crf': '16', 'threads': '2'}
            for index, depth in enumerate(depths):
                check()
                gray = (np.clip((depth - lo) / span, 0, 1) * 255).round().astype(np.uint8)
                frame = av.VideoFrame.from_ndarray(_clay_relief(gray), format='rgb24')
                frame.pts, frame.time_base = index, Fraction(video['fps_den'], video['fps_num'])
                for packet in stream.encode(frame):
                    container.mux(packet)
            for packet in stream.encode(None):
                container.mux(packet)
        return dict(low=float(lo), high=float(hi), near='white', torch=torch.__version__,
                    av=av.__version__, ffmpeg=av.library_versions,
                    attention='ordinary-torch-matmul-softmax', device=device)
    finally:
        # Close mappings before deleting or moving files (Windows).
        depths._mmap.close()
        small._mmap.close()
        del depths, small
        raw_path.unlink(missing_ok=True)
        small_path.unlink(missing_ok=True)


def _convert(request, emit, check):
    source, root, work = Path(request['source']), Path(request['root']), Path(request['work'])
    device = request.get('device', 'auto')
    if device == 'auto':
        import torch
        device = 'cuda' if torch.cuda.is_available() else 'cpu'
    if device not in ('cpu', 'cuda'):
        raise ValueError('Depth reference device must be cpu, cuda or auto')
    emit(phase='depth_reference', detail='Checking depth reference cache')
    initial_hash, initial_size = digest(source, check), source.stat().st_size
    video = inspect_video(source, check)
    config = identity(source, video, device, check)
    if config['source_sha256'] != initial_hash or config['source_size'] != initial_size:
        raise ValueError('Source video changed during depth inspection')
    cache = Path(request['cache'])
    key = cache_key(config)
    target = cache / key
    with _lock(cache / '.locks' / (key + '.lock'), check):
        found = _cached(target, config, check)
        if found is not None:
            return dict(path=str(found), cache_hit=True, key=key)
        weights = ensure_weights(root, emit, check)
        # Parent releases idle Comfy/engine/encoder models only on a cache miss.
        emit(phase=STAGE, stage=STAGE, label='Extracting depth from the reference video',
             done=0, total=video['frames'], unit='frames', detail='Loading depth model', waiting_for_device=True)
        while not (work / 'device-ready').is_file():
            check()
            time.sleep(.05)
        stage = work / 'result'
        stage.mkdir()
        with _lock(Path(os.environ.get('FREEVIDEO_LOCK_PATH', root / 'engine.lock')), check, inherit=True):
            metrics = _infer(source, stage, video, weights, device, emit, check)
        observed = _validate(stage / 'depth.mp4', video, check)
        if digest(source, check) != config['source_sha256'] or source.stat().st_size != config['source_size']:
            raise ValueError('Source video changed during depth conversion')
        manifest = dict(config=config, source_pts=video['pts'], source_time_base=video['time_base'], output=observed,
                        output_sha256=digest(stage / 'depth.mp4', check), output_bytes=(stage / 'depth.mp4').stat().st_size,
                        normalization=metrics, render='clay-relief-v1', license='Apache-2.0',
                        license_sources=['https://github.com/DepthAnything/Depth-Anything-V2/tree/'+catalog()['source_commit'],
                                         'https://huggingface.co/'+catalog()['repo']+'/tree/'+catalog()['revision']])
        (stage / 'manifest.json').write_text(canonical_json(manifest), encoding='utf-8')
        check()
        if target.exists():
            # Retain invalid old artifacts separately; never accept a partial directory.
            target.rename(cache / (key + '.rejected-' + uuid.uuid4().hex))
        os.replace(stage, target)
        return dict(path=str(target / 'depth.mp4'), cache_hit=False, key=key)


def convert(source, *, root=None, cache=None, device='auto', python=None, environ=None,
            progress=None, interrupted=None, release_models=None):
    """Convert in a supervised child; return a validated cache path after it exits.

    Progress carries stage=depth_reference and done/total in frames. Cancellation
    stops the whole child tree and removes only this request's temporary files.
    """
    root = Path(root or data_root()).resolve()
    cache = Path(cache or cache_directory(root)).resolve()
    cache.mkdir(parents=True, exist_ok=True)
    def check():
        if interrupted:
            interrupted()
    check()
    with tempfile.TemporaryDirectory(prefix='.depth-', dir=cache) as directory:
        work = Path(directory)
        request = dict(source=str(Path(source).resolve()), root=str(root), cache=str(cache), device=device, work=str(work))
        (work / 'request.json').write_text(canonical_json(request), encoding='utf-8')
        command = processes.module_command('freevideo_engine.depth_reference', '--request', work / 'request.json')
        if python is not None:
            command[0] = str(python)
        with (work / 'worker.log').open('wb') as log:
            child = processes.popen(command, env=environ, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT,
                                    start_new_session=True, supervise=True, pass_fds=processes.descriptors())
            last, released = None, False
            def poll():
                nonlocal last, released
                check()
                try:
                    value = json.loads((work / 'progress.json').read_text(encoding='utf-8'))
                except (OSError, ValueError):
                    return
                if value.get('waiting_for_device') and not released:
                    if release_models:
                        release_models()
                    check()
                    (work / 'device-ready').touch()
                    released = True
                if progress and value != last:
                    progress({key: val for key, val in value.items() if key != 'waiting_for_device'})
                last = value
            try:
                while child.poll() is None:
                    poll()
                    try:
                        child.wait(timeout=.1)
                    except subprocess.TimeoutExpired:
                        pass
                poll()
                if child.returncode:
                    try:
                        error = json.loads((work / 'error.json').read_text(encoding='utf-8'))['error']
                    except (OSError, ValueError, KeyError):
                        error = 'Depth reference worker exited with code %s' % child.returncode
                    raise RuntimeError(error)
                result = json.loads((work / 'result.json').read_text(encoding='utf-8'))
                check()
                return Path(result['path'])
            except BaseException:
                (work / 'cancelled').touch()
                processes.stop(child)
                raise


def prepare_media(media, *, root, machine, environ, progress, interrupted, release_models):
    """Execution only. With depth off, return the original object unchanged."""
    if not any(row.get('depth') is True for row in media.get('references', [])):
        return media
    result = dict(media, references=[])
    for row in media.get('references', []):
        if 'depth' in row and (type(row['depth']) is not bool or row.get('kind') != 'video'):
            raise ValueError('depth must be a boolean on a video reference')
        if row.get('depth') is not True:
            result['references'].append(row)
            continue
        converted = convert(row['path'], root=root, python=machine['python'], environ=environ,
                            device='cpu' if machine.get('device_backend') in ('cpu', 'mps') else 'cuda',
                            progress=progress, interrupted=interrupted, release_models=release_models)
        value = {key: val for key, val in row.items() if key != 'depth'}
        value['path'] = str(converted)
        value.setdefault('early_steps', 3)
        result['references'].append(value)
    return result


def main():
    import argparse
    from .monitoring import save
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--request', required=True, type=Path)
    args = parser.parse_args()
    request = json.loads(args.request.read_text(encoding='utf-8'))
    work = Path(request['work'])
    def check():
        if (work / 'cancelled').exists():
            raise InterruptedError('Depth reference conversion cancelled')
    state = dict(stage=STAGE, phase=STAGE, timing_phase='encoding', label='Extracting depth from the reference video',
                 done=0, total=None, unit='frames', references=True)
    def emit(**values):
        state.update(values)
        save(work / 'progress.json', state)
    try:
        with processes.worker_signals():
            result = _convert(request, emit, check)
        save(work / 'result.json', result)
    except Exception as error:
        save(work / 'error.json', dict(error=type(error).__name__ + ': ' + str(error)))
        return 1
    return 0


if __name__ == '__main__':
    raise SystemExit(main())