Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

10.0 KB
Raw Blame
"""Small, Torch-free media/adapter request contract shared by CLI and ComfyUI."""
import hashlib
import json
import math
from pathlib import Path


TASKS = ('t2va', 'i2va', 'l2va', 'fl2va', 'ref2va', 'ref2va_audio', 'ref2va_av')


def digest(path):
    value = hashlib.sha256()
    with Path(path).open('rb') as stream:
        for block in iter(lambda: stream.read(1024 * 1024), b''):
            value.update(block)
    return value.hexdigest()


def read(path):
    if path is None:
        return {}
    path = Path(path).resolve()
    if path.stat().st_size > 1024 * 1024:
        raise ValueError('Media request JSON exceeds 1 MiB')
    value = json.loads(path.read_text(encoding='utf-8'))
    if not isinstance(value, dict) or value.get('version') != 1:
        raise ValueError('Expected a version 1 FreeVideo media request')
    if 'early_steps' in value:
        raise ValueError('early_steps is only allowed on video references')
    if set(value) - {'version', 'first', 'last', 'references', 'loras', 'conditioning_info'}:
        raise ValueError('Unknown media request fields')
    result = {'version': 1}
    def file_info(name):
        if not isinstance(name, str) or not name:
            raise ValueError('Expected a media/LoRA file path')
        source = Path(name).expanduser()
        if not source.is_absolute():
            source = path.parent / source
        source = source.resolve(strict=True)
        if not source.is_file() or not source.stat().st_size:
            raise ValueError('Input file is empty or not a file: ' + str(source))
        return {'path': str(source), 'sha256': digest(source), 'bytes': source.stat().st_size}
    for anchor in ('first', 'last'):
        if value.get(anchor):
            result[anchor] = file_info(value[anchor])
    for field in ('references', 'loras'):
        entries = value.get(field)
        if entries is None:
            entries = []
        if not isinstance(entries, list) or len(entries) > 32:
            raise ValueError('At most 32 ' + field + ' per request')
        result[field] = []
        for entry in entries:
            if not isinstance(entry, dict):
                raise ValueError('Invalid ' + field + ' entry')
            if field != 'references' and 'early_steps' in entry:
                raise ValueError('early_steps is only allowed on video references')
            allowed = ({'path', 'kind', 'early_steps', 'sound', 'guide', 'voice_only', 'keep_source_audio', 'use', 'range', 'segment'} if field == 'references'
                       else {'path', 'strength', 'alpha'})
            if set(entry) - allowed:
                raise ValueError('Unknown ' + field + ' fields')
            row = file_info(entry.get('path'))
            if field == 'references':
                if entry.get('kind') not in ('image', 'video', 'audio'):
                    raise ValueError('Reference kind must be image, video or audio')
                row['kind'] = entry['kind']
                if 'early_steps' in entry:
                    validate_early_steps(entry)
                    row['early_steps'] = entry['early_steps']
                if 'sound' in entry:
                    validate_sound(entry)
                    if not entry['sound']:
                        row['sound'] = False
                if 'guide' in entry:
                    validate_guide(entry)
                    # A false marker is not stored, so requests without a guide
                    # keep exactly the same rows and cache key as before.
                    if entry['guide']:
                        row['guide'] = True
                if 'voice_only' in entry:
                    validate_voice_only(entry)
                    if entry['voice_only']:
                        row['voice_only'] = True
                if 'keep_source_audio' in entry:
                    validate_keep_source_audio(entry, row['path'])
                    if entry['keep_source_audio']:
                        row['keep_source_audio'] = True
                from .original_audio import fields
                row.update(fields(entry))
            else:
                if Path(row['path']).suffix.lower() != '.safetensors':
                    raise ValueError('LoRA requires safetensors; pickle adapters are not loaded')
                for name, default in (('strength', 1.), ('alpha', None)):
                    number = entry.get(name, default)
                    if (number is None and name == 'strength') or (number is not None and
                            (isinstance(number, bool) or not isinstance(number, (int, float)) or not math.isfinite(number))):
                        raise ValueError('LoRA ' + name + ' must be finite')
                    row[name] = number
            result[field].append(row)
    from .original_audio import original
    original(result)
    if result['references'] and any(result.get(k) for k in ('first', 'last')):
        raise ValueError('Reference and first/last keyframe layouts cannot be mixed in one request; '
                         'mark the clip that carries the motion with "guide": true instead')
    guides = [row for row in result['references'] if row.get('guide')]
    if len(guides) > 1:
        raise ValueError('At most one guide clip per request')
    # The packed layout needs the guide after the other references.
    result['references'].sort(key=lambda row: bool(row.get('guide')))
    if value.get('conditioning_info') is not None:
        info = value['conditioning_info']
        if not isinstance(info, dict) or info.get('task') not in TASKS:
            raise ValueError('Invalid preencoded conditioning information')
        result['conditioning_info'] = info
    return result


def task_for(media):
    if media.get('conditioning_info'):
        return media['conditioning_info']['task']
    if any(row.get('early_steps') != 0 for row in media.get('references', [])):
        return 'ref2va'
    if media.get('first') and media.get('last'):
        return 'fl2va'
    return 'i2va' if media.get('first') else 'l2va' if media.get('last') else 't2va'


def validate_early_steps(row):
    if row.get('kind') != 'video':
        raise ValueError('early_steps is only allowed on video references')
    if type(row['early_steps']) is not int or row['early_steps'] < 0:
        raise ValueError('Video reference early_steps must be an integer >= 0')


def validate_sound(row):
    if row.get('kind') != 'video':
        raise ValueError('sound is only allowed on video references')
    if type(row['sound']) is not bool:
        raise ValueError('Video reference sound must be true or false')


def validate_guide(row):
    """A guide is a video reference that shares the generated clip's time origin.

    It is encoded like a silent video reference, so it cannot carry its own
    audio or a delayed start.
    """
    if row.get('kind') != 'video':
        raise ValueError('guide is only allowed on video references')
    if type(row['guide']) is not bool:
        raise ValueError('Video reference guide must be true or false')
    if row['guide'] and 'early_steps' in row:
        raise ValueError('A guide clip cannot use early_steps')


def validate_voice_only(row):
    """A voice-only video supplies only its sound track, read as an audio reference; its picture is not encoded."""
    if row.get('kind') != 'video':
        raise ValueError('voice_only is only allowed on video references')
    if type(row['voice_only']) is not bool:
        raise ValueError('Video reference voice_only must be true or false')
    if row['voice_only'] and (row.get('guide') or row.get('sound') is False or 'early_steps' in row):
        raise ValueError('A voice-only clip cannot be a guide, silent or use early_steps')


def validate_keep_source_audio(row, path):
    """Keeping the guide's sound needs a guide clip that has an audio stream."""
    if type(row['keep_source_audio']) is not bool:
        raise ValueError('keep_source_audio must be true or false')
    if not row['keep_source_audio']:
        return
    if row.get('guide') is not True:
        raise ValueError('keep_source_audio is only allowed on a guide clip')
    from .guide_audio import has_audio
    if not has_audio(path):
        raise ValueError('keep_source_audio needs a guide clip with an audio stream')


def reference_stages(media):
    """Active references at each boundary; zero-step inputs never reach an encoder."""
    refs = media.get('references', [])
    for row in refs:
        if 'early_steps' in row:
            validate_early_steps(row)
    boundaries = sorted({0} | {row['early_steps'] for row in refs if 'early_steps' in row})
    return [(step, dict(media, references=[row for row in refs
             if 'early_steps' not in row or step < row['early_steps']])) for step in boundaries]


def cache_key(prompt, media, canvas):
    # LoRA affects the denoiser, not the encoder. Paths do not identify content.
    if not any(media.get(k) for k in ('first', 'last', 'references')):
        return prompt
    identity = {'version': 1, 'prompt': prompt, 'task': task_for(media),
                'width': canvas['width'], 'height': canvas['height'], 'frames': canvas['frames']}
    for key in ('first', 'last', 'references'):
        value = media.get(key)
        if value:
            identity[key] = [{k: v for k, v in row.items() if k not in ('path', 'keep_source_audio')}
                             for row in value] if isinstance(value, list) else value['sha256']
            if key == 'references':
                from .original_audio import identity as audio_identity
                for row in identity[key]:
                    if row.get('original_audio'):
                        row['original_audio'] = audio_identity(row['original_audio'])
    return 'multimodal:' + json.dumps(identity, sort_keys=True, ensure_ascii=False)


def verify_files(media):
    rows = [media[k] for k in ('first', 'last') if media.get(k)]
    rows += media.get('references', []) + media.get('loras', [])
    for row in rows:
        path = Path(row['path'])
        if path.stat().st_size != row['bytes'] or digest(path) != row['sha256']:
            raise ValueError('Input changed after inspection: ' + str(path))