Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

17.6 KB
Raw Blame
"""Normalize schema-validated job values; derive geometry, sampling and dependencies."""
import json
import math
import re
import secrets
from . import schema as contract
from .length import single_frame_limit

QUALITY, ASPECTS = contract.QUALITY, contract.ASPECTS
KINDS = tuple(contract.KINDS)
FRAMES = ('first', 'last')
# A part that continues a video carries at most this many reference pictures, and no other reference media.
CONTINUED_PICTURES = 3
MAX_SEED = contract.generate()['properties']['seed']['maximum']
JOB_ID = re.compile(contract.JOB['pattern'].strip('^$'))
LIBRARY_ID = re.compile(contract.SAVED['pattern'].strip('^$'))
SLUG = re.compile(contract.SLUG['pattern'].strip('^$'))


class SpecError(ValueError):
    def __init__(self, errors):
        self.errors = errors if isinstance(errors, list) else [errors]
        super().__init__('; '.join('%s: %s' % (e['path'] or '/', e['message']) for e in self.errors))


def fail(path, message):
    raise SpecError(contract.diagnostic(path, message, None))


def import_name(name):
    """Timeline imports accept video/audio content, with no HTTP filesystem paths."""
    from ..comfy_assets import media_kind
    problems = contract.issues(contract.import_upload()['properties']['name'], name, '/name')
    if problems:
        raise SpecError(problems)
    try:
        if media_kind(name) not in ('video', 'audio'):
            raise ValueError
    except ValueError:
        error = SpecError(contract.diagnostic('/name', 'a supported video or audio filename', None))
        error.code = 'invalid_media'
        raise error from None
    return name


# Field checks for values no route schema describes: screenplays and filming records.
def _bool(value, path):
    if type(value) is not bool:
        fail(path, 'must be true or false')
    return value


def _int(value, path, low, high):
    if type(value) is not int or not low <= value <= high:
        fail(path, 'must be an integer from %d to %d' % (low, high))
    return value


def _number(value, path, low, high):
    if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) \
            or not low <= value <= high:
        fail(path, 'must be a number from %g to %g' % (low, high))
    return float(value)


def _text(value, path, limit, *, empty=False):
    if not isinstance(value, str) or (not empty and not value.strip()) or len(value) > limit or '\x00' in value:
        fail(path, 'must be a non-empty string of at most %d characters' % limit if not empty
             else 'must be a string of at most %d characters' % limit)
    return value


def check(value, *, local=False):
    """One job as submission accepts it: checked against the shared definition, then normalized.

    local=True admits local file paths, exactly as Jobs.submit does for in-process callers;
    the default is the stricter content-only boundary of external HTTP.
    """
    contract.validate(contract.job_schema(local=local), value)
    return normalize(value)


def new_id():
    return 'j' + secrets.token_hex(6)


def media_item(value, path, *, image=False, frame_default=None, early=False):
    if isinstance(value, str):
        value = {'file': value}
    value = dict(value) if isinstance(value, dict) else value
    if isinstance(value, dict) and frame_default is not None and any(k in value for k in ('from_job', 'library')):
        value.setdefault('frame', frame_default)
    errors = contract.issues(contract.media(image, local=True, early=early), value, path)
    if errors:
        raise SpecError(errors)
    if 'file' in value and re.fullmatch(contract.CHAT_ATTACHMENT_NAME['pattern'], value['file']):
        value['file'] = 'freevideo/agent/assets/' + value['file']
    return value


CUT = re.compile(r'\[Shot\s+\d+\]\s*At\s+(\d{1,2}):(\d{1,2}(?:\.\d+)?)')


def cut_times(text):
    """Seconds of each "[Shot N] At MM:SS.mmm" cut in a prompt; none for a prompt taken from another job."""
    if not isinstance(text, str):
        return []
    return [int(minutes) * 60 + float(seconds) for minutes, seconds in CUT.findall(text)]


def dependencies(item):
    """Job ids a normalized spec reads from."""
    found = set(item.get('after', ()))
    def walk(value):
        if isinstance(value, dict):
            if isinstance(value.get('from_job'), str):
                found.add(value['from_job'])
            for key, child in value.items():
                if key != 'edit':
                    walk(child)
        elif isinstance(value, list):
            for child in value:
                walk(child)
    walk({key: value for key, value in item.items() if key not in ('after', 'metadata')})
    return sorted(found)


def _common(value):
    return {k: value[k] for k in ('label', 'project', 'metadata', 'after') if k in value}


def canvas_size(value):
    if 'aspect' in value or 'megapixels' in value:
        a, b = map(int, value.get('aspect', '16:9').split(':'))
        total = value.get('megapixels', contract.generate()['properties']['megapixels']['default']) * 1024 ** 2
        width = round(math.sqrt(total * a / b) / 32) * 32
        height = round(math.sqrt(total * b / a) / 32) * 32
        properties = contract.generate()['properties']
        if any(contract.issues(properties[k], v) for k, v in (('width', width), ('height', height))):
            fail('/aspect', 'an aspect and megapixels producing dimensions within the width/height contract')
        return width, height
    properties = contract.generate()['properties']
    return tuple(value.get(k, properties[k]['default']) for k in ('width', 'height'))


def sampling(value):
    from .. import tiers
    mac = tiers.on_mac()
    quality = value.get('quality', contract.generate()['properties']['quality']['default'])
    two, base, refine = QUALITY[quality]
    custom = value.get('sampling', {})
    if two and custom.get('two_pass') is False and 'base_steps' not in custom:
        # One pass always samples 20 steps unless the caller gives its own count.
        base = 20
    two, base, refine = custom.get('two_pass', two), custom.get('base_steps', base), custom.get('refine_steps', refine)
    note = None
    if two:
        from ..two_pass import one_pass_note
        # Only the 8-step first pass goes two-pass; any other count is one pass, said once in the answer.
        note = one_pass_note(base)
        two = two and note is None
    # A level's step cache and upscale factor stay only with that level's own plan; custom sampling
    # runs every step and upscales 2x.
    own = QUALITY[quality] == (two, base, refine)
    cache = tiers.step_cache(quality, mac) if own else None
    factor = tiers.upscale_factor(quality, mac) if own and two else 2
    if custom:
        quality = tiers.name_for(base, two, refine, cache, mac, factor) or 'custom'
    return dict(quality=quality, two_pass=two, base_steps=base, refine_steps=refine,
                **({'step_cache': cache} if cache else {}), **({'upscale_factor': factor} if factor != 2 else {}),
                **({'sampling_note': note} if note else {}))


def generation_media(value):
    if not value:
        return {}
    result = {role: media_item(value[role], '/media/' + role, image=True) for role in FRAMES if value.get(role) is not None}
    if value.get('references'):
        result['references'] = [media_item(v, '/media/references/%d' % i, early=True) for i, v in enumerate(value['references'])]
    return result


def still_picture(item):
    """A reference that is one picture: a frame of a video, or a picture file."""
    if isinstance(item, str):
        item = {'file': item}
    if not isinstance(item, dict):
        return False
    if 'frame' in item:
        return True
    from ..comfy_assets import EXTENSIONS
    source = item.get('file') or item.get('path')
    return isinstance(source, str) and source.lower().endswith(tuple(EXTENSIONS['image']))


def task(media):
    if media.get('references'):
        return 'ref2va'
    if 'first' in media and 'last' in media:
        return 'fl2va'
    return 'i2va' if 'first' in media else 'l2va' if 'last' in media else 't2va'


def saved_length(canvas, spec):
    """Saved length: one generation keeps its aligned canvas; multiple blocks are joined to the planned
    whole seconds. A continuation saves only what it adds."""
    from ..chain_plan import PlanError, plan
    continued = bool(spec.get('continue_from'))
    try:
        planned = plan(spec['seconds'], max_frames=single_frame_limit(spec), continued=continued)
    except PlanError:
        return canvas['frames'], canvas['seconds']
    if planned['single'] and not continued:
        return canvas['frames'], canvas['seconds']
    return planned['frames'], planned['frames'] / canvas['fps']


def normalize(value, *, port=None):
    # Local Python callers retain local media refs. HTTP validates its stricter
    # (content-only) schema first; the client adapts local refs before transport.
    contract.validate_job(value, local=True)
    kind = value.get('kind', 'generate')
    value = {k: v for k, v in value.items() if v is not None or k == 'seed'}
    original = value
    value = contract.defaults(contract.KINDS[kind](local=True), value)
    result = dict(kind=kind, **_common(value))
    if kind == 'generate':
        result['force_regenerate'] = value['force_regenerate']
        if value.get('release_gpu_after'):
            result['release_gpu_after'] = True
        if value.get('refine_from'):
            ref = value['refine_from']
            result['refine_from'] = {'from_job' if JOB_ID.fullmatch(ref) else 'library': ref}
        else:
            from ..geometry import geometry
            width, height = canvas_size(original)
            canvas = geometry(width, height, seconds=value['seconds'])
            result.update(prompt=value['prompt'], width=width, height=height, seconds=float(value['seconds']),
                          seed_random=value['seed'] is None,
                          seed=secrets.randbelow(MAX_SEED + 1) if value['seed'] is None else value['seed'],
                          preview=value['preview'], media=generation_media(value.get('media')), **sampling(value))
            result['task'] = task(result['media'])
            if value.get('continue_from'):
                result['continue_from'] = dict(value['continue_from'])
            if value['first_frame_reference']:
                result['first_frame_reference'] = True
            if value.get('part_prompts'):
                result['part_prompts'] = list(value['part_prompts'])
            result['loras'] = [dict(name=row, strength=1.) if isinstance(row, str) else {'strength': 1., **row}
                               for row in value['loras']]
            if port is not None:
                result['single_max_frames'] = single_frame_limit(result, port=port)
            result['frames'], result['output_seconds'] = saved_length(canvas, result)
    elif kind == 'assemble':
        result['clips'] = []
        for row in value['clips']:
            if isinstance(row, str):
                row = {'from_job' if JOB_ID.fullmatch(row) else 'library' if LIBRARY_ID.fullmatch(row) else 'file': row}
            result['clips'].append(dict(row))
        result.update({k: value[k] for k in ('transition', 'transition_seconds', 'width', 'height', 'audio', 'name') if k in value})
    elif kind == 'vlm':
        rows = value['questions'] if isinstance(value['questions'], list) else [value['questions']]
        result['questions'] = []
        for row in rows:
            row = {'text': row} if isinstance(row, str) else row
            item = dict(text=row['text'], images=[media_item(v, '/questions/images', image=True) for v in row.get('images', [])])
            if row.get('video'):
                video = row['video']
                item['frames'] = video.get('frames', 6) if isinstance(video, dict) else 6
                item['video'] = media_item({k: v for k, v in video.items() if k != 'frames'} if isinstance(video, dict) else video, '/questions/video')
            result['questions'].append(item)
        result.update({k: value[k] for k in ('format', 'max_tokens', 'system') if k in value})
    elif kind == 'image':
        from ..image_gen.loras import validate_selection
        loras = validate_selection(original.get('loras', []))
        if loras:
            result['loras'] = loras
        width, height = ((value['width'], value['height']) if 'width' in value else
                         contract.IMAGE_ASPECTS[value.get('aspect', '1:1')])
        result.update(prompt=value.get('prompt', ''), width=width, height=height,
                      images=[item if isinstance(item, str) else item['file'] for item in value.get('images', [])])
        if value.get('seed') is not None:
            result['seed'] = value['seed']
        if value.get('people'):
            result['people'] = [dict({k: person[k] for k in ('image', 'cast', 'name', 'facing') if k in person},
                                     x=person.get('x', .5), height=person.get('height', .62),
                                     framing=person.get('framing', 'full'),
                                     feet=person.get('feet', max(.9, person.get('height', .62))))
                                for person in value['people']]
            if value.get('scene_image'):
                result['scene_image'] = value['scene_image']
    elif kind == 'export':
        result.update(editor_project=value['editor_project'], revision=value['revision'])
    elif kind == 'listen':
        result.update(editor_project=value['editor_project'], revision=value['revision'], mode=value['mode'],
                      language=value.get('language', 'auto'), **{k: value[k] for k in ('lines', 'range') if k in value})
    elif kind == 'music':
        result.update(editor_project=value['editor_project'], revision=value['revision'], caption=value['caption'],
                      **{k: value[k] for k in ('seed', 'quiet') if k in value})
    elif kind == 'stems':
        result.update(editor_project=value['editor_project'], revision=value['revision'], clip_id=value['clip_id'],
                      lay=value.get('lay', True))
    elif kind == 'finish_sound':
        result.update(editor_project=value['editor_project'], revision=value['revision'],
                      keep_dialogue_volume=bool(value.get('keep_dialogue_volume', False)),
                      **({'music': dict(value['music'])} if 'music' in value else {}))
    elif kind == 'voice_lines':
        result.update(editor_project=value['editor_project'], revision=value['revision'], selection=dict(value['selection']),
                      language=value.get('language', 'auto'), separate_first=bool(value.get('separate_first', False)),
                      **({'script': [dict(entry) for entry in value['script']]} if value.get('script') else {}))
    elif kind == 'voice':
        result.update(editor_project=value['editor_project'], revision=value['revision'], selection=dict(value['selection']),
                      lines=[dict(line) for line in value['lines']], language=value.get('language', 'auto'),
                      voice=dict(value.get('voice') or {}), lay=value.get('lay', True),
                      **{k: value[k] for k in ('seed',) if k in value})
    elif kind == 'image_edit':
        result.update(image=value['image'], revision=value['revision'])
    else:
        result.update(text=value['text'], seconds=float(value['seconds']), media=generation_media(value.get('media')))
    result['depends_on'] = dependencies(result)
    return result


def comfy_prompt(spec, node, *, prompt, assets, refine_from=''):
    """The ComfyUI API-format graph for one generate job; media are files in ComfyUI input."""
    from ..lora_triggers import apply_triggers
    prompt = apply_triggers(prompt, spec.get('loras', []))
    inputs = dict(text=prompt, width=spec['width'], height=spec['height'], seconds=spec['seconds'],
                  seed=spec['seed'], two_pass=spec['two_pass'], base_steps=spec['base_steps'],
                  refine_steps=spec['refine_steps'], force_regenerate=spec['force_regenerate'],
                  preview=spec['preview'], refine_from=refine_from)
    if spec.get('first_frame_reference'):
        # Every part of the video carries it; the part that continues nothing ignores it.
        inputs['first_frame_reference'] = True
    if spec.get('step_cache'):
        inputs['step_cache'] = spec['step_cache']
    if spec.get('upscale_factor', 2) != 2 and spec['two_pass'] and not spec['preview']:
        inputs['upscale_factor'] = '%g' % spec['upscale_factor']
    graph = {node: {'class_type': 'FreeVideoGenerate', 'inputs': inputs,
                    '_meta': {'title': 'FreeVideo · ' + (spec.get('label') or 'Agent job')}}}
    if assets:
        graph[node + 'm'] = {'class_type': 'FreeVideoMedia', 'inputs': {'assets': json.dumps(assets)},
                             '_meta': {'title': 'FreeVideo · Media'}}
        inputs['media'] = [node + 'm', 0]
    if spec.get('loras'):
        graph[node + 'l'] = {'class_type': 'FreeVideoLoRAStack',
                             'inputs': {'adapters': json.dumps([dict(r, enabled=True) for r in spec['loras']
                                                               if r.get('enabled', True)])},
                             '_meta': {'title': 'FreeVideo · LoRAs'}}
        inputs['loras'] = [node + 'l', 0]
    return graph


def upscale_prompt(node, preview_video):
    """A second pass of a finished preview; the bridge restores the preview's own request."""
    inputs = dict(text='(continues the preview request)', width=1344, height=768, seconds=10., seed=0,
                  two_pass=True, base_steps=8, refine_steps=3, force_regenerate=False, preview=False,
                  refine_from=preview_video)
    return {node: {'class_type': 'FreeVideoGenerate', 'inputs': inputs, '_meta': {'title': 'FreeVideo · Second pass'}}}