Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

6.8 KB
Raw Blame
"""First frames from a scene followed by character references and numeric positions.

The default uses a scene-first prompt (pos2), without writing a collage. It aims
for the scene, requested framing and approximate positions; face similarity and
exact coordinates are not guaranteed. Collage helpers remain available for experiments.
"""
from pathlib import Path

from PIL import Image
from .staging import position_prompt, scene_prompt

MAX_PEOPLE = 3  # the model takes 4 pictures: the people (or none) plus the layout or the scene
TOLERANCE = 9


def cut_out(picture, step=TOLERANCE):
    """The person on their plain backdrop as an RGBA picture.

    The backdrop is what the edge reaches by changing less than `step` between neighbouring pixels,
    so a vignette or gradient backdrop goes as well; it is found on a small copy and scaled back.
    """
    from collections import deque
    rgb = picture.convert('RGB')
    scale = min(1., 384 / max(rgb.size))
    small = rgb.resize((max(2, round(rgb.width * scale)), max(2, round(rgb.height * scale))), Image.BILINEAR)
    width, height = small.size
    pixels = list(small.getdata())
    gone = bytearray(width * height)
    queue = deque()
    for x in range(width):
        queue.extend((x, width * (height - 1) + x))
    for y in range(height):
        queue.extend((width * y, width * y + width - 1))
    for index in queue:
        gone[index] = 1
    while queue:
        index = queue.popleft()
        r, g, b = pixels[index]
        x = index % width
        for other in (index - 1 if x else -1, index + 1 if x < width - 1 else -1, index - width, index + width):
            if 0 <= other < len(pixels) and not gone[other]:
                r2, g2, b2 = pixels[other]
                if abs(r - r2) + abs(g - g2) + abs(b - b2) < step:
                    gone[other] = 1
                    queue.append(other)
    mask = Image.new('L', small.size)
    mask.putdata([0 if flag else 255 for flag in gone])
    mask = mask.resize(rgb.size, Image.BILINEAR).point(lambda v: 255 if v > 127 else 0)
    result = rgb.convert('RGBA')
    result.putalpha(mask)
    box = mask.getbbox()
    return result.crop(box) if box else result


def collage(scene, people, width, height):
    """The scene scaled to cover the canvas with each person pasted: people = [(picture, x, height, feet)] in 0-1."""
    canvas = scene.convert('RGB')
    scale = max(width / canvas.width, height / canvas.height)
    canvas = canvas.resize((max(width, round(canvas.width * scale)), max(height, round(canvas.height * scale))))
    left, top = (canvas.width - width) // 2, (canvas.height - height) // 2
    canvas = canvas.crop((left, top, left + width, top + height))
    for picture, x, tall, feet in people:
        person = cut_out(picture)
        size = max(1, round(tall * height))
        person = person.resize((max(1, round(person.width * size / person.height)), size))
        at = (round(x * width - person.width / 2), round(feet * height - person.height))
        canvas.paste(person, at, person)
    return canvas


def _check(people):
    if not isinstance(people, list) or not 1 <= len(people) <= MAX_PEOPLE:
        raise ValueError('invalid_request')
    rows = []
    for person in people:
        if not isinstance(person, dict) or not isinstance(person.get('image'), str):
            raise ValueError('invalid_media')
        x = person.get('x', .5)
        tall = person.get('height', .62)
        feet = person.get('feet', max(.9, tall) if type(tall) in (int, float) else .9)
        if any(type(v) not in (int, float) or not 0 <= v <= 1 for v in (x, tall, feet)) or tall < .05:
            raise ValueError('invalid_request')
        if feet < tall or person.get('facing', 'right') not in ('left', 'right') \
                or person.get('framing', 'full') not in ('full', 'half'):
            raise ValueError('invalid_request')
        rows.append(dict(image=person['image'], x=float(x), height=float(tall), feet=float(feet),
                         name=str(person.get('name') or '')[:40],
                         framing=person.get('framing', 'full'),
                         **({'facing': person['facing']} if 'facing' in person else {})))
    return rows


def build(value, input_directory):
    """A generate request for a first frame. value: width, height, people=[{image, x, height, feet, name}],
    and the scene as scene_image (a file name) or scene (text). Pictures are file names in the input directory
    (or freevideo-image/<name>). facing=left/right and framing=full/half are optional per person."""
    if not isinstance(value, dict):
        raise ValueError('invalid_request')
    people = _check(value.get('people'))
    width, height = value.get('width'), value.get('height')
    if any(type(v) is not int or not 256 <= v <= 2048 or v % 32 for v in (width, height)):
        raise ValueError('invalid_dimensions')
    text = value.get('scene') if isinstance(value.get('scene'), str) else ''
    directory = Path(input_directory).resolve()

    def load(name):
        folder, _, leaf = name.rpartition('/')
        if not leaf or leaf in ('.', '..') or folder not in ('', 'freevideo-image') or any(c in name for c in ('\\', ':', '\0')):
            raise ValueError('invalid_media')
        path = (directory / folder / leaf) if folder else directory / leaf
        if path.parent.is_symlink() or path.is_symlink() or not path.is_file() \
                or path.resolve().parent != (directory / folder if folder else directory):
            raise ValueError('invalid_media')
        try:
            with Image.open(path) as image:
                return image.convert('RGBA')
        except OSError:
            raise ValueError('invalid_media') from None

    names = [person['image'] for person in people]
    scene_name = value.get('scene_image')
    if isinstance(scene_name, str) and scene_name:
        names.insert(0, scene_name)
        prompt = scene_prompt(people)
    else:
        if not text.strip():
            raise ValueError('invalid_prompt')
        first_image = 1
        prompt = 'The scene: ' + text.strip().rstrip('.') + '. '
        prompt += ' '.join('Image %d is a character reference%s.' %
                          (i, ' for ' + p['name'] if p['name'] else '') for i, p in enumerate(people, first_image))
        prompt += ' ' + position_prompt(people, first_image=first_image)
        prompt += (' Render one natural photograph with only these %d people, matching the scene lighting, '
                   'with soft contact shadows and no cut-out edges. Use the character references for appearance and clothing.'
                   % len(people))
    for name in names:
        load(name)
    if len(prompt) > 2000:
        raise ValueError('invalid_prompt')
    return dict(prompt=prompt, width=width, height=height, seed=value.get('seed'), images=names,
                **({'loras': value['loras']} if 'loras' in value else {}))