Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Agent first-frame staging: scene reference, character cards, framing and facing, with a...

Bowen Xue89da9ea29efd
diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py
index 4bada09..09ed9a1 100644
--- a/freevideo_engine/agent/guide.py
+++ b/freevideo_engine/agent/guide.py
@@ -242,7 +242,9 @@ RULES = [
'Pictures. When the user wants a picture, or a first frame or reference picture that does not exist yet, make '
'it with freevideo_generate_image; to change a picture, pass it in images. The result is an input file: use '
'result.file as a first frame, as a reference, or in images again. When where people stand in the shot matters, '
- 'make the first frame with people (cast pictures and places) and generate the video from it.',
+ 'make the first frame with scene_image and people (cast card IDs or pictures, x, height, facing left/right, '
+ 'framing full/half), then generate the video from result.file. Use dimensions divisible by 32. '
+ 'This controls the scene, requested framing and approximate positions; face similarity is not guaranteed.',
'Cut by default. Wherever the film can change shot, make the new shot from the reference pictures it should '
'have (cast cards, scene pictures) instead of continuing the previous video. Continue a video only for a '
'long take that cannot be cut, an action that must stay unbroken, or sound that must stay unbroken, and '
diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py
index 827e42b..73da1fd 100644
--- a/freevideo_engine/agent/jobs.py
+++ b/freevideo_engine/agent/jobs.py
@@ -797,6 +797,7 @@ class Jobs:
item = substitute(item, names)
try:
normalized = specs.normalize(item, port=_Warmed(self.port))
+ self.resolve_image_cast(normalized)
except specs.SpecError as error:
problems.extend(schema.prefix_diagnostic(e, prefix) for e in error.errors)
continue
@@ -1853,6 +1854,30 @@ class Jobs:
return failure
# Local image generation ----------------------------------------------------------------------------
+ def resolve_image_cast(self, spec):
+ """Pin character card pictures before saving the job, including across restarts/replays."""
+ if spec['kind'] != 'image':
+ return
+ for i, person in enumerate(spec.get('people', [])):
+ identity = person.get('cast')
+ if identity is None:
+ continue
+ path = '/people/%d/cast' % i
+ try:
+ card = self.cast.get(identity)
+ except JobError as error:
+ if error.code != 'not_found':
+ raise
+ specs.fail(path, 'an existing character card ID from GET /freevideo/agent/v1/cast')
+ if card['kind'] != 'character':
+ specs.fail(path, 'a character card, rather than a setting card')
+ picture = next((row for row in card['images'] if row['label'] == 'front'), None)
+ if picture is None:
+ specs.fail(path, 'a character card with a front picture; add or generate one first')
+ person.pop('cast')
+ person['image'] = picture['media']['file']
+ person.setdefault('name', card['name'][:40])
+
def wait_image(self, row, code, *, waiting_for=None):
if code not in IMAGE_WAIT_ERRORS:
return False
diff --git a/freevideo_engine/agent/mcp.py b/freevideo_engine/agent/mcp.py
index e94b1f8..956f4e1 100644
--- a/freevideo_engine/agent/mcp.py
+++ b/freevideo_engine/agent/mcp.py
@@ -50,7 +50,7 @@ def tool_list():
dict(name='freevideo_schema', title='Job JSON Schemas', description='JSON Schemas of every job kind and of image edit recipes.', annotations=read),
dict(name='freevideo_plan', title='Plan a job', description="Validate a job without running it: actual frames/seconds, sampling passes, warnings and a time estimate from this computer's history. Same fields as freevideo_generate.", annotations=read),
dict(name='freevideo_generate', title='Generate a video', description='Generate one video from an H3 prompt, up to 60 seconds. Under 30 s defaults to one generation if the device supports this size and quality; unknown limits use 15 s. Otherwise the video is cut into as few parts as possible, each at least 10 s, and how many parts depends on how long this computer can generate in one run; the parts continue and join automatically. Prefer a new shot with its reference pictures wherever the film can cut; continue only a take, action or sound that must stay unbroken. Use this for any single video, also after freevideo_enhance_prompt; the drama tools are only for a film of several separately chosen shots. Optionally from first/last frames or references (files, local paths, or frames of earlier jobs). Returns the job; with wait, its result (video path and URL).', annotations=write),
- dict(name='freevideo_generate_image', title='Generate a picture', description='Generate a picture from a prompt; with images, change or combine input pictures; with people, make a first frame with cast members standing where you say. The result is an input file to use as a first frame, as a reference, or in images again.', annotations=write),
+ dict(name='freevideo_generate_image', title='Generate a picture', description='Generate a picture from a prompt; with images, change or combine input pictures; with people and scene_image, make a first frame with the scene and the people near the places you say. The result is an input file to use as a first frame, as a reference, or in images again.', annotations=write),
dict(name='freevideo_submit_batch', title='Submit a storyboard', description='Submit several jobs at once. Each row is a job object with "kind" (generate default, assemble, vlm, enhance, image) and an optional "ref"; later rows use "@ref" wherever a job id is expected (from_job, after, clips, refine_from). FreeVideo runs them in dependency order.', annotations=write),
dict(name='freevideo_job', title='Job status', description='Status, progress, result or error of a job.', annotations=read),
dict(name='freevideo_wait', title='Wait for a job or the user', description="Wait at most 25 seconds for a job (with id) or, without id, at most 50 seconds for what the user says in FreeVideo (inbox). again=true means nothing new yet: call it again. A job wait returns early when the user says something. If the job is still active, call it again with the same ID; do not resubmit.", annotations=read),
diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py
index eb07103..5055247 100644
--- a/freevideo_engine/agent/schema.py
+++ b/freevideo_engine/agent/schema.py
@@ -228,20 +228,28 @@ def vlm(*, local=False):
def image(*, local=False):
return obj(dict(kind={'const': 'image', 'type': 'string'},
- prompt=dict(type='string', minLength=1, maxLength=2000),
+ prompt=dict(type='string', minLength=1, maxLength=2000,
+ description='Scene description; required unless scene_image supplies the scene.'),
width=number(256, 2048, integer=True), height=number(256, 2048, integer=True),
aspect=enum(IMAGE_ASPECTS, description='Instead of width and height; 1:1 when all three are omitted.'),
seed=nullable(number(-2**63, 2**64 - 1, integer=True)),
images=array({'oneOf': [FILE, obj(dict(file=FILE), ('file',))]}, high=4, description='Pictures to change '
'or combine: input files, such as result.file of an earlier picture job. The prompt calls them image 1, image 2.'),
- people=array(obj(dict(image=FILE, name=string(40), x=number(0, 1), height=number(.05, 1), feet=number(0, 1)),
- ('image',)), 1, 3, description='Makes a first frame with people standing where you say. image is '
- "a full-body picture on a plain backdrop, such as a cast card's front picture. From 0 to 1: x is the person's "
- 'centre across the width (default 0.5), height their size (0.62) and feet where they stand down the height '
- '(0.9). The prompt describes the scene, or scene_image shows it. Not together with images.'),
+ people=array(obj(dict(image=FILE, cast=CAST, name=string(40), x=number(0, 1), height=number(.05, 1),
+ feet=number(0, 1), facing=enum(('left', 'right')),
+ framing=enum(('full', 'half'), default='full')),
+ oneOf=[{'required': ['image']}, {'required': ['cast']}]), 1, 3,
+ description='First frame with people near the places you say; faces may differ. Give image (an input '
+ 'file) or cast (a character card ID; its front picture is used). From 0 to 1: x is the '
+ "centre across the width (default 0.5), height the visible figure over the frame height (0.62; head to waist "
+ 'for half), feet the figure bottom from the top (default max(0.9, height)). facing is the screen left/right; '
+ 'framing is full or half. Width and height: multiples of 32. The prompt or scene_image gives the scene. Not with images.'),
scene_image=FILE,
**common()),
- ('kind', 'prompt'), allOf=[{'if': {'required': ['width']}, 'then': {'required': ['height']}},
+ ('kind',), allOf=[{'if': {'not': {'required': ['scene_image']}}, 'then': {'required': ['prompt']}},
+ {'if': {'required': ['people']}, 'then': {'properties': {
+ key: number(256, 2048, integer=True, multipleOf=32) for key in ('width', 'height')}}},
+ {'if': {'required': ['width']}, 'then': {'required': ['height']}},
{'if': {'required': ['height']}, 'then': {'required': ['width']}},
{'if': {'required': ['aspect']},
'then': {'not': {'anyOf': [{'required': ['width']}, {'required': ['height']}]}}}],
@@ -1180,6 +1188,9 @@ def cross_issues(schema, value, path):
bad('images', 'omitted when people is supplied', value['images'])
if value.get('scene_image') and not value.get('people'):
bad('scene_image', 'omitted unless people is supplied', value['scene_image'])
+ for i, person in enumerate(value.get('people', [])):
+ if 'feet' in person and person['feet'] < person.get('height', .62):
+ bad('people/%d/feet' % i, 'at least the figure height, so the figure fits inside the frame', person['feet'])
if kind == 'generate':
if value.get('continue_from') and isinstance(value.get('media'), dict) and any(
v is not None for k, v in value['media'].items() if k != 'references'):
diff --git a/freevideo_engine/agent/spec.py b/freevideo_engine/agent/spec.py
index a5d51c0..72d221f 100644
--- a/freevideo_engine/agent/spec.py
+++ b/freevideo_engine/agent/spec.py
@@ -269,13 +269,15 @@ def normalize(value, *, port=None):
elif kind == 'image':
width, height = ((value['width'], value['height']) if 'width' in value else
contract.IMAGE_ASPECTS[value.get('aspect', '1:1')])
- result.update(prompt=value['prompt'], width=width, height=height,
+ result.update(prompt=value.get('prompt', ''), width=width, height=height,
images=[item if isinstance(item, str) else item['file'] for item in value.get('images', [])])
if value.get('seed') is not None:
result['seed'] = value['seed']
if value.get('people'):
- result['people'] = [dict(image=person['image'], x=person.get('x', .5), height=person.get('height', .62),
- feet=person.get('feet', .9), **({'name': person['name']} if person.get('name') else {}))
+ result['people'] = [dict({k: person[k] for k in ('image', 'cast', 'name', 'facing') if k in person},
+ x=person.get('x', .5), height=person.get('height', .62),
+ framing=person.get('framing', 'full'),
+ feet=person.get('feet', max(.9, person.get('height', .62))))
for person in value['people']]
if value.get('scene_image'):
result['scene_image'] = value['scene_image']
diff --git a/freevideo_engine/image_gen/compose.py b/freevideo_engine/image_gen/compose.py
index 5272b6b..0dc0653 100644
--- a/freevideo_engine/image_gen/compose.py
+++ b/freevideo_engine/image_gen/compose.py
@@ -1,14 +1,13 @@
-"""First frames with placed people: a scene, some character pictures and where each one stands.
+"""First frames from a scene followed by character references and numeric positions.
-With a scene picture the people are cut out of their plain backdrops and pasted at the asked places;
-the result is a rough layout, and the image model renders it as one clean photograph (the layout is
-passed as a reference, so the places and sizes carry over). With only a scene description the same
-places are written into the prompt. Either way the answer is a normal generate request.
+The default uses a scene-first prompt (pos2), without writing a collage. It aims
+for the scene, requested framing and approximate positions; face similarity and
+exact coordinates are not guaranteed. Collage helpers remain available for experiments.
"""
from pathlib import Path
-import uuid
from PIL import Image
+from .staging import position_prompt, scene_prompt
MAX_PEOPLE = 3 # the model takes 4 pictures: the people (or none) plus the layout or the scene
TOLERANCE = 9
@@ -78,51 +77,39 @@ def _check(people):
raise ValueError('invalid_media')
x = person.get('x', .5)
tall = person.get('height', .62)
- feet = person.get('feet', .9)
+ feet = person.get('feet', max(.9, tall) if type(tall) in (int, float) else .9)
if any(type(v) not in (int, float) or not 0 <= v <= 1 for v in (x, tall, feet)) or tall < .05:
raise ValueError('invalid_request')
+ if feet < tall or person.get('facing', 'right') not in ('left', 'right') \
+ or person.get('framing', 'full') not in ('full', 'half'):
+ raise ValueError('invalid_request')
rows.append(dict(image=person['image'], x=float(x), height=float(tall), feet=float(feet),
- name=str(person.get('name') or '')[:40]))
+ name=str(person.get('name') or '')[:40],
+ framing=person.get('framing', 'full'),
+ **({'facing': person['facing']} if 'facing' in person else {})))
return rows
-def _where(x):
- return 'on the left' if x < .38 else 'on the right' if x > .62 else 'in the middle'
-
-
-def _positions(people):
- """Left-to-right sentence; ordering words place the people better than percentages do."""
- order = sorted(range(len(people)), key=lambda i: people[i]['x'])
- if len(people) == 1:
- return 'The person shown in image 1 stands %s of the frame, full body, feet on the floor.' % _where(people[0]['x'])
- words = ['on the left', 'in the middle', 'on the right'] if len(people) == 3 else ['on the left', 'on the right']
- rows = ['%s stands the person shown in image %d' % (word, i + 1) for word, i in zip(words, order)]
- rows[0] = rows[0][0].upper() + rows[0][1:]
- return '; '.join(rows) + '. Show everyone head to toe, standing apart with empty floor between them, with natural shadows.'
-
-
def build(value, input_directory):
"""A generate request for a first frame. value: width, height, people=[{image, x, height, feet, name}],
and the scene as scene_image (a file name) or scene (text). Pictures are file names in the input directory
- (or freevideo-image/<name>); the layout picture, if any, is written to freevideo-image/."""
+ (or freevideo-image/<name>). facing=left/right and framing=full/half are optional per person."""
if not isinstance(value, dict):
raise ValueError('invalid_request')
people = _check(value.get('people'))
width, height = value.get('width'), value.get('height')
- if any(type(v) is not int or not 256 <= v <= 2048 for v in (width, height)):
+ if any(type(v) is not int or not 256 <= v <= 2048 or v % 32 for v in (width, height)):
raise ValueError('invalid_dimensions')
- width, height = width // 32 * 32, height // 32 * 32
text = value.get('scene') if isinstance(value.get('scene'), str) else ''
directory = Path(input_directory).resolve()
def load(name):
folder, _, leaf = name.rpartition('/')
- if not leaf or folder not in ('', 'freevideo-image') or any(c in name for c in ('\\', ':', '\0')):
+ if not leaf or leaf in ('.', '..') or folder not in ('', 'freevideo-image') or any(c in name for c in ('\\', ':', '\0')):
raise ValueError('invalid_media')
path = (directory / folder / leaf) if folder else directory / leaf
- parent = directory / folder if folder else directory
- if (parent.is_symlink() or path.is_symlink() or not path.is_file()
- or path.resolve().parent != parent):
+ if path.parent.is_symlink() or path.is_symlink() or not path.is_file() \
+ or path.resolve().parent != (directory / folder if folder else directory):
raise ValueError('invalid_media')
try:
with Image.open(path) as image:
@@ -132,34 +119,22 @@ def build(value, input_directory):
names = [person['image'] for person in people]
scene_name = value.get('scene_image')
- intro = ' '.join('Image %d is the character %s.' % (i + 1, p['name'] or str(i + 1)) for i, p in enumerate(people))
if isinstance(scene_name, str) and scene_name:
- layout = collage(load(scene_name), [(load(p['image']), p['x'], p['height'], p['feet']) for p in people],
- width, height)
- folder = directory / 'freevideo-image'
- if folder.is_symlink() or folder.resolve().parent != directory:
- raise ValueError('invalid_media')
- leaf = 'layout-' + uuid.uuid4().hex + '.png'
- try:
- folder.mkdir(exist_ok=True)
- layout.save(folder / leaf)
- except OSError as error:
- # A runtime layout failure must reach the service's diagnostic-report
- # path, rather than being returned as a malformed request.
- raise ValueError('generation_failed') from error
- names.append('freevideo-image/' + leaf)
- n = len(people)
- prompt = ('%s Image %d is a rough layout with the characters pasted on the scene. Render image %d as one clean '
- 'photograph of the same place with the same people. Keep the scene, keep everyone at exactly the same '
- 'position and size, keep each face, hairstyle and clothes as in their own image. Remove every cut-out '
- 'edge and white outline. Relight everyone to the lighting of the room, ground them with soft contact '
- 'shadows on the floor and slight floor reflections so they stand in the space instead of floating.'
- % (intro, n + 1, n + 1))
+ names.insert(0, scene_name)
+ prompt = scene_prompt(people)
else:
if not text.strip():
raise ValueError('invalid_prompt')
- prompt = '%s %s Keep each face and outfit exactly as in their own image. %s' % (
- intro, _positions(people), 'The scene: ' + text.strip().rstrip('.') + '.')
+ first_image = 1
+ prompt = 'The scene: ' + text.strip().rstrip('.') + '. '
+ prompt += ' '.join('Image %d is a character reference%s.' %
+ (i, ' for ' + p['name'] if p['name'] else '') for i, p in enumerate(people, first_image))
+ prompt += ' ' + position_prompt(people, first_image=first_image)
+ prompt += (' Render one natural photograph with only these %d people, matching the scene lighting, '
+ 'with soft contact shadows and no cut-out edges. Use the character references for appearance and clothing.'
+ % len(people))
+ for name in names:
+ load(name)
if len(prompt) > 2000:
raise ValueError('invalid_prompt')
return dict(prompt=prompt, width=width, height=height, seed=value.get('seed'), images=names)
diff --git a/freevideo_engine/image_gen/staging.py b/freevideo_engine/image_gen/staging.py
new file mode 100644
index 0000000..b0fdca2
--- /dev/null
+++ b/freevideo_engine/image_gen/staging.py
@@ -0,0 +1,65 @@
+"""Pure coordinate-to-prompt mapping for first frames; no files or model calls.
+
+Keep this policy separate from compose's reference loading so later staging
+experiments can replace it without changing the Agent request/result contract.
+People are validated by compose before reaching this function.
+"""
+
+
+def scene_prompt(people):
+ """The measured pos2 wording, with scene first and integer percentages.
+
+ Equal feet and unequal full-body heights express the two-person depth cue
+ from pos2 shot 8. Reference order never changes to describe that depth.
+ Explicit framing/facing extensions follow the complete baseline prompt.
+ """
+ labels = [person.get('name', '').strip() or f'person {i}'
+ for i, person in enumerate(people, 1)]
+ sentences = [
+ 'Image 1 is an empty scene; the background of the result must be exactly this scene '
+ '(its walls, floor, windows and lighting), not the plain backdrop of the character photos.']
+ sentences.extend(f'Image {i} is the character {name}.' for i, name in enumerate(labels, 2))
+ positions = [f'{name} centred at {person["x"] * 100:.0f} percent of the image width, '
+ f'full body, about {person["height"] * 100:.0f} percent of the image height'
+ for name, person in zip(labels, people)]
+ feet = [f'{person["feet"] * 100:.0f}' for person in people]
+ if len(set(feet)) == 1:
+ positions.append(f'all feet at about {feet[0]} percent of the image height, whole body from head to shoes')
+ else:
+ positions.extend(f'{name} feet at about {foot} percent of the image height, whole body from head to shoes'
+ for name, foot in zip(labels, feet))
+ sentences.append('Place the characters in the scene: ' + '; '.join(positions) + '.')
+ heights = [round(person['height'] * 100) for person in people]
+ if len(people) == 2 and len(set(feet)) == 1 and heights[0] != heights[1] \
+ and all(person.get('framing', 'full') == 'full' for person in people):
+ near, far = (0, 1) if heights[0] > heights[1] else (1, 0)
+ sentences.append(f'{labels[near]} is closer to the camera and larger, '
+ f'{labels[far]} is farther away and smaller.')
+ sentences.append("Keep each face and the clothes as in the character's own image and match the lighting of the scene.")
+ for index, person in enumerate(people, 2):
+ if 'facing' in person or person.get('framing', 'full') != 'full':
+ sentences.append(position_prompt([person], first_image=index))
+ return ' '.join(sentences)
+
+
+def position_prompt(people, *, first_image=2):
+ """Describe positions in screen coordinates, preserving input/reference order.
+
+ Height means the visible figure's fraction of frame height. For a half-body
+ shot this is the head-to-waist figure, rather than an off-screen full body.
+ """
+ sentences = []
+ for index, person in enumerate(people, first_image):
+ x, height = person['x'], person['height']
+ side = 'on the screen left' if x < .5 else 'on the screen right' if x > .5 else 'in the screen centre'
+ full = person.get('framing', 'full') == 'full'
+ figure = 'full-body' if full else 'visible upper-body'
+ sentences.append(
+ f'The person in image {index} stands {side}, centred at x={x:g} '
+ f'({x * 100:g}% of frame width from the left edge), with {figure} height '
+ f'{height:g} ({height * 100:g}% of frame height), facing screen {person.get("facing", "right")}. '
+ + ('Show their full body from head to shoes, with both shoes inside the frame.' if full else
+ 'Show them from head to waist, a half-body view.'))
+ if 'feet' in person:
+ sentences.append(f'The bottom of this visible figure is at {person["feet"] * 100:g}% of frame height from the top.')
+ return ' '.join(sentences)