Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Add reference image attachments to agent chat

Bowen Xue38ea40d48a2d
diff --git a/freevideo_engine/agent/attachments.py b/freevideo_engine/agent/attachments.py
new file mode 100644
index 0000000..2767b45
--- /dev/null
+++ b/freevideo_engine/agent/attachments.py
@@ -0,0 +1,66 @@
+"""Chat images in the retained input assets directory, with server-owned filenames."""
+import json
+import os
+from pathlib import Path
+import re
+import threading
+import uuid
+
+from . import media, schema
+
+FOLDER = 'freevideo/agent/assets'
+_LOCK = threading.RLock()
+
+
+def path(input_root, name):
+ if not isinstance(name, str) or not re.fullmatch(schema.CHAT_ATTACHMENT_NAME['pattern'], name):
+ raise ValueError('Invalid attachment name')
+ root = Path(input_root).resolve()
+ target = root / FOLDER / name
+ # Reject links, including links to another location inside the input directory.
+ for part in (target, *target.parents):
+ if part == root:
+ break
+ if part.is_symlink():
+ raise ValueError('Attachment paths cannot contain links')
+ if target.resolve() != target:
+ raise ValueError('Attachment leaves assets')
+ return target
+
+
+def _metadata(target):
+ result = target.with_name('.' + target.name + '.json')
+ if result.is_symlink():
+ raise ValueError('Invalid attachment metadata')
+ return result
+
+
+def publish(input_root, temporary, name, original_name):
+ """Publish decoded bytes atomically; the first filename for identical content wins."""
+ with _LOCK:
+ target = path(input_root, name)
+ target.parent.mkdir(parents=True, exist_ok=True)
+ metadata = _metadata(target)
+ if not metadata.exists():
+ pending = metadata.with_name('.attachment-' + uuid.uuid4().hex + '.tmp')
+ try:
+ pending.write_text(json.dumps(dict(original_name=original_name), ensure_ascii=False), encoding='utf-8')
+ os.replace(pending, metadata)
+ finally:
+ pending.unlink(missing_ok=True)
+ os.replace(temporary, target)
+ return json.loads(metadata.read_text(encoding='utf-8'))['original_name']
+
+
+def read(input_root, name):
+ """Reinspect stored bytes; never use dimensions or filenames sent with a message."""
+ with _LOCK:
+ target = path(input_root, name)
+ if not target.is_file() or target.stat().st_size > schema.MAX_CHAT_ATTACHMENT_BYTES:
+ raise ValueError('No saved attachment within the upload limit')
+ info, suffix = media.validate_upload(target)
+ if suffix not in ('.png', '.jpg', '.webp') or info['kind'] != 'image':
+ raise ValueError('Not a chat image')
+ metadata = _metadata(target)
+ original_name = json.loads(metadata.read_text(encoding='utf-8'))['original_name'] if metadata.exists() else name
+ return dict(name=name, original_name=original_name, width=info['width'], height=info['height'])
diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py
index 09ed9a1..2a4268d 100644
--- a/freevideo_engine/agent/guide.py
+++ b/freevideo_engine/agent/guide.py
@@ -69,6 +69,7 @@ GUIDE = {
'itself; read the project afterwards and check that one clip covers the whole film.',
],
'references': [
+ "User message attachments are reference images by default. Use each name directly in media.references, media.first or media.last according to the user's words; if the intended role is unclear, ask one question. Text can be empty when a message has attachments.",
'Tags are numbered per kind in reference order: <Picture 1>, <Picture 2>, <Video 1>, <Audio 1>. '
'For keyframes the first frame is <Picture 1> and the last frame the next picture.',
'Name subjects <Subject 1>, <Subject 2> in subject_definitions and reuse those labels in every section.',
@@ -211,6 +212,7 @@ GUIDE = {
# How an agent is expected to work once a user has asked for something. Only what this version can honour.
RULES = [
+ "User message attachments are reference images by default. Use each name directly in media.references, media.first or media.last according to the user's words; if the intended role is unclear, ask one question. Text can be empty when a message has attachments.",
'Finish the job. When the user asks for a video, keep going until the finished video exists. Decide ordinary '
'things yourself: how to split shots, how many takes, which take to keep. Tell the user what you decided in '
'one or two sentences at the end, not along the way. Name every video you submit: label is a short name of what '
@@ -259,6 +261,7 @@ RULES = [
RULES_ZH = [
+ '用户消息里的 attachments 默认当参考图。按用户的话把每张图的 name 直接用在 media.references、media.first 或 media.last;拿不准用途时问一句。有附件的消息 text 可以为空串。',
'把任务做完。用户要求生成视频时,持续推进,直到成片生成。如何拆分镜头、拍几条、选哪一条等普通决定,'
'由你自己做。最后用一两句话告诉用户你做了哪些决定,不要每做一步都请用户决定。'
'提交的每条视频都起个名字:label 写拍的是什么,用用户的语言,2 到 8 个字,例如“清晨海边”。',
diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.py
index 3be38e2..450dc54 100644
--- a/freevideo_engine/agent/routes.py
+++ b/freevideo_engine/agent/routes.py
@@ -22,7 +22,7 @@ import time
import uuid
from . import API_VERSION, enabled
-from . import assets
+from . import assets, attachments
from . import private_file
from . import spec as specs
from .. import tiers as quality_tiers
@@ -433,6 +433,7 @@ def register():
connection = dict(connected=False, client=None, via=None, last_seen_at=None)
# The conversation in the Autopilot view: the page and the connected agent talk in the agent's latest session.
chat = dict(client=None, waiting=0, folds={}, told={})
+ chat_writes = asyncio.Lock()
server._freevideo_agent_connection = connection
def problem(code, message=None, status=None, **details):
@@ -473,7 +474,12 @@ def register():
valid = definition.get('lan') and proxy_identity.check_csrf(request)
else:
valid = valid_page_session(request)
- if not valid:
+ elif definition.get('page_cookie'):
+ session = page_sessions.get(request.cookies.get('freevideo_agent_page', ''))
+ valid = session is not None and session[1] >= time.monotonic()
+ else:
+ valid = True
+ if not valid:
raise JobError('page_session_required', 'Page session required')
header = request.headers.get('Authorization', '')
given = header[7:].strip() if header[:7].lower() == 'bearer ' else request.headers.get('X-FreeVideo-Agent', '')
@@ -522,7 +528,7 @@ def register():
return await handler(request)
except specs.SpecError as error:
value = errors.validation(error.errors, getattr(error, 'code', 'invalid_request'))
- return reply({'error': value}, errors.ERRORS[value['code']][0])
+ return reply({'error': value}, getattr(error, 'status', errors.ERRORS[value['code']][0]))
except JobError as error:
value = error.view()
if value.get('next') and request.match_info.get('id'):
@@ -888,7 +894,8 @@ def register():
fold['seq'] = row['seq']
base = dict(seq=row['seq'], at=row['at'], type=row['type'])
if row['type'] == 'message':
- fold['items'].append(dict(base, role='user' if row['by'] == 'user' else 'agent', text=row['text']))
+ fold['items'].append(dict(base, role='user' if row['by'] == 'user' else 'agent', text=row['text'],
+ **({'attachments': row['attachments']} if row.get('attachments') else {})))
if row['by'] == 'user':
fold['said'] = row['text']
if row.get('mid'):
@@ -1017,6 +1024,75 @@ def register():
active=sum(1 for r in rows if r['status'] in ACTIVE))
return reply(await asyncio.to_thread(read))
+ def chat_not_connected():
+ return reply({'error': dict(code='not_connected', message='No agent is connected.', retryable=True,
+ error_zh='尚未连接 Agent', error_en='No agent is connected.',
+ details=[], next='Connect an agent in the Autopilot view, then say it again.',
+ request_id=errors.request_id())}, 409)
+
+ @routes.post('/freevideo/agent/chat/attach')
+ @guarded
+ async def chat_attach(request):
+ from . import media
+ if work_sessions() is None or chat['client'] is None:
+ return chat_not_connected()
+ limit = contract.MAX_CHAT_ATTACHMENT_BYTES
+ if request.content_length is not None and request.content_length > limit:
+ raise JobError('too_large', 'Upload exceeds limit', limit_bytes=limit)
+ with tempfile.TemporaryDirectory(prefix='.chat-attach-', dir=port.input_root) as temporary:
+ # The suffix selects the image decoder; Pillow recognizes the actual content format.
+ target = Path(temporary) / 'image.png'
+ size, digest = 0, hashlib.sha256()
+ with target.open('xb') as output:
+ while True:
+ chunk = await request.content.read(min(256 * 1024, limit - size + 1))
+ if not chunk:
+ break
+ size += len(chunk)
+ if size > limit:
+ raise JobError('too_large', 'Upload exceeds limit', limit_bytes=limit)
+ output.write(chunk)
+ digest.update(chunk)
+ async with upload_decoders:
+ try:
+ info, suffix = await asyncio.to_thread(media.validate_upload, target)
+ if suffix not in ('.png', '.jpg', '.webp'):
+ raise ValueError('Only PNG, JPEG and WebP chat images are supported')
+ except (OSError, ValueError, SyntaxError):
+ raise JobError('invalid_media', 'Image cannot be decoded within the upload limits') from None
+ name = digest.hexdigest() + suffix
+ original = await asyncio.to_thread(attachments.publish, port.input_root, target, name, request[query_key]['name'])
+ return reply(dict(name=name, original_name=original, width=info['width'], height=info['height'], bytes=size), 201)
+
+ @routes.get('/freevideo/agent/chat/attachment')
+ @guarded
+ async def chat_attachment(request):
+ name = request[query_key]['name']
+ try:
+ path = attachments.path(port.input_root, name)
+ async with upload_decoders:
+ await asyncio.to_thread(attachments.read, port.input_root, name)
+ except (OSError, ValueError, SyntaxError):
+ raise JobError('not_found', 'No saved chat image') from None
+ kind = {'.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.webp': 'image/webp'}[path.suffix]
+ return web.FileResponse(path, headers={'Content-Type': kind, 'Cache-Control': 'private, no-store',
+ 'X-Content-Type-Options': 'nosniff'})
+
+ def chat_attachments(values):
+ result, seen = [], set()
+ for i, item in enumerate(values):
+ field = '/attachments/%d/name' % i
+ if item['name'] in seen:
+ raise specs.SpecError(contract.diagnostic(field, 'a distinct attachment name', item['name']))
+ seen.add(item['name'])
+ try:
+ result.append(attachments.read(port.input_root, item['name']))
+ except (OSError, ValueError, SyntaxError):
+ error = specs.SpecError(contract.diagnostic(field, 'an existing uploaded image', item['name']))
+ error.status = 400
+ raise error from None
+ return result
+
@routes.post('/freevideo/agent/chat')
@guarded
async def chat_say(request):
@@ -1026,17 +1102,22 @@ def register():
if store is None or client is None:
return None
before = store.current_for(client)
- sid = chat_session(store, value.get('session')) or store.create(client=client, by='user')['id']
- fold = chat_fold(store, sid)
- if value['id'] in fold['sent']:
+ sid = chat_session(store, value.get('session'))
+ fold = chat_fold(store, sid) if sid else None
+ if fold is not None and value['id'] in fold['sent']:
seq = fold['sent'][value['id']]
first = next(i for i in fold['items'] if i['seq'] == seq)
return 200, dict(seq=seq, id=value['id'], at=first['at'], session=sid,
read=seq <= store.read_until(sid, client), agent=agent_state())
+ pictures = chat_attachments(value.get('attachments', []))
+ sid = sid or store.create(client=client, by='user')['id']
+ fold = fold or chat_fold(store, sid)
if store.stopped(sid):
# Speaking again continues a conversation the user stopped earlier.
store.append(sid, dict(type='control', action='resumed'), by='user')
message = dict(type='message', text=value['text'], mid=value['id'])
+ if pictures:
+ message['attachments'] = pictures
if value.get('whitebox'):
message['whitebox'] = True
row = store.append(sid, message, by='user')
@@ -1046,13 +1127,11 @@ def register():
fold['ids'].pop(fold['sent'].pop(next(iter(fold['sent']))), None)
chat_moved(store, before)
return 201, dict(seq=row['seq'], id=value['id'], at=row['at'], session=sid, read=False, agent=agent_state())
- done = await asyncio.to_thread(write)
+ async with chat_writes, upload_decoders:
+ done = await asyncio.to_thread(write)
if done is None:
# Nothing is kept: no agent has connected since FreeVideo started, so nobody would read it.
- return reply({'error': dict(code='not_connected', message='No agent is connected.', retryable=True,
- error_zh='尚未连接 Agent', error_en='No agent is connected.',
- details=[], next='Connect an agent in the Autopilot view, then say it again.',
- request_id=errors.request_id())}, 409)
+ return chat_not_connected()
return reply(done[1], done[0])
@routes.post('/freevideo/agent/chat/abandon')
@@ -1520,6 +1599,8 @@ def register():
value['text'] = chat_text(value['text'])
client = session_client(request, by)
def write():
+ if value.get('attachments'):
+ value['attachments'] = chat_attachments(value['attachments'])
row = sessions.append(sid, value, by=by, client=client)
here = sessions.current_for(client) if client and value['type'] == 'message' else None
if here not in (None, sid):
diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py
index 2ee30dc..ad76ce1 100644
--- a/freevideo_engine/agent/schema.py
+++ b/freevideo_engine/agent/schema.py
@@ -24,6 +24,8 @@ IMAGE_ASPECTS = {'1:1': (1024, 1024), '16:9': (1344, 768), '9:16': (768, 1344),
MAX_UPLOAD_BYTES = 2 * 1024 ** 3
MAX_JSON_UPLOAD_BYTES = 96 * 1024 ** 2
MAX_BODY_BYTES = 1024 * 1024
+MAX_CHAT_ATTACHMENT_BYTES = 10 * 1024 ** 2
+MAX_CHAT_ATTACHMENTS = 4
def obj(properties=None, required=(), **kw):
@@ -70,6 +72,22 @@ CLIENT_WAIT = number(0, 25, default=25)
# Waiting for the user's words is longer: a person types slowly, and each wait is one tool call.
INBOX_WAIT = number(0, default=0, description='x-clamp caps accepted wait values at 50 seconds without rejecting larger values.', **{'x-clamp': 50})
CHAT_ID = string(64, pattern=r'^[A-Za-z0-9_-]{1,64}$')
+CHAT_ATTACHMENT_NAME = string(69, pattern=r'^[0-9a-f]{64}\.(png|jpg|jpeg|webp)$')
+CHAT_ATTACHMENT = obj(dict(name=CHAT_ATTACHMENT_NAME, original_name=string(255),
+ width=number(1, integer=True), height=number(1, integer=True)),
+ ('name', 'original_name', 'width', 'height'))
+
+
+def chat_request():
+ # Upload metadata supplied by the page is ignored; only the stored name is authoritative.
+ attachment = obj(dict(name=CHAT_ATTACHMENT_NAME, original_name={}, width={}, height={}, bytes={}), ('name',))
+ return obj(dict(text=dict(type='string', minLength=0, maxLength=8000), id=CHAT_ID,
+ session=SESSION, whitebox={'type': 'boolean'},
+ attachments=array(attachment, 0, MAX_CHAT_ATTACHMENTS)), ('text', 'id'),
+ **{'if': {'required': ['attachments'], 'properties': {'attachments': {'minItems': 1}}},
+ 'else': {'properties': {'text': string(8000)}}})
+
+
EMPTY = obj()
@@ -447,7 +465,7 @@ def contracts(*, local=False):
('/connect/reset', 'POST', EMPTY), ('/chat', 'GET', None), ('/chat/now', 'GET', None),
('/chat/video', 'GET', None), ('/chat/list', 'GET', None), ('/chat/new', 'POST', EMPTY),
('/chat/delete', 'POST', obj({'session': SESSION, 'stop': {'type': 'boolean'}}, ('session',))),
- ('/chat', 'POST', obj({'text': string(8000), 'id': CHAT_ID, 'session': SESSION, 'whitebox': {'type': 'boolean'}}, ('text', 'id'))),
+ ('/chat', 'POST', chat_request()),
('/chat/abandon', 'POST', obj({'session': SESSION})),
('/chat/confirm', 'POST', obj({'id': string(12, pattern=r'^cf[0-9a-f]{10}$'), 'approve': {'type': 'boolean'},
'session': SESSION}, ('id', 'approve')))):
@@ -461,6 +479,13 @@ def contracts(*, local=False):
query=obj({'client_id': string()}) if suffix != '/freevideo/progress' else EMPTY,
**({'path': obj({'id': string()}, ('id',))} if '{id}' in suffix else {}))
routes['GET', '/freevideo/agent/activity']['query'] = obj({'cursor': string(80)})
+ routes['POST', '/freevideo/agent/chat/attach'] = route(
+ 'Upload one chat reference image as raw bytes (at most 10 MiB); requires a page cookie and CSRF.',
+ query=obj({'name': string(255, pattern=r'^[^/\\:\x00-\x1f]+$')}, ('name',)),
+ auth=False, local=True, raw_body=True, status=201)
+ routes['GET', '/freevideo/agent/chat/attachment'] = route(
+ 'Read a saved chat image using the page cookie.', query=obj({'name': CHAT_ATTACHMENT_NAME}, ('name',)),
+ auth=False, local=True, page_cookie=True)
routes['GET', '/freevideo/agent/chat']['query'] = obj({'after': number(0, integer=True), 'session': SESSION})
routes['GET', '/freevideo/agent/chat/video']['query'] = obj({'id': string(43, pattern=r'^[0-9]{4}-[0-9]{2}-[0-9]{2}/[0-9a-f]{32}$')}, ('id',))
for method, suffix, summary, extra in (
@@ -806,7 +831,7 @@ SESSION_OPERATIONS = {
'create': ('POST', '/sessions', 'Create a work conversation for messages and plans shared with the user.'),
'list': ('GET', '/sessions', 'List work conversations, most recently updated first.'),
'get': ('GET', '/sessions/{id}', 'Read conversation metadata and whether the user stopped it.'),
- 'events': ('GET', '/sessions/{id}/events', 'Read events after a sequence number; wait up to 25 seconds or request an SSE stream.'),
+ 'events': ('GET', '/sessions/{id}/events', 'Read events, including user image attachments (name, original_name, width, height); text may be empty with attachments. Wait up to 25 seconds or request an SSE stream.'),
'post': ('POST', '/sessions/{id}/events', 'Append a message or plan for the user. Steps are recorded by the server.'),
'stop': ('POST', '/sessions/{id}/stop', 'Only the page user may stop a work conversation.'),
'resume': ('POST', '/sessions/{id}/resume', 'Only the page user may continue a work conversation.'),
@@ -848,6 +873,7 @@ def session_event(*, stored=False, agent_only=False):
1, 20000), total_seconds=number(0)), ('type', 'shots'))]
if not agent_only:
variants[0]['properties']['mid'] = CHAT_ID
+ variants[0]['properties']['attachments'] = array(CHAT_ATTACHMENT, 1, MAX_CHAT_ATTACHMENTS)
variants[0]['properties']['whitebox'] = {'type': 'boolean', 'description': 'The user chose a white-model preview for this request.'}
from .errors import ERRORS
variants += [obj(dict(type={'const': 'step'}, id=string(12, pattern=r'^st[0-9a-f]{10}$'),
@@ -1354,6 +1380,11 @@ def openapi():
**{str(status): {'description': 'Agent error', 'content': {'application/json': {'schema': {
'type': 'object', 'properties': {'error': {'$ref': '#/components/schemas/Error'}}}}}}
for status in sorted({r[0] for r in ERRORS.values() if r[0] >= 400})}})
+ if contract.get('auth') is False and (method != 'GET' or contract.get('page_cookie')):
+ operation['security'] = [{'page': []}]
+ if contract.get('raw_body'):
+ operation['requestBody'] = dict(required=True, content={'application/octet-stream': {'schema': {
+ 'type': 'string', 'format': 'binary', 'maxLength': MAX_CHAT_ATTACHMENT_BYTES}}})
if contract.get('page_or_token'):
operation['security'] = [{'page': []}] if suffix.endswith(('/stop', '/resume')) else [{'bearer': []}, {'page': []}]
if contract['body'] is not None:
@@ -1370,6 +1401,9 @@ def openapi():
for k in re.findall(r'\{(.*?)\}', path)]
if contract.get('idempotent'):
operation['parameters'].append(dict(name='Idempotency-Key', **{'in': 'header'}, required=False, schema=REF))
+ if suffix == '/freevideo/agent/chat/attach':
+ operation['parameters'].append(dict(name='X-FreeVideo-CSRF', **{'in': 'header'}, required=True,
+ schema=string(128), description='The page session CSRF token.'))
if contract.get('page_or_token'):
operation['parameters'].append(dict(name='X-FreeVideo-CSRF', **{'in': 'header'}, required=False,
schema=string(128), description='Required with the freevideo_agent_page cookie, including reads.'))
@@ -1405,6 +1439,11 @@ def openapi():
elif suffix == '/imports':
response_schema = obj({'import': import_view()}, ('import',)) if method == 'POST' else obj(
{'items': array(import_view())}, ('items',))
+ elif suffix == '/freevideo/agent/chat/attach':
+ response_schema = obj(dict(CHAT_ATTACHMENT['properties'], bytes=number(1, MAX_CHAT_ATTACHMENT_BYTES, integer=True)),
+ (*CHAT_ATTACHMENT['required'], 'bytes'))
+ elif suffix == '/freevideo/agent/chat/attachment':
+ content_type, response_schema = 'image/png', {'type': 'string', 'format': 'binary'}
elif suffix == '/editor/sources/content':
content_type, response_schema = 'application/octet-stream', {'type': 'string', 'format': 'binary'}
elif suffix.endswith('/file') or suffix.endswith('/thumbnail'):
@@ -1412,6 +1451,8 @@ def openapi():
response_schema = {'type': 'string', 'format': 'binary'}
success = operation['responses'][str(contract.get('status', 200))]
success['content'] = {content_type: {'schema': response_schema}}
+ if suffix == '/freevideo/agent/chat/attachment':
+ success['content'].update({kind: {'schema': response_schema} for kind in ('image/jpeg', 'image/webp')})
if suffix == '/sessions/{id}/events' and method == 'GET':
success['content']['text/event-stream'] = {'schema': {'type': 'string'}}
if contract.get('idempotent'):
diff --git a/freevideo_engine/agent/spec.py b/freevideo_engine/agent/spec.py
index 72d221f..76c7bf8 100644
--- a/freevideo_engine/agent/spec.py
+++ b/freevideo_engine/agent/spec.py
@@ -93,6 +93,8 @@ def media_item(value, path, *, image=False, frame_default=None, early=False):
errors = contract.issues(contract.media(image, local=True, early=early), value, path)
if errors:
raise SpecError(errors)
+ if 'file' in value and re.fullmatch(contract.CHAT_ATTACHMENT_NAME['pattern'], value['file']):
+ value['file'] = 'freevideo/agent/assets/' + value['file']
return value