Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
Add reference image attachments to agent chat
Bowen Xue38ea40d48a2d
diff --git a/freevideo_engine/agent/attachments.py b/freevideo_engine/agent/attachments.pynew file mode 100644index 0000000..2767b45--- /dev/null+++ b/freevideo_engine/agent/attachments.py@@ -0,0 +1,66 @@+"""Chat images in the retained input assets directory, with server-owned filenames."""+import json+import os+from pathlib import Path+import re+import threading+import uuid++from . import media, schema++FOLDER = 'freevideo/agent/assets'+_LOCK = threading.RLock()+++def path(input_root, name):+ if not isinstance(name, str) or not re.fullmatch(schema.CHAT_ATTACHMENT_NAME['pattern'], name):+ raise ValueError('Invalid attachment name')+ root = Path(input_root).resolve()+ target = root / FOLDER / name+ # Reject links, including links to another location inside the input directory.+ for part in (target, *target.parents):+ if part == root:+ break+ if part.is_symlink():+ raise ValueError('Attachment paths cannot contain links')+ if target.resolve() != target:+ raise ValueError('Attachment leaves assets')+ return target+++def _metadata(target):+ result = target.with_name('.' + target.name + '.json')+ if result.is_symlink():+ raise ValueError('Invalid attachment metadata')+ return result+++def publish(input_root, temporary, name, original_name):+ """Publish decoded bytes atomically; the first filename for identical content wins."""+ with _LOCK:+ target = path(input_root, name)+ target.parent.mkdir(parents=True, exist_ok=True)+ metadata = _metadata(target)+ if not metadata.exists():+ pending = metadata.with_name('.attachment-' + uuid.uuid4().hex + '.tmp')+ try:+ pending.write_text(json.dumps(dict(original_name=original_name), ensure_ascii=False), encoding='utf-8')+ os.replace(pending, metadata)+ finally:+ pending.unlink(missing_ok=True)+ os.replace(temporary, target)+ return json.loads(metadata.read_text(encoding='utf-8'))['original_name']+++def read(input_root, name):+ """Reinspect stored bytes; never use dimensions or filenames sent with a message."""+ with _LOCK:+ target = path(input_root, name)+ if not target.is_file() or target.stat().st_size > schema.MAX_CHAT_ATTACHMENT_BYTES:+ raise ValueError('No saved attachment within the upload limit')+ info, suffix = media.validate_upload(target)+ if suffix not in ('.png', '.jpg', '.webp') or info['kind'] != 'image':+ raise ValueError('Not a chat image')+ metadata = _metadata(target)+ original_name = json.loads(metadata.read_text(encoding='utf-8'))['original_name'] if metadata.exists() else name+ return dict(name=name, original_name=original_name, width=info['width'], height=info['height'])diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.pyindex 09ed9a1..2a4268d 100644--- a/freevideo_engine/agent/guide.py+++ b/freevideo_engine/agent/guide.py@@ -69,6 +69,7 @@ GUIDE = {'itself; read the project afterwards and check that one clip covers the whole film.',],'references': [+ "User message attachments are reference images by default. Use each name directly in media.references, media.first or media.last according to the user's words; if the intended role is unclear, ask one question. Text can be empty when a message has attachments.",'Tags are numbered per kind in reference order: <Picture 1>, <Picture 2>, <Video 1>, <Audio 1>. ''For keyframes the first frame is <Picture 1> and the last frame the next picture.','Name subjects <Subject 1>, <Subject 2> in subject_definitions and reuse those labels in every section.',@@ -211,6 +212,7 @@ GUIDE = {# How an agent is expected to work once a user has asked for something. Only what this version can honour.RULES = [+ "User message attachments are reference images by default. Use each name directly in media.references, media.first or media.last according to the user's words; if the intended role is unclear, ask one question. Text can be empty when a message has attachments.",'Finish the job. When the user asks for a video, keep going until the finished video exists. Decide ordinary ''things yourself: how to split shots, how many takes, which take to keep. Tell the user what you decided in ''one or two sentences at the end, not along the way. Name every video you submit: label is a short name of what '@@ -259,6 +261,7 @@ RULES = [RULES_ZH = [+ '用户消息里的 attachments 默认当参考图。按用户的话把每张图的 name 直接用在 media.references、media.first 或 media.last;拿不准用途时问一句。有附件的消息 text 可以为空串。','把任务做完。用户要求生成视频时,持续推进,直到成片生成。如何拆分镜头、拍几条、选哪一条等普通决定,''由你自己做。最后用一两句话告诉用户你做了哪些决定,不要每做一步都请用户决定。''提交的每条视频都起个名字:label 写拍的是什么,用用户的语言,2 到 8 个字,例如“清晨海边”。',diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.pyindex 3be38e2..450dc54 100644--- a/freevideo_engine/agent/routes.py+++ b/freevideo_engine/agent/routes.py@@ -22,7 +22,7 @@ import timeimport uuidfrom . import API_VERSION, enabled-from . import assets+from . import assets, attachmentsfrom . import private_filefrom . import spec as specsfrom .. import tiers as quality_tiers@@ -433,6 +433,7 @@ def register():connection = dict(connected=False, client=None, via=None, last_seen_at=None)# The conversation in the Autopilot view: the page and the connected agent talk in the agent's latest session.chat = dict(client=None, waiting=0, folds={}, told={})+ chat_writes = asyncio.Lock()server._freevideo_agent_connection = connectiondef problem(code, message=None, status=None, **details):@@ -473,7 +474,12 @@ def register():valid = definition.get('lan') and proxy_identity.check_csrf(request)else:valid = valid_page_session(request)- if not valid:+ elif definition.get('page_cookie'):+ session = page_sessions.get(request.cookies.get('freevideo_agent_page', ''))+ valid = session is not None and session[1] >= time.monotonic()+ else:+ valid = True+ if not valid:raise JobError('page_session_required', 'Page session required')header = request.headers.get('Authorization', '')given = header[7:].strip() if header[:7].lower() == 'bearer ' else request.headers.get('X-FreeVideo-Agent', '')@@ -522,7 +528,7 @@ def register():return await handler(request)except specs.SpecError as error:value = errors.validation(error.errors, getattr(error, 'code', 'invalid_request'))- return reply({'error': value}, errors.ERRORS[value['code']][0])+ return reply({'error': value}, getattr(error, 'status', errors.ERRORS[value['code']][0]))except JobError as error:value = error.view()if value.get('next') and request.match_info.get('id'):@@ -888,7 +894,8 @@ def register():fold['seq'] = row['seq']base = dict(seq=row['seq'], at=row['at'], type=row['type'])if row['type'] == 'message':- fold['items'].append(dict(base, role='user' if row['by'] == 'user' else 'agent', text=row['text']))+ fold['items'].append(dict(base, role='user' if row['by'] == 'user' else 'agent', text=row['text'],+ **({'attachments': row['attachments']} if row.get('attachments') else {})))if row['by'] == 'user':fold['said'] = row['text']if row.get('mid'):@@ -1017,6 +1024,75 @@ def register():active=sum(1 for r in rows if r['status'] in ACTIVE))return reply(await asyncio.to_thread(read))+ def chat_not_connected():+ return reply({'error': dict(code='not_connected', message='No agent is connected.', retryable=True,+ error_zh='尚未连接 Agent', error_en='No agent is connected.',+ details=[], next='Connect an agent in the Autopilot view, then say it again.',+ request_id=errors.request_id())}, 409)++ @routes.post('/freevideo/agent/chat/attach')+ @guarded+ async def chat_attach(request):+ from . import media+ if work_sessions() is None or chat['client'] is None:+ return chat_not_connected()+ limit = contract.MAX_CHAT_ATTACHMENT_BYTES+ if request.content_length is not None and request.content_length > limit:+ raise JobError('too_large', 'Upload exceeds limit', limit_bytes=limit)+ with tempfile.TemporaryDirectory(prefix='.chat-attach-', dir=port.input_root) as temporary:+ # The suffix selects the image decoder; Pillow recognizes the actual content format.+ target = Path(temporary) / 'image.png'+ size, digest = 0, hashlib.sha256()+ with target.open('xb') as output:+ while True:+ chunk = await request.content.read(min(256 * 1024, limit - size + 1))+ if not chunk:+ break+ size += len(chunk)+ if size > limit:+ raise JobError('too_large', 'Upload exceeds limit', limit_bytes=limit)+ output.write(chunk)+ digest.update(chunk)+ async with upload_decoders:+ try:+ info, suffix = await asyncio.to_thread(media.validate_upload, target)+ if suffix not in ('.png', '.jpg', '.webp'):+ raise ValueError('Only PNG, JPEG and WebP chat images are supported')+ except (OSError, ValueError, SyntaxError):+ raise JobError('invalid_media', 'Image cannot be decoded within the upload limits') from None+ name = digest.hexdigest() + suffix+ original = await asyncio.to_thread(attachments.publish, port.input_root, target, name, request[query_key]['name'])+ return reply(dict(name=name, original_name=original, width=info['width'], height=info['height'], bytes=size), 201)++ @routes.get('/freevideo/agent/chat/attachment')+ @guarded+ async def chat_attachment(request):+ name = request[query_key]['name']+ try:+ path = attachments.path(port.input_root, name)+ async with upload_decoders:+ await asyncio.to_thread(attachments.read, port.input_root, name)+ except (OSError, ValueError, SyntaxError):+ raise JobError('not_found', 'No saved chat image') from None+ kind = {'.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.webp': 'image/webp'}[path.suffix]+ return web.FileResponse(path, headers={'Content-Type': kind, 'Cache-Control': 'private, no-store',+ 'X-Content-Type-Options': 'nosniff'})++ def chat_attachments(values):+ result, seen = [], set()+ for i, item in enumerate(values):+ field = '/attachments/%d/name' % i+ if item['name'] in seen:+ raise specs.SpecError(contract.diagnostic(field, 'a distinct attachment name', item['name']))+ seen.add(item['name'])+ try:+ result.append(attachments.read(port.input_root, item['name']))+ except (OSError, ValueError, SyntaxError):+ error = specs.SpecError(contract.diagnostic(field, 'an existing uploaded image', item['name']))+ error.status = 400+ raise error from None+ return result+@routes.post('/freevideo/agent/chat')@guardedasync def chat_say(request):@@ -1026,17 +1102,22 @@ def register():if store is None or client is None:return Nonebefore = store.current_for(client)- sid = chat_session(store, value.get('session')) or store.create(client=client, by='user')['id']- fold = chat_fold(store, sid)- if value['id'] in fold['sent']:+ sid = chat_session(store, value.get('session'))+ fold = chat_fold(store, sid) if sid else None+ if fold is not None and value['id'] in fold['sent']:seq = fold['sent'][value['id']]first = next(i for i in fold['items'] if i['seq'] == seq)return 200, dict(seq=seq, id=value['id'], at=first['at'], session=sid,read=seq <= store.read_until(sid, client), agent=agent_state())+ pictures = chat_attachments(value.get('attachments', []))+ sid = sid or store.create(client=client, by='user')['id']+ fold = fold or chat_fold(store, sid)if store.stopped(sid):# Speaking again continues a conversation the user stopped earlier.store.append(sid, dict(type='control', action='resumed'), by='user')message = dict(type='message', text=value['text'], mid=value['id'])+ if pictures:+ message['attachments'] = picturesif value.get('whitebox'):message['whitebox'] = Truerow = store.append(sid, message, by='user')@@ -1046,13 +1127,11 @@ def register():fold['ids'].pop(fold['sent'].pop(next(iter(fold['sent']))), None)chat_moved(store, before)return 201, dict(seq=row['seq'], id=value['id'], at=row['at'], session=sid, read=False, agent=agent_state())- done = await asyncio.to_thread(write)+ async with chat_writes, upload_decoders:+ done = await asyncio.to_thread(write)if done is None:# Nothing is kept: no agent has connected since FreeVideo started, so nobody would read it.- return reply({'error': dict(code='not_connected', message='No agent is connected.', retryable=True,- error_zh='尚未连接 Agent', error_en='No agent is connected.',- details=[], next='Connect an agent in the Autopilot view, then say it again.',- request_id=errors.request_id())}, 409)+ return chat_not_connected()return reply(done[1], done[0])@routes.post('/freevideo/agent/chat/abandon')@@ -1520,6 +1599,8 @@ def register():value['text'] = chat_text(value['text'])client = session_client(request, by)def write():+ if value.get('attachments'):+ value['attachments'] = chat_attachments(value['attachments'])row = sessions.append(sid, value, by=by, client=client)here = sessions.current_for(client) if client and value['type'] == 'message' else Noneif here not in (None, sid):diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.pyindex 2ee30dc..ad76ce1 100644--- a/freevideo_engine/agent/schema.py+++ b/freevideo_engine/agent/schema.py@@ -24,6 +24,8 @@ IMAGE_ASPECTS = {'1:1': (1024, 1024), '16:9': (1344, 768), '9:16': (768, 1344),MAX_UPLOAD_BYTES = 2 * 1024 ** 3MAX_JSON_UPLOAD_BYTES = 96 * 1024 ** 2MAX_BODY_BYTES = 1024 * 1024+MAX_CHAT_ATTACHMENT_BYTES = 10 * 1024 ** 2+MAX_CHAT_ATTACHMENTS = 4def obj(properties=None, required=(), **kw):@@ -70,6 +72,22 @@ CLIENT_WAIT = number(0, 25, default=25)# Waiting for the user's words is longer: a person types slowly, and each wait is one tool call.INBOX_WAIT = number(0, default=0, description='x-clamp caps accepted wait values at 50 seconds without rejecting larger values.', **{'x-clamp': 50})CHAT_ID = string(64, pattern=r'^[A-Za-z0-9_-]{1,64}$')+CHAT_ATTACHMENT_NAME = string(69, pattern=r'^[0-9a-f]{64}\.(png|jpg|jpeg|webp)$')+CHAT_ATTACHMENT = obj(dict(name=CHAT_ATTACHMENT_NAME, original_name=string(255),+ width=number(1, integer=True), height=number(1, integer=True)),+ ('name', 'original_name', 'width', 'height'))+++def chat_request():+ # Upload metadata supplied by the page is ignored; only the stored name is authoritative.+ attachment = obj(dict(name=CHAT_ATTACHMENT_NAME, original_name={}, width={}, height={}, bytes={}), ('name',))+ return obj(dict(text=dict(type='string', minLength=0, maxLength=8000), id=CHAT_ID,+ session=SESSION, whitebox={'type': 'boolean'},+ attachments=array(attachment, 0, MAX_CHAT_ATTACHMENTS)), ('text', 'id'),+ **{'if': {'required': ['attachments'], 'properties': {'attachments': {'minItems': 1}}},+ 'else': {'properties': {'text': string(8000)}}})++EMPTY = obj()@@ -447,7 +465,7 @@ def contracts(*, local=False):('/connect/reset', 'POST', EMPTY), ('/chat', 'GET', None), ('/chat/now', 'GET', None),('/chat/video', 'GET', None), ('/chat/list', 'GET', None), ('/chat/new', 'POST', EMPTY),('/chat/delete', 'POST', obj({'session': SESSION, 'stop': {'type': 'boolean'}}, ('session',))),- ('/chat', 'POST', obj({'text': string(8000), 'id': CHAT_ID, 'session': SESSION, 'whitebox': {'type': 'boolean'}}, ('text', 'id'))),+ ('/chat', 'POST', chat_request()),('/chat/abandon', 'POST', obj({'session': SESSION})),('/chat/confirm', 'POST', obj({'id': string(12, pattern=r'^cf[0-9a-f]{10}$'), 'approve': {'type': 'boolean'},'session': SESSION}, ('id', 'approve')))):@@ -461,6 +479,13 @@ def contracts(*, local=False):query=obj({'client_id': string()}) if suffix != '/freevideo/progress' else EMPTY,**({'path': obj({'id': string()}, ('id',))} if '{id}' in suffix else {}))routes['GET', '/freevideo/agent/activity']['query'] = obj({'cursor': string(80)})+ routes['POST', '/freevideo/agent/chat/attach'] = route(+ 'Upload one chat reference image as raw bytes (at most 10 MiB); requires a page cookie and CSRF.',+ query=obj({'name': string(255, pattern=r'^[^/\\:\x00-\x1f]+$')}, ('name',)),+ auth=False, local=True, raw_body=True, status=201)+ routes['GET', '/freevideo/agent/chat/attachment'] = route(+ 'Read a saved chat image using the page cookie.', query=obj({'name': CHAT_ATTACHMENT_NAME}, ('name',)),+ auth=False, local=True, page_cookie=True)routes['GET', '/freevideo/agent/chat']['query'] = obj({'after': number(0, integer=True), 'session': SESSION})routes['GET', '/freevideo/agent/chat/video']['query'] = obj({'id': string(43, pattern=r'^[0-9]{4}-[0-9]{2}-[0-9]{2}/[0-9a-f]{32}$')}, ('id',))for method, suffix, summary, extra in (@@ -806,7 +831,7 @@ SESSION_OPERATIONS = {'create': ('POST', '/sessions', 'Create a work conversation for messages and plans shared with the user.'),'list': ('GET', '/sessions', 'List work conversations, most recently updated first.'),'get': ('GET', '/sessions/{id}', 'Read conversation metadata and whether the user stopped it.'),- 'events': ('GET', '/sessions/{id}/events', 'Read events after a sequence number; wait up to 25 seconds or request an SSE stream.'),+ 'events': ('GET', '/sessions/{id}/events', 'Read events, including user image attachments (name, original_name, width, height); text may be empty with attachments. Wait up to 25 seconds or request an SSE stream.'),'post': ('POST', '/sessions/{id}/events', 'Append a message or plan for the user. Steps are recorded by the server.'),'stop': ('POST', '/sessions/{id}/stop', 'Only the page user may stop a work conversation.'),'resume': ('POST', '/sessions/{id}/resume', 'Only the page user may continue a work conversation.'),@@ -848,6 +873,7 @@ def session_event(*, stored=False, agent_only=False):1, 20000), total_seconds=number(0)), ('type', 'shots'))]if not agent_only:variants[0]['properties']['mid'] = CHAT_ID+ variants[0]['properties']['attachments'] = array(CHAT_ATTACHMENT, 1, MAX_CHAT_ATTACHMENTS)variants[0]['properties']['whitebox'] = {'type': 'boolean', 'description': 'The user chose a white-model preview for this request.'}from .errors import ERRORSvariants += [obj(dict(type={'const': 'step'}, id=string(12, pattern=r'^st[0-9a-f]{10}$'),@@ -1354,6 +1380,11 @@ def openapi():**{str(status): {'description': 'Agent error', 'content': {'application/json': {'schema': {'type': 'object', 'properties': {'error': {'$ref': '#/components/schemas/Error'}}}}}}for status in sorted({r[0] for r in ERRORS.values() if r[0] >= 400})}})+ if contract.get('auth') is False and (method != 'GET' or contract.get('page_cookie')):+ operation['security'] = [{'page': []}]+ if contract.get('raw_body'):+ operation['requestBody'] = dict(required=True, content={'application/octet-stream': {'schema': {+ 'type': 'string', 'format': 'binary', 'maxLength': MAX_CHAT_ATTACHMENT_BYTES}}})if contract.get('page_or_token'):operation['security'] = [{'page': []}] if suffix.endswith(('/stop', '/resume')) else [{'bearer': []}, {'page': []}]if contract['body'] is not None:@@ -1370,6 +1401,9 @@ def openapi():for k in re.findall(r'\{(.*?)\}', path)]if contract.get('idempotent'):operation['parameters'].append(dict(name='Idempotency-Key', **{'in': 'header'}, required=False, schema=REF))+ if suffix == '/freevideo/agent/chat/attach':+ operation['parameters'].append(dict(name='X-FreeVideo-CSRF', **{'in': 'header'}, required=True,+ schema=string(128), description='The page session CSRF token.'))if contract.get('page_or_token'):operation['parameters'].append(dict(name='X-FreeVideo-CSRF', **{'in': 'header'}, required=False,schema=string(128), description='Required with the freevideo_agent_page cookie, including reads.'))@@ -1405,6 +1439,11 @@ def openapi():elif suffix == '/imports':response_schema = obj({'import': import_view()}, ('import',)) if method == 'POST' else obj({'items': array(import_view())}, ('items',))+ elif suffix == '/freevideo/agent/chat/attach':+ response_schema = obj(dict(CHAT_ATTACHMENT['properties'], bytes=number(1, MAX_CHAT_ATTACHMENT_BYTES, integer=True)),+ (*CHAT_ATTACHMENT['required'], 'bytes'))+ elif suffix == '/freevideo/agent/chat/attachment':+ content_type, response_schema = 'image/png', {'type': 'string', 'format': 'binary'}elif suffix == '/editor/sources/content':content_type, response_schema = 'application/octet-stream', {'type': 'string', 'format': 'binary'}elif suffix.endswith('/file') or suffix.endswith('/thumbnail'):@@ -1412,6 +1451,8 @@ def openapi():response_schema = {'type': 'string', 'format': 'binary'}success = operation['responses'][str(contract.get('status', 200))]success['content'] = {content_type: {'schema': response_schema}}+ if suffix == '/freevideo/agent/chat/attachment':+ success['content'].update({kind: {'schema': response_schema} for kind in ('image/jpeg', 'image/webp')})if suffix == '/sessions/{id}/events' and method == 'GET':success['content']['text/event-stream'] = {'schema': {'type': 'string'}}if contract.get('idempotent'):diff --git a/freevideo_engine/agent/spec.py b/freevideo_engine/agent/spec.pyindex 72d221f..76c7bf8 100644--- a/freevideo_engine/agent/spec.py+++ b/freevideo_engine/agent/spec.py@@ -93,6 +93,8 @@ def media_item(value, path, *, image=False, frame_default=None, early=False):errors = contract.issues(contract.media(image, local=True, early=early), value, path)if errors:raise SpecError(errors)+ if 'file' in value and re.fullmatch(contract.CHAT_ATTACHMENT_NAME['pattern'], value['file']):+ value['file'] = 'freevideo/agent/assets/' + value['file']return value