Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
Agent: subtitles edit operation and a listen job for automatic subtitles
Bowen Xue10a4e11e3adc
diff --git a/freevideo_engine/agent/comfy.py b/freevideo_engine/agent/comfy.pyindex 2fe945f..3fafea3 100644--- a/freevideo_engine/agent/comfy.py+++ b/freevideo_engine/agent/comfy.py@@ -169,6 +169,10 @@ class ComfyPort:"""The editor page's export queue (editor_export.register), shared so exports take turns."""return getattr(self.server, '_freevideo_editor_export', None)+ def editor_ears(self):+ """The auto-subtitle ears queue (subtitle_ears.register): time lines against the sound or write down speech."""+ return getattr(self.server, '_freevideo_subtitle_ears', None)+def queue_busy(self):running, pending = self.snapshot()return bool(running or pending)diff --git a/freevideo_engine/agent/editor_runtime.py b/freevideo_engine/agent/editor_runtime.pyindex bdb703d..6af6b6f 100644--- a/freevideo_engine/agent/editor_runtime.py+++ b/freevideo_engine/agent/editor_runtime.py@@ -103,7 +103,7 @@ globalThis.__call = text => {// before; the first rejection stops the batch and nothing is returned to save.globalThis.__batch = text => {const {project, operations} = JSON.parse(text), model = __load('editor_model'), actions = __load('editor_actions');- const by = {by: 'agent'}, results = [];+ const texts = __load('editor_text_model'), by = {by: 'agent'}, results = [];const ids = record => new Set(record.tracks.flatMap(track => track.clips.map(clip => clip.id)));let record;try { record = model.parse(project); }@@ -138,6 +138,8 @@ globalThis.__batch = text => {case 'describe': record = model.describe(record, actions.linkedIds(record, op.clip_ids), op.text, by); break;case 'set_ref': record = model.setRef(record, actions.linkedIds(record, op.clip_ids), op.ref); break;case 'use_version': record = model.restoreVersion(record, op.clip_id, op.index, by); break;+ // One undo on the page; rejects as subtitles / range / text / locked / overlap.+ case 'subtitles': record = texts.addSubtitles(record, op.cues, {replace: op.replace === true}); break;default: throw new model.EditError('invalid', 'unknown operation');}} catch (error) {diff --git a/freevideo_engine/agent/errors.py b/freevideo_engine/agent/errors.pyindex bbfad5e..b5b3fdb 100644--- a/freevideo_engine/agent/errors.py+++ b/freevideo_engine/agent/errors.py@@ -106,6 +106,10 @@ ERRORS.update({'image_changed': (409, False, 'The image changed since you read it.', 'Apply your operations to error.image and send its revision.'),'layer_locked': (409, False, 'An operation changes a layer the user locked.', 'Leave the locked layer unchanged; ask the user to unlock it if it must change.'),'export_failed': (409, True, 'The export did not finish.', 'Read the project, then submit the export again with a new ref.'),+ 'no_speech_model': (409, False, 'No speech model is installed on this machine.',+ 'Do not retry. Write the subtitle lines yourself with the subtitles edit, or tell the user the model is not installed.'),+ 'listen_failed': (409, True, 'Listening to the sound did not finish.',+ 'Read the project, then submit the listening job again with a new ref; if it fails again, write the lines yourself.'),'clip_source_unusable': (409, False, 'A clip\'s source is missing, or cannot be read as far as the clip uses it.','Read the project and check each clip\'s source: it must exist and play, and in + frames must stay ''within its real length. Correct the project, then export the new revision.'),diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.pyindex f46cc7e..85d2feb 100644--- a/freevideo_engine/agent/guide.py+++ b/freevideo_engine/agent/guide.py@@ -94,6 +94,7 @@ GUIDE = {'POST /projects/{id}/operations sends revision and 1-64 operations, using whole frames at 24 fps. The batch saves once or changes nothing.','On 409 project_changed, recompute edits from error.project and its current_revision, then send a new request. On clip_busy wait for generation to finish.','POST /projects/{id}/generate reserves pending parts for clip_ids or a start/frames range and fills them as the returned job finishes. POST /projects/{id}/export renders the revision you read; wait for its job.',+ 'Subtitles: POST /projects/{id}/listen with mode transcribe (writes down what is said) or align (times the lines you give) and the revision you read; wait for the job and read result.lines, each with start and end frames. Then add them with a subtitles operation of cues {start, frames, text}; replace true swaps the whole track. With no speech model the job fails with no_speech_model: write the lines yourself.','Reuse ref or Idempotency-Key only for the identical request. Source video/audio/thumbnail URLs require the same Agent token.',],'project_routes': [dict(operation=name, method=method, path=path, description=summary)@@ -265,6 +266,9 @@ CAPABILITIES_ZH = {'先读取 revision 和片段标识;所选部分会立即预留,返回的任务完成后填入结果',('POST', '/projects/{id}/export'): '将保存的某个版本导出为 MP4,存入已保存视频。''先读取 revision,等待返回的任务完成,再读取结果视频',+ ('POST', '/projects/{id}/listen'): '听剪辑工程的声音,得到字幕行及其时间:对给定字幕行定时,或直接转写。'+ '先读取 revision,等待返回的任务完成,再用 subtitles 编辑把字幕行加入时间线;'+ '没有安装语音模型时返回 no_speech_model,改为自己写字幕行',('POST', '/images'): '像图片编辑页面一样打开图片;提交 source,会话会保留源图片副本和图层文档',('GET', '/images/{id}'): '读取图片编辑会话的图层文档、选区和 revision;编辑前先读取 revision 和图层标识',('PUT', '/images/{id}'): '替换整个图片文档;先读取会话并提交其 revision。'diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.pyindex d507f7e..a7631a9 100644--- a/freevideo_engine/agent/jobs.py+++ b/freevideo_engine/agent/jobs.py@@ -395,10 +395,10 @@ class Jobs:records = list(latest.values())for row in records:if (isinstance(row, dict) and specs.JOB_ID.fullmatch(str(row.get('id')))- and row.get('kind') in ('generate', 'assemble', 'export', 'image_edit') + GPU_KINDS+ and row.get('kind') in ('generate', 'assemble', 'export', 'listen', 'image_edit') + GPU_KINDSand row.get('status') in ACTIVE + DONE and isinstance(row.get('spec'), dict)and type(row.get('created_at')) in (int, float) and math.isfinite(row['created_at'])):- if row.get('status') == 'running' and row['kind'] in ('assemble', 'export', 'image_edit') + GPU_KINDS:+ if row.get('status') == 'running' and row['kind'] in ('assemble', 'export', 'listen', 'image_edit') + GPU_KINDS:# This process restarted while the work ran in it; nothing continues it.self.finish(row, 'failed', error=dict(code='interrupted',message='ComfyUI restarted while this job ran; submit it again'), save=False)@@ -1120,6 +1120,9 @@ class Jobs:if row['kind'] == 'export':from .projects import run_exportresult = run_export(self.port, row)+ elif row['kind'] == 'listen':+ from .projects import run_listen+ result = run_listen(self.port, row)elif row['kind'] == 'image_edit':from .images import run_renderresult = run_render(self, row)diff --git a/freevideo_engine/agent/projects.py b/freevideo_engine/agent/projects.pyindex 4d6f5f3..348d5b8 100644--- a/freevideo_engine/agent/projects.py+++ b/freevideo_engine/agent/projects.py@@ -114,6 +114,7 @@ class Projects:def __init__(self, root, *, directory=None):self.root = root # The ComfyUI output folder, where the page keeps FreeVideo/.editorself.lock = threading.Lock()+ self.listen_lock = threading.Lock()self.receipts = {} # (client, ref) -> {digest, result}; in memory, bounded; the page save keeps the durable one.self.prepared = {} # job id -> the reserved project, until its chain id is written into it.@@ -341,6 +342,24 @@ class Projects:self._remember(ref, digest, result, client)return result+ def listen(self, identity, value, *, jobs, ref=None, client=None):+ """Queue a listening job (align or transcribe) for the revision the caller read; a stale revision starts nothing."""+ schema.validate(schema.project_schema('listen'), value)+ if (value['mode'] == 'align') != ('lines' in value):+ raise _invalid('/lines', 'lines for mode align, and none for transcribe', value.get('lines'))+ with self.listen_lock: # the same key sent twice at once queues one job+ digest, replay = self._receipt('projects:listen', dict(id=identity, body=value), ref, client)+ if replay is not None:+ return replay+ stored = self._load(identity)['project']+ if stored['revision'] != value['revision']:+ raise ProjectError('project_changed', project=stored)+ job = jobs.submit(dict(kind='listen', editor_project=identity,+ **{k: v for k, v in value.items() if k != 'ref'}), client=client)[0]+ result = dict(job=jobs.view(job), jobs=[jobs.view(job)])+ self._remember(ref, digest, result, client)+ return result+def _update(self, identity, change, *, tries=4):"""Read, change and save; a page save in between makes it read again. None when nothing changed."""for _ in range(tries):@@ -448,3 +467,49 @@ def run_export(port, row, *, poll=.25):relative = view['path']return dict(library_id=library_id(relative), video=library_view(port, relative),editor_project=spec['editor_project'], revision=spec['revision'])+++# Listen ----------------------------------------------------------------------------------------------+LISTEN_REASONS = {'no_engine': 'no_speech_model', 'language': 'no_speech_model', 'no_sound': 'invalid_request', 'too_long': 'invalid_request',+ 'source_missing': 'clip_source_unusable', 'unreadable': 'clip_source_unusable',+ 'disk_full': 'storage_unavailable'}+LISTEN_MESSAGES = {+ 'no_engine': 'No speech model is installed on this machine; write the subtitle lines yourself with the subtitles edit',+ 'no_sound': 'The timeline has no sound to listen to',+ 'too_long': 'The sound is too long for one listening job; give a shorter range'}+++def run_listen(port, row, *, poll=.25):+ """A listening job: the page's subtitle ears (align or transcribe) over one saved revision."""+ spec = row['spec']+ queue = port.editor_ears()+ if queue is None:+ raise JobError('not_ready', 'The subtitle service is not running')+ request = dict(kind=spec['mode'], id=spec['editor_project'], revision=spec['revision'], language=spec['language'])+ request.update({k: spec[k] for k in ('lines', 'range') if k in spec})+ try:+ view = queue.submit(request)+ except FileNotFoundError as error:+ raise JobError('not_found', 'The project does not exist') from error+ except (ValueError, TypeError) as error:+ raise JobError('invalid_request', 'This project cannot be listened to') from error+ except Exception as error:+ if getattr(error, 'reason', None) is None:+ raise+ raise JobError('busy', 'The subtitle queue is full; retry shortly') from error+ identity, cancelled = view['id'], False+ row['_phase'] = 'listening'+ while view['state'] == 'running':+ if row.get('_cancel') and not cancelled:+ view, cancelled = queue.cancel(identity), True+ continue+ row['_detail'] = json.dumps(dict(fraction=round(view['fraction'], 3)))+ time.sleep(poll)+ view = queue.get(identity)+ if view['state'] == 'cancelled':+ raise JobError('cancelled', 'Cancelled')+ if view['state'] != 'done':+ reason = view.get('reason') or 'failed'+ raise JobError(LISTEN_REASONS.get(reason, 'listen_failed'),+ LISTEN_MESSAGES.get(reason) or 'Listening failed (%s)' % reason)+ return dict(editor_project=spec['editor_project'], revision=spec['revision'], **view['result'])diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.pyindex 92d5a33..2638001 100644--- a/freevideo_engine/agent/routes.py+++ b/freevideo_engine/agent/routes.py@@ -154,7 +154,7 @@ def features(continuation, vlm):'queue.shared': True, 'queue.busy': True, 'chain.auto': bool(continuation),'enhance': bool(vlm.get('ready')), 'drama.film': True, 'images.edit': rules,'images.layers': rules, 'images.heal': rules, 'images.render': rules, 'projects.read': True,- **{'projects.' + name: rules for name in ('insert', 'split', 'move', 'trim', 'remove', 'export', 'generate')}}+ **{'projects.' + name: rules for name in ('insert', 'split', 'move', 'trim', 'remove', 'export', 'generate', 'listen')}}def capabilities(port, jobs):@@ -192,6 +192,7 @@ def capabilities(port, jobs):'vlm': 'Ask the local vision-language model about images or sampled video frames','enhance': 'Rewrite a rough idea into a full H3 prompt with the local VLM','export': 'Render a saved revision of an editing project to a video',+ 'listen': 'Listen to the sound of an editing project for subtitle lines and times','image_edit': 'Render a saved revision of an image session to a picture'},limits=dict(width=[256, 4096, 32], height=[256, 4096, 32], seconds=[1.625, 60 if continuation else BLOCK_SECONDS],seconds_tested=[4, 15],@@ -1633,7 +1634,7 @@ def register():raise JobError('idempotency_conflict', 'Conflicting keys')args = [] if operation == 'create' else [identity]extra = dict(client=request.headers.get('X-FreeVideo-Client', '')[:80] or None)- if operation in ('generate', 'export'):+ if operation in ('generate', 'export', 'listen'):extra['jobs'] = jobsif operation == 'generate':def capability(size):@@ -1643,9 +1644,9 @@ def register():return Noneextra['capability'] = capabilityresult = await asyncio.to_thread(getattr(projects, operation), *args, value, ref=key or ref, **extra)- if operation in ('generate', 'export'):+ if operation in ('generate', 'export', 'listen'):await jobs.tick()- return reply(result, 201 if operation in ('create', 'generate', 'export') else 200)+ return reply(result, 201 if operation in ('create', 'generate', 'export', 'listen') else 200)return guarded(handler)for operation, (method, suffix, _) in contract.PROJECT_OPERATIONS.items():diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.pyindex 518812e..38410e1 100644--- a/freevideo_engine/agent/schema.py+++ b/freevideo_engine/agent/schema.py@@ -209,6 +209,23 @@ def export(*, local=False):**{'x-job-kind': 'export'})+LISTEN_FIELDS = dict(+ mode=enum(('align', 'transcribe'), description='align: time the lines you give against the sound; transcribe: write down what is said.'),+ language=string(8, pattern=r'^(auto|[a-z]{2,3})$', default='auto', description='Spoken language, e.g. zh or en; auto detects it.'),+ lines=array(string(2000), 1, 1000, description='align only: the lines to time, in order.'),+ range=obj(dict(start=EDITOR_FRAME, frames=number(1, 10**7, integer=True)), ('start', 'frames'),+ description='Timeline frames to listen to; omit for the whole timeline.'))+++def listen(*, local=False):+ return obj(dict(kind={'const': 'listen', 'type': 'string'},+ editor_project=string(34, pattern=r'^e-[0-9a-f]{32}$', description='The editing project whose sound to listen to (GET /projects).'),+ revision=number(1, 2**53 - 1, integer=True, description='The saved revision you read; a later save fails the job with project_changed.'),+ **LISTEN_FIELDS, **common()), ('kind', 'editor_project', 'revision', 'mode'),+ description='Listen to the timeline sound with the local speech model. The result lists each line with its start and end frame; nothing is saved.',+ **{'x-job-kind': 'listen'})++def image_edit(*, local=False):return obj(dict(kind={'const': 'image_edit', 'type': 'string'},image=dict(IMAGE, description='The image session to render (POST /images).'),@@ -218,7 +235,7 @@ def image_edit(*, local=False):**{'x-job-kind': 'image_edit'})-KINDS = dict(generate=generate, assemble=assemble, vlm=vlm, enhance=enhance, export=export, image_edit=image_edit)+KINDS = dict(generate=generate, assemble=assemble, vlm=vlm, enhance=enhance, export=export, listen=listen, image_edit=image_edit)def job_schema(*, local=False, refs=False):@@ -320,7 +337,7 @@ def contracts(*, local=False):if method == 'GET':extra['query'] = shapeelse:- extra.update(body=shape, idempotent=True, **({'status': 201} if name in ('create', 'generate', 'export') else {}))+ extra.update(body=shape, idempotent=True, **({'status': 201} if name in ('create', 'generate', 'export', 'listen') else {}))routes[method, suffix] = route(summary, **extra)for name, (method, suffix, summary) in IMAGE_OPERATIONS.items():shape = image_schema(name)@@ -623,10 +640,12 @@ PROJECT_OPERATIONS = {'operations': ('POST', '/projects/{id}/operations', 'Apply 1-64 timeline edits atomically and save once. First get revision and clip IDs; on project_changed recompute from error.project, and on clip_busy wait for generation to finish.'),'generate': ('POST', '/projects/{id}/generate', 'Make new picture and sound for selected clips or a frame range, as the page Generate does. First get revision and clip IDs; the parts are reserved at once and fill in as the returned job finishes.'),'export': ('POST', '/projects/{id}/export', 'Render one saved revision to an MP4 in the saved videos. First get the revision; wait for the returned job, then read its result video.'),+ 'listen': ('POST', '/projects/{id}/listen', 'Listen to the timeline sound for subtitles: mode align times lines you give, transcribe writes down what is said. First get the revision; wait for the returned job, then add the lines with the subtitles edit. Nothing is saved; with no speech model for the language the call fails with no_speech_model, so write the lines by hand.'),}PROJECT_TOOLS = dict(list='freevideo_projects', get='freevideo_project_read', create='freevideo_project_create',write='freevideo_project_write', operations='freevideo_editor_operations',- generate='freevideo_project_generate', export='freevideo_export')+ generate='freevideo_project_generate', export='freevideo_export',+ listen='freevideo_project_listen')SESSION_OPERATIONS = {'create': ('POST', '/sessions', 'Create a work conversation for messages and plans shared with the user.'),@@ -727,6 +746,8 @@ def project_schema(name):settings=settings, ref=REF), ('revision', 'target'))if name == 'export':return obj(dict(revision=revision, ref=REF), ('revision',))+ if name == 'listen':+ return obj(dict(revision=revision, **LISTEN_FIELDS, ref=REF), ('revision', 'mode'))if name != 'operations':return EMPTYfields = {@@ -741,6 +762,9 @@ def project_schema(name):'describe': (dict(clip_ids=ids, text=dict(type='string', maxLength=8000)), ('clip_ids', 'text'), 'Set the description of selected clips. Use clip IDs from this revision and one paragraph of text.'),'set_ref': (dict(clip_ids=ids, ref=dict(type='string', maxLength=128, pattern=r'^[A-Za-z0-9_-]{0,128}$')), ('clip_ids', 'ref'), 'Set an outside shot reference on selected clips. Use clip IDs from this revision; an empty ref removes it.'),'use_version': (dict(clip_id=CLIP, index=number(0, 49, integer=True)), ('clip_id', 'index'), 'Restore a stored clip version. First get the clip ID and its versions array; index is zero based.'),+ 'subtitles': (dict(cues=array(obj(dict(start=EDITOR_FRAME, frames=number(1, 10**7, integer=True), text=dict(type='string', minLength=1, maxLength=400)),+ ('start', 'frames', 'text')), 1, 500), replace=dict(type='boolean')), ('cues',),+ 'Add subtitle lines as one undo step on the subtitle track (created if missing). Each cue is a start frame, a length in frames and one or two lines of text; cues may not overlap each other or lines already there. replace: true swaps the whole track.'),}operations = [obj(dict(op={'const': op}, **props), ('op', *required), description=description)for op, (props, required, description) in fields.items()]diff --git a/freevideo_engine/agent/spec.py b/freevideo_engine/agent/spec.pyindex d2f75c6..8a9608c 100644--- a/freevideo_engine/agent/spec.py+++ b/freevideo_engine/agent/spec.py@@ -213,6 +213,9 @@ def normalize(value):result.update({k: value[k] for k in ('format', 'max_tokens', 'system') if k in value})elif kind == 'export':result.update(editor_project=value['editor_project'], revision=value['revision'])+ elif kind == 'listen':+ result.update(editor_project=value['editor_project'], revision=value['revision'], mode=value['mode'],+ language=value.get('language', 'auto'), **{k: value[k] for k in ('lines', 'range') if k in value})elif kind == 'image_edit':result.update(image=value['image'], revision=value['revision'])else: