Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Agent: subtitles edit operation and a listen job for automatic subtitles

Bowen Xue10a4e11e3adc
diff --git a/freevideo_engine/agent/comfy.py b/freevideo_engine/agent/comfy.py
index 2fe945f..3fafea3 100644
--- a/freevideo_engine/agent/comfy.py
+++ b/freevideo_engine/agent/comfy.py
@@ -169,6 +169,10 @@ class ComfyPort:
"""The editor page's export queue (editor_export.register), shared so exports take turns."""
return getattr(self.server, '_freevideo_editor_export', None)
+ def editor_ears(self):
+ """The auto-subtitle ears queue (subtitle_ears.register): time lines against the sound or write down speech."""
+ return getattr(self.server, '_freevideo_subtitle_ears', None)
+
def queue_busy(self):
running, pending = self.snapshot()
return bool(running or pending)
diff --git a/freevideo_engine/agent/editor_runtime.py b/freevideo_engine/agent/editor_runtime.py
index bdb703d..6af6b6f 100644
--- a/freevideo_engine/agent/editor_runtime.py
+++ b/freevideo_engine/agent/editor_runtime.py
@@ -103,7 +103,7 @@ globalThis.__call = text => {
// before; the first rejection stops the batch and nothing is returned to save.
globalThis.__batch = text => {
const {project, operations} = JSON.parse(text), model = __load('editor_model'), actions = __load('editor_actions');
- const by = {by: 'agent'}, results = [];
+ const texts = __load('editor_text_model'), by = {by: 'agent'}, results = [];
const ids = record => new Set(record.tracks.flatMap(track => track.clips.map(clip => clip.id)));
let record;
try { record = model.parse(project); }
@@ -138,6 +138,8 @@ globalThis.__batch = text => {
case 'describe': record = model.describe(record, actions.linkedIds(record, op.clip_ids), op.text, by); break;
case 'set_ref': record = model.setRef(record, actions.linkedIds(record, op.clip_ids), op.ref); break;
case 'use_version': record = model.restoreVersion(record, op.clip_id, op.index, by); break;
+ // One undo on the page; rejects as subtitles / range / text / locked / overlap.
+ case 'subtitles': record = texts.addSubtitles(record, op.cues, {replace: op.replace === true}); break;
default: throw new model.EditError('invalid', 'unknown operation');
}
} catch (error) {
diff --git a/freevideo_engine/agent/errors.py b/freevideo_engine/agent/errors.py
index bbfad5e..b5b3fdb 100644
--- a/freevideo_engine/agent/errors.py
+++ b/freevideo_engine/agent/errors.py
@@ -106,6 +106,10 @@ ERRORS.update({
'image_changed': (409, False, 'The image changed since you read it.', 'Apply your operations to error.image and send its revision.'),
'layer_locked': (409, False, 'An operation changes a layer the user locked.', 'Leave the locked layer unchanged; ask the user to unlock it if it must change.'),
'export_failed': (409, True, 'The export did not finish.', 'Read the project, then submit the export again with a new ref.'),
+ 'no_speech_model': (409, False, 'No speech model is installed on this machine.',
+ 'Do not retry. Write the subtitle lines yourself with the subtitles edit, or tell the user the model is not installed.'),
+ 'listen_failed': (409, True, 'Listening to the sound did not finish.',
+ 'Read the project, then submit the listening job again with a new ref; if it fails again, write the lines yourself.'),
'clip_source_unusable': (409, False, 'A clip\'s source is missing, or cannot be read as far as the clip uses it.',
'Read the project and check each clip\'s source: it must exist and play, and in + frames must stay '
'within its real length. Correct the project, then export the new revision.'),
diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py
index f46cc7e..85d2feb 100644
--- a/freevideo_engine/agent/guide.py
+++ b/freevideo_engine/agent/guide.py
@@ -94,6 +94,7 @@ GUIDE = {
'POST /projects/{id}/operations sends revision and 1-64 operations, using whole frames at 24 fps. The batch saves once or changes nothing.',
'On 409 project_changed, recompute edits from error.project and its current_revision, then send a new request. On clip_busy wait for generation to finish.',
'POST /projects/{id}/generate reserves pending parts for clip_ids or a start/frames range and fills them as the returned job finishes. POST /projects/{id}/export renders the revision you read; wait for its job.',
+ 'Subtitles: POST /projects/{id}/listen with mode transcribe (writes down what is said) or align (times the lines you give) and the revision you read; wait for the job and read result.lines, each with start and end frames. Then add them with a subtitles operation of cues {start, frames, text}; replace true swaps the whole track. With no speech model the job fails with no_speech_model: write the lines yourself.',
'Reuse ref or Idempotency-Key only for the identical request. Source video/audio/thumbnail URLs require the same Agent token.',
],
'project_routes': [dict(operation=name, method=method, path=path, description=summary)
@@ -265,6 +266,9 @@ CAPABILITIES_ZH = {
'先读取 revision 和片段标识;所选部分会立即预留,返回的任务完成后填入结果',
('POST', '/projects/{id}/export'): '将保存的某个版本导出为 MP4,存入已保存视频。'
'先读取 revision,等待返回的任务完成,再读取结果视频',
+ ('POST', '/projects/{id}/listen'): '听剪辑工程的声音,得到字幕行及其时间:对给定字幕行定时,或直接转写。'
+ '先读取 revision,等待返回的任务完成,再用 subtitles 编辑把字幕行加入时间线;'
+ '没有安装语音模型时返回 no_speech_model,改为自己写字幕行',
('POST', '/images'): '像图片编辑页面一样打开图片;提交 source,会话会保留源图片副本和图层文档',
('GET', '/images/{id}'): '读取图片编辑会话的图层文档、选区和 revision;编辑前先读取 revision 和图层标识',
('PUT', '/images/{id}'): '替换整个图片文档;先读取会话并提交其 revision。'
diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py
index d507f7e..a7631a9 100644
--- a/freevideo_engine/agent/jobs.py
+++ b/freevideo_engine/agent/jobs.py
@@ -395,10 +395,10 @@ class Jobs:
records = list(latest.values())
for row in records:
if (isinstance(row, dict) and specs.JOB_ID.fullmatch(str(row.get('id')))
- and row.get('kind') in ('generate', 'assemble', 'export', 'image_edit') + GPU_KINDS
+ and row.get('kind') in ('generate', 'assemble', 'export', 'listen', 'image_edit') + GPU_KINDS
and row.get('status') in ACTIVE + DONE and isinstance(row.get('spec'), dict)
and type(row.get('created_at')) in (int, float) and math.isfinite(row['created_at'])):
- if row.get('status') == 'running' and row['kind'] in ('assemble', 'export', 'image_edit') + GPU_KINDS:
+ if row.get('status') == 'running' and row['kind'] in ('assemble', 'export', 'listen', 'image_edit') + GPU_KINDS:
# This process restarted while the work ran in it; nothing continues it.
self.finish(row, 'failed', error=dict(code='interrupted',
message='ComfyUI restarted while this job ran; submit it again'), save=False)
@@ -1120,6 +1120,9 @@ class Jobs:
if row['kind'] == 'export':
from .projects import run_export
result = run_export(self.port, row)
+ elif row['kind'] == 'listen':
+ from .projects import run_listen
+ result = run_listen(self.port, row)
elif row['kind'] == 'image_edit':
from .images import run_render
result = run_render(self, row)
diff --git a/freevideo_engine/agent/projects.py b/freevideo_engine/agent/projects.py
index 4d6f5f3..348d5b8 100644
--- a/freevideo_engine/agent/projects.py
+++ b/freevideo_engine/agent/projects.py
@@ -114,6 +114,7 @@ class Projects:
def __init__(self, root, *, directory=None):
self.root = root # The ComfyUI output folder, where the page keeps FreeVideo/.editor
self.lock = threading.Lock()
+ self.listen_lock = threading.Lock()
self.receipts = {} # (client, ref) -> {digest, result}; in memory, bounded; the page save keeps the durable one.
self.prepared = {} # job id -> the reserved project, until its chain id is written into it.
@@ -341,6 +342,24 @@ class Projects:
self._remember(ref, digest, result, client)
return result
+ def listen(self, identity, value, *, jobs, ref=None, client=None):
+ """Queue a listening job (align or transcribe) for the revision the caller read; a stale revision starts nothing."""
+ schema.validate(schema.project_schema('listen'), value)
+ if (value['mode'] == 'align') != ('lines' in value):
+ raise _invalid('/lines', 'lines for mode align, and none for transcribe', value.get('lines'))
+ with self.listen_lock: # the same key sent twice at once queues one job
+ digest, replay = self._receipt('projects:listen', dict(id=identity, body=value), ref, client)
+ if replay is not None:
+ return replay
+ stored = self._load(identity)['project']
+ if stored['revision'] != value['revision']:
+ raise ProjectError('project_changed', project=stored)
+ job = jobs.submit(dict(kind='listen', editor_project=identity,
+ **{k: v for k, v in value.items() if k != 'ref'}), client=client)[0]
+ result = dict(job=jobs.view(job), jobs=[jobs.view(job)])
+ self._remember(ref, digest, result, client)
+ return result
+
def _update(self, identity, change, *, tries=4):
"""Read, change and save; a page save in between makes it read again. None when nothing changed."""
for _ in range(tries):
@@ -448,3 +467,49 @@ def run_export(port, row, *, poll=.25):
relative = view['path']
return dict(library_id=library_id(relative), video=library_view(port, relative),
editor_project=spec['editor_project'], revision=spec['revision'])
+
+
+# Listen ----------------------------------------------------------------------------------------------
+LISTEN_REASONS = {'no_engine': 'no_speech_model', 'language': 'no_speech_model', 'no_sound': 'invalid_request', 'too_long': 'invalid_request',
+ 'source_missing': 'clip_source_unusable', 'unreadable': 'clip_source_unusable',
+ 'disk_full': 'storage_unavailable'}
+LISTEN_MESSAGES = {
+ 'no_engine': 'No speech model is installed on this machine; write the subtitle lines yourself with the subtitles edit',
+ 'no_sound': 'The timeline has no sound to listen to',
+ 'too_long': 'The sound is too long for one listening job; give a shorter range'}
+
+
+def run_listen(port, row, *, poll=.25):
+ """A listening job: the page's subtitle ears (align or transcribe) over one saved revision."""
+ spec = row['spec']
+ queue = port.editor_ears()
+ if queue is None:
+ raise JobError('not_ready', 'The subtitle service is not running')
+ request = dict(kind=spec['mode'], id=spec['editor_project'], revision=spec['revision'], language=spec['language'])
+ request.update({k: spec[k] for k in ('lines', 'range') if k in spec})
+ try:
+ view = queue.submit(request)
+ except FileNotFoundError as error:
+ raise JobError('not_found', 'The project does not exist') from error
+ except (ValueError, TypeError) as error:
+ raise JobError('invalid_request', 'This project cannot be listened to') from error
+ except Exception as error:
+ if getattr(error, 'reason', None) is None:
+ raise
+ raise JobError('busy', 'The subtitle queue is full; retry shortly') from error
+ identity, cancelled = view['id'], False
+ row['_phase'] = 'listening'
+ while view['state'] == 'running':
+ if row.get('_cancel') and not cancelled:
+ view, cancelled = queue.cancel(identity), True
+ continue
+ row['_detail'] = json.dumps(dict(fraction=round(view['fraction'], 3)))
+ time.sleep(poll)
+ view = queue.get(identity)
+ if view['state'] == 'cancelled':
+ raise JobError('cancelled', 'Cancelled')
+ if view['state'] != 'done':
+ reason = view.get('reason') or 'failed'
+ raise JobError(LISTEN_REASONS.get(reason, 'listen_failed'),
+ LISTEN_MESSAGES.get(reason) or 'Listening failed (%s)' % reason)
+ return dict(editor_project=spec['editor_project'], revision=spec['revision'], **view['result'])
diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.py
index 92d5a33..2638001 100644
--- a/freevideo_engine/agent/routes.py
+++ b/freevideo_engine/agent/routes.py
@@ -154,7 +154,7 @@ def features(continuation, vlm):
'queue.shared': True, 'queue.busy': True, 'chain.auto': bool(continuation),
'enhance': bool(vlm.get('ready')), 'drama.film': True, 'images.edit': rules,
'images.layers': rules, 'images.heal': rules, 'images.render': rules, 'projects.read': True,
- **{'projects.' + name: rules for name in ('insert', 'split', 'move', 'trim', 'remove', 'export', 'generate')}}
+ **{'projects.' + name: rules for name in ('insert', 'split', 'move', 'trim', 'remove', 'export', 'generate', 'listen')}}
def capabilities(port, jobs):
@@ -192,6 +192,7 @@ def capabilities(port, jobs):
'vlm': 'Ask the local vision-language model about images or sampled video frames',
'enhance': 'Rewrite a rough idea into a full H3 prompt with the local VLM',
'export': 'Render a saved revision of an editing project to a video',
+ 'listen': 'Listen to the sound of an editing project for subtitle lines and times',
'image_edit': 'Render a saved revision of an image session to a picture'},
limits=dict(width=[256, 4096, 32], height=[256, 4096, 32], seconds=[1.625, 60 if continuation else BLOCK_SECONDS],
seconds_tested=[4, 15],
@@ -1633,7 +1634,7 @@ def register():
raise JobError('idempotency_conflict', 'Conflicting keys')
args = [] if operation == 'create' else [identity]
extra = dict(client=request.headers.get('X-FreeVideo-Client', '')[:80] or None)
- if operation in ('generate', 'export'):
+ if operation in ('generate', 'export', 'listen'):
extra['jobs'] = jobs
if operation == 'generate':
def capability(size):
@@ -1643,9 +1644,9 @@ def register():
return None
extra['capability'] = capability
result = await asyncio.to_thread(getattr(projects, operation), *args, value, ref=key or ref, **extra)
- if operation in ('generate', 'export'):
+ if operation in ('generate', 'export', 'listen'):
await jobs.tick()
- return reply(result, 201 if operation in ('create', 'generate', 'export') else 200)
+ return reply(result, 201 if operation in ('create', 'generate', 'export', 'listen') else 200)
return guarded(handler)
for operation, (method, suffix, _) in contract.PROJECT_OPERATIONS.items():
diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py
index 518812e..38410e1 100644
--- a/freevideo_engine/agent/schema.py
+++ b/freevideo_engine/agent/schema.py
@@ -209,6 +209,23 @@ def export(*, local=False):
**{'x-job-kind': 'export'})
+LISTEN_FIELDS = dict(
+ mode=enum(('align', 'transcribe'), description='align: time the lines you give against the sound; transcribe: write down what is said.'),
+ language=string(8, pattern=r'^(auto|[a-z]{2,3})$', default='auto', description='Spoken language, e.g. zh or en; auto detects it.'),
+ lines=array(string(2000), 1, 1000, description='align only: the lines to time, in order.'),
+ range=obj(dict(start=EDITOR_FRAME, frames=number(1, 10**7, integer=True)), ('start', 'frames'),
+ description='Timeline frames to listen to; omit for the whole timeline.'))
+
+
+def listen(*, local=False):
+ return obj(dict(kind={'const': 'listen', 'type': 'string'},
+ editor_project=string(34, pattern=r'^e-[0-9a-f]{32}$', description='The editing project whose sound to listen to (GET /projects).'),
+ revision=number(1, 2**53 - 1, integer=True, description='The saved revision you read; a later save fails the job with project_changed.'),
+ **LISTEN_FIELDS, **common()), ('kind', 'editor_project', 'revision', 'mode'),
+ description='Listen to the timeline sound with the local speech model. The result lists each line with its start and end frame; nothing is saved.',
+ **{'x-job-kind': 'listen'})
+
+
def image_edit(*, local=False):
return obj(dict(kind={'const': 'image_edit', 'type': 'string'},
image=dict(IMAGE, description='The image session to render (POST /images).'),
@@ -218,7 +235,7 @@ def image_edit(*, local=False):
**{'x-job-kind': 'image_edit'})
-KINDS = dict(generate=generate, assemble=assemble, vlm=vlm, enhance=enhance, export=export, image_edit=image_edit)
+KINDS = dict(generate=generate, assemble=assemble, vlm=vlm, enhance=enhance, export=export, listen=listen, image_edit=image_edit)
def job_schema(*, local=False, refs=False):
@@ -320,7 +337,7 @@ def contracts(*, local=False):
if method == 'GET':
extra['query'] = shape
else:
- extra.update(body=shape, idempotent=True, **({'status': 201} if name in ('create', 'generate', 'export') else {}))
+ extra.update(body=shape, idempotent=True, **({'status': 201} if name in ('create', 'generate', 'export', 'listen') else {}))
routes[method, suffix] = route(summary, **extra)
for name, (method, suffix, summary) in IMAGE_OPERATIONS.items():
shape = image_schema(name)
@@ -623,10 +640,12 @@ PROJECT_OPERATIONS = {
'operations': ('POST', '/projects/{id}/operations', 'Apply 1-64 timeline edits atomically and save once. First get revision and clip IDs; on project_changed recompute from error.project, and on clip_busy wait for generation to finish.'),
'generate': ('POST', '/projects/{id}/generate', 'Make new picture and sound for selected clips or a frame range, as the page Generate does. First get revision and clip IDs; the parts are reserved at once and fill in as the returned job finishes.'),
'export': ('POST', '/projects/{id}/export', 'Render one saved revision to an MP4 in the saved videos. First get the revision; wait for the returned job, then read its result video.'),
+ 'listen': ('POST', '/projects/{id}/listen', 'Listen to the timeline sound for subtitles: mode align times lines you give, transcribe writes down what is said. First get the revision; wait for the returned job, then add the lines with the subtitles edit. Nothing is saved; with no speech model for the language the call fails with no_speech_model, so write the lines by hand.'),
}
PROJECT_TOOLS = dict(list='freevideo_projects', get='freevideo_project_read', create='freevideo_project_create',
write='freevideo_project_write', operations='freevideo_editor_operations',
- generate='freevideo_project_generate', export='freevideo_export')
+ generate='freevideo_project_generate', export='freevideo_export',
+ listen='freevideo_project_listen')
SESSION_OPERATIONS = {
'create': ('POST', '/sessions', 'Create a work conversation for messages and plans shared with the user.'),
@@ -727,6 +746,8 @@ def project_schema(name):
settings=settings, ref=REF), ('revision', 'target'))
if name == 'export':
return obj(dict(revision=revision, ref=REF), ('revision',))
+ if name == 'listen':
+ return obj(dict(revision=revision, **LISTEN_FIELDS, ref=REF), ('revision', 'mode'))
if name != 'operations':
return EMPTY
fields = {
@@ -741,6 +762,9 @@ def project_schema(name):
'describe': (dict(clip_ids=ids, text=dict(type='string', maxLength=8000)), ('clip_ids', 'text'), 'Set the description of selected clips. Use clip IDs from this revision and one paragraph of text.'),
'set_ref': (dict(clip_ids=ids, ref=dict(type='string', maxLength=128, pattern=r'^[A-Za-z0-9_-]{0,128}$')), ('clip_ids', 'ref'), 'Set an outside shot reference on selected clips. Use clip IDs from this revision; an empty ref removes it.'),
'use_version': (dict(clip_id=CLIP, index=number(0, 49, integer=True)), ('clip_id', 'index'), 'Restore a stored clip version. First get the clip ID and its versions array; index is zero based.'),
+ 'subtitles': (dict(cues=array(obj(dict(start=EDITOR_FRAME, frames=number(1, 10**7, integer=True), text=dict(type='string', minLength=1, maxLength=400)),
+ ('start', 'frames', 'text')), 1, 500), replace=dict(type='boolean')), ('cues',),
+ 'Add subtitle lines as one undo step on the subtitle track (created if missing). Each cue is a start frame, a length in frames and one or two lines of text; cues may not overlap each other or lines already there. replace: true swaps the whole track.'),
}
operations = [obj(dict(op={'const': op}, **props), ('op', *required), description=description)
for op, (props, required, description) in fields.items()]
diff --git a/freevideo_engine/agent/spec.py b/freevideo_engine/agent/spec.py
index d2f75c6..8a9608c 100644
--- a/freevideo_engine/agent/spec.py
+++ b/freevideo_engine/agent/spec.py
@@ -213,6 +213,9 @@ def normalize(value):
result.update({k: value[k] for k in ('format', 'max_tokens', 'system') if k in value})
elif kind == 'export':
result.update(editor_project=value['editor_project'], revision=value['revision'])
+ elif kind == 'listen':
+ result.update(editor_project=value['editor_project'], revision=value['revision'], mode=value['mode'],
+ language=value.get('language', 'auto'), **{k: value[k] for k in ('lines', 'range') if k in value})
elif kind == 'image_edit':
result.update(image=value['image'], revision=value['revision'])
else: