Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Long videos: name the part-description parser apart from the voice reference's line finder

Bowen Xueaafae0aa4c71
diff --git a/freevideo_engine/chain_runner.py b/freevideo_engine/chain_runner.py
index e5e4179..64e809d 100644
--- a/freevideo_engine/chain_runner.py
+++ b/freevideo_engine/chain_runner.py
@@ -212,6 +212,30 @@ class ChainRunner:
self.wake.set()
return self._view(record)
+ def set_text(self, chain_id, block_id, text):
+ """The description of a part that has not started; the user's own words, used as written."""
+ if not isinstance(text, str) or not text.strip() or len(text) > 20000:
+ raise ValueError('Text must be a nonempty string of at most 20000 characters')
+ with self.lock:
+ record = self.records[chain_id]
+ block = next((row for row in record['blocks'] if row['id'] == block_id), None)
+ if block is None:
+ raise KeyError(block_id)
+ if (record.get('split') or {}).get('state') in ('pending', 'running'):
+ raise ChainBusy('The descriptions of the parts are not written yet')
+ if block['state'] != 'pending' or block['prompt_id'] is not None:
+ raise ChainBusy('This part has already started')
+ previous = block['inputs']['text']
+ block['inputs']['text'] = text
+ try:
+ from .lora_triggers import apply_chain_triggers
+ apply_chain_triggers(record)
+ except Exception:
+ block['inputs']['text'] = previous
+ raise ValueError('Text must be at most 20000 characters with LoRA triggers') from None
+ self._save(record)
+ return self._view(record)
+
def resume(self, chain_id, client_id=None):
with self.lock:
record = self.records[chain_id]
@@ -231,6 +255,12 @@ class ChainRunner:
seconds=None, output=None, report_id=None, error=None,
fraction=0., remaining_seconds=None, continuation=None,
skip_frames=0, skip_audio_samples=0)
+ split = record.get('split')
+ if (split is not None and split['state'] == 'failed' and split['reason'] in ('busy', 'memory')
+ and all(row['state'] != 'done' for row in record['blocks'])
+ and len({row['inputs']['text'] for row in record['blocks']}) == 1):
+ # Busy and memory pass; nothing has started and no part was rewritten, so split again.
+ split.update(state='pending', reason=None)
if client_id is not None:
record['client_id'] = client_id
record['state'] = 'queued'
@@ -491,8 +521,18 @@ class ChainRunner:
else:
prompt_id, node = block['prompt_id'], record['node']
action = 'status'
+ split = record.get('split')
if action == 'join':
pass
+ elif (prompt_id is None and split is not None and split['state'] == 'pending'
+ and record['state'] != 'cancelling' and block['index'] == 0):
+ # Before the first part: one description per part, so later parts carry on instead of replaying.
+ split['state'] = 'running'
+ record['state'] = 'running'
+ self._save(record)
+ value = dict(text=block['inputs']['text'], continued=block['after'] is not None,
+ seconds=[row['new_frames'] / chain_plan.FPS for row in record['blocks']])
+ action = 'split'
elif prompt_id is None:
self.progress_messages.pop(identity, None)
self.progress_values.get(identity, {}).pop('remaining_seconds', None)
@@ -520,6 +560,11 @@ class ChainRunner:
if action == 'join':
self._join(identity, snapshot)
return
+ if action == 'split':
+ self._split(record, value)
+ if record['split']['state'] != 'pending':
+ self._step_chain(identity) # The first part starts in the same pass.
+ return
if action == 'submit':
try:
for key, source in sources.items():
@@ -587,6 +632,39 @@ class ChainRunner:
block['state'] = desired
self._save(record)
+ def _split(self, record, value):
+ def cancelled():
+ return record['state'] == 'cancelling' or self.closed.is_set()
+ split = getattr(self.host, 'split_texts', None)
+ parts, reason = None, 'unavailable'
+ if split is not None:
+ try:
+ parts, reason = split(value, cancelled)
+ except Exception:
+ logging.exception('[FreeVideo] Splitting the description failed')
+ parts, reason = None, 'failed'
+ with self.lock:
+ if parts is None and cancelled():
+ record['split']['state'] = 'pending' # Split again when resumed.
+ elif parts is not None and len(parts) == len(record['blocks']) and all(
+ isinstance(part, str) and part.strip() for part in parts):
+ for row, part in zip(record['blocks'], parts):
+ row['inputs']['text'] = part
+ try:
+ from .lora_triggers import apply_chain_triggers
+ apply_chain_triggers(record)
+ record['split'].update(state='done', reason=None)
+ except Exception:
+ for row in record['blocks']:
+ row['inputs']['text'] = value['text']
+ record['split'].update(state='failed', reason='failed')
+ else:
+ # The whole description goes to every part, as before; the page says why.
+ if parts is not None:
+ reason = 'unusable'
+ record['split'].update(state='failed', reason=reason if reason in store.SPLIT_REASONS else 'failed')
+ self._save(record)
+
def _join(self, identity, snapshot):
final_state = snapshot.get('join_final_state', 'done')
def cancelled():
diff --git a/freevideo_engine/chain_store.py b/freevideo_engine/chain_store.py
index 065a8ad..468e49e 100644
--- a/freevideo_engine/chain_store.py
+++ b/freevideo_engine/chain_store.py
@@ -16,6 +16,16 @@ ACTIVE = ('queued', 'running', 'cancelling')
CHAIN_STATES = (*ACTIVE, 'cancelled', 'failed', 'interrupted', 'done')
BLOCK_STATES = ('pending', 'queued', 'running', 'done', 'cancelled', 'failed', 'interrupted')
OUTPUT_STATES = ('none', 'pending', 'joining', 'done', 'failed')
+# pending: split before the first part; given: the request gave one text per part; none: nothing to split.
+SPLIT_STATES = ('pending', 'running', 'done', 'failed', 'given', 'none')
+SPLIT_REASONS = {
+ 'model_missing': 'The prompt model is not installed, so every part used the whole description.',
+ 'busy': 'The prompt model was busy, so every part used the whole description.',
+ 'memory': 'There was not enough memory to split the description, so every part used the whole description.',
+ 'unusable': 'The description could not be split into parts, so every part used the whole description.',
+ 'unavailable': 'Splitting is not available here, so every part used the whole description.',
+ 'failed': 'Splitting the description failed, so every part used the whole description.',
+}
# Parts one video may be continued through: the first part and four continuations.
MAX_PARTS = 5
@@ -63,7 +73,7 @@ def validate(payload):
raise ValueError('Invalid extra data or client ID')
if ('blocks' in payload) == ('seconds' in payload):
raise ValueError('Specify blocks or seconds, not both')
- planned = None
+ planned = split = None
join = payload.get('join', 'seconds' in payload)
if type(join) is not bool:
raise ValueError('Join must be a bool')
@@ -81,9 +91,19 @@ def validate(payload):
raise ValueError('Text must be a string')
if type(seed) is not int or seed < 0:
raise ValueError('A nonnegative integer seed is required')
+ # Each part plays its own share of the description; one text for all parts replays the story
+ # from the start in every part. Without texts the runner splits it before the first part.
+ texts = payload.get('texts')
+ if texts is not None and (not isinstance(texts, list) or len(texts) != len(planned['blocks'])
+ or any(not isinstance(row, str) or not row.strip() for row in texts)):
+ raise ValueError('Texts must give one description for each planned part')
rows = [dict(id='b%s' % (index + 1), after='b%s' % index if index else source,
- frames=row['frames'], text=text, seed=seed + index)
+ frames=row['frames'], text=texts[index] if texts else text, seed=seed + index)
for index, row in enumerate(planned['blocks'])]
+ if len(rows) > 1:
+ split = dict(state='given' if texts else 'pending' if text.strip() else 'none', reason=None)
+ elif 'texts' in payload:
+ raise ValueError('Texts are given with seconds; blocks carry their own text')
else:
rows = payload['blocks']
if not isinstance(rows, list) or not 1 <= len(rows) <= chain_plan.MAX_BLOCKS:
@@ -143,7 +163,7 @@ def validate(payload):
updated=now, state='queued', client_id=client,
snapshot=dict(prompt=deepcopy(prompt), extra_data=deepcopy(extra)), blocks=blocks,
plan=planned, join=join, output=None, output_state='pending' if join else 'none',
- output_error=None, output_blocks=0)
+ output_error=None, output_blocks=0, split=split)
try:
json.dumps(record, allow_nan=False)
except (ValueError, TypeError, OverflowError) as error:
@@ -166,7 +186,7 @@ def summary(record):
return dict({key: deepcopy(record[key]) for key in
('version', 'id', 'node', 'created', 'updated', 'state',
'join', 'output', 'output_state', 'output_error', 'output_blocks')},
- output_seams=deepcopy(record.get('output_seams')),
+ output_seams=deepcopy(record.get('output_seams')), split=deepcopy(record.get('split')),
count=len(blocks), done=sum(row['state'] == 'done' for row in blocks),
current=current, elapsed_seconds=sum(row['seconds'] or 0 for row in blocks if row['state'] == 'done'),
last_report_id=record.get('last_report_id') or next((row['report_id'] for row in reversed(blocks)
@@ -224,6 +244,13 @@ def load_all(directory, limit=200, *, identity=None):
if (type(record['output_blocks']) is not int or not 0 <= record['output_blocks'] <= len(record['blocks'])
or record['output'] is None and record['output_blocks'] != 0):
raise ValueError('Invalid chain output block count')
+ split = record.setdefault('split', None)
+ if split is not None and (not isinstance(split, dict) or set(split) != {'state', 'reason'}
+ or split['state'] not in SPLIT_STATES
+ or split['reason'] is not None and split['reason'] not in SPLIT_REASONS):
+ raise ValueError('Invalid chain split')
+ if split is not None and split['state'] == 'running':
+ split['state'] = 'pending' # Stopped while splitting: split again when resumed.
seams = record.get('output_seams')
if seams is not None and (not isinstance(seams, dict) or set(seams) != {'expected', 'kept'}
or any(type(value) is not int or value < 0 for value in seams.values())):
diff --git a/freevideo_engine/comfy_chain.py b/freevideo_engine/comfy_chain.py
index 3afecef..f1c86cb 100644
--- a/freevideo_engine/comfy_chain.py
+++ b/freevideo_engine/comfy_chain.py
@@ -6,6 +6,7 @@ import math
import os
from pathlib import Path, PurePosixPath
import ssl
+import time
from urllib import error, request
from .chain_runner import ChainBusy, ChainPartTooShort, ChainRunner, ChainTooLong
@@ -177,6 +178,54 @@ class ComfyHost:
return join_outputs([block['output'] for block in blocks], record['id'], cancelled,
frames=frames, replace=record.get('output'), **kept)
+ def split_texts(self, value, cancelled):
+ """One description per part from the local prompt model: (parts, None) or (None, reason)."""
+ service = getattr(self.server, '_freevideo_prompt_vlm', None)
+ if service is None:
+ return None, 'unavailable'
+ # A rewrite or a single video the user started a moment ago finishes first; busy is said only after
+ # ten minutes of waiting. Up to ten minutes waiting plus fifteen splitting hold the runner thread; there
+ # is one active chain, so nothing else waits behind it.
+ waiting = time.monotonic() + 600
+ while True:
+ try:
+ job = service.start('split', value)['job']
+ except ValueError as failure:
+ if str(failure) != 'busy' or time.monotonic() > waiting or cancelled():
+ return None, {'model_missing': 'model_missing', 'busy': 'busy'}.get(str(failure), 'failed')
+ time.sleep(2)
+ continue
+ parts, reason = self._split_job(service, job, cancelled)
+ if reason != 'busy' or time.monotonic() > waiting or cancelled():
+ return parts, reason
+ time.sleep(2)
+
+ def _split_job(self, service, job, cancelled):
+ deadline = time.monotonic() + 900
+ while True:
+ try:
+ row = service.status(job)
+ except ValueError:
+ return None, 'failed'
+ phase = row.get('phase')
+ if phase == 'complete':
+ return row.get('parts'), None
+ if phase in ('failed', 'cancelled'):
+ error = row.get('error')
+ return None, ('memory' if error in ('ram_space', 'disk_space') else 'busy' if error == 'busy'
+ else 'unusable' if error == 'split_failed' else 'failed')
+ if cancelled() or time.monotonic() > deadline:
+ service.cancel(job)
+ stopping = time.monotonic() + 30
+ try:
+ while (time.monotonic() < stopping
+ and service.status(job).get('phase') not in ('complete', 'failed', 'cancelled')):
+ time.sleep(.2)
+ except ValueError:
+ pass
+ return None, 'failed'
+ time.sleep(.5)
+
def lead(self, output, previous):
"""A finished block's seam sound, read while its record is still in the shared store."""
from .comfy_continuation import audio_lead_in
@@ -332,6 +381,19 @@ def register(host_factory=None):
except KeyError:
return response({'error': 'Unknown chain'}, 404)
+ @server.routes.post('/freevideo/chain/{id}/blocks/{block}/text')
+ async def text(request):
+ try:
+ value = await body(request)
+ return response({'chain': runner.set_text(request.match_info['id'], request.match_info['block'],
+ value.get('text') if isinstance(value, dict) else None)})
+ except KeyError:
+ return response({'error': 'Unknown chain or part'}, 404)
+ except ChainBusy as failure:
+ return response({'error': str(failure)}, 409)
+ except ValueError as failure:
+ return response({'error': str(failure)}, 400)
+
@server.routes.post('/freevideo/chain/{id}/resume')
async def resume(request):
try:
diff --git a/freevideo_engine/prompt_vlm/rules.py b/freevideo_engine/prompt_vlm/rules.py
index 32d576b..e1bfa77 100644
--- a/freevideo_engine/prompt_vlm/rules.py
+++ b/freevideo_engine/prompt_vlm/rules.py
@@ -411,3 +411,214 @@ def validate_answers(answers, value):
if 'error' in answer and answer['error'] not in ('invalid_json', 'context_length'):
raise ValueError('invalid_output')
return answers
+
+
+# Long videos: one description per part --------------------------------------------------------------------
+# Every part of a long video is generated from its own text. Handing each part the whole description makes
+# the model play the story from the start again in every part: the opening pause, the first line and the
+# first action come back after each seam. Each part instead gets the shared look and sound plus only what
+# happens during its own seconds.
+#
+# The small local model only sorts the events: spoken lines become placeholders it places, and the shared
+# description, the sound sections and the words themselves are put back by code, so they cannot drift.
+SPLIT_MAX_PARTS = 8
+CONTINUE = 'Continuing directly from the previous moment, with no pause: everything already in progress keeps going.'
+SPEAKER = re.compile(r'(?:\(S\d+(?:,S\d+)*\)\s*)?<d>(\[[A-Za-z]+\])?\s*([^<\n]{1,400}?)\s*</d>')
+
+
+def split_request(value):
+ if not isinstance(value, dict) or set(value) - {'text', 'seconds', 'mode', 'continued'}:
+ raise ValueError('invalid_request')
+ text, seconds = value.get('text'), value.get('seconds')
+ if type(value.get('continued', False)) is not bool:
+ raise ValueError('invalid_request')
+ if not isinstance(text, str) or not text.strip() or len(text) > 12000:
+ raise ValueError('invalid_prompt')
+ if (not isinstance(seconds, list) or not 2 <= len(seconds) <= SPLIT_MAX_PARTS or
+ any(type(s) not in (float, int) or not math.isfinite(s) or not 1 <= s <= 60 for s in seconds)):
+ raise ValueError('invalid_duration')
+ return dict(mode='split', text=text, seconds=[float(s) for s in seconds],
+ continued=value.get('continued', False))
+
+
+def speech_seconds(words):
+ """Rough speaking time: about 2.6 English words or 4.5 CJK characters a second."""
+ cjk = len(re.findall(r'[぀-ヿ㐀-鿿가-힯]', words))
+ return round(cjk / 4.5 + len(re.findall(r'[A-Za-z0-9À-ɏ]+', words)) / 2.6, 1)
+
+
+def part_source(text):
+ """(body, sound sections, lines): the description with each spoken line as [L1], [L2], ...
+
+ H3 prompts keep their sound sections apart; they describe the whole video and are repeated as written.
+ """
+ body, sound = text.strip(), ''
+ found = re.search(r'(?m)^overall_soundscape:', body)
+ if body.startswith('integrated_multimodal_description:') and found:
+ body, sound = body[:found.start()].strip(), body[found.start():].strip()
+ body = body[len('integrated_multimodal_description:'):].strip()
+ body = re.sub(r'^\[Shot 1\]\s*', '', body)
+ lines = []
+ def tagged(match):
+ lines.append(match.group(0))
+ return '[L%d]' % len(lines)
+ body = SPEAKER.sub(tagged, body)
+ spoken = set(spoken_lines(body))
+ def quoted(match):
+ words = next(group for group in match.groups() if group is not None).strip()
+ if words not in spoken:
+ return match.group(0)
+ lines.append(match.group(0))
+ return '[L%d]' % len(lines)
+ if spoken:
+ body = QUOTED.sub(quoted, body)
+ numbers = [int(n) for n in re.findall(r'\[L(\d+)\]', body)]
+ if numbers != list(range(1, len(lines) + 1)):
+ raise ValueError('invalid_prompt')
+ return body, sound, lines
+
+
+def split_instruction(value):
+ count = len(value['seconds'])
+ rows, start = [], 0.
+ for i, length in enumerate(value['seconds'], 1):
+ rows.append(f'Part {i}: {start:.0f}-{start + length:.0f} s')
+ start += length
+ opening = 'continues an earlier video' if value.get('continued') else 'is the opening'
+ return f"""A long video is made of {count} parts generated one after another; each part continues where the
+previous one ended. Sort the user's description into what stays the same and what happens in each part:
+{chr(10).join(rows)}
+Shared: who is in the scene and how they look, the place, the lighting, the camera, the style and the
+background sound. Copy these words from the description. Leave out every action, event, change and every [L] marker.
+Part N: only what happens during that part, in the order of the description, so each action happens
+once. Part 1 {opening}; each later part picks up where the previous part ended. When the story is
+shorter than the video, the later parts continue the same activity calmly instead of repeating it.
+The markers [L1], [L2], ... stand for spoken lines; their speaking time is given. Put each marker in
+exactly one part, in order, where it is said, as "he says [L2]" or "她说 [L2]"; spread them so each
+part's speech fits its seconds. Do not write any other speech.
+Example for a 2-part video. Description: A girl in a yellow raincoat stands on a wooden pier at dawn,
+filmed in a wide shot, gulls crying. She waves and shouts [L1], runs to the end of the pier and jumps
+into the sea. Speaking time: [L1] 2 s
+Shared: A girl in a yellow raincoat on a wooden pier at dawn, filmed in a wide shot, gulls crying.
+Part 1: She waves and shouts [L1], then starts running along the pier.
+Part 2: She reaches the end of the pier and jumps into the sea.
+Write in the language of the description. Answer with exactly these lines and nothing else:
+Shared: ...
+{chr(10).join(f'Part {i}: ...' for i in range(1, count + 1))}"""
+
+
+def split_user_text(value):
+ body, _, lines = part_source(value['text'])
+ text = f'Description of the whole {sum(value["seconds"]):.0f}-second video:\n{body}'
+ if lines:
+ text += '\n\nSpeaking time: ' + ', '.join('[L%d] %.0f s' % (i, max(1, speech_seconds(line)))
+ for i, line in enumerate(lines, 1))
+ return text
+
+
+def split_answer(text, count):
+ if not isinstance(text, str):
+ raise ValueError('invalid_output')
+ text = text.strip().strip('`').strip()
+ found = list(re.finditer(r'(?m)^[ \t*#]*(Shared|Part[ \t]+(\d+))[ \t*]*[::]', text))
+ names = [m.group(2) or 'shared' for m in found]
+ if names != ['shared'] + [str(i) for i in range(1, count + 1)] or text[:found[0].start()].strip(' \t\n*#'):
+ raise ValueError('invalid_output')
+ ends = [m.start() for m in found[1:]] + [len(text)]
+ values = [' '.join(text[m.end():end].split()).strip('* ') for m, end in zip(found, ends)]
+ return values[0], values[1:]
+
+
+def place_lines(body, lines, seconds):
+ """Spoken lines to parts, in order, when the model did not place them.
+
+ A line starts halfway between where an even spread of the speech puts it and where it stands in the
+ story between the first and the last line; a line that would run into the next part starts early enough to fit.
+ """
+ total, lengths = sum(seconds), [max(1., speech_seconds(line)) for line in lines]
+ ends = [sum(seconds[:i + 1]) for i in range(len(seconds))]
+ gap = max(0., total - sum(lengths)) / len(lines)
+ first = body.index('[L1]')
+ span = max(1, body.index('[L%d]' % len(lines)) - first) # Scenery after the last line is not story time.
+ placed, clock = [], 0.
+ for index, length in enumerate(lengths):
+ even = sum(lengths[:index]) + index * gap
+ story = (body.index('[L%d]' % (index + 1)) - first) / span * total
+ start = max(clock, min((even + story) / 2, total - sum(lengths[index:])))
+ part = next((n for n, end in enumerate(ends) if start < end), len(ends) - 1)
+ opening = ends[part - 1] if part else 0.
+ if start + length > ends[part] and part < len(ends) - 1:
+ if ends[part] - length >= max(clock, opening):
+ start = ends[part] - length
+ elif start - opening > seconds[part] / 2: # A long line starting early in a part stays there.
+ part, start = part + 1, max(start, ends[part])
+ placed.append(part)
+ clock = start + length
+ return placed
+
+
+def spoken(line, words):
+ """A line put back by code: H3 lines carry their own speaker tag; plain quotes need a speech verb."""
+ if '<d>' in line:
+ return line
+ return ('说:' if language(words) in ('Chinese', 'Japanese') else 'says: ') + line
+
+
+def validate_split(text, value, *, strict=False):
+ """Part texts built from the model's sorting; every spoken line kept once, in order.
+
+ A part with nothing new carries on from the previous one; a misplaced line is placed by speaking time.
+ Only an answer without the expected layout or a usable shared description fails; strict also
+ fails when the model did not place every line once and in order, so it can be asked again.
+ """
+ body, sound, lines = part_source(value['text'])
+ shared, events = split_answer(text, len(value['seconds']))
+ shared = ' '.join(re.sub(r'\[L\d+\]', '', shared).split())
+ if not 20 <= len(shared) <= 4000:
+ raise ValueError('invalid_output')
+ seen, cleaned = {plain(shared), plain(body)}, []
+ for event in events:
+ key = plain(re.sub(r'\[L\d+\]', '', event))
+ if len(event) > 4000 or not key or key in seen or key in plain(shared):
+ key = ''
+ seen.add(key)
+ cleaned.append(event if key else ' '.join(re.findall(r'\[L\d+\]', event)))
+ events = cleaned
+ placed = [int(n) for event in events for n in re.findall(r'\[L(\d+)\]', event)]
+ if placed != list(range(1, len(lines) + 1)) and strict:
+ raise ValueError('invalid_output')
+ if placed != list(range(1, len(lines) + 1)):
+ # The model lost or reordered a line: keep its events, place the lines by time.
+ events = [' '.join(re.sub(r'\s*\[L\d+\]\s*[::,,]?', ' ', event).split()).rstrip('::,,') for event in events]
+ for number, part in enumerate(place_lines(body, lines, value['seconds']), 1):
+ events[part] = (events[part] + ' [P%d]' % number).strip()
+ def restore(words):
+ # [L] stands where the model put a line in its own words; [P] where code placed it.
+ return re.sub(r'\[([LP])(\d+)\]', lambda m: (lines[int(m.group(2)) - 1] if m.group(1) == 'L'
+ else spoken(lines[int(m.group(2)) - 1], shared)), words)
+ parts = []
+ for index, event in enumerate(events):
+ if index == 0 and not re.sub(r'\[[LP]\d+\]', '', event).strip():
+ # Nothing sorted into the opening: it is the user's own description, without later lines.
+ mine = set(re.findall(r'\[[LP](\d+)\]', event))
+ words = re.sub(r'\[L(\d+)\]', lambda m: m.group(0) if m.group(1) in mine else '', body)
+ words = restore(' '.join(words.split()))
+ else:
+ words = ' '.join(([CONTINUE] if index or value.get('continued') else []) + [shared] +
+ ([restore(event)] if event else []))
+ if sound:
+ words = 'integrated_multimodal_description: [Shot 1] ' + words + '\n\n' + sound
+ parts.append(words)
+ return parts
+
+
+def validate_parts(parts, value):
+ """Worker output shape for a split job: one non-empty text per part."""
+ if (not isinstance(parts, list) or len(parts) != len(value['seconds'])
+ or any(not isinstance(part, str) or not 40 <= len(part) <= 20000 for part in parts)):
+ raise ValueError('invalid_output')
+ return parts
+
+
+SPLIT_RETRY = ('Answer again with exactly one "Shared:" line and the "Part N:" lines, every [L] marker exactly once '
+ 'and in order, and no marker in Shared.')
diff --git a/freevideo_engine/prompt_vlm/service.py b/freevideo_engine/prompt_vlm/service.py
index fc64c89..cad2b22 100644
--- a/freevideo_engine/prompt_vlm/service.py
+++ b/freevideo_engine/prompt_vlm/service.py
@@ -262,7 +262,7 @@ class Service:
def start(self, action, value):
received_epoch = time.time()
- if action not in ('install', 'rewrite', 'ask'):
+ if action not in ('install', 'rewrite', 'ask', 'split'):
raise ValueError('invalid_request')
if self.busy():
raise ValueError('busy')
@@ -274,6 +274,10 @@ class Service:
if not assets.ready(root):
raise ValueError('model_missing')
value = self.ask_images(rules.ask_request(value))
+ elif action == 'split':
+ if not assets.ready(root):
+ raise ValueError('model_missing')
+ value = rules.split_request(value)
else:
if not assets.ready(root):
raise ValueError('model_missing')
@@ -290,7 +294,7 @@ class Service:
with self.mutex:
if self.busy():
raise ValueError('busy')
- if action in ('rewrite', 'ask'):
+ if action != 'install':
self.prestart_ticket += 1 # Invalidate pending imports before the worker starts.
self.key = secrets.token_urlsafe(24)
self.stop.clear()
@@ -326,6 +330,7 @@ class Service:
if self.result_expires and time.monotonic() > self.result_expires:
self.state.pop('text', None)
self.state.pop('answers', None)
+ self.state.pop('parts', None)
self.state.update(phase='failed', error='expired')
row = dict(self.state)
# The browser may immediately queue video work or the next rewrite.
@@ -334,6 +339,7 @@ class Service:
row['phase'] = 'finishing'
row.pop('text', None)
row.pop('answers', None)
+ row.pop('parts', None)
return row
def cancel(self, key):
@@ -368,7 +374,7 @@ class Service:
from ..background_compile import COMPILE
COMPILE.stop()
self.release_idle_image_worker()
- if action in ('rewrite', 'ask'):
+ if action != 'install':
from ..resident_process import OWNER
if OWNER.active:
raise BlockingIOError('busy')
@@ -388,13 +394,14 @@ class Service:
except BlockingIOError:
self.progress(dict(phase='failed', error='busy'))
except Exception as error:
- known = {'disk_space', 'ram_space', 'rewrite_failed', 'worker_exit', 'timeout', 'worker_cleanup_failed'}
+ known = {'disk_space', 'ram_space', 'rewrite_failed', 'split_failed', 'worker_exit', 'timeout', 'worker_cleanup_failed'}
self.progress(dict(phase='failed', error=str(error) if str(error) in known
else 'download_failed' if action == 'install'
- else 'ask_failed' if action == 'ask' else 'rewrite_failed',
+ else 'ask_failed' if action == 'ask' else 'split_failed' if action == 'split'
+ else 'rewrite_failed',
exception=type(error).__name__))
finally:
- if action in ('rewrite', 'ask'):
+ if action != 'install':
self.stop_prestart(self.prewarm_state.get('reason', 'none'))
if self.stop.is_set():
self.progress(dict(phase='cancelled'))
@@ -564,6 +571,8 @@ class Service:
if result.get('phase') == 'complete':
if value.get('mode') == 'ask':
rules.validate_answers(result.get('answers'), value)
+ elif value.get('mode') == 'split':
+ rules.validate_parts(result.get('parts'), value)
else:
rules.validate_output(result.get('text'), value)
self.progress(dict(result, worker_returncode=child.returncode))
diff --git a/freevideo_engine/prompt_vlm/worker.py b/freevideo_engine/prompt_vlm/worker.py
index 0fe96ce..3459536 100644
--- a/freevideo_engine/prompt_vlm/worker.py
+++ b/freevideo_engine/prompt_vlm/worker.py
@@ -343,6 +343,8 @@ def run(folder, value, *, stats=None):
stats = {} if stats is None else stats
if value.get('mode') == 'ask':
return ask(folder, value, stats)
+ if value.get('mode') == 'split':
+ return split(folder, value, stats)
return rewrite(folder, value, stats)
@@ -409,6 +411,62 @@ def rewrite(folder, value, stats):
emit(phase='complete', text=rewritten, diagnostics=dict(stats))
+def split(folder, value, stats):
+ """One description per part of a long video; text leaves only via stdout."""
+ from .rules import SPLIT_RETRY, split_instruction, split_user_text, validate_split
+ count = len(value['seconds'])
+ limit = min(2 * OUTPUT_TOKENS - 200, 600 + 300 * count) # Within the cache reserved for a rewrite and its retry.
+ stats.update(stage='imports', pid=os.getpid(), mode='split', part_count=count, dtype='bfloat16',
+ attention='sdpa', kv_cache='dynamic', backend='transformers', accelerations=[],
+ persistent_model=False)
+ torch, checkpoint = begin(stats)
+ from transformers import AutoProcessor
+ checkpoint('preprocess')
+ preparing = time.monotonic()
+ processor = AutoProcessor.from_pretrained(folder, local_files_only=True, trust_remote_code=False)
+ messages = [{'role': 'system', 'content': split_instruction(value)},
+ {'role': 'user', 'content': [{'type': 'text', 'text': split_user_text(value)}]}]
+ inputs = tokenize(processor, messages)
+ if inputs.input_ids.shape[-1] > 6500:
+ raise ValueError('context_length')
+ stats.update(input_tokens=int(inputs.input_ids.shape[-1]), preprocess_seconds=time.monotonic() - preparing)
+ model, device = load_model(folder, torch, stats, checkpoint)
+ loaded = time.monotonic()
+ checkpoint('prefill', 'splitting')
+ shown = [0.]
+ def on_token(timing):
+ stats.update(output_tokens=timing['output_tokens'], decode_seconds=timing['decode_seconds'])
+ if time.monotonic() - shown[0] > .5:
+ checkpoint('decode', 'splitting')
+ shown[0] = time.monotonic()
+ ids, _ = generate(torch, model, inputs.to(device), limit, loaded, on_token)
+ answers = [processor.decode(ids, skip_special_tokens=True)] if len(ids) < limit else []
+ stats.update(split_seconds=time.monotonic() - loaded, output_tokens=len(ids))
+ parts = None
+ try:
+ parts = validate_split(answers[0], value, strict=True)
+ except (ValueError, IndexError):
+ # One more try with the rules restated; resending the wrong answer would not fit the cache.
+ retry = [messages[0], {'role': 'user', 'content': [{'type': 'text', 'text': (
+ split_user_text(value) + '\n\n' + SPLIT_RETRY)}]}]
+ retry_inputs = tokenize(processor, retry)
+ if retry_inputs.input_ids.shape[-1] + limit <= stats.get('kv_positions', 10 ** 9):
+ ids, _ = generate(torch, model, retry_inputs.to(device), limit, time.monotonic(), lambda timing: None)
+ stats.update(split_retry=True, retry_output_tokens=len(ids), split_seconds=time.monotonic() - loaded)
+ if len(ids) < limit:
+ answers.insert(0, processor.decode(ids, skip_special_tokens=True))
+ for strict in (True, False):
+ for answer in answers:
+ try:
+ parts = parts or validate_split(answer, value, strict=strict)
+ except ValueError:
+ pass
+ if parts is None:
+ raise ValueError('split_failed')
+ stats['stage'] = 'complete'
+ emit(phase='complete', parts=parts, diagnostics=dict(stats))
+
+
def ask(folder, value, stats):
"""Answer each question with one loaded model; answers leave only via stdout."""
from .rules import ask_instruction, image_tokens, parse_json, ASK_JSON_RETRY