Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Queue editor continuations on the server

Bowen Xue29ab41ac2bcd
diff --git a/freevideo_engine/chain_runner.py b/freevideo_engine/chain_runner.py
index 64e809d..2194ded 100644
--- a/freevideo_engine/chain_runner.py
+++ b/freevideo_engine/chain_runner.py
@@ -118,12 +118,61 @@ class ChainRunner:
if isinstance(value, str):
source = next(row for row in record['blocks'] if row['id'] == value)
sources[key] = dict(block=value, output=source['output'])
+ elif isinstance(value, dict) and 'chain' in value:
+ source = self._referenced_block(value)
+ sources[key] = dict(work=source['output'], output=source['output'])
elif value is not None:
sources[key] = dict(work=value['work'], output=value['work'])
else:
sources[key] = None
return sources
+ def _referenced_block(self, value):
+ source = self.records.get(value['chain'])
+ if source is None:
+ # A recent dependent may outlive the list's bounded startup window.
+ try:
+ self.load(value['chain'])
+ except KeyError:
+ raise ValueError('Referenced chain does not exist') from None
+ source = self.records[value['chain']]
+ block = next((row for row in source['blocks'] if row['id'] == value['block']), None)
+ if block is None:
+ raise ValueError('Referenced chain block does not exist')
+ return block
+
+ def _dependency_state(self, record):
+ waiting = False
+ for block in record['blocks']:
+ for key in ('after', 'before'):
+ value = block[key]
+ if not isinstance(value, dict) or 'chain' not in value:
+ continue
+ try:
+ source = self._referenced_block(value)
+ except ValueError:
+ return 'interrupted'
+ parent = self.records[value['chain']]
+ if parent['state'] in ('cancelling', 'cancelled', 'failed', 'interrupted'):
+ return 'interrupted'
+ if parent['state'] == 'done':
+ if source['state'] != 'done' or not source['output']:
+ return 'interrupted'
+ else:
+ waiting = True
+ return 'queued' if waiting else 'ready'
+
+ def _interrupt_dependants(self):
+ # Repeat so a stopped ancestor reaches descendants in any receipt order.
+ changed = True
+ while changed:
+ changed = False
+ for waiting in list(self.records.values()):
+ if waiting['state'] == 'queued' and self._dependency_state(waiting) == 'interrupted':
+ waiting['state'] = 'interrupted'
+ self._save(waiting)
+ changed = True
+
def _elapsed(self, block):
start = self.starts.get(block['prompt_id'])
return max(0., self.clock() - start) if start is not None else 0.
@@ -161,13 +210,21 @@ class ChainRunner:
rest.append(identity)
continue
depth[identity] = 1 + sum(depth[value] if isinstance(value, str)
- else self._work_parts(value['work'], seen + (record['id'],))
+ else self._source_parts(value, seen + (record['id'],))
for value in sources)
if len(rest) == len(pending):
raise ValueError('Block references cannot contain a cycle or self reference')
pending = rest
return depth
+ def _source_parts(self, value, seen):
+ if 'chain' not in value:
+ return self._work_parts(value['work'], seen)
+ self._referenced_block(value)
+ if value['chain'] in seen:
+ raise ValueError('Chain references cannot contain a cycle')
+ return self._depths(self.records[value['chain']], seen)[value['block']]
+
def _work_parts(self, work, seen):
key = _work_key(work)
for record in list(self.records.values()):
@@ -209,6 +266,7 @@ class ChainRunner:
if record['state'] in store.ACTIVE and record['state'] != 'cancelling':
record['state'] = 'cancelling'
self._save(record)
+ self._interrupt_dependants()
self.wake.set()
return self._view(record)
@@ -469,6 +527,8 @@ class ChainRunner:
self.success.pop(prompt_id, None)
self.cancel_sent.discard(prompt_id)
self._save(record)
+ if record['state'] in ('cancelled', 'failed', 'interrupted'):
+ self._interrupt_dependants()
def step(self):
"""Run one scheduler pass; tests can drive the same loop without sleeping."""
@@ -490,11 +550,16 @@ class ChainRunner:
self._save(record)
if self.active is not None and self.records[self.active]['state'] not in store.ACTIVE:
self.active = None
- while self.active is None and self.queue:
- identity = self.queue.pop(0)
- waiting = self.records[identity]
- if waiting['state'] == 'queued' or waiting['state'] == 'cancelling' and waiting['output_state'] == 'joining':
- self.active = identity
+ self._interrupt_dependants()
+ for queued_id in list(self.queue):
+ waiting = self.records[queued_id]
+ if waiting['state'] not in store.ACTIVE:
+ self.queue.remove(queued_id)
+ elif self.active is None and (waiting['state'] == 'queued'
+ and self._dependency_state(waiting) == 'ready'
+ or waiting['state'] == 'cancelling' and waiting['output_state'] == 'joining'):
+ self.queue.remove(queued_id)
+ self.active = queued_id
identity = self.active
for recovering_id, snapshot in recovering:
if not self.closed.is_set():
diff --git a/freevideo_engine/chain_store.py b/freevideo_engine/chain_store.py
index 468e49e..2c2c5d4 100644
--- a/freevideo_engine/chain_store.py
+++ b/freevideo_engine/chain_store.py
@@ -30,7 +30,7 @@ SPLIT_REASONS = {
MAX_PARTS = 5
-def reference(value):
+def reference(value, queue=False):
if value is None or isinstance(value, str) and value:
return value
if isinstance(value, dict) and set(value) == {'work'}:
@@ -41,6 +41,12 @@ def reference(value):
and '..' not in posix.parts and '..' not in windows.parts
and posix.suffix.lower() == '.mp4'):
return deepcopy(value)
+ if queue and isinstance(value, dict) and set(value) == {'chain', 'block'}:
+ if (isinstance(value['chain'], str) and re.fullmatch(r'c-[0-9a-f]{12}', value['chain'])
+ and isinstance(value['block'], str) and value['block']):
+ return deepcopy(value)
+ if queue:
+ raise ValueError('References must be block IDs, relative MP4 work paths, or queued chain outputs')
raise ValueError('References must be block IDs or relative MP4 work paths without ..')
@@ -73,6 +79,9 @@ def validate(payload):
raise ValueError('Invalid extra data or client ID')
if ('blocks' in payload) == ('seconds' in payload):
raise ValueError('Specify blocks or seconds, not both')
+ queue = payload.get('queue', False)
+ if type(queue) is not bool:
+ raise ValueError('Queue must be a bool')
planned = split = None
join = payload.get('join', 'seconds' in payload)
if type(join) is not bool:
@@ -118,7 +127,7 @@ def validate(payload):
part = row.get('part', identity)
if not isinstance(part, str):
raise ValueError('Block part must be a string')
- after, before = reference(row.get('after')), reference(row.get('before'))
+ after, before = reference(row.get('after'), queue), reference(row.get('before'), queue)
text, frames = row.get('text'), row.get('frames')
if not isinstance(text, str) or len(text) > 20000:
raise ValueError('Block text must be a string of at most 20000 characters')
@@ -164,6 +173,8 @@ def validate(payload):
snapshot=dict(prompt=deepcopy(prompt), extra_data=deepcopy(extra)), blocks=blocks,
plan=planned, join=join, output=None, output_state='pending' if join else 'none',
output_error=None, output_blocks=0, split=split)
+ if queue:
+ record['queue'] = True
try:
json.dumps(record, allow_nan=False)
except (ValueError, TypeError, OverflowError) as error:
@@ -194,7 +205,9 @@ def summary(record):
def detail(record):
- return dict(summary(record), blocks=deepcopy(record['blocks']), plan=deepcopy(record['plan']))
+ return dict(summary(record), blocks=deepcopy(record['blocks']), plan=deepcopy(record['plan']),
+ generation_size={key: record['snapshot']['prompt'][record['node']]['inputs'].get(key)
+ for key in ('width', 'height')})
def save(directory, record):
@@ -277,7 +290,7 @@ def load_all(directory, limit=200, *, identity=None):
continuation=None, skip_frames=0, skip_audio_samples=0)
snapshot = record['snapshot']
normalized = validate(dict(node=record['node'], client_id=record['client_id'], **snapshot,
- blocks=[{key: row[key] for key in ('id', 'part', 'after', 'before', 'fill', 'context')}
+ queue=record.get('queue', False), blocks=[{key: row[key] for key in ('id', 'part', 'after', 'before', 'fill', 'context')}
| row['inputs']
for row in record['blocks']], join=record['join']))
if record['state'] not in CHAIN_STATES:
diff --git a/web/editor.js b/web/editor.js
index d0dda2c..eab8237 100644
--- a/web/editor.js
+++ b/web/editor.js
@@ -337,8 +337,8 @@ export function openEditor({api, app, from = null, panel = {}, components: given
} catch (value) { if (value instanceof model.EditError) return null; throw value; }
}
const jobActive = () => generationBusy || [...jobStatuses.values()].some(job => ['queued', 'running', 'cancelling'].includes(job.state));
- // Parts pressed while they cannot be sent yet: the engine is still starting, or they continue a part
- // that is not finished. They are planned at once, shown queued, and sent in order when they can be.
+ // Only an engine that has not answered yet leaves parts held locally.
+ // Once ready, the server owns waiting parts and their dependencies.
// Parts, and ranges not yet planned into parts, whose request is being sent.
const held = [], sending = new Set(), sendingRanges = new Set();
const overlaps = (a, b) => a.start < b.start + b.frames && a.start + a.frames > b.start;
@@ -352,10 +352,9 @@ export function openEditor({api, app, from = null, panel = {}, components: given
return state.record.tracks[0].clips.some(clip => (generating(clip) || isHeld(clip.id) || sending.has(clip.id))
&& clip.start < range.start + range.frames && clip.start + clip.frames > range.start);
}
- function waitingFor(selection) {
+ function waitingFor() {
if (!store.generationAvailable) return 'ready';
- const before = partBefore(selectionRange(selection).start);
- return generationBusy || generating(before) || before && isHeld(before.id) ? 'before' : null;
+ return null;
}
// The nearest spot at or after the chosen one where a part of the same length fits.
function emptySpot(selection) {
@@ -424,9 +423,8 @@ export function openEditor({api, app, from = null, panel = {}, components: given
const entry = held[0], clips = entry.ids.map(id => model.find(state.record, id)).filter(clip => clip && clip.pending && !generating(clip));
if (!clips.length) { held.shift(); markHeld(entry.ids, false); rememberView(); continue; }
const selection = {kind: 'clip', ids: clips.map(clip => clip.id)}, before = partBefore(Math.min(...clips.map(clip => clip.start)));
- if (generating(before)) break;
held.shift(); markHeld(entry.ids, false); rememberView();
- if (before && !before.source && before.pending) { genbar?.say(t('The part before did not finish, so this one did not start.', '上一段没有完成,这一段没有开始。')); continue; }
+ if (before && !before.source && before.pending && !before.job) { genbar?.say(t('The part before did not finish, so this one did not start.', '上一段没有完成,这一段没有开始。')); continue; }
try { await actions.generate({text: entry.text, selection}); }
catch (value) { const reason = error(value, 'generate'); if (reason && !['chain_too_long', 'chain_part_too_short'].includes(reason.reason)) genbar?.say(reason.say); }
}
@@ -771,8 +769,13 @@ export function openEditor({api, app, from = null, panel = {}, components: given
// leaves whatever the user has moved on to, and what they typed there.
submittedSelection = shown && state.selection === selection ? state.select({kind: 'clip', ids}) : state.selection; await completeSources();
const paths = Object.fromEntries([...store.sources].map(([key, value]) => [key, value.path]));
- const request = model.generationRequest(prepared, ids, {text, paths});
- const follow = continuedSize(model.generationBefore(prepared, ids), store.sources), edit = capability.generationSize || generationSize(state.record);
+ const request = model.generationRequest(prepared, ids, {text, paths, queue: true});
+ const beforeKeys = model.generationBefore(prepared, ids), sizes = new Map(store.sources);
+ for (const key of beforeKeys.filter(key => key?.startsWith('chain:'))) {
+ const job = await store.chain(key.split(':')[1]);
+ sizes.set(key, job?.generation_size);
+ }
+ const follow = continuedSize(beforeKeys, sizes), edit = capability.generationSize || generationSize(state.record);
Object.assign(frozen.prompt[frozen.node].inputs, follow || {});
// Parts made at another size than the edit's have that size's limit.
const limit = (follow && (follow.width !== edit.width || follow.height !== edit.height)
@@ -790,7 +793,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given
const castRows = panel.references?.available() ? panel.references.rows?.() ?? [] : [];
const blocks = request.blocks.map(({before, ...block}, i) => ({...block, seed: seed + i,
text: expandCast(block.text, castRows, {continued: Boolean(block.after)}).text}));
- const job = await store.startChain({...frozen, blocks});
+ const job = await store.startChain({...frozen, blocks, queue: true});
if (!job && !store.generationAvailable) {
if (epoch === projectEpoch) {
const restoreSelection = submittedSelection !== priorSelection && state.selection === submittedSelection;
@@ -823,6 +826,17 @@ export function openEditor({api, app, from = null, panel = {}, components: given
async press(text) {
if (disposed || closing || initializing) return {say: t('Opening the edit…', '正在打开…')};
let selection = state.selection, said = null;
+ if (actions.selectionHeld()) return {say: t('Queued', '排队中')};
+ if (busySelection(selection)) return {say: t('This part is generating.', '这一段正在生成。')};
+ // A preceding POST must supply its durable ID before a dependant can reference it.
+ const epoch = projectEpoch, awaiting = selectionRange(selection);
+ if (generationBusy) {
+ if (awaiting) sendingRanges.add(awaiting);
+ updateTransport();
+ try { while (generationBusy) await generationDone; }
+ finally { sendingRanges.delete(awaiting); }
+ }
+ if (disposed || closing || epoch !== projectEpoch) return;
if (!selectionRange(selection)) return {say: t('Drag on the timeline to choose a part first.', '先在时间线上拖出一段。')};
if (actions.selectionHeld()) return {say: t('Queued', '排队中')};
if (busySelection(selection)) return {say: t('This part is generating.', '这一段正在生成。')};
diff --git a/web/editor_model.js b/web/editor_model.js
index 9bb4507..1fa226c 100644
--- a/web/editor_model.js
+++ b/web/editor_model.js
@@ -690,6 +690,8 @@ function remember(clip, source = null) {
// A range drawn on its own starts a new video, even when it touches the part before it.
const continuing = (clip, before) => !!clip && !!before && before.id !== clip.id && end(before) === clip.start
&& (clip.follows === before.id || (clip.part !== null && clip.part === before.part));
+const chainSource = clip => !!clip.job && !!clip.pending;
+const continuationKey = clip => chainSource(clip) ? `chain:${clip.job}:${clip.id}` : clip.source ? key(clip.source) : null;
const wholeEnd = clip => !!clip.source && clip.extend && clip.in + clip.frames === clip.length;
const partBefore = (record, clip) => trackBy(record, 'v1').clips.find(before => continuing(clip, before)) ?? null;
// Whether the part is drawn attached to the one before it on its track.
@@ -943,7 +945,7 @@ function generationPlan(record, clips) {
return videos.flatMap(video => videoPlan(record, video, replacing));
}
function videoPlan(record, clips, replacing) {
- const first = clips[0], before = trackBy(record, 'v1').clips.find(clip => continuing(first, clip) && wholeEnd(clip));
+ const first = clips[0], before = trackBy(record, 'v1').clips.find(clip => continuing(first, clip) && (chainSource(clip) || wholeEnd(clip)));
// Revealing a predecessor's hidden tail reduces the new visible range.
const credit = clips.every(clip => clip.target_frames !== undefined && !clip.source)
? clips.reduce((sum, clip) => sum + clip.target_frames - clip.frames, 0) : 0;
@@ -998,7 +1000,7 @@ export function prepare(record, ids) {
export function generationBefore(record, ids) {
const clips = generationClips(record, ids);
return generationPlan(record, clips).filter(row => !row.before || !clips.some(clip => clip.id === row.before.id))
- .map(row => row.before ? key(row.before.source) : null);
+ .map(row => row.before ? continuationKey(row.before) : null);
}
// An imported video is continued from the copy the editor plays, which is upright and at 24 fps.
function continuedPath(source, path) {
@@ -1006,7 +1008,7 @@ function continuedPath(source, path) {
check(path === `FreeVideo/.editor/imports/${source.id}/preview.mp4`, 'source');
return path;
}
-export function generationRequest(record, ids, {text, paths = {}} = {}) {
+export function generationRequest(record, ids, {text, paths = {}, queue = false} = {}) {
check(object(paths), 'source');
if (text !== undefined) check(validText(text), 'text');
const clips = generationClips(record, ids), rows = generationPlan(record, clips), plan = {}, blocks = [];
@@ -1014,7 +1016,8 @@ export function generationRequest(record, ids, {text, paths = {}} = {}) {
let after = null;
if (row.before) {
if (clips.some(clip => clip.id === row.before.id)) after = row.before.id;
- else { const path = paths[key(row.before.source)]; check(typeof path === 'string' && path.length > 0, 'source', 'missing source path'); after = {work: continuedPath(row.before.source, path)}; }
+ else if (queue && chainSource(row.before)) after = {chain: row.before.job, block: row.before.id};
+ else { const path = paths[continuationKey(row.before)]; check(typeof path === 'string' && path.length > 0, 'source', 'missing source path'); after = {work: continuedPath(row.before.source, path)}; }
}
blocks.push({id: row.clip.id, part: row.clip.part ?? row.clip.id, after, before: null, text: text ?? row.clip.text, frames: row.request_frames});
plan[row.clip.id] = {target_frames: row.target_frames, net_frames: row.net_frames, request_frames: row.request_frames, audio: row.clip.link};