Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

Editor: a continued part can start with a preview and carry on with the second pass

Bowen Xueec2733788254
diff --git a/freevideo_engine/comfy_bridge.py b/freevideo_engine/comfy_bridge.py
index cf1fa8a..012ea9a 100644
--- a/freevideo_engine/comfy_bridge.py
+++ b/freevideo_engine/comfy_bridge.py
@@ -1274,6 +1274,12 @@ def generate(prompt, width, height, seconds, seed, output_directory, *,
width, height = previous['geometry']['width'], previous['geometry']['height']
seconds, seed = previous['geometry']['seconds'], previous['seed']
two_pass, preview = True, False
+ # A continued preview's second pass continues the same video the same way.
+ continue_from = argument(arguments, '--continue-from')
+ continue_from = None if continue_from is None else Path(continue_from)
+ frames = int(argument(arguments, '--frames')) if '--frames' in arguments else None
+ first_frame_reference = '--first-frame-reference' in arguments
+ voice_reference = '--voice-reference' in arguments
base_steps = int(argument(arguments, '--base-steps', 8))
refine_steps = int(argument(arguments, '--refine-steps', 3))
# The second pass never uses a step cache; the preview's own command keeps its flag.
@@ -1296,7 +1302,7 @@ def generate(prompt, width, height, seconds, seed, output_directory, *,
def check_ending():
from .continuation_tail import check
from .two_pass import sampling_canvas
- check(continue_from, sampling_canvas(canvas)[0])
+ check(continue_from, sampling_canvas(canvas, staged=bool(preview or upscale))[0])
if step_cache == 'fbcache' and machine.get('device_backend') == 'mps':
# Only the audio-video EasyCache is ported to Mac: the request samples every step,
@@ -1304,24 +1310,31 @@ def generate(prompt, width, height, seconds, seed, output_directory, *,
logging.warning('[FreeVideo] The first-block cache is not available on Mac yet; sampling every step.')
step_cache = None
if continue_from is not None:
- validate_route(backend=machine.get('device_backend', 'cuda'))
- from .continuation import MISSING, prepare, summary
+ validate_route(preview=preview, refine_from=refine_from, backend=machine.get('device_backend', 'cuda'))
+ from .continuation import MISSING, prepare, restore_snapshot, summary
from .two_pass import sampling_canvas
- # A continuation takes no media or preview, so its sampled size is known here; the snapshot
+ # A continuation takes no media, so its sampled size is known here; the snapshot
# of the previous segment was written for that canvas, not for the requested size.
- try:
- continuation = prepare(continue_from, sampling_canvas(canvas)[0], directory=root / 'continuations')
- except ValueError as error:
- if str(error) != MISSING:
- raise
- # A video with no stored context (single-pass, an earlier version, a replaced file) is
- # continued from its own ending, which the engine reads and encodes. Only a video whose
- # ending cannot be used stops here, before anything is loaded.
- check_ending()
+ started = sampling_canvas(canvas, staged=bool(preview or upscale))[0]
+ preview_encoded = False
+ if upscale:
+ from .preview import continuation_snapshot, source as preview_source
+ snapshot, digest, preview_encoded = continuation_snapshot(preview_source(upscale[0]), continue_from)
+ continuation = restore_snapshot(snapshot, digest, started)
+ else:
+ try:
+ continuation = prepare(continue_from, started, directory=root / 'continuations')
+ except ValueError as error:
+ if str(error) != MISSING:
+ raise
+ # A video with no stored context (single-pass, an earlier version, a replaced file) is
+ # continued from its own ending, which the engine reads and encodes. Only a video whose
+ # ending cannot be used stops here, before anything is loaded.
+ check_ending()
# The engine decides the same way from the same record: whether the film's first frame is there to use.
# An ending encoded from the file has no first frame of the film to give.
anchored = False
- if continue_from is not None and continuation is not None and first_frame_reference:
+ if continue_from is not None and continuation is not None and first_frame_reference and not preview_encoded:
from .continuation import chain_first_frame_picture
try:
anchored = chain_first_frame_picture(continuation, continue_from) is not None
@@ -1365,7 +1378,7 @@ def generate(prompt, width, height, seconds, seed, output_directory, *,
command.append('--first-frame-reference')
if voice_reference:
command.append('--voice-reference')
- if continuation is not None:
+ if continuation is not None and not upscale:
from .continuation import persist_snapshot
continuation = persist_snapshot(continuation, run / 'continuation-source.safetensors')
command += ['--continuation-snapshot', continuation['snapshot'],
@@ -1378,7 +1391,7 @@ def generate(prompt, width, height, seconds, seed, output_directory, *,
command += ['--' + field.replace('_', '-'), str(reserve)]
if preview:
command.append('--preview')
- if hide_tail:
+ if hide_tail and '--hide-tail' not in command:
command.append('--hide-tail')
state = {'status': 'starting', 'geometry': canvas, 'seed': seed, 'installation': str(root),
'command': command, 'output': str(output), 'source': str(source), 'resources': resources}
diff --git a/freevideo_engine/comfy_chain.py b/freevideo_engine/comfy_chain.py
index 5d15cfb..5d1207f 100644
--- a/freevideo_engine/comfy_chain.py
+++ b/freevideo_engine/comfy_chain.py
@@ -274,6 +274,12 @@ def continuation_supported():
return supported()[0]
+def continuation_preview():
+ """Whether a part that continues can be made as a preview first, on this device."""
+ from .continuation import previews_supported
+ return continuation_supported() and previews_supported()
+
+
def continue_seconds():
"""The length a new continuation starts with where this device sets it: short parts on Apple silicon, where
a full part takes long and often does not fit; None elsewhere, where the page goes by the measured limits."""
@@ -470,7 +476,7 @@ def register(host_factory=None):
@server.routes.get('/freevideo/chains')
async def listing(request):
value = {'chains': runner.list(asking(request)), 'continuation': continuation_available(),
- 'continue_seconds': continue_seconds()}
+ 'continue_seconds': continue_seconds(), 'continuation_preview': continuation_preview()}
if fake:
value['fake'] = True
return response(value)
diff --git a/freevideo_engine/comfy_nodes.py b/freevideo_engine/comfy_nodes.py
index 9733125..bffdce0 100644
--- a/freevideo_engine/comfy_nodes.py
+++ b/freevideo_engine/comfy_nodes.py
@@ -177,10 +177,11 @@ class FreeVideoGenerate(io.ComfyNode):
try:
options = request_options(continue_from, node_frames(frames))
if options.get('continue_from'):
- # A continued part turns on two passes, skips the preview and takes keyframes as pictures (execute).
- two_pass, preview = True, False
+ # A continued part turns on two passes and takes keyframes as pictures; only NVIDIA keeps its preview (execute).
+ from .continuation import previews_supported
+ two_pass, preview = True, preview is True and previews_supported()
base_steps, refine_steps = part_steps(base_steps, refine_steps, step_cache)[:2]
- options.update(two_pass=True, preview=False, refine_from=refine_from or None,
+ options.update(two_pass=True, preview=preview, refine_from=refine_from or None,
media=dict(first=False, last=False, conditioning_info=False))
comfy_bridge.validate_request(text, width, height, seconds, seed, **options,
allow_empty_prompt=allow_empty_prompt is True)
@@ -231,8 +232,10 @@ class FreeVideoGenerate(io.ComfyNode):
two_pass = True
notes.append('Continuing a video needs "Two-pass acceleration", so it is on for this part.')
if options.get('continue_from') and preview:
- preview = False
- notes.append('A continued part is made in full, without stopping at the preview.')
+ from .continuation import previews_supported
+ if not previews_supported():
+ preview = False
+ notes.append('A continued part is made in full, without stopping at the preview.')
from .two_pass import validate_steps
if not refine_from:
validate_steps(base_steps, refine_steps, two_pass)
diff --git a/freevideo_engine/continuation.py b/freevideo_engine/continuation.py
index c268e91..c47273d 100644
--- a/freevideo_engine/continuation.py
+++ b/freevideo_engine/continuation.py
@@ -86,8 +86,7 @@ def audio_context():
return validate_audio_context(os.environ.get('FREEVIDEO_CONTINUATION_AUDIO', AUDIO_CONTEXT_DEFAULT))
-def supported():
- """Platform capability, without loading Torch or probing a GPU."""
+def _machine_backend():
from .paths import data_root
try:
machine = json.loads((data_root() / 'machine.json').read_text(encoding='utf-8'))
@@ -95,11 +94,21 @@ def supported():
machine = {}
if not isinstance(machine, dict):
machine = {}
- if machine.get('device_backend', 'cuda') not in ('cuda', 'mps'):
+ return machine.get('device_backend', 'cuda')
+
+
+def supported():
+ """Platform capability, without loading Torch or probing a GPU."""
+ if _machine_backend() not in ('cuda', 'mps'):
return False, 'unsupported_device'
return True, None
+def previews_supported(backend=None):
+ """A continued part can stop at its first pass and be finished from it: NVIDIA only."""
+ return (backend or _machine_backend()) == 'cuda'
+
+
def validate_frames(frames):
if type(frames) is not int or frames < 39 or (frames - 5) % 17:
raise ValueError(INVALID_FRAMES)
@@ -131,9 +140,9 @@ def validate_route(*, task='t2va', two_pass=True, preview=False, refine_from=Non
valid_task = not sound and task in (('ref2va', 'ref2va_av') if anchor else ('t2va', 'ref2va', 'ref2va_audio', 'ref2va_av'))
if not supported()[0] or backend not in (None, 'cuda', 'mps'):
raise ValueError(UNSUPPORTED_DEVICE)
- if refine_from:
+ if refine_from and not previews_supported(backend):
raise ValueError(UNSUPPORTED_REFINE)
- if not two_pass or preview:
+ if not two_pass or (preview and not previews_supported(backend)):
raise ValueError(UNSUPPORTED_PASSES)
# Keyframes reach a continued part as pictures (comfy_media.continued); here they are a request error.
if (not valid_task or any(media.get(key) for key in ('first', 'last', 'conditioning_info'))
diff --git a/freevideo_engine/generate.py b/freevideo_engine/generate.py
index b91e6f4..160c32a 100644
--- a/freevideo_engine/generate.py
+++ b/freevideo_engine/generate.py
@@ -540,7 +540,12 @@ def _run(args):
preview=preview, refine_from=getattr(args, 'refine_from', None), media=media,
original_audio=original_audio is not None and 'segment' in original_audio)
# Reject absent/mismatched context before text encoding or model loading.
- if getattr(args, 'continuation_snapshot', None):
+ preview_encoded = False
+ if refine_source:
+ from .preview import continuation_snapshot
+ snapshot, digest, preview_encoded = continuation_snapshot(refine_source, continue_from)
+ continuation = restore_snapshot(snapshot, digest, canvas)
+ elif getattr(args, 'continuation_snapshot', None):
continuation = restore_snapshot(args.continuation_snapshot,
getattr(args, 'continuation_snapshot_sha256', None), canvas)
else:
@@ -566,7 +571,8 @@ def _run(args):
from .continuation import anchor_media, chain_first_frame_picture
try:
# A context encoded from the file carries no first frame; whoever planned this run saw none either.
- kept = None if 'tail' in continuation else chain_first_frame_picture(continuation, continue_from)
+ kept = (None if 'tail' in continuation or preview_encoded
+ else chain_first_frame_picture(continuation, continue_from))
except ValueError:
kept = None
first_frame['reason'] = 'unreadable'
@@ -577,9 +583,15 @@ def _run(args):
if args.out.resolve().exists():
raise FileExistsError(args.out.resolve())
args.out.resolve().parent.mkdir(parents=True, exist_ok=True)
- continue_anchor = args.out.resolve().with_suffix('.first-frame.png')
picture, picture_width, picture_height = kept
- continue_anchor.write_bytes(picture)
+ # A second pass names the same picture its preview was made with.
+ continue_anchor = None
+ if refine_source:
+ from .preview import continuation_anchor
+ continue_anchor = continuation_anchor(refine_source)
+ if continue_anchor is None:
+ continue_anchor = args.out.resolve().with_suffix('.first-frame.png')
+ continue_anchor.write_bytes(picture)
media = anchor_media(media, continue_anchor)
validate_route(task=getattr(args, 'task', None) or 'ref2va', anchor=True, two_pass=two_pass,
preview=preview, refine_from=getattr(args, 'refine_from', None), media=media)
@@ -667,7 +679,7 @@ def _run(args):
validate_route(task=profile['engine'].get('task', 't2va'), anchor=anchored,
original_audio=original_audio is not None)
planned = continuation_plan(canvas, two_pass, base_steps=base_steps, refine_steps=refine_steps,
- scale=upscale_factor(args))
+ staged=bool(preview or refine_source), scale=upscale_factor(args))
if two_pass and not planned['enabled'] and base_steps != 8:
raise ValueError(planned['reason'] + ' A continuation needs two-pass generation.')
continuation_first = planned['first']
diff --git a/freevideo_engine/preview.py b/freevideo_engine/preview.py
index 028405c..f868bd1 100644
--- a/freevideo_engine/preview.py
+++ b/freevideo_engine/preview.py
@@ -4,6 +4,7 @@ The first pass is kept beside the preview MP4, so an upscale continues from the
same latents: the learned latent upscale, the refinement steps and a full
decode, as the uninterrupted request would have run them. Torch-free.
"""
+import hashlib
import json
from pathlib import Path
@@ -51,6 +52,31 @@ def source(output, native=False):
return result
+def continuation_snapshot(source_paths, continue_from=None):
+ """The ending a continued preview was made from: (path, sha256, encoded from the file).
+
+ Its second pass continues that same record, whatever the store holds now."""
+ request = json.loads(Path(source_paths['request']).read_text(encoding='utf-8'))
+ value = request.get('continuation')
+ if not isinstance(value, dict):
+ raise ValueError('This preview does not continue a video. Generate the preview again.')
+ if continue_from is not None and Path(value.get('source_video', '')).resolve() != Path(continue_from).resolve():
+ raise ValueError('This preview continues another video. Generate the preview again.')
+ tail = value.get('tail')
+ path = Path((tail if isinstance(tail, dict) else value).get('snapshot') or '')
+ if not path.is_file():
+ raise ValueError('This preview no longer has the video ending it continues. Generate the preview again.')
+ return str(path), hashlib.sha256(path.read_bytes()).hexdigest(), isinstance(tail, dict)
+
+
+def continuation_anchor(source_paths):
+ """The first-frame picture a continued preview was made with, when it was and is still there."""
+ request = json.loads(Path(source_paths['request']).read_text(encoding='utf-8'))
+ anchor = (request.get('continuation') or {}).get('anchor')
+ path = Path(anchor['path']) if isinstance(anchor, dict) and anchor.get('path') else None
+ return path if path is not None and path.is_file() else None
+
+
def upscale_reads(cache, video, upscaler_checkpoint):
"""Files the upscale of this preview reads from disk, in the order it needs them.
diff --git a/web/editor.js b/web/editor.js
index 3788593..9fc8c16 100644
--- a/web/editor.js
+++ b/web/editor.js
@@ -1044,30 +1044,30 @@ export function openEditor({api, app, from = null, panel = {}, components: given
else genbar?.update(mostRecentJob ? jobStatuses.get(mostRecentJob) ?? null : null);
},
onPass: listener => panel.passMode?.subscribe(listener) || (() => {}),
- // A part made as a half-resolution preview: only one that stands alone, as the next part continues from
- // this one's last frames, which the second pass changes.
+ // A part made as a half-resolution preview: one part only, as the next part continues from this one's last
+ // frames, which the second pass changes. It may continue the part before it where the engine can.
previewFirst(selection = state.selection) {
if (actions.passMode()?.mode !== 'preview' || music.target(selection)) return false;
if ((state.record.settings?.base_steps ?? 8) !== 8) return false;
const run = partRun(selection), chosen = run ? clipChoice(run) : selection, range = selectionRange(chosen); if (!range) return false;
- if (continuesEarlier(range)) return false;
+ if (continuesEarlier(range) && !store.continuationPreview) return false;
try {
// The parts Generate would send, planned as it plans them.
const selected = selectionClips(chosen), planned = selected.length && selected.every(clip => clip.pending && !clip.source);
const next = planned ? state.record : model.planRange(state.record, {...range, target: 'both', maxSeconds: capability.maxSeconds, maxFollowSeconds: capability.maxFollowSeconds});
const ids = planned ? selected.map(clip => clip.id) : next.tracks[0].clips.filter(clip => clip.start >= range.start && clip.start < range.start + range.frames
&& clip.pending && !['queued', 'running'].includes(clip.pending)).map(clip => clip.id);
- return ids.length === 1 && !model.generationBefore(model.prepare(next, ids), ids).some(Boolean);
+ return ids.length === 1 && (store.continuationPreview || !model.generationBefore(model.prepare(next, ids), ids).some(Boolean));
} catch (value) { if (value instanceof model.EditError) return false; throw value; }
},
- // The chosen preview part, shown from its first frame, whose second pass can replace it in place.
+ // The chosen preview part, shown from its first new frame, whose second pass can replace it in place.
secondPassTarget(selection = state.selection) {
if (selection?.kind !== 'clip' || !store.generationAvailable) return null;
const clips = selectionClips(selection);
if (clips.length !== 1) return null;
const clip = clips[0], mate = clip.link ? model.find(state.record, clip.link) : null;
// From the start of the preview: the second pass makes the same frames again, so a part cut short at its end still shows the same moment.
- if (!clip.source || clip.pending || mate?.pending || clip.in !== 0 || clip.frames > clip.length) return null;
+ if (!clip.source || clip.pending || mate?.pending || clip.in !== (clip.splice?.frames ?? 0) || clip.frames > clip.length) return null;
const metadata = store.sources.get(sourceKey(clip.source));
return metadata?.preview === true && metadata.path ? {clip, path: metadata.path} : null;
},
@@ -1155,8 +1155,8 @@ export function openEditor({api, app, from = null, panel = {}, components: given
}
const follow = continuedSize(beforeKeys, sizes), edit = capability.generationSize || generationSize(state.record);
Object.assign(frozen.prompt[frozen.node].inputs, follow || {});
- // A preview is one part that continues nothing; any other part samples both passes, whatever the panel shows.
- const inputs = frozen.prompt[frozen.node].inputs, alone = request.blocks.length === 1 && !beforeKeys.some(Boolean);
+ // A preview is one part; any other part samples both passes, whatever the panel shows.
+ const inputs = frozen.prompt[frozen.node].inputs, alone = request.blocks.length === 1 && (store.continuationPreview || !beforeKeys.some(Boolean));
if (secondPass) {
if (!alone) throw new model.EditError('unsupported');
Object.assign(inputs, {two_pass: true, preview: false, refine_from: secondPass});
diff --git a/web/editor_store.js b/web/editor_store.js
index 30974eb..40441f2 100644
--- a/web/editor_store.js
+++ b/web/editor_store.js
@@ -178,6 +178,8 @@ export function createEditorStore(api, {t = en => en, notify = () => {}, onGener
let recovered = null, capabilityPoll = null;
// The length a new continuation starts with where the engine names one (a device that continues in short parts).
let continueSeconds = null;
+ // Whether a part that continues can be made as a preview first, once the engine has said so.
+ let continuationPreview = false;
let generationAvailable = false, generationGone = false, generationProbe = null, generationRetry = null, generationAttempts = 0, generationStalled = false, generationAsking = false;
let disposed = false; const uploads = new Set();
const session = holdSession();
@@ -264,6 +266,7 @@ export function createEditorStore(api, {t = en => en, notify = () => {}, onGener
if (activeProject === id) { activeProject = null; clock.clearTimeout(sessionTimer); sessionTimer = null; }
},
get continueSeconds() { return standIn ? standIn.continueSeconds ?? null : continueSeconds; },
+ get continuationPreview() { return standIn ? standIn.continuationPreview === true : continuationPreview; },
get generationAvailable() { return standIn ? standIn.generationAvailable ?? true : generationAvailable; },
// The server said for certain it cannot generate (not just a missed answer that is asked again).
get generationGone() { return !standIn && generationGone; },
@@ -301,6 +304,7 @@ export function createEditorStore(api, {t = en => en, notify = () => {}, onGener
if (!response.ok) return null;
const body = await response.json();
if (Number.isInteger(body?.continue_seconds) && body.continue_seconds > 0) continueSeconds = body.continue_seconds;
+ continuationPreview = body?.continuation_preview === true;
return Array.isArray(body?.chains) ? true : null;
})
.catch(() => null), timeout]);