Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.
"""Join finished chain blocks end to end into one saved video.
The picture is copied packet by packet, except for a few closing frames when
needed to reach an exact requested length with matching codec parameters. The
sound is decoded and encoded once as a single stream: every block's sound
follows the previous block's sound, not its own first frame, which keeps a
long chain in step.
Original soundtracks instead bypass every block's sound and mux the selected
source range once, with a sample-identity receipt.
"""
from datetime import datetime
from fractions import Fraction
import json
import logging
import os
from pathlib import Path
import shutil
import uuid
FPS = 24
SEAM_TAIL = 12000
SEAM_HEAD = 240
class JoinCancelled(InterruptedError):
pass
class _VoiceFailed(Exception):
"""The optional voice step failed partway: the join is done again without it."""
def _voice_step(call, *args):
try:
return call(*args)
except JoinCancelled:
raise
except Exception as error:
raise _VoiceFailed() from error
class _ExactEndFailed(Exception):
pass
def _check(cancelled):
if cancelled is not None and cancelled():
raise JoinCancelled()
def _shape(stream):
context = stream.codec_context
return (context.name, context.width, context.height, context.pix_fmt, stream.time_base,
stream.average_rate, bytes(context.extradata or b''))
def _last_kept(path, wanted):
"""The frame a file can end on at or just after `wanted` frames, or None to keep all of it.
Packets are copied, not encoded, so a file may only end on a frame that later frames do not
complete: one that is shown after everything decoded before it. That is at most a few frames on.
"""
import av
with av.open(str(path)) as container:
video = container.streams.video[0]
step = int(Fraction(1, video.average_rate) / video.time_base)
start = latest = None
for packet in container.demux(video):
if packet.dts is None:
continue
if start is None:
start = video.start_time if video.start_time is not None else packet.pts
if latest is None or packet.pts > latest:
latest = packet.pts
if (latest - start) // step >= wanted - 1:
return (latest - start) // step
return None
def _exact_tail(source, target, wanted, cancelled=None):
"""Prepare at most three closing frames, or leave the packet-copy ending alone."""
import av
temporary = target.with_name('.join-tail-' + uuid.uuid4().hex + '.mp4')
try:
with av.open(str(source)) as container:
video = container.streams.video[0]
shape = _shape(video)
if shape[0] != 'h264':
return None
step = int(Fraction(1, video.average_rate) / video.time_base)
begin = video.start_time
prefix = count = 0
latest = previous_dts = None
for packet in container.demux(video):
_check(cancelled)
if packet.dts is None:
continue
if begin is None:
begin = packet.pts
if latest is None or packet.pts > latest:
latest = packet.pts
shown = (latest - begin) // step
if shown >= wanted - 1:
break
prefix = shown + 1
count += 1
previous_dts = packet.dts
if count != prefix or not 1 <= wanted - prefix <= 3:
return None
# Seek to the preceding key frame; only the remaining shown frames are encoded.
container.seek(begin + prefix * step, stream=video, backward=True)
with av.open(str(temporary), 'w', format='mp4') as output:
tail = output.add_stream('libx264', rate=FPS)
tail.width, tail.height, tail.pix_fmt = shape[1:4]
tail.time_base = shape[4]
# decode_to_file's encode_video uses these libx264 defaults and yuv420p.
encoded = 0
for frame in container.decode(video):
_check(cancelled)
position = (frame.pts - begin) // step
if position < prefix:
continue
if position != prefix + encoded:
return None
frame.pts, frame.time_base = encoded, Fraction(1, FPS)
frame.pict_type = av.video.frame.PictureType.NONE
for packet in tail.encode(frame):
output.mux(packet)
encoded += 1
if encoded == wanted - prefix:
break
if encoded != wanted - prefix:
return None
for packet in tail.encode(None):
output.mux(packet)
with av.open(str(temporary)) as container:
tail = container.streams.video[0]
if _shape(tail) != shape:
return None
packets = [packet for packet in container.demux(tail) if packet.dts is not None]
if len(packets) != encoded or not packets[0].is_keyframe:
return None
if sorted(packet.pts for packet in packets) != [number * step for number in range(encoded)]:
return None
# A one- or two-frame encode has less delay than the block. Keep the
# copied decoding cadence while leaving the new display timestamps unchanged.
shift = previous_dts - begin + step - (packets[0].dts + prefix * step)
for packet in packets:
packet.dts += shift
if packet.dts > packet.pts:
return None
stamps = [previous_dts - begin] + [packet.dts + prefix * step for packet in packets]
if any(later <= earlier for earlier, later in zip(stamps, stamps[1:])):
return None
_check(cancelled)
return prefix, packets
except JoinCancelled:
raise
except Exception:
return None
finally:
temporary.unlink(missing_ok=True)
def join_files(sources, target, cancelled=None, frames=None, leads=None, expected=None, speaking=None, *,
original_audio=None):
"""Write `target` from `sources` in order; returns the joined frame and sample counts.
`frames`: stop at this length when a compatible short tail can close it exactly,
otherwise keep the packet-copy ending at or just after it.
`leads`: optional `leads(following, previous)` returning a seam lead kept by the chain;
the shared continuation record is read only when it has none.
`expected`: how many seams should be repaired; only reported back, the caller decides.
The returned `seam_notes` list describes unrepaired seams, numbered from one,
without source paths or exception messages.
`original_audio`: mux the original selection and close the picture on its
source-sample clock; a rounded completed prefix keeps its existing frames.
"""
sources = list(sources)
try:
return _join_ending(sources, target, cancelled, frames, leads, expected, speaking, original_audio)
except _VoiceFailed:
logging.warning('[FreeVideo] The voice of the parts could not be evened out; they are joined as they are',
exc_info=True)
return _join_ending(sources, target, cancelled, frames, leads, expected, None, original_audio)
def _join_ending(sources, target, cancelled, frames, leads, expected, speaking, original_audio=None):
try:
return _join_files(sources, target, cancelled, frames, leads=leads, expected=expected, speaking=speaking,
original_audio=original_audio)
except _ExactEndFailed:
# A late mux or validation error must also leave the original ending available.
return _join_files(sources, target, cancelled, frames, exact=False, leads=leads, expected=expected,
speaking=speaking, original_audio=original_audio)
def _difference(number, shape, other):
"""Why part `number` cannot join part 1, in words."""
if shape[1:3] != other[1:3]:
return f'Part {number} is {shape[1]}x{shape[2]} but part 1 is {other[1]}x{other[2]}'
for name, index in (('codec', 0), ('pixel format', 3)):
if shape[index] != other[index]:
return f'Part {number} uses {shape[index]} {name} but part 1 uses {other[index]}'
if shape[4:6] != other[4:6]:
return f'Part {number} does not have the frame rate of part 1'
return f'Part {number} was encoded differently from part 1'
def _conform(source, shape, directory, cancelled):
"""A copy of `source` with the first block's picture settings, or None when it cannot have them.
The picture is encoded again with the encoder's defaults; the sound is copied packet by packet.
"""
import av
from .comfy_metadata import _copy_stream
temporary = directory / ('.join-conform-' + uuid.uuid4().hex + '.mp4')
try:
with av.open(str(source)) as container, av.open(str(temporary), 'w', format='mp4') as output:
video = container.streams.video[0]
if abs(float(video.average_rate or 0) - FPS) > .01:
return None
picture = output.add_stream('libx264', rate=FPS)
picture.width, picture.height, picture.pix_fmt = shape[1:4]
picture.time_base = shape[4]
sound = container.streams.audio[0] if container.streams.audio else None
copy = _copy_stream(output, sound) if sound is not None else None
number = 0
for packet in container.demux(video, *([sound] if sound is not None else [])):
_check(cancelled)
if packet.stream is sound:
if packet.dts is not None:
packet.stream = copy
output.mux(packet)
continue
for frame in packet.decode():
frame = frame.reformat(width=shape[1], height=shape[2], format=shape[3])
frame.pts, frame.time_base = number, Fraction(1, FPS)
frame.pict_type = av.video.frame.PictureType.NONE
for encoded in picture.encode(frame):
output.mux(encoded)
number += 1
for encoded in picture.encode(None):
output.mux(encoded)
with av.open(str(temporary)) as check:
if not number or _shape(check.streams.video[0]) != shape:
temporary.unlink(missing_ok=True)
return None
return temporary
except JoinCancelled:
temporary.unlink(missing_ok=True)
raise
except Exception:
temporary.unlink(missing_ok=True)
logging.warning('[FreeVideo] A part could not be matched to the first part', exc_info=True)
return None
def _join_files(sources, target, cancelled, frames, exact=True, leads=None, expected=None, speaking=None,
original_audio=None):
import av
from .comfy_metadata import _copy_stream
audio_lead_in = apply_audio_lead_in = None
if original_audio is None:
try:
from .comfy_continuation import audio_lead_in, apply_audio_lead_in
except ImportError:
audio_lead_in = apply_audio_lead_in = None
if leads is not None and audio_lead_in is not None:
stored = audio_lead_in
def audio_lead_in(video, previous=None):
if expected is not None:
# The caller named the seams that must be repaired and keeps their sound itself;
# a shared record for any other pair of videos must not stand in for them.
return leads(video, previous)
return leads(video, previous) or stored(video, previous=previous)
sources, target = [Path(source) for source in sources], Path(target)
if len(sources) < (2 if original_audio is None else 1):
raise ValueError('Joining needs at least two videos')
if target.exists() or target.is_symlink():
raise ValueError('The joined video already exists')
target.parent.mkdir(parents=True, exist_ok=True)
temporary = target.with_name('.join-' + uuid.uuid4().hex + '.mp4')
wanted, frames, samples = frames, 0, 0
used_tail = False
blocks_joined = 0
conformed = []
try:
with av.open(str(sources[0])) as first:
if len(first.streams.video) != 1:
raise ValueError('A block has no single picture stream')
shape = _shape(first.streams.video[0])
sound = first.streams.audio[0] if first.streams.audio and original_audio is None else None
rate = sound.codec_context.sample_rate if sound is not None else None
layout = sound.codec_context.layout.name if sound is not None else None
bit_rate = max(sound.codec_context.bit_rate or 0, 192000) if sound is not None else None
step = Fraction(1, shape[5]) / shape[4]
if shape[5] != FPS or step.denominator != 1:
raise ValueError('The blocks are not 24 frames per second videos')
step = int(step)
originals = {}
for number, source in enumerate(sources[1:], 2):
with av.open(str(source)) as container:
if len(container.streams.video) != 1 or bool(container.streams.audio) != (sound is not None):
continue
other = _shape(container.streams.video[0])
if other == shape:
continue
copy = _conform(source, shape, target.parent, cancelled)
if copy is None:
raise ValueError(_difference(number, other, shape))
conformed.append(copy)
originals[str(copy)] = str(source)
sources[number - 1] = copy
if originals:
# Seam sound records belong to the blocks as made, not to the matched copies.
lead_reader = audio_lead_in
if lead_reader is not None:
def audio_lead_in(video, previous=None):
return lead_reader(originals.get(str(video), video),
originals.get(str(previous), previous) if previous is not None else None)
voice = _voice(sources, speaking, cancelled) if sound is not None else None
with av.open(str(temporary), 'w', format='mp4',
options={'movflags': 'use_metadata_tags+faststart'}) as output:
picture = audio = fifo = joined_audio = None
for index, source in enumerate(sources):
_check(cancelled)
if wanted is not None and frames >= wanted and picture is not None:
break
last = _last_kept(source, wanted - frames) if wanted is not None else None
with av.open(str(source)) as container:
if len(container.streams.video) != 1 or _shape(container.streams.video[0]) != shape:
raise ValueError('The blocks differ in size or encoding')
if original_audio is None and bool(container.streams.audio) != (sound is not None):
raise ValueError('Only some of the blocks have sound')
video = container.streams.video[0]
tail = (_exact_tail(source, target, wanted - frames, cancelled)
if exact and last is not None and last + 1 > wanted - frames else None)
copied_last = tail[0] - 1 if tail is not None else last
if tail is not None:
used_tail = True
last = wanted - frames - 1
if picture is None:
picture = _copy_stream(output, video)
if sound is not None:
audio = output.add_stream('aac', rate=rate)
audio.layout = layout
audio.bit_rate = bit_rate
audio.codec_context.time_base = Fraction(1, rate)
fifo = av.AudioFifo()
joined_audio = _JoinedAudio(output, audio, fifo, rate, layout,
audio_lead_in, apply_audio_lead_in, cancelled, voice)
joined_audio.seams_total = len(sources) - 1
streams = [video] + ([container.streams.audio[0]] if sound is not None else [])
shift, count = None, 0
# A decoder returns the encoder's padding after the last sample; the stream's own
# length says where the sound ends. Left in, every block would end 20 ms late.
length = streams[-1].duration if sound is not None else None
limit = round(length * streams[-1].time_base * rate) if length is not None else None
if joined_audio is not None:
following = sources[index + 1] if last is None and index + 1 < len(sources) else None
# Selecting the final frame of a whole block leaves its existing sound limit alone.
total_frames = (frames + last + 1 if last is not None
and (not video.frames or last + 1 < video.frames) else None)
joined_audio.start(source, streams[-1], limit, last, following, total_frames)
for packet in container.demux(*streams):
if packet.dts is None:
continue
if packet.stream is video:
if shift is None:
# The first packet of a file is its first shown frame's key frame.
begin = video.start_time if video.start_time is not None else packet.pts
shift = frames * step - begin
if copied_last is not None and (packet.pts - begin) // step > copied_last:
continue
packet.pts += shift
packet.dts += shift
packet.stream = picture
output.mux(packet)
count += 1
if not count % 48:
_check(cancelled)
continue
for decoded in packet.decode():
joined_audio.write(decoded)
if tail is not None:
for packet in tail[1]:
_check(cancelled)
packet.pts += (frames + tail[0]) * step
packet.dts += (frames + tail[0]) * step
packet.stream = picture
output.mux(packet)
count += 1
if joined_audio is not None:
joined_audio.end()
if not count:
raise ValueError('A block has no picture')
frames += count
blocks_joined += 1
if audio is not None:
samples = joined_audio.finish()
for packet in audio.encode(None):
output.mux(packet)
_check(cancelled)
with av.open(str(temporary)) as check:
if check.streams.video[0].frames != frames:
raise ValueError('The joined video is incomplete')
receipt = None
if original_audio is not None:
from .original_audio import mux
required = ((original_audio['end_sample'] - original_audio['start_sample']) * FPS
+ original_audio['sample_rate'] - 1) // original_audio['sample_rate']
if frames < required:
# Completed prefixes end on nearest-sample global boundaries.
# Rounding up can put the endpoint half a sample past the
# last frame; it does not require a further picture frame.
count = original_audio['end_sample'] - original_audio['start_sample']
if 2 * (count * FPS - frames * original_audio['sample_rate']) > FPS:
raise ValueError('Original audio is longer than the joined picture')
required = frames
receipt = mux(temporary, temporary, original_audio, video_frames=required)
_check(cancelled)
rate, samples = receipt['sample_rate'], receipt['samples_per_channel']
with av.open(str(temporary)) as check:
video = check.streams.video[0]
frames = video.frames
if (video.duration is None or
abs(video.duration * video.time_base - Fraction(samples, rate)) > Fraction(1, rate)):
raise ValueError('The joined picture did not end on the selected audio sample')
if frames != required:
raise ValueError('The joined picture did not reach the selected audio endpoint')
os.replace(temporary, target)
return dict(**({'original_audio': receipt} if receipt is not None else {}),
exact_end=wanted is not None and frames == wanted, frames=frames, width=shape[1], height=shape[2],
audio_samples=samples, sample_rate=rate,
seams_repaired=joined_audio.seams_repaired if joined_audio is not None else 0,
seams_expected=expected,
seam_notes=(joined_audio.seam_notes if joined_audio is not None else
[dict(seam=seam, step='no_audio' if seam < blocks_joined else 'not_joined',
reason='previous block has no sound track' if seam < blocks_joined
else 'next block is outside the requested frame range')
for seam in range(1, len(sources))] if original_audio is None else []),
voice=[[round(setting[0], 1), round(setting[1], 1)] if setting is not None else None
for setting in voice] if voice is not None and audio is not None else None)
except (JoinCancelled, _VoiceFailed):
raise
except Exception as error:
if used_tail:
raise _ExactEndFailed() from error
raise
finally:
temporary.unlink(missing_ok=True)
for copy in conformed:
copy.unlink(missing_ok=True)
def _voice(sources, speaking, cancelled):
"""Each block's voice setting, or None: an optional step that never stops the join."""
if speaking is None:
return None
try:
from .chain_level import plan
return plan(sources, speaking, lambda: cancelled is not None and cancelled())
except Exception:
logging.warning('[FreeVideo] The voice of the parts could not be measured; they are joined as they are')
return None
class _JoinedAudio:
"""Keep native seam windows out of the encoder until the next block's head is ready."""
def __init__(self, output, stream, fifo, rate, layout, read_lead, apply_lead, cancelled=None, voice=None):
self.output, self.stream, self.fifo = output, stream, fifo
self.shaper = None
if voice is not None:
from .chain_level import Shaper
self.shaper = _voice_step(Shaper, voice, rate, stream.codec_context.layout.nb_channels)
self.rate, self.layout = rate, layout
self.read_lead, self.apply_lead = read_lead, apply_lead
self.cancelled, self.next_head = cancelled, None
self.warned = False
self.written = self.seams_repaired = 0
self.seam = self.seams_total = 0
self.seam_notes = []
def start(self, source, stream, limit, last=None, following=None, total_frames=None):
import av
context = stream.codec_context
self.seam += 1
self.source_format = context.sample_rate, context.layout.name, getattr(context.layout, 'nb_channels', None)
self.source, self.following = str(source), str(following) if following is not None else None
self.limit, self.heard, self.kept = limit, 0, 0
# end() has released the previous tail and flushed its resampler. Count
# both encoded samples and those still waiting in the encoder queue.
joined = self.written + self.fifo.samples + (self.shaper.pending if self.shaper is not None else 0) \
if total_frames is not None else 0
if self.shaper is not None:
_voice_step(self.shaper.mark)
if last is not None:
cut = (max(0, round(total_frames * self.rate / FPS) - joined) if total_frames is not None
else round((last + 1) * self.rate / FPS))
self.limit = min(self.limit, cut) if self.limit is not None else cut
self.tail, self.released = None, False
self.head = self.next_head[1] if self.next_head is not None and self.next_head[0] == self.source else None
self.next_head, self.head_position = None, 0
self.resampler = av.AudioResampler(format='fltp', layout=self.layout, rate=self.rate)
self.native = (av.AudioResampler(format='fltp', layout='stereo', rate=32000)
if context.sample_rate == 32000 and context.layout.name == 'stereo' else None)
self.native_limit = round(stream.duration * stream.time_base * 32000) if stream.duration is not None else None
if last is not None:
cut = (max(0, round(total_frames * 32000 / FPS) - round(joined * 32000 / self.rate))
if total_frames is not None else round((last + 1) * 32000 / FPS))
self.native_limit = min(self.native_limit, cut) if self.native_limit is not None else cut
def write(self, decoded):
if self.native is None:
self._encode(self.resampler.resample(decoded))
else:
for frame in self.native.resample(decoded):
self._native(frame)
def _native(self, frame):
import numpy as np
data = np.array(frame.to_ndarray().T, dtype=np.float32, order='C', copy=True)
if self.head is not None:
count = min(SEAM_HEAD - self.head_position, len(data))
data[:count] = self.head[self.head_position:self.head_position + count]
self.head_position += count
if self.head_position == SEAM_HEAD:
self.head = None
if self.released:
self._queue(data)
return
keep = len(data) if self.native_limit is None else max(0, min(len(data), self.native_limit - self.kept))
self.kept += keep
self._hold(data[:keep])
if self.native_limit is not None and self.kept >= self.native_limit:
self._release()
if keep < len(data):
# Keep decoder padding available to the existing resampler; the output limit still removes it.
self._queue(data[keep:])
def _hold(self, data):
import numpy as np
if not len(data):
return
if self.tail is not None:
data = np.concatenate((self.tail, data))
if len(data) > SEAM_TAIL:
self._queue(data[:-SEAM_TAIL])
self.tail = data[-SEAM_TAIL:].copy()
def _release(self):
tail = self.tail
if tail is not None and len(tail) == SEAM_TAIL and self.following is not None:
stage = 'reading next block sound'
try:
# Keep the two-argument head reader usable by existing adapters. The
# cancellation callback also carries this call's optional diagnostic sink.
def checked():
return self.cancelled is not None and self.cancelled()
checked.seam_note = self._note
head = _audio_head(self.following, checked)
if head is not None:
stage = 'reading seam record'
lead = self.read_lead(self.following, previous=self.source) if self.read_lead is not None else None
if lead is not None:
stage = 'copying seam windows'
repaired_tail, repaired_head = tail.copy(), head.copy()
stage = 'applying seam record'
if self.apply_lead(repaired_tail, repaired_head, lead):
tail = repaired_tail
self.next_head = self.following, repaired_head
self.seams_repaired += 1
else:
self._note('apply_rejected', 'seam record was rejected by apply')
self._warn()
else:
self._note('no_record', 'seam record is unavailable' if self.read_lead is not None
else 'seam record reader is unavailable')
else:
self._note('next_head', 'next block effective head is unavailable')
except JoinCancelled:
raise
except Exception as error:
name = type(error).__name__
self._note('exception', '%s raised %s' % (stage, name), stage=stage, exception=name)
self._warn()
elif self.following is not None or self.seam <= self.seams_total:
if tail is None or len(tail) != SEAM_TAIL:
count = len(tail) if tail is not None else 0
self._note('previous_tail', 'previous block effective tail has %d samples, needs %d' %
(count, SEAM_TAIL), samples=count, required=SEAM_TAIL)
else:
self._note('not_joined', 'next block is outside the requested frame range')
if tail is not None:
self._queue(tail)
self.tail, self.released = None, True
def _warn(self):
if not self.warned:
logging.warning('[FreeVideo] A chain sound seam could not be repaired; the blocks are joined as they are'
+ _seam_reasons(self.seam_notes[-1:]))
self.warned = True
def _note(self, step, reason, **values):
if not any(note['seam'] == self.seam for note in self.seam_notes):
self.seam_notes.append(dict(seam=self.seam, step=step, reason=reason, **values))
def _queue(self, data):
import av
import numpy as np
frame = av.AudioFrame.from_ndarray(np.ascontiguousarray(data.T), format='fltp', layout='stereo')
frame.sample_rate = 32000
self._encode(self.resampler.resample(frame))
def _encode(self, frames):
self.heard, self.written = _encode(self.output, self.stream, self.fifo, frames, self.heard, self.limit,
self.written, self.rate, self.layout, shaper=self.shaper)
def end(self):
if self.native is not None:
for frame in self.native.resample(None):
self._native(frame)
if not self.released:
self._release()
elif self.following is not None or self.seam <= self.seams_total:
rate, layout, channels = self.source_format
self._note('previous_format', 'previous block sound is %d Hz %s, needs 32000 Hz stereo' % (rate, layout),
sample_rate=rate, layout=layout, channels=channels, required_sample_rate=32000,
required_channels=2)
self._encode(self.resampler.resample(None))
def finish(self):
for seam in range(self.seam + 1, self.seams_total + 1):
self.seam_notes.append(dict(seam=seam, step='not_joined',
reason='next block is outside the requested frame range'))
return _encode(self.output, self.stream, self.fifo, [], 0, None,
self.written, self.rate, self.layout, final=True, shaper=self.shaper)[1]
def _audio_head(source, cancelled):
"""Decode just the next block's kept native head, without changing its files."""
import av
import numpy as np
note = getattr(cancelled, 'seam_note', lambda *args, **kwargs: None)
_check(cancelled)
with av.open(source) as container:
if not container.streams.audio:
note('no_audio', 'next block has no sound track')
return None
stream = container.streams.audio[0]
if stream.codec_context.sample_rate != 32000 or stream.codec_context.layout.name != 'stereo':
context = stream.codec_context
note('next_format', 'next block sound is %d Hz %s, needs 32000 Hz stereo' %
(context.sample_rate, context.layout.name), sample_rate=context.sample_rate,
layout=context.layout.name, channels=context.layout.nb_channels,
required_sample_rate=32000, required_channels=2)
return None
if stream.duration is not None and round(stream.duration * stream.time_base * 32000) < SEAM_HEAD:
count = round(stream.duration * stream.time_base * 32000)
note('next_head', 'next block effective head has %d samples, needs %d' % (count, SEAM_HEAD),
samples=count, required=SEAM_HEAD)
return None
resampler = av.AudioResampler(format='fltp', layout='stereo', rate=32000)
chunks, count = [], 0
for packet in container.demux(stream):
_check(cancelled)
if packet.dts is None:
continue
for decoded in packet.decode():
for frame in resampler.resample(decoded):
keep = min(frame.samples, SEAM_HEAD - count)
chunks.append(frame.to_ndarray()[:, :keep].T.copy())
count += keep
if count == SEAM_HEAD:
return np.ascontiguousarray(np.concatenate(chunks), dtype=np.float32)
note('next_head', 'next block effective head has %d samples, needs %d' % (count, SEAM_HEAD),
samples=count, required=SEAM_HEAD)
return None
def _seam_reasons(notes):
# A seam with no record, or one the cut never reached, is usually a seam nobody asked to repair: name the others first.
notes = sorted(notes, key=lambda note: note['step'] in ('no_record', 'not_joined'))
reasons = ['seam %d: %s' % (note['seam'], note['reason']) for note in notes[:3]]
if len(notes) > 3:
reasons.append('and %d more' % (len(notes) - 3))
return ' (' + '; '.join(reasons) + ')' if reasons else ''
def _encode(output, stream, fifo, resampled, heard, limit, written, rate, layout, final=False, shaper=None):
"""Queue one block's resampled sound up to its length and encode whole encoder frames; `final` writes the rest."""
import av
import numpy as np
def queue(data):
if data is not None:
frame = av.AudioFrame.from_ndarray(np.ascontiguousarray(data), format='fltp', layout=layout)
frame.sample_rate = rate
fifo.write(frame)
for frame in resampled:
keep = frame.samples if limit is None else min(frame.samples, limit - heard)
if keep <= 0:
continue
if keep < frame.samples:
frame = av.AudioFrame.from_ndarray(np.ascontiguousarray(frame.to_ndarray()[:, :keep]),
format='fltp', layout=layout)
frame.sample_rate = rate
heard += keep
frame.pts = None # the queue counts samples itself: blocks follow one another without gaps
if shaper is not None:
queue(_voice_step(shaper.push, frame.to_ndarray()))
else:
fifo.write(frame)
if final and shaper is not None:
queue(_voice_step(shaper.flush))
size = stream.codec_context.frame_size or 1024
while fifo.samples >= size or final and fifo.samples:
frame = fifo.read(min(size, fifo.samples))
frame.pts, frame.sample_rate, frame.time_base = written, rate, Fraction(1, rate)
written += frame.samples
for packet in stream.encode(frame):
output.mux(packet)
return heard, written
def _request_seconds(sources):
total = 0.
for source in sources:
try:
value = json.loads(source.with_suffix('.request.json').read_text(encoding='utf-8')).get('request_seconds')
except (OSError, ValueError, AttributeError):
return None
if type(value) not in (int, float) or not 0 <= value < float('inf'):
return None
total += value
return total
def _replace_target(root, replace, identity, sources):
from .comfy_library import _video, _report
try:
if not isinstance(replace, str) or not replace.startswith('FreeVideo/') or not replace.endswith('/video.mp4'):
return None
target = _video(root, replace[len('FreeVideo/'):-len('/video.mp4')])
if target != root / replace:
return None
report = _report(target)
if target.is_file() and target not in sources and report.get('chain_join', {}).get('chain') == identity:
return target
except (OSError, ValueError, AttributeError, TypeError):
pass
return None
def join_outputs(outputs, identity, cancelled=None, frames=None, replace=None, leads=None, expected_seams=None,
speaking=None, *, original_audio=None):
"""Join relative block outputs into a saved video, replacing only this chain's earlier join."""
import folder_paths
root = Path(folder_paths.get_output_directory()).resolve()
sources = []
for output in outputs:
if not isinstance(output, str) or not output:
raise ValueError('A block has no video')
source = (root / output).resolve()
if not source.is_relative_to(root) or not source.is_file():
raise ValueError('A block video is missing')
sources.append(source)
if leads is not None and len(set(sources)) != len(sources):
# The chain's seam sounds are looked up per video; one video in two places would take the wrong one.
raise ValueError('Two blocks share one video')
# A saved video is a folder the library lists once its report says it is complete.
target = _replace_target(root, replace, identity, sources)
replacing = target is not None
folder = target.parent if replacing else root / 'FreeVideo' / datetime.now().strftime('%Y-%m-%d') / uuid.uuid4().hex
target = folder / 'video.mp4'
staged = target.with_name('.join-result-' + uuid.uuid4().hex + '.mp4') if replacing else target
path = target.with_suffix('.request.json')
temporary = path.with_name(path.name + '.' + uuid.uuid4().hex[:8] + '.tmp')
backup = target.with_name('.join-backup-' + uuid.uuid4().hex + '.mp4') if replacing else None
try:
kept = None
if leads is not None:
# The chain asks for its own output strings, never a path rebuilt from the disk.
originals = {str(source): output for source, output in zip(sources, outputs)}
def kept(following, previous):
return leads(originals.get(str(following)), originals.get(str(previous)) if previous is not None else None)
joined = join_files(sources, staged, cancelled, frames, leads=kept, expected=expected_seams, speaking=speaking,
**({'original_audio': original_audio} if original_audio is not None else {}))
# The original selection replaces every block's sound, so there are no seam sounds to repair.
if original_audio is None and expected_seams is not None and joined['seams_repaired'] < expected_seams:
# A seam sound left unrepaired is never reported as a finished video.
raise ValueError('Only %d of %d seam sounds could be repaired' % (joined['seams_repaired'], expected_seams)
+ _seam_reasons(joined.get('seam_notes', [])))
report = dict(success=True, geometry=dict(width=joined['width'], height=joined['height'], frames=joined['frames'],
fps=FPS, seconds=joined['frames'] / FPS),
request_seconds=_request_seconds(sources), video={}, encoding={},
chain_join=dict(chain=identity, blocks=len(sources), seams_repaired=joined['seams_repaired'],
exact_end=joined['exact_end']))
if expected_seams is not None:
report['chain_join']['seams_expected'] = expected_seams
if joined.get('voice') is not None:
report['chain_join']['voice'] = joined['voice']
if original_audio is not None:
report['video']['original_audio'] = joined['original_audio']
report['geometry']['seconds'] = joined['original_audio']['duration_seconds']
with temporary.open('w', encoding='utf-8') as stream:
json.dump(report, stream, ensure_ascii=False)
stream.flush()
os.fsync(stream.fileno())
_check(cancelled)
if replacing:
try:
os.link(target, backup)
except OSError:
shutil.copyfile(target, backup)
_check(cancelled)
os.replace(staged, target)
try:
os.replace(temporary, path)
except BaseException:
if replacing:
os.replace(backup, target)
raise
if replacing:
try:
target.with_suffix('.thumbnail.jpg').unlink(missing_ok=True)
except OSError:
pass
except BaseException:
if not replacing:
for name in ('video.mp4', 'video.request.json'):
(folder / name).unlink(missing_ok=True)
try:
folder.rmdir()
except OSError:
pass
raise
finally:
temporary.unlink(missing_ok=True)
if replacing:
staged.unlink(missing_ok=True)
backup.unlink(missing_ok=True)
return target.relative_to(root).as_posix()