Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Implement Video Concatenate (CORE-436) (#16267)

rattusd537de93a0ac
diff --git a/comfy_api/latest/__init__.py b/comfy_api/latest/__init__.py
index 294ad425e..73ddcb7e9 100644
--- a/comfy_api/latest/__init__.py
+++ b/comfy_api/latest/__init__.py
@@ -4,7 +4,7 @@ from comfy_api.internal import ComfyAPIBase
from comfy_api.internal.singleton import ProxiedSingleton
from comfy_api.internal.async_to_sync import create_sync_class
from ._input import ImageInput, AudioInput, MaskInput, LatentInput, VideoInput
-from ._input_impl import VideoFromFile, VideoFromComponents
+from ._input_impl import VideoFromFile, VideoFromComponents, VideoFromList
from ._util import VideoCodec, VideoContainer, VideoComponents, MESH, VOXEL, SPLAT, File3D
from . import _io_public as io
from . import _ui_public as ui
@@ -136,6 +136,7 @@ class Input:
class InputImpl:
VideoFromFile = VideoFromFile
VideoFromComponents = VideoFromComponents
+ VideoFromList = VideoFromList
class Types:
VideoCodec = VideoCodec
diff --git a/comfy_api/latest/_input_impl/__init__.py b/comfy_api/latest/_input_impl/__init__.py
index 02901b8b9..9f4f56344 100644
--- a/comfy_api/latest/_input_impl/__init__.py
+++ b/comfy_api/latest/_input_impl/__init__.py
@@ -1,7 +1,8 @@
-from .video_types import VideoFromFile, VideoFromComponents
+from .video_types import VideoFromFile, VideoFromComponents, VideoFromList
__all__ = [
# Implementations
"VideoFromFile",
"VideoFromComponents",
+ "VideoFromList",
]
diff --git a/comfy_api/latest/_input_impl/video_types.py b/comfy_api/latest/_input_impl/video_types.py
index d02c61ac4..d32b48260 100644
--- a/comfy_api/latest/_input_impl/video_types.py
+++ b/comfy_api/latest/_input_impl/video_types.py
@@ -394,7 +394,9 @@ class VideoFromFile(VideoInput):
duration_from_start = min(raw_duration, -self.__start_time)
else:
duration_from_start = raw_duration - self.__start_time
- duration_seconds = min(self.__duration, duration_from_start)
+ duration_seconds = (
+ min(self.__duration, duration_from_start) if self.__duration else duration_from_start
+ )
estimated_frames = int(round(duration_seconds * float(video_stream.average_rate)))
if estimated_frames > 0:
return estimated_frames
@@ -403,7 +405,7 @@ class VideoFromFile(VideoInput):
start_time, duration = self.get_active_trim_window()
frame_count = 1
start_pts = int(start_time / video_stream.time_base)
- end_pts = int((start_time + duration) / video_stream.time_base)
+ end_pts = int((start_time + duration) / video_stream.time_base) if duration else None
container.seek(start_pts, stream=video_stream)
frame_iterator = (
container.decode(video_stream)
@@ -416,7 +418,7 @@ class VideoFromFile(VideoInput):
else:
raise ValueError(f"Could not determine frame count for file '{self.__file}'\nNo frames exist for start_time {self.__start_time}")
for frame in frame_iterator:
- if frame.pts >= end_pts:
+ if end_pts is not None and frame.pts >= end_pts:
break
frame_count += 1
return frame_count
@@ -709,6 +711,10 @@ class VideoFromFile(VideoInput):
crf: float | None = None,
color_space: str | None = None,
preset: str | None = None,
+ preserve_source_timestamps: bool = True,
+ frame_rate: Fraction | None = None,
+ audio_sample_rate: int | None = None,
+ audio_layout: str | None = None,
):
"""Re-encode one frame at a time; peak memory does not scale with video length."""
open_kwargs, output_format, output_codec = video_output_config(path, format, codec)
@@ -730,7 +736,7 @@ class VideoFromFile(VideoInput):
source_color_space = video_stream_color_space(video_stream)
preserve_source_color = source_color_space is not None
pix_fmt = "yuv420p10le" if bit_depth >= 10 else "yuv420p"
- rate = Fraction(video_stream.average_rate) if video_stream.average_rate else Fraction(1)
+ rate = frame_rate or (Fraction(video_stream.average_rate) if video_stream.average_rate else Fraction(1))
resampler = None
sample_rate = 0
@@ -748,10 +754,9 @@ class VideoFromFile(VideoInput):
logging.warning("Audio stream parameters could not be determined; ignoring audio.")
audio_stream = None
if audio_stream is not None:
- if output_format == VideoContainer.WEBM:
- sample_rate = 48000
+ sample_rate = 48000 if output_format == VideoContainer.WEBM else audio_sample_rate or sample_rate
audio_time_base = Fraction(1, sample_rate)
- layout = {1: "mono", 2: "stereo", 6: "5.1"}.get(channels, "stereo")
+ layout = audio_layout or {1: "mono", 2: "stereo", 6: "5.1"}.get(channels, "stereo")
resampler = av.audio.resampler.AudioResampler(format="fltp", layout=layout, rate=sample_rate)
if duration:
duration_cap = math.ceil(duration * sample_rate)
@@ -879,9 +884,6 @@ class VideoFromFile(VideoInput):
# Add metadata before writing any streams
write_output_metadata(container, output, metadata)
out_video = output.add_stream(VIDEO_ENCODERS[output_codec], rate=rate)
- # no B-frames: reordering makes mp4 sample durations follow decode order,
- # so irregular-VFR spans and trim windows land wrong
- out_video.codec_context.max_b_frames = 0
out_video.width = out_width
out_video.height = out_height
out_video.pix_fmt = pix_fmt
@@ -890,8 +892,10 @@ class VideoFromFile(VideoInput):
copy_color_properties(video_stream, out_video.codec_context)
elif color_space is not None:
set_video_color_properties(out_video.codec_context, color_space)
- # source pts pass through (rebased to 0), so variable frame rate survives
- out_video.codec_context.time_base = video_stream.time_base
+ # Preserve source timing; B-frame reordering shortens irregular-VFR spans.
+ if preserve_source_timestamps:
+ out_video.codec_context.max_b_frames = 0
+ out_video.codec_context.time_base = video_stream.time_base
if audio_stream is not None:
audio_codec = "libopus" if output_format == VideoContainer.WEBM else "aac"
out_audio = output.add_stream(audio_codec, rate=sample_rate, layout=layout)
@@ -1210,3 +1214,360 @@ class VideoFromComponents(VideoInput):
return None
#TODO Consider tracking duration and trimming at time of save?
return VideoFromFile(self.get_stream_source(), start_time=start_time, duration=duration)
+
+
+class VideoFromList(VideoInput):
+ def __init__(
+ self,
+ videos: list[VideoInput],
+ complete_audio: AudioInput | None = None,
+ codec: VideoCodec = VideoCodec.AUTO,
+ ):
+ self.videos = []
+ inherited_audio = None
+ for video in videos:
+ if isinstance(video, VideoFromList):
+ if video.complete_audio is not None:
+ inherited_audio = video.complete_audio
+ self.videos.extend(video.videos)
+ elif not isinstance(video, VideoFromFile):
+ buffer = io.BytesIO()
+ video.save_to(buffer, format=VideoContainer.MKV, codec=codec)
+ self.videos.append(VideoFromFile(buffer))
+ else:
+ self.videos.append(video)
+ if not self.videos:
+ raise ValueError("Concatenate Video requires at least one input")
+ self.complete_audio = complete_audio if complete_audio is not None else inherited_audio
+ self.__buffer = None
+
+ def get_components(self) -> VideoComponents:
+ components = [video.get_components() for video in self.videos]
+ frame_rate = components[0].frame_rate
+ if any(component.frame_rate != frame_rate for component in components[1:]):
+ raise ValueError("Cannot materialize accumulated videos with different frame rates")
+ try:
+ images = torch.cat([component.images for component in components])
+ except RuntimeError as error:
+ raise ValueError("Accumulated videos have incompatible frame dimensions") from error
+
+ if self.complete_audio is not None:
+ sample_rate = int(self.complete_audio["sample_rate"])
+ audio = AudioInput({
+ "waveform": self.complete_audio["waveform"][..., :round(len(images) / frame_rate * sample_rate)],
+ "sample_rate": sample_rate,
+ })
+ elif any(component.audio is not None for component in components):
+ if any(component.audio is None for component in components):
+ raise ValueError("Cannot materialize accumulated videos with missing audio segments")
+ sample_rate = int(components[0].audio["sample_rate"])
+ channels = components[0].audio["waveform"].shape[1]
+ if any(
+ int(component.audio["sample_rate"]) != sample_rate
+ or component.audio["waveform"].shape[1] != channels
+ for component in components[1:]
+ ):
+ raise ValueError("Cannot materialize accumulated videos with incompatible audio")
+ audio = AudioInput({
+ "waveform": torch.cat(
+ [component.audio["waveform"] for component in components], dim=-1
+ )[..., :round(len(images) / frame_rate * sample_rate)],
+ "sample_rate": sample_rate,
+ })
+ else:
+ audio = None
+
+ alphas = [component.alpha for component in components]
+ alpha = torch.cat(alphas) if all(value is not None for value in alphas) else None
+ return VideoComponents(
+ images=images,
+ audio=audio,
+ frame_rate=frame_rate,
+ metadata=components[0].metadata,
+ alpha=alpha,
+ )
+
+ def get_dimensions(self):
+ dimensions = [video.get_dimensions() for video in self.videos]
+ mismatches = [
+ f"chunk {index} is {width}x{height}"
+ for index, (width, height) in enumerate(dimensions[1:], 1)
+ if (width, height) != dimensions[0]
+ ]
+ if mismatches:
+ width, height = dimensions[0]
+ raise ValueError(
+ f"Accumulated videos have incompatible frame dimensions: chunk 0 is {width}x{height}; "
+ + "; ".join(mismatches)
+ )
+ return dimensions[0]
+
+ def get_bit_depth(self):
+ return self.videos[0].get_bit_depth()
+
+ def get_color_space(self):
+ return self.videos[0].get_color_space()
+
+ def get_duration(self):
+ return sum(video.get_duration() for video in self.videos)
+
+ def get_frame_count(self):
+ return sum(video.get_frame_count() for video in self.videos)
+
+ def get_frame_rate(self):
+ return self.videos[0].get_frame_rate()
+
+ def save_to(self, path, format=VideoContainer.AUTO, codec=VideoCodec.AUTO, metadata=None,
+ bit_depth=None, crf=None, color_space=None, preset=None):
+ open_kwargs, output_format, output_codec = video_output_config(path, format, codec)
+ requested_codec = VideoCodec(codec)
+ output = None
+ output_video = None
+ output_audio = None
+ audio_resampler = None
+ audio_pts = 0
+ signature = None
+ video_offset = Fraction()
+
+ def mux_audio_frames(frames, end_pts):
+ nonlocal audio_pts
+ for frame in frames:
+ remaining = end_pts - audio_pts
+ if remaining <= 0:
+ return
+ if frame.samples > remaining:
+ frame = av.AudioFrame.from_ndarray(
+ frame.to_ndarray()[..., :remaining], format="fltp", layout=audio_layout
+ )
+ frame.sample_rate = output_audio.rate
+ frame.pts = audio_pts
+ frame.time_base = Fraction(1, frame.sample_rate)
+ audio_pts += frame.samples
+ output.mux(output_audio.encode(frame))
+
+ def write_audio_frames(frames, end_pts):
+ for frame in frames:
+ mux_audio_frames(audio_resampler.resample(frame), end_pts)
+
+ def video_packets(container, stream):
+ pending = []
+ for packet in container.demux(stream):
+ if packet.dts is None:
+ if packet.pts is not None:
+ pending.append(packet)
+ continue
+ if pending:
+ dts = packet.dts
+ for leading in reversed(pending):
+ dts -= leading.duration or packet.duration or 1
+ leading.dts = dts
+ yield from pending
+ pending.clear()
+ yield packet
+ if pending:
+ if all(first.pts <= second.pts for first, second in zip(pending, pending[1:])):
+ for packet in pending:
+ packet.dts = packet.pts
+ else:
+ dts = 0
+ for packet in reversed(pending):
+ dts -= packet.duration or 1
+ packet.dts = dts
+ yield from pending
+
+ def encoded_signature(container):
+ video_stream = container.streams.video[0]
+ audio_stream = None if self.complete_audio is not None else last_decodable_audio_stream(container)
+ return (
+ video_stream.codec.canonical_name,
+ video_stream.codec_context.extradata,
+ video_stream.width,
+ video_stream.height,
+ video_stream_bit_depth(video_stream),
+ video_stream_color_space(video_stream),
+ audio_stream.sample_rate if audio_stream else None,
+ audio_stream.layout.name if audio_stream else None,
+ )
+
+ source_signatures = []
+ shared_encode = crf is not None or color_space is not None
+ for video in self.videos:
+ with av.open(video.get_stream_source()) as container:
+ current_signature = encoded_signature(container)
+ source_signatures.append(current_signature)
+ video_stream = container.streams.video[0]
+ shared_encode |= (
+ (requested_codec != VideoCodec.AUTO and current_signature[0] != requested_codec.value)
+ or (bit_depth is not None and current_signature[4] != bit_depth)
+ or (output_format == VideoContainer.WEBM and video_stream.codec.canonical_name not in WEBM_STREAM_CODECS["video"])
+ or video.get_active_trim_window() != (0.0, 0.0)
+ or video.get_dimensions() != (video_stream.width, video_stream.height)
+ )
+ shared_encode |= any(
+ current_signature != source_signatures[0]
+ for current_signature in source_signatures[1:]
+ )
+ target_codec = requested_codec if requested_codec != VideoCodec.AUTO else output_codec
+ target_bit_depth = bit_depth if bit_depth is not None else source_signatures[0][4]
+ target_color_space = color_space if color_space is not None else source_signatures[0][5]
+ target_frame_rate = Fraction(self.videos[0].get_frame_rate())
+ target_audio_rate = source_signatures[0][6]
+ target_audio_layout = source_signatures[0][7]
+
+ try:
+ for index, video in enumerate(self.videos):
+ container = av.open(video.get_stream_source())
+ current_signature = encoded_signature(container)
+ if shared_encode:
+ scratch = io.BytesIO()
+ try:
+ video._save_transcoded(
+ container,
+ scratch,
+ format=output_format,
+ codec=target_codec,
+ metadata=None,
+ bit_depth=target_bit_depth,
+ crf=crf,
+ color_space=target_color_space,
+ preset=preset,
+ preserve_source_timestamps=False,
+ frame_rate=target_frame_rate,
+ audio_sample_rate=target_audio_rate,
+ audio_layout=target_audio_layout,
+ )
+ finally:
+ container.close()
+ scratch.seek(0)
+ container = av.open(scratch)
+ current_signature = encoded_signature(container)
+ if signature is not None and current_signature != signature:
+ container.close()
+ fields = ("codec", "extradata", "width", "height", "bit depth", "color space", "audio rate", "audio layout")
+ differences = "; ".join(
+ f"{field}: expected {expected!r}, got {actual!r}"
+ for field, expected, actual in zip(fields, signature, current_signature)
+ if expected != actual
+ )
+ raise ValueError(f"Video chunk {index} could not be encoded compatibly: {differences}")
+ with container:
+ video_stream = container.streams.video[0]
+ audio_stream = None if self.complete_audio is not None else last_decodable_audio_stream(container)
+ if index == 0:
+ signature = current_signature
+ output = av.open(path, **open_kwargs)
+ write_output_metadata(container, output, metadata)
+ output_video = output.add_stream_from_template(video_stream, opaque=True)
+ if self.complete_audio is not None:
+ source_rate = int(self.complete_audio["sample_rate"])
+ channels = self.complete_audio["waveform"].shape[1]
+ layout = {1: "mono", 2: "stereo", 6: "5.1"}.get(channels, "stereo")
+ elif audio_stream is not None:
+ source_rate = audio_stream.sample_rate
+ layout = audio_stream.layout.name
+ else:
+ source_rate = layout = None
+ if layout is not None:
+ target_rate = 48000 if output_format == VideoContainer.WEBM else source_rate
+ audio_layout = layout
+ output_audio = output.add_stream(
+ "libopus" if output_format == VideoContainer.WEBM else "aac",
+ rate=target_rate,
+ layout=layout,
+ )
+ audio_resampler = av.AudioResampler(format="fltp", layout=layout, rate=target_rate)
+ hevc_filter = isobmff_hevc_filter(output, video_stream, output_video)
+ video_end = video_offset
+ origin = None
+ for packet in video_packets(container, video_stream):
+ time_base = packet.time_base
+ if origin is None:
+ origin = Fraction(packet.pts if packet.pts is not None else packet.dts) * time_base
+ if packet.pts is not None:
+ packet.pts = int((Fraction(packet.pts) * time_base - origin + video_offset) / time_base)
+ video_end = max(video_end, Fraction(packet.pts + (packet.duration or 0)) * time_base)
+ packet.dts = int((Fraction(packet.dts) * time_base - origin + video_offset) / time_base)
+ packet.stream = output_video
+ for packet in filter_hevc_packet(hevc_filter, packet) if hevc_filter else (packet,):
+ packet.stream = output_video
+ output.mux(packet)
+ video_offset = video_end
+ if audio_stream is not None:
+ container.seek(0)
+ write_audio_frames(
+ itertools.chain.from_iterable(packet.decode() for packet in container.demux(audio_stream)),
+ round(float(video_offset) * output_audio.rate),
+ )
+
+ if output_audio is not None:
+ if self.complete_audio is not None:
+ source_rate = int(self.complete_audio["sample_rate"])
+ waveform = self.complete_audio["waveform"][0, :, : round(source_rate * float(video_offset))]
+ frame = av.AudioFrame.from_ndarray(
+ waveform.float().cpu().contiguous().numpy(), format="fltp", layout=audio_layout
+ )
+ frame.sample_rate = source_rate
+ write_audio_frames((frame,), round(float(video_offset) * output_audio.rate))
+ mux_audio_frames(audio_resampler.resample(None), round(float(video_offset) * output_audio.rate))
+ output.mux(output_audio.encode(None))
+ except BaseException:
+ if output is not None:
+ output.close()
+ if isinstance(path, (str, os.PathLike)) and os.path.exists(path):
+ os.remove(path)
+ raise
+ else:
+ if output is not None:
+ output.close()
+
+ def get_stream_source(self):
+ if self.__buffer is None:
+ self.__buffer = io.BytesIO()
+ self.save_to(self.__buffer, format=VideoContainer.MP4)
+ self.__buffer.seek(0)
+ return self.__buffer
+
+ def as_trimmed(self, start_time=None, duration=None, strict_duration=False):
+ total_duration = self.get_duration()
+ start_time = float(start_time or 0)
+ if start_time < 0:
+ start_time = max(total_duration + start_time, 0)
+ available = total_duration - start_time
+ if available < 0 or duration is not None and duration < 0:
+ return None
+ if strict_duration and duration and duration > available:
+ return None
+ duration = min(float(duration), available) if duration else available
+
+ selected = []
+ offset = start_time
+ remaining = duration
+ for video in self.videos:
+ video_duration = video.get_duration()
+ if offset >= video_duration:
+ offset -= video_duration
+ continue
+ selected.append(video.as_trimmed(offset, min(video_duration - offset, remaining), False))
+ remaining -= video_duration - offset
+ if remaining <= 0:
+ break
+ offset = 0
+ if not selected:
+ return None
+
+ audio = self.complete_audio
+ if audio is not None:
+ sample_rate = int(audio["sample_rate"])
+ start_sample = round(start_time * sample_rate)
+ end_sample = start_sample + round(duration * sample_rate)
+ audio = AudioInput({
+ "waveform": audio["waveform"][..., start_sample:end_sample],
+ "sample_rate": sample_rate,
+ })
+ return VideoFromList(selected, audio)
+
+ def as_cropped(self, x=0, y=0, width=0, height=0):
+ return VideoFromList(
+ [video.as_cropped(x, y, width, height) for video in self.videos],
+ self.complete_audio,
+ )
diff --git a/comfy_extras/nodes_video.py b/comfy_extras/nodes_video.py
index 5da594ba8..70876a96b 100644
--- a/comfy_extras/nodes_video.py
+++ b/comfy_extras/nodes_video.py
@@ -231,6 +231,14 @@ class CreateVideo(io.ComfyNode):
optional=True,
tooltip="Colorspace of the input images. HDR selects BT.2020/HLG and HDR PQ selects BT.2020/PQ.",
),
+ io.Combo.Input(
+ "codec",
+ options=["none", *Types.VideoCodec.as_input()],
+ default="none",
+ advanced=True,
+ optional=True,
+ tooltip="Optionally encode the video immediately. None keeps the images in tensor form; Auto uses H.264.",
+ ),
],
outputs=[
io.Video.Output(),
@@ -239,17 +247,63 @@ class CreateVideo(io.ComfyNode):
@classmethod
def execute(
- cls, images: Input.Image, fps: float, audio: Optional[Input.Audio] = None, bit_depth: int | str = "auto", color_space: str = "sRGB",
+ cls, images: Input.Image, fps: float, audio: Optional[Input.Audio] = None, bit_depth: int | str = "auto", color_space: str = "sRGB", codec: str = "none",
) -> io.NodeOutput:
if bit_depth == "auto":
bit_depth = 10 if color_space in ("HDR", "HDR PQ") else 8
- return io.NodeOutput(
- InputImpl.VideoFromComponents(
- Types.VideoComponents(images=images, audio=audio, frame_rate=Fraction(fps)),
- bit_depth=bit_depth,
- color_space=color_space,
- )
+ video = InputImpl.VideoFromComponents(
+ Types.VideoComponents(images=images, audio=audio, frame_rate=Fraction(fps)),
+ bit_depth=bit_depth,
+ color_space=color_space,
)
+ if codec != "none":
+ video = InputImpl.VideoFromList([video], codec=Types.VideoCodec(codec))
+ return io.NodeOutput(video)
+
+
+class ConcatenateVideo(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="ConcatenateVideo",
+ display_name="Concatenate Video",
+ category="video",
+ essentials_category="Video Tools",
+ description="Concatenates videos in order without decoding compatible encoded inputs.",
+ inputs=[
+ io.Autogrow.Input(
+ "videos",
+ template=io.Autogrow.TemplatePrefix(
+ io.Video.Input("video", tooltip="A video segment to append."),
+ prefix="video",
+ min=1,
+ max=100,
+ ),
+ tooltip="Video segments to concatenate in input order.",
+ ),
+ io.Combo.Input(
+ "codec",
+ options=Types.VideoCodec.as_input(),
+ default="auto",
+ advanced=True,
+ tooltip="Codec used to encode videos tensors. Auto uses H.264; already encoded videos remain unchanged.",
+ ),
+ io.Audio.Input(
+ "complete_audio",
+ optional=True,
+ advanced=True,
+ tooltip="Optional complete soundtrack for the concatenated video. Overrides audio carried by the input videos.",
+ ),
+ ],
+ outputs=[io.Video.Output(tooltip="The concatenated video.")],
+ is_input_list=True,
+ )
+
+ @classmethod
+ def execute(cls, videos: io.Autogrow.Type, codec=None, complete_audio=None) -> io.NodeOutput:
+ videos = [video for group in videos.values() for video in group]
+ audio = complete_audio[0] if complete_audio else None
+ return io.NodeOutput(InputImpl.VideoFromList(videos, audio, Types.VideoCodec(codec[0] if codec else "auto")))
class GetVideoComponents(io.ComfyNode):
@classmethod
@@ -509,6 +563,7 @@ class VideoExtension(ComfyExtension):
SaveWEBM,
SaveVideo,
CreateVideo,
+ ConcatenateVideo,
GetVideoComponents,
LoadVideo,
VideoSlice,
diff --git a/tests-unit/comfy_api_test/video_accumulation_test.py b/tests-unit/comfy_api_test/video_accumulation_test.py
new file mode 100644
index 000000000..aca278488
--- /dev/null
+++ b/tests-unit/comfy_api_test/video_accumulation_test.py
@@ -0,0 +1,286 @@
+import io
+import gc
+import os
+import tempfile
+import weakref
+from fractions import Fraction
+
+import av
+import torch
+
+from comfy_api.input_impl.video_types import VideoFromComponents, VideoFromFile, VideoFromList
+from comfy_api.input.basic_types import AudioInput
+from comfy_api.util.video_types import VideoCodec, VideoComponents
+from comfy_extras.nodes_video import ConcatenateVideo, CreateVideo
+
+
+def test_tensor_video_encodes_to_list_owned_buffer():
+ images = torch.zeros((2, 16, 16, 3))
+ images_ref = weakref.ref(images)
+ source = VideoFromComponents(VideoComponents(images=images, frame_rate=Fraction(8)))
+
+ video = VideoFromList([source])
+ encoded = video.videos[0]
+ buffer = encoded.get_stream_source()
+ trimmed = video.as_trimmed(0, 0.125)
+ del images, source, video, encoded
+ gc.collect()
+
+ assert isinstance(trimmed, VideoFromList)
+ assert images_ref() is None
+ assert isinstance(buffer, io.BytesIO)
+ assert buffer.getbuffer().nbytes > 0
+
+
+def test_accumulate_flattens_groups_and_eagerly_encodes_tensors():
+ images = [torch.full((1, 16, 16, 3), value) for value in (0.1, 0.5, 0.9)]
+ references = [weakref.ref(image) for image in images]
+ videos = [VideoFromComponents(VideoComponents(images=image, frame_rate=Fraction(8))) for image in images]
+ nested = VideoFromList(videos[:2])
+
+ result = ConcatenateVideo.execute({"inputs0": [nested], "inputs1": [videos[2]]}).result[0]
+ del images, videos, nested
+ gc.collect()
+
+ assert len(result.videos) == 3
+ assert all(isinstance(video, VideoFromFile) for video in result.videos)
+ assert all(reference() is None for reference in references)
+
+
+def test_concatenate_video_schema_and_intermediate_codec(monkeypatch):
+ encoded_codecs = []
+
+ def record_save(self, path, **kwargs):
+ encoded_codecs.append(kwargs["codec"])
+ path.write(b"")
+
+ monkeypatch.setattr(VideoFromComponents, "save_to", record_save)
+ source = VideoFromComponents(
+ VideoComponents(images=torch.zeros((1, 16, 16, 3)), frame_rate=Fraction(8))
+ )
+ ConcatenateVideo.execute({"inputs0": [source]}, codec=["av1"])
+
+ schema = ConcatenateVideo.define_schema()
+ inputs = {input.id: input for input in schema.inputs}
+ assert encoded_codecs == [VideoCodec.AV1]
+ assert inputs["codec"].advanced and inputs["complete_audio"].advanced
+ assert schema.description and schema.outputs[0].tooltip
+ assert all(input.tooltip for input in schema.inputs)
+ assert inputs["inputs"].template.input.tooltip
+
+
+def test_create_video_optional_eager_encoding(monkeypatch):
+ encoded_codecs = []
+
+ def record_save(self, path, **kwargs):
+ encoded_codecs.append(kwargs["codec"])
+ path.write(b"")
+
+ monkeypatch.setattr(VideoFromComponents, "save_to", record_save)
+ video = CreateVideo.execute(torch.zeros((1, 16, 16, 3)), 8, codec="av1").result[0]
+
+ codec_input = next(input for input in CreateVideo.define_schema().inputs if input.id == "codec")
+ assert isinstance(video, VideoFromList)
+ assert encoded_codecs == [VideoCodec.AV1]
+ assert codec_input.options == ["none", "auto", "h264", "av1"]
+ assert codec_input.default == "none"
+ assert codec_input.advanced and codec_input.optional
+
+
+def test_nested_complete_audio_uses_most_recent_override():
+ source = VideoFromComponents(
+ VideoComponents(images=torch.zeros((1, 16, 16, 3)), frame_rate=Fraction(8))
+ )
+ audios = [
+ {"waveform": torch.full((1, 1, 1000), value), "sample_rate": 8000}
+ for value in (1, 2, 3)
+ ]
+ nested = [VideoFromList([source], audio) for audio in audios[:2]]
+
+ assert VideoFromList(nested).complete_audio is audios[1]
+ assert VideoFromList(nested, audios[2]).complete_audio is audios[2]
+
+
+def test_accumulated_video_packet_concatenates_file_backed_inputs():
+ class NoMaterializeVideo(VideoFromFile):
+ def get_components(self):
+ raise AssertionError("file-backed concatenation decoded video frames")
+
+ def save_to(self, *args, **kwargs):
+ raise AssertionError("compatible file-backed video was rewritten")
+
+ with tempfile.TemporaryDirectory() as directory:
+ sources = []
+ for index, extension in enumerate(("mkv", "mp4")):
+ source = os.path.join(directory, f"source{index}.{extension}")
+ VideoFromComponents(
+ VideoComponents(images=torch.full((2, 16, 16, 3), index / 2), frame_rate=Fraction(8))
+ ).save_to(source)
+ sources.append(NoMaterializeVideo(source))
+
+ output = os.path.join(directory, "output.mp4")
+ VideoFromList(sources).save_to(output)
+ with av.open(output) as container:
+ assert sum(1 for _ in container.decode(video=0)) == 4
+
+
+def test_accumulated_video_reencodes_all_chunks_with_shared_configuration():
+ class RewriteTrackingVideo(VideoFromFile):
+ rewritten = False
+
+ def _save_transcoded(self, *args, **kwargs):
+ self.rewritten = True
+ return super()._save_transcoded(*args, **kwargs)
+
+ with tempfile.TemporaryDirectory() as directory:
+ sources = []
+ for bit_depth in (8, 10):
+ source = os.path.join(directory, f"source-{bit_depth}-bit.mp4")
+ VideoFromComponents(
+ VideoComponents(images=torch.zeros((2, 16, 16, 3)), frame_rate=Fraction(8)),
+ bit_depth=bit_depth,
+ ).save_to(source)
+ sources.append(RewriteTrackingVideo(source))
+
+ output = os.path.join(directory, "output.mp4")
+ VideoFromList(sources).save_to(output)
+
+ assert all(source.rewritten for source in sources)
+ with av.open(output) as container:
+ assert sum(1 for _ in container.decode(video=0)) == 4
+
+
+def test_accumulated_video_reencodes_audio_to_shared_rate_and_layout():
+ with tempfile.TemporaryDirectory() as directory:
+ sources = []
+ for index, (sample_rate, channels) in enumerate(((8000, 1), (16000, 2))):
+ source = os.path.join(directory, f"source-{index}.mp4")
+ audio = AudioInput({
+ "waveform": torch.zeros((1, channels, sample_rate // 4)),
+ "sample_rate": sample_rate,
+ })
+ VideoFromComponents(
+ VideoComponents(
+ images=torch.zeros((2, 16, 16, 3)),
+ frame_rate=Fraction(8),
+ audio=audio,
+ )
+ ).save_to(source)
+ sources.append(VideoFromFile(source))
+
+ output = os.path.join(directory, "output.mp4")
+ VideoFromList(sources).save_to(output)
+
+ with av.open(output) as container:
+ assert container.streams.audio[0].sample_rate == 8000
+ assert container.streams.audio[0].layout.name == "mono"
+
+
+def test_accumulated_video_stream_source_is_buffered_and_reused():
+ video = VideoFromList([
+ VideoFromComponents(VideoComponents(images=torch.zeros((1, 16, 16, 3)), frame_rate=Fraction(8)))
+ ])
+
+ first = video.get_stream_source()
+ second = video.get_stream_source()
+
+ assert first == second
+ assert isinstance(first, io.BytesIO)
+ assert first.getbuffer().nbytes > 0
+
+
+def test_accumulated_video_continuously_encodes_audio_and_allows_override():
+ audio = AudioInput({"waveform": torch.zeros((1, 2, 2000)), "sample_rate": 8000})
+ videos = [
+ VideoFromComponents(
+ VideoComponents(images=torch.zeros((2, 16, 16, 3)), frame_rate=Fraction(8), audio=audio)
+ )
+ for _ in range(2)
+ ]
+ override = AudioInput({"waveform": torch.ones((1, 1, 8000)), "sample_rate": 8000})
+
+ with tempfile.TemporaryDirectory() as directory:
+ embedded_path = os.path.join(directory, "embedded.mp4")
+ override_path = os.path.join(directory, "override.mp4")
+ VideoFromList(videos).save_to(embedded_path)
+ VideoFromList(videos, override).save_to(override_path)
+
+ with av.open(embedded_path) as embedded, av.open(override_path) as overridden:
+ embedded_audio = embedded.streams.audio[0]
+ overridden_audio = overridden.streams.audio[0]
+ assert embedded_audio.layout.name == "stereo"
+ assert overridden_audio.layout.name == "mono"
+ assert float(embedded_audio.duration * embedded_audio.time_base) <= 0.6
+ assert float(overridden_audio.duration * overridden_audio.time_base) <= 0.6
+
+
+def test_accumulated_video_metadata_and_explicit_materialization():
+ videos = [
+ VideoFromComponents(
+ VideoComponents(images=torch.full((2, 16, 16, 3), value), frame_rate=Fraction(8))
+ )
+ for value in (0.0, 0.5)
+ ]
+ video = VideoFromList(videos)
+
+ assert video.get_dimensions() == (16, 16)
+ assert video.get_duration() == 0.5
+ assert video.get_frame_count() == 4
+ assert video.get_frame_rate() == 8
+ assert video.get_components().images.shape == (4, 16, 16, 3)
+
+
+def test_accumulated_video_reports_each_incompatible_dimension():
+ videos = [
+ VideoFromComponents(
+ VideoComponents(images=torch.zeros((1, height, width, 3)), frame_rate=Fraction(8))
+ )
+ for width, height in ((16, 16), (24, 16), (16, 24))
+ ]
+ video = VideoFromList(videos)
+
+ try:
+ video.get_dimensions()
+ except ValueError as error:
+ assert str(error) == (
+ "Accumulated videos have incompatible frame dimensions: "
+ "chunk 0 is 16x16; chunk 1 is 24x16; chunk 2 is 16x24"
+ )
+ else:
+ raise AssertionError("Expected incompatible dimensions to fail")
+
+
+def test_accumulated_video_trims_across_file_boundaries_without_materializing():
+ class NoMaterializeVideo(VideoFromFile):
+ def get_components(self):
+ raise AssertionError("trim materialized video frames")
+
+ with tempfile.TemporaryDirectory() as directory:
+ sources = []
+ for index in range(2):
+ source = os.path.join(directory, f"source{index}.mp4")
+ VideoFromComponents(
+ VideoComponents(images=torch.zeros((2, 16, 16, 3)), frame_rate=Fraction(8))
+ ).save_to(source)
+ sources.append(NoMaterializeVideo(source))
+
+ trimmed = VideoFromList(sources).as_trimmed(0.125, 0.25, strict_duration=True)
+
+ assert isinstance(trimmed, VideoFromList)
+ assert len(trimmed.videos) == 2
+ assert trimmed.get_duration() == 0.25
+
+
+def test_accumulated_video_trim_slices_complete_audio():
+ video = VideoFromList(
+ [
+ VideoFromComponents(
+ VideoComponents(images=torch.zeros((4, 16, 16, 3)), frame_rate=Fraction(4))
+ )
+ ],
+ AudioInput({"waveform": torch.arange(8000).reshape(1, 1, -1), "sample_rate": 8000}),
+ )
+
+ trimmed = video.as_trimmed(0.25, 0.5)
+
+ assert torch.equal(trimmed.complete_audio["waveform"], torch.arange(2000, 6000).reshape(1, 1, -1))