The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add a VideoSlice node (#12107)
AustinMrozdbe70b682199
diff --git a/comfy_api/latest/_input/video_types.py b/comfy_api/latest/_input/video_types.pyindex e634a0311..451e9526e 100644--- a/comfy_api/latest/_input/video_types.py+++ b/comfy_api/latest/_input/video_types.py@@ -34,6 +34,21 @@ class VideoInput(ABC):"""pass+ @abstractmethod+ def as_trimmed(+ self,+ start_time: float | None = None,+ duration: float | None = None,+ strict_duration: bool = False,+ ) -> VideoInput | None:+ """+ Create a new VideoInput which is trimmed to have the corresponding start_time and duration++ Returns:+ A new VideoInput, or None if the result would have negative duration+ """+ pass+def get_stream_source(self) -> Union[str, io.BytesIO]:"""Get a streamable source for the video. This allows processing withoutdiff --git a/comfy_api/latest/_input_impl/video_types.py b/comfy_api/latest/_input_impl/video_types.pyindex 1405d0b81..3463ed1c9 100644--- a/comfy_api/latest/_input_impl/video_types.py+++ b/comfy_api/latest/_input_impl/video_types.py@@ -6,6 +6,7 @@ from typing import Optionalfrom .._input import AudioInput, VideoInputimport avimport io+import itertoolsimport jsonimport numpy as npimport math@@ -29,7 +30,6 @@ def container_to_output_format(container_format: str | None) -> str | None:formats = container_format.split(",")return formats[0]-def get_open_write_kwargs(dest: str | io.BytesIO, container_format: str, to_format: str | None) -> dict:@@ -57,12 +57,14 @@ class VideoFromFile(VideoInput):Class representing video input from a file."""- def __init__(self, file: str | io.BytesIO):+ def __init__(self, file: str | io.BytesIO, *, start_time: float=0, duration: float=0):"""Initialize the VideoFromFile object based off of either a path on disk or a BytesIO objectcontaining the file contents."""self.__file = file+ self.__start_time = start_time+ self.__duration = durationdef get_stream_source(self) -> str | io.BytesIO:"""@@ -96,6 +98,16 @@ class VideoFromFile(VideoInput):Returns:Duration in seconds"""+ raw_duration = self._get_raw_duration()+ if self.__start_time < 0:+ duration_from_start = min(raw_duration, -self.__start_time)+ else:+ duration_from_start = raw_duration - self.__start_time+ if self.__duration:+ return min(self.__duration, duration_from_start)+ return duration_from_start++ def _get_raw_duration(self) -> float:if isinstance(self.__file, io.BytesIO):self.__file.seek(0)with av.open(self.__file, mode="r") as container:@@ -113,9 +125,13 @@ class VideoFromFile(VideoInput):if video_stream and video_stream.average_rate:frame_count = 0container.seek(0)- for packet in container.demux(video_stream):- for _ in packet.decode():- frame_count += 1+ frame_iterator = (+ container.decode(video_stream)+ if video_stream.codec.capabilities & 0x100+ else container.demux(video_stream)+ )+ for packet in frame_iterator:+ frame_count += 1if frame_count > 0:return float(frame_count / video_stream.average_rate)@@ -131,36 +147,54 @@ class VideoFromFile(VideoInput):with av.open(self.__file, mode="r") as container:video_stream = self._get_first_video_stream(container)- # 1. Prefer the frames field if available- if video_stream.frames and video_stream.frames > 0:+ # 1. Prefer the frames field if available and usable+ if (+ video_stream.frames+ and video_stream.frames > 0+ and not self.__start_time+ and not self.__duration+ ):return int(video_stream.frames)# 2. Try to estimate from duration and average_rate using only metadata- if container.duration is not None and video_stream.average_rate:- duration_seconds = float(container.duration / av.time_base)- estimated_frames = int(round(duration_seconds * float(video_stream.average_rate)))- if estimated_frames > 0:- return estimated_frames-if (getattr(video_stream, "duration", None) is not Noneand getattr(video_stream, "time_base", None) is not Noneand video_stream.average_rate):- duration_seconds = float(video_stream.duration * video_stream.time_base)+ raw_duration = float(video_stream.duration * video_stream.time_base)+ if self.__start_time < 0:+ duration_from_start = min(raw_duration, -self.__start_time)+ else:+ duration_from_start = raw_duration - self.__start_time+ duration_seconds = min(self.__duration, duration_from_start)estimated_frames = int(round(duration_seconds * float(video_stream.average_rate)))if estimated_frames > 0:return estimated_frames# 3. Last resort: decode frames and count them (streaming)- frame_count = 0- container.seek(0)- for packet in container.demux(video_stream):- for _ in packet.decode():- frame_count += 1-- if frame_count == 0:- raise ValueError(f"Could not determine frame count for file '{self.__file}'")+ if self.__start_time < 0:+ start_time = max(self._get_raw_duration() + self.__start_time, 0)+ else:+ start_time = self.__start_time+ frame_count = 1+ start_pts = int(start_time / video_stream.time_base)+ end_pts = int((start_time + self.__duration) / video_stream.time_base)+ container.seek(start_pts, stream=video_stream)+ frame_iterator = (+ container.decode(video_stream)+ if video_stream.codec.capabilities & 0x100+ else container.demux(video_stream)+ )+ for frame in frame_iterator:+ if frame.pts >= start_pts:+ break+ else:+ raise ValueError(f"Could not determine frame count for file '{self.__file}'\nNo frames exist for start_time {self.__start_time}")+ for frame in frame_iterator:+ if frame.pts >= end_pts:+ break+ frame_count += 1return frame_countdef get_frame_rate(self) -> Fraction:@@ -199,9 +233,21 @@ class VideoFromFile(VideoInput):return container.format.namedef get_components_internal(self, container: InputContainer) -> VideoComponents:+ video_stream = self._get_first_video_stream(container)+ if self.__start_time < 0:+ start_time = max(self._get_raw_duration() + self.__start_time, 0)+ else:+ start_time = self.__start_time# Get video framesframes = []- for frame in container.decode(video=0):+ start_pts = int(start_time / video_stream.time_base)+ end_pts = int((start_time + self.__duration) / video_stream.time_base)+ container.seek(start_pts, stream=video_stream)+ for frame in container.decode(video_stream):+ if frame.pts < start_pts:+ continue+ if self.__duration and frame.pts >= end_pts:+ breakimg = frame.to_ndarray(format='rgb24') # shape: (H, W, 3)img = torch.from_numpy(img) / 255.0 # shape: (H, W, 3)frames.append(img)@@ -209,31 +255,44 @@ class VideoFromFile(VideoInput):images = torch.stack(frames) if len(frames) > 0 else torch.zeros(0, 3, 0, 0)# Get frame rate- video_stream = next(s for s in container.streams if s.type == 'video')- frame_rate = Fraction(video_stream.average_rate) if video_stream and video_stream.average_rate else Fraction(1)+ frame_rate = Fraction(video_stream.average_rate) if video_stream.average_rate else Fraction(1)# Get audio if availableaudio = None- try:- container.seek(0) # Reset the container to the beginning- for stream in container.streams:- if stream.type != 'audio':- continue- assert isinstance(stream, av.AudioStream)- audio_frames = []- for packet in container.demux(stream):- for frame in packet.decode():- assert isinstance(frame, av.AudioFrame)- audio_frames.append(frame.to_ndarray()) # shape: (channels, samples)- if len(audio_frames) > 0:- audio_data = np.concatenate(audio_frames, axis=1) # shape: (channels, total_samples)- audio_tensor = torch.from_numpy(audio_data).unsqueeze(0) # shape: (1, channels, total_samples)- audio = AudioInput({- "waveform": audio_tensor,- "sample_rate": int(stream.sample_rate) if stream.sample_rate else 1,- })- except StopIteration:- pass # No audio stream+ container.seek(start_pts, stream=video_stream)+ # Use last stream for consistency+ if len(container.streams.audio):+ audio_stream = container.streams.audio[-1]+ audio_frames = []+ resample = av.audio.resampler.AudioResampler(format='fltp').resample+ frames = itertools.chain.from_iterable(+ map(resample, container.decode(audio_stream))+ )++ has_first_frame = False+ for frame in frames:+ offset_seconds = start_time - frame.pts * audio_stream.time_base+ to_skip = int(offset_seconds * audio_stream.sample_rate)+ if to_skip < frame.samples:+ has_first_frame = True+ break+ if has_first_frame:+ audio_frames.append(frame.to_ndarray()[..., to_skip:])++ for frame in frames:+ if frame.time > start_time + self.__duration:+ break+ audio_frames.append(frame.to_ndarray()) # shape: (channels, samples)+ if len(audio_frames) > 0:+ audio_data = np.concatenate(audio_frames, axis=1) # shape: (channels, total_samples)+ if self.__duration:+ audio_data = audio_data[..., :int(self.__duration * audio_stream.sample_rate)]++ audio_tensor = torch.from_numpy(audio_data).unsqueeze(0) # shape: (1, channels, total_samples)+ audio = AudioInput({+ "waveform": audio_tensor,+ "sample_rate": int(audio_stream.sample_rate) if audio_stream.sample_rate else 1,+ })metadata = container.metadatareturn VideoComponents(images=images, audio=audio, frame_rate=frame_rate, metadata=metadata)@@ -250,7 +309,7 @@ class VideoFromFile(VideoInput):path: str | io.BytesIO,format: VideoContainer = VideoContainer.AUTO,codec: VideoCodec = VideoCodec.AUTO,- metadata: Optional[dict] = None+ metadata: Optional[dict] = None,):if isinstance(self.__file, io.BytesIO):self.__file.seek(0) # Reset the BytesIO object to the beginning@@ -262,15 +321,14 @@ class VideoFromFile(VideoInput):reuse_streams = Falseif codec != VideoCodec.AUTO and codec != video_encoding and video_encoding is not None:reuse_streams = False+ if self.__start_time or self.__duration:+ reuse_streams = Falseif not reuse_streams:components = self.get_components_internal(container)video = VideoFromComponents(components)return video.save_to(- path,- format=format,- codec=codec,- metadata=metadata+ path, format=format, codec=codec, metadata=metadata)streams = container.streams@@ -304,10 +362,21 @@ class VideoFromFile(VideoInput):output_container.mux(packet)def _get_first_video_stream(self, container: InputContainer):- video_stream = next((s for s in container.streams if s.type == "video"), None)- if video_stream is None:- raise ValueError(f"No video stream found in file '{self.__file}'")- return video_stream+ if len(container.streams.video):+ return container.streams.video[0]+ raise ValueError(f"No video stream found in file '{self.__file}'")++ def as_trimmed(+ self, start_time: float = 0, duration: float = 0, strict_duration: bool = True+ ) -> VideoInput | None:+ trimmed = VideoFromFile(+ self.get_stream_source(),+ start_time=start_time + self.__start_time,+ duration=duration,+ )+ if trimmed.get_duration() < duration and strict_duration:+ return None+ return trimmedclass VideoFromComponents(VideoInput):@@ -322,7 +391,7 @@ class VideoFromComponents(VideoInput):return VideoComponents(images=self.__components.images,audio=self.__components.audio,- frame_rate=self.__components.frame_rate+ frame_rate=self.__components.frame_rate,)def save_to(@@ -330,7 +399,7 @@ class VideoFromComponents(VideoInput):path: str,format: VideoContainer = VideoContainer.AUTO,codec: VideoCodec = VideoCodec.AUTO,- metadata: Optional[dict] = None+ metadata: Optional[dict] = None,):if format != VideoContainer.AUTO and format != VideoContainer.MP4:raise ValueError("Only MP4 format is supported for now")@@ -357,7 +426,10 @@ class VideoFromComponents(VideoInput):audio_stream: Optional[av.AudioStream] = Noneif self.__components.audio:audio_sample_rate = int(self.__components.audio['sample_rate'])- audio_stream = output.add_stream('aac', rate=audio_sample_rate)+ waveform = self.__components.audio['waveform']+ waveform = waveform[0, :, :math.ceil((audio_sample_rate / frame_rate) * self.__components.images.shape[0])]+ layout = {1: 'mono', 2: 'stereo', 6: '5.1'}.get(waveform.shape[0], 'stereo')+ audio_stream = output.add_stream('aac', rate=audio_sample_rate, layout=layout)# Encode videofor i, frame in enumerate(self.__components.images):@@ -372,12 +444,21 @@ class VideoFromComponents(VideoInput):output.mux(packet)if audio_stream and self.__components.audio:- waveform = self.__components.audio['waveform']- waveform = waveform[:, :, :math.ceil((audio_sample_rate / frame_rate) * self.__components.images.shape[0])]- frame = av.AudioFrame.from_ndarray(waveform.movedim(2, 1).reshape(1, -1).float().cpu().numpy(), format='flt', layout='mono' if waveform.shape[1] == 1 else 'stereo')+ frame = av.AudioFrame.from_ndarray(waveform.float().cpu().numpy(), format='fltp', layout=layout)frame.sample_rate = audio_sample_rateframe.pts = 0output.mux(audio_stream.encode(frame))# Flush encoderoutput.mux(audio_stream.encode(None))++ def as_trimmed(+ self,+ start_time: float | None = None,+ duration: float | None = None,+ strict_duration: bool = True,+ ) -> VideoInput | None:+ if self.get_duration() < start_time + duration:+ return None+ #TODO Consider tracking duration and trimming at time of save?+ return VideoFromFile(self.get_stream_source(), start_time=start_time, duration=duration)diff --git a/comfy_extras/nodes_video.py b/comfy_extras/nodes_video.pyindex ccf7b63d3..cd765a7c1 100644--- a/comfy_extras/nodes_video.py+++ b/comfy_extras/nodes_video.py@@ -202,6 +202,56 @@ class LoadVideo(io.ComfyNode):return True+class VideoSlice(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="Video Slice",+ display_name="Video Slice",+ search_aliases=[+ "trim video duration",+ "skip first frames",+ "frame load cap",+ "start time",+ ],+ category="image/video",+ inputs=[+ io.Video.Input("video"),+ io.Float.Input(+ "start_time",+ default=0.0,+ max=1e5,+ min=-1e5,+ step=0.001,+ tooltip="Start time in seconds",+ ),+ io.Float.Input(+ "duration",+ default=0.0,+ min=0.0,+ step=0.001,+ tooltip="Duration in seconds, or 0 for unlimited duration",+ ),+ io.Boolean.Input(+ "strict_duration",+ default=False,+ tooltip="If True, when the specified duration is not possible, an error will be raised.",+ ),+ ],+ outputs=[+ io.Video.Output(),+ ],+ )++ @classmethod+ def execute(cls, video: io.Video.Type, start_time: float, duration: float, strict_duration: bool) -> io.NodeOutput:+ trimmed = video.as_trimmed(start_time, duration, strict_duration=strict_duration)+ if trimmed is not None:+ return io.NodeOutput(trimmed)+ raise ValueError(+ f"Failed to slice video:\nSource duration: {video.get_duration()}\nStart time: {start_time}\nTarget duration: {duration}"+ )+class VideoExtension(ComfyExtension):@override@@ -212,6 +262,7 @@ class VideoExtension(ComfyExtension):CreateVideo,GetVideoComponents,LoadVideo,+ VideoSlice,]async def comfy_entrypoint() -> VideoExtension: