Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Unsloth Studio: Cosmos3-Nano video continuation (extend a clip from its last or first frames) (#16741)

Leo Borcherding34905ec7b2b9
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index dcf8eeeb7ed..c68ee21eb96 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -274,6 +274,8 @@ from .video_families import (
local_cosmos3_pipeline_family,
local_ltx_pipeline_family,
pipeline_available_video_families,
+ COSMOS3_CONTINUATION_FRAMES,
+ DEFAULT_VIDEO_CONTINUATION_KEEP,
VideoFamily,
default_video_generation_params,
detect_video_family,
@@ -281,6 +283,7 @@ from .video_families import (
snap_num_frames,
snap_video_size,
supported_video_family_names,
+ validate_video_continuation,
validate_video_flow_controls,
validate_video_keyframe_conditioning,
validate_video_reference_conditioning,
@@ -1076,6 +1079,9 @@ def _decode_b64_media(data: Optional[str]) -> bytes:
return blob
+VIDEO_CONTINUATION_CONDITIONING = "v2v"
+
+
class _VideoGenerationCancelled(Exception):
"""Unwinds a denoise loop that has no cooperative interrupt (no step callback);
generate() maps it to the VIDEO_CANCELLED_MSG sentinel the routes 409 on."""
@@ -1676,6 +1682,7 @@ class _VideoResolvedInputs:
references: Any
flow_shift: Optional[float]
audio_flow_shift: Optional[float]
+ continuation: Any = None
@dataclass
@@ -13189,6 +13196,8 @@ class VideoBackend:
reference_image_size: Optional[str] = None,
flow_shift: Optional[float] = None,
audio_flow_shift: Optional[float] = None,
+ continuation_video: Optional[str] = None,
+ continuation_keep: Optional[str] = None,
video_id: Optional[str] = None,
decoded_reference_audios: Optional[Any] = None,
decoded_reference_images: Optional[Any] = None,
@@ -13264,6 +13273,19 @@ class VideoBackend:
run_num_frames = k * step + offset
if references:
conditioning = h3_conditioning_mode(has_references = True)
+ continuation, canvas_w, canvas_h = self._resolve_continuation(
+ state.family,
+ continuation_video,
+ continuation_keep,
+ requested_size = (width, height),
+ canvas = (canvas_w, canvas_h),
+ fps = fps,
+ num_frames = run_num_frames,
+ has_keyframes = bool(resolved_first_frame or last_frame),
+ has_references = bool(references),
+ )
+ if continuation is not None:
+ conditioning = VIDEO_CONTINUATION_CONDITIONING
resolved_inputs = _VideoResolvedInputs(
state = state,
first_frame = first_pil,
@@ -13274,6 +13296,7 @@ class VideoBackend:
references = references,
flow_shift = shift,
audio_flow_shift = audio_shift,
+ continuation = continuation,
)
with self._lock:
if self._state is not state:
@@ -13556,6 +13579,8 @@ class VideoBackend:
reference_image_size: Optional[str] = None,
flow_shift: Optional[float] = None,
audio_flow_shift: Optional[float] = None,
+ continuation_video: Optional[str] = None,
+ continuation_keep: Optional[str] = None,
cancel_event: Optional[threading.Event] = None,
_resolved_inputs: Optional[_VideoResolvedInputs] = None,
# None = on unless UNSLOTH_DIFFUSION_PREVIEW=0.
@@ -13614,6 +13639,7 @@ class VideoBackend:
)
fam = state.family
if _resolved_inputs is None:
+ requested_size = (width, height)
first_pil, last_pil, width, height, conditioning = self._resolve_keyframes(
fam, state.h3_task, first_frame, last_frame, width, height
)
@@ -13630,6 +13656,19 @@ class VideoBackend:
)
if references:
conditioning = h3_conditioning_mode(has_references = True)
+ continuation, width, height = self._resolve_continuation(
+ fam,
+ continuation_video,
+ continuation_keep,
+ requested_size = requested_size,
+ canvas = (width, height),
+ fps = fps,
+ num_frames = num_frames,
+ has_keyframes = bool(first_frame or last_frame),
+ has_references = bool(references),
+ )
+ if continuation is not None:
+ conditioning = VIDEO_CONTINUATION_CONDITIONING
shift, audio_shift = self._resolve_flow_shifts(
fam, state.engine, flow_shift, audio_flow_shift
)
@@ -13640,6 +13679,7 @@ class VideoBackend:
height = _resolved_inputs.height
conditioning = _resolved_inputs.conditioning
references = _resolved_inputs.references
+ continuation = _resolved_inputs.continuation
shift = _resolved_inputs.flow_shift
audio_shift = _resolved_inputs.audio_flow_shift
frames = snap_num_frames(fam, num_frames or fam.default_num_frames)
@@ -14055,6 +14095,9 @@ class VideoBackend:
if references:
from .video_minimax_h3 import h3_diffusers_references
kwargs["references"] = h3_diffusers_references(references)
+ if continuation is not None:
+ kwargs["video"], kwargs["condition_video_keep"] = continuation
+ request_shape["continuation"] = continuation[1]
# Set scheduler shifts every run so prior requests cannot leak state.
for component, value in (
("scheduler", shift),
@@ -14563,6 +14606,47 @@ class VideoBackend:
conditioning = h3_conditioning_mode(has_first = first is not None, has_last = last is not None)
return first, last, width, height, conditioning
+ @staticmethod
+ def _resolve_continuation(
+ fam: VideoFamily,
+ video: Optional[str],
+ keep: Optional[str],
+ *,
+ requested_size: tuple[Optional[int], Optional[int]],
+ canvas: tuple[int, int],
+ fps: Optional[int],
+ num_frames: Optional[int],
+ has_keyframes: bool,
+ has_references: bool,
+ ) -> tuple[Any, int, int]:
+ """Omitting both dimensions takes the preset nearest the clip's aspect, as a first frame does."""
+ validate_video_continuation(
+ fam,
+ has_video = bool(video),
+ keep = keep,
+ has_keyframes = has_keyframes,
+ has_references = has_references,
+ num_frames = (
+ snap_num_frames(fam, num_frames or fam.default_num_frames) if video else None
+ ),
+ )
+ if not video:
+ return None, *canvas
+ from .video_cosmos3 import decode_continuation_clip, fit_continuation_frames
+
+ validate_video_fps(fam, fps)
+ keep = keep or DEFAULT_VIDEO_CONTINUATION_KEEP
+ clip = decode_continuation_clip(
+ _decode_b64_media(video),
+ keep = keep,
+ fps = int(fps or fam.default_fps),
+ count = COSMOS3_CONTINUATION_FRAMES,
+ )
+ width, height = canvas
+ if requested_size == (None, None):
+ width, height = first_frame_canvas(fam, *clip[0].size)
+ return (fit_continuation_frames(clip, width, height), keep), width, height
+
@staticmethod
def _resolve_ltx25_first_frame(
fam: VideoFamily,
@@ -15416,6 +15500,7 @@ class VideoBackend:
"supports_first_frame": False,
"supports_last_frame": False,
"supports_references": False,
+ "supports_video_continuation": False,
"h3_task": None,
"defaults": None,
"resolved": None,
@@ -15492,6 +15577,7 @@ class VideoBackend:
"supports_last_frame": video_family_accepts_last_frame(fam)
and state.h3_task != H3_TASK_REFERENCES,
"supports_references": fam.supports_references and state.h3_task == H3_TASK_REFERENCES,
+ "supports_video_continuation": fam.supports_video_continuation,
"h3_task": state.h3_task,
"defaults": {
"steps": default_steps,
diff --git a/studio/backend/core/inference/video_cosmos3.py b/studio/backend/core/inference/video_cosmos3.py
index 8f24b1980ff..05cefb428c6 100644
--- a/studio/backend/core/inference/video_cosmos3.py
+++ b/studio/backend/core/inference/video_cosmos3.py
@@ -269,3 +269,102 @@ def seam_safe_tile_strides(pipe: Any, height: int, width: int):
for (tile_key, stride_key), (tile, stride) in zip(keys, old):
setattr(vae, tile_key, tile)
setattr(vae, stride_key, stride)
+
+
+# Same caps as a MiniMax-H3 reference video and a keyframe image: 15 s of source, 4096 px a side.
+CONTINUATION_MAX_SECONDS = 15.0
+CONTINUATION_MAX_SIDE = 4096
+
+
+def _upright(image: Any, frame: Any) -> Any:
+ # PyAV's rotation is counterclockwise degrees, the same sense as PIL's rotate (matches ffmpeg autorotate).
+ rotation = int(getattr(frame, "rotation", 0) or 0) % 360
+ return image.rotate(rotation, expand = True) if rotation else image
+
+
+def decode_continuation_clip(blob: bytes, *, keep: str, fps: int, count: int) -> list:
+ """Frames land on that clock by whole-frame drop and duplicate, picked by presentation time so a variable-frame-rate
+ clip keeps even spacing; a clip already at ``fps`` passes through unchanged."""
+ import collections
+ import math
+
+ import av
+
+ from .video_minimax_h3 import _open_reference_media
+
+ picked: Any = [] if keep == "first" else collections.deque(maxlen = count)
+ try:
+ container = _open_reference_media(av, blob)
+ except av.error.FFmpegError as exc:
+ raise ValueError("That clip could not be read as a video.") from exc
+ with container:
+ if not container.streams.video:
+ raise ValueError("That clip carries no video track.")
+ stream = container.streams.video[0]
+ source_fps = float(stream.average_rate or stream.guessed_rate or fps)
+ if source_fps <= 0:
+ source_fps = float(fps)
+ max_source_frames = math.floor(CONTINUATION_MAX_SECONDS * source_fps + 1e-6)
+ next_target = 0
+ decoded = 0
+ held: Any = None
+ origin: Optional[float] = None
+
+ def _take(frame: Any, until: float) -> None:
+ nonlocal next_target
+ while next_target / fps < until - 1e-6:
+ if keep != "first" or len(picked) < count:
+ picked.append(frame)
+ next_target += 1
+
+ try:
+ for index, frame in enumerate(container.decode(video = 0)):
+ decoded = index + 1
+ stamp = frame.time if frame.time is not None else index / source_fps
+ if origin is None:
+ origin = stamp
+ # Count and timestamps both: a VFR clip's average rate can understate a long gap.
+ if (
+ decoded > max_source_frames
+ or stamp - origin + 1 / source_fps > CONTINUATION_MAX_SECONDS + 1e-6
+ ):
+ raise ValueError(
+ f"The clip to continue runs longer than {CONTINUATION_MAX_SECONDS:g} seconds. Trim it first."
+ )
+ if max(frame.width, frame.height) > CONTINUATION_MAX_SIDE:
+ raise ValueError(
+ f"The clip to continue is {frame.width}x{frame.height}; the limit is "
+ f"{CONTINUATION_MAX_SIDE} px a side."
+ )
+ # "first" still decodes to the end, so both ends are held to the same limits.
+ if held is not None:
+ _take(held, max(stamp - origin, 0.0))
+ held = frame
+ if held is not None:
+ # The last frame lasts one nominal frame, as the clip's duration counts it.
+ _take(held, max(stamp - origin, 0.0) + 1 / source_fps)
+ except av.error.FFmpegError as exc:
+ raise ValueError("That clip could not be decoded.") from exc
+ frames = [
+ _upright(frame.to_image().convert("RGB"), frame) for frame in list(picked)[:count]
+ ]
+ if len(frames) < count:
+ raise ValueError(
+ f"This clip has {len(frames)} frames at {fps} fps, and continuing it needs at least {count}. "
+ "Use a longer clip."
+ )
+ return frames
+
+
+def fit_continuation_frames(frames: list, width: int, height: int) -> list:
+ """The pipeline resizes onto the canvas itself, so a clip already at the canvas aspect passes through untouched."""
+ if not frames:
+ return frames
+ src_w, src_h = frames[0].size
+ crop_w = min(src_w, round(src_h * width / height))
+ crop_h = min(src_h, round(src_w * height / width))
+ if (crop_w, crop_h) == (src_w, src_h):
+ return frames
+ left, top = (src_w - crop_w) // 2, (src_h - crop_h) // 2
+ box = (left, top, left + crop_w, top + crop_h)
+ return [frame.crop(box) for frame in frames]
diff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.py
index d76f6b9463d..8fab2605e09 100644
--- a/studio/backend/core/inference/video_families.py
+++ b/studio/backend/core/inference/video_families.py
@@ -337,6 +337,7 @@ class VideoFamily:
unsupported_checkpoints: tuple[tuple[str, str], ...] = field(default_factory = tuple)
# Omni-reference conditioning. MiniMax-H3 loads this in a separate denoiser partition.
supports_references: bool = False
+ supports_video_continuation: bool = False
# Guidance-distilled families expose neither CFG nor a negative prompt.
supports_cfg: bool = True
# True when Unsloth can TRAIN a LoRA on this family (every such name must resolve through get_trainer).
@@ -1049,6 +1050,8 @@ _FAMILIES: tuple[VideoFamily, ...] = (
supports_torch_compile = False,
i2v_pipeline_class = "Cosmos3OmniPipeline",
i2v_fits_first_frame = False,
+ # Nano only: Edge's card lists text, image and action input, no video.
+ supports_video_continuation = True,
scheduler_overrides = _COSMOS3_NANO_SCHEDULER,
cudnn_benchmark = False,
vae_fused_encoder = False,
@@ -1679,6 +1682,41 @@ def validate_video_keyframe_conditioning(
)
+VIDEO_CONTINUATION_KEEP = ("last", "first")
+DEFAULT_VIDEO_CONTINUATION_KEEP = "last"
+# Pixel frames the pipeline keeps clean: latent indexes (0, 1) under the Wan2.2 VAE's 4x temporal compression.
+COSMOS3_CONTINUATION_FRAMES = 5
+
+
+def validate_video_continuation(
+ fam: VideoFamily,
+ *,
+ has_video: bool,
+ keep: Optional[str] = None,
+ has_keyframes: bool = False,
+ has_references: bool = False,
+ num_frames: Optional[int] = None,
+) -> None:
+ """Pure, so the route can judge an auto-switch target before evicting anything."""
+ if not has_video:
+ if keep is not None:
+ raise ValueError("continuation_keep needs a continuation_video.")
+ return
+ if not fam.supports_video_continuation:
+ raise ValueError(f"{fam.name} takes no input video, so it cannot continue a clip.")
+ if keep is not None and keep not in VIDEO_CONTINUATION_KEEP:
+ raise ValueError("continuation_keep must be 'last' or 'first'.")
+ if has_keyframes or has_references:
+ raise ValueError(
+ "A clip to continue cannot be combined with a start frame, end frame or references."
+ )
+ if num_frames is not None and num_frames <= COSMOS3_CONTINUATION_FRAMES:
+ raise VideoShapeError(
+ f"Continuing a clip keeps its {COSMOS3_CONTINUATION_FRAMES} frames, so the length must be more than "
+ f"{COSMOS3_CONTINUATION_FRAMES} frames to generate anything; got {num_frames}."
+ )
+
+
def validate_video_flow_controls(
fam: VideoFamily,
flow_shift: Optional[float],
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index a03f43b71ed..651e2275574 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -7641,6 +7641,21 @@ class VideoGenerateRequest(BaseModel):
"cover-cropped onto the canvas.",
)
+ continuation_video: Optional[str] = Field(
+ None,
+ max_length = 96 * 1024 * 1024,
+ description = "Base64/data-URL video to continue (Cosmos3-Nano only, status "
+ "supports_video_continuation): the clip keeps 5 frames from one end of it and generates "
+ "what follows. At most 15 seconds; frames are resampled onto fps and centre-cropped to "
+ "the canvas aspect. With width and height omitted the canvas is the preset nearest the "
+ "clip's aspect. Cannot be combined with keyframes or references.",
+ )
+ continuation_keep: Optional[Literal["last", "first"]] = Field(
+ None,
+ description = "Which end of continuation_video to keep: last (default) continues the "
+ "clip from its final frames; first keeps its opening frames and generates on from there.",
+ )
+
# Separate lists preserve the model's image, video, then audio packing order.
reference_images: Optional[list[str]] = Field(
None,
@@ -7708,6 +7723,14 @@ class VideoGenerateRequest(BaseModel):
raise ValueError(
"reference audio needs at least one reference image or video to go with"
)
+ if self.continuation_video and (
+ images or videos or audios or self.first_frame or self.last_frame
+ ):
+ raise ValueError(
+ "A clip to continue cannot be combined with a start frame, end frame or references."
+ )
+ if self.continuation_keep is not None and not self.continuation_video:
+ raise ValueError("continuation_keep needs a continuation_video.")
if (images or videos or audios) and (self.first_frame or self.last_frame):
raise ValueError(
"keyframes and references cannot be combined: MiniMax-H3 runs them against "
@@ -8069,6 +8092,11 @@ class VideoStatusResponse(BaseModel):
"(gates the reference-to-video controls). Never true at the same time as "
"supports_keyframes: MiniMax-H3 serves the two from different denoiser partitions.",
)
+ supports_video_continuation: bool = Field(
+ False,
+ description = "Whether the LOADED checkpoint takes continuation_video (Cosmos3-Nano "
+ "only): it keeps frames from one end of a clip and generates what happens next",
+ )
h3_task: Optional[str] = Field(
None,
description = "The MiniMax-H3 denoiser partition that is up: fl2va | ref2va, or null "
diff --git a/studio/backend/routes/video.py b/studio/backend/routes/video.py
index 641963c4fcc..860abdcb855 100644
--- a/studio/backend/routes/video.py
+++ b/studio/backend/routes/video.py
@@ -731,6 +731,8 @@ async def generate_video(
)
from core.inference.video_minimax_h3 import is_h3_native
from core.inference.video_families import (
+ snap_num_frames,
+ validate_video_continuation,
validate_video_flow_controls,
validate_video_keyframe_conditioning,
validate_video_reference_conditioning,
@@ -767,6 +769,12 @@ async def generate_video(
validate_video_flow_controls(
fam, request.flow_shift, request.audio_flow_shift, engine = engine
)
+ validate_video_continuation(
+ fam,
+ has_video = bool(request.continuation_video),
+ keep = request.continuation_keep,
+ num_frames = snap_num_frames(fam, request.num_frames or fam.default_num_frames),
+ )
# Before the backend is resolved: the requested model may be the one this brings up.
try:
@@ -804,6 +812,8 @@ async def generate_video(
reference_image_size = request.reference_image_size,
flow_shift = request.flow_shift,
audio_flow_shift = request.audio_flow_shift,
+ continuation_video = request.continuation_video,
+ continuation_keep = request.continuation_keep,
live_preview = request.live_preview,
)
# Authorize the reservation-pinned token: a gap load must not render weights of another account.
diff --git a/studio/backend/tests/test_video_cosmos3.py b/studio/backend/tests/test_video_cosmos3.py
index 0afa9181da4..fc8cf132ced 100644
--- a/studio/backend/tests/test_video_cosmos3.py
+++ b/studio/backend/tests/test_video_cosmos3.py
@@ -1488,3 +1488,313 @@ def test_generate_renders_and_exports_at_the_chosen_fps(cosmos3_runtime, repo, f
with pytest.raises(vf.VideoShapeError, match = "frame rate"):
backend.generate(prompt = "p", steps = 1, fps = bad)
assert backend.generate_progress()["active"] is False
+
+
+# Video continuation (Cosmos3-Nano only): keep frames from one end of a clip, generate what follows.
+
+
+def _clip(
+ count,
+ fps = 24,
+ size = (208, 120),
+):
+ """An mp4 whose frame i is solid gray 20*i, so a decoded frame names its source index."""
+ av = pytest.importorskip("av")
+ import io
+
+ import numpy as np
+
+ for codec in ("libx264", "mpeg4"):
+ buf = io.BytesIO()
+ try:
+ with av.open(buf, "w", format = "mp4") as container:
+ stream = container.add_stream(codec, rate = fps)
+ stream.width, stream.height = size
+ stream.pix_fmt = "yuv420p"
+ if codec == "libx264":
+ stream.options = {"qp": "0"}
+ for i in range(count):
+ pixels = np.full((size[1], size[0], 3), (20 * i) % 256, np.uint8)
+ for packet in stream.encode(av.VideoFrame.from_ndarray(pixels, format = "rgb24")):
+ container.mux(packet)
+ for packet in stream.encode():
+ container.mux(packet)
+ except av.error.FFmpegError:
+ continue
+ return buf.getvalue()
+ pytest.skip("PyAV has no mp4 video encoder")
+
+
+def _clip_url(*args, **kwargs):
+ import base64
+ return "data:video/mp4;base64," + base64.b64encode(_clip(*args, **kwargs)).decode()
+
+
+def _indexes(frames):
+ import numpy as np
+ return [int(round(float(np.asarray(f).mean()) / 20)) for f in frames]
+
+
+def test_only_nano_continues_a_clip():
+ assert {f.name for f in vf._FAMILIES if f.supports_video_continuation} == {"cosmos3-nano"}
+
+
+ "name, kwargs, error, fragment",
+ [
+ ("cosmos3-nano", dict(has_video = True), None, None),
+ ("cosmos3-nano", dict(has_video = True, keep = "first", num_frames = 9), None, None),
+ ("cosmos3-nano", dict(has_video = False), None, None),
+ ("cosmos3-edge", dict(has_video = False), None, None),
+ ("cosmos3-edge", dict(has_video = True), ValueError, "takes no input video"),
+ ("wan2.2-ti2v-5b", dict(has_video = True), ValueError, "takes no input video"),
+ ("cosmos3-nano", dict(has_video = False, keep = "last"), ValueError, "needs a"),
+ ("cosmos3-nano", dict(has_video = True, keep = "middle"), ValueError, "'last' or 'first'"),
+ ("cosmos3-nano", dict(has_video = True, has_keyframes = True), ValueError, "start frame"),
+ ("cosmos3-nano", dict(has_video = True, has_references = True), ValueError, "references"),
+ ("cosmos3-nano", dict(has_video = True, num_frames = 5), vf.VideoShapeError, "more than 5"),
+ ],
+)
+def test_continuation_rules(name, kwargs, error, fragment):
+ if error is None:
+ vf.validate_video_continuation(_fam(name), **kwargs)
+ return
+ with pytest.raises(error, match = fragment):
+ vf.validate_video_continuation(_fam(name), **kwargs)
+
+
+ "count, source_fps, fps, keep, expected",
+ [
+ (12, 24, 24, "last", [7, 8, 9, 10, 11]),
+ (12, 24, 24, "first", [0, 1, 2, 3, 4]),
+ # Resampled onto the output clock: 48 fps source at 24 fps keeps every other frame.
+ (12, 48, 24, "last", [2, 4, 6, 8, 10]),
+ (12, 12, 24, "first", [0, 0, 1, 1, 2]),
+ (5, 24, 24, "last", [0, 1, 2, 3, 4]),
+ ],
+)
+def test_the_clip_keeps_five_frames_from_the_chosen_end(count, source_fps, fps, keep, expected):
+ frames = vc3.decode_continuation_clip(_clip(count, fps = source_fps), keep = keep, fps = fps, count = 5)
+ assert _indexes(frames) == expected
+ assert all(f.mode == "RGB" and f.size == (208, 120) for f in frames)
+
+
+def _vfr_clip(stamps_ms, size = (208, 120)):
+ """An mp4 like _clip, but frame i is shown at stamps_ms[i]."""
+ av = pytest.importorskip("av")
+ import io
+
+ import numpy as np
+
+ for codec in ("libx264", "mpeg4"):
+ buf = io.BytesIO()
+ try:
+ with av.open(buf, "w", format = "mp4") as container:
+ stream = container.add_stream(codec, rate = 1000)
+ stream.width, stream.height = size
+ stream.pix_fmt = "yuv420p"
+ if codec == "libx264":
+ stream.options = {"qp": "0"}
+ for i, ms in enumerate(stamps_ms):
+ pixels = np.full((size[1], size[0], 3), (20 * i) % 256, np.uint8)
+ frame = av.VideoFrame.from_ndarray(pixels, format = "rgb24")
+ frame.pts = ms
+ for packet in stream.encode(frame):
+ container.mux(packet)
+ for packet in stream.encode():
+ container.mux(packet)
+ except av.error.FFmpegError:
+ continue
+ return buf.getvalue()
+ pytest.skip("PyAV has no mp4 video encoder")
+
+
+def test_a_variable_frame_rate_clip_is_resampled_by_timestamp():
+ # 50 fps for 6 frames, then 8 fps: index-based picking at the average rate would give [0, 0, 1, 1, 2].
+ clip = _vfr_clip([0, 20, 40, 60, 80, 100, 225, 350, 475, 600, 725, 850])
+ frames = vc3.decode_continuation_clip(clip, keep = "first", fps = 24, count = 5)
+ assert _indexes(frames) == [0, 2, 4, 5, 5]
+
+
+def test_a_long_timestamp_gap_is_refused_even_when_the_rate_hides_it(monkeypatch):
+ pytest.importorskip("av")
+ from core.inference import video_minimax_h3
+
+ real_open = video_minimax_h3._open_reference_media
+
+ class _Stream:
+ def __init__(self, stream):
+ self._stream = stream
+ self.average_rate = 1000 # reported far above the timestamped rate
+ self.guessed_rate = None
+
+ class _Container:
+ def __init__(self, container):
+ self._container = container
+ self.streams = types.SimpleNamespace(video = [_Stream(container.streams.video[0])])
+
+ def decode(self, **kwargs):
+ return self._container.decode(**kwargs)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc):
+ return self._container.__exit__(*exc)
+
+ monkeypatch.setattr(
+ video_minimax_h3,
+ "_open_reference_media",
+ lambda av_mod, blob: _Container(real_open(av_mod, blob)),
+ )
+ monkeypatch.setattr(vc3, "CONTINUATION_MAX_SECONDS", 0.5)
+ clip = _vfr_clip([0, 20, 40, 60, 80, 900])
+ with pytest.raises(ValueError, match = "longer than 0.5 seconds"):
+ vc3.decode_continuation_clip(clip, keep = "first", fps = 24, count = 5)
+ vc3.decode_continuation_clip(_vfr_clip([0, 20, 40, 60, 80, 400]), keep = "first", fps = 24, count = 5)
+
+
+def test_short_long_and_unreadable_clips_are_refused(monkeypatch):
+ with pytest.raises(
+ ValueError,
+ match = r"^This clip has 4 frames at 24 fps, and continuing it needs at least 5\. Use a longer clip\.$",
+ ):
+ vc3.decode_continuation_clip(_clip(4), keep = "last", fps = 24, count = 5)
+ monkeypatch.setattr(vc3, "CONTINUATION_MAX_SECONDS", 0.5)
+ vc3.decode_continuation_clip(_clip(12), keep = "last", fps = 24, count = 5)
+ for keep in ("last", "first"):
+ with pytest.raises(ValueError, match = "longer than 0.5 seconds"):
+ vc3.decode_continuation_clip(_clip(13), keep = keep, fps = 24, count = 5)
+ monkeypatch.setattr(vc3, "CONTINUATION_MAX_SIDE", 200)
+ with pytest.raises(ValueError, match = "208x120"):
+ vc3.decode_continuation_clip(_clip(6), keep = "first", fps = 24, count = 5)
+ with pytest.raises(ValueError, match = "could not be"):
+ vc3.decode_continuation_clip(b"not a video", keep = "last", fps = 24, count = 5)
+
+
[email protected]("rotation, size", [(90, (120, 208)), (-90, (120, 208)), (180, (208, 120))])
+def test_a_phone_clip_is_upright_as_a_player_shows_it(tmp_path, rotation, size):
+ # Phones store portrait clips landscape with a display rotation; ffmpeg's autorotate is the reference.
+ import shutil
+ import subprocess
+
+ import numpy as np
+ from PIL import Image
+
+ if shutil.which("ffmpeg") is None:
+ pytest.skip("needs the ffmpeg CLI to tag a display rotation")
+ plain, rotated, ref = tmp_path / "plain.mp4", tmp_path / "rot.mp4", tmp_path / "ref.png"
+ av = pytest.importorskip("av")
+ with av.open(str(plain), "w", format = "mp4") as container:
+ stream = container.add_stream("mpeg4", rate = 24)
+ stream.width, stream.height = 208, 120
+ stream.pix_fmt = "yuv420p"
+ pixels = np.zeros((120, 208, 3), np.uint8)
+ pixels[:40, :64] = 255
+ for _ in range(6):
+ for packet in stream.encode(av.VideoFrame.from_ndarray(pixels, format = "rgb24")):
+ container.mux(packet)
+ for packet in stream.encode():
+ container.mux(packet)
+ run = lambda *a: subprocess.run(["ffmpeg", "-loglevel", "error", "-y", *a], check = True)
+ run("-display_rotation", str(rotation), "-i", str(plain), "-c", "copy", str(rotated))
+ run("-i", str(rotated), "-frames:v", "1", str(ref))
+ frames = vc3.decode_continuation_clip(rotated.read_bytes(), keep = "first", fps = 24, count = 5)
+ assert frames[0].size == size
+ want = np.asarray(Image.open(ref).convert("RGB"), np.int16)
+ assert np.abs(np.asarray(frames[0], np.int16) - want).mean() < 8
+
+
+def test_frames_are_cropped_to_the_canvas_aspect_only_when_it_differs():
+ from PIL import Image
+
+ same = [Image.new("RGB", (1664, 960))]
+ assert vc3.fit_continuation_frames(same, 832, 480)[0] is same[0]
+ wide = [Image.new("RGB", (1280, 720))]
+ assert vc3.fit_continuation_frames(wide, 832, 480)[0].size == (1248, 720)
+ tall = [Image.new("RGB", (720, 720))]
+ assert vc3.fit_continuation_frames(tall, 832, 480)[0].size == (720, 415)
+
+
+def test_status_offers_continuation_on_nano_only(cosmos3_runtime):
+ assert VideoBackend().status()["supports_video_continuation"] is False
+ for repo, offered in ((NANO, True), (EDGE, False)):
+ assert _loaded(repo)[1]["supports_video_continuation"] is offered
+
+
+ "keep, size, canvas, expected",
+ [
+ (None, None, (832, 480), [7, 8, 9, 10, 11]),
+ ("first", None, (832, 480), [0, 1, 2, 3, 4]),
+ ("last", (1280, 720), (1280, 720), [7, 8, 9, 10, 11]),
+ ],
+)
+def test_nano_passes_the_kept_frames_to_the_pipeline(cosmos3_runtime, keep, size, canvas, expected):
+ backend, _status = _loaded(NANO)
+ pipe = backend._state.pipe
+ request = {"continuation_video": _clip_url(12), "num_frames": 33, "steps": 2, "seed": 0}
+ if keep:
+ request["continuation_keep"] = keep
+ if size:
+ request["width"], request["height"] = size
+ result = backend.generate(prompt = "the robot keeps walking", **request)
+ passed = pipe.passed
+ assert passed["condition_video_keep"] == (keep or "last") and "image" not in passed
+ assert _indexes(passed["video"]) == expected
+ # 208x120 is the 832x480 aspect; at 1280x720 it is centre-cropped to 16:9 and the pipeline resizes it.
+ assert {f.size for f in passed["video"]} == {(208, 120) if canvas == (832, 480) else (208, 117)}
+ assert (passed["width"], passed["height"], passed["num_frames"]) == (*canvas, 33)
+ assert result["conditioning"] == "v2v"
+ backend.generate(prompt = "p", steps = 1)
+ assert not {"video", "condition_video_keep"} & set(pipe.passed)
+
+
+ "axis, canvas", [({"width": 832}, (832, 720)), ({"height": 480}, (1280, 480))]
+)
+def test_one_supplied_axis_is_kept_rather_than_matched_to_the_clip(cosmos3_runtime, axis, canvas):
+ backend, _status = _loaded(NANO)
+ clip = _clip_url(12, size = (120, 208))
+ backend.generate(prompt = "p", steps = 1, num_frames = 9, continuation_video = clip, **axis)
+ passed = backend._state.pipe.passed
+ assert (passed["width"], passed["height"]) == canvas
+
+
+def test_begin_generate_resolves_the_clip_before_reserving(cosmos3_runtime):
+ backend, _status = _loaded(NANO)
+ with pytest.raises(vf.VideoShapeError, match = "more than 5"):
+ backend.begin_generate(prompt = "p", continuation_video = _clip_url(12), num_frames = 5)
+ with pytest.raises(ValueError, match = "has 3"):
+ backend.begin_generate(prompt = "p", continuation_video = _clip_url(3))
+ with pytest.raises(ValueError, match = "start frame"):
+ backend.begin_generate(
+ prompt = "p", continuation_video = _clip_url(12), first_frame = _data_url(64, 64)
+ )
+ assert backend._state.pipe.passed is None and backend.generate_progress()["active"] is False
+
+
+def test_edge_refuses_a_clip_without_rendering(cosmos3_runtime):
+ backend, _status = _loaded(EDGE)
+ for call in (backend.begin_generate, backend.generate):
+ with pytest.raises(ValueError, match = "takes no input video"):
+ call(prompt = "p", steps = 2, continuation_video = _clip_url(12))
+ assert backend._state.pipe.passed is None and backend.generate_progress()["active"] is False
+
+
+def test_the_request_model_keeps_continuation_apart_from_keyframes_and_references():
+ from pydantic import ValidationError
+
+ from models.inference import VideoGenerateRequest
+
+ ok = VideoGenerateRequest(prompt = "p", continuation_video = "data:,x", continuation_keep = "first")
+ assert ok.continuation_keep == "first"
+ for extra, fragment in (
+ ({"continuation_keep": "last"}, "needs a continuation_video"),
+ ({"continuation_video": "x", "first_frame": "x"}, "cannot be combined"),
+ ({"continuation_video": "x", "reference_images": ["x"]}, "cannot be combined"),
+ ({"continuation_video": "x", "continuation_keep": "middle"}, "last"),
+ ):
+ with pytest.raises(ValidationError, match = fragment):
+ VideoGenerateRequest(prompt = "p", **extra)
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/model-capabilities.ts b/studio/frontend/src/features/model-picker/components/model-selector/model-capabilities.ts
index 33de839a8d5..21a264fc5f1 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/model-capabilities.ts
+++ b/studio/frontend/src/features/model-picker/components/model-selector/model-capabilities.ts
@@ -35,7 +35,7 @@ const IMAGE_GEN_TAGS = new Set([
"image-text-to-image",
"inpainting",
]);
-// Only tags the Video page can run; video-to-video is absent (backend takes reference images).
+// Only tags the Video page can run; video-to-video is absent (Nano's continuation is gated on status).
const VIDEO_GEN_TAGS = new Set([
"text-to-video",
"image-to-video",
diff --git a/studio/frontend/src/features/video/api.ts b/studio/frontend/src/features/video/api.ts
index 8a828b843f7..dc0171ec301 100644
--- a/studio/frontend/src/features/video/api.ts
+++ b/studio/frontend/src/features/video/api.ts
@@ -97,6 +97,7 @@ export interface VideoStatus {
supports_first_frame?: boolean;
supports_last_frame?: boolean;
supports_references?: boolean;
+ supports_video_continuation?: boolean;
h3_task?: string | null;
defaults?: VideoGenerationDefaults | null;
// One or two sentences on how the loaded model reads prompts; null when it has no tip.
@@ -213,8 +214,12 @@ export interface VideoGenerateRequest {
// Video schedule shift, and the audio one (Diffusers engine only).
flow_shift?: number;
audio_flow_shift?: number;
+ continuation_video?: string;
+ continuation_keep?: VideoContinuationKeep;
}
+export type VideoContinuationKeep = "last" | "first";
+
export interface GalleryVideo {
id: string;
url: string;
diff --git a/studio/frontend/src/features/video/video-continuation.ts b/studio/frontend/src/features/video/video-continuation.ts
new file mode 100644
index 00000000000..4cee0e8cd51
--- /dev/null
+++ b/studio/frontend/src/features/video/video-continuation.ts
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import type { VideoContinuationKeep, VideoStatus } from "./api.ts";
+
+// The pipeline keeps 5 frames of the clip (two latent frames); the rest of the length is generated.
+export const CONTINUATION_KEPT_FRAMES = 5;
+// Mirrors the backend's CONTINUATION_MAX_SECONDS; the backend refuses a longer clip.
+export const CONTINUATION_MAX_SECONDS = 15;
+
+export const CONTINUATION_KEEP_OPTIONS: ReadonlyArray<{
+ value: VideoContinuationKeep;
+ label: string;
+}> = [
+ { value: "last", label: "From its last frames" },
+ { value: "first", label: "From its first frames" },
+];
+
+export function supportsVideoContinuation(
+ status: Pick<VideoStatus, "supports_video_continuation"> | null | undefined,
+): boolean {
+ return status?.supports_video_continuation === true;
+}
+
+export function continuationLengthError(numFrames: number): string | null {
+ return numFrames > CONTINUATION_KEPT_FRAMES
+ ? null
+ : `Continuing a clip keeps ${CONTINUATION_KEPT_FRAMES} of its frames; pick a longer length so there is something to generate`;
+}
+
+/** A note for a picked clip the backend will refuse as too long, or null (unknown durations pass). */
+export function continuationDurationWarning(durationSeconds: number | undefined): string | null {
+ if (durationSeconds === undefined || durationSeconds <= CONTINUATION_MAX_SECONDS) return null;
+ return `This clip is ${durationSeconds.toFixed(1)} s; clips to continue can be up to ${CONTINUATION_MAX_SECONDS} s. Trim it first.`;
+}
+
+export function continuationRequest(
+ supported: boolean,
+ clipDataUrl: string | null | undefined,
+ keep: VideoContinuationKeep,
+): { continuation_video?: string; continuation_keep?: VideoContinuationKeep } {
+ if (!supported || !clipDataUrl) return {};
+ return { continuation_video: clipDataUrl, continuation_keep: keep };
+}
diff --git a/studio/frontend/src/features/video/video-page.tsx b/studio/frontend/src/features/video/video-page.tsx
index 93130f6eacb..91db37f4fa9 100644
--- a/studio/frontend/src/features/video/video-page.tsx
+++ b/studio/frontend/src/features/video/video-page.tsx
@@ -250,6 +250,13 @@ import {
} from "@/lib/image-crop";
import { ReferenceImageEditor } from "@/components/image-crop-editor";
import { type ReferenceMedia, ReferenceMediaPicker } from "./reference-picker";
+import {
+ CONTINUATION_KEEP_OPTIONS,
+ continuationDurationWarning,
+ continuationLengthError,
+ continuationRequest,
+ supportsVideoContinuation,
+} from "./video-continuation";
import {
defaultReferenceVideoTrim,
H3_REFERENCE_MAX_SECONDS,
@@ -259,6 +266,7 @@ import {
import {
type GalleryVideo,
type VideoGenerateProgress,
+ type VideoContinuationKeep,
type VideoReferenceVideo,
type VideoLoadProgress,
type VideoLoadRequest,
@@ -499,6 +507,7 @@ const CONDITIONING_LABELS: Record<string, string> = {
l2va: "To end frame",
fl2va: "Start to end frame",
ref2va: "From references",
+ v2v: "Continued a clip",
};
function clipMeta(video: GalleryVideo): string {
@@ -1267,6 +1276,8 @@ function VideoGenerator({
// MiniMax-H3 keyframes as data URLs: the frame the clip starts from, the frame it ends on, or both.
const [firstFrame, setFirstFrame] = useState<string | null>(null);
const [lastFrame, setLastFrame] = useState<string | null>(null);
+ const [continuationClip, setContinuationClip] = useState<ReferenceMedia | null>(null);
+ const [continuationKeep, setContinuationKeep] = useState<VideoContinuationKeep>("last");
const [keyframeAspect, setKeyframeAspect] = useState<[number, number] | null>(null);
// Separate lists preserve Ref2VA's image, video, then audio request order.
const [referenceImages, setReferenceImages] = useState<StagedReferenceImage[]>([]);
@@ -1570,6 +1581,17 @@ function VideoGenerator({
if (status?.loaded && !supportsFirstFrame) setFirstFrame(null);
if (status?.loaded && !supportsLastFrame) setLastFrame(null);
}, [status?.loaded, supportsFirstFrame, supportsLastFrame]);
+ const supportsContinuation = supportsVideoContinuation(status);
+ const liveInputs = useRef({ firstFrame, lastFrame, continuationClip });
+ liveInputs.current = { firstFrame, lastFrame, continuationClip };
+ const dropContinuationClip = useCallback((why: string) => {
+ if (!liveInputs.current.continuationClip) return;
+ setContinuationClip(null);
+ toast.info(`Removed the clip to continue: ${why}`);
+ }, []);
+ useEffect(() => {
+ if (status?.loaded && !supportsContinuation) setContinuationClip(null);
+ }, [status?.loaded, supportsContinuation]);
useEffect(() => {
if (status?.loaded && !supportsReferences) {
setReferenceImages([]);
@@ -1582,24 +1604,34 @@ function VideoGenerator({
if (!canPickReferenceSize) setReferenceImageSize("match");
}, [canPickReferenceSize]);
+ // A clip to continue sizes the canvas like a start frame (the two exclude each other).
+ const canvasClip = canvasKeyframe ? null : supportsContinuation ? (continuationClip?.dataUrl ?? null) : null;
+ const canvasSource = canvasKeyframe ?? canvasClip;
useEffect(() => {
- if (!canvasKeyframe) {
+ if (!canvasSource) {
setKeyframeAspect(null);
return;
}
let cancelled = false;
- const img = new Image();
- img.onload = () => {
- if (!cancelled) setKeyframeAspect([img.naturalWidth, img.naturalHeight]);
- };
- img.onerror = () => {
- if (!cancelled) setKeyframeAspect(null);
+ const done = (size: [number, number] | null) => {
+ if (!cancelled) setKeyframeAspect(size && size[0] > 0 && size[1] > 0 ? size : null);
};
- img.src = canvasKeyframe;
+ if (canvasKeyframe) {
+ const img = new Image();
+ img.onload = () => done([img.naturalWidth, img.naturalHeight]);
+ img.onerror = () => done(null);
+ img.src = canvasKeyframe;
+ } else {
+ const media = document.createElement("video");
+ media.preload = "metadata";
+ media.onloadedmetadata = () => done([media.videoWidth, media.videoHeight]);
+ media.onerror = () => done(null);
+ media.src = canvasSource;
+ }
return () => {
cancelled = true;
};
- }, [canvasKeyframe]);
+ }, [canvasSource, canvasKeyframe]);
const matchedResolution = useMemo(
() =>
@@ -1609,7 +1641,7 @@ function VideoGenerator({
[keyframeAspect, status?.defaults],
);
- const sourceSized = matchSource && canvasKeyframe != null && matchedResolution != null;
+ const sourceSized = matchSource && canvasSource != null && matchedResolution != null;
const shownSize = sourceSized ? matchedResolution : videoSize;
const livePreviewBox = { maxWidth: shownSize[0], maxHeight: shownSize[1] };
@@ -1772,12 +1804,12 @@ function VideoGenerator({
// Select "match source" only after the staged keyframe passes the aspect-ratio check.
const hadKeyframeRef = useRef(false);
useEffect(() => {
- const has = canvasKeyframe != null;
+ const has = canvasSource != null;
if (has === hadKeyframeRef.current) return;
if (has && !matchedResolution) return;
hadKeyframeRef.current = has;
setMatchSource(has);
- }, [canvasKeyframe, matchedResolution]);
+ }, [canvasSource, matchedResolution]);
const prevFamilyRef = useRef<string | null>(null);
const prevFpsRef = useRef(fps);
@@ -3691,6 +3723,12 @@ function VideoGenerator({
return;
}
}
+ const stagedClip = supportsContinuation ? continuationClip : null;
+ const lengthError = stagedClip ? continuationLengthError(numFrames) : null;
+ if (lengthError) {
+ toast.error(lengthError);
+ return;
+ }
// Pinned now, even if random, so the recipe records it.
let resolvedSeed: number | undefined;
if (seed.trim()) {
@@ -3757,6 +3795,7 @@ function VideoGenerator({
canPickAudioFlowShift && audioFlowShift != null && audioFlowShift !== defaultAudioFlowShift
? audioFlowShift
: undefined,
+ ...continuationRequest(supportsContinuation, stagedClip?.dataUrl, continuationKeep),
live_preview: livePreview,
});
} catch (err) {
@@ -3798,6 +3837,9 @@ function VideoGenerator({
audioFlowShift,
defaultAudioFlowShift,
canPickAudioFlowShift,
+ supportsContinuation,
+ continuationClip,
+ continuationKeep,
startGenPoll,
resyncAfterGenerateRefusal,
]);
@@ -4170,6 +4212,8 @@ function VideoGenerator({
<p className="text-xs leading-snug text-muted-foreground">
{supportsReferences
? "Generate a video from a prompt and reference pictures, videos or audio"
+ : supportsContinuation
+ ? "Generate a video from a prompt, animate a start frame, or continue a clip"
: supportsLastFrame
? "Generate a video from a prompt, or from a start and end frame"
: supportsFirstFrame
@@ -4234,7 +4278,10 @@ function VideoGenerator({
<span className="text-ui-11 text-muted-foreground/70">Start frame</span>
<ImageDropzone
value={firstFrame}
- onChange={setFirstFrame}
+ onChange={(next) => {
+ setFirstFrame(next);
+ if (next) dropContinuationClip("a start frame replaces it");
+ }}
label="Click or drop"
removeLabel="Remove start frame"
/>
@@ -4245,7 +4292,10 @@ function VideoGenerator({
<span className="text-ui-11 text-muted-foreground/70">End frame</span>
<ImageDropzone
value={lastFrame}
- onChange={setLastFrame}
+ onChange={(next) => {
+ setLastFrame(next);
+ if (next) dropContinuationClip("an end frame replaces it");
+ }}
label="Click or drop"
removeLabel="Remove end frame"
/>
@@ -4540,6 +4590,70 @@ function VideoGenerator({
</div>
)}
+ {supportsContinuation && (
+ <div className="grid gap-2" data-testid="video-continuation">
+ <span className="flex items-center gap-1 text-xs font-medium text-muted-foreground">
+ Continue a clip
+ <InfoHint>
+ Optional, and instead of a start frame. The new clip starts on 5 frames of this
+ one and generates what happens next; describe that in the prompt. Up to 15
+ seconds. Its frames are matched to the frame rate; Match source sizes the video to
+ the clip's shape, and any other resolution centre-crops it.
+ </InfoHint>
+ </span>
+ <ReferenceMediaPicker
+ kind="video"
+ value={continuationClip}
+ label="Clip to continue"
+ onChange={(next) => {
+ setContinuationClip(next);
+ // Live values, not the closure: an upload that finishes late holds stale ones.
+ if (next && (liveInputs.current.firstFrame || liveInputs.current.lastFrame)) {
+ setFirstFrame(null);
+ setLastFrame(null);
+ toast.info("Removed the start frame: the clip to continue replaces it");
+ }
+ }}
+ />
+ {continuationClip && (
+ <>
+ <video
+ controls={true}
+ muted={true}
+ preload="metadata"
+ src={continuationClip.dataUrl}
+ className="max-h-36 w-full rounded-lg bg-black object-contain"
+ />
+ {continuationDurationWarning(continuationClip.durationSeconds) && (
+ <p className="text-ui-11 leading-snug text-destructive">
+ {continuationDurationWarning(continuationClip.durationSeconds)}
+ </p>
+ )}
+ <Field
+ label="Continue"
+ hint="From its last frames picks up where the clip ends. From its first frames keeps its opening 5 frames and generates a new version of what follows."
+ >
+ <Select
+ value={continuationKeep}
+ onValueChange={(v) => setContinuationKeep(v as VideoContinuationKeep)}
+ >
+ <SelectTrigger aria-label="Which end of the clip to continue from">
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ {CONTINUATION_KEEP_OPTIONS.map((option) => (
+ <SelectItem key={option.value} value={option.value}>
+ {option.label}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ </Field>
+ </>
+ )}
+ </div>
+ )}
+
{cropPictureIndex !== null && referenceImages[cropPictureIndex] && (
<ReferenceImageEditor
key={cropPictureIndex}
@@ -4559,7 +4673,7 @@ function VideoGenerator({
<Field
label="Aspect ratio"
- hint="Pick a ratio to lock the proportions, then set the size below. Flip swaps width and height. With a keyframe staged, Match source keeps the picture's own shape."
+ hint="Pick a ratio to lock the proportions, then set the size below. Flip swaps width and height. With a keyframe or a clip to continue staged, Match source keeps its own shape."
>
<AspectRatioSelect
value={sourceSized ? MATCH_SOURCE : aspect}
@@ -4567,7 +4681,7 @@ function VideoGenerator({
portrait={sourceSized ? shownSize[1] > shownSize[0] : portrait}
onFlip={flipDimensions}
extraItems={
- canvasKeyframe && matchedResolution ? (
+ canvasSource && matchedResolution ? (
<SelectItem value={MATCH_SOURCE}>Match source</SelectItem>
) : null
}
diff --git a/studio/frontend/tests/unsloth-support-cosmos3.test.ts b/studio/frontend/tests/unsloth-support-cosmos3.test.ts
index a8bac132d01..ecb449cfe13 100644
--- a/studio/frontend/tests/unsloth-support-cosmos3.test.ts
+++ b/studio/frontend/tests/unsloth-support-cosmos3.test.ts
@@ -28,3 +28,13 @@ test("only nvidia's Cosmos3 Edge is routed to the Video page; Nano, siblings and
assert.deepEqual(classifyUnslothSupport({ modelId, ...HUB_CARD }), expected, modelId);
}
});
+
+test("a video-to-video pipeline tag is not routed to the Video page (Nano's continuation is gated on status)", () => {
+ const result = classifyUnslothSupport({
+ modelId: "someone/any-video-to-video",
+ pipelineTag: "video-to-video",
+ libraryName: "diffusers",
+ tags: ["diffusers"],
+ }) as { supportedIn?: string };
+ assert.notEqual(result.supportedIn, "video");
+});
diff --git a/studio/frontend/tests/video-continuation.test.ts b/studio/frontend/tests/video-continuation.test.ts
new file mode 100644
index 00000000000..19db627d5d7
--- /dev/null
+++ b/studio/frontend/tests/video-continuation.test.ts
@@ -0,0 +1,109 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import test from "node:test";
+
+import {
+ CONTINUATION_KEEP_OPTIONS,
+ CONTINUATION_MAX_SECONDS,
+ continuationDurationWarning,
+ continuationLengthError,
+ continuationRequest,
+ supportsVideoContinuation,
+} from "../src/features/video/video-continuation.ts";
+
+const page = await readFile(
+ new URL("../src/features/video/video-page.tsx", import.meta.url),
+ "utf8",
+);
+
+test("only a status that declares it offers continuation (Cosmos3-Nano); older backends do not", () => {
+ assert.equal(supportsVideoContinuation({ supports_video_continuation: true }), true);
+ for (const status of [
+ { supports_video_continuation: false },
+ {},
+ null,
+ undefined,
+ ]) {
+ assert.equal(supportsVideoContinuation(status), false);
+ }
+});
+
+test("continuing from the end of the clip is the default choice", () => {
+ assert.deepEqual(
+ CONTINUATION_KEEP_OPTIONS.map((option) => option.value),
+ ["last", "first"],
+ );
+ assert.match(page, /useState<VideoContinuationKeep>\("last"\)/);
+});
+
+test("the request carries the clip only when the model takes one and one is staged", () => {
+ assert.deepEqual(continuationRequest(true, "data:video/mp4;base64,AA", "first"), {
+ continuation_video: "data:video/mp4;base64,AA",
+ continuation_keep: "first",
+ });
+ assert.deepEqual(continuationRequest(false, "data:video/mp4;base64,AA", "last"), {});
+ assert.deepEqual(continuationRequest(true, null, "last"), {});
+ assert.deepEqual(continuationRequest(true, undefined, "last"), {});
+});
+
+test("a length that is all kept frames is refused before Generate", () => {
+ assert.match(continuationLengthError(5) ?? "", /keeps 5 of its frames/);
+ assert.equal(continuationLengthError(9), null);
+ assert.equal(continuationLengthError(189), null);
+});
+
+test("the clip section renders only for a family that declares it, and a clip and a keyframe exclude each other", () => {
+ const start = page.indexOf('data-testid="video-continuation"');
+ assert.ok(start > 0);
+ const head = page.slice(0, start);
+ assert.match(head.slice(head.lastIndexOf("{supports")), /^\{supportsContinuation && \(/);
+ const section = page.slice(start, page.indexOf("{cropPictureIndex !== null", start));
+ assert.match(section, /setFirstFrame\(null\);\s*setLastFrame\(null\);/);
+ const keyframes = page.slice(
+ page.indexOf('data-testid="video-keyframes"'),
+ page.indexOf("{supportsReferences && ("),
+ );
+ assert.equal(keyframes.match(/if \(next\) dropContinuationClip\(/g)?.length, 2);
+ // Late uploads read the live inputs, not the closure they were started with.
+ assert.match(section, /liveInputs\.current\.firstFrame \|\| liveInputs\.current\.lastFrame/);
+ assert.match(page, /if \(!liveInputs\.current\.continuationClip\) return;\s*setContinuationClip\(null\);/);
+ assert.match(page, /if \(status\?\.loaded && !supportsContinuation\) setContinuationClip\(null\);/);
+});
+
+test("the payload spreads the continuation fields and the gallery labels a continued clip", () => {
+ assert.match(
+ page,
+ /\.\.\.continuationRequest\(supportsContinuation, stagedClip\?\.dataUrl, continuationKeep\)/,
+ );
+ assert.match(page, /v2v: "Continued a clip"/);
+});
+
+test("a picked clip past the 15 s cap is flagged before Generate; unknown or shorter clips are not", () => {
+ assert.equal(CONTINUATION_MAX_SECONDS, 15);
+ assert.equal(continuationDurationWarning(undefined), null);
+ assert.equal(continuationDurationWarning(3.9), null);
+ assert.equal(continuationDurationWarning(15), null);
+ assert.match(continuationDurationWarning(32.04) ?? "", /32\.0 s.*up to 15 s/);
+ assert.match(page, /continuationDurationWarning\(continuationClip\.durationSeconds\)/);
+});
+
+test("a start frame and a clip replace each other, and the user is told which one went", () => {
+ assert.match(page, /Removed the start frame: the clip to continue replaces it/);
+ assert.match(page, /Removed the clip to continue: \$\{why\}/);
+ assert.match(page, /dropContinuationClip\("a start frame replaces it"\)/);
+ assert.match(page, /dropContinuationClip\("an end frame replaces it"\)/);
+});
+
+test("a staged clip sizes the canvas through Match source, as a start frame does", () => {
+ assert.match(
+ page,
+ /const canvasClip = canvasKeyframe \? null : supportsContinuation \? \(continuationClip\?\.dataUrl \?\? null\) : null;/,
+ );
+ assert.match(page, /const canvasSource = canvasKeyframe \?\? canvasClip;/);
+ assert.match(page, /media\.onloadedmetadata = \(\) => done\(\[media\.videoWidth, media\.videoHeight\]\);/);
+ assert.match(page, /const sourceSized = matchSource && canvasSource != null && matchedResolution != null;/);
+ assert.match(page, /canvasSource && matchedResolution \? \(\s*<SelectItem value=\{MATCH_SOURCE\}>Match source<\/SelectItem>/);
+});