Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

23.5 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""The /video/generate gate rejects off-family shapes with 422 instead of silently snapping them."""

from __future__ import annotations

import threading
import time
from dataclasses import replace

import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient

import core.inference.video as video_module
import core.inference.video_families as video_families_module
import core.inference.video_gallery as gallery_module
from auth.authentication import get_current_subject
from core.inference.video_families import (
    _FAMILIES,
    MAX_VIDEO_NUM_FRAMES,
    VIDEO_NOT_LOADED_MSG,
    detect_video_family,
    format_video_resolution_presets,
    snap_num_frames,
    snap_video_size,
    validate_video_request_shape,
    video_custom_size_bounds,
)
from core.inference.video_minimax_h3 import h3_conditioning_mode
from models.inference import VideoReferenceVideo
from routes.video import router as video_router

LTX2 = detect_video_family("Lightricks/LTX-2")
# Wan is the contrast family: 768x512 is an LTX-2 preset and below Wan TI2V's trained area.
WAN_TI2V_5B = detect_video_family("Wan-AI/Wan2.2-TI2V-5B-Diffusers")


@pytest.mark.parametrize("fam", _FAMILIES, ids = lambda f: f.name)
def test_every_declared_preset_is_accepted(fam):
    """Every declared preset must be accepted, since a rejected one is a dead UI control."""
    for width, height in fam.resolution_presets:
        validate_video_request_shape(
            fam, width = width, height = height, num_frames = fam.default_num_frames
        )


@pytest.mark.parametrize("fam", _FAMILIES, ids = lambda f: f.name)
def test_256x256_is_rejected_and_the_message_names_the_real_presets(fam):
    """Snap leaves 256x256 alone, so a family that does not list it must refuse it (Cosmos3's 256p tier lists it)."""
    if (256, 256) in fam.resolution_presets:
        validate_video_request_shape(fam, width = 256, height = 256)
        return
    with pytest.raises(ValueError) as excinfo:
        validate_video_request_shape(fam, width = 256, height = 256)
    message = str(excinfo.value)
    assert "256x256" in message
    assert fam.name in message
    for width, height in fam.resolution_presets:
        assert f"{width}x{height}" in message


@pytest.mark.parametrize("fam", _FAMILIES, ids = lambda f: f.name)
def test_the_default_frame_count_is_on_its_own_lattice(fam):
    validate_video_request_shape(fam, num_frames = fam.default_num_frames)


def test_off_lattice_frame_count_is_rejected_with_the_straddling_counts():
    with pytest.raises(ValueError) as excinfo:
        validate_video_request_shape(LTX2, num_frames = 100)
    message = str(excinfo.value)
    assert "97" in message and "105" in message
    assert str(LTX2.default_num_frames) in message
    validate_video_request_shape(LTX2, num_frames = 97)
    validate_video_request_shape(LTX2, num_frames = 105)


def test_the_refusal_never_suggests_a_count_past_the_request_ceiling():
    """Never suggest a frame count past the request ceiling: the model's le bound would 422 it again."""
    with pytest.raises(ValueError) as excinfo:
        validate_video_request_shape(LTX2, num_frames = 1024)
    message = str(excinfo.value)
    assert "1017" in message
    assert "1025" not in message
    assert "the nearest supported count is" in message
    validate_video_request_shape(LTX2, num_frames = 1017)
    with pytest.raises(ValueError) as excinfo:
        validate_video_request_shape(LTX2, num_frames = 100)
    assert "the nearest supported counts are" in str(excinfo.value)


def test_the_request_ceiling_and_the_gate_share_one_constant():
    """A drifted pair would silently reintroduce the dead-end suggestion above."""
    from models.inference import VideoGenerateRequest

    field = VideoGenerateRequest.model_fields["num_frames"]
    ceiling = next(m.le for m in field.metadata if hasattr(m, "le"))
    assert ceiling == MAX_VIDEO_NUM_FRAMES


def test_wan_lattice_is_step_4_not_step_8():
    """Per-family, not one hardcoded rule: 85 is valid on Wan's 4k+1 and invalid on LTX-2's 8k+1."""
    wan = detect_video_family("Wan-AI/Wan2.2-T2V-A14B-Diffusers")
    validate_video_request_shape(wan, num_frames = 85)
    with pytest.raises(ValueError):
        validate_video_request_shape(LTX2, num_frames = 85)


def test_the_lattice_reads_frame_offset_not_a_hardcoded_one():
    """MiniMax-H3's frame lattice is 17k + 5, not 17k + 1, so the offset must come from the family."""
    h3 = detect_video_family("MiniMaxAI/MiniMax-H3")
    assert (h3.frame_step, h3.frame_offset) == (17, 5)
    for count in (124, 243, 345):
        assert (count - h3.frame_offset) % h3.frame_step == 0
        validate_video_request_shape(h3, num_frames = count)
    with pytest.raises(ValueError) as excinfo:
        validate_video_request_shape(h3, num_frames = 137)
    message = str(excinfo.value)
    assert "k * 17 + 5" in message
    assert "124" in message and "141" in message


def test_a_suggested_count_never_falls_outside_the_family_range():
    """Naming a lattice point the family cannot load is the same dead end as naming one
    past the request ceiling. H3 starts at 124, so 90 (a real 17k+5 point) is not an answer."""
    h3 = detect_video_family("MiniMaxAI/MiniMax-H3")
    with pytest.raises(ValueError) as excinfo:
        validate_video_request_shape(h3, num_frames = 100)
    message = str(excinfo.value)
    assert "90" not in message
    assert "124" in message
    with pytest.raises(ValueError) as excinfo:
        validate_video_request_shape(h3, num_frames = 400)
    assert "supported counts run from 124 to 345" in str(excinfo.value)


def test_the_frame_gate_enforces_the_range_it_names():
    """Frame counts outside the family's trained window must be refused, not snapped to its edges."""
    h3 = detect_video_family("MiniMaxAI/MiniMax-H3")
    assert (h3.min_num_frames, h3.max_num_frames) == (124, 345)
    for count in (5, 90, 107, 362, 872):
        assert (count - h3.frame_offset) % h3.frame_step == 0
        with pytest.raises(ValueError) as excinfo:
            validate_video_request_shape(h3, num_frames = count)
        assert "counts run from 124 to 345" in str(excinfo.value)
    for count in (124, 243, 345):
        validate_video_request_shape(h3, num_frames = count)
    for count in (1, 9, 121, 1017):
        validate_video_request_shape(LTX2, num_frames = count)


def test_omitted_fields_are_always_valid():
    """None means "use the family default", which is valid by construction."""
    validate_video_request_shape(LTX2)
    validate_video_request_shape(LTX2, num_frames = None)


def test_a_half_specified_size_resolves_against_the_default_preset():
    """generate() fills a missing side from presets[0], so the check must judge that same pair."""
    validate_video_request_shape(LTX2, width = 768)
    validate_video_request_shape(LTX2, height = 512)
    # 1216 alone resolves to 1216x512: off the presets but on /32 and inside the area band.
    validate_video_request_shape(LTX2, width = 1216)
    with pytest.raises(ValueError) as excinfo:
        validate_video_request_shape(LTX2, width = 2048)
    assert "2048x512" in str(excinfo.value)


def test_a_custom_size_on_the_multiple_and_inside_the_band_is_accepted():
    validate_video_request_shape(LTX2, width = 1024, height = 576, num_frames = 97)
    validate_video_request_shape(LTX2, width = 576, height = 1024)
    validate_video_request_shape(LTX2, width = 640, height = 640)
    validate_video_request_shape(
        replace(LTX2, resolution_multiple = 16), width = 848, height = 480, num_frames = 97
    )


def test_a_size_off_the_multiple_is_refused_with_the_multiple_named():
    with pytest.raises(ValueError) as excinfo:
        validate_video_request_shape(LTX2, width = 1000, height = 576)
    message = str(excinfo.value)
    assert "1000x576" in message and "multiples of 32" in message
    assert "768x512" in message
    with pytest.raises(ValueError):
        validate_video_request_shape(LTX2, width = 848, height = 480)


def test_a_size_outside_the_area_band_or_too_narrow_is_refused():
    min_pixels, max_pixels, max_aspect = video_custom_size_bounds(LTX2)
    assert max_pixels == 1216 * 704
    assert min_pixels == 768 * 512
    assert max_aspect == 4.0
    with pytest.raises(ValueError, match = "pixels"):
        validate_video_request_shape(LTX2, width = 1280, height = 1024)
    with pytest.raises(ValueError, match = "pixels"):
        validate_video_request_shape(LTX2, width = 512, height = 512)
    # A family trained at one area still gets a band down to half of it.
    single = replace(LTX2, resolution_presets = ((832, 480), (480, 832)))
    assert video_custom_size_bounds(single)[0] == 832 * 480 // 2
    validate_video_request_shape(single, width = 512, height = 512)
    with pytest.raises(ValueError, match = "aspect ratio"):
        validate_video_request_shape(LTX2, width = 1536, height = 352)


def test_the_status_payload_reports_the_custom_size_bounds():
    defaults = video_module._custom_size_defaults(LTX2)
    assert defaults == {
        "custom_size_min_pixels": 768 * 512,
        "custom_size_max_pixels": 1216 * 704,
        "custom_size_max_aspect": 4.0,
    }
    assert video_module._custom_size_defaults(replace(LTX2, resolution_presets = ())) == {}


def test_presets_spelled_as_lists_still_match():
    """Status presets come back as lists; the list form must still match, or every size 422s."""
    fam = replace(LTX2, resolution_presets = tuple([w, h] for w, h in LTX2.resolution_presets))
    for width, height in LTX2.resolution_presets:
        validate_video_request_shape(fam, width = width, height = height)


def test_a_family_with_no_presets_is_left_to_the_snap():
    """No presets means nothing to enforce, so the old silent snap stays in charge, not a 422."""
    fam = replace(LTX2, resolution_presets = ())
    validate_video_request_shape(fam, width = 256, height = 256)
    with pytest.raises(ValueError):
        validate_video_request_shape(fam, num_frames = 100)


def test_snapping_helpers_are_untouched():
    """The validator is additive: internal callers still get the flooring snap."""
    assert snap_video_size(LTX2, 250, 250) == (224, 224)
    assert snap_num_frames(LTX2, 100) == 97
    assert format_video_resolution_presets(LTX2) == "768x512, 1216x704, 704x1216, 512x768"


def test_reference_video_trim_schema_requires_one_bounded_interval():
    reference = VideoReferenceVideo(
        video = "data:video/mp4;base64,AA==",
        trim_start_seconds = 4.0,
        trim_end_seconds = 19.0,
    )
    assert reference.trim_start_seconds == 4.0
    assert reference.trim_end_seconds == 19.0

    for values, message in (
        ({"trim_start_seconds": 4.0}, "provided together"),
        ({"trim_start_seconds": 4.0, "trim_end_seconds": 5.0}, "2 to 15 seconds"),
        ({"trim_start_seconds": 4.0, "trim_end_seconds": 20.0}, "2 to 15 seconds"),
    ):
        with pytest.raises(ValueError, match = message):
            VideoReferenceVideo(video = "data:video/mp4;base64,AA==", **values)


class _ShapeFakeBackend(video_module.VideoBackend):
    """Real _VideoLoadState and job machinery over a stub generate() that mirrors shape resolution."""

    def load_as(self, fam) -> None:
        self._state = video_module._VideoLoadState(
            pipe = object(),
            family = fam,
            repo_id = f"unsloth/{fam.name}",
            base_repo = fam.base_repo,
            device = "cpu",
            dtype = "bfloat16",
            kind = "pipeline",
        )

    def generate(
        self,
        *,
        prompt,
        seed = None,
        cancel_event = None,
        **kwargs,
    ):
        state = self._state
        if state is None:
            raise RuntimeError(VIDEO_NOT_LOADED_MSG)
        fam = state.family
        default = fam.resolution_presets[0] if fam.resolution_presets else (768, 512)
        width, height = snap_video_size(
            fam, kwargs.get("width") or default[0], kwargs.get("height") or default[1]
        )
        frames = snap_num_frames(fam, kwargs.get("num_frames") or fam.default_num_frames)
        fps = int(kwargs.get("fps") or fam.default_fps)
        return {
            "mp4_bytes": b"MP4-FAKE-BYTES",
            "seed": 4242 if seed is None else seed,
            "repo_id": state.repo_id,
            "width": width,
            "height": height,
            "num_frames": frames,
            "fps": fps,
            "duration_s": frames / fps,
            "has_audio": fam.has_audio,
            "steps": int(kwargs.get("steps") or fam.default_steps),
            "guidance": fam.default_guidance,
            # Derived from the shared helper so the stub matches gallery-accepted conditioning modes.
            "conditioning": h3_conditioning_mode(),
            "flow_shift": kwargs.get("flow_shift"),
            "audio_flow_shift": kwargs.get("audio_flow_shift"),
        }


@pytest.fixture
def backend(monkeypatch):
    fake = _ShapeFakeBackend()
    monkeypatch.setattr(video_module, "get_video_backend", lambda: fake)
    return fake


@pytest.fixture
def client(backend, monkeypatch, tmp_path):
    monkeypatch.setattr(gallery_module, "gallery_dir", lambda: tmp_path)
    app = FastAPI()
    app.include_router(video_router, prefix = "/api/inference")
    app.dependency_overrides[get_current_subject] = lambda: "test-user"
    return TestClient(app)


def _payload(**overrides) -> dict:
    return {"prompt": "a cat", **overrides}


def _wait_terminal(client, timeout = 5.0) -> dict:
    """Generation is async, so the outcome is only observable by polling generate-progress."""
    deadline = time.monotonic() + timeout
    progress: dict = {}
    while time.monotonic() < deadline:
        progress = client.get("/api/inference/video/generate-progress").json()
        if progress.get("phase") in ("completed", "failed"):
            return progress
        time.sleep(0.01)
    raise AssertionError(f"generation never reached a terminal state: {progress}")


def test_generate_rejects_256x256_with_422_naming_the_presets(client, backend):
    """A valid-looking 256x256 the loaded model cannot render is refused, not silently denoised."""
    backend.load_as(LTX2)
    resp = client.post("/api/inference/video/generate", json = _payload(width = 256, height = 256))
    assert resp.status_code == 422, resp.text
    detail = resp.json()["detail"]
    assert "256x256" in detail
    assert "768x512" in detail and "1216x704" in detail
    progress = client.get("/api/inference/video/generate-progress").json()
    assert "active" in progress, (
        f"generate-progress answered {progress}, which is not the declared shape -- the "
        "account gate hid the poll (see routes.video._generation_hidden)"
    )
    assert progress["active"] is False and progress.get("phase") is None


@pytest.mark.parametrize("fam", _FAMILIES, ids = lambda f: f.name)
def test_generate_accepts_every_declared_preset_of_the_loaded_family(client, backend, fam):
    """Every size the interface can offer for this family round-trips to a saved clip."""
    backend.load_as(fam)
    for width, height in fam.resolution_presets:
        resp = client.post(
            "/api/inference/video/generate",
            json = _payload(width = width, height = height, num_frames = fam.default_num_frames),
        )
        assert resp.status_code == 200, (fam.name, width, height, resp.text)
        record = _wait_terminal(client)["video"]
        assert (record["width"], record["height"]) == (width, height)


def test_generate_renders_a_custom_size_and_refuses_one_off_the_multiple(client, backend):
    backend.load_as(LTX2)
    resp = client.post("/api/inference/video/generate", json = _payload(width = 1024, height = 576))
    assert resp.status_code == 200, resp.text
    record = _wait_terminal(client)["video"]
    assert (record["width"], record["height"]) == (1024, 576)
    resp = client.post("/api/inference/video/generate", json = _payload(width = 1000, height = 576))
    assert resp.status_code == 422, resp.text
    assert "multiples of 32" in resp.json()["detail"]


def test_generate_rejects_an_off_lattice_frame_count_with_422(client, backend):
    backend.load_as(LTX2)
    resp = client.post("/api/inference/video/generate", json = _payload(num_frames = 100))
    assert resp.status_code == 422, resp.text
    detail = resp.json()["detail"]
    assert "97" in detail and "105" in detail


def test_generate_with_nothing_loaded_still_reports_not_loaded_not_a_shape_error(client):
    """With no model loaded there is no family to judge, so the request still gets the 409."""
    resp = client.post("/api/inference/video/generate", json = _payload(width = 256, height = 256))
    assert resp.status_code == 409
    assert resp.json()["detail"] == VIDEO_NOT_LOADED_MSG


def test_generate_for_a_family_without_presets_still_snaps(client, backend):
    """Backwards compatibility: an odd size against a family that declares no presets
    is accepted and floored to the family multiple, the pre-change behaviour."""
    backend.load_as(replace(LTX2, resolution_presets = ()))
    resp = client.post("/api/inference/video/generate", json = _payload(width = 250, height = 250))
    assert resp.status_code == 200, resp.text
    record = _wait_terminal(client)["video"]
    assert (record["width"], record["height"]) == (224, 224)


def test_a_family_without_presets_still_enforces_its_frame_lattice(client, backend):
    """Presets only waive size; frame_step is enforced even with no presets, so off-lattice counts 422."""
    backend.load_as(replace(LTX2, resolution_presets = ()))
    resp = client.post("/api/inference/video/generate", json = _payload(num_frames = 100))
    assert resp.status_code == 422, resp.text
    assert "97" in resp.json()["detail"] and "105" in resp.json()["detail"]
    assert (
        client.post("/api/inference/video/generate", json = _payload(num_frames = 97)).status_code
        == 200
    )


def test_generate_omitting_the_shape_uses_the_family_defaults(client, backend):
    """The common API call sends no size at all; it must not be caught by the gate."""
    backend.load_as(LTX2)
    resp = client.post("/api/inference/video/generate", json = _payload())
    assert resp.status_code == 200, resp.text
    record = _wait_terminal(client)["video"]
    assert (record["width"], record["height"]) == LTX2.resolution_presets[0]
    assert record["num_frames"] == LTX2.default_num_frames


def test_the_coarse_pydantic_bounds_still_reject_out_of_range_sizes(client, backend):
    """The coarse request-model bounds still 422 before any family is consulted."""
    backend.load_as(LTX2)
    for body in (_payload(width = 16), _payload(height = 4096), _payload(num_frames = 0)):
        assert client.post("/api/inference/video/generate", json = body).status_code == 422


def test_the_shape_is_judged_under_the_lock_that_reserves_the_state(backend, monkeypatch):
    """Judge the shape under the state lock, or a concurrent load can swap the family mid-check."""
    backend.load_as(LTX2)
    held: list[bool] = []
    real = video_families_module.validate_video_request_shape

    def _spy(*args, **kwargs):
        # Probe from another thread: a Lock does not say which thread owns it.
        probe: list[bool] = []
        watcher = threading.Thread(target = lambda: probe.append(backend._lock.acquire(False)))
        watcher.start()
        watcher.join(5)
        if probe and probe[0]:
            backend._lock.release()
        held.append(not (probe and probe[0]))
        return real(*args, **kwargs)

    monkeypatch.setattr(video_module, "validate_video_request_shape", _spy)
    backend.begin_generate(prompt = "a cat", width = 768, height = 512)

    assert held == [True], "the shape was judged outside the lock that owns the state"


def test_a_family_swap_cannot_slip_between_the_check_and_the_job(backend):
    """The family resident at lock time decides, so the job that runs is the one that was judged."""
    backend.load_as(LTX2)
    backend.begin_generate(prompt = "a cat", width = 768, height = 512)  # accepted for LTX-2
    backend.cancel_generate()
    backend._generate_job_active = False

    backend.load_as(WAN_TI2V_5B)
    with pytest.raises(ValueError):
        backend.begin_generate(prompt = "a cat", width = 768, height = 512)
    assert not backend._generate_job_active, "a refused shape must not reserve the job slot"


def _tiny_png_b64() -> str:
    """A 2x1 PNG, enough for the keyframe branch of the request model."""
    import base64
    import io

    from PIL import Image

    buf = io.BytesIO()
    Image.new("RGB", (2, 1), (255, 0, 0)).save(buf, format = "PNG")
    return base64.b64encode(buf.getvalue()).decode()


@pytest.mark.parametrize("axes", [{"width": 768}, {"height": 512}])
def test_a_half_specified_canvas_stays_valid_without_a_keyframe(axes):
    """Paired-axes check must not run on requests without a keyframe, or one-axis clients get 422s."""
    from models.inference import VideoGenerateRequest

    req = VideoGenerateRequest(prompt = "a cat", **axes)
    assert (req.width, req.height) == (axes.get("width"), axes.get("height"))


@pytest.mark.parametrize("axes", [{"width": 768}, {"height": 512}])
def test_a_half_specified_canvas_is_refused_with_a_keyframe(axes):
    """With a keyframe, a half-given canvas makes _resolve_keyframes drop the sent axis; refuse it."""
    import pydantic
    from models.inference import VideoGenerateRequest

    with pytest.raises(pydantic.ValidationError, match = "sent together"):
        VideoGenerateRequest(prompt = "a cat", first_frame = _tiny_png_b64(), **axes)


def test_both_axes_and_neither_stay_valid_with_a_keyframe():
    """The two shapes the rule exists to allow: an explicit canvas, and "match source"."""
    from models.inference import VideoGenerateRequest

    frame = _tiny_png_b64()
    assert (
        VideoGenerateRequest(prompt = "a cat", first_frame = frame, width = 768, height = 512).width == 768
    )
    assert VideoGenerateRequest(prompt = "a cat", first_frame = frame).width is None


@pytest.mark.parametrize("count", [107, 362])
def test_an_on_lattice_count_outside_the_family_range_is_refused(count):
    """The frame gate must check the range too: snap_num_frames clamps on-lattice counts as well."""
    from core.inference.video_families import (
        VideoShapeError,
        detect_video_family,
        snap_num_frames,
        validate_video_request_shape,
    )

    fam = detect_video_family("", override = "minimax-h3")
    assert (count - fam.frame_offset) % fam.frame_step == 0, "the point of the case is on-lattice"
    assert snap_num_frames(fam, count) != count, "and that the snap would have moved it"
    with pytest.raises(VideoShapeError, match = "not a supported frame count"):
        validate_video_request_shape(fam, num_frames = count)


@pytest.mark.parametrize("count", [124, 141, 345])
def test_in_range_lattice_counts_still_pass(count):
    """Endpoints and an interior count stay valid; the range check does not narrow the family."""
    from core.inference.video_families import detect_video_family, validate_video_request_shape

    fam = detect_video_family("", override = "minimax-h3")
    validate_video_request_shape(fam, num_frames = count)


def test_a_family_without_a_declared_range_is_unaffected():
    """Every pre-existing family declares min 1 and no max, so the added bound must be inert for
    them: the request model's own ceiling stays the only upper limit."""
    from core.inference.video_families import detect_video_family, validate_video_request_shape

    fam = detect_video_family("", override = "ltx-2")
    assert fam.min_num_frames == 1 and fam.max_num_frames is None
    for k in (0, 1, 5, 20):
        validate_video_request_shape(fam, num_frames = k * fam.frame_step + fam.frame_offset)