Skip to content

AI media harness for planning, generating, and editing locally

TypeScript
20.8 KB
Raw Blame
use super::*;
use crate::slopfab::{config::Configuration, events::NativeEvent, references::ReferenceVideos, types::{GenerationRequest, ProgressEvent}};
use std::sync::{Arc, Mutex, atomic::AtomicBool};

#[test]
#[ignore = "requires local H3 VAE/upscaler weights and a GPU; set SLOPUS_UPSCALE_WEIGHTS"]
fn saved_latent_upscale_gpu_smoke() {
    use std::io::Write;
    use crate::settings::{ProviderOption, ProviderSetting};
    let root = std::path::PathBuf::from(std::env::var("SLOPUS_UPSCALE_WEIGHTS").unwrap());
    let folder = tempfile::tempdir().unwrap();
    let source = folder.path().join("source.safetensors");
    let output = folder.path().join("upscaled.safetensors");
    let video_bytes = 7 * 2 * 96 * 4;
    let audio_bytes = 37 * 2 * 32 * 4;
    let header = serde_json::json!({"__metadata__":{"slopfab_latents":"h3-av-v1", "fps":"24", "sampled":"1", "frames":"22", "width":"64", "height":"32"},
        "video_rows":{"dtype":"F32", "shape":[14,96], "data_offsets":[0,video_bytes]},
        "audio_rows":{"dtype":"F32", "shape":[74,32], "data_offsets":[video_bytes,video_bytes+audio_bytes]}}).to_string();
    let mut file = std::fs::File::create(&source).unwrap();
    file.write_all(&(header.len() as u64).to_le_bytes()).unwrap();
    file.write_all(header.as_bytes()).unwrap();
    file.write_all(&vec![0; video_bytes + audio_bytes]).unwrap();
    drop(file);
    let original = std::fs::read(&source).unwrap();
    let mut options = std::collections::BTreeMap::new();
    for (key, name) in [("latentUpscaler", "minimax_h3_latent_upscaler_3d_conv_v1_fp16.safetensors"),
        ("videoVae", "vae/minimax_h3_video_vae_fp16.safetensors"), ("audioVae", "vae/minimax_h3_audio_vae_fp32.safetensors")] {
        options.insert(key.into(), ProviderOption::String(root.join(name).to_string_lossy().into_owned()));
    }
    let dll = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../lib/slopfab").join(crate::slopfab::DLL_FILE_NAME);
    options.insert("dllPath".into(), ProviderOption::String(dll.to_string_lossy().into_owned()));
    let settings = std::collections::BTreeMap::from([("slopfab".into(), ProviderSetting { enabled: true, model: None, options })]);
    let request = GenerationRequest { job_id: "upscale-smoke".into(),
        latent_upscale_source: Some(crate::slopfab::LatentUpscaleSource { path: source.to_string_lossy().into_owned(), scale: 2.0 }),
        save_latents_path: Some(output.clone()), ..Default::default() };
    let item = QueueItem { request, configuration: Configuration::from_settings(&settings), references: ReferenceVideos::default(),
        events: Arc::new(|_| {}), cancel: Arc::new(AtomicBool::new(false)) };
    let (metadata, rendered) = run_generation(&item).unwrap();
    assert_eq!((metadata.width, metadata.height, metadata.frames, metadata.steps_computed), (128, 64, 22, 0));
    assert_eq!(rendered.frame(0).unwrap().unwrap().len(), 128 * 64 * 4);
    assert!(!rendered.audio_bytes().is_empty());
    assert!(output.is_file());
    assert_eq!(std::fs::read(source).unwrap(), original);
}

#[test]
#[ignore = "requires local music weights and a GPU; set SLOPUS_MUSIC_WEIGHTS"]
fn music_gpu_smoke() {
    use crate::settings::{ProviderOption, ProviderSetting};
    let root = std::path::PathBuf::from(std::env::var("SLOPUS_MUSIC_WEIGHTS").unwrap());
    for family in ["yue2", "minimax_music3"] {
        let paths = if family == "yue2" { vec![("transformer", "yue2_3b_int8_convrot.safetensors")] } else {
            vec![("transformer", "minimax_music3_dit_int8_convrot.safetensors"), ("textEncoder", "text_encoder/minimax_music3_text_encoder_pruned_int8_convrot.safetensors"), ("audioVae", "minimax_music3_dav.safetensors")]
        };
        let mut options = std::collections::BTreeMap::from([("modelType".into(), ProviderOption::String(family.into()))]);
        for (key, file) in paths { options.insert(key.into(), ProviderOption::String(root.join(file).to_string_lossy().into_owned())); }
        let settings = std::collections::BTreeMap::from([("slopfab".into(), ProviderSetting { enabled: true, model: None, options })]);
        let request = GenerationRequest { job_id: format!("smoke-{family}"), prompt: "Quiet solo piano, instrumental".into(), steps: 2, seed: 42,
            music: Some(crate::slopfab::music::MusicSettings { duration: 2.0, planning: if family == "yue2" { "off" } else { "full" }.into(), ..Default::default() }), ..Default::default() };
        let item = QueueItem { request, configuration: Configuration::from_settings(&settings), references: ReferenceVideos::default(),
            events: Arc::new(|_| {}), cancel: Arc::new(AtomicBool::new(false)) };
        let (metadata, output) = run_generation(&item).unwrap();
        assert_eq!(metadata.frames, 0);
        assert_eq!(metadata.audio_channels, 2);
        println!("{family}: {} seconds, {} Hz stereo", metadata.duration_seconds, metadata.audio_sample_rate);
        assert!((1.9..=2.1).contains(&metadata.duration_seconds));
        assert!(output.frame(0).unwrap().is_none());
        assert!(!output.audio_bytes().is_empty());
    }
}

struct Pixels(u8);
impl rendered::FrameSource for Pixels {
    fn frame_rgba(&self, _: u32, _: u32, _: u32) -> Result<Vec<u8>, String> { Ok(vec![self.0, 20, 30, 255]) }
}

#[test]
fn continuation_handoff_keeps_every_joined_frame_and_audio_sample() {
    struct IndexedFrames;
    impl rendered::FrameSource for IndexedFrames {
        fn frame_rgba(&self, index: u32, _: u32, _: u32) -> Result<Vec<u8>, String> { Ok(vec![index as u8, 0, 0, 255]) }
    }
    let mut item = item();
    item.request.frames = 119;
    item.request.continuation_path = Some("source.safetensors".into());
    item.request.image_edit = None;
    let audio: Vec<f32> = (0..486).map(|n| n as f32).collect();
    let expected_audio: Vec<u8> = audio.iter().flat_map(|n| n.to_le_bytes()).collect();
    let output = ffi::Output { frames: 243, video_float_count: 243 * 3, width: 1, height: 1, channels: 3,
        fps: 24.0, audio, audio_channels: 2, audio_sample_rate: 24, seconds_conditioning: 0.0, seconds_denoise: 0.0,
        seconds_video_decode: 0.0, seconds_audio_decode: 0.0, seconds_total: 0.0, steps_computed: 2, steps_skipped: 0 };
    let (metadata, video) = collect_output(&item, output, Box::new(IndexedFrames), "test".into()).unwrap();
    assert_eq!(metadata.frames, 243);
    assert_eq!(metadata.audio_samples, 486);
    assert_eq!(video.frame_count, 243);
    for index in [0, 123, 124, 242] { assert_eq!(video.frame(index).unwrap().unwrap()[0], index as u8); }
    assert_eq!(video.audio_bytes(), expected_audio);
}
fn result(value: u8) -> (OutputMetadata, rendered::RenderedVideo) {
    let metadata = OutputMetadata { frames: 1, width: 1, height: 1, fps: 1.0, audio_channels: 0, audio_sample_rate: 0, audio_samples: 0,
        reference_count: 0, timing_profile: "test".into(), seconds_conditioning: 0.0, seconds_denoise: 1.0, seconds_video_decode: 0.0,
        seconds_audio_decode: 0.0, seconds_total: 1.0, steps_computed: 2, steps_skipped: 0, duration_seconds: 1.0, boundary: "test" };
    (metadata, rendered::from_source("edit", Box::new(Pixels(value)), vec![], 1, 1, 1, 3, 3, 1.0, 0, 0).unwrap())
}
fn item() -> QueueItem {
    let request: GenerationRequest = serde_json::from_value(serde_json::json!({
        "jobId":"edit", "prompt":"first", "frames":1, "stillImage":true, "steps":2, "seed":1, "canvasWidth":1, "canvasHeight":1,
        "imageEdit":{"sourceRelativePath":"media/source.png","edits":[
            {"prompt":"first","x":0,"y":0,"width":1,"height":1},
            {"prompt":"second","x":0,"y":0,"width":1,"height":1}
        ]}
    })).unwrap();
    QueueItem { request, configuration: Configuration::from_settings(&Default::default()), references: ReferenceVideos::default(),
        events: Arc::new(|_| {}), cancel: Arc::new(AtomicBool::new(false)) }
}

#[test]
fn cropped_edit_sequence_merges_each_result_into_the_original_canvas() {
    let folder = tempfile::tempdir().unwrap();
    let path = folder.path().join("source.png");
    let original = image::RgbaImage::from_pixel(129, 97, image::Rgba([10, 20, 30, 100]));
    original.save(&path).unwrap();
    let bytes = std::fs::read(&path).unwrap();
    let mut item = item();
    item.request = serde_json::from_value(serde_json::json!({
        "jobId":"crop-sequence", "prompt":"first", "frames":1, "stillImage":true, "steps":4, "seed":42,
        "canvasWidth":129, "canvasHeight":97, "imageEdit":{"sourceRelativePath":"source.png", "regionSize":{"width":64,"height":64}, "edits":[
            {"prompt":"first", "x":65,"y":33,"width":32,"height":32,"feather":0},
            {"prompt":"second", "x":81,"y":49,"width":32,"height":32,"feather":0}
        ]}
    })).unwrap();
    item.request.image_edit_path = Some(path.clone());
    item.request.reference_paths = vec![path.to_string_lossy().into_owned()];
    let mut calls = 0;
    let mut crop_references = Vec::new();
    let (metadata, merged) = run_edit_sequence(&item, |stage| {
        assert_eq!((stage.request.canvas_width, stage.request.canvas_height), (32, 32));
        let input = stage.request.image_edit_pixels.as_ref().unwrap();
        let reference = std::path::PathBuf::from(&stage.request.reference_paths[0]);
        assert_ne!(reference, path);
        assert_eq!(image::open(&reference).unwrap().to_rgb8().as_raw(), input.as_ref());
        crop_references.push(reference);
        if calls == 1 { assert!(input.chunks_exact(3).any(|pixel| pixel == [200, 0, 0])); }
        calls += 1;
        let color = if calls == 1 { [200, 0, 0, 255] } else { [0, 200, 0, 255] };
        let (mut metadata, _) = result(0);
        metadata.width = 32; metadata.height = 32;
        Ok((metadata, super::super::image_regions::rendered_image("crop-sequence", image::RgbaImage::from_pixel(32, 32, image::Rgba(color)))?))
    }).unwrap();
    assert_eq!(calls, 2);
    assert!(crop_references.iter().all(|path| !path.exists()));
    assert_eq!((metadata.width, metadata.height, merged.width, merged.height), (129, 97, 129, 97));
    let pixels = merged.frame(0).unwrap().unwrap();
    for y in 0..97 { for x in 0..129 {
        let expected = if (81..113).contains(&x) && (49..81).contains(&y) { [0, 200, 0, 255] }
            else if (65..97).contains(&x) && (33..65).contains(&y) { [200, 0, 0, 255] } else { [10, 20, 30, 100] };
        assert_eq!(&pixels[(y * 129 + x) * 4..(y * 129 + x + 1) * 4], &expected);
    }}
    assert_eq!(std::fs::read(&path).unwrap(), bytes);
    assert!(run_edit_sequence(&item, |_| Err("inference failed".into())).is_err());
    assert_eq!(std::fs::read(path).unwrap(), bytes);
}

fn single_shot_item() -> (QueueItem, tempfile::TempPath) {
    let source = super::super::continuation::tests::archive(124).into_temp_path();
    let mut item = item();
    item.request.image_edit = None;
    item.request.still_image = false;
    item.request.frames = 48;
    item.request.single_shot = Some(crate::slopfab::types::SingleShotRequest { ending_prompt: "Waves goodbye".into(), ending_path: None });
    item.request.reference_paths = vec!["reference.png".into()];
    item.request.continuation_path = Some(source.to_path_buf());
    item.request.continuation_relative_path = Some("latents/source.safetensors".into());
    item.request.continuation_source_frames = Some(124);
    item.request.continuation_overlap_frames = Some(22);
    item.request.continuation_lock_overlap = true;
    item.request.save_latents_path = Some("final.safetensors".into());
    (item, source)
}

#[test]
fn single_shot_with_context_runs_one_pass_without_an_ending_anchor() {
    let (mut item, _source) = single_shot_item();
    let context = super::super::continuation::tests::archive(56).into_temp_path();
    item.request.keystone_path = Some(context.to_path_buf());
    item.request.single_shot.as_mut().unwrap().ending_prompt.clear();
    let mut calls = 0;
    let (metadata, _) = run_single_shot(&item, |stage| {
        calls += 1;
        assert_eq!(stage.request.keystone_path, item.request.keystone_path);
        assert_eq!(stage.request.continuation_path, item.request.continuation_path);
        assert_eq!(stage.request.save_latents_path, item.request.save_latents_path);
        assert_eq!(stage.request.frames, 48);
        assert!(stage.request.single_shot.as_ref().unwrap().ending_path.is_none());
        Ok(result(1))
    }).unwrap();
    assert_eq!(calls, 1);
    assert_eq!(metadata.seconds_total, 1.0);
}

#[test]
fn single_shot_generates_ending_first_then_attaches_it_with_the_starting_source() {
    for (continued, noise) in [(false, None), (true, None), (true, Some(0.35)), (true, Some(1.0))] {
        let (mut item, source) = single_shot_item();
        item.request.continuation_source_noise = noise;
        item.request.generation_effects = Some(serde_json::from_value(serde_json::json!({
            "tone": {"contrast":0.9,"brightness":0.1,"saturation":0.95,"temperature":0},
            "sharpness": {"strength":0.5}, "startStep":2,"endStep":4,
            "region":{"x":0,"y":0,"width":50,"height":100,"invert":false}
        })).unwrap());
        let original_source = std::fs::read(&source).unwrap();
        if !continued {
            item.request.continuation_path = None;
            item.request.continuation_relative_path = None;
            item.request.continuation_source_frames = None;
        }
        let mut calls = 0;
        let mut ending_path = std::path::PathBuf::new();
        let generated_ending = super::super::continuation::tests::archive(if continued { 56 } else { 22 });
        let expected_anchor = super::super::continuation::tail_archive(generated_ending.path(), 22).unwrap();
        let expected_start = super::super::continuation::tail_archive(&source, 22).unwrap();
        let mut temporary_paths = Vec::new();
        let (metadata, _) = run_single_shot(&item, |stage| {
            calls += 1;
            assert_eq!(stage.request.continuation_source_noise, None);
            assert_eq!(stage.request.reference_paths, item.request.reference_paths);
            assert_eq!(stage.request.seed, item.request.seed);
            if calls == 1 {
                let mut expected = item.request.generation_effects.clone().unwrap();
                expected.tone = None;
                assert_eq!(stage.request.generation_effects, Some(expected));
                assert!(stage.request.single_shot.is_none());
                assert_eq!(stage.request.prompt, "Waves goodbye");
                assert_eq!(stage.request.frames, 22);
                if continued {
                    let path = stage.request.continuation_path.as_ref().unwrap();
                    assert_ne!(path, &source.to_path_buf());
                    assert_eq!(std::fs::read(path).unwrap(), std::fs::read(&expected_start).unwrap());
                    temporary_paths.push(path.clone());
                    assert_eq!(stage.request.continuation_from.as_deref(), Some("end"));
                    assert_eq!(stage.request.continuation_source_frames, Some(22));
                    assert_eq!(stage.request.continuation_overlap_frames, Some(22));
                    assert!(stage.request.continuation_lock_overlap);
                } else {
                    assert!(stage.request.continuation_path.is_none());
                }
                assert!(stage.request.continuation_relative_path.is_none());
                ending_path = stage.request.save_latents_path.clone().unwrap();
                std::fs::copy(generated_ending.path(), &ending_path).unwrap();
            } else {
                assert_eq!(stage.request.generation_effects, item.request.generation_effects);
                assert_eq!(stage.request.prompt, item.request.prompt);
                assert_eq!(stage.request.frames, 48);
                assert_eq!(stage.request.continuation_path, item.request.continuation_path);
                assert_eq!(stage.request.save_latents_path, item.request.save_latents_path);
                let anchor = stage.request.single_shot.as_ref().unwrap().ending_path.as_ref().unwrap();
                assert_eq!(std::fs::read(anchor).unwrap(), std::fs::read(&expected_anchor).unwrap());
                temporary_paths.push(anchor.clone());
                assert!(ending_path.is_file());
            }
            Ok(result(1))
        }).unwrap();
        assert_eq!(calls, 2);
        assert_eq!(metadata.seconds_total, 2.0);
        assert_eq!(metadata.steps_computed, 4);
        assert!(!ending_path.exists());
        assert!(temporary_paths.iter().all(|path| !path.exists()));
        assert_eq!(std::fs::read(&source).unwrap(), original_source);
    }
}

#[test]
fn single_shot_stops_on_ending_failure_missing_latents_or_cancellation_and_cleans_up() {
    for mode in ["failure", "missing", "cancel-before", "cancel-between", "full-failure"] {
        let (item, _source) = single_shot_item();
        let generated_ending = super::super::continuation::tests::archive(56);
        if mode == "cancel-before" { item.cancel.store(true, Ordering::Release); }
        let mut calls = 0;
        let mut path = None;
        let error = run_single_shot(&item, |stage| {
            calls += 1;
            if calls == 2 { return Err("full shot failed".into()); }
            let ending = stage.request.save_latents_path.clone().unwrap();
            path = Some(ending.clone());
            if mode != "missing" { std::fs::copy(generated_ending.path(), &ending).unwrap(); }
            if mode == "failure" { return Err("ending failed".into()); }
            if mode == "cancel-between" { item.cancel.store(true, Ordering::Release); }
            Ok(result(1))
        }).err().unwrap();
        assert!(!error.is_empty());
        assert_eq!(calls, if mode == "cancel-before" { 0 } else if mode == "full-failure" { 2 } else { 1 });
        assert!(path.is_none_or(|path| !path.exists()));
    }
}

#[test]
fn sequential_image_edits_feed_previous_pixels_and_report_combined_progress() {
    let mut item = item();
    let progress = Arc::new(Mutex::new(Vec::new()));
    let sink = progress.clone();
    item.events = Arc::new(move |event| { if let NativeEvent::Progress(event) = event { sink.lock().unwrap().push(event); } });
    let mut calls = 0;
    let (metadata, output) = run_edit_sequence(&item, |stage| {
        calls += 1;
        assert_eq!(stage.request.image_edit.as_ref().unwrap().edits.len(), 1);
        if calls == 1 {
            assert_eq!(stage.request.prompt, "first");
            assert!(stage.request.image_edit_pixels.is_none());
        } else {
            assert_eq!(stage.request.prompt, "second");
            assert_eq!(stage.request.image_edit_pixels.as_ref().unwrap().as_slice(), &[1, 20, 30]);
        }
        (stage.events)(NativeEvent::Progress(ProgressEvent { job_id: "edit".into(), stage: "denoising", step: 2, total_steps: 2,
            planned_steps: 2, elapsed_seconds: 1.0, frames: 1, canvas_width: 1, canvas_height: 1, reference_count: 0, timing_profile: "test".into() }));
        Ok(result(calls))
    }).unwrap();
    assert_eq!(calls, 2);
    assert_eq!(output.frame(0).unwrap().unwrap(), [2, 20, 30, 255]);
    assert_eq!(metadata.seconds_total, 2.0);
    assert_eq!(metadata.steps_computed, 4);
    let progress = progress.lock().unwrap();
    assert_eq!((progress[0].step, progress[1].step, progress[1].total_steps), (2, 4, 4));
    assert_eq!(progress[1].elapsed_seconds, 2.0);
}

#[test]
fn image_edits_use_their_own_reference_pictures_and_refmods() {
    let mut item = item();
    item.request.reference_paths = vec!["legacy.png".into()];
    item.request.refmods = vec![serde_json::from_value(serde_json::json!({"path":"legacy.safetensors","strength":1.0,"copies":1})).unwrap()];
    let steps = &mut item.request.image_edit.as_mut().unwrap().edits;
    steps[0].reference_paths = Some(vec!["source.png".into(), "balloon.png".into()]);
    steps[0].refmods = Some(vec![serde_json::from_value(serde_json::json!({"path":"balloon.safetensors","strength":0.75,"copies":2})).unwrap()]);
    steps[1].reference_paths = Some(vec!["source.png".into()]);
    steps[1].refmods = Some(vec![]);
    let mut calls = 0;
    run_edit_sequence(&item, |stage| {
        calls += 1;
        if calls == 1 {
            assert_eq!(stage.request.reference_paths, ["source.png", "balloon.png"]);
            assert_eq!(stage.request.refmods[0].path, "balloon.safetensors");
            assert_eq!(stage.request.refmods[0].strength, 0.75);
            assert_eq!(stage.request.refmods[0].copies, 2);
        } else {
            assert_eq!(stage.request.reference_paths, ["source.png"]);
            assert!(stage.request.refmods.is_empty());
            assert!(stage.request.image_edit_pixels.is_some());
        }
        Ok(result(calls))
    }).unwrap();
    assert_eq!(calls, 2);
}

#[test]
fn cancellation_and_failures_do_not_publish_partial_image_edits() {
    for cancel in [false, true] {
        let item = item();
        let mut calls = 0;
        let result = run_edit_sequence(&item, |_| {
            calls += 1;
            if cancel { item.cancel.store(true, Ordering::Release); Ok(result(1)) }
            else { Err("inpainting failed".into()) }
        });
        assert!(result.is_err());
        assert_eq!(calls, 1);
    }
}