AI media harness for planning, generating, and editing locally
TypeScript
Add blank reference for H3 text images
Bitti6e12e009d925
diff --git a/src-tauri/core/src/slopfab/generation.rs b/src-tauri/core/src/slopfab/generation.rsindex 0a186f8..871ebf7 100644--- a/src-tauri/core/src/slopfab/generation.rs+++ b/src-tauri/core/src/slopfab/generation.rs@@ -229,7 +229,7 @@ fn run_single_generation(}),);let request = RequestHandle::new(&api)?;- configure_request(+ let _blank_reference = configure_request(&api,&request,&item.request,diff --git a/src-tauri/core/src/slopfab/h3.rs b/src-tauri/core/src/slopfab/h3.rsindex a4923c2..e2192b4 100644--- a/src-tauri/core/src/slopfab/h3.rs+++ b/src-tauri/core/src/slopfab/h3.rs@@ -7,7 +7,9 @@ use super::{references::ReferenceVideos,types::GenerationRequest,};-use std::path::Path;+use std::{io::Write, path::Path};++/// Keep the returned temporary reference alive through planning or generation.pub(super) fn configure_request(api: &ffi::Api,handle: &RequestHandle,@@ -16,12 +18,12 @@ pub(super) fn configure_request(platform: ComputePlatform,purpose: RequestPurpose,references: &ReferenceVideos,-) -> Result<(), String> {+) -> Result<Option<tempfile::TempPath>, String> {configuration.validate_inputs(request)?;if platform == ComputePlatform::Vulkan && configuration.sampling.as_ref().map_err(Clone::clone)?.sampler.as_deref() == Some("ab2") {return Err("AB2 sampling requires CUDA. Select a CUDA GPU or worker, or choose another sampler in Settings.".into());}- if request.music.is_some() { return super::music::configure(api, handle, request, configuration, platform); }+ if request.music.is_some() { return super::music::configure(api, handle, request, configuration, platform).map(|()| None); }if request.single_shot.is_some() && request.keystone_path.is_none() { api.require_end_anchor()?; }let source_noise = super::continuation::source_noise(request);if source_noise.is_some() { api.require_continuation_source_init()?; }@@ -229,6 +231,10 @@ pub(super) fn configure_request(}}attach_references(api, handle, request, configuration, references)?;+ let blank_reference = blank_image_reference(request, configuration)?;+ if let Some(path) = &blank_reference {+ api.add_reference(handle, path)?;+ }for refmod in &request.refmods {if refmod.path.trim().is_empty()|| !refmod.strength.is_finite()@@ -261,7 +267,26 @@ pub(super) fn configure_request(}}}- Ok(())+ Ok(blank_reference)+}++fn blank_image_reference(+ request: &GenerationRequest,+ configuration: &Configuration,+) -> Result<Option<tempfile::TempPath>, String> {+ // A blank reference improves H3 Image's text-only results. Preserve the+ // prompt and use ordinary reference conditioning, never image editing.+ if configuration.model_type != "h3_image" || !request.still_image+ || request.image_edit.is_some() || request.reference_count() != 0+ || !request.refmods.is_empty() {+ return Ok(None);+ }+ let mut file = tempfile::Builder::new().prefix("slopus-h3-reference-").suffix(".ppm")+ .tempfile().map_err(|error| format!("Could not prepare image reference: {error}"))?;+ // PPM is supported by the runtime on every platform without external codecs.+ file.write_all(b"P6\n64 64\n255\n").and_then(|()| file.write_all(&[255; 64 * 64 * 3]))+ .map_err(|error| format!("Could not write image reference: {error}"))?;+ Ok(Some(file.into_temp_path()))}pub(super) fn attach_references(@@ -300,3 +325,32 @@ pub(super) const ANIMATE_PROMPT: &str =include_str!("../../../assets/viggle-animate-fixed-prompt.txt");// SlopFab rounds up to 17*k + 5 frames but rejects Animate plans over 360.pub(super) const ANIMATE_MAX_ALIGNED_FRAMES: i32 = (360 - 5) / 17 * 17 + 5;++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn blank_reference_skips_other_models_and_existing_conditioning() {+ let mut configuration = Configuration::from_settings(&Default::default());+ let request = GenerationRequest { still_image: true, ..Default::default() };+ for model in ["minimax-h3", "moss", "fibo2", "krea2"] {+ configuration.model_type = model.into();+ assert!(blank_image_reference(&request, &configuration).unwrap().is_none());+ }+ configuration.model_type = "h3_image".into();+ for patch in [+ serde_json::json!({"stillImage": false}),+ serde_json::json!({"referencePaths": ["image.png"]}),+ serde_json::json!({"referenceVideoIds": ["video"]}),+ serde_json::json!({"referenceAudioIds": ["audio"]}),+ serde_json::json!({"refmods": [{"path": "reference.safetensors", "strength": 1, "copies": 1}]}),+ serde_json::json!({"imageEdit": {"sourceRelativePath": "source.png", "edits": []}}),+ ] {+ let mut value = serde_json::to_value(&request).unwrap();+ value.as_object_mut().unwrap().extend(patch.as_object().unwrap().clone());+ let request = serde_json::from_value(value).unwrap();+ assert!(blank_image_reference(&request, &configuration).unwrap().is_none());+ }+ }+}diff --git a/src-tauri/core/src/slopfab/icons.rs b/src-tauri/core/src/slopfab/icons.rsindex e29f83e..b6dc914 100644--- a/src-tauri/core/src/slopfab/icons.rs+++ b/src-tauri/core/src/slopfab/icons.rs@@ -53,7 +53,7 @@ pub(super) fn write_reference_icon(..Default::default()};let handle = RequestHandle::new(api)?;- configure_request(+ let _blank_reference = configure_request(api,&handle,&request,diff --git a/src-tauri/core/src/slopfab/planning.rs b/src-tauri/core/src/slopfab/planning.rsindex 20a2f5d..493b585 100644--- a/src-tauri/core/src/slopfab/planning.rs+++ b/src-tauri/core/src/slopfab/planning.rs@@ -35,7 +35,7 @@ pub fn resolve_plan(let platform = configuration.platform(detect_platform(&api));let handle = RequestHandle::new(&api)?;let region = super::image_regions::prepare_first(request)?;- configure_request(+ let _blank_reference = configure_request(&api,&handle,region.as_ref().map_or(request, |region| ®ion.request),diff --git a/src-tauri/core/src/slopfab/tests.rs b/src-tauri/core/src/slopfab/tests.rsindex e491cda..79540bd 100644--- a/src-tauri/core/src/slopfab/tests.rs+++ b/src-tauri/core/src/slopfab/tests.rs@@ -1960,9 +1960,11 @@ fn h3_image_plans_six_steps_on_cuda_and_vulkan_with_masked_edits_and_outpaintingcanvas_width: 128, canvas_height: 128, ..Default::default() };for platform in [ComputePlatform::Cuda13, ComputePlatform::Vulkan] {for purpose in [RequestPurpose::Plan, RequestPurpose::Generate] {- for mode in ["generate", "mask", "outpaint"] {+ for mode in ["generate", "references", "mask", "outpaint"] {let mut request = request.clone();- if mode != "generate" {+ if mode == "references" {+ request.reference_paths = vec!["reference.png".into(); 16];+ } else if mode != "generate" {request.image_edit = Some(serde_json::from_value(serde_json::json!({"sourceRelativePath":"source.png", "edits":[{"prompt":"Edit <Picture 1>. Make it blue", "x":32, "y":32, "width":64, "height":64,"invertMask": mode == "outpaint", "feather": 0@@ -1970,9 +1972,32 @@ fn h3_image_plans_six_steps_on_cuda_and_vulkan_with_masked_edits_and_outpaintingrequest.image_edit_pixels = Some(vec![128; 128 * 128 * 3].into());}let handle = RequestHandle::new(&api).unwrap();- configure_request(&api, &handle, &request, &configuration, platform, purpose, &ReferenceVideos::default()).unwrap();+ let blank = configure_request(&api, &handle, &request, &configuration, platform, purpose, &ReferenceVideos::default()).unwrap();let plan = api.resolve(&handle).unwrap();assert_eq!((plan.canvas_width, plan.canvas_height, plan.aligned_frames, plan.num_model_evaluations), (128, 128, 1, 6));+ assert_eq!(blank.is_some(), mode == "generate");+ let description = api.describe(&handle).unwrap();+ let conditioning = match mode {+ "generate" => "image editing (1 references)",+ "references" => "image editing (16 references)",+ _ => "pictures 1\n",+ };+ assert!(description.contains(conditioning), "{description}");+ if let Some(blank) = blank {+ let path = blank.to_path_buf();+ let bytes = std::fs::read(&path).unwrap();+ assert!(bytes.starts_with(b"P6\n64 64\n255\n"));+ assert_eq!(&bytes[b"P6\n64 64\n255\n".len()..], &[255; 64 * 64 * 3]);+ // Automatic conditioning must match a manually supplied+ // blank reference, including the unchanged text prompt.+ let mut manual = request.clone();+ manual.reference_paths.push(path.to_string_lossy().into_owned());+ let manual_handle = RequestHandle::new(&api).unwrap();+ assert!(configure_request(&api, &manual_handle, &manual, &configuration, platform, purpose, &ReferenceVideos::default()).unwrap().is_none());+ assert_eq!(api.describe(&manual_handle).unwrap(), description);+ drop(blank);+ assert!(!path.exists());+ }}}}@@ -2024,7 +2049,7 @@ fn h3_image_supports_twenty_mipixel_canvases() {let request = GenerationRequest { still_image: true, frames: 1, steps: 6, prompt: "A landscape".into(),canvas_width: width, canvas_height: height, ..Default::default() };let handle = RequestHandle::new(&api).unwrap();- configure_request(&api, &handle, &request, &configuration, platform, RequestPurpose::Plan, &ReferenceVideos::default()).unwrap();+ let _blank_reference = configure_request(&api, &handle, &request, &configuration, platform, RequestPurpose::Plan, &ReferenceVideos::default()).unwrap();if valid {let plan = api.resolve(&handle).unwrap();assert_eq!((plan.canvas_width, plan.canvas_height, plan.aligned_frames, plan.num_model_evaluations), (width, height, 1, 6));