Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Add blank reference for H3 text images

Bitti6e12e009d925
diff --git a/src-tauri/core/src/slopfab/generation.rs b/src-tauri/core/src/slopfab/generation.rs
index 0a186f8..871ebf7 100644
--- a/src-tauri/core/src/slopfab/generation.rs
+++ b/src-tauri/core/src/slopfab/generation.rs
@@ -229,7 +229,7 @@ fn run_single_generation(
}),
);
let request = RequestHandle::new(&api)?;
- configure_request(
+ let _blank_reference = configure_request(
&api,
&request,
&item.request,
diff --git a/src-tauri/core/src/slopfab/h3.rs b/src-tauri/core/src/slopfab/h3.rs
index a4923c2..e2192b4 100644
--- a/src-tauri/core/src/slopfab/h3.rs
+++ b/src-tauri/core/src/slopfab/h3.rs
@@ -7,7 +7,9 @@ use super::{
references::ReferenceVideos,
types::GenerationRequest,
};
-use std::path::Path;
+use std::{io::Write, path::Path};
+
+/// Keep the returned temporary reference alive through planning or generation.
pub(super) fn configure_request(
api: &ffi::Api,
handle: &RequestHandle,
@@ -16,12 +18,12 @@ pub(super) fn configure_request(
platform: ComputePlatform,
purpose: RequestPurpose,
references: &ReferenceVideos,
-) -> Result<(), String> {
+) -> Result<Option<tempfile::TempPath>, String> {
configuration.validate_inputs(request)?;
if platform == ComputePlatform::Vulkan && configuration.sampling.as_ref().map_err(Clone::clone)?.sampler.as_deref() == Some("ab2") {
return Err("AB2 sampling requires CUDA. Select a CUDA GPU or worker, or choose another sampler in Settings.".into());
}
- if request.music.is_some() { return super::music::configure(api, handle, request, configuration, platform); }
+ if request.music.is_some() { return super::music::configure(api, handle, request, configuration, platform).map(|()| None); }
if request.single_shot.is_some() && request.keystone_path.is_none() { api.require_end_anchor()?; }
let source_noise = super::continuation::source_noise(request);
if source_noise.is_some() { api.require_continuation_source_init()?; }
@@ -229,6 +231,10 @@ pub(super) fn configure_request(
}
}
attach_references(api, handle, request, configuration, references)?;
+ let blank_reference = blank_image_reference(request, configuration)?;
+ if let Some(path) = &blank_reference {
+ api.add_reference(handle, path)?;
+ }
for refmod in &request.refmods {
if refmod.path.trim().is_empty()
|| !refmod.strength.is_finite()
@@ -261,7 +267,26 @@ pub(super) fn configure_request(
}
}
}
- Ok(())
+ Ok(blank_reference)
+}
+
+fn blank_image_reference(
+ request: &GenerationRequest,
+ configuration: &Configuration,
+) -> Result<Option<tempfile::TempPath>, String> {
+ // A blank reference improves H3 Image's text-only results. Preserve the
+ // prompt and use ordinary reference conditioning, never image editing.
+ if configuration.model_type != "h3_image" || !request.still_image
+ || request.image_edit.is_some() || request.reference_count() != 0
+ || !request.refmods.is_empty() {
+ return Ok(None);
+ }
+ let mut file = tempfile::Builder::new().prefix("slopus-h3-reference-").suffix(".ppm")
+ .tempfile().map_err(|error| format!("Could not prepare image reference: {error}"))?;
+ // PPM is supported by the runtime on every platform without external codecs.
+ file.write_all(b"P6\n64 64\n255\n").and_then(|()| file.write_all(&[255; 64 * 64 * 3]))
+ .map_err(|error| format!("Could not write image reference: {error}"))?;
+ Ok(Some(file.into_temp_path()))
}
pub(super) fn attach_references(
@@ -300,3 +325,32 @@ pub(super) const ANIMATE_PROMPT: &str =
include_str!("../../../assets/viggle-animate-fixed-prompt.txt");
// SlopFab rounds up to 17*k + 5 frames but rejects Animate plans over 360.
pub(super) const ANIMATE_MAX_ALIGNED_FRAMES: i32 = (360 - 5) / 17 * 17 + 5;
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn blank_reference_skips_other_models_and_existing_conditioning() {
+ let mut configuration = Configuration::from_settings(&Default::default());
+ let request = GenerationRequest { still_image: true, ..Default::default() };
+ for model in ["minimax-h3", "moss", "fibo2", "krea2"] {
+ configuration.model_type = model.into();
+ assert!(blank_image_reference(&request, &configuration).unwrap().is_none());
+ }
+ configuration.model_type = "h3_image".into();
+ for patch in [
+ serde_json::json!({"stillImage": false}),
+ serde_json::json!({"referencePaths": ["image.png"]}),
+ serde_json::json!({"referenceVideoIds": ["video"]}),
+ serde_json::json!({"referenceAudioIds": ["audio"]}),
+ serde_json::json!({"refmods": [{"path": "reference.safetensors", "strength": 1, "copies": 1}]}),
+ serde_json::json!({"imageEdit": {"sourceRelativePath": "source.png", "edits": []}}),
+ ] {
+ let mut value = serde_json::to_value(&request).unwrap();
+ value.as_object_mut().unwrap().extend(patch.as_object().unwrap().clone());
+ let request = serde_json::from_value(value).unwrap();
+ assert!(blank_image_reference(&request, &configuration).unwrap().is_none());
+ }
+ }
+}
diff --git a/src-tauri/core/src/slopfab/icons.rs b/src-tauri/core/src/slopfab/icons.rs
index e29f83e..b6dc914 100644
--- a/src-tauri/core/src/slopfab/icons.rs
+++ b/src-tauri/core/src/slopfab/icons.rs
@@ -53,7 +53,7 @@ pub(super) fn write_reference_icon(
..Default::default()
};
let handle = RequestHandle::new(api)?;
- configure_request(
+ let _blank_reference = configure_request(
api,
&handle,
&request,
diff --git a/src-tauri/core/src/slopfab/planning.rs b/src-tauri/core/src/slopfab/planning.rs
index 20a2f5d..493b585 100644
--- a/src-tauri/core/src/slopfab/planning.rs
+++ b/src-tauri/core/src/slopfab/planning.rs
@@ -35,7 +35,7 @@ pub fn resolve_plan(
let platform = configuration.platform(detect_platform(&api));
let handle = RequestHandle::new(&api)?;
let region = super::image_regions::prepare_first(request)?;
- configure_request(
+ let _blank_reference = configure_request(
&api,
&handle,
region.as_ref().map_or(request, |region| &region.request),
diff --git a/src-tauri/core/src/slopfab/tests.rs b/src-tauri/core/src/slopfab/tests.rs
index e491cda..79540bd 100644
--- a/src-tauri/core/src/slopfab/tests.rs
+++ b/src-tauri/core/src/slopfab/tests.rs
@@ -1960,9 +1960,11 @@ fn h3_image_plans_six_steps_on_cuda_and_vulkan_with_masked_edits_and_outpainting
canvas_width: 128, canvas_height: 128, ..Default::default() };
for platform in [ComputePlatform::Cuda13, ComputePlatform::Vulkan] {
for purpose in [RequestPurpose::Plan, RequestPurpose::Generate] {
- for mode in ["generate", "mask", "outpaint"] {
+ for mode in ["generate", "references", "mask", "outpaint"] {
let mut request = request.clone();
- if mode != "generate" {
+ if mode == "references" {
+ request.reference_paths = vec!["reference.png".into(); 16];
+ } else if mode != "generate" {
request.image_edit = Some(serde_json::from_value(serde_json::json!({"sourceRelativePath":"source.png", "edits":[{
"prompt":"Edit <Picture 1>. Make it blue", "x":32, "y":32, "width":64, "height":64,
"invertMask": mode == "outpaint", "feather": 0
@@ -1970,9 +1972,32 @@ fn h3_image_plans_six_steps_on_cuda_and_vulkan_with_masked_edits_and_outpainting
request.image_edit_pixels = Some(vec![128; 128 * 128 * 3].into());
}
let handle = RequestHandle::new(&api).unwrap();
- configure_request(&api, &handle, &request, &configuration, platform, purpose, &ReferenceVideos::default()).unwrap();
+ let blank = configure_request(&api, &handle, &request, &configuration, platform, purpose, &ReferenceVideos::default()).unwrap();
let plan = api.resolve(&handle).unwrap();
assert_eq!((plan.canvas_width, plan.canvas_height, plan.aligned_frames, plan.num_model_evaluations), (128, 128, 1, 6));
+ assert_eq!(blank.is_some(), mode == "generate");
+ let description = api.describe(&handle).unwrap();
+ let conditioning = match mode {
+ "generate" => "image editing (1 references)",
+ "references" => "image editing (16 references)",
+ _ => "pictures 1\n",
+ };
+ assert!(description.contains(conditioning), "{description}");
+ if let Some(blank) = blank {
+ let path = blank.to_path_buf();
+ let bytes = std::fs::read(&path).unwrap();
+ assert!(bytes.starts_with(b"P6\n64 64\n255\n"));
+ assert_eq!(&bytes[b"P6\n64 64\n255\n".len()..], &[255; 64 * 64 * 3]);
+ // Automatic conditioning must match a manually supplied
+ // blank reference, including the unchanged text prompt.
+ let mut manual = request.clone();
+ manual.reference_paths.push(path.to_string_lossy().into_owned());
+ let manual_handle = RequestHandle::new(&api).unwrap();
+ assert!(configure_request(&api, &manual_handle, &manual, &configuration, platform, purpose, &ReferenceVideos::default()).unwrap().is_none());
+ assert_eq!(api.describe(&manual_handle).unwrap(), description);
+ drop(blank);
+ assert!(!path.exists());
+ }
}
}
}
@@ -2024,7 +2049,7 @@ fn h3_image_supports_twenty_mipixel_canvases() {
let request = GenerationRequest { still_image: true, frames: 1, steps: 6, prompt: "A landscape".into(),
canvas_width: width, canvas_height: height, ..Default::default() };
let handle = RequestHandle::new(&api).unwrap();
- configure_request(&api, &handle, &request, &configuration, platform, RequestPurpose::Plan, &ReferenceVideos::default()).unwrap();
+ let _blank_reference = configure_request(&api, &handle, &request, &configuration, platform, RequestPurpose::Plan, &ReferenceVideos::default()).unwrap();
if valid {
let plan = api.resolve(&handle).unwrap();
assert_eq!((plan.canvas_width, plan.canvas_height, plan.aligned_frames, plan.num_model_evaluations), (width, height, 1, 6));