Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Support latent upscale requests

Bitti074e971c4afd
diff --git a/src-tauri/core/src/slopfab/config.rs b/src-tauri/core/src/slopfab/config.rs
index 6cd58a6..d3b1d89 100644
--- a/src-tauri/core/src/slopfab/config.rs
+++ b/src-tauri/core/src/slopfab/config.rs
@@ -22,6 +22,7 @@ pub(super) struct Configuration {
pub(super) motion_cache: bool,
pub(super) animate: bool,
pub(super) prompt_embedding: Option<PathBuf>,
+ pub(super) latent_upscaler: Option<PathBuf>,
pub(super) models: [(i32, &'static str, Option<PathBuf>); 5],
pub(super) loras: Result<Vec<LoraAdapter>, String>,
pub(super) step_override: Result<Option<i32>, String>,
@@ -57,6 +58,7 @@ impl Configuration {
Some(ProviderOption::Boolean(true))
),
prompt_embedding: option("promptEmbedding"),
+ latent_upscaler: option("latentUpscaler"),
attention: match string_option("attention") {
Some("exact") => "exact",
Some("flash2") => "flash2",
diff --git a/src-tauri/core/src/slopfab/events.rs b/src-tauri/core/src/slopfab/events.rs
index c31176f..7b72f80 100644
--- a/src-tauri/core/src/slopfab/events.rs
+++ b/src-tauri/core/src/slopfab/events.rs
@@ -61,6 +61,7 @@ pub(super) fn stage_name(stage: i32) -> &'static str {
6 => "audioDecode",
7 => "delivering",
8 => "finished",
+ 9 => "upscaling",
_ => "unknown",
}
}
diff --git a/src-tauri/core/src/slopfab/ffi/mod.rs b/src-tauri/core/src/slopfab/ffi/mod.rs
index 494e266..4610fd4 100644
--- a/src-tauri/core/src/slopfab/ffi/mod.rs
+++ b/src-tauri/core/src/slopfab/ffi/mod.rs
@@ -139,6 +139,14 @@ impl Api {
self.check(r)?;
self.inner.set_steps(r.pointer.as_ptr(), v)
}
+ pub fn set_latent_upscaler(&self, r: &RequestHandle, path: Option<&Path>) -> Result<(), String> {
+ self.check(r)?;
+ self.inner.set_latent_upscaler(r.pointer.as_ptr(), path)
+ }
+ #[cfg(test)]
+ pub fn disable_latent_upscaler_for_test(&mut self) {
+ Arc::get_mut(&mut self.inner).expect("configure API before creating handles").disable_latent_upscaler_for_test();
+ }
pub fn set_audio_steps(&self, r: &RequestHandle, steps: i32) -> Result<(), String> {
self.check(r)?;
self.inner.set_audio_steps(r.pointer.as_ptr(), steps)
diff --git a/src-tauri/core/src/slopfab/ffi/raw/mod.rs b/src-tauri/core/src/slopfab/ffi/raw/mod.rs
index ee847f8..97d148e 100644
--- a/src-tauri/core/src/slopfab/ffi/raw/mod.rs
+++ b/src-tauri/core/src/slopfab/ffi/raw/mod.rs
@@ -90,6 +90,7 @@ pub struct Api {
set_video_transition: Option<unsafe extern "C" fn(*mut Request, i32) -> i32>,
set_steps: unsafe extern "C" fn(*mut Request, i32) -> i32,
set_audio_steps: Option<unsafe extern "C" fn(*mut Request, i32) -> i32>,
+ set_latent_upscaler: Option<unsafe extern "C" fn(*mut Request, *const c_char, f32, i32) -> i32>,
set_sampling_settings: Option<unsafe extern "C" fn(*mut Request, *const c_char) -> i32>,
set_seed: unsafe extern "C" fn(*mut Request, u64) -> i32,
set_model: unsafe extern "C" fn(*mut Request, i32, *const c_char) -> i32,
@@ -207,6 +208,9 @@ impl Api {
"slopfab_request_set_steps",
unsafe extern "C" fn(*mut Request, i32) -> i32
),
+ set_latent_upscaler: library
+ .get::<unsafe extern "C" fn(*mut Request, *const c_char, f32, i32) -> i32>(b"slopfab_request_set_latent_upscaler\0")
+ .ok().map(|symbol| *symbol),
set_audio_steps: library
.get::<unsafe extern "C" fn(*mut Request, i32) -> i32>(b"slopfab_request_set_audio_steps\0")
.ok().map(|symbol| *symbol),
@@ -457,6 +461,17 @@ impl Api {
pub fn set_steps(&self, r: *mut Request, v: i32) -> Result<(), String> {
self.error(unsafe { (self.set_steps)(r, v) })
}
+ pub fn set_latent_upscaler(&self, r: *mut Request, path: Option<&Path>) -> Result<(), String> {
+ let Some(set) = self.set_latent_upscaler else {
+ return if path.is_some() { Err("Latent upscale requires SlopFab API 1.24 or later. Update the generation runtime.".into()) } else { Ok(()) };
+ };
+ let path = path_cstring(path.unwrap_or_else(|| Path::new("")))?;
+ self.error(unsafe { set(r, path.as_ptr(), 2.0, 1) })
+ }
+ #[cfg(test)]
+ pub fn disable_latent_upscaler_for_test(&mut self) {
+ self.set_latent_upscaler = None;
+ }
pub fn set_audio_steps(&self, r: *mut Request, steps: i32) -> Result<(), String> {
let Some(set) = self.set_audio_steps else {
return if steps == 0 { Ok(()) } else { Err("Separate audio steps require SlopFab API 1.21 or later. Update the generation runtime.".into()) };
diff --git a/src-tauri/core/src/slopfab/ffi/tests.rs b/src-tauri/core/src/slopfab/ffi/tests.rs
index 143abea..6e92789 100644
--- a/src-tauri/core/src/slopfab/ffi/tests.rs
+++ b/src-tauri/core/src/slopfab/ffi/tests.rs
@@ -100,3 +100,15 @@ fn reference_registries_are_independent_sessions() {
.append_reference_video(&id, &[0; 16], 2, 2, 1.0)
.is_err());
}
+
+#[test]
+fn older_runtimes_reject_latent_upscale_but_allow_normal_generation() {
+ let mut api = Api::load(&crate::slopfab::default_dll_path()).unwrap();
+ api.disable_latent_upscaler_for_test();
+ let request = RequestHandle::new(&api).unwrap();
+ assert!(api.set_latent_upscaler(&request, Some(Path::new("upscaler.safetensors"))).unwrap_err().contains("API 1.24"));
+ api.set_latent_upscaler(&request, None).unwrap();
+ api.set_frames(&request, 48).unwrap();
+ api.set_resolution(&request, 64, 32).unwrap();
+ assert!(api.resolve(&request).is_ok());
+}
diff --git a/src-tauri/core/src/slopfab/h3.rs b/src-tauri/core/src/slopfab/h3.rs
index b9f6abd..86d5728 100644
--- a/src-tauri/core/src/slopfab/h3.rs
+++ b/src-tauri/core/src/slopfab/h3.rs
@@ -77,8 +77,22 @@ pub(super) fn configure_request(
api.set_image_edit_invert_mask(handle)?;
}
} else {
- api.set_resolution(handle, request.canvas_width, request.canvas_height)?;
+ // H3 diffusion needs complete 32-pixel patches. The encoder fits the
+ // 2x decoded result to the exact selected output dimensions.
+ let (width, height) = if request.latent_upscale {
+ ((request.canvas_width / 64 + i32::from(request.canvas_width % 64 != 0)) * 32,
+ (request.canvas_height / 64 + i32::from(request.canvas_height % 64 != 0)) * 32)
+ } else { (request.canvas_width, request.canvas_height) };
+ api.set_resolution(handle, width, height)?;
}
+ let upscaler = if request.latent_upscale {
+ let path = configuration.latent_upscaler.as_deref().ok_or("Download Latent upscale weights or choose a local file in Settings > Generator > Other weights first.")?;
+ if !path.is_file() {
+ return Err(format!("Latent upscale weights are missing: {}", path.display()));
+ }
+ Some(path)
+ } else { None };
+ api.set_latent_upscaler(handle, upscaler)?;
api.set_inference_backend(handle, platform.backend())?;
api.set_attention(handle, configuration.attention)?;
let lock_overlap = request.continuation_path.is_some() && request.continuation_lock_overlap;
diff --git a/src-tauri/core/src/slopfab/icons.rs b/src-tauri/core/src/slopfab/icons.rs
index 041393d..a407dd4 100644
--- a/src-tauri/core/src/slopfab/icons.rs
+++ b/src-tauri/core/src/slopfab/icons.rs
@@ -117,6 +117,7 @@ pub fn generate_reference_icon_batch(
motion_cache: false,
animate: false,
prompt_embedding: None,
+ latent_upscaler: None,
models: [
(0, "transformer", Some(batch.transformer.clone())),
(1, "textEncoder", Some(batch.text_encoder.clone())),
diff --git a/src-tauri/core/src/slopfab/planning.rs b/src-tauri/core/src/slopfab/planning.rs
index 00b6efa..958691e 100644
--- a/src-tauri/core/src/slopfab/planning.rs
+++ b/src-tauri/core/src/slopfab/planning.rs
@@ -42,8 +42,8 @@ pub fn resolve_plan(
let description = api.describe(&handle)?;
let (_, frames) = scene_frame_window(request, plan.aligned_frames)?;
Ok(ResolvedPlan {
- canvas_width: plan.canvas_width,
- canvas_height: plan.canvas_height,
+ canvas_width: if request.latent_upscale { request.canvas_width } else { plan.canvas_width },
+ canvas_height: if request.latent_upscale { request.canvas_height } else { plan.canvas_height },
aligned_frames: frames,
duration_seconds: plan.duration_seconds * frames as f64 / plan.aligned_frames as f64,
model_evaluations: plan.num_model_evaluations * request.image_edit.as_ref().map_or(1, |edit| edit.edits.len() as i32),
@@ -58,6 +58,11 @@ pub fn resolve_plan(
pub(super) fn validate_generation_controls(request: &GenerationRequest) -> Result<(), String> {
super::continuation::validate(request)?;
+ if request.latent_upscale && (request.still_image || request.image_edit.is_some()
+ || request.canvas_width < 64 || request.canvas_height < 64
+ || request.canvas_width % 32 != 0 || request.canvas_height % 32 != 0) {
+ return Err("Latent upscale requires a video canvas of at least 64 pixels per side, in multiples of 32.".into());
+ }
if let Some(edit) = &request.image_edit {
if !request.still_image || request.frames != 1 || request.continuation_relative_path.is_some() || request.continuation_path.is_some() || request.video_transition.is_some()
|| edit.edits.is_empty() || edit.edits.len() > 500 || request.canvas_width > 8192 || request.canvas_height > 8192
diff --git a/src-tauri/core/src/slopfab/tests.rs b/src-tauri/core/src/slopfab/tests.rs
index cecfd17..9ee63b0 100644
--- a/src-tauri/core/src/slopfab/tests.rs
+++ b/src-tauri/core/src/slopfab/tests.rs
@@ -1279,3 +1279,41 @@ fn linux_packages_find_the_runtime_in_the_resource_folder() {
std::fs::write(bin.join(super::DLL_FILE_NAME), b"\x7fELF").unwrap();
assert_eq!(super::runtime_beside(&bin), bin.join(super::DLL_FILE_NAME));
}
+
+#[test]
+fn latent_upscale_uses_half_aligned_canvas_for_preview_and_execution() {
+ let folder = tempfile::tempdir().unwrap();
+ let weights = folder.path().join("latent-upscaler.safetensors");
+ std::fs::write(&weights, b"planning does not load the weights").unwrap();
+ let mut configuration = Configuration::from_settings(&BTreeMap::new());
+ configuration.latent_upscaler = Some(weights);
+ let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../lib/slopfab").join(super::DLL_FILE_NAME);
+ let api = ffi::Api::load(&path).unwrap();
+ for platform in [ComputePlatform::Cuda13, ComputePlatform::Vulkan] {
+ for purpose in [RequestPurpose::Plan, RequestPurpose::Generate] {
+ for (width, height, half_width, half_height) in [(1344, 768, 672, 384), (736, 416, 384, 224), (416, 736, 224, 384)] {
+ let handle = RequestHandle::new(&api).unwrap();
+ let mut request = GenerationRequest { prompt: "Ocean waves".into(), frames: 48, steps: 4,
+ latent_upscale: true, seed: 1, canvas_width: width, canvas_height: height, ..Default::default() };
+ validate_generation_controls(&request).unwrap();
+ configure_request(&api, &handle, &request, &configuration, platform, purpose, &ReferenceVideos::default()).unwrap();
+ let plan = api.resolve(&handle).unwrap();
+ assert_eq!((plan.canvas_width, plan.canvas_height), (half_width, half_height));
+ request.latent_upscale = false;
+ configure_request(&api, &handle, &request, &configuration, platform, purpose, &ReferenceVideos::default()).unwrap();
+ let normal = api.resolve(&handle).unwrap();
+ assert_eq!((normal.canvas_width, normal.canvas_height), (width, height));
+ assert_eq!(normal.aligned_frames, plan.aligned_frames);
+ }
+ }
+ }
+ let request = GenerationRequest { prompt: "Ocean waves".into(), frames: 48, steps: 4, latent_upscale: true,
+ canvas_width: 736, canvas_height: 416, ..Default::default() };
+ configuration.latent_upscaler = None;
+ let handle = RequestHandle::new(&api).unwrap();
+ assert!(configure_request(&api, &handle, &request, &configuration, ComputePlatform::Vulkan,
+ RequestPurpose::Plan, &ReferenceVideos::default()).unwrap_err().contains("Other weights"));
+ assert!(validate_generation_controls(&GenerationRequest { still_image: true, ..request.clone() }).is_err());
+ assert!(validate_generation_controls(&GenerationRequest { canvas_width: 65, ..request }).is_err());
+ assert_eq!(stage_name(9), "upscaling");
+}
diff --git a/src-tauri/core/src/slopfab/types.rs b/src-tauri/core/src/slopfab/types.rs
index 4efeab7..98aacef 100644
--- a/src-tauri/core/src/slopfab/types.rs
+++ b/src-tauri/core/src/slopfab/types.rs
@@ -56,6 +56,9 @@ pub struct GenerationRequest {
pub steps: i32,
#[serde(default)]
pub audio_steps: Option<i32>,
+ /// Canvas dimensions describe the final output; denoise at half size when enabled.
+ #[serde(default)]
+ pub latent_upscale: bool,
/// -1 asks the host to draw a fresh seed for this generation.
pub seed: i64,
pub canvas_width: i32,
diff --git a/src-tauri/core/src/worker/protocol.rs b/src-tauri/core/src/worker/protocol.rs
index 5d191f7..6aaf5aa 100644
--- a/src-tauri/core/src/worker/protocol.rs
+++ b/src-tauri/core/src/worker/protocol.rs
@@ -9,11 +9,12 @@ use std::collections::BTreeMap;
pub const SERVICE_TYPE: &str = "_slopus-worker._tcp.local.";
/// Bumped on any incompatible change to the routes or bodies below.
+// v8 carries latent upscaling and its model weights.
// v7 carries separate audio steps; older workers would silently ignore them.
// v6 added custom upscale weights.
// v5 added overlap locking.
// v4 added SeedVR2 export streams. v3 added inverted image masks.
-pub const PROTOCOL_VERSION: u32 = 7;
+pub const PROTOCOL_VERSION: u32 = 8;
/// Streaming SeedVR2 restoration; model paths are resolved on the worker.
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -158,7 +159,7 @@ pub enum PathKind {
Input,
}
-const MODEL_OPTIONS: [&str; 5] = ["transformer", "textEncoder", "videoVae", "audioVae", "promptEmbedding"];
+const MODEL_OPTIONS: [&str; 6] = ["transformer", "textEncoder", "videoVae", "audioVae", "promptEmbedding", "latentUpscaler"];
/// Visits every file path a generation names, in a fixed order. The client
/// swaps local paths for tokens with it, and the worker swaps tokens back
@@ -171,6 +172,7 @@ pub fn visit_paths(
if let Some(slopfab) = settings.get_mut("slopfab") {
// The engine library always comes from the machine that runs it.
slopfab.options.remove("dllPath");
+ if !request.latent_upscale { slopfab.options.remove("latentUpscaler"); }
for name in MODEL_OPTIONS {
if let Some(ProviderOption::String(value)) = slopfab.options.get_mut(name) {
if !value.trim().is_empty() {
@@ -276,6 +278,33 @@ mod tests {
assert_eq!(parse_file_token("C:/project/a.png"), None);
}
+ #[test]
+ fn latent_upscale_weights_are_transferred_only_for_enabled_jobs() {
+ for enabled in [false, true] {
+ let mut request = GenerationRequest { latent_upscale: enabled, ..Default::default() };
+ let mut settings = settings(&[("latentUpscaler", "C:/weights/upscaler.safetensors")]);
+ let mut count = 0;
+ visit_paths(&mut request, &mut settings, |kind, path| {
+ assert_eq!(kind, PathKind::Model);
+ *path = file_token(0);
+ count += 1;
+ Ok(())
+ }).unwrap();
+ assert_eq!(count, usize::from(enabled));
+ let wire = serde_json::to_vec(&request).unwrap();
+ let restored: GenerationRequest = serde_json::from_slice(&wire).unwrap();
+ assert_eq!(restored.latent_upscale, enabled);
+ visit_paths(&mut request, &mut settings, |_, path| {
+ assert_eq!(parse_file_token(path), Some(0));
+ *path = "/worker/upscaler.safetensors".into();
+ Ok(())
+ }).unwrap();
+ if enabled {
+ assert_eq!(settings["slopfab"].options["latentUpscaler"], ProviderOption::String("/worker/upscaler.safetensors".into()));
+ }
+ }
+ }
+
#[test]
fn upload_paths_cannot_leave_their_folder() {
assert_eq!(safe_relative_path("tokenizer/vocab.json").unwrap(), std::path::Path::new("tokenizer").join("vocab.json"));
diff --git a/src-tauri/src/project/scenes.rs b/src-tauri/src/project/scenes.rs
index d931c1b..8bb785d 100644
--- a/src-tauri/src/project/scenes.rs
+++ b/src-tauri/src/project/scenes.rs
@@ -119,6 +119,8 @@ pub(crate) struct GenerationJob {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub(crate) audio_steps: Option<i32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
+ pub(crate) latent_upscale: Option<bool>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
pub(crate) seed: Option<i64>,
/// Base guide §4.6 and §4.7 — per-prompt fields, so they sit on the scene
/// rather than on a shot. Absent means the compiler writes its own
@@ -178,6 +180,7 @@ impl GenerationJob {
duration_seconds: Some(defaults.duration_seconds),
steps: None,
audio_steps: None,
+ latent_upscale: None,
seed: None,
soundscape: None,
music: None,
diff --git a/src-tauri/src/tests/scenes.rs b/src-tauri/src/tests/scenes.rs
index 432d00e..2ab948e 100644
--- a/src-tauri/src/tests/scenes.rs
+++ b/src-tauri/src/tests/scenes.rs
@@ -497,3 +497,15 @@ fn timeline_flags_fall_back_to_the_frontend_defaults() {
The webview names the scene, and the scene names the file. That is the
whole of the caller's influence over this path, and these pin it.
------------------------------------------------------------------- */
+
+#[test]
+fn scene_latent_upscale_survives_save_and_preserves_legacy_defaults() {
+ for latent_upscale in [None, Some(false), Some(true)] {
+ let folder = tempfile::tempdir().unwrap();
+ let mut config = scene_fixture();
+ config.generation_jobs[0].latent_upscale = latent_upscale;
+ write_project(folder.path(), &config).unwrap();
+ let restored = read_project(folder.path()).unwrap().config;
+ assert_eq!(restored.generation_jobs[0].latent_upscale, latent_upscale);
+ }
+}
diff --git a/src-tauri/src/worker/client.rs b/src-tauri/src/worker/client.rs
index 8eb1494..1f18699 100644
--- a/src-tauri/src/worker/client.rs
+++ b/src-tauri/src/worker/client.rs
@@ -1063,6 +1063,32 @@ mod tests {
assert!(normalize_address("").is_err());
}
+ #[test]
+ fn latent_upscale_jobs_transfer_custom_weights_or_download_on_the_worker() {
+ let folder = tempfile::tempdir().unwrap();
+ let local = folder.path().join("latent-upscaler.safetensors");
+ fs::write(&local, b"weights").unwrap();
+ for path in ["https://huggingface.co/LBH-123-AI/Minimax_h3_latent_Upscaler/resolve/main/minimax_h3_latent_upscaler_3d_conv_v1/minimax_h3_latent_upscaler_3d_conv_v1_fp16.safetensors".to_string(), local.to_string_lossy().into_owned()] {
+ let settings = BTreeMap::from([("slopfab".into(), ProviderSetting { enabled: true, model: None,
+ options: BTreeMap::from([("latentUpscaler".into(), ProviderOption::String(path.clone()))]) })]);
+ let request = GenerationRequest { latent_upscale: true, canvas_width: 736, canvas_height: 416, ..Default::default() };
+ let (job, uploads) = build_job(&request, &settings, &[]).unwrap();
+ assert!(job.request.latent_upscale);
+ assert_eq!((job.request.canvas_width, job.request.canvas_height), (736, 416));
+ assert_eq!(job.settings["slopfab"].options["latentUpscaler"], ProviderOption::String(file_token(0)));
+ assert_eq!(job.files.len(), 1);
+ if path.starts_with("https:") {
+ assert_eq!(job.files[0], FileRef::Url { url: path, lora: false });
+ assert!(uploads.is_empty());
+ } else {
+ assert!(matches!(job.files[0], FileRef::Upload { .. }));
+ assert_eq!(uploads.len(), 1);
+ }
+ let (ordinary, _) = build_job(&GenerationRequest::default(), &settings, &[]).unwrap();
+ assert!(ordinary.files.is_empty());
+ }
+ }
+
#[test]
fn jobs_send_urls_for_downloads_and_uploads_for_local_files() {
let folder = tempfile::tempdir().unwrap();