AI media harness for planning, generating, and editing locally
TypeScript
Support latent upscale requests
Bitti074e971c4afd
diff --git a/src-tauri/core/src/slopfab/config.rs b/src-tauri/core/src/slopfab/config.rsindex 6cd58a6..d3b1d89 100644--- a/src-tauri/core/src/slopfab/config.rs+++ b/src-tauri/core/src/slopfab/config.rs@@ -22,6 +22,7 @@ pub(super) struct Configuration {pub(super) motion_cache: bool,pub(super) animate: bool,pub(super) prompt_embedding: Option<PathBuf>,+ pub(super) latent_upscaler: Option<PathBuf>,pub(super) models: [(i32, &'static str, Option<PathBuf>); 5],pub(super) loras: Result<Vec<LoraAdapter>, String>,pub(super) step_override: Result<Option<i32>, String>,@@ -57,6 +58,7 @@ impl Configuration {Some(ProviderOption::Boolean(true))),prompt_embedding: option("promptEmbedding"),+ latent_upscaler: option("latentUpscaler"),attention: match string_option("attention") {Some("exact") => "exact",Some("flash2") => "flash2",diff --git a/src-tauri/core/src/slopfab/events.rs b/src-tauri/core/src/slopfab/events.rsindex c31176f..7b72f80 100644--- a/src-tauri/core/src/slopfab/events.rs+++ b/src-tauri/core/src/slopfab/events.rs@@ -61,6 +61,7 @@ pub(super) fn stage_name(stage: i32) -> &'static str {6 => "audioDecode",7 => "delivering",8 => "finished",+ 9 => "upscaling",_ => "unknown",}}diff --git a/src-tauri/core/src/slopfab/ffi/mod.rs b/src-tauri/core/src/slopfab/ffi/mod.rsindex 494e266..4610fd4 100644--- a/src-tauri/core/src/slopfab/ffi/mod.rs+++ b/src-tauri/core/src/slopfab/ffi/mod.rs@@ -139,6 +139,14 @@ impl Api {self.check(r)?;self.inner.set_steps(r.pointer.as_ptr(), v)}+ pub fn set_latent_upscaler(&self, r: &RequestHandle, path: Option<&Path>) -> Result<(), String> {+ self.check(r)?;+ self.inner.set_latent_upscaler(r.pointer.as_ptr(), path)+ }+ #[cfg(test)]+ pub fn disable_latent_upscaler_for_test(&mut self) {+ Arc::get_mut(&mut self.inner).expect("configure API before creating handles").disable_latent_upscaler_for_test();+ }pub fn set_audio_steps(&self, r: &RequestHandle, steps: i32) -> Result<(), String> {self.check(r)?;self.inner.set_audio_steps(r.pointer.as_ptr(), steps)diff --git a/src-tauri/core/src/slopfab/ffi/raw/mod.rs b/src-tauri/core/src/slopfab/ffi/raw/mod.rsindex ee847f8..97d148e 100644--- a/src-tauri/core/src/slopfab/ffi/raw/mod.rs+++ b/src-tauri/core/src/slopfab/ffi/raw/mod.rs@@ -90,6 +90,7 @@ pub struct Api {set_video_transition: Option<unsafe extern "C" fn(*mut Request, i32) -> i32>,set_steps: unsafe extern "C" fn(*mut Request, i32) -> i32,set_audio_steps: Option<unsafe extern "C" fn(*mut Request, i32) -> i32>,+ set_latent_upscaler: Option<unsafe extern "C" fn(*mut Request, *const c_char, f32, i32) -> i32>,set_sampling_settings: Option<unsafe extern "C" fn(*mut Request, *const c_char) -> i32>,set_seed: unsafe extern "C" fn(*mut Request, u64) -> i32,set_model: unsafe extern "C" fn(*mut Request, i32, *const c_char) -> i32,@@ -207,6 +208,9 @@ impl Api {"slopfab_request_set_steps",unsafe extern "C" fn(*mut Request, i32) -> i32),+ set_latent_upscaler: library+ .get::<unsafe extern "C" fn(*mut Request, *const c_char, f32, i32) -> i32>(b"slopfab_request_set_latent_upscaler\0")+ .ok().map(|symbol| *symbol),set_audio_steps: library.get::<unsafe extern "C" fn(*mut Request, i32) -> i32>(b"slopfab_request_set_audio_steps\0").ok().map(|symbol| *symbol),@@ -457,6 +461,17 @@ impl Api {pub fn set_steps(&self, r: *mut Request, v: i32) -> Result<(), String> {self.error(unsafe { (self.set_steps)(r, v) })}+ pub fn set_latent_upscaler(&self, r: *mut Request, path: Option<&Path>) -> Result<(), String> {+ let Some(set) = self.set_latent_upscaler else {+ return if path.is_some() { Err("Latent upscale requires SlopFab API 1.24 or later. Update the generation runtime.".into()) } else { Ok(()) };+ };+ let path = path_cstring(path.unwrap_or_else(|| Path::new("")))?;+ self.error(unsafe { set(r, path.as_ptr(), 2.0, 1) })+ }+ #[cfg(test)]+ pub fn disable_latent_upscaler_for_test(&mut self) {+ self.set_latent_upscaler = None;+ }pub fn set_audio_steps(&self, r: *mut Request, steps: i32) -> Result<(), String> {let Some(set) = self.set_audio_steps else {return if steps == 0 { Ok(()) } else { Err("Separate audio steps require SlopFab API 1.21 or later. Update the generation runtime.".into()) };diff --git a/src-tauri/core/src/slopfab/ffi/tests.rs b/src-tauri/core/src/slopfab/ffi/tests.rsindex 143abea..6e92789 100644--- a/src-tauri/core/src/slopfab/ffi/tests.rs+++ b/src-tauri/core/src/slopfab/ffi/tests.rs@@ -100,3 +100,15 @@ fn reference_registries_are_independent_sessions() {.append_reference_video(&id, &[0; 16], 2, 2, 1.0).is_err());}++#[test]+fn older_runtimes_reject_latent_upscale_but_allow_normal_generation() {+ let mut api = Api::load(&crate::slopfab::default_dll_path()).unwrap();+ api.disable_latent_upscaler_for_test();+ let request = RequestHandle::new(&api).unwrap();+ assert!(api.set_latent_upscaler(&request, Some(Path::new("upscaler.safetensors"))).unwrap_err().contains("API 1.24"));+ api.set_latent_upscaler(&request, None).unwrap();+ api.set_frames(&request, 48).unwrap();+ api.set_resolution(&request, 64, 32).unwrap();+ assert!(api.resolve(&request).is_ok());+}diff --git a/src-tauri/core/src/slopfab/h3.rs b/src-tauri/core/src/slopfab/h3.rsindex b9f6abd..86d5728 100644--- a/src-tauri/core/src/slopfab/h3.rs+++ b/src-tauri/core/src/slopfab/h3.rs@@ -77,8 +77,22 @@ pub(super) fn configure_request(api.set_image_edit_invert_mask(handle)?;}} else {- api.set_resolution(handle, request.canvas_width, request.canvas_height)?;+ // H3 diffusion needs complete 32-pixel patches. The encoder fits the+ // 2x decoded result to the exact selected output dimensions.+ let (width, height) = if request.latent_upscale {+ ((request.canvas_width / 64 + i32::from(request.canvas_width % 64 != 0)) * 32,+ (request.canvas_height / 64 + i32::from(request.canvas_height % 64 != 0)) * 32)+ } else { (request.canvas_width, request.canvas_height) };+ api.set_resolution(handle, width, height)?;}+ let upscaler = if request.latent_upscale {+ let path = configuration.latent_upscaler.as_deref().ok_or("Download Latent upscale weights or choose a local file in Settings > Generator > Other weights first.")?;+ if !path.is_file() {+ return Err(format!("Latent upscale weights are missing: {}", path.display()));+ }+ Some(path)+ } else { None };+ api.set_latent_upscaler(handle, upscaler)?;api.set_inference_backend(handle, platform.backend())?;api.set_attention(handle, configuration.attention)?;let lock_overlap = request.continuation_path.is_some() && request.continuation_lock_overlap;diff --git a/src-tauri/core/src/slopfab/icons.rs b/src-tauri/core/src/slopfab/icons.rsindex 041393d..a407dd4 100644--- a/src-tauri/core/src/slopfab/icons.rs+++ b/src-tauri/core/src/slopfab/icons.rs@@ -117,6 +117,7 @@ pub fn generate_reference_icon_batch(motion_cache: false,animate: false,prompt_embedding: None,+ latent_upscaler: None,models: [(0, "transformer", Some(batch.transformer.clone())),(1, "textEncoder", Some(batch.text_encoder.clone())),diff --git a/src-tauri/core/src/slopfab/planning.rs b/src-tauri/core/src/slopfab/planning.rsindex 00b6efa..958691e 100644--- a/src-tauri/core/src/slopfab/planning.rs+++ b/src-tauri/core/src/slopfab/planning.rs@@ -42,8 +42,8 @@ pub fn resolve_plan(let description = api.describe(&handle)?;let (_, frames) = scene_frame_window(request, plan.aligned_frames)?;Ok(ResolvedPlan {- canvas_width: plan.canvas_width,- canvas_height: plan.canvas_height,+ canvas_width: if request.latent_upscale { request.canvas_width } else { plan.canvas_width },+ canvas_height: if request.latent_upscale { request.canvas_height } else { plan.canvas_height },aligned_frames: frames,duration_seconds: plan.duration_seconds * frames as f64 / plan.aligned_frames as f64,model_evaluations: plan.num_model_evaluations * request.image_edit.as_ref().map_or(1, |edit| edit.edits.len() as i32),@@ -58,6 +58,11 @@ pub fn resolve_plan(pub(super) fn validate_generation_controls(request: &GenerationRequest) -> Result<(), String> {super::continuation::validate(request)?;+ if request.latent_upscale && (request.still_image || request.image_edit.is_some()+ || request.canvas_width < 64 || request.canvas_height < 64+ || request.canvas_width % 32 != 0 || request.canvas_height % 32 != 0) {+ return Err("Latent upscale requires a video canvas of at least 64 pixels per side, in multiples of 32.".into());+ }if let Some(edit) = &request.image_edit {if !request.still_image || request.frames != 1 || request.continuation_relative_path.is_some() || request.continuation_path.is_some() || request.video_transition.is_some()|| edit.edits.is_empty() || edit.edits.len() > 500 || request.canvas_width > 8192 || request.canvas_height > 8192diff --git a/src-tauri/core/src/slopfab/tests.rs b/src-tauri/core/src/slopfab/tests.rsindex cecfd17..9ee63b0 100644--- a/src-tauri/core/src/slopfab/tests.rs+++ b/src-tauri/core/src/slopfab/tests.rs@@ -1279,3 +1279,41 @@ fn linux_packages_find_the_runtime_in_the_resource_folder() {std::fs::write(bin.join(super::DLL_FILE_NAME), b"\x7fELF").unwrap();assert_eq!(super::runtime_beside(&bin), bin.join(super::DLL_FILE_NAME));}++#[test]+fn latent_upscale_uses_half_aligned_canvas_for_preview_and_execution() {+ let folder = tempfile::tempdir().unwrap();+ let weights = folder.path().join("latent-upscaler.safetensors");+ std::fs::write(&weights, b"planning does not load the weights").unwrap();+ let mut configuration = Configuration::from_settings(&BTreeMap::new());+ configuration.latent_upscaler = Some(weights);+ let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../lib/slopfab").join(super::DLL_FILE_NAME);+ let api = ffi::Api::load(&path).unwrap();+ for platform in [ComputePlatform::Cuda13, ComputePlatform::Vulkan] {+ for purpose in [RequestPurpose::Plan, RequestPurpose::Generate] {+ for (width, height, half_width, half_height) in [(1344, 768, 672, 384), (736, 416, 384, 224), (416, 736, 224, 384)] {+ let handle = RequestHandle::new(&api).unwrap();+ let mut request = GenerationRequest { prompt: "Ocean waves".into(), frames: 48, steps: 4,+ latent_upscale: true, seed: 1, canvas_width: width, canvas_height: height, ..Default::default() };+ validate_generation_controls(&request).unwrap();+ configure_request(&api, &handle, &request, &configuration, platform, purpose, &ReferenceVideos::default()).unwrap();+ let plan = api.resolve(&handle).unwrap();+ assert_eq!((plan.canvas_width, plan.canvas_height), (half_width, half_height));+ request.latent_upscale = false;+ configure_request(&api, &handle, &request, &configuration, platform, purpose, &ReferenceVideos::default()).unwrap();+ let normal = api.resolve(&handle).unwrap();+ assert_eq!((normal.canvas_width, normal.canvas_height), (width, height));+ assert_eq!(normal.aligned_frames, plan.aligned_frames);+ }+ }+ }+ let request = GenerationRequest { prompt: "Ocean waves".into(), frames: 48, steps: 4, latent_upscale: true,+ canvas_width: 736, canvas_height: 416, ..Default::default() };+ configuration.latent_upscaler = None;+ let handle = RequestHandle::new(&api).unwrap();+ assert!(configure_request(&api, &handle, &request, &configuration, ComputePlatform::Vulkan,+ RequestPurpose::Plan, &ReferenceVideos::default()).unwrap_err().contains("Other weights"));+ assert!(validate_generation_controls(&GenerationRequest { still_image: true, ..request.clone() }).is_err());+ assert!(validate_generation_controls(&GenerationRequest { canvas_width: 65, ..request }).is_err());+ assert_eq!(stage_name(9), "upscaling");+}diff --git a/src-tauri/core/src/slopfab/types.rs b/src-tauri/core/src/slopfab/types.rsindex 4efeab7..98aacef 100644--- a/src-tauri/core/src/slopfab/types.rs+++ b/src-tauri/core/src/slopfab/types.rs@@ -56,6 +56,9 @@ pub struct GenerationRequest {pub steps: i32,#[serde(default)]pub audio_steps: Option<i32>,+ /// Canvas dimensions describe the final output; denoise at half size when enabled.+ #[serde(default)]+ pub latent_upscale: bool,/// -1 asks the host to draw a fresh seed for this generation.pub seed: i64,pub canvas_width: i32,diff --git a/src-tauri/core/src/worker/protocol.rs b/src-tauri/core/src/worker/protocol.rsindex 5d191f7..6aaf5aa 100644--- a/src-tauri/core/src/worker/protocol.rs+++ b/src-tauri/core/src/worker/protocol.rs@@ -9,11 +9,12 @@ use std::collections::BTreeMap;pub const SERVICE_TYPE: &str = "_slopus-worker._tcp.local.";/// Bumped on any incompatible change to the routes or bodies below.+// v8 carries latent upscaling and its model weights.// v7 carries separate audio steps; older workers would silently ignore them.// v6 added custom upscale weights.// v5 added overlap locking.// v4 added SeedVR2 export streams. v3 added inverted image masks.-pub const PROTOCOL_VERSION: u32 = 7;+pub const PROTOCOL_VERSION: u32 = 8;/// Streaming SeedVR2 restoration; model paths are resolved on the worker.#[derive(Debug, Clone, Serialize, Deserialize)]@@ -158,7 +159,7 @@ pub enum PathKind {Input,}-const MODEL_OPTIONS: [&str; 5] = ["transformer", "textEncoder", "videoVae", "audioVae", "promptEmbedding"];+const MODEL_OPTIONS: [&str; 6] = ["transformer", "textEncoder", "videoVae", "audioVae", "promptEmbedding", "latentUpscaler"];/// Visits every file path a generation names, in a fixed order. The client/// swaps local paths for tokens with it, and the worker swaps tokens back@@ -171,6 +172,7 @@ pub fn visit_paths(if let Some(slopfab) = settings.get_mut("slopfab") {// The engine library always comes from the machine that runs it.slopfab.options.remove("dllPath");+ if !request.latent_upscale { slopfab.options.remove("latentUpscaler"); }for name in MODEL_OPTIONS {if let Some(ProviderOption::String(value)) = slopfab.options.get_mut(name) {if !value.trim().is_empty() {@@ -276,6 +278,33 @@ mod tests {assert_eq!(parse_file_token("C:/project/a.png"), None);}+ #[test]+ fn latent_upscale_weights_are_transferred_only_for_enabled_jobs() {+ for enabled in [false, true] {+ let mut request = GenerationRequest { latent_upscale: enabled, ..Default::default() };+ let mut settings = settings(&[("latentUpscaler", "C:/weights/upscaler.safetensors")]);+ let mut count = 0;+ visit_paths(&mut request, &mut settings, |kind, path| {+ assert_eq!(kind, PathKind::Model);+ *path = file_token(0);+ count += 1;+ Ok(())+ }).unwrap();+ assert_eq!(count, usize::from(enabled));+ let wire = serde_json::to_vec(&request).unwrap();+ let restored: GenerationRequest = serde_json::from_slice(&wire).unwrap();+ assert_eq!(restored.latent_upscale, enabled);+ visit_paths(&mut request, &mut settings, |_, path| {+ assert_eq!(parse_file_token(path), Some(0));+ *path = "/worker/upscaler.safetensors".into();+ Ok(())+ }).unwrap();+ if enabled {+ assert_eq!(settings["slopfab"].options["latentUpscaler"], ProviderOption::String("/worker/upscaler.safetensors".into()));+ }+ }+ }+#[test]fn upload_paths_cannot_leave_their_folder() {assert_eq!(safe_relative_path("tokenizer/vocab.json").unwrap(), std::path::Path::new("tokenizer").join("vocab.json"));diff --git a/src-tauri/src/project/scenes.rs b/src-tauri/src/project/scenes.rsindex d931c1b..8bb785d 100644--- a/src-tauri/src/project/scenes.rs+++ b/src-tauri/src/project/scenes.rs@@ -119,6 +119,8 @@ pub(crate) struct GenerationJob {#[serde(default, skip_serializing_if = "Option::is_none")]pub(crate) audio_steps: Option<i32>,#[serde(default, skip_serializing_if = "Option::is_none")]+ pub(crate) latent_upscale: Option<bool>,+ #[serde(default, skip_serializing_if = "Option::is_none")]pub(crate) seed: Option<i64>,/// Base guide §4.6 and §4.7 — per-prompt fields, so they sit on the scene/// rather than on a shot. Absent means the compiler writes its own@@ -178,6 +180,7 @@ impl GenerationJob {duration_seconds: Some(defaults.duration_seconds),steps: None,audio_steps: None,+ latent_upscale: None,seed: None,soundscape: None,music: None,diff --git a/src-tauri/src/tests/scenes.rs b/src-tauri/src/tests/scenes.rsindex 432d00e..2ab948e 100644--- a/src-tauri/src/tests/scenes.rs+++ b/src-tauri/src/tests/scenes.rs@@ -497,3 +497,15 @@ fn timeline_flags_fall_back_to_the_frontend_defaults() {The webview names the scene, and the scene names the file. That is thewhole of the caller's influence over this path, and these pin it.------------------------------------------------------------------- */++#[test]+fn scene_latent_upscale_survives_save_and_preserves_legacy_defaults() {+ for latent_upscale in [None, Some(false), Some(true)] {+ let folder = tempfile::tempdir().unwrap();+ let mut config = scene_fixture();+ config.generation_jobs[0].latent_upscale = latent_upscale;+ write_project(folder.path(), &config).unwrap();+ let restored = read_project(folder.path()).unwrap().config;+ assert_eq!(restored.generation_jobs[0].latent_upscale, latent_upscale);+ }+}diff --git a/src-tauri/src/worker/client.rs b/src-tauri/src/worker/client.rsindex 8eb1494..1f18699 100644--- a/src-tauri/src/worker/client.rs+++ b/src-tauri/src/worker/client.rs@@ -1063,6 +1063,32 @@ mod tests {assert!(normalize_address("").is_err());}+ #[test]+ fn latent_upscale_jobs_transfer_custom_weights_or_download_on_the_worker() {+ let folder = tempfile::tempdir().unwrap();+ let local = folder.path().join("latent-upscaler.safetensors");+ fs::write(&local, b"weights").unwrap();+ for path in ["https://huggingface.co/LBH-123-AI/Minimax_h3_latent_Upscaler/resolve/main/minimax_h3_latent_upscaler_3d_conv_v1/minimax_h3_latent_upscaler_3d_conv_v1_fp16.safetensors".to_string(), local.to_string_lossy().into_owned()] {+ let settings = BTreeMap::from([("slopfab".into(), ProviderSetting { enabled: true, model: None,+ options: BTreeMap::from([("latentUpscaler".into(), ProviderOption::String(path.clone()))]) })]);+ let request = GenerationRequest { latent_upscale: true, canvas_width: 736, canvas_height: 416, ..Default::default() };+ let (job, uploads) = build_job(&request, &settings, &[]).unwrap();+ assert!(job.request.latent_upscale);+ assert_eq!((job.request.canvas_width, job.request.canvas_height), (736, 416));+ assert_eq!(job.settings["slopfab"].options["latentUpscaler"], ProviderOption::String(file_token(0)));+ assert_eq!(job.files.len(), 1);+ if path.starts_with("https:") {+ assert_eq!(job.files[0], FileRef::Url { url: path, lora: false });+ assert!(uploads.is_empty());+ } else {+ assert!(matches!(job.files[0], FileRef::Upload { .. }));+ assert_eq!(uploads.len(), 1);+ }+ let (ordinary, _) = build_job(&GenerationRequest::default(), &settings, &[]).unwrap();+ assert!(ordinary.files.is_empty());+ }+ }+#[test]fn jobs_send_urls_for_downloads_and_uploads_for_local_files() {let folder = tempfile::tempdir().unwrap();