Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Add refmod export to the slopfab runtime

Bitti1241fed07a5c
diff --git a/src-tauri/core/src/slopfab/ffi/mod.rs b/src-tauri/core/src/slopfab/ffi/mod.rs
index a959b78..13f4c92 100644
--- a/src-tauri/core/src/slopfab/ffi/mod.rs
+++ b/src-tauri/core/src/slopfab/ffi/mod.rs
@@ -57,6 +57,12 @@ impl Api {
.disable_animate_for_test();
}
#[cfg(test)]
+ pub fn disable_refmod_export_for_test(&mut self) {
+ Arc::get_mut(&mut self.inner)
+ .expect("configure API before creating handles")
+ .disable_refmod_export_for_test();
+ }
+ #[cfg(test)]
pub fn disable_motion_cache_for_test(&mut self) {
Arc::get_mut(&mut self.inner)
.expect("configure API before creating handles")
@@ -186,6 +192,10 @@ impl Api {
self.inner
.add_refmod(r.pointer.as_ptr(), path, strength, copies)
}
+ pub fn export_refmod(&self, r: &RequestHandle, output: &Path, name: &str, description: &str) -> Result<(), String> {
+ self.check(r)?;
+ self.inner.export_refmod(r.pointer.as_ptr(), output, name, description)
+ }
pub fn resolve(&self, request: &RequestHandle) -> Result<Plan, String> {
self.check(request)?;
self.inner.resolve(request.pointer.as_ptr())
diff --git a/src-tauri/core/src/slopfab/ffi/raw/mod.rs b/src-tauri/core/src/slopfab/ffi/raw/mod.rs
index e987067..93eaf5f 100644
--- a/src-tauri/core/src/slopfab/ffi/raw/mod.rs
+++ b/src-tauri/core/src/slopfab/ffi/raw/mod.rs
@@ -63,6 +63,7 @@ type SetMotionCache =
unsafe extern "C" fn(*mut Request, i32, f32, f32, i32, i32, f32, f32, i32, i32) -> i32;
type SetImageEditPath = unsafe extern "C" fn(*mut Request, *const c_char, i32, i32, i32, i32, f32, i32) -> i32;
type SetImageEditRgb = unsafe extern "C" fn(*mut Request, *const u8, usize, i32, i32, usize, i32, i32, i32, i32, f32, i32) -> i32;
+type ExportRefmod = unsafe extern "C" fn(*const Request, *const c_char, *const c_char, *const c_char, i32) -> i32;
pub struct Api {
_library: Library,
@@ -92,6 +93,7 @@ pub struct Api {
add_lora: Option<unsafe extern "C" fn(*mut Request, *const c_char, f32) -> i32>,
prepare_lora_grid: Option<unsafe extern "C" fn(*const c_char, i32, i32) -> i32>,
add_refmod: Option<unsafe extern "C" fn(*mut Request, *const c_char, f32, i32) -> i32>,
+ export_refmod: Option<ExportRefmod>,
add_reference: unsafe extern "C" fn(*mut Request, *const c_char) -> i32,
add_reference_audio: Option<AddReferenceAudio>,
set_attention: unsafe extern "C" fn(*mut Request, *const c_char) -> i32,
@@ -240,6 +242,7 @@ impl Api {
)
.ok()
.map(|symbol| *symbol),
+ export_refmod: library.get::<ExportRefmod>(b"slopfab_export_refmod\0").ok().map(|symbol| *symbol),
add_reference_audio: library
.get::<AddReferenceAudio>(b"slopfab_request_add_reference_audio_f32\0")
.ok()
@@ -514,6 +517,19 @@ impl Api {
let path = path_cstring(path)?;
self.error(unsafe { add(r, path.as_ptr(), strength, copies) })
}
+ pub fn export_refmod(&self, r: *mut Request, output: &Path, name: &str, description: &str) -> Result<(), String> {
+ let export = self.export_refmod.ok_or("Refmod export requires slopfab.dll API 1.18 or later. Update the runtime.")?;
+ let output = path_cstring(output)?;
+ let name = CString::new(name).map_err(|_| "Refmod name contains a null byte.".to_string())?;
+ let description = CString::new(description).map_err(|_| "Refmod description contains a null byte.".to_string())?;
+ let description = if description.is_empty() { ptr::null() } else { description.as_ptr() };
+ // short_edge 0 selects the runtime's native 768-pixel reference size.
+ self.error(unsafe { export(r, output.as_ptr(), name.as_ptr(), description, 0) })
+ }
+ #[cfg(test)]
+ pub fn disable_refmod_export_for_test(&mut self) {
+ self.export_refmod = None;
+ }
pub fn add_reference_audio(
&self,
r: *mut Request,
diff --git a/src-tauri/core/src/slopfab/h3.rs b/src-tauri/core/src/slopfab/h3.rs
index 9c8700e..0469bd3 100644
--- a/src-tauri/core/src/slopfab/h3.rs
+++ b/src-tauri/core/src/slopfab/h3.rs
@@ -118,6 +118,35 @@ pub(super) fn configure_request(
api.set_model(handle, *id, path)?;
}
}
+ attach_references(api, handle, request, configuration, references)?;
+ for refmod in &request.refmods {
+ if refmod.path.trim().is_empty()
+ || !refmod.strength.is_finite()
+ || !(0.0..=1.0).contains(&refmod.strength)
+ || !(1..=10).contains(&refmod.copies)
+ {
+ return Err("Refmods need a path, strength 0 to 1 and copies 1 to 10.".into());
+ }
+ if refmod.strength == 0.0 {
+ continue;
+ }
+ api.add_refmod(
+ handle,
+ Path::new(&refmod.path),
+ refmod.strength,
+ refmod.copies as i32,
+ )?;
+ }
+ Ok(())
+}
+
+pub(super) fn attach_references(
+ api: &ffi::Api,
+ handle: &RequestHandle,
+ request: &GenerationRequest,
+ configuration: &Configuration,
+ references: &ReferenceVideos,
+) -> Result<(), String> {
for path in &request.reference_paths {
api.add_reference(handle, Path::new(path))?;
}
@@ -139,24 +168,6 @@ pub(super) fn configure_request(
let audio = references.reference_audio(id)?;
api.add_reference_audio(handle, &audio.samples, audio.channels, audio.sample_rate)?;
}
- for refmod in &request.refmods {
- if refmod.path.trim().is_empty()
- || !refmod.strength.is_finite()
- || !(0.0..=1.0).contains(&refmod.strength)
- || !(1..=10).contains(&refmod.copies)
- {
- return Err("Refmods need a path, strength 0 to 1 and copies 1 to 10.".into());
- }
- if refmod.strength == 0.0 {
- continue;
- }
- api.add_refmod(
- handle,
- Path::new(&refmod.path),
- refmod.strength,
- refmod.copies as i32,
- )?;
- }
Ok(())
}
diff --git a/src-tauri/core/src/slopfab/mod.rs b/src-tauri/core/src/slopfab/mod.rs
index 07ae254..480eb3c 100644
--- a/src-tauri/core/src/slopfab/mod.rs
+++ b/src-tauri/core/src/slopfab/mod.rs
@@ -20,6 +20,7 @@ mod icons;
mod planning;
mod platform;
mod references;
+mod refmod_export;
mod runtime;
mod status;
mod types;
@@ -29,6 +30,7 @@ pub use types::*;
pub use events::NativeEvent;
pub use icons::{generate_reference_icon_batch, ReferenceIconBatchConfig, ReferenceIconSpec};
pub use planning::resolve_plan;
+pub use refmod_export::export_refmod;
pub use platform::{probe_backend, BackendProbe};
pub use status::{status, worker_status};
#[cfg(windows)]
diff --git a/src-tauri/core/src/slopfab/refmod_export.rs b/src-tauri/core/src/slopfab/refmod_export.rs
new file mode 100644
index 0000000..0ff56ff
--- /dev/null
+++ b/src-tauri/core/src/slopfab/refmod_export.rs
@@ -0,0 +1,53 @@
+//! Encodes a reference's raw images, video and sound into a RefMod file.
+use super::{
+ config::Configuration,
+ ffi::{self, RequestHandle},
+ h3::attach_references,
+ platform::detect_platform,
+ references::ReferenceVideos,
+ types::GenerationRequest,
+};
+use crate::settings::ProviderSetting;
+use std::{collections::BTreeMap, path::Path};
+
+pub fn export_refmod(
+ request: &GenerationRequest,
+ settings: &BTreeMap<String, ProviderSetting>,
+ references: &ReferenceVideos,
+ output: &Path,
+ name: &str,
+ description: &str,
+) -> Result<(), String> {
+ if request.reference_count() == 0 {
+ return Err("Add an image, video or sound to export a refmod.".into());
+ }
+ let _models = super::MODEL_ACCESS
+ .read()
+ .map_err(|_| "Model access lock failed.")?;
+ let configuration = Configuration::from_settings(settings);
+ let api = ffi::Api::load(&configuration.dll_path)?;
+ api.version()?;
+ export_with(&api, request, &configuration, references, output, name, description)
+}
+
+pub(super) fn export_with(
+ api: &ffi::Api,
+ request: &GenerationRequest,
+ configuration: &Configuration,
+ references: &ReferenceVideos,
+ output: &Path,
+ name: &str,
+ description: &str,
+) -> Result<(), String> {
+ let platform = configuration.platform(detect_platform(api));
+ let handle = RequestHandle::new(api)?;
+ api.set_inference_backend(&handle, platform.backend())?;
+ // Only the video and audio VAEs encode references.
+ for (id, _, path) in &configuration.models[3..] {
+ if let Some(path) = path {
+ api.set_model(&handle, *id, path)?;
+ }
+ }
+ attach_references(api, &handle, request, configuration, references)?;
+ api.export_refmod(&handle, output, name, description)
+}
diff --git a/src-tauri/core/src/slopfab/tests.rs b/src-tauri/core/src/slopfab/tests.rs
index a69235a..39f77ba 100644
--- a/src-tauri/core/src/slopfab/tests.rs
+++ b/src-tauri/core/src/slopfab/tests.rs
@@ -647,6 +647,31 @@ fn refmods_attach_for_plans_and_icons_or_report_an_older_dll() {
.is_err());
}
+#[test]
+fn refmod_export_needs_raw_media_and_vaes_or_reports_an_older_dll() {
+ let references = ReferenceVideos::default();
+ let settings = BTreeMap::new();
+ let root = tempfile::tempdir().unwrap();
+ let output = root.path().join("voice.safetensors");
+ let mut request = GenerationRequest { job_id: "refmod-export-test".into(), ..Default::default() };
+ let empty = export_refmod(&request, &settings, &references, &output, "Voice", "").unwrap_err();
+ assert!(empty.contains("Add an image, video or sound"), "{empty}");
+ let audio = references
+ .create_reference_audio(&vec![0; 32_000 * 2 * 4], 1, 32_000)
+ .unwrap();
+ request.reference_audio_ids = vec![audio.clone()];
+ // The runtime refuses a request without VAEs before any GPU work.
+ let missing = export_refmod(&request, &settings, &references, &output, "Voice", "A voice").unwrap_err();
+ assert!(missing.contains("VAE path is missing"), "{missing}");
+ assert!(!output.exists());
+ let configuration = Configuration::from_settings(&settings);
+ let mut api = ffi::Api::load(&configuration.dll_path).unwrap();
+ api.disable_refmod_export_for_test();
+ let older = refmod_export::export_with(&api, &request, &configuration, &references, &output, "Voice", "").unwrap_err();
+ assert!(older.contains("API 1.18"), "{older}");
+ references.release_reference_audios(&[audio]).unwrap();
+}
+
#[test]
fn motion_cache_reaches_cuda_and_vulkan_requests_and_changes_timing_profile() {
let mut configuration = Configuration::from_settings(&BTreeMap::from([(