Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Export reference refmods locally and on LAN workers

Bitti1c0bb3644d45
diff --git a/src-tauri/core/src/worker/protocol.rs b/src-tauri/core/src/worker/protocol.rs
index ef298c5..5a7c001 100644
--- a/src-tauri/core/src/worker/protocol.rs
+++ b/src-tauri/core/src/worker/protocol.rs
@@ -81,6 +81,15 @@ pub struct StartJob {
pub save_latents: bool,
}
+/// A refmod export. The job carries raw references and the VAEs only.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RefmodExport {
+ pub job: RemoteJob,
+ pub name: String,
+ pub description: String,
+}
+
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct WireEvent {
diff --git a/src-tauri/core/src/worker/server.rs b/src-tauri/core/src/worker/server.rs
index 1b5b36f..c615896 100644
--- a/src-tauri/core/src/worker/server.rs
+++ b/src-tauri/core/src/worker/server.rs
@@ -406,6 +406,18 @@ impl Worker {
let (request, settings) = self.materialize(job)?;
json(&slopfab::resolve_plan(&request, &settings, &self.runtime.references)?)
}
+ (Method::Post, ["refmods"]) => {
+ let export: RefmodExport = body(request)?;
+ let id = generated_file_stem(&export.job.request.job_id)?;
+ let (request, settings) = self.materialize(export.job)?;
+ let directory = self.data.join("refmods");
+ fs::create_dir_all(&directory).map_err(|error| error.to_string())?;
+ let path = directory.join(format!("{id}.safetensors"));
+ let refmod = slopfab::export_refmod(&request, &settings, &self.runtime.references, &path, &export.name, &export.description)
+ .and_then(|_| fs::read(&path).map_err(|error| error.to_string()));
+ let _ = fs::remove_file(&path);
+ Ok(bytes(refmod?, "application/octet-stream"))
+ }
(Method::Post, ["jobs", id]) => {
let start: StartJob = body(request)?;
self.start_job(id, start)?;
diff --git a/src-tauri/src/app.rs b/src-tauri/src/app.rs
index 7b3d27c..8e628a3 100644
--- a/src-tauri/src/app.rs
+++ b/src-tauri/src/app.rs
@@ -114,6 +114,7 @@ pub fn run() {
.manage(slopfab::SlopfabRuntime::default())
.manage(weights::WeightDownloads::default())
.manage(worker::Workers::default())
+ .manage(commands::generation::RefmodExportTargets::default())
.manage(ExitGuard::default())
.manage(native_shell::Backdrop::default())
.manage(native_shell::SystemAccent::default())
@@ -217,6 +218,8 @@ pub fn run() {
commands::generation::release_reference_videos,
commands::generation::create_reference_audio,
commands::generation::release_reference_audios,
+ commands::generation::choose_refmod_export_path,
+ commands::generation::export_reference_refmod,
commands::media::import_media_files,
commands::media::read_project_file,
commands::media::read_external_media_file,
diff --git a/src-tauri/src/commands/generation.rs b/src-tauri/src/commands/generation.rs
index 50f2220..036357a 100644
--- a/src-tauri/src/commands/generation.rs
+++ b/src-tauri/src/commands/generation.rs
@@ -2,6 +2,7 @@ use crate::media::artifacts::*;
use crate::project::{validation::validate_and_normalize_config, *};
use crate::{diagnostics, slopfab, worker::Workers};
use tauri::{AppHandle, Emitter};
+use tauri_plugin_dialog::DialogExt;
/// Generation commands run on the native queue here, or on the LAN worker
/// selected in Settings → Workers. The webview cannot tell the difference.
async fn blocking<T: Send + 'static>(
@@ -282,6 +283,71 @@ pub(crate) async fn enqueue_slopfab_generation(
.await
}
+/// Refmod destinations picked in the save dialog during this run. An export
+/// writes only to one of these, so the webview cannot name its own target.
+#[derive(Clone, Default)]
+pub(crate) struct RefmodExportTargets(std::sync::Arc<std::sync::Mutex<std::collections::HashSet<String>>>);
+
+#[tauri::command]
+pub(crate) async fn choose_refmod_export_path(
+ app: AppHandle,
+ targets: tauri::State<'_, RefmodExportTargets>,
+ name: String,
+) -> Result<Option<String>, String> {
+ let targets = targets.inner().clone();
+ blocking(move || {
+ let Some(destination) = app.dialog().file()
+ .set_title("Export refmod")
+ .add_filter("Refmod", &["safetensors"])
+ .set_file_name(format!("{}.safetensors", paths::safe_folder_name(&name)))
+ .blocking_save_file() else {
+ return Ok(None);
+ };
+ let destination = destination.into_path().map_err(|_| "Choose a local refmod destination.")?;
+ let destination = if destination.extension().is_some_and(|value| value.eq_ignore_ascii_case("safetensors")) {
+ destination
+ } else {
+ destination.with_extension("safetensors")
+ };
+ let destination = paths::display_path(&destination);
+ targets.0.lock().map_err(|_| "Refmod export lock failed.")?.insert(destination.clone());
+ Ok(Some(destination))
+ })
+ .await
+}
+
+/// Encodes a reference's prepared images, video and sound into a refmod file.
+#[tauri::command]
+pub(crate) async fn export_reference_refmod(
+ state: tauri::State<'_, slopfab::SlopfabRuntime>,
+ workers: tauri::State<'_, Workers>,
+ targets: tauri::State<'_, RefmodExportTargets>,
+ request: slopfab::GenerationRequest,
+ config: ProjectConfig,
+ output_path: String,
+ name: String,
+ description: String,
+) -> Result<(), String> {
+ let (runtime, workers, targets) = (state.inner().clone(), workers.inner().clone(), targets.inner().clone());
+ blocking(move || {
+ if !targets.0.lock().map_err(|_| "Refmod export lock failed.")?.remove(&output_path) {
+ return Err("Choose where to save the refmod before exporting it.".into());
+ }
+ let output = std::path::Path::new(&output_path);
+ let context = serde_json::json!({ "jobId": request.job_id, "referenceCount": request.reference_count() });
+ let result = validate_and_normalize_config(config).and_then(|config| match workers.active()? {
+ Some(worker) => workers.export_refmod(&worker, &request, &config.provider_settings, &name, &description, output),
+ None => slopfab::export_refmod(&request, &config.provider_settings, &runtime.references, output, &name, &description),
+ });
+ match &result {
+ Ok(()) => diagnostics::info("slopfab", "refmod.exported", "Reference exported as a refmod.", context),
+ Err(error) => diagnostics::error("slopfab", "refmod.export_failed", error, context),
+ }
+ result
+ })
+ .await
+}
+
fn prepare_image_edit_path(request: &mut slopfab::GenerationRequest, folder_path: Option<&str>) -> Result<(), String> {
if let Some(edit) = &request.image_edit {
let root = crate::project::paths::ProjectRoot::open(folder_path.ok_or("Image edits require a project folder.")?)?;
diff --git a/src-tauri/src/worker/client.rs b/src-tauri/src/worker/client.rs
index a15e6a3..754cf23 100644
--- a/src-tauri/src/worker/client.rs
+++ b/src-tauri/src/worker/client.rs
@@ -571,6 +571,25 @@ impl Workers {
result
}
+ /// Encodes a refmod on the worker and saves the file it returns.
+ pub(crate) fn export_refmod(&self, connection: &Connection, request: &GenerationRequest, settings: &BTreeMap<String, ProviderSetting>, name: &str, description: &str, destination: &Path) -> Result<(), String> {
+ // Export needs the VAEs only; keep the other weights off the network.
+ let mut settings = settings.clone();
+ if let Some(slopfab) = settings.get_mut("slopfab") {
+ slopfab.options.retain(|option, _| matches!(option.as_str(), "videoVae" | "audioVae"));
+ }
+ let (job, sources) = build_job(request, &settings)?;
+ let cancel = self.register(&request.job_id, connection)?;
+ let result = self.send_files(connection, &job, &sources, &cancel).and_then(|_| {
+ let export = RefmodExport { job, name: name.into(), description: description.into() };
+ let response = connection.send(connection.request(reqwest::Method::POST, "/v1/refmods").json(&export))
+ .map_err(|error| if error.contains("Unknown worker route") { format!("Update worker {} to export refmods.", connection.name) } else { error })?;
+ save_response(response, destination, "the refmod")
+ });
+ self.unregister(&request.job_id);
+ result
+ }
+
/// Starts a job on the worker and follows it on a background thread. The
/// webview receives the same queued/progress/framesReady events, and the
/// frames wait in this process's render store like a local render.
@@ -645,18 +664,8 @@ impl Workers {
/// demand as the webview encodes them.
fn collect(&self, connection: &Connection, id: &str, latents: Option<&Path>) -> Result<Option<String>, String> {
if let Some(destination) = latents {
- let mut response = connection.send(connection.request(reqwest::Method::GET, &format!("/v1/jobs/{id}/latents")))?;
- let temporary = destination.with_extension("safetensors.part");
- let copied = File::create(&temporary)
- .and_then(|mut file| io::copy(&mut response, &mut file).and_then(|bytes| file.sync_all().map(|_| bytes)))
- .map_err(|error| format!("Could not save latents from the worker: {error}"));
- match copied {
- Ok(_) => fs::rename(&temporary, destination).map_err(|error| error.to_string())?,
- Err(error) => {
- let _ = fs::remove_file(&temporary);
- return Err(error);
- }
- }
+ let response = connection.send(connection.request(reqwest::Method::GET, &format!("/v1/jobs/{id}/latents")))?;
+ save_response(response, destination, "latents")?;
}
let summary: rendered::RenderedSummary = connection.get(&format!("/v1/jobs/{id}/summary"))?;
let audio: Vec<f32> = connection.bytes(&format!("/v1/jobs/{id}/audio"))?
@@ -686,6 +695,21 @@ impl Workers {
}
}
+/// Streams a worker's file beside its destination, then moves it into place.
+fn save_response(mut response: Response, destination: &Path, what: &str) -> Result<(), String> {
+ let temporary = destination.with_extension("safetensors.part");
+ let copied = File::create(&temporary)
+ .and_then(|mut file| io::copy(&mut response, &mut file).and_then(|bytes| file.sync_all().map(|_| bytes)))
+ .map_err(|error| format!("Could not save {what} from the worker: {error}"));
+ match copied {
+ Ok(_) => fs::rename(&temporary, destination).map_err(|error| error.to_string()),
+ Err(error) => {
+ let _ = fs::remove_file(&temporary);
+ Err(error)
+ }
+ }
+}
+
/// Frames stay on the worker until the encoder asks for them, and the
/// worker's copy is released with this one.
struct RemoteFrames {
@@ -959,6 +983,51 @@ mod tests {
}
}
+ #[test]
+ fn refmod_exports_send_only_vaes_and_save_the_worker_file() {
+ let server = tiny_http::Server::http("127.0.0.1:0").unwrap();
+ let address = server.server_addr().to_ip().unwrap().to_string();
+ let posted = Arc::new(Mutex::new(None::<RefmodExport>));
+ let seen = posted.clone();
+ std::thread::spawn(move || {
+ for mut request in server.incoming_requests() {
+ let url = request.url().to_string();
+ let body: Vec<u8> = match url.as_str() {
+ "/v1/uploads/missing" => br#"{"missing":[]}"#.to_vec(),
+ "/v1/refmods" => {
+ *seen.lock().unwrap() = serde_json::from_reader(request.as_reader()).ok();
+ b"refmod bytes".to_vec()
+ }
+ _ => { let _ = request.respond(tiny_http::Response::empty(404)); continue; }
+ };
+ let _ = request.respond(tiny_http::Response::from_data(body));
+ }
+ });
+ let info = WorkerInfo {
+ id: "00112233aabbccdd".into(), name: "Stub".into(), version: "0".into(), protocol: PROTOCOL_VERSION,
+ requires_token: false, gpus: Vec::new(),
+ runtime: WorkerRuntime { state: "ready".into(), version: None, platform: None, cuda_available: false, cuda_device_names: Vec::new(), detail: String::new() },
+ };
+ let connection = Connection { base: format!("http://{address}"), token: None, name: "Stub".into(), address, info, http: http_client(None) };
+ let folder = tempfile::tempdir().unwrap();
+ let file = |name: &str| {
+ let path = folder.path().join(name);
+ fs::write(&path, name).unwrap();
+ path.to_string_lossy().into_owned()
+ };
+ let options = ["transformer", "videoVae", "audioVae"].into_iter()
+ .map(|name| (name.to_string(), ProviderOption::String(file(&format!("{name}.safetensors"))))).collect();
+ let settings = BTreeMap::from([("slopfab".to_string(), ProviderSetting { enabled: true, model: None, options })]);
+ let request = GenerationRequest { job_id: "refmod-export".into(), reference_paths: vec![file("hero.png")], ..Default::default() };
+ let destination = folder.path().join("hero.safetensors");
+ Workers::default().export_refmod(&connection, &request, &settings, "Hero", "A hero", &destination).unwrap();
+ assert_eq!(fs::read(&destination).unwrap(), b"refmod bytes");
+ let export = posted.lock().unwrap().take().unwrap();
+ assert_eq!((export.name.as_str(), export.description.as_str()), ("Hero", "A hero"));
+ assert_eq!(export.job.files.len(), 3, "the transformer stays on this computer");
+ assert!(!export.job.settings["slopfab"].options.contains_key("transformer"));
+ }
+
/// Drives a running worker through the same path as the app. Needs
/// SLOPUS_E2E_WORKER (host:port) and SLOPUS_E2E_WEIGHTS (a folder of
/// Slopus downloads: transformer, text encoder and VAEs).