AI media harness for planning, generating, and editing locally
TypeScript
Add standalone audio references to the native backend
Bitti07499052c7af
diff --git a/src-tauri/core/src/models.rs b/src-tauri/core/src/models.rsindex d34a8c6..7abef78 100644--- a/src-tauri/core/src/models.rs+++ b/src-tauri/core/src/models.rs@@ -7,6 +7,7 @@ pub struct ModelCapabilities {pub min_steps: i32,pub max_image_references: usize,pub max_video_references: usize,+ pub max_audio_references: usize,pub max_references: usize,pub continuation_overlap: i32,pub frame_stride: i32,@@ -17,6 +18,7 @@ pub const H3_CAPABILITIES: ModelCapabilities = ModelCapabilities {min_steps: 2,max_image_references: 9,max_video_references: 3,+ max_audio_references: 3,max_references: 12,continuation_overlap: 22,frame_stride: 17,diff --git a/src-tauri/core/src/slopfab/config.rs b/src-tauri/core/src/slopfab/config.rsindex 6bf77ca..6cd58a6 100644--- a/src-tauri/core/src/slopfab/config.rs+++ b/src-tauri/core/src/slopfab/config.rs@@ -111,8 +111,9 @@ impl Configuration {|| !request.refmods.is_empty()|| !request.reference_paths.is_empty()|| request.reference_video_ids.len() != expected+ || !request.reference_audio_ids.is_empty(){- return Err("Extend/Bridge requires exactly one/two video references without frame anchors, Animate, continuation or refmods.".into());+ return Err("Extend/Bridge requires exactly one/two video references without frame anchors, sound references, Animate, continuation or refmods.".into());}}if self.animate && self.motion_cache {@@ -130,8 +131,8 @@ impl Configuration {"Animate requires exactly one repainted frame of the driving scene.".into(),);}- if request.continuation_path.is_some() || !request.refmods.is_empty() {- return Err("Animate does not support scene continuation or refmods.".into());+ if request.continuation_path.is_some() || !request.refmods.is_empty() || !request.reference_audio_ids.is_empty() {+ return Err("Animate does not support scene continuation, sound references or refmods.".into());}} else if request.prompt.trim().is_empty() {return Err("Generation prompt cannot be empty.".into());diff --git a/src-tauri/core/src/slopfab/ffi/mod.rs b/src-tauri/core/src/slopfab/ffi/mod.rsindex 954b3d9..a959b78 100644--- a/src-tauri/core/src/slopfab/ffi/mod.rs+++ b/src-tauri/core/src/slopfab/ffi/mod.rs@@ -160,6 +160,17 @@ impl Api {self.check(r)?;self.inner.add_reference(r.pointer.as_ptr(), path)}+ pub fn add_reference_audio(+ &self,+ r: &RequestHandle,+ samples: &[f32],+ channels: i32,+ sample_rate: i32,+ ) -> Result<(), String> {+ self.check(r)?;+ self.inner+ .add_reference_audio(r.pointer.as_ptr(), samples, channels, sample_rate)+ }pub fn add_lora(&self, r: &RequestHandle, path: &Path, strength: f32) -> Result<(), String> {self.check(r)?;self.inner.add_lora(r.pointer.as_ptr(), path, strength)diff --git a/src-tauri/core/src/slopfab/ffi/raw/mod.rs b/src-tauri/core/src/slopfab/ffi/raw/mod.rsindex db63b2a..e987067 100644--- a/src-tauri/core/src/slopfab/ffi/raw/mod.rs+++ b/src-tauri/core/src/slopfab/ffi/raw/mod.rs@@ -57,6 +57,7 @@ pub struct Output {pub steps_computed: i32,pub steps_skipped: i32,}+type AddReferenceAudio = unsafe extern "C" fn(*mut Request, *const f32, usize, i32, i32) -> i32;pub type ProgressFn = Option<unsafe extern "C" fn(*const Progress, *mut c_void)>;type SetMotionCache =unsafe extern "C" fn(*mut Request, i32, f32, f32, i32, i32, f32, f32, i32, i32) -> i32;@@ -92,6 +93,7 @@ pub struct Api {prepare_lora_grid: Option<unsafe extern "C" fn(*const c_char, i32, i32) -> i32>,add_refmod: Option<unsafe extern "C" fn(*mut Request, *const c_char, f32, i32) -> i32>,add_reference: unsafe extern "C" fn(*mut Request, *const c_char) -> i32,+ add_reference_audio: Option<AddReferenceAudio>,set_attention: unsafe extern "C" fn(*mut Request, *const c_char) -> i32,set_motion_cache: Option<SetMotionCache>,set_inference_backend: unsafe extern "C" fn(*mut Request, i32) -> i32,@@ -238,6 +240,10 @@ impl Api {).ok().map(|symbol| *symbol),+ add_reference_audio: library+ .get::<AddReferenceAudio>(b"slopfab_request_add_reference_audio_f32\0")+ .ok()+ .map(|symbol| *symbol),set_attention: symbol!("slopfab_request_set_attention",unsafe extern "C" fn(*mut Request, *const c_char) -> i32@@ -508,6 +514,16 @@ impl Api {let path = path_cstring(path)?;self.error(unsafe { add(r, path.as_ptr(), strength, copies) })}+ pub fn add_reference_audio(+ &self,+ r: *mut Request,+ samples: &[f32],+ channels: i32,+ sample_rate: i32,+ ) -> Result<(), String> {+ let add = self.add_reference_audio.ok_or("This slopfab.dll does not support audio references. Update the runtime.")?;+ self.error(unsafe { add(r, samples.as_ptr(), samples.len(), channels, sample_rate) })+ }pub fn resolve(&self, request: *mut Request) -> Result<Plan, String> {let mut value = Plan {canvas_width: 0,diff --git a/src-tauri/core/src/slopfab/h3.rs b/src-tauri/core/src/slopfab/h3.rsindex 05b07dc..00dfe70 100644--- a/src-tauri/core/src/slopfab/h3.rs+++ b/src-tauri/core/src/slopfab/h3.rs@@ -132,6 +132,12 @@ pub(super) fn configure_request(.attach(handle, &configuration.dll_path)?;}}+ // Standalone audio follows the videos, so slopfab numbers it after every+ // video soundtrack. The prompt compiler numbers <Audio N> the same way.+ for id in &request.reference_audio_ids {+ let audio = references.reference_audio(id)?;+ api.add_reference_audio(handle, &audio.samples, audio.channels, audio.sample_rate)?;+ }for refmod in &request.refmods {if refmod.path.trim().is_empty()|| !refmod.strength.is_finite()diff --git a/src-tauri/core/src/slopfab/planning.rs b/src-tauri/core/src/slopfab/planning.rsindex 7866e0b..4f1964d 100644--- a/src-tauri/core/src/slopfab/planning.rs+++ b/src-tauri/core/src/slopfab/planning.rs@@ -69,12 +69,13 @@ pub(super) fn validate_generation_controls(request: &GenerationRequest) -> Resullet limits = crate::models::H3_CAPABILITIES;if request.reference_paths.len() > limits.max_image_references|| request.reference_video_ids.len() > limits.max_video_references+ || request.reference_audio_ids.len() > limits.max_audio_references|| request.reference_count() > limits.max_references{- return Err("Use at most nine images and three videos per generation.".into());+ return Err("Use at most nine images, three videos and three sounds per generation.".into());}- if request.still_image && !request.reference_video_ids.is_empty() {- return Err("Video references cannot be used for still-image generation.".into());+ if request.still_image && (!request.reference_video_ids.is_empty() || !request.reference_audio_ids.is_empty()) {+ return Err("Video and sound references cannot be used for still-image generation.".into());}if request.steps < limits.min_steps {return Err("Generation step count must be at least 2.".into());diff --git a/src-tauri/core/src/slopfab/references.rs b/src-tauri/core/src/slopfab/references.rsindex 50dd054..c675bbf 100644--- a/src-tauri/core/src/slopfab/references.rs+++ b/src-tauri/core/src/slopfab/references.rs@@ -10,9 +10,18 @@ use std::{#[derive(Clone, Default)]pub struct ReferenceVideos {pub(super) videos: Arc<Mutex<HashMap<String, ffi::ReferenceVideoHandle>>>,+ pub(super) audios: Arc<Mutex<HashMap<String, Arc<ReferenceAudio>>>>,next: Arc<std::sync::atomic::AtomicU64>,}+/// Decoded standalone reference audio. The C API copies the samples when a+/// request adds them, so the registry only keeps them until release.+pub struct ReferenceAudio {+ pub samples: Vec<f32>,+ pub channels: i32,+ pub sample_rate: i32,+}+impl ReferenceVideos {pub fn create_reference_video(&self,@@ -73,6 +82,58 @@ impl ReferenceVideos {.set_audio(&samples, channels, sample_rate)}+ pub fn create_reference_audio(+ &self,+ bytes: &[u8],+ channels: i32,+ sample_rate: i32,+ ) -> Result<String, String> {+ if !(1..=2).contains(&channels) || sample_rate <= 0 {+ return Err("Reference audio must be mono or stereo with a positive sample rate.".into());+ }+ let samples = float_samples(bytes)?;+ if samples.is_empty() || samples.len() % channels as usize != 0 {+ return Err("Reference audio must contain complete sample frames.".into());+ }+ let mut audios = self+ .audios+ .lock()+ .map_err(|_| "Reference audio lock failed.")?;+ if audios.len() >= 3 {+ return Err("At most three audio references can be prepared at once.".into());+ }+ let id = self.next.fetch_add(1, Ordering::Relaxed).to_string();+ audios.insert(+ id.clone(),+ Arc::new(ReferenceAudio {+ samples,+ channels,+ sample_rate,+ }),+ );+ Ok(id)+ }++ pub fn release_reference_audios(&self, ids: &[String]) -> Result<(), String> {+ let mut audios = self+ .audios+ .lock()+ .map_err(|_| "Reference audio lock failed.")?;+ for id in ids {+ audios.remove(id);+ }+ Ok(())+ }++ pub(super) fn reference_audio(&self, id: &str) -> Result<Arc<ReferenceAudio>, String> {+ self.audios+ .lock()+ .map_err(|_| "Reference audio lock failed.")?+ .get(id)+ .cloned()+ .ok_or_else(|| "The prepared reference audio no longer exists. Please retry generation.".into())+ }+pub fn release_reference_videos(&self, ids: &[String]) -> Result<(), String> {let mut videos = self.videos@@ -84,3 +145,13 @@ impl ReferenceVideos {Ok(())}}++fn float_samples(bytes: &[u8]) -> Result<Vec<f32>, String> {+ if bytes.len() % 4 != 0 {+ return Err("Reference audio must contain complete float32 samples.".into());+ }+ Ok(bytes+ .chunks_exact(4)+ .map(|chunk| f32::from_le_bytes(chunk.try_into().unwrap()))+ .collect())+}diff --git a/src-tauri/core/src/slopfab/tests.rs b/src-tauri/core/src/slopfab/tests.rsindex 626fb8e..f2e2178 100644--- a/src-tauri/core/src/slopfab/tests.rs+++ b/src-tauri/core/src/slopfab/tests.rs@@ -865,6 +865,63 @@ fn vulkan_video_references_prepare_and_attach_for_planning_and_generation() {result.unwrap();}+#[test]+fn standalone_audio_references_attach_after_videos_and_need_visual_media() {+ let references = ReferenceVideos::default();+ let settings = BTreeMap::new();+ let video = references.create_reference_video(2.0, &settings).unwrap();+ let audio = references+ .create_reference_audio(&vec![0; 32_000 * 2 * 4], 1, 32_000)+ .unwrap();+ let result = (|| -> Result<(), String> {+ references.append_reference_video(&video, &vec![127; 64 * 64 * 4], 64, 64, 0.0)?;+ let mut request = GenerationRequest {+ job_id: "audio-reference-test".into(),+ prompt: "A scene using <Video 1> and <Audio 1>.".into(),+ frames: 120,+ steps: 12,+ seed: 1,+ canvas_width: 736,+ canvas_height: 416,+ reference_video_ids: vec![video.clone()],+ reference_audio_ids: vec![audio.clone()],+ ..Default::default()+ };+ let configuration = Configuration::from_settings(&settings);+ let api = ffi::Api::load(&configuration.dll_path)?;+ let plan = |request: &GenerationRequest| -> Result<String, String> {+ let handle = RequestHandle::new(&api)?;+ configure_request(&api, &handle, request, &configuration, ComputePlatform::Vulkan, RequestPurpose::Plan, &references)?;+ api.resolve(&handle)?;+ api.describe(&handle)+ };+ let description = plan(&request)?;+ assert!(description.contains("reference audios 1"));+ request.reference_video_ids.clear();+ assert!(plan(&request).unwrap_err().contains("image or video reference"));+ Ok(())+ })();+ references.release_reference_videos(&[video]).unwrap();+ references.release_reference_audios(&[audio.clone()]).unwrap();+ assert!(references.reference_audio(&audio).is_err());+ result.unwrap();+}++#[test]+fn reference_audio_registry_validates_pcm_layout() {+ let references = ReferenceVideos::default();+ assert!(references.create_reference_audio(&[0; 6], 1, 32_000).is_err());+ assert!(references.create_reference_audio(&[0; 12], 2, 32_000).is_err());+ assert!(references.create_reference_audio(&[0; 8], 3, 32_000).is_err());+ assert!(references.create_reference_audio(&[0; 8], 1, 0).is_err());+ let ids: Vec<String> = (0..3)+ .map(|_| references.create_reference_audio(&[0; 8], 2, 48_000).unwrap())+ .collect();+ assert!(references.create_reference_audio(&[0; 8], 2, 48_000).is_err());+ references.release_reference_audios(&ids).unwrap();+ assert!(references.create_reference_audio(&[0; 8], 2, 48_000).is_ok());+}+#[test]fn video_reference_c_api_copies_inputs_and_retains_attached_snapshot() {let path = default_dll_path();diff --git a/src-tauri/core/src/slopfab/types.rs b/src-tauri/core/src/slopfab/types.rsindex 2e58b7e..8a9a931 100644--- a/src-tauri/core/src/slopfab/types.rs+++ b/src-tauri/core/src/slopfab/types.rs@@ -63,6 +63,8 @@ pub struct GenerationRequest {#[serde(default)]pub reference_video_ids: Vec<String>,#[serde(default)]+ pub reference_audio_ids: Vec<String>,+ #[serde(default)]pub refmods: Vec<RefmodInput>,#[serde(default)]pub continuation_relative_path: Option<String>,@@ -102,7 +104,7 @@ pub struct RefmodInput {impl GenerationRequest {pub fn reference_count(&self) -> usize {- self.reference_paths.len() + self.reference_video_ids.len()+ self.reference_paths.len() + self.reference_video_ids.len() + self.reference_audio_ids.len()}pub fn refmod_count(&self) -> usize {self.refmods.iter().filter(|refmod| refmod.strength > 0.0).count()diff --git a/src-tauri/core/src/worker/protocol.rs b/src-tauri/core/src/worker/protocol.rsindex bac0071..ef298c5 100644--- a/src-tauri/core/src/worker/protocol.rs+++ b/src-tauri/core/src/worker/protocol.rs@@ -9,7 +9,7 @@ use std::collections::BTreeMap;pub const SERVICE_TYPE: &str = "_slopus-worker._tcp.local.";/// Bumped on any incompatible change to the routes or bodies below.-pub const PROTOCOL_VERSION: u32 = 1;+pub const PROTOCOL_VERSION: u32 = 2;pub const DEFAULT_PORT: u16 = 47321;pub const TOKEN_HEADER: &str = "x-slopus-token";diff --git a/src-tauri/core/src/worker/server.rs b/src-tauri/core/src/worker/server.rsindex 9096e99..1b5b36f 100644--- a/src-tauri/core/src/worker/server.rs+++ b/src-tauri/core/src/worker/server.rs@@ -469,6 +469,18 @@ impl Worker {self.runtime.references.set_reference_video_audio(id, &samples, channels, rate)?;json(&true)}+ (Method::Post, ["reference-audios"]) => {+ let channels = parsed_header(request, "x-reference-channels")?;+ let rate = parsed_header(request, "x-reference-rate")?;+ let mut samples = Vec::new();+ request.as_reader().take(1024 * 1024 * 1024).read_to_end(&mut samples).map_err(|error| error.to_string())?;+ json(&self.runtime.references.create_reference_audio(&samples, channels, rate)?)+ }+ (Method::Post, ["reference-audios", "release"]) => {+ let ids: Vec<String> = body(request)?;+ self.runtime.references.release_reference_audios(&ids)?;+ json(&true)+ }_ => Err(HttpError(404, "Unknown worker route.".into())),}}diff --git a/src-tauri/src/agent/tests_transport.rs b/src-tauri/src/agent/tests_transport.rsindex 0d8f2c5..21c3feb 100644--- a/src-tauri/src/agent/tests_transport.rs+++ b/src-tauri/src/agent/tests_transport.rs@@ -38,6 +38,7 @@ fn another_model_supplies_its_own_policy_and_vocabulary() {min_steps: 1,max_image_references: 1,max_video_references: 0,+ max_audio_references: 0,max_references: 1,continuation_overlap: 0,frame_stride: 1,diff --git a/src-tauri/src/app.rs b/src-tauri/src/app.rsindex b0724af..b47a7f3 100644--- a/src-tauri/src/app.rs+++ b/src-tauri/src/app.rs@@ -203,6 +203,8 @@ pub fn run() {commands::generation::append_reference_video,commands::generation::set_reference_video_audio,commands::generation::release_reference_videos,+ commands::generation::create_reference_audio,+ commands::generation::release_reference_audios,commands::media::import_media_files,commands::media::read_project_file,commands::media::read_external_media_file,diff --git a/src-tauri/src/commands/generation.rs b/src-tauri/src/commands/generation.rsindex 17b808f..50f2220 100644--- a/src-tauri/src/commands/generation.rs+++ b/src-tauri/src/commands/generation.rs@@ -68,15 +68,17 @@ pub(crate) async fn append_reference_video(let bytes = bytes.clone();let (runtime, workers) = (state.inner().clone(), workers.inner().clone());blocking(move || match workers.active()? {- Some(worker) => worker.post_bytes(- &format!("/v1/reference-videos/{id}/frames"),- &[- ("x-reference-width", width.to_string()),- ("x-reference-height", height.to_string()),- ("x-reference-time", timestamp.to_string()),- ],- bytes,- ),+ Some(worker) => worker+ .post_bytes::<bool>(+ &format!("/v1/reference-videos/{id}/frames"),+ &[+ ("x-reference-width", width.to_string()),+ ("x-reference-height", height.to_string()),+ ("x-reference-time", timestamp.to_string()),+ ],+ bytes,+ )+ .map(|_| ()),None => runtime.references.append_reference_video(&id, &bytes, width, height, timestamp),@@ -102,9 +104,60 @@ pub(crate) async fn set_reference_video_audio(.map_err(|_| "Invalid sample rate.")?;let bytes = bytes.clone();let (runtime, workers) = (state.inner().clone(), workers.inner().clone());+ blocking(move || match workers.active()? {+ Some(worker) => worker+ .post_bytes::<bool>(+ &format!("/v1/reference-videos/{id}/audio"),+ &[+ ("x-reference-channels", channels.to_string()),+ ("x-reference-rate", rate.to_string()),+ ],+ bytes,+ )+ .map(|_| ()),+ None => runtime+ .references+ .set_reference_video_audio(&id, &bytes, channels, rate),+ })+ .await+}++#[tauri::command]+pub(crate) async fn release_reference_videos(+ state: tauri::State<'_, slopfab::SlopfabRuntime>,+ workers: tauri::State<'_, Workers>,+ ids: Vec<String>,+) -> Result<(), String> {+ let (runtime, workers) = (state.inner().clone(), workers.inner().clone());+ blocking(move || match workers.active()? {+ Some(worker) => worker+ .post::<bool>("/v1/reference-videos/release", &ids)+ .map(|_| ()),+ None => runtime.references.release_reference_videos(&ids),+ })+ .await+}++#[tauri::command]+pub(crate) async fn create_reference_audio(+ state: tauri::State<'_, slopfab::SlopfabRuntime>,+ workers: tauri::State<'_, Workers>,+ request: tauri::ipc::Request<'_>,+) -> Result<String, String> {+ let tauri::ipc::InvokeBody::Raw(bytes) = request.body() else {+ return Err("Reference audio requires a binary body.".into());+ };+ let channels: i32 = reference_video_header(&request, "x-reference-channels")?+ .parse()+ .map_err(|_| "Invalid audio channels.")?;+ let rate: i32 = reference_video_header(&request, "x-reference-rate")?+ .parse()+ .map_err(|_| "Invalid sample rate.")?;+ let bytes = bytes.clone();+ let (runtime, workers) = (state.inner().clone(), workers.inner().clone());blocking(move || match workers.active()? {Some(worker) => worker.post_bytes(- &format!("/v1/reference-videos/{id}/audio"),+ "/v1/reference-audios",&[("x-reference-channels", channels.to_string()),("x-reference-rate", rate.to_string()),@@ -113,13 +166,13 @@ pub(crate) async fn set_reference_video_audio(),None => runtime.references- .set_reference_video_audio(&id, &bytes, channels, rate),+ .create_reference_audio(&bytes, channels, rate),}).await}#[tauri::command]-pub(crate) async fn release_reference_videos(+pub(crate) async fn release_reference_audios(state: tauri::State<'_, slopfab::SlopfabRuntime>,workers: tauri::State<'_, Workers>,ids: Vec<String>,@@ -127,9 +180,9 @@ pub(crate) async fn release_reference_videos(let (runtime, workers) = (state.inner().clone(), workers.inner().clone());blocking(move || match workers.active()? {Some(worker) => worker- .post::<bool>("/v1/reference-videos/release", &ids)+ .post::<bool>("/v1/reference-audios/release", &ids).map(|_| ()),- None => runtime.references.release_reference_videos(&ids),+ None => runtime.references.release_reference_audios(&ids),}).await}diff --git a/src-tauri/src/worker/client.rs b/src-tauri/src/worker/client.rsindex 34a786a..81ce151 100644--- a/src-tauri/src/worker/client.rs+++ b/src-tauri/src/worker/client.rs@@ -177,12 +177,12 @@ impl Connection {self.send(self.request(reqwest::Method::POST, path).json(body))?.json().map_err(|error| format!("Worker {}: {error}", self.name))}- pub(crate) fn post_bytes(&self, path: &str, headers: &[(&str, String)], bytes: Vec<u8>) -> Result<(), String> {+ pub(crate) fn post_bytes<T: DeserializeOwned>(&self, path: &str, headers: &[(&str, String)], bytes: Vec<u8>) -> Result<T, String> {let mut builder = self.request(reqwest::Method::POST, path).body(bytes);for (name, value) in headers {builder = builder.header(*name, value);}- self.send(builder).map(|_| ())+ self.send(builder)?.json().map_err(|error| format!("Worker {}: {error}", self.name))}fn bytes(&self, path: &str) -> Result<Vec<u8>, String> {