Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Add standalone audio references to the native backend

Bitti07499052c7af
diff --git a/src-tauri/core/src/models.rs b/src-tauri/core/src/models.rs
index d34a8c6..7abef78 100644
--- a/src-tauri/core/src/models.rs
+++ b/src-tauri/core/src/models.rs
@@ -7,6 +7,7 @@ pub struct ModelCapabilities {
pub min_steps: i32,
pub max_image_references: usize,
pub max_video_references: usize,
+ pub max_audio_references: usize,
pub max_references: usize,
pub continuation_overlap: i32,
pub frame_stride: i32,
@@ -17,6 +18,7 @@ pub const H3_CAPABILITIES: ModelCapabilities = ModelCapabilities {
min_steps: 2,
max_image_references: 9,
max_video_references: 3,
+ max_audio_references: 3,
max_references: 12,
continuation_overlap: 22,
frame_stride: 17,
diff --git a/src-tauri/core/src/slopfab/config.rs b/src-tauri/core/src/slopfab/config.rs
index 6bf77ca..6cd58a6 100644
--- a/src-tauri/core/src/slopfab/config.rs
+++ b/src-tauri/core/src/slopfab/config.rs
@@ -111,8 +111,9 @@ impl Configuration {
|| !request.refmods.is_empty()
|| !request.reference_paths.is_empty()
|| request.reference_video_ids.len() != expected
+ || !request.reference_audio_ids.is_empty()
{
- return Err("Extend/Bridge requires exactly one/two video references without frame anchors, Animate, continuation or refmods.".into());
+ return Err("Extend/Bridge requires exactly one/two video references without frame anchors, sound references, Animate, continuation or refmods.".into());
}
}
if self.animate && self.motion_cache {
@@ -130,8 +131,8 @@ impl Configuration {
"Animate requires exactly one repainted frame of the driving scene.".into(),
);
}
- if request.continuation_path.is_some() || !request.refmods.is_empty() {
- return Err("Animate does not support scene continuation or refmods.".into());
+ if request.continuation_path.is_some() || !request.refmods.is_empty() || !request.reference_audio_ids.is_empty() {
+ return Err("Animate does not support scene continuation, sound references or refmods.".into());
}
} else if request.prompt.trim().is_empty() {
return Err("Generation prompt cannot be empty.".into());
diff --git a/src-tauri/core/src/slopfab/ffi/mod.rs b/src-tauri/core/src/slopfab/ffi/mod.rs
index 954b3d9..a959b78 100644
--- a/src-tauri/core/src/slopfab/ffi/mod.rs
+++ b/src-tauri/core/src/slopfab/ffi/mod.rs
@@ -160,6 +160,17 @@ impl Api {
self.check(r)?;
self.inner.add_reference(r.pointer.as_ptr(), path)
}
+ pub fn add_reference_audio(
+ &self,
+ r: &RequestHandle,
+ samples: &[f32],
+ channels: i32,
+ sample_rate: i32,
+ ) -> Result<(), String> {
+ self.check(r)?;
+ self.inner
+ .add_reference_audio(r.pointer.as_ptr(), samples, channels, sample_rate)
+ }
pub fn add_lora(&self, r: &RequestHandle, path: &Path, strength: f32) -> Result<(), String> {
self.check(r)?;
self.inner.add_lora(r.pointer.as_ptr(), path, strength)
diff --git a/src-tauri/core/src/slopfab/ffi/raw/mod.rs b/src-tauri/core/src/slopfab/ffi/raw/mod.rs
index db63b2a..e987067 100644
--- a/src-tauri/core/src/slopfab/ffi/raw/mod.rs
+++ b/src-tauri/core/src/slopfab/ffi/raw/mod.rs
@@ -57,6 +57,7 @@ pub struct Output {
pub steps_computed: i32,
pub steps_skipped: i32,
}
+type AddReferenceAudio = unsafe extern "C" fn(*mut Request, *const f32, usize, i32, i32) -> i32;
pub type ProgressFn = Option<unsafe extern "C" fn(*const Progress, *mut c_void)>;
type SetMotionCache =
unsafe extern "C" fn(*mut Request, i32, f32, f32, i32, i32, f32, f32, i32, i32) -> i32;
@@ -92,6 +93,7 @@ pub struct Api {
prepare_lora_grid: Option<unsafe extern "C" fn(*const c_char, i32, i32) -> i32>,
add_refmod: Option<unsafe extern "C" fn(*mut Request, *const c_char, f32, i32) -> i32>,
add_reference: unsafe extern "C" fn(*mut Request, *const c_char) -> i32,
+ add_reference_audio: Option<AddReferenceAudio>,
set_attention: unsafe extern "C" fn(*mut Request, *const c_char) -> i32,
set_motion_cache: Option<SetMotionCache>,
set_inference_backend: unsafe extern "C" fn(*mut Request, i32) -> i32,
@@ -238,6 +240,10 @@ impl Api {
)
.ok()
.map(|symbol| *symbol),
+ add_reference_audio: library
+ .get::<AddReferenceAudio>(b"slopfab_request_add_reference_audio_f32\0")
+ .ok()
+ .map(|symbol| *symbol),
set_attention: symbol!(
"slopfab_request_set_attention",
unsafe extern "C" fn(*mut Request, *const c_char) -> i32
@@ -508,6 +514,16 @@ impl Api {
let path = path_cstring(path)?;
self.error(unsafe { add(r, path.as_ptr(), strength, copies) })
}
+ pub fn add_reference_audio(
+ &self,
+ r: *mut Request,
+ samples: &[f32],
+ channels: i32,
+ sample_rate: i32,
+ ) -> Result<(), String> {
+ let add = self.add_reference_audio.ok_or("This slopfab.dll does not support audio references. Update the runtime.")?;
+ self.error(unsafe { add(r, samples.as_ptr(), samples.len(), channels, sample_rate) })
+ }
pub fn resolve(&self, request: *mut Request) -> Result<Plan, String> {
let mut value = Plan {
canvas_width: 0,
diff --git a/src-tauri/core/src/slopfab/h3.rs b/src-tauri/core/src/slopfab/h3.rs
index 05b07dc..00dfe70 100644
--- a/src-tauri/core/src/slopfab/h3.rs
+++ b/src-tauri/core/src/slopfab/h3.rs
@@ -132,6 +132,12 @@ pub(super) fn configure_request(
.attach(handle, &configuration.dll_path)?;
}
}
+ // Standalone audio follows the videos, so slopfab numbers it after every
+ // video soundtrack. The prompt compiler numbers <Audio N> the same way.
+ for id in &request.reference_audio_ids {
+ let audio = references.reference_audio(id)?;
+ api.add_reference_audio(handle, &audio.samples, audio.channels, audio.sample_rate)?;
+ }
for refmod in &request.refmods {
if refmod.path.trim().is_empty()
|| !refmod.strength.is_finite()
diff --git a/src-tauri/core/src/slopfab/planning.rs b/src-tauri/core/src/slopfab/planning.rs
index 7866e0b..4f1964d 100644
--- a/src-tauri/core/src/slopfab/planning.rs
+++ b/src-tauri/core/src/slopfab/planning.rs
@@ -69,12 +69,13 @@ pub(super) fn validate_generation_controls(request: &GenerationRequest) -> Resul
let limits = crate::models::H3_CAPABILITIES;
if request.reference_paths.len() > limits.max_image_references
|| request.reference_video_ids.len() > limits.max_video_references
+ || request.reference_audio_ids.len() > limits.max_audio_references
|| request.reference_count() > limits.max_references
{
- return Err("Use at most nine images and three videos per generation.".into());
+ return Err("Use at most nine images, three videos and three sounds per generation.".into());
}
- if request.still_image && !request.reference_video_ids.is_empty() {
- return Err("Video references cannot be used for still-image generation.".into());
+ if request.still_image && (!request.reference_video_ids.is_empty() || !request.reference_audio_ids.is_empty()) {
+ return Err("Video and sound references cannot be used for still-image generation.".into());
}
if request.steps < limits.min_steps {
return Err("Generation step count must be at least 2.".into());
diff --git a/src-tauri/core/src/slopfab/references.rs b/src-tauri/core/src/slopfab/references.rs
index 50dd054..c675bbf 100644
--- a/src-tauri/core/src/slopfab/references.rs
+++ b/src-tauri/core/src/slopfab/references.rs
@@ -10,9 +10,18 @@ use std::{
#[derive(Clone, Default)]
pub struct ReferenceVideos {
pub(super) videos: Arc<Mutex<HashMap<String, ffi::ReferenceVideoHandle>>>,
+ pub(super) audios: Arc<Mutex<HashMap<String, Arc<ReferenceAudio>>>>,
next: Arc<std::sync::atomic::AtomicU64>,
}
+/// Decoded standalone reference audio. The C API copies the samples when a
+/// request adds them, so the registry only keeps them until release.
+pub struct ReferenceAudio {
+ pub samples: Vec<f32>,
+ pub channels: i32,
+ pub sample_rate: i32,
+}
+
impl ReferenceVideos {
pub fn create_reference_video(
&self,
@@ -73,6 +82,58 @@ impl ReferenceVideos {
.set_audio(&samples, channels, sample_rate)
}
+ pub fn create_reference_audio(
+ &self,
+ bytes: &[u8],
+ channels: i32,
+ sample_rate: i32,
+ ) -> Result<String, String> {
+ if !(1..=2).contains(&channels) || sample_rate <= 0 {
+ return Err("Reference audio must be mono or stereo with a positive sample rate.".into());
+ }
+ let samples = float_samples(bytes)?;
+ if samples.is_empty() || samples.len() % channels as usize != 0 {
+ return Err("Reference audio must contain complete sample frames.".into());
+ }
+ let mut audios = self
+ .audios
+ .lock()
+ .map_err(|_| "Reference audio lock failed.")?;
+ if audios.len() >= 3 {
+ return Err("At most three audio references can be prepared at once.".into());
+ }
+ let id = self.next.fetch_add(1, Ordering::Relaxed).to_string();
+ audios.insert(
+ id.clone(),
+ Arc::new(ReferenceAudio {
+ samples,
+ channels,
+ sample_rate,
+ }),
+ );
+ Ok(id)
+ }
+
+ pub fn release_reference_audios(&self, ids: &[String]) -> Result<(), String> {
+ let mut audios = self
+ .audios
+ .lock()
+ .map_err(|_| "Reference audio lock failed.")?;
+ for id in ids {
+ audios.remove(id);
+ }
+ Ok(())
+ }
+
+ pub(super) fn reference_audio(&self, id: &str) -> Result<Arc<ReferenceAudio>, String> {
+ self.audios
+ .lock()
+ .map_err(|_| "Reference audio lock failed.")?
+ .get(id)
+ .cloned()
+ .ok_or_else(|| "The prepared reference audio no longer exists. Please retry generation.".into())
+ }
+
pub fn release_reference_videos(&self, ids: &[String]) -> Result<(), String> {
let mut videos = self
.videos
@@ -84,3 +145,13 @@ impl ReferenceVideos {
Ok(())
}
}
+
+fn float_samples(bytes: &[u8]) -> Result<Vec<f32>, String> {
+ if bytes.len() % 4 != 0 {
+ return Err("Reference audio must contain complete float32 samples.".into());
+ }
+ Ok(bytes
+ .chunks_exact(4)
+ .map(|chunk| f32::from_le_bytes(chunk.try_into().unwrap()))
+ .collect())
+}
diff --git a/src-tauri/core/src/slopfab/tests.rs b/src-tauri/core/src/slopfab/tests.rs
index 626fb8e..f2e2178 100644
--- a/src-tauri/core/src/slopfab/tests.rs
+++ b/src-tauri/core/src/slopfab/tests.rs
@@ -865,6 +865,63 @@ fn vulkan_video_references_prepare_and_attach_for_planning_and_generation() {
result.unwrap();
}
+#[test]
+fn standalone_audio_references_attach_after_videos_and_need_visual_media() {
+ let references = ReferenceVideos::default();
+ let settings = BTreeMap::new();
+ let video = references.create_reference_video(2.0, &settings).unwrap();
+ let audio = references
+ .create_reference_audio(&vec![0; 32_000 * 2 * 4], 1, 32_000)
+ .unwrap();
+ let result = (|| -> Result<(), String> {
+ references.append_reference_video(&video, &vec![127; 64 * 64 * 4], 64, 64, 0.0)?;
+ let mut request = GenerationRequest {
+ job_id: "audio-reference-test".into(),
+ prompt: "A scene using <Video 1> and <Audio 1>.".into(),
+ frames: 120,
+ steps: 12,
+ seed: 1,
+ canvas_width: 736,
+ canvas_height: 416,
+ reference_video_ids: vec![video.clone()],
+ reference_audio_ids: vec![audio.clone()],
+ ..Default::default()
+ };
+ let configuration = Configuration::from_settings(&settings);
+ let api = ffi::Api::load(&configuration.dll_path)?;
+ let plan = |request: &GenerationRequest| -> Result<String, String> {
+ let handle = RequestHandle::new(&api)?;
+ configure_request(&api, &handle, request, &configuration, ComputePlatform::Vulkan, RequestPurpose::Plan, &references)?;
+ api.resolve(&handle)?;
+ api.describe(&handle)
+ };
+ let description = plan(&request)?;
+ assert!(description.contains("reference audios 1"));
+ request.reference_video_ids.clear();
+ assert!(plan(&request).unwrap_err().contains("image or video reference"));
+ Ok(())
+ })();
+ references.release_reference_videos(&[video]).unwrap();
+ references.release_reference_audios(&[audio.clone()]).unwrap();
+ assert!(references.reference_audio(&audio).is_err());
+ result.unwrap();
+}
+
+#[test]
+fn reference_audio_registry_validates_pcm_layout() {
+ let references = ReferenceVideos::default();
+ assert!(references.create_reference_audio(&[0; 6], 1, 32_000).is_err());
+ assert!(references.create_reference_audio(&[0; 12], 2, 32_000).is_err());
+ assert!(references.create_reference_audio(&[0; 8], 3, 32_000).is_err());
+ assert!(references.create_reference_audio(&[0; 8], 1, 0).is_err());
+ let ids: Vec<String> = (0..3)
+ .map(|_| references.create_reference_audio(&[0; 8], 2, 48_000).unwrap())
+ .collect();
+ assert!(references.create_reference_audio(&[0; 8], 2, 48_000).is_err());
+ references.release_reference_audios(&ids).unwrap();
+ assert!(references.create_reference_audio(&[0; 8], 2, 48_000).is_ok());
+}
+
#[test]
fn video_reference_c_api_copies_inputs_and_retains_attached_snapshot() {
let path = default_dll_path();
diff --git a/src-tauri/core/src/slopfab/types.rs b/src-tauri/core/src/slopfab/types.rs
index 2e58b7e..8a9a931 100644
--- a/src-tauri/core/src/slopfab/types.rs
+++ b/src-tauri/core/src/slopfab/types.rs
@@ -63,6 +63,8 @@ pub struct GenerationRequest {
#[serde(default)]
pub reference_video_ids: Vec<String>,
#[serde(default)]
+ pub reference_audio_ids: Vec<String>,
+ #[serde(default)]
pub refmods: Vec<RefmodInput>,
#[serde(default)]
pub continuation_relative_path: Option<String>,
@@ -102,7 +104,7 @@ pub struct RefmodInput {
impl GenerationRequest {
pub fn reference_count(&self) -> usize {
- self.reference_paths.len() + self.reference_video_ids.len()
+ self.reference_paths.len() + self.reference_video_ids.len() + self.reference_audio_ids.len()
}
pub fn refmod_count(&self) -> usize {
self.refmods.iter().filter(|refmod| refmod.strength > 0.0).count()
diff --git a/src-tauri/core/src/worker/protocol.rs b/src-tauri/core/src/worker/protocol.rs
index bac0071..ef298c5 100644
--- a/src-tauri/core/src/worker/protocol.rs
+++ b/src-tauri/core/src/worker/protocol.rs
@@ -9,7 +9,7 @@ use std::collections::BTreeMap;
pub const SERVICE_TYPE: &str = "_slopus-worker._tcp.local.";
/// Bumped on any incompatible change to the routes or bodies below.
-pub const PROTOCOL_VERSION: u32 = 1;
+pub const PROTOCOL_VERSION: u32 = 2;
pub const DEFAULT_PORT: u16 = 47321;
pub const TOKEN_HEADER: &str = "x-slopus-token";
diff --git a/src-tauri/core/src/worker/server.rs b/src-tauri/core/src/worker/server.rs
index 9096e99..1b5b36f 100644
--- a/src-tauri/core/src/worker/server.rs
+++ b/src-tauri/core/src/worker/server.rs
@@ -469,6 +469,18 @@ impl Worker {
self.runtime.references.set_reference_video_audio(id, &samples, channels, rate)?;
json(&true)
}
+ (Method::Post, ["reference-audios"]) => {
+ let channels = parsed_header(request, "x-reference-channels")?;
+ let rate = parsed_header(request, "x-reference-rate")?;
+ let mut samples = Vec::new();
+ request.as_reader().take(1024 * 1024 * 1024).read_to_end(&mut samples).map_err(|error| error.to_string())?;
+ json(&self.runtime.references.create_reference_audio(&samples, channels, rate)?)
+ }
+ (Method::Post, ["reference-audios", "release"]) => {
+ let ids: Vec<String> = body(request)?;
+ self.runtime.references.release_reference_audios(&ids)?;
+ json(&true)
+ }
_ => Err(HttpError(404, "Unknown worker route.".into())),
}
}
diff --git a/src-tauri/src/agent/tests_transport.rs b/src-tauri/src/agent/tests_transport.rs
index 0d8f2c5..21c3feb 100644
--- a/src-tauri/src/agent/tests_transport.rs
+++ b/src-tauri/src/agent/tests_transport.rs
@@ -38,6 +38,7 @@ fn another_model_supplies_its_own_policy_and_vocabulary() {
min_steps: 1,
max_image_references: 1,
max_video_references: 0,
+ max_audio_references: 0,
max_references: 1,
continuation_overlap: 0,
frame_stride: 1,
diff --git a/src-tauri/src/app.rs b/src-tauri/src/app.rs
index b0724af..b47a7f3 100644
--- a/src-tauri/src/app.rs
+++ b/src-tauri/src/app.rs
@@ -203,6 +203,8 @@ pub fn run() {
commands::generation::append_reference_video,
commands::generation::set_reference_video_audio,
commands::generation::release_reference_videos,
+ commands::generation::create_reference_audio,
+ commands::generation::release_reference_audios,
commands::media::import_media_files,
commands::media::read_project_file,
commands::media::read_external_media_file,
diff --git a/src-tauri/src/commands/generation.rs b/src-tauri/src/commands/generation.rs
index 17b808f..50f2220 100644
--- a/src-tauri/src/commands/generation.rs
+++ b/src-tauri/src/commands/generation.rs
@@ -68,15 +68,17 @@ pub(crate) async fn append_reference_video(
let bytes = bytes.clone();
let (runtime, workers) = (state.inner().clone(), workers.inner().clone());
blocking(move || match workers.active()? {
- Some(worker) => worker.post_bytes(
- &format!("/v1/reference-videos/{id}/frames"),
- &[
- ("x-reference-width", width.to_string()),
- ("x-reference-height", height.to_string()),
- ("x-reference-time", timestamp.to_string()),
- ],
- bytes,
- ),
+ Some(worker) => worker
+ .post_bytes::<bool>(
+ &format!("/v1/reference-videos/{id}/frames"),
+ &[
+ ("x-reference-width", width.to_string()),
+ ("x-reference-height", height.to_string()),
+ ("x-reference-time", timestamp.to_string()),
+ ],
+ bytes,
+ )
+ .map(|_| ()),
None => runtime
.references
.append_reference_video(&id, &bytes, width, height, timestamp),
@@ -102,9 +104,60 @@ pub(crate) async fn set_reference_video_audio(
.map_err(|_| "Invalid sample rate.")?;
let bytes = bytes.clone();
let (runtime, workers) = (state.inner().clone(), workers.inner().clone());
+ blocking(move || match workers.active()? {
+ Some(worker) => worker
+ .post_bytes::<bool>(
+ &format!("/v1/reference-videos/{id}/audio"),
+ &[
+ ("x-reference-channels", channels.to_string()),
+ ("x-reference-rate", rate.to_string()),
+ ],
+ bytes,
+ )
+ .map(|_| ()),
+ None => runtime
+ .references
+ .set_reference_video_audio(&id, &bytes, channels, rate),
+ })
+ .await
+}
+
+#[tauri::command]
+pub(crate) async fn release_reference_videos(
+ state: tauri::State<'_, slopfab::SlopfabRuntime>,
+ workers: tauri::State<'_, Workers>,
+ ids: Vec<String>,
+) -> Result<(), String> {
+ let (runtime, workers) = (state.inner().clone(), workers.inner().clone());
+ blocking(move || match workers.active()? {
+ Some(worker) => worker
+ .post::<bool>("/v1/reference-videos/release", &ids)
+ .map(|_| ()),
+ None => runtime.references.release_reference_videos(&ids),
+ })
+ .await
+}
+
+#[tauri::command]
+pub(crate) async fn create_reference_audio(
+ state: tauri::State<'_, slopfab::SlopfabRuntime>,
+ workers: tauri::State<'_, Workers>,
+ request: tauri::ipc::Request<'_>,
+) -> Result<String, String> {
+ let tauri::ipc::InvokeBody::Raw(bytes) = request.body() else {
+ return Err("Reference audio requires a binary body.".into());
+ };
+ let channels: i32 = reference_video_header(&request, "x-reference-channels")?
+ .parse()
+ .map_err(|_| "Invalid audio channels.")?;
+ let rate: i32 = reference_video_header(&request, "x-reference-rate")?
+ .parse()
+ .map_err(|_| "Invalid sample rate.")?;
+ let bytes = bytes.clone();
+ let (runtime, workers) = (state.inner().clone(), workers.inner().clone());
blocking(move || match workers.active()? {
Some(worker) => worker.post_bytes(
- &format!("/v1/reference-videos/{id}/audio"),
+ "/v1/reference-audios",
&[
("x-reference-channels", channels.to_string()),
("x-reference-rate", rate.to_string()),
@@ -113,13 +166,13 @@ pub(crate) async fn set_reference_video_audio(
),
None => runtime
.references
- .set_reference_video_audio(&id, &bytes, channels, rate),
+ .create_reference_audio(&bytes, channels, rate),
})
.await
}
#[tauri::command]
-pub(crate) async fn release_reference_videos(
+pub(crate) async fn release_reference_audios(
state: tauri::State<'_, slopfab::SlopfabRuntime>,
workers: tauri::State<'_, Workers>,
ids: Vec<String>,
@@ -127,9 +180,9 @@ pub(crate) async fn release_reference_videos(
let (runtime, workers) = (state.inner().clone(), workers.inner().clone());
blocking(move || match workers.active()? {
Some(worker) => worker
- .post::<bool>("/v1/reference-videos/release", &ids)
+ .post::<bool>("/v1/reference-audios/release", &ids)
.map(|_| ()),
- None => runtime.references.release_reference_videos(&ids),
+ None => runtime.references.release_reference_audios(&ids),
})
.await
}
diff --git a/src-tauri/src/worker/client.rs b/src-tauri/src/worker/client.rs
index 34a786a..81ce151 100644
--- a/src-tauri/src/worker/client.rs
+++ b/src-tauri/src/worker/client.rs
@@ -177,12 +177,12 @@ impl Connection {
self.send(self.request(reqwest::Method::POST, path).json(body))?.json().map_err(|error| format!("Worker {}: {error}", self.name))
}
- pub(crate) fn post_bytes(&self, path: &str, headers: &[(&str, String)], bytes: Vec<u8>) -> Result<(), String> {
+ pub(crate) fn post_bytes<T: DeserializeOwned>(&self, path: &str, headers: &[(&str, String)], bytes: Vec<u8>) -> Result<T, String> {
let mut builder = self.request(reqwest::Method::POST, path).body(bytes);
for (name, value) in headers {
builder = builder.header(*name, value);
}
- self.send(builder).map(|_| ())
+ self.send(builder)?.json().map_err(|error| format!("Worker {}: {error}", self.name))
}
fn bytes(&self, path: &str) -> Result<Vec<u8>, String> {