AI media harness for planning, generating, and editing locally
TypeScript
mod handles;
mod raw;
#[cfg(test)]
mod tests;
#[cfg(test)]
mod lcs_tests;
pub use handles::{FinishedGeneration, GenerationHandle, ProgressSink, Output};
pub use handles::PreviewSink;
pub use raw::{cuda_device_names, gpu_devices, has_cuda_device, Plan, Progress};
pub use raw::VideoDecodeTimings;
use std::{
path::{Path, PathBuf},
ptr::NonNull,
sync::Arc,
};
#[derive(Clone)]
pub struct Api {
inner: Arc<raw::Api>,
path: PathBuf,
}
impl Api {
pub fn require_saved_latent_upscale(&self) -> Result<(), String> { self.inner.require_saved_latent_upscale() }
pub fn load(path: &Path) -> Result<Self, String> {
let inner = raw::Api::load(path)?;
inner.version()?;
Ok(Self {
inner: Arc::new(inner),
path: path.to_owned(),
})
}
pub fn require_lcs(&self) -> Result<(), String> { self.inner.require_lcs() }
pub fn require_output_tone(&self) -> Result<(), String> { self.inner.require_output_tone() }
pub fn set_lcs(&self, r: &RequestHandle, path: &Path, settings: &str) -> Result<(), String> {
self.check(r)?; self.inner.set_lcs(r.pointer.as_ptr(), path, settings)
}
pub fn set_lcs_mask(&self, r: &RequestHandle, mask: &[f32]) -> Result<(), String> {
self.check(r)?; self.inner.set_lcs_mask(r.pointer.as_ptr(), mask)
}
pub fn calibrate_lcs(&self, r: &RequestHandle, path: &Path, progress: &mut dyn FnMut(i32, i32) -> bool) -> Result<(), String> {
self.check(r)?; self.inner.calibrate_lcs(r.pointer.as_ptr(), path, progress)
}
pub fn version(&self) -> Result<String, String> {
self.inner.version()
}
pub fn set_cuda_version(&self, v: &str) -> Result<(), String> {
self.inner.set_cuda_version(v)
}
pub fn cuda_loaded_major(&self) -> Result<i32, String> {
self.inner.cuda_loaded_major()
}
pub fn clear_reused_models(&self) -> Result<(), String> {
self.inner.clear_reused_models()
}
pub fn prepare_lora_grid(
&self,
path: &Path,
width: i32,
allow_download: bool,
) -> Result<(), String> {
self.inner.prepare_lora_grid(path, width, allow_download)
}
#[cfg(test)]
pub fn disable_outpainting_for_test(&mut self) {
Arc::get_mut(&mut self.inner)
.expect("configure API before creating handles")
.disable_outpainting_for_test();
}
#[cfg(test)]
pub fn disable_lora_preparation_for_test(&mut self) {
Arc::get_mut(&mut self.inner)
.expect("configure API before creating handles")
.disable_lora_preparation_for_test();
}
#[cfg(test)]
pub fn disable_animate_for_test(&mut self) {
Arc::get_mut(&mut self.inner)
.expect("configure API before creating handles")
.disable_animate_for_test();
}
#[cfg(test)]
pub fn disable_refmod_export_for_test(&mut self) {
Arc::get_mut(&mut self.inner)
.expect("configure API before creating handles")
.disable_refmod_export_for_test();
}
#[cfg(test)]
pub fn disable_motion_cache_for_test(&mut self) {
Arc::get_mut(&mut self.inner)
.expect("configure API before creating handles")
.disable_motion_cache_for_test();
}
fn check(&self, r: &RequestHandle) -> Result<(), String> {
if Arc::ptr_eq(&self.inner, &r.api.inner) {
Ok(())
} else {
Err("Request belongs to a different runtime instance.".into())
}
}
pub fn set_prompt(&self, r: &RequestHandle, v: &str) -> Result<(), String> {
self.check(r)?;
self.inner.set_prompt(r.pointer.as_ptr(), v)
}
pub fn set_animate(&self, r: &RequestHandle, preserve_audio: bool) -> Result<(), String> {
self.check(r)?;
self.inner.set_animate(r.pointer.as_ptr(), preserve_audio)
}
pub fn set_prompt_embedding(&self, r: &RequestHandle, path: &Path) -> Result<(), String> {
self.check(r)?;
self.inner.set_prompt_embedding(r.pointer.as_ptr(), path)
}
pub fn set_resolution(&self, r: &RequestHandle, w: i32, h: i32) -> Result<(), String> {
self.check(r)?;
self.inner.set_resolution(r.pointer.as_ptr(), w, h)
}
pub fn set_frames(&self, r: &RequestHandle, v: i32) -> Result<(), String> {
self.check(r)?;
self.inner.set_frames(r.pointer.as_ptr(), v)
}
pub fn set_video_transition(&self, r: &RequestHandle, mode: i32) -> Result<(), String> {
self.check(r)?;
self.inner.set_video_transition(r.pointer.as_ptr(), mode)
}
pub fn set_still_image(&self, r: &RequestHandle) -> Result<(), String> {
self.check(r)?;
self.inner.set_still_image(r.pointer.as_ptr())
}
#[cfg(test)]
pub fn disable_image_mask_for_test(&mut self) {
Arc::get_mut(&mut self.inner).expect("configure API before creating handles").disable_image_mask_for_test();
}
pub fn set_image_edit_mask(&self, r: &RequestHandle, mask: &[u8], width: i32, height: i32) -> Result<(), String> {
self.check(r)?;
self.inner.set_image_edit_mask(r.pointer.as_ptr(), mask, width, height)
}
pub fn set_image_edit_path(&self, r: &RequestHandle, path: &Path, bounds: [i32; 4], strength: f32, feather: i32) -> Result<(), String> {
self.check(r)?;
self.inner.set_image_edit_path(r.pointer.as_ptr(), path, bounds, strength, feather)
}
pub fn set_image_edit_rgb(&self, r: &RequestHandle, pixels: &[u8], width: i32, height: i32, bounds: [i32; 4], strength: f32, feather: i32) -> Result<(), String> {
self.check(r)?;
self.inner.set_image_edit_rgb(r.pointer.as_ptr(), pixels, width, height, bounds, strength, feather)
}
pub fn set_outpaint_blend_overlap(&self, r: &RequestHandle, value: i32) -> Result<(), String> {
self.check(r)?;
self.inner.set_outpaint_blend_overlap(r.pointer.as_ptr(), value)
}
#[cfg(test)]
pub fn disable_outpaint_refinement_for_test(&mut self) {
Arc::get_mut(&mut self.inner).expect("configure API before creating handles").disable_outpaint_refinement_for_test();
}
pub fn set_outpaint_langevin_steps(&self, r: &RequestHandle, value: i32) -> Result<(), String> {
self.check(r)?;
self.inner.set_outpaint_langevin_steps(r.pointer.as_ptr(), value)
}
pub fn set_image_edit_invert_mask(&self, r: &RequestHandle) -> Result<(), String> {
self.check(r)?;
self.inner.set_image_edit_invert_mask(r.pointer.as_ptr())
}
pub fn set_save_latents(&self, r: &RequestHandle, path: &Path) -> Result<(), String> {
self.check(r)?;
self.inner.set_save_latents(r.pointer.as_ptr(), path)
}
pub fn set_latent_bridge_files(&self, r: &RequestHandle, left: &Path, right: &Path, left_margin: i32, right_margin: i32, context: i32) -> Result<(), String> {
self.check(r)?;
self.inner.set_latent_bridge_files(r.pointer.as_ptr(), left, right, left_margin, right_margin, context)
}
pub fn require_end_anchor(&self) -> Result<(), String> { self.inner.require_end_anchor() }
pub fn set_keystone_file(&self, r: &RequestHandle, path: &Path, lock_after: i32) -> Result<(), String> {
self.check(r)?;
self.inner.set_keystone_file(r.pointer.as_ptr(), path, lock_after)
}
pub fn require_continuation_source_init(&self) -> Result<(), String> { self.inner.require_continuation_source_init() }
pub fn set_continuation_source_init(&self, r: &RequestHandle, noise: f32) -> Result<(), String> {
self.check(r)?;
self.inner.set_continuation_source_init(r.pointer.as_ptr(), noise)
}
pub fn set_end_anchor_file(&self, r: &RequestHandle, path: &Path) -> Result<(), String> {
self.check(r)?;
self.inner.set_end_anchor_file(r.pointer.as_ptr(), path)
}
pub fn set_continuation_file(
&self,
r: &RequestHandle,
path: &Path,
overlap: i32,
) -> Result<(), String> {
self.check(r)?;
self.inner
.set_continuation_file(r.pointer.as_ptr(), path, overlap)
}
pub fn set_steps(&self, r: &RequestHandle, v: i32) -> Result<(), String> {
self.check(r)?;
self.inner.set_steps(r.pointer.as_ptr(), v)
}
pub fn set_latent_upscaler(&self, r: &RequestHandle, path: Option<&Path>) -> Result<(), String> {
self.check(r)?;
self.inner.set_latent_upscaler(r.pointer.as_ptr(), path)
}
#[cfg(test)]
pub fn disable_latent_upscaler_for_test(&mut self) {
Arc::get_mut(&mut self.inner).expect("configure API before creating handles").disable_latent_upscaler_for_test();
}
pub fn set_audio_steps(&self, r: &RequestHandle, steps: i32) -> Result<(), String> {
self.check(r)?;
self.inner.set_audio_steps(r.pointer.as_ptr(), steps)
}
pub fn set_continuation_lock_overlap(&self, r: &RequestHandle, enabled: bool) -> Result<(), String> {
self.check(r)?;
self.inner.set_continuation_lock_overlap(r.pointer.as_ptr(), enabled)
}
pub fn set_continuation_regenerate_frames(&self, r: &RequestHandle, frames: i32) -> Result<(), String> {
self.check(r)?;
self.inner.set_continuation_regenerate_frames(r.pointer.as_ptr(), frames)
}
pub fn set_seed(&self, r: &RequestHandle, v: u64) -> Result<(), String> {
self.check(r)?;
self.inner.set_seed(r.pointer.as_ptr(), v)
}
pub fn set_sampling(&self, r: &RequestHandle, video: Option<f32>, audio: Option<f32>, sampler: Option<&str>, scheduler: Option<&str>) -> Result<(), String> {
self.check(r)?;
self.inner.set_sampling(r.pointer.as_ptr(), video, audio, sampler, scheduler)
}
pub fn set_dmad_sampling(&self, r: &RequestHandle) -> Result<(), String> {
self.check(r)?;
self.inner.set_dmad_sampling(r.pointer.as_ptr())
}
pub fn set_pika_sampling(&self, r: &RequestHandle) -> Result<(), String> {
self.check(r)?;
self.inner.set_pika_sampling(r.pointer.as_ptr())
}
pub fn set_verbose(&self, r: &RequestHandle, v: bool) -> Result<(), String> {
self.check(r)?;
self.inner.set_verbose(r.pointer.as_ptr(), v)
}
pub fn set_reuse_models(&self, r: &RequestHandle, v: bool) -> Result<(), String> {
self.check(r)?;
self.inner.set_reuse_models(r.pointer.as_ptr(), v)
}
pub fn set_attention(&self, r: &RequestHandle, v: &str) -> Result<(), String> {
self.check(r)?;
self.inner.set_attention(r.pointer.as_ptr(), v)
}
pub fn set_motion_cache(&self, r: &RequestHandle, enabled: bool) -> Result<(), String> {
self.check(r)?;
self.inner.set_motion_cache(r.pointer.as_ptr(), enabled)
}
pub fn set_inference_backend(&self, r: &RequestHandle, backend: i32) -> Result<(), String> {
self.check(r)?;
self.inner
.set_inference_backend(r.pointer.as_ptr(), backend)
}
pub fn set_model(&self, r: &RequestHandle, id: i32, path: &Path) -> Result<(), String> {
self.check(r)?;
self.inner.set_model(r.pointer.as_ptr(), id, path)
}
#[cfg(test)]
pub fn disable_image_guidance_for_test(&mut self) {
Arc::get_mut(&mut self.inner).expect("configure API before creating handles").disable_image_guidance_for_test();
}
pub fn set_image_guidance(&self, r: &RequestHandle, scale: f32, negative_prompt: &str) -> Result<(), String> {
self.check(r)?;
self.inner.set_image_guidance(r.pointer.as_ptr(), scale, negative_prompt, &[22])
}
pub fn set_krea_guidance(&self, r: &RequestHandle, scale: f32, negative_prompt: &str) -> Result<(), String> {
self.check(r)?;
self.inner.set_image_guidance(r.pointer.as_ptr(), scale, negative_prompt, &[])
}
pub fn set_music_settings(&self, r: &RequestHandle, settings: &str) -> Result<(), String> {
self.check(r)?; self.inner.set_music_settings(r.pointer.as_ptr(), settings)
}
pub fn set_vulkan_portable(&self, r: &RequestHandle) -> Result<(), String> {
self.check(r)?; self.inner.set_vulkan_portable(r.pointer.as_ptr())
}
pub fn set_model_family(&self, r: &RequestHandle, family: &str) -> Result<(), String> {
self.check(r)?;
self.inner.set_model_family(r.pointer.as_ptr(), family)
}
pub fn add_reference(&self, r: &RequestHandle, path: &Path) -> Result<(), String> {
self.check(r)?;
self.inner.add_reference(r.pointer.as_ptr(), path)
}
pub fn add_reference_audio(
&self,
r: &RequestHandle,
samples: &[f32],
channels: i32,
sample_rate: i32,
) -> Result<(), String> {
self.check(r)?;
self.inner
.add_reference_audio(r.pointer.as_ptr(), samples, channels, sample_rate)
}
pub fn add_lora(&self, r: &RequestHandle, path: &Path, strength: f32) -> Result<(), String> {
self.check(r)?;
self.inner.add_lora(r.pointer.as_ptr(), path, strength)
}
pub fn add_refmod(
&self,
r: &RequestHandle,
path: &Path,
strength: f32,
copies: i32,
) -> Result<(), String> {
self.check(r)?;
self.inner
.add_refmod(r.pointer.as_ptr(), path, strength, copies)
}
pub fn export_refmod(&self, r: &RequestHandle, output: &Path, name: &str, description: &str) -> Result<(), String> {
self.check(r)?;
self.inner.export_refmod(r.pointer.as_ptr(), output, name, description)
}
pub fn resolve(&self, request: &RequestHandle) -> Result<Plan, String> {
self.check(request)?;
self.inner.resolve(request.pointer.as_ptr())
}
pub fn describe(&self, request: &RequestHandle) -> Result<String, String> {
self.check(request)?;
self.inner.describe(request.pointer.as_ptr())
}
}
/// Owns its library; native requests cannot outlive the DLL or cross instances.
pub struct RequestHandle {
api: Api,
pointer: NonNull<raw::Request>,
}
impl RequestHandle {
pub fn new(api: &Api) -> Result<Self, String> {
let pointer = NonNull::new(api.inner.create_request()?).ok_or("Empty native request.")?;
Ok(Self {
api: api.clone(),
pointer,
})
}
pub fn start(self, callback: Option<ProgressSink>) -> Result<GenerationHandle, String> {
GenerationHandle::start(self, callback, None)
}
pub fn start_with_preview(self, callback: Option<ProgressSink>, preview: Option<(PathBuf, PreviewSink)>) -> Result<GenerationHandle, String> {
GenerationHandle::start(self, callback, preview)
}
}
impl Drop for RequestHandle {
fn drop(&mut self) {
self.api.inner.destroy_request(self.pointer.as_ptr());
}
}
pub struct ReferenceVideoHandle(raw::ReferenceVideoHandle);
impl ReferenceVideoHandle {
pub fn new(api: Api, duration: f64, path: PathBuf) -> Result<Self, String> {
if api.path != path {
return Err("Reference video runtime mismatch.".into());
}
raw::ReferenceVideoHandle::new(api.inner, duration, path).map(Self)
}
pub fn append(&mut self, bytes: &[u8], w: i32, h: i32, time: f64) -> Result<(), String> {
self.0.append(bytes, w, h, time)
}
pub fn set_audio(&mut self, samples: &[f32], channels: i32, rate: i32) -> Result<(), String> {
self.0.set_audio(samples, channels, rate)
}
pub fn has_audio(&self) -> bool {
self.0.has_audio()
}
pub fn attach(&self, request: &RequestHandle, path: &Path) -> Result<(), String> {
if request.api.path != path {
return Err("Reference video runtime mismatch.".into());
}
self.0.attach(request.pointer.as_ptr(), path)
}
}