Skip to content

AI media harness for planning, generating, and editing locally

TypeScript
15.5 KB
Raw Blame
mod handles;
mod raw;
#[cfg(test)]
mod tests;
#[cfg(test)]
mod lcs_tests;
pub use handles::{FinishedGeneration, GenerationHandle, ProgressSink, Output};
pub use handles::PreviewSink;
pub use raw::{cuda_device_names, gpu_devices, has_cuda_device, Plan, Progress};
pub use raw::VideoDecodeTimings;
use std::{
    path::{Path, PathBuf},
    ptr::NonNull,
    sync::Arc,
};

#[derive(Clone)]
pub struct Api {
    inner: Arc<raw::Api>,
    path: PathBuf,
}
impl Api {
    pub fn require_saved_latent_upscale(&self) -> Result<(), String> { self.inner.require_saved_latent_upscale() }
    pub fn load(path: &Path) -> Result<Self, String> {
        let inner = raw::Api::load(path)?;
        inner.version()?;
        Ok(Self {
            inner: Arc::new(inner),
            path: path.to_owned(),
        })
    }
    pub fn require_lcs(&self) -> Result<(), String> { self.inner.require_lcs() }
    pub fn require_output_tone(&self) -> Result<(), String> { self.inner.require_output_tone() }
    pub fn set_lcs(&self, r: &RequestHandle, path: &Path, settings: &str) -> Result<(), String> {
        self.check(r)?; self.inner.set_lcs(r.pointer.as_ptr(), path, settings)
    }
    pub fn set_lcs_mask(&self, r: &RequestHandle, mask: &[f32]) -> Result<(), String> {
        self.check(r)?; self.inner.set_lcs_mask(r.pointer.as_ptr(), mask)
    }
    pub fn calibrate_lcs(&self, r: &RequestHandle, path: &Path, progress: &mut dyn FnMut(i32, i32) -> bool) -> Result<(), String> {
        self.check(r)?; self.inner.calibrate_lcs(r.pointer.as_ptr(), path, progress)
    }
    pub fn version(&self) -> Result<String, String> {
        self.inner.version()
    }
    pub fn set_cuda_version(&self, v: &str) -> Result<(), String> {
        self.inner.set_cuda_version(v)
    }
    pub fn cuda_loaded_major(&self) -> Result<i32, String> {
        self.inner.cuda_loaded_major()
    }
    pub fn clear_reused_models(&self) -> Result<(), String> {
        self.inner.clear_reused_models()
    }
    pub fn prepare_lora_grid(
        &self,
        path: &Path,
        width: i32,
        allow_download: bool,
    ) -> Result<(), String> {
        self.inner.prepare_lora_grid(path, width, allow_download)
    }
    #[cfg(test)]
    pub fn disable_outpainting_for_test(&mut self) {
        Arc::get_mut(&mut self.inner)
            .expect("configure API before creating handles")
            .disable_outpainting_for_test();
    }
    #[cfg(test)]
    pub fn disable_lora_preparation_for_test(&mut self) {
        Arc::get_mut(&mut self.inner)
            .expect("configure API before creating handles")
            .disable_lora_preparation_for_test();
    }
    #[cfg(test)]
    pub fn disable_animate_for_test(&mut self) {
        Arc::get_mut(&mut self.inner)
            .expect("configure API before creating handles")
            .disable_animate_for_test();
    }
    #[cfg(test)]
    pub fn disable_refmod_export_for_test(&mut self) {
        Arc::get_mut(&mut self.inner)
            .expect("configure API before creating handles")
            .disable_refmod_export_for_test();
    }
    #[cfg(test)]
    pub fn disable_motion_cache_for_test(&mut self) {
        Arc::get_mut(&mut self.inner)
            .expect("configure API before creating handles")
            .disable_motion_cache_for_test();
    }
    fn check(&self, r: &RequestHandle) -> Result<(), String> {
        if Arc::ptr_eq(&self.inner, &r.api.inner) {
            Ok(())
        } else {
            Err("Request belongs to a different runtime instance.".into())
        }
    }
    pub fn set_prompt(&self, r: &RequestHandle, v: &str) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_prompt(r.pointer.as_ptr(), v)
    }
    pub fn set_animate(&self, r: &RequestHandle, preserve_audio: bool) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_animate(r.pointer.as_ptr(), preserve_audio)
    }
    pub fn set_prompt_embedding(&self, r: &RequestHandle, path: &Path) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_prompt_embedding(r.pointer.as_ptr(), path)
    }
    pub fn set_resolution(&self, r: &RequestHandle, w: i32, h: i32) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_resolution(r.pointer.as_ptr(), w, h)
    }
    pub fn set_frames(&self, r: &RequestHandle, v: i32) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_frames(r.pointer.as_ptr(), v)
    }
    pub fn set_video_transition(&self, r: &RequestHandle, mode: i32) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_video_transition(r.pointer.as_ptr(), mode)
    }
    pub fn set_still_image(&self, r: &RequestHandle) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_still_image(r.pointer.as_ptr())
    }
    #[cfg(test)]
    pub fn disable_image_mask_for_test(&mut self) {
        Arc::get_mut(&mut self.inner).expect("configure API before creating handles").disable_image_mask_for_test();
    }
    pub fn set_image_edit_mask(&self, r: &RequestHandle, mask: &[u8], width: i32, height: i32) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_image_edit_mask(r.pointer.as_ptr(), mask, width, height)
    }
    pub fn set_image_edit_path(&self, r: &RequestHandle, path: &Path, bounds: [i32; 4], strength: f32, feather: i32) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_image_edit_path(r.pointer.as_ptr(), path, bounds, strength, feather)
    }
    pub fn set_image_edit_rgb(&self, r: &RequestHandle, pixels: &[u8], width: i32, height: i32, bounds: [i32; 4], strength: f32, feather: i32) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_image_edit_rgb(r.pointer.as_ptr(), pixels, width, height, bounds, strength, feather)
    }
    pub fn set_outpaint_blend_overlap(&self, r: &RequestHandle, value: i32) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_outpaint_blend_overlap(r.pointer.as_ptr(), value)
    }
    #[cfg(test)]
    pub fn disable_outpaint_refinement_for_test(&mut self) {
        Arc::get_mut(&mut self.inner).expect("configure API before creating handles").disable_outpaint_refinement_for_test();
    }
    pub fn set_outpaint_langevin_steps(&self, r: &RequestHandle, value: i32) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_outpaint_langevin_steps(r.pointer.as_ptr(), value)
    }
    pub fn set_image_edit_invert_mask(&self, r: &RequestHandle) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_image_edit_invert_mask(r.pointer.as_ptr())
    }
    pub fn set_save_latents(&self, r: &RequestHandle, path: &Path) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_save_latents(r.pointer.as_ptr(), path)
    }
    pub fn set_latent_bridge_files(&self, r: &RequestHandle, left: &Path, right: &Path, left_margin: i32, right_margin: i32, context: i32) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_latent_bridge_files(r.pointer.as_ptr(), left, right, left_margin, right_margin, context)
    }
    pub fn require_end_anchor(&self) -> Result<(), String> { self.inner.require_end_anchor() }
    pub fn set_keystone_file(&self, r: &RequestHandle, path: &Path, lock_after: i32) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_keystone_file(r.pointer.as_ptr(), path, lock_after)
    }
    pub fn require_continuation_source_init(&self) -> Result<(), String> { self.inner.require_continuation_source_init() }
    pub fn set_continuation_source_init(&self, r: &RequestHandle, noise: f32) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_continuation_source_init(r.pointer.as_ptr(), noise)
    }
    pub fn set_end_anchor_file(&self, r: &RequestHandle, path: &Path) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_end_anchor_file(r.pointer.as_ptr(), path)
    }
    pub fn set_continuation_file(
        &self,
        r: &RequestHandle,
        path: &Path,
        overlap: i32,
    ) -> Result<(), String> {
        self.check(r)?;
        self.inner
            .set_continuation_file(r.pointer.as_ptr(), path, overlap)
    }
    pub fn set_steps(&self, r: &RequestHandle, v: i32) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_steps(r.pointer.as_ptr(), v)
    }
    pub fn set_latent_upscaler(&self, r: &RequestHandle, path: Option<&Path>) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_latent_upscaler(r.pointer.as_ptr(), path)
    }
    #[cfg(test)]
    pub fn disable_latent_upscaler_for_test(&mut self) {
        Arc::get_mut(&mut self.inner).expect("configure API before creating handles").disable_latent_upscaler_for_test();
    }
    pub fn set_audio_steps(&self, r: &RequestHandle, steps: i32) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_audio_steps(r.pointer.as_ptr(), steps)
    }
    pub fn set_continuation_lock_overlap(&self, r: &RequestHandle, enabled: bool) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_continuation_lock_overlap(r.pointer.as_ptr(), enabled)
    }
    pub fn set_continuation_regenerate_frames(&self, r: &RequestHandle, frames: i32) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_continuation_regenerate_frames(r.pointer.as_ptr(), frames)
    }
    pub fn set_seed(&self, r: &RequestHandle, v: u64) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_seed(r.pointer.as_ptr(), v)
    }
    pub fn set_sampling(&self, r: &RequestHandle, video: Option<f32>, audio: Option<f32>, sampler: Option<&str>, scheduler: Option<&str>) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_sampling(r.pointer.as_ptr(), video, audio, sampler, scheduler)
    }
    pub fn set_dmad_sampling(&self, r: &RequestHandle) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_dmad_sampling(r.pointer.as_ptr())
    }
    pub fn set_pika_sampling(&self, r: &RequestHandle) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_pika_sampling(r.pointer.as_ptr())
    }
    pub fn set_verbose(&self, r: &RequestHandle, v: bool) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_verbose(r.pointer.as_ptr(), v)
    }
    pub fn set_reuse_models(&self, r: &RequestHandle, v: bool) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_reuse_models(r.pointer.as_ptr(), v)
    }
    pub fn set_attention(&self, r: &RequestHandle, v: &str) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_attention(r.pointer.as_ptr(), v)
    }
    pub fn set_motion_cache(&self, r: &RequestHandle, enabled: bool) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_motion_cache(r.pointer.as_ptr(), enabled)
    }
    pub fn set_inference_backend(&self, r: &RequestHandle, backend: i32) -> Result<(), String> {
        self.check(r)?;
        self.inner
            .set_inference_backend(r.pointer.as_ptr(), backend)
    }
    pub fn set_model(&self, r: &RequestHandle, id: i32, path: &Path) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_model(r.pointer.as_ptr(), id, path)
    }
    #[cfg(test)]
    pub fn disable_image_guidance_for_test(&mut self) {
        Arc::get_mut(&mut self.inner).expect("configure API before creating handles").disable_image_guidance_for_test();
    }
    pub fn set_image_guidance(&self, r: &RequestHandle, scale: f32, negative_prompt: &str) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_image_guidance(r.pointer.as_ptr(), scale, negative_prompt, &[22])
    }
    pub fn set_krea_guidance(&self, r: &RequestHandle, scale: f32, negative_prompt: &str) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_image_guidance(r.pointer.as_ptr(), scale, negative_prompt, &[])
    }
    pub fn set_music_settings(&self, r: &RequestHandle, settings: &str) -> Result<(), String> {
        self.check(r)?; self.inner.set_music_settings(r.pointer.as_ptr(), settings)
    }
    pub fn set_vulkan_portable(&self, r: &RequestHandle) -> Result<(), String> {
        self.check(r)?; self.inner.set_vulkan_portable(r.pointer.as_ptr())
    }
    pub fn set_model_family(&self, r: &RequestHandle, family: &str) -> Result<(), String> {
        self.check(r)?;
        self.inner.set_model_family(r.pointer.as_ptr(), family)
    }
    pub fn add_reference(&self, r: &RequestHandle, path: &Path) -> Result<(), String> {
        self.check(r)?;
        self.inner.add_reference(r.pointer.as_ptr(), path)
    }
    pub fn add_reference_audio(
        &self,
        r: &RequestHandle,
        samples: &[f32],
        channels: i32,
        sample_rate: i32,
    ) -> Result<(), String> {
        self.check(r)?;
        self.inner
            .add_reference_audio(r.pointer.as_ptr(), samples, channels, sample_rate)
    }
    pub fn add_lora(&self, r: &RequestHandle, path: &Path, strength: f32) -> Result<(), String> {
        self.check(r)?;
        self.inner.add_lora(r.pointer.as_ptr(), path, strength)
    }
    pub fn add_refmod(
        &self,
        r: &RequestHandle,
        path: &Path,
        strength: f32,
        copies: i32,
    ) -> Result<(), String> {
        self.check(r)?;
        self.inner
            .add_refmod(r.pointer.as_ptr(), path, strength, copies)
    }
    pub fn export_refmod(&self, r: &RequestHandle, output: &Path, name: &str, description: &str) -> Result<(), String> {
        self.check(r)?;
        self.inner.export_refmod(r.pointer.as_ptr(), output, name, description)
    }
    pub fn resolve(&self, request: &RequestHandle) -> Result<Plan, String> {
        self.check(request)?;
        self.inner.resolve(request.pointer.as_ptr())
    }
    pub fn describe(&self, request: &RequestHandle) -> Result<String, String> {
        self.check(request)?;
        self.inner.describe(request.pointer.as_ptr())
    }
}

/// Owns its library; native requests cannot outlive the DLL or cross instances.
pub struct RequestHandle {
    api: Api,
    pointer: NonNull<raw::Request>,
}
impl RequestHandle {
    pub fn new(api: &Api) -> Result<Self, String> {
        let pointer = NonNull::new(api.inner.create_request()?).ok_or("Empty native request.")?;
        Ok(Self {
            api: api.clone(),
            pointer,
        })
    }
    pub fn start(self, callback: Option<ProgressSink>) -> Result<GenerationHandle, String> {
        GenerationHandle::start(self, callback, None)
    }
    pub fn start_with_preview(self, callback: Option<ProgressSink>, preview: Option<(PathBuf, PreviewSink)>) -> Result<GenerationHandle, String> {
        GenerationHandle::start(self, callback, preview)
    }
}
impl Drop for RequestHandle {
    fn drop(&mut self) {
        self.api.inner.destroy_request(self.pointer.as_ptr());
    }
}

pub struct ReferenceVideoHandle(raw::ReferenceVideoHandle);
impl ReferenceVideoHandle {
    pub fn new(api: Api, duration: f64, path: PathBuf) -> Result<Self, String> {
        if api.path != path {
            return Err("Reference video runtime mismatch.".into());
        }
        raw::ReferenceVideoHandle::new(api.inner, duration, path).map(Self)
    }
    pub fn append(&mut self, bytes: &[u8], w: i32, h: i32, time: f64) -> Result<(), String> {
        self.0.append(bytes, w, h, time)
    }
    pub fn set_audio(&mut self, samples: &[f32], channels: i32, rate: i32) -> Result<(), String> {
        self.0.set_audio(samples, channels, rate)
    }
    pub fn has_audio(&self) -> bool {
        self.0.has_audio()
    }
    pub fn attach(&self, request: &RequestHandle, path: &Path) -> Result<(), String> {
        if request.api.path != path {
            return Err("Reference video runtime mismatch.".into());
        }
        self.0.attach(request.pointer.as_ptr(), path)
    }
}