AI media harness for planning, generating, and editing locally
TypeScript
use super::{
config::Configuration,
ffi::{self, RequestHandle},
h3::configure_request,
planning::RequestPurpose,
platform::*,
references::ReferenceVideos,
types::GenerationRequest,
};
use serde::Deserialize;
use std::path::PathBuf;
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ReferenceIconSpec {
pub id: String,
pub prompt: String,
pub seed: u64,
pub destination: PathBuf,
}
#[derive(Debug)]
pub struct ReferenceIconBatchConfig {
pub dll_path: PathBuf,
pub transformer: PathBuf,
pub text_encoder: PathBuf,
pub video_vae: PathBuf,
pub backend: String,
}
pub(super) fn write_reference_icon(
api: &ffi::Api,
configuration: &Configuration,
platform: ComputePlatform,
spec: &ReferenceIconSpec,
) -> Result<(), String> {
let _models = super::MODEL_ACCESS
.read()
.map_err(|_| "Model access lock failed.")?;
let request = GenerationRequest {
job_id: spec.id.clone(),
prompt: spec.prompt.clone(),
frames: 1,
still_image: true,
steps: crate::reference_icons::STEPS,
seed: i64::try_from(spec.seed)
.map_err(|_| format!("Icon '{}' seed is too large.", spec.id))?,
canvas_width: crate::reference_icons::RENDER_SIZE as i32,
canvas_height: crate::reference_icons::RENDER_SIZE as i32,
reference_paths: Vec::new(),
refmods: Vec::new(),
reference_video_ids: Vec::new(),
..Default::default()
};
let handle = RequestHandle::new(api)?;
let _blank_reference = configure_request(
api,
&handle,
&request,
configuration,
platform,
RequestPurpose::Generate,
&ReferenceVideos::default(),
)?;
api.resolve(&handle)?;
let mut generation = handle.start(None)?;
while !generation
.wait(-1)
.map_err(|error| format!("Icon '{}': {error}", spec.id))?
{}
let generation = generation.finish()?;
let output = generation.output()?;
let render_size = crate::reference_icons::RENDER_SIZE;
if output.width != render_size as i32
|| output.height != render_size as i32
|| output.frames != 1
{
return Err(format!(
"Icon '{}' returned {}x{} with {} frames.",
spec.id, output.width, output.height, output.frames
));
}
let rgba = generation.frame_rgba8(0, render_size, render_size)?;
drop(generation);
let jpeg = crate::reference_icons::encode_jpeg(&rgba)?;
if let Some(parent) = spec.destination.parent() {
std::fs::create_dir_all(parent).map_err(|error| {
format!(
"Could not create icon folder '{}': {error}",
parent.display()
)
})?;
}
std::fs::write(&spec.destination, jpeg).map_err(|error| {
format!(
"Could not create icon '{}': {error}",
spec.destination.display()
)
})
}
pub fn generate_reference_icon_batch(
specs: &[ReferenceIconSpec],
batch: &ReferenceIconBatchConfig,
) -> Result<(), String> {
let backend = match batch.backend.as_str() {
"cuda" => ComputePlatform::Cuda13,
"vulkan" => ComputePlatform::Vulkan,
value => return Err(format!("Unsupported icon generation backend '{value}'.")),
};
let configuration = Configuration {
model_type: "minimax-h3".into(),
krea_raw: false,
sampling: Ok(Default::default()),
editing_model: true,
dll_path: batch.dll_path.clone(),
loras: Ok(Vec::new()),
step_override: Ok(None),
sampling_preset: Ok(None),
sigma_shifts: Ok(Default::default()),
vulkan: backend == ComputePlatform::Vulkan,
attention: "sage2",
motion_cache: false,
animate: false,
prompt_embedding: None,
latent_upscaler: None,
preview_vae: None,
models: [
(0, "transformer", Some(batch.transformer.clone())),
(1, "textEncoder", Some(batch.text_encoder.clone())),
(2, "tokenizer", None),
(3, "videoVae", Some(batch.video_vae.clone())),
// Icons keep one picture and need no decoded soundtrack.
(4, "audioVae", None),
],
};
let api = ffi::Api::load(&configuration.dll_path)?;
api.version()?;
let platform = if backend == ComputePlatform::Cuda13 {
detect_platform(&api)
} else {
backend
};
let result = specs.iter().enumerate().try_for_each(|(index, spec)| {
let started = std::time::Instant::now();
write_reference_icon(&api, &configuration, platform, spec)?;
println!(
"[{}/{}] {}: generated in {:.1}s",
index + 1,
specs.len(),
spec.id,
started.elapsed().as_secs_f64()
);
Ok(())
});
let cleared = api.clear_reused_models();
result.and(cleared)
}