AI media harness for planning, generating, and editing locally
TypeScript
use super::{default_dll_path, platform::ComputePlatform, types::GenerationRequest};
use crate::settings::{ProviderOption, ProviderSetting};
use serde::Deserialize;
use std::{collections::BTreeMap, path::PathBuf};
#[derive(Debug, Clone, Deserialize)]
pub(super) struct LoraAdapter {
pub(super) path: String,
pub(super) strength: f32,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub(super) enum SamplingPreset {
Dmad4Step,
Pika6Step,
}
#[derive(Clone, Debug, Default)]
pub(super) struct SigmaShifts {
pub(super) video: Option<f32>,
pub(super) audio: Option<f32>,
pub(super) video_override: Option<f32>,
pub(super) audio_override: Option<f32>,
}
#[derive(Clone)]
pub(super) struct Configuration {
pub(super) sampling: Result<SamplingChoices, String>,
pub(super) model_type: String,
pub(super) krea_raw: bool,
pub(super) editing_model: bool,
pub(super) dll_path: PathBuf,
pub(super) vulkan: bool,
pub(super) attention: &'static str,
pub(super) motion_cache: bool,
pub(super) animate: bool,
pub(super) prompt_embedding: Option<PathBuf>,
pub(super) latent_upscaler: Option<PathBuf>,
pub(super) preview_vae: Option<PathBuf>,
pub(super) models: [(i32, &'static str, Option<PathBuf>); 5],
pub(super) loras: Result<Vec<LoraAdapter>, String>,
pub(super) step_override: Result<Option<i32>, String>,
pub(super) sampling_preset: Result<Option<SamplingPreset>, String>,
pub(super) sigma_shifts: Result<SigmaShifts, String>,
}
#[derive(Clone, Debug, Default)]
pub(super) struct SamplingChoices {
pub sampler: Option<String>,
pub scheduler: Option<String>,
}
impl Configuration {
pub(super) fn from_settings(settings: &BTreeMap<String, ProviderSetting>) -> Self {
let slopfab = settings.get("slopfab");
let string_option = |name: &str| {
slopfab.and_then(|setting| match setting.options.get(name) {
Some(ProviderOption::String(value)) => Some(value.as_str()),
_ => None,
})
};
let option = |name: &str| {
slopfab.and_then(|setting| match setting.options.get(name) {
Some(ProviderOption::String(value)) if !value.trim().is_empty() => {
Some(PathBuf::from(value))
}
_ => None,
})
};
let shift = |name: &str| -> Result<Option<f32>, String> {
match slopfab.and_then(|setting| setting.options.get(name)) {
None => Ok(None),
Some(ProviderOption::Number(value)) if value.is_finite() && *value >= f32::from_bits(1) as f64 && *value <= f32::MAX as f64 => Ok(Some(*value as f32)),
_ => Err(format!("{name} must be a finite, positive float32 value.")),
}
};
Self {
sampling: (|| {
let read = |name: &str| match slopfab.and_then(|setting| setting.options.get(name)) {
None => Ok(None),
Some(ProviderOption::String(value)) if value == "default" => Ok(None),
Some(ProviderOption::String(value)) => Ok(Some(value.clone())),
_ => Err(format!("{name} must be a supported sampling option.")),
};
Ok(SamplingChoices { sampler: read("sampler")?, scheduler: read("scheduler")? })
})(),
editing_model: !matches!(slopfab.and_then(|setting| setting.options.get("editingModel")), Some(ProviderOption::Boolean(false))),
model_type: string_option("modelType").unwrap_or("minimax-h3").to_owned(),
krea_raw: string_option("kreaVariant") == Some("raw"),
sigma_shifts: (|| Ok(SigmaShifts { video: shift("videoSigmaShift")?, audio: shift("audioSigmaShift")?,
video_override: shift("videoSigmaShiftOverride")?, audio_override: shift("audioSigmaShiftOverride")? }))(),
// `dllPath` is no longer written by the app and has no settings UI.
// It is still READ so a project or a test that carries one keeps
// working; with none, use the DLL beside the executable.
dll_path: option("dllPath").unwrap_or_else(default_dll_path),
vulkan: string_option("inferenceBackend") == Some("vulkan"),
animate: string_option("generationMode") == Some("animate"),
motion_cache: matches!(
slopfab.and_then(|setting| setting.options.get("motionCache")),
Some(ProviderOption::Boolean(true))
),
prompt_embedding: option("promptEmbedding"),
latent_upscaler: option("latentUpscaler"),
preview_vae: option("previewVae"),
attention: match string_option("attention") {
Some("exact") => "exact",
Some("flash2") => "flash2",
Some("sol") => "sol",
_ => "sage2",
},
models: [
(0, "transformer", option("transformer")),
(1, "textEncoder", option("textEncoder")),
(2, "tokenizer", option("tokenizer")),
(3, "videoVae", option("videoVae")),
(4, "audioVae", option("audioVae")),
],
loras: match slopfab.and_then(|setting| setting.options.get("loras")) {
None => Ok(Vec::new()),
Some(ProviderOption::String(value)) => {
serde_json::from_str::<Vec<LoraAdapter>>(value)
.map_err(|error| format!("Invalid LoRA settings: {error}"))
}
_ => Err("LoRA settings must be a JSON array of paths and strengths.".into()),
},
step_override: match slopfab.and_then(|setting| setting.options.get("stepOverride")) {
None => Ok(None),
Some(ProviderOption::Number(value))
if value.is_finite()
&& value.fract() == 0.0
&& *value >= 2.0
&& *value <= i32::MAX as f64 =>
{
Ok(Some(*value as i32))
}
_ => Err("LoRA step override must be a whole number from 2 to 2147483647.".into()),
},
sampling_preset: match slopfab.and_then(|setting| setting.options.get("samplingPreset")) {
None => Ok(None),
Some(ProviderOption::String(value)) if value == "dmad-4step" => Ok(Some(SamplingPreset::Dmad4Step)),
Some(ProviderOption::String(value)) if value == "pika-6step" => Ok(Some(SamplingPreset::Pika6Step)),
_ => Err("Unsupported LoRA sampling recipe.".into()),
},
}
}
pub(super) fn validate_inputs(&self, request: &GenerationRequest) -> Result<(), String> {
let sampling = self.sampling.as_ref().map_err(Clone::clone)?;
let features = crate::models::generator_features(&self.model_type);
for (label, value, allowed) in [("sampler", &sampling.sampler, &features.samplers), ("scheduler", &sampling.scheduler, &features.schedulers)] {
if value.as_ref().is_some_and(|value| !allowed.contains(value)) {
return Err(format!("{} does not support the selected {label}.", features.label));
}
}
if self.sampling_preset.as_ref().map_err(Clone::clone)?.is_some() && (sampling.sampler.is_some() || sampling.scheduler.is_some()) {
return Err("The active LoRA recipe controls the sampler and scheduler.".into());
}
if self.animate && sampling.sampler.as_deref().is_some_and(|sampler| sampler != "euler") {
return Err("Animate requires Euler sampling.".into());
}
if self.model_type == "minimax-h3" && request.image_edit.is_some()
&& sampling.sampler.as_deref().is_some_and(|sampler| !["euler", "renoise"].contains(&sampler)) {
return Err("MiniMax H3 image editing requires Euler or re-noising sampling.".into());
}
if self.motion_cache && sampling.sampler.as_deref().is_some_and(|sampler| sampler != "euler") {
return Err("MotionCache requires Euler sampling.".into());
}
if request.latent_upscale_source.is_some() {
if self.model_type != "minimax-h3" { return Err("Saved latent upscaling requires MiniMax H3.".into()); }
for (label, path) in [("latent upscaler", self.latent_upscaler.as_ref()), ("video VAE", self.models[3].2.as_ref()), ("audio VAE", self.models[4].2.as_ref())] {
if !path.is_some_and(|path| path.is_file()) { return Err(format!("Download the {label} weights before upscaling.")); }
}
return Ok(());
}
if request.single_shot.is_some() && (self.model_type != "minimax-h3" || self.animate) {
return Err("Single Shot requires a MiniMax H3 prompt generator.".into());
}
if request.music.is_some() { return super::music::validate(request, self); }
let features = crate::models::generator_features(&self.model_type);
if let Some(negative) = &request.negative_prompt {
if negative.contains('\0') { return Err("Negative prompt cannot contain null characters.".into()); }
if !negative.is_empty() && (!features.negative_prompt || !request.still_image) {
return Err(format!("{} does not support negative prompts for this generation.", features.label));
}
}
if let Some(scale) = request.guidance_scale {
crate::models::validate_cfg_scale(f64::from(scale))?;
if features.cfg.is_none() || !request.still_image { return Err(format!("{} does not support configurable CFG for this generation.", features.label)); }
}
if let Some(effects) = &request.generation_effects {
if effects.native_settings()?.is_some() && !features.generation_effects { return Err(format!("{} does not support generation effects.", features.label)); }
}
if request.still_image && !features.image_generation || !request.still_image && !features.video_generation {
return Err(format!("{} does not support {} generation.", features.label, if request.still_image { "image" } else { "video" }));
}
if (!features.image_editing || !self.editing_model) && request.image_edit.is_some() {
return Err(format!("This {} generator does not support image editing.", features.label));
}
if features.requires_image_input && request.image_edit.is_none() && request.reference_paths.is_empty() {
return Err(format!("{} requires a source image or image references.", features.label));
}
if request.still_image && features.image_references && request.reference_paths.len() > features.max_image_references {
return Err(format!("{} supports at most {} reference images.", features.label, features.max_image_references));
}
if !features.refmods && (!request.refmods.is_empty() || request.image_edit.as_ref().is_some_and(|edit| edit.edits.iter().any(|step| step.refmods.as_ref().is_some_and(|items| !items.is_empty())))) {
return Err(format!("{} does not support refmods.", features.label));
}
if let Some(edit) = &request.image_edit {
if !features.edit_image_references && (!request.reference_paths.is_empty() || edit.edits.iter().any(|step| step.reference_paths.as_ref().is_some_and(|paths| !paths.is_empty()))) {
return Err(format!("{} edits use exactly one source image without additional image references.", features.label));
}
if self.model_type == "fibo2" && edit.edits.iter().any(|step| step.invert_mask) {
return Err("Fibo outpainting requires an explicit mask rather than H3 inverted-mask conditioning.".into());
}
if !features.image_outpainting && edit.edits.iter().any(|step| step.invert_mask) {
return Err(format!("{} does not support Zoom or outpainting.", features.label));
}
}
if !features.sigma_shifts {
let shifts = self.sigma_shifts.as_ref().map_err(Clone::clone)?;
if [shifts.video, shifts.audio, shifts.video_override, shifts.audio_override, request.video_sigma_shift, request.audio_sigma_shift].iter().any(Option::is_some)
|| request.audio_steps.is_some() || self.sampling_preset.as_ref().map_err(Clone::clone)?.is_some() {
return Err(format!("{} uses its own sampling schedule without H3 overrides.", features.label));
}
}
if self.model_type == "fibo2" && (self.animate || request.latent_upscale || request.latent_bridge.is_some()
|| request.continuation_path.is_some() || request.continuation_relative_path.is_some() || request.save_latents_path.is_some()
|| self.attention != "flash2") {
return Err("Fibo requires Flash attention, without H3 video or latent features.".into());
}
if !features.image_references && (!request.reference_paths.is_empty() || !request.refmods.is_empty()) {
return Err(format!("{} does not support image references or refmods.", features.label));
}
if !features.loras && self.loras.as_ref().map_err(Clone::clone)?.iter().any(|lora| lora.strength != 0.0) {
return Err(format!("{} does not support LoRAs.", features.label));
}
if !features.motion_cache && self.motion_cache {
return Err(format!("{} does not support MotionCache.", features.label));
}
if self.model_type == "moss" && (self.animate || request.latent_upscale || request.latent_bridge.is_some()
|| request.continuation_path.is_some() || request.continuation_relative_path.is_some()) {
return Err("MOSS supports text-to-image generation only.".into());
}
if self.model_type == "krea2" {
let shifts = self.sigma_shifts.as_ref().map_err(Clone::clone)?;
if self.animate || request.latent_upscale || request.latent_bridge.is_some()
|| request.continuation_path.is_some() || request.continuation_relative_path.is_some()
|| request.save_latents_path.is_some() || request.keystone_path.is_some()
|| request.video_transition.is_some() || !request.reference_video_ids.is_empty()
|| !request.reference_audio_ids.is_empty() || request.audio_steps.is_some()
|| shifts.audio.is_some() || shifts.audio_override.is_some() || request.audio_sigma_shift.is_some()
|| self.sampling_preset.as_ref().map_err(Clone::clone)?.is_some() || self.attention != "flash2" {
return Err("Krea 2 requires Flash attention and text-to-image generation without audio or H3 conditioning.".into());
}
}
if self.model_type == "h3_image" {
if self.generation_steps(request.steps)? != 6 {
return Err("MiniMax H3 Image requires its fixed six-step sampling schedule.".into());
}
if self.animate || request.latent_upscale || request.latent_bridge.is_some()
|| request.continuation_path.is_some() || request.continuation_relative_path.is_some()
|| request.save_latents_path.is_some() || request.keystone_path.is_some()
|| request.video_transition.is_some() || !request.reference_video_ids.is_empty()
|| !request.reference_audio_ids.is_empty() || self.preview_vae.is_some()
|| self.prompt_embedding.is_some() || self.attention != "flash2" {
return Err("MiniMax H3 Image requires Flash attention without video, audio, previews or H3 latent features.".into());
}
if let Some(edit) = &request.image_edit {
for step in &edit.edits {
if step.strength.is_some_and(|strength| strength != 1.0)
|| step.outpaint_langevin_steps.is_some_and(|steps| steps != 0) {
return Err("MiniMax H3 Image edits require strength 1 and zero Langevin steps.".into());
}
if step.reference_paths.as_ref().unwrap_or(&request.reference_paths).len() + 1 > 16 {
return Err("MiniMax H3 Image supports at most 16 pictures including the edit source.".into());
}
}
}
}
if let Some(mode) = request.video_transition.as_deref() {
let expected = match mode {
"extend" => 1,
"bridge" => 2,
_ => return Err("Unsupported video transition mode.".into()),
};
if self.animate
|| request.still_image
|| request.continuation_path.is_some()
|| request.continuation_relative_path.is_some()
|| !request.refmods.is_empty()
|| !request.reference_paths.is_empty()
|| request.reference_video_ids.len() != expected
|| !request.reference_audio_ids.is_empty()
{
return Err("Extend/Bridge requires exactly one/two video references without frame anchors, sound references, Animate, continuation or refmods.".into());
}
}
if self.animate && self.motion_cache {
return Err("MotionCache is unavailable in Animate mode.".into());
}
if self.animate {
if request.still_image {
return Err("Animate requires video generation.".into());
}
if request.reference_video_ids.len() != 1 {
return Err("Animate requires exactly one reference video.".into());
}
if request.reference_paths.len() != 1 {
return Err(
"Animate requires exactly one repainted frame of the driving scene.".into(),
);
}
if request.continuation_path.is_some() || request.latent_bridge.is_some() || !request.refmods.is_empty() || !request.reference_audio_ids.is_empty() {
return Err("Animate does not support scene continuation, sound references or refmods.".into());
}
} else if request.prompt.trim().is_empty() {
return Err("Generation prompt cannot be empty.".into());
}
Ok(())
}
pub(super) fn generation_steps(&self, fallback: i32) -> Result<i32, String> {
if self.sampling_preset.clone()? == Some(SamplingPreset::Dmad4Step) {
return Ok(4);
}
if self.sampling_preset.clone()? == Some(SamplingPreset::Pika6Step) {
return Ok(6);
}
self.step_override
.clone()
.map(|steps| steps.unwrap_or(fallback))
}
pub(super) fn platform(&self, detected: ComputePlatform) -> ComputePlatform {
if self.vulkan {
ComputePlatform::Vulkan
} else {
detected
}
}
pub(super) fn timing_profile(&self, version: &str, platform: ComputePlatform) -> String {
let models = self
.models
.iter()
.filter_map(|(_, id, path)| {
path.as_ref().map(|path| {
let file = path
.file_name()
.and_then(|value| value.to_str())
.unwrap_or("unknown");
format!("{id}={file}")
})
})
.collect::<Vec<_>>()
.join(";");
let adapters = self
.loras
.as_ref()
.map(|loras| {
loras
.iter()
.map(|lora| format!("{}@{}", lora.path, lora.strength))
.collect::<Vec<_>>()
.join(";")
})
.unwrap_or_default();
format!(
"slopfab={version}|platform={}|attention={}|motionCache={}|{models}|loras={adapters}|steps={:?}|sampling={:?}",
platform.label(),
self.attention,
self.motion_cache,
self.step_override,
self.sampling_preset
)
}
}