Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Support generation sigma shifts in runtime

Bitticf77a5b38d7a
diff --git a/src-tauri/core/src/slopfab/config.rs b/src-tauri/core/src/slopfab/config.rs
index 81b26b1..32c617a 100644
--- a/src-tauri/core/src/slopfab/config.rs
+++ b/src-tauri/core/src/slopfab/config.rs
@@ -14,6 +14,14 @@ pub(super) enum SamplingPreset {
Dmad4Step,
}
+#[derive(Clone, Debug, Default)]
+pub(super) struct SigmaShifts {
+ pub(super) video: Option<f32>,
+ pub(super) audio: Option<f32>,
+ pub(super) video_override: Option<f32>,
+ pub(super) audio_override: Option<f32>,
+}
+
#[derive(Clone)]
pub(super) struct Configuration {
pub(super) dll_path: PathBuf,
@@ -27,6 +35,7 @@ pub(super) struct Configuration {
pub(super) loras: Result<Vec<LoraAdapter>, String>,
pub(super) step_override: Result<Option<i32>, String>,
pub(super) sampling_preset: Result<Option<SamplingPreset>, String>,
+ pub(super) sigma_shifts: Result<SigmaShifts, String>,
}
impl Configuration {
@@ -46,7 +55,16 @@ impl Configuration {
_ => None,
})
};
+ let shift = |name: &str| -> Result<Option<f32>, String> {
+ match slopfab.and_then(|setting| setting.options.get(name)) {
+ None => Ok(None),
+ Some(ProviderOption::Number(value)) if value.is_finite() && *value >= f32::from_bits(1) as f64 && *value <= f32::MAX as f64 => Ok(Some(*value as f32)),
+ _ => Err(format!("{name} must be a finite, positive float32 value.")),
+ }
+ };
Self {
+ sigma_shifts: (|| Ok(SigmaShifts { video: shift("videoSigmaShift")?, audio: shift("audioSigmaShift")?,
+ video_override: shift("videoSigmaShiftOverride")?, audio_override: shift("audioSigmaShiftOverride")? }))(),
// `dllPath` is no longer written by the app and has no settings UI.
// It is still READ so a project or a test that carries one keeps
// working; with none, use the DLL beside the executable.
diff --git a/src-tauri/core/src/slopfab/ffi/mod.rs b/src-tauri/core/src/slopfab/ffi/mod.rs
index 34cdd01..1069654 100644
--- a/src-tauri/core/src/slopfab/ffi/mod.rs
+++ b/src-tauri/core/src/slopfab/ffi/mod.rs
@@ -164,6 +164,10 @@ impl Api {
self.check(r)?;
self.inner.set_seed(r.pointer.as_ptr(), v)
}
+ pub fn set_sigma_shifts(&self, r: &RequestHandle, video: Option<f32>, audio: Option<f32>) -> Result<(), String> {
+ self.check(r)?;
+ self.inner.set_sigma_shifts(r.pointer.as_ptr(), video, audio)
+ }
pub fn set_dmad_sampling(&self, r: &RequestHandle) -> Result<(), String> {
self.check(r)?;
self.inner.set_dmad_sampling(r.pointer.as_ptr())
diff --git a/src-tauri/core/src/slopfab/ffi/raw/mod.rs b/src-tauri/core/src/slopfab/ffi/raw/mod.rs
index ca3f5f4..d018c61 100644
--- a/src-tauri/core/src/slopfab/ffi/raw/mod.rs
+++ b/src-tauri/core/src/slopfab/ffi/raw/mod.rs
@@ -375,6 +375,22 @@ impl Api {
let settings = CString::new(r#"{"version":1,"sampler":"renoise","video_sigma_shift":12,"audio_sigma_shift":2,"base_sigmas":[1,0.75,0.5,0.25,0]}"#).unwrap();
self.error(unsafe { set(r, settings.as_ptr()) })
}
+ pub fn set_sigma_shifts(&self, r: *mut Request, video: Option<f32>, audio: Option<f32>) -> Result<(), String> {
+ let Some(set) = self.set_sampling_settings else {
+ return if video.is_none() && audio.is_none() { Ok(()) }
+ else { Err("Sigma shifts require slopfab.dll API 1.16 or later. Update the runtime.".into()) };
+ };
+ let mut settings = serde_json::json!({ "version": 1 });
+ for (key, value) in [("video_sigma_shift", video), ("audio_sigma_shift", audio)] {
+ if let Some(value) = value {
+ if !value.is_finite() || value <= 0.0 { return Err("Sigma shifts must be finite and positive.".into()); }
+ settings[key] = serde_json::json!(value);
+ }
+ }
+ // Replaces the previous recipe, including clearing stale shifts on reused handles.
+ let settings = CString::new(settings.to_string()).map_err(|error| error.to_string())?;
+ self.error(unsafe { set(r, settings.as_ptr()) })
+ }
fn error(&self, code: i32) -> Result<(), String> {
if code == 0 {
Ok(())
diff --git a/src-tauri/core/src/slopfab/h3.rs b/src-tauri/core/src/slopfab/h3.rs
index 11bd5b7..459fa62 100644
--- a/src-tauri/core/src/slopfab/h3.rs
+++ b/src-tauri/core/src/slopfab/h3.rs
@@ -57,6 +57,10 @@ pub(super) fn configure_request(
let dmad = configuration.sampling_preset.as_ref().map_err(Clone::clone)? == &Some(SamplingPreset::Dmad4Step);
if dmad {
api.set_dmad_sampling(handle)?;
+ } else {
+ let shifts = configuration.sigma_shifts.as_ref().map_err(Clone::clone)?;
+ api.set_sigma_shifts(handle, shifts.video_override.or(request.video_sigma_shift).or(shifts.video),
+ shifts.audio_override.or(request.audio_sigma_shift).or(shifts.audio))?;
}
// Sampling presets replace overrides, so apply the audio setting afterward.
api.set_audio_steps(handle, request.audio_steps.unwrap_or(0))?;
diff --git a/src-tauri/core/src/slopfab/icons.rs b/src-tauri/core/src/slopfab/icons.rs
index 6750070..fef6160 100644
--- a/src-tauri/core/src/slopfab/icons.rs
+++ b/src-tauri/core/src/slopfab/icons.rs
@@ -113,6 +113,7 @@ pub fn generate_reference_icon_batch(
loras: Ok(Vec::new()),
step_override: Ok(None),
sampling_preset: Ok(None),
+ sigma_shifts: Ok(Default::default()),
vulkan: backend == ComputePlatform::Vulkan,
attention: "sage2",
motion_cache: false,
diff --git a/src-tauri/core/src/slopfab/planning.rs b/src-tauri/core/src/slopfab/planning.rs
index cee4bfc..bc97fb2 100644
--- a/src-tauri/core/src/slopfab/planning.rs
+++ b/src-tauri/core/src/slopfab/planning.rs
@@ -109,6 +109,11 @@ pub(super) fn validate_generation_controls(request: &GenerationRequest) -> Resul
if request.audio_steps.is_some_and(|steps| !(2..=1_000_000).contains(&steps)) {
return Err("Audio step count must be a whole number from 2 to 1000000.".into());
}
+ for shift in [request.video_sigma_shift, request.audio_sigma_shift].into_iter().flatten() {
+ if !shift.is_finite() || shift <= 0.0 {
+ return Err("Sigma shifts must be finite and positive.".into());
+ }
+ }
if request.seed < -1 {
return Err("Generation seed must be -1 or greater.".into());
}
diff --git a/src-tauri/core/src/slopfab/tests.rs b/src-tauri/core/src/slopfab/tests.rs
index 9ee63b0..b8de9af 100644
--- a/src-tauri/core/src/slopfab/tests.rs
+++ b/src-tauri/core/src/slopfab/tests.rs
@@ -49,6 +49,7 @@ fn dmad_recipe_reaches_preview_and_execution_on_both_backends() {
let request = GenerationRequest {
prompt: "A polar bear plays the violin in the snow.".into(),
frames: 124, steps: 30, seed: 42, canvas_width: 1344, canvas_height: 768,
+ video_sigma_shift: Some(7.0), audio_sigma_shift: Some(4.0),
..Default::default()
};
for platform in [ComputePlatform::Cuda13, ComputePlatform::Vulkan] {
@@ -1317,3 +1318,54 @@ fn latent_upscale_uses_half_aligned_canvas_for_preview_and_execution() {
assert!(validate_generation_controls(&GenerationRequest { canvas_width: 65, ..request }).is_err());
assert_eq!(stage_name(9), "upscaling");
}
+
+#[test]
+fn sigma_shifts_reach_planning_execution_and_reset() {
+ let mut configuration = Configuration::from_settings(&BTreeMap::new());
+ configuration.sigma_shifts = Ok(SigmaShifts { video: Some(9.0), audio: Some(3.5), ..Default::default() });
+ let api = ffi::Api::load(&configuration.dll_path).unwrap();
+ for platform in [ComputePlatform::Cuda13, ComputePlatform::Vulkan] {
+ for purpose in [RequestPurpose::Plan, RequestPurpose::Generate] {
+ let handle = RequestHandle::new(&api).unwrap();
+ let mut request = GenerationRequest { prompt: "Ocean waves".into(), frames: 48, steps: 4,
+ video_sigma_shift: Some(8.5), audio_sigma_shift: Some(2.5), seed: 1,
+ canvas_width: 64, canvas_height: 32, ..Default::default() };
+ for (video, audio) in [(8.5, 2.5), (6.0, 2.0), (9.0, 3.5), (12.0, 3.0)] {
+ configure_request(&api, &handle, &request, &configuration, platform, purpose, &ReferenceVideos::default()).unwrap();
+ let description = api.describe(&handle).unwrap();
+ assert!(description.contains(&format!("shift {video}")), "{description}");
+ assert!(description.contains(&format!("shift {audio}")), "{description}");
+ if video == 8.5 {
+ let shifts = configuration.sigma_shifts.as_mut().unwrap();
+ shifts.video_override = Some(6.0);
+ shifts.audio_override = Some(2.0);
+ } else if video == 6.0 {
+ configuration.sigma_shifts = Ok(SigmaShifts { video: Some(9.0), audio: Some(3.5), ..Default::default() });
+ request.video_sigma_shift = None;
+ request.audio_sigma_shift = None;
+ } else {
+ configuration.sigma_shifts = Ok(SigmaShifts::default());
+ }
+ }
+ configuration.sigma_shifts = Ok(SigmaShifts { video: Some(9.0), audio: Some(3.5), ..Default::default() });
+ for invalid in [0.0, -1.0, f32::INFINITY, f32::NAN] {
+ request.video_sigma_shift = Some(invalid);
+ assert!(validate_generation_controls(&request).is_err());
+ request.video_sigma_shift = None;
+ request.audio_sigma_shift = Some(invalid);
+ assert!(validate_generation_controls(&request).is_err());
+ }
+ }
+ }
+}
+
+#[test]
+fn sigma_provider_settings_validate_float32_bounds() {
+ for key in ["videoSigmaShift", "audioSigmaShift", "videoSigmaShiftOverride", "audioSigmaShiftOverride"] {
+ for value in [0.0, -1.0, f64::INFINITY, 1e39, 1e-50] {
+ let settings = BTreeMap::from([("slopfab".into(), ProviderSetting { enabled: true, model: None,
+ options: BTreeMap::from([(key.into(), ProviderOption::Number(value))]) })]);
+ assert!(Configuration::from_settings(&settings).sigma_shifts.is_err());
+ }
+ }
+}
diff --git a/src-tauri/core/src/slopfab/types.rs b/src-tauri/core/src/slopfab/types.rs
index 0d94733..a0eed2e 100644
--- a/src-tauri/core/src/slopfab/types.rs
+++ b/src-tauri/core/src/slopfab/types.rs
@@ -70,6 +70,10 @@ pub struct GenerationRequest {
pub steps: i32,
#[serde(default)]
pub audio_steps: Option<i32>,
+ #[serde(default)]
+ pub video_sigma_shift: Option<f32>,
+ #[serde(default)]
+ pub audio_sigma_shift: Option<f32>,
/// Canvas dimensions describe the final output; denoise at half size when enabled.
#[serde(default)]
pub latent_upscale: bool,
diff --git a/src-tauri/src/project/scenes.rs b/src-tauri/src/project/scenes.rs
index 66ac5a8..155ebe2 100644
--- a/src-tauri/src/project/scenes.rs
+++ b/src-tauri/src/project/scenes.rs
@@ -125,6 +125,10 @@ pub(crate) struct GenerationJob {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub(crate) audio_steps: Option<i32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
+ pub(crate) video_sigma_shift: Option<f64>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub(crate) audio_sigma_shift: Option<f64>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
pub(crate) latent_upscale: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub(crate) seed: Option<i64>,
@@ -189,6 +193,8 @@ impl GenerationJob {
duration_seconds: Some(defaults.duration_seconds),
steps: None,
audio_steps: None,
+ video_sigma_shift: None,
+ audio_sigma_shift: None,
latent_upscale: None,
seed: None,
soundscape: None,
diff --git a/src-tauri/src/project/validation/scenes.rs b/src-tauri/src/project/validation/scenes.rs
index 5137211..2619868 100644
--- a/src-tauri/src/project/validation/scenes.rs
+++ b/src-tauri/src/project/validation/scenes.rs
@@ -114,6 +114,11 @@ pub(super) fn validate(config: &mut ProjectConfig) -> Result<(), String> {
if job.audio_steps.is_some_and(|steps| !(2..=1_000_000).contains(&steps)) {
return Err(format!("Scene '{}' must use 2 to 1000000 audio steps.", job.id));
}
+ for shift in [job.video_sigma_shift, job.audio_sigma_shift].into_iter().flatten() {
+ if !shift.is_finite() || shift < f32::from_bits(1) as f64 || shift > f32::MAX as f64 {
+ return Err(format!("Scene '{}' sigma shifts must be finite, positive float32 values.", job.id));
+ }
+ }
if job.seed.is_some_and(|seed| seed < -1) {
return Err(format!("Scene '{}' has a seed below -1.", job.id));
}
diff --git a/src-tauri/src/tests/scenes.rs b/src-tauri/src/tests/scenes.rs
index 2cbee9a..3138544 100644
--- a/src-tauri/src/tests/scenes.rs
+++ b/src-tauri/src/tests/scenes.rs
@@ -537,3 +537,23 @@ fn scene_latent_upscale_survives_save_and_preserves_legacy_defaults() {
assert_eq!(restored.generation_jobs[0].latent_upscale, latent_upscale);
}
}
+
+#[test]
+fn scene_sigma_shifts_round_trip_and_validate() {
+ let mut config = scene_fixture();
+ config.generation_jobs[0].video_sigma_shift = Some(8.5);
+ config.generation_jobs[0].audio_sigma_shift = Some(2.5);
+ let folder = tempfile::tempdir().unwrap();
+ write_project(folder.path(), &config).unwrap();
+ let restored = read_project(folder.path()).unwrap().config;
+ assert_eq!(restored.generation_jobs[0].video_sigma_shift, Some(8.5));
+ assert_eq!(restored.generation_jobs[0].audio_sigma_shift, Some(2.5));
+ for invalid in [0.0, -1.0, f64::INFINITY, 1e39, 1e-50] {
+ let mut invalid_config = config.clone();
+ invalid_config.generation_jobs[0].video_sigma_shift = Some(invalid);
+ assert!(validate_and_normalize_config(invalid_config).is_err());
+ let mut invalid_config = config.clone();
+ invalid_config.generation_jobs[0].audio_sigma_shift = Some(invalid);
+ assert!(validate_and_normalize_config(invalid_config).is_err());
+ }
+}
diff --git a/src-tauri/src/worker/client.rs b/src-tauri/src/worker/client.rs
index 008bf91..8f4e917 100644
--- a/src-tauri/src/worker/client.rs
+++ b/src-tauri/src/worker/client.rs
@@ -1142,6 +1142,8 @@ mod tests {
continuation_source_frames: Some(85),
continuation_lock_overlap: true,
audio_steps: Some(17),
+ video_sigma_shift: Some(8.5),
+ audio_sigma_shift: Some(2.5),
..Default::default()
};
let (job, sources) = build_job(&request, &settings, &[]).unwrap();
@@ -1158,6 +1160,8 @@ mod tests {
assert_eq!(wire["request"]["continuationSourceFrames"], 85);
assert_eq!(wire["request"]["continuationLockOverlap"], true);
assert_eq!(wire["request"]["audioSteps"], 17);
+ assert_eq!(wire["request"]["videoSigmaShift"], 8.5);
+ assert_eq!(wire["request"]["audioSigmaShift"], 2.5);
assert_eq!(sources.len(), 2);
assert!(build_job(&GenerationRequest { reference_paths: vec![folder.path().join("missing.png").to_string_lossy().into_owned()], ..Default::default() }, &BTreeMap::new(), &[]).is_err());
}