AI media harness for planning, generating, and editing locally
TypeScript
use super::{
events::{progress_sink, CallbackContext},
ffi::{self, RequestHandle},
h3::configure_request,
planning::{scene_frame_window, RequestPurpose},
platform::detect_platform,
runtime::QueueItem,
types::OutputMetadata,
};
use crate::{diagnostics, rendered};
use std::sync::atomic::Ordering;
#[cfg(test)]
#[path = "generation_tests.rs"]
mod tests;
struct FrameSource {
generation: ffi::FinishedGeneration,
}
impl rendered::FrameSource for FrameSource {
fn frame_rgba(&self, index: u32, width: u32, height: u32) -> Result<Vec<u8>, String> {
self.generation.frame_rgba8(index, width, height)
}
}
pub(super) fn run_generation(
item: &QueueItem,
) -> Result<(OutputMetadata, rendered::RenderedVideo), String> {
if item.request.latent_upscale_source.is_some() { return run_latent_upscale(item); }
if item.request.single_shot.is_some() {
if item.request.keystone_path.is_none() {
let api = ffi::Api::load(&item.configuration.dll_path)?;
api.require_end_anchor()?;
if item.request.generation_effects.as_ref().map(|e| e.native_settings()).transpose()?.flatten()
.is_some_and(|settings| settings.get("contrast").is_some()) { api.require_output_tone()?; }
}
return run_single_shot(item, run_single_generation);
}
run_edit_sequence(item, run_single_generation)
}
fn run_single_shot(
item: &QueueItem,
mut generate: impl FnMut(&QueueItem) -> Result<(OutputMetadata, rendered::RenderedVideo), String>,
) -> Result<(OutputMetadata, rendered::RenderedVideo), String> {
let single = item.request.single_shot.as_ref().ok_or("Missing Single Shot settings.")?;
if item.cancel.load(Ordering::Acquire) { return Err("Single Shot generation cancelled.".into()); }
// Keystone supplies both original-clip anchors in a single continuation pass.
// A separately generated ending is incompatible with those anchors.
if item.request.keystone_path.is_some() { return generate(item); }
let temporary = tempfile::tempdir().map_err(|error| format!("Could not prepare ending latents: {error}"))?;
let ending_path = temporary.path().join("ending.safetensors");
// Condition the ending on the same source as the full shot. An independent
// ending can force a cut to an unrelated subject/scene despite a locked start.
// Only its tail is needed here; avoid decoding the entire previous chain.
let starting_tail = item.request.continuation_path.as_deref()
.map(|path| super::continuation::tail_archive(path, 22)).transpose()?;
let mut ending_tail = None;
let mut elapsed = 0.0;
let mut timings = [0.0; 4];
let mut computed = 0;
let mut skipped = 0;
for index in 0..2 {
if item.cancel.load(Ordering::Acquire) { return Err("Single Shot generation cancelled.".into()); }
let mut request = item.request.clone();
// Clear legacy source initialization before removing single_shot for
// the ending pass. Both passes use fresh noise between fixed anchors.
request.continuation_source_noise = None;
if index == 0 {
// The ending is conditioning for the full pass. Grade Tone only
// once, after the main clip and this neutral ending are joined.
if let Some(effects) = request.generation_effects.as_mut() { effects.tone = None; }
request.single_shot = None;
request.prompt = single.ending_prompt.clone();
request.frames = 22;
request.continuation_relative_path = None;
request.continuation_path = starting_tail.as_ref().map(|path| path.to_path_buf());
request.continuation_source_frames = starting_tail.as_ref().map(|_| 22);
request.continuation_overlap_frames = starting_tail.as_ref().map(|_| 22);
request.continuation_from = starting_tail.as_ref().map(|_| "end".into());
request.continuation_lock_overlap = starting_tail.is_some();
request.save_latents_path = Some(ending_path.clone());
} else {
request.single_shot.as_mut().unwrap().ending_path = Some(ending_tail.as_ref()
.map_or_else(|| ending_path.clone(), |path: &tempfile::TempPath| path.to_path_buf()));
}
let sink = item.events.clone();
let offset = elapsed;
let stage = QueueItem { request, configuration: item.configuration.clone(), references: item.references.clone(), cancel: item.cancel.clone(),
events: std::sync::Arc::new(move |event| {
let event = match event {
super::events::NativeEvent::Progress(mut progress) => {
let steps = progress.planned_steps.max(1);
progress.step = index * steps + progress.step.clamp(0, steps);
progress.total_steps = steps * 2;
progress.planned_steps = progress.total_steps;
progress.elapsed_seconds += offset;
progress.stage = if index == 0 { "singleShotEnding" } else { "singleShotFull" };
progress.timing_profile.push_str("|singleShot=2");
super::events::NativeEvent::Progress(progress)
}
event => event,
};
sink(event);
}),
};
let (mut metadata, result) = generate(&stage)?;
if item.cancel.load(Ordering::Acquire) { return Err("Single Shot generation cancelled.".into()); }
elapsed += metadata.seconds_total;
timings[0] += metadata.seconds_conditioning;
timings[1] += metadata.seconds_denoise;
timings[2] += metadata.seconds_video_decode;
timings[3] += metadata.seconds_audio_decode;
computed += metadata.steps_computed;
skipped += metadata.steps_skipped;
if index == 1 {
metadata.seconds_total = elapsed;
metadata.seconds_conditioning = timings[0]; metadata.seconds_denoise = timings[1];
metadata.seconds_video_decode = timings[2]; metadata.seconds_audio_decode = timings[3];
metadata.steps_computed = computed; metadata.steps_skipped = skipped;
metadata.timing_profile.push_str("|singleShot=2");
return Ok((metadata, result));
}
// The ending is only an anchor, never a separate scene/history result.
// Release its decoded buffers before allocating the full generation.
drop(result);
if !ending_path.is_file() { return Err("The ending generation did not save its latents.".into()); }
if starting_tail.is_some() {
// Continuation saves the joined source + generated clip. Anchor only
// its generated ending, never the retained source at its beginning.
ending_tail = Some(super::continuation::tail_archive(&ending_path, 22)?);
}
}
unreachable!()
}
fn run_edit_sequence(
item: &QueueItem,
mut generate: impl FnMut(&QueueItem) -> Result<(OutputMetadata, rendered::RenderedVideo), String>,
) -> Result<(OutputMetadata, rendered::RenderedVideo), String> {
let Some(edit) = &item.request.image_edit else { return generate(item); };
if item.cancel.load(Ordering::Acquire) { return Err("Image editing cancelled.".into()); }
let count = edit.edits.len();
let mut canvas = if edit.region_size.is_some() { Some(super::image_regions::load_source(&item.request)?) } else { None };
let mut pixels = None;
let mut elapsed = 0.0;
let mut computed = 0;
let mut skipped = 0;
for (index, step) in edit.edits.iter().enumerate() {
if item.cancel.load(Ordering::Acquire) { return Err("Image editing cancelled.".into()); }
let mut request = item.request.clone();
request.prompt = step.prompt.clone();
if let Some(paths) = &step.reference_paths { request.reference_paths = paths.clone(); }
if let Some(refmods) = &step.refmods { request.refmods = refmods.clone(); }
request.image_edit.as_mut().unwrap().edits = vec![step.clone()];
request.image_edit_pixels = pixels.take();
let region = canvas.as_ref().map(|source| super::image_regions::RegionEdit::prepare(&request, source)).transpose()?;
if let Some(region) = ®ion { request = region.request.clone(); }
let cropped = region.is_some();
let sink = item.events.clone();
let offset = elapsed;
let stage = QueueItem {
request, configuration: item.configuration.clone(), references: item.references.clone(), cancel: item.cancel.clone(),
events: std::sync::Arc::new(move |event| {
let event = match event {
// A crop preview has a different coordinate system from the
// original canvas; retain the source until the merged result.
super::events::NativeEvent::Preview(_) if cropped => return,
super::events::NativeEvent::Progress(mut progress) if progress.stage != "calibrating" => {
let per_edit = progress.planned_steps.max(1);
progress.step = index as i32 * per_edit + progress.step.clamp(0, per_edit);
progress.total_steps = per_edit * count as i32;
progress.planned_steps = progress.total_steps;
progress.elapsed_seconds += offset;
progress.timing_profile.push_str(&format!("|imageEdits={count}"));
super::events::NativeEvent::Progress(progress)
}
event => event,
};
sink(event);
}),
};
let (mut metadata, result) = generate(&stage)?;
if item.cancel.load(Ordering::Acquire) { return Err("Image editing cancelled.".into()); }
if let (Some(region), Some(canvas)) = (®ion, &mut canvas) { region.merge(canvas, &result)?; }
elapsed += metadata.seconds_total;
computed += metadata.steps_computed;
skipped += metadata.steps_skipped;
if index + 1 == count {
metadata.seconds_total = elapsed;
metadata.steps_computed = computed;
metadata.steps_skipped = skipped;
metadata.timing_profile.push_str(&format!("|imageEdits={count}"));
if let Some(canvas) = canvas {
metadata.width = canvas.width() as i32;
metadata.height = canvas.height() as i32;
return Ok((metadata, super::image_regions::rendered_image(&item.request.job_id, canvas)?));
}
return Ok((metadata, result));
}
if canvas.is_some() { continue; }
let rgba = result.frame(0)?.ok_or("Image edit returned no pixels.")?;
// Copy only the completed still, then release its generation before
// starting the next edit. No intermediate file or lossy encode.
pixels = Some(std::sync::Arc::new(rgba.chunks_exact(4).flat_map(|pixel| pixel[..3].iter().copied()).collect()));
}
Err("Add an image edit before generating.".into())
}
fn run_single_generation(
item: &QueueItem,
) -> Result<(OutputMetadata, rendered::RenderedVideo), String> {
let _models = super::MODEL_ACCESS
.read()
.map_err(|_| "Model access lock failed.")?;
if item.cancel.load(Ordering::Acquire) {
return Err("Generation was cancelled before it started.".into());
}
let api = ffi::Api::load(&item.configuration.dll_path)?;
let version = api.version()?;
let platform = item.configuration.platform(detect_platform(&api));
let timing_profile = item.configuration.timing_profile(&version, platform);
diagnostics::debug(
"slopfab",
"generation.runtime_ready",
"slopfab runtime loaded.",
serde_json::json!({
"jobId": item.request.job_id, "version": version, "platform": platform.label(),
}),
);
let request = RequestHandle::new(&api)?;
let _blank_reference = configure_request(
&api,
&request,
&item.request,
&item.configuration,
platform,
RequestPurpose::Generate,
&item.references,
)?;
let effects_active = super::lcs::prepare(&api, &request, item, platform)?;
let plan = api.resolve(&request)?;
if effects_active {
if let Some(mask) = item.request.generation_effects.as_ref().unwrap().mask(plan.latent_width / 2, plan.latent_height / 2)? {
api.set_lcs_mask(&request, &mask)?;
}
}
if item.cancel.load(Ordering::Acquire) { return Err("Generation cancelled.".into()); }
diagnostics::debug(
"slopfab",
"generation.request_resolved",
"slopfab resolved the generation request.",
serde_json::json!({ "jobId": item.request.job_id }),
);
let context = CallbackContext {
events: item.events.clone(),
job_id: item.request.job_id.clone(),
frames: item.request.frames,
canvas_width: item.request.canvas_width,
canvas_height: item.request.canvas_height,
reference_count: item.request.reference_count(),
timing_profile: timing_profile.clone(),
planned_steps: plan.num_model_evaluations,
};
let preview = item.configuration.preview_vae.as_ref()
.filter(|path| path.is_file() && item.configuration.model_type == "minimax-h3" && item.request.music.is_none())
.map(|path| (path.clone(), super::events::preview_sink(item.events.clone(), item.request.job_id.clone())));
let mut generation = request.start_with_preview(Some(progress_sink(context)), preview)?;
diagnostics::debug(
"slopfab",
"generation.api_started",
"slopfab accepted the generation request.",
serde_json::json!({ "jobId": item.request.job_id }),
);
loop {
if item.cancel.load(Ordering::Acquire) {
generation.cancel();
}
if generation.wait(100)? {
break;
}
}
let generation = generation.finish()?;
let output = generation.output()?;
// Diagnostics must never turn a successfully generated image into a failure.
match generation.video_decode_timings() {
Ok(Some(timings)) => diagnostics::info("slopfab", "generation.video_decode", "VAE decode timing breakdown.", serde_json::json!({
"jobId": item.request.job_id, "backend": platform.label(), "runtimeVersion": version,
"stillImage": item.request.still_image, "width": output.width, "height": output.height, "frames": output.frames,
"secondsTotal": output.seconds_video_decode, "timings": timings,
})),
Ok(None) => {}, // Older runtimes keep their aggregate decode timing.
Err(error) => diagnostics::warn("slopfab", "generation.decode_timing_unavailable", &error, serde_json::json!({ "jobId": item.request.job_id })),
}
let source = Box::new(FrameSource { generation });
collect_output(item, output, source, timing_profile)
}
fn run_latent_upscale(item: &QueueItem) -> Result<(OutputMetadata, rendered::RenderedVideo), String> {
let _models = super::MODEL_ACCESS.read().map_err(|_| "Model access lock failed.")?;
if item.cancel.load(Ordering::Acquire) { return Err("Upscaling cancelled.".into()); }
let config = &item.configuration;
config.validate_inputs(&item.request)?;
let plan = super::latent_upscale::plan(&item.request)?;
let api = ffi::Api::load(&config.dll_path)?;
let platform = config.platform(detect_platform(&api));
let timing_profile = format!("{}|savedLatentUpscale", config.timing_profile(&api.version()?, platform));
let source = item.request.latent_upscale_source.as_ref().unwrap();
let context = CallbackContext { events: item.events.clone(), job_id: item.request.job_id.clone(), frames: plan.aligned_frames,
canvas_width: plan.canvas_width, canvas_height: plan.canvas_height, reference_count: 0, timing_profile: timing_profile.clone(), planned_steps: 0 };
let mut generation = ffi::GenerationHandle::upscale(&api, &source.path, config.latent_upscaler.as_ref().unwrap(), source.scale,
config.models[3].2.as_ref().unwrap(), config.models[4].2.as_ref().unwrap(), platform.backend(), progress_sink(context))?;
loop {
if item.cancel.load(Ordering::Acquire) { generation.cancel(); }
if generation.wait(100)? { break; }
}
let generation = generation.finish()?;
if item.cancel.load(Ordering::Acquire) { return Err("Upscaling cancelled.".into()); }
let destination = item.request.save_latents_path.as_ref().ok_or("Missing destination for enlarged latents.")?;
if destination.exists() { return Err("Latents already exist. Start a new upscale to preserve them.".into()); }
generation.save_latents(destination)?;
let output = generation.output()?;
collect_output(item, output, Box::new(FrameSource { generation }), timing_profile)
}
fn collect_output(item: &QueueItem, output: ffi::Output, source: Box<dyn rendered::FrameSource>, timing_profile: String) -> Result<(OutputMetadata, rendered::RenderedVideo), String> {
let frames = if item.request.music.is_some() { 0 } else { scene_frame_window(&item.request, output.frames)?.1 };
// The VAE decoded both sides of the boundary together. Keep that complete
// output, including the source-side pixels and soundtrack near the join.
let audio = output.audio;
let metadata = OutputMetadata {
frames, width: output.width, height: output.height, fps: output.fps,
audio_channels: output.audio_channels, audio_sample_rate: output.audio_sample_rate,
audio_samples: audio.len(),
reference_count: item.request.reference_count(),
timing_profile,
seconds_conditioning: output.seconds_conditioning,
seconds_denoise: output.seconds_denoise,
seconds_video_decode: output.seconds_video_decode,
seconds_audio_decode: output.seconds_audio_decode,
seconds_total: output.seconds_total,
steps_computed: output.steps_computed,
steps_skipped: output.steps_skipped,
duration_seconds: if item.request.music.is_some() && output.audio_channels > 0 && output.audio_sample_rate > 0 {
audio.len() as f64 / output.audio_channels as f64 / output.audio_sample_rate as f64
} else if output.fps > 0.0 {
frames as f64 / output.fps
} else {
0.0
},
boundary: if item.request.music.is_some() { "Music samples are ready to save as WAV." } else { "Latents are saved in the project. Decoded buffers remain owned by slopfab; the webview encodes the full joined video and audio on demand." },
};
let pictures = rendered::from_source(
&item.request.job_id,
source,
audio,
frames as u32,
output.width.max(0) as u32,
output.height.max(0) as u32,
output.channels.max(0) as u32,
output.video_float_count,
output.fps,
output.audio_channels.max(0) as u32,
output.audio_sample_rate.max(0) as u32,
)?;
Ok((metadata, pictures))
}