Skip to content

AI media harness for planning, generating, and editing locally

TypeScript
19.5 KB
Raw Blame
use crate::media::artifacts::*;
use crate::project::{validation::validate_and_normalize_config, *};
use crate::{diagnostics, slopfab, worker::Workers};
use tauri::{AppHandle, Emitter};
use tauri_plugin_dialog::DialogExt;
/// Generation commands run on the native queue here, or on the LAN worker
/// selected in Settings → Workers. The webview cannot tell the difference.
async fn blocking<T: Send + 'static>(
    work: impl FnOnce() -> Result<T, String> + Send + 'static,
) -> Result<T, String> {
    tauri::async_runtime::spawn_blocking(work)
        .await
        .map_err(|error| error.to_string())?
}

#[tauri::command]
pub(crate) async fn create_reference_video(
    state: tauri::State<'_, slopfab::SlopfabRuntime>,
    workers: tauri::State<'_, Workers>,
    duration_seconds: f64,
    config: ProjectConfig,
) -> Result<String, String> {
    let (runtime, workers) = (state.inner().clone(), workers.inner().clone());
    blocking(move || {
        let config = validate_and_normalize_config(config)?;
        match workers.active()? {
            Some(worker) => worker.post(
                "/v1/reference-videos",
                &serde_json::json!({ "durationSeconds": duration_seconds }),
            ),
            None => runtime
                .references
                .create_reference_video(duration_seconds, &config.provider_settings),
        }
    })
    .await
}

pub(crate) fn reference_video_header<'a>(
    request: &'a tauri::ipc::Request<'_>,
    name: &str,
) -> Result<&'a str, String> {
    request
        .headers()
        .get(name)
        .and_then(|value| value.to_str().ok())
        .ok_or_else(|| format!("Missing or invalid {name} header."))
}

#[tauri::command]
pub(crate) async fn append_reference_video(
    state: tauri::State<'_, slopfab::SlopfabRuntime>,
    workers: tauri::State<'_, Workers>,
    request: tauri::ipc::Request<'_>,
) -> Result<(), String> {
    let tauri::ipc::InvokeBody::Raw(bytes) = request.body() else {
        return Err("Reference frames require a binary body.".into());
    };
    let id = reference_video_header(&request, "x-reference-id")?.to_string();
    let width: i32 = reference_video_header(&request, "x-reference-width")?
        .parse()
        .map_err(|_| "Invalid frame width.")?;
    let height: i32 = reference_video_header(&request, "x-reference-height")?
        .parse()
        .map_err(|_| "Invalid frame height.")?;
    let timestamp: f64 = reference_video_header(&request, "x-reference-time")?
        .parse()
        .map_err(|_| "Invalid frame timestamp.")?;
    let bytes = bytes.clone();
    let (runtime, workers) = (state.inner().clone(), workers.inner().clone());
    blocking(move || match workers.active()? {
        Some(worker) => worker
            .post_bytes::<bool>(
                &format!("/v1/reference-videos/{id}/frames"),
                &[
                    ("x-reference-width", width.to_string()),
                    ("x-reference-height", height.to_string()),
                    ("x-reference-time", timestamp.to_string()),
                ],
                bytes,
            )
            .map(|_| ()),
        None => runtime
            .references
            .append_reference_video(&id, &bytes, width, height, timestamp),
    })
    .await
}

#[tauri::command]
pub(crate) async fn set_reference_video_audio(
    state: tauri::State<'_, slopfab::SlopfabRuntime>,
    workers: tauri::State<'_, Workers>,
    request: tauri::ipc::Request<'_>,
) -> Result<(), String> {
    let tauri::ipc::InvokeBody::Raw(bytes) = request.body() else {
        return Err("Reference audio requires a binary body.".into());
    };
    let id = reference_video_header(&request, "x-reference-id")?.to_string();
    let channels: i32 = reference_video_header(&request, "x-reference-channels")?
        .parse()
        .map_err(|_| "Invalid audio channels.")?;
    let rate: i32 = reference_video_header(&request, "x-reference-rate")?
        .parse()
        .map_err(|_| "Invalid sample rate.")?;
    let bytes = bytes.clone();
    let (runtime, workers) = (state.inner().clone(), workers.inner().clone());
    blocking(move || match workers.active()? {
        Some(worker) => worker
            .post_bytes::<bool>(
                &format!("/v1/reference-videos/{id}/audio"),
                &[
                    ("x-reference-channels", channels.to_string()),
                    ("x-reference-rate", rate.to_string()),
                ],
                bytes,
            )
            .map(|_| ()),
        None => runtime
            .references
            .set_reference_video_audio(&id, &bytes, channels, rate),
    })
    .await
}

#[tauri::command]
pub(crate) async fn release_reference_videos(
    state: tauri::State<'_, slopfab::SlopfabRuntime>,
    workers: tauri::State<'_, Workers>,
    ids: Vec<String>,
) -> Result<(), String> {
    let (runtime, workers) = (state.inner().clone(), workers.inner().clone());
    blocking(move || match workers.active()? {
        Some(worker) => worker
            .post::<bool>("/v1/reference-videos/release", &ids)
            .map(|_| ()),
        None => runtime.references.release_reference_videos(&ids),
    })
    .await
}

#[tauri::command]
pub(crate) async fn create_reference_audio(
    state: tauri::State<'_, slopfab::SlopfabRuntime>,
    workers: tauri::State<'_, Workers>,
    request: tauri::ipc::Request<'_>,
) -> Result<String, String> {
    let tauri::ipc::InvokeBody::Raw(bytes) = request.body() else {
        return Err("Reference audio requires a binary body.".into());
    };
    let channels: i32 = reference_video_header(&request, "x-reference-channels")?
        .parse()
        .map_err(|_| "Invalid audio channels.")?;
    let rate: i32 = reference_video_header(&request, "x-reference-rate")?
        .parse()
        .map_err(|_| "Invalid sample rate.")?;
    let bytes = bytes.clone();
    let (runtime, workers) = (state.inner().clone(), workers.inner().clone());
    blocking(move || match workers.active()? {
        Some(worker) => worker.post_bytes(
            "/v1/reference-audios",
            &[
                ("x-reference-channels", channels.to_string()),
                ("x-reference-rate", rate.to_string()),
            ],
            bytes,
        ),
        None => runtime
            .references
            .create_reference_audio(&bytes, channels, rate),
    })
    .await
}

#[tauri::command]
pub(crate) async fn release_reference_audios(
    state: tauri::State<'_, slopfab::SlopfabRuntime>,
    workers: tauri::State<'_, Workers>,
    ids: Vec<String>,
) -> Result<(), String> {
    let (runtime, workers) = (state.inner().clone(), workers.inner().clone());
    blocking(move || match workers.active()? {
        Some(worker) => worker
            .post::<bool>("/v1/reference-audios/release", &ids)
            .map(|_| ()),
        None => runtime.references.release_reference_audios(&ids),
    })
    .await
}

#[tauri::command]
pub(crate) async fn resolve_slopfab_plan(
    state: tauri::State<'_, slopfab::SlopfabRuntime>,
    workers: tauri::State<'_, Workers>,
    mut request: slopfab::GenerationRequest,
    config: ProjectConfig,
    folder_path: Option<String>,
) -> Result<serde_json::Value, String> {
    let (runtime, workers) = (state.inner().clone(), workers.inner().clone());
    blocking(move || {
        prepare_continuation_path(&mut request, folder_path.as_deref())?;
        prepare_image_edit_path(&mut request, folder_path.as_deref())?;
        let context = serde_json::json!({
            "jobId": request.job_id, "frames": request.frames, "steps": request.steps,
            "canvasWidth": request.canvas_width, "canvasHeight": request.canvas_height,
            "referenceCount": request.reference_count(), "refmodCount": request.refmod_count(), "inputChars": request.prompt.chars().count(),
        });
        diagnostics::info(
            "slopfab",
            "plan.requested",
            "Generation plan requested.",
            context.clone(),
        );
        let result = validate_and_normalize_config(config).and_then(|config| {
            match workers.active()? {
                Some(worker) => workers.plan(&worker, &request, &config.provider_settings),
                None => slopfab::resolve_plan(&request, &config.provider_settings, &runtime.references)
                    .and_then(|plan| serde_json::to_value(plan).map_err(|error| error.to_string())),
            }
        });
        match &result {
            Ok(plan) => diagnostics::info(
                "slopfab",
                "plan.resolved",
                "Generation plan resolved.",
                serde_json::json!({
                    "jobId": request.job_id, "frames": plan["alignedFrames"],
                    "canvasWidth": plan["canvasWidth"], "canvasHeight": plan["canvasHeight"],
                    "durationSeconds": plan["durationSeconds"], "modelEvaluations": plan["modelEvaluations"],
                }),
            ),
            Err(error) => diagnostics::error("slopfab", "plan.failed", error, context),
        }
        result
    })
    .await
}

#[tauri::command]
pub(crate) async fn enqueue_slopfab_generation(
    app: AppHandle,
    state: tauri::State<'_, slopfab::SlopfabRuntime>,
    workers: tauri::State<'_, Workers>,
    mut request: slopfab::GenerationRequest,
    config: ProjectConfig,
    folder_path: String,
) -> Result<(), String> {
    let (runtime, workers) = (state.inner().clone(), workers.inner().clone());
    blocking(move || {
        prepare_continuation_path(&mut request, Some(&folder_path))?;
        prepare_image_edit_path(&mut request, Some(&folder_path))?;
        if !request.still_image && request.music.is_none() {
            request.save_latents_path = Some(generated_latent_destination(&folder_path, &request.job_id)?);
        }
        let job_id = request.job_id.clone();
        let result = validate_and_normalize_config(config).and_then(|config| {
            match workers.active()? {
                Some(worker) => workers.enqueue(worker, request, &config.provider_settings),
                None => runtime.enqueue(
                    std::sync::Arc::new(move |event| match event {
                        slopfab::NativeEvent::Job(value) => {
                            let _ = app.emit("slopfab-job", value);
                        }
                        slopfab::NativeEvent::Progress(value) => {
                            let _ = app.emit("slopfab-progress", value);
                        }
                        slopfab::NativeEvent::Preview(value) => {
                            let _ = app.emit("slopfab-preview", value);
                        }
                    }),
                    request,
                    &config.provider_settings,
                ),
            }
        });
        if let Err(error) = &result {
            diagnostics::error(
                "slopfab",
                "generation.enqueue_failed",
                error,
                serde_json::json!({ "jobId": job_id }),
            );
        }
        result
    })
    .await
}

/// Refmod destinations picked in the save dialog during this run. An export
/// writes only to one of these, so the webview cannot name its own target.
#[derive(Clone, Default)]
pub(crate) struct RefmodExportTargets(std::sync::Arc<std::sync::Mutex<std::collections::HashSet<String>>>);

#[tauri::command]
pub(crate) async fn choose_refmod_export_path(
    app: AppHandle,
    targets: tauri::State<'_, RefmodExportTargets>,
    name: String,
) -> Result<Option<String>, String> {
    let targets = targets.inner().clone();
    blocking(move || {
        let Some(destination) = app.dialog().file()
            .set_title("Export refmod")
            .add_filter("Refmod", &["safetensors"])
            .set_file_name(format!("{}.safetensors", paths::safe_folder_name(&name)))
            .blocking_save_file() else {
            return Ok(None);
        };
        let destination = destination.into_path().map_err(|_| "Choose a local refmod destination.")?;
        let destination = if destination.extension().is_some_and(|value| value.eq_ignore_ascii_case("safetensors")) {
            destination
        } else {
            destination.with_extension("safetensors")
        };
        let destination = paths::display_path(&destination);
        targets.0.lock().map_err(|_| "Refmod export lock failed.")?.insert(destination.clone());
        Ok(Some(destination))
    })
    .await
}

/// Encodes a reference's prepared images, video and sound into a refmod file.
#[tauri::command]
pub(crate) async fn export_reference_refmod(
    state: tauri::State<'_, slopfab::SlopfabRuntime>,
    workers: tauri::State<'_, Workers>,
    targets: tauri::State<'_, RefmodExportTargets>,
    request: slopfab::GenerationRequest,
    config: ProjectConfig,
    output_path: String,
    name: String,
    description: String,
) -> Result<(), String> {
    let (runtime, workers, targets) = (state.inner().clone(), workers.inner().clone(), targets.inner().clone());
    blocking(move || {
        if !targets.0.lock().map_err(|_| "Refmod export lock failed.")?.remove(&output_path) {
            return Err("Choose where to save the refmod before exporting it.".into());
        }
        let output = std::path::Path::new(&output_path);
        let context = serde_json::json!({ "jobId": request.job_id, "referenceCount": request.reference_count() });
        let result = validate_and_normalize_config(config).and_then(|config| match workers.active()? {
            Some(worker) => workers.export_refmod(&worker, &request, &config.provider_settings, &name, &description, output),
            None => slopfab::export_refmod(&request, &config.provider_settings, &runtime.references, output, &name, &description),
        });
        match &result {
            Ok(()) => diagnostics::info("slopfab", "refmod.exported", "Reference exported as a refmod.", context),
            Err(error) => diagnostics::error("slopfab", "refmod.export_failed", error, context),
        }
        result
    })
    .await
}

fn prepare_image_edit_path(request: &mut slopfab::GenerationRequest, folder_path: Option<&str>) -> Result<(), String> {
    if let Some(edit) = &request.image_edit {
        if !(1..=8192).contains(&request.canvas_width) || !(1..=8192).contains(&request.canvas_height) {
            return Err("Image edit dimensions must be between 1 and 8192 pixels.".into());
        }
        let root = crate::project::paths::ProjectRoot::open(folder_path.ok_or("Image edits require a project folder.")?)?;
        let mut path = root.existing(&edit.source_relative_path)?;
        let (width, height) = ::image::image_dimensions(&path).map_err(|error| format!("Could not read source image: {error}"))?;
        if edit.region_size.is_some() && (width != request.canvas_width as u32 || height != request.canvas_height as u32) {
            return Err("The source image dimensions changed. Reopen the image before editing.".into());
        }
        if width != request.canvas_width as u32 || height != request.canvas_height as u32 {
            // Native inpainting takes its canvas dimensions from the source file.
            // Prepare the selected size before planning or uploading to a LAN worker.
            use std::hash::{Hash, Hasher};
            let mut hash = std::collections::hash_map::DefaultHasher::new();
            (&request.job_id, &path, request.canvas_width, request.canvas_height).hash(&mut hash);
            let stem = generated_file_stem(&request.job_id)?;
            let directory = root.directory(&format!("cache/image-edits/{stem}"))?;
            let destination = directory.join(format!("{:016x}.png", hash.finish()));
            let resized = ::image::open(&path).map_err(|error| format!("Could not read source image: {error}"))?
                .resize_exact(request.canvas_width as u32, request.canvas_height as u32, ::image::imageops::FilterType::Lanczos3);
            let mut temporary = tempfile::NamedTempFile::new_in(&directory).map_err(|error| error.to_string())?;
            resized.write_to(&mut temporary, ::image::ImageFormat::Png).map_err(|error| error.to_string())?;
            temporary.persist(&destination).map_err(|error| error.to_string())?;
            path = destination;
        }
        request.image_edit_path = Some(path);
    }
    Ok(())
}

#[cfg(test)]
mod image_edit_tests {
    use super::*;

    #[test]
    fn image_edit_prepares_selected_dimensions_without_changing_the_original() {
        let folder = tempfile::tempdir().unwrap();
        std::fs::write(folder.path().join("slopus.json"), "{}").unwrap();
        let source = folder.path().join("source.png");
        ::image::RgbaImage::from_pixel(101, 77, ::image::Rgba([80, 120, 160, 128])).save(&source).unwrap();
        let original = std::fs::read(&source).unwrap();
        let mut request: slopfab::GenerationRequest = serde_json::from_value(serde_json::json!({
            "jobId":"edit-test", "prompt":"Warm up the lighting", "frames":1, "stillImage":true,
            "steps":32, "seed":123456, "canvasWidth":96, "canvasHeight":64,
            "imageEdit":{"sourceRelativePath":"source.png", "edits":[{"prompt":"Warm up the lighting", "x":0,"y":0,"width":96,"height":64}]}
        })).unwrap();
        for (width, height) in [(96, 64), (192, 128), (192, 128), (101, 77)] {
            request.canvas_width = width;
            request.canvas_height = height;
            prepare_image_edit_path(&mut request, folder.path().to_str()).unwrap();
            let prepared = ::image::open(request.image_edit_path.as_ref().unwrap()).unwrap().to_rgba8();
            assert_eq!(prepared.dimensions(), (width as u32, height as u32));
            assert_eq!(prepared.get_pixel(0, 0).0, [80, 120, 160, 128]);
            assert_eq!(std::fs::read(&source).unwrap(), original);
        }
        request.canvas_width = -1;
        assert!(prepare_image_edit_path(&mut request, folder.path().to_str()).is_err());
        request.canvas_width = 8193;
        assert!(prepare_image_edit_path(&mut request, folder.path().to_str()).is_err());
    }

    #[test]
    fn regional_edit_keeps_the_original_file_for_local_and_worker_generation() {
        let folder = tempfile::tempdir().unwrap();
        std::fs::write(folder.path().join("slopus.json"), "{}").unwrap();
        let source = folder.path().join("source.png");
        ::image::RgbaImage::from_pixel(1001, 809, ::image::Rgba([80, 120, 160, 128])).save(&source).unwrap();
        let mut request: slopfab::GenerationRequest = serde_json::from_value(serde_json::json!({
            "jobId":"regional", "prompt":"Flower", "frames":1, "stillImage":true, "steps":4, "seed":42, "canvasWidth":1001, "canvasHeight":809,
            "imageEdit":{"sourceRelativePath":"source.png", "regionSize":{"width":672,"height":544},
                "edits":[{"prompt":"Flower", "x":700,"y":600,"width":100,"height":100}]}
        })).unwrap();
        prepare_image_edit_path(&mut request, folder.path().to_str()).unwrap();
        assert_eq!(std::fs::canonicalize(request.image_edit_path.as_ref().unwrap()).unwrap(), std::fs::canonicalize(source).unwrap());
        assert!(!folder.path().join("cache/image-edits").exists());
        request.canvas_width = 672;
        assert!(prepare_image_edit_path(&mut request, folder.path().to_str()).unwrap_err().contains("dimensions changed"));
    }
}

#[tauri::command]
pub(crate) async fn cancel_slopfab_generation(
    state: tauri::State<'_, slopfab::SlopfabRuntime>,
    workers: tauri::State<'_, Workers>,
    job_id: String,
) -> Result<bool, String> {
    let (runtime, workers) = (state.inner().clone(), workers.inner().clone());
    // A job sent to a worker is cancelled there, whichever worker is selected now.
    blocking(move || Ok(workers.cancel(&job_id) || runtime.cancel(&job_id))).await
}