Skip to content

AI media harness for planning, generating, and editing locally

TypeScript
38.5 KB
Raw Blame
//! The headless worker: the desktop app's generation queue behind a small
//! HTTP API. Every route runs on its own thread; generation itself stays on
//! the one serial native queue.
use super::{
    discovery,
    protocol::*,
    store::FileStore,
};
use crate::{
    diagnostics,
    names::generated_file_stem,
    settings::{ProviderOption, ProviderSetting},
    rendered, slopfab,
};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use std::{
    collections::{BTreeMap, HashMap},
    fs,
    io::Read,
    path::PathBuf,
    sync::{
        atomic::{AtomicBool, Ordering},
        Arc, Condvar, Mutex,
    },
    time::{Duration, Instant},
};
use tiny_http::{Header, Method, Request, Response, Server};

/// Which GPU backend the worker runs generation on.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum Backend {
    /// CUDA 13, then CUDA 12, when available; otherwise Vulkan.
    #[default]
    Auto,
    /// CUDA 13, then CUDA 12; an error when neither loads.
    Cuda,
    Cuda13,
    Cuda12,
    Vulkan,
}

impl Backend {
    fn is_cuda(self) -> bool {
        matches!(self, Self::Cuda | Self::Cuda13 | Self::Cuda12)
    }

    /// The engine's CUDA toolkit request.
    pub fn cuda_version(self) -> &'static str {
        match self {
            Self::Cuda13 => "13",
            Self::Cuda12 => "12",
            _ => "auto",
        }
    }
}

impl std::str::FromStr for Backend {
    type Err = String;
    fn from_str(value: &str) -> Result<Self, String> {
        match value.to_ascii_lowercase().replace(['-', ' ', '.'], "").as_str() {
            "auto" => Ok(Self::Auto),
            "cuda" => Ok(Self::Cuda),
            "cuda13" => Ok(Self::Cuda13),
            "cuda12" => Ok(Self::Cuda12),
            "vulkan" => Ok(Self::Vulkan),
            _ => Err(format!("Unknown backend '{value}'. Use auto, cuda, cuda13, cuda12 or vulkan.")),
        }
    }
}

/// The backend to run on, and a warning to show when Auto had to fall back.
/// Forcing CUDA where it cannot load is an error rather than a silent Vulkan run.
pub fn choose_backend(requested: Backend, probe: &slopfab::BackendProbe) -> Result<(bool, Option<String>), String> {
    let problem = probe.cuda_problem.as_deref().unwrap_or("CUDA was not found");
    let flag = match requested {
        Backend::Cuda13 => "--backend cuda13",
        Backend::Cuda12 => "--backend cuda12",
        _ => "--backend cuda",
    };
    let install = match requested {
        _ if !cfg!(windows) => "Install the NVIDIA driver and CUDA 13 cuBLAS (the Linux runtime is built for CUDA 13)",
        Backend::Cuda13 => "Install the CUDA 13 toolkit",
        Backend::Cuda12 => "Install the CUDA 12.8 toolkit",
        _ => "Install CUDA 12.8 (RTX 30/40) or CUDA 13 (RTX 50)",
    };
    match (requested, probe.cuda) {
        (Backend::Vulkan, _) => Ok((true, None)),
        (_, Some(_)) => Ok((false, None)),
        (backend, None) if backend.is_cuda() && probe.nvidia_gpu => Err(format!("{flag}: CUDA is not available ({problem}). {install}, or use --backend vulkan.")),
        (backend, None) if backend.is_cuda() => Err(format!("{flag}: no NVIDIA GPU was found. Use --backend vulkan.")),
        (Backend::Auto, None) if probe.nvidia_gpu => Ok((true, Some(format!(
            "WARNING: This NVIDIA GPU supports CUDA, but CUDA was not detected ({problem}). Using the slower Vulkan backend. {} for faster generation.",
            if cfg!(windows) { "Install CUDA 12.8 (RTX 30/40) or CUDA 13 (RTX 50)" } else { "Install CUDA 13 cuBLAS" }
        )))),
        (_, None) => Ok((true, None)),
    }
}

#[derive(Debug, Clone)]
pub struct WorkerOptions {
    pub port: u16,
    pub name: Option<String>,
    pub token: Option<String>,
    pub data: Option<PathBuf>,
    pub weights: Vec<PathBuf>,
    pub backend: Backend,
    pub advertise: bool,
}

impl Default for WorkerOptions {
    fn default() -> Self {
        Self { port: DEFAULT_PORT, name: None, token: None, data: None, weights: Vec::new(), backend: Backend::Auto, advertise: true }
    }
}

struct JobLog {
    events: Mutex<Vec<WireEvent>>,
    changed: Condvar,
    latents: Option<PathBuf>,
    created: Instant,
    released: AtomicBool,
}

impl JobLog {
    fn push(&self, kind: &str, payload: serde_json::Value) {
        // The client can disappear while native generation is finishing.
        // Its result is stored before this callback runs, so a release that
        // raced with that hand-off must also discard the late result.
        if self.released.load(Ordering::Acquire) {
            if kind == "job" && payload["state"] == "framesReady" {
                if let Some(id) = payload["jobId"].as_str() {
                    rendered::release(id);
                }
            }
            return;
        }
        if let Ok(mut events) = self.events.lock() {
            if kind == "preview" || kind == "job" && matches!(payload["state"].as_str(), Some("failed" | "cancelled" | "framesReady")) {
                // Keep sequence cursors stable but retain only the latest preview payload.
                for event in events.iter_mut().filter(|event| event.kind == "preview") {
                    event.payload = serde_json::Value::Null;
                }
            }
            let seq = events.len() as u64 + 1;
            events.push(WireEvent { seq, kind: kind.into(), payload });
        }
        self.changed.notify_all();
    }

    fn release(&self, id: &str) {
        self.released.store(true, Ordering::Release);
        rendered::release(id);
    }

    /// Long poll: returns as soon as anything after `after` exists.
    fn after(&self, after: u64, wait: Duration) -> Vec<WireEvent> {
        let Ok(events) = self.events.lock() else { return Vec::new() };
        let (events, _) = self
            .changed
            .wait_timeout_while(events, wait, |events| events.len() as u64 <= after)
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        events.iter().filter(|event| event.seq > after).cloned().collect()
    }
}

struct Preparation {
    status: Mutex<FilePreparation>,
    cancel: Arc<AtomicBool>,
    created: Instant,
}

struct Worker {
    id: String,
    name: String,
    token: Option<String>,
    vulkan: bool,
    data: PathBuf,
    store: Arc<FileStore>,
    runtime: slopfab::SlopfabRuntime,
    jobs: Mutex<HashMap<String, Arc<JobLog>>>,
    preparations: Mutex<HashMap<String, Arc<Preparation>>>,
    upscales: Mutex<HashMap<String, Arc<crate::upscale_stream::Session>>>,
}

struct HttpError(u16, String);

impl From<String> for HttpError {
    fn from(message: String) -> Self {
        Self(400, message)
    }
}
impl From<&str> for HttpError {
    fn from(message: &str) -> Self {
        Self(400, message.into())
    }
}

type Reply = Result<Response<Box<dyn Read + Send>>, HttpError>;

fn json<T: Serialize>(value: &T) -> Reply {
    let body = serde_json::to_vec(value).map_err(|error| HttpError(500, error.to_string()))?;
    Ok(bytes(body, "application/json"))
}

fn bytes(body: Vec<u8>, kind: &str) -> Response<Box<dyn Read + Send>> {
    let length = body.len();
    Response::new(
        200.into(),
        vec![Header::from_bytes("Content-Type", kind).unwrap()],
        Box::new(std::io::Cursor::new(body)) as Box<dyn Read + Send>,
        Some(length),
        None,
    )
}

fn body<T: DeserializeOwned>(request: &mut Request) -> Result<T, HttpError> {
    let mut text = Vec::new();
    request
        .as_reader()
        .take(64 * 1024 * 1024)
        .read_to_end(&mut text)
        .map_err(|error| HttpError(400, error.to_string()))?;
    serde_json::from_slice(&text).map_err(|error| HttpError(400, format!("Invalid request body: {error}")))
}

fn header<'a>(request: &'a Request, name: &str) -> Option<&'a str> {
    request
        .headers()
        .iter()
        .find(|header| header.field.as_str().as_str().eq_ignore_ascii_case(name))
        .map(|header| header.value.as_str())
}

fn parsed_header<T: std::str::FromStr>(request: &Request, name: &str) -> Result<T, HttpError> {
    header(request, name)
        .and_then(|value| value.parse().ok())
        .ok_or_else(|| HttpError(400, format!("Missing or invalid {name} header.")))
}

fn data_directory(options: &WorkerOptions) -> PathBuf {
    options.data.clone().unwrap_or_else(|| {
        if cfg!(windows) {
            std::env::var_os("LOCALAPPDATA").map(PathBuf::from).unwrap_or_else(std::env::temp_dir).join("Slopus Worker")
        } else {
            // XDG base directories: $XDG_DATA_HOME, else ~/.local/share.
            std::env::var_os("XDG_DATA_HOME").map(PathBuf::from)
                .or_else(|| std::env::var_os("HOME").map(|home| PathBuf::from(home).join(".local/share")))
                .unwrap_or_else(std::env::temp_dir)
                .join("slopus-worker")
        }
    })
}

fn worker_id(data: &std::path::Path) -> Result<String, String> {
    let path = data.join("worker-id");
    if let Ok(id) = fs::read_to_string(&path) {
        if valid_key(id.trim()) {
            return Ok(id.trim().to_string());
        }
    }
    let mut random = [0u8; 8];
    getrandom::fill(&mut random).map_err(|error| error.to_string())?;
    let id = format!("{:016x}", u64::from_le_bytes(random));
    fs::write(&path, &id).map_err(|error| format!("Could not write {}: {error}", path.display()))?;
    Ok(id)
}

pub fn computer_name() -> String {
    std::env::var("COMPUTERNAME")
        .or_else(|_| std::env::var("HOSTNAME"))
        .ok()
        // Shells set HOSTNAME without exporting it, so services rarely see it.
        .or_else(|| fs::read_to_string("/etc/hostname").ok().map(|name| name.trim().to_string()))
        .filter(|name| !name.trim().is_empty())
        .unwrap_or_else(|| "Slopus worker".into())
}

/// Runs the worker until the process exits.
pub fn run(options: WorkerOptions) -> Result<(), String> {
    let data = data_directory(&options);
    fs::create_dir_all(&data).map_err(|error| format!("Could not create {}: {error}", data.display()))?;
    if let Err(error) = diagnostics::initialize_in(&data.join("logs")) {
        eprintln!("Could not initialize diagnostic logging: {error}");
    }
    let roots = crate::weights::worker_weight_roots(&options.weights, &data)?;
    // Vulkan needs no CUDA probe; the engine then keeps its default toolkit choice.
    let probed = if options.backend == Backend::Vulkan { Err(String::new()) }
        else { slopfab::probe_backend(&slopfab::default_dll_path(), options.backend.cuda_version()) };
    let (vulkan, backend) = match probed {
        Ok(probe) => {
            let (vulkan, warning) = choose_backend(options.backend, &probe)?;
            if let Some(warning) = warning {
                eprintln!("{warning}");
                diagnostics::warn("worker", "backend.fallback", &warning, serde_json::json!({}));
            }
            (vulkan, if vulkan { "Vulkan" } else { probe.cuda.unwrap_or("CUDA") })
        }
        Err(_) if options.backend == Backend::Vulkan => (true, "Vulkan"),
        // The engine could not load; the worker still answers and reports why.
        Err(_) => (false, "unavailable"),
    };
    let worker = Arc::new(Worker {
        id: worker_id(&data)?,
        name: options.name.clone().unwrap_or_else(computer_name),
        token: options.token.clone().filter(|token| !token.is_empty()),
        vulkan,
        store: Arc::new(FileStore::new(&data, roots.clone())?),
        data,
        runtime: slopfab::SlopfabRuntime::default(),
        jobs: Mutex::default(),
        preparations: Mutex::default(),
        upscales: Mutex::default(),
    });
    let server = Server::http(("0.0.0.0", options.port)).map_err(|error| format!("Could not listen on port {}: {error}", options.port))?;
    let cleanup = Arc::downgrade(&worker);
    std::thread::spawn(move || loop {
        std::thread::sleep(Duration::from_secs(60));
        let Some(worker) = cleanup.upgrade() else { break };
        if let Ok(mut sessions) = worker.upscales.lock() {
            sessions.retain(|_, session| {
                let keep = session.idle_for() < Duration::from_secs(30 * 60);
                if !keep { session.cancel(); }
                keep
            });
        };
    });
    let info = worker.info();
    println!("Slopus worker '{}' ({}) listening on port {}.", info.name, info.id, options.port);
    println!("Engine: {} {}", info.runtime.state, info.runtime.version.as_deref().unwrap_or(""));
    println!("Backend: {backend}{}", if options.backend == Backend::Auto { " (automatic; override with --backend)" } else { "" });
    println!("Weights: {}", roots.iter().map(|root| root.display().to_string()).collect::<Vec<_>>().join("; "));
    println!("Data: {}", worker.data.display());
    if worker.token.is_some() {
        println!("Clients must enter this worker's access token.");
    }
    diagnostics::info("worker", "started", "Slopus worker started.", serde_json::json!({ "port": options.port, "id": info.id }));
    let _advertisement = if options.advertise {
        match discovery::advertise(&info, options.port) {
            Ok(daemon) => Some(daemon),
            Err(error) => {
                eprintln!("mDNS advertisement failed; clients must add this worker by address: {error}");
                None
            }
        }
    } else {
        None
    };
    for request in server.incoming_requests() {
        let worker = worker.clone();
        std::thread::spawn(move || worker.handle(request));
    }
    Ok(())
}

impl Worker {
    fn info(&self) -> WorkerInfo {
        let status = slopfab::status(&BTreeMap::new());
        let loaded = matches!(status.state, "ready" | "modelsMissing");
        WorkerInfo {
            id: self.id.clone(),
            name: self.name.clone(),
            version: env!("CARGO_PKG_VERSION").into(),
            protocol: PROTOCOL_VERSION,
            requires_token: self.token.is_some(),
            gpus: crate::weights::hardware_devices(),
            runtime: WorkerRuntime {
                state: if loaded { "ready".into() } else { status.state.into() },
                version: status.version,
                platform: if self.vulkan && loaded { Some("Vulkan".into()) } else { status.platform.map(Into::into) },
                cuda_available: status.cuda_available,
                cuda_device_names: status.cuda_device_names,
                detail: if loaded { format!("Engine loaded from {}.", status.dll_path) } else { status.detail },
            },
        }
    }

    fn handle(&self, mut request: Request) {
        let started = Instant::now();
        let method = request.method().clone();
        let url = request.url().to_string();
        let reply = self.route(&mut request, &method, &url);
        let response = match reply {
            Ok(response) => response,
            Err(HttpError(status, message)) => {
                if status >= 500 || !url.ends_with("/events") {
                    diagnostics::warn("worker", "request.failed", &message, serde_json::json!({ "url": url, "status": status }));
                }
                let body = serde_json::to_vec(&serde_json::json!({ "error": message })).unwrap_or_default();
                bytes(body, "application/json").with_status_code(status)
            }
        };
        let _ = request.respond(response);
        if started.elapsed() > Duration::from_secs(30) && !url.contains("/events") {
            diagnostics::debug("worker", "request.slow", "Slow worker request.", serde_json::json!({ "url": url, "ms": started.elapsed().as_millis() as u64 }));
        }
    }

    fn route(&self, request: &mut Request, method: &Method, url: &str) -> Reply {
        let (path, query) = url.split_once('?').unwrap_or((url, ""));
        let parts: Vec<&str> = path.trim_matches('/').split('/').collect();
        if parts.first() != Some(&"v1") {
            return Err(HttpError(404, "Unknown worker route.".into()));
        }
        if !matches!(parts.as_slice(), ["v1", "info"]) {
            if let Some(token) = &self.token {
                if header(request, TOKEN_HEADER) != Some(token.as_str()) {
                    return Err(HttpError(401, "This worker requires an access token. Enter it in Settings → Workers.".into()));
                }
            }
        }
        match (method, &parts[1..]) {
            (Method::Get, ["info"]) => json(&self.info()),
            (Method::Post, ["upscales", id]) => {
                generated_file_stem(id)?;
                let options: StartUpscale = body(request)?;
                if !(1..=5).contains(&options.segment_frames) || options.width < 16 || options.height < 16 {
                    return Err("Invalid SeedVR2 dimensions or segment size.".into());
                }
                if self.vulkan { return Err("SeedVR2 requires a CUDA worker.".into()); }
                let mut sessions = self.upscales.lock().map_err(|_| "Lock failed.")?;
                sessions.retain(|_, session| {
                    let keep = session.idle_for() < Duration::from_secs(30 * 60);
                    if !keep { session.cancel(); }
                    keep
                });
                if !sessions.is_empty() { return Err("Another upscale export is running on this worker.".into()); }
                let store = self.store.clone();
                let session = crate::upscale_stream::Session::start((options.input_width, options.input_height), (options.width, options.height), move |stop, read, write| {
                    let files = [options.model.unwrap_or(FileRef::Url { url: SEEDVR2_MODEL.into(), lora: false }),
                        options.vae.unwrap_or(FileRef::Url { url: SEEDVR2_VAE.into(), lora: false })];
                    store.prepare(&files, stop, |_| {})?;
                    let config = slopfab::upscale::UpscaleConfig { method: slopfab::upscale::UpscaleMethod::Seedvr2,
                        model_path: store.resolve(&files[0])?.to_string_lossy().into_owned(),
                        vae_path: Some(store.resolve(&files[1])?.to_string_lossy().into_owned()) };
                    slopfab::upscale::run(&config, (options.input_width, options.input_height), (options.width, options.height), options.segment_frames as i32, stop, read, write)
                })?;
                sessions.insert(id.to_string(), session);
                json(&true)
            }
            (Method::Post, ["upscales", id, "frames"]) => {
                let session = self.upscale(id)?;
                let mut pixels = Vec::new();
                request.as_reader().take(512 * 1024 * 1024 + 1).read_to_end(&mut pixels).map_err(|e| e.to_string())?;
                session.push(pixels)?;
                json(&true)
            }
            (Method::Post, ["upscales", id, "finish"]) => { self.upscale(id)?.finish()?; json(&true) }
            (Method::Get, ["upscales", id, "frames"]) => Ok(bytes(self.upscale(id)?.read()?, "application/octet-stream")),
            (Method::Get, ["upscales", id]) => { self.upscale(id)?.touch(); json(&true) }
            (Method::Delete, ["upscales", id]) => {
                if let Some(session) = self.upscales.lock().map_err(|_| "Lock failed.")?.remove(*id) { session.cancel(); }
                json(&true)
            }
            (Method::Post, ["uploads", "missing"]) => {
                let files: Vec<FileRef> = body(request)?;
                json(&MissingUploads { missing: self.store.missing(&files)? })
            }
            (Method::Put, ["uploads", key, entry @ ..]) => {
                let entry = entry.iter().map(|part| percent_decode(part)).collect::<Result<Vec<_>, _>>()?.join("/");
                let bytes = request.body_length().ok_or("Uploads need a Content-Length.")? as u64;
                let key = key.to_string();
                self.store.receive(&key, &entry, bytes, request.as_reader())?;
                json(&true)
            }
            (Method::Post, ["prepare", id]) => {
                let files: Vec<FileRef> = body(request)?;
                self.start_preparation(id, files)?;
                json(&true)
            }
            (Method::Get, ["prepare", id]) => {
                let preparation = self.preparations.lock().map_err(|_| "Lock failed.")?.get(*id).cloned()
                    .ok_or(HttpError(404, "No preparation with this id.".into()))?;
                let status = preparation.status.lock().map_err(|_| "Lock failed.")?.clone();
                json(&status)
            }
            (Method::Post, ["prepare", id, "cancel"]) => {
                if let Some(preparation) = self.preparations.lock().map_err(|_| "Lock failed.")?.get(*id) {
                    preparation.cancel.store(true, Ordering::Relaxed);
                }
                json(&true)
            }
            (Method::Post, ["plan"]) => {
                let job: RemoteJob = body(request)?;
                let (request, settings) = self.materialize(job)?;
                json(&slopfab::resolve_plan(&request, &settings, &self.runtime.references)?)
            }
            (Method::Post, ["refmods"]) => {
                let export: RefmodExport = body(request)?;
                let id = generated_file_stem(&export.job.request.job_id)?;
                let (request, settings) = self.materialize(export.job)?;
                let directory = self.data.join("refmods");
                fs::create_dir_all(&directory).map_err(|error| error.to_string())?;
                let path = directory.join(format!("{id}.safetensors"));
                let refmod = slopfab::export_refmod(&request, &settings, &self.runtime.references, &path, &export.name, &export.description)
                    .and_then(|_| fs::read(&path).map_err(|error| error.to_string()));
                let _ = fs::remove_file(&path);
                Ok(bytes(refmod?, "application/octet-stream"))
            }
            (Method::Post, ["jobs", id]) => {
                let start: StartJob = body(request)?;
                self.start_job(id, start)?;
                json(&true)
            }
            (Method::Get, ["jobs", id, "events"]) => {
                let after = query.split('&').find_map(|pair| pair.strip_prefix("after=")).and_then(|value| value.parse().ok()).unwrap_or(0);
                let log = self.job(id)?;
                json(&log.after(after, Duration::from_secs(20)))
            }
            (Method::Post, ["jobs", id, "cancel"]) => json(&self.runtime.cancel(id)),
            (Method::Get, ["jobs", id, "summary"]) => {
                json(&rendered::summary(id).ok_or(HttpError(404, "The frames of this job are not in memory.".into()))?)
            }
            (Method::Get, ["jobs", id, "frames", index]) => {
                let index: u32 = index.parse().map_err(|_| "Invalid frame index.")?;
                let frame = rendered::frame(id, index)?.ok_or(HttpError(404, format!("Frame {index} of {id} is not in memory.")))?;
                Ok(bytes(frame, "application/octet-stream"))
            }
            (Method::Get, ["jobs", id, "audio"]) => {
                Ok(bytes(rendered::audio(id).ok_or(HttpError(404, "The audio of this job is not in memory.".into()))?, "application/octet-stream"))
            }
            (Method::Get, ["jobs", id, "latents"]) => {
                let log = self.job(id)?;
                let path = log.latents.clone().ok_or(HttpError(404, "This job saves no latents.".into()))?;
                let file = fs::File::open(&path).map_err(|error| HttpError(404, format!("Latents are unavailable: {error}")))?;
                let length = file.metadata().map(|meta| meta.len() as usize).ok();
                Ok(Response::new(200.into(), vec![Header::from_bytes("Content-Type", "application/octet-stream").unwrap()],
                    Box::new(file) as Box<dyn Read + Send>, length, None))
            }
            (Method::Delete, ["jobs", id]) => {
                self.release_job(id);
                json(&true)
            }
            (Method::Post, ["reference-videos"]) => {
                #[derive(Deserialize)]
                #[serde(rename_all = "camelCase")]
                struct Create { duration_seconds: f64 }
                let create: Create = body(request)?;
                json(&self.runtime.references.create_reference_video(create.duration_seconds, &BTreeMap::new())?)
            }
            (Method::Post, ["reference-videos", "release"]) => {
                let ids: Vec<String> = body(request)?;
                self.runtime.references.release_reference_videos(&ids)?;
                json(&true)
            }
            (Method::Post, ["reference-videos", id, "frames"]) => {
                let width = parsed_header(request, "x-reference-width")?;
                let height = parsed_header(request, "x-reference-height")?;
                let time = parsed_header(request, "x-reference-time")?;
                let mut pixels = Vec::new();
                request.as_reader().take(512 * 1024 * 1024).read_to_end(&mut pixels).map_err(|error| error.to_string())?;
                self.runtime.references.append_reference_video(id, &pixels, width, height, time)?;
                json(&true)
            }
            (Method::Post, ["reference-videos", id, "audio"]) => {
                let channels = parsed_header(request, "x-reference-channels")?;
                let rate = parsed_header(request, "x-reference-rate")?;
                let mut samples = Vec::new();
                request.as_reader().take(1024 * 1024 * 1024).read_to_end(&mut samples).map_err(|error| error.to_string())?;
                self.runtime.references.set_reference_video_audio(id, &samples, channels, rate)?;
                json(&true)
            }
            (Method::Post, ["reference-audios"]) => {
                let channels = parsed_header(request, "x-reference-channels")?;
                let rate = parsed_header(request, "x-reference-rate")?;
                let mut samples = Vec::new();
                request.as_reader().take(1024 * 1024 * 1024).read_to_end(&mut samples).map_err(|error| error.to_string())?;
                json(&self.runtime.references.create_reference_audio(&samples, channels, rate)?)
            }
            (Method::Post, ["reference-audios", "release"]) => {
                let ids: Vec<String> = body(request)?;
                self.runtime.references.release_reference_audios(&ids)?;
                json(&true)
            }
            _ => Err(HttpError(404, "Unknown worker route.".into())),
        }
    }

    fn job(&self, id: &str) -> Result<Arc<JobLog>, HttpError> {
        self.jobs.lock().map_err(|_| "Lock failed.")?.get(id).cloned().ok_or(HttpError(404, "No job with this id.".into()))
    }

    fn upscale(&self, id: &str) -> Result<Arc<crate::upscale_stream::Session>, HttpError> {
        self.upscales.lock().map_err(|_| "Lock failed.")?.get(id).cloned().ok_or(HttpError(404, "Upscale session has ended.".into()))
    }

    fn start_preparation(&self, id: &str, files: Vec<FileRef>) -> Result<(), String> {
        generated_file_stem(id)?;
        let preparation = Arc::new(Preparation {
            status: Mutex::new(FilePreparation { state: "running".into(), ..Default::default() }),
            cancel: Arc::new(AtomicBool::new(false)),
            created: Instant::now(),
        });
        {
            let mut preparations = self.preparations.lock().map_err(|_| "Lock failed.")?;
            // Keep finished results long enough for their client to read them.
            preparations.retain(|_, entry| entry.created.elapsed() < Duration::from_secs(60 * 60)
                || entry.status.lock().is_ok_and(|status| status.state == "running"));
            if preparations.get(id).is_some_and(|entry| entry.status.lock().is_ok_and(|status| status.state != "running")) {
                preparations.remove(id);
            }
            if preparations.contains_key(id) {
                return Err("Files for this job are already being prepared.".into());
            }
            preparations.insert(id.to_string(), preparation.clone());
        }
        let store = self.store.clone();
        let shared = preparation;
        std::thread::spawn(move || {
            let result = store.prepare(&files, &shared.cancel, |status| {
                if let Ok(mut current) = shared.status.lock() {
                    *current = status.clone();
                }
            });
            if let Ok(mut status) = shared.status.lock() {
                match result {
                    Ok(()) => status.state = "done".into(),
                    Err(_) if shared.cancel.load(Ordering::Relaxed) => status.state = "cancelled".into(),
                    Err(error) => {
                        status.state = "failed".into();
                        status.error = Some(error);
                    }
                }
            }
        });
        Ok(())
    }

    /// Turns a client's job into one this machine can run: tokens become this
    /// worker's own files, and machine choices are this worker's.
    fn materialize(&self, job: RemoteJob) -> Result<(slopfab::GenerationRequest, BTreeMap<String, ProviderSetting>), String> {
        let paths = job.files.iter().map(|file| self.store.resolve(file)).collect::<Result<Vec<_>, _>>()?;
        let mut request = job.request;
        let mut settings = job.settings;
        settings.retain(|name, _| name == "slopfab");
        visit_paths(&mut request, &mut settings, |_, value| {
            let index = parse_file_token(value).ok_or("A worker job can only name files sent with it.")?;
            *value = paths.get(index).ok_or("A worker job names a file it did not send.")?.to_string_lossy().into_owned();
            Ok(())
        })?;
        if let Some(slopfab) = settings.get_mut("slopfab") {
            // The backend is a property of the computer that runs the job.
            slopfab.options.remove("inferenceBackend");
            if self.vulkan {
                slopfab.options.insert("inferenceBackend".into(), ProviderOption::String("vulkan".into()));
            }
        }
        let file = |index: Option<usize>| index.map(|index| paths.get(index).cloned().ok_or("A worker job names a file it did not send.")).transpose();
        request.continuation_path = file(job.continuation)?;
        request.keystone_path = file(job.keystone)?;
        request.bridge_left_path = file(job.bridge_left)?;
        request.bridge_right_path = file(job.bridge_right)?;
        if request.latent_bridge.is_some() && (request.bridge_left_path.is_none() || request.bridge_right_path.is_none()) {
            return Err("Long Shot requires both neighboring latent archives.".into());
        }
        request.image_edit_path = file(job.image_edit_source)?;
        if request.image_edit.is_some() && request.image_edit_path.is_none() {
            return Err("Image edits require their source image.".into());
        }
        request.image_edit_pixels = None;
        request.save_latents_path = None;
        Ok((request, settings))
    }

    fn start_job(&self, id: &str, start: StartJob) -> Result<(), String> {
        generated_file_stem(id)?;
        if start.job.request.job_id != id {
            return Err("The job id does not match its request.".into());
        }
        self.prune_jobs();
        let (mut request, settings) = self.materialize(start.job)?;
        let latents = if start.save_latents && !request.still_image && request.music.is_none() {
            let directory = self.data.join("jobs").join(id);
            let _ = fs::remove_dir_all(&directory);
            fs::create_dir_all(&directory).map_err(|error| error.to_string())?;
            Some(directory.join("latents.safetensors"))
        } else {
            None
        };
        request.save_latents_path = latents.clone();
        let log = Arc::new(JobLog { events: Mutex::default(), changed: Condvar::new(), latents, created: Instant::now(), released: AtomicBool::new(false) });
        {
            let mut jobs = self.jobs.lock().map_err(|_| "Lock failed.")?;
            if jobs.contains_key(id) {
                return Err("This job was already started on the worker.".into());
            }
            jobs.insert(id.to_string(), log.clone());
        }
        let sink = log.clone();
        let result = self.runtime.enqueue(
            Arc::new(move |event| match event {
                slopfab::NativeEvent::Job(value) => sink.push("job", serde_json::to_value(value).unwrap_or_default()),
                slopfab::NativeEvent::Progress(value) => sink.push("progress", serde_json::to_value(value).unwrap_or_default()),
                slopfab::NativeEvent::Preview(value) => sink.push("preview", serde_json::to_value(value).unwrap_or_default()),
            }),
            request,
            &settings,
        );
        if result.is_err() {
            self.release_job(id);
        }
        result
    }

    fn release_job(&self, id: &str) {
        self.runtime.cancel(id);
        let log = self.jobs.lock().ok().and_then(|mut jobs| jobs.remove(id));
        if let Some(log) = log {
            log.release(id);
        } else {
            rendered::release(id);
        }
        if generated_file_stem(id).is_ok() {
            let _ = fs::remove_dir_all(self.data.join("jobs").join(id));
        }
    }

    /// Clients release their jobs; this catches the ones that vanished.
    fn prune_jobs(&self) {
        let stale: Vec<String> = self.jobs.lock().map(|jobs| jobs.iter()
            .filter(|(_, log)| log.created.elapsed() > Duration::from_secs(12 * 60 * 60))
            .map(|(id, _)| id.clone()).collect()).unwrap_or_default();
        for id in stale {
            self.release_job(&id);
        }
    }
}

fn percent_decode(part: &str) -> Result<String, String> {
    let bytes = part.as_bytes();
    let mut decoded = Vec::with_capacity(bytes.len());
    let mut index = 0;
    while index < bytes.len() {
        if bytes[index] == b'%' {
            let hex = part.get(index + 1..index + 3).ok_or("Invalid escape in upload path.")?;
            decoded.push(u8::from_str_radix(hex, 16).map_err(|_| "Invalid escape in upload path.")?);
            index += 3;
        } else {
            decoded.push(bytes[index]);
            index += 1;
        }
    }
    String::from_utf8(decoded).map_err(|_| "Upload paths must be UTF-8.".into())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn preview_log_retains_only_latest_pixels_without_reusing_cursors() {
        let log = JobLog { events: Mutex::default(), changed: Condvar::new(), latents: None,
            created: Instant::now(), released: AtomicBool::new(false) };
        log.push("preview", serde_json::json!({"frames": ["first"]}));
        log.push("progress", serde_json::json!({"step": 1}));
        log.push("preview", serde_json::json!({"frames": ["second"]}));
        let events = log.after(0, Duration::ZERO);
        assert!(events[0].payload.is_null());
        assert_eq!(events[2].seq, 3);
        assert_eq!(events[2].payload["frames"][0], "second");
        log.push("job", serde_json::json!({"state": "framesReady"}));
        assert!(log.after(2, Duration::ZERO)[0].payload.is_null());
    }

    struct TrackedFrames(Arc<AtomicBool>);

    impl rendered::FrameSource for TrackedFrames {
        fn frame_rgba(&self, _: u32, _: u32, _: u32) -> Result<Vec<u8>, String> {
            Ok(vec![0, 0, 0, 255])
        }
    }

    impl Drop for TrackedFrames {
        fn drop(&mut self) { self.0.store(true, Ordering::Release); }
    }

    #[test]
    fn releasing_a_worker_job_discards_frames_that_finish_after_release() {
        let _serial = rendered::TEST_LOCK.lock().unwrap();
        let id = "worker-release-before-finish";
        let log = JobLog { events: Mutex::default(), changed: Condvar::new(), latents: None,
            created: Instant::now(), released: AtomicBool::new(false) };
        log.release(id);
        let dropped = Arc::new(AtomicBool::new(false));
        rendered::keep(rendered::from_source(id, Box::new(TrackedFrames(dropped.clone())),
            vec![], 1, 1, 1, 3, 3, 24.0, 0, 0).unwrap());
        log.push("job", serde_json::json!({ "jobId": id, "state": "framesReady" }));
        assert!(dropped.load(Ordering::Acquire));
        assert!(rendered::summary(id).is_none());
        assert!(log.after(0, Duration::ZERO).is_empty());
    }

    #[test]
    fn releasing_a_finished_worker_job_discards_its_frames() {
        let _serial = rendered::TEST_LOCK.lock().unwrap();
        let id = "worker-release-after-finish";
        let log = JobLog { events: Mutex::default(), changed: Condvar::new(), latents: None,
            created: Instant::now(), released: AtomicBool::new(false) };
        let dropped = Arc::new(AtomicBool::new(false));
        rendered::keep(rendered::from_source(id, Box::new(TrackedFrames(dropped.clone())),
            vec![], 1, 1, 1, 3, 3, 24.0, 0, 0).unwrap());
        log.push("job", serde_json::json!({ "jobId": id, "state": "framesReady" }));
        assert_eq!(log.after(0, Duration::ZERO).len(), 1);
        log.release(id);
        assert!(dropped.load(Ordering::Acquire));
        assert!(rendered::summary(id).is_none());
    }

    fn probe(nvidia_gpu: bool, cuda: Option<&'static str>) -> slopfab::BackendProbe {
        slopfab::BackendProbe { nvidia_gpu, cuda, cuda_problem: (nvidia_gpu && cuda.is_none()).then(|| "cublas64_12.dll not found".into()) }
    }

    #[test]
    fn auto_uses_cuda_when_it_loads_and_warns_before_falling_back_on_nvidia() {
        assert_eq!(choose_backend(Backend::Auto, &probe(true, Some("CUDA 13"))).unwrap(), (false, None));
        let (vulkan, warning) = choose_backend(Backend::Auto, &probe(true, None)).unwrap();
        assert!(vulkan);
        assert!(warning.unwrap().contains("cublas64_12.dll not found"));
        assert_eq!(choose_backend(Backend::Auto, &probe(false, None)).unwrap(), (true, None), "AMD and Intel use Vulkan quietly");
    }

    #[test]
    fn a_requested_cuda_major_must_load() {
        assert_eq!(Backend::Cuda13.cuda_version(), "13");
        assert_eq!(Backend::Cuda12.cuda_version(), "12");
        assert_eq!(Backend::Auto.cuda_version(), "auto");
        assert_eq!("CUDA-12".parse::<Backend>().unwrap(), Backend::Cuda12);
        assert_eq!("cuda13".parse::<Backend>().unwrap(), Backend::Cuda13);
        assert_eq!(choose_backend(Backend::Cuda12, &probe(true, Some("CUDA 12"))).unwrap(), (false, None));
        let error = choose_backend(Backend::Cuda13, &probe(true, None)).unwrap_err();
        let hint = if cfg!(windows) { "CUDA 13 toolkit" } else { "CUDA 13 cuBLAS" };
        assert!(error.starts_with("--backend cuda13") && error.contains(hint), "{error}");
    }

    #[test]
    fn an_explicit_backend_overrides_detection_but_cuda_must_exist() {
        assert_eq!(choose_backend(Backend::Vulkan, &probe(true, Some("CUDA 12"))).unwrap(), (true, None));
        assert_eq!(choose_backend(Backend::Cuda, &probe(true, Some("CUDA 12"))).unwrap(), (false, None));
        assert!(choose_backend(Backend::Cuda, &probe(true, None)).unwrap_err().contains("CUDA is not available"));
        assert!(choose_backend(Backend::Cuda, &probe(false, None)).unwrap_err().contains("no NVIDIA GPU"));
        assert_eq!("VULKAN".parse::<Backend>().unwrap(), Backend::Vulkan);
        assert!("metal".parse::<Backend>().is_err());
    }
}