Skip to content

AI media harness for planning, generating, and editing locally

TypeScript
67.8 KB
Raw Blame
//! The desktop side of LAN workers: which worker is selected, how to reach it,
//! and running a generation there as if the native queue were local. The
//! webview keeps using the same commands and events either way.
use slopus_core::worker::{
    discovery::{self, Announced},
    protocol::*,
};
use crate::{
    diagnostics,
    project::ProviderSetting,
    rendered,
    slopfab::{self, GenerationRequest, GpuDevice},
};
use slopus_core::weights;
use reqwest::blocking::{Body, Client, Response};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use std::{
    collections::{BTreeMap, HashMap},
    fs::{self, File},
    io::{self, Read},
    path::{Path, PathBuf},
    sync::{
        atomic::{AtomicBool, Ordering},
        Arc, Mutex, OnceLock,
    },
    time::{Duration, Instant, UNIX_EPOCH},
};
use tauri::{AppHandle, Emitter};

#[derive(Default, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
struct Saved {
    /// The selected worker's id. None runs generation on this computer.
    selected: Option<String>,
    /// Addresses added by hand, for networks where mDNS does not reach.
    manual: Vec<String>,
    /// Where each worker was last reached, so a selection survives a restart
    /// before mDNS has answered.
    known: BTreeMap<String, KnownWorker>,
    tokens: BTreeMap<String, String>,
}

#[derive(Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
struct KnownWorker {
    name: String,
    address: String,
}

#[derive(Clone)]
struct Probe {
    info: Result<WorkerInfo, String>,
    at: Instant,
}

/// Never dropped, unlike `Workers`: test binaries have no Windows manifest
/// and must not link the webview runtime that an AppHandle's drop pulls in.
static APP: OnceLock<AppHandle> = OnceLock::new();

#[derive(Default)]
struct Inner {
    file: OnceLock<PathBuf>,
    saved: Mutex<Saved>,
    announced: Mutex<Vec<Announced>>,
    probes: Mutex<HashMap<String, Probe>>,
    /// Jobs sent from here, with the worker each one went to.
    jobs: Mutex<HashMap<String, (Arc<AtomicBool>, Connection)>>,
    daemon: Mutex<Option<mdns_sd::ServiceDaemon>>,
}

#[derive(Clone, Default)]
pub struct Workers(Arc<Inner>);

#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct WorkerView {
    /// The worker id, or the address of a manual worker never reached.
    key: String,
    id: Option<String>,
    name: String,
    address: Option<String>,
    online: bool,
    compatible: bool,
    discovered: bool,
    manual: bool,
    selected: bool,
    version: Option<String>,
    requires_token: bool,
    has_token: bool,
    gpus: Vec<GpuDevice>,
    runtime: Option<WorkerRuntime>,
    error: Option<String>,
}

#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct WorkerList {
    selected_id: Option<String>,
    workers: Vec<WorkerView>,
}

/// A selected worker that answered. Cheap to clone; one per remote call.
#[derive(Clone)]
pub(crate) struct Connection {
    base: String,
    token: Option<String>,
    pub(crate) name: String,
    pub(crate) address: String,
    pub(crate) info: WorkerInfo,
    http: Client,
}

/// A worker the list should show, before anyone has asked it anything.
struct Candidate {
    id: Option<String>,
    name: String,
    addresses: Vec<String>,
    discovered: bool,
    manual: bool,
}

const PROBE_FRESH: Duration = Duration::from_secs(10);

fn normalize_address(address: &str) -> Result<String, String> {
    let trimmed = address.trim().trim_start_matches("http://").trim_end_matches('/');
    if trimmed.is_empty() || trimmed.contains(['/', '?', '#', '@', ' ']) {
        return Err("Enter a worker address like 192.168.1.20 or worker-pc:47321.".into());
    }
    let has_port = trimmed.rsplit_once(':').is_some_and(|(host, port)| !host.is_empty() && !host.ends_with(':') && port.parse::<u16>().is_ok());
    Ok(if has_port { trimmed.to_string() } else { format!("{trimmed}:{DEFAULT_PORT}") })
}

fn socket_address(address: &std::net::SocketAddr) -> String {
    match address {
        std::net::SocketAddr::V6(v6) => format!("[{}]:{}", v6.ip(), v6.port()),
        std::net::SocketAddr::V4(v4) => v4.to_string(),
    }
}

fn http_client(timeout: Option<Duration>) -> Client {
    Client::builder()
        .connect_timeout(Duration::from_secs(4))
        .timeout(timeout)
        .build()
        .unwrap_or_default()
}

fn worker_error(name: &str, response: Response) -> String {
    let status = response.status();
    #[derive(Deserialize)]
    struct Failure { error: String }
    match response.json::<Failure>() {
        Ok(failure) => format!("Worker {name}: {}", failure.error),
        Err(_) => format!("Worker {name} answered {status}."),
    }
}

impl Connection {
    pub(crate) fn upscale(&self, id: &str, options: &StartUpscale, stop: &AtomicBool,
        read: &mut (dyn FnMut() -> Result<Option<Vec<u8>>, String> + Send),
        write: &mut (dyn FnMut(Vec<u8>) -> Result<(), String> + Send)) -> Result<u64, String> {
        if !self.info.runtime.cuda_available { return Err(format!("Worker {} needs CUDA for SeedVR2.", self.name)); }
        self.post::<bool>(&format!("/v1/upscales/{id}"), options)?;
        let finished = AtomicBool::new(false);
        let failed = AtomicBool::new(false);
        let result = std::thread::scope(|scope| {
            let monitor = scope.spawn(|| {
                let mut ticks = 0;
                while !finished.load(Ordering::Relaxed) {
                    if stop.load(Ordering::Relaxed) || failed.load(Ordering::Relaxed) {
                        let _ = self.send(self.request(reqwest::Method::DELETE, &format!("/v1/upscales/{id}")).timeout(Duration::from_secs(5)));
                        break;
                    }
                    ticks += 1;
                    if ticks % 50 == 0 {
                        let _ = self.send(self.request(reqwest::Method::GET, &format!("/v1/upscales/{id}")).timeout(Duration::from_secs(5)));
                    }
                    std::thread::sleep(Duration::from_millis(100));
                }
            });
            let output = scope.spawn(|| -> Result<u64, String> {
                let result = (|| {
                    let mut count = 0;
                    loop {
                        let bytes = self.bytes(&format!("/v1/upscales/{id}/frames"))?;
                        if bytes.is_empty() { return Ok(count); }
                        write(bytes)?;
                        count += 1;
                    }
                })();
                if result.is_err() { failed.store(true, Ordering::Relaxed); }
                result
            });
            let input = (|| -> Result<(), String> {
                while let Some(frame) = read()? {
                    if stop.load(Ordering::Relaxed) || failed.load(Ordering::Relaxed) { return Err("Upscaler stream stopped.".into()); }
                    self.post_bytes::<bool>(&format!("/v1/upscales/{id}/frames"), &[], frame)?;
                }
                self.post::<bool>(&format!("/v1/upscales/{id}/finish"), &())?;
                Ok(())
            })();
            if input.is_err() { failed.store(true, Ordering::Relaxed); }
            let output = output.join().unwrap_or_else(|_| Err("Worker upscale reader panicked.".into()));
            finished.store(true, Ordering::Relaxed);
            let _ = monitor.join();
            output.and_then(|count| input.map(|_| count))
        });
        let _ = self.send(self.request(reqwest::Method::DELETE, &format!("/v1/upscales/{id}")).timeout(Duration::from_secs(5)));
        result
    }

    fn request(&self, method: reqwest::Method, path: &str) -> reqwest::blocking::RequestBuilder {
        let builder = self.http.request(method, format!("{}{path}", self.base));
        match &self.token {
            Some(token) => builder.header(TOKEN_HEADER, token),
            None => builder,
        }
    }

    fn send(&self, builder: reqwest::blocking::RequestBuilder) -> Result<Response, String> {
        let response = builder.send().map_err(|error| format!("Could not reach worker {}: {error}", self.name))?;
        if response.status().is_success() { Ok(response) } else { Err(worker_error(&self.name, response)) }
    }

    pub(crate) fn get<T: DeserializeOwned>(&self, path: &str) -> Result<T, String> {
        self.send(self.request(reqwest::Method::GET, path))?.json().map_err(|error| format!("Worker {}: {error}", self.name))
    }

    pub(crate) fn post<T: DeserializeOwned>(&self, path: &str, body: &impl Serialize) -> Result<T, String> {
        self.send(self.request(reqwest::Method::POST, path).json(body))?.json().map_err(|error| format!("Worker {}: {error}", self.name))
    }

    pub(crate) fn post_bytes<T: DeserializeOwned>(&self, path: &str, headers: &[(&str, String)], bytes: Vec<u8>) -> Result<T, String> {
        let mut builder = self.request(reqwest::Method::POST, path).body(bytes);
        for (name, value) in headers {
            builder = builder.header(*name, value);
        }
        self.send(builder)?.json().map_err(|error| format!("Worker {}: {error}", self.name))
    }

    fn bytes(&self, path: &str) -> Result<Vec<u8>, String> {
        let response = self.send(self.request(reqwest::Method::GET, path))?;
        response.bytes().map(|bytes| bytes.to_vec()).map_err(|error| format!("Worker {}: {error}", self.name))
    }

    fn release(&self, job_id: &str) {
        let _ = self.send(self.request(reqwest::Method::DELETE, &format!("/v1/jobs/{job_id}")));
    }
}

fn probe(address: &str, token: Option<&str>) -> Result<WorkerInfo, String> {
    let mut builder = http_client(Some(Duration::from_secs(3))).get(format!("http://{address}/v1/info"));
    if let Some(token) = token {
        builder = builder.header(TOKEN_HEADER, token);
    }
    let response = builder.send().map_err(|error| format!("Not reachable: {error}"))?;
    if !response.status().is_success() {
        return Err(format!("Answered {}.", response.status()));
    }
    let info: WorkerInfo = response.json().map_err(|_| "This address is not a Slopus worker.".to_string())?;
    if !valid_key(&info.id) {
        return Err("This address is not a Slopus worker.".into());
    }
    Ok(info)
}

impl Workers {
    /// Reuse the generation upload cache for explicitly selected local upscaler files.
    pub(crate) fn prepare_upscale_files(&self, connection: &Connection, id: &str,
        config: &slopfab::upscale::UpscaleConfig, stop: &AtomicBool) -> Result<[FileRef; 2], String> {
        if !connection.info.runtime.cuda_available { return Err(format!("Worker {} needs CUDA for SeedVR2.", connection.name)); }
        let mut sources = HashMap::new();
        let mut source = |path: &str, url: &str| {
            if path.trim().is_empty() { Ok(FileRef::Url { url: url.into(), lora: false }) }
            else { file_ref(PathKind::Input, path, &mut sources) }
        };
        let files = [source(&config.model_path, SEEDVR2_MODEL)?, source(config.vae_path.as_deref().unwrap_or(""), SEEDVR2_VAE)?];
        let uploads: Vec<_> = files.iter().filter(|file| matches!(file, FileRef::Upload { .. })).cloned().collect();
        if stop.load(Ordering::Acquire) { return Err("Export cancelled.".into()); }
        if !uploads.is_empty() {
            let job = RemoteJob { request: GenerationRequest { job_id: id.into(), ..Default::default() },
                settings: BTreeMap::new(), files: uploads, continuation: None, keystone: None, image_edit_source: None, bridge_left: None, bridge_right: None };
            let cancel = Arc::new(AtomicBool::new(false));
            let finished = AtomicBool::new(false);
            std::thread::scope(|scope| {
                scope.spawn(|| {
                    while !finished.load(Ordering::Acquire) {
                        if stop.load(Ordering::Acquire) { cancel.store(true, Ordering::Release); break; }
                        std::thread::sleep(Duration::from_millis(100));
                    }
                });
                let result = self.send_files(connection, &job, &sources, &cancel);
                finished.store(true, Ordering::Release);
                result
            })?;
        }
        if stop.load(Ordering::Acquire) { return Err("Export cancelled.".into()); }
        Ok(files)
    }

    /// Loads the saved selection and starts listening for workers.
    pub fn start(&self, app: &AppHandle, data_directory: &Path) {
        let _ = APP.set(app.clone());
        let file = data_directory.join("workers.json");
        if let Some(saved) = fs::read(&file).ok().and_then(|bytes| serde_json::from_slice::<Saved>(&bytes).ok()) {
            if let Ok(mut current) = self.0.saved.lock() {
                *current = saved;
            }
        }
        let _ = self.0.file.set(file);
        let inner = Arc::downgrade(&self.0);
        match discovery::browse(move |announced| {
            let Some(inner) = inner.upgrade() else { return };
            if let Ok(mut current) = inner.announced.lock() {
                *current = announced;
            }
            if let Some(app) = APP.get() {
                let _ = app.emit("workers-changed", ());
            }
        }) {
            Ok(daemon) => {
                if let Ok(mut current) = self.0.daemon.lock() {
                    *current = Some(daemon);
                }
            }
            Err(error) => diagnostics::warn("workers", "discovery.failed", &error, serde_json::json!({})),
        }
    }

    fn save(&self, saved: &Saved) {
        if let Some(file) = self.0.file.get() {
            if let Ok(bytes) = serde_json::to_vec_pretty(saved) {
                if let Err(error) = crate::storage::atomic::write_atomically(file, &bytes) {
                    diagnostics::warn("workers", "save.failed", &error, serde_json::json!({}));
                }
            }
        }
    }

    fn saved(&self) -> Saved {
        self.0.saved.lock().map(|saved| saved.clone()).unwrap_or_default()
    }

    fn update(&self, change: impl FnOnce(&mut Saved)) -> Saved {
        let saved = match self.0.saved.lock() {
            Ok(mut saved) => {
                change(&mut saved);
                saved.clone()
            }
            Err(_) => return Saved::default(),
        };
        self.save(&saved);
        self.emit("workers-changed", serde_json::Value::Null);
        saved
    }

    fn probe_cached(&self, address: &str, token: Option<&str>, fresh: Duration) -> Result<WorkerInfo, String> {
        if let Some(probe) = self.0.probes.lock().ok().and_then(|probes| probes.get(address).cloned()) {
            if probe.at.elapsed() < fresh {
                return probe.info;
            }
        }
        let info = probe(address, token);
        if let Ok(mut probes) = self.0.probes.lock() {
            probes.insert(address.to_string(), Probe { info: info.clone(), at: Instant::now() });
        }
        info
    }

    /// Every address a worker id might answer on, best first.
    fn addresses_for(&self, id: &str, saved: &Saved) -> Vec<String> {
        // The address that last answered first, then what mDNS announces.
        let mut addresses: Vec<String> = saved.known.get(id).map(|known| known.address.clone()).into_iter().collect();
        for address in self.0.announced.lock().map(|announced| announced.iter()
            .filter(|worker| worker.id == id)
            .flat_map(|worker| worker.addresses.iter().map(socket_address))
            .collect::<Vec<_>>()).unwrap_or_default() {
            if !addresses.contains(&address) {
                addresses.push(address);
            }
        }
        addresses
    }

    /// The selected worker, or None to generate on this computer. A selected
    /// worker that does not answer is an error rather than a silent fallback:
    /// the user chose where their GPU time is spent.
    pub(crate) fn active(&self) -> Result<Option<Connection>, String> {
        let saved = self.saved();
        let Some(id) = saved.selected.clone() else { return Ok(None) };
        let name = saved.known.get(&id).map(|known| known.name.clone()).unwrap_or_else(|| "worker".into());
        let token = saved.tokens.get(&id).cloned();
        let mut last_error = None;
        for address in self.addresses_for(&id, &saved) {
            match self.probe_cached(&address, token.as_deref(), PROBE_FRESH) {
                Ok(info) if info.id == id => {
                    if info.protocol != PROTOCOL_VERSION {
                        return Err(format!("Worker {} uses a different Slopus version ({}). Update it to match this app.", info.name, info.version));
                    }
                    if info.requires_token && token.is_none() {
                        return Err(format!("Worker {} requires an access token. Enter it in Settings → Workers.", info.name));
                    }
                    if saved.known.get(&id).is_none_or(|known| known.address != address || known.name != info.name) {
                        let (worker_name, worker_address) = (info.name.clone(), address.clone());
                        self.update(|saved| { saved.known.insert(id.clone(), KnownWorker { name: worker_name, address: worker_address }); });
                    }
                    return Ok(Some(Connection {
                        base: format!("http://{address}"),
                        token,
                        name: info.name.clone(),
                        address,
                        info,
                        http: http_client(None),
                    }));
                }
                Ok(_) => last_error = Some("Another worker now answers at its last address.".to_string()),
                Err(error) => last_error = Some(error),
            }
        }
        Err(format!(
            "The selected worker {name} is offline{}. Start it, or choose This computer in Settings → Workers.",
            last_error.map(|error| format!(" ({error})")).unwrap_or_default()
        ))
    }

    pub fn list(&self) -> WorkerList {
        let saved = self.saved();
        let announced = self.0.announced.lock().map(|announced| announced.clone()).unwrap_or_default();
        let mut candidates: Vec<Candidate> = announced.iter()
            .map(|worker| Candidate { id: Some(worker.id.clone()), name: worker.name.clone(), addresses: worker.addresses.iter().map(socket_address).collect(), discovered: true, manual: false })
            .collect();
        for address in &saved.manual {
            candidates.push(Candidate { id: None, name: address.clone(), addresses: vec![address.clone()], discovered: false, manual: true });
        }
        if let Some(id) = &saved.selected {
            if let Some(known) = saved.known.get(id) {
                candidates.push(Candidate { id: Some(id.clone()), name: known.name.clone(), addresses: vec![known.address.clone()], discovered: false, manual: false });
            }
        }
        let probes: Vec<_> = std::thread::scope(|scope| {
            let handles: Vec<_> = candidates.iter().map(|Candidate { id, addresses, .. }| {
                let token = id.as_ref().and_then(|id| saved.tokens.get(id)).cloned();
                scope.spawn(move || {
                    let mut last = Err("No address.".to_string());
                    for address in addresses {
                        last = self.probe_cached(address, token.as_deref(), Duration::from_secs(3)).map(|info| (info, address.clone()));
                        if last.is_ok() { break; }
                    }
                    last
                })
            }).collect();
            handles.into_iter().map(|handle| handle.join().unwrap_or_else(|_| Err("Probe failed.".into()))).collect()
        });
        let mut views: Vec<WorkerView> = Vec::new();
        for (Candidate { id, name, addresses, discovered, manual }, probe) in candidates.into_iter().zip(probes) {
            let (id, view) = match probe {
                Ok((info, address)) => (Some(info.id.clone()), WorkerView {
                    key: info.id.clone(), id: Some(info.id.clone()), name: info.name.clone(), address: Some(address),
                    online: true, compatible: info.protocol == PROTOCOL_VERSION, discovered, manual,
                    selected: saved.selected.as_deref() == Some(info.id.as_str()), version: Some(info.version.clone()),
                    requires_token: info.requires_token, has_token: saved.tokens.contains_key(&info.id),
                    gpus: info.gpus.clone(), runtime: Some(info.runtime.clone()), error: None,
                }),
                Err(error) => (id.clone(), WorkerView {
                    key: id.clone().unwrap_or_else(|| addresses.first().cloned().unwrap_or_default()),
                    selected: id.is_some() && saved.selected == id, has_token: id.as_ref().is_some_and(|id| saved.tokens.contains_key(id)),
                    id, name, address: addresses.first().cloned(), online: false, compatible: false, discovered, manual,
                    version: None, requires_token: false, gpus: Vec::new(), runtime: None, error: Some(error),
                }),
            };
            match views.iter_mut().find(|view| id.is_some() && view.id == id) {
                Some(existing) => {
                    existing.discovered |= view.discovered;
                    existing.manual |= view.manual;
                    if !existing.online && view.online {
                        let (discovered, manual) = (existing.discovered, existing.manual);
                        *existing = WorkerView { discovered, manual, ..view };
                    }
                }
                None => views.push(view),
            }
        }
        views.sort_by(|a, b| b.online.cmp(&a.online).then_with(|| a.name.to_lowercase().cmp(&b.name.to_lowercase())));
        WorkerList { selected_id: saved.selected, workers: views }
    }

    pub fn select(&self, id: Option<String>) -> Result<WorkerList, String> {
        if let Some(id) = &id {
            let list = self.list();
            let worker = list.workers.iter().find(|worker| worker.id.as_ref() == Some(id))
                .ok_or("This worker is not available.")?;
            if !worker.online {
                return Err(format!("Worker {} is offline.", worker.name));
            }
            if !worker.compatible {
                return Err(format!("Worker {} runs a different Slopus version. Update it to match this app.", worker.name));
            }
            let known = KnownWorker { name: worker.name.clone(), address: worker.address.clone().unwrap_or_default() };
            self.update(|saved| {
                saved.known.insert(id.clone(), known);
                saved.selected = Some(id.clone());
            });
        } else {
            self.update(|saved| saved.selected = None);
        }
        diagnostics::info("workers", "selected", "Generation worker selected.", serde_json::json!({ "remote": id.is_some() }));
        Ok(self.list())
    }

    pub fn add(&self, address: &str) -> Result<WorkerList, String> {
        let address = normalize_address(address)?;
        let info = probe(&address, None).map_err(|error| format!("{address}: {error}"))?;
        if let Ok(mut probes) = self.0.probes.lock() {
            probes.insert(address.clone(), Probe { info: Ok(info.clone()), at: Instant::now() });
        }
        self.update(|saved| {
            if !saved.manual.contains(&address) {
                saved.manual.push(address.clone());
            }
            saved.known.insert(info.id.clone(), KnownWorker { name: info.name.clone(), address: address.clone() });
        });
        Ok(self.list())
    }

    pub fn remove(&self, key: &str) -> WorkerList {
        self.update(|saved| {
            let address = saved.known.get(key).map(|known| known.address.clone());
            saved.manual.retain(|manual| manual != key && Some(manual) != address.as_ref());
            saved.known.remove(key);
            saved.tokens.remove(key);
            if saved.selected.as_deref() == Some(key) {
                saved.selected = None;
            }
        });
        self.list()
    }

    pub fn set_token(&self, id: &str, token: &str) -> WorkerList {
        let token = token.trim().to_string();
        self.update(|saved| {
            if token.is_empty() { saved.tokens.remove(id); } else { saved.tokens.insert(id.to_string(), token.clone()); }
        });
        if let Ok(mut probes) = self.0.probes.lock() {
            probes.clear();
        }
        self.list()
    }

    fn emit(&self, event: &str, payload: serde_json::Value) {
        #[cfg(test)]
        {
            eprintln!("{event}: {payload}");
            if payload["state"] == "failed" {
                tests::FAILED.lock().unwrap().push(payload["detail"].to_string());
            }
        }
        #[cfg(not(test))]
        if let Some(app) = APP.get() {
            let _ = app.emit(event, payload);
        }
    }

    fn register(&self, job_id: &str, connection: &Connection) -> Result<Arc<AtomicBool>, String> {
        let mut jobs = self.0.jobs.lock().map_err(|_| "Worker job lock failed.")?;
        if jobs.contains_key(job_id) {
            return Err("This generation job is already queued.".into());
        }
        let cancel = Arc::new(AtomicBool::new(false));
        jobs.insert(job_id.to_string(), (cancel.clone(), connection.clone()));
        Ok(cancel)
    }

    fn unregister(&self, job_id: &str) {
        if let Ok(mut jobs) = self.0.jobs.lock() {
            jobs.remove(job_id);
        }
    }

    /// Cancels a job this computer sent to a worker, on the worker it went
    /// to even if another one is selected now. False when it sent none.
    pub(crate) fn cancel(&self, job_id: &str) -> bool {
        let Some((flag, connection)) = self.0.jobs.lock().ok().and_then(|jobs| jobs.get(job_id).cloned()) else { return false };
        flag.store(true, Ordering::Release);
        let job_id = job_id.to_string();
        std::thread::spawn(move || {
            let _ = connection.post::<serde_json::Value>(&format!("/v1/jobs/{job_id}/cancel"), &());
            let _ = connection.post::<serde_json::Value>(&format!("/v1/prepare/{job_id}/cancel"), &());
        });
        true
    }

    /// Uploads what only this computer has, then lets the worker download the
    /// rest. Both report progress as queue events on the job.
    fn send_files(&self, connection: &Connection, job: &RemoteJob, sources: &HashMap<String, PathBuf>, cancel: &Arc<AtomicBool>) -> Result<(), String> {
        let request = &job.request;
        let missing: MissingUploads = connection.post("/v1/uploads/missing", &job.files)?;
        let entries: Vec<(&String, &UploadEntry, bool)> = job.files.iter().flat_map(|file| match file {
            FileRef::Upload { key, entries, directory, .. } => entries.iter()
                .filter(|entry| missing.missing.iter().any(|(k, path)| k == key && *path == entry.path))
                .map(|entry| (key, entry, *directory)).collect(),
            FileRef::Url { .. } => Vec::new(),
        }).collect();
        let total: u64 = entries.iter().map(|(_, entry, _)| entry.bytes).sum();
        let sent = Arc::new(Mutex::new((0u64, Instant::now())));
        for (key, entry, directory) in entries {
            if cancel.load(Ordering::Acquire) {
                return Err("Generation was cancelled before it started.".into());
            }
            let root = sources.get(key).ok_or("Upload source missing.")?;
            let local = if directory { root.join(safe_relative_path(&entry.path)?) } else { root.clone() };
            let file = File::open(&local).map_err(|error| format!("Could not read {}: {error}", local.display()))?;
            if file.metadata().map(|meta| meta.len()).ok() != Some(entry.bytes) {
                return Err(format!("{} changed while it was being sent. Try again.", local.display()));
            }
            let reader = Progress { inner: file, cancel: cancel.clone(), sent: sent.clone(), report: {
                let workers = self.clone();
                let request = request.clone();
                let name = entry.path.clone();
                Box::new(move |done| workers.emit("slopfab-progress", transfer_event(&request, "workerUpload", format!("Sending {name} to the worker"), done, Some(total))))
            } };
            let encoded = entry.path.split('/').map(encode_segment).collect::<Vec<_>>().join("/");
            connection.send(connection.request(reqwest::Method::PUT, &format!("/v1/uploads/{key}/{encoded}"))
                .body(Body::sized(reader, entry.bytes)))
                .map_err(|error| if cancel.load(Ordering::Acquire) { "Generation was cancelled before it started.".into() } else { error })?;
        }
        if !job.files.iter().any(|file| matches!(file, FileRef::Url { .. })) {
            return Ok(());
        }
        let id = &request.job_id;
        connection.post::<bool>(&format!("/v1/prepare/{id}"), &job.files)?;
        loop {
            std::thread::sleep(Duration::from_millis(400));
            if cancel.load(Ordering::Acquire) {
                let _ = connection.post::<bool>(&format!("/v1/prepare/{id}/cancel"), &());
                return Err("Generation was cancelled before it started.".into());
            }
            let status: FilePreparation = connection.get(&format!("/v1/prepare/{id}"))?;
            match status.state.as_str() {
                "done" => return Ok(()),
                "failed" => return Err(format!("Worker {} could not get the weights: {}", connection.name, status.error.unwrap_or_default())),
                "cancelled" => return Err("Generation was cancelled before it started.".into()),
                _ => {
                    let file = status.file.as_deref().and_then(|url| url.rsplit('/').next()).unwrap_or("weights");
                    let detail = format!("Worker downloading {file} ({} of {})", (status.completed + 1).min(status.files), status.files);
                    self.emit("slopfab-progress", transfer_event(request, "workerDownload", detail, status.downloaded, status.total));
                }
            }
        }
    }

    pub(crate) fn plan(&self, connection: &Connection, request: &GenerationRequest, settings: &BTreeMap<String, ProviderSetting>) -> Result<serde_json::Value, String> {
        let (job, sources) = build_job(request, settings, &connection.info.gpus)?;
        let cancel = self.register(&request.job_id, connection)?;
        let result = self.send_files(connection, &job, &sources, &cancel)
            .and_then(|_| connection.post::<serde_json::Value>("/v1/plan", &job));
        self.unregister(&request.job_id);
        result
    }

    /// Encodes a refmod on the worker and saves the file it returns.
    pub(crate) fn export_refmod(&self, connection: &Connection, request: &GenerationRequest, settings: &BTreeMap<String, ProviderSetting>, name: &str, description: &str, destination: &Path) -> Result<(), String> {
        // Export needs the VAEs only; keep the other weights off the network.
        let mut settings = settings.clone();
        if let Some(slopfab) = settings.get_mut("slopfab") {
            slopfab.options.retain(|option, _| matches!(option.as_str(), "videoVae" | "audioVae" | "workerWeightSources"));
        }
        let (job, sources) = build_job(request, &settings, &connection.info.gpus)?;
        let cancel = self.register(&request.job_id, connection)?;
        let result = self.send_files(connection, &job, &sources, &cancel).and_then(|_| {
            let export = RefmodExport { job, name: name.into(), description: description.into() };
            let response = connection.send(connection.request(reqwest::Method::POST, "/v1/refmods").json(&export))
                .map_err(|error| if error.contains("Unknown worker route") { format!("Update worker {} to export refmods.", connection.name) } else { error })?;
            save_response(response, destination, "the refmod")
        });
        self.unregister(&request.job_id);
        result
    }

    /// Starts a job on the worker and follows it on a background thread. The
    /// webview receives the same queued/progress/framesReady events, and the
    /// frames wait in this process's render store like a local render.
    pub(crate) fn enqueue(&self, connection: Connection, request: GenerationRequest, settings: &BTreeMap<String, ProviderSetting>) -> Result<(), String> {
        crate::media::artifacts::generated_file_stem(&request.job_id)?;
        let (job, sources) = build_job(&request, settings, &connection.info.gpus)?;
        let cancel = self.register(&request.job_id, &connection)?;
        let workers = self.clone();
        let latents = request.save_latents_path.clone();
        diagnostics::info("workers", "job.sending", "Generation is being sent to a worker.", serde_json::json!({
            "jobId": request.job_id, "files": job.files.len(),
        }));
        std::thread::Builder::new().name("worker-job".into()).spawn(move || {
            let id = request.job_id.clone();
            let result = workers.send_files(&connection, &job, &sources, &cancel)
                .and_then(|_| connection.post::<bool>(&format!("/v1/jobs/{id}"), &StartJob { job, save_latents: latents.is_some() }))
                .and_then(|_| workers.follow(&connection, &id, latents.as_deref()));
            if let Err(error) = result {
                let state = if cancel.load(Ordering::Acquire) { "cancelled" } else { "failed" };
                diagnostics::warn("workers", "job.failed", &error, serde_json::json!({ "jobId": id, "state": state }));
                workers.emit("slopfab-job", serde_json::json!({ "jobId": id, "state": state, "detail": error, "output": null }));
                connection.release(&id);
            }
            workers.unregister(&id);
        }).map_err(|error| error.to_string())?;
        Ok(())
    }

    fn follow(&self, connection: &Connection, id: &str, latents: Option<&Path>) -> Result<(), String> {
        let mut after = 0;
        let mut failures = 0;
        loop {
            let events: Vec<WireEvent> = match connection.get(&format!("/v1/jobs/{id}/events?after={after}")) {
                Ok(events) => { failures = 0; events }
                Err(error) => {
                    failures += 1;
                    if failures >= 15 {
                        return Err(format!("Lost the connection to worker {}: {error}", connection.name));
                    }
                    std::thread::sleep(Duration::from_secs(2));
                    continue;
                }
            };
            for event in events {
                after = event.seq;
                if event.kind == "preview" {
                    if !event.payload.is_null() { self.emit("slopfab-preview", event.payload); }
                    continue;
                }
                if event.kind == "progress" {
                    self.emit("slopfab-progress", event.payload);
                    continue;
                }
                let mut payload = event.payload;
                match payload.get("state").and_then(|state| state.as_str()) {
                    Some("framesReady") => {
                        let evicted = self.collect(connection, id, latents)?;
                        if let Some(job) = evicted.filter(|job| job != id) {
                            payload["detail"] = format!("Frames and audio are ready to be encoded. The frames of {job} were dropped to make room and that scene would have to be rendered again.").into();
                        }
                        self.emit("slopfab-job", payload);
                        return Ok(());
                    }
                    Some("failed") | Some("cancelled") => {
                        self.emit("slopfab-job", payload);
                        connection.release(id);
                        return Ok(());
                    }
                    _ => self.emit("slopfab-job", payload),
                }
            }
        }
    }

    /// Brings a finished job home: latents into the project now, frames on
    /// demand as the webview encodes them.
    fn collect(&self, connection: &Connection, id: &str, latents: Option<&Path>) -> Result<Option<String>, String> {
        if let Some(destination) = latents {
            let response = connection.send(connection.request(reqwest::Method::GET, &format!("/v1/jobs/{id}/latents")))?;
            save_response(response, destination, "latents")?;
        }
        let summary: rendered::RenderedSummary = connection.get(&format!("/v1/jobs/{id}/summary"))?;
        let audio: Vec<f32> = connection.bytes(&format!("/v1/jobs/{id}/audio"))?
            .chunks_exact(4).map(|chunk| f32::from_le_bytes(chunk.try_into().unwrap())).collect();
        let values = summary.frame_count as usize * summary.width as usize * summary.height as usize * 3;
        let video = rendered::from_source(
            id,
            Box::new(RemoteFrames { connection: connection.clone(), job_id: id.to_string() }),
            audio,
            summary.frame_count,
            summary.width,
            summary.height,
            3,
            values,
            summary.fps,
            summary.audio_channels,
            summary.audio_sample_rate,
        )?;
        diagnostics::info("workers", "job.collected", "Worker generation is ready to encode.", serde_json::json!({
            "jobId": id, "frames": summary.frame_count, "latents": latents.is_some(),
        }));
        Ok(rendered::keep(video))
    }

    pub(crate) fn status(&self, connection: &Connection, settings: &BTreeMap<String, ProviderSetting>) -> slopfab::SlopfabStatus {
        slopfab::worker_status(settings, &connection.info.runtime, format!("Worker {} ({})", connection.name, connection.address))
    }
}

/// Streams a worker's file beside its destination, then moves it into place.
fn save_response(mut response: Response, destination: &Path, what: &str) -> Result<(), String> {
    let temporary = destination.with_extension("safetensors.part");
    let copied = File::create(&temporary)
        .and_then(|mut file| io::copy(&mut response, &mut file).and_then(|bytes| file.sync_all().map(|_| bytes)))
        .map_err(|error| format!("Could not save {what} from the worker: {error}"));
    match copied {
        Ok(_) => fs::rename(&temporary, destination).map_err(|error| error.to_string()),
        Err(error) => {
            let _ = fs::remove_file(&temporary);
            Err(error)
        }
    }
}

/// Frames stay on the worker until the encoder asks for them, and the
/// worker's copy is released with this one.
struct RemoteFrames {
    connection: Connection,
    job_id: String,
}

impl rendered::FrameSource for RemoteFrames {
    fn frame_rgba(&self, index: u32, width: u32, height: u32) -> Result<Vec<u8>, String> {
        let frame = self.connection.bytes(&format!("/v1/jobs/{}/frames/{index}", self.job_id))?;
        if frame.len() != width as usize * height as usize * 4 {
            return Err(format!("Worker {} sent an incomplete frame {index}.", self.connection.name));
        }
        Ok(frame)
    }
}

impl Drop for RemoteFrames {
    fn drop(&mut self) {
        let (connection, job_id) = (self.connection.clone(), self.job_id.clone());
        std::thread::spawn(move || connection.release(&job_id));
    }
}

struct Progress {
    inner: File,
    cancel: Arc<AtomicBool>,
    sent: Arc<Mutex<(u64, Instant)>>,
    report: Box<dyn Fn(u64) + Send>,
}

impl Read for Progress {
    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
        if self.cancel.load(Ordering::Acquire) {
            return Err(io::Error::new(io::ErrorKind::Interrupted, "cancelled"));
        }
        let count = self.inner.read(buffer)?;
        if let Ok(mut sent) = self.sent.lock() {
            sent.0 += count as u64;
            if sent.1.elapsed() >= Duration::from_millis(250) {
                sent.1 = Instant::now();
                (self.report)(sent.0);
            }
        }
        Ok(count)
    }
}

fn encode_segment(segment: &str) -> String {
    segment.bytes().map(|byte| match byte {
        b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (byte as char).to_string(),
        _ => format!("%{byte:02X}"),
    }).collect()
}

fn transfer_event(request: &GenerationRequest, stage: &str, detail: String, done: u64, total: Option<u64>) -> serde_json::Value {
    serde_json::json!({
        "jobId": request.job_id, "stage": stage, "step": 0, "totalSteps": 0, "plannedSteps": 0,
        "elapsedSeconds": 0.0, "frames": request.frames, "canvasWidth": request.canvas_width,
        "canvasHeight": request.canvas_height, "referenceCount": request.reference_count(),
        "timingProfile": "", "detail": detail, "transferred": done, "transferTotal": total,
    })
}

fn is_url(value: &str) -> bool {
    let lower = value.trim().to_ascii_lowercase();
    lower.starts_with("https://") || lower.starts_with("http://")
}

fn modified(meta: &fs::Metadata) -> u128 {
    meta.modified().ok().and_then(|time| time.duration_since(UNIX_EPOCH).ok()).map_or(0, |time| time.as_nanos())
}

/// Identifies upload content by name, size and modification time, so an
/// unchanged file is sent once and an edited one is sent again.
fn upload_key(name: &str, files: &[(String, u64, u128)]) -> String {
    let mut hash = 0xcbf29ce484222325u64;
    let mut feed = |bytes: &[u8]| for byte in bytes { hash = (hash ^ *byte as u64).wrapping_mul(0x100000001b3); };
    feed(name.as_bytes());
    for (path, bytes, modified) in files {
        feed(path.as_bytes());
        feed(&bytes.to_le_bytes());
        feed(&modified.to_le_bytes());
    }
    format!("{hash:016x}")
}

fn walk(root: &Path, relative: &str, out: &mut Vec<(String, u64, u128)>) -> Result<(), String> {
    let directory = if relative.is_empty() { root.to_path_buf() } else { root.join(relative) };
    let mut entries = fs::read_dir(&directory).map_err(|error| format!("Could not read {}: {error}", directory.display()))?
        .collect::<Result<Vec<_>, _>>().map_err(|error| error.to_string())?;
    entries.sort_by_key(|entry| entry.file_name());
    for entry in entries {
        let name = entry.file_name().to_string_lossy().into_owned();
        let path = if relative.is_empty() { name } else { format!("{relative}/{name}") };
        let meta = entry.metadata().map_err(|error| error.to_string())?;
        if meta.is_dir() {
            walk(root, &path, out)?;
        } else if meta.is_file() {
            out.push((path, meta.len(), modified(&meta)));
        }
        if out.len() > 10_000 {
            return Err(format!("{} has too many files to send to a worker.", root.display()));
        }
    }
    Ok(())
}

fn file_ref(kind: PathKind, value: &str, sources: &mut HashMap<String, PathBuf>) -> Result<FileRef, String> {
    let lora = kind == PathKind::Lora;
    if is_url(value) {
        return Ok(FileRef::Url { url: value.trim().to_string(), lora });
    }
    let path = PathBuf::from(value);
    if kind != PathKind::Input {
        if let Some(url) = weights::recorded_download_url(&path) {
            return Ok(FileRef::Url { url, lora });
        }
    }
    let meta = fs::metadata(&path).map_err(|_| format!("{} was not found on this computer.", path.display()))?;
    let name = path.file_name().map(|name| name.to_string_lossy().into_owned()).unwrap_or_else(|| "file".into());
    safe_relative_path(&name)?;
    let (directory, files) = if meta.is_dir() {
        if kind != PathKind::Folder {
            return Err(format!("{} is a folder, not a file.", path.display()));
        }
        let mut files = Vec::new();
        walk(&path, "", &mut files)?;
        (true, files)
    } else {
        (false, vec![(name.clone(), meta.len(), modified(&meta))])
    };
    let key = upload_key(&name, &files);
    sources.insert(key.clone(), path);
    Ok(FileRef::Upload {
        key, name, directory, lora,
        entries: files.into_iter().map(|(path, bytes, _)| UploadEntry { path, bytes }).collect(),
    })
}

/// Replaces every path of a resolved local request with a token for the worker.
pub(crate) fn build_job(request: &GenerationRequest, settings: &BTreeMap<String, ProviderSetting>, gpus: &[GpuDevice]) -> Result<(RemoteJob, HashMap<String, PathBuf>), String> {
    let mut request = request.clone();
    let mut settings: BTreeMap<String, ProviderSetting> = settings.iter()
        .filter(|(name, _)| name.as_str() == "slopfab").map(|(name, value)| (name.clone(), value.clone())).collect();
    super::variants::select(&mut settings, gpus)?;
    if let Some(slopfab) = settings.get_mut("slopfab") {
        if matches!(slopfab.options.get("previewVae"), Some(slopus_core::settings::ProviderOption::String(path)) if !Path::new(path).is_file()) {
            slopfab.options.remove("previewVae");
        }
    }
    let mut files = Vec::new();
    let mut indices: HashMap<String, usize> = HashMap::new();
    let mut sources = HashMap::new();
    let mut add = |kind: PathKind, value: &str, files: &mut Vec<FileRef>| -> Result<usize, String> {
        if let Some(index) = indices.get(value) {
            return Ok(*index);
        }
        files.push(file_ref(kind, value, &mut sources)?);
        indices.insert(value.to_string(), files.len() - 1);
        Ok(files.len() - 1)
    };
    visit_paths(&mut request, &mut settings, |kind, value| {
        *value = file_token(add(kind, value, &mut files)?);
        Ok(())
    })?;
    let continuation = request.continuation_path.take()
        .map(|path| add(PathKind::Input, &path.to_string_lossy(), &mut files)).transpose()?;
    let keystone = request.keystone_path.take()
        .map(|path| add(PathKind::Input, &path.to_string_lossy(), &mut files)).transpose()?;
    let image_edit_source = request.image_edit_path.take()
        .map(|path| add(PathKind::Input, &path.to_string_lossy(), &mut files)).transpose()?;
    request.save_latents_path = None;
    request.image_edit_pixels = None;
    let bridge_left = request.bridge_left_path.take()
        .map(|path| add(PathKind::Input, &path.to_string_lossy(), &mut files)).transpose()?;
    let bridge_right = request.bridge_right_path.take()
        .map(|path| add(PathKind::Input, &path.to_string_lossy(), &mut files)).transpose()?;
    Ok((RemoteJob { request, settings, files, continuation, keystone, image_edit_source, bridge_left, bridge_right }, sources))
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::project::ProviderOption;

    pub(super) static FAILED: Mutex<Vec<String>> = Mutex::new(Vec::new());

    #[test]
    fn upscale_stream_sends_and_receives_frames_with_bounded_backpressure() {
        let server = tiny_http::Server::http("127.0.0.1:0").unwrap();
        let address = server.server_addr().to_ip().unwrap().to_string();
        let folder = tempfile::tempdir().unwrap();
        let store = Arc::new(slopus_core::worker::store::FileStore::new(&folder.path().join("worker"), Vec::new()).unwrap());
        let uploads = Arc::new(std::sync::atomic::AtomicUsize::new(0));
        let uploaded = uploads.clone();
        let config = slopfab::upscale::UpscaleConfig { method: slopfab::upscale::UpscaleMethod::Seedvr2,
            model_path: folder.path().join("model.safetensors").to_string_lossy().into_owned(),
            vae_path: Some(folder.path().join("vae.safetensors").to_string_lossy().into_owned()) };
        fs::write(&config.model_path, b"custom model").unwrap();
        fs::write(config.vae_path.as_ref().unwrap(), b"custom vae").unwrap();
        let session = slopus_core::upscale_stream::Session::start((1, 1), (1, 1), |_, read, write| {
            // Like SeedVR2, wait for a temporal segment before writing output.
            let mut frames = Vec::new();
            while let Some(frame) = read()? { frames.push(frame); }
            let count = frames.len() as u64;
            for frame in frames { write(frame)?; }
            Ok(count)
        }).unwrap();
        let server_thread = std::thread::spawn(move || {
            while let Some(mut request) = server.recv_timeout(Duration::from_secs(5)).unwrap() {
                let session = session.clone();
                let store = store.clone();
                let uploaded = uploaded.clone();
                let remove = request.method() == &tiny_http::Method::Delete;
                std::thread::spawn(move || {
                    assert!(request.headers().iter().any(|header| header.field.equiv(TOKEN_HEADER) && header.value.as_str() == "test-token"));
                    let response = match (request.method().clone(), request.url()) {
                        (tiny_http::Method::Post, "/v1/uploads/missing") => {
                            let files: Vec<FileRef> = serde_json::from_reader(request.as_reader()).unwrap();
                            serde_json::to_vec(&MissingUploads { missing: store.missing(&files).unwrap() }).unwrap()
                        }
                        (tiny_http::Method::Put, path) if path.starts_with("/v1/uploads/") => {
                            let parts: Vec<_> = path.split('/').map(str::to_owned).collect();
                            store.receive(&parts[3], &parts[4], request.body_length().unwrap() as u64, request.as_reader()).unwrap();
                            uploaded.fetch_add(1, Ordering::Relaxed);
                            b"true".to_vec()
                        }
                        (tiny_http::Method::Post, "/v1/upscales/test") => {
                            let options: StartUpscale = serde_json::from_reader(request.as_reader()).unwrap();
                            assert_eq!(fs::read(store.resolve(options.model.as_ref().unwrap()).unwrap()).unwrap(), b"custom model");
                            assert_eq!(fs::read(store.resolve(options.vae.as_ref().unwrap()).unwrap()).unwrap(), b"custom vae");
                            b"true".to_vec()
                        }
                        (tiny_http::Method::Post, "/v1/upscales/test/frames") => {
                            let mut bytes = Vec::new(); request.as_reader().read_to_end(&mut bytes).unwrap();
                            session.push(bytes).unwrap(); b"true".to_vec()
                        }
                        (tiny_http::Method::Post, "/v1/upscales/test/finish") => { session.finish().unwrap(); b"true".to_vec() }
                        (tiny_http::Method::Get, "/v1/upscales/test/frames") => session.read().unwrap(),
                        (tiny_http::Method::Delete, "/v1/upscales/test") => { session.cancel(); b"true".to_vec() }
                        _ => panic!("Unexpected upscale request"),
                    };
                    request.respond(tiny_http::Response::from_data(response)).unwrap();
                });
                if remove { break; }
            }
        });
        let info = WorkerInfo { id: "00112233aabbccdd".into(), name: "Stub".into(), version: "0".into(), protocol: PROTOCOL_VERSION,
            requires_token: true, gpus: Vec::new(), runtime: WorkerRuntime { state: "ready".into(), version: None,
                platform: None, cuda_available: true, cuda_device_names: Vec::new(), detail: String::new() } };
        let connection = Connection { base: format!("http://{address}"), token: Some("test-token".into()), name: "Stub".into(), address, info, http: http_client(Some(Duration::from_secs(5))) };
        let workers = Workers::default();
        let files = workers.prepare_upscale_files(&connection, "test", &config, &AtomicBool::new(false)).unwrap();
        assert_eq!(workers.prepare_upscale_files(&connection, "test", &config, &AtomicBool::new(false)).unwrap(), files);
        assert_eq!(uploads.load(Ordering::Relaxed), 2, "unchanged weights are uploaded only once");
        assert!(workers.prepare_upscale_files(&connection, "test", &config, &AtomicBool::new(true)).is_err());
        let default_config = slopfab::upscale::UpscaleConfig { model_path: String::new(), vae_path: None, ..config };
        let defaults = workers.prepare_upscale_files(&connection, "test", &default_config, &AtomicBool::new(false)).unwrap();
        assert_eq!(defaults, [FileRef::Url { url: SEEDVR2_MODEL.into(), lora: false }, FileRef::Url { url: SEEDVR2_VAE.into(), lora: false }]);
        let [model, vae] = files;
        let mut source = vec![vec![1; 4], vec![2; 4], vec![3; 4], vec![4; 4], vec![5; 4]].into_iter();
        let mut frames = Vec::new();
        let count = connection.upscale("test", &StartUpscale { input_width: 1, input_height: 1, width: 1, height: 1, segment_frames: 5, model: Some(model), vae: Some(vae) },
            &AtomicBool::new(false), &mut || Ok(source.next()), &mut |frame| { frames.push(frame); Ok(()) }).unwrap();
        assert_eq!(count, 5);
        assert_eq!(frames, vec![vec![1; 4], vec![2; 4], vec![3; 4], vec![4; 4], vec![5; 4]]);
        server_thread.join().unwrap();
    }

    #[test]
    fn addresses_get_the_default_port_and_reject_urls_with_paths() {
        assert_eq!(normalize_address("192.168.1.20").unwrap(), format!("192.168.1.20:{DEFAULT_PORT}"));
        assert_eq!(normalize_address(" http://pc:9000/ ").unwrap(), "pc:9000");
        assert!(normalize_address("pc/v1/info").is_err());
        assert!(normalize_address("").is_err());
    }

    #[test]
    fn long_shot_transfers_both_anchors_without_serializing_host_paths() {
        let folder = tempfile::tempdir().unwrap();
        let left = folder.path().join("left.safetensors");
        let right = folder.path().join("right.safetensors");
        fs::write(&left, b"left archive").unwrap();
        fs::write(&right, b"right archive").unwrap();
        let request = GenerationRequest { bridge_left_path: Some(left), bridge_right_path: Some(right),
            latent_bridge: Some(slopfab::LatentBridgeRequest { left_margin_frames: 17, right_margin_frames: 34, context_frames: 22, ..Default::default() }),
            ..Default::default() };
        let (job, uploads) = build_job(&request, &BTreeMap::new(), &[]).unwrap();
        assert_eq!(job.files.len(), 2);
        assert_eq!(uploads.len(), 2);
        assert_eq!(job.bridge_left, Some(0));
        assert_eq!(job.bridge_right, Some(1));
        assert!(job.request.bridge_left_path.is_none());
        assert!(job.request.bridge_right_path.is_none());
        let wire = serde_json::to_value(&job).unwrap();
        assert_eq!(wire["request"]["latentBridge"]["rightMarginFrames"], 34);
        assert!(wire["request"].get("bridgeLeftPath").is_none());
    }

    #[test]
    fn latent_upscale_jobs_transfer_custom_weights_or_download_on_the_worker() {
        let folder = tempfile::tempdir().unwrap();
        let local = folder.path().join("latent-upscaler.safetensors");
        fs::write(&local, b"weights").unwrap();
        for path in ["https://huggingface.co/LBH-123-AI/Minimax_h3_latent_Upscaler/resolve/main/minimax_h3_latent_upscaler_3d_conv_v1/minimax_h3_latent_upscaler_3d_conv_v1_fp16.safetensors".to_string(), local.to_string_lossy().into_owned()] {
            let settings = BTreeMap::from([("slopfab".into(), ProviderSetting { enabled: true, model: None,
                options: BTreeMap::from([("latentUpscaler".into(), ProviderOption::String(path.clone()))]) })]);
            let request = GenerationRequest { latent_upscale: true, canvas_width: 736, canvas_height: 416, ..Default::default() };
            let (job, uploads) = build_job(&request, &settings, &[]).unwrap();
            assert!(job.request.latent_upscale);
            assert_eq!((job.request.canvas_width, job.request.canvas_height), (736, 416));
            assert_eq!(job.settings["slopfab"].options["latentUpscaler"], ProviderOption::String(file_token(0)));
            assert_eq!(job.files.len(), 1);
            if path.starts_with("https:") {
                assert_eq!(job.files[0], FileRef::Url { url: path, lora: false });
                assert!(uploads.is_empty());
            } else {
                assert!(matches!(job.files[0], FileRef::Upload { .. }));
                assert_eq!(uploads.len(), 1);
            }
            let (ordinary, _) = build_job(&GenerationRequest::default(), &settings, &[]).unwrap();
            assert!(ordinary.files.is_empty());
        }
    }

    #[test]
    fn jobs_send_urls_for_downloads_and_uploads_for_local_files() {
        let folder = tempfile::tempdir().unwrap();
        let image = folder.path().join("ref one.png");
        fs::write(&image, b"png").unwrap();
        let tokenizer = folder.path().join("tokenizer");
        fs::create_dir_all(tokenizer.join("nested")).unwrap();
        fs::write(tokenizer.join("vocab.json"), b"{}").unwrap();
        fs::write(tokenizer.join("nested/merges.txt"), b"m").unwrap();
        let options = BTreeMap::from([
            ("transformer".to_string(), ProviderOption::String("https://example.com/t.safetensors".into())),
            ("tokenizer".to_string(), ProviderOption::String(tokenizer.to_string_lossy().into_owned())),
            ("attention".to_string(), ProviderOption::String("sage2".into())),
        ]);
        let settings = BTreeMap::from([
            ("slopfab".to_string(), ProviderSetting { enabled: true, model: None, options }),
            ("openrouter".to_string(), ProviderSetting { enabled: true, model: None, options: BTreeMap::from([("apiKey".to_string(), ProviderOption::String("secret".into()))]) }),
        ]);
        let request = GenerationRequest {
            job_id: "work-1".into(),
            reference_paths: vec![image.to_string_lossy().into_owned(), image.to_string_lossy().into_owned()],
            continuation_path: Some(image.clone()),
            keystone_path: Some(image.clone()),
            continuation_from: Some("start".into()),
            continuation_overlap_frames: Some(39),
            continuation_source_frames: Some(85),
            continuation_lock_overlap: true,
            audio_steps: Some(17),
            video_sigma_shift: Some(8.5),
            audio_sigma_shift: Some(2.5),
            generation_effects: Some(serde_json::from_str(include_str!("../../../fixtures/generation-effects.json")).unwrap()),
            ..Default::default()
        };
        let (job, sources) = build_job(&request, &settings, &[]).unwrap();
        assert!(!job.settings.contains_key("openrouter"), "credentials never leave this computer");
        assert_eq!(job.files.len(), 3, "a repeated path is sent once");
        assert_eq!(job.files[0], FileRef::Url { url: "https://example.com/t.safetensors".into(), lora: false });
        let FileRef::Upload { directory: true, entries, .. } = &job.files[1] else { panic!("tokenizer folder") };
        assert_eq!(entries.iter().map(|entry| entry.path.as_str()).collect::<Vec<_>>(), vec!["nested/merges.txt", "vocab.json"]);
        assert_eq!(job.request.reference_paths, vec![file_token(2), file_token(2)]);
        assert_eq!(job.continuation, Some(2));
        assert_eq!(job.keystone, Some(2));
        let wire = serde_json::to_value(&job).unwrap();
        assert!(wire["request"].get("keystonePath").is_none());
        assert_eq!(wire["keystone"], 2);
        assert_eq!(wire["request"]["continuationFrom"], "start");
        assert_eq!(wire["request"]["continuationOverlapFrames"], 39);
        assert_eq!(wire["request"]["continuationSourceFrames"], 85);
        assert_eq!(wire["request"]["continuationLockOverlap"], true);
        assert_eq!(wire["request"]["audioSteps"], 17);
        assert_eq!(wire["request"]["videoSigmaShift"], 8.5);
        assert_eq!(wire["request"]["audioSigmaShift"], 2.5);
        assert_eq!(wire["request"]["generationEffects"]["color"]["strength"], 0.5);
        assert_eq!(sources.len(), 2);
        assert!(build_job(&GenerationRequest { reference_paths: vec![folder.path().join("missing.png").to_string_lossy().into_owned()], ..Default::default() }, &BTreeMap::new(), &[]).is_err());
    }

    #[test]
    fn finished_jobs_bring_latents_home_and_release_the_worker_copy_with_the_frames() {
        let server = tiny_http::Server::http("127.0.0.1:0").unwrap();
        let address = server.server_addr().to_ip().unwrap().to_string();
        let released = Arc::new(AtomicBool::new(false));
        let seen = released.clone();
        std::thread::spawn(move || {
            for request in server.incoming_requests() {
                let url = request.url().to_string();
                let body: Vec<u8> = match url.as_str() {
                    "/v1/jobs/work-collect/latents" => b"latent bytes".to_vec(),
                    "/v1/jobs/work-collect/summary" => serde_json::to_vec(&serde_json::json!({
                        "jobId": "work-collect", "width": 2, "height": 1, "frameCount": 2, "fps": 24.0,
                        "audioChannels": 1, "audioSampleRate": 48000, "audioSamples": 2,
                    })).unwrap(),
                    "/v1/jobs/work-collect/audio" => [0.5f32, -0.5].iter().flat_map(|sample| sample.to_le_bytes()).collect(),
                    "/v1/jobs/work-collect/frames/1" => vec![9; 8],
                    "/v1/jobs/work-collect" => { seen.store(true, Ordering::Release); b"true".to_vec() }
                    _ => { let _ = request.respond(tiny_http::Response::empty(404)); continue; }
                };
                let _ = request.respond(tiny_http::Response::from_data(body));
            }
        });
        let info = WorkerInfo {
            id: "00112233aabbccdd".into(), name: "Stub".into(), version: "0".into(), protocol: PROTOCOL_VERSION,
            requires_token: false, gpus: Vec::new(),
            runtime: WorkerRuntime { state: "ready".into(), version: None, platform: None, cuda_available: false, cuda_device_names: Vec::new(), detail: String::new() },
        };
        let connection = Connection { base: format!("http://{address}"), token: None, name: "Stub".into(), address, info, http: http_client(None) };
        let folder = tempfile::tempdir().unwrap();
        let latents = folder.path().join("work-collect.safetensors");
        Workers::default().collect(&connection, "work-collect", Some(&latents)).unwrap();
        assert_eq!(fs::read(&latents).unwrap(), b"latent bytes");
        assert!(!folder.path().join("work-collect.safetensors.part").exists());
        assert_eq!(rendered::summary("work-collect").unwrap().audio_samples, 2);
        assert_eq!(rendered::frame("work-collect", 1).unwrap().unwrap(), vec![9; 8]);
        assert!(rendered::frame("work-collect", 0).is_err(), "a missing frame is an error, not a blank picture");
        assert!(!released.load(Ordering::Acquire));
        assert!(rendered::release("work-collect"));
        let started = Instant::now();
        while !released.load(Ordering::Acquire) {
            assert!(started.elapsed() < Duration::from_secs(5), "the worker copy was not released");
            std::thread::sleep(Duration::from_millis(20));
        }
    }

    #[test]
    fn refmod_exports_send_only_vaes_and_save_the_worker_file() {
        let server = tiny_http::Server::http("127.0.0.1:0").unwrap();
        let address = server.server_addr().to_ip().unwrap().to_string();
        let posted = Arc::new(Mutex::new(None::<RefmodExport>));
        let seen = posted.clone();
        std::thread::spawn(move || {
            for mut request in server.incoming_requests() {
                let url = request.url().to_string();
                let body: Vec<u8> = match url.as_str() {
                    "/v1/uploads/missing" => br#"{"missing":[]}"#.to_vec(),
                    "/v1/refmods" => {
                        *seen.lock().unwrap() = serde_json::from_reader(request.as_reader()).ok();
                        b"refmod bytes".to_vec()
                    }
                    _ => { let _ = request.respond(tiny_http::Response::empty(404)); continue; }
                };
                let _ = request.respond(tiny_http::Response::from_data(body));
            }
        });
        let info = WorkerInfo {
            id: "00112233aabbccdd".into(), name: "Stub".into(), version: "0".into(), protocol: PROTOCOL_VERSION,
            requires_token: false, gpus: Vec::new(),
            runtime: WorkerRuntime { state: "ready".into(), version: None, platform: None, cuda_available: false, cuda_device_names: Vec::new(), detail: String::new() },
        };
        let connection = Connection { base: format!("http://{address}"), token: None, name: "Stub".into(), address, info, http: http_client(None) };
        let folder = tempfile::tempdir().unwrap();
        let file = |name: &str| {
            let path = folder.path().join(name);
            fs::write(&path, name).unwrap();
            path.to_string_lossy().into_owned()
        };
        let options = ["transformer", "videoVae", "audioVae"].into_iter()
            .map(|name| (name.to_string(), ProviderOption::String(file(&format!("{name}.safetensors"))))).collect();
        let settings = BTreeMap::from([("slopfab".to_string(), ProviderSetting { enabled: true, model: None, options })]);
        let request = GenerationRequest { job_id: "refmod-export".into(), reference_paths: vec![file("hero.png")], ..Default::default() };
        let destination = folder.path().join("hero.safetensors");
        Workers::default().export_refmod(&connection, &request, &settings, "Hero", "A hero", &destination).unwrap();
        assert_eq!(fs::read(&destination).unwrap(), b"refmod bytes");
        let export = posted.lock().unwrap().take().unwrap();
        assert_eq!((export.name.as_str(), export.description.as_str()), ("Hero", "A hero"));
        assert_eq!(export.job.files.len(), 3, "the transformer stays on this computer");
        assert!(!export.job.settings["slopfab"].options.contains_key("transformer"));
    }

    /// Drives a running worker through the same path as the app. Needs
    /// SLOPUS_E2E_WORKER (host:port) and SLOPUS_E2E_WEIGHTS (a folder of
    /// Slopus downloads: transformer, text encoder and VAEs).
    #[test]
    #[ignore]
    fn end_to_end_with_a_running_worker() {
        let address = std::env::var("SLOPUS_E2E_WORKER").expect("SLOPUS_E2E_WORKER");
        let weights = PathBuf::from(std::env::var("SLOPUS_E2E_WEIGHTS").expect("SLOPUS_E2E_WEIGHTS"));
        let find = |part: &str| fs::read_dir(&weights).unwrap().filter_map(Result::ok).map(|entry| entry.path())
            .find(|path| path.to_string_lossy().contains(part) && path.extension().is_some_and(|ext| ext == "safetensors"))
            .unwrap_or_else(|| panic!("no {part} weights")).to_string_lossy().into_owned();
        let folder = tempfile::tempdir().unwrap();
        let reference = folder.path().join("reference.png");
        image::RgbImage::from_fn(256, 256, |x, y| image::Rgb([x as u8, y as u8, 128])).save(&reference).unwrap();
        let options = BTreeMap::from([
            ("transformer".to_string(), ProviderOption::String(find("ref2va_Pruned"))),
            ("textEncoder".to_string(), ProviderOption::String(find("qwen3vl"))),
            ("videoVae".to_string(), ProviderOption::String(find("video_vae_fp16"))),
            ("audioVae".to_string(), ProviderOption::String(find("audio_vae"))),
            ("attention".to_string(), ProviderOption::String("sage2".into())),
        ]);
        let settings = BTreeMap::from([("slopfab".to_string(), ProviderSetting { enabled: true, model: None, options })]);
        let workers = Workers::default();
        let list = workers.add(&address).unwrap();
        let id = list.workers[0].id.clone().unwrap();
        workers.select(Some(id)).unwrap();
        let connection = workers.active().unwrap().expect("selected worker");
        let job_id = format!("work-e2e-{}", std::process::id());
        let request = GenerationRequest {
            job_id: job_id.clone(), prompt: "A red apple on a wooden table.".into(), frames: 1, still_image: true,
            steps: 8, seed: 7, canvas_width: 512, canvas_height: 512,
            reference_paths: vec![reference.to_string_lossy().into_owned()],
            ..Default::default()
        };
        let (job, _) = build_job(&request, &settings, &connection.info.gpus).unwrap();
        assert!(job.files.iter().filter(|file| matches!(file, FileRef::Url { .. })).count() == 4, "downloads are sent as URLs");
        let plan = workers.plan(&connection, &request, &settings).unwrap();
        eprintln!("plan: {plan}");
        workers.enqueue(connection, request, &settings).unwrap();
        let started = Instant::now();
        let summary = loop {
            if let Some(summary) = rendered::summary(&job_id) { break summary; }
            if let Some(error) = FAILED.lock().unwrap().first() { panic!("worker job failed: {error}"); }
            assert!(started.elapsed() < Duration::from_secs(900), "worker job timed out");
            std::thread::sleep(Duration::from_millis(500));
        };
        assert_eq!((summary.width, summary.height, summary.frame_count), (512, 512, 1));
        let frame = rendered::frame(&job_id, 0).unwrap().unwrap();
        assert_eq!(frame.len(), 512 * 512 * 4);
        image::RgbaImage::from_raw(512, 512, frame).unwrap().save(std::env::temp_dir().join("slopus-worker-e2e.png")).unwrap();
        assert!(rendered::release(&job_id));
    }
}