Skip to content

AI media harness for planning, generating, and editing locally

TypeScript
22.7 KB
Raw Blame
//! What a finished generation leaves behind, between slopfab and the encoder.
//!
//! slopfab hands back raw pictures — planar RGB floats, one channel after
//! another, with every frame of a channel contiguous — and frees them when the
//! generation handle is destroyed. Nothing in this process can turn those into
//! an MP4: encoding is the webview's job, because WebCodecs owns the OS encoder
//! and Slopus ships no FFmpeg by design (see docs/architecture.md, and
//! CLAUDE.md on the licensing reason).
//!
//! So this module is the hand-off. It keeps the finished generation handle
//! alive and asks slopfab to convert only the frame the consumer currently
//! needs. Still images release the handle after copying RGBA; videos release
//! it after WebCodecs consumes the samples, before muxing and writing the file.
//!
//! ## What this costs
//!
//! Only one RGBA frame is allocated per IPC request. Slopfab's decoded planar
//! floats remain owned by its generation handle, avoiding the former second
//! whole-video allocation and eager conversion pass.
//!
//! [`CAPACITY`] is what stops that growing without limit.

use std::{
    collections::VecDeque,
    sync::{LazyLock, Mutex},
};

/// How many finished renders may wait for the webview at once.
///
/// The queue is serial and the webview claims a render as soon as the event
/// announcing it lands, so one is the normal number and two is slack for a
/// window that was busy. Beyond that the OLDEST is dropped — its frames are
/// gone and the scene has to be rendered again. That is the honest trade: the
/// alternative is a process that grows by a gigabyte per abandoned render until
/// the machine gives out.
const CAPACITY: usize = 2;

pub trait FrameSource: Send {
    fn frame_rgba(&self, index: u32, width: u32, height: u32) -> Result<Vec<u8>, String>;
}

/// A finished render whose slopfab source lives until its samples are consumed.
pub struct RenderedVideo {
    pub job_id: String,
    pub width: u32,
    pub height: u32,
    pub frame_count: u32,
    pub fps: f64,
    source: Box<dyn FrameSource>,
    /// Interleaved samples, as slopfab returned them.
    audio: Vec<f32>,
    pub audio_channels: u32,
    pub audio_sample_rate: u32,
}

impl RenderedVideo {
    fn summary(&self) -> RenderedSummary {
        RenderedSummary {
            job_id: self.job_id.clone(),
            width: self.width,
            height: self.height,
            frame_count: self.frame_count,
            fps: self.fps,
            audio_channels: self.audio_channels,
            audio_sample_rate: self.audio_sample_rate,
            audio_samples: self.audio.len() as u32,
        }
    }

    fn into_still(self) -> Result<(RenderedSummary, Vec<u8>), String> {
        if self.frame_count != 1 {
            return Err("Expected a single still image.".into());
        }
        let summary = self.summary();
        let pixels = self.frame(0)?.ok_or("The generated image has no pixels.")?;
        // Dropping self releases the native generation before image encoding
        // allocates its own buffers. Errors release the same ownership.
        Ok((summary, pixels))
    }

    /// One frame's pixels, converted on demand, or None past the last frame.
    pub fn frame(&self, index: u32) -> Result<Option<Vec<u8>>, String> {
        if index >= self.frame_count {
            return Ok(None);
        }
        self.source
            .frame_rgba(index, self.width, self.height)
            .map(Some)
    }

    /// The soundtrack as little-endian f32 bytes, which is what the webview's
    /// `Float32Array` reads without a conversion of its own.
    pub fn audio_bytes(&self) -> Vec<u8> {
        let mut bytes = Vec::with_capacity(self.audio.len() * 4);
        for sample in &self.audio {
            bytes.extend_from_slice(&sample.to_le_bytes());
        }
        bytes
    }
}

/// How the floats slopfab returns map onto 0–255.
///
/// The C API documents the layout (planar RGB) but not the range, and the two
/// conventions a decoder can use — 0…1 and −1…1 — differ by a factor of two and
/// an offset. Getting it wrong is not subtle: a signed picture read as unsigned
/// is blown-out and half black. So it is READ off the pictures rather than
/// assumed, from the first frame, which is the smallest sample that can settle
/// it: a picture with any meaningfully negative value cannot be 0…1.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[cfg(test)]
pub enum Range {
    /// 0.0 is black, 1.0 is white.
    Unit,
    /// −1.0 is black, 1.0 is white.
    Signed,
}

#[cfg(test)]
impl Range {
    /// Anything below this is a negative the 0…1 convention has no room for.
    /// Not zero, because a decoder that clamps at zero still lands a whisker
    /// under it now and then.
    const NEGATIVE: f32 = -0.02;

    pub fn of(sample: &[f32]) -> Self {
        if sample.iter().any(|value| *value < Self::NEGATIVE) {
            Self::Signed
        } else {
            Self::Unit
        }
    }

    fn to_byte(self, value: f32) -> u8 {
        // NaN maps to black rather than to whatever `as u8` does with it, which
        // is 0 on every target Rust supports but is not something to lean on.
        if value.is_nan() {
            return 0;
        }
        let unit = match self {
            Self::Unit => value,
            Self::Signed => (value + 1.0) / 2.0,
        };
        (unit.clamp(0.0, 1.0) * 255.0).round() as u8
    }
}

/// Turns slopfab's planar float pictures into the RGBA the webview can encode.
///
/// The C API declares the layout as `[channels][frames][height][width]`: all
/// red frames first, then all green frames, then all blue frames. In
/// particular, it is not `[frames][channels][height][width]`. Treating it as
/// frame-major makes an RGB render look like the same third-length motion clip
/// played three times, once from each channel plane.
///
/// One channel is greyscale and is written to all three; more than three means
/// the first three are the colour and the rest is not ours to interpret.
#[cfg(test)]
pub fn rgba_from_planar(
    video: &[f32],
    frame_count: u32,
    width: u32,
    height: u32,
    channels: u32,
) -> Result<Vec<u8>, String> {
    let (frames, width_px, height_px, planes) = (
        frame_count as usize,
        width as usize,
        height as usize,
        channels as usize,
    );
    if frames == 0 || width_px == 0 || height_px == 0 || planes == 0 {
        return Err("The render returned no pictures.".to_string());
    }
    let pixels = width_px * height_px;
    let expected = frames
        .checked_mul(planes)
        .and_then(|value| value.checked_mul(pixels))
        .ok_or_else(|| "The render's dimensions do not fit in memory.".to_string())?;
    if video.len() < expected {
        return Err(format!(
            "The render says it has {frames} frames of {width_px}×{height_px}×{planes}, which is {expected} values, but only {} came back.",
            video.len()
        ));
    }
    let channel_stride = frames * pixels;
    let range = if (0..planes).any(|channel| {
        let start = channel * channel_stride;
        Range::of(&video[start..start + pixels]) == Range::Signed
    }) {
        Range::Signed
    } else {
        Range::Unit
    };
    let mut rgba = vec![255u8; frames * pixels * 4];
    for frame in 0..frames {
        let frame_offset = frame * pixels;
        let red = &video[frame_offset..frame_offset + pixels];
        let green = if planes >= 2 {
            &video[channel_stride + frame_offset..channel_stride + frame_offset + pixels]
        } else {
            red
        };
        let blue = if planes >= 3 {
            &video[2 * channel_stride + frame_offset..2 * channel_stride + frame_offset + pixels]
        } else {
            red
        };
        let out = &mut rgba[frame * pixels * 4..(frame + 1) * pixels * 4];
        for pixel in 0..pixels {
            out[pixel * 4] = range.to_byte(red[pixel]);
            out[pixel * 4 + 1] = range.to_byte(green[pixel]);
            out[pixel * 4 + 2] = range.to_byte(blue[pixel]);
            // Alpha was filled with 255 up front; a generated frame is opaque.
        }
    }
    Ok(rgba)
}

/// Builds the hand-off around a source that owns the finished slopfab handle.
#[allow(clippy::too_many_arguments)]
pub fn from_source(
    job_id: &str,
    source: Box<dyn FrameSource>,
    audio: Vec<f32>,
    frame_count: u32,
    width: u32,
    height: u32,
    channels: u32,
    video_float_count: usize,
    fps: f64,
    audio_channels: u32,
    audio_sample_rate: u32,
) -> Result<RenderedVideo, String> {
    let expected_video_values = (frame_count as usize)
        .checked_mul(width as usize)
        .and_then(|value| value.checked_mul(height as usize))
        .and_then(|value| value.checked_mul(channels as usize))
        .ok_or_else(|| "The render's dimensions do not fit in memory.".to_string())?;
    let audio_only = frame_count == 0 && width == 0 && height == 0 && video_float_count == 0 && !audio.is_empty();
    if !audio_only && (frame_count == 0 || width == 0 || height == 0 || channels != 3) {
        return Err(format!(
            "The render returned {frame_count} frames of {width}x{height}x{channels}."
        ));
    }
    if video_float_count < expected_video_values {
        return Err(format!(
            "The decoded video needs {expected_video_values} values but only {video_float_count} came back."
        ));
    }
    if !audio.is_empty()
        && (audio_channels == 0
            || audio_sample_rate == 0
            || audio.len() % audio_channels as usize != 0)
    {
        return Err(format!(
            "The render returned {} audio samples with {audio_channels} channels at {audio_sample_rate} Hz.",
            audio.len()
        ));
    }
    Ok(RenderedVideo {
        job_id: job_id.to_string(),
        width,
        height,
        frame_count,
        fps,
        source,
        audio,
        audio_channels,
        audio_sample_rate,
    })
}

#[cfg(test)]
struct OwnedRgba(Vec<u8>);

#[cfg(test)]
impl FrameSource for OwnedRgba {
    fn frame_rgba(&self, index: u32, width: u32, height: u32) -> Result<Vec<u8>, String> {
        let size = width as usize * height as usize * 4;
        let start = index as usize * size;
        self.0
            .get(start..start + size)
            .map(<[u8]>::to_vec)
            .ok_or_else(|| format!("Frame {index} is outside the owned test video."))
    }
}

#[cfg(test)]
#[allow(clippy::too_many_arguments)]
fn from_output(
    job_id: &str,
    video: &[f32],
    audio: &[f32],
    frame_count: u32,
    width: u32,
    height: u32,
    channels: u32,
    fps: f64,
    audio_channels: u32,
    audio_sample_rate: u32,
) -> Result<RenderedVideo, String> {
    let rgba = rgba_from_planar(video, frame_count, width, height, channels)?;
    from_source(
        job_id,
        Box::new(OwnedRgba(rgba)),
        audio.to_vec(),
        frame_count,
        width,
        height,
        channels,
        video.len(),
        fps,
        audio_channels,
        audio_sample_rate,
    )
}

static WAITING: LazyLock<Mutex<VecDeque<RenderedVideo>>> = LazyLock::new(Default::default);

// Tests that exercise the shared capacity must not evict one another's jobs.
#[cfg(test)]
pub(crate) static TEST_LOCK: Mutex<()> = Mutex::new(());

/// Puts a finished render where the webview can fetch it. Returns what had to
/// be dropped to make room, so the caller can say so rather than let a render
/// disappear quietly.
pub fn keep(video: RenderedVideo) -> Option<String> {
    let Ok(mut waiting) = WAITING.lock() else {
        return None;
    };
    // A second run of the same scene replaces the first: the frames the user
    // wants are the ones they just watched being made.
    waiting.retain(|held| held.job_id != video.job_id);
    waiting.push_back(video);
    if waiting.len() > CAPACITY {
        return waiting.pop_front().map(|dropped| dropped.job_id);
    }
    None
}

/// Reads something out of one waiting render. Nothing is copied for a job that
/// is not there — the answer is None and the caller says so.
fn with<T>(job_id: &str, read: impl FnOnce(&RenderedVideo) -> T) -> Option<T> {
    let waiting = WAITING.lock().ok()?;
    waiting.iter().find(|held| held.job_id == job_id).map(read)
}

pub fn frame(job_id: &str, index: u32) -> Result<Option<Vec<u8>>, String> {
    let waiting = WAITING
        .lock()
        .map_err(|_| "Finished-render lock failed.".to_string())?;
    match waiting.iter().find(|held| held.job_id == job_id) {
        Some(video) => video.frame(index),
        None => Ok(None),
    }
}

pub fn audio(job_id: &str) -> Option<Vec<u8>> {
    with(job_id, RenderedVideo::audio_bytes)
}

/// What the webview needs before it can encode: the shape of the pictures it is
/// about to ask for, one at a time, and how much sound came with them.
///
/// Read from the STORE rather than from the event that announced the render.
/// The event is a notification — it can be missed, it can arrive at a window
/// that is reloading — and the only authority on what can still be encoded is
/// what is still held.
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct RenderedSummary {
    pub job_id: String,
    pub width: u32,
    pub height: u32,
    pub frame_count: u32,
    pub fps: f64,
    pub audio_channels: u32,
    pub audio_sample_rate: u32,
    /// Interleaved samples across every channel, so an empty soundtrack is
    /// distinguishable from one that has not been read yet.
    pub audio_samples: u32,
}

pub fn summary(job_id: &str) -> Option<RenderedSummary> {
    with(job_id, RenderedVideo::summary)
}

/// Transfers a still's pixels to its finalizer and immediately releases the
/// generation. The caller owns everything needed to encode the image now.
pub fn take_still(job_id: &str) -> Result<(RenderedSummary, Vec<u8>), String> {
    let video = {
        let mut waiting = WAITING
            .lock()
            .map_err(|_| "Finished-render lock failed.".to_string())?;
        let index = waiting.iter().position(|video| video.job_id == job_id)
            .ok_or("The generated image is no longer in memory.")?;
        waiting.remove(index).unwrap()
    };
    video.into_still()
}

/// Hands the memory back. Called when the webview has written the file, and
/// when it has given up on writing it — a render nobody can encode is still a
/// gigabyte nobody should be holding.
pub fn release(job_id: &str) -> bool {
    let Ok(mut waiting) = WAITING.lock() else {
        return false;
    };
    let before = waiting.len();
    waiting.retain(|held| held.job_id != job_id);
    waiting.len() != before
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::sync::{atomic::{AtomicBool, Ordering}, Arc};

    #[test]
    fn audio_only_results_keep_samples_without_requesting_video_frames() {
        let dropped = Arc::new(AtomicBool::new(false));
        let result = from_source("music", Box::new(TrackedStill { dropped: dropped.clone(), fail: true }),
            vec![0.25, -0.25], 0, 0, 0, 0, 0, 0.0, 2, 48000).unwrap();
        assert!(result.frame(0).unwrap().is_none());
        assert_eq!(result.audio_bytes(), [0.25_f32.to_le_bytes(), (-0.25_f32).to_le_bytes()].concat());
        assert_eq!(result.summary().audio_samples, 2);
        drop(result);
        assert!(dropped.load(Ordering::Acquire));
    }

    struct TrackedStill {
        dropped: Arc<AtomicBool>,
        fail: bool,
    }

    impl FrameSource for TrackedStill {
        fn frame_rgba(&self, _: u32, _: u32, _: u32) -> Result<Vec<u8>, String> {
            assert!(!self.dropped.load(Ordering::Acquire));
            if self.fail { Err("Frame conversion failed.".into()) } else { Ok(vec![20, 40, 60, 255]) }
        }
    }

    impl Drop for TrackedStill {
        fn drop(&mut self) { self.dropped.store(true, Ordering::Release); }
    }

    #[test]
    fn consuming_a_still_releases_its_source_before_the_encoder_receives_pixels() {
        let _serial = TEST_LOCK.lock().unwrap();
        let dropped = Arc::new(AtomicBool::new(false));
        let mut video = sample("consume-still");
        video.source = Box::new(TrackedStill { dropped: dropped.clone(), fail: false });
        keep(video);
        let (summary, pixels) = take_still("consume-still").unwrap();
        assert!(dropped.load(Ordering::Acquire));
        assert_eq!(summary.frame_count, 1);
        assert_eq!(pixels, vec![20, 40, 60, 255]);
        assert!(!release("consume-still"));
    }

    #[test]
    fn consuming_a_still_releases_its_source_when_conversion_fails() {
        let _serial = TEST_LOCK.lock().unwrap();
        let dropped = Arc::new(AtomicBool::new(false));
        let mut video = sample("failed-still");
        video.source = Box::new(TrackedStill { dropped: dropped.clone(), fail: true });
        keep(video);
        assert_eq!(take_still("failed-still").unwrap_err(), "Frame conversion failed.");
        assert!(dropped.load(Ordering::Acquire));
        assert!(!release("failed-still"));
    }

    fn planar(
        frames: usize,
        width: usize,
        height: usize,
        channels: usize,
        value: impl Fn(usize) -> f32,
    ) -> Vec<f32> {
        (0..frames * channels * width * height).map(value).collect()
    }

    #[test]
    fn planes_become_pixels_in_the_right_order() {
        // One 2×1 frame: red plane, then green, then blue.
        let video = vec![
            1.0, 0.0, /* R */ 0.0, 1.0, /* G */ 0.0, 0.0, /* B */
        ];
        let rgba = rgba_from_planar(&video, 1, 2, 1, 3).unwrap();
        assert_eq!(rgba, vec![255, 0, 0, 255, 0, 255, 0, 255]);
    }

    #[test]
    fn channel_major_frames_are_not_read_as_repeated_temporal_blocks() {
        // [R0, R1, G0, G1, B0, B1], exactly as slopfab's C API returns it.
        // Distinct channels and frames make a channel/frame transposition
        // visible instead of letting a greyscale fixture hide it.
        let video = vec![0.1, 0.4, 0.2, 0.5, 0.3, 0.6];
        let rgba = rgba_from_planar(&video, 2, 1, 1, 3).unwrap();
        assert_eq!(rgba, vec![26, 51, 77, 255, 102, 128, 153, 255]);
    }

    #[test]
    fn a_signed_picture_is_recognised_and_shifted_rather_than_clipped() {
        // −1…1: mid grey is 0.0 here and would read as black on the 0…1 rule.
        assert_eq!(Range::of(&[-1.0, 0.0, 1.0]), Range::Signed);
        let video = vec![-1.0, 0.0, 1.0, -1.0, 0.0, 1.0, -1.0, 0.0, 1.0];
        let rgba = rgba_from_planar(&video, 1, 3, 1, 3).unwrap();
        assert_eq!(
            &rgba[..12],
            &[0, 0, 0, 255, 128, 128, 128, 255, 255, 255, 255, 255]
        );
    }

    #[test]
    fn a_unit_picture_keeps_its_own_scale() {
        assert_eq!(Range::of(&[0.0, 0.5, 1.0]), Range::Unit);
        let video = vec![0.0, 0.5, 1.0, 0.0, 0.5, 1.0, 0.0, 0.5, 1.0];
        let rgba = rgba_from_planar(&video, 1, 3, 1, 3).unwrap();
        assert_eq!(
            &rgba[..12],
            &[0, 0, 0, 255, 128, 128, 128, 255, 255, 255, 255, 255]
        );
    }

    #[test]
    fn out_of_range_and_not_a_number_land_inside_the_byte() {
        let video = vec![
            2.0,
            f32::NAN,
            -0.001,
            2.0,
            f32::NAN,
            -0.001,
            2.0,
            f32::NAN,
            -0.001,
        ];
        let rgba = rgba_from_planar(&video, 1, 3, 1, 3).unwrap();
        assert_eq!(
            &rgba[..12],
            &[255, 255, 255, 255, 0, 0, 0, 255, 0, 0, 0, 255]
        );
    }

    #[test]
    fn one_plane_is_greyscale_rather_than_a_red_picture() {
        let rgba = rgba_from_planar(&[0.5], 1, 1, 1, 1).unwrap();
        assert_eq!(rgba, vec![128, 128, 128, 255]);
    }

    #[test]
    fn a_short_buffer_is_refused_rather_than_read_past_its_end() {
        let video = planar(2, 4, 4, 3, |_| 0.5);
        assert!(rgba_from_planar(&video, 3, 4, 4, 3).is_err());
        assert!(rgba_from_planar(&video, 2, 4, 4, 3).is_ok());
        assert!(rgba_from_planar(&[], 0, 4, 4, 3).is_err());
    }

    #[test]
    fn malformed_interleaved_audio_is_refused_at_the_handoff() {
        assert!(from_output(
            "bad-audio",
            &[0.5, 0.5, 0.5],
            &[0.1, 0.2, 0.3],
            1,
            1,
            1,
            3,
            24.0,
            2,
            32_000
        )
        .is_err());
        assert!(from_output(
            "bad-rate",
            &[0.5, 0.5, 0.5],
            &[0.1, 0.2],
            1,
            1,
            1,
            3,
            24.0,
            2,
            0
        )
        .is_err());
    }

    fn sample(job_id: &str) -> RenderedVideo {
        from_output(
            job_id,
            &[0.5, 0.5, 0.5],
            &[0.25, -0.25],
            1,
            1,
            1,
            3,
            24.0,
            2,
            48_000,
        )
        .unwrap()
    }

    #[test]
    fn a_render_is_held_until_it_is_released() {
        let _serial = TEST_LOCK.lock().unwrap();
        release("held");
        assert!(frame("held", 0).unwrap().is_none());
        keep(sample("held"));
        assert_eq!(frame("held", 0).unwrap().unwrap(), vec![128, 128, 128, 255]);
        assert_eq!(
            summary("held").unwrap(),
            RenderedSummary {
                job_id: "held".into(),
                width: 1,
                height: 1,
                frame_count: 1,
                fps: 24.0,
                audio_channels: 2,
                audio_sample_rate: 48_000,
                audio_samples: 2
            },
        );
        // Past the last frame is an absence, not a panic and not a short read.
        assert!(frame("held", 1).unwrap().is_none());
        // f32 little-endian, which is what the webview reads it back as.
        assert_eq!(
            audio("held").unwrap(),
            0.25f32
                .to_le_bytes()
                .iter()
                .chain(&(-0.25f32).to_le_bytes())
                .copied()
                .collect::<Vec<u8>>()
        );
        assert!(release("held"));
        assert!(!release("held"));
        assert!(frame("held", 0).unwrap().is_none());
    }

    #[test]
    fn a_second_run_of_one_scene_replaces_the_first() {
        let _serial = TEST_LOCK.lock().unwrap();
        release("same");
        keep(sample("same"));
        keep(sample("same"));
        assert!(release("same"));
        assert!(!release("same"), "one job may only ever hold one render");
    }

    #[test]
    fn the_oldest_render_is_dropped_rather_than_the_process_growing_without_limit() {
        let _serial = TEST_LOCK.lock().unwrap();
        for id in ["cap-1", "cap-2", "cap-3"] {
            release(id);
        }
        assert_eq!(keep(sample("cap-1")), None);
        assert_eq!(keep(sample("cap-2")), None);
        // Whatever else is waiting from another test, pushing past the cap
        // reports what it evicted rather than losing it silently.
        let evicted = keep(sample("cap-3"));
        assert!(evicted.is_some());
        for id in ["cap-1", "cap-2", "cap-3"] {
            release(id);
        }
    }
}