Skip to content

AI media harness for planning, generating, and editing locally

TypeScript
10.0 KB
Raw Blame
use super::{capture::*, protocol::parse_turn_result, types::*};
use base64::{engine::general_purpose::STANDARD, Engine as _};
use serde_json::{json, Value};
use std::{
    io::Cursor,
    sync::atomic::{AtomicBool, Ordering},
    time::Duration,
};

fn frame() -> CapturedFrame {
    let mut png = Cursor::new(Vec::new());
    image::RgbaImage::from_pixel(2, 2, image::Rgba([255, 0, 0, 255]))
        .write_to(&mut png, image::ImageFormat::Png)
        .unwrap();
    CapturedFrame {
        png_base64: STANDARD.encode(png.into_inner()),
        time_ms: 1500.0,
        width: 2,
        height: 2,
    }
}

#[test]
fn capture_contract_is_read_only_and_requires_an_explicit_valid_time() {
    let parsed = parse_turn_result(r#"{"kind":"inspect","requests":[{"op":"timeline.capture","at":1.5},{"op":"generator.list"}]}"#).unwrap();
    assert!(matches!(parsed, AgentTurnResult::Inspect { requests } if requests.len() == 2));
    for request in [
        json!({"op":"timeline.capture"}),
        json!({"op":"timeline.capture","at":-1}),
        json!({"op":"timeline.capture","at":1,"path":"private.png"}),
    ] {
        assert!(
            parse_turn_result(&json!({"kind":"inspect","requests":[request]}).to_string()).is_err()
        );
    }
    assert!(parse_turn_result(
        "{\"op\":\"timeline.capture\",\"at\":1}\n{\"op\":\"commit\",\"summary\":\"Capture\"}"
    )
    .is_err());
}

#[test]
fn media_capture_contract_round_trips_ids_defaults_and_explicit_takes() {
    for request in [
        json!({"op":"scene.capture","scene":"scene-1","generation":"take-2","at":1.5}),
        json!({"op":"shot.capture","scene":"scene-1","shot":"shot-2"}),
        json!({"op":"reference.capture","reference":"ref-1","at":0.5}),
        json!({"op":"reference.capture","reference":"ref-1","image":"front"}),
        json!({"op":"image.capture","image":"edited-image"}),
        json!({"op":"motion.capture","motion":"motion-1","at":2.5}),
    ] {
        let parsed =
            parse_turn_result(&json!({"kind":"inspect","requests":[request]}).to_string()).unwrap();
        let AgentTurnResult::Inspect { requests } = parsed else {
            panic!("inspection expected")
        };
        let InspectionRequest::Frame(target) = &requests[0] else {
            panic!("frame expected")
        };
        let payload = CaptureRequest {
            request_id: "turn".into(),
            capture_id: "capture".into(),
            target: target.clone(),
        };
        let encoded = serde_json::to_value(payload).unwrap();
        assert_eq!(encoded["target"]["op"], request["op"]);
        for (key, value) in request.as_object().unwrap() {
            assert_eq!(&encoded["target"][key], value);
        }
    }
    for request in [
        json!({"op":"scene.capture","scene":""}),
        json!({"op":"scene.capture","scene":"s","generation":" "}),
        json!({"op":"shot.capture","scene":"s"}),
        json!({"op":"shot.capture","scene":"s","shot":"s","at":-1}),
        json!({"op":"reference.capture","reference":"r","path":"D:/private.png"}),
        json!({"op":"image.capture","image":"i","at":1}),
        json!({"op":"motion.capture","motion":"m","at":-1}),
    ] {
        assert!(
            parse_turn_result(&json!({"kind":"inspect","requests":[request]}).to_string()).is_err()
        );
    }
}

#[test]
fn capture_bridge_matches_turns_and_discards_cancelled_or_invalid_replies() {
    let bridge = CaptureBridge::default();
    let cancel = AtomicBool::new(false);
    let result = bridge
        .request("one", FrameRead::Timeline { at: 1.5 }, &cancel, |request| {
            assert!(bridge
                .complete(
                    "other".into(),
                    request.capture_id.clone(),
                    Some(frame()),
                    None
                )
                .is_err());
            bridge.complete(
                request.request_id,
                request.capture_id.clone(),
                Some(frame()),
                None,
            )?;
            assert!(bridge
                .complete("one".into(), request.capture_id, Some(frame()), None)
                .is_err());
            Ok(())
        })
        .unwrap();
    assert_eq!(result.time_ms, 1500.0);
    let result = bridge.request("one", FrameRead::Timeline { at: 1.5 }, &cancel, |request| {
        let mut invalid = frame();
        invalid.width = 1537;
        bridge.complete(request.request_id, request.capture_id, Some(invalid), None)
    });
    assert!(result.unwrap_err().contains("oversized"));
    let mut pending = None;
    let result = bridge.request("one", FrameRead::Timeline { at: 1.5 }, &cancel, |request| {
        pending = Some(request);
        cancel.store(true, Ordering::Release);
        Ok(())
    });
    assert!(result.unwrap_err().contains("cancelled"));
    let request = pending.unwrap();
    assert!(bridge
        .complete(request.request_id, request.capture_id, Some(frame()), None)
        .is_err());
    assert!(bridge
        .request(
            "one",
            FrameRead::Timeline { at: 1.5 },
            &AtomicBool::new(false),
            |_| Err("No window".into())
        )
        .unwrap_err()
        .contains("No window"));
}

#[test]
fn cli_providers_receive_actual_images_and_codex_files_are_temporary() {
    use super::providers::{cli_provider_for, images::attach_cli_images};
    let config =
        serde_json::from_str(include_str!("../../../fixtures/project-v1-complete.json")).unwrap();
    let root = std::path::Path::new(".");
    let images = [frame()];
    let mut codex = cli_provider_for(ProviderId::Codex)
        .unwrap()
        .command_spec(root, root, &config, "Inspect", None)
        .unwrap();
    let files = attach_cli_images(ProviderId::Codex, &mut codex, &images)
        .unwrap()
        .unwrap();
    let path = files.path().join("capture-1.png");
    assert_eq!(
        std::fs::read(&path).unwrap(),
        images[0].png_bytes().unwrap()
    );
    assert!(codex.args.contains(&"--image".into()));
    assert_eq!(codex.args.last().unwrap(), "-");
    assert!(codex.stdin_payload.unwrap().contains("1.500 seconds"));
    drop(files);
    assert!(!path.exists());
    let mut claude = cli_provider_for(ProviderId::Claude)
        .unwrap()
        .command_spec(root, root, &config, "Inspect", None)
        .unwrap();
    assert!(attach_cli_images(ProviderId::Claude, &mut claude, &images)
        .unwrap()
        .is_none());
    assert!(claude.args.contains(&"--input-format".into()));
    let message: Value = serde_json::from_str(&claude.stdin_payload.unwrap()).unwrap();
    assert_eq!(message["type"], "user");
    assert_eq!(
        message["message"]["content"][2]["source"]["data"],
        images[0].png_base64
    );
}

#[test]
fn endpoint_receives_capture_and_keeps_it_on_validation_retries_without_project_writes() {
    let project = tempfile::tempdir().unwrap();
    let fixture = include_str!("../../../fixtures/project-v1-complete.json");
    let project_path = project.path().join("slopus.json");
    std::fs::write(&project_path, fixture).unwrap();
    let server = tiny_http::Server::http("127.0.0.1:0").unwrap();
    let endpoint = format!("http://{}/v1", server.server_addr());
    let handle = std::thread::spawn(move || {
        let mut bodies = Vec::new();
        for result in [
            json!({"kind":"inspect","requests":[{"op":"motion.capture","motion":"motion-1","at":1.5}]}),
            json!({"kind":"answer","content":""}),
            json!({"kind":"answer","content":"The frame is red."}),
        ] {
            let mut request = server
                .recv_timeout(Duration::from_secs(10))
                .unwrap()
                .expect("provider request");
            let mut body = String::new();
            request.as_reader().read_to_string(&mut body).unwrap();
            bodies.push(serde_json::from_str::<Value>(&body).unwrap());
            request
                .respond(tiny_http::Response::from_string(
                    json!({"choices":[{"message":{"content":result.to_string()}}]}).to_string(),
                ))
                .unwrap();
        }
        bodies
    });
    let mut config: crate::project::ProjectConfig = serde_json::from_str(fixture).unwrap();
    config.provider_settings.insert("local".into(), serde_json::from_value(json!({"enabled":true,"model":"vision-model","options":{"endpoint":endpoint,"apiKey":"private-secret"}})).unwrap());
    let capture_count = std::cell::Cell::new(0);
    let result = super::AgentRuntime::default()
        .run_with_capture(
            AgentTurnRequest {
                request_id: "capture-turn".into(),
                scope: super::AgentScope::Project,
                folder_path: project.path().to_string_lossy().into(),
                provider: ProviderId::Local,
                prompt: "Look at 1.5 seconds".into(),
                config,
                conversation: vec![],
                generators: Default::default(),
            },
            |_| {},
            |at, _| {
                assert!(matches!(at, FrameRead::Motion { motion, at } if motion == "motion-1" && *at == 1.5));
                // The provider's failed final answer must reuse the capture, not render again.
                capture_count.set(capture_count.get() + 1);
                Ok(frame())
            },
        )
        .unwrap();
    assert!(matches!(result.result, AgentTurnResult::Answer { .. }));
    assert_eq!(capture_count.get(), 1);
    let bodies = handle.join().unwrap();
    assert!(bodies[0]["messages"][1]["content"].is_string());
    for body in &bodies[1..] {
        let content = &body["messages"][1]["content"];
        assert!(content[0]["text"]
            .as_str()
            .unwrap()
            .contains("start of this agent turn"));
        assert!(content[1]["text"]
            .as_str()
            .unwrap()
            .contains("1.500 seconds"));
        assert_eq!(
            content[2]["image_url"]["url"],
            format!("data:image/png;base64,{}", frame().png_base64)
        );
        assert!(!body.to_string().contains("private-secret"));
    }
    assert_eq!(std::fs::read_to_string(project_path).unwrap(), fixture);
}