Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

13.4 KB
Raw Blame
//! Media generation agent tools.
//!
//! `media_generate_image` and `media_generate_video` are TinyAgents'
//! [`GenerateImageTool`] / [`GenerateVideoTool`] bound to the managed
//! generators from [`super::provider`], under the names the `media` tool pack
//! and the image/video agents' allowlists already use. `media_list_models`
//! lists what the generators can run.

use std::collections::HashSet;
use std::path::{Component, Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::Duration;

use async_trait::async_trait;
use serde_json::{json, Value};
use tinyagents_harness::media::{GenerateImageTool, GenerateVideoTool, MediaOutput};
use tinyagents_harness::tinyinference_image::ImageGenerator;
use tinyagents_harness::tinyinference_video::{VideoGenerator, WaitPolicy};
use tinyagents_harness_tools::media::MediaJobProgress;
use tinytools::{PermissionLevel, Tool, ToolCategory, ToolResult};

use super::artifact_tool::MediaArtifactTool;
use super::provider::MediaGenerators;
use crate::artifacts::ArtifactKind;
use crate::config::Config;

/// Image tool name (pinned by the `media` pack and agent allowlists).
pub const IMAGE_TOOL_NAME: &str = "media_generate_image";
/// Video tool name.
pub const VIDEO_TOOL_NAME: &str = "media_generate_video";
/// Model-listing tool name.
pub const LIST_MODELS_TOOL_NAME: &str = "media_list_models";

/// Poll cadence and budget for a video job.
const VIDEO_POLL_INTERVAL: Duration = Duration::from_secs(5);
const VIDEO_WAIT_BUDGET: Duration = Duration::from_secs(600);

pub type MediaReferenceAllowlist = Arc<Mutex<HashSet<PathBuf>>>;

const IMAGE_DESCRIPTION: &str = "Generate or edit images from a text prompt via OpenRouter \
     (default model: Seedream 5.0 Lite). Pass `references` (https URLs, workspace paths, or \
     paths returned by earlier media calls) \
     to edit, restyle, or keep a subject consistent. Saves each image in the Files folder and \
     returns its local file path. Billed per call: after an error that \
     says the call was billed, do not call again — report it to the user.";

const VIDEO_DESCRIPTION: &str = "Generate a short video clip via OpenRouter (default model: \
     Seedance 2.0 Mini, 4–15 s, 480p/720p, optional audio). Optionally start from \
     `first_frame` or end on `last_frame` (URL, workspace path, or path returned by an earlier \
     media call). Blocks until the clip is \
     ready (minutes) and saves it in the Files folder. Billed per call: if it times out, \
     call again with `resume_job_id` instead of submitting a new job.";

/// Builds the media tools over `generators` for a configuration: the
/// reference-file policy, the files roots and the video wait come from
/// `root_config`; `allowed_references` is the session's shared allowlist of
/// artifact paths. The generators come from the host that owns the media
/// service (the core has none of its own).
pub fn media_tools_for_config(
    generators: MediaGenerators,
    root_config: &Config,
    action_dir: &Path,
    allowed_references: MediaReferenceAllowlist,
) -> Vec<Box<dyn Tool>> {
    media_tools_from_with_allowlist(
        generators,
        action_dir,
        &root_config.workspace_dir,
        crate::artifacts::FileRoots::from_config(root_config),
        // Each poll's job state feeds the call's progress heartbeat
        // (`super::progress`, wired per call by `MediaArtifactTool`), so a
        // multi-minute wait is not silent.
        WaitPolicy::new(VIDEO_POLL_INTERVAL, VIDEO_WAIT_BUDGET),
        allowed_references,
    )
}

/// Builds the tool set over any generators (the managed ones in production,
/// mocks in tests), writing under `action_dir`; `video_wait` bounds each
/// video job.
pub fn media_tools_from(
    generators: MediaGenerators,
    action_dir: &Path,
    workspace_dir: &Path,
    files_dir: impl Into<crate::artifacts::FileRoots>,
    video_wait: WaitPolicy,
) -> Vec<Box<dyn Tool>> {
    media_tools_from_with_allowlist(
        generators,
        action_dir,
        workspace_dir,
        files_dir,
        video_wait,
        Arc::new(Mutex::new(HashSet::new())),
    )
}

fn media_tools_from_with_allowlist(
    generators: MediaGenerators,
    action_dir: &Path,
    workspace_dir: &Path,
    files_dir: impl Into<crate::artifacts::FileRoots>,
    video_wait: WaitPolicy,
    allowed_artifacts: MediaReferenceAllowlist,
) -> Vec<Box<dyn Tool>> {
    let files = files_dir.into();
    let generators = Arc::new(generators);
    let Some(blobs) = crate::storage::sessions::artifact_blobs_for(workspace_dir) else {
        log::warn!("[media] scoped blob storage unavailable; media tools are not registered");
        return Vec::new();
    };
    let output = MediaOutput::new(action_dir, blobs).with_reference_policy(reference_policy(
        action_dir,
        workspace_dir,
        Arc::clone(&allowed_artifacts),
    ));
    let output_blobs = Arc::clone(&output.blobs);
    let image_tool = |image: &Arc<dyn ImageGenerator>| {
        GenerateImageTool::new(Arc::clone(image), output.clone())
            .with_name(IMAGE_TOOL_NAME)
            .with_description(IMAGE_DESCRIPTION)
            .with_permission_level(PermissionLevel::Execute)
            .with_category(ToolCategory::Workflow)
    };
    let tools: Vec<Box<dyn Tool>> = vec![
        Box::new(
            MediaArtifactTool::new(
                image_tool(&generators.image),
                ArtifactKind::Image,
                workspace_dir.to_path_buf(),
                files.clone(),
            )
            .with_blobs(Arc::clone(&output_blobs))
            .with_reference_allowlist(Arc::clone(&allowed_artifacts))
            // A tool per call, around generators that disclose to that call's
            // chat.
            .with_per_call_inner({
                let generators = Arc::clone(&generators);
                let output = output.clone();
                move |_observer, context| {
                    let approval =
                        crate::agent::tinyagents::host::run_context::approval_for_tool(context);
                    let routed = generators.routed(approval.chat.as_ref());
                    GenerateImageTool::new(routed.image, output.clone())
                        .with_name(IMAGE_TOOL_NAME)
                        .with_description(IMAGE_DESCRIPTION)
                        .with_permission_level(PermissionLevel::Execute)
                        .with_category(ToolCategory::Workflow)
                }
            }),
        ),
        Box::new(
            MediaArtifactTool::new(
                video_tool(&generators.video, &output, video_wait.clone()),
                ArtifactKind::Video,
                workspace_dir.to_path_buf(),
                files,
            )
            .with_blobs(output_blobs)
            .with_reference_allowlist(Arc::clone(&allowed_artifacts))
            .with_per_call_inner({
                let generators = Arc::clone(&generators);
                let output = output.clone();
                move |progress, context| {
                    let approval =
                        crate::agent::tinyagents::host::run_context::approval_for_tool(context);
                    let routed = generators.routed(approval.chat.as_ref());
                    let observer = Arc::new(
                        move |status: &tinyagents_harness::tinyinference_video::VideoJobStatus| {
                            progress(&MediaJobProgress {
                                id: status.id.clone(),
                                state: status.state.to_string(),
                                outputs: status.outputs,
                                output_indices: status.output_indices.clone(),
                                cost_usd: status.cost_usd,
                                error: status.error.clone(),
                            });
                        },
                    );
                    video_tool(
                        &routed.video,
                        &output,
                        video_wait.clone().with_progress(observer),
                    )
                }
            }),
        ),
        Box::new(MediaListModelsTool {
            image: Arc::clone(&generators.image),
            video: Arc::clone(&generators.video),
        }),
    ];
    tracing::debug!("[media_generation] registered {} media tools", tools.len());
    tools
}

/// The video generation tool over `video`, with `wait` bounding each job.
fn video_tool(
    video: &Arc<dyn VideoGenerator>,
    output: &MediaOutput,
    wait: WaitPolicy,
) -> GenerateVideoTool {
    GenerateVideoTool::new(Arc::clone(video), output.clone())
        .with_name(VIDEO_TOOL_NAME)
        .with_description(VIDEO_DESCRIPTION)
        .with_permission_level(PermissionLevel::Execute)
        .with_category(ToolCategory::Workflow)
        .with_wait_policy(wait)
}

/// Local reference files may be read and uploaded only from the action
/// directory or the workspace directory, never through `..`, and never from
/// an always-forbidden location (credential stores, system roots).
pub(crate) fn reference_policy(
    action_dir: &Path,
    workspace_dir: &Path,
    allowed_artifacts: MediaReferenceAllowlist,
) -> tinyagents_harness::media::ReferencePathPolicy {
    let roots: Vec<PathBuf> = vec![action_dir.to_path_buf(), workspace_dir.to_path_buf()];
    Arc::new(move |path: &Path| {
        if path.components().any(|c| matches!(c, Component::ParentDir)) {
            return Err(format!(
                "reference path {} may not contain '..'",
                path.display()
            ));
        }
        if crate::security::SecurityPolicy::is_always_forbidden(path) {
            return Err(format!(
                "reference path {} is in a protected location",
                path.display()
            ));
        }
        let is_filed_artifact = allowed_artifacts
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner)
            .contains(path);
        if !is_filed_artifact && !roots.iter().any(|root| path.starts_with(root)) {
            return Err(format!(
                "reference path {} is outside the workspace; use a URL or a file inside the workspace",
                path.display()
            ));
        }
        Ok(path.to_path_buf())
    })
}

/// Lists the image and video models the generators can run.
pub struct MediaListModelsTool {
    image: Arc<dyn ImageGenerator>,
    video: Arc<dyn VideoGenerator>,
}

#[async_trait]
impl Tool for MediaListModelsTool {
    fn name(&self) -> &str {
        LIST_MODELS_TOOL_NAME
    }

    fn description(&self) -> &str {
        "List the image and video generation models available to media_generate_image and \
         media_generate_video, with their ids. Use only when the user asks for a specific model \
         or style the default model cannot do."
    }

    fn parameters_schema(&self) -> Value {
        json!({
            "type": "object",
            "properties": {
                "kind": { "type": "string", "enum": ["image", "video", "all"], "description": "Which catalog (default all)." },
                "search": { "type": "string", "description": "Case-insensitive substring filter on id or name." }
            }
        })
    }

    fn category(&self) -> ToolCategory {
        ToolCategory::Workflow
    }

    fn permission_level(&self) -> PermissionLevel {
        PermissionLevel::ReadOnly
    }

    async fn execute(&self, args: Value) -> anyhow::Result<ToolResult> {
        let kind = args.get("kind").and_then(Value::as_str).unwrap_or("all");
        let search = args
            .get("search")
            .and_then(Value::as_str)
            .map(str::to_ascii_lowercase);
        let keep = |id: &str, name: Option<&str>| {
            search.as_deref().is_none_or(|needle| {
                id.to_ascii_lowercase().contains(needle)
                    || name.is_some_and(|n| n.to_ascii_lowercase().contains(needle))
            })
        };
        let mut out = serde_json::Map::new();
        if kind != "video" {
            match self.image.list_models().await {
                Ok(models) => {
                    let list: Vec<Value> = models
                        .iter()
                        .filter(|m| keep(&m.id, m.name.as_deref()))
                        .map(|m| json!({ "id": m.id, "name": m.name }))
                        .collect();
                    out.insert(
                        "image".into(),
                        json!({ "default": self.image.default_model(), "models": list }),
                    );
                }
                Err(error) => {
                    return Ok(ToolResult::error(format!(
                        "Listing image models failed: {error}"
                    )));
                }
            }
        }
        if kind != "image" {
            match self.video.list_models().await {
                Ok(models) => {
                    let list: Vec<Value> = models
                        .iter()
                        .filter(|m| keep(&m.id, m.name.as_deref()))
                        .map(|m| json!({ "id": m.id, "name": m.name }))
                        .collect();
                    out.insert(
                        "video".into(),
                        json!({ "default": self.video.default_model(), "models": list }),
                    );
                }
                Err(error) => {
                    return Ok(ToolResult::error(format!(
                        "Listing video models failed: {error}"
                    )));
                }
            }
        }
        Ok(ToolResult::json(Value::Object(out)))
    }
}

#[cfg(test)]
#[path = "tools_tests.rs"]
mod tools_tests;