Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

17.3 KB
Raw Blame
//! Legacy attachment marker compatibility and configuration adapters.
//!
//! New uploads are handled by `attachments`: originals live beneath
//! the acting workspace and provider bytes are prepared only on request copies.
//! The older stash remains readable so existing conversations can resume.

use crate::core::runtime::DispatchCtx;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::OnceLock;
#[cfg(feature = "documents")]
use std::time::Duration;

use async_trait::async_trait;
use reqwest::Client;

use crate::config::{build_runtime_proxy_client_with_timeouts, MultimodalConfig};
use tinyagents_session::transcript::{TranscriptMessage, TranscriptPart};

use tinyagents_harness::multimodal::{
    config::ImageLimits,
    markers,
    payload::sha256_prefix,
    resolve::{resolve_image, TextExtractor},
    BlobAttachmentStash,
};

/// Hard upper bound on how long the `tinydocs` module may spend extracting a
/// PDF's text layer before the attempt is abandoned and the file degrades to a
/// metadata-only reference.
///
/// The crate's [`TextExtractor`] carries no deadline by design: the cost is set
/// by the document rather than by anything validated beforehand, and only a
/// host knows how long an attachment is worth waiting for. PDFs that choke the
/// parser — extremely large, encrypted, malformed — must not stall a chat turn.
#[cfg(feature = "documents")]
const PDF_EXTRACTION_TIMEOUT: Duration = Duration::from_secs(60);

// ── Config mapping ───────────────────────────────────────────────────────
// Follows the `session_config_from` precedent in `openhuman::tinyagents::config`:
// the crate owns the struct, the host maps its schema into it. The clamping
// stays crate-side, so these are plain field copies — if they ever grow a rule,
// that rule belongs in the crate instead.

/// Map the host image config onto the crate's limits.
fn image_limits(config: &MultimodalConfig) -> ImageLimits {
    ImageLimits {
        max_images: config.max_images,
        max_image_size_mb: config.max_image_size_mb,
        allow_remote_fetch: config.allow_remote_fetch,
    }
}

/// The HTTP client remote references are fetched with.
///
/// Built per call rather than cached because the runtime proxy configuration
/// can change under a running core, and a stale client would keep dialling the
/// old one.
fn remote_client() -> Client {
    build_runtime_proxy_client_with_timeouts("provider.ollama", 30, 10)
}

// ── Text extraction ──────────────────────────────────────────────────────
/// The host's [`TextExtractor`]: PDF text through the `tinydocs` module,
#[cfg_attr(feature = "documents", doc = "bounded by [`PDF_EXTRACTION_TIMEOUT`].")]
#[cfg_attr(
    not(feature = "documents"),
    doc = "bounded by the module's own deadline."
)]
///
/// Claims `application/pdf` and nothing else. The native module runs in the
/// core process; a wait deadline does not cancel its blocking parser work.
pub struct DocumentsTextExtractor<'a> {
    /// The context the extraction reads its config under.
    #[cfg_attr(not(feature = "documents"), allow(dead_code))]
    pub ctx: &'a DispatchCtx,
}

#[async_trait]
impl TextExtractor for DocumentsTextExtractor<'_> {
    fn handles(&self, mime: &str) -> bool {
        mime == "application/pdf"
    }

    /// Extract a PDF's text layer.
    ///
    /// The parsing runs in the module, which owns its own blocking pool, so
    /// there is no `spawn_blocking` here — but the deadline stays.
    ///
    /// The bytes ride a bus stream rather than a JSON frame: a `.pdf` is
    /// bounded only by what the multimodal config accepted, which is far past
    /// what a frame holds.
    ///
    /// Every failure — an unavailable module, a damaged document, or the
    /// deadline — is reported to TinyAgents, which selects its built-in
    /// metadata fallback rather than surfacing the parser error.
    #[cfg(feature = "documents")]
    async fn extract(&self, _mime: &str, bytes: &[u8]) -> Result<String, String> {
        use crate::modules::documents;

        match tokio::time::timeout(PDF_EXTRACTION_TIMEOUT, async {
            let config = crate::config::ops::load_current_or_init(self.ctx)
                .await
                .map_err(|error| format!("config unavailable for pdf extraction: {error}"))?;
            documents::extract_text(&config, bytes)
                .await
                .map_err(|error| error.to_string())
        })
        .await
        {
            Ok(Ok(text)) => Ok(text),
            Ok(Err(error)) => Err(error.to_string()),
            Err(_) => Err(format!(
                "pdf extraction exceeded {}s timeout",
                PDF_EXTRACTION_TIMEOUT.as_secs()
            )),
        }
    }

    /// Disabled variant when the `documents` feature is off: no PDF parser is
    /// compiled in, so signal failure and let TinyAgents choose its built-in
    /// metadata fallback — the same path a parse error or timeout takes.
    #[cfg(not(feature = "documents"))]
    async fn extract(&self, _mime: &str, _bytes: &[u8]) -> Result<String, String> {
        log::debug!(
            "[multimodal] pdf text extraction skipped: built without the `documents` feature"
        );
        Err("pdf text extraction disabled (built without the `documents` feature)".to_string())
    }
}

// ── Marker helpers over `TranscriptMessage` ────────────────────────────────────

// ── Ingress ──────────────────────────────────────────────────────────────

/// Ingress-time image stashing. Replaces every `[IMAGE:data:…]` marker with a
/// `[Image: image #att:<id>]` placeholder and stashes the decoded canonical
/// data URI, so the multi-MB base64 never persists. Idempotent (no `[IMAGE:`
/// ⇒ no-op).
pub async fn stash_image_attachments(message: &str, image_config: &MultimodalConfig) -> String {
    if !message.contains(markers::IMAGE_MARKER_PREFIX) {
        return message.to_string();
    }
    let (cleaned, image_refs) = markers::parse_image_markers(message);
    if image_refs.is_empty() {
        return message.to_string();
    }

    let images = image_limits(image_config);
    let (max_images, _) = images.effective();
    let max_image_bytes = images.max_image_bytes();
    let client = remote_client();

    let mut placeholders = Vec::with_capacity(image_refs.len());
    for (idx, reference) in image_refs.iter().enumerate() {
        // Enforce the per-turn image cap at ingress: over-cap markers degrade
        // to a text placeholder and are never read or stashed (rewriting the
        // markers are rewritten before they reach the owner request wrapper,
        // and bounds stash growth per ingress message).
        if idx >= max_images {
            placeholders.push("[Image: (over image limit)]".to_string());
            continue;
        }
        match resolve_image(reference, &images, max_image_bytes, &client).await {
            Ok(data_uri) => {
                let id = sha256_prefix(data_uri.as_bytes());
                let written = match stash() {
                    Some(stash) => stash.write(&id, &data_uri).await,
                    None => Err(anyhow::anyhow!("no attachment stash for this tenant")),
                };
                match written {
                    Ok(stashed) => tracing::debug!(
                        target: "multimodal",
                        id = %id,
                        len = stashed.len,
                        "[multimodal::images][stash] persisted attachment to the blob store"
                    ),
                    Err(err) => tracing::warn!(
                        target: "multimodal",
                        id = %id,
                        reason = %err,
                        "[multimodal::images][stash] failed to persist attachment; placeholder will not rehydrate"
                    ),
                }
                placeholders.push(markers::image_placeholder(&id));
            }
            Err(err) => {
                tracing::warn!(
                    target: "multimodal",
                    reason = %err,
                    "[multimodal::images][ingress] image could not be normalized; emitting bare placeholder"
                );
                placeholders.push("[Image: (could not be processed)]".to_string());
            }
        }
    }

    let mut out = cleaned.trim().to_string();
    for p in &placeholders {
        if !out.is_empty() {
            out.push('\n');
        }
        out.push_str(p);
    }
    tracing::info!(
        target: "multimodal",
        images = placeholders.len(),
        before_chars = message.chars().count(),
        after_chars = out.chars().count(),
        "[multimodal::images][ingress] stashed image attachments — data URI replaced with placeholder before persistence"
    );
    out
}

// ── Placeholders over `TranscriptMessage` ──────────────────────────────────────

/// True if any message carries an `[Image: … #att:<id>]` sidecar placeholder.
pub fn has_image_placeholders(messages: &[TranscriptMessage]) -> bool {
    messages.iter().any(row_has_image_placeholders)
}

/// Whether a row's text (its `content`, or any text part of a user row with
/// typed image parts) carries a sidecar placeholder.
fn row_has_image_placeholders(row: &TranscriptMessage) -> bool {
    markers::text_has_image_placeholders(&row.content)
        || row.parts.as_deref().is_some_and(|parts| {
            parts.iter().any(|part| {
                matches!(part, TranscriptPart::Text { text } if markers::text_has_image_placeholders(text))
            })
        })
}

/// Rehydrate `[Image: … #att:<id>]` placeholders back into `[IMAGE:…]` markers,
/// returning a provider-only copy. An attachment stashed in the blob store
/// comes back as an inline `data:` URI; a legacy sidecar file in the
/// attachments directory as its path (re-read at dispatch). Placeholders whose
/// id is in neither (evicted, or written by a different workspace) keep their
/// text. The attachment host migrates managed sidecars into acting
/// workspace originals before provider resolution, including text fallbacks.
pub async fn rehydrate_image_placeholders(
    messages: &[TranscriptMessage],
) -> Vec<TranscriptMessage> {
    let index = build_attachment_index(messages).await;
    messages
        .iter()
        .map(|m| {
            if !row_has_image_placeholders(m) {
                return m.clone();
            }
            TranscriptMessage {
                content: markers::rehydrate_placeholders_in_text(&m.content, &index),
                parts: m.parts.as_ref().map(|parts| {
                    parts
                        .iter()
                        .map(|part| match part {
                            TranscriptPart::Text { text } => TranscriptPart::Text {
                                text: markers::rehydrate_placeholders_in_text(text, &index),
                            },
                            image => image.clone(),
                        })
                        .collect()
                }),
                cache_breakpoints: Vec::new(),
                ..m.clone()
            }
        })
        .collect()
}

// ── The attachment stash ─────────────────────────────────────────────────

/// Process-global legacy attachments directory (`<workspace>/attachments`),
/// installed once at core startup via [`init_attachments_dir`]. Sidecar files
/// written before the blob stash stay readable from here.
static ATTACHMENTS_DIR: OnceLock<PathBuf> = OnceLock::new();

/// The workspace whose storage backend holds the blob stash; the parent of the
/// directory given to [`init_attachments_dir`].
static ATTACHMENTS_WORKSPACE: OnceLock<PathBuf> = OnceLock::new();

/// Install the legacy attachments directory (`<workspace>/attachments`) and,
/// from its parent, the workspace the blob stash lives in. Call once at core
/// startup. Idempotent — first writer wins.
pub fn init_attachments_dir(dir: PathBuf) {
    if let Some(workspace) = dir.parent() {
        let _ = ATTACHMENTS_WORKSPACE.set(workspace.to_path_buf());
    }
    let _ = ATTACHMENTS_DIR.set(dir);
}

/// Resolve the legacy attachments dir, falling back to a **per-user private**
/// dir when unset (CLI / direct invocation / tests that never called
/// [`init_attachments_dir`]).
pub(crate) fn attachments_dir() -> PathBuf {
    ATTACHMENTS_DIR
        .get()
        .cloned()
        .unwrap_or_else(fallback_attachments_dir)
}

/// The calling tenant's blob-backed stash. It exists only after
/// [`init_attachments_dir`] supplies the workspace storage binding; absent
/// that binding, ingress keeps the placeholder but cannot promise rehydration.
/// The mechanism (dedup'd writes, cap eviction, TTL sweep, id index) is
/// `tinyagents_harness::multimodal::BlobAttachmentStash`; the cap and TTL are
/// this host's policy (none: references stay resolvable).
fn stash() -> Option<BlobAttachmentStash> {
    stash_for_workspace(ATTACHMENTS_WORKSPACE.get().map(PathBuf::as_path))
}

fn stash_for_workspace(workspace: Option<&Path>) -> Option<BlobAttachmentStash> {
    workspace.and_then(crate::storage::sessions::attachment_stash_for)
}

/// Per-user fallback attachments dir used only when [`init_attachments_dir`]
/// was never called. Uses the OS user cache dir (e.g. `~/Library/Caches/…`,
/// `~/.cache/…`) so legacy sidecars aren't looked for in a world-readable
/// shared `temp_dir()` on multi-user hosts. Only falls back to `temp_dir()`
/// when no user cache dir can be resolved at all.
fn fallback_attachments_dir() -> PathBuf {
    directories::BaseDirs::new()
        .map(|d| d.cache_dir().join("openhuman-attachments"))
        .unwrap_or_else(|| std::env::temp_dir().join("openhuman-attachments"))
}

/// `id -> marker target` for the placeholders in `messages`: the blob stash's
/// bytes as a `data:` URI, else (for an id the stash lacks) a legacy sidecar
/// file in the attachments directory, as its path. The marker target is carried
/// as a `PathBuf` only because that is the harness's index type; it is rendered
/// verbatim inside `[IMAGE:…]`.
async fn build_attachment_index(messages: &[TranscriptMessage]) -> HashMap<String, PathBuf> {
    let mut wanted = Vec::new();
    for row in messages {
        wanted.extend(placeholder_ids(&row.content));
        for part in row.parts.iter().flatten() {
            if let TranscriptPart::Text { text } = part {
                wanted.extend(placeholder_ids(text));
            }
        }
    }
    wanted.sort();
    wanted.dedup();
    let mut index = HashMap::new();
    if wanted.is_empty() {
        return index;
    }
    if let Some(stash) = stash() {
        for id in &wanted {
            match stash.read_data_uri(id).await {
                Ok(Some(data_uri)) => {
                    index.insert(id.clone(), PathBuf::from(data_uri));
                }
                Ok(None) => {}
                Err(err) => tracing::warn!(
                    target: "multimodal",
                    id = %id,
                    reason = %err,
                    "[multimodal::images][stash] could not read attachment; trying legacy sidecars"
                ),
            }
        }
    }
    if index.len() < wanted.len() {
        for (id, path) in legacy_attachment_index() {
            if wanted.contains(&id) {
                index.entry(id).or_insert(path);
            }
        }
    }
    index
}

/// The `#att:<id>` ids of the sidecar placeholders in `text`.
fn placeholder_ids(text: &str) -> Vec<String> {
    markers::extract_image_placeholders_in_text(text)
        .iter()
        .filter_map(|placeholder| {
            let after = placeholder.split(markers::IMAGE_STASH_REF).nth(1)?;
            let id = after.trim_end_matches(']').trim();
            (!id.is_empty()).then(|| id.to_string())
        })
        .collect()
}

/// `id -> path` of the legacy sidecar files directly under the attachments
/// directory (written before the blob stash). Symlinks and nested entries are
/// ignored.
fn legacy_attachment_index() -> HashMap<String, PathBuf> {
    legacy_attachment_index_in(&attachments_dir())
}

fn legacy_attachment_index_in(dir: &std::path::Path) -> HashMap<String, PathBuf> {
    let Ok(entries) = std::fs::read_dir(dir) else {
        return HashMap::new();
    };
    entries
        .flatten()
        .filter(|entry| entry.file_type().is_ok_and(|kind| kind.is_file()))
        .filter_map(|entry| {
            let path = entry.path();
            let stem = path.file_stem()?.to_str()?.to_string();
            let valid = !stem.is_empty()
                && stem
                    .bytes()
                    .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_'));
            valid.then_some((stem, path))
        })
        .collect()
}

/// Compatibility no-op: durable conversation attachments are not swept.
pub async fn sweep_stale_attachments() {
    // Retained compatibility API. Conversation references do not expire.
}

#[cfg(test)]
#[path = "multimodal_tests.rs"]
mod tests;