Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
chore(agents): describe images for the haiku trigger judge (#7054)
Wolf Mermelstein43fd7826b2af
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex 86746f52f9..edaed7ebcb 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -340,6 +340,7 @@"crates/secretsmanager_client","crates/share_permission_db_utils","crates/sqs_worker",+ "crates/static_file","crates/static_file_service_client","crates/sync_service","crates/sync_service_client",@@ -491,6 +492,7 @@"crates/soup","crates/sqs_client","crates/sqs_worker",+ "crates/static_file","crates/static_file_service_client","crates/sync_service","crates/sync_service_client",@@ -797,6 +799,7 @@"crates/remote_env_var","crates/share_permission_db_utils","crates/sqs_worker",+ "crates/static_file","crates/static_file_service_client","crates/system_properties","crates/workspace-hack"@@ -876,6 +879,7 @@"crates/remote_env_var","crates/share_permission_db_utils","crates/sqs_worker",+ "crates/static_file","crates/static_file_service_client","crates/system_properties","crates/workspace-hack",@@ -2513,6 +2517,7 @@"crates/share_permission_db_utils","crates/sqs_client","crates/sqs_worker",+ "crates/static_file","crates/static_file_service_client","crates/sync_service","crates/sync_service_client",@@ -3680,6 +3685,7 @@"crates/soup_realtime","crates/sqs_client","crates/sqs_worker",+ "crates/static_file","crates/static_file_service_client","crates/sync_service","crates/sync_service_client",@@ -9956,6 +9962,7 @@"crates/share_permission_db_utils","crates/sqs_client","crates/sqs_worker",+ "crates/static_file","crates/static_file_service_client","crates/sync_service","crates/sync_service_client",diff --git a/Cargo.lock b/Cargo.lockindex 13168aeff7..c0eb7686bf 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -412,6 +412,7 @@ dependencies = ["serde","serde_json","sqlx",+ "static_file","static_file_service_client","tokio","tokio-retry",@@ -565,6 +566,7 @@ dependencies = ["agent_session","ai_usage","anyhow",+ "async-trait","bot_id","bots","channel_sender",@@ -577,11 +579,13 @@ dependencies = ["macro_user_id","macro_uuid","messages",+ "mime","mockall","model_owner","serde","serde_json","sqlx",+ "static_file","strum 0.27.2","thiserror 2.0.18","tokio",@@ -612,6 +616,7 @@ dependencies = ["rdkafka","serde_json","sqlx",+ "static_file","tokio","tracing","workspace-hack",diff --git a/crates/agent/src/completion.rs b/crates/agent/src/completion.rsindex e5e2015947..f5ac2810e6 100644--- a/crates/agent/src/completion.rs+++ b/crates/agent/src/completion.rs@@ -11,6 +11,7 @@ use ai_usage::{UsageContext, UsageRecorder};use genai_telemetry::ContentPolicy;use rig_agent::agent::{AgentBuilder, PromptResponse};use rig_agent::completion::Prompt;+use rig_core::OneOrMany;use rig_core::completion::CompletionModel;use rig_core::message::Message;@@ -134,6 +135,45 @@ pub async fn complete_with_history<M: ToString>(Ok(response.output)}+/// Ask `model` to answer `instruction` about one image and return its text.+///+/// `image_bytes` are the original file. They are downscaled to WebP before the+/// model sees them. Token usage is recorded the same way as [`complete`].+#[tracing::instrument(skip(model, system_prompt, image_bytes, recorder, ctx), err)]+pub async fn complete_about_image<M: ToString>(+ model: M,+ system_prompt: &str,+ instruction: &str,+ image_bytes: Vec<u8>,+ recorder: &dyn UsageRecorder,+ ctx: UsageContext,+) -> anyhow::Result<String> {+ let message = image_user_message(instruction, image_bytes)?;+ complete_with_history(model, system_prompt, vec![message], recorder, ctx).await+}++/// A user turn whose text is `instruction` and whose second block is the image.+pub(crate) fn image_user_message(+ instruction: &str,+ image_bytes: Vec<u8>,+) -> anyhow::Result<Message> {+ use attachment::image::ImageData;+ use rig_core::message::{ImageMediaType, UserContent};++ let ImageData::Base64(image) = ImageData::try_from_bytes(image_bytes)? else {+ anyhow::bail!("image bytes did not normalize to a base64 image");+ };+ let content = OneOrMany::many([+ UserContent::text(instruction),+ UserContent::image_base64(+ image.base64_data().to_owned(),+ Some(ImageMediaType::WEBP),+ None,+ ),+ ])?;+ Ok(Message::User { content })+}+/// GenAI telemetry for a one-shot completion: named after the feature making/// it, reporting the routed model, with no conversation (a one-shot has none).fn telemetry_for(ctx: &UsageContext, routed: &RoutedModel<'_>) -> GenAiContext {diff --git a/crates/agent/src/completion/test.rs b/crates/agent/src/completion/test.rsindex d6d324d811..3c2f800650 100644--- a/crates/agent/src/completion/test.rs+++ b/crates/agent/src/completion/test.rs@@ -105,3 +105,44 @@ async fn a_failed_one_shot_completion_marks_its_chat_span() {assert!(string_attribute(chat, attr::INPUT_MESSAGES).is_some());assert_eq!(string_attribute(chat, attr::OUTPUT_MESSAGES), None);}++/// A minimal valid 1x1 PNG.+#[rustfmt::skip]+const ONE_BY_ONE_PNG: &[u8] = &[+ 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D,+ 0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01,+ 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53, 0xDE, 0x00, 0x00, 0x00,+ 0x0C, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9C, 0x63, 0xF8, 0xCF, 0xC0, 0x00,+ 0x00, 0x03, 0x01, 0x01, 0x00, 0xC9, 0xFE, 0x92, 0xEF, 0x00, 0x00, 0x00,+ 0x00, 0x49, 0x45, 0x4E, 0x44, 0xAE, 0x42, 0x60, 0x82,+];++#[test]+fn image_user_message_sends_the_instruction_and_a_webp() {+ use rig_core::message::{DocumentSourceKind, ImageMediaType, UserContent};++ let message =+ image_user_message("Describe this image.", ONE_BY_ONE_PNG.to_vec()).expect("a valid png");+ let Message::User { content } = message else {+ panic!("expected a user message");+ };+ let blocks: Vec<_> = content.into_iter().collect();+ assert_eq!(blocks.len(), 2);++ let UserContent::Text(text) = &blocks[0] else {+ panic!("expected the instruction first");+ };+ assert_eq!(text.text, "Describe this image.");++ let UserContent::Image(image) = &blocks[1] else {+ panic!("expected the image second");+ };+ assert_eq!(image.media_type, Some(ImageMediaType::WEBP));+ assert!(matches!(image.data, DocumentSourceKind::Base64(_)));+}++#[test]+fn image_user_message_rejects_bytes_that_are_not_an_image() {+ image_user_message("Describe this image.", b"not an image".to_vec())+ .expect_err("non-image bytes");+}diff --git a/crates/agent/src/lib.rs b/crates/agent/src/lib.rsindex 6599d821ad..6a7f59def6 100644--- a/crates/agent/src/lib.rs+++ b/crates/agent/src/lib.rs@@ -21,7 +21,7 @@ pub mod types;pub use accumulator::StreamAccumulator;pub use agent_loop::{AgentLoop, Session};-pub use completion::{complete, complete_with_history};+pub use completion::{complete, complete_about_image, complete_with_history};pub use convert::{merge_consecutive_parts, to_rig_messages};pub use error::AgentError;pub use hook::{FinishedUserTool, PendingUserTool, UserToolFinisher};diff --git a/crates/agent_trigger/Cargo.toml b/crates/agent_trigger/Cargo.tomlindex 6d61788616..24ace5c8bb 100644--- a/crates/agent_trigger/Cargo.toml+++ b/crates/agent_trigger/Cargo.toml@@ -9,7 +9,7 @@ version = "0.1.0"# models_search → models_soup → notification → rdkafka, and the self-hosted# daemon only needs domain events from this crate. Services that construct# the adapters enable this; do not add it to coding_agent_worker.-outbound = ["dep:agent", "dep:ai_usage", "dep:lexical_client", "dep:sqlx", "channels/ports"]+outbound = ["dep:agent", "dep:ai_usage", "dep:async-trait", "dep:lexical_client", "dep:sqlx", "dep:static_file", "channels/ports"][dependencies]agent = { path = "../agent", optional = true }@@ -20,6 +20,7 @@ bot_id = { path = "../bot_id" }entity_access = { path = "../entity_access", default-features = false, features = ["ports"] }bots = { path = "../bots", default-features = false, features = ["ports"] }lexical_client = { path = "../lexical_client", optional = true }+static_file = { path = "../static_file", optional = true, default-features = false }chrono = { workspace = true }# Only the domain vocabulary (`Event`, `MacroEvent`, `TopicEvent`). Asking for# `outbound` here would pull rdkafka — and so librdkafka and openssl — into@@ -40,13 +41,15 @@ strum = { workspace = true }serde_json = { workspace = true }thiserror = { workspace = true }tracing = { workspace = true }-tokio = { workspace = true, features = ["rt"] }+tokio = { workspace = true, features = ["rt", "time"] }+async-trait = { workspace = true, optional = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }[dev-dependencies]messages = { path = "../messages", features = ["test-utils"] }agent_session = { path = "../agent_session", features = ["test-utils"] }bot_id = { path = "../bot_id", features = ["test-utils"] }+mime = "0.3"mockall = { workspace = true }model_owner = { path = "../model_owner" }tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }diff --git a/crates/agent_trigger/src/outbound/fast_model_judge.rs b/crates/agent_trigger/src/outbound/fast_model_judge.rsindex 29d4474743..356c8df2ff 100644--- a/crates/agent_trigger/src/outbound/fast_model_judge.rs+++ b/crates/agent_trigger/src/outbound/fast_model_judge.rs@@ -1,5 +1,8 @@//! Implicit-trigger judgement backed by the fast agent model.+#[cfg(test)]+mod test;+use std::sync::Arc;use agent::structured_output::{DynamicSchema, dynamic_structured_completion};@@ -12,6 +15,8 @@ use serde_json::json;use crate::domain::service::ImplicitTriggerJudge;+use super::image_caption::{ImageCaptioner, append_image_blurbs, blurbs_for_attachments};+static SYSTEM_PROMPT: &str = "\You decide whether a channel message is addressed to an AI coding agent.@@ -28,7 +33,14 @@ its work addressed to other people, or unrelated discussion.You are given the thread around the agent's part in it, as lines of \'[speaker] message' where the agent's own messages are marked '[agent]'. Some \-messages may be hidden; judge on what you are shown.";+messages may be hidden; judge on what you are shown.++Attached images are not shown to you. Each one is written into the message \+as <this is an image of ...>, a description of the picture rather than words \+the author typed. Treat that description as what the image shows. \+<this is an image> means a picture was attached but could not be described. \+Those descriptions appear on the message to judge; the thread transcript \+does not repeat them.";#[derive(Debug, Deserialize)]struct JudgeOutput {@@ -42,14 +54,19 @@ struct JudgeOutput {pub struct FastModelTriggerJudge {model: PredefinedModel,recorder: Arc<dyn UsageRecorder>,+ images: Arc<dyn ImageCaptioner>,}impl FastModelTriggerJudge {/// Creates a judge using the fast agent model.- pub fn new(recorder: Arc<dyn UsageRecorder>) -> Self {+ ///+ /// `images` turns attached pictures into the `<this is an image of ...>`+ /// blurbs the text-only model reads. The forwarded message is unchanged.+ pub fn new(recorder: Arc<dyn UsageRecorder>, images: impl ImageCaptioner + 'static) -> Self {Self {model: PredefinedModel::Fast,recorder,+ images: Arc::new(images),}}}@@ -89,14 +106,10 @@ impl ImplicitTriggerJudge for FastModelTriggerJudge {None => UsageContext::system(AiFeature::Automation),};- let prompt = if transcript.is_empty() {- format!("The message to judge:\n{}", posted.content)- } else {- format!(- "The thread so far:\n{transcript}\nThe message to judge:\n{}",- posted.content- )- };+ let blurbs =+ blurbs_for_attachments(self.images.as_ref(), &posted.attachments, ctx.clone()).await;+ let content = append_image_blurbs(&posted.content, &blurbs);+ let prompt = judge_user_prompt(transcript, &content);let value = dynamic_structured_completion(self.model,@@ -114,3 +127,12 @@ impl ImplicitTriggerJudge for FastModelTriggerJudge {Ok(output.addressed_to_agent)}}++/// The user prompt the judge scores. Image blurbs are already part of `content`.+pub(crate) fn judge_user_prompt(transcript: &str, content: &str) -> String {+ if transcript.is_empty() {+ format!("The message to judge:\n{content}")+ } else {+ format!("The thread so far:\n{transcript}\nThe message to judge:\n{content}")+ }+}diff --git a/crates/agent_trigger/src/outbound/fast_model_judge/test.rs b/crates/agent_trigger/src/outbound/fast_model_judge/test.rsnew file mode 100644index 0000000000..c16573c648--- /dev/null+++ b/crates/agent_trigger/src/outbound/fast_model_judge/test.rs@@ -0,0 +1,20 @@+use super::*;++#[test]+fn the_judge_is_told_what_an_image_blurb_means() {+ assert!(SYSTEM_PROMPT.contains("<this is an image of ...>"));+ assert!(SYSTEM_PROMPT.contains("<this is an image>"));+}++#[test]+fn an_image_blurb_is_part_of_the_message_being_judged() {+ let content = "see this\n<this is an image of a frog>";+ assert_eq!(+ judge_user_prompt("", content),+ "The message to judge:\nsee this\n<this is an image of a frog>"+ );+ assert_eq!(+ judge_user_prompt("[agent] on it\n", "see this"),+ "The thread so far:\n[agent] on it\n\nThe message to judge:\nsee this"+ );+}diff --git a/crates/agent_trigger/src/outbound/image_caption.rs b/crates/agent_trigger/src/outbound/image_caption.rsnew file mode 100644index 0000000000..5b4dc0ecac--- /dev/null+++ b/crates/agent_trigger/src/outbound/image_caption.rs@@ -0,0 +1,204 @@+//! Turns attached images into short phrases a text-only judge can read.++#[cfg(test)]+mod test;++use std::sync::Arc;+use std::time::Duration;++use agent::PredefinedModel;+use ai_usage::{UsageContext, UsageRecorder};+use async_trait::async_trait;+use messages::domain::events::MessageEventAttachment;+use static_file::domain::ports::StaticFileRepo;++/// Channel attachment entity type for an image stored as a static file.+const STATIC_IMAGE: &str = "static/image";++/// How many images get a model description. Further images are still marked,+/// so the judge can tell they exist, without a vision call per picture.+const MAX_DESCRIBED_IMAGES: usize = 8;++/// Bound on one description, download included, so a stuck file cannot hold+/// the trigger consumer.+const CAPTION_TIMEOUT: Duration = Duration::from_secs(8);++/// Longest phrase kept from a caption. A rambling answer would crowd out the+/// message the judge is actually scoring.+const MAX_PHRASE_CHARS: usize = 160;++const CAPTION_SYSTEM: &str = "\+Reply with only a short noun phrase describing the image, as if finishing \+\"this is an image of\". Examples: \"a frog\", \"a screenshot of a compiler \+error\". No sentence, no quotes, no trailing punctuation.";++const CAPTION_INSTRUCTION: &str = "Describe this image.";++/// Describes one static-file image as a short noun phrase, such as "a frog".+///+/// `None` means the picture could not be described. Callers still tell the+/// judge that an image was attached.+#[async_trait]+pub trait ImageCaptioner: Send + Sync {+ /// A short noun phrase for `file_id`, or `None` when it cannot be described.+ async fn caption(&self, file_id: &str, ctx: UsageContext) -> Option<String>;+}++/// Captions images with the fast model after downloading them from static files.+pub struct VisionImageCaptioner<F> {+ files: F,+ model: PredefinedModel,+ recorder: Arc<dyn UsageRecorder>,+}++impl<F> VisionImageCaptioner<F> {+ /// Caption images read through `files`, billing `recorder`.+ pub fn new(files: F, recorder: Arc<dyn UsageRecorder>) -> Self {+ Self {+ files,+ model: PredefinedModel::Fast,+ recorder,+ }+ }+}++#[async_trait]+impl<F> ImageCaptioner for VisionImageCaptioner<F>+where+ F: StaticFileRepo,+{+ async fn caption(&self, file_id: &str, ctx: UsageContext) -> Option<String> {+ let bytes = match self.files.read(file_id).await {+ Ok(bytes) => bytes,+ Err(error) => {+ tracing::warn!(error = ?error, file_id, "failed to read an attached image");+ return None;+ }+ };+ match agent::complete_about_image(+ self.model,+ CAPTION_SYSTEM,+ CAPTION_INSTRUCTION,+ bytes,+ self.recorder.as_ref(),+ ctx,+ )+ .await+ {+ Ok(raw) => normalize_caption(&raw),+ Err(error) => {+ tracing::warn!(error = ?error, file_id, "failed to describe an attached image");+ None+ }+ }+ }+}++/// Blurbs for the image attachments on a message, in attachment order.+///+/// Each described image becomes `<this is an image of a frog>`. An image that+/// cannot be described, or that is past [`MAX_DESCRIBED_IMAGES`], becomes+/// `<this is an image>`.+pub(crate) async fn blurbs_for_attachments(+ captioner: &dyn ImageCaptioner,+ attachments: &[MessageEventAttachment],+ ctx: UsageContext,+) -> Vec<String> {+ let image_ids: Vec<&str> = attachments+ .iter()+ .filter(|attachment| attachment.entity_type == STATIC_IMAGE)+ .map(|attachment| attachment.entity_id.as_str())+ .collect();++ let mut blurbs = Vec::with_capacity(image_ids.len());+ for (index, file_id) in image_ids.iter().copied().enumerate() {+ if index >= MAX_DESCRIBED_IMAGES {+ blurbs.push(image_blurb(None));+ continue;+ }+ let phrase =+ match tokio::time::timeout(CAPTION_TIMEOUT, captioner.caption(file_id, ctx.clone()))+ .await+ {+ Ok(phrase) => phrase,+ Err(_elapsed) => {+ tracing::warn!(file_id, "image caption timed out");+ None+ }+ };+ blurbs.push(image_blurb(phrase.as_deref()));+ }+ blurbs+}++/// `<this is an image of {phrase}>`, or `<this is an image>` when there is no phrase.+pub(crate) fn image_blurb(phrase: Option<&str>) -> String {+ match phrase.map(str::trim).filter(|phrase| !phrase.is_empty()) {+ Some(phrase) => format!("<this is an image of {phrase}>"),+ None => "<this is an image>".to_owned(),+ }+}++/// Append image blurbs to message text. An image-only message is just the blurbs.+pub(crate) fn append_image_blurbs(content: &str, blurbs: &[String]) -> String {+ if blurbs.is_empty() {+ return content.to_owned();+ }+ let blurbs = blurbs.join("\n");+ if content.trim().is_empty() {+ blurbs+ } else {+ format!("{}\n{blurbs}", content.trim_end())+ }+}++/// The first line of a caption, reduced to the noun phrase the blurb wraps.+pub(crate) fn normalize_caption(raw: &str) -> Option<String> {+ let mut phrase = raw.lines().next()?.trim().to_owned();+ phrase = strip_wrapping_quotes(&phrase).to_owned();+ for prefix in ["this is an image of ", "an image of ", "image of "] {+ if let Some(rest) = strip_prefix_ignore_ascii_case(&phrase, prefix) {+ phrase = rest.to_owned();+ break;+ }+ }+ phrase.retain(|ch| ch != '<' && ch != '>');+ let phrase = phrase+ .split_whitespace()+ .collect::<Vec<_>>()+ .join(" ")+ .trim_end_matches(['.', '!', '?', ',', ';', ':'])+ .trim()+ .to_owned();+ if phrase.is_empty() {+ return None;+ }+ let phrase = phrase.chars().take(MAX_PHRASE_CHARS).collect::<String>();+ let phrase = phrase.trim_end().to_owned();+ if phrase.is_empty() {+ None+ } else {+ Some(phrase)+ }+}++fn strip_wrapping_quotes(phrase: &str) -> &str {+ let mut chars = phrase.chars();+ let (Some(open), Some(close)) = (chars.next(), chars.next_back()) else {+ return phrase;+ };+ if (open == '"' && close == '"') || (open == '\'' && close == '\'') {+ phrase[open.len_utf8()..phrase.len() - close.len_utf8()].trim()+ } else {+ phrase+ }+}++fn strip_prefix_ignore_ascii_case<'a>(phrase: &'a str, prefix: &str) -> Option<&'a str> {+ let rest = phrase.get(prefix.len()..)?;+ if phrase[..prefix.len()].eq_ignore_ascii_case(prefix) {+ Some(rest.trim())+ } else {+ None+ }+}diff --git a/crates/agent_trigger/src/outbound/image_caption/test.rs b/crates/agent_trigger/src/outbound/image_caption/test.rsnew file mode 100644index 0000000000..bc0bf2caaf--- /dev/null+++ b/crates/agent_trigger/src/outbound/image_caption/test.rs@@ -0,0 +1,165 @@+use super::*;+use chrono::Utc;+use macro_user_id::cowlike::CowLike as _;+use macro_user_id::user_id::MacroUserIdStr;+use macro_uuid::Uuid;+use std::sync::Mutex;++fn user() -> MacroUserIdStr<'static> {+ MacroUserIdStr::parse_from_str("macro|[email protected]")+ .expect("valid user id")+ .into_owned()+}++fn ctx() -> UsageContext {+ UsageContext::new(ai_usage::AiFeature::Automation, user())+}++fn attachment(entity_type: &str, entity_id: &str) -> MessageEventAttachment {+ MessageEventAttachment {+ attachment_id: Uuid::from_u128(1),+ entity_type: entity_type.to_owned(),+ entity_id: entity_id.to_owned(),+ created_at: Utc::now(),+ }+}++struct Scripted {+ calls: Mutex<Vec<String>>,+ phrase: Option<String>,+}++#[async_trait]+impl ImageCaptioner for Scripted {+ async fn caption(&self, file_id: &str, _ctx: UsageContext) -> Option<String> {+ self.calls.lock().expect("calls").push(file_id.to_owned());+ self.phrase.clone()+ }+}++#[test]+fn a_caption_becomes_the_blurb_the_judge_reads() {+ assert_eq!(image_blurb(Some("a frog")), "<this is an image of a frog>");+ assert_eq!(image_blurb(None), "<this is an image>");+ assert_eq!(image_blurb(Some(" ")), "<this is an image>");+}++#[test]+fn captions_are_reduced_to_a_noun_phrase() {+ assert_eq!(normalize_caption("a frog").as_deref(), Some("a frog"));+ assert_eq!(normalize_caption("a frog.").as_deref(), Some("a frog"));+ assert_eq!(normalize_caption("\"a frog\"").as_deref(), Some("a frog"));+ assert_eq!(+ normalize_caption("This is an image of a frog.").as_deref(),+ Some("a frog")+ );+ assert_eq!(+ normalize_caption("a frog\nignore the rest").as_deref(),+ Some("a frog")+ );+ assert_eq!(normalize_caption("<a frog>").as_deref(), Some("a frog"));+ assert_eq!(normalize_caption(" \n"), None);+ assert_eq!(normalize_caption(""), None);+}++#[test]+fn blurbs_follow_the_authors_words() {+ let blurbs = vec!["<this is an image of a frog>".to_owned()];+ assert_eq!(+ append_image_blurbs("see this", &blurbs),+ "see this\n<this is an image of a frog>"+ );+ assert_eq!(+ append_image_blurbs(" ", &blurbs),+ "<this is an image of a frog>"+ );+ assert_eq!(append_image_blurbs("see this", &[]), "see this");+}++#[tokio::test]+async fn only_images_are_described_and_a_failed_caption_stays_visible() {+ let captioner = Scripted {+ calls: Mutex::new(Vec::new()),+ phrase: Some("a frog".to_owned()),+ };+ let attachments = vec![+ attachment("document", "doc-1"),+ attachment("static/video", "clip-1"),+ attachment("static/image", "frog"),+ ];++ let blurbs = blurbs_for_attachments(&captioner, &attachments, ctx()).await;++ assert_eq!(blurbs, vec!["<this is an image of a frog>".to_owned()]);+ assert_eq!(captioner.calls.lock().expect("calls").as_slice(), ["frog"]);++ let silent = Scripted {+ calls: Mutex::new(Vec::new()),+ phrase: None,+ };+ let blurbs =+ blurbs_for_attachments(&silent, &[attachment("static/image", "unreadable")], ctx()).await;+ assert_eq!(blurbs, vec!["<this is an image>".to_owned()]);+}++#[tokio::test]+async fn images_past_the_description_cap_are_marked_without_another_caption() {+ let captioner = Scripted {+ calls: Mutex::new(Vec::new()),+ phrase: Some("a frog".to_owned()),+ };+ let attachments: Vec<_> = (0..MAX_DESCRIBED_IMAGES + 1)+ .map(|index| attachment("static/image", &format!("img-{index}")))+ .collect();++ let blurbs = blurbs_for_attachments(&captioner, &attachments, ctx()).await;++ assert_eq!(blurbs.len(), MAX_DESCRIBED_IMAGES + 1);+ assert!(+ blurbs[..MAX_DESCRIBED_IMAGES]+ .iter()+ .all(|blurb| blurb == "<this is an image of a frog>")+ );+ assert_eq!(+ blurbs.last().map(String::as_str),+ Some("<this is an image>")+ );+ assert_eq!(+ captioner.calls.lock().expect("calls").len(),+ MAX_DESCRIBED_IMAGES+ );+}++struct MissingFile;++impl StaticFileRepo for MissingFile {+ async fn content_type(&self, _file_id: &str) -> anyhow::Result<mime::Mime> {+ Ok(mime::IMAGE_PNG)+ }++ async fn read(&self, file_id: &str) -> anyhow::Result<Vec<u8>> {+ anyhow::bail!("no file {file_id}")+ }+}++struct GarbageFile;++impl StaticFileRepo for GarbageFile {+ async fn content_type(&self, _file_id: &str) -> anyhow::Result<mime::Mime> {+ Ok(mime::APPLICATION_OCTET_STREAM)+ }++ async fn read(&self, _file_id: &str) -> anyhow::Result<Vec<u8>> {+ Ok(b"not an image".to_vec())+ }+}++#[tokio::test]+async fn a_missing_or_undecodable_image_is_not_described() {+ let recorder = Arc::new(ai_usage::NoOpUsageRecorder);+ let missing = VisionImageCaptioner::new(MissingFile, recorder.clone());+ assert_eq!(missing.caption("gone", ctx()).await, None);++ let garbage = VisionImageCaptioner::new(GarbageFile, recorder);+ assert_eq!(garbage.caption("bad", ctx()).await, None);+}diff --git a/crates/agent_trigger/src/outbound/mod.rs b/crates/agent_trigger/src/outbound/mod.rsindex c2b198e12b..19ecaec7d5 100644--- a/crates/agent_trigger/src/outbound/mod.rs+++ b/crates/agent_trigger/src/outbound/mod.rs@@ -3,11 +3,13 @@mod bot_repo_agent_lookup;mod channel_type_lookup;mod fast_model_judge;+mod image_caption;mod lexical_explicit_reply_extractor;mod message_thread_history;pub use bot_repo_agent_lookup::BotRepoAgentLookup;pub use channel_type_lookup::ChannelRepoTypeLookup;pub use fast_model_judge::FastModelTriggerJudge;+pub use image_caption::{ImageCaptioner, VisionImageCaptioner};pub use lexical_explicit_reply_extractor::LexicalExplicitReplyExtractor;pub use message_thread_history::MessageThreadHistory;diff --git a/services/agent_harness_service/Cargo.toml b/services/agent_harness_service/Cargo.tomlindex 44cc8d0e66..07dd796e14 100644--- a/services/agent_harness_service/Cargo.toml+++ b/services/agent_harness_service/Cargo.toml@@ -59,6 +59,7 @@ lexical_client = { path = "../../crates/lexical_client" }lexical_mention_extractor = { path = "../../crates/lexical_mention_extractor" }macro_service_urls = { path = "../../crates/macro_service_urls" }messages = { path = "../../crates/messages", features = ["broker", "delivery"] }+static_file = { path = "../../crates/static_file" }static_file_service_client = { path = "../../crates/static_file_service_client" }macro_tower_layers = { path = "../../crates/macro_tower_layers" }notification = { path = "../../crates/notification" }diff --git a/services/agent_harness_service/src/trigger.rs b/services/agent_harness_service/src/trigger.rsindex dc058eb749..b58c6a2067 100644--- a/services/agent_harness_service/src/trigger.rs+++ b/services/agent_harness_service/src/trigger.rs@@ -8,7 +8,7 @@ use agent_trigger::domain::sources::{};use agent_trigger::outbound::{BotRepoAgentLookup, ChannelRepoTypeLookup, FastModelTriggerJudge,- LexicalExplicitReplyExtractor, MessageThreadHistory,+ LexicalExplicitReplyExtractor, MessageThreadHistory, VisionImageCaptioner,};use bots::outbound::pg_bots_repo::PgBotsRepo;use channels::outbound::pg_channels_repo::PgChannelsRepo;@@ -18,7 +18,7 @@ use macro_event_broker::{KafkaConsumerAdapter, KafkaEventPublisher, MacroEventBrokerService, MacroEventCollection,MacroEventConsumerService,};-use macro_service_urls::LexicalServiceUrl;+use macro_service_urls::{LexicalServiceUrl, StaticFileServiceUrl};use messages::outbound::pg_message_repo::PgMessageRepository;use rdkafka::consumer::CommitMode;use rdkafka::message::{BorrowedMessage, Message as _};@@ -91,13 +91,18 @@ async fn run(source: TriggerEventSource,) -> anyhow::Result<()> {let lexical = LexicalClient::new(internal_api_key, LexicalServiceUrl::new()?.to_string());+ let recorder = ai_usage::pg_recorder(pool.clone());+ let images = VisionImageCaptioner::new(+ static_file::outbound::CdnStaticFileRepo::new(StaticFileServiceUrl::new()?.to_string()),+ recorder.clone(),+ );let trigger = AgentTriggerService::new(PgAgentSessionRepo::new(pool.clone()),BotRepoAgentLookup::new(PgBotsRepo::new(pool.clone())),BotRepoAgentLookup::new(PgBotsRepo::new(pool.clone())),BotRepoAgentLookup::new(PgBotsRepo::new(pool.clone())),LexicalExplicitReplyExtractor::new(lexical),- FastModelTriggerJudge::new(ai_usage::pg_recorder(pool.clone())),+ FastModelTriggerJudge::new(recorder, images),MessageThreadHistory::new(std::sync::Arc::new(messages::domain::service::MessageService::new(PgMessageRepository::new(pool.clone()),diff --git a/services/agent_trigger_service/Cargo.toml b/services/agent_trigger_service/Cargo.tomlindex 92c694a9d4..1ca67e4eeb 100644--- a/services/agent_trigger_service/Cargo.toml+++ b/services/agent_trigger_service/Cargo.toml@@ -28,6 +28,7 @@ messages = { path = "../../crates/messages", features = ["broker", "outbound"] }rdkafka = { workspace = true }serde_json = { workspace = true }sqlx = { workspace = true }+static_file = { path = "../../crates/static_file" }tokio = { workspace = true, features = ["macros", "rt-multi-thread", "signal"] }tracing = { workspace = true }workspace-hack = { version = "0.1", path = "../../crates/workspace-hack" }diff --git a/services/agent_trigger_service/src/main.rs b/services/agent_trigger_service/src/main.rsindex ffd2eced65..b861e20054 100644--- a/services/agent_trigger_service/src/main.rs+++ b/services/agent_trigger_service/src/main.rs@@ -12,7 +12,7 @@ use agent_trigger::domain::service::AgentTriggerService;use agent_trigger::domain::sources::{ChannelTriggerEvents, MessageTriggerEvents, TriggerEvents};use agent_trigger::outbound::{BotRepoAgentLookup, ChannelRepoTypeLookup, FastModelTriggerJudge,- LexicalExplicitReplyExtractor, MessageThreadHistory,+ LexicalExplicitReplyExtractor, MessageThreadHistory, VisionImageCaptioner,};use anyhow::Context as _;use bots::outbound::pg_bots_repo::PgBotsRepo;@@ -25,7 +25,7 @@ use macro_event_broker::{KafkaConsumerAdapter, KafkaEventPublisher, MacroEventBrokerService, MacroEventCollection,MacroEventConsumerService,};-use macro_service_urls::LexicalServiceUrl;+use macro_service_urls::{LexicalServiceUrl, StaticFileServiceUrl};use messages::outbound::pg_message_repo::PgMessageRepository;use rdkafka::consumer::CommitMode;use rdkafka::message::{BorrowedMessage, Message as _};@@ -89,13 +89,18 @@ async fn run() -> anyhow::Result<()> {config.internal_api_key.clone(),LexicalServiceUrl::new()?.to_string(),);+ let recorder = ai_usage::pg_recorder(pool.clone());+ let images = VisionImageCaptioner::new(+ static_file::outbound::CdnStaticFileRepo::new(StaticFileServiceUrl::new()?.to_string()),+ recorder.clone(),+ );let trigger = AgentTriggerService::new(PgAgentSessionRepo::new(pool.clone()),BotRepoAgentLookup::new(PgBotsRepo::new(pool.clone())),BotRepoAgentLookup::new(PgBotsRepo::new(pool.clone())),BotRepoAgentLookup::new(PgBotsRepo::new(pool.clone())),LexicalExplicitReplyExtractor::new(lexical),- FastModelTriggerJudge::new(ai_usage::pg_recorder(pool.clone())),+ FastModelTriggerJudge::new(recorder, images),MessageThreadHistory::new(std::sync::Arc::new(messages::domain::service::MessageService::new(PgMessageRepository::new(pool.clone()),