Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

chore(agents): describe images for the haiku trigger judge (#7054)

Wolf Mermelstein43fd7826b2af
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json
index 86746f52f9..edaed7ebcb 100644
--- a/.github/workspace-dep-closures.json
+++ b/.github/workspace-dep-closures.json
@@ -340,6 +340,7 @@
"crates/secretsmanager_client",
"crates/share_permission_db_utils",
"crates/sqs_worker",
+ "crates/static_file",
"crates/static_file_service_client",
"crates/sync_service",
"crates/sync_service_client",
@@ -491,6 +492,7 @@
"crates/soup",
"crates/sqs_client",
"crates/sqs_worker",
+ "crates/static_file",
"crates/static_file_service_client",
"crates/sync_service",
"crates/sync_service_client",
@@ -797,6 +799,7 @@
"crates/remote_env_var",
"crates/share_permission_db_utils",
"crates/sqs_worker",
+ "crates/static_file",
"crates/static_file_service_client",
"crates/system_properties",
"crates/workspace-hack"
@@ -876,6 +879,7 @@
"crates/remote_env_var",
"crates/share_permission_db_utils",
"crates/sqs_worker",
+ "crates/static_file",
"crates/static_file_service_client",
"crates/system_properties",
"crates/workspace-hack",
@@ -2513,6 +2517,7 @@
"crates/share_permission_db_utils",
"crates/sqs_client",
"crates/sqs_worker",
+ "crates/static_file",
"crates/static_file_service_client",
"crates/sync_service",
"crates/sync_service_client",
@@ -3680,6 +3685,7 @@
"crates/soup_realtime",
"crates/sqs_client",
"crates/sqs_worker",
+ "crates/static_file",
"crates/static_file_service_client",
"crates/sync_service",
"crates/sync_service_client",
@@ -9956,6 +9962,7 @@
"crates/share_permission_db_utils",
"crates/sqs_client",
"crates/sqs_worker",
+ "crates/static_file",
"crates/static_file_service_client",
"crates/sync_service",
"crates/sync_service_client",
diff --git a/Cargo.lock b/Cargo.lock
index 13168aeff7..c0eb7686bf 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -412,6 +412,7 @@ dependencies = [
"serde",
"serde_json",
"sqlx",
+ "static_file",
"static_file_service_client",
"tokio",
"tokio-retry",
@@ -565,6 +566,7 @@ dependencies = [
"agent_session",
"ai_usage",
"anyhow",
+ "async-trait",
"bot_id",
"bots",
"channel_sender",
@@ -577,11 +579,13 @@ dependencies = [
"macro_user_id",
"macro_uuid",
"messages",
+ "mime",
"mockall",
"model_owner",
"serde",
"serde_json",
"sqlx",
+ "static_file",
"strum 0.27.2",
"thiserror 2.0.18",
"tokio",
@@ -612,6 +616,7 @@ dependencies = [
"rdkafka",
"serde_json",
"sqlx",
+ "static_file",
"tokio",
"tracing",
"workspace-hack",
diff --git a/crates/agent/src/completion.rs b/crates/agent/src/completion.rs
index e5e2015947..f5ac2810e6 100644
--- a/crates/agent/src/completion.rs
+++ b/crates/agent/src/completion.rs
@@ -11,6 +11,7 @@ use ai_usage::{UsageContext, UsageRecorder};
use genai_telemetry::ContentPolicy;
use rig_agent::agent::{AgentBuilder, PromptResponse};
use rig_agent::completion::Prompt;
+use rig_core::OneOrMany;
use rig_core::completion::CompletionModel;
use rig_core::message::Message;
@@ -134,6 +135,45 @@ pub async fn complete_with_history<M: ToString>(
Ok(response.output)
}
+/// Ask `model` to answer `instruction` about one image and return its text.
+///
+/// `image_bytes` are the original file. They are downscaled to WebP before the
+/// model sees them. Token usage is recorded the same way as [`complete`].
+#[tracing::instrument(skip(model, system_prompt, image_bytes, recorder, ctx), err)]
+pub async fn complete_about_image<M: ToString>(
+ model: M,
+ system_prompt: &str,
+ instruction: &str,
+ image_bytes: Vec<u8>,
+ recorder: &dyn UsageRecorder,
+ ctx: UsageContext,
+) -> anyhow::Result<String> {
+ let message = image_user_message(instruction, image_bytes)?;
+ complete_with_history(model, system_prompt, vec![message], recorder, ctx).await
+}
+
+/// A user turn whose text is `instruction` and whose second block is the image.
+pub(crate) fn image_user_message(
+ instruction: &str,
+ image_bytes: Vec<u8>,
+) -> anyhow::Result<Message> {
+ use attachment::image::ImageData;
+ use rig_core::message::{ImageMediaType, UserContent};
+
+ let ImageData::Base64(image) = ImageData::try_from_bytes(image_bytes)? else {
+ anyhow::bail!("image bytes did not normalize to a base64 image");
+ };
+ let content = OneOrMany::many([
+ UserContent::text(instruction),
+ UserContent::image_base64(
+ image.base64_data().to_owned(),
+ Some(ImageMediaType::WEBP),
+ None,
+ ),
+ ])?;
+ Ok(Message::User { content })
+}
+
/// GenAI telemetry for a one-shot completion: named after the feature making
/// it, reporting the routed model, with no conversation (a one-shot has none).
fn telemetry_for(ctx: &UsageContext, routed: &RoutedModel<'_>) -> GenAiContext {
diff --git a/crates/agent/src/completion/test.rs b/crates/agent/src/completion/test.rs
index d6d324d811..3c2f800650 100644
--- a/crates/agent/src/completion/test.rs
+++ b/crates/agent/src/completion/test.rs
@@ -105,3 +105,44 @@ async fn a_failed_one_shot_completion_marks_its_chat_span() {
assert!(string_attribute(chat, attr::INPUT_MESSAGES).is_some());
assert_eq!(string_attribute(chat, attr::OUTPUT_MESSAGES), None);
}
+
+/// A minimal valid 1x1 PNG.
+#[rustfmt::skip]
+const ONE_BY_ONE_PNG: &[u8] = &[
+ 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D,
+ 0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01,
+ 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53, 0xDE, 0x00, 0x00, 0x00,
+ 0x0C, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9C, 0x63, 0xF8, 0xCF, 0xC0, 0x00,
+ 0x00, 0x03, 0x01, 0x01, 0x00, 0xC9, 0xFE, 0x92, 0xEF, 0x00, 0x00, 0x00,
+ 0x00, 0x49, 0x45, 0x4E, 0x44, 0xAE, 0x42, 0x60, 0x82,
+];
+
+#[test]
+fn image_user_message_sends_the_instruction_and_a_webp() {
+ use rig_core::message::{DocumentSourceKind, ImageMediaType, UserContent};
+
+ let message =
+ image_user_message("Describe this image.", ONE_BY_ONE_PNG.to_vec()).expect("a valid png");
+ let Message::User { content } = message else {
+ panic!("expected a user message");
+ };
+ let blocks: Vec<_> = content.into_iter().collect();
+ assert_eq!(blocks.len(), 2);
+
+ let UserContent::Text(text) = &blocks[0] else {
+ panic!("expected the instruction first");
+ };
+ assert_eq!(text.text, "Describe this image.");
+
+ let UserContent::Image(image) = &blocks[1] else {
+ panic!("expected the image second");
+ };
+ assert_eq!(image.media_type, Some(ImageMediaType::WEBP));
+ assert!(matches!(image.data, DocumentSourceKind::Base64(_)));
+}
+
+#[test]
+fn image_user_message_rejects_bytes_that_are_not_an_image() {
+ image_user_message("Describe this image.", b"not an image".to_vec())
+ .expect_err("non-image bytes");
+}
diff --git a/crates/agent/src/lib.rs b/crates/agent/src/lib.rs
index 6599d821ad..6a7f59def6 100644
--- a/crates/agent/src/lib.rs
+++ b/crates/agent/src/lib.rs
@@ -21,7 +21,7 @@ pub mod types;
pub use accumulator::StreamAccumulator;
pub use agent_loop::{AgentLoop, Session};
-pub use completion::{complete, complete_with_history};
+pub use completion::{complete, complete_about_image, complete_with_history};
pub use convert::{merge_consecutive_parts, to_rig_messages};
pub use error::AgentError;
pub use hook::{FinishedUserTool, PendingUserTool, UserToolFinisher};
diff --git a/crates/agent_trigger/Cargo.toml b/crates/agent_trigger/Cargo.toml
index 6d61788616..24ace5c8bb 100644
--- a/crates/agent_trigger/Cargo.toml
+++ b/crates/agent_trigger/Cargo.toml
@@ -9,7 +9,7 @@ version = "0.1.0"
# models_search → models_soup → notification → rdkafka, and the self-hosted
# daemon only needs domain events from this crate. Services that construct
# the adapters enable this; do not add it to coding_agent_worker.
-outbound = ["dep:agent", "dep:ai_usage", "dep:lexical_client", "dep:sqlx", "channels/ports"]
+outbound = ["dep:agent", "dep:ai_usage", "dep:async-trait", "dep:lexical_client", "dep:sqlx", "dep:static_file", "channels/ports"]
[dependencies]
agent = { path = "../agent", optional = true }
@@ -20,6 +20,7 @@ bot_id = { path = "../bot_id" }
entity_access = { path = "../entity_access", default-features = false, features = ["ports"] }
bots = { path = "../bots", default-features = false, features = ["ports"] }
lexical_client = { path = "../lexical_client", optional = true }
+static_file = { path = "../static_file", optional = true, default-features = false }
chrono = { workspace = true }
# Only the domain vocabulary (`Event`, `MacroEvent`, `TopicEvent`). Asking for
# `outbound` here would pull rdkafka — and so librdkafka and openssl — into
@@ -40,13 +41,15 @@ strum = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
tracing = { workspace = true }
-tokio = { workspace = true, features = ["rt"] }
+tokio = { workspace = true, features = ["rt", "time"] }
+async-trait = { workspace = true, optional = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
[dev-dependencies]
messages = { path = "../messages", features = ["test-utils"] }
agent_session = { path = "../agent_session", features = ["test-utils"] }
bot_id = { path = "../bot_id", features = ["test-utils"] }
+mime = "0.3"
mockall = { workspace = true }
model_owner = { path = "../model_owner" }
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
diff --git a/crates/agent_trigger/src/outbound/fast_model_judge.rs b/crates/agent_trigger/src/outbound/fast_model_judge.rs
index 29d4474743..356c8df2ff 100644
--- a/crates/agent_trigger/src/outbound/fast_model_judge.rs
+++ b/crates/agent_trigger/src/outbound/fast_model_judge.rs
@@ -1,5 +1,8 @@
//! Implicit-trigger judgement backed by the fast agent model.
+#[cfg(test)]
+mod test;
+
use std::sync::Arc;
use agent::structured_output::{DynamicSchema, dynamic_structured_completion};
@@ -12,6 +15,8 @@ use serde_json::json;
use crate::domain::service::ImplicitTriggerJudge;
+use super::image_caption::{ImageCaptioner, append_image_blurbs, blurbs_for_attachments};
+
static SYSTEM_PROMPT: &str = "\
You decide whether a channel message is addressed to an AI coding agent.
@@ -28,7 +33,14 @@ its work addressed to other people, or unrelated discussion.
You are given the thread around the agent's part in it, as lines of \
'[speaker] message' where the agent's own messages are marked '[agent]'. Some \
-messages may be hidden; judge on what you are shown.";
+messages may be hidden; judge on what you are shown.
+
+Attached images are not shown to you. Each one is written into the message \
+as <this is an image of ...>, a description of the picture rather than words \
+the author typed. Treat that description as what the image shows. \
+<this is an image> means a picture was attached but could not be described. \
+Those descriptions appear on the message to judge; the thread transcript \
+does not repeat them.";
#[derive(Debug, Deserialize)]
struct JudgeOutput {
@@ -42,14 +54,19 @@ struct JudgeOutput {
pub struct FastModelTriggerJudge {
model: PredefinedModel,
recorder: Arc<dyn UsageRecorder>,
+ images: Arc<dyn ImageCaptioner>,
}
impl FastModelTriggerJudge {
/// Creates a judge using the fast agent model.
- pub fn new(recorder: Arc<dyn UsageRecorder>) -> Self {
+ ///
+ /// `images` turns attached pictures into the `<this is an image of ...>`
+ /// blurbs the text-only model reads. The forwarded message is unchanged.
+ pub fn new(recorder: Arc<dyn UsageRecorder>, images: impl ImageCaptioner + 'static) -> Self {
Self {
model: PredefinedModel::Fast,
recorder,
+ images: Arc::new(images),
}
}
}
@@ -89,14 +106,10 @@ impl ImplicitTriggerJudge for FastModelTriggerJudge {
None => UsageContext::system(AiFeature::Automation),
};
- let prompt = if transcript.is_empty() {
- format!("The message to judge:\n{}", posted.content)
- } else {
- format!(
- "The thread so far:\n{transcript}\nThe message to judge:\n{}",
- posted.content
- )
- };
+ let blurbs =
+ blurbs_for_attachments(self.images.as_ref(), &posted.attachments, ctx.clone()).await;
+ let content = append_image_blurbs(&posted.content, &blurbs);
+ let prompt = judge_user_prompt(transcript, &content);
let value = dynamic_structured_completion(
self.model,
@@ -114,3 +127,12 @@ impl ImplicitTriggerJudge for FastModelTriggerJudge {
Ok(output.addressed_to_agent)
}
}
+
+/// The user prompt the judge scores. Image blurbs are already part of `content`.
+pub(crate) fn judge_user_prompt(transcript: &str, content: &str) -> String {
+ if transcript.is_empty() {
+ format!("The message to judge:\n{content}")
+ } else {
+ format!("The thread so far:\n{transcript}\nThe message to judge:\n{content}")
+ }
+}
diff --git a/crates/agent_trigger/src/outbound/fast_model_judge/test.rs b/crates/agent_trigger/src/outbound/fast_model_judge/test.rs
new file mode 100644
index 0000000000..c16573c648
--- /dev/null
+++ b/crates/agent_trigger/src/outbound/fast_model_judge/test.rs
@@ -0,0 +1,20 @@
+use super::*;
+
+#[test]
+fn the_judge_is_told_what_an_image_blurb_means() {
+ assert!(SYSTEM_PROMPT.contains("<this is an image of ...>"));
+ assert!(SYSTEM_PROMPT.contains("<this is an image>"));
+}
+
+#[test]
+fn an_image_blurb_is_part_of_the_message_being_judged() {
+ let content = "see this\n<this is an image of a frog>";
+ assert_eq!(
+ judge_user_prompt("", content),
+ "The message to judge:\nsee this\n<this is an image of a frog>"
+ );
+ assert_eq!(
+ judge_user_prompt("[agent] on it\n", "see this"),
+ "The thread so far:\n[agent] on it\n\nThe message to judge:\nsee this"
+ );
+}
diff --git a/crates/agent_trigger/src/outbound/image_caption.rs b/crates/agent_trigger/src/outbound/image_caption.rs
new file mode 100644
index 0000000000..5b4dc0ecac
--- /dev/null
+++ b/crates/agent_trigger/src/outbound/image_caption.rs
@@ -0,0 +1,204 @@
+//! Turns attached images into short phrases a text-only judge can read.
+
+#[cfg(test)]
+mod test;
+
+use std::sync::Arc;
+use std::time::Duration;
+
+use agent::PredefinedModel;
+use ai_usage::{UsageContext, UsageRecorder};
+use async_trait::async_trait;
+use messages::domain::events::MessageEventAttachment;
+use static_file::domain::ports::StaticFileRepo;
+
+/// Channel attachment entity type for an image stored as a static file.
+const STATIC_IMAGE: &str = "static/image";
+
+/// How many images get a model description. Further images are still marked,
+/// so the judge can tell they exist, without a vision call per picture.
+const MAX_DESCRIBED_IMAGES: usize = 8;
+
+/// Bound on one description, download included, so a stuck file cannot hold
+/// the trigger consumer.
+const CAPTION_TIMEOUT: Duration = Duration::from_secs(8);
+
+/// Longest phrase kept from a caption. A rambling answer would crowd out the
+/// message the judge is actually scoring.
+const MAX_PHRASE_CHARS: usize = 160;
+
+const CAPTION_SYSTEM: &str = "\
+Reply with only a short noun phrase describing the image, as if finishing \
+\"this is an image of\". Examples: \"a frog\", \"a screenshot of a compiler \
+error\". No sentence, no quotes, no trailing punctuation.";
+
+const CAPTION_INSTRUCTION: &str = "Describe this image.";
+
+/// Describes one static-file image as a short noun phrase, such as "a frog".
+///
+/// `None` means the picture could not be described. Callers still tell the
+/// judge that an image was attached.
+#[async_trait]
+pub trait ImageCaptioner: Send + Sync {
+ /// A short noun phrase for `file_id`, or `None` when it cannot be described.
+ async fn caption(&self, file_id: &str, ctx: UsageContext) -> Option<String>;
+}
+
+/// Captions images with the fast model after downloading them from static files.
+pub struct VisionImageCaptioner<F> {
+ files: F,
+ model: PredefinedModel,
+ recorder: Arc<dyn UsageRecorder>,
+}
+
+impl<F> VisionImageCaptioner<F> {
+ /// Caption images read through `files`, billing `recorder`.
+ pub fn new(files: F, recorder: Arc<dyn UsageRecorder>) -> Self {
+ Self {
+ files,
+ model: PredefinedModel::Fast,
+ recorder,
+ }
+ }
+}
+
+#[async_trait]
+impl<F> ImageCaptioner for VisionImageCaptioner<F>
+where
+ F: StaticFileRepo,
+{
+ async fn caption(&self, file_id: &str, ctx: UsageContext) -> Option<String> {
+ let bytes = match self.files.read(file_id).await {
+ Ok(bytes) => bytes,
+ Err(error) => {
+ tracing::warn!(error = ?error, file_id, "failed to read an attached image");
+ return None;
+ }
+ };
+ match agent::complete_about_image(
+ self.model,
+ CAPTION_SYSTEM,
+ CAPTION_INSTRUCTION,
+ bytes,
+ self.recorder.as_ref(),
+ ctx,
+ )
+ .await
+ {
+ Ok(raw) => normalize_caption(&raw),
+ Err(error) => {
+ tracing::warn!(error = ?error, file_id, "failed to describe an attached image");
+ None
+ }
+ }
+ }
+}
+
+/// Blurbs for the image attachments on a message, in attachment order.
+///
+/// Each described image becomes `<this is an image of a frog>`. An image that
+/// cannot be described, or that is past [`MAX_DESCRIBED_IMAGES`], becomes
+/// `<this is an image>`.
+pub(crate) async fn blurbs_for_attachments(
+ captioner: &dyn ImageCaptioner,
+ attachments: &[MessageEventAttachment],
+ ctx: UsageContext,
+) -> Vec<String> {
+ let image_ids: Vec<&str> = attachments
+ .iter()
+ .filter(|attachment| attachment.entity_type == STATIC_IMAGE)
+ .map(|attachment| attachment.entity_id.as_str())
+ .collect();
+
+ let mut blurbs = Vec::with_capacity(image_ids.len());
+ for (index, file_id) in image_ids.iter().copied().enumerate() {
+ if index >= MAX_DESCRIBED_IMAGES {
+ blurbs.push(image_blurb(None));
+ continue;
+ }
+ let phrase =
+ match tokio::time::timeout(CAPTION_TIMEOUT, captioner.caption(file_id, ctx.clone()))
+ .await
+ {
+ Ok(phrase) => phrase,
+ Err(_elapsed) => {
+ tracing::warn!(file_id, "image caption timed out");
+ None
+ }
+ };
+ blurbs.push(image_blurb(phrase.as_deref()));
+ }
+ blurbs
+}
+
+/// `<this is an image of {phrase}>`, or `<this is an image>` when there is no phrase.
+pub(crate) fn image_blurb(phrase: Option<&str>) -> String {
+ match phrase.map(str::trim).filter(|phrase| !phrase.is_empty()) {
+ Some(phrase) => format!("<this is an image of {phrase}>"),
+ None => "<this is an image>".to_owned(),
+ }
+}
+
+/// Append image blurbs to message text. An image-only message is just the blurbs.
+pub(crate) fn append_image_blurbs(content: &str, blurbs: &[String]) -> String {
+ if blurbs.is_empty() {
+ return content.to_owned();
+ }
+ let blurbs = blurbs.join("\n");
+ if content.trim().is_empty() {
+ blurbs
+ } else {
+ format!("{}\n{blurbs}", content.trim_end())
+ }
+}
+
+/// The first line of a caption, reduced to the noun phrase the blurb wraps.
+pub(crate) fn normalize_caption(raw: &str) -> Option<String> {
+ let mut phrase = raw.lines().next()?.trim().to_owned();
+ phrase = strip_wrapping_quotes(&phrase).to_owned();
+ for prefix in ["this is an image of ", "an image of ", "image of "] {
+ if let Some(rest) = strip_prefix_ignore_ascii_case(&phrase, prefix) {
+ phrase = rest.to_owned();
+ break;
+ }
+ }
+ phrase.retain(|ch| ch != '<' && ch != '>');
+ let phrase = phrase
+ .split_whitespace()
+ .collect::<Vec<_>>()
+ .join(" ")
+ .trim_end_matches(['.', '!', '?', ',', ';', ':'])
+ .trim()
+ .to_owned();
+ if phrase.is_empty() {
+ return None;
+ }
+ let phrase = phrase.chars().take(MAX_PHRASE_CHARS).collect::<String>();
+ let phrase = phrase.trim_end().to_owned();
+ if phrase.is_empty() {
+ None
+ } else {
+ Some(phrase)
+ }
+}
+
+fn strip_wrapping_quotes(phrase: &str) -> &str {
+ let mut chars = phrase.chars();
+ let (Some(open), Some(close)) = (chars.next(), chars.next_back()) else {
+ return phrase;
+ };
+ if (open == '"' && close == '"') || (open == '\'' && close == '\'') {
+ phrase[open.len_utf8()..phrase.len() - close.len_utf8()].trim()
+ } else {
+ phrase
+ }
+}
+
+fn strip_prefix_ignore_ascii_case<'a>(phrase: &'a str, prefix: &str) -> Option<&'a str> {
+ let rest = phrase.get(prefix.len()..)?;
+ if phrase[..prefix.len()].eq_ignore_ascii_case(prefix) {
+ Some(rest.trim())
+ } else {
+ None
+ }
+}
diff --git a/crates/agent_trigger/src/outbound/image_caption/test.rs b/crates/agent_trigger/src/outbound/image_caption/test.rs
new file mode 100644
index 0000000000..bc0bf2caaf
--- /dev/null
+++ b/crates/agent_trigger/src/outbound/image_caption/test.rs
@@ -0,0 +1,165 @@
+use super::*;
+use chrono::Utc;
+use macro_user_id::cowlike::CowLike as _;
+use macro_user_id::user_id::MacroUserIdStr;
+use macro_uuid::Uuid;
+use std::sync::Mutex;
+
+fn user() -> MacroUserIdStr<'static> {
+ MacroUserIdStr::parse_from_str("macro|[email protected]")
+ .expect("valid user id")
+ .into_owned()
+}
+
+fn ctx() -> UsageContext {
+ UsageContext::new(ai_usage::AiFeature::Automation, user())
+}
+
+fn attachment(entity_type: &str, entity_id: &str) -> MessageEventAttachment {
+ MessageEventAttachment {
+ attachment_id: Uuid::from_u128(1),
+ entity_type: entity_type.to_owned(),
+ entity_id: entity_id.to_owned(),
+ created_at: Utc::now(),
+ }
+}
+
+struct Scripted {
+ calls: Mutex<Vec<String>>,
+ phrase: Option<String>,
+}
+
+#[async_trait]
+impl ImageCaptioner for Scripted {
+ async fn caption(&self, file_id: &str, _ctx: UsageContext) -> Option<String> {
+ self.calls.lock().expect("calls").push(file_id.to_owned());
+ self.phrase.clone()
+ }
+}
+
+#[test]
+fn a_caption_becomes_the_blurb_the_judge_reads() {
+ assert_eq!(image_blurb(Some("a frog")), "<this is an image of a frog>");
+ assert_eq!(image_blurb(None), "<this is an image>");
+ assert_eq!(image_blurb(Some(" ")), "<this is an image>");
+}
+
+#[test]
+fn captions_are_reduced_to_a_noun_phrase() {
+ assert_eq!(normalize_caption("a frog").as_deref(), Some("a frog"));
+ assert_eq!(normalize_caption("a frog.").as_deref(), Some("a frog"));
+ assert_eq!(normalize_caption("\"a frog\"").as_deref(), Some("a frog"));
+ assert_eq!(
+ normalize_caption("This is an image of a frog.").as_deref(),
+ Some("a frog")
+ );
+ assert_eq!(
+ normalize_caption("a frog\nignore the rest").as_deref(),
+ Some("a frog")
+ );
+ assert_eq!(normalize_caption("<a frog>").as_deref(), Some("a frog"));
+ assert_eq!(normalize_caption(" \n"), None);
+ assert_eq!(normalize_caption(""), None);
+}
+
+#[test]
+fn blurbs_follow_the_authors_words() {
+ let blurbs = vec!["<this is an image of a frog>".to_owned()];
+ assert_eq!(
+ append_image_blurbs("see this", &blurbs),
+ "see this\n<this is an image of a frog>"
+ );
+ assert_eq!(
+ append_image_blurbs(" ", &blurbs),
+ "<this is an image of a frog>"
+ );
+ assert_eq!(append_image_blurbs("see this", &[]), "see this");
+}
+
+#[tokio::test]
+async fn only_images_are_described_and_a_failed_caption_stays_visible() {
+ let captioner = Scripted {
+ calls: Mutex::new(Vec::new()),
+ phrase: Some("a frog".to_owned()),
+ };
+ let attachments = vec![
+ attachment("document", "doc-1"),
+ attachment("static/video", "clip-1"),
+ attachment("static/image", "frog"),
+ ];
+
+ let blurbs = blurbs_for_attachments(&captioner, &attachments, ctx()).await;
+
+ assert_eq!(blurbs, vec!["<this is an image of a frog>".to_owned()]);
+ assert_eq!(captioner.calls.lock().expect("calls").as_slice(), ["frog"]);
+
+ let silent = Scripted {
+ calls: Mutex::new(Vec::new()),
+ phrase: None,
+ };
+ let blurbs =
+ blurbs_for_attachments(&silent, &[attachment("static/image", "unreadable")], ctx()).await;
+ assert_eq!(blurbs, vec!["<this is an image>".to_owned()]);
+}
+
+#[tokio::test]
+async fn images_past_the_description_cap_are_marked_without_another_caption() {
+ let captioner = Scripted {
+ calls: Mutex::new(Vec::new()),
+ phrase: Some("a frog".to_owned()),
+ };
+ let attachments: Vec<_> = (0..MAX_DESCRIBED_IMAGES + 1)
+ .map(|index| attachment("static/image", &format!("img-{index}")))
+ .collect();
+
+ let blurbs = blurbs_for_attachments(&captioner, &attachments, ctx()).await;
+
+ assert_eq!(blurbs.len(), MAX_DESCRIBED_IMAGES + 1);
+ assert!(
+ blurbs[..MAX_DESCRIBED_IMAGES]
+ .iter()
+ .all(|blurb| blurb == "<this is an image of a frog>")
+ );
+ assert_eq!(
+ blurbs.last().map(String::as_str),
+ Some("<this is an image>")
+ );
+ assert_eq!(
+ captioner.calls.lock().expect("calls").len(),
+ MAX_DESCRIBED_IMAGES
+ );
+}
+
+struct MissingFile;
+
+impl StaticFileRepo for MissingFile {
+ async fn content_type(&self, _file_id: &str) -> anyhow::Result<mime::Mime> {
+ Ok(mime::IMAGE_PNG)
+ }
+
+ async fn read(&self, file_id: &str) -> anyhow::Result<Vec<u8>> {
+ anyhow::bail!("no file {file_id}")
+ }
+}
+
+struct GarbageFile;
+
+impl StaticFileRepo for GarbageFile {
+ async fn content_type(&self, _file_id: &str) -> anyhow::Result<mime::Mime> {
+ Ok(mime::APPLICATION_OCTET_STREAM)
+ }
+
+ async fn read(&self, _file_id: &str) -> anyhow::Result<Vec<u8>> {
+ Ok(b"not an image".to_vec())
+ }
+}
+
+#[tokio::test]
+async fn a_missing_or_undecodable_image_is_not_described() {
+ let recorder = Arc::new(ai_usage::NoOpUsageRecorder);
+ let missing = VisionImageCaptioner::new(MissingFile, recorder.clone());
+ assert_eq!(missing.caption("gone", ctx()).await, None);
+
+ let garbage = VisionImageCaptioner::new(GarbageFile, recorder);
+ assert_eq!(garbage.caption("bad", ctx()).await, None);
+}
diff --git a/crates/agent_trigger/src/outbound/mod.rs b/crates/agent_trigger/src/outbound/mod.rs
index c2b198e12b..19ecaec7d5 100644
--- a/crates/agent_trigger/src/outbound/mod.rs
+++ b/crates/agent_trigger/src/outbound/mod.rs
@@ -3,11 +3,13 @@
mod bot_repo_agent_lookup;
mod channel_type_lookup;
mod fast_model_judge;
+mod image_caption;
mod lexical_explicit_reply_extractor;
mod message_thread_history;
pub use bot_repo_agent_lookup::BotRepoAgentLookup;
pub use channel_type_lookup::ChannelRepoTypeLookup;
pub use fast_model_judge::FastModelTriggerJudge;
+pub use image_caption::{ImageCaptioner, VisionImageCaptioner};
pub use lexical_explicit_reply_extractor::LexicalExplicitReplyExtractor;
pub use message_thread_history::MessageThreadHistory;
diff --git a/services/agent_harness_service/Cargo.toml b/services/agent_harness_service/Cargo.toml
index 44cc8d0e66..07dd796e14 100644
--- a/services/agent_harness_service/Cargo.toml
+++ b/services/agent_harness_service/Cargo.toml
@@ -59,6 +59,7 @@ lexical_client = { path = "../../crates/lexical_client" }
lexical_mention_extractor = { path = "../../crates/lexical_mention_extractor" }
macro_service_urls = { path = "../../crates/macro_service_urls" }
messages = { path = "../../crates/messages", features = ["broker", "delivery"] }
+static_file = { path = "../../crates/static_file" }
static_file_service_client = { path = "../../crates/static_file_service_client" }
macro_tower_layers = { path = "../../crates/macro_tower_layers" }
notification = { path = "../../crates/notification" }
diff --git a/services/agent_harness_service/src/trigger.rs b/services/agent_harness_service/src/trigger.rs
index dc058eb749..b58c6a2067 100644
--- a/services/agent_harness_service/src/trigger.rs
+++ b/services/agent_harness_service/src/trigger.rs
@@ -8,7 +8,7 @@ use agent_trigger::domain::sources::{
};
use agent_trigger::outbound::{
BotRepoAgentLookup, ChannelRepoTypeLookup, FastModelTriggerJudge,
- LexicalExplicitReplyExtractor, MessageThreadHistory,
+ LexicalExplicitReplyExtractor, MessageThreadHistory, VisionImageCaptioner,
};
use bots::outbound::pg_bots_repo::PgBotsRepo;
use channels::outbound::pg_channels_repo::PgChannelsRepo;
@@ -18,7 +18,7 @@ use macro_event_broker::{
KafkaConsumerAdapter, KafkaEventPublisher, MacroEventBrokerService, MacroEventCollection,
MacroEventConsumerService,
};
-use macro_service_urls::LexicalServiceUrl;
+use macro_service_urls::{LexicalServiceUrl, StaticFileServiceUrl};
use messages::outbound::pg_message_repo::PgMessageRepository;
use rdkafka::consumer::CommitMode;
use rdkafka::message::{BorrowedMessage, Message as _};
@@ -91,13 +91,18 @@ async fn run(
source: TriggerEventSource,
) -> anyhow::Result<()> {
let lexical = LexicalClient::new(internal_api_key, LexicalServiceUrl::new()?.to_string());
+ let recorder = ai_usage::pg_recorder(pool.clone());
+ let images = VisionImageCaptioner::new(
+ static_file::outbound::CdnStaticFileRepo::new(StaticFileServiceUrl::new()?.to_string()),
+ recorder.clone(),
+ );
let trigger = AgentTriggerService::new(
PgAgentSessionRepo::new(pool.clone()),
BotRepoAgentLookup::new(PgBotsRepo::new(pool.clone())),
BotRepoAgentLookup::new(PgBotsRepo::new(pool.clone())),
BotRepoAgentLookup::new(PgBotsRepo::new(pool.clone())),
LexicalExplicitReplyExtractor::new(lexical),
- FastModelTriggerJudge::new(ai_usage::pg_recorder(pool.clone())),
+ FastModelTriggerJudge::new(recorder, images),
MessageThreadHistory::new(
std::sync::Arc::new(messages::domain::service::MessageService::new(
PgMessageRepository::new(pool.clone()),
diff --git a/services/agent_trigger_service/Cargo.toml b/services/agent_trigger_service/Cargo.toml
index 92c694a9d4..1ca67e4eeb 100644
--- a/services/agent_trigger_service/Cargo.toml
+++ b/services/agent_trigger_service/Cargo.toml
@@ -28,6 +28,7 @@ messages = { path = "../../crates/messages", features = ["broker", "outbound"] }
rdkafka = { workspace = true }
serde_json = { workspace = true }
sqlx = { workspace = true }
+static_file = { path = "../../crates/static_file" }
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "signal"] }
tracing = { workspace = true }
workspace-hack = { version = "0.1", path = "../../crates/workspace-hack" }
diff --git a/services/agent_trigger_service/src/main.rs b/services/agent_trigger_service/src/main.rs
index ffd2eced65..b861e20054 100644
--- a/services/agent_trigger_service/src/main.rs
+++ b/services/agent_trigger_service/src/main.rs
@@ -12,7 +12,7 @@ use agent_trigger::domain::service::AgentTriggerService;
use agent_trigger::domain::sources::{ChannelTriggerEvents, MessageTriggerEvents, TriggerEvents};
use agent_trigger::outbound::{
BotRepoAgentLookup, ChannelRepoTypeLookup, FastModelTriggerJudge,
- LexicalExplicitReplyExtractor, MessageThreadHistory,
+ LexicalExplicitReplyExtractor, MessageThreadHistory, VisionImageCaptioner,
};
use anyhow::Context as _;
use bots::outbound::pg_bots_repo::PgBotsRepo;
@@ -25,7 +25,7 @@ use macro_event_broker::{
KafkaConsumerAdapter, KafkaEventPublisher, MacroEventBrokerService, MacroEventCollection,
MacroEventConsumerService,
};
-use macro_service_urls::LexicalServiceUrl;
+use macro_service_urls::{LexicalServiceUrl, StaticFileServiceUrl};
use messages::outbound::pg_message_repo::PgMessageRepository;
use rdkafka::consumer::CommitMode;
use rdkafka::message::{BorrowedMessage, Message as _};
@@ -89,13 +89,18 @@ async fn run() -> anyhow::Result<()> {
config.internal_api_key.clone(),
LexicalServiceUrl::new()?.to_string(),
);
+ let recorder = ai_usage::pg_recorder(pool.clone());
+ let images = VisionImageCaptioner::new(
+ static_file::outbound::CdnStaticFileRepo::new(StaticFileServiceUrl::new()?.to_string()),
+ recorder.clone(),
+ );
let trigger = AgentTriggerService::new(
PgAgentSessionRepo::new(pool.clone()),
BotRepoAgentLookup::new(PgBotsRepo::new(pool.clone())),
BotRepoAgentLookup::new(PgBotsRepo::new(pool.clone())),
BotRepoAgentLookup::new(PgBotsRepo::new(pool.clone())),
LexicalExplicitReplyExtractor::new(lexical),
- FastModelTriggerJudge::new(ai_usage::pg_recorder(pool.clone())),
+ FastModelTriggerJudge::new(recorder, images),
MessageThreadHistory::new(
std::sync::Arc::new(messages::domain::service::MessageService::new(
PgMessageRepository::new(pool.clone()),