Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(mcp): return image blocks for markdown content (#5827)
teo8db7cb648c30
diff --git a/Cargo.lock b/Cargo.lockindex a258c0825c..250a4e2ca7 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -9729,6 +9729,8 @@ dependencies = ["ai_toolset","ai_usage","anyhow",+ "async-trait",+ "attachment","aws-sdk-secretsmanager","aws-sdk-sqs","axum",@@ -9765,6 +9767,7 @@ dependencies = ["readonly_pool","redis","reminders",+ "reqwest 0.13.4","rmcp","search_service_client","secretsmanager_client",@@ -9776,6 +9779,7 @@ dependencies = ["tokio","tokio-util","tracing",+ "url","workspace-hack",]diff --git a/services/mcp_service/Cargo.toml b/services/mcp_service/Cargo.tomlindex 3820a84cbf..c3070502be 100644--- a/services/mcp_service/Cargo.toml+++ b/services/mcp_service/Cargo.toml@@ -18,6 +18,8 @@ ai_tools = { path = "../../crates/ai_tools" }ai_toolset = { path = "../../crates/ai_toolset" }ai_usage = { path = "../../crates/ai_usage" }anyhow = { workspace = true }+async-trait = { workspace = true }+attachment = { path = "../../crates/attachment" }aws-sdk-secretsmanager = { workspace = true }aws-sdk-sqs = { workspace = true }axum = { workspace = true, features = ["tower-log"] }@@ -58,6 +60,7 @@ prompt = { path = "../../crates/prompt" }readonly_pool = { path = "../../crates/readonly_pool" }redis = { workspace = true, features = ["aio", "tokio-comp"] }reminders = { path = "../../crates/reminders", default-features = false, features = ["ports"] }+reqwest = { workspace = true }rmcp = { workspace = true, features = ["transport-streamable-http-server"] }search_service_client = { path = "../../crates/search_service_client" }secretsmanager_client = { path = "../../crates/secretsmanager_client" }@@ -75,6 +78,7 @@ sync_service_client = { path = "../../crates/sync_service_client" }tokio = { workspace = true, features = ["signal"] }tokio-util = { workspace = true, features = ["rt"] }tracing = { workspace = true }+url = { workspace = true }macro_service_urls = { path = "../../crates/macro_service_urls" }workspace-hack = { version = "0.1", path = "../../crates/workspace-hack" }diff --git a/services/mcp_service/src/main.rs b/services/mcp_service/src/main.rsindex e91c60d824..811d25bb86 100644--- a/services/mcp_service/src/main.rs+++ b/services/mcp_service/src/main.rs@@ -6,6 +6,7 @@mod config;mod context;+mod markdown_images;mod tool_service;use anyhow::Context;use config::Config;diff --git a/services/mcp_service/src/markdown_images.rs b/services/mcp_service/src/markdown_images.rsnew file mode 100644index 0000000000..b12bfccfc7--- /dev/null+++ b/services/mcp_service/src/markdown_images.rs@@ -0,0 +1,285 @@+use ai_tools::ToolServiceContext;+use async_trait::async_trait;+use attachment::image::ImageData;+use documents::domain::{+ models::LocationQueryParams, ports::DocumentService, response::LocationResponseV3,+};+use entity_access::domain::{+ models::{EntityType, ViewAccessLevel},+ ports::EntityAccessService,+};+use macro_user_id::user_id::MacroUserIdStr;+use rmcp::model::{CallToolResult, Content};+use std::net::IpAddr;+use std::sync::OnceLock;+use url::Url;++#[cfg(test)]+mod test;++const MAX_MARKDOWN_IMAGES: usize = 8;+const MAX_IMAGE_BYTES: usize = 8 * 1024 * 1024;++#[derive(Debug, Clone, PartialEq, Eq)]+pub(crate) struct ResolvedImage {+ pub data: String,+ pub mime_type: String,+}++#[async_trait]+pub(crate) trait MarkdownImageResolver: Send + Sync {+ async fn resolve_static(&self, url: &str) -> Option<ResolvedImage>;+ async fn resolve_dss(&self, user_id: &MacroUserIdStr<'_>, id: &str) -> Option<ResolvedImage>;+}++#[cfg(test)]+#[async_trait]+impl MarkdownImageResolver for () {+ async fn resolve_static(&self, _url: &str) -> Option<ResolvedImage> {+ None+ }++ async fn resolve_dss(&self, _user_id: &MacroUserIdStr<'_>, _id: &str) -> Option<ResolvedImage> {+ None+ }+}++#[async_trait]+impl MarkdownImageResolver for ToolServiceContext {+ async fn resolve_static(&self, url: &str) -> Option<ResolvedImage> {+ fetch_public_and_encode(url).await+ }++ async fn resolve_dss(&self, user_id: &MacroUserIdStr<'_>, id: &str) -> Option<ResolvedImage> {+ let ctx = &self.document_tool_context;+ let receipt = ctx+ .entity_access_service+ .generate_entity_access_receipt::<ViewAccessLevel>(+ user_id,+ None,+ id,+ EntityType::Document,+ )+ .await+ .inspect_err(|error| {+ tracing::warn!(error=?error, id, "mcp dss image access denied");+ })+ .ok()?;+ let document = ctx+ .service+ .internal_get_basic_document(id)+ .await+ .inspect_err(|error| {+ tracing::warn!(error=?error, id, "mcp dss image document missing");+ })+ .ok()?;+ let location = ctx+ .service+ .get_document_location(+ &document,+ receipt,+ LocationQueryParams {+ get_converted_docx_url: Some(true),+ document_version_id: None,+ },+ )+ .await+ .inspect_err(|error| {+ tracing::warn!(error=?error, id, "mcp dss image location failed");+ })+ .ok()?;+ let LocationResponseV3::PresignedUrl { presigned_url, .. } = location else {+ tracing::warn!(id, "mcp dss image had no presigned url");+ return None;+ };+ fetch_and_encode(&presigned_url).await+ }+}++pub(crate) async fn tool_result_with_images<R: MarkdownImageResolver>(+ resolver: &R,+ user_id: &MacroUserIdStr<'_>,+ value: serde_json::Value,+) -> CallToolResult {+ let images = resolve_markdown_images(resolver, user_id, &value).await;+ let mut result = CallToolResult::structured(value);+ result.content.extend(+ images+ .into_iter()+ .map(|image| Content::image(image.data, image.mime_type)),+ );+ result+}++async fn resolve_markdown_images<R: MarkdownImageResolver>(+ resolver: &R,+ user_id: &MacroUserIdStr<'_>,+ value: &serde_json::Value,+) -> Vec<ResolvedImage> {+ let refs = markdown_image_refs(value);+ if refs.len() > MAX_MARKDOWN_IMAGES {+ tracing::warn!(+ count = refs.len(),+ max = MAX_MARKDOWN_IMAGES,+ "skipping markdown images beyond the MCP cap"+ );+ }++ let mut images = Vec::new();+ for image_ref in refs.into_iter().take(MAX_MARKDOWN_IMAGES) {+ let resolved = match image_ref {+ ImageRef::Static(url) => resolver.resolve_static(&url).await,+ ImageRef::Dss(id) => resolver.resolve_dss(user_id, &id).await,+ };+ if let Some(image) = resolved {+ images.push(image);+ }+ }+ images+}++#[derive(Debug, PartialEq, Eq)]+enum ImageRef {+ Static(String),+ Dss(String),+}++fn markdown_image_refs(value: &serde_json::Value) -> Vec<ImageRef> {+ let Some(nodes) = value+ .get("content")+ .and_then(|content| content.get("markdown"))+ .and_then(|markdown| markdown.as_array())+ else {+ return Vec::new();+ };++ nodes+ .iter()+ .filter_map(|node| {+ let node_type = node.get("type")?.as_str()?;+ match node_type {+ "staticImage" => node+ .get("url")+ .and_then(|url| url.as_str())+ .map(|url| ImageRef::Static(url.to_owned())),+ "dssImage" => node+ .get("id")+ .and_then(|id| id.as_str())+ .map(|id| ImageRef::Dss(id.to_owned())),+ _ => None,+ }+ })+ .collect()+}++async fn fetch_public_and_encode(url: &str) -> Option<ResolvedImage> {+ match fetch_public_and_encode_inner(url).await {+ Ok(image) => Some(image),+ Err(error) => {+ tracing::warn!(error=?error, url, "failed to fetch public markdown image for MCP");+ None+ }+ }+}++async fn fetch_public_and_encode_inner(url: &str) -> anyhow::Result<ResolvedImage> {+ let url = assert_public_http_url(url).await?;+ let response = static_http_client().get(url.as_str()).send().await?;+ encode_response(url.as_str(), response).await+}++async fn fetch_and_encode(url: &str) -> Option<ResolvedImage> {+ match fetch_and_encode_inner(url).await {+ Ok(image) => Some(image),+ Err(error) => {+ tracing::warn!(error=?error, url, "failed to fetch markdown image for MCP");+ None+ }+ }+}++async fn fetch_and_encode_inner(url: &str) -> anyhow::Result<ResolvedImage> {+ let url = macro_aws_config::transform_aws_url_for_internal_fetch(url);+ let url = url.as_str();+ let response = reqwest::get(url).await?;+ encode_response(url, response).await+}++async fn encode_response(url: &str, response: reqwest::Response) -> anyhow::Result<ResolvedImage> {+ if !response.status().is_success() {+ anyhow::bail!("failed to fetch {url}: HTTP {}", response.status());+ }+ let bytes = read_body_capped(response, MAX_IMAGE_BYTES).await?;+ match ImageData::try_from_bytes(bytes)? {+ ImageData::Base64(image) => Ok(ResolvedImage {+ data: image.base64_data().to_owned(),+ mime_type: "image/webp".to_owned(),+ }),+ ImageData::StaticUrl(_) => anyhow::bail!("image encoder returned a url"),+ }+}++async fn read_body_capped(mut response: reqwest::Response, max: usize) -> anyhow::Result<Vec<u8>> {+ if response+ .content_length()+ .is_some_and(|content_length| content_length > max as u64)+ {+ anyhow::bail!("image exceeds {max} bytes");+ }+ let mut out = Vec::new();+ while let Some(chunk) = response.chunk().await? {+ if out.len().saturating_add(chunk.len()) > max {+ anyhow::bail!("image exceeds {max} bytes");+ }+ out.extend_from_slice(&chunk);+ }+ Ok(out)+}++fn static_http_client() -> &'static reqwest::Client {+ static CLIENT: OnceLock<reqwest::Client> = OnceLock::new();+ CLIENT.get_or_init(|| {+ reqwest::Client::builder()+ .redirect(reqwest::redirect::Policy::none())+ .build()+ .expect("static image client")+ })+}++async fn assert_public_http_url(raw: &str) -> anyhow::Result<Url> {+ let url = Url::parse(raw)?;+ if url.scheme() != "http" && url.scheme() != "https" {+ anyhow::bail!("only http/https image urls");+ }+ let host = url+ .host_str()+ .ok_or_else(|| anyhow::anyhow!("missing host"))?;+ let port = url.port_or_known_default().unwrap_or(80);+ for addr in tokio::net::lookup_host((host, port)).await? {+ if is_private_ip(addr.ip()) {+ anyhow::bail!("private image url");+ }+ }+ Ok(url)+}++fn is_private_ip(ip: IpAddr) -> bool {+ match ip {+ IpAddr::V4(v4) => {+ v4.is_loopback()+ || v4.is_private()+ || v4.is_link_local()+ || v4.is_unspecified()+ || v4.is_broadcast()+ }+ IpAddr::V6(v6) => {+ if let Some(mapped_v4) = v6.to_ipv4_mapped() {+ return is_private_ip(IpAddr::V4(mapped_v4));+ }+ v6.is_loopback()+ || v6.is_unspecified()+ || v6.is_unique_local()+ || v6.is_unicast_link_local()+ }+ }+}diff --git a/services/mcp_service/src/markdown_images/test.rs b/services/mcp_service/src/markdown_images/test.rsnew file mode 100644index 0000000000..77e252f061--- /dev/null+++ b/services/mcp_service/src/markdown_images/test.rs@@ -0,0 +1,278 @@+use super::*;+use macro_user_id::user_id::MacroUserIdStr;+use serde_json::json;+use std::collections::HashMap;+use std::sync::atomic::{AtomicUsize, Ordering};++const TINY_PNG: &[u8] = &[+ 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52,+ 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, 0x00, 0x1F, 0x15, 0xC4,+ 0x89, 0x00, 0x00, 0x00, 0x0A, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9C, 0x63, 0x00, 0x01, 0x00, 0x00,+ 0x05, 0x00, 0x01, 0x0D, 0x0A, 0x2D, 0xB4, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, 0x44, 0xAE,+ 0x42, 0x60, 0x82,+];++fn user_id() -> MacroUserIdStr<'static> {+ MacroUserIdStr::try_from_email("[email protected]").unwrap()+}++fn read_content_markdown(nodes: serde_json::Value) -> serde_json::Value {+ json!({+ "content": { "markdown": nodes },+ "comments": []+ })+}++#[derive(Clone, Default)]+struct FakeResolver {+ static_images: HashMap<String, ResolvedImage>,+ dss_images: HashMap<String, ResolvedImage>,+}++#[async_trait]+impl MarkdownImageResolver for FakeResolver {+ async fn resolve_static(&self, url: &str) -> Option<ResolvedImage> {+ self.static_images.get(url).cloned()+ }++ async fn resolve_dss(&self, _user_id: &MacroUserIdStr<'_>, id: &str) -> Option<ResolvedImage> {+ self.dss_images.get(id).cloned()+ }+}++struct CountingResolver {+ hits: AtomicUsize,+}++#[async_trait]+impl MarkdownImageResolver for CountingResolver {+ async fn resolve_static(&self, _url: &str) -> Option<ResolvedImage> {+ self.hits.fetch_add(1, Ordering::SeqCst);+ Some(ResolvedImage {+ data: "x".into(),+ mime_type: "image/webp".into(),+ })+ }++ async fn resolve_dss(&self, _user_id: &MacroUserIdStr<'_>, _id: &str) -> Option<ResolvedImage> {+ None+ }+}++#[test]+fn markdown_image_refs_collect_static_and_dss_nodes() {+ let value = read_content_markdown(json!([+ {+ "type": "generic",+ "nodeId": "n1",+ "content": "Long tag names are hidden in the dropdown.",+ "tag": "paragraph"+ },+ { "type": "staticImage", "url": "https://static.example/tag-dropdown.png" },+ { "type": "dssImage", "id": "bb7ee066-7f29-4282-ae9e-beca188d033e" }+ ]));++ assert_eq!(+ markdown_image_refs(&value),+ vec![+ ImageRef::Static("https://static.example/tag-dropdown.png".into()),+ ImageRef::Dss("bb7ee066-7f29-4282-ae9e-beca188d033e".into()),+ ]+ );+}++#[test]+fn markdown_image_refs_ignore_plain_text_and_unrelated_json() {+ assert!(+ markdown_image_refs(&json!({ "content": { "text": "hello" }, "comments": [] })).is_empty()+ );+ assert!(markdown_image_refs(&json!({ "items": [] })).is_empty());+}++#[tokio::test]+async fn tool_result_keeps_structured_json_and_appends_image_blocks() {+ let resolver = FakeResolver {+ static_images: HashMap::from([(+ "https://static.example/tag-dropdown.png".into(),+ ResolvedImage {+ data: "aaa".into(),+ mime_type: "image/webp".into(),+ },+ )]),+ dss_images: HashMap::from([(+ "img-1".into(),+ ResolvedImage {+ data: "bbb".into(),+ mime_type: "image/webp".into(),+ },+ )]),+ };+ let value = read_content_markdown(json!([+ {+ "type": "generic",+ "nodeId": "n1",+ "content": "Long tag names are hidden in the dropdown.",+ "tag": "paragraph"+ },+ { "type": "staticImage", "url": "https://static.example/tag-dropdown.png" },+ { "type": "dssImage", "id": "img-1" }+ ]));++ let result = tool_result_with_images(&resolver, &user_id(), value.clone()).await;++ assert_eq!(result.structured_content.as_ref(), Some(&value));+ assert_eq!(result.is_error, Some(false));+ assert_eq!(result.content.len(), 3);+ let expected_text = value.to_string();+ assert_eq!(+ result.content[0].as_text().map(|text| text.text.as_str()),+ Some(expected_text.as_str())+ );++ let first_image = result.content[1]+ .as_image()+ .expect("staticImage must become an MCP image block");+ assert_eq!(first_image.data, "aaa");+ assert_eq!(first_image.mime_type, "image/webp");++ let second_image = result.content[2]+ .as_image()+ .expect("dssImage must become an MCP image block");+ assert_eq!(second_image.data, "bbb");+ assert_eq!(second_image.mime_type, "image/webp");+}++#[tokio::test]+async fn tool_result_without_images_stays_structured_text() {+ let value = read_content_markdown(json!([+ { "type": "generic", "nodeId": "n1", "content": "no pictures", "tag": "paragraph" }+ ]));++ let result = tool_result_with_images(&FakeResolver::default(), &user_id(), value.clone()).await;++ assert_eq!(result.content.len(), 1);+ assert!(result.content[0].as_text().is_some());+ assert!(result.content[0].as_image().is_none());+}++#[tokio::test]+async fn unresolved_images_are_omitted_from_content_blocks() {+ let value = read_content_markdown(json!([+ { "type": "staticImage", "url": "https://missing.example/gone.png" }+ ]));++ let result = tool_result_with_images(&FakeResolver::default(), &user_id(), value).await;++ assert_eq!(result.content.len(), 1);+ assert!(result.content[0].as_image().is_none());+}++#[tokio::test]+async fn fetch_and_encode_returns_webp_bytes() {+ let app = axum::Router::new().route(+ "/",+ axum::routing::get(|| async {+ ([(axum::http::header::CONTENT_TYPE, "image/png")], TINY_PNG)+ }),+ );+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let addr = listener.local_addr().unwrap();+ tokio::spawn(async move {+ axum::serve(listener, app).await.unwrap();+ });++ let image = fetch_and_encode(&format!("http://{addr}/"))+ .await+ .expect("png should encode as a webp image block");+ assert_eq!(image.mime_type, "image/webp");+ assert!(!image.data.is_empty());+}++#[tokio::test]+async fn resolve_markdown_images_caps_count() {+ let resolver = CountingResolver {+ hits: AtomicUsize::new(0),+ };+ let markdown: Vec<serde_json::Value> = (0..12)+ .map(+ |i| json!({ "type": "staticImage", "url": format!("https://cdn.example.com/{i}.png") }),+ )+ .collect();++ let images = resolve_markdown_images(+ &resolver,+ &user_id(),+ &read_content_markdown(serde_json::Value::Array(markdown)),+ )+ .await;++ assert_eq!(images.len(), MAX_MARKDOWN_IMAGES);+ assert_eq!(resolver.hits.load(Ordering::SeqCst), MAX_MARKDOWN_IMAGES);+}++#[tokio::test]+async fn assert_public_http_url_rejects_non_http_and_private_hosts() {+ assert!(assert_public_http_url("http://1.1.1.1/a.png").await.is_ok());+ assert!(assert_public_http_url("file:///etc/passwd").await.is_err());+ assert!(+ assert_public_http_url("https://127.0.0.1/a.png")+ .await+ .is_err()+ );+ assert!(+ assert_public_http_url("http://169.254.169.254/latest/meta-data")+ .await+ .is_err()+ );+ assert!(+ assert_public_http_url("http://localhost/a.png")+ .await+ .is_err()+ );+ assert!(assert_public_http_url("https://[::1]/a.png").await.is_err());+ assert!(+ assert_public_http_url("https://10.0.0.1/a.png")+ .await+ .is_err()+ );+}++#[tokio::test]+async fn fetch_public_and_encode_does_not_fetch_private_urls() {+ assert!(+ fetch_public_and_encode("http://127.0.0.1/should-not-fetch.png")+ .await+ .is_none()+ );+ assert!(+ fetch_public_and_encode("file:///etc/passwd")+ .await+ .is_none()+ );+}++#[tokio::test]+async fn fetch_and_encode_rejects_oversized_content_length() {+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0")+ .await+ .expect("bind");+ let port = listener.local_addr().expect("addr").port();+ tokio::spawn(async move {+ let (mut stream, _) = listener.accept().await.expect("accept");+ use tokio::io::{AsyncReadExt, AsyncWriteExt};+ let mut buf = vec![0u8; 1024];+ let _ = stream.read(&mut buf).await;+ let header = format!(+ "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",+ MAX_IMAGE_BYTES + 1+ );+ let _ = stream.write_all(header.as_bytes()).await;+ });++ assert!(+ fetch_and_encode(&format!("http://127.0.0.1:{port}/big.png"))+ .await+ .is_none(),+ "oversized Content-Length must not be downloaded"+ );+}diff --git a/services/mcp_service/src/tool_service.rs b/services/mcp_service/src/tool_service.rsindex 8ab6da3874..740a2d99b3 100644--- a/services/mcp_service/src/tool_service.rs+++ b/services/mcp_service/src/tool_service.rs@@ -1,3 +1,4 @@+use crate::markdown_images::{MarkdownImageResolver, tool_result_with_images};use ai_toolset::{AsyncToolCollection, RequestContext, ToolSet};use macro_user_id::user_id::MacroUserIdStr;use rmcp::{@@ -84,7 +85,7 @@ mod test;impl<Context> ServerHandler for AuthenticatedToolService<Context>where- Context: Clone + Send + Sync + 'static,+ Context: Clone + Send + Sync + MarkdownImageResolver + 'static,{fn get_info(&self) -> ServerInfo {let mut info = ServerInfo::new(ServerCapabilities::builder().enable_tools().build());@@ -136,7 +137,7 @@ where) -> Result<rmcp::model::CallToolResult, rmcp::ErrorData> {let user_id = Self::authenticated_user_id(&context.extensions)?;- let request_context = RequestContext::new(user_id);+ let request_context = RequestContext::new(user_id.clone());let arguments = request.arguments@@ -162,7 +163,7 @@ where})?;match result {- Ok(value) => Ok(rmcp::model::CallToolResult::structured(value)),+ Ok(value) => Ok(tool_result_with_images(&self.context, &user_id, value).await),Err(error) => Ok(rmcp::model::CallToolResult::error(vec![Content::text(error.description,)])),