Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

feat(agent): support file and voice note content in user messages

Steven Enamakel557f615f920a
diff --git a/crates/openhuman-core/src/agent/message_convert.rs b/crates/openhuman-core/src/agent/message_convert.rs
index 5c5b9aa778..f60ea59a13 100644
--- a/crates/openhuman-core/src/agent/message_convert.rs
+++ b/crates/openhuman-core/src/agent/message_convert.rs
@@ -190,6 +190,11 @@ pub(crate) fn user_message_from_parts(
}
for part in parts {
match part {
+ tinychannels_bus::ContentPart::Text { text, .. } => {
+ if !text.is_empty() {
+ content.push(ContentBlock::Text(text.clone()));
+ }
+ }
tinychannels_bus::ContentPart::Image {
media,
caption,
@@ -210,9 +215,54 @@ pub(crate) fn user_message_from_parts(
mime_type: media.content_type.clone(),
}));
}
+ tinychannels_bus::ContentPart::File {
+ media, caption, ..
+ } => {
+ let path = media
+ .path
+ .clone()
+ .ok_or_else(|| "resolved media attachment has no trusted path".to_string())?;
+ if let Some(caption) = caption.as_ref().filter(|text| !text.is_empty()) {
+ content.push(ContentBlock::Text(caption.clone()));
+ }
+ content.push(ContentBlock::Document(MediaRef::Path {
+ path,
+ media_type: media.content_type.clone(),
+ }));
+ }
+ tinychannels_bus::ContentPart::VoiceNote { voice_note } => {
+ let transcript = voice_note
+ .transcript
+ .as_ref()
+ .filter(|transcript| {
+ transcript.state == tinychannels_bus::VoiceTranscriptState::Complete
+ })
+ .map(|transcript| transcript.text.trim())
+ .filter(|text| !text.is_empty())
+ .ok_or_else(|| "voice transcript is unavailable".to_owned())?;
+ content.push(ContentBlock::Text(transcript.to_owned()));
+ }
+ tinychannels_bus::ContentPart::Audio { .. }
+ | tinychannels_bus::ContentPart::Video { .. } => {
+ return Err("resolved audio/video attachment is not supported".into());
+ }
+ tinychannels_bus::ContentPart::Sticker { .. }
+ | tinychannels_bus::ContentPart::Location { .. }
+ | tinychannels_bus::ContentPart::Contact { .. }
+ | tinychannels_bus::ContentPart::Buttons { .. }
+ | tinychannels_bus::ContentPart::Poll { .. }
+ | tinychannels_bus::ContentPart::Presentation { .. }
+ | tinychannels_bus::ContentPart::Native { .. } => {
+ return Err("resolved attachment has an unsupported kind".into());
+ }
+ }
+ }
+ Ok(Message::User(UserMessage { content }))
+}
+
+/*
tinychannels_bus::ContentPart::Audio { media, .. }
- | tinychannels_bus::ContentPart::Video { media, .. }
- | tinychannels_bus::ContentPart::File { media, .. } => {
+ | tinychannels_bus::ContentPart::Video { media, .. } => {
let path = media
.path
.clone()
@@ -232,6 +282,7 @@ pub(crate) fn user_message_from_parts(
}
}
Ok(Message::User(UserMessage { content }))
+*/
}
/// The inverse of [`user_message_from_text`]: a user message's text with each
diff --git a/crates/openhuman-core/src/channels/runtime/dispatch/processor/native_media.rs b/crates/openhuman-core/src/channels/runtime/dispatch/processor/native_media.rs
new file mode 100644
index 0000000000..8a7bcfe90a
--- /dev/null
+++ b/crates/openhuman-core/src/channels/runtime/dispatch/processor/native_media.rs
@@ -0,0 +1,328 @@
+//! Resolve typed native-channel media through TinyChannels' scoped media port.
+
+use std::sync::Arc;
+
+use tinyinference_llm::message::Message;
+use tinychannels_bus::{
+ module_media::{MediaReadBegin, MediaReadChunk, MediaReadFinish}, ChannelInboundEnvelope,
+ ContentPart, HostVoiceScope, MediaKind, MediaReference,
+};
+use tinybus::module::ModuleClient;
+
+use crate::{
+ channels::context::{module_client, ChannelRuntimeContext},
+ core::runtime::DispatchCtx,
+};
+
+const MAX_MEDIA_BYTES: usize = tinychannels_bus::module_media::MEDIA_UPLOAD_MAX_BYTES;
+
+pub(super) fn requires_typed_preparation(envelope: &ChannelInboundEnvelope) -> bool {
+ envelope.parts.iter().any(|part| {
+ matches!(
+ part,
+ ContentPart::Image { .. }
+ | ContentPart::File { .. }
+ | ContentPart::Audio { .. }
+ | ContentPart::Video { .. }
+ | ContentPart::VoiceNote { .. }
+ )
+ })
+}
+
+/// Resolve only opaque references carried by the trusted module envelope.
+/// Provider URLs and paths are never used as a filesystem or network source.
+pub(super) async fn resolve_typed_message(
+ ctx: &ChannelRuntimeContext,
+ envelope: &ChannelInboundEnvelope,
+ thread_id: &str,
+) -> Result<Option<Message>, String> {
+ if !requires_typed_preparation(envelope) {
+ return Ok(None);
+ }
+ let account = envelope
+ .channel
+ .account_id
+ .as_deref()
+ .filter(|account| !account.trim().is_empty())
+ .ok_or_else(|| "native media has no configured account scope".to_owned())?;
+ if envelope.channel.id.trim().is_empty()
+ || envelope.conversation.id.trim().is_empty()
+ || envelope.message_id.trim().is_empty()
+ {
+ return Err("native media has an incomplete conversation scope".into());
+ }
+ let config = ctx
+ .config
+ .as_deref()
+ .ok_or_else(|| "native media requires runtime configuration".to_owned())?;
+ let mut parts = envelope.parts.clone();
+ let mut image_count = 0usize;
+ let mut file_count = 0usize;
+ let (max_images, image_mb) = config.multimodal.effective_limits();
+ let (configured_files, file_mb, _) = config.multimodal_files.effective_limits();
+ let max_files = if config.multimodal_files.max_files == 0 {
+ 0
+ } else {
+ configured_files
+ };
+ let client = module_client(ctx).await;
+ let dispatch = ctx.dispatch_ctx();
+ let scope = HostVoiceScope {
+ configured_instance: account.to_owned(),
+ channel: envelope.channel.clone(),
+ conversation: envelope.conversation.clone(),
+ turn_id: envelope.message_id.clone(),
+ };
+
+ for part in &mut parts {
+ let (media, image, name) = match part {
+ ContentPart::Image { media, .. } => {
+ image_count += 1;
+ (media, true, media.file_name.clone().unwrap_or_else(|| "image".into()))
+ }
+ ContentPart::File {
+ media, file_name, ..
+ } => {
+ file_count += 1;
+ (
+ media,
+ false,
+ file_name
+ .clone()
+ .or_else(|| media.file_name.clone())
+ .unwrap_or_else(|| "attachment".into()),
+ )
+ }
+ ContentPart::Audio { .. } | ContentPart::Video { .. } => {
+ return Err("native audio/video attachments are not supported".into());
+ }
+ ContentPart::VoiceNote { voice_note } => {
+ let transcript = voice_note
+ .transcript
+ .as_ref()
+ .filter(|transcript| {
+ transcript.state == tinychannels_bus::VoiceTranscriptState::Complete
+ })
+ .map(|transcript| transcript.text.trim())
+ .filter(|text| !text.is_empty())
+ .ok_or_else(|| "native voice transcript is unavailable".to_owned())?;
+ let replacement = ContentPart::Text {
+ text: transcript.to_owned(),
+ markdown_dialect: None,
+ };
+ *part = replacement;
+ continue;
+ }
+ ContentPart::Text { .. } => continue,
+ _ => return Err("native channel content kind is not supported".into()),
+ };
+ let byte_limit = if image {
+ image_mb.saturating_mul(1024 * 1024).min(MAX_MEDIA_BYTES)
+ } else {
+ if max_files == 0 || file_count > max_files {
+ return Err("native file attachments are disabled or exceed their count limit".into());
+ }
+ file_mb.saturating_mul(1024 * 1024).min(MAX_MEDIA_BYTES)
+ };
+ if image_count > max_images {
+ return Err("native image attachments exceed their count limit".into());
+ }
+ validate_reference(media, image, byte_limit, &config.multimodal_files.allowed_mime_types)?;
+ let bytes = read_scoped_media(
+ &client,
+ &dispatch,
+ envelope.channel.id.as_str(),
+ &scope,
+ media,
+ byte_limit,
+ )
+ .await?;
+ let mime = media
+ .content_type
+ .as_deref()
+ .ok_or_else(|| "native media has no content type".to_owned())?;
+ let staged = crate::agent::attachments::store_scoped_channel_media(
+ config,
+ thread_id,
+ &name,
+ mime,
+ &bytes,
+ )
+ .await
+ .map_err(|_| "native media could not be staged safely".to_owned())?;
+ media.path = Some(staged.path);
+ media.url = None;
+ media.local_cache_path = None;
+ media.opaque_id = None;
+ media.content_type = Some(staged.mime);
+ media.size_bytes = Some(staged.size_bytes as u64);
+ }
+
+ let text = if parts
+ .iter()
+ .any(|part| matches!(part, ContentPart::Text { .. }))
+ {
+ String::new()
+ } else {
+ envelope.text.clone()
+ };
+ crate::agent::message_convert::user_message_from_parts(&text, &parts).map(Some)
+}
+
+fn validate_reference(
+ media: &MediaReference,
+ image: bool,
+ byte_limit: usize,
+ allowed_file_mimes: &[String],
+) -> Result<(), String> {
+ let opaque_id = media
+ .opaque_id
+ .as_deref()
+ .filter(|id| !id.trim().is_empty())
+ .ok_or_else(|| "native media reference is missing its opaque id".to_owned())?;
+ if opaque_id.len() > tinychannels_bus::module_media::MEDIA_UPLOAD_BEGIN_MAX_BYTES
+ || media.path.is_some()
+ || media.url.is_some()
+ || media.local_cache_path.is_some()
+ {
+ return Err("native media reference is not an opaque module reference".into());
+ }
+ if media.size_bytes.is_some_and(|size| size == 0 || size > byte_limit as u64) {
+ return Err("native media exceeds its declared byte limit".into());
+ }
+ let mime = media
+ .content_type
+ .as_deref()
+ .filter(|mime| !mime.trim().is_empty())
+ .ok_or_else(|| "native media has no content type".to_owned())?;
+ if image {
+ if !matches!(mime, "image/png" | "image/jpeg" | "image/webp" | "image/gif") {
+ return Err("native image MIME type is unsupported".into());
+ }
+ } else if !allowed_file_mimes.iter().any(|allowed| allowed == mime) {
+ return Err("native file MIME type is unsupported".into());
+ }
+ Ok(())
+}
+
+async fn read_scoped_media(
+ client: &ModuleClient,
+ dispatch: &DispatchCtx,
+ instance: &str,
+ scope: &HostVoiceScope,
+ media: &MediaReference,
+ max_bytes: usize,
+) -> Result<Vec<u8>, String> {
+ let begin = client
+ .call_confidential::<tinychannels_bus::MediaReadInfo>(
+ "tinychannels",
+ tinychannels_bus::methods::BEGIN_MEDIA_READ,
+ (MediaReadBegin {
+ instance: instance.to_owned(),
+ scope: scope.clone(),
+ media: media.clone(),
+ max_bytes: max_bytes as u64,
+ },),
+ )
+ .await
+ .map_err(|_| "native media could not be resolved in this turn".to_owned())?;
+ let mut lease = MediaReadLease::new(client.clone(), dispatch.clone(), begin.handle.clone());
+ if begin.byte_count == 0
+ || begin.byte_count > max_bytes
+ || begin.byte_count > MAX_MEDIA_BYTES
+ || begin.media.opaque_id != media.opaque_id
+ || begin.content_type != media.content_type.as_deref().unwrap_or_default()
+ || media.size_bytes.is_some_and(|size| size != begin.byte_count as u64)
+ {
+ return Err("native media read metadata does not match its scoped reference".into());
+ }
+ let expected_chunks = begin.byte_count.div_ceil(tinychannels_bus::module_media::MEDIA_READ_CHUNK_BYTES);
+ if begin.chunk_count as usize != expected_chunks {
+ return Err("native media read chunk count is invalid".into());
+ }
+ let mut bytes = Vec::with_capacity(begin.byte_count);
+ for sequence in 0..begin.chunk_count {
+ let chunk = client
+ .call_confidential::<tinychannels_bus::MediaReadChunkResult>(
+ "tinychannels",
+ tinychannels_bus::methods::READ_MEDIA_CHUNK,
+ (MediaReadChunk {
+ handle: begin.handle.clone(),
+ sequence,
+ },),
+ )
+ .await
+ .map_err(|_| "native media read was interrupted".to_owned())?;
+ let expected_done = sequence + 1 == begin.chunk_count;
+ if chunk.sequence != sequence || chunk.done != expected_done {
+ return Err("native media read returned an invalid chunk sequence".into());
+ }
+ if bytes.len().saturating_add(chunk.bytes.len()) > begin.byte_count
+ || bytes.len().saturating_add(chunk.bytes.len()) > max_bytes
+ {
+ return Err("native media bytes exceed their declared limit".into());
+ }
+ bytes.extend(chunk.bytes);
+ }
+ if bytes.len() != begin.byte_count {
+ return Err("native media bytes do not match their declared size".into());
+ }
+ lease.abort().await?;
+ Ok(bytes)
+}
+
+struct MediaReadLease {
+ client: ModuleClient,
+ dispatch: DispatchCtx,
+ handle: Option<tinychannels_bus::MediaReadHandle>,
+}
+
+impl MediaReadLease {
+ fn new(
+ client: ModuleClient,
+ dispatch: DispatchCtx,
+ handle: tinychannels_bus::MediaReadHandle,
+ ) -> Self {
+ Self {
+ client,
+ dispatch,
+ handle: Some(handle),
+ }
+ }
+
+ async fn abort(&mut self) -> Result<(), String> {
+ let Some(handle) = self.handle.take() else {
+ return Ok(());
+ };
+ self.client
+ .call_confidential::<()>(
+ "tinychannels",
+ tinychannels_bus::methods::ABORT_MEDIA_READ,
+ (MediaReadFinish(handle),),
+ )
+ .await
+ .map_err(|_| "native media read cleanup failed".to_owned())
+ }
+}
+
+impl Drop for MediaReadLease {
+ fn drop(&mut self) {
+ let Some(handle) = self.handle.take() else {
+ return;
+ };
+ let client = self.client.clone();
+ self.dispatch.spawn(async move {
+ let _ = client
+ .call_confidential::<()>(
+ "tinychannels",
+ tinychannels_bus::methods::ABORT_MEDIA_READ,
+ (MediaReadFinish(handle),),
+ )
+ .await;
+ });
+ }
+}
+
+#[cfg(test)]
+#[path = "native_media_tests.rs"]
+mod tests;