The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
feat(agent): support file and voice note content in user messages
Steven Enamakel557f615f920a
diff --git a/crates/openhuman-core/src/agent/message_convert.rs b/crates/openhuman-core/src/agent/message_convert.rsindex 5c5b9aa778..f60ea59a13 100644--- a/crates/openhuman-core/src/agent/message_convert.rs+++ b/crates/openhuman-core/src/agent/message_convert.rs@@ -190,6 +190,11 @@ pub(crate) fn user_message_from_parts(}for part in parts {match part {+ tinychannels_bus::ContentPart::Text { text, .. } => {+ if !text.is_empty() {+ content.push(ContentBlock::Text(text.clone()));+ }+ }tinychannels_bus::ContentPart::Image {media,caption,@@ -210,9 +215,54 @@ pub(crate) fn user_message_from_parts(mime_type: media.content_type.clone(),}));}+ tinychannels_bus::ContentPart::File {+ media, caption, ..+ } => {+ let path = media+ .path+ .clone()+ .ok_or_else(|| "resolved media attachment has no trusted path".to_string())?;+ if let Some(caption) = caption.as_ref().filter(|text| !text.is_empty()) {+ content.push(ContentBlock::Text(caption.clone()));+ }+ content.push(ContentBlock::Document(MediaRef::Path {+ path,+ media_type: media.content_type.clone(),+ }));+ }+ tinychannels_bus::ContentPart::VoiceNote { voice_note } => {+ let transcript = voice_note+ .transcript+ .as_ref()+ .filter(|transcript| {+ transcript.state == tinychannels_bus::VoiceTranscriptState::Complete+ })+ .map(|transcript| transcript.text.trim())+ .filter(|text| !text.is_empty())+ .ok_or_else(|| "voice transcript is unavailable".to_owned())?;+ content.push(ContentBlock::Text(transcript.to_owned()));+ }+ tinychannels_bus::ContentPart::Audio { .. }+ | tinychannels_bus::ContentPart::Video { .. } => {+ return Err("resolved audio/video attachment is not supported".into());+ }+ tinychannels_bus::ContentPart::Sticker { .. }+ | tinychannels_bus::ContentPart::Location { .. }+ | tinychannels_bus::ContentPart::Contact { .. }+ | tinychannels_bus::ContentPart::Buttons { .. }+ | tinychannels_bus::ContentPart::Poll { .. }+ | tinychannels_bus::ContentPart::Presentation { .. }+ | tinychannels_bus::ContentPart::Native { .. } => {+ return Err("resolved attachment has an unsupported kind".into());+ }+ }+ }+ Ok(Message::User(UserMessage { content }))+}++/*tinychannels_bus::ContentPart::Audio { media, .. }- | tinychannels_bus::ContentPart::Video { media, .. }- | tinychannels_bus::ContentPart::File { media, .. } => {+ | tinychannels_bus::ContentPart::Video { media, .. } => {let path = media.path.clone()@@ -232,6 +282,7 @@ pub(crate) fn user_message_from_parts(}}Ok(Message::User(UserMessage { content }))+*/}/// The inverse of [`user_message_from_text`]: a user message's text with eachdiff --git a/crates/openhuman-core/src/channels/runtime/dispatch/processor/native_media.rs b/crates/openhuman-core/src/channels/runtime/dispatch/processor/native_media.rsnew file mode 100644index 0000000000..8a7bcfe90a--- /dev/null+++ b/crates/openhuman-core/src/channels/runtime/dispatch/processor/native_media.rs@@ -0,0 +1,328 @@+//! Resolve typed native-channel media through TinyChannels' scoped media port.++use std::sync::Arc;++use tinyinference_llm::message::Message;+use tinychannels_bus::{+ module_media::{MediaReadBegin, MediaReadChunk, MediaReadFinish}, ChannelInboundEnvelope,+ ContentPart, HostVoiceScope, MediaKind, MediaReference,+};+use tinybus::module::ModuleClient;++use crate::{+ channels::context::{module_client, ChannelRuntimeContext},+ core::runtime::DispatchCtx,+};++const MAX_MEDIA_BYTES: usize = tinychannels_bus::module_media::MEDIA_UPLOAD_MAX_BYTES;++pub(super) fn requires_typed_preparation(envelope: &ChannelInboundEnvelope) -> bool {+ envelope.parts.iter().any(|part| {+ matches!(+ part,+ ContentPart::Image { .. }+ | ContentPart::File { .. }+ | ContentPart::Audio { .. }+ | ContentPart::Video { .. }+ | ContentPart::VoiceNote { .. }+ )+ })+}++/// Resolve only opaque references carried by the trusted module envelope.+/// Provider URLs and paths are never used as a filesystem or network source.+pub(super) async fn resolve_typed_message(+ ctx: &ChannelRuntimeContext,+ envelope: &ChannelInboundEnvelope,+ thread_id: &str,+) -> Result<Option<Message>, String> {+ if !requires_typed_preparation(envelope) {+ return Ok(None);+ }+ let account = envelope+ .channel+ .account_id+ .as_deref()+ .filter(|account| !account.trim().is_empty())+ .ok_or_else(|| "native media has no configured account scope".to_owned())?;+ if envelope.channel.id.trim().is_empty()+ || envelope.conversation.id.trim().is_empty()+ || envelope.message_id.trim().is_empty()+ {+ return Err("native media has an incomplete conversation scope".into());+ }+ let config = ctx+ .config+ .as_deref()+ .ok_or_else(|| "native media requires runtime configuration".to_owned())?;+ let mut parts = envelope.parts.clone();+ let mut image_count = 0usize;+ let mut file_count = 0usize;+ let (max_images, image_mb) = config.multimodal.effective_limits();+ let (configured_files, file_mb, _) = config.multimodal_files.effective_limits();+ let max_files = if config.multimodal_files.max_files == 0 {+ 0+ } else {+ configured_files+ };+ let client = module_client(ctx).await;+ let dispatch = ctx.dispatch_ctx();+ let scope = HostVoiceScope {+ configured_instance: account.to_owned(),+ channel: envelope.channel.clone(),+ conversation: envelope.conversation.clone(),+ turn_id: envelope.message_id.clone(),+ };++ for part in &mut parts {+ let (media, image, name) = match part {+ ContentPart::Image { media, .. } => {+ image_count += 1;+ (media, true, media.file_name.clone().unwrap_or_else(|| "image".into()))+ }+ ContentPart::File {+ media, file_name, ..+ } => {+ file_count += 1;+ (+ media,+ false,+ file_name+ .clone()+ .or_else(|| media.file_name.clone())+ .unwrap_or_else(|| "attachment".into()),+ )+ }+ ContentPart::Audio { .. } | ContentPart::Video { .. } => {+ return Err("native audio/video attachments are not supported".into());+ }+ ContentPart::VoiceNote { voice_note } => {+ let transcript = voice_note+ .transcript+ .as_ref()+ .filter(|transcript| {+ transcript.state == tinychannels_bus::VoiceTranscriptState::Complete+ })+ .map(|transcript| transcript.text.trim())+ .filter(|text| !text.is_empty())+ .ok_or_else(|| "native voice transcript is unavailable".to_owned())?;+ let replacement = ContentPart::Text {+ text: transcript.to_owned(),+ markdown_dialect: None,+ };+ *part = replacement;+ continue;+ }+ ContentPart::Text { .. } => continue,+ _ => return Err("native channel content kind is not supported".into()),+ };+ let byte_limit = if image {+ image_mb.saturating_mul(1024 * 1024).min(MAX_MEDIA_BYTES)+ } else {+ if max_files == 0 || file_count > max_files {+ return Err("native file attachments are disabled or exceed their count limit".into());+ }+ file_mb.saturating_mul(1024 * 1024).min(MAX_MEDIA_BYTES)+ };+ if image_count > max_images {+ return Err("native image attachments exceed their count limit".into());+ }+ validate_reference(media, image, byte_limit, &config.multimodal_files.allowed_mime_types)?;+ let bytes = read_scoped_media(+ &client,+ &dispatch,+ envelope.channel.id.as_str(),+ &scope,+ media,+ byte_limit,+ )+ .await?;+ let mime = media+ .content_type+ .as_deref()+ .ok_or_else(|| "native media has no content type".to_owned())?;+ let staged = crate::agent::attachments::store_scoped_channel_media(+ config,+ thread_id,+ &name,+ mime,+ &bytes,+ )+ .await+ .map_err(|_| "native media could not be staged safely".to_owned())?;+ media.path = Some(staged.path);+ media.url = None;+ media.local_cache_path = None;+ media.opaque_id = None;+ media.content_type = Some(staged.mime);+ media.size_bytes = Some(staged.size_bytes as u64);+ }++ let text = if parts+ .iter()+ .any(|part| matches!(part, ContentPart::Text { .. }))+ {+ String::new()+ } else {+ envelope.text.clone()+ };+ crate::agent::message_convert::user_message_from_parts(&text, &parts).map(Some)+}++fn validate_reference(+ media: &MediaReference,+ image: bool,+ byte_limit: usize,+ allowed_file_mimes: &[String],+) -> Result<(), String> {+ let opaque_id = media+ .opaque_id+ .as_deref()+ .filter(|id| !id.trim().is_empty())+ .ok_or_else(|| "native media reference is missing its opaque id".to_owned())?;+ if opaque_id.len() > tinychannels_bus::module_media::MEDIA_UPLOAD_BEGIN_MAX_BYTES+ || media.path.is_some()+ || media.url.is_some()+ || media.local_cache_path.is_some()+ {+ return Err("native media reference is not an opaque module reference".into());+ }+ if media.size_bytes.is_some_and(|size| size == 0 || size > byte_limit as u64) {+ return Err("native media exceeds its declared byte limit".into());+ }+ let mime = media+ .content_type+ .as_deref()+ .filter(|mime| !mime.trim().is_empty())+ .ok_or_else(|| "native media has no content type".to_owned())?;+ if image {+ if !matches!(mime, "image/png" | "image/jpeg" | "image/webp" | "image/gif") {+ return Err("native image MIME type is unsupported".into());+ }+ } else if !allowed_file_mimes.iter().any(|allowed| allowed == mime) {+ return Err("native file MIME type is unsupported".into());+ }+ Ok(())+}++async fn read_scoped_media(+ client: &ModuleClient,+ dispatch: &DispatchCtx,+ instance: &str,+ scope: &HostVoiceScope,+ media: &MediaReference,+ max_bytes: usize,+) -> Result<Vec<u8>, String> {+ let begin = client+ .call_confidential::<tinychannels_bus::MediaReadInfo>(+ "tinychannels",+ tinychannels_bus::methods::BEGIN_MEDIA_READ,+ (MediaReadBegin {+ instance: instance.to_owned(),+ scope: scope.clone(),+ media: media.clone(),+ max_bytes: max_bytes as u64,+ },),+ )+ .await+ .map_err(|_| "native media could not be resolved in this turn".to_owned())?;+ let mut lease = MediaReadLease::new(client.clone(), dispatch.clone(), begin.handle.clone());+ if begin.byte_count == 0+ || begin.byte_count > max_bytes+ || begin.byte_count > MAX_MEDIA_BYTES+ || begin.media.opaque_id != media.opaque_id+ || begin.content_type != media.content_type.as_deref().unwrap_or_default()+ || media.size_bytes.is_some_and(|size| size != begin.byte_count as u64)+ {+ return Err("native media read metadata does not match its scoped reference".into());+ }+ let expected_chunks = begin.byte_count.div_ceil(tinychannels_bus::module_media::MEDIA_READ_CHUNK_BYTES);+ if begin.chunk_count as usize != expected_chunks {+ return Err("native media read chunk count is invalid".into());+ }+ let mut bytes = Vec::with_capacity(begin.byte_count);+ for sequence in 0..begin.chunk_count {+ let chunk = client+ .call_confidential::<tinychannels_bus::MediaReadChunkResult>(+ "tinychannels",+ tinychannels_bus::methods::READ_MEDIA_CHUNK,+ (MediaReadChunk {+ handle: begin.handle.clone(),+ sequence,+ },),+ )+ .await+ .map_err(|_| "native media read was interrupted".to_owned())?;+ let expected_done = sequence + 1 == begin.chunk_count;+ if chunk.sequence != sequence || chunk.done != expected_done {+ return Err("native media read returned an invalid chunk sequence".into());+ }+ if bytes.len().saturating_add(chunk.bytes.len()) > begin.byte_count+ || bytes.len().saturating_add(chunk.bytes.len()) > max_bytes+ {+ return Err("native media bytes exceed their declared limit".into());+ }+ bytes.extend(chunk.bytes);+ }+ if bytes.len() != begin.byte_count {+ return Err("native media bytes do not match their declared size".into());+ }+ lease.abort().await?;+ Ok(bytes)+}++struct MediaReadLease {+ client: ModuleClient,+ dispatch: DispatchCtx,+ handle: Option<tinychannels_bus::MediaReadHandle>,+}++impl MediaReadLease {+ fn new(+ client: ModuleClient,+ dispatch: DispatchCtx,+ handle: tinychannels_bus::MediaReadHandle,+ ) -> Self {+ Self {+ client,+ dispatch,+ handle: Some(handle),+ }+ }++ async fn abort(&mut self) -> Result<(), String> {+ let Some(handle) = self.handle.take() else {+ return Ok(());+ };+ self.client+ .call_confidential::<()>(+ "tinychannels",+ tinychannels_bus::methods::ABORT_MEDIA_READ,+ (MediaReadFinish(handle),),+ )+ .await+ .map_err(|_| "native media read cleanup failed".to_owned())+ }+}++impl Drop for MediaReadLease {+ fn drop(&mut self) {+ let Some(handle) = self.handle.take() else {+ return;+ };+ let client = self.client.clone();+ self.dispatch.spawn(async move {+ let _ = client+ .call_confidential::<()>(+ "tinychannels",+ tinychannels_bus::methods::ABORT_MEDIA_READ,+ (MediaReadFinish(handle),),+ )+ .await;+ });+ }+}++#[cfg(test)]+#[path = "native_media_tests.rs"]+mod tests;