Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

Merge remote-tracking branch 'upstream/main' into epic-7964-ci-green

Steven Enamakel878553364d3d
diff --cc crates/openhuman-core/src/agent/message_convert.rs
index 0aec7559bc,63c61cee18..e7ac6e4fa1
--- a/crates/openhuman-core/src/agent/message_convert.rs
+++ b/crates/openhuman-core/src/agent/message_convert.rs
@@@ -144,110 -144,60 +144,154 @@@ pub(crate) fn user_message_from_text(te
tinyagents_runtime::transcript::user_message_from_text(text)
}
+/// Build the user turn from marker-free persisted text and trusted media refs.
+/// Callers must resolve opaque refs against the exact persisted message under
+/// its DispatchCtx before passing parts here; paths on these internal parts are
+/// never accepted from the RPC caller.
+pub(crate) fn user_message_from_parts(
+ text: &str,
+ parts: &[tinychannels_bus::ContentPart],
+) -> Result<Message, String> {
+ let mut content = Vec::new();
+ if !text.is_empty() {
+ content.push(ContentBlock::Text(text.to_string()));
+ }
+ for part in parts {
+ match part {
+ tinychannels_bus::ContentPart::Text { text, .. } => {
+ if !text.is_empty() {
+ content.push(ContentBlock::Text(text.clone()));
+ }
+ }
+ tinychannels_bus::ContentPart::Image {
+ media,
+ caption,
+ alt_text,
+ } => {
+ let path = media
+ .path
+ .clone()
+ .ok_or_else(|| "resolved image attachment has no trusted path".to_string())?;
+ if let Some(caption) = caption.as_ref().filter(|text| !text.is_empty()) {
+ content.push(ContentBlock::Text(caption.clone()));
+ }
+ if let Some(alt) = alt_text.as_ref().filter(|text| !text.is_empty()) {
+ content.push(ContentBlock::Text(alt.clone()));
+ }
+ content.push(ContentBlock::Image(ImageRef {
+ url: path,
+ mime_type: media.content_type.clone(),
+ }));
+ }
+ tinychannels_bus::ContentPart::File { media, caption, .. } => {
+ let path = media
+ .path
+ .clone()
+ .ok_or_else(|| "resolved media attachment has no trusted path".to_string())?;
+ if let Some(caption) = caption.as_ref().filter(|text| !text.is_empty()) {
+ content.push(ContentBlock::Text(caption.clone()));
+ }
+ content.push(ContentBlock::Document(MediaRef::Path {
+ path,
+ media_type: media.content_type.clone(),
+ }));
+ }
+ tinychannels_bus::ContentPart::VoiceNote { voice_note } => {
+ let transcript = voice_note
+ .transcript
+ .as_ref()
+ .filter(|transcript| {
+ transcript.state == tinychannels_bus::VoiceTranscriptState::Complete
+ })
+ .map(|transcript| transcript.text.trim())
+ .filter(|text| !text.is_empty())
+ .ok_or_else(|| "voice transcript is unavailable".to_owned())?;
+ content.push(ContentBlock::Text(transcript.to_owned()));
+ }
+ tinychannels_bus::ContentPart::Audio { media, .. }
+ | tinychannels_bus::ContentPart::Video { media, .. } => {
+ let path = media
+ .path
+ .clone()
+ .ok_or_else(|| "resolved media attachment has no trusted path".to_string())?;
+ let media_ref = MediaRef::Path {
+ path,
+ media_type: media.content_type.clone(),
+ };
+ let block = match part {
+ tinychannels_bus::ContentPart::Audio { .. } => ContentBlock::Audio(media_ref),
+ _ => ContentBlock::Video(media_ref),
+ };
+ content.push(block);
+ }
+ tinychannels_bus::ContentPart::Sticker { .. }
+ | tinychannels_bus::ContentPart::Location { .. }
+ | tinychannels_bus::ContentPart::Contact { .. }
+ | tinychannels_bus::ContentPart::Buttons { .. }
+ | tinychannels_bus::ContentPart::Poll { .. }
+ | tinychannels_bus::ContentPart::Presentation { .. }
+ | tinychannels_bus::ContentPart::Native { .. } => {
+ return Err("resolved attachment has an unsupported kind".into());
+ }
+ }
+ }
+ Ok(Message::User(UserMessage { content }))
+}
+
/// The inverse of [`user_message_from_text`]: a user message's text with each
- /// image block rendered back as an `[IMAGE:<url>]` marker. A text-only message
- /// is exactly [`Message::text`].
+ /// uploaded file rendered back as its `[ATTACHMENT:…]` reference and each
+ /// other image block as an `[IMAGE:<url>]` marker. A text-only message is
+ /// exactly [`Message::text`].
+ ///
+ /// An upload must come back as its reference, not as the display form of its
+ /// block: a `[DOCUMENT:<path>]` marker lifts back to plain text, so a turn
+ /// that renders and re-lifts its message would hand the model the path
+ /// instead of the file.
pub(crate) fn user_text_with_markers(msg: &Message) -> String {
- match msg {
- Message::User(_) => tinyagents_runtime::transcript::message_to_native_transcript(msg)
- .expect("user messages have a native transcript projection")
- .display_content(),
- other => other.text(),
+ let Message::User(user) = msg else {
+ return msg.text();
+ };
+ let mut text = String::new();
+ let mut run = Vec::new();
+ let mut blocks = user.content.iter().peekable();
+ while let Some(block) = blocks.next() {
+ if let ContentBlock::Text(description) = block {
+ if let Some(upload) = blocks.peek().and_then(|next| uploaded(description, next)) {
+ text.push_str(&display_text(std::mem::take(&mut run)));
+ text.push_str(&upload.marker());
+ blocks.next();
+ continue;
+ }
+ }
+ run.push(block.clone());
+ }
+ text.push_str(&display_text(run));
+ text
+ }
+
+ /// The display text of user `content`: its text, image blocks as markers.
+ fn display_text(content: Vec<ContentBlock>) -> String {
+ if content.is_empty() {
+ return String::new();
}
+ let message = Message::User(UserMessage { content });
+ tinyagents_runtime::transcript::message_to_native_transcript(&message)
+ .expect("user messages have a native transcript projection")
+ .display_content()
+ }
+
+ /// The upload a `description` block followed by `block` stands for: the
+ /// pair [`user_message_from_text`] makes of one `[ATTACHMENT:…]` reference.
+ fn uploaded(description: &str, block: &ContentBlock) -> Option<crate::attachments::Attachment> {
+ let path = match block {
+ ContentBlock::Image(image) => &image.url,
+ ContentBlock::Audio(MediaRef::Path { path, .. })
+ | ContentBlock::Video(MediaRef::Path { path, .. })
+ | ContentBlock::Document(MediaRef::Path { path, .. }) => path,
+ _ => return None,
+ };
+ crate::attachments::Attachment::from_description(description)
+ .filter(|upload| upload.path == *path)
}
/// Build the content blocks for a user turn, lifting any `[IMAGE:…]` markers out