Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

fix(agent): preserve uploaded files across the user message text round trip

Steven Enamakel7e7f45627875
diff --git a/crates/openhuman-core/src/agent/message_convert.rs b/crates/openhuman-core/src/agent/message_convert.rs
index 1de8bef9dd..63c61cee18 100644
--- a/crates/openhuman-core/src/agent/message_convert.rs
+++ b/crates/openhuman-core/src/agent/message_convert.rs
@@ -145,15 +145,59 @@ pub(crate) fn user_message_from_text(text: &str) -> Message {
}
/// The inverse of [`user_message_from_text`]: a user message's text with each
-/// image block rendered back as an `[IMAGE:<url>]` marker. A text-only message
-/// is exactly [`Message::text`].
+/// uploaded file rendered back as its `[ATTACHMENT:…]` reference and each
+/// other image block as an `[IMAGE:<url>]` marker. A text-only message is
+/// exactly [`Message::text`].
+///
+/// An upload must come back as its reference, not as the display form of its
+/// block: a `[DOCUMENT:<path>]` marker lifts back to plain text, so a turn
+/// that renders and re-lifts its message would hand the model the path
+/// instead of the file.
pub(crate) fn user_text_with_markers(msg: &Message) -> String {
- match msg {
- Message::User(_) => tinyagents_runtime::transcript::message_to_native_transcript(msg)
- .expect("user messages have a native transcript projection")
- .display_content(),
- other => other.text(),
+ let Message::User(user) = msg else {
+ return msg.text();
+ };
+ let mut text = String::new();
+ let mut run = Vec::new();
+ let mut blocks = user.content.iter().peekable();
+ while let Some(block) = blocks.next() {
+ if let ContentBlock::Text(description) = block {
+ if let Some(upload) = blocks.peek().and_then(|next| uploaded(description, next)) {
+ text.push_str(&display_text(std::mem::take(&mut run)));
+ text.push_str(&upload.marker());
+ blocks.next();
+ continue;
+ }
+ }
+ run.push(block.clone());
+ }
+ text.push_str(&display_text(run));
+ text
+}
+
+/// The display text of user `content`: its text, image blocks as markers.
+fn display_text(content: Vec<ContentBlock>) -> String {
+ if content.is_empty() {
+ return String::new();
}
+ let message = Message::User(UserMessage { content });
+ tinyagents_runtime::transcript::message_to_native_transcript(&message)
+ .expect("user messages have a native transcript projection")
+ .display_content()
+}
+
+/// The upload a `description` block followed by `block` stands for: the
+/// pair [`user_message_from_text`] makes of one `[ATTACHMENT:…]` reference.
+fn uploaded(description: &str, block: &ContentBlock) -> Option<crate::attachments::Attachment> {
+ let path = match block {
+ ContentBlock::Image(image) => &image.url,
+ ContentBlock::Audio(MediaRef::Path { path, .. })
+ | ContentBlock::Video(MediaRef::Path { path, .. })
+ | ContentBlock::Document(MediaRef::Path { path, .. }) => path,
+ _ => return None,
+ };
+ crate::attachments::Attachment::from_description(description)
+ .filter(|upload| upload.path == *path)
}
/// Build the content blocks for a user turn, lifting any `[IMAGE:…]` markers out
diff --git a/crates/openhuman-core/src/agent/message_convert_tests.rs b/crates/openhuman-core/src/agent/message_convert_tests.rs
index 4492897ec2..7660fa7657 100644
--- a/crates/openhuman-core/src/agent/message_convert_tests.rs
+++ b/crates/openhuman-core/src/agent/message_convert_tests.rs
@@ -40,6 +40,55 @@ fn durable_upload_parts_survive_message_row_replay_without_inline_payloads() {
.any(|part| matches!(part, ContentBlock::Document(MediaRef::Path { .. }))));
assert!(!serde_json::to_string(&row).unwrap().contains("base64"));
}
+// A turn renders its user message to text and lifts it back after
+// enrichment, so every upload must come back as the same typed block, not as
+// a `[DOCUMENT:<path>]` display marker the provider wrapper never resolves.
+#[test]
+fn uploaded_files_survive_the_text_round_trip_a_turn_takes() {
+ let mut text = String::from("Current Date & Time: now\n\n");
+ for (name, mime) in [
+ ("Q3; final.pdf", "application/pdf"),
+ ("receipt.png", "image/png"),
+ ("clip.mp3", "audio/mpeg"),
+ ("video.mp4", "video/mp4"),
+ ] {
+ let attachment = crate::attachments::Attachment {
+ path: format!("uploads/thread/id/{name}"),
+ name: name.into(),
+ mime: mime.into(),
+ size_bytes: 600,
+ };
+ text.push_str(&attachment.marker());
+ text.push('\n');
+ }
+ text.push_str("summarise it");
+ let message = user_message_from_text(&text);
+
+ let rendered = user_text_with_markers(&message);
+
+ assert_eq!(rendered, text);
+ assert_eq!(user_message_from_text(&rendered), message);
+ let Message::User(user) = message else {
+ panic!("user message expected")
+ };
+ assert!(user.content.iter().any(|part| matches!(
+ part,
+ ContentBlock::Document(MediaRef::Path { path, .. }) if path == "uploads/thread/id/Q3; final.pdf"
+ )));
+}
+
+// Text that only looks like an upload's description, with no upload after
+// it, stays text: the round trip invents no attachment.
+#[test]
+fn a_typed_upload_description_without_its_file_stays_text() {
+ let text =
+ "[Uploaded file: a.pdf; application/pdf; 3 bytes; workspace path: uploads/t/i/a.pdf]";
+ let message = Message::User(UserMessage {
+ content: vec![ContentBlock::Text(text.into())],
+ });
+ assert_eq!(user_text_with_markers(&message), text);
+}
+
use tinyinference_llm::model::ModelRequest;
// #5359: a user turn whose text carries an inline `[IMAGE:data:…]` marker
diff --git a/crates/openhuman-core/src/attachments/mod.rs b/crates/openhuman-core/src/attachments/mod.rs
index 50aa36c312..b85859307f 100644
--- a/crates/openhuman-core/src/attachments/mod.rs
+++ b/crates/openhuman-core/src/attachments/mod.rs
@@ -92,6 +92,24 @@ impl Attachment {
self.name, self.mime, self.size_bytes, self.path
)
}
+
+ /// The attachment `text` is the [`description`](Self::description) of,
+ /// when it is exactly one. The name may hold anything, so the fixed
+ /// fields are read from the right and the result must describe itself
+ /// back byte for byte.
+ pub(crate) fn from_description(text: &str) -> Option<Self> {
+ let body = text.strip_prefix("[Uploaded file: ")?.strip_suffix(']')?;
+ let (head, path) = body.rsplit_once("; workspace path: ")?;
+ let (head, size) = head.strip_suffix(" bytes")?.rsplit_once("; ")?;
+ let (name, mime) = head.rsplit_once("; ")?;
+ let attachment = Self {
+ path: path.into(),
+ name: name.into(),
+ mime: mime.into(),
+ size_bytes: size.parse().ok()?,
+ };
+ (attachment.description() == text).then_some(attachment)
+ }
}
pub(crate) enum Segment {