Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

feat(session_host): add enriched typed user input handling

Steven Enamakel0953828200ee
diff --git a/crates/openhuman-core/src/agent/session_host/runtime_session.rs b/crates/openhuman-core/src/agent/session_host/runtime_session.rs
index 81f732bf5f..d4605ccda8 100644
--- a/crates/openhuman-core/src/agent/session_host/runtime_session.rs
+++ b/crates/openhuman-core/src/agent/session_host/runtime_session.rs
@@ -24,7 +24,7 @@ use tinyagents_runtime::{
CommitReceipt, ResumePreparation, SessionBuilder, SessionTerminal, ToolSnapshot,
TranscriptTarget, TurnPreparation,
};
-use tinyinference_llm::message::Message;
+use tinyinference_llm::message::{ContentBlock, Message, UserMessage};
use super::announcement_notes::{
integration_announcement_note, mcp_announcement_note, skill_announcement_note,
@@ -65,6 +65,41 @@ pub(super) struct OpenHumanSessionState {
pending_post_commit: Option<tokio::task::JoinHandle<()>>,
}
+/// Keep host-added context while retaining already-resolved, ordered media
+/// blocks on a trusted typed user input. The string enrichment path remains
+/// authoritative for text-only turns.
+fn enriched_typed_user_input(
+ original: &Message,
+ original_user_message: &str,
+ enriched: &str,
+) -> Result<Option<Message>, ()> {
+ let Message::User(user) = original else {
+ return Ok(None);
+ };
+ let has_typed_media = user.content.iter().any(|block| {
+ matches!(
+ block,
+ ContentBlock::Image(_)
+ | ContentBlock::Audio(_)
+ | ContentBlock::Video(_)
+ | ContentBlock::Document(_)
+ )
+ });
+ if !has_typed_media {
+ return Ok(None);
+ }
+ // `enrich_request` only prepends host context and the turn preamble to
+ // `original_user_message`. If that invariant changes, fail the turn rather
+ // than downgrade trusted typed media into a string.
+ let host_prefix = enriched.strip_suffix(original_user_message).ok_or(())?;
+ let mut content = Vec::with_capacity(user.content.len() + 1);
+ if !host_prefix.is_empty() {
+ content.push(ContentBlock::Text(host_prefix.to_owned()));
+ }
+ content.extend(user.content.iter().cloned());
+ Ok(Some(Message::User(UserMessage { content })))
+}
+
/// Owned host-only inputs used by the async runtime preparation hook.
/// It deliberately excludes transcript/history/tool snapshots, which belong to
/// `tinyagents_runtime::Session`.
@@ -839,7 +874,8 @@ impl OpenHumanSessionHost {
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.context_window = context_window;
- let original_user_message = user_text_with_markers(&request.input);
+ let original_input = request.input.clone();
+ let original_user_message = user_text_with_markers(&original_input);
prelude.begin_user_effects(request, &options.run_context.data);
let turn = state
.lock()
@@ -863,7 +899,19 @@ impl OpenHumanSessionHost {
)),
);
options.run_context.data.memory_turn = memory_turn;
- request.input = user_message_from_text(&enriched);
+ request.input = match enriched_typed_user_input(
+ &original_input,
+ &original_user_message,
+ &enriched,
+ ) {
+ Ok(Some(input)) => input,
+ Ok(None) => user_message_from_text(&enriched),
+ Err(()) => {
+ return Err(tinyagents_runtime::RuntimeError::Driver(
+ "host enrichment could not preserve typed user media".into(),
+ ));
+ }
+ };
let mut preparation = prelude
.prepare(turn::prepare_prefix_for_turn(
new_session,