The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
use super::*;
use std::sync::Mutex;
#[derive(Default)]
pub(super) struct Probe {
pub(super) requests: Mutex<Vec<ModelRequest>>,
pub(super) profile: ModelProfile,
}
#[async_trait]
impl ChatModel<()> for Probe {
fn profile(&self) -> Option<&ModelProfile> {
Some(&self.profile)
}
fn supports_input(&self, m: InputModality, t: &str, s: InputSource) -> bool {
m == InputModality::Image
&& matches!(t, "image/png" | "image/jpeg")
&& s == InputSource::Base64
}
async fn invoke(
&self,
_: &(),
request: ModelRequest,
) -> tinyinference_llm::Result<ModelResponse> {
self.requests.lock().unwrap().push(request);
Err(tinyinference_llm::Error::Model("probe".into()))
}
}
#[tokio::test]
async fn local_native_resolution_changes_only_ephemeral_request() {
let temp = tempfile::tempdir().unwrap();
tokio::fs::write(temp.path().join("sample.png"), b"original-png")
.await
.unwrap();
let mut config = Config::default();
config.action_dir = temp.path().into();
config.workspace_dir = temp.path().join("internal");
let probe = Arc::new(Probe::default());
let wrapper = wrap(
&crate::core::runtime::DispatchCtx::host_default(),
probe.clone(),
Arc::new(config),
"gpt-4o",
"openai",
);
let request = ModelRequest::new(vec![Message::User(
tinyinference_llm::message::UserMessage {
content: vec![ContentBlock::Image(ImageRef {
url: "sample.png".into(),
mime_type: Some("image/png".into()),
})],
},
)]);
let durable = request.clone();
assert!(wrapper.invoke(&(), request).await.is_err());
assert!(serde_json::to_string(&durable)
.unwrap()
.contains("sample.png"));
let captured = probe.requests.lock().unwrap().clone();
assert_eq!(captured.len(), 1);
assert!(serde_json::to_string(&captured[0])
.unwrap()
.contains("data:image/png;base64,"));
assert_eq!(
tokio::fs::read(temp.path().join("sample.png"))
.await
.unwrap(),
b"original-png"
);
}
#[cfg(unix)]
#[tokio::test]
async fn blocked_attachment_fails_before_model_call() {
let mut config = Config::default();
config.multimodal_files.max_files = 0;
let probe = Arc::new(Probe::default());
let wrapper = wrap(
&crate::core::runtime::DispatchCtx::host_default(),
probe.clone(),
Arc::new(config),
"gpt-4o",
"openai",
);
let request = ModelRequest::new(vec![Message::User(
tinyinference_llm::message::UserMessage {
content: vec![ContentBlock::Image(ImageRef {
url: "../secret.png".into(),
mime_type: Some("image/png".into()),
})],
},
)]);
assert!(wrapper.invoke(&(), request).await.is_err());
assert!(probe.requests.lock().unwrap().is_empty());
}
#[test]
fn unknown_model_does_not_inherit_transport_vision() {
let model = wrap(
&crate::core::runtime::DispatchCtx::host_default(),
Arc::new(Probe::default()),
Arc::new(Config::default()),
"unknown-model",
"cloud",
);
assert!(!model.profile().unwrap().modalities.image_in);
}
#[tokio::test]
async fn persisted_image_without_mime_uses_magic_over_extension() {
let temp = tempfile::tempdir().unwrap();
tokio::fs::write(temp.path().join("renamed.png"), [0xff, 0xd8, 0xff, 0xe0])
.await
.unwrap();
let mut config = Config::default();
config.action_dir = temp.path().into();
let probe = Arc::new(Probe::default());
let wrapper = wrap(
&crate::core::runtime::DispatchCtx::host_default(),
probe.clone(),
Arc::new(config),
"gpt-4o",
"openai",
);
let request = ModelRequest::new(vec![Message::User(
tinyinference_llm::message::UserMessage {
content: vec![ContentBlock::Image(ImageRef {
url: "renamed.png".into(),
mime_type: None,
})],
},
)]);
assert!(wrapper.invoke(&(), request).await.is_err());
let captured = probe.requests.lock().unwrap().clone();
assert_eq!(captured.len(), 1);
assert!(serde_json::to_string(&captured[0])
.unwrap()
.contains("data:image/jpeg;base64,"));
}
#[tokio::test]
async fn too_many_typed_images_reject_before_read_or_inference() {
let probe = Arc::new(Probe::default());
let wrapper = wrap(
&crate::core::runtime::DispatchCtx::host_default(),
probe.clone(),
Arc::new(Config::default()),
"gpt-4o",
"openai",
);
let request = ModelRequest::new(vec![Message::User(
tinyinference_llm::message::UserMessage {
content: (0..5)
.map(|_| {
ContentBlock::Image(ImageRef {
url: "missing.png".into(),
mime_type: None,
})
})
.collect(),
},
)]);
let error = wrapper.invoke(&(), request).await.unwrap_err();
assert!(error.to_string().contains("attachment count"));
assert!(probe.requests.lock().unwrap().is_empty());
}
#[tokio::test]
async fn external_channel_cannot_rehydrate_a_local_image() {
use crate::security::turn_origin::AgentTurnOrigin;
let probe = Arc::new(Probe::default());
let wrapper = wrap(
&crate::core::runtime::DispatchCtx::host_default(),
probe.clone(),
Arc::new(Config::default()),
"gpt-4o",
"openai",
);
let mut request = ModelRequest::new(vec![Message::User(
tinyinference_llm::message::UserMessage {
content: vec![ContentBlock::Image(ImageRef {
url: "missing.png".into(),
mime_type: None,
})],
},
)]);
let origin = AgentTurnOrigin::ExternalChannel {
sender_name: None,
channel: "telegram".into(),
sender: None,
reply_target: "chat".into(),
message_id: "message".into(),
history_key: None,
channel_session: None,
};
let mut context = crate::agent::tinyagents::host::OpenHumanRunContext::new();
context.origin = Some(origin);
super::super::attach_request_scope(&mut request, &context);
let error = wrapper.invoke(&(), request).await.unwrap_err();
assert!(error.to_string().contains("external channel"));
assert!(probe.requests.lock().unwrap().is_empty());
}
#[tokio::test]
async fn unknown_video_retains_path_as_metadata_without_specialist() {
let temp = tempfile::tempdir().unwrap();
tokio::fs::write(temp.path().join("clip.mp4"), b"video-original")
.await
.unwrap();
let mut config = Config::default();
config.action_dir = temp.path().into();
let probe = Arc::new(Probe::default());
let wrapper = wrap(
&crate::core::runtime::DispatchCtx::host_default(),
probe.clone(),
Arc::new(config),
"unknown",
"custom",
);
let request = ModelRequest::new(vec![Message::User(
tinyinference_llm::message::UserMessage {
content: vec![ContentBlock::Video(MediaRef::Path {
path: "clip.mp4".into(),
media_type: Some("video/mp4".into()),
})],
},
)]);
assert!(wrapper.invoke(&(), request).await.is_err());
let captured = probe.requests.lock().unwrap().clone();
let text = captured[0].messages[0].text();
assert!(text.contains("clip.mp4"));
assert!(text.contains("size=\"14 B\""));
assert!(!serde_json::to_string(&captured[0])
.unwrap()
.contains("base64"));
}
#[tokio::test]
async fn derivative_cache_survives_model_recreation_and_invalidates_changed_content() {
let temp = tempfile::tempdir().unwrap();
let original = temp.path().join("uploads/thread/id/original.txt");
tokio::fs::create_dir_all(original.parent().unwrap())
.await
.unwrap();
tokio::fs::write(&original, b"original").await.unwrap();
let mut config = Config::default();
config.action_dir = temp.path().into();
let config = Arc::new(config);
let make = || AttachmentModel {
config: config.clone(),
core: None,
fallback_cache: Mutex::new(Default::default()),
};
let first = make();
let scope = super::super::AttachmentAccessScope::default();
let key = first.fallback_key("uploads/thread/id/original.txt", "text/plain", b"original");
first
.save_fallback(
"uploads/thread/id/original.txt",
&key,
"derived readout",
&scope,
)
.await;
let resumed = make();
assert_eq!(
resumed
.cached_fallback("uploads/thread/id/original.txt", &key, &scope)
.await
.as_deref(),
Some("derived readout")
);
let changed = first.fallback_key("uploads/thread/id/original.txt", "text/plain", b"modified");
assert!(resumed
.cached_fallback("uploads/thread/id/original.txt", &changed, &scope)
.await
.is_none());
assert_eq!(tokio::fs::read(original).await.unwrap(), b"original");
assert!(tokio::fs::read_dir(temp.path().join("uploads/thread/id"))
.await
.unwrap()
.next_entry()
.await
.unwrap()
.is_some());
}
#[tokio::test]
async fn cache_is_not_written_beside_unmanaged_local_files() {
let temp = tempfile::tempdir().unwrap();
tokio::fs::write(temp.path().join("original.txt"), b"original")
.await
.unwrap();
let mut config = Config::default();
config.action_dir = temp.path().into();
let model = AttachmentModel {
config: Arc::new(config),
core: None,
fallback_cache: Mutex::new(Default::default()),
};
model
.save_fallback(
"original.txt",
"key",
"derived",
&super::super::AttachmentAccessScope::default(),
)
.await;
assert!(!temp.path().join(".openhuman-intake.txt").exists());
}
#[test]
fn endpoint_discovery_denial_overrides_offline_vision_facts() {
use crate::config::schema::cloud_providers::{AuthStyle, CloudProviderCreds};
use tinyinference_llm::model::discover::{model_limits_cache, LimitSource, ModelLimits};
let temp = tempfile::tempdir().unwrap();
let mut config = Config::default();
config.config_path = temp.path().join("config.toml");
config.workspace_dir = temp.path().join("workspace");
config.cloud_providers = vec![CloudProviderCreds {
id: "p_attachment_probe".into(),
slug: "attachment-probe".into(),
label: "test".into(),
endpoint: "http://127.0.0.1:19991/v1".into(),
auth_style: AuthStyle::None,
..Default::default()
}];
let request = crate::inference::context_window::model_limits_request(
"chat",
"attachment-probe:gpt-4o",
"gpt-4o",
&config,
)
.unwrap();
model_limits_cache().insert_discovered_variant(
&request.endpoint,
&request.model,
&request.cache_variant(),
Some(ModelLimits {
context_window: None,
max_output_tokens: None,
input_modalities: Some(vec!["text".into()]),
source: LimitSource::ProviderListing,
}),
);
let model = wrap(
&crate::core::runtime::DispatchCtx::host_default(),
Arc::new(Probe::default()),
Arc::new(config),
"gpt-4o",
"attachment-probe",
);
assert!(!model.profile().unwrap().modalities.image_in);
}
#[tokio::test]
async fn legacy_inline_source_is_bounded_and_malformed_base64_rejects_before_primary() {
let probe = Arc::new(Probe::default());
let mut config = Config::default();
config.multimodal.max_image_size_mb = 1;
let wrapper = wrap(
&crate::core::runtime::DispatchCtx::host_default(),
probe.clone(),
Arc::new(config),
"gpt-4o",
"openai",
);
for source in [
"data:image/png;base64,%%%".to_owned(),
format!("data:image/png;base64,{}", "A".repeat(2 * 1024 * 1024)),
] {
let request = ModelRequest::new(vec![Message::User(
tinyinference_llm::message::UserMessage {
content: vec![ContentBlock::Image(ImageRef {
url: source,
mime_type: Some("image/png".into()),
})],
},
)]);
assert!(wrapper.invoke(&(), request).await.is_err());
}
assert!(probe.requests.lock().unwrap().is_empty());
}
#[tokio::test]
async fn disabled_remote_replay_is_rejected_before_inference() {
let probe = Arc::new(Probe::default());
let wrapper = wrap(
&crate::core::runtime::DispatchCtx::host_default(),
probe.clone(),
Arc::new(Config::default()),
"unknown",
"custom",
);
for block in [
ContentBlock::Image(ImageRef {
url: "https://example.invalid/image.png".into(),
mime_type: Some("image/png".into()),
}),
ContentBlock::Document(MediaRef::Url {
url: "https://example.invalid/file.pdf".into(),
media_type: Some("application/pdf".into()),
}),
] {
let request = ModelRequest::new(vec![Message::User(
tinyinference_llm::message::UserMessage {
content: vec![block],
},
)]);
let error = wrapper.invoke(&(), request).await.unwrap_err();
assert!(error
.to_string()
.contains("remote attachment fetch is disabled"));
}
assert!(probe.requests.lock().unwrap().is_empty());
}
#[tokio::test]
async fn inline_native_replay_materializes_original_and_keeps_durable_message_unchanged() {
let temp = tempfile::tempdir().unwrap();
let mut config = Config::default();
config.action_dir = temp.path().into();
let probe = Arc::new(Probe::default());
let wrapper = wrap(
&crate::core::runtime::DispatchCtx::host_default(),
probe.clone(),
Arc::new(config),
"gpt-4o",
"openai",
);
let bytes = b"\x89PNG\r\n\x1a\noriginal";
let request = ModelRequest::new(vec![Message::User(
tinyinference_llm::message::UserMessage {
content: vec![ContentBlock::Image(ImageRef {
url: format!("data:image/png;base64,{}", STANDARD.encode(bytes)),
mime_type: None,
})],
},
)]);
let durable = request.clone();
assert!(wrapper.invoke(&(), request).await.is_err());
let captured = probe.requests.lock().unwrap().clone();
assert_eq!(captured.len(), 1);
let text = captured[0].messages[0].text();
let path = text
.split("workspace path: ")
.nth(1)
.unwrap()
.trim_end_matches(']');
assert_eq!(
tokio::fs::read(temp.path().join(path)).await.unwrap(),
bytes
);
assert_eq!(durable.messages[0].text(), "");
}
#[tokio::test]
async fn derivative_cache_cannot_overwrite_original_named_like_cache() {
let temp = tempfile::tempdir().unwrap();
let mut config = Config::default();
config.action_dir = temp.path().into();
let source = format!(
"data:text/plain;name=.openhuman-intake.txt;base64,{}",
STANDARD.encode(b"original")
);
let staged = super::super::stage(
&format!("[FILE:{source}]"),
"thread",
&config,
&super::super::AttachmentAccessScope::default(),
)
.await
.unwrap();
let (_, files) = super::super::parse(&staged);
assert_eq!(files.len(), 1);
assert_eq!(files[0].name, ".openhuman-intake.txt");
let probe = Arc::new(Probe::default());
let wrapper = wrap(
&crate::core::runtime::DispatchCtx::host_default(),
probe.clone(),
Arc::new(config.clone()),
"unknown",
"custom",
);
let request = ModelRequest::new(vec![Message::User(
tinyinference_llm::message::UserMessage {
content: vec![ContentBlock::Document(MediaRef::Path {
path: files[0].path.clone(),
media_type: Some("text/plain".into()),
})],
},
)]);
assert!(wrapper.invoke(&(), request).await.is_err());
assert_eq!(
tokio::fs::read(temp.path().join(&files[0].path))
.await
.unwrap(),
b"original"
);
}
struct InlineVision {
profile: ModelProfile,
calls: std::sync::atomic::AtomicUsize,
}
#[async_trait]
impl ChatModel<()> for InlineVision {
fn profile(&self) -> Option<&ModelProfile> {
Some(&self.profile)
}
fn supports_input(&self, m: InputModality, mime: &str, source: InputSource) -> bool {
m == InputModality::Image && mime == "image/png" && source == InputSource::Base64
}
async fn invoke(
&self,
_: &(),
request: ModelRequest,
) -> tinyinference_llm::Result<ModelResponse> {
assert!(
matches!(&request.messages[0],Message::User(user) if user.content.iter().any(|b|matches!(b,ContentBlock::Image(_))))
);
self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Ok(ModelResponse::assistant("specialist image readout"))
}
}
#[tokio::test]
async fn unknown_model_legacy_inline_replay_uses_specialist_once_across_model_recreation() {
let temp = tempfile::tempdir().unwrap();
let mut config = Config::default();
config.action_dir = temp.path().into();
let config = Arc::new(config);
let mut profile = ModelProfile::default();
profile.provider = Some("injected".into());
profile.modalities.image_in = true;
let specialist = Arc::new(InlineVision {
profile,
calls: std::sync::atomic::AtomicUsize::new(0),
});
let _guard = crate::inference::provider::factory::test_provider_override::install_model(
specialist.clone(),
);
let request = ModelRequest::new(vec![Message::User(
tinyinference_llm::message::UserMessage {
content: vec![ContentBlock::Image(ImageRef {
url: format!(
"data:image/png;base64,{}",
STANDARD.encode(b"\x89PNG\r\n\x1a\noriginal")
),
mime_type: None,
})],
},
)]);
for _ in 0..2 {
let main = Arc::new(Probe::default());
let wrapper = wrap(
&crate::core::runtime::DispatchCtx::host_default(),
main.clone(),
config.clone(),
"unknown-model",
"custom",
);
assert!(wrapper.invoke(&(), request.clone()).await.is_err());
let captured = main.requests.lock().unwrap().clone();
assert_eq!(captured.len(), 1);
assert!(captured[0].messages[0]
.text()
.contains("specialist image readout"));
assert!(!serde_json::to_string(&captured[0])
.unwrap()
.contains("base64"));
}
assert_eq!(
specialist.calls.load(std::sync::atomic::Ordering::SeqCst),
1
);
}
#[tokio::test]
async fn legacy_audio_base64_unsupported_transport_becomes_workspace_reference() {
let temp = tempfile::tempdir().unwrap();
let mut config = Config::default();
config.action_dir = temp.path().into();
let main = Arc::new(Probe::default());
let wrapper = wrap(
&crate::core::runtime::DispatchCtx::host_default(),
main.clone(),
Arc::new(config),
"unknown",
"custom",
);
let request = ModelRequest::new(vec![Message::User(
tinyinference_llm::message::UserMessage {
content: vec![ContentBlock::Audio(MediaRef::Base64 {
data: STANDARD.encode(b"original-audio"),
media_type: "audio/wav".into(),
})],
},
)]);
assert!(wrapper.invoke(&(), request).await.is_err());
let captured = main.requests.lock().unwrap().clone();
assert_eq!(captured.len(), 1);
let text = captured[0].messages[0].text();
assert!(text.contains("audio/wav"));
assert!(text.contains("workspace path: uploads/legacy-media/"));
assert!(!serde_json::to_string(&captured[0])
.unwrap()
.contains("base64"));
}