The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
use super::*;
use crate::inference::context_window::model_limits_request;
use crate::inference::provider::factory::{
create_chat_model_from_string, create_chat_model_with_identity_in_for_inspection,
make_openhuman_backend_model_for_thread_in, PROVIDER_OPENHUMAN,
};
use async_trait::async_trait;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Mutex;
use tinyinference_llm::model::discover::DiscoveryRequest;
use tinyinference_llm::model::ChatModel;
use tinyinference_llm::model::{ModelRequest, ModelResponse};
use tinyinference_llm::providers::overrides::CatalogRequest;
use tinyinference_llm::providers::overrides::{
CatalogEntry, ChatModelEntry, ProviderModelMetadata,
};
type SeenModelRequests = Arc<Mutex<Vec<(String, Option<String>, Option<String>)>>>;
struct FakeModel;
#[async_trait]
impl ChatModel<()> for FakeModel {
async fn invoke(
&self,
_state: &(),
_request: ModelRequest,
) -> tinyinference_llm::Result<ModelResponse> {
Ok(ModelResponse::assistant("from the registered model"))
}
}
fn fake_entry(seen: SeenModelRequests) -> ChatModelEntry<ProviderContext> {
ChatModelEntry {
build: Arc::new(move |request| {
seen.lock().unwrap().push((
request.role.to_string(),
request.model_override.map(str::to_string),
request.thread_id.map(str::to_string),
));
Ok((
Arc::new(FakeModel) as Arc<dyn ChatModel<()>>,
"fake-model".to_string(),
))
}),
probe: None,
models: Some(ProviderModelMetadata {
default_model: Arc::new(|| Some("host/fake-default".to_string())),
role_for_model: Arc::new(|model| {
(model == "hint:reasoning").then(|| "reasoning".to_string())
}),
is_alias: Arc::new(|model| model == "hint:reasoning"),
is_raw_model: Arc::new(|model| model != "hint:reasoning"),
supports_vision: Arc::new(|_| false),
resolve_model: Arc::new(|_, config, model| match model {
Some("hint:reasoning") => Some("host/resolved-reasoning".to_string()),
Some(model) if !model.starts_with("hint:") => Some(model.to_string()),
Some(_) => None,
None => config
.config
.default_model
.clone()
.or_else(|| Some("host/fake-default".to_string())),
}),
context_window: Arc::new(|_| None),
default_pricing: Arc::new(|| None),
without_reasoning: Arc::new(|request| request),
}),
}
}
#[test]
fn nothing_registered_resolves_no_provider() {
let _guard = crate::inference::inference_test_guard();
assert!(chat_model(PROVIDER_OPENHUMAN).is_none());
assert!(embedding(PROVIDER_OPENHUMAN).is_none());
assert!(catalog(PROVIDER_OPENHUMAN).is_none());
}
#[tokio::test]
async fn a_registered_chat_model_serves_the_managed_provider_for_a_thread() {
let _guard = crate::inference::inference_test_guard_async().await;
let _exclusive = crate::inference::provider::factory::test_provider_override::exclusive();
let seen = Arc::default();
register_chat_model(PROVIDER_OPENHUMAN, fake_entry(Arc::clone(&seen)));
let built = make_openhuman_backend_model_for_thread_in(
None,
"chat",
&Config::default(),
Some("hint:reasoning"),
true,
Some("thread-1"),
&EgressLedger::none(),
);
let library = crate::core::runtime::CoreContext::for_test_with_host_kind(
crate::core::types::HostKind::Library,
crate::core::runtime::DomainSet::full(),
None,
);
let one_shot = crate::core::runtime::CoreContext::scope(library, async {
create_chat_model_from_string("chat", PROVIDER_OPENHUMAN, &Config::default(), 0.5)
})
.await;
unregister_chat_model(PROVIDER_OPENHUMAN);
let (model, resolved) = built.expect("registered model builds");
assert_eq!(resolved, "fake-model");
let reply = model
.invoke(&(), ModelRequest::new(vec![]))
.await
.expect("invoke");
assert_eq!(reply.text(), "from the registered model");
assert!(one_shot.is_ok());
let seen = seen.lock().unwrap();
assert_eq!(
seen[0],
(
"chat".to_string(),
Some("host/resolved-reasoning".to_string()),
Some("thread-1".to_string())
)
);
assert_eq!(seen[1].1.as_deref(), Some("host/fake-default"));
assert_eq!(
seen.len(),
2,
"both construction paths consulted the registry"
);
}
#[test]
fn a_registered_embedding_factory_builds_the_managed_embedding_model() {
let _guard = crate::inference::inference_test_guard();
let calls = Arc::new(AtomicUsize::new(0));
let counter = Arc::clone(&calls);
register_embedding(
PROVIDER_OPENHUMAN,
Arc::new(move |request| {
assert_eq!(request.dimensions, 8);
counter.fetch_add(1, Ordering::SeqCst);
Ok(Box::new(tinyinference_embeddings::NoopEmbeddingModel))
}),
);
let built = crate::inference::embedding_host::create_embedding_provider_with_credentials(
"cloud",
"any-model",
8,
"",
None,
);
unregister_embedding(PROVIDER_OPENHUMAN);
assert!(built.is_ok());
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn a_registered_catalog_answers_the_model_listing_for_its_provider() {
register_catalog(
"fake-catalog",
CatalogEntry {
list: Arc::new(|_| {
Box::pin(async { Ok(serde_json::json!({ "models": [{ "id": "registered" }] })) })
}),
limits: None,
},
);
let listing =
crate::inference::provider::models::catalog_listing::list_configured_models_from_config(
"fake-catalog",
&Config::default(),
)
.await;
unregister_catalog("fake-catalog");
assert_eq!(listing.unwrap().value["models"][0]["id"], "registered");
}
#[tokio::test]
async fn without_a_catalog_an_unknown_provider_still_errors() {
let error =
crate::inference::provider::models::catalog_listing::list_configured_models_from_config(
"fake-catalog",
&Config::default(),
)
.await
.unwrap_err();
assert!(error.contains("no cloud provider"));
}
#[test]
fn a_registered_catalog_supplies_the_managed_limits_request() {
register_catalog(
PROVIDER_OPENHUMAN,
CatalogEntry {
list: Arc::new(|_| Box::pin(async { Err("unused".to_string()) })),
limits: Some(Arc::new(|model, _| {
Some(DiscoveryRequest::new("https://catalog.example/v1", model))
})),
},
);
let request =
model_limits_request("chat", PROVIDER_OPENHUMAN, "some-model", &Config::default());
unregister_catalog(PROVIDER_OPENHUMAN);
assert!(request.is_some());
}
/// With nothing registered under the managed provider id, selecting it yields
/// the `BACKEND_UNAVAILABLE:` sentinel instead of a panic or a hosted model.
#[test]
fn selecting_the_managed_provider_with_nothing_registered_is_backend_unavailable() {
let _guard = crate::inference::inference_test_guard();
let _exclusive = crate::inference::provider::factory::test_provider_override::exclusive();
let error = make_openhuman_backend_model_for_thread_in(
None,
"chat",
&Config::default(),
Some("any-model"),
true,
None,
&EgressLedger::none(),
)
.err()
.expect("no managed provider is registered");
assert!(
error
.to_string()
.starts_with(crate::core::observability::BACKEND_UNAVAILABLE_PREFIX),
"unexpected error: {error}"
);
let one_shot =
create_chat_model_from_string("chat", PROVIDER_OPENHUMAN, &Config::default(), 0.5);
assert!(one_shot.is_err());
}
/// A configured entry that authenticates with the managed session must hand a
/// registered catalog the key stored for that entry, so a signed-out user with
/// their own key still gets a listing.
#[tokio::test]
async fn a_registered_catalog_receives_the_entrys_provider_key() {
use crate::config::schema::cloud_providers::{AuthStyle, CloudProviderCreds};
use crate::config::schema::{EphemeralRoute, EPHEMERAL_ROUTE_SLUG};
let seen = Arc::new(Mutex::new(String::new()));
crate::config::schema::cloud_providers::register_hosted_cloud_provider_policy(
Arc::new(|slug| slug == EPHEMERAL_ROUTE_SLUG),
Arc::new(|style| style == AuthStyle::OpenhumanJwt),
);
let sink = Arc::clone(&seen);
register_catalog(
PROVIDER_OPENHUMAN,
CatalogEntry {
list: Arc::new(move |request: CatalogRequest<ProviderContext>| {
*sink.lock().unwrap() = request
.context
.catalog
.as_ref()
.unwrap()
.request
.provider_key
.clone();
Box::pin(async { Ok(serde_json::json!({ "models": [] })) })
}),
limits: None,
},
);
let config = Config {
cloud_providers: vec![CloudProviderCreds {
id: "p_ephemeral".into(),
slug: EPHEMERAL_ROUTE_SLUG.into(),
endpoint: "https://self-hosted.example/v1".into(),
auth_style: AuthStyle::OpenhumanJwt,
..Default::default()
}],
ephemeral_route: Some(EphemeralRoute {
endpoint: "https://self-hosted.example/v1".into(),
api_key: "entry-key".into(),
headers: vec![],
..EphemeralRoute::default()
}),
..Config::default()
};
let listing =
crate::inference::provider::models::catalog_listing::list_configured_models_from_config(
"p_ephemeral",
&config,
)
.await;
unregister_catalog(PROVIDER_OPENHUMAN);
listing.unwrap();
assert_eq!(seen.lock().unwrap().as_str(), "entry-key");
}
#[test]
fn inspection_model_factory_uses_the_callers_context_before_process_fallback() {
use crate::core::slots::Extensions;
use crate::inference::registry::{
register_inspection_chat_model, register_inspection_chat_model_in,
unregister_inspection_chat_model, InspectionChatModelBuild,
PreparedInspectionChatModelFactory,
};
let _guard = crate::inference::inference_test_guard();
let factory = |endpoint: &'static str| -> PreparedInspectionChatModelFactory {
Arc::new(move |request| {
Ok(InspectionChatModelBuild {
model: Arc::new(FakeModel),
model_id: format!(
"{endpoint}:{}",
request.context.config.default_model.as_deref().unwrap()
),
endpoint: format!("https://{endpoint}.example/v1"),
credential: format!("{endpoint}-credential"),
})
})
};
register_inspection_chat_model(PROVIDER_OPENHUMAN, factory("process"));
register_chat_model(PROVIDER_OPENHUMAN, fake_entry(Arc::default()));
let a = Extensions::new();
register_inspection_chat_model_in(&a, PROVIDER_OPENHUMAN, factory("context-a"));
register_chat_model_in(&a, PROVIDER_OPENHUMAN, fake_entry(Arc::default()));
let context_a = crate::core::runtime::DispatchCtx::new(
crate::core::runtime::CoreContext::for_test_with_host_kind_and_extensions(
crate::core::types::HostKind::Library,
crate::core::runtime::DomainSet::full(),
None,
a,
),
);
let b = Extensions::new();
register_inspection_chat_model_in(&b, PROVIDER_OPENHUMAN, factory("context-b"));
register_chat_model_in(&b, PROVIDER_OPENHUMAN, fake_entry(Arc::default()));
let context_b = crate::core::runtime::DispatchCtx::new(
crate::core::runtime::CoreContext::for_test_with_host_kind_and_extensions(
crate::core::types::HostKind::Library,
crate::core::runtime::DomainSet::full(),
None,
b,
),
);
let mut config_a = Config::default();
config_a.default_model = Some("context-a".into());
let mut config_b = Config::default();
config_b.default_model = Some("context-b".into());
let mut config_process = Config::default();
config_process.default_model = Some("process".into());
let build_a =
create_chat_model_with_identity_in_for_inspection(&context_a, "chat", &config_a, 0.0)
.expect("context A inspection provider");
let build_b =
create_chat_model_with_identity_in_for_inspection(&context_b, "chat", &config_b, 0.0)
.expect("context B inspection provider");
let build_process = create_chat_model_with_identity_in_for_inspection(
&crate::core::runtime::DispatchCtx::ambient(),
"chat",
&config_process,
0.0,
)
.expect("process inspection provider");
unregister_inspection_chat_model(PROVIDER_OPENHUMAN);
unregister_chat_model(PROVIDER_OPENHUMAN);
assert_eq!(build_a.model_id, "context-a:context-a");
assert_eq!(build_b.model_id, "context-b:context-b");
assert_eq!(build_process.model_id, "process:process");
}