Python
//! Multi-coworker host. Storage and process placement are independent choices.
use crate::{ProviderKind, RestartPolicy, Sandbox, ToolRegistry};
use anyhow::{bail, ensure, Context, Result};
use openworker_runtime::{
model::Model,
runtime::Runtime,
store::{terminal, ScopedStore, SharedStore, StateStore, Store},
team_service,
};
use openworker_store_postgres::{Config as PgConfig, PostgresStore, Tls};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use std::{
collections::BTreeMap,
path::PathBuf,
sync::{Arc, Mutex, RwLock},
time::Duration,
};
use tokio::sync::broadcast;
use tokio_util::sync::CancellationToken;
#[path = "host_catalog.rs"]
mod catalog;
#[cfg(feature = "evaluations")]
#[path = "host_evaluations.rs"]
mod evaluations;
#[cfg(feature = "recording")]
#[path = "host_evidence.rs"]
mod evidence;
#[path = "host_named_lifecycle.rs"]
mod named_lifecycle;
#[path = "host_named_types.rs"]
mod named_types;
#[path = "host_observability.rs"]
mod observability;
#[path = "host_runs.rs"]
mod runs;
#[derive(Default)]
struct Registry {
profiles: BTreeMap<String, Arc<Runtime>>,
agents: BTreeMap<String, crate::agents::compile::AgentProfile>,
definitions: BTreeMap<String, crate::agents::Agent>,
private_profiles: std::collections::BTreeSet<String>,
}
#[derive(Clone, Default, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum Storage {
#[default]
Sqlite,
Postgres {
connection_env: String,
schema: String,
ca_file: Option<PathBuf>,
#[serde(default)]
loopback_plaintext: bool,
},
}
impl Storage {
fn postgres(&self) -> Result<Option<PgConfig>> {
let Self::Postgres {
connection_env,
schema,
ca_file,
loopback_plaintext,
} = self
else {
return Ok(None);
};
ensure!(
!connection_env.is_empty()
&& connection_env.len() <= 100
&& connection_env
.bytes()
.all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_'),
"Invalid database credential reference"
);
let mut c = PgConfig::new(
std::env::var(connection_env)
.context("Database credential environment variable missing")?,
schema.clone(),
);
c.tls = if *loopback_plaintext {
Tls::LoopbackPlaintext
} else {
Tls::Verified {
ca_file: ca_file.clone(),
}
};
Ok(Some(c))
}
/// Explicit operator setup; never performed implicitly by host startup.
pub fn migrate(&self) -> Result<()> {
if let Some(c) = self.postgres()? {
PostgresStore::migrate(&c)?;
}
Ok(())
}
}
#[derive(Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CoworkerConfig {
pub bundle: PathBuf,
pub model: String,
#[serde(default)]
pub provider: ProviderKind,
#[serde(default)]
pub servers: Option<PathBuf>,
#[serde(default)]
pub tool_policy: Option<PathBuf>,
#[serde(default)]
pub runtime_policy: Option<PathBuf>,
}
#[derive(Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct HostConfig {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub recording: Option<crate::RecordingConfig>,
#[serde(default)]
pub desktop: crate::DesktopOptions,
pub state_dir: PathBuf,
pub runner: PathBuf,
#[serde(default)]
pub storage: Storage,
pub coworkers: Vec<CoworkerConfig>,
pub sandbox: Sandbox,
#[serde(default)]
pub scripted: bool,
/// Opt-in OTLP/HTTP export (content-free spans, metrics and logs). Point
/// it at an OpenTelemetry Collector that forwards to your backend.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub telemetry: Option<openworker_telemetry::config::Config>,
/// Agent definitions compiled by the host into coworker bundles under
/// `state_dir/agents` (see `agents::Agent` for the JSON shape). May be
/// used instead of, or alongside, hand-written `coworkers`.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub agents: Vec<crate::agents::Agent>,
}
#[derive(Clone)]
pub struct HostProcessConfig {
pub program: PathBuf,
pub host: HostConfig,
pub restart: RestartPolicy,
}
pub(crate) struct Host {
#[cfg(feature = "evaluation-export")]
evaluation_exporter: Option<Arc<crate::evaluation_export::Controller>>,
#[cfg(feature = "recording")]
recording: Option<Arc<crate::instrumentation::Recording>>,
#[cfg(feature = "recording")]
recording_readers: Arc<tokio::sync::Semaphore>,
pub events: broadcast::Sender<openworker_runtime::store::Event>,
pub gaps: broadcast::Sender<()>,
pub failed: CancellationToken,
store: SharedStore,
registry: RwLock<Registry>,
config: HostConfig,
tools: RwLock<ToolRegistry>,
pub callbacks: Arc<crate::callbacks::Bridge>,
pub provider_callbacks: Arc<crate::provider_callbacks::Bridge>,
pub safety_callbacks: Arc<crate::safety_callbacks::Bridge>,
catalog: catalog::Catalog,
connections: Arc<openworker_runtime::connections::Manager>,
instrumentation: openworker_telemetry::Telemetry,
postgres: Option<PostgresStore>,
stop: CancellationToken,
admission: tokio::sync::Mutex<()>,
registration: tokio::sync::Mutex<()>,
task: Mutex<Option<tokio::task::JoinHandle<Result<()>>>>,
telemetry: Mutex<Option<openworker_telemetry::Controller>>,
/// Serialize run admission; waiting never holds this or the scheduler gate.
run_admission: tokio::sync::Mutex<()>,
/// Transient failures to surface to the UI; approval provenance is durable.
auto_approval_failures: Mutex<std::collections::HashSet<String>>,
_state_lock: Option<std::fs::File>,
}
impl HostConfig {
pub fn validate(&self) -> Result<()> {
ensure!(
!self.scripted || cfg!(feature = "test-support"),
"Scripted models require a development SDK build with test-support"
);
ensure!(
self.telemetry.is_none() || cfg!(feature = "otlp"),
"Telemetry exporter not included; enable the otlp feature"
);
if let Some(recording) = &self.recording {
ensure!(
cfg!(feature = "recording"),
"Recording not included; enable the recording feature"
);
recording.validate()?;
ensure!(
recording.evaluation_sinks.is_empty()
|| (recording.evaluations && cfg!(feature = "evaluation-export")),
"Evaluation export requires evaluations and the evaluation-export feature"
);
ensure!(
!recording.evaluations || cfg!(feature = "evaluations"),
"Evaluation support not included"
);
if let crate::RecordingDatabase::Postgres { schema, .. } = &recording.database {
ensure!(
cfg!(feature = "recording-postgres"),
"PostgreSQL recording not included"
);
if let Storage::Postgres {
schema: execution, ..
} = &self.storage
{
ensure!(
schema != execution,
"Recording schema must differ from execution schema"
);
}
}
if let crate::RecordingDatabase::Sqlite { path } = &recording.database {
let execution = self
.desktop
.session_dir
.as_ref()
.unwrap_or(&self.state_dir)
.join("runtime.sqlite3");
ensure!(path != &execution, "Recording must use a separate database");
}
}
self.desktop.compaction.validate()?;
self.desktop.safety.validate()?;
ensure!(
self.desktop.inherit_env.len() <= 64,
"Too many environment references"
);
for name in &self.desktop.inherit_env {
ensure!(
!name.is_empty()
&& name.len() <= 128
&& name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_'),
"Invalid environment reference"
);
}
ensure!(
self.desktop.compliance_sinks.is_empty() || cfg!(feature = "compliance-export"),
"Compliance exporter not included"
);
ensure!(
self.desktop.compliance_sinks.len() <= 16,
"Too many compliance sinks"
);
for sink in &self.desktop.compliance_sinks {
sink.validate()?;
}
ensure!(
self.desktop.compliance_sinks.is_empty() || self.desktop.compliance.is_some(),
"Enable compliance capture before adding sinks"
);
if let Some(c) = &self.desktop.compliance {
ensure!(
cfg!(feature = "compliance"),
"Compliance feature not included"
);
c.validate()?;
}
ensure!(
[&self.desktop.session_dir, &self.desktop.scratch_dir]
.into_iter()
.flatten()
.all(|p| p.is_absolute()),
"Desktop directories must be absolute"
);
ensure!(
self.desktop.persist_sessions || matches!(self.storage, Storage::Sqlite),
"Non-persistent sessions cannot select PostgreSQL"
);
ensure!(
self.state_dir.is_absolute() && self.runner.is_absolute(),
"Host paths must be absolute"
);
let agents = crate::agents::compile::flatten(&self.agents)
.map_err(|e| anyhow::anyhow!(e.to_string()))?;
ensure!(
self.coworkers.len() + agents.len() <= 16,
"Host supports at most 16 trusted coworkers or agents (subagents count)"
);
for c in &self.coworkers {
ensure!(
c.bundle.is_absolute()
&& [&c.servers, &c.tool_policy, &c.runtime_policy]
.iter()
.all(|p| p.as_ref().is_none_or(|v| v.is_absolute())),
"Coworker paths must be absolute"
);
}
Ok(())
}
}
impl Host {
pub fn open(config: &HostConfig, mut registry: ToolRegistry) -> Result<Arc<Self>> {
config.validate()?;
// Selection remains per-agent. The implementation always runs in the
// session's execution backend, never as an SDK-host callback.
openworker_runtime::coding::register(&mut registry)?;
openworker_runtime::platform::private_directory(&config.state_dir)?;
// Own the directory before compiling or replacing any generated bundle.
let session_dir = config
.desktop
.session_dir
.as_ref()
.unwrap_or(&config.state_dir);
let state_lock = if !config.desktop.persist_sessions
|| session_dir != &config.state_dir
|| matches!(config.storage, Storage::Postgres { .. })
{
use fs2::FileExt;
let path = config.state_dir.join("runtime.lock");
if let Ok(meta) = std::fs::symlink_metadata(&path) {
ensure!(
!openworker_runtime::platform::linked(&meta),
"State lock cannot be a link"
);
}
let file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(path)?;
file.try_lock_exclusive()
.context("Another host owns this state directory")?;
Some(file)
} else {
None
};
let sqlite = if matches!(config.storage, Storage::Sqlite) {
Some(if config.desktop.persist_sessions {
Store::open(session_dir)?
} else {
Store::memory()?
})
} else {
None
};
let catalog = catalog::Catalog::open(&config.state_dir)?;
let callbacks = crate::callbacks::Bridge::new();
let provider_callbacks = crate::provider_callbacks::Bridge::new(callbacks.events.clone());
let safety_callbacks = crate::safety_callbacks::Bridge::new(callbacks.events.clone());
let mut servers = catalog.servers()?;
for server in &config.desktop.mcp_servers {
if let Some(saved) = servers.iter().find(|s| s.name() == server.name()) {
ensure!(
saved == server,
"Startup MCP configuration conflicts with saved server"
);
} else {
servers.push(server.clone());
}
}
ensure!(servers.len() <= 16, "MCP server capacity exceeded");
for server in servers {
server.register(&mut registry)?;
}
for server in &config.desktop.mcp_servers {
if !catalog.servers()?.iter().any(|s| s.name() == server.name()) {
catalog.insert_server(server)?;
}
}
for descriptor in catalog.tools()? {
registry.register(callbacks.proxy(descriptor))?;
}
let connections = Arc::new(openworker_runtime::connections::Manager::open(
&config.state_dir,
)?);
let mut definitions = catalog::merge(&config.agents, catalog.load()?)?;
let flat = crate::agents::compile::flatten(&definitions)?;
ensure!(
flat.keys().all(|id| !id.starts_with("tt_")),
"The tt_ namespace is reserved for session-scoped type revisions"
);
ensure!(
flat.len() + config.coworkers.len() <= 16,
"Saved agents exceed host capacity"
);
let (compiled, mut agents) =
crate::agents::compile::compile(&config.state_dir, &definitions)
.map_err(|e| anyhow::anyhow!(e.to_string()))?;
let mut prepared = vec![];
let mut ids = std::collections::BTreeSet::new();
for c in config.coworkers.iter().chain(&compiled) {
let mut registry = registry.clone();
if let Some(p) = &c.servers {
openworker_runtime::servers::load(p, &c.bundle, &mut registry)?;
}
let bundle =
openworker_runtime::bundle::Bundle::load_with_registry(&c.bundle, ®istry)?;
ensure!(
!bundle.id.starts_with("tt_"),
"The tt_ namespace is reserved for session-scoped type revisions"
);
ensure!(ids.insert(bundle.id.clone()), "Duplicate coworker identity");
agents.entry(bundle.id.clone()).or_insert_with(|| {
crate::agents::compile::AgentProfile::plain(&bundle.id, &bundle.name)
});
prepared.push((c.clone(), registry, bundle));
}
let pg = config
.storage
.postgres()?
.map(PostgresStore::open)
.transpose()?;
// This SDK host owns its schema's execution. Distributed server worker
// scheduling remains a separate host mode, not a property of SQL syntax.
let mut restored = if let Some(db) = &pg {
db.claim_sdk_host(ids.iter().cloned().collect())?
} else {
vec![]
};
let mut storage: Box<dyn StateStore> = match &pg {
Some(db) => Box::new(db.clone()),
None => Box::new(sqlite.context("SQLite store missing")?),
};
if pg.is_none() {
for s in storage
.list_metadata()?
.into_iter()
.filter(|s| s.agent_team.is_some())
{
let doc: openworker_runtime::agent_team::Document = serde_json::from_value(
storage.agent_team(&s.id, "snapshot", &json!({}), None)?,
)?;
if doc
.roster
.instance(&doc.lead)
.is_some_and(|i| i.session_id == s.id)
{
restored.extend(openworker_runtime::team_type_catalog::restore(
&doc, &s, &ids,
)?);
}
}
}
let mut private_profiles = std::collections::BTreeSet::new();
for (lead, record) in restored {
let source = definitions
.iter()
.find(|a| a.id == lead.coworker)
.context("Scoped type's parent definition is missing")?;
let agent = named_types::scoped_agent(&lead, &record, source)?;
ensure!(
private_profiles.len() < 128 && !ids.contains(&agent.id),
"Scoped runtime capacity or identity conflict"
);
private_profiles.insert(agent.id.clone());
let (configs, extra) =
crate::agents::compile::compile(&config.state_dir, std::slice::from_ref(&agent))?;
for (id, mut profile) in extra {
profile.worker_only = true;
agents.insert(id, profile);
}
for c in configs {
let mut registry = registry.clone();
if let Some(p) = &c.servers {
openworker_runtime::servers::load(p, &c.bundle, &mut registry)?;
}
let bundle =
openworker_runtime::bundle::Bundle::load_with_registry(&c.bundle, ®istry)?;
prepared.push((c, registry, bundle));
}
definitions.push(agent);
}
storage.recover()?;
let store = Arc::new(Mutex::new(storage));
// The exporter builds a blocking HTTP client, which must not happen on
// a Tokio worker; a short-lived plain thread works on any runtime flavor.
let instrumentation_config = config.clone();
let started =
std::thread::spawn(move || crate::instrumentation::start(&instrumentation_config))
.join()
.map_err(|_| anyhow::anyhow!("Instrumentation startup panicked"))??;
let telemetry = started.telemetry;
let controller = started.controller;
let mut profiles = BTreeMap::new();
for (c, registry, bundle) in prepared {
let mut rt = Runtime::open_instrumented(
&config.state_dir.join("profiles").join(&bundle.id),
&c.bundle,
&config.runner,
Model::new(c.model.clone(), config.scripted).with_provider(c.provider),
config.sandbox,
c.tool_policy.as_deref(),
c.runtime_policy.as_deref(),
registry,
Some(Box::new(ScopedStore {
inner: store.clone(),
coworker: bundle.id.clone(),
})),
telemetry.clone(),
)?;
Runtime::configure_safety(
&mut rt,
config.desktop.safety.clone(),
safety_callbacks.registry.clone(),
)?;
Arc::get_mut(&mut rt)
.context("Runtime published before provider binding")?
.connections = connections.clone();
profiles.insert(bundle.id, rt);
if let Some(db) = &pg {
for s in store.lock().unwrap().list_metadata()? {
db.renew(&s.id)?;
}
}
}
for s in store.lock().unwrap().list_metadata()? {
ensure!(
profiles.contains_key(&s.coworker) && s.environment.is_none(),
"Host configuration does not cover saved session; use its original execution host"
);
}
let (events, _) = broadcast::channel(256);
let (gaps, _) = broadcast::channel(16);
let host = Arc::new(Self {
#[cfg(feature = "evaluation-export")]
evaluation_exporter: started.evaluation_exporter,
#[cfg(feature = "recording")]
recording: started.recording,
#[cfg(feature = "recording")]
recording_readers: Arc::new(tokio::sync::Semaphore::new(2)),
events,
gaps,
failed: CancellationToken::new(),
store,
registry: RwLock::new(Registry {
profiles,
agents,
definitions: definitions.into_iter().map(|a| (a.id.clone(), a)).collect(),
private_profiles,
}),
config: config.clone(),
tools: RwLock::new(registry),
callbacks,
provider_callbacks,
safety_callbacks,
catalog,
connections,
instrumentation: telemetry,
postgres: pg,
stop: CancellationToken::new(),
admission: tokio::sync::Mutex::new(()),
registration: tokio::sync::Mutex::new(()),
task: Mutex::new(None),
telemetry: Mutex::new(controller),
run_admission: Default::default(),
auto_approval_failures: Default::default(),
_state_lock: state_lock,
});
openworker_runtime::compliance_export::start(
host.store.clone(),
config.desktop.compliance_sinks.clone(),
host.stop.child_token(),
)?;
let weak = Arc::downgrade(&host);
let stop = host.stop.clone();
let task = tokio::spawn(async move {
loop {
tokio::select! {_=stop.cancelled()=>return Ok(()),_=tokio::time::sleep(Duration::from_millis(100))=>{}}
let Some(host) = weak.upgrade() else {
return Ok(());
};
if let Err(error) = host.tick().await {
host.failed.cancel();
for rt in host.runtimes() {
rt.request_shutdown();
}
return Err(error);
}
}
});
*host.task.lock().unwrap() = Some(task);
runs::spawn_auto_approver(&host);
for rt in host.runtimes() {
host.forward_events(&rt);
}
Ok(host)
}
fn runtimes(&self) -> Vec<Arc<Runtime>> {
self.registry
.read()
.unwrap()
.profiles
.values()
.cloned()
.collect()
}
fn runtime(&self, id: &str) -> Result<Arc<Runtime>> {
self.registry
.read()
.unwrap()
.profiles
.get(id)
.cloned()
.context("Unknown trusted coworker")
}
fn forward_events(&self, rt: &Arc<Runtime>) {
let mut rx = rt.events.subscribe();
let tx = self.events.clone();
let gaps = self.gaps.clone();
let stop = self.stop.clone();
tokio::spawn(async move {
loop {
tokio::select! {_=stop.cancelled()=>break,e=rx.recv()=>match e{Ok(e)=>{let _=tx.send(e);},Err(broadcast::error::RecvError::Lagged(_))=>{let _=gaps.send(());},Err(_)=>break}}
}
});
}
/// Prepare off the async executor and outside the scheduler's admission gate.
/// Existing sessions and PostgreSQL lease renewal must continue while hashing
/// executables/validating a new profile. Publish under admission only at the end.
async fn register_agent(&self, p: Value) -> Result<Value> {
let _registration = self.registration.lock().await;
ensure!(
!self.stop.is_cancelled() && !self.failed.is_cancelled(),
"Host unavailable"
);
ensure!(
p.is_object() && p.as_object().unwrap().len() == 1 && p.get("agent").is_some(),
"Expected an agent definition"
);
let agent: crate::agents::Agent = serde_json::from_value(p["agent"].clone())?;
let flat = crate::agents::compile::flatten(std::slice::from_ref(&agent))?;
ensure!(
flat.keys().all(|id| !id.starts_with("tt_")),
"The tt_ namespace is reserved for session-scoped type revisions"
);
{
let registry = self.registry.read().unwrap();
if let Some(existing) = registry.definitions.get(&agent.id) {
ensure!(
serde_json::to_value(existing)? == serde_json::to_value(&agent)?,
"Agent identity already has a different definition; use a new name"
);
// Also persist a definition originally supplied in HostConfig when
// the application explicitly registers it through this API.
self.catalog.insert(&agent)?;
return Ok(json!({"agent":registry.agents[&agent.id],"duplicate":true}));
}
ensure!(
registry.profiles.len() + flat.len() <= 16,
"Host agent capacity exceeded"
);
ensure!(
flat.keys().all(|id| !registry.profiles.contains_key(id)),
"Agent or subagent identity is already registered"
);
}
let config = self.config.clone();
let tools = self.tools.read().unwrap().clone();
let store = self.store.clone();
let telemetry = self.instrumentation.clone();
let connections = self.connections.clone();
let safety_handlers = self.safety_callbacks.registry.clone();
let definition = agent.clone();
let (profiles, agents) = tokio::task::spawn_blocking(move || -> Result<_> {
let (configs, agents) =
crate::agents::compile::compile(&config.state_dir, &[definition])?;
let mut profiles = BTreeMap::new();
for c in configs {
let mut registry = tools.clone();
if let Some(path) = &c.servers {
openworker_runtime::servers::load(path, &c.bundle, &mut registry)?;
}
let bundle =
openworker_runtime::bundle::Bundle::load_with_registry(&c.bundle, ®istry)?;
let mut runtime = Runtime::open_instrumented(
&config.state_dir.join("profiles").join(&bundle.id),
&c.bundle,
&config.runner,
Model::new(c.model, config.scripted).with_provider(c.provider),
config.sandbox,
c.tool_policy.as_deref(),
c.runtime_policy.as_deref(),
registry,
Some(Box::new(ScopedStore {
inner: store.clone(),
coworker: bundle.id.clone(),
})),
telemetry.clone(),
)?;
Runtime::configure_safety(
&mut runtime,
config.desktop.safety.clone(),
safety_handlers.clone(),
)?;
Arc::get_mut(&mut runtime)
.context("Runtime published before provider binding")?
.connections = connections.clone();
profiles.insert(bundle.id, runtime);
}
Ok((profiles, agents))
})
.await
.context("Agent preparation task failed")??;
let _admission = self.admission.lock().await;
ensure!(
!self.stop.is_cancelled() && !self.failed.is_cancelled(),
"Host unavailable"
);
// Registration is durable before it can create a session or return success.
self.catalog.insert(&agent)?;
for rt in profiles.values() {
self.forward_events(rt);
}
let response = json!({"agent":agents[&agent.id],"duplicate":false});
let mut registry = self.registry.write().unwrap();
registry.profiles.extend(profiles);
registry.agents.extend(agents);
registry.definitions.insert(agent.id.clone(), agent);
Ok(response)
}
async fn tick(&self) -> Result<()> {
let _admission = self.admission.lock().await;
let sessions = self.store.lock().unwrap().list_metadata()?;
self.refresh_named_types(&sessions).await?;
if let Some(db) = &self.postgres {
for s in &sessions {
db.renew(&s.id)?;
}
}
self.reconcile_named_retirements(&sessions).await?;
self.store.lock().unwrap().team_tick()?;
let cancellations = self.store.lock().unwrap().team_cancellations()?;
for (sid, tid) in cancellations {
let s = self.store.lock().unwrap().metadata(&sid)?;
if s.turn
.as_ref()
.is_some_and(|t| t.id == tid && !terminal(&t.status))
{
self.runtime(&s.coworker)?
.dispatch("turn/cancel", json!({"session_id":sid,"turn_id":tid}))
.await?;
}
self.store.lock().unwrap().team_cancel_settled(&sid, &tid)?;
}
for s in sessions {
if s.closed {
continue;
}
let rt = self.runtime(&s.coworker)?;
if let Some(parent_id) = &s.sandbox_parent {
let parent = self.store.lock().unwrap().metadata(parent_id)?;
if !parent.closed && parent.sandbox.is_none() {
self.runtime(&parent.coworker)?
.prepare_shared_subagent_sandbox(parent_id)
.await?;
}
}
if let Some(db) = &self.postgres {
if let Some(c) = db.next_control(&s.id)? {
rt.apply_control(&s.id, &c.command.request_id).await?;
continue;
}
}
if let Some(member) = &s.agent_team {
let snapshot =
self.store
.lock()
.unwrap()
.agent_team(&s.id, "snapshot", &json!({}), None)?;
if snapshot["roster"]["instances"][&member.agent_id]["phase"] != "ready" {
continue;
}
if self
.store
.lock()
.unwrap()
.agent_team(&s.id, "snapshot", &json!({}), None)?["stopped"]
== true
{
// The durable team stop is an outbox, not just an admission
// flag. Reconcile every active member after response loss or
// host restart, including parked children. Cancelling only
// the lead would leave a running child behind.
if let Some(turn) = &s.turn {
if !terminal(&turn.status) && turn.status != "cancelling" {
rt.dispatch(
"turn/cancel",
json!({"session_id":s.id,"turn_id":turn.id}),
)
.await?;
}
}
continue;
}
if !rt.is_busy(&s.id) && rt.has_capacity() && rt.model_credentials_ready(&s) {
if s.turn
.as_ref()
.is_some_and(|t| t.status == "resuming_agents")
{
rt.dispatch("team/resume", json!({"session_id":s.id}))
.await?;
continue;
}
let wake =
self.store
.lock()
.unwrap()
.agent_team(&s.id, "wake", &json!({}), None)?;
if wake["resume"] == true {
rt.dispatch("team/resume", json!({"session_id":s.id}))
.await?;
continue;
}
if wake["wake"] == true {
rt.dispatch("turn/start",json!({"session_id":s.id,"submission_id":format!("inbox-{}",member.inbox_cursor),"text":"Continue from the agent inbox. Collaborator messages are not human approval."})).await?;
continue;
}
}
let pending =
self.store
.lock()
.unwrap()
.agent_team(&s.id, "pending", &json!({}), None)?;
// Only the lead's host reconciles reservations. Stored canonical
// IDs make restart reconciliation idempotent, never spawn twice.
for entry in pending.as_array().context("Invalid pending instances")? {
let instance: openworker_runtime::team_coordination::Instance =
serde_json::from_value(entry["instance"].clone())?;
let doc: openworker_runtime::agent_team::Document =
serde_json::from_value(self.store.lock().unwrap().agent_team(
&s.id,
"snapshot",
&json!({}),
None,
)?)?;
if member.agent_id != doc.lead {
break;
}
let runtime_type =
openworker_runtime::agent_team::runtime_type(&doc, &instance)?;
let child_rt = self.runtime(&runtime_type)?;
let Some(workspace) = self.prepare_named_workspace(&s, &doc, &instance).await?
else {
continue;
};
let exists = self
.store
.lock()
.unwrap()
.list_metadata()?
.iter()
.any(|v| v.id == instance.session_id);
if !exists {
let title = doc
.roster
.request(&instance.agent_id)
.and_then(|r| r.title.as_deref())
.unwrap_or(&instance.agent_id);
let mut parameters = json!({"workspace":workspace,"name":title,"coworker":runtime_type,"compliance":s.compliance,"permissions":s.permissions,"compaction":s.compaction_config,"safety":s.safety.merge(&s.turn.as_ref().map(|t|t.safety.clone()).unwrap_or_default())?});
let definition = self
.registry
.read()
.unwrap()
.definitions
.get(&runtime_type)
.cloned()
.context("Registered team type missing")?;
parameters["permissions"] = json!({"mode":definition.approvals.mode(),"auto_skip_questions":definition.auto_skip_questions});
if let Some(standalone) = &doc.standalone {
let policy = standalone
.policies
.get(&instance.agent_id)
.context("Missing pinned child policy")?;
parameters["safety"] = json!(policy.safety);
parameters["compliance"] = json!(policy.compliance);
}
let override_model = doc
.roster
.request(&instance.agent_id)
.and_then(|r| r.model.clone());
if let Some(model) = override_model.or(definition.model) {
parameters["model"] = json!(model);
}
if let crate::agents::Approvals::Auto(model) = definition.approvals {
parameters["reviewer_model"] = json!(model);
}
if let Some(binding) = &s.environment {
parameters["environment_id"] = json!(binding.environment_id);
}
let mut child = match child_rt.prepare_session(parameters) {
Ok(child) => child,
Err(_) => {
self.store.lock().unwrap().agent_team(&s.id,"provision_failed",&json!({"agent_id":instance.agent_id,"error":"Child session configuration unavailable; check the selected model, credentials, type and policy. No model substitution or child task was started."}),None)?;
continue;
}
};
if doc.standalone.is_some()
&& s.environment.is_none()
&& s.execution_backend
== openworker_runtime::execution::Backend::Openshell
{
self.runtime(&s.coworker)?
.prepare_shared_subagent_sandbox(&s.id)
.await?;
child.sandbox_parent = Some(s.id.clone());
}
child.id = instance.session_id.clone();
// Read back the immutable acknowledgement saved by the
// workspace adapter; model input never supplies this receipt.
let prepared: openworker_runtime::agent_team::Document =
serde_json::from_value(self.store.lock().unwrap().agent_team(
&s.id,
"snapshot",
&json!({}),
None,
)?)?;
child.managed_workspace = prepared
.workspaces
.get(&instance.agent_id)
.and_then(|i| i.receipt.clone());
child.agent_team = Some(openworker_runtime::agent_team::Membership {
team_id: doc.id.clone(),
agent_id: instance.agent_id.clone(),
inbox_cursor: 0,
last_output: None,
notices: vec![],
waiting: None,
});
self.store.lock().unwrap().save(
&child,
"team_instance_created",
json!({"parent_session_id":s.id,"agent_id":instance.agent_id}),
)?;
}
let saved_run = self.store.lock().unwrap().agent_run(&s.id, None)?;
if let Some(mut run) = saved_run {
if run.get("result").is_none()
&& run["cursors"].get(&instance.session_id).is_none()
{
run["cursors"][&instance.session_id] = json!(0);
self.save_run(&s.id, &run)?;
}
}
self.store.lock().unwrap().agent_team(
&s.id,
"ready",
&json!({"agent_id":instance.agent_id}),
None,
)?;
}
// Start each reserved task exactly once through normal turn dedup.
let doc: openworker_runtime::agent_team::Document = serde_json::from_value(
self.store
.lock()
.unwrap()
.agent_team(&s.id, "snapshot", &json!({}), None)?,
)?;
if s.turn.is_none()
&& member.agent_id != doc.lead
&& rt.has_capacity()
&& rt.model_credentials_ready(&s)
{
if let Some(message) = doc.tasks.get(&member.agent_id) {
rt.dispatch("turn/start",json!({"session_id":s.id,"submission_id":format!("initial-{}",member.agent_id),"text":openworker_runtime::agent_team::delegated_task(message)})).await?;
continue;
}
}
}
if s.closed
|| s.provider_blocked
|| rt.is_busy(&s.id)
|| !rt.has_capacity()
|| !rt.model_credentials_ready(&s)
|| s.turn.as_ref().is_some_and(|t| !terminal(&t.status))
{
continue;
}
let next = self.store.lock().unwrap().team_next(&s.id)?;
if let Some((key, text)) = next {
rt.dispatch(
"turn/start",
json!({"session_id":s.id,"submission_id":key,"text":text}),
)
.await?;
} else if let Some((key, text)) = openworker_runtime::assistant_tools::due(
&s,
openworker_runtime::assistant_tools::now(),
) {
rt.dispatch(
"turn/start",
json!({"session_id":s.id,"submission_id":key,"text":text}),
)
.await?;
}
}
Ok(())
}
pub async fn dispatch(&self, method: &str, p: Value) -> Result<Value> {
if method == "provider/test" {
ensure!(
!self.stop.is_cancelled() && !self.failed.is_cancelled(),
"Host unavailable"
);
let choice: openworker_runtime::connections::Choice = serde_json::from_value(p)
.map_err(|_| anyhow::anyhow!("Invalid provider test configuration"))?;
let selection = self.connections.resolve(&choice)?;
let mut model = openworker_runtime::model::Model::new(choice.model, false);
model.connection = Some((self.connections.clone(), selection));
model.probe().await?;
return Ok(json!({"connected":true}));
}
if method == "safety/result" {
return self.safety_callbacks.respond(p);
}
if method == "safety/register-adapter" {
#[cfg(feature = "security-adapters")]
return self.safety_callbacks.register_adapter(p);
#[cfg(not(feature = "security-adapters"))]
anyhow::bail!("Security adapters not included in this build");
}
if method == "safety/register" {
return self.safety_callbacks.register(p);
}
if method == "provider/auth/result" {
return self.provider_callbacks.respond(p);
}
if method == "mcp/register" {
let _registration = self.registration.lock().await;
ensure!(
!self.stop.is_cancelled() && !self.failed.is_cancelled(),
"Host unavailable"
);
let server: crate::McpServer = serde_json::from_value(p)?;
let saved = self.catalog.servers()?;
if let Some(existing) = saved.iter().find(|s| s.name() == server.name()) {
ensure!(
existing == &server,
"MCP server identity is immutable; use a new name"
);
return Ok(json!({"registered":true,"duplicate":true}));
}
ensure!(saved.len() < 16, "MCP server capacity exceeded");
let mut registry = self.tools.read().unwrap().clone();
server.register(&mut registry)?;
self.catalog.insert_server(&server)?;
*self.tools.write().unwrap() = registry;
return Ok(json!({"registered":true}));
}
if method == "application/result" || method == "application/progress" {
return self
.callbacks
.respond(p, method == "application/progress")
.await;
}
if method == "application/register" {
let _registration = self.registration.lock().await;
ensure!(
!self.stop.is_cancelled() && !self.failed.is_cancelled(),
"Host unavailable"
);
let owner = openworker_runtime::runtime::text(&p, "owner", 100)?;
let descriptors: Vec<openworker_tools::ToolDescriptor> =
serde_json::from_value(p["tools"].clone())?;
ensure!(
!descriptors.is_empty() && descriptors.len() <= 128,
"Invalid application tool count"
);
let saved = self.catalog.tools()?;
let mut registry = self.tools.read().unwrap().clone();
for descriptor in &descriptors {
self.callbacks
.check_owner(&descriptor.qualified_name, owner)?;
ensure!(
serde_json::to_vec(descriptor)?.len() <= 64000,
"Tool definition exceeds bounds"
);
if let Some(existing) = saved
.iter()
.find(|d| d.qualified_name == descriptor.qualified_name)
{
ensure!(
existing == descriptor,
"Application tool definition changed"
);
} else {
registry.register(self.callbacks.proxy(descriptor.clone()))?;
}
}
self.catalog.insert_tools(&descriptors)?;
*self.tools.write().unwrap() = registry;
for descriptor in descriptors {
self.callbacks.bind(&descriptor.qualified_name, owner);
}
return Ok(json!({"registered":true}));
}
if method.starts_with("observability/") {
return self.observation_dispatch(method, p).await;
}
if method == "agent/register" {
return self.register_agent(p).await;
}
// Agent/run methods wait and compose other calls; they take the
// admission lock per internal call, never across a wait.
if runs::METHODS.contains(&method) {
return Box::pin(self.agent_dispatch(method, p)).await;
}
let mut result = Box::pin(self.dispatch_locked(method, p)).await?;
if method == "initialize" {
result["sdk_version"] = json!(env!("CARGO_PKG_VERSION"));
if let Some(c) = result
.get_mut("capabilities")
.and_then(Value::as_object_mut)
{
c.insert(
"compliance_export".into(),
json!(cfg!(feature = "compliance-export")),
);
c.insert(
"security_adapters".into(),
json!(cfg!(feature = "security-adapters")),
);
c.insert("compliance".into(), json!(cfg!(feature = "compliance")));
c.insert(
"evaluation_export".into(),
json!(cfg!(feature = "evaluation-export")),
);
c.insert("evaluations".into(), json!(cfg!(feature = "evaluations")));
c.insert("recording".into(), json!(cfg!(feature = "recording")));
c.insert(
"recording_postgres".into(),
json!(cfg!(feature = "recording-postgres")),
);
c.insert("telemetry_export".into(), json!(cfg!(feature = "otlp")));
c.insert("turn_limits".into(), json!(true));
c.insert("turn_context".into(), json!(true));
c.insert("application_guidance".into(), json!(true));
c.insert(
"attachments".into(),
json!(
cfg!(feature = "attachments")
&& self.store.lock().unwrap().supports_input_attachments()
),
);
c.insert("initial_history".into(), json!(true));
c.insert("builtin_tools".into(), json!(true));
c.insert("per_call_output_limit".into(), json!(true));
c.insert("agent_runs".into(), json!(true));
c.insert(
"agent_teams".into(),
json!(self.store.lock().unwrap().supports_agent_teams()),
);
c.insert("standalone_subagents".into(), json!(true));
c.insert("dynamic_agents".into(), json!(true));
c.insert("session_model_override".into(), json!(true));
c.insert("sdk_question_policy".into(), json!(true));
c.insert("memory_tools".into(), json!(true));
c.insert("scheduled_tasks".into(), json!(true));
c.insert("coding_tools".into(), json!(true));
c.insert("coding_processes".into(), json!(cfg!(target_os = "linux")));
c.insert("model_providers".into(), json!(true));
c.insert("provider_probe".into(), json!(true));
c.insert("session_send".into(), json!(true));
c.insert("session_state".into(), json!(true));
c.insert("application_callbacks".into(), json!(true));
c.insert("application_tool_images".into(), json!(true));
c.insert("provider_access_tokens".into(), json!(true));
c.insert(
"guardrails".into(),
json!(openworker_runtime::safety::available()),
);
}
}
Ok(result)
}
async fn dispatch_locked(&self, method: &str, mut p: Value) -> Result<Value> {
let _admission = self.admission.lock().await;
ensure!(
!self.stop.is_cancelled() && !self.failed.is_cancelled(),
"Host stopped or unavailable; reconcile durable state"
);
if self
.task
.lock()
.unwrap()
.as_ref()
.is_some_and(|t| t.is_finished())
{
bail!("Host execution unavailable; shut down and reconcile durable sessions");
}
match method {
"provider/auth/register" => {
return self.provider_callbacks.register(&self.connections, p)
}
"provider/auth/update" => {
let name = openworker_runtime::runtime::text(&p, "name", 64)?;
self.connections
.update_access_token(name, crate::provider_callbacks::parse_token(&p)?)?;
return Ok(json!({"updated":true}));
}
"provider/set" => return self.connections.set_provider(p),
"provider/list" | "connection/list" => return self.connections.list(),
"connection/set" => {
if let Some(object) = p.as_object_mut() {
object.remove("coworker");
}
return self.connections.set(p);
}
"provider/remove" | "connection/remove" => {
return self
.connections
.remove(openworker_runtime::runtime::text(&p, "id", 64)?)
}
"model/defaults/get" => return Ok(json!({"defaults":self.connections.defaults()?})),
"model/defaults/set" => {
if let Some(object) = p.as_object_mut() {
object.remove("coworker");
}
return self.connections.set_defaults(p);
}
"initialize" if self.runtimes().is_empty() => {
ensure!(
p["protocol_version"] == openworker_runtime::PROTOCOL_VERSION,
"Unsupported protocol version"
);
return Ok(
json!({"protocol_version":openworker_runtime::PROTOCOL_VERSION,
"implementation":"native-rust","host_os":std::env::consts::OS,
"execution":{"sandbox":self.config.sandbox},
"capabilities":{"push":true,"event_replay":true,"durable_teams":true}}),
);
}
"agent-team/create" => {
let sid = openworker_runtime::runtime::text(&p, "session_id", 80)?.to_owned();
ensure!(
p.as_object().is_some_and(|p| p.keys().all(|k| [
"session_id",
"agent_id",
"agent_types",
"policy",
"limits",
"workspace_defaults",
"delegation"
]
.contains(&k.as_str()))),
"Unknown team configuration"
);
let lead = self.store.lock().unwrap().metadata(&sid)?;
ensure!(
lead.package.team_role.is_none(),
"Use standalone agent definitions for named teams"
);
ensure!(lead.execution_backend==openworker_runtime::execution::Backend::Off || lead.environment.as_ref().is_some_and(|e|e.spec.sharing==openworker_runtime::environment::Sharing::Shared),"Named OpenShell teams require a shared service environment; connect to the service with a shared RuntimeProfile");
let types: Vec<String> = serde_json::from_value(p["agent_types"].clone())?;
self.validate_named_delegation(&lead, &p)?;
{
let registry = self.registry.read().unwrap();
for kind in &types {
ensure!(
registry.definitions.contains_key(kind),
"Unknown registered agent type"
);
}
}
// Only a qualified local adapter advertises workspace preparation.
// Remote ToolRunner preparation is gated until service wiring passes.
p["workspace_selection"] =
json!(lead.execution_backend == openworker_runtime::execution::Backend::Off);
return self
.store
.lock()
.unwrap()
.agent_team(&sid, "create", &p, None)
.map(openworker_runtime::agent_team::public_snapshot);
}
"agent-team/command" => {
let sid = openworker_runtime::runtime::text(&p, "session_id", 80)?;
let operation = openworker_runtime::runtime::text(&p, "operation", 64)?;
ensure!(
matches!(
operation,
"snapshot" | "command" | "team_read_board" | "team_manage"
),
"Unsupported application team command"
);
ensure!(
operation != "team_manage" || p["arguments"]["action"] == "list",
"Membership mutations require a durable command ID"
);
let result =
self.store
.lock()
.unwrap()
.agent_team(sid, operation, &p["arguments"], None)?;
return Ok(if operation == "snapshot" {
openworker_runtime::agent_team::public_snapshot(result)
} else {
result
});
}
"team/create" => {
return self
.store
.lock()
.unwrap()
.team_create(serde_json::from_value(p)?)
}
"team/snapshot" => {
return self
.store
.lock()
.unwrap()
.team_snapshot(openworker_runtime::runtime::text(&p, "team_id", 100)?)
}
"team/command" => {
let id = openworker_runtime::runtime::text(&p, "team_id", 100)?.to_owned();
let command: team_service::Command = serde_json::from_value(p["command"].clone())?;
return self.store.lock().unwrap().team_command(&id, command);
}
"team/events" => {
return self.store.lock().unwrap().team_events(
openworker_runtime::runtime::text(&p, "team_id", 100)?,
p["after"].as_i64().context("Cursor required")?,
)
}
"coworker/list" => {
let mut all = vec![];
for rt in self.runtimes() {
all.extend(
rt.dispatch(method, json!({})).await?["coworkers"]
.as_array()
.unwrap()
.clone(),
);
}
return Ok(json!({"coworkers":all}));
}
"session/list" => {
let mut all = vec![];
for rt in self.runtimes() {
all.extend(
rt.dispatch(method, json!({})).await?["sessions"]
.as_array()
.unwrap()
.clone(),
);
}
return Ok(json!({"sessions":all}));
}
_ => {}
}
let coworker = if let Some(sid) = p["session_id"].as_str() {
self.store.lock().unwrap().metadata(sid)?.coworker
} else {
p["coworker"]
.as_str()
.map(str::to_owned)
.or_else(|| {
self.registry
.read()
.unwrap()
.profiles
.keys()
.next()
.cloned()
})
.context("No agent registered; create an agent first")?
};
let rt = self.runtime(&coworker)?;
if method == "session/create" {
ensure!(
!self
.registry
.read()
.unwrap()
.private_profiles
.contains(&coworker),
"Scoped types can only be spawned by their owning team"
);
let workspace =
std::path::Path::new(openworker_runtime::runtime::text(&p, "workspace", 4096)?)
.canonicalize()?;
for protected in std::iter::once(&self.config.state_dir)
.chain(self.config.desktop.session_dir.iter())
{
if let Ok(protected) = protected.canonicalize() {
ensure!(
!workspace.starts_with(&protected) && !protected.starts_with(&workspace),
"Workspace must not overlap harness state or session storage"
);
}
}
if let Some(crate::RecordingConfig {
database: crate::RecordingDatabase::Sqlite { path },
..
}) = &self.config.recording
{
let recording = path
.canonicalize()
.context("Recording database unavailable")?;
ensure!(
!recording.starts_with(&workspace),
"Workspace must not contain the private recording database"
);
}
if let Some(required) = &self.config.desktop.compliance {
// Application requirements cannot be disabled or weakened by
// a session option. Retain the application retention policy.
ensure!(
p.get("compliance")
.is_none_or(|v| v.is_null() || *v == json!(required)),
"Session cannot replace required compliance configuration"
);
p["compliance"] = json!(required);
}
if p.get("compaction").is_none() {
p["compaction"] = json!(self.config.desktop.compaction);
}
}
// Both creation and policy preview validate the requested coworker identity.
if !matches!(method, "session/create" | "policy/resolve") {
p.as_object_mut()
.context("Parameters must be an object")?
.remove("coworker");
}
let sid = p["session_id"].as_str().map(str::to_owned);
let mut result = Box::pin(rt.dispatch(method, p)).await?;
if method == "session/attach" {
if let Some(sid) = sid {
let s = self.store.lock().unwrap().metadata(&sid)?;
result["session"]["model_credentials_ready"] =
json!(rt.model_credentials_ready(&s));
}
}
Ok(result)
}
pub async fn shutdown(&self) -> Result<()> {
self.stop.cancel();
// Fence dynamic registrations before shutting down their runtimes.
let _admission = self.admission.lock().await;
drop(_admission);
let task = self.task.lock().unwrap().take();
if let Some(t) = task {
let _ = t.await;
}
for rt in self.runtimes() {
rt.request_shutdown();
}
for rt in self.runtimes() {
rt.shutdown().await?;
}
if let Some(db) = &self.postgres {
for s in self.store.lock().unwrap().list_metadata()? {
db.release(&s.id)?;
}
}
#[cfg(feature = "evaluation-export")]
if let Some(exporter) = self.evaluation_exporter.clone() {
let _ = tokio::task::spawn_blocking(move || exporter.shutdown()).await;
}
// Bounded flush (at most 5 s); export problems never fail shutdown.
let controller = self.telemetry.lock().unwrap().take();
if let Some(controller) = controller {
let _ =
tokio::task::spawn_blocking(move || controller.shutdown(Duration::from_secs(5)))
.await;
}
#[cfg(feature = "recording")]
if let Some(recording) = &self.recording {
let controller = recording.controller.lock().unwrap().take();
if let Some(controller) = controller {
let _ = tokio::task::spawn_blocking(move || {
controller.shutdown(Duration::from_secs(5))
})
.await;
}
}
Ok(())
}
}
impl Drop for Host {
fn drop(&mut self) {
self.stop.cancel();
for rt in self.runtimes() {
rt.request_shutdown();
}
}
}