Python
//! Bounded worker role; HTTP/tenant authorization stays outside this module.
//! Each slot has one store incarnation and one runtime for one session. Recovery
//! therefore cannot interrupt a sibling session running on this worker.
use crate::{Config, PostgresStore};
use anyhow::{bail, Context, Result};
use openworker_runtime::telemetry::{Gauge, Telemetry};
use openworker_runtime::{execution::Backend, runtime::Runtime, store::StateStore};
use std::{
sync::{
atomic::{AtomicBool, AtomicU64, Ordering},
Arc,
},
time::{Duration, SystemTime, UNIX_EPOCH},
};
use tokio::{
task::JoinSet,
time::{interval, MissedTickBehavior},
};
use tokio_util::sync::CancellationToken;
/// Host-owned planned-stop handle. Request is immediate; acknowledgement means
/// all admissions already in flight have crossed the serialized boundary.
/// One-shot: use a fresh handle for each worker lifetime. Race acknowledgement
/// with the run result: an immediate failure/abort may exit without acknowledging.
#[derive(Clone, Default)]
pub struct Drain(Arc<Intake>);
#[derive(Default)]
struct Intake {
gate: tokio::sync::RwLock<()>,
request: CancellationToken,
acknowledged: CancellationToken,
draining: AtomicBool,
expired: AtomicBool,
}
impl Drain {
pub fn request(&self) {
self.0.request.cancel();
}
pub fn is_requested(&self) -> bool {
self.0.request.is_cancelled()
}
pub async fn acknowledged(&self) {
self.0.acknowledged.cancelled().await;
}
pub fn is_acknowledged(&self) -> bool {
self.0.draining.load(Ordering::Acquire)
}
}
struct DrainWatch(tokio::task::JoinHandle<()>);
impl Drop for DrainWatch {
fn drop(&mut self) {
self.0.abort();
}
}
async fn watch_drain(drain: Drain, stop: CancellationToken, timeout: Duration) {
tokio::select! {biased; _=stop.cancelled()=>return, _=drain.0.request.cancelled()=>{}}
let deadline = tokio::time::sleep(timeout);
tokio::pin!(deadline);
tokio::select! {biased;
_=stop.cancelled()=>return,
_=&mut deadline=>{drain.0.expired.store(true,Ordering::Release);stop.cancel();return;},
permit=drain.0.gate.write()=>{
drain.0.draining.store(true,Ordering::Release);
drain.0.acknowledged.cancel();
drop(permit);
}
}
tokio::select! {biased; _=stop.cancelled()=>{}, _=deadline=>{drain.0.expired.store(true,Ordering::Release);stop.cancel();}}
}
#[derive(Clone, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Selector {
pub coworker: String,
pub bundle_digest: String,
pub backend: Backend,
pub scripted: bool,
pub fallback_model: String,
#[serde(default)]
pub model_provider: openworker_runtime::provider::Kind,
#[serde(default)]
pub model_connections: std::collections::BTreeMap<String, String>,
/// None is an explicitly local/unsandboxed pool, never a remote fallback.
pub environment_provider: Option<String>,
pub environment_identity: Option<String>,
pub environment_profile: Option<String>,
}
impl Selector {
pub fn validate(&self) -> Result<()> {
crate::submissions::validate_id(&self.coworker, 120)?;
crate::submissions::validate_id(&self.bundle_digest, 128)?;
crate::submissions::validate_id(&self.fallback_model, 200)?;
anyhow::ensure!(
self.model_connections.len() <= 16,
"Too many model connection identities"
);
for (id, identity) in &self.model_connections {
crate::submissions::validate_id(id, 64)?;
crate::submissions::validate_id(identity, 128)?;
}
if let Some(p) = &self.environment_provider {
crate::submissions::validate_id(p, 80)?;
}
if self.environment_provider.is_some() != self.environment_identity.is_some()
|| self.environment_provider.is_some() != self.environment_profile.is_some()
{
bail!("Remote scheduling requires provider, deployment identity and profile");
}
for p in [&self.environment_identity, &self.environment_profile]
.into_iter()
.flatten()
{
crate::submissions::validate_id(p, 512)?;
}
Ok(())
}
}
#[derive(Clone)]
pub struct Options {
pub slots: usize,
pub poll_interval: Duration,
pub shutdown_timeout: Duration,
}
impl Default for Options {
fn default() -> Self {
Self {
slots: 4,
poll_interval: Duration::from_millis(250),
shutdown_timeout: Duration::from_secs(45),
}
}
}
/// Must configure the matching coworker, model connections and provider registry
/// before returning. Use a session-private local controller directory, not the
/// remote workspace. The supplied store already owns exactly this session.
pub type Factory = Arc<dyn Fn(&str, PostgresStore) -> Result<Arc<Runtime>> + Send + Sync>;
pub type Catalog = Arc<dyn Fn() -> Result<Vec<(Selector, Factory)>> + Send + Sync>;
pub struct Worker {
catalog: Option<Catalog>,
config: Config,
selector: Selector,
options: Options,
factory: Factory,
}
impl Worker {
pub fn new(
config: Config,
selector: Selector,
options: Options,
factory: Factory,
) -> Result<Self> {
config.validate()?;
selector.validate()?;
if selector.environment_provider.is_none() && !cfg!(feature = "test-support") {
bail!(
"Server worker requires a bound environment; local execution is test-support only"
);
}
if !(1..=32).contains(&options.slots)
|| !(Duration::from_millis(50)..=Duration::from_secs(5))
.contains(&options.poll_interval)
|| !(Duration::from_secs(1)..=Duration::from_secs(120))
.contains(&options.shutdown_timeout)
|| config.lease_duration < config.timeout * 5
{
bail!("Invalid worker bounds; lease must be at least five database timeouts");
}
Ok(Self {
config,
selector,
options,
factory,
catalog: None,
})
}
pub fn with_catalog(mut self, catalog: Catalog) -> Self {
self.catalog = Some(catalog);
self
}
/// Stops on the first database/lease/runtime failure. A poisoned connection
/// is never silently reopened to replay work. Deployment supervision may
/// restart the worker; new incarnations inspect durable state after expiry.
pub async fn run(self, shutdown: CancellationToken) -> Result<()> {
self.run_with_drain(Drain::default(), shutdown, Duration::from_secs(300))
.await
}
/// Planned drain leaves running execution alone until it completes/parks or
/// the deadline expires. `shutdown` remains an immediate interruption signal.
/// Storage/lease failures always interrupt immediately. The drain deadline
/// starts at request observation, INCLUDING time spent awaiting admission.
pub async fn run_with_drain(
self,
drain: Drain,
shutdown: CancellationToken,
timeout: Duration,
) -> Result<()> {
anyhow::ensure!(
(Duration::from_secs(1)..=Duration::from_secs(3600)).contains(&timeout),
"Drain timeout must be 1–3600 seconds"
);
let stop = shutdown.child_token();
let _stop_on_drop = StopOnDrop(stop.clone());
let _watch = DrainWatch(tokio::spawn(watch_drain(
drain.clone(),
stop.clone(),
timeout,
)));
let occupied = Arc::new(AtomicU64::new(0));
let _metrics = WorkerMetrics::start(
self.config.telemetry.clone(),
self.options.slots as u64,
occupied.clone(),
drain.clone(),
);
let mut slots = JoinSet::new();
for _ in 0..self.options.slots {
let (config, selector, options, factory, stop, occupied, drain) = (
self.config.clone(),
self.selector.clone(),
self.options.clone(),
self.factory.clone(),
stop.clone(),
occupied.clone(),
drain.clone(),
);
let catalog = self.catalog.clone();
slots.spawn(async move {
slot(
config, selector, options, factory, stop, occupied, drain, catalog,
)
.await
});
}
let mut error = None;
while let Some(result) = slots.join_next().await {
match result {
Ok(Ok(())) => {}
Ok(Err(e)) => {
error.get_or_insert(e);
stop.cancel();
}
Err(e) => {
error.get_or_insert(e.into());
stop.cancel();
}
}
}
if drain.is_requested() && !stop.is_cancelled() {
let _permit = drain.0.gate.write().await;
drain.0.draining.store(true, Ordering::Release);
drain.0.acknowledged.cancel();
}
if let Some(e) = error {
Err(e)
} else if drain.0.expired.load(Ordering::Acquire) {
bail!("Worker drain deadline expired; interrupted execution, inspect durable results before retrying")
} else {
Ok(())
}
}
}
struct Occupied(Arc<AtomicU64>);
impl Occupied {
fn enter(n: Arc<AtomicU64>) -> Self {
n.fetch_add(1, Ordering::Relaxed);
Self(n)
}
}
impl Drop for Occupied {
fn drop(&mut self) {
self.0.fetch_sub(1, Ordering::Relaxed);
}
}
struct WorkerMetrics {
telemetry: Telemetry,
task: Option<tokio::task::JoinHandle<()>>,
}
fn report_worker(t: &Telemetry, capacity: u64, active: u64, draining: bool) {
t.observe_many(&[
(Gauge::WorkerCapacity, capacity),
(Gauge::WorkerActive, active),
(Gauge::WorkerDraining, u64::from(draining)),
(
Gauge::WorkerHeartbeat,
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs(),
),
]);
}
impl WorkerMetrics {
fn start(telemetry: Telemetry, capacity: u64, occupied: Arc<AtomicU64>, drain: Drain) -> Self {
let task = if telemetry.enabled() {
let t = telemetry.clone();
Some(tokio::spawn(async move {
let mut tick = interval(Duration::from_secs(5));
tick.set_missed_tick_behavior(MissedTickBehavior::Skip);
loop {
tick.tick().await;
report_worker(
&t,
if drain.is_requested() { 0 } else { capacity },
occupied.load(Ordering::Relaxed),
drain.is_acknowledged(),
);
}
}))
} else {
None
};
Self { telemetry, task }
}
}
impl Drop for WorkerMetrics {
fn drop(&mut self) {
if let Some(task) = &self.task {
task.abort();
report_worker(&self.telemetry, 0, 0, false);
}
}
}
#[allow(clippy::too_many_arguments)] // Explicit worker ownership, limits and cancellation handles.
async fn slot(
config: Config,
selector: Selector,
options: Options,
factory: Factory,
stop: CancellationToken,
occupied: Arc<AtomicU64>,
drain: Drain,
catalog: Option<Catalog>,
) -> Result<()> {
let mut turn = 0usize;
let mut spare = Some(PostgresStore::open(config.clone())?);
let mut tick = interval(options.poll_interval);
tick.set_missed_tick_behavior(MissedTickBehavior::Skip);
loop {
tokio::select! {biased; _=stop.cancelled()=>return Ok(()), _=tick.tick()=>{} }
let permit = drain.0.gate.read().await;
if drain.is_requested() {
return Ok(());
}
let store = spare.take().context("Worker slot missing store")?;
let mut selected = None;
let candidates = match &catalog {
Some(get) => get()?,
None => vec![(selector.clone(), factory.clone())],
};
for offset in 0..candidates.len() {
let index = (turn + offset) % candidates.len();
let (selector, factory) = &candidates[index];
selector.validate()?;
if let Some(lease) = store.claim_next(selector.clone())? {
selected = Some((lease, selector.clone(), factory.clone()));
turn = index + 1;
break;
}
}
let claim = selected;
drop(permit);
let Some((lease, selector, factory)) = claim else {
spare = Some(store);
continue;
};
if drain.is_requested() {
store.release(&lease.session_id)?;
return Ok(());
}
let _occupied = Occupied::enter(occupied.clone());
execute(
&lease.session_id,
store,
&config,
&selector,
&options,
&factory,
Lifecycle {
stop: &stop,
drain: &drain,
},
)
.await?;
if stop.is_cancelled() {
return Ok(());
}
spare = Some(PostgresStore::open(config.clone())?);
}
}
struct StopOnDrop(CancellationToken);
impl Drop for StopOnDrop {
fn drop(&mut self) {
self.0.cancel();
}
}
struct RuntimeStop(Arc<Runtime>);
impl Drop for RuntimeStop {
fn drop(&mut self) {
self.0.request_shutdown();
}
}
#[derive(Clone, Copy)]
struct Lifecycle<'a> {
stop: &'a CancellationToken,
drain: &'a Drain,
}
async fn execute(
sid: &str,
store: PostgresStore,
config: &Config,
selector: &Selector,
options: &Options,
factory: &Factory,
lifecycle: Lifecycle<'_>,
) -> Result<()> {
let span = config.telemetry.span(
openworker_runtime::telemetry::Operation::Worker,
Some(sid),
None,
);
let result = openworker_runtime::telemetry::scope(
span.context(),
execute_inner(sid, store, config, selector, options, factory, lifecycle),
)
.await;
span.finish(if result.is_ok() {
openworker_runtime::telemetry::Outcome::Ok
} else {
openworker_runtime::telemetry::Outcome::Error
});
result
}
async fn execute_inner(
sid: &str,
store: PostgresStore,
config: &Config,
selector: &Selector,
options: &Options,
factory: &Factory,
Lifecycle { stop, drain }: Lifecycle<'_>,
) -> Result<()> {
let heartbeat_stop = CancellationToken::new();
let guard = StopOnDrop(heartbeat_stop.clone());
let renew = store.clone();
let heartbeat_sid = sid.to_owned();
let period = config.lease_duration / 4;
let mut heartbeat = tokio::spawn(async move {
let mut tick = interval(period);
tick.set_missed_tick_behavior(MissedTickBehavior::Skip);
loop {
tokio::select! {biased; _=heartbeat_stop.cancelled()=>return anyhow::Ok(()),_=tick.tick()=>{renew.renew(&heartbeat_sid)?;} }
}
});
// Hashing/config validation may do filesystem work; keep heartbeats runnable.
let runtime = match tokio::task::block_in_place(|| factory(sid, store.clone())) {
Ok(rt) => rt,
Err(e) => {
drop(guard);
let _ = heartbeat.await;
return Err(e.context("Worker initialization failed; claim retained until expiry"));
}
};
let _runtime_stop = RuntimeStop(runtime.clone());
let work = async {
if runtime.execution_digest() != selector.bundle_digest {
bail!("Worker factory identity differs from scheduling selector");
}
let mut tick = interval(options.poll_interval);
tick.set_missed_tick_behavior(MissedTickBehavior::Skip);
loop {
tick.tick().await;
let _admission = drain.0.gate.read().await;
if drain.is_requested() && !runtime.is_busy(sid) {
return Ok(());
}
if let Some(control) = store.next_control(sid)? {
if drain.is_requested()
&& matches!(
control.command.action,
openworker_runtime::control::Action::Response { .. }
)
{
continue;
}
// Runtime returns a durable rejected/applied receipt or pending
// while the current job settles. Never resubmit caller payload.
runtime
.apply_control(sid, &control.command.request_id)
.await?;
} else if !runtime.is_busy(sid) {
let metadata = store.clone().metadata(sid)?;
if let Some(member) = &metadata.agent_team {
let doc: openworker_runtime::agent_team::Document = serde_json::from_value(
store
.clone()
.agent_team(sid, "snapshot", &serde_json::json!({}), None)?,
)?;
if doc.roster.instance(&member.agent_id).is_some_and(|i| {
i.phase == openworker_runtime::team_coordination::Phase::Retiring
}) {
if metadata
.turn
.as_ref()
.is_none_or(|t| openworker_runtime::store::terminal(&t.status))
{
if runtime
.dispatch("session/close", serde_json::json!({"session_id":sid}))
.await
.is_ok()
{
store.clone().agent_team(
sid,
"retirement_settle",
&serde_json::json!({"agent_id":member.agent_id}),
None,
)?;
return Ok(());
} else if doc.retirements[&member.agent_id].pending_reason.as_deref()
!= Some("cleanup_not_confirmed")
{
store.clone().agent_team(sid,"retirement_error",&serde_json::json!({"agent_id":member.agent_id,"reason":"cleanup_not_confirmed"}),None)?;
}
} else if metadata
.turn
.as_ref()
.is_some_and(|t| t.status == "resuming_agents")
{
runtime
.dispatch("team/resume", serde_json::json!({"session_id":sid}))
.await?;
continue;
} else if member.waiting.is_some() {
let wake = store.clone().agent_team(
sid,
"wake",
&serde_json::json!({}),
None,
)?;
if wake["resume"] == true {
runtime
.dispatch("team/resume", serde_json::json!({"session_id":sid}))
.await?;
continue; // Keep the owning runtime alive for this continuation.
}
}
return Ok(()); // Parked work keeps durable state, not a worker slot.
}
if doc.stopped {
return Ok(());
}
if metadata
.turn
.as_ref()
.is_some_and(|t| t.status == "resuming_agents")
{
runtime
.dispatch("team/resume", serde_json::json!({"session_id":sid}))
.await?;
continue;
}
let wake =
store
.clone()
.agent_team(sid, "wake", &serde_json::json!({}), None)?;
if wake["resume"] == true {
runtime
.dispatch("team/resume", serde_json::json!({"session_id":sid}))
.await?;
continue;
}
if wake["wake"] == true {
store.enqueue_agent_inbox_wake(sid)?;
}
if metadata
.turn
.as_ref()
.is_some_and(|t| t.status == "waiting_agents")
{
return Ok(());
}
}
if drain.is_requested() {
return Ok(());
}
let next = match store.next_turn(sid) {
Ok(next) => next,
Err(e)
if matches!(
e.downcast_ref::<crate::SubmissionError>(),
Some(
crate::SubmissionError::ProviderBlocked
| crate::SubmissionError::Closed
)
) =>
{
return Ok(())
}
Err(e) => return Err(e),
};
if let Some(next) = next {
let content = store.submission_input(sid, &next.submission_id)?;
let mut request = serde_json::to_value(next)?;
if let Some(content) = content {
request["content"] = serde_json::to_value(content)?;
}
if let Err(error) = runtime.dispatch("turn/start", request).await {
// A control can arrive after the peek. Acceptance has
// committed nothing: re-read the priority path next tick,
// without restarting this worker or changing task IDs.
if error.downcast_ref::<crate::SubmissionError>()
!= Some(&crate::SubmissionError::ControlsPending)
{
return Err(error);
}
}
} else {
return Ok(());
}
}
}
};
let mut heartbeat_selected = false;
let result = tokio::select! {biased;
_=stop.cancelled()=>Ok(()),
outcome=&mut heartbeat=>{heartbeat_selected=true;match outcome {Ok(Ok(()))=>Err(anyhow::anyhow!("Heartbeat stopped unexpectedly")),Ok(Err(e))=>Err(e.context("Lease renewal failed; stopping worker")),Err(e)=>Err(e.into())}},
r=work=>r,
};
// Cancel already-running futures/close only this session's attachments. A
// lost lease fences writes; it cannot undo an external effect already sent.
let cleanup = tokio::time::timeout(options.shutdown_timeout, runtime.shutdown()).await;
drop(guard);
let heartbeat_cleanup = if !heartbeat_selected {
heartbeat
.await
.context("Heartbeat task failed")
.and_then(|v| v)
} else {
Ok(())
};
result?;
heartbeat_cleanup?;
cleanup.context("Worker shutdown incomplete; host must terminate worker, lease retained")??;
store.release(sid)?;
Ok(())
}
#[cfg(test)]
mod metric_tests {
use super::*;
#[tokio::test]
async fn drain_ack_waits_for_admission_and_deadline_includes_that_wait() {
let drain = Drain::default();
let stop = CancellationToken::new();
let permit = drain.0.gate.read().await;
let task = tokio::spawn(watch_drain(
drain.clone(),
stop.clone(),
Duration::from_secs(1),
));
drain.request();
tokio::task::yield_now().await;
assert!(!drain.is_acknowledged());
tokio::time::timeout(Duration::from_secs(2), stop.cancelled())
.await
.unwrap();
assert!(drain.0.expired.load(Ordering::Acquire));
assert!(!drain.is_acknowledged());
drop(permit);
task.await.unwrap();
}
#[tokio::test]
async fn acknowledgement_follows_inflight_admission_and_abort_is_independent() {
let drain = Drain::default();
let stop = CancellationToken::new();
let permit = drain.0.gate.read().await;
let task = tokio::spawn(watch_drain(
drain.clone(),
stop.clone(),
Duration::from_secs(5),
));
drain.request();
tokio::task::yield_now().await;
assert!(!drain.is_acknowledged());
drop(permit);
tokio::time::timeout(Duration::from_secs(1), drain.acknowledged())
.await
.unwrap();
assert!(drain.is_acknowledged());
assert!(!stop.is_cancelled());
stop.cancel();
task.await.unwrap();
assert!(!drain.0.expired.load(Ordering::Acquire));
}
#[test]
fn occupied_slots_release_on_normal_exit_and_unwind() {
let count = Arc::new(AtomicU64::new(0));
let a = Occupied::enter(count.clone());
let b = Occupied::enter(count.clone());
assert_eq!(count.load(Ordering::Relaxed), 2);
drop(a);
assert_eq!(count.load(Ordering::Relaxed), 1);
let _ = std::panic::catch_unwind(|| {
let _c = Occupied::enter(count.clone());
panic!("fixture");
});
assert_eq!(count.load(Ordering::Relaxed), 1);
drop(b);
assert_eq!(count.load(Ordering::Relaxed), 0);
}
}