Skip to content
Python
30.7 KB
Raw Blame
//! Service management never runs tools in the API. Definitions select from the
//! operator-installed bundle; controller and existing workers own all execution.
use crate::{
    http::{ApiError, App},
    Config,
};
use anyhow::{ensure, Context, Result};
use axum::{
    extract::Path,
    http::{HeaderMap, StatusCode},
    Extension, Json,
};
use openworker_runtime::store::StateStore;
use openworker_runtime::{
    bundle::{digest, Bundle},
    environment::{Management, Provider},
    runtime::Runtime,
};
use openworker_store_postgres::{
    service_catalog::{Admission, Entry, Profile, Release},
    PostgresStore, SubmissionError,
};
use serde::Deserialize;
use serde_json::{json, Value};
use std::sync::Arc;

pub fn installation(config: &Config) -> Result<String> {
    let base = config.load_bundle()?;
    config.validate_git_grants()?;
    config.validate_repository_grants()?;
    let mut identity = json!({"bundle":base.digest,"runner":digest(&std::fs::read(&config.runner)?),"gateway":config.openshell.endpoint,"gatewayIdentity":config.openshell.identity,"workspace":config.openshell.workspace,"image":config.openshell.image});
    if let Some(registry) = &base.registry_digest {
        identity["registry"] = json!(registry);
    }
    if !config.openshell.git_grants.is_empty() {
        identity["gitGrants"] = json!(config.openshell.git_grants);
    }
    if !config.repository_grants.is_empty() {
        identity["repositoryGrants"] = json!(config.repository_grants);
    }
    Ok(digest(&serde_json::to_vec(&identity)?))
}
pub fn provider(
    config: &Config,
    profile: &Entry,
) -> Result<Arc<openworker_runtime::environment_openshell::OpenShell>> {
    ensure!(
        profile.installation == installation(config)?,
        "Installed service bundle/provider changed; new release required"
    );
    let p: Profile =
        serde_json::from_value(profile.definition.clone()).map_err(anyhow::Error::from)?;
    p.validate()?;
    let c = config.profile_config(&p)?;
    c.provider()
}
fn copy_tree(from: &std::path::Path, to: &std::path::Path) -> Result<()> {
    std::fs::create_dir_all(to)?;
    for e in std::fs::read_dir(from)? {
        let e = e?;
        let ty = e.file_type()?;
        ensure!(!ty.is_symlink(), "Installed skill symlink rejected");
        if ty.is_dir() {
            copy_tree(&e.path(), &to.join(e.file_name()))?;
        } else {
            std::fs::copy(e.path(), to.join(e.file_name()))?;
        }
    }
    Ok(())
}
/// Content-addressed installed bundle cache. Generated files contain no secrets.
pub fn release_config(config: &Config, release: &Entry, profile: &Entry) -> Result<Config> {
    ensure!(
        release.installation == installation(config)?
            && release.installation == profile.installation,
        "Release installation changed"
    );
    let r: Release =
        serde_json::from_value(release.definition.clone()).map_err(anyhow::Error::from)?;
    r.validate()?;
    let p: Profile =
        serde_json::from_value(profile.definition.clone()).map_err(anyhow::Error::from)?;
    p.validate()?;
    ensure!(
        r.runtime_profiles.contains(&p.id),
        "Release/profile incompatible"
    );
    ensure!(
        !r.subagents.as_ref().is_some_and(|c| c.enabled)
            || p.sharing == openworker_runtime::environment::Sharing::Shared,
        "Standalone sub-agents require a shared RuntimeProfile"
    );
    let base = config.load_bundle()?;
    let mut manifest = base.manifest.clone();
    manifest.id = format!("svc-{}", &release.digest[..24]);
    manifest.name = r.id.clone();
    manifest.version = release.digest.clone();
    manifest.default_model = r.default_model;
    manifest.limits = r.limits;
    manifest.standalone_subagents = r.subagents;
    manifest.memory_scope = None;
    let tools = base.tools.select(&r.tools)?;
    let manifest_text = format!(
        "---\n{}\n---\n{}\n",
        serde_json::to_string(&manifest)?,
        r.instructions
    );
    let tools_text = base.selection_text(&tools)?;
    let dir = publish_bundle(config, &release.digest, &base, &manifest_text, &tools_text)?;
    config.repository_grant(&p)?;
    let mut c = config.profile_config(&p)?;
    c.bundle = dir;
    c.openshell.cpu = Some(p.cpu);
    c.openshell.memory_mb = Some(p.memory_mb);
    Ok(c)
}
/// Both published releases and scoped types use deterministic, verified files.
pub(crate) fn publish_bundle(
    config: &Config,
    key: &str,
    base: &Bundle,
    manifest_text: &str,
    tools_text: &str,
) -> Result<std::path::PathBuf> {
    ensure!(
        key.len() == 64 && key.bytes().all(|b| b.is_ascii_hexdigit()),
        "Invalid cache key"
    );
    let root = config.state_root.join("service-bundles");
    std::fs::create_dir_all(&root)?;
    let dir = root.join(key);
    if !dir.exists() {
        let tmp = root.join(openworker_runtime::id("prepare"));
        std::fs::create_dir_all(tmp.join("coworker"))?;
        std::fs::write(tmp.join("coworker/manifest.md"), manifest_text)?;
        std::fs::write(tmp.join("tools.json"), tools_text)?;
        if base.root.join("coworker/skills").exists() {
            copy_tree(
                &base.root.join("coworker/skills"),
                &tmp.join("coworker/skills"),
            )?;
        }
        if let Err(e) = std::fs::rename(&tmp, &dir) {
            std::fs::remove_dir_all(&tmp)?;
            ensure!(dir.exists(), "Cannot publish generated bundle: {e}");
        }
    }
    ensure!(
        !std::fs::symlink_metadata(&dir)?.file_type().is_symlink(),
        "Bundle cache symlink rejected"
    );
    ensure!(
        std::fs::read_to_string(dir.join("coworker/manifest.md"))? == manifest_text
            && std::fs::read_to_string(dir.join("tools.json"))? == tools_text,
        "Installed release cache changed"
    );
    Ok(dir)
}

pub(crate) async fn put_profile(
    Extension(app): Extension<App>,
    headers: HeaderMap,
    Json(p): Json<Profile>,
) -> Result<Json<Value>, ApiError> {
    p.validate()?;
    app.config.profile_config(&p)?;
    Ok(Json(json!(app.store.catalog_put(
        &crate::service::key(&app, &headers, "profile")?,
        "profile",
        json!(p),
        &installation(&app.config)?
    )?)))
}
pub(crate) async fn put_release(
    Extension(app): Extension<App>,
    headers: HeaderMap,
    Json(r): Json<Release>,
) -> Result<Json<Value>, ApiError> {
    r.validate()?;
    if app.service.as_ref().is_some_and(|s| s.agent == r.id) {
        return Err(ApiError(SubmissionError::Conflict.into()));
    }
    let base = app.config.load_bundle()?;
    if base.tools.select(&r.tools).is_err() {
        return Err(ApiError(SubmissionError::InvalidRequest.into()));
    }
    for m in r
        .default_model
        .iter()
        .chain(r.reviewer_model.iter())
        .chain(&r.model_overrides)
    {
        m.validate()?;
        app.runtime.connections.resolve(m)?;
    }
    for p in &r.runtime_profiles {
        app.store.catalog_get("profile", p)?;
    }
    Ok(Json(json!(app.store.catalog_put(
        &crate::service::key(&app, &headers, "release")?,
        "release",
        json!(r),
        &installation(&app.config)?
    )?)))
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Activate {
    pub alias: Option<String>,
}
pub(crate) async fn activate_profile(
    Extension(app): Extension<App>,
    Path(id): Path<String>,
    headers: HeaderMap,
    Json(a): Json<Activate>,
) -> Result<Json<Value>, ApiError> {
    Ok(Json(json!(app.store.catalog_activate(
        &crate::service::key(&app, &headers, "activate-profile")?,
        "profile",
        &id,
        a.alias
    )?)))
}
pub(crate) async fn activate_release(
    Extension(app): Extension<App>,
    Path(id): Path<String>,
    headers: HeaderMap,
    Json(a): Json<Activate>,
) -> Result<Json<Value>, ApiError> {
    if a.alias
        .as_ref()
        .is_some_and(|alias| app.service.as_ref().is_some_and(|s| s.agent == *alias))
    {
        return Err(ApiError(SubmissionError::Conflict.into()));
    }
    Ok(Json(json!(app.store.catalog_activate(
        &crate::service::key(&app, &headers, "activate-release")?,
        "release",
        &id,
        a.alias
    )?)))
}
pub(crate) async fn get_profile(
    Extension(app): Extension<App>,
    Path(id): Path<String>,
) -> Result<Json<Value>, ApiError> {
    Ok(Json(json!(app.store.catalog_get("profile", &id)?)))
}
pub(crate) async fn get_environment(
    Extension(app): Extension<App>,
    Path(id): Path<String>,
) -> Result<Json<Value>, ApiError> {
    Ok(Json(app.store.service_environment(&id)?))
}
pub(crate) async fn get_operation(
    Extension(app): Extension<App>,
    Path(id): Path<String>,
) -> Result<Json<Value>, ApiError> {
    Ok(Json(json!(app.store.service_operation(&id)?)))
}
pub(crate) async fn admit(
    app: App,
    headers: HeaderMap,
    input: Admission,
) -> Result<(StatusCode, Json<Value>), ApiError> {
    admit_kind(app, headers, input, "session").await
}
pub(crate) async fn admit_kind(
    app: App,
    headers: HeaderMap,
    mut input: Admission,
    kind: &str,
) -> Result<(StatusCode, Json<Value>), ApiError> {
    let release = app.store.catalog_get("release", &input.agent)?;
    let r: Release =
        serde_json::from_value(release.definition.clone()).map_err(anyhow::Error::from)?;
    let implicit_default = input.runtime_profile.is_none()
        && input.environment_id.is_none()
        && r.default_runtime_profile.is_none();
    let profile_id = if let Some(env) = &input.environment_id {
        app.store.service_environment(env)?["runtimeProfile"]
            .as_str()
            .context("Environment profile missing")?
            .to_owned()
    } else {
        input
            .runtime_profile
            .clone()
            .or(r.default_runtime_profile)
            .unwrap_or_else(|| "default".into())
    };
    let profile = match app.store.catalog_get("profile", &profile_id) {
        Ok(p) => p,
        Err(e)
            if implicit_default
                && profile_id == "default"
                && e.downcast_ref::<SubmissionError>() == Some(&SubmissionError::NotFound) =>
        {
            let default = app
                .service
                .as_ref()
                .and_then(|s| s.default_runtime_profile.as_ref())
                .ok_or_else(|| ApiError(SubmissionError::InvalidRequest.into()))?;
            app.store.catalog_get("profile", default)?
        }
        Err(e) => return Err(e.into()),
    };
    let p: Profile =
        serde_json::from_value(profile.definition.clone()).map_err(anyhow::Error::from)?;
    let spec = provider(&app.config, &profile)?.spec(p.sharing, Management::Harness);
    let request = crate::service::key(&app, &headers, kind)?;
    if let Some(text) = &input.initial_message {
        input.initial_message =
            Some(crate::service_safety::inspect(&app, None, &request, text).await?);
    }
    let op = app
        .store
        .service_admit(&request, input, release, profile, spec)?;
    Ok((
        if op.status == "complete" {
            StatusCode::OK
        } else {
            StatusCode::ACCEPTED
        },
        Json(json!({"operation":op})),
    ))
}
/// Private controller. A claim permits one create only; all subsequent visits
/// inspect the original remote name/UID and never issue another create.
pub async fn control_once(config: &Config, store: &PostgresStore) -> Result<bool> {
    control_matching(config, store, &installation(config)?).await
}
async fn control_matching(config: &Config, store: &PostgresStore, installed: &str) -> Result<bool> {
    let Some((op, create)) = store.service_claim_for(Some(installed))? else {
        return Ok(false);
    };
    let work:Result<()>=async {
        let rel=store.catalog_get("release",&op.release_id)?;let profile=store.catalog_get("profile",&op.profile_id)?;
        let provider=provider(config,&profile)?;let mut env=store.clone().environment(&op.environment_id)?;
        if op.status!="ready"{
            let result=if create {provider.create(env.clone()).await}else{
                let raw=env.id.strip_prefix("env_").context("Bad environment ID")?;
                provider.bind_existing(&env,&format!("owr-{}",&raw[..15])).await
            };
            match result {Ok(binding)=>{env=store.complete_environment(&env.id,env.revision,Some(binding))?;store.service_update_operation(&op.id,"ready",None)?;},Err(_)=>{store.complete_environment(&env.id,env.revision,None)?;store.service_update_operation(&op.id,"unknown",Some("Provisioning outcome unknown; inspecting original environment only".into()))?;return Ok(());}}
        }
        provider.inspect(env.binding.clone().context("Environment not ready")?).await?;
        let c=release_config(config,&rel,&profile)?;let rt=c.api_runtime(&format!("admit-{}",&op.id[3..19]),store.clone())?;
        let input=store.service_input(&op.id)?;let r:Release=serde_json::from_value(rel.definition.clone())?;let p:Profile=serde_json::from_value(profile.definition.clone())?;
        let mut parameters=json!({"coworker":rt.coworker(),"workspace":"/sandbox","environment_id":env.id,"name":input.name.unwrap_or(op.release_id),"permissions":{"mode":p.approval_mode,"auto_skip_questions":p.auto_skip_questions}});
        if let Some(model)=input.model{parameters["model"]=json!(model);}
        if let Some(model)=r.reviewer_model{parameters["reviewer_model"]=json!(model);}
        let configured=store.service_operation_configuration(&op.id)?;parameters["compaction"]=json!(configured.configuration.compaction);
        let session=rt.prepare_session(parameters)?;
        let service=config.service.as_ref().context("Service config required")?;
        let result=store.service_finish(&op.id,session,&service.id,&service.namespace);rt.shutdown().await?;result?;Ok(())
    }.await;
    if let Err(error) = work {
        if error
            .downcast_ref::<openworker_store_postgres::auth::Error>()
            .is_some()
        {
            store.service_update_operation(
                &op.id,
                "failed",
                Some("Admission authorization no longer valid".into()),
            )?;
        } else {
            return Err(error);
        }
    }
    Ok(true)
}
pub async fn controller(
    config: Config,
    store: PostgresStore,
    stop: tokio_util::sync::CancellationToken,
) -> Result<()> {
    config.require_execution()?;
    let service = config
        .service
        .as_ref()
        .context("Service configuration required")?;
    service.validate()?;
    store.bind_service(&service.id, &service.namespace)?;
    let installed = installation(&config)?;
    let mut type_cursor = String::new();
    let mut placement_cursor = String::new();
    // Bounded source I/O has its own task so a slow bundle cannot block
    // cancellation, environment cleanup or other control-plane reconciliation.
    // JoinSet aborts this task if its owning controller exits unexpectedly.
    let mut source_tasks = tokio::task::JoinSet::new();
    if let Some(artifacts) = config.artifacts.as_ref().map(|c| c.build()).transpose()? {
        #[cfg(feature = "trusted-git")]
        if !config.repository_grants.is_empty() {
            let (c, s, i, a, cancel) = (
                config.clone(),
                store.clone(),
                installed.clone(),
                artifacts.clone(),
                stop.clone(),
            );
            source_tasks.spawn(async move {while !cancel.is_cancelled(){if !crate::service_team_publications::prepare_once(&c,&s,&i,&a,&cancel).await?{tokio::select!{_=cancel.cancelled()=>break,_=tokio::time::sleep(std::time::Duration::from_millis(500))=>{}}}}Ok::<_,anyhow::Error>(())});
        }
        let (config, store, installed, stop) = (
            config.clone(),
            store.clone(),
            installed.clone(),
            stop.clone(),
        );
        let (import_config, import_store, import_installed, import_stop, import_artifacts) = (
            config.clone(),
            store.clone(),
            installed.clone(),
            stop.clone(),
            artifacts.clone(),
        );
        let (
            submission_config,
            submission_store,
            submission_installed,
            submission_stop,
            submission_artifacts,
        ) = (
            config.clone(),
            store.clone(),
            installed.clone(),
            stop.clone(),
            artifacts.clone(),
        );
        let (integration_config, integration_store, integration_installed, integration_stop) = (
            config.clone(),
            store.clone(),
            installed.clone(),
            stop.clone(),
        );
        source_tasks.spawn(async move {
            while !integration_stop.is_cancelled() {
                if !crate::service_team_integrations::prepare_once(&integration_config,&integration_store,&integration_installed).await? {
                    tokio::select!{_=integration_stop.cancelled()=>break,_=tokio::time::sleep(std::time::Duration::from_millis(500))=>{}}
                }
            }
            Ok::<_,anyhow::Error>(())
        });
        source_tasks.spawn(async move {
            while !submission_stop.is_cancelled() {
                if !crate::service_team_submissions::prepare_once(&submission_config,&submission_store,&submission_installed,Some(&submission_artifacts),&submission_stop).await? {
                    tokio::select! {_=submission_stop.cancelled()=>break,_=tokio::time::sleep(std::time::Duration::from_millis(500))=>{}}
                }
            }
            Ok::<_,anyhow::Error>(())
        });
        source_tasks.spawn(async move {
            while !stop.is_cancelled() {
                if !crate::service_team_sources::prepare_once(&config, &store, &installed, Some(&artifacts), &stop).await? {
                    tokio::select! { _=stop.cancelled()=>break, _=tokio::time::sleep(std::time::Duration::from_millis(500))=>{} }
                }
            }
            Ok::<_,anyhow::Error>(())
        });
        source_tasks.spawn(async move {
            while !import_stop.is_cancelled() {
                if !crate::service_team_imports::prepare_once(&import_config,&import_store,&import_installed,&import_artifacts,&import_stop).await? {
                    tokio::select! { _=import_stop.cancelled()=>break, _=tokio::time::sleep(std::time::Duration::from_millis(500))=>{} }
                }
            }
            Ok::<_,anyhow::Error>(())
        });
    }
    while !stop.is_cancelled() {
        crate::service_team_types::refresh(&config, &store, &installed, &mut type_cursor).await?;
        crate::service_team_placements::refresh(&config, &store, &installed, &mut placement_cursor)
            .await?;
        if let Some(result) = source_tasks.try_join_next() {
            result??;
        }
        crate::service_team_workspaces::prepare_once(&config, &store, &installed).await?;
        delete_once(&config, &store, &installed).await?;
        store.provision_agent_team_child_for(Some(&installed))?;
        store.reconcile_stopped_agent_team()?;
        store.reconcile_unstarted_retirement()?;
        if !control_matching(&config, &store, &installed).await? {
            tokio::select! {_=stop.cancelled()=>break,_=tokio::time::sleep(std::time::Duration::from_millis(500))=>{}}
        }
    }
    source_tasks.shutdown().await;
    Ok(())
}

pub fn selector(
    config: &Config,
    rt: &Arc<Runtime>,
) -> Result<openworker_store_postgres::worker::Selector> {
    let env = config.provider()?.spec(
        openworker_runtime::environment::Sharing::Exclusive,
        Management::Harness,
    );
    Ok(openworker_store_postgres::worker::Selector {
        coworker: rt.coworker().into(),
        bundle_digest: rt.execution_digest().into(),
        backend: openworker_runtime::execution::Backend::Openshell,
        scripted: config.scripted,
        fallback_model: config.model.clone(),
        model_provider: config.model_provider,
        model_connections: config
            .model_connections
            .iter()
            .map(|c| {
                Ok((
                    c.id.clone(),
                    rt.connections
                        .resolve(&openworker_runtime::connections::Choice::new(
                            &c.id,
                            "validation",
                        ))?
                        .identity,
                ))
            })
            .collect::<Result<_>>()?,
        environment_provider: Some(env.provider),
        environment_identity: Some(env.provider_identity),
        environment_profile: Some(env.profile_digest),
    })
}
pub async fn worker(
    config: Config,
    store: PostgresStore,
    slots: usize,
    stop: tokio_util::sync::CancellationToken,
) -> Result<()> {
    worker_with_options(
        config,
        store,
        slots,
        stop,
        tokio_util::sync::CancellationToken::new(),
        "service-worker".into(),
        None,
        300,
    )
    .await
}
#[allow(clippy::too_many_arguments)] // Worker resource and shutdown ownership are explicit.
pub async fn worker_with_options(
    config: Config,
    store: PostgresStore,
    slots: usize,
    stop: tokio_util::sync::CancellationToken,
    abort: tokio_util::sync::CancellationToken,
    node: String,
    health_listen: Option<std::net::SocketAddr>,
    drain_seconds: u64,
) -> Result<()> {
    use openworker_store_postgres::worker::{Catalog, Factory, Options, Worker};
    let _abort_guard = abort.clone().drop_guard();
    config.require_execution()?;
    anyhow::ensure!(
        health_listen.is_none_or(|a| a.ip().is_loopback()),
        "Worker health must bind loopback"
    );
    anyhow::ensure!(
        openworker_runtime::bundle::safe_name(&node),
        "Invalid worker node name"
    );
    let provider = config.provider()?;
    provider.check_gateway().await?;
    let drain = openworker_store_postgres::worker::Drain::default();
    let health = crate::readiness::Readiness::new(drain.clone());
    let health_stop = tokio_util::sync::CancellationToken::new();
    let _health_guard = health_stop.clone().drop_guard();
    let health_server = if let Some(address) = health_listen {
        let listener = tokio::net::TcpListener::bind(address).await?;
        let router = health.router();
        let done = health_stop.clone();
        Some(tokio::spawn(async move {
            axum::serve(listener, router)
                .with_graceful_shutdown(done.cancelled_owned())
                .await
        }))
    } else {
        None
    };
    let signal_stop = stop.clone();
    let signal_abort = abort.clone();
    let signal_drain = drain.clone();
    let signal = tokio::spawn(async move {
        tokio::select! {_=signal_stop.cancelled()=>signal_drain.request(),_=signal_abort.cancelled()=>{}}
    });

    let service = config
        .service
        .as_ref()
        .context("Service configuration required")?;
    service.validate()?;
    store.bind_service(&service.id, &service.namespace)?;
    let base = config.runtime("service-worker-base", store.clone())?;
    let base_selector = selector(&config, &base)?;
    base.shutdown().await?;
    let cache = Arc::new(std::sync::Mutex::new(std::collections::BTreeMap::<
        String,
        (openworker_store_postgres::worker::Selector, Factory),
    >::new()));
    let cached = cache.clone();
    let catalog: Catalog = Arc::new(move || Ok(cached.lock().unwrap().values().cloned().collect()));
    // Slot connections must retain the process-owned instrumentation. Rebuilding
    // only database credentials here silently disabled model/tool/turn recording
    // for catalog-deployed agents, while the monitor still emitted storage spans.
    let mut db = config.database()?;
    db.telemetry = store.telemetry();
    let default_config = config.clone();
    let default_factory: Factory = Arc::new(move |sid, s| {
        default_config.runtime(&format!("svc-{}", &digest(sid.as_bytes())[..16]), s)
    });
    cache.lock().unwrap().insert(
        "legacy".into(),
        (base_selector.clone(), default_factory.clone()),
    );
    let work = Worker::new(
        db,
        base_selector.clone(),
        Options {
            slots,
            ..Default::default()
        },
        default_factory,
    )?
    .with_catalog(catalog)
    .run_with_drain(
        drain.clone(),
        abort.clone(),
        std::time::Duration::from_secs(drain_seconds),
    );
    let installed = installation(&config)?;
    let monitor = async {
        let mut type_cursor = String::new();
        while !abort.is_cancelled() {
            provider.check_gateway().await?;
            store.advertise_worker(
                &node,
                base_selector.clone(),
                slots as u16,
                drain.is_requested(),
            )?;
            health.checked();
            let entries = store.catalog_list()?;
            for release in entries
                .iter()
                .filter(|e| e.kind == "release" && e.activated && e.installation == installed)
            {
                let r: Release = serde_json::from_value(release.definition.clone())
                    .map_err(anyhow::Error::from)?;
                for pid in r.runtime_profiles {
                    let key = format!("{}:{pid}", release.digest);
                    if cache.lock().unwrap().contains_key(&key) {
                        continue;
                    }
                    let profile = entries
                        .iter()
                        .find(|p| p.kind == "profile" && p.id == pid && p.activated)
                        .context("Active profile missing")?;
                    let c = release_config(&config, release, profile)?;
                    let rt = c.runtime(
                        &format!("template-{}", &digest(key.as_bytes())[..16]),
                        store.clone(),
                    )?;
                    let select = selector(&c, &rt)?;
                    rt.shutdown().await?;
                    let factory: Factory = Arc::new(move |sid, s| {
                        c.runtime(&format!("svc-{}", &digest(sid.as_bytes())[..16]), s)
                    });
                    cache.lock().unwrap().insert(key, (select, factory));
                }
            }
            let page = store.service_team_type_page(&installed, &type_cursor, 8)?;
            if page.is_empty() {
                type_cursor.clear()
            }
            for team in page {
                type_cursor = team.document.id.clone();
                if !team.release.activated || !team.profile.activated {
                    continue;
                }
                let Some(types) = &team.document.type_catalog else {
                    continue;
                };
                for record in
                    types.records.values().flatten().filter(|r| {
                        r.status == openworker_runtime::team_type_catalog::Status::Approved
                    })
                {
                    let runtime_id = record.runtime_id();
                    if !store.service_team_type_ready(&team.document.id, &runtime_id)? {
                        continue;
                    }
                    let mut profiles = std::collections::BTreeSet::from([team.profile.id.clone()]);
                    if team.document.placement_selection {
                        profiles.extend(types.ceiling.profiles.keys().cloned());
                    }
                    for pid in profiles {
                        let key = format!("{runtime_id}:{pid}");
                        if cache.lock().unwrap().contains_key(&key) {
                            continue;
                        }
                        let Some(profile) = entries
                            .iter()
                            .find(|p| p.kind == "profile" && p.id == pid && p.activated)
                        else {
                            continue;
                        };
                        let c = crate::service_team_types::placed_config(
                            &config, &team, record, profile,
                        )?;
                        let rt = c.runtime(
                            &format!("private-{}", &digest(key.as_bytes())[..20]),
                            store.clone(),
                        )?;
                        let select = selector(&c, &rt)?;
                        rt.shutdown().await?;
                        store.verify_service_team_type_runtime(
                            &team.document.id,
                            &runtime_id,
                            &select.bundle_digest,
                        )?;
                        let factory: Factory = Arc::new(move |sid, s| {
                            c.runtime(&format!("svc-{}", &digest(sid.as_bytes())[..16]), s)
                        });
                        cache.lock().unwrap().insert(key, (select, factory));
                    }
                }
            }
            tokio::select! {_=abort.cancelled()=>break,_=tokio::time::sleep(std::time::Duration::from_secs(1))=>{}}
        }
        Ok::<_, anyhow::Error>(())
    };
    tokio::pin!(work);
    tokio::pin!(monitor);
    let result = tokio::select! {result=&mut work=>result,result=&mut monitor=>{health.clear();abort.cancel();let joined=work.await;result.and(joined)}};
    abort.cancel();
    signal.abort();
    health.clear();
    health_stop.cancel();
    if let Some(server) = health_server {
        let _ = server.await;
    }
    result
}

/// The API records intent only; gateway credentials stay on private controllers.
pub(crate) async fn delete_environment(
    Extension(app): Extension<App>,
    Path(id): Path<String>,
    headers: HeaderMap,
) -> Result<Json<Value>, ApiError> {
    app.store.service_environment(&id)?;
    let key = crate::service::key(&app, &headers, &format!("delete-environment:{id}"))?;
    let operation = &key[..32];
    let (record, _) = app
        .store
        .begin_environment_delete(&id, operation, &app.actor)?;
    if record.phase != openworker_runtime::environment::Phase::Deleted {
        app.store.service_queue_delete(&id, operation)?;
    }
    Ok(Json(json!({"id":id,"phase":record.phase})))
}
async fn delete_once(config: &Config, store: &PostgresStore, installed: &str) -> Result<()> {
    let Some((record, key, profile, fresh)) = store.service_claim_delete(installed)? else {
        return Ok(());
    };
    let provider = provider(config, &store.catalog_get("profile", &profile)?)?;
    let binding = record.binding.context("Environment binding missing")?;
    // A lost connection is not retry permission. Only the first claim can delete;
    // after that, read the original object's state and never delete a replacement.
    let deleted = if fresh {
        provider.delete(binding, key.clone()).await.is_ok()
    } else {
        provider.confirm_deleted(binding).await.unwrap_or(false)
    };
    if deleted {
        store.complete_environment_delete(&record.id, &key, record.revision)?;
    } else {
        store.service_recheck_delete(&record.id)?;
    }
    Ok(())
}

pub(crate) async fn get_release(
    Extension(app): Extension<App>,
    Path(id): Path<String>,
) -> Result<Json<Value>, ApiError> {
    Ok(Json(json!(app.store.catalog_get("release", &id)?)))
}