Python
//! Shared kind/verified-EKS installer. The local alias forbids cloud bindings.
//! Namespace UIDs, Helm receipts and input hashes protect existing installations.
use super::*;
use std::{process::Stdio, time::Duration};
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
process::Command as Process,
};
const OWNER: &str = "openworker.io/installer-id";
#[path = "deployment_cluster_platform.rs"]
mod platform;
const CHART_PINS: &[&str] = &[
"5a82e27d6cf4089fa16d96c87d9fc0fe62f65fa05341465bc749ba5ab91abb33", // 0.1.3
"5ce3e63648d1a80f96f748063e44444069667ae9049457b8aded35cdb2a6171a", // 0.1.7 official OCI archive
];
#[derive(Subcommand)]
pub enum Command {
Platform {
#[arg(long)]
state: PathBuf,
#[arg(long)]
accept_plan: String,
#[arg(long)]
allow_experimental_cloud: bool,
},
Plan {
#[arg(long)]
config: PathBuf,
#[arg(long)]
out: PathBuf,
},
Apply {
#[arg(long)]
state: PathBuf,
#[arg(long)]
accept_plan: String,
#[arg(long)]
openshell_only: bool,
/// Only after independently proving an uncertain operation had no effect.
#[arg(long)]
confirm_no_effect: Option<String>,
/// Confirm original hook-free service resources are ready after Helm timed out.
#[arg(long)]
reconcile_ready_release: Option<String>,
},
/// Record the original terminal failed release for cleanup; never reinstall it.
RecordFailedRelease {
#[arg(long)]
state: PathBuf,
#[arg(long)]
accept_plan: String,
#[arg(long)]
release: String,
},
Status {
#[arg(long)]
state: PathBuf,
},
/// Create only declared Secrets in the owned namespace, via private stdin.
Secrets {
#[arg(long)]
state: PathBuf,
#[arg(long)]
input: PathBuf,
#[arg(long)]
accept_plan: String,
},
Destroy {
#[arg(long)]
state: PathBuf,
#[arg(long)]
deployment_id: String,
},
}
#[derive(Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct Config {
schema_version: u16,
kubeconfig: PathBuf,
context: String,
helm: PathBuf,
service: Service,
openshell: Shell,
#[serde(default, skip_serializing_if = "Option::is_none")]
cloud: Option<cloud::Binding>,
#[serde(default, skip_serializing_if = "Option::is_none")]
azure: Option<azure::InstallationBinding>,
#[serde(default, skip_serializing_if = "Option::is_none")]
platform: Option<platform::Artifact>,
}
#[derive(Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct Service {
namespace: String,
release: String,
chart: PathBuf,
values_file: PathBuf,
fixture_database: bool,
}
#[derive(Clone, Serialize, Deserialize)]
#[serde(tag = "mode", rename_all = "lowercase", deny_unknown_fields)]
enum Shell {
#[serde(rename_all = "camelCase")]
Install {
namespace: String,
release: String,
fullname: String,
archive: PathBuf,
values_file: PathBuf,
},
Existing {
namespace: String,
release: String,
fullname: String,
},
}
impl Shell {
fn scope(&self) -> (&str, &str, &str) {
match self {
Self::Install {
namespace,
release,
fullname,
..
}
| Self::Existing {
namespace,
release,
fullname,
} => (namespace, release, fullname),
}
}
}
#[derive(Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct Plan {
schema_version: u16,
id: String,
config: Config,
hashes: BTreeMap<String, String>,
cluster_uid: String,
digest: String,
}
#[derive(Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct Journal {
schema_version: u16,
plan_digest: String,
phase: String,
namespaces: BTreeMap<String, String>,
operations: BTreeMap<String, Operation>,
#[serde(default)]
confirmed_no_effect: Vec<Operation>,
}
#[derive(Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct Operation {
status: String,
request_id: String,
receipt: Option<Value>,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
label_provenance: bool,
}
struct Client<'a> {
c: &'a Config,
}
pub(super) async fn exec(bin: &Path, a: &[String], input: Option<Vec<u8>>) -> Result<Vec<u8>> {
exec_with_env(bin, a, input, &BTreeMap::new()).await
}
pub(super) async fn exec_with_env(
bin: &Path,
a: &[String],
input: Option<Vec<u8>>,
env: &BTreeMap<String, String>,
) -> Result<Vec<u8>> {
let mut child = Process::new(bin)
.envs(env)
.args(a)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true)
.spawn()
.context("Local installer tool could not start")?;
let stdout = child.stdout.take().unwrap();
let stderr = child.stderr.take().unwrap();
let mut stdin = child.stdin.take().unwrap();
async fn bytes<R: tokio::io::AsyncRead + Unpin>(s: R, max: u64) -> std::io::Result<Vec<u8>> {
let mut b = vec![];
s.take(max + 1).read_to_end(&mut b).await?;
Ok(b)
}
let operation = async {
tokio::try_join!(
child.wait(),
bytes(stdout, 8 * 1024 * 1024),
bytes(stderr, 65536),
async move {
if let Some(v) = input {
let v = zeroize::Zeroizing::new(v);
stdin.write_all(&v).await?;
}
drop(stdin);
Ok(())
}
)
};
match tokio::time::timeout(Duration::from_secs(240), operation).await {
Ok(Ok((status, out, err, _))) => {
ensure!(
out.len() <= 8 * 1024 * 1024 && err.len() <= 65536,
"Local tool output limit exceeded; inspect original operation"
);
ensure!(status.success(),"Local tool failed; result may be unknown. Inspect scoped resources before retrying. Tool output withheld to protect credentials");
Ok(out)
}
_ => {
let _ = child.kill().await;
bail!("Local tool timed out/failed; inspect original operation before retrying")
}
}
}
fn args(a: &[&str]) -> Vec<String> {
a.iter().map(|s| s.to_string()).collect()
}
impl Client<'_> {
async fn kube(&self, a: Vec<String>, input: Option<Vec<u8>>) -> Result<Vec<u8>> {
let mut all = args(&[
"--kubeconfig",
self.c
.kubeconfig
.to_str()
.context("Invalid kubeconfig path")?,
"--context",
&self.c.context,
]);
all.extend(a);
exec(Path::new("kubectl"), &all, input).await
}
async fn get(&self, kind: &str, name: &str, namespace: Option<&str>) -> Result<Option<Value>> {
let mut a = args(&["get", kind, name, "--ignore-not-found=true", "-o", "json"]);
if let Some(ns) = namespace {
a.extend(args(&["-n", ns]));
}
let b = self.kube(a, None).await?;
if b.iter().all(u8::is_ascii_whitespace) {
return Ok(None);
}
Ok(Some(
serde_json::from_slice(&b).context("Invalid cluster response")?,
))
}
async fn helm(&self, a: Vec<String>) -> Result<Vec<u8>> {
let mut all = a;
all.extend(args(&[
"--kubeconfig",
self.c.kubeconfig.to_str().unwrap(),
"--kube-context",
&self.c.context,
]));
exec(&self.c.helm, &all, None).await
}
async fn release(&self, ns: &str, name: &str) -> Result<Option<Value>> {
let rows: Value = serde_json::from_slice(
&self
.helm(args(&[
"list", "-n", ns, "--all", "--max", "256", "-o", "json",
]))
.await?,
)?;
let rows = rows.as_array().context("Invalid Helm list")?;
ensure!(rows.len() < 256, "Release list truncated");
if !rows.iter().any(|v| v["name"] == name) {
return Ok(None);
}
let status: Value = serde_json::from_slice(
&self
.helm(args(&["status", name, "-n", ns, "-o", "json"]))
.await?,
)?;
let revision = status["version"]
.as_u64()
.context("Helm revision unavailable")?;
let secret = self
.get(
"secret",
&format!("sh.helm.release.v1.{name}.v{revision}"),
Some(ns),
)
.await?
.context("Helm release receipt missing")?;
ensure!(
secret["metadata"]["uid"]
.as_str()
.is_some_and(|s| !s.is_empty()),
"Helm release UID missing"
);
let mut receipt = json!({"name":name,"namespace":ns,"revision":revision,"status":status["info"]["status"],"description":status["info"]["description"],"uid":secret["metadata"]["uid"]});
if secret["metadata"]["labels"][OWNER].is_string()
&& secret["metadata"]["labels"]["openworker.io/operation-id"].is_string()
{
receipt["installerId"] = secret["metadata"]["labels"][OWNER].clone();
receipt["operationId"] =
secret["metadata"]["labels"]["openworker.io/operation-id"].clone();
}
Ok(Some(receipt))
}
async fn cluster(&self) -> Result<String> {
let ns = self
.get("namespace", "kube-system", None)
.await?
.context("Cluster identity missing")?;
Ok(ns["metadata"]["uid"]
.as_str()
.context("Cluster UID missing")?
.to_string())
}
}
fn read_json(p: &Path) -> Result<Value> {
let parent = p
.parent()
.filter(|p| !p.as_os_str().is_empty())
.unwrap_or(Path::new("."));
Ok(serde_json::from_slice(&super::read_file(
parent,
p.file_name()
.and_then(|s| s.to_str())
.context("Invalid file path")?,
)?)?)
}
fn bounded_name(s: &str) -> bool {
(3..=50).contains(&s.len())
&& s.as_bytes()[0].is_ascii_lowercase()
&& s.as_bytes().last().unwrap().is_ascii_alphanumeric()
&& s.bytes()
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')
}
fn hash_path(path: &Path) -> Result<String> {
let meta = fs::symlink_metadata(path)?;
ensure!(
!meta.file_type().is_symlink(),
"Installer inputs cannot be links"
);
if meta.is_file() {
ensure!(meta.len() <= 4 * 1024 * 1024, "Installer input too large");
return Ok(hash(&fs::read(path)?));
}
ensure!(meta.is_dir(), "Invalid chart input");
fn walk(root: &Path, p: &Path, files: &mut BTreeMap<String, String>) -> Result<()> {
for (index, entry) in fs::read_dir(p)?.enumerate() {
ensure!(index < 200, "Chart directory entry limit exceeded");
let path = entry?.path();
let m = fs::symlink_metadata(&path)?;
ensure!(!m.file_type().is_symlink(), "Chart links forbidden");
if m.is_dir() {
ensure!(
path.components().count() - root.components().count() < 12,
"Chart nesting too deep"
);
walk(root, &path, files)?;
} else {
ensure!(m.is_file() && m.len() <= 1024 * 1024, "Invalid chart file");
files.insert(
path.strip_prefix(root)?
.to_str()
.context("Invalid chart filename")?
.to_owned(),
hash(&fs::read(path)?),
);
ensure!(files.len() <= 200, "Chart file limit exceeded");
}
}
Ok(())
}
let mut files = BTreeMap::new();
walk(path, path, &mut files)?;
ensure!(!files.is_empty(), "Empty chart");
Ok(hash(&serde_json::to_vec(&files)?))
}
fn pinned(s: &str) -> bool {
s.rsplit_once("@sha256:").is_some_and(|(name, h)| {
!name.is_empty()
&& h.len() == 64
&& h.bytes()
.all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase())
})
}
fn inputs(c: &Config) -> Result<BTreeMap<String, String>> {
ensure!(
!(c.cloud.is_some() && c.azure.is_some()),
"Only one cloud installation binding is allowed"
);
ensure!(
c.schema_version == 1
&& (c.cloud.is_some() || c.azure.is_some() || c.context.starts_with("kind-"))
&& c.kubeconfig.is_absolute(),
"Explicit reviewed cluster kubeconfig/context required"
);
let (ns, release, fullname) = c.openshell.scope();
for name in [
&c.service.namespace,
&c.service.release,
ns,
release,
fullname,
] {
ensure!(bounded_name(name), "Invalid namespace/release name");
}
ensure!(
(c.service.namespace.starts_with("ow-service-m4-")
|| ((c.cloud.is_some() || c.azure.is_some()) && c.service.namespace == "ow-service"))
&& c.service.namespace != ns,
"Dedicated local service namespace required"
);
let v = read_json(&c.service.values_file)?;
ensure!(
v["image"].as_str().is_some_and(pinned),
"Service image must be digest pinned"
);
ensure!(
v["config"]["service"]["id"]
.as_str()
.is_some_and(|s| s.starts_with("ows-")
&& s.len() <= 50
&& s.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-')),
"Invalid service identity"
);
let expected = format!("https://{fullname}.{ns}.svc.cluster.local:8080");
ensure!(
v["config"]["openshell"]["endpoint"] == expected,
"Service and gateway endpoint differ"
);
let mut hashes = BTreeMap::from([
("service-chart".into(), hash_path(&c.service.chart)?),
("service-values".into(), hash_path(&c.service.values_file)?),
]);
if c.cloud.is_some() || c.azure.is_some() {
hashes.insert("cluster-kubeconfig".into(), hash_path(&c.kubeconfig)?);
}
if let Some(artifact) = &c.platform {
platform::validate(artifact)?;
hashes.insert("platform".into(), artifact.sha256.clone());
}
if let Shell::Install {
archive,
values_file,
..
} = &c.openshell
{
let sha = hash_path(archive)?;
ensure!(
CHART_PINS.contains(&sha.as_str()),
"OpenShell archive differs from reviewed 0.1.3/0.1.7 pins"
);
let v = read_json(values_file)?;
ensure!(
v["fullnameOverride"] == fullname
&& v["server"]["auth"]["allowUnauthenticatedUsers"] == false,
"OpenShell must use the explicit name and authenticated gateway"
);
for pointer in [
"/gateway/image/digest",
"/sandboxRuntime/image/digest",
"/supervisor/image/digest",
"/sandbox/image/digest",
] {
ensure!(
v.pointer(pointer)
.and_then(Value::as_str)
.is_some_and(|s| pinned(&format!("image@{s}"))),
"Trusted OpenShell images must be digest pinned"
);
}
hashes.insert("openshell-chart".into(), sha);
hashes.insert("openshell-values".into(), hash_path(values_file)?);
}
Ok(hashes)
}
fn digest(p: &Plan) -> Result<String> {
Ok(hash(&serde_json::to_vec(&(
&p.schema_version,
&p.id,
&p.config,
&p.hashes,
&p.cluster_uid,
))?))
}
fn save(root: &Path, name: &str, value: &impl Serialize) -> Result<()> {
let path = root.join(format!(".{name}-{}", uuid::Uuid::new_v4().simple()));
let mut o = fs::OpenOptions::new();
o.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
o.mode(0o600);
}
let mut f = o.open(&path)?;
f.write_all(&json_bytes(value)?)?;
f.sync_all()?;
fs::rename(path, root.join(name))?;
#[cfg(unix)]
fs::File::open(root)?.sync_all()?;
Ok(())
}
fn load(root: &Path) -> Result<(Plan, Journal, fs::File)> {
let m = fs::symlink_metadata(root)?;
ensure!(
m.is_dir() && !m.file_type().is_symlink(),
"Invalid installer state directory"
);
let lock_path = root.join("installer.lock");
if let Ok(meta) = fs::symlink_metadata(&lock_path) {
ensure!(
meta.is_file() && !meta.file_type().is_symlink(),
"Invalid installer lock"
);
}
let mut o = fs::OpenOptions::new();
o.read(true).write(true).create(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
o.mode(0o600);
}
let lock = o.open(lock_path)?;
lock.try_lock()
.map_err(|_| anyhow::anyhow!("Another installer process holds this journal"))?;
let p: Plan = serde_json::from_slice(&super::read_file(root, "local-plan.json")?)?;
ensure!(
p.schema_version == 1 && p.digest == digest(&p)?,
"Local plan digest mismatch"
);
let j: Journal = serde_json::from_slice(&super::read_file(root, "journal.json")?)?;
ensure!(
j.schema_version == 1 && j.plan_digest == p.digest,
"Journal/plan mismatch"
);
Ok((p, j, lock))
}
fn begin(root: &Path, j: &mut Journal, step: &str) -> Result<()> {
ensure!(
!j.operations.contains_key(step),
"Operation already has an intent"
);
j.operations.insert(
step.into(),
Operation {
status: "intent".into(),
request_id: uuid::Uuid::new_v4().to_string(),
receipt: None,
label_provenance: ["openshell-release", "service-release"].contains(&step),
},
);
save(root, "journal.json", j)
}
fn complete(root: &Path, j: &mut Journal, step: &str, receipt: Value) -> Result<()> {
let op = j
.operations
.get_mut(step)
.context("Operation intent missing")?;
op.status = "complete".into();
op.receipt = Some(receipt);
save(root, "journal.json", j)
}
async fn namespace(
client: &Client<'_>,
p: &Plan,
root: &Path,
j: &mut Journal,
name: &str,
service_id: &str,
) -> Result<()> {
let step = format!("namespace:{name}");
let existing = client.get("namespace", name, None).await?;
if let Some(ns) = existing {
ensure!(
ns["metadata"]["labels"][OWNER] == p.id,
"Namespace is not owned by this installer"
);
let uid = ns["metadata"]["uid"]
.as_str()
.context("Namespace UID missing")?;
if let Some(expected) = j.namespaces.get(name) {
ensure!(expected == uid, "Namespace identity changed");
}
ensure!(
j.operations.contains_key(&step),
"Refusing to adopt a namespace without an original intent"
);
j.namespaces.insert(name.into(), uid.into());
complete(root, j, &step, json!({"uid":uid}))?;
return Ok(());
}
ensure!(
!j.operations.contains_key(&step),
"Namespace outcome unknown or owned namespace missing; inspect before retrying"
);
begin(root, j, &step)?;
let ns = json!({"apiVersion":"v1","kind":"Namespace","metadata":{"name":name,"labels":{OWNER:p.id,"openworker.io/qualification":service_id}}});
client
.kube(args(&["create", "-f", "-"]), Some(serde_json::to_vec(&ns)?))
.await?;
let ns = client
.get("namespace", name, None)
.await?
.context("Namespace creation outcome unknown")?;
ensure!(
ns["metadata"]["labels"][OWNER] == p.id,
"Namespace ownership mismatch"
);
let uid = ns["metadata"]["uid"]
.as_str()
.context("Namespace UID missing")?;
j.namespaces.insert(name.into(), uid.into());
complete(root, j, &step, json!({"uid":uid}))
}
async fn owned(client: &Client<'_>, p: &Plan, j: &Journal, name: &str) -> Result<Option<Value>> {
let ns = client.get("namespace", name, None).await?;
if let Some(v) = &ns {
ensure!(
v["metadata"]["labels"][OWNER] == p.id
&& j.namespaces.get(name).map(String::as_str) == v["metadata"]["uid"].as_str(),
"Namespace ownership/UID mismatch"
);
}
Ok(ns)
}
struct ReleaseInstall<'a> {
key: &'a str,
ns: &'a str,
release: &'a str,
chart: &'a Path,
values: &'a Path,
reconcile_ready: bool,
}
async fn install_release(
client: &Client<'_>,
p: &Plan,
root: &Path,
j: &mut Journal,
install: ReleaseInstall<'_>,
) -> Result<()> {
let ReleaseInstall {
key,
ns,
release,
chart,
values,
reconcile_ready,
} = install;
owned(client, p, j, ns)
.await?
.context("Owned namespace missing")?;
if let Some(mut r) = client.release(ns, release).await? {
let op = j
.operations
.get(key)
.context("Refusing to adopt an existing Helm release")?;
ensure!(
if op.label_provenance {
r["installerId"] == p.id && r["operationId"] == op.request_id
} else {
r["description"] == format!("ows:{}:{}", p.id, op.request_id)
},
"Helm release provenance mismatch"
);
if let Some(original) = &op.receipt {
ensure!(
original["uid"] == r["uid"] && original["revision"] == r["revision"],
"Helm release identity/revision changed"
);
}
if r["status"] != "deployed" {
ensure!(
key == "service-release"
&& r["status"] == "failed"
&& (reconcile_ready
|| op
.receipt
.as_ref()
.is_some_and(|v| v["qualifiedAfterTimeout"] == true)),
"Helm release is pending/failed; inspect original release, no automatic reinstall"
);
let details: Value = serde_json::from_slice(
&client
.helm(args(&["status", release, "-n", ns, "-o", "json"]))
.await?,
)?;
ensure!(
details
.get("hooks")
.is_none_or(|v| v.is_null() || v.as_array().is_some_and(Vec::is_empty)),
"Hooks may have unknown effects; failed release requires manual review"
);
ready_service(client, p).await?;
let job = client
.get("job", &format!("{release}-bootstrap"), Some(ns))
.await?
.context("Original bootstrap Job not observed")?;
ensure!(
job["status"]["succeeded"].as_u64().is_some_and(|n| n > 0),
"Bootstrap Job is not complete"
);
r["qualifiedAfterTimeout"] = json!(true);
}
complete(root, j, key, r)?;
return Ok(());
}
ensure!(!j.operations.contains_key(key),"Helm operation outcome unknown: no release observed. Confirm no effect explicitly before reissuing");
begin(root, j, key)?;
let nonce = &j.operations[key].request_id;
client
.helm(args(&[
"install",
release,
chart.to_str().context("Invalid chart path")?,
"-n",
ns,
"-f",
values.to_str().context("Invalid values path")?,
"--description",
&format!("ows:{}:{nonce}", p.id),
"--labels",
&format!("{OWNER}={},openworker.io/operation-id={nonce}", p.id),
"--wait",
"--wait-for-jobs",
"--timeout",
"180s",
]))
.await?;
let r = client
.release(ns, release)
.await?
.context("Helm installation outcome unknown")?;
ensure!(
r["status"] == "deployed" && r["installerId"] == p.id && r["operationId"] == *nonce,
"Helm installation not confirmed"
);
complete(root, j, key, r)
}
async fn prerequisites(client: &Client<'_>, eks: bool) -> Result<()> {
let crd = client
.get("crd", "sandboxes.agents.x-k8s.io", None)
.await?
.context("Agent Sandbox CRD required")?;
ensure!(
crd["spec"]["versions"]
.as_array()
.is_some_and(|a| a.iter().any(|v| v["served"] == true
&& ["v1alpha1", "v1beta1"].contains(&v["name"].as_str().unwrap_or("")))),
"Supported Agent Sandbox API required"
);
let controller = client
.get(
"deployment",
"agent-sandbox-controller",
Some("agent-sandbox-system"),
)
.await?
.context("Local qualification requires the Agent Sandbox controller")?;
ensure!(
controller["status"]["availableReplicas"]
.as_u64()
.is_some_and(|n| n > 0),
"Agent Sandbox controller must be available"
);
let classes: Value = serde_json::from_slice(
&client
.kube(args(&["get", "storageclasses", "-o", "json"]), None)
.await?,
)?;
ensure!(
classes["items"]
.as_array()
.is_some_and(|a| a.iter().any(|s| s["metadata"]["annotations"]
["storageclass.kubernetes.io/is-default-class"]
== "true")),
"Default storage provisioner required"
);
if client.c.azure.is_some() {
azure::installation_storage(&classes)?;
return Ok(());
}
if eks {
let binding = client.c.cloud.as_ref().context("Cloud binding required")?;
let (deployment, _) = cloud::install_scope(binding)?;
let defaults: Vec<_> = classes["items"]
.as_array()
.unwrap()
.iter()
.filter(|s| {
s["metadata"]["annotations"]["storageclass.kubernetes.io/is-default-class"]
== "true"
})
.collect();
ensure!(
defaults.len() == 1
&& defaults[0]["provisioner"] == "ebs.csi.aws.com"
&& defaults[0]["reclaimPolicy"] == "Delete"
&& defaults[0]["volumeBindingMode"] == "WaitForFirstConsumer"
&& defaults[0]["parameters"]["encrypted"] == "true"
&& defaults[0]["parameters"]["tagSpecification_1"]
== format!("ows:deployment-id={deployment}"),
"Cloud default storage must be the sole encrypted, ownership-tagged EBS class"
);
cloud::verify_network(binding).await?;
return Ok(());
}
let ds: Value = serde_json::from_slice(
&client
.kube(args(&["get", "daemonsets", "-A", "-o", "json"]), None)
.await?,
)?;
ensure!(
ds["items"]
.as_array()
.is_some_and(|a| a.iter().any(|s| s["metadata"]["name"] == "calico-node"
&& s["status"]["desiredNumberScheduled"]
.as_u64()
.is_some_and(|n| n > 0 && s["status"]["numberReady"] == n))),
"Local qualification requires ready Calico; functional enforcement must also be probed"
);
Ok(())
}
async fn ready_service(client: &Client<'_>, p: &Plan) -> Result<()> {
let values: Value = read_json(&p.config.service.values_file)?;
let mut roles = vec!["api", "worker", "controller", "platform"];
if values["managementUI"]["enabled"] == true {
roles.push("ui");
}
for role in roles {
client
.kube(
args(&[
"-n",
&p.config.service.namespace,
"rollout",
"status",
&format!("deployment/{}-{role}", p.config.service.release),
"--timeout=180s",
]),
None,
)
.await?;
}
Ok(())
}
async fn quiet_database(client: &Client<'_>, p: &Plan) -> Result<()> {
let ns = &p.config.service.namespace;
if !p.config.service.fixture_database {
let observed: Value = serde_json::from_slice(
&client
.kube(
args(&[
"-n",
ns,
"exec",
&format!("deployment/{}-worker", p.config.service.release),
"-c",
"harness",
"--",
"/bin/sh",
"/opt/openworker/entrypoint.sh",
"service-install-status",
]),
None,
)
.await?,
)?;
ensure!(
observed["schemaVersion"] == 1
&& [
"openSessions",
"liveEnvironments",
"pendingAdmissions",
"queuedTurns"
]
.iter()
.all(|key| observed[*key] == 0),
"Close sessions, delete environments and finish pending admissions before teardown"
);
return Ok(());
}
let exists = client.get("deployment", "postgres", Some(ns)).await?;
if exists.is_none() {
ensure!(
client
.release(ns, &p.config.service.release)
.await?
.is_none(),
"Database unavailable for safe teardown"
);
return Ok(());
}
let v = read_json(&p.config.service.values_file)?;
let schema = v["config"]["schema"]
.as_str()
.context("Schema unavailable")?;
ensure!(
schema.starts_with("ow_")
&& schema.len() <= 48
&& schema
.bytes()
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_'),
"Invalid fixture schema"
);
let sql = |q: &str| {
args(&[
"-n",
ns,
"exec",
"deployment/postgres",
"--",
"psql",
"-U",
"openworker",
"-d",
"openworker",
"-Atc",
q,
])
};
let exists=String::from_utf8(client.kube(sql(&format!("SELECT to_regclass('{schema}.sessions') IS NOT NULL, to_regclass('{schema}.environments') IS NOT NULL")),None).await?)?;
if exists.trim() == "f|f" {
return Ok(());
}
ensure!(
exists.trim() == "t|t",
"Partial schema requires manual cleanup review"
);
let q=format!("SELECT (SELECT count(*) FROM {schema}.environments WHERE data->>'phase' IS DISTINCT FROM 'deleted')+(SELECT count(*) FROM {schema}.sessions WHERE NOT COALESCE((data->>'closed')::boolean,FALSE))");
let count = String::from_utf8(client.kube(sql(&q), None).await?)?;
ensure!(
count.trim() == "0",
"Close sessions and delete their environments before teardown"
);
Ok(())
}
async fn stage_secrets(
client: &Client<'_>,
p: &Plan,
root: &Path,
j: &mut Journal,
input: &Path,
) -> Result<()> {
owned(client, p, j, &p.config.service.namespace)
.await?
.context("Owned service namespace missing")?;
let meta = fs::symlink_metadata(input)?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
ensure!(
meta.permissions().mode() & 0o077 == 0,
"Secret input must be private (0600)"
);
}
ensure!(
meta.is_file() && !meta.file_type().is_symlink(),
"Secret input must be a private regular file"
);
let mut values = read_json(input)?;
let object = values.as_object().context("Invalid secret input")?;
ensure!(
object
.keys()
.all(|k| ["database", "apiTLS", "model"].contains(&k.as_str())),
"Only database, apiTLS and model inputs are accepted"
);
if let Some(binding) = &p.config.cloud {
ensure!(
values["database"].get("database-url").is_none(),
"Cloud database URL comes from the bound managed secret"
);
values["database"]["database-url"] = json!(cloud::database_url(binding).await?);
}
if let Some(binding) = &p.config.azure {
azure::installation_database(
binding,
values["database"]["database-url"]
.as_str()
.context("Azure database URL required in private secret input")?,
)?;
}
for (group, required, allowed) in [
(
"database",
vec!["database-url", "pg-ca.crt"],
vec!["database-url", "pg-ca.crt"],
),
(
"apiTLS",
vec!["tls.crt", "tls.key"],
vec!["tls.crt", "tls.key", "ca.crt"],
),
] {
let data = values[group]
.as_object()
.context("Required secret group missing")?;
ensure!(
required.iter().all(|k| data
.get(*k)
.is_some_and(|v| v.as_str().is_some_and(|s| !s.is_empty())))
&& data.keys().all(|k| allowed.contains(&k.as_str())),
"Secret group has missing/unsupported entries"
);
}
let v = read_json(&p.config.service.values_file)?;
let secret_name = |key: &str, default: &str| -> Result<String> {
let name = v["secrets"][key].as_str().unwrap_or(default);
ensure!(bounded_name(name), "Invalid declared secret name");
Ok(name.into())
};
let (ns, release, _) = p.config.openshell.scope();
let peer = client
.release(ns, release)
.await?
.context("Gateway release absent")?;
let receipt = j
.operations
.get(if matches!(p.config.openshell, Shell::Existing { .. }) {
"external-openshell"
} else {
"openshell-release"
})
.and_then(|o| o.receipt.as_ref())
.context("Gateway identity not confirmed")?;
ensure!(peer == *receipt, "Gateway release identity changed");
let tls = client
.get("secret", "openshell-client-tls", Some(ns))
.await?
.context("Gateway client TLS unavailable")?;
let material = tls["data"]
.as_object()
.context("Gateway TLS material missing")?;
ensure!(
["ca.crt", "tls.crt", "tls.key"].iter().all(|k| material
.get(*k)
.is_some_and(|v| v.as_str().is_some_and(|s| !s.is_empty()))),
"Incomplete gateway client TLS"
);
let mut secrets = BTreeMap::new();
secrets.insert(secret_name("openshell","ow-service-openshell")?,json!({"ca.crt":material["ca.crt"],"tls.crt":material["tls.crt"],"tls.key":material["tls.key"]}));
let encode = |data: &Value| -> Result<Value> {
use base64::Engine;
let mut result = serde_json::Map::new();
for (k, v) in data.as_object().context("Invalid secret data")? {
let text = v.as_str().context("Secret values must be strings")?;
ensure!(text.len() <= 256 * 1024, "Secret value too large");
result.insert(
k.clone(),
json!(base64::engine::general_purpose::STANDARD.encode(text)),
);
}
Ok(Value::Object(result))
};
secrets.insert(
secret_name("database", "ow-service-database")?,
encode(&values["database"])?,
);
secrets.insert(
secret_name("apiTLS", "ow-service-tls")?,
encode(&values["apiTLS"])?,
);
if !values["model"].is_null() {
ensure!(
v["secrets"]["model"]
.as_str()
.is_some_and(|s| !s.is_empty()),
"Declare secrets.model in Helm values so the worker mounts the provided model keys"
);
ensure!(
values["model"].as_object().is_some_and(|m| !m.is_empty()
&& m.keys()
.all(|k| ["openai", "anthropic"].contains(&k.as_str()))),
"Unsupported model secret entries"
);
secrets.insert(
secret_name("model", "ow-service-model")?,
encode(&values["model"])?,
);
} else {
ensure!(
v["secrets"]["model"].as_str().is_none_or(str::is_empty),
"Declared model Secret requires explicit private input"
);
}
fn token(root: &Path, name: &str) -> Result<String> {
if root.join(name).exists() {
return Ok(String::from_utf8(super::read_file(root, name)?)?);
}
let value = format!(
"{}{}",
uuid::Uuid::new_v4().simple(),
uuid::Uuid::new_v4().simple()
);
let mut options = fs::OpenOptions::new();
options.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
let mut f = options.open(root.join(name))?;
f.write_all(value.as_bytes())?;
f.sync_all()?;
Ok(value)
}
let admin = token(root, "bootstrap-token")?;
let probe = token(root, "probe-token")?;
// Persist the observer spec once: retrying staging must not silently extend its expiry.
let spec_file = root.join("probe-spec.json");
if !spec_file.exists() {
let data = json!({"principal":"readiness","role":"observer","scope":{"kind":"all"},"expires_at":super::now()?+86400});
save(root, "probe-spec.json", &data)?;
}
let spec = String::from_utf8(super::read_file(root, "probe-spec.json")?)?;
secrets.insert(
secret_name("bootstrap", "ow-service-bootstrap")?,
encode(&json!({"admin-token":admin,"probe":probe,"probe.json":spec}))?,
);
secrets.insert(
secret_name("probe", "ow-service-probe")?,
encode(&json!({"curl.conf":format!("header = \"Authorization: Bearer {probe}\"\n")}))?,
);
ensure!(
secrets.len() == if values["model"].is_null() { 5 } else { 6 },
"Declared Secret names must be distinct"
);
for (name, data) in secrets {
let key = format!("secret:{name}");
let existing = client
.get("secret", &name, Some(&p.config.service.namespace))
.await?;
if let Some(secret) = existing {
let op = j
.operations
.get(&key)
.context("Refusing to adopt an existing Secret")?;
ensure!(
secret["metadata"]["annotations"]["openworker.io/operation-id"] == op.request_id
&& secret["metadata"]["labels"][OWNER] == p.id
&& secret["data"] == data,
"Secret ownership/content changed; no automatic overwrite"
);
if let Some(receipt) = &op.receipt {
ensure!(
receipt["uid"] == secret["metadata"]["uid"],
"Secret UID changed"
);
}
complete(root, j, &key, json!({"uid":secret["metadata"]["uid"]}))?;
} else {
ensure!(
!j.operations.contains_key(&key),
"Secret creation outcome unknown; inspect original intent"
);
begin(root, j, &key)?;
let secret = json!({"apiVersion":"v1","kind":"Secret","metadata":{"name":name,"namespace":p.config.service.namespace,"labels":{OWNER:p.id},"annotations":{"openworker.io/operation-id":j.operations[&key].request_id}},"type":"Opaque","data":data});
client
.kube(
args(&["create", "-f", "-"]),
Some(serde_json::to_vec(&secret)?),
)
.await?;
let actual = client
.get("secret", &name, Some(&p.config.service.namespace))
.await?
.context("Secret creation not confirmed")?;
ensure!(
actual["data"] == secret["data"]
&& actual["metadata"]["annotations"] == secret["metadata"]["annotations"],
"Secret creation receipt mismatch"
);
complete(root, j, &key, json!({"uid":actual["metadata"]["uid"]}))?;
}
}
crate::output(
json!({"secretsReady":true,"valuesPersistedInJournal":false,"bootstrapTokenFile":root.join("bootstrap-token")}),
)
}
async fn delete_namespace(
client: &Client<'_>,
p: &Plan,
root: &Path,
j: &mut Journal,
ns: &str,
) -> Result<()> {
let key = format!("delete-namespace:{ns}");
let existing = owned(client, p, j, ns).await?;
if existing.is_none() {
ensure!(
j.operations.contains_key(&key),
"Namespace absent without destruction intent"
);
return complete(root, j, &key, json!({"absent":true}));
}
if !j.operations.contains_key(&key) {
begin(root, j, &key)?;
}
// The DELETE body carries a UID precondition: checking a name before delete
// alone would allow a replacement namespace to be deleted in a race.
let options =
json!({"apiVersion":"v1","kind":"DeleteOptions","preconditions":{"uid":j.namespaces[ns]}});
client
.kube(
args(&[
"delete",
"--raw",
&format!("/api/v1/namespaces/{ns}"),
"-f",
"-",
]),
Some(serde_json::to_vec(&options)?),
)
.await?;
for _ in 0..180 {
if owned(client, p, j, ns).await?.is_none() {
return complete(root, j, &key, json!({"absent":true}));
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
bail!("Namespace deletion pending; resume the same journal")
}
async fn uninstall(
client: &Client<'_>,
p: &Plan,
root: &Path,
j: &mut Journal,
key: &str,
ns: &str,
release: &str,
) -> Result<()> {
owned(client, p, j, ns)
.await?
.context("Owned namespace missing before uninstall")?;
let delete = format!("uninstall:{key}");
if let Some(r) = client.release(ns, release).await? {
let receipt = j
.operations
.get(key)
.and_then(|o| o.receipt.as_ref())
.context("Release identity was never confirmed")?;
ensure!(
r["uid"] == receipt["uid"] && r["revision"] == receipt["revision"],
"Release identity changed before uninstall"
);
ensure!(
!j.operations.contains_key(&delete),
"Uninstall outcome unknown; inspect original release instead of replaying"
);
begin(root, j, &delete)?;
client
.helm(args(&[
"uninstall",
release,
"-n",
ns,
"--wait",
"--timeout",
"180s",
]))
.await?;
} else if !j.operations.contains_key(&delete) {
begin(root, j, &delete)?;
}
ensure!(
client.release(ns, release).await?.is_none(),
"Uninstall not confirmed"
);
complete(root, j, &delete, json!({"absent":true}))
}
/// Verify a real teardown on the originally bound cluster. Not a user-supplied
/// boolean receipt: check current kube-system UID and absence over TLS.
pub(super) async fn verify_teardown(
root: &Path,
cloud_state: &Path,
profile: &str,
kubeconfig: &Value,
) -> Result<()> {
let (p, j, _lock) = load(root)?;
ensure!(
j.phase == "destroyed",
"Installation teardown has not completed"
);
let binding = p
.config
.cloud
.as_ref()
.context("Cloud-bound installation receipt required")?;
ensure!(
binding.state.canonicalize()? == cloud_state.canonicalize()? && binding.profile == profile,
"Installation belongs to a different cloud lifecycle/profile"
);
ensure!(
super::up::original_kubeconfig(&read_json(&p.config.kubeconfig)?)? == *kubeconfig,
"Installation cluster endpoint/CA/profile changed"
);
let client = Client { c: &p.config };
ensure!(
client.cluster().await? == p.cluster_uid,
"Installation cluster identity changed"
);
ensure!(
client
.get("namespace", &p.config.service.namespace, None)
.await?
.is_none(),
"Service namespace still exists"
);
if let Shell::Install { namespace, .. } = &p.config.openshell {
ensure!(
client.get("namespace", namespace, None).await?.is_none(),
"Owned gateway namespace still exists"
);
}
Ok(())
}
pub(super) async fn run(command: &Command) -> Result<()> {
run_scoped(command, true).await
}
pub(super) async fn run_cluster(command: &Command) -> Result<()> {
run_scoped(command, false).await
}
async fn run_scoped(command: &Command, local_only: bool) -> Result<()> {
if let Command::Plan { config, out } = command {
let mut c: Config = serde_json::from_value(read_json(config)?)?;
c.kubeconfig = c.kubeconfig.canonicalize()?;
c.service.chart = c.service.chart.canonicalize()?;
c.service.values_file = c.service.values_file.canonicalize()?;
if let Some(platform) = &mut c.platform {
platform.file = platform.file.canonicalize()?;
}
if let Shell::Install {
archive,
values_file,
..
} = &mut c.openshell
{
*archive = archive.canonicalize()?;
*values_file = values_file.canonicalize()?;
}
let hashes = inputs(&c)?;
ensure!(
!local_only || (c.cloud.is_none() && c.azure.is_none() && c.service.fixture_database),
"Use deploy cluster for managed-database/cloud installations"
);
if let Some(binding) = &c.cloud {
cloud::verify_installation(
binding,
&c.kubeconfig,
&c.context,
&c.service.values_file,
true,
)
.await?;
}
if let Some(binding) = &c.azure {
azure::verify_installation(
binding,
&c.kubeconfig,
&c.context,
&c.service.values_file,
true,
)
.await?;
}
let client = Client { c: &c };
let cluster_uid = client.cluster().await?;
let mut p = Plan {
schema_version: 1,
id: format!("ows-local-{}", uuid::Uuid::new_v4().simple()),
config: c,
hashes,
cluster_uid,
digest: String::new(),
};
p.digest = digest(&p)?;
let mut d = fs::DirBuilder::new();
#[cfg(unix)]
{
use std::os::unix::fs::DirBuilderExt;
d.mode(0o700);
}
d.create(out)
.context("Local state directory must not exist")?;
save(out, "local-plan.json", &p)?;
save(
out,
"journal.json",
&Journal {
schema_version: 1,
plan_digest: p.digest.clone(),
phase: "planned".into(),
..Default::default()
},
)?;
return crate::output(
json!({"deploymentId":p.id,"planDigest":p.digest,"state":"planned","scope":if p.config.cloud.is_some(){"aws-eks"}else if p.config.azure.is_some(){"azure-aks"}else{"local-kind"}}),
);
}
let state = match command {
Command::Apply { state, .. }
| Command::Platform { state, .. }
| Command::RecordFailedRelease { state, .. }
| Command::Status { state }
| Command::Secrets { state, .. }
| Command::Destroy { state, .. } => state,
_ => unreachable!(),
};
let (p, mut j, _lock) = load(state)?;
ensure!(
!local_only
|| (p.config.cloud.is_none()
&& p.config.azure.is_none()
&& p.config.service.fixture_database),
"Use deploy cluster for managed-database/cloud installations"
);
if let Some(binding) = &p.config.cloud {
cloud::verify_installation(
binding,
&p.config.kubeconfig,
&p.config.context,
&p.config.service.values_file,
!matches!(
command,
Command::Status { .. }
| Command::Destroy { .. }
| Command::RecordFailedRelease { .. }
),
)
.await?;
}
if let Some(binding) = &p.config.azure {
azure::verify_installation(
binding,
&p.config.kubeconfig,
&p.config.context,
&p.config.service.values_file,
!matches!(
command,
Command::Status { .. }
| Command::Destroy { .. }
| Command::RecordFailedRelease { .. }
),
)
.await?;
}
let client = Client { c: &p.config };
ensure!(
client.cluster().await? == p.cluster_uid,
"Cluster identity changed"
);
if let Command::Status { .. } = command {
let mut observed = BTreeMap::new();
for name in j.namespaces.keys() {
observed.insert(name.clone(), owned(&client,&p,&j,name).await?.map(|v|json!({"uid":v["metadata"]["uid"],"terminating":v["metadata"].get("deletionTimestamp").is_some()})));
}
let (ns, release, _) = p.config.openshell.scope();
let gateway = client.release(ns, release).await?;
let service = client
.release(&p.config.service.namespace, &p.config.service.release)
.await?;
return crate::output(
json!({"deploymentId":p.id,"phase":j.phase,"operations":j.operations,"confirmedNoEffect":j.confirmed_no_effect,"observedNamespaces":observed,"observedGateway":gateway,"observedService":service,"clusterIdentityVerified":true,"readOnly":true}),
);
}
ensure!(
inputs(&p.config)? == p.hashes,
"Installer inputs changed; regenerate and review a new plan"
);
if let Command::RecordFailedRelease {
accept_plan,
release: key,
..
} = command
{
ensure!(accept_plan == &p.digest, "Accepted plan digest mismatch");
let (ns, release) = match key.as_str() {
"openshell-release" if matches!(p.config.openshell, Shell::Install { .. }) => {
let (ns, release, _) = p.config.openshell.scope();
(ns, release)
}
"service-release" => (
&p.config.service.namespace[..],
&p.config.service.release[..],
),
_ => bail!("Invalid original release operation"),
};
owned(&client, &p, &j, ns)
.await?
.context("Owned namespace missing")?;
let op = j
.operations
.get(key)
.context("No original install intent; refusing adoption")?;
ensure!(
["intent", "failed-confirmed"].contains(&op.status.as_str()),
"Only a failed original install may be recorded"
);
let r = client
.release(ns, release)
.await?
.context("Original release absent; outcome remains unknown")?;
ensure!(
r["status"] == "failed" && r["revision"] == 1,
"Expected original terminal failed install revision 1"
);
ensure!(
if op.label_provenance {
r["installerId"] == p.id && r["operationId"] == op.request_id
} else {
r["description"] == format!("ows:{}:{}", p.id, op.request_id)
},
"Helm release provenance mismatch"
);
if let Some(original) = &op.receipt {
ensure!(
original["uid"] == r["uid"] && original["revision"] == r["revision"],
"Helm release identity/revision changed"
);
}
// A failed hook can have side effects. This receipt permits identity-checked
// cleanup only; it must never authorize retrying the original installation.
let op = j.operations.get_mut(key).unwrap();
op.receipt = Some(r);
op.status = "failed-confirmed".into();
save(state, "journal.json", &j)?;
return crate::output(
json!({"deploymentId":p.id,"operation":key,"status":"failed-confirmed","cleanupOnly":true,"remoteMutation":false}),
);
}
if let Command::Secrets {
input, accept_plan, ..
} = command
{
ensure!(*accept_plan == p.digest, "Accepted plan digest mismatch");
ensure!(
j.phase == "openshell_ready",
"Prepare the authenticated gateway before handing off Secrets"
);
if let Some(binding) = &p.config.cloud {
cloud::installation_intent(binding)?;
}
if let Some(binding) = &p.config.azure {
azure::installation_intent(binding).await?;
}
return stage_secrets(&client, &p, state, &mut j, input).await;
}
if let Command::Platform {
accept_plan,
allow_experimental_cloud,
..
} = command
{
ensure!(*accept_plan == p.digest, "Accepted plan digest mismatch");
ensure!(
["planned", "platform_ready", "platform_configuring"].contains(&j.phase.as_str()),
"Platform preparation must precede gateway/service installation"
);
if let Some(binding) = &p.config.cloud {
cloud::installation_intent(binding)?;
}
if let Some(binding) = &p.config.azure {
azure::installation_intent(binding).await?;
}
return platform::apply(&client, &p, state, &mut j, *allow_experimental_cloud).await;
}
if let Command::Apply {
accept_plan,
openshell_only,
confirm_no_effect,
reconcile_ready_release,
..
} = command
{
ensure!(accept_plan == &p.digest, "Accepted plan digest mismatch");
ensure!(
reconcile_ready_release
.as_ref()
.is_none_or(|s| s == "service-release"),
"Only a hook-free original service release may be readiness-reconciled"
);
ensure!(
!["destroying", "destroyed"].contains(&j.phase.as_str()),
"Deployment is being/has been destroyed"
);
if let Some(binding) = &p.config.cloud {
cloud::installation_intent(binding)?;
}
if let Some(binding) = &p.config.azure {
azure::installation_intent(binding).await?;
}
prerequisites(&client, p.config.cloud.is_some()).await?;
if let Some(step) = confirm_no_effect {
let old = j
.operations
.get(step)
.context("No such uncertain operation")?;
ensure!(
old.status == "intent" && step.ends_with("-release"),
"Only an unconfirmed Helm install may be reset"
);
let (ns, release, _) = if step == "openshell-release" {
p.config.openshell.scope()
} else if step == "service-release" {
(
&p.config.service.namespace[..],
&p.config.service.release[..],
"",
)
} else {
bail!("Invalid recovery step")
};
ensure!(
client.release(ns, release).await?.is_none(),
"A release exists; reconcile it instead"
);
ensure!(
j.confirmed_no_effect.len() < 32,
"Recovery history limit reached"
);
j.confirmed_no_effect
.push(j.operations.remove(step).unwrap());
save(state, "journal.json", &j)?;
}
let v = read_json(&p.config.service.values_file)?;
let service_id = v["config"]["service"]["id"].as_str().unwrap();
namespace(
&client,
&p,
state,
&mut j,
&p.config.service.namespace,
service_id,
)
.await?;
let (ns, release, _) = p.config.openshell.scope();
match &p.config.openshell {
Shell::Install {
archive,
values_file,
..
} => {
namespace(&client, &p, state, &mut j, ns, service_id).await?;
install_release(
&client,
&p,
state,
&mut j,
ReleaseInstall {
key: "openshell-release",
ns,
release,
chart: archive,
values: values_file,
reconcile_ready: false,
},
)
.await?;
}
Shell::Existing { .. } => {
let r = client
.release(ns, release)
.await?
.context("Existing gateway release absent")?;
ensure!(
r["status"] == "deployed",
"Existing gateway is not deployed"
);
if let Some(op) = j.operations.get("external-openshell") {
ensure!(
op.receipt.as_ref() == Some(&r),
"Existing gateway changed during installation"
);
} else {
begin(state, &mut j, "external-openshell")?;
complete(state, &mut j, "external-openshell", r)?;
}
}
}
j.phase = "openshell_ready".into();
save(state, "journal.json", &j)?;
if !openshell_only {
for key in ["database", "openshell", "apiTLS", "bootstrap", "probe"] {
let name = v["secrets"][key].as_str().unwrap_or(match key {
"database" => "ow-service-database",
"openshell" => "ow-service-openshell",
"apiTLS" => "ow-service-tls",
"bootstrap" => "ow-service-bootstrap",
_ => "ow-service-probe",
});
client.get("secret",name,Some(&p.config.service.namespace)).await?.context("Required service Secret absent; stage credentials/fixtures before service installation")?;
}
install_release(
&client,
&p,
state,
&mut j,
ReleaseInstall {
key: "service-release",
ns: &p.config.service.namespace,
release: &p.config.service.release,
chart: &p.config.service.chart,
values: &p.config.service.values_file,
reconcile_ready: reconcile_ready_release
.as_ref()
.is_some_and(|s| s == "service-release"),
},
)
.await?;
ready_service(&client, &p).await?;
j.phase = "installed".into();
save(state, "journal.json", &j)?;
}
return crate::output(
json!({"deploymentId":p.id,"phase":j.phase,"functionalSandboxQualification":"required_separately"}),
);
}
if let Command::Destroy { deployment_id, .. } = command {
ensure!(deployment_id == &p.id, "Deployment identity mismatch");
if j.phase == "destroyed" {
return crate::output(json!({"deploymentId":p.id,"phase":"destroyed"}));
}
if owned(&client, &p, &j, &p.config.service.namespace)
.await?
.is_some()
{
let service_dispatched = j.operations.contains_key("service-release");
if !service_dispatched {
for role in ["api", "worker"] {
ensure!(client.get("deployment", &format!("{}-{role}",p.config.service.release), Some(&p.config.service.namespace)).await?.is_none(),
"Unexpected service process without an installer intent; inspect before cleanup");
}
}
if service_dispatched && j.phase != "destroying" {
quiet_database(&client, &p).await?;
}
if client
.get(
"deployment",
&format!("{}-api", p.config.service.release),
Some(&p.config.service.namespace),
)
.await?
.is_some()
{
client
.kube(
args(&[
"-n",
&p.config.service.namespace,
"scale",
&format!("deployment/{}-api", p.config.service.release),
"--replicas=0",
]),
None,
)
.await?;
client
.kube(
args(&[
"-n",
&p.config.service.namespace,
"wait",
"--for=delete",
"pod",
"-l",
&format!("app={}-api", p.config.service.release),
"--timeout=90s",
]),
None,
)
.await?;
if j.phase != "destroying" {
quiet_database(&client, &p).await?;
}
}
j.phase = "destroying".into();
save(state, "journal.json", &j)?;
uninstall(
&client,
&p,
state,
&mut j,
"service-release",
&p.config.service.namespace,
&p.config.service.release,
)
.await?;
delete_namespace(&client, &p, state, &mut j, &p.config.service.namespace).await?;
} else {
ensure!(
j.operations.is_empty()
|| j.operations
.contains_key(&format!("delete-namespace:{}", p.config.service.namespace)),
"Service namespace unexpectedly missing"
);
}
if let Shell::Install {
namespace: ns,
release,
..
} = &p.config.openshell
{
if owned(&client, &p, &j, ns).await?.is_some() {
let sandboxes: Value = serde_json::from_slice(
&client
.kube(
args(&["get", "sandboxes.agents.x-k8s.io", "-n", ns, "-o", "json"]),
None,
)
.await?,
)?;
ensure!(
sandboxes["items"].as_array().is_some_and(Vec::is_empty),
"Owned OpenShell still has sandboxes; remove them through the service first"
);
uninstall(&client, &p, state, &mut j, "openshell-release", ns, release).await?;
delete_namespace(&client, &p, state, &mut j, ns).await?;
}
}
j.phase = "destroyed".into();
save(state, "journal.json", &j)?;
return crate::output(
json!({"deploymentId":p.id,"phase":j.phase,"existingGatewayPreserved":matches!(p.config.openshell,Shell::Existing{..})}),
);
}
unreachable!()
}
pub(crate) async fn verify_azure_teardown(
root: &Path,
binding: &azure::InstallationBinding,
) -> Result<()> {
let (p, j, _lock) = load(root)?;
ensure!(
j.phase == "destroyed"
&& p.config.azure.as_ref() == Some(binding)
&& p.config.cloud.is_none(),
"Original Azure shared-installer teardown binding required"
);
let client = Client { c: &p.config };
ensure!(
client.cluster().await? == p.cluster_uid,
"Installation cluster UID changed"
);
ensure!(
client
.get("namespace", &p.config.service.namespace, None)
.await?
.is_none(),
"Service namespace still exists"
);
if let Shell::Install { namespace, .. } = &p.config.openshell {
ensure!(
client.get("namespace", namespace, None).await?.is_none(),
"Owned OpenShell namespace still exists"
);
}
Ok(())
}