Skip to content
Python
104 KB
Raw Blame
//! One durable control object outside AKS serializes effects with Blob ETag CAS.
//! A lost connection is an unknown outcome, returned to the caller/LLM. Never
//! replay a write automatically: read/check the original operation first.
use super::api::{Azure, Reply};
use super::*;
use reqwest::Method;

#[derive(Subcommand)]
pub enum Lifecycle {
    Prepare {
        #[arg(long)]
        plan: PathBuf,
        #[arg(long)]
        out: PathBuf,
    },
    /// Observe the original ARM deployment/resources and checkpoint observations.
    Reconcile(Remote),
    /// Explicit experimental submission of one fixed ARM stage; no automatic replay.
    Apply {
        #[command(flatten)]
        remote: Remote,
        #[arg(long, value_enum)]
        stage: Stage,
    },
    /// Begin/advance deletion of one exact owned resource per invocation.
    Destroy(Remote),
    /// Read shared durable control and sanitized status, never raw provider errors.
    Status {
        #[arg(long)]
        state: PathBuf,
    },
}
#[derive(Args)]
pub struct Remote {
    #[arg(long)]
    state: PathBuf,
    #[arg(long)]
    accept: String,
    #[arg(long)]
    allow_experimental_cloud: bool,
}
#[derive(Subcommand)]
pub enum Cleanup {
    /// Export independent Container Apps scheduled-job/control-store ARM graph.
    Package {
        #[arg(long)]
        state: PathBuf,
        #[arg(long)]
        out: PathBuf,
    },
    /// Create the control Blob once, after an administrator installed its resources.
    Initialize(Remote),
    /// Read the shared control heartbeat/configuration; no cloud writes.
    Verify {
        #[arg(long)]
        state: PathBuf,
    },
    /// Pin a reviewed replacement image before an administrator updates the same job.
    PrepareUpgrade {
        #[command(flatten)]
        remote: Remote,
        #[arg(long)]
        image: String,
    },
    /// One scheduled iteration outside AKS. Uses Container Apps managed identity.
    Tick {
        #[arg(long)]
        control: String,
        #[arg(long)]
        tenant: String,
        #[arg(long)]
        subscription: String,
        #[arg(long)]
        client_id: String,
    },
}
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, clap::ValueEnum)]
#[serde(rename_all = "snake_case")]
pub enum Stage {
    Foundation,
    Cluster,
}
impl Stage {
    fn name(self) -> &'static str {
        match self {
            Self::Foundation => "foundation",
            Self::Cluster => "cluster",
        }
    }
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "snake_case")]
enum Phase {
    Prepared,
    Intent,
    Unknown,
    Creating,
    Active,
    Failed,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct Operation {
    stage: Stage,
    phase: Phase,
    request_id: String,
    correlation_id: Option<String>,
    operation_url: Option<String>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct Owned {
    id: String,
    incarnation: String,
    deleted: bool,
    delete_request: Option<String>,
    delete_accepted: bool,
    operation_url: Option<String>,
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    rejected_dns_deletes: Vec<RejectedDnsDelete>,
}
// A bounded, provider-confirmed no-effect response is different from an unknown
// mutation. Keep every original request; old ambiguous receipts stay parked.
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct RejectedDnsDelete {
    request_id: String,
    observed_at_unix: u64,
    retry_after_unix: u64,
    status: u16,
    code: String,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct GuardReceipt {
    created_at: String,
    principal_id: String,
    client_id: String,
}
impl GuardReceipt {
    fn validate(&self) -> Result<()> {
        ensure!(
            chrono::DateTime::parse_from_rfc3339(&self.created_at).is_ok()
                || chrono::NaiveDateTime::parse_from_str(&self.created_at, "%Y-%m-%dT%H:%M:%S%.f")
                    .is_ok(),
            "Invalid controller creation receipt"
        );
        for value in [&self.principal_id, &self.client_id] {
            ensure!(
                uuid::Uuid::parse_str(value).is_ok_and(|u| !u.is_nil() && u.to_string() == *value),
                "Invalid controller identity receipt"
            );
        }
        Ok(())
    }
}
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "snake_case")]
enum StopPhase {
    Intent,
    Unknown,
    Accepted,
    Absent,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct Stop {
    request_id: String,
    phase: StopPhase,
    operation_url: Option<String>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub(super) struct State {
    schema_version: u16,
    plan: Plan,
    bundle_digest: String,
    nonce: String,
    operations: Vec<Operation>,
    resources: Vec<Owned>,
    heartbeat_unix: Option<u64>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    guard: Option<GuardReceipt>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    image_upgrade: Option<ImageUpgrade>,
    destroy_requested: bool,
    completed_unix: Option<u64>,
    job_stop: Option<Stop>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    installation_digest: Option<String>,
    #[serde(default)]
    installation_closed: bool,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct ImageUpgrade {
    previous_image: String,
    image: String,
    request_id: String,
    prepared_at_unix: u64,
    controller: GuardReceipt,
}
fn controller_image(s: &State) -> &str {
    s.image_upgrade
        .as_ref()
        .map(|u| u.image.as_str())
        .unwrap_or(&s.plan.azure_infrastructure.as_ref().unwrap().guard_image)
}
fn validate_image_upgrade(s: &State, image: &str) -> Result<()> {
    let original = &s.plan.azure_infrastructure.as_ref().unwrap().guard_image;
    let (repo, _) = original
        .split_once("@sha256:")
        .context("Original image is not pinned")?;
    let (next, digest) = image
        .split_once("@sha256:")
        .context("Replacement image must be pinned")?;
    ensure!(
        next == repo
            && digest.len() == 64
            && digest
                .bytes()
                .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)),
        "Replacement must be a SHA256 digest in the original cleanup repository"
    );
    Ok(())
}
fn digest(p: &Plan) -> Result<String> {
    let files = super::super::artifacts(p)?;
    let hashes: BTreeMap<_, _> = files.into_iter().map(|(n, b)| (n, hash(&b))).collect();
    Ok(hash(&serde_json::to_vec(&hashes)?))
}
pub(super) fn control_url(p: &Plan) -> String {
    format!(
        "https://{}ctl.blob.core.windows.net/control/{}.json",
        graph::names(p).storage,
        p.deployment_id
    )
}
fn job_id(p: &Plan) -> String {
    format!(
        "{}/providers/Microsoft.App/jobs/{}-expiry",
        graph::control_scope(p),
        graph::names(p).prefix
    )
}
fn environment_id(p: &Plan) -> String {
    p.azure_infrastructure
        .as_ref()
        .unwrap()
        .guard_environment_resource_id
        .clone()
        .unwrap_or_else(|| {
            format!(
                "{}/providers/Microsoft.App/managedEnvironments/{}-expiry",
                graph::control_scope(p),
                graph::names(p).prefix
            )
        })
}
fn arm_deployment(s: &State, stage: Stage) -> String {
    format!(
        "{}/providers/Microsoft.Resources/deployments/{}-{}",
        scope(&s.plan),
        graph::names(&s.plan).prefix,
        stage.name()
    )
}
impl State {
    fn validate(&self) -> Result<()> {
        self.plan.validate()?;
        ensure!(
            self.schema_version == 1
                && self.plan.azure_infrastructure.is_some()
                && self.bundle_digest == digest(&self.plan)?,
            "Control graph identity mismatch"
        );
        ensure!(
            uuid::Uuid::parse_str(&self.nonce)
                .is_ok_and(|u| !u.is_nil() && u.to_string() == self.nonce),
            "Invalid ownership nonce"
        );
        ensure!(
            self.operations.len() == 2
                && self.operations[0].stage == Stage::Foundation
                && self.operations[1].stage == Stage::Cluster,
            "Invalid stage state"
        );
        for op in &self.operations {
            ensure!(
                uuid::Uuid::parse_str(&op.request_id)
                    .is_ok_and(|u| !u.is_nil() && u.to_string() == op.request_id),
                "Invalid persisted request"
            );
            if matches!(op.phase, Phase::Creating | Phase::Active | Phase::Failed) {
                ensure!(
                    op.correlation_id
                        .as_ref()
                        .is_some_and(|c| uuid::Uuid::parse_str(c).is_ok()),
                    "Missing ARM correlation receipt"
                );
            }
        }
        if let Some(receipt) = &self.guard {
            receipt.validate()?;
        }
        if let Some(u) = &self.image_upgrade {
            validate_image_upgrade(self, &u.image)?;
            ensure!(
                u.previous_image == self.plan.azure_infrastructure.as_ref().unwrap().guard_image
                    && self.guard.as_ref() == Some(&u.controller)
                    && uuid::Uuid::parse_str(&u.request_id).is_ok_and(|id| !id.is_nil())
                    && u.prepared_at_unix >= self.plan.created_at_unix
                    && u.prepared_at_unix < self.plan.expires_at_unix,
                "Invalid pinned controller upgrade receipt"
            );
        }

        ensure!(
            self.heartbeat_unix.is_none() || self.guard.is_some(),
            "Heartbeat requires an installer-pinned controller receipt"
        );
        if let Some(digest) = &self.installation_digest {
            ensure!(
                digest.len() == 64
                    && digest
                        .bytes()
                        .all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(&c)),
                "Invalid installation receipt digest"
            );
        }
        ensure!(
            !self.installation_closed || self.installation_digest.is_some(),
            "Closed installation lacks original mutation receipt"
        );
        if let Some(stop) = &self.job_stop {
            ensure!(
                self.completed_unix.is_some()
                    && self.guard.is_some()
                    && uuid::Uuid::parse_str(&stop.request_id)
                        .is_ok_and(|u| !u.is_nil() && u.to_string() == stop.request_id),
                "Controller stop requires original completion/identity/request receipts"
            );
        }
        let inventory = graph::inventory(&self.plan);
        let mut ids = std::collections::BTreeSet::new();
        for r in &self.resources {
            ensure!(
                ids.insert(&r.id)
                    && inventory.iter().any(|i| i["id"].as_str() == Some(&r.id))
                    && !r.incarnation.is_empty()
                    && r.incarnation.len() <= 128,
                "Unexpected or duplicate ownership receipt"
            );
            if let Some(request) = &r.delete_request {
                ensure!(
                    uuid::Uuid::parse_str(request)
                        .is_ok_and(|u| !u.is_nil() && u.to_string() == *request),
                    "Invalid deletion request UUID"
                );
            }
            let mut rejected = std::collections::BTreeSet::new();
            ensure!(
                r.rejected_dns_deletes.len() <= 5,
                "DNS dependency retry limit exceeded"
            );
            for rejection in &r.rejected_dns_deletes {
                ensure!(
                    r.id.contains("/providers/Microsoft.Network/privateDnsZones/")
                        && rejection.status == 409
                        && rejection.code == "CannotDeleteResource"
                        && uuid::Uuid::parse_str(&rejection.request_id).is_ok_and(|u| !u.is_nil())
                        && rejected.insert(&rejection.request_id)
                        && rejection.retry_after_unix
                            == rejection.observed_at_unix.saturating_add(60),
                    "Invalid DNS no-effect receipt"
                );
            }
            if r.deleted {
                ensure!(
                    r.delete_request.is_some(),
                    "Absence without delete intent is not cleanup proof"
                );
            }
        }
        ensure!(
            self.resources.len() <= inventory.len(),
            "Invalid receipt count"
        );
        Ok(())
    }
    fn status(&self) -> Value {
        let stopped = self
            .job_stop
            .as_ref()
            .is_some_and(|r| r.phase == StopPhase::Absent);
        let attention = self
            .job_stop
            .as_ref()
            .is_some_and(|r| matches!(r.phase, StopPhase::Intent | StopPhase::Unknown));
        json!({"deploymentId":self.plan.deployment_id,"bundleDigest":self.bundle_digest,"control":control_url(&self.plan),"stages":self.operations,"ownedResourceCount":self.resources.len(),"deletedResourceCount":self.resources.iter().filter(|r|r.deleted).count(),"heartbeatUnix":self.heartbeat_unix,"controllerReceipt":self.guard,"controllerImage":controller_image(self),"controllerUpgrade":self.image_upgrade,"controllerStop":self.job_stop,"controllerStopped":stopped,"requiresAttention":attention,"workloadAndControllerComplete":self.completed_unix.is_some()&&stopped,"workloadCompletedUnix":self.completed_unix,"destroyRequested":self.destroy_requested,"completedUnix":self.completed_unix,"retainedControlResources":true,"liveQualified":false})
    }
}
fn local(path: &Path) -> Result<State> {
    let bytes = super::super::read_file(path, "azure-state.json")?;
    let s: State = serde_json::from_slice(&bytes)?;
    s.validate()?;
    Ok(s)
}
fn private_dir(path: &Path) -> Result<()> {
    let mut d = fs::DirBuilder::new();
    #[cfg(unix)]
    {
        use std::os::unix::fs::DirBuilderExt;
        d.mode(0o700);
    }
    d.create(path)
        .context("Use a fresh private output directory")?;
    Ok(())
}
fn put(path: &Path, name: &str, bytes: &[u8]) -> Result<()> {
    let mut o = fs::OpenOptions::new();
    o.write(true).create_new(true);
    #[cfg(unix)]
    {
        use std::os::unix::fs::OpenOptionsExt;
        o.mode(0o600);
    }
    let mut f = o.open(path.join(name))?;
    f.write_all(bytes)?;
    f.sync_all()?;
    Ok(())
}

trait Cloud {
    async fn read_control(&mut self, url: &str) -> Result<Option<(String, State)>>;
    async fn save_control(
        &mut self,
        url: &str,
        etag: Option<&str>,
        state: &State,
    ) -> Result<String>;
    async fn arm(
        &mut self,
        method: Method,
        id: &str,
        api: &str,
        body: Option<&Value>,
        request: &str,
    ) -> Result<Reply>;
}
impl Cloud for Azure {
    async fn read_control(&mut self, url: &str) -> Result<Option<(String, State)>> {
        let r = self.blob(Method::GET, url, None, None).await?;
        if r.status == 404 {
            return Ok(None);
        }
        ensure!(r.status == 200, "Control read rejected");
        let s: State = serde_json::from_value(r.body.context("Control body missing")?)?;
        s.validate()?;
        ensure!(control_url(&s.plan) == url, "Control location mismatch");
        Ok(Some((r.etag.context("Control ETag required")?, s)))
    }
    async fn save_control(&mut self, url: &str, etag: Option<&str>, s: &State) -> Result<String> {
        s.validate()?;
        let r = self
            .blob(Method::PUT, url, etag, Some(&serde_json::to_value(s)?))
            .await?;
        ensure!(
            r.status == 201,
            "Control CAS conflict/unknown; reread before continuing"
        );
        r.etag.context("Control acknowledgement lacks ETag")
    }
    async fn arm(
        &mut self,
        method: Method,
        id: &str,
        api: &str,
        body: Option<&Value>,
        request: &str,
    ) -> Result<Reply> {
        Azure::arm(self, method, id, api, body, request).await
    }
}
async fn save(c: &mut impl Cloud, s: &State, etag: &mut String) -> Result<()> {
    *etag = c.save_control(&control_url(&s.plan), Some(etag), s).await?;
    Ok(())
}
fn owned_body(s: &State, r: &Value, expected: &Value) -> Result<String> {
    ensure!(
        r["id"]
            .as_str()
            .is_some_and(|id| id.eq_ignore_ascii_case(expected["id"].as_str().unwrap()))
            && r["tags"]["ows-deployment"] == s.plan.deployment_id
            && r["tags"]["ows-nonce"] == s.nonce
            && r["tags"]["ows-bundle"] == s.bundle_digest,
        "Resource ownership mismatch; names/tags alone are not receipts"
    );
    // AKS resourceUID distinguishes delete/recreate under the same ARM ID. Some
    // live responses omit systemData; preserve older frozen graph descriptors and
    // accept this documented immutable identity only for managedClusters.
    let receipt = r
        .pointer(expected["incarnationPointer"].as_str().unwrap())
        .filter(|v| v.as_str().is_some_and(|s| !s.is_empty()))
        .or_else(|| {
            expected["id"]
                .as_str()
                .filter(|id| id.contains("/providers/Microsoft.ContainerService/managedClusters/"))
                .and_then(|_| r.pointer("/properties/resourceUID"))
        })
        .and_then(Value::as_str)
        .filter(|v| !v.is_empty() && v.len() <= 128)
        .context("Provider immutable incarnation receipt unavailable; refuse cleanup")?;
    Ok(receipt.to_owned())
}
async fn observe(c: &mut impl Cloud, s: &mut State, e: &mut String) -> Result<()> {
    for index in 0..s.operations.len() {
        if s.operations[index].phase == Phase::Prepared {
            continue;
        }
        let op = s.operations[index].clone();
        let r = c
            .arm(
                Method::GET,
                &arm_deployment(s, op.stage),
                "2022-09-01",
                None,
                &op.request_id,
            )
            .await?;
        if r.status == 404 {
            continue;
        } // Absence never grants a second PUT.
        ensure!(r.status == 200, "ARM observation rejected");
        let v = r.body.context("Missing deployment observation")?;
        let p = &v["properties"];
        ensure!(
            p["parameters"]["ownershipNonce"]["value"] == s.nonce
                && p["parameters"]["bundleDigest"]["value"] == s.bundle_digest,
            "Original deployment identity mismatch"
        );
        let correlation = p["correlationId"]
            .as_str()
            .filter(|v| uuid::Uuid::parse_str(v).is_ok())
            .context("ARM correlation receipt missing")?
            .to_owned();
        if let Some(old) = &op.correlation_id {
            ensure!(old == &correlation, "Original deployment was replaced");
        }
        s.operations[index].correlation_id = Some(correlation);
        s.operations[index].phase = match p["provisioningState"].as_str() {
            Some("Succeeded") => Phase::Active,
            Some("Failed" | "Canceled") => Phase::Failed,
            _ => Phase::Creating,
        };
        save(c, s, e).await?;
    }
    for expected in graph::inventory(&s.plan) {
        let id = expected["id"].as_str().unwrap();
        let stage = if expected["stage"] == "cluster" {
            Stage::Cluster
        } else {
            Stage::Foundation
        };
        if s.operations
            .iter()
            .find(|o| o.stage == stage)
            .unwrap()
            .correlation_id
            .is_none()
        {
            continue;
        }
        let r = c
            .arm(
                Method::GET,
                id,
                expected["apiVersion"].as_str().unwrap(),
                None,
                &s.nonce,
            )
            .await?;
        let prior = s.resources.iter().position(|r| r.id == id);
        if r.status == 404 {
            if let Some(i) = prior {
                ensure!(
                    s.resources[i].delete_request.is_some(),
                    "Owned resource disappeared without delete intent"
                );
                s.resources[i].deleted = true;
                save(c, s, e).await?;
            }
            continue;
        }
        ensure!(r.status == 200, "Resource observation rejected");
        let incarnation = owned_body(
            s,
            &r.body.context("Missing resource observation")?,
            &expected,
        )?;
        if let Some(i) = prior {
            ensure!(
                s.resources[i].incarnation == incarnation && !s.resources[i].deleted,
                "Owned resource was replaced; no deletion allowed"
            );
        } else {
            s.resources.push(Owned {
                id: id.into(),
                incarnation,
                deleted: false,
                delete_request: None,
                delete_accepted: false,
                operation_url: None,
                rejected_dns_deletes: vec![],
            });
            save(c, s, e).await?;
        }
    }
    Ok(())
}
async fn apply(
    c: &mut impl Cloud,
    s: &mut State,
    e: &mut String,
    stage: Stage,
    at: u64,
) -> Result<()> {
    ensure!(
        at < s.plan.expires_at_unix && !s.destroy_requested && s.completed_unix.is_none(),
        "Creation expired or cleanup requested"
    );
    ensure!(
        s.guard.is_some() && s.heartbeat_unix.is_some_and(|t| t <= at && at - t <= 600),
        "Independent cleanup heartbeat is missing/stale"
    );
    let i = if stage == Stage::Foundation { 0 } else { 1 };
    ensure!(
        s.operations[i].phase == Phase::Prepared,
        "Original request already started; reconcile, never automatically retry"
    );
    if stage == Stage::Cluster {
        ensure!(
            s.operations[0].phase == Phase::Active,
            "Foundation must complete before AKS"
        );
    }
    let request = s.operations[i].request_id.clone();
    let r = c
        .arm(
            Method::GET,
            &arm_deployment(s, stage),
            "2022-09-01",
            None,
            &request,
        )
        .await?;
    ensure!(
        r.status == 404,
        "Generated deployment already exists; do not adopt/overwrite"
    );
    for item in graph::inventory(&s.plan)
        .into_iter()
        .filter(|r| r["stage"] == stage.name())
    {
        let r = c
            .arm(
                Method::GET,
                item["id"].as_str().unwrap(),
                item["apiVersion"].as_str().unwrap(),
                None,
                &request,
            )
            .await?;
        ensure!(
            r.status == 404,
            "Existing workload resource refused; no name/tag adoption"
        );
    }
    s.operations[i].phase = Phase::Intent;
    save(c, s, e).await?; // durable CAS before network effect
    let files = graph::artifacts(&s.plan)?;
    let template: Value =
        serde_json::from_slice(&files[&format!("azure-{}.arm.json", stage.name())])?;
    let mut parameters =
        json!({"ownershipNonce":{"value":s.nonce},"bundleDigest":{"value":s.bundle_digest}});
    if stage == Stage::Foundation {
        let v: Value = serde_json::from_slice(&files["azure-foundation.parameters.json"])?;
        parameters["postgresPassword"] = v["parameters"]["postgresPassword"].clone();
    }
    let body =
        json!({"properties":{"mode":"Incremental","template":template,"parameters":parameters}});
    match c
        .arm(
            Method::PUT,
            &arm_deployment(s, stage),
            "2022-09-01",
            Some(&body),
            &request,
        )
        .await
    {
        Ok(r) if [200, 201, 202].contains(&r.status) => {
            s.operations[i].operation_url = r.operation;
            s.operations[i].phase = Phase::Unknown;
            save(c, s, e).await?;
            observe(c, s, e).await
        }
        _ => {
            s.operations[i].phase = Phase::Unknown;
            save(c, s, e).await?;
            bail!("ARM outcome unknown/rejected. Observe the original deployment; no automatic resubmission.")
        }
    }
}
async fn destroy_one(c: &mut impl Cloud, s: &mut State, e: &mut String, at: u64) -> Result<()> {
    ensure!(s.plan.expires_workload(), "Persistent deployment data is retained. Review a separate data-retention/teardown plan before destruction.");
    if !s.destroy_requested {
        s.destroy_requested = true;
        save(c, s, e).await?;
    }
    observe(c, s, e).await?;
    // Never race an unfinished ARM create against cleanup, or let a 404 create
    // observation erase an uncertain intent. A human can inspect Azure activity.
    ensure!(
        s.operations
            .iter()
            .all(|o| matches!(o.phase, Phase::Prepared | Phase::Active | Phase::Failed)),
        "Creation is still running/unknown; observe before cleanup"
    );
    for expected in graph::inventory(&s.plan) {
        let Some(i) = s
            .resources
            .iter()
            .position(|r| r.id == expected["id"].as_str().unwrap() && !r.deleted)
        else {
            continue;
        };
        if let Some(request) = &s.resources[i].delete_request {
            let Some(rejected) = s.resources[i]
                .rejected_dns_deletes
                .last()
                .filter(|r| &r.request_id == request)
            else {
                // GET only after an accepted or unknown mutation, including old
                // receipts. A lost response must never authorize another DELETE.
                return Ok(());
            };
            ensure!(s.resources[i].rejected_dns_deletes.len() < 5,
                "DNS dependencies repeatedly rejected deletion; inspect original resource and receipts");
            if at < rejected.retry_after_unix {
                return Ok(());
            }
        }
        let id = s.resources[i].id.clone();
        let api = expected["apiVersion"].as_str().unwrap();
        let r = c.arm(Method::GET, &id, api, None, &s.nonce).await?;
        ensure!(
            r.status == 200,
            "Expected owned resource absent; reconcile first"
        );
        let incarnation = owned_body(
            s,
            &r.body.context("Missing ownership observation")?,
            &expected,
        )?;
        ensure!(
            incarnation == s.resources[i].incarnation,
            "Resource incarnation changed; refuse deletion"
        );
        if id.contains("/providers/Microsoft.Network/privateDnsZones/") {
            // VNet deletion acknowledges before DNS links finish disappearing.
            // Wait without a DELETE intent; never retry a rejected/unknown mutation.
            let links = c
                .arm(
                    Method::GET,
                    &format!("{id}/virtualNetworkLinks"),
                    "2020-06-01",
                    None,
                    &s.nonce,
                )
                .await?;
            ensure!(links.status == 200, "DNS link observation unavailable");
            let body = links.body.context("DNS link observation missing")?;
            ensure!(
                body.get("nextLink").is_none_or(Value::is_null),
                "DNS link inventory incomplete"
            );
            let items = body["value"]
                .as_array()
                .context("DNS link inventory malformed")?;
            if !items.is_empty() {
                return Ok(());
            }
        }
        let request = uuid::Uuid::new_v4().to_string();
        s.resources[i].delete_request = Some(request.clone());
        s.resources[i].delete_accepted = false;
        s.resources[i].operation_url = None;
        save(c, s, e).await?;
        let r = c.arm(Method::DELETE, &id, api, None, &request).await;
        match r {
            Ok(r) if [200, 202, 204].contains(&r.status) => {
                s.resources[i].delete_accepted = true;
                s.resources[i].operation_url = r.operation;
                save(c, s, e).await?;
            }
            Ok(r) if id.contains("/providers/Microsoft.Network/privateDnsZones/")
                && r.status == 409
                && r.body.as_ref().and_then(|v| v.pointer("/error/code")).and_then(Value::as_str)
                    == Some("CannotDeleteResource") => {
                // Azure's empty link listing can precede dependency deletion.
                // Only this exact rejected response permits a future NEW attempt,
                // after backoff and fresh ownership/dependency checks above.
                s.resources[i].rejected_dns_deletes.push(RejectedDnsDelete {
                    request_id: request, observed_at_unix: at,
                    retry_after_unix: at.saturating_add(60), status: 409,
                    code: "CannotDeleteResource".into(),
                });
                save(c, s, e).await?;
            }
            _ => bail!("Delete outcome unknown/rejected. Observe this exact resource; no automatic replay."),
        }
        return Ok(());
    }
    s.completed_unix = Some(at);
    save(c, s, e).await?;
    Ok(())
}
/// Read the original controller after a persisted stop attempt. Never retry DELETE.
async fn reconcile_stop(c: &mut impl Cloud, s: &mut State, e: &mut String) -> Result<()> {
    let Some(stop) = s.job_stop.as_ref() else {
        return Ok(());
    };
    if stop.phase == StopPhase::Absent {
        return Ok(());
    }
    let r = c
        .arm(Method::GET, &job_id(&s.plan), "2024-03-01", None, &s.nonce)
        .await?;
    if r.status == 404 {
        s.job_stop.as_mut().unwrap().phase = StopPhase::Absent;
        save(c, s, e).await?;
        return Ok(());
    }
    ensure!(
        r.status == 200,
        "Controller stop observation unavailable; outcome remains pending"
    );
    guard_receipt(s, &r.body.context("Controller stop observation missing")?)?;
    // A surviving controller after an uncertain request remains visibly unknown;
    // an acknowledged DELETE is still pending until a read proves absence.
    if stop.phase == StopPhase::Intent {
        s.job_stop.as_mut().unwrap().phase = StopPhase::Unknown;
        save(c, s, e).await?;
    }
    Ok(())
}
async fn heartbeat(c: &mut impl Cloud, s: &mut State, e: &mut String, at: u64) -> Result<()> {
    if s.job_stop.is_some() {
        return reconcile_stop(c, s, e).await;
    }
    s.heartbeat_unix = Some(at);
    save(c, s, e).await?;
    if s.completed_unix.is_none() {
        if (s.plan.expires_workload() && at >= s.plan.expires_at_unix) || s.destroy_requested {
            destroy_one(c, s, e, at).await?;
        } else {
            observe(c, s, e).await?;
        }
    }
    if s.completed_unix.is_some() {
        ensure!(
            s.guard.is_some(),
            "Installer controller receipt missing; refuse self-stop"
        );
        verify_job(c, s).await?;
        let request = uuid::Uuid::new_v4().to_string();
        s.job_stop = Some(Stop {
            request_id: request.clone(),
            phase: StopPhase::Intent,
            operation_url: None,
        });
        save(c, s, e).await?;
        let response = c
            .arm(
                Method::DELETE,
                &job_id(&s.plan),
                "2024-03-01",
                None,
                &request,
            )
            .await;
        match response {
            Ok(r) if [200, 202, 204].contains(&r.status) => {
                let stop = s.job_stop.as_mut().unwrap();
                stop.phase = StopPhase::Accepted;
                stop.operation_url = r.operation;
                save(c, s, e).await?;
                reconcile_stop(c, s, e).await?;
            }
            Ok(r) if r.status == 404 => {
                s.job_stop.as_mut().unwrap().phase = StopPhase::Absent;
                save(c, s, e).await?;
            }
            _ => {
                s.job_stop.as_mut().unwrap().phase = StopPhase::Unknown;
                save(c, s, e).await?;
                bail!("Controller stop outcome unknown/rejected; inspect the original request. DELETE will not be replayed.")
            }
        }
    }
    Ok(())
}
async fn tick(c: &mut impl Cloud, s: &mut State, e: &mut String, at: u64) -> Result<()> {
    ensure!(s.guard.is_some(),"Controller receipt must be pinned by explicit initialization; a scheduled job cannot adopt itself");
    if s.job_stop.is_none() {
        verify_job(c, s).await?;
    }
    heartbeat(c, s, e, at).await
}
fn tick_identity(s: &State, client_id: &str) -> Result<()> {
    let receipt = s
        .guard
        .as_ref()
        .context("Installer-pinned controller identity required")?;
    ensure!(
        uuid::Uuid::parse_str(client_id)
            .is_ok_and(|u| !u.is_nil() && u.to_string() == receipt.client_id),
        "Scheduled execution identity differs from pinned controller"
    );
    Ok(())
}
async fn initialize(c: &mut impl Cloud, s: &mut State) -> Result<()> {
    ensure!(
        s.guard.is_none() && s.heartbeat_unix.is_none() && s.job_stop.is_none(),
        "Use the original uninitialized local state"
    );
    ensure!(
        c.read_control(&control_url(&s.plan)).await?.is_none(),
        "Control already exists; reconcile initialization instead of overwriting"
    );
    // The trusted explicit installer, rather than the first scheduled execution,
    // pins provider metadata before the create-only control write.
    s.guard = Some(verify_job(c, s).await?);
    c.save_control(&control_url(&s.plan), None, s).await?;
    Ok(())
}

pub(super) fn up_package(root: &Path, out: &Path) -> Result<()> {
    let s = local(root)?;
    let files = control_artifacts(&s)?;
    private_dir(out)?;
    for (name, bytes) in files {
        put(out, &name, &bytes)?;
    }
    Ok(())
}
pub(super) async fn up_initialize_control(root: &Path) -> Result<bool> {
    let mut expected = local(root)?;
    let mut api = Azure::new(&expected.plan, None)?;
    let path = root.join("up-operations/control-init.json");
    if let Some((_etag, shared)) = api.read_control(&control_url(&expected.plan)).await? {
        ensure!(
            shared.nonce == expected.nonce && shared.bundle_digest == expected.bundle_digest,
            "Shared cleanup identity mismatch"
        );
        verify_job(&mut api, &shared).await?;
        super::super::up::write_private(
            &path,
            &json!({"phase":"observed","nonce":shared.nonce,"bundleDigest":shared.bundle_digest}),
        )?;
        return Ok(shared
            .heartbeat_unix
            .is_some_and(|at| now().is_ok_and(|n| at <= n && n - at <= 600)));
    }
    ensure!(!path.exists(),"Control initialization outcome unknown; observe original Blob instead of replaying creation");
    super::super::up::write_private(
        &path,
        &json!({"phase":"intent","nonce":expected.nonce,"bundleDigest":expected.bundle_digest}),
    )?;
    initialize(&mut api, &mut expected).await?;
    Ok(false)
}
pub(super) fn up_local(root: &Path) -> Result<(Plan, String, String)> {
    let s = local(root)?;
    Ok((s.plan, s.bundle_digest, s.nonce))
}
pub(crate) async fn lifecycle(command: &Lifecycle) -> Result<()> {
    match command {
        Lifecycle::Prepare { plan, out } => {
            outside(out, plan)?;
            let (p, m) = inspect(plan)?;
            ensure!(
                p.azure_infrastructure.is_some() && now()? < p.expires_at_unix,
                "Unexpired Azure graph required"
            );
            let s = State {
                schema_version: 1,
                plan: p,
                bundle_digest: m.bundle_digest,
                nonce: uuid::Uuid::new_v4().to_string(),
                operations: [Stage::Foundation, Stage::Cluster]
                    .into_iter()
                    .map(|stage| Operation {
                        stage,
                        phase: Phase::Prepared,
                        request_id: uuid::Uuid::new_v4().to_string(),
                        correlation_id: None,
                        operation_url: None,
                    })
                    .collect(),
                resources: vec![],
                heartbeat_unix: None,
                guard: None,
                image_upgrade: None,
                destroy_requested: false,
                completed_unix: None,
                job_stop: None,
                installation_digest: None,
                installation_closed: false,
            };
            s.validate()?;
            private_dir(out)?;
            put(out, "azure-state.json", &json_bytes(&s)?)?;
            crate::output(s.status())
        }
        Lifecycle::Status { state } => {
            let s = local(state)?;
            let mut c = Azure::new(&s.plan, None)?;
            let (_, s) = c
                .read_control(&control_url(&s.plan))
                .await?
                .context("Shared control is not initialized")?;
            crate::output(s.status())
        }
        Lifecycle::Apply { remote, stage } => run(remote, Some(*stage), false).await,
        Lifecycle::Reconcile(remote) => run(remote, None, false).await,
        Lifecycle::Destroy(remote) => run(remote, None, true).await,
    }
}
async fn run(o: &Remote, stage: Option<Stage>, destroy: bool) -> Result<()> {
    let expected = local(&o.state)?;
    ensure!(
        o.allow_experimental_cloud && o.accept == expected.bundle_digest,
        "Explicit experimental cloud flag and exact bundle digest required"
    );
    let mut c = Azure::new(&expected.plan, None)?;
    let (mut e, mut s) = c
        .read_control(&control_url(&expected.plan))
        .await?
        .context("Initialize independent control before workload creation")?;
    ensure!(
        s.nonce == expected.nonce && s.bundle_digest == expected.bundle_digest,
        "Local/shared control identity mismatch"
    );
    if let Some(stage) = stage {
        verify_job(&mut c, &s).await?;
        apply(&mut c, &mut s, &mut e, stage, now()?).await?;
    } else if destroy {
        ensure!(
            s.installation_digest.is_none() || s.installation_closed,
            "Close verified Kubernetes installation before manual infrastructure destruction"
        );
        destroy_one(&mut c, &mut s, &mut e, now()?).await?;
    } else {
        observe(&mut c, &mut s, &mut e).await?;
        reconcile_stop(&mut c, &mut s, &mut e).await?;
    }
    crate::output(s.status())
}
pub(crate) async fn cleanup(command: &Cleanup) -> Result<()> {
    match command {
        Cleanup::Package { state, out } => {
            outside(out, state)?;
            let s = local(state)?;
            let files = control_artifacts(&s)?;
            private_dir(out)?;
            for (n, b) in files {
                put(out, &n, &b)?;
            }
            crate::output(
                json!({"deploymentId":s.plan.deployment_id,"bundleDigest":s.bundle_digest,"path":out,"cloudExecuted":false,"liveQualified":false}),
            )
        }
        Cleanup::Initialize(o) => {
            let s = local(&o.state)?;
            ensure!(
                o.allow_experimental_cloud
                    && o.accept == s.bundle_digest
                    && now()? < s.plan.expires_at_unix,
                "Unexpired explicit control initialization confirmation required"
            );
            let mut c = Azure::new(&s.plan, None)?;
            let mut s = s;
            initialize(&mut c, &mut s).await?;
            crate::output(s.status())
        }
        Cleanup::Verify { state } => {
            let s = local(state)?;
            let mut c = Azure::new(&s.plan, None)?;
            let (_, v) = c
                .read_control(&control_url(&s.plan))
                .await?
                .context("Control unavailable")?;
            ensure!(s.nonce == v.nonce, "Control identity mismatch");
            if v.completed_unix.is_some() || v.job_stop.is_some() {
                return crate::output(
                    json!({"heartbeatReady":false,"status":v.status(),"next":"lifecycle_reconcile","liveQualified":false,"workloadApplyReady":false}),
                );
            }
            let at = now()?;
            ensure!(
                v.heartbeat_unix.is_some_and(|t| t <= at && at - t <= 600),
                "Independent cleanup heartbeat stale/missing"
            );
            verify_job(&mut c, &v).await?;
            crate::output(
                json!({"heartbeatReady":true,"independentOfAks":true,"liveQualified":false,"workloadApplyReady":false}),
            )
        }
        Cleanup::PrepareUpgrade { remote, image } => {
            let expected = local(&remote.state)?;
            ensure!(
                remote.allow_experimental_cloud && remote.accept == expected.bundle_digest,
                "Explicit original bundle confirmation required"
            );
            let mut c = Azure::new(&expected.plan, None)?;
            let (mut e, mut s) = c
                .read_control(&control_url(&expected.plan))
                .await?
                .context("Shared control missing")?;
            ensure!(
                s.nonce == expected.nonce && s.bundle_digest == expected.bundle_digest,
                "Control identity mismatch"
            );
            prepare_image_upgrade(&mut c, &mut s, &mut e, image, now()?).await?;
            crate::output(
                json!({"status":s.status(),"jobImageUpdateRequired":true,"jobId":job_id(&s.plan),"image":image,"resourceOwnershipUnchanged":true}),
            )
        }
        Cleanup::Tick {
            control,
            tenant,
            subscription,
            client_id,
        } => {
            ensure!(
                uuid::Uuid::parse_str(client_id).is_ok_and(|u| !u.is_nil()),
                "Exact cleanup client ID required"
            );
            let mut c = Azure::target(tenant, subscription, Some(client_id.clone()))?;
            let (mut e, mut s) = c
                .read_control(control)
                .await?
                .context("Control not initialized")?;
            let p = &s.plan;
            let Target::Azure {
                tenant: t,
                subscription: sub,
                ..
            } = &p.target
            else {
                unreachable!()
            };
            ensure!(
                t == tenant && sub == subscription && control_url(p) == *control,
                "Cleanup target mismatch"
            );
            tick_identity(&s, client_id)?;
            tick(&mut c, &mut s, &mut e, now()?).await?;
            crate::output(s.status())
        }
    }
}
async fn prepare_image_upgrade(
    c: &mut impl Cloud,
    s: &mut State,
    e: &mut String,
    image: &str,
    at: u64,
) -> Result<()> {
    validate_image_upgrade(s, image)?;
    ensure!(
        at >= s.plan.created_at_unix
            && at < s.plan.expires_at_unix
            && !s.destroy_requested
            && s.completed_unix.is_none()
            && s.job_stop.is_none(),
        "Controller is closing or expired"
    );
    if let Some(u) = &s.image_upgrade {
        ensure!(
            u.image == image,
            "Another controller upgrade is already pinned; reconcile it"
        );
        return Ok(());
    }
    let controller = verify_job(c, s).await?;
    ensure!(
        s.guard.as_ref() == Some(&controller),
        "Initialized controller receipt required"
    );
    s.image_upgrade = Some(ImageUpgrade {
        previous_image: controller_image(s).into(),
        image: image.into(),
        request_id: uuid::Uuid::new_v4().to_string(),
        prepared_at_unix: at,
        controller,
    });
    s.heartbeat_unix = None; // Old image/configuration cannot authorize new work.
    save(c, s, e).await
}
fn control_artifacts(s: &State) -> Result<BTreeMap<String, Vec<u8>>> {
    let p = &s.plan;
    let n = graph::names(p);
    let c = p.azure_infrastructure.as_ref().unwrap();
    let Target::Azure {
        location,
        tenant,
        subscription,
    } = &p.target
    else {
        unreachable!()
    };
    let control_scope = graph::control_scope(p);
    let control_account = format!("{}ctl", n.storage);
    let identity_name = format!("{}-cleanup", n.prefix);
    let identity_id=format!("{control_scope}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identity_name}");
    let account_id =
        format!("{control_scope}/providers/Microsoft.Storage/storageAccounts/{control_account}");
    let environment_id = environment_id(p);
    let recipient = format!("[reference('{identity_id}', '2023-01-31').principalId]");
    let client = format!("[reference('{identity_id}', '2023-01-31').clientId]");
    let mut workload_actions = std::collections::BTreeSet::new();
    for item in graph::inventory(p) {
        let parts: Vec<_> = item["id"]
            .as_str()
            .unwrap()
            .split("/providers/")
            .nth(1)
            .unwrap()
            .split('/')
            .collect();
        let kind = format!("{}/{}", parts[0], parts[1]);
        for verb in ["read", "delete"] {
            workload_actions.insert(format!("{kind}/{verb}"));
        }
    }
    workload_actions.insert("Microsoft.Resources/deployments/read".into());
    workload_actions.insert("Microsoft.Network/privateDnsZones/virtualNetworkLinks/read".into());
    let role_id = format!(
        "/subscriptions/{subscription}/providers/Microsoft.Authorization/roleDefinitions/{}",
        guid(p, "cleanup-role")
    );
    let custom_role = json!({"properties":{"roleName":format!("{}-cleanup",n.prefix),"type":"CustomRole","assignableScopes":[scope(p)],"permissions":[{"actions":workload_actions,"notActions":[],"dataActions":[],"notDataActions":[]}]}});
    let cleanup_assignment = json!({"properties":{"principalId":recipient,"principalType":"ServicePrincipal","roleDefinitionId":role_id}});
    let blob_scope = format!("{account_id}/blobServices/default/containers/control");
    let builtin=format!("/subscriptions/{subscription}/providers/Microsoft.Authorization/roleDefinitions/ba92f5b4-2d11-453d-a403-e96b0029c9fe");
    let mut control_resources = vec![
        json!({"type":"Microsoft.ManagedIdentity/userAssignedIdentities","apiVersion":"2023-01-31","name":identity_name,"location":location}),
        json!({"type":"Microsoft.Storage/storageAccounts","apiVersion":"2023-05-01","name":control_account,"location":location,"sku":{"name":"Standard_LRS"},"kind":"StorageV2","properties":{"allowSharedKeyAccess":false,"allowBlobPublicAccess":false,"minimumTlsVersion":"TLS1_2","supportsHttpsTrafficOnly":true,"publicNetworkAccess":"Enabled"}}),
        json!({"type":"Microsoft.Storage/storageAccounts/blobServices/containers","apiVersion":"2023-05-01","name":format!("{control_account}/default/control"),"dependsOn":[account_id],"properties":{"publicAccess":"None"}}),
        json!({"type":"Microsoft.App/managedEnvironments","apiVersion":"2024-03-01","name":format!("{}-expiry",n.prefix),"location":location,"properties":{"appLogsConfiguration":{"destination":null}}}),
        json!({"type":"Microsoft.App/jobs","apiVersion":"2024-03-01","name":format!("{}-expiry",n.prefix),"location":location,"tags":{"ows-deployment":p.deployment_id,"ows-nonce":s.nonce,"ows-bundle":s.bundle_digest},"dependsOn":[identity_id,environment_id],"identity":{"type":"UserAssigned","userAssignedIdentities":{identity_id.clone():{}}},"properties":{"environmentId":environment_id,"configuration":{"triggerType":"Schedule","replicaTimeout":240,"replicaRetryLimit":0,"scheduleTriggerConfig":{"cronExpression":"*/5 * * * *","parallelism":1,"replicaCompletionCount":1}},"template":{"containers":[{"name":"cleanup","image":controller_image(s),"command":["/opt/openworker/ows"],"args":["deploy","azure","cleanup","tick","--control",control_url(p),"--tenant",tenant,"--subscription",subscription,"--client-id",client],"resources":{"cpu":0.25,"memory":"0.5Gi"}}]}}}),
    ];
    if c.guard_environment_resource_id.is_some() {
        control_resources.retain(|r| r["type"] != "Microsoft.App/managedEnvironments");
        control_resources.last_mut().unwrap()["dependsOn"] = json!([identity_id]);
    }
    if let Some(registry) = &c.guard_registry_resource_id {
        let server = format!("{}.azurecr.io", registry.rsplit('/').next().unwrap());
        control_resources.last_mut().unwrap()["properties"]["configuration"]["registries"] =
            json!([{"server":server,"identity":identity_id}]);
    }
    let template = json!({"$schema":"https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#","contentVersion":"1.0.0.0","resources":control_resources});
    let blob_binding = json!({"schemaVersion":1,"scope":blob_scope,"roleDefinitionId":builtin,"assignments":[{"guid":guid(p,"cleanup-control"),"principalId":recipient,"principalType":"ServicePrincipal"},{"guid":guid(p,"setup-control"),"principalId":p.azure_bootstrap.as_ref().unwrap().setup_object_id,"principalType":p.azure_bootstrap.as_ref().unwrap().setup_principal_type.azure()}],"resolveIdentityAfterControlDeployment":true});
    let job_role = json!({"properties":{"roleName":format!("{}-cleanup-stop",n.prefix),"type":"CustomRole","assignableScopes":[control_scope],"permissions":[{"actions":["Microsoft.App/jobs/read","Microsoft.App/jobs/delete"],"notActions":[],"dataActions":[],"notDataActions":[]}]},"assignmentScope":job_id(p),"principalId":recipient,"roleDefinitionGuid":guid(p,"cleanup-stop-role"),"assignmentGuid":guid(p,"cleanup-stop-assignment")});
    let readme=format!("# Independent Azure expiry control\n\nOffline export. Reviewed administrator deploys `azure-control.arm.json` into existing `{control_scope}`, resolves the actual cleanup managed-identity principal, and grants the exact exported Blob bindings, workload cleanup role and own-job stop role. Role/assignment files are REST payloads, not ARM templates; review their fixed IDs/scopes. ows never applies these privileged grants implicitly.\n\nThen initialize shared control exactly once, wait for a scheduled heartbeat and verify it before creating workload infrastructure. Cleanup uses the same versioned Blob with ETag compare-and-swap as the CLI. It survives AKS and invoking-laptop loss. No account key/SAS is allowed; the control endpoint is public HTTPS with private OAuth-only Blob data. Customer policy/network must permit this controller path.\n\nThe pinned external image must contain this matching native `ows` binary. Schedule: five minutes, one parallel invocation, platform retry count zero. CAS protects concurrent ticks/CLI. Unknown mutation outcomes remain parked; a subsequent tick reads original state/resources instead of replaying a write. Azure can reject a request before effect or lose its response after effect; neither is automatic retry authorization.\n\nWorkload deletions compare exact IDs, nonce, bundle digest and immutable creation receipts. No customer RG, guessed `MC_*` group or foreign resource is deleted. Missing ownership receipts, an active/uncertain create, replacement or ambiguous delete pauses cleanup for operator inspection; TTL is not an unconditional spending cap. A concurrent privileged administrator can replace a resource between read and delete because ARM has no universal conditional DELETE contract; do not operate on these dedicated owned IDs during teardown.\n\nAfter workload teardown, the job deletes its own exact job ID once. Control Blob/account, cleanup managed identity, consumption environment and reviewed role grants are retained for audit/recovery; the customer explicitly removes them after reviewing the checkpoint. Retention may incur small storage charges. This is workload expiry qualification, not a claim that every control byte/cost vanishes.\n\nNo real Azure deployment, IAM, scheduling, managed-identity endpoint, quotas, prices or AKS/OpenShell runtime has been tested. `liveQualified:false` and `applyReady:false` are intentional.\n");
    Ok(BTreeMap::from([
        ("azure-control.arm.json".into(), json_bytes(&template)?),
        ("azure-cleanup-role.json".into(), json_bytes(&custom_role)?),
        (
            "azure-cleanup-assignment.json".into(),
            json_bytes(&cleanup_assignment)?,
        ),
        (
            "azure-control-bindings.json".into(),
            json_bytes(&blob_binding)?,
        ),
        (
            "azure-cleanup-stop-role.json".into(),
            json_bytes(&job_role)?,
        ),
        (
            "azure-control-manifest.json".into(),
            json_bytes(
                &json!({"deploymentId":p.deployment_id,"bundleDigest":s.bundle_digest,"control":control_url(p),"controlScope":control_scope,"cleanupRoleDefinitionId":role_id,"cleanupRoleUrl":format!("https://management.azure.com{}/providers/Microsoft.Authorization/roleDefinitions/{}?api-version=2022-04-01",scope(p),guid(p,"cleanup-role")),"cleanupAssignmentUrl":format!("https://management.azure.com{}/providers/Microsoft.Authorization/roleAssignments/{}?api-version=2022-04-01",scope(p),guid(p,"cleanup-workload")),"initializationRequired":true,"retainedControlOwnedByCustomer":true,"liveQualified":false}),
            )?,
        ),
        (
            "azure-control-registry-binding.json".into(),
            json_bytes(&match &c.guard_registry_resource_id {
                Some(registry) => {
                    json!({"scope":registry,"principalId":recipient,"principalType":"ServicePrincipal","roleDefinitionId":format!("/subscriptions/{subscription}/providers/Microsoft.Authorization/roleDefinitions/7f951dda-4ed3-4680-a7ca-43fe172d538d"),"assignmentGuid":guid(p,"cleanup-registry-pull"),"resolveIdentityAfterControlDeployment":true,"automaticGrant":false})
                }
                None => json!({"required":false}),
            })?,
        ),
        ("AZURE-CLEANUP.md".into(), readme.into_bytes()),
    ]))
}

#[cfg(test)]
mod tests {
    use super::*;
    pub(super) fn state() -> State {
        let mut p = super::super::super::tests::plan();
        p.schema_version = 2;
        p.topology.architecture = "amd64".into();
        p.target = Target::Azure {
            tenant: "11111111-2222-4333-8444-555555555555".into(),
            subscription: "66666666-7777-4888-8999-aaaaaaaaaaaa".into(),
            location: "westus2".into(),
        };
        p.azure_bootstrap=Some(serde_json::from_value(json!({"resourceGroup":"customer-workload","setupObjectId":"bbbbbbbb-cccc-4ddd-8eee-ffffffffffff","setupPrincipalType":"ServicePrincipal"})).unwrap());
        p.azure_infrastructure=Some(serde_json::from_value(json!({"kubernetesVersion":"1.33.4","nodeVmSize":"Standard_D4s_v5","sshPublicKey":"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","adminGroupObjectIds":["11111111-1111-4111-8111-111111111111"],"controlResourceGroup":"customer-control","guardImage":format!("ghcr.io/test/ows@sha256:{}","a".repeat(64)),"postgresPassword":{"keyVaultId":"/subscriptions/66666666-7777-4888-8999-aaaaaaaaaaaa/resourceGroups/customer-secrets/providers/Microsoft.KeyVault/vaults/customer-vault","secretName":"postgres","secretVersion":"a".repeat(32)}})).unwrap());
        let s = State {
            schema_version: 1,
            bundle_digest: digest(&p).unwrap(),
            plan: p,
            nonce: uuid::Uuid::new_v4().to_string(),
            operations: [Stage::Foundation, Stage::Cluster]
                .into_iter()
                .map(|stage| Operation {
                    stage,
                    phase: Phase::Prepared,
                    request_id: uuid::Uuid::new_v4().to_string(),
                    correlation_id: None,
                    operation_url: None,
                })
                .collect(),
            resources: vec![],
            heartbeat_unix: Some(150),
            guard: Some(fixture_guard()),
            image_upgrade: None,
            destroy_requested: false,
            completed_unix: None,
            job_stop: None,
            installation_digest: None,
            installation_closed: false,
        };
        s.validate().unwrap();
        s
    }
    fn reply(status: u16, body: Option<Value>) -> Reply {
        Reply {
            status,
            body,
            etag: None,
            operation: None,
        }
    }
    struct Fake {
        control: State,
        revision: u64,
        objects: BTreeMap<String, Value>,
        writes: Vec<(Method, String)>,
        lose_put: bool,
        lose_delete: bool,
        conflict: bool,
        dns_rejections: usize,
        dns_error_code: String,
    }
    impl Fake {
        fn new(s: State) -> Self {
            let mut objects = BTreeMap::new();
            objects.insert(job_id(&s.plan), fixture_job(&s));
            Self {
                control: s,
                revision: 1,
                objects,
                writes: vec![],
                lose_put: false,
                lose_delete: false,
                conflict: false,
                dns_rejections: 0,
                dns_error_code: "CannotDeleteResource".into(),
            }
        }
        fn etag(&self) -> String {
            format!("\"{}\"", self.revision)
        }
        fn puts(&self) -> usize {
            self.writes
                .iter()
                .filter(|(m, _)| *m == Method::PUT)
                .count()
        }
        fn deletes(&self) -> usize {
            self.writes
                .iter()
                .filter(|(m, _)| *m == Method::DELETE)
                .count()
        }
    }
    impl Cloud for Fake {
        async fn read_control(&mut self, _: &str) -> Result<Option<(String, State)>> {
            Ok(Some((self.etag(), self.control.clone())))
        }
        async fn save_control(&mut self, _: &str, etag: Option<&str>, s: &State) -> Result<String> {
            s.validate()?;
            ensure!(
                !self.conflict && etag == Some(self.etag().as_str()),
                "CAS conflict"
            );
            self.revision += 1;
            self.control = s.clone();
            Ok(self.etag())
        }
        async fn arm(
            &mut self,
            method: Method,
            id: &str,
            _: &str,
            body: Option<&Value>,
            request: &str,
        ) -> Result<Reply> {
            if method == Method::GET {
                if id.ends_with("/virtualNetworkLinks") && !self.objects.contains_key(id) {
                    return Ok(reply(200, Some(json!({"value":[]}))));
                }
                return Ok(self
                    .objects
                    .get(id)
                    .cloned()
                    .map(|v| reply(200, Some(v)))
                    .unwrap_or_else(|| reply(404, None)));
            }
            self.writes.push((method.clone(), id.into()));
            if method == Method::PUT {
                let op = self
                    .control
                    .operations
                    .iter()
                    .find(|o| o.request_id == request)
                    .unwrap();
                assert_eq!(
                    op.phase,
                    Phase::Intent,
                    "Intent must be durable before mutation"
                );
                let stage = op.stage;
                let params = body.unwrap()["properties"]["parameters"].clone();
                self.objects.insert(id.into(),json!({"properties":{"parameters":params,"correlationId":"12345678-1234-4234-8234-123456789abc","provisioningState":"Succeeded"}}));
                for item in graph::inventory(&self.control.plan)
                    .into_iter()
                    .filter(|r| r["stage"] == stage.name())
                {
                    let rid = item["id"].as_str().unwrap();
                    let mut v = json!({"id":rid,"tags":{"ows-deployment":self.control.plan.deployment_id,"ows-nonce":self.control.nonce,"ows-bundle":self.control.bundle_digest},"systemData":{"createdAt":"2026-10-31T00:00:00Z"},"properties":{"creationTime":"2026-10-31T00:00:00Z","creationDate":"2026-10-31T00:00:00Z","resourceGuid":"receipt-guid","internalId":"opaque-zone-identity","principalId":"12345678-1234-4234-8234-123456789abc"}});
                    v["etag"] = json!("ignored-mutable-etag");
                    self.objects.insert(rid.into(), v);
                }
                if self.lose_put {
                    bail!("response lost after effect")
                }
                return Ok(reply(201, None));
            }
            if method == Method::DELETE {
                if id == job_id(&self.control.plan) {
                    assert!(
                        self.control.completed_unix.is_some()
                            && self
                                .control
                                .job_stop
                                .as_ref()
                                .is_some_and(|stop| stop.request_id == request)
                    );
                } else {
                    assert!(
                        self.control
                            .resources
                            .iter()
                            .any(|r| r.id == id && r.delete_request.as_deref() == Some(request)),
                        "Delete intent must be durable"
                    );
                }
                if id.contains("/privateDnsZones/") && self.dns_rejections > 0 {
                    self.dns_rejections -= 1;
                    return Ok(reply(
                        409,
                        Some(json!({"error":{"code":self.dns_error_code}})),
                    ));
                }
                self.objects.remove(id);
                if self.lose_delete {
                    self.lose_delete = false;
                    bail!("delete response lost after effect")
                };
                return Ok(reply(202, None));
            }
            bail!("Unexpected method")
        }
    }
    async fn active() -> (Fake, State, String) {
        let mut s = state();
        let mut c = Fake::new(s.clone());
        let mut e = c.etag();
        apply(&mut c, &mut s, &mut e, Stage::Foundation, 150)
            .await
            .unwrap();
        assert_eq!(s.operations[0].phase, Phase::Active);
        (c, s, e)
    }
    #[tokio::test]
    async fn lost_create_response_is_observed_without_replaying_original_put() {
        let mut s = state();
        let mut c = Fake::new(s.clone());
        c.lose_put = true;
        let mut e = c.etag();
        assert!(apply(&mut c, &mut s, &mut e, Stage::Foundation, 150)
            .await
            .is_err());
        assert_eq!(c.puts(), 1);
        assert_eq!(s.operations[0].phase, Phase::Unknown);
        assert!(apply(&mut c, &mut s, &mut e, Stage::Foundation, 150)
            .await
            .is_err());
        assert_eq!(c.puts(), 1);
        let (mut e, mut restarted) = c.read_control("").await.unwrap().unwrap();
        observe(&mut c, &mut restarted, &mut e).await.unwrap();
        assert_eq!(restarted.operations[0].phase, Phase::Active);
        assert_eq!(restarted.resources.len(), 10);
        assert_eq!(c.puts(), 1);
    }
    #[tokio::test]
    async fn cas_failure_prevents_any_infrastructure_write() {
        let mut s = state();
        let mut c = Fake::new(s.clone());
        c.conflict = true;
        let mut e = c.etag();
        assert!(apply(&mut c, &mut s, &mut e, Stage::Foundation, 150)
            .await
            .is_err());
        assert!(c.writes.is_empty());
        assert_eq!(c.control.operations[0].phase, Phase::Prepared);
    }
    #[tokio::test]
    async fn lost_delete_response_is_reconciled_after_restart_and_never_repeated() {
        let (mut c, mut s, mut e) = active().await;
        c.lose_delete = true;
        assert!(destroy_one(&mut c, &mut s, &mut e, 160).await.is_err());
        let deleted = c.writes.last().unwrap().1.clone();
        let (mut e, mut restarted) = c.read_control("").await.unwrap().unwrap();
        destroy_one(&mut c, &mut restarted, &mut e, 161)
            .await
            .unwrap();
        assert!(restarted
            .resources
            .iter()
            .any(|r| r.id == deleted && r.deleted));
        assert_eq!(
            c.writes
                .iter()
                .filter(|(m, id)| *m == Method::DELETE && id == &deleted)
                .count(),
            1
        );
    }
    #[tokio::test]
    async fn network_receipts_use_live_provider_fields_and_delete_vnet_before_dns() {
        let (mut c, mut s, mut e) = active().await;
        let mut protected = Vec::new();
        for item in graph::inventory(&s.plan) {
            let id = item["id"].as_str().unwrap();
            if id.contains("/privateEndpoints/") || id.contains("/privateDnsZones/") {
                let v = c.objects.get_mut(id).unwrap();
                v.as_object_mut().unwrap().remove("systemData");
                protected.push((
                    id.to_owned(),
                    item["incarnationPointer"].as_str().unwrap().to_owned(),
                ));
            }
        }
        observe(&mut c, &mut s, &mut e).await.unwrap();
        // Neither a mutable ETag nor ownership tags may replace the provider's identity.
        for (id, pointer) in protected {
            let original = c.objects[&id].pointer(&pointer).unwrap().clone();
            *c.objects
                .get_mut(&id)
                .unwrap()
                .pointer_mut(&pointer)
                .unwrap() = Value::Null;
            assert!(observe(&mut c, &mut s, &mut e).await.is_err());
            *c.objects
                .get_mut(&id)
                .unwrap()
                .pointer_mut(&pointer)
                .unwrap() = json!("replaced");
            assert!(observe(&mut c, &mut s, &mut e).await.is_err());
            *c.objects
                .get_mut(&id)
                .unwrap()
                .pointer_mut(&pointer)
                .unwrap() = original;
        }
        for _ in 0..15 {
            heartbeat(&mut c, &mut s, &mut e, 201).await.unwrap();
        }
        let deleted: Vec<_> = c
            .writes
            .iter()
            .filter(|(method, _)| *method == Method::DELETE)
            .map(|(_, id)| id)
            .collect();
        let network = deleted
            .iter()
            .position(|id| id.contains("/virtualNetworks/"))
            .unwrap();
        assert!(deleted
            .iter()
            .enumerate()
            .filter(|(_, id)| id.contains("/privateDnsZones/"))
            .all(|(i, _)| i > network));
    }

    #[tokio::test]
    async fn dns_cleanup_waits_for_provider_link_convergence_without_delete_intent() {
        let (mut c, mut s, mut e) = active().await;
        let zone = graph::inventory(&s.plan)
            .into_iter()
            .find(|r| r["id"].as_str().unwrap().contains("/privateDnsZones/"))
            .unwrap()["id"]
            .as_str()
            .unwrap()
            .to_owned();
        let links = format!("{zone}/virtualNetworkLinks");
        c.objects
            .insert(links.clone(), json!({"value":[{"id":"still-converging"}]}));
        for _ in 0..15 {
            heartbeat(&mut c, &mut s, &mut e, 201).await.unwrap();
        }
        assert!(!c
            .writes
            .iter()
            .any(|(m, id)| *m == Method::DELETE && id == &zone));
        assert!(s
            .resources
            .iter()
            .find(|r| r.id == zone)
            .unwrap()
            .delete_request
            .is_none());
        assert!(s.completed_unix.is_none());
        c.objects.remove(&links);
        for _ in 0..15 {
            heartbeat(&mut c, &mut s, &mut e, 201).await.unwrap();
        }
        assert!(s.completed_unix.is_some());
    }

    async fn blocked_dns() -> (Fake, State, String, String) {
        let (mut c, mut s, mut e) = active().await;
        c.dns_rejections = 1;
        for _ in 0..15 {
            destroy_one(&mut c, &mut s, &mut e, 201).await.unwrap();
            if s.resources
                .iter()
                .any(|r| !r.rejected_dns_deletes.is_empty())
            {
                break;
            }
        }
        let zone = s
            .resources
            .iter()
            .find(|r| !r.rejected_dns_deletes.is_empty())
            .unwrap()
            .id
            .clone();
        (c, s, e, zone)
    }
    #[tokio::test]
    async fn dns_proven_dependency_rejection_survives_restart_and_rechecks_before_new_request() {
        let (mut c, s, _, zone) = blocked_dns().await;
        let old = s
            .resources
            .iter()
            .find(|r| r.id == zone)
            .unwrap()
            .delete_request
            .clone();
        let (mut e, mut s) = c.read_control("").await.unwrap().unwrap();
        let before = c.deletes();
        destroy_one(&mut c, &mut s, &mut e, 260).await.unwrap();
        assert_eq!(c.deletes(), before, "backoff must survive restart");
        c.objects.insert(
            format!("{zone}/virtualNetworkLinks"),
            json!({"value":[{"id":"still-there"}]}),
        );
        destroy_one(&mut c, &mut s, &mut e, 261).await.unwrap();
        assert_eq!(c.deletes(), before);
        c.objects.remove(&format!("{zone}/virtualNetworkLinks"));
        destroy_one(&mut c, &mut s, &mut e, 262).await.unwrap();
        let resource = s.resources.iter().find(|r| r.id == zone).unwrap();
        assert_ne!(resource.delete_request, old);
        assert_eq!(resource.rejected_dns_deletes[0].request_id, old.unwrap());
        assert!(resource.delete_accepted);
        assert_eq!(c.deletes(), before + 1);
    }
    #[tokio::test]
    async fn dns_retry_refuses_replacement_unknown_response_and_other_conflicts() {
        let (mut c, mut s, mut e, zone) = blocked_dns().await;
        c.objects.get_mut(&zone).unwrap()["properties"]["internalId"] = json!("replacement");
        let count = c.deletes();
        assert!(destroy_one(&mut c, &mut s, &mut e, 300).await.is_err());
        assert_eq!(c.deletes(), count);
        let (mut c, mut s, mut e, zone) = blocked_dns().await;
        // Model a lost/legacy response: no proven-rejection receipt exists.
        s.resources
            .iter_mut()
            .find(|r| r.id == zone)
            .unwrap()
            .rejected_dns_deletes
            .clear();
        save(&mut c, &s, &mut e).await.unwrap();
        let count = c.deletes();
        destroy_one(&mut c, &mut s, &mut e, 400).await.unwrap();
        assert_eq!(c.deletes(), count);
        let (mut c, mut s, mut e) = active().await;
        c.dns_rejections = 1;
        c.dns_error_code = "AnotherConflict".into();
        for _ in 0..15 {
            if destroy_one(&mut c, &mut s, &mut e, 201).await.is_err() {
                break;
            }
        }
        assert!(s
            .resources
            .iter()
            .all(|r| r.rejected_dns_deletes.is_empty()));
        let count = c.deletes();
        destroy_one(&mut c, &mut s, &mut e, 400).await.unwrap();
        assert_eq!(c.deletes(), count);
    }
    #[tokio::test]
    async fn dns_dependency_attempts_are_bounded_and_keep_all_receipts() {
        let (mut c, mut s, mut e, _) = blocked_dns().await;
        c.dns_rejections = 20;
        for at in [261, 321, 381, 441] {
            destroy_one(&mut c, &mut s, &mut e, at).await.unwrap();
        }
        let count = c.deletes();
        assert!(destroy_one(&mut c, &mut s, &mut e, 501).await.is_err());
        assert_eq!(c.deletes(), count);
        assert_eq!(
            s.resources
                .iter()
                .map(|r| r.rejected_dns_deletes.len())
                .sum::<usize>(),
            5
        );
    }

    #[tokio::test]
    async fn aks_resource_uid_is_required_when_creation_metadata_is_absent() {
        let (mut c, mut s, mut e) = active().await;
        // The foundation is active; apply the cluster, then model the live API shape.
        apply(&mut c, &mut s, &mut e, Stage::Cluster, 150)
            .await
            .unwrap();
        let id = graph::inventory(&s.plan)
            .into_iter()
            .find(|r| r["stage"] == "cluster")
            .unwrap()["id"]
            .as_str()
            .unwrap()
            .to_owned();
        let v = c.objects.get_mut(&id).unwrap();
        v.as_object_mut().unwrap().remove("systemData");
        v["properties"]["resourceUID"] = json!("6af38caa5deaf50001940f2c");
        // First real observation must pin UID, never switch an existing receipt.
        s.resources.retain(|r| r.id != id);
        observe(&mut c, &mut s, &mut e).await.unwrap();
        assert_eq!(
            s.resources.iter().find(|r| r.id == id).unwrap().incarnation,
            "6af38caa5deaf50001940f2c"
        );
        c.objects.get_mut(&id).unwrap()["properties"]["resourceUID"] = json!("replacement");
        assert!(observe(&mut c, &mut s, &mut e).await.is_err());
        c.objects.get_mut(&id).unwrap()["properties"]["resourceUID"] = Value::Null;
        assert!(observe(&mut c, &mut s, &mut e).await.is_err());
    }
    #[tokio::test]
    async fn controller_upgrade_preserves_ownership_and_refuses_old_image_or_replacement() {
        let (mut c, mut s, mut e) = active().await;
        let original = serde_json::to_value((
            &s.plan,
            &s.resources,
            &s.operations,
            &s.guard,
            &s.nonce,
            &s.bundle_digest,
        ))
        .unwrap();
        let image = format!("ghcr.io/test/ows@sha256:{}", "b".repeat(64));
        prepare_image_upgrade(&mut c, &mut s, &mut e, &image, 160)
            .await
            .unwrap();
        assert_eq!(
            original,
            serde_json::to_value((
                &s.plan,
                &s.resources,
                &s.operations,
                &s.guard,
                &s.nonce,
                &s.bundle_digest
            ))
            .unwrap()
        );
        assert!(s.heartbeat_unix.is_none());
        assert!(verify_job(&mut c, &s).await.is_err());
        prepare_image_upgrade(&mut c, &mut s, &mut e, &image, 161)
            .await
            .unwrap();
        assert!(
            prepare_image_upgrade(&mut c, &mut s, &mut e, "foreign/image:latest", 162)
                .await
                .is_err()
        );
        c.objects.get_mut(&job_id(&s.plan)).unwrap()["properties"]["template"]["containers"][0]
            ["image"] = json!(image);
        verify_job(&mut c, &s).await.unwrap();
        c.objects.get_mut(&job_id(&s.plan)).unwrap()["systemData"]["createdAt"] = json!("replaced");
        assert!(verify_job(&mut c, &s).await.is_err());
    }

    #[tokio::test]
    async fn replacement_or_missing_incarnation_refuses_deletion() {
        let (mut c, mut s, mut e) = active().await;
        let id = s.resources[0].id.clone();
        let pointer = graph::inventory(&s.plan)
            .into_iter()
            .find(|r| r["id"] == id)
            .unwrap()["incarnationPointer"]
            .as_str()
            .unwrap()
            .to_owned();
        *c.objects
            .get_mut(&id)
            .unwrap()
            .pointer_mut(&pointer)
            .unwrap() = json!("replacement");
        assert!(destroy_one(&mut c, &mut s, &mut e, 160).await.is_err());
        assert_eq!(c.deletes(), 0);
    }
    #[tokio::test]
    async fn persistent_workload_is_not_deleted_at_provisioning_deadline() {
        let mut s = state();
        s.plan.auto_expire = Some(false);
        s.bundle_digest = digest(&s.plan).unwrap();
        let mut c = Fake::new(s.clone());
        let mut e = c.etag();
        apply(&mut c, &mut s, &mut e, Stage::Foundation, 150)
            .await
            .unwrap();
        heartbeat(&mut c, &mut s, &mut e, 500).await.unwrap();
        assert_eq!(c.deletes(), 0);
        assert!(destroy_one(&mut c, &mut s, &mut e, 500).await.is_err());
        assert_eq!(c.deletes(), 0);
    }

    #[tokio::test]
    async fn expiry_runs_outside_cluster_and_stops_its_job_after_workload_cleanup() {
        let (mut c, mut s, mut e) = active().await;
        for _ in 0..15 {
            heartbeat(&mut c, &mut s, &mut e, 201).await.unwrap();
        }
        assert!(s.completed_unix.is_some());
        assert!(s.resources.iter().all(|r| r.deleted));
        assert_eq!(c.deletes(), 11);
        assert!(s.job_stop.is_some());
        heartbeat(&mut c, &mut s, &mut e, 202).await.unwrap();
        assert_eq!(c.deletes(), 11);
        assert!(
            !c.writes.iter().any(|(_, id)| id == &scope(&s.plan)),
            "Customer RG must never be deleted"
        );
    }
    #[tokio::test]
    async fn expiry_and_missing_guard_prevent_creates_and_unknown_404_stays_parked() {
        let mut s = state();
        let mut c = Fake::new(s.clone());
        let mut e = c.etag();
        assert!(apply(&mut c, &mut s, &mut e, Stage::Foundation, 201)
            .await
            .is_err());
        s.heartbeat_unix = None;
        assert!(apply(&mut c, &mut s, &mut e, Stage::Foundation, 150)
            .await
            .is_err());
        s.operations[0].phase = Phase::Unknown;
        save(&mut c, &s, &mut e).await.unwrap();
        observe(&mut c, &mut s, &mut e).await.unwrap();
        assert_eq!(s.operations[0].phase, Phase::Unknown);
        assert!(destroy_one(&mut c, &mut s, &mut e, 201).await.is_err());
        assert!(c.writes.is_empty());
    }
    #[test]
    fn azure_creation_receipt_preserves_timezone_less_fractional_timestamp() {
        let mut s = state();
        s.guard = None;
        let mut body = fixture_job(&s);
        body["systemData"]["createdAt"] = json!("2026-11-10T18:49:20.2965605");
        let receipt = guard_body(&s, &body).unwrap();
        assert_eq!(receipt.created_at, "2026-11-10T18:49:20.2965605");
        s.guard = Some(receipt);
        body["systemData"]["createdAt"] = json!("2026-11-10T18:49:20.2965606");
        assert!(guard_body(&s, &body).is_err());
        s.guard = None;
        body["systemData"]["createdAt"] = json!("not-a-timestamp");
        assert!(guard_body(&s, &body).is_err());
    }
    #[test]
    fn private_registry_pulls_bind_cleanup_identity_and_reject_credential_drift() {
        let mut s = state();
        let input = s.plan.azure_infrastructure.as_mut().unwrap();
        input.guard_registry_resource_id = Some("/subscriptions/66666666-7777-4888-8999-aaaaaaaaaaaa/resourceGroups/customer-control/providers/Microsoft.ContainerRegistry/registries/controlacr".into());
        input.guard_image = format!("controlacr.azurecr.io/cleanup@sha256:{}", "a".repeat(64));
        s.bundle_digest = digest(&s.plan).unwrap();
        let job = fixture_job(&s);
        assert!(guard_body(&s, &job).is_ok());
        // Azure's completed GET normalizes omitted registry credentials to "".
        for empty in [Value::Null, json!("")] {
            let mut normalized = job.clone();
            normalized["properties"]["configuration"]["registries"][0]["username"] = empty.clone();
            normalized["properties"]["configuration"]["registries"][0]["passwordSecretRef"] = empty;
            assert!(guard_body(&s, &normalized).is_ok());
        }

        let files = control_artifacts(&s).unwrap();
        let binding: Value =
            serde_json::from_slice(&files["azure-control-registry-binding.json"]).unwrap();
        assert_eq!(
            binding["scope"],
            s.plan
                .azure_infrastructure
                .as_ref()
                .unwrap()
                .guard_registry_resource_id
                .as_ref()
                .unwrap()
                .as_str()
        );
        for (field, value) in [
            ("identity", json!("system")),
            ("server", json!("foreign.azurecr.io")),
            ("passwordSecretRef", json!("password")),
            ("username", json!("admin")),
        ] {
            let mut changed = job.clone();
            changed["properties"]["configuration"]["registries"][0][field] = value;
            assert!(guard_body(&s, &changed).is_err());
        }
    }
    #[test]
    fn control_package_has_no_keys_no_cluster_dependency_and_retains_audit_control() {
        let s = state();
        let files = control_artifacts(&s).unwrap();
        let v: Value = serde_json::from_slice(&files["azure-control.arm.json"]).unwrap();
        let job = v["resources"]
            .as_array()
            .unwrap()
            .iter()
            .find(|v| v["type"] == "Microsoft.App/jobs")
            .unwrap();
        assert_eq!(job["properties"]["configuration"]["replicaRetryLimit"], 0);
        let environment = v["resources"]
            .as_array()
            .unwrap()
            .iter()
            .find(|r| r["type"] == "Microsoft.App/managedEnvironments")
            .unwrap();
        assert_eq!(
            environment["properties"]["appLogsConfiguration"]["destination"],
            Value::Null
        );
        assert!(!job.to_string().contains("managedClusters"));
        let grants: String = files
            .iter()
            .filter(|(n, _)| n.contains("role"))
            .map(|(_, b)| String::from_utf8_lossy(b))
            .collect();
        assert!(!grants.contains('*'));
        assert!(!grants.contains("roleAssignments/write"));
        assert!(!grants.contains("resourceGroups/delete"));
        let mut altered = s.clone();
        altered.nonce = "".into();
        assert!(altered.validate().is_err());
    }
}

pub(super) fn local_plan(root: &Path) -> Result<Plan> {
    Ok(local(root)?.plan)
}
pub(super) async fn installation_snapshot(
    root: &Path,
    provisioning: bool,
) -> Result<(Plan, Azure, Value)> {
    let expected = local(root)?;
    let mut c = Azure::new(&expected.plan, None)?;
    let (_, s) = c
        .read_control(&control_url(&expected.plan))
        .await?
        .context("Shared Azure control missing")?;
    ensure!(
        s.nonce == expected.nonce
            && s.bundle_digest == expected.bundle_digest
            && s.operations.iter().all(|o| o.phase == Phase::Active),
        "Original infrastructure stages are not active"
    );
    if provisioning {
        let at = now()?;
        ensure!(
            at < s.plan.expires_at_unix
                && !s.destroy_requested
                && s.completed_unix.is_none()
                && s.heartbeat_unix.is_some_and(|t| t <= at && at - t <= 600),
            "Expired/closing infrastructure or stale independent cleanup"
        );
    }
    let entry = graph::inventory(&s.plan)
        .into_iter()
        .find(|v| v["stage"] == "cluster")
        .unwrap();
    let id = entry["id"].as_str().unwrap();
    let r = c
        .arm(
            Method::GET,
            id,
            entry["apiVersion"].as_str().unwrap(),
            None,
            &s.nonce,
        )
        .await?;
    ensure!(r.status == 200, "Original AKS cluster missing");
    let cluster = r.body.context("AKS response missing")?;
    let incarnation = owned_body(&s, &cluster, &entry)?;
    ensure!(
        s.resources
            .iter()
            .any(|r| r.id == id && r.incarnation == incarnation && !r.deleted),
        "AKS ownership receipt missing/changed"
    );
    Ok((s.plan, c, cluster))
}
pub(super) async fn installation_mark(root: &Path, digest: &str, closed: bool) -> Result<()> {
    ensure!(
        digest.len() == 64
            && digest
                .bytes()
                .all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(&c)),
        "Invalid installation binding digest"
    );
    let expected = local(root)?;
    let mut c = Azure::new(&expected.plan, None)?;
    let (mut e, mut s) = c
        .read_control(&control_url(&expected.plan))
        .await?
        .context("Shared control missing")?;
    ensure!(
        s.nonce == expected.nonce && s.bundle_digest == expected.bundle_digest,
        "Azure installation identity mismatch"
    );
    if closed {
        ensure!(
            s.installation_digest.as_deref() == Some(digest),
            "Original installation mutation receipt missing"
        );
        s.installation_closed = true;
    } else {
        ensure!(
            now()? < s.plan.expires_at_unix && !s.destroy_requested && !s.installation_closed,
            "Infrastructure is closing/expired"
        );
        if let Some(old) = &s.installation_digest {
            ensure!(
                old == digest,
                "Another installation binding already has mutation intent"
            );
        }
        s.installation_digest = Some(digest.to_owned());
    }
    save(&mut c, &s, &mut e).await
}

#[cfg(test)]
pub(super) fn fixture_plan() -> Plan {
    tests::state().plan
}
#[cfg(test)]
pub(super) fn fixture_state_dir(root: &Path) -> Plan {
    let s = tests::state();
    private_dir(root).unwrap();
    put(root, "azure-state.json", &json_bytes(&s).unwrap()).unwrap();
    s.plan
}

async fn verify_job(c: &mut impl Cloud, s: &State) -> Result<GuardReceipt> {
    let r = c
        .arm(Method::GET, &job_id(&s.plan), "2024-03-01", None, &s.nonce)
        .await?;
    ensure!(r.status == 200, "Independent cleanup job unavailable");
    guard_body(s, &r.body.context("Cleanup job body missing")?)
}
fn guard_receipt(s: &State, body: &Value) -> Result<GuardReceipt> {
    let stamp = body["systemData"]["createdAt"]
        .as_str()
        .context("Controller immutable creation receipt missing")?
        .to_owned();
    let mi = format!(
        "{}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{}-cleanup",
        graph::control_scope(&s.plan),
        graph::names(&s.plan).prefix
    );
    let identities = body["identity"]["userAssignedIdentities"]
        .as_object()
        .context("Controller identity map missing")?;
    ensure!(
        identities.len() == 1,
        "Exactly one controller managed identity required"
    );
    let identity = identities
        .iter()
        .find(|(id, _)| id.eq_ignore_ascii_case(&mi))
        .map(|(_, value)| value)
        .context("Controller identity scope mismatch")?;
    let receipt = GuardReceipt {
        created_at: stamp,
        principal_id: identity["principalId"]
            .as_str()
            .context("Controller principal receipt missing")?
            .to_owned(),
        client_id: identity["clientId"]
            .as_str()
            .context("Controller client receipt missing")?
            .to_owned(),
    };
    receipt.validate()?;
    ensure!(
        body["id"]
            .as_str()
            .is_some_and(|id| id.eq_ignore_ascii_case(&job_id(&s.plan)))
            && body["tags"]["ows-deployment"] == s.plan.deployment_id
            && body["tags"]["ows-nonce"] == s.nonce
            && body["tags"]["ows-bundle"] == s.bundle_digest
            && s.guard.as_ref().is_none_or(|old| old == &receipt),
        "Independent cleanup controller replaced/identity changed"
    );
    Ok(receipt)
}
fn guard_body(s: &State, body: &Value) -> Result<GuardReceipt> {
    let receipt = guard_receipt(s, body)?;
    let resources = &body["properties"]["template"]["containers"];
    let container = resources
        .as_array()
        .filter(|v| v.len() == 1)
        .map(|v| &v[0])
        .context("Exactly one cleanup container required")?;
    let config = &body["properties"]["configuration"];
    let args = container["args"]
        .as_array()
        .context("Cleanup arguments missing")?;
    let Target::Azure {
        tenant,
        subscription,
        ..
    } = &s.plan.target
    else {
        unreachable!()
    };
    let mi = format!(
        "{}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{}-cleanup",
        graph::control_scope(&s.plan),
        graph::names(&s.plan).prefix
    );
    ensure!(
        body["properties"]["provisioningState"] == "Succeeded"
            && body["properties"]["environmentId"]
                .as_str()
                .is_some_and(|id| id.eq_ignore_ascii_case(&environment_id(&s.plan)))
            && container["image"] == controller_image(s)
            && container["command"] == json!(["/opt/openworker/ows"])
            && args.len() == 12
            && args[..4]
                == json!(["deploy", "azure", "cleanup", "tick"])
                    .as_array()
                    .unwrap()[..]
            && args[4] == "--control"
            && args[5] == control_url(&s.plan)
            && args[6] == "--tenant"
            && args[7] == *tenant
            && args[8] == "--subscription"
            && args[9] == *subscription
            && args[10] == "--client-id"
            && args[11] == receipt.client_id
            && body["identity"]["type"] == "UserAssigned"
            && body["identity"]["userAssignedIdentities"]
                .as_object()
                .is_some_and(|m| m.len() == 1 && m.keys().any(|id| id.eq_ignore_ascii_case(&mi)))
            && config["triggerType"] == "Schedule"
            && config["replicaRetryLimit"] == 0
            && config["scheduleTriggerConfig"]["cronExpression"] == "*/5 * * * *"
            && config["scheduleTriggerConfig"]["parallelism"] == 1
            && config["scheduleTriggerConfig"]["replicaCompletionCount"] == 1,
        "Independent cleanup job configuration drift"
    );
    if let Some(registry) = &s
        .plan
        .azure_infrastructure
        .as_ref()
        .unwrap()
        .guard_registry_resource_id
    {
        let entries = config["registries"]
            .as_array()
            .filter(|v| v.len() == 1)
            .context("Exactly one managed-identity cleanup registry required")?;
        let entry = &entries[0];
        ensure!(
            entry["server"] == format!("{}.azurecr.io", registry.rsplit('/').next().unwrap())
                && entry["identity"]
                    .as_str()
                    .is_some_and(|v| v.eq_ignore_ascii_case(&mi))
                && entry
                    .get("username")
                    .is_none_or(|v| v.is_null() || v.as_str() == Some(""))
                && entry
                    .get("passwordSecretRef")
                    .is_none_or(|v| v.is_null() || v.as_str() == Some("")),
            "Cleanup registry authentication drift"
        );
    } else {
        ensure!(
            config
                .get("registries")
                .is_none_or(|v| v.is_null() || v.as_array().is_some_and(Vec::is_empty)),
            "Unexpected cleanup registry credentials"
        );
    }
    Ok(receipt)
}

#[cfg(test)]
mod guard_tests {
    use super::*;
    #[test]
    fn existing_control_environment_is_external_and_bound_to_exact_scope() {
        let mut s = tests::state();
        let env = format!(
            "{}/providers/Microsoft.App/managedEnvironments/existing-control",
            graph::control_scope(&s.plan)
        );
        s.plan
            .azure_infrastructure
            .as_mut()
            .unwrap()
            .guard_environment_resource_id = Some(env.clone());
        s.bundle_digest = digest(&s.plan).unwrap();
        s.validate().unwrap();
        let files = control_artifacts(&s).unwrap();
        let role: Value = serde_json::from_slice(&files["azure-cleanup-role.json"]).unwrap();
        assert!(role["properties"]["permissions"][0]["actions"]
            .as_array()
            .unwrap()
            .contains(&json!(
                "Microsoft.Network/privateDnsZones/virtualNetworkLinks/read"
            )));
        let template: Value = serde_json::from_slice(&files["azure-control.arm.json"]).unwrap();
        assert!(!template["resources"]
            .as_array()
            .unwrap()
            .iter()
            .any(|r| r["type"] == "Microsoft.App/managedEnvironments"));
        let job = template["resources"]
            .as_array()
            .unwrap()
            .iter()
            .find(|r| r["type"] == "Microsoft.App/jobs")
            .unwrap();
        assert_eq!(job["properties"]["environmentId"], env);
        assert_eq!(job["dependsOn"].as_array().unwrap().len(), 1);
        let mut live = fixture_job(&s);
        guard_body(&s, &live).unwrap();
        live["properties"]["environmentId"] = json!("/foreign/environment");
        assert!(guard_body(&s, &live).is_err());
        s.plan
            .azure_infrastructure
            .as_mut()
            .unwrap()
            .guard_environment_resource_id = Some(format!(
            "{}/providers/Microsoft.App/managedEnvironments/existing-control",
            scope(&s.plan)
        ));
        assert!(s.plan.validate().is_err());
    }

    #[tokio::test]
    async fn cleanup_configuration_drift_blocks_workload_submission() {
        struct ReadOnly {
            body: Value,
        }
        impl Cloud for ReadOnly {
            async fn read_control(&mut self, _: &str) -> Result<Option<(String, State)>> {
                bail!("unexpected control read")
            }
            async fn save_control(
                &mut self,
                _: &str,
                _: Option<&str>,
                _: &State,
            ) -> Result<String> {
                bail!("unexpected mutation")
            }
            async fn arm(
                &mut self,
                m: Method,
                _: &str,
                _: &str,
                _: Option<&Value>,
                _: &str,
            ) -> Result<Reply> {
                ensure!(m == Method::GET, "Guard verification must only read");
                Ok(Reply {
                    status: 200,
                    etag: None,
                    operation: None,
                    body: Some(self.body.clone()),
                })
            }
        }
        let s = tests::state();
        let body = fixture_job(&s);
        verify_job(&mut ReadOnly { body: body.clone() }, &s)
            .await
            .unwrap();
        for pointer in [
            "/properties/template/containers/0/image",
            "/properties/template/containers/0/args/5",
            "/properties/configuration/scheduleTriggerConfig/cronExpression",
        ] {
            let mut changed = body.clone();
            *changed.pointer_mut(pointer).unwrap() = json!("drift");
            assert!(verify_job(&mut ReadOnly { body: changed }, &s)
                .await
                .is_err());
        }
        let mut replaced = body;
        replaced["systemData"]["createdAt"] = json!("2026-10-31T00:00:01Z");
        assert!(
            verify_job(&mut ReadOnly { body: replaced }, &s)
                .await
                .is_err(),
            "Same-name controller replacement must never be deleted"
        );
        let mut invalid = s;
        invalid.installation_closed = true;
        assert!(invalid.validate().is_err());
    }
}

#[cfg(test)]
fn fixture_job(s: &State) -> Value {
    let files = control_artifacts(s).unwrap();
    let graph: Value = serde_json::from_slice(&files["azure-control.arm.json"]).unwrap();
    let mut body = graph["resources"]
        .as_array()
        .unwrap()
        .iter()
        .find(|r| r["type"] == "Microsoft.App/jobs")
        .unwrap()
        .clone();
    body["properties"]["template"]["containers"][0]["args"][11] =
        json!("12345678-1234-4234-8234-123456789abc");
    body["id"] = json!(job_id(&s.plan));
    body["properties"]["provisioningState"] = json!("Succeeded");
    body["systemData"] = json!({"createdAt":"2026-10-31T00:00:00Z"});
    let mi = format!(
        "{}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{}-cleanup",
        graph::control_scope(&s.plan),
        graph::names(&s.plan).prefix
    );
    body["identity"]["userAssignedIdentities"][&mi] =
        json!({"clientId":fixture_guard().client_id,"principalId":fixture_guard().principal_id});
    body
}

#[cfg(test)]
fn fixture_guard() -> GuardReceipt {
    GuardReceipt {
        created_at: "2026-10-31T00:00:00Z".into(),
        principal_id: "22345678-1234-4234-8234-123456789abc".into(),
        client_id: "12345678-1234-4234-8234-123456789abc".into(),
    }
}

#[cfg(test)]
mod review_tests {
    use super::*;
    #[derive(Clone, Copy)]
    enum Delete {
        Success,
        Rejected,
        LostBefore,
        LostAfter,
        AcceptedPending,
    }
    struct Fixture {
        stored: Option<State>,
        revision: u64,
        job: Option<Value>,
        deletes: usize,
        reads: usize,
        saves: usize,
        behavior: Delete,
        lose_initial_control: bool,
    }
    impl Fixture {
        fn new(s: &State, initialized: bool) -> Self {
            Self {
                stored: initialized.then(|| s.clone()),
                revision: 1,
                job: Some(fixture_job(s)),
                deletes: 0,
                reads: 0,
                saves: 0,
                behavior: Delete::Success,
                lose_initial_control: false,
            }
        }
        fn etag(&self) -> String {
            format!("\"{}\"", self.revision)
        }
    }
    impl Cloud for Fixture {
        async fn read_control(&mut self, _: &str) -> Result<Option<(String, State)>> {
            Ok(self.stored.as_ref().map(|s| (self.etag(), s.clone())))
        }
        async fn save_control(&mut self, _: &str, e: Option<&str>, s: &State) -> Result<String> {
            s.validate()?;
            ensure!(
                match &self.stored {
                    None => e.is_none(),
                    Some(_) => e == Some(self.etag().as_str()),
                },
                "CAS conflict"
            );
            let initial = self.stored.is_none();
            self.stored = Some(s.clone());
            self.revision += 1;
            self.saves += 1;
            if initial && self.lose_initial_control {
                bail!("control response lost after effect")
            };
            Ok(self.etag())
        }
        async fn arm(
            &mut self,
            m: Method,
            id: &str,
            _: &str,
            _: Option<&Value>,
            request: &str,
        ) -> Result<Reply> {
            let s = self.stored.as_ref();
            let expected = s.map(|s| job_id(&s.plan));
            if let Some(expected) = expected {
                ensure!(id == expected, "Unexpected resource");
            }
            if m == Method::GET {
                self.reads += 1;
                return Ok(Reply {
                    status: if self.job.is_some() { 200 } else { 404 },
                    body: self.job.clone(),
                    etag: None,
                    operation: None,
                });
            }
            ensure!(m == Method::DELETE, "Unexpected mutation");
            let s = self.stored.as_ref().unwrap();
            assert!(
                s.job_stop.as_ref().is_some_and(
                    |stop| stop.phase == StopPhase::Intent && stop.request_id == request
                ),
                "Stop intent must be durable before DELETE"
            );
            self.deletes += 1;
            match self.behavior {
                Delete::Success=>{self.job=None;Ok(Reply{status:200,body:None,etag:None,operation:None})},
                Delete::Rejected=>Ok(Reply{status:403,body:None,etag:None,operation:None}),
                Delete::LostBefore=>bail!("connection lost before observed effect"),
                Delete::LostAfter=>{self.job=None;bail!("connection lost after committed effect")},
                Delete::AcceptedPending=>Ok(Reply{status:202,body:None,etag:None,operation:Some("https://management.azure.com/subscriptions/66666666-7777-4888-8999-aaaaaaaaaaaa/providers/Microsoft.App/locations/westus2/operationResults/exact?api-version=2024-03-01".into())}),
            }
        }
    }
    fn pending_install() -> State {
        let mut s = tests::state();
        s.guard = None;
        s.heartbeat_unix = None;
        s
    }
    fn completed() -> State {
        let mut s = tests::state();
        s.destroy_requested = true;
        s.completed_unix = Some(160);
        s
    }
    #[tokio::test]
    async fn installer_pins_provider_receipt_before_create_only_control_and_first_tick_rejects_replacement(
    ) {
        let mut s = pending_install();
        let mut f = Fixture::new(&s, false);
        initialize(&mut f, &mut s).await.unwrap();
        assert_eq!(s.guard, Some(fixture_guard()));
        assert_eq!(f.stored.as_ref().unwrap().guard, s.guard);
        assert_eq!(s.heartbeat_unix, None);
        assert_eq!(f.saves, 1);
        assert!(initialize(&mut f, &mut pending_install()).await.is_err());
        let mut e = f.etag();
        f.job.as_mut().unwrap()["systemData"]["createdAt"] = json!("2026-10-31T00:00:01Z");
        assert!(tick(&mut f, &mut s, &mut e, 150).await.is_err());
        assert_eq!(s.heartbeat_unix, None);
        assert_eq!(f.saves, 1);
        assert_eq!(f.deletes, 0);
    }
    #[tokio::test]
    async fn scheduled_job_cannot_bootstrap_its_own_receipt_or_accept_a_wrong_client_argument() {
        let mut s = pending_install();
        let mut f = Fixture::new(&s, true);
        let mut e = f.etag();
        assert!(tick(&mut f, &mut s, &mut e, 150).await.is_err());
        assert_eq!(f.reads, 0);
        assert_eq!(f.saves, 0);
        let mut f = Fixture::new(&s, false);
        f.job.as_mut().unwrap()["properties"]["template"]["containers"][0]["args"][11] =
            json!("33333333-3333-4333-8333-333333333333");
        assert!(initialize(&mut f, &mut s).await.is_err());
        assert!(f.stored.is_none());
        assert_eq!(f.saves, 0);
        let bound = tests::state();
        tick_identity(&bound, &fixture_guard().client_id).unwrap();
        assert!(tick_identity(&bound, "33333333-3333-4333-8333-333333333333").is_err());
        let mut bad = fixture_job(&bound);
        bad["properties"]["provisioningState"] = json!("Deleting");
        assert!(guard_body(&bound, &bad).is_err());
        assert!(
            guard_receipt(&bound, &bad).is_ok(),
            "Deleting owned job remains observable during stop reconciliation"
        );
    }
    #[tokio::test]
    async fn lost_initialization_reply_keeps_the_original_pinned_receipt_and_never_overwrites() {
        let mut s = pending_install();
        let mut f = Fixture::new(&s, false);
        f.lose_initial_control = true;
        assert!(initialize(&mut f, &mut s).await.is_err());
        let receipt = f.stored.as_ref().unwrap().guard.clone();
        assert_eq!(receipt, Some(fixture_guard()));
        assert!(initialize(&mut f, &mut pending_install()).await.is_err());
        assert_eq!(f.saves, 1);
        assert_eq!(f.stored.as_ref().unwrap().guard, receipt);
    }
    #[tokio::test]
    async fn rejected_or_unknown_stop_remains_visible_after_restart_without_repeating_delete() {
        for behavior in [Delete::Rejected, Delete::LostBefore] {
            let mut s = completed();
            let mut f = Fixture::new(&s, true);
            f.behavior = behavior;
            let mut e = f.etag();
            assert!(heartbeat(&mut f, &mut s, &mut e, 170).await.is_err());
            assert_eq!(s.job_stop.as_ref().unwrap().phase, StopPhase::Unknown);
            let saved_heartbeat = s.heartbeat_unix;
            let (_, saved) = f.read_control("").await.unwrap().unwrap();
            let mut s: State =
                serde_json::from_slice(&serde_json::to_vec(&saved).unwrap()).unwrap();
            tick(&mut f, &mut s, &mut e, 180).await.unwrap();
            assert_eq!(f.deletes, 1);
            assert_eq!(s.heartbeat_unix, saved_heartbeat);
            assert_eq!(s.status()["requiresAttention"], true);
            assert_eq!(s.status()["controllerStopped"], false);
            assert_eq!(s.status()["workloadAndControllerComplete"], false);
            assert_eq!(s.status()["controllerStop"]["phase"], "unknown");
        }
    }
    #[tokio::test]
    async fn lost_stop_reply_after_effect_is_confirmed_absent_on_read_without_replay() {
        let mut s = completed();
        let mut f = Fixture::new(&s, true);
        f.behavior = Delete::LostAfter;
        let mut e = f.etag();
        assert!(heartbeat(&mut f, &mut s, &mut e, 170).await.is_err());
        let mut restarted = f.stored.clone().unwrap();
        tick(&mut f, &mut restarted, &mut e, 180).await.unwrap();
        assert_eq!(f.deletes, 1);
        assert_eq!(
            restarted.job_stop.as_ref().unwrap().phase,
            StopPhase::Absent
        );
        assert_eq!(restarted.status()["controllerStopped"], true);
        assert_eq!(restarted.status()["requiresAttention"], false);
    }
    #[tokio::test]
    async fn accepted_stop_is_pending_until_absence_and_persisted_intent_never_replays() {
        let mut s = completed();
        let mut f = Fixture::new(&s, true);
        f.behavior = Delete::AcceptedPending;
        let mut e = f.etag();
        heartbeat(&mut f, &mut s, &mut e, 170).await.unwrap();
        assert_eq!(s.job_stop.as_ref().unwrap().phase, StopPhase::Accepted);
        assert_eq!(s.status()["workloadAndControllerComplete"], false);
        assert!(s.job_stop.as_ref().unwrap().operation_url.is_some());
        tick(&mut f, &mut s, &mut e, 180).await.unwrap();
        assert_eq!(f.deletes, 1);
        f.job = None;
        reconcile_stop(&mut f, &mut s, &mut e).await.unwrap();
        assert_eq!(s.job_stop.as_ref().unwrap().phase, StopPhase::Absent);
        assert_eq!(s.status()["workloadAndControllerComplete"], true);
        let mut s = completed();
        s.job_stop = Some(Stop {
            request_id: uuid::Uuid::new_v4().to_string(),
            phase: StopPhase::Intent,
            operation_url: None,
        });
        let mut f = Fixture::new(&s, true);
        let mut e = f.etag();
        tick(&mut f, &mut s, &mut e, 180).await.unwrap();
        assert_eq!(f.deletes, 0);
        assert_eq!(s.job_stop.as_ref().unwrap().phase, StopPhase::Unknown);
        assert_eq!(s.status()["requiresAttention"], true);
    }
}