Python
fix(azure): pin controller upgrades and recognize AKS resource identity
Rohit P98d6c02f404a
diff --git a/harness/service-cli/src/deployment_azure_lifecycle.rs b/harness/service-cli/src/deployment_azure_lifecycle.rsindex a181194e..92c6e1f8 100644--- a/harness/service-cli/src/deployment_azure_lifecycle.rs+++ b/harness/service-cli/src/deployment_azure_lifecycle.rs@@ -55,6 +55,13 @@ pub enum Cleanup {#[arg(long)]state: PathBuf,},+ /// Pin a reviewed replacement image before an administrator updates the same job.+ PrepareUpgrade {+ #[command(flatten)]+ remote: Remote,+ #[arg(long)]+ image: String,+ },/// One scheduled iteration outside AKS. Uses Container Apps managed identity.Tick {#[arg(long)]@@ -161,6 +168,8 @@ pub(super) struct State {heartbeat_unix: Option<u64>,#[serde(default, skip_serializing_if = "Option::is_none")]guard: Option<GuardReceipt>,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ image_upgrade: Option<ImageUpgrade>,destroy_requested: bool,completed_unix: Option<u64>,job_stop: Option<Stop>,@@ -169,6 +178,39 @@ pub(super) struct State {#[serde(default)]installation_closed: bool,}+#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+struct ImageUpgrade {+ previous_image: String,+ image: String,+ request_id: String,+ prepared_at_unix: u64,+ controller: GuardReceipt,+}+fn controller_image(s: &State) -> &str {+ s.image_upgrade+ .as_ref()+ .map(|u| u.image.as_str())+ .unwrap_or(&s.plan.azure_infrastructure.as_ref().unwrap().guard_image)+}+fn validate_image_upgrade(s: &State, image: &str) -> Result<()> {+ let original = &s.plan.azure_infrastructure.as_ref().unwrap().guard_image;+ let (repo, _) = original+ .split_once("@sha256:")+ .context("Original image is not pinned")?;+ let (next, digest) = image+ .split_once("@sha256:")+ .context("Replacement image must be pinned")?;+ ensure!(+ next == repo+ && digest.len() == 64+ && digest+ .bytes()+ .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)),+ "Replacement must be a SHA256 digest in the original cleanup repository"+ );+ Ok(())+}fn digest(p: &Plan) -> Result<String> {let files = super::super::artifacts(p)?;let hashes: BTreeMap<_, _> = files.into_iter().map(|(n, b)| (n, hash(&b))).collect();@@ -248,6 +290,18 @@ impl State {if let Some(receipt) = &self.guard {receipt.validate()?;}+ if let Some(u) = &self.image_upgrade {+ validate_image_upgrade(self, &u.image)?;+ ensure!(+ u.previous_image == self.plan.azure_infrastructure.as_ref().unwrap().guard_image+ && self.guard.as_ref() == Some(&u.controller)+ && uuid::Uuid::parse_str(&u.request_id).is_ok_and(|id| !id.is_nil())+ && u.prepared_at_unix >= self.plan.created_at_unix+ && u.prepared_at_unix < self.plan.expires_at_unix,+ "Invalid pinned controller upgrade receipt"+ );+ }+ensure!(self.heartbeat_unix.is_none() || self.guard.is_some(),"Heartbeat requires an installer-pinned controller receipt"@@ -313,7 +367,7 @@ impl State {.job_stop.as_ref().is_some_and(|r| matches!(r.phase, StopPhase::Intent | StopPhase::Unknown));- json!({"deploymentId":self.plan.deployment_id,"bundleDigest":self.bundle_digest,"control":control_url(&self.plan),"stages":self.operations,"ownedResourceCount":self.resources.len(),"deletedResourceCount":self.resources.iter().filter(|r|r.deleted).count(),"heartbeatUnix":self.heartbeat_unix,"controllerReceipt":self.guard,"controllerStop":self.job_stop,"controllerStopped":stopped,"requiresAttention":attention,"workloadAndControllerComplete":self.completed_unix.is_some()&&stopped,"workloadCompletedUnix":self.completed_unix,"destroyRequested":self.destroy_requested,"completedUnix":self.completed_unix,"retainedControlResources":true,"liveQualified":false})+ json!({"deploymentId":self.plan.deployment_id,"bundleDigest":self.bundle_digest,"control":control_url(&self.plan),"stages":self.operations,"ownedResourceCount":self.resources.len(),"deletedResourceCount":self.resources.iter().filter(|r|r.deleted).count(),"heartbeatUnix":self.heartbeat_unix,"controllerReceipt":self.guard,"controllerImage":controller_image(self),"controllerUpgrade":self.image_upgrade,"controllerStop":self.job_stop,"controllerStopped":stopped,"requiresAttention":attention,"workloadAndControllerComplete":self.completed_unix.is_some()&&stopped,"workloadCompletedUnix":self.completed_unix,"destroyRequested":self.destroy_requested,"completedUnix":self.completed_unix,"retainedControlResources":true,"liveQualified":false})}}fn local(path: &Path) -> Result<State> {@@ -412,8 +466,18 @@ fn owned_body(s: &State, r: &Value, expected: &Value) -> Result<String> {&& r["tags"]["ows-bundle"] == s.bundle_digest,"Resource ownership mismatch; names/tags alone are not receipts");+ // AKS resourceUID distinguishes delete/recreate under the same ARM ID. Some+ // live responses omit systemData; preserve older frozen graph descriptors and+ // accept this documented immutable identity only for managedClusters.let receipt = r.pointer(expected["incarnationPointer"].as_str().unwrap())+ .filter(|v| v.as_str().is_some_and(|s| !s.is_empty()))+ .or_else(|| {+ expected["id"]+ .as_str()+ .filter(|id| id.contains("/providers/Microsoft.ContainerService/managedClusters/"))+ .and_then(|_| r.pointer("/properties/resourceUID"))+ }).and_then(Value::as_str).filter(|v| !v.is_empty() && v.len() <= 128).context("Provider immutable incarnation receipt unavailable; refuse cleanup")?;@@ -841,6 +905,7 @@ pub(crate) async fn lifecycle(command: &Lifecycle) -> Result<()> {resources: vec![],heartbeat_unix: None,guard: None,+ image_upgrade: None,destroy_requested: false,completed_unix: None,job_stop: None,@@ -946,6 +1011,26 @@ pub(crate) async fn cleanup(command: &Cleanup) -> Result<()> {json!({"heartbeatReady":true,"independentOfAks":true,"liveQualified":false,"workloadApplyReady":false}),)}+ Cleanup::PrepareUpgrade { remote, image } => {+ let expected = local(&remote.state)?;+ ensure!(+ remote.allow_experimental_cloud && remote.accept == expected.bundle_digest,+ "Explicit original bundle confirmation required"+ );+ let mut c = Azure::new(&expected.plan, None)?;+ let (mut e, mut s) = c+ .read_control(&control_url(&expected.plan))+ .await?+ .context("Shared control missing")?;+ ensure!(+ s.nonce == expected.nonce && s.bundle_digest == expected.bundle_digest,+ "Control identity mismatch"+ );+ prepare_image_upgrade(&mut c, &mut s, &mut e, image, now()?).await?;+ crate::output(+ json!({"status":s.status(),"jobImageUpdateRequired":true,"jobId":job_id(&s.plan),"image":image,"resourceOwnershipUnchanged":true}),+ )+ }Cleanup::Tick {control,tenant,@@ -980,6 +1065,44 @@ pub(crate) async fn cleanup(command: &Cleanup) -> Result<()> {}}}+async fn prepare_image_upgrade(+ c: &mut impl Cloud,+ s: &mut State,+ e: &mut String,+ image: &str,+ at: u64,+) -> Result<()> {+ validate_image_upgrade(s, image)?;+ ensure!(+ at >= s.plan.created_at_unix+ && at < s.plan.expires_at_unix+ && !s.destroy_requested+ && s.completed_unix.is_none()+ && s.job_stop.is_none(),+ "Controller is closing or expired"+ );+ if let Some(u) = &s.image_upgrade {+ ensure!(+ u.image == image,+ "Another controller upgrade is already pinned; reconcile it"+ );+ return Ok(());+ }+ let controller = verify_job(c, s).await?;+ ensure!(+ s.guard.as_ref() == Some(&controller),+ "Initialized controller receipt required"+ );+ s.image_upgrade = Some(ImageUpgrade {+ previous_image: controller_image(s).into(),+ image: image.into(),+ request_id: uuid::Uuid::new_v4().to_string(),+ prepared_at_unix: at,+ controller,+ });+ s.heartbeat_unix = None; // Old image/configuration cannot authorize new work.+ save(c, s, e).await+}fn control_artifacts(s: &State) -> Result<BTreeMap<String, Vec<u8>>> {let p = &s.plan;let n = graph::names(p);@@ -1031,7 +1154,7 @@ fn control_artifacts(s: &State) -> Result<BTreeMap<String, Vec<u8>>> {json!({"type":"Microsoft.Storage/storageAccounts","apiVersion":"2023-05-01","name":control_account,"location":location,"sku":{"name":"Standard_LRS"},"kind":"StorageV2","properties":{"allowSharedKeyAccess":false,"allowBlobPublicAccess":false,"minimumTlsVersion":"TLS1_2","supportsHttpsTrafficOnly":true,"publicNetworkAccess":"Enabled"}}),json!({"type":"Microsoft.Storage/storageAccounts/blobServices/containers","apiVersion":"2023-05-01","name":format!("{control_account}/default/control"),"dependsOn":[account_id],"properties":{"publicAccess":"None"}}),json!({"type":"Microsoft.App/managedEnvironments","apiVersion":"2024-03-01","name":format!("{}-expiry",n.prefix),"location":location,"properties":{"appLogsConfiguration":{"destination":null}}}),- json!({"type":"Microsoft.App/jobs","apiVersion":"2024-03-01","name":format!("{}-expiry",n.prefix),"location":location,"tags":{"ows-deployment":p.deployment_id,"ows-nonce":s.nonce,"ows-bundle":s.bundle_digest},"dependsOn":[identity_id,environment_id],"identity":{"type":"UserAssigned","userAssignedIdentities":{identity_id.clone():{}}},"properties":{"environmentId":environment_id,"configuration":{"triggerType":"Schedule","replicaTimeout":240,"replicaRetryLimit":0,"scheduleTriggerConfig":{"cronExpression":"*/5 * * * *","parallelism":1,"replicaCompletionCount":1}},"template":{"containers":[{"name":"cleanup","image":c.guard_image,"command":["/opt/openworker/ows"],"args":["deploy","azure","cleanup","tick","--control",control_url(p),"--tenant",tenant,"--subscription",subscription,"--client-id",client],"resources":{"cpu":0.25,"memory":"0.5Gi"}}]}}}),+ json!({"type":"Microsoft.App/jobs","apiVersion":"2024-03-01","name":format!("{}-expiry",n.prefix),"location":location,"tags":{"ows-deployment":p.deployment_id,"ows-nonce":s.nonce,"ows-bundle":s.bundle_digest},"dependsOn":[identity_id,environment_id],"identity":{"type":"UserAssigned","userAssignedIdentities":{identity_id.clone():{}}},"properties":{"environmentId":environment_id,"configuration":{"triggerType":"Schedule","replicaTimeout":240,"replicaRetryLimit":0,"scheduleTriggerConfig":{"cronExpression":"*/5 * * * *","parallelism":1,"replicaCompletionCount":1}},"template":{"containers":[{"name":"cleanup","image":controller_image(s),"command":["/opt/openworker/ows"],"args":["deploy","azure","cleanup","tick","--control",control_url(p),"--tenant",tenant,"--subscription",subscription,"--client-id",client],"resources":{"cpu":0.25,"memory":"0.5Gi"}}]}}}),];if c.guard_environment_resource_id.is_some() {control_resources.retain(|r| r["type"] != "Microsoft.App/managedEnvironments");@@ -1112,6 +1235,7 @@ mod tests {resources: vec![],heartbeat_unix: Some(150),guard: Some(fixture_guard()),+ image_upgrade: None,destroy_requested: false,completed_unix: None,job_stop: None,@@ -1417,6 +1541,80 @@ mod tests {assert!(s.completed_unix.is_some());}+ #[tokio::test]+ async fn aks_resource_uid_is_required_when_creation_metadata_is_absent() {+ let (mut c, mut s, mut e) = active().await;+ // The foundation is active; apply the cluster, then model the live API shape.+ apply(&mut c, &mut s, &mut e, Stage::Cluster, 150)+ .await+ .unwrap();+ let id = graph::inventory(&s.plan)+ .into_iter()+ .find(|r| r["stage"] == "cluster")+ .unwrap()["id"]+ .as_str()+ .unwrap()+ .to_owned();+ let v = c.objects.get_mut(&id).unwrap();+ v.as_object_mut().unwrap().remove("systemData");+ v["properties"]["resourceUID"] = json!("6af38caa5deaf50001940f2c");+ // First real observation must pin UID, never switch an existing receipt.+ s.resources.retain(|r| r.id != id);+ observe(&mut c, &mut s, &mut e).await.unwrap();+ assert_eq!(+ s.resources.iter().find(|r| r.id == id).unwrap().incarnation,+ "6af38caa5deaf50001940f2c"+ );+ c.objects.get_mut(&id).unwrap()["properties"]["resourceUID"] = json!("replacement");+ assert!(observe(&mut c, &mut s, &mut e).await.is_err());+ c.objects.get_mut(&id).unwrap()["properties"]["resourceUID"] = Value::Null;+ assert!(observe(&mut c, &mut s, &mut e).await.is_err());+ }+ #[tokio::test]+ async fn controller_upgrade_preserves_ownership_and_refuses_old_image_or_replacement() {+ let (mut c, mut s, mut e) = active().await;+ let original = serde_json::to_value((+ &s.plan,+ &s.resources,+ &s.operations,+ &s.guard,+ &s.nonce,+ &s.bundle_digest,+ ))+ .unwrap();+ let image = format!("ghcr.io/test/ows@sha256:{}", "b".repeat(64));+ prepare_image_upgrade(&mut c, &mut s, &mut e, &image, 160)+ .await+ .unwrap();+ assert_eq!(+ original,+ serde_json::to_value((+ &s.plan,+ &s.resources,+ &s.operations,+ &s.guard,+ &s.nonce,+ &s.bundle_digest+ ))+ .unwrap()+ );+ assert!(s.heartbeat_unix.is_none());+ assert!(verify_job(&mut c, &s).await.is_err());+ prepare_image_upgrade(&mut c, &mut s, &mut e, &image, 161)+ .await+ .unwrap();+ assert!(+ prepare_image_upgrade(&mut c, &mut s, &mut e, "foreign/image:latest", 162)+ .await+ .is_err()+ );+ c.objects.get_mut(&job_id(&s.plan)).unwrap()["properties"]["template"]["containers"][0]+ ["image"] = json!(image);+ verify_job(&mut c, &s).await.unwrap();+ c.objects.get_mut(&job_id(&s.plan)).unwrap()["systemData"]["createdAt"] = json!("replaced");+ assert!(verify_job(&mut c, &s).await.is_err());+ }+#[tokio::test]async fn replacement_or_missing_incarnation_refuses_deletion() {let (mut c, mut s, mut e) = active().await;@@ -1755,7 +1953,7 @@ fn guard_body(s: &State, body: &Value) -> Result<GuardReceipt> {&& body["properties"]["environmentId"].as_str().is_some_and(|id| id.eq_ignore_ascii_case(&environment_id(&s.plan)))- && container["image"] == s.plan.azure_infrastructure.as_ref().unwrap().guard_image+ && container["image"] == controller_image(s)&& container["command"] == json!(["/opt/openworker/ows"])&& args.len() == 12&& args[..4]