Python
fix(azure): reconcile proven DNS dependency rejections before bounded retry
Rohit P426ba6fa0399
diff --git a/harness/service-cli/src/deployment_azure_lifecycle.rs b/harness/service-cli/src/deployment_azure_lifecycle.rsindex 92c6e1f8..02a4c644 100644--- a/harness/service-cli/src/deployment_azure_lifecycle.rs+++ b/harness/service-cli/src/deployment_azure_lifecycle.rs@@ -116,6 +116,19 @@ struct Owned {delete_request: Option<String>,delete_accepted: bool,operation_url: Option<String>,+ #[serde(default, skip_serializing_if = "Vec::is_empty")]+ rejected_dns_deletes: Vec<RejectedDnsDelete>,+}+// A bounded, provider-confirmed no-effect response is different from an unknown+// mutation. Keep every original request; old ambiguous receipts stay parked.+#[derive(Clone, Debug, Serialize, Deserialize)]+#[serde(rename_all = "camelCase", deny_unknown_fields)]+struct RejectedDnsDelete {+ request_id: String,+ observed_at_unix: u64,+ retry_after_unix: u64,+ status: u16,+ code: String,}#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]#[serde(rename_all = "camelCase", deny_unknown_fields)]@@ -345,6 +358,23 @@ impl State {"Invalid deletion request UUID");}+ let mut rejected = std::collections::BTreeSet::new();+ ensure!(+ r.rejected_dns_deletes.len() <= 5,+ "DNS dependency retry limit exceeded"+ );+ for rejection in &r.rejected_dns_deletes {+ ensure!(+ r.id.contains("/providers/Microsoft.Network/privateDnsZones/")+ && rejection.status == 409+ && rejection.code == "CannotDeleteResource"+ && uuid::Uuid::parse_str(&rejection.request_id).is_ok_and(|u| !u.is_nil())+ && rejected.insert(&rejection.request_id)+ && rejection.retry_after_unix+ == rejection.observed_at_unix.saturating_add(60),+ "Invalid DNS no-effect receipt"+ );+ }if r.deleted {ensure!(r.delete_request.is_some(),@@ -581,6 +611,7 @@ async fn observe(c: &mut impl Cloud, s: &mut State, e: &mut String) -> Result<()delete_request: None,delete_accepted: false,operation_url: None,+ rejected_dns_deletes: vec![],});save(c, s, e).await?;}@@ -703,9 +734,22 @@ async fn destroy_one(c: &mut impl Cloud, s: &mut State, e: &mut String, at: u64)else {continue;};- if s.resources[i].delete_request.is_some() {- return Ok(());- } // GET only on subsequent ticks; even a lost DELETE is not replay permission.+ if let Some(request) = &s.resources[i].delete_request {+ let Some(rejected) = s.resources[i]+ .rejected_dns_deletes+ .last()+ .filter(|r| &r.request_id == request)+ else {+ // GET only after an accepted or unknown mutation, including old+ // receipts. A lost response must never authorize another DELETE.+ return Ok(());+ };+ ensure!(s.resources[i].rejected_dns_deletes.len() < 5,+ "DNS dependencies repeatedly rejected deletion; inspect original resource and receipts");+ if at < rejected.retry_after_unix {+ return Ok(());+ }+ }let id = s.resources[i].id.clone();let api = expected["apiVersion"].as_str().unwrap();let r = c.arm(Method::GET, &id, api, None, &s.nonce).await?;@@ -749,9 +793,32 @@ async fn destroy_one(c: &mut impl Cloud, s: &mut State, e: &mut String, at: u64)}let request = uuid::Uuid::new_v4().to_string();s.resources[i].delete_request = Some(request.clone());+ s.resources[i].delete_accepted = false;+ s.resources[i].operation_url = None;save(c, s, e).await?;let r = c.arm(Method::DELETE, &id, api, None, &request).await;- match r {Ok(r) if [200,202,204].contains(&r.status)=>{s.resources[i].delete_accepted=true;s.resources[i].operation_url=r.operation;save(c,s,e).await?;},_=>bail!("Delete outcome unknown/rejected. Observe this exact resource; no automatic replay.")}+ match r {+ Ok(r) if [200, 202, 204].contains(&r.status) => {+ s.resources[i].delete_accepted = true;+ s.resources[i].operation_url = r.operation;+ save(c, s, e).await?;+ }+ Ok(r) if id.contains("/providers/Microsoft.Network/privateDnsZones/")+ && r.status == 409+ && r.body.as_ref().and_then(|v| v.pointer("/error/code")).and_then(Value::as_str)+ == Some("CannotDeleteResource") => {+ // Azure's empty link listing can precede dependency deletion.+ // Only this exact rejected response permits a future NEW attempt,+ // after backoff and fresh ownership/dependency checks above.+ s.resources[i].rejected_dns_deletes.push(RejectedDnsDelete {+ request_id: request, observed_at_unix: at,+ retry_after_unix: at.saturating_add(60), status: 409,+ code: "CannotDeleteResource".into(),+ });+ save(c, s, e).await?;+ }+ _ => bail!("Delete outcome unknown/rejected. Observe this exact resource; no automatic replay."),+ }return Ok(());}s.completed_unix = Some(at);@@ -1261,6 +1328,8 @@ mod tests {lose_put: bool,lose_delete: bool,conflict: bool,+ dns_rejections: usize,+ dns_error_code: String,}impl Fake {fn new(s: State) -> Self {@@ -1274,6 +1343,8 @@ mod tests {lose_put: false,lose_delete: false,conflict: false,+ dns_rejections: 0,+ dns_error_code: "CannotDeleteResource".into(),}}fn etag(&self) -> String {@@ -1374,6 +1445,13 @@ mod tests {"Delete intent must be durable");}+ if id.contains("/privateDnsZones/") && self.dns_rejections > 0 {+ self.dns_rejections -= 1;+ return Ok(reply(+ 409,+ Some(json!({"error":{"code":self.dns_error_code}})),+ ));+ }self.objects.remove(id);if self.lose_delete {self.lose_delete = false;@@ -1541,6 +1619,109 @@ mod tests {assert!(s.completed_unix.is_some());}+ async fn blocked_dns() -> (Fake, State, String, String) {+ let (mut c, mut s, mut e) = active().await;+ c.dns_rejections = 1;+ for _ in 0..15 {+ destroy_one(&mut c, &mut s, &mut e, 201).await.unwrap();+ if s.resources+ .iter()+ .any(|r| !r.rejected_dns_deletes.is_empty())+ {+ break;+ }+ }+ let zone = s+ .resources+ .iter()+ .find(|r| !r.rejected_dns_deletes.is_empty())+ .unwrap()+ .id+ .clone();+ (c, s, e, zone)+ }+ #[tokio::test]+ async fn dns_proven_dependency_rejection_survives_restart_and_rechecks_before_new_request() {+ let (mut c, s, _, zone) = blocked_dns().await;+ let old = s+ .resources+ .iter()+ .find(|r| r.id == zone)+ .unwrap()+ .delete_request+ .clone();+ let (mut e, mut s) = c.read_control("").await.unwrap().unwrap();+ let before = c.deletes();+ destroy_one(&mut c, &mut s, &mut e, 260).await.unwrap();+ assert_eq!(c.deletes(), before, "backoff must survive restart");+ c.objects.insert(+ format!("{zone}/virtualNetworkLinks"),+ json!({"value":[{"id":"still-there"}]}),+ );+ destroy_one(&mut c, &mut s, &mut e, 261).await.unwrap();+ assert_eq!(c.deletes(), before);+ c.objects.remove(&format!("{zone}/virtualNetworkLinks"));+ destroy_one(&mut c, &mut s, &mut e, 262).await.unwrap();+ let resource = s.resources.iter().find(|r| r.id == zone).unwrap();+ assert_ne!(resource.delete_request, old);+ assert_eq!(resource.rejected_dns_deletes[0].request_id, old.unwrap());+ assert!(resource.delete_accepted);+ assert_eq!(c.deletes(), before + 1);+ }+ #[tokio::test]+ async fn dns_retry_refuses_replacement_unknown_response_and_other_conflicts() {+ let (mut c, mut s, mut e, zone) = blocked_dns().await;+ c.objects.get_mut(&zone).unwrap()["properties"]["internalId"] = json!("replacement");+ let count = c.deletes();+ assert!(destroy_one(&mut c, &mut s, &mut e, 300).await.is_err());+ assert_eq!(c.deletes(), count);+ let (mut c, mut s, mut e, zone) = blocked_dns().await;+ // Model a lost/legacy response: no proven-rejection receipt exists.+ s.resources+ .iter_mut()+ .find(|r| r.id == zone)+ .unwrap()+ .rejected_dns_deletes+ .clear();+ save(&mut c, &s, &mut e).await.unwrap();+ let count = c.deletes();+ destroy_one(&mut c, &mut s, &mut e, 400).await.unwrap();+ assert_eq!(c.deletes(), count);+ let (mut c, mut s, mut e) = active().await;+ c.dns_rejections = 1;+ c.dns_error_code = "AnotherConflict".into();+ for _ in 0..15 {+ if destroy_one(&mut c, &mut s, &mut e, 201).await.is_err() {+ break;+ }+ }+ assert!(s+ .resources+ .iter()+ .all(|r| r.rejected_dns_deletes.is_empty()));+ let count = c.deletes();+ destroy_one(&mut c, &mut s, &mut e, 400).await.unwrap();+ assert_eq!(c.deletes(), count);+ }+ #[tokio::test]+ async fn dns_dependency_attempts_are_bounded_and_keep_all_receipts() {+ let (mut c, mut s, mut e, _) = blocked_dns().await;+ c.dns_rejections = 20;+ for at in [261, 321, 381, 441] {+ destroy_one(&mut c, &mut s, &mut e, at).await.unwrap();+ }+ let count = c.deletes();+ assert!(destroy_one(&mut c, &mut s, &mut e, 501).await.is_err());+ assert_eq!(c.deletes(), count);+ assert_eq!(+ s.resources+ .iter()+ .map(|r| r.rejected_dns_deletes.len())+ .sum::<usize>(),+ 5+ );+ }+#[tokio::test]async fn aks_resource_uid_is_required_when_creation_metadata_is_absent() {let (mut c, mut s, mut e) = active().await;