Python
//! Isolated protocol/transport fixtures. Never use cloud credentials or endpoints.
use super::{
control::*,
engine::{self, Cloud, Mode},
*,
};
use std::collections::BTreeMap;
pub(super) fn control() -> Control {
let mut p:Plan=serde_json::from_value(json!({"schemaVersion":2,"name":"gcp-test","deploymentId":"ows-0123456789abcdef0123456789abcdef","target":{"provider":"gcp","project":"schema-only-project","location":"us-central1"},"createdAtUnix":1793373000u64,"expiresAtUnix":1793376600u64,"budget":{"authorizedUsd":100,"hardCap":false,"estimate":null},"topology":{"clusterMode":"new","nodeCount":2,"maxWorkerReplicas":2,"architecture":"amd64","access":"private","nodeAutoscaling":false}})).unwrap();
p.gcp_bootstrap = Some(Identity {
project_number: "123456789012".into(),
operator_member: "user:[email protected]".into(),
});
p.gcp_infrastructure=Some(serde_json::from_value(json!({"zone":"us-central1-a","kubernetesVersion":"1.33.5-gke.1162000","terraformVersion":"1.5.7","googleProviderVersion":"6.44.0","nodeMachineType":"e2-standard-2","nodeDiskGb":40,"postgresVersion":"POSTGRES_16","postgresTier":"db-f1-micro","postgresDiskGb":10,"subnetCidr":"10.70.0.0/16","podCidr":"10.72.0.0/14","serviceCidr":"10.76.0.0/20","controlPlaneCidr":"172.16.0.0/28","sqlCidr":"10.80.0.0/16"})).unwrap());
p.validate().unwrap();
let hashes: BTreeMap<_, _> = super::super::artifacts(&p)
.unwrap()
.iter()
.map(|(n, b)| (n.clone(), hash(b)))
.collect();
let c=Control{schema_version:1,source_sha256:hash(&source_zip(&p).unwrap()),plan:p,bundle_digest:hash(&serde_json::to_vec(&hashes).unwrap()),source_generation:"123".into(),request_id:"11111111-1111-4111-8111-111111111111".into(),guard_image:format!("us-central1-docker.pkg.dev/schema-only-project/ow-0123456789ab-control-images/ows@sha256:{}","a".repeat(64))};
c.validate().unwrap();
c
}
fn timestamp(at: u64) -> String {
chrono::DateTime::<chrono::Utc>::from_timestamp(at as i64, 0)
.unwrap()
.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)
}
fn operation(c: &Control, verb: &str, at: u64) -> Value {
json!({"name":format!("{}/operations/{verb}-original",c.parent()),"metadata":{"target":c.deployment(),"verb":verb,"createTime":timestamp(at)},"done":false})
}
struct Fake {
c: Control,
j: Checkpoint,
g: u64,
at: u64,
d: Option<Value>,
ops: BTreeMap<String, Value>,
creates: u32,
deletes: u32,
saves: u32,
save_fail: Option<(u32, bool)>,
lost_create: bool,
lost_delete: bool,
source_ok: bool,
uid: String,
absent: bool,
revision_drift: bool,
allowed: bool,
paused: bool,
lost_pause: bool,
dependent_gone: bool,
}
impl Fake {
fn new() -> Self {
let c = control();
let at = c.plan.created_at_unix + 1;
let mut j = Checkpoint::new(&c).unwrap();
let uid = uuid::Uuid::new_v4().to_string();
j.guard_uid = Some(uid.clone());
j.heartbeat_at = Some(at);
Self {
c,
j,
g: 1,
at,
d: None,
ops: BTreeMap::new(),
creates: 0,
deletes: 0,
saves: 0,
save_fail: None,
lost_create: false,
lost_delete: false,
source_ok: true,
uid,
absent: false,
revision_drift: false,
allowed: true,
paused: false,
lost_pause: false,
dependent_gone: true,
}
}
async fn step(&mut self, mode: Mode) -> Result<Value> {
let c = self.c.clone();
let mut j = self.j.clone();
let mut g = self.g.to_string();
engine::drive(self, &c, &mut j, &mut g, mode, self.at).await
}
fn complete_create(&mut self, error: bool) {
self.ops.get_mut("create").unwrap()["done"] = json!(true);
if error {
self.ops.get_mut("create").unwrap()["error"] =
json!({"code":7,"message":"DO_NOT_EXPORT_PROVIDER_ERROR"});
}
self.d.as_mut().unwrap()["state"] = json!(if error { "FAILED" } else { "ACTIVE" });
}
async fn active(&mut self) {
self.step(Mode::Apply).await.unwrap();
self.complete_create(false);
self.step(Mode::Reconcile).await.unwrap();
assert_eq!(self.j.phase, Phase::Active);
}
fn complete_delete(&mut self, error: bool) {
self.ops.get_mut("delete").unwrap()["done"] = json!(true);
if error {
self.ops.get_mut("delete").unwrap()["error"] = json!({"code":7});
} else {
self.d = None;
}
}
}
impl Cloud for Fake {
async fn deployment(&mut self) -> Result<Option<Value>> {
Ok(self.d.clone())
}
async fn operation(&mut self, name: &str) -> Result<Value> {
self.ops
.values()
.find(|v| v["name"] == name)
.cloned()
.context("missing original operation")
}
async fn revision(&mut self, name: &str) -> Result<Value> {
Ok(
json!({"name":name,"serviceAccount":self.c.executor(),"terraformBlueprint":if self.revision_drift{json!({"gcsSource":"gs://other/source"})}else{self.c.request()["terraformBlueprint"].clone()}}),
)
}
async fn operations(&mut self) -> Result<Vec<Value>> {
Ok(self.ops.values().cloned().collect())
}
async fn source(&mut self) -> Result<()> {
ensure!(self.source_ok, "source unavailable");
Ok(())
}
async fn guard(&mut self) -> Result<String> {
Ok(self.uid.clone())
}
async fn authorize_create(&mut self) -> Result<()> {
ensure!(self.allowed, "not qualified");
Ok(())
}
async fn create(&mut self) -> Result<Value> {
assert_eq!(self.j.phase, Phase::CreateIntent);
assert!(self.j.create_intent_at.is_some());
self.creates += 1;
let op = operation(&self.c, "create", self.at);
self.ops.insert("create".into(), op.clone());
let mut d = self.c.request();
d["name"] = json!(self.c.deployment());
d["createTime"] = json!(timestamp(self.at));
d["latestRevision"] = json!(format!("{}/revisions/original", self.c.deployment()));
d["state"] = json!("CREATING");
self.d = Some(d);
ensure!(!self.lost_create, "create reply lost");
Ok(op)
}
async fn delete(&mut self, request: &str) -> Result<Value> {
assert_eq!(self.j.phase, Phase::DeleteIntent);
assert_eq!(request, self.j.delete_request_id);
self.deletes += 1;
let op = operation(&self.c, "delete", self.at);
self.ops.insert("delete".into(), op.clone());
self.d.as_mut().unwrap()["state"] = json!("DELETING");
ensure!(!self.lost_delete, "delete reply lost");
Ok(op)
}
async fn save(&mut self, g: &str, j: &Checkpoint) -> Result<String> {
self.saves += 1;
ensure!(g == self.g.to_string(), "generation conflict");
j.validate(&self.c)?;
let fail = self.save_fail.filter(|(n, _)| *n == self.saves);
if fail.is_none_or(|(_, after)| after) {
self.g += 1;
self.j = j.clone();
}
ensure!(fail.is_none(), "CAS reply lost");
Ok(self.g.to_string())
}
async fn dependent_assets(&mut self) -> Result<Vec<Asset>> {
Ok(vec![Asset {
path: format!(
"/compute/v1/projects/{}/zones/us-central1-a/disks/owned",
project(&self.c.plan)
),
id: "1234".into(),
}])
}
async fn assets_absent(&mut self, _: &[Asset]) -> Result<bool> {
Ok(self.dependent_gone)
}
async fn scheduler_paused(&mut self) -> Result<bool> {
Ok(self.paused)
}
async fn pause_scheduler(&mut self) -> Result<()> {
self.paused = true;
ensure!(!self.lost_pause, "pause reply lost");
Ok(())
}
async fn named_resources_absent(&mut self) -> Result<bool> {
Ok(self.absent)
}
}
#[tokio::test]
async fn full_lifecycle_persists_original_receipts_and_does_not_claim_unproved_cleanup() {
let mut f = Fake::new();
f.active().await;
assert_eq!(f.creates, 1);
f.step(Mode::Destroy).await.unwrap();
assert_eq!(f.deletes, 1);
f.complete_delete(false);
let r = f.step(Mode::Reconcile).await.unwrap();
assert_eq!(r["providerDeletionConfirmed"], true);
f.step(Mode::Reconcile).await.unwrap();
assert_eq!(f.j.phase, Phase::ProviderDeleted);
f.absent = true;
let r = f.step(Mode::Reconcile).await.unwrap();
assert_eq!(f.j.phase, Phase::NamedResourcesAbsent);
assert_eq!(r["allCleanupComplete"], false);
assert_eq!(r["managedDependentAbsenceQualified"], false);
f.step(Mode::Destroy).await.unwrap();
assert_eq!((f.creates, f.deletes), (1, 1));
}
#[tokio::test]
async fn lost_create_reply_is_read_reconciled_after_restart_never_replayed() {
let mut f = Fake::new();
f.lost_create = true;
assert!(f.step(Mode::Apply).await.is_err());
assert_eq!(f.j.phase, Phase::CreateIntent);
f.complete_create(false);
f.step(Mode::Apply).await.unwrap();
assert_eq!(f.j.phase, Phase::Active);
assert_eq!(f.creates, 1);
// No operation/absence proof cannot be turned into permission to recreate.
let mut f = Fake::new();
f.lost_create = true;
assert!(f.step(Mode::Apply).await.is_err());
f.ops.clear();
f.d = None;
assert!(f.step(Mode::Apply).await.is_err());
assert_eq!(f.creates, 1);
}
#[tokio::test]
async fn lost_delete_reply_recovers_original_operation_without_another_delete() {
let mut f = Fake::new();
f.active().await;
f.lost_delete = true;
assert!(f.step(Mode::Destroy).await.is_err());
assert_eq!(f.j.phase, Phase::DeleteIntent);
f.complete_delete(false);
f.step(Mode::Destroy).await.unwrap();
assert_eq!(f.deletes, 1);
let mut f = Fake::new();
f.active().await;
f.lost_delete = true;
assert!(f.step(Mode::Destroy).await.is_err());
f.ops.remove("delete");
f.d = None;
assert!(f.step(Mode::Destroy).await.is_err());
assert_eq!(f.deletes, 1);
}
#[tokio::test]
async fn checkpoint_failure_before_or_after_write_never_replays_an_unknown_effect() {
for after in [false, true] {
let mut f = Fake::new();
f.save_fail = Some((1, after));
assert!(f.step(Mode::Apply).await.is_err());
assert_eq!(f.creates, 0);
f.save_fail = None;
if after {
assert!(f.step(Mode::Apply).await.is_err());
assert_eq!(f.creates, 0);
} else {
f.step(Mode::Apply).await.unwrap();
assert_eq!(f.creates, 1);
}
}
let mut f = Fake::new();
f.save_fail = Some((2, false));
assert!(f.step(Mode::Apply).await.is_err());
assert_eq!(f.creates, 1);
f.save_fail = None;
f.complete_create(false);
f.step(Mode::Reconcile).await.unwrap();
assert_eq!(f.creates, 1);
}
#[tokio::test]
async fn independent_tick_enforces_expiry_and_runs_without_local_journal_or_cluster() {
let mut f = Fake::new();
f.active().await;
f.step(Mode::Tick).await.unwrap();
assert_eq!(f.deletes, 0);
f.at = f.c.plan.expires_at_unix + 1;
f.step(Mode::Tick).await.unwrap();
assert_eq!(f.deletes, 1);
assert_eq!(f.j.heartbeat_at, Some(f.at));
f.complete_delete(false);
f.step(Mode::Tick).await.unwrap();
let mut f = Fake::new();
f.at = f.c.plan.expires_at_unix;
f.step(Mode::Tick).await.unwrap();
assert_eq!(f.j.phase, Phase::Unused);
assert_eq!((f.creates, f.deletes), (0, 0));
}
#[tokio::test]
async fn concurrent_checkpoint_writers_are_fenced_before_provider_mutation() {
let mut f = Fake::new();
let mut stale = f.j.clone();
let mut g = f.g.to_string();
let c = f.c.clone();
f.step(Mode::Apply).await.unwrap();
assert!(engine::drive(
&mut f,
&c,
&mut stale,
&mut g,
Mode::Apply,
c.plan.created_at_unix + 1
)
.await
.is_err());
assert_eq!(f.creates, 1);
}
#[tokio::test]
async fn ownership_revision_source_and_guard_drift_block_destructive_actions() {
let mut missing = Fake::new();
missing.active().await;
missing.ops.clear();
assert!(missing.step(Mode::Destroy).await.is_err());
assert_eq!(missing.deletes, 0);
let mut drift = Fake::new();
drift.active().await;
drift.revision_drift = true;
assert!(drift.step(Mode::Destroy).await.is_err());
assert_eq!(drift.deletes, 0);
for scenario in ["time", "revision", "source", "executor", "request"] {
let mut f = Fake::new();
f.active().await;
let d = f.d.as_mut().unwrap();
match scenario {
"time" => d["createTime"] = json!(timestamp(f.at + 1)),
"revision" => {
d["latestRevision"] = json!(format!("{}/revisions/changed", f.c.deployment()))
}
"source" => d["terraformBlueprint"]["gcsSource"] = json!("gs://other/source"),
"executor" => d["serviceAccount"] = json!("foreign"),
_ => d["annotations"]["openworker.dev/request"] = json!("foreign"),
};
assert!(f.step(Mode::Destroy).await.is_err(), "{scenario}");
assert_eq!(f.deletes, 0);
}
let mut f = Fake::new();
f.source_ok = false;
assert!(f.step(Mode::Apply).await.is_err());
assert_eq!(f.creates, 0);
let mut f = Fake::new();
f.uid = uuid::Uuid::new_v4().to_string();
assert!(f.step(Mode::Tick).await.is_err());
assert_eq!(f.saves, 0);
let mut f = Fake::new();
f.j.heartbeat_at = Some(f.at - 181);
assert!(f.step(Mode::Apply).await.is_err());
assert_eq!(f.creates, 0);
}
#[tokio::test]
async fn operation_ambiguity_failure_and_permission_gates_keep_receipts() {
let mut f = Fake::new();
f.lost_create = true;
assert!(f.step(Mode::Apply).await.is_err());
let mut duplicate = f.ops["create"].clone();
duplicate["name"] = json!(format!("{}/operations/another", f.c.parent()));
f.ops.insert("duplicate".into(), duplicate);
assert!(f.step(Mode::Reconcile).await.is_err());
assert_eq!(f.creates, 1);
let mut f = Fake::new();
f.allowed = false;
assert!(f.step(Mode::Apply).await.is_err());
assert_eq!(f.creates, 0);
assert_eq!(f.j.phase, Phase::Prepared);
let mut f = Fake::new();
f.step(Mode::Apply).await.unwrap();
f.complete_create(true);
f.step(Mode::Reconcile).await.unwrap();
assert_eq!(f.j.phase, Phase::CreateFailed);
f.step(Mode::Destroy).await.unwrap();
f.complete_delete(true);
f.step(Mode::Reconcile).await.unwrap();
assert_eq!(f.j.phase, Phase::DeleteFailed);
assert!(f.step(Mode::Destroy).await.is_err());
assert_eq!(f.deletes, 1);
}
#[tokio::test]
async fn missing_or_changed_provider_receipts_never_mean_successful_deletion() {
let mut f = Fake::new();
f.active().await;
f.step(Mode::Destroy).await.unwrap();
f.d = None;
assert!(f.step(Mode::Reconcile).await.is_err());
assert_eq!(f.j.phase, Phase::Deleting);
f.ops.get_mut("delete").unwrap()["metadata"]["target"] =
json!("projects/other/locations/r/deployments/foreign");
assert!(f.step(Mode::Reconcile).await.is_err());
assert_eq!(f.deletes, 1);
}
#[test]
fn source_archive_has_fixed_names_and_control_identity_is_content_bound() {
let c = control();
let a = source_zip(&c.plan).unwrap();
assert_eq!(a, source_zip(&c.plan).unwrap());
assert_eq!(&a[..4], b"PK\x03\x04");
assert_eq!(&a[a.len() - 22..a.len() - 18], b"PK\x05\x06");
for scenario in ["generation", "source", "image", "digest", "project"] {
let mut bad = c.clone();
match scenario {
"generation" => bad.source_generation = "0".into(),
"source" => bad.source_sha256 = "b".repeat(64),
"image" => bad.guard_image = "ghcr.io/other:latest".into(),
"digest" => bad.bundle_digest = "b".repeat(64),
_ => {
if let Target::Gcp { project, .. } = &mut bad.plan.target {
*project = "another-project".into()
}
}
};
assert!(bad.validate().is_err(), "{scenario}");
}
}
#[tokio::test]
async fn captured_dependents_and_scheduler_pause_must_be_confirmed_after_expiry() {
let mut f = Fake::new();
f.active().await;
f.step(Mode::Destroy).await.unwrap();
assert_eq!(f.j.assets.as_ref().unwrap().len(), 1);
f.complete_delete(false);
f.step(Mode::Reconcile).await.unwrap();
f.absent = true;
f.dependent_gone = false;
f.step(Mode::Reconcile).await.unwrap();
assert_eq!(f.j.phase, Phase::ProviderDeleted);
assert!(!f.paused);
f.dependent_gone = true;
f.step(Mode::Reconcile).await.unwrap();
assert_eq!(f.j.phase, Phase::NamedResourcesAbsent);
f.lost_pause = true;
assert!(f.step(Mode::Reconcile).await.is_err());
assert_eq!(f.j.phase, Phase::PauseIntent);
assert!(f.paused);
let result = f.step(Mode::Reconcile).await.unwrap();
assert_eq!(f.j.phase, Phase::Complete);
assert_eq!(result["schedulerPaused"], true);
assert_eq!(f.deletes, 1);
let mut f = Fake::new();
f.at = f.c.plan.expires_at_unix;
f.step(Mode::Tick).await.unwrap();
f.step(Mode::Tick).await.unwrap();
assert_eq!(f.j.phase, Phase::UnusedComplete);
assert!(f.paused);
assert_eq!(f.deletes, 0);
}