Skip to content
Python
9.5 KB
Raw Blame
//! Restartable orchestration of native lifecycle stages. A stage receipt does not
//! replace provider ownership receipts. Every resumed effect is observed first.
use super::*;
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
#[serde(rename_all = "snake_case")]
pub(super) enum Stage {
    Bundle,
    Preflight,
    Permissions,
    Control,
    Foundation,
    IdentityBindings,
    Cluster,
    Images,
    PrivateAccess,
    Platform,
    OpenShell,
    Secrets,
    Service,
    Verify,
}
pub(super) const ORDER: [Stage; 14] = [
    Stage::Bundle,
    Stage::Permissions,
    Stage::Preflight,
    Stage::Control,
    Stage::Foundation,
    Stage::IdentityBindings,
    Stage::Cluster,
    Stage::Images,
    Stage::PrivateAccess,
    Stage::Platform,
    Stage::OpenShell,
    Stage::Secrets,
    Stage::Service,
    Stage::Verify,
];
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "snake_case")]
pub(super) enum Phase {
    Intent,
    Pending,
    Complete,
    Blocked,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub(super) struct Step {
    pub request_id: String,
    pub phase: Phase,
    pub receipt: Option<Value>,
    pub action: Option<String>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub(super) struct Journal {
    pub schema_version: u16,
    pub review_digest: String,
    pub deployment_id: String,
    pub stages: BTreeMap<Stage, Step>,
}
#[derive(Debug)]
pub(super) enum Observation {
    /// Independently proven no request/effect; native receipt contract permits start.
    Absent,
    Pending,
    Complete(Value),
    Blocked(String),
    /// No deletion/recreation/replay is permitted on this observation.
    Unknown(String),
}
pub(super) trait Driver {
    async fn observe(&mut self, stage: Stage, step: Option<&Step>) -> Result<Observation>;
    async fn execute(&mut self, stage: Stage, request_id: &str) -> Result<Observation>;
}
impl Journal {
    pub fn open(path: &Path, review: &Review) -> Result<Self> {
        if !path.exists() {
            return Ok(Self {
                schema_version: 1,
                review_digest: review.digest.clone(),
                deployment_id: review.deployment_id.clone(),
                stages: BTreeMap::new(),
            });
        }
        let j: Self = serde_json::from_slice(&fs::read(path)?)?;
        ensure!(
            j.schema_version == 1
                && j.review_digest == review.digest
                && j.deployment_id == review.deployment_id,
            "Deployment journal identity mismatch"
        );
        for step in j.stages.values() {
            ensure!(
                uuid::Uuid::parse_str(&step.request_id).is_ok_and(|id| !id.is_nil()),
                "Invalid stage request receipt"
            );
        }
        Ok(j)
    }
    pub fn complete(&self) -> bool {
        ORDER.iter().all(|s| {
            self.stages
                .get(s)
                .is_some_and(|v| v.phase == Phase::Complete)
        })
    }
}
fn record(
    j: &mut Journal,
    path: &Path,
    stage: Stage,
    phase: Phase,
    receipt: Option<Value>,
    action: Option<String>,
) -> Result<()> {
    let step = j.stages.entry(stage).or_insert_with(|| Step {
        request_id: uuid::Uuid::new_v4().to_string(),
        phase: Phase::Intent,
        receipt: None,
        action: None,
    });
    step.phase = phase;
    step.receipt = receipt;
    step.action = action;
    write_private(path, j)
}
/// Advances at most one cloud stage. Effects are never retried because a local
/// process failed: the native adapter must first prove no-effect or completion.
pub(super) async fn advance(
    j: &mut Journal,
    path: &Path,
    driver: &mut impl Driver,
) -> Result<bool> {
    let Some(stage) = ORDER
        .iter()
        .find(|s| j.stages.get(s).is_none_or(|v| v.phase != Phase::Complete))
        .copied()
    else {
        return Ok(true);
    };
    let observed = driver.observe(stage, j.stages.get(&stage)).await?;
    let outcome = match observed {
        Observation::Absent => {
            record(j, path, stage, Phase::Intent, None, None)?;
            let id = j.stages[&stage].request_id.clone();
            // On failure the durable intent remains. The next invocation calls
            // observe, never execute directly.
            driver.execute(stage, &id).await?
        }
        other => other,
    };
    match outcome {
        Observation::Complete(receipt) => {
            record(j, path, stage, Phase::Complete, Some(receipt), None)?
        }
        Observation::Pending => record(j, path, stage, Phase::Pending, None, None)?,
        Observation::Blocked(action) => {
            record(j, path, stage, Phase::Blocked, None, Some(action.clone()))?;
            bail!("{action}");
        }
        Observation::Unknown(action) => {
            record(j, path, stage, Phase::Pending, None, Some(action.clone()))?;
            bail!("Outcome unknown; inspect original stage {stage:?}. {action}");
        }
        Observation::Absent => bail!(
            "Stage did not provide a durable outcome after execution; reconcile before continuing"
        ),
    }
    Ok(j.complete())
}
#[cfg(test)]
mod tests {
    use super::*;
    struct Fake {
        effects: usize,
        lost: bool,
        remote: bool,
        unknown: bool,
    }
    impl Driver for Fake {
        async fn observe(&mut self, _: Stage, _: Option<&Step>) -> Result<Observation> {
            Ok(if self.unknown {
                Observation::Unknown("read failed".into())
            } else if self.remote {
                Observation::Complete(json!({"original":true}))
            } else {
                Observation::Absent
            })
        }
        async fn execute(&mut self, _: Stage, _: &str) -> Result<Observation> {
            self.effects += 1;
            self.remote = true;
            if self.lost {
                bail!("lost after effect")
            }
            Ok(Observation::Pending)
        }
    }
    fn journal() -> Journal {
        Journal {
            schema_version: 1,
            review_digest: "digest".into(),
            deployment_id: "original".into(),
            stages: BTreeMap::new(),
        }
    }
    #[tokio::test]
    async fn lost_effect_is_reconciled_after_process_restart_without_duplicate() {
        let dir = tempfile::tempdir().unwrap();
        let p = dir.path().join("journal.json");
        let mut j = journal();
        let mut d = Fake {
            effects: 0,
            lost: true,
            remote: false,
            unknown: false,
        };
        assert!(advance(&mut j, &p, &mut d).await.is_err());
        assert_eq!(j.stages[&Stage::Bundle].phase, Phase::Intent);
        let id = j.stages[&Stage::Bundle].request_id.clone();
        let mut j: Journal = serde_json::from_slice(&fs::read(&p).unwrap()).unwrap();
        advance(&mut j, &p, &mut d).await.unwrap();
        assert_eq!(d.effects, 1);
        assert_eq!(j.stages[&Stage::Bundle].request_id, id);
        assert_eq!(j.stages[&Stage::Bundle].phase, Phase::Complete);
    }
    #[tokio::test]
    async fn unknown_observation_never_executes_and_blocks_following_stages() {
        let dir = tempfile::tempdir().unwrap();
        let p = dir.path().join("journal.json");
        let mut j = journal();
        let mut d = Fake {
            effects: 0,
            lost: false,
            remote: false,
            unknown: true,
        };
        assert!(advance(&mut j, &p, &mut d).await.is_err());
        assert_eq!(d.effects, 0);
        assert_eq!(j.stages.len(), 1);
    }
}

#[cfg(test)]
mod boundary_tests {
    use super::*;
    struct Fault {
        effects: BTreeMap<Stage, usize>,
        fail: Stage,
        lost: bool,
    }
    impl Driver for Fault {
        async fn observe(&mut self, s: Stage, _: Option<&Step>) -> Result<Observation> {
            Ok(if self.effects.contains_key(&s) {
                Observation::Complete(json!({"original":true}))
            } else {
                Observation::Absent
            })
        }
        async fn execute(&mut self, s: Stage, _: &str) -> Result<Observation> {
            *self.effects.entry(s).or_default() += 1;
            if s == self.fail && !self.lost {
                self.lost = true;
                bail!("crash after effect")
            }
            Ok(Observation::Complete(json!({"ready":true})))
        }
    }
    #[tokio::test]
    async fn every_stage_survives_restart_without_repeating_effects() {
        for failed in ORDER {
            let temp = tempfile::tempdir().unwrap();
            let path = temp.path().join("journal.json");
            let mut j = Journal {
                schema_version: 1,
                review_digest: "r".into(),
                deployment_id: "d".into(),
                stages: BTreeMap::new(),
            };
            let mut driver = Fault {
                effects: BTreeMap::new(),
                fail: failed,
                lost: false,
            };
            for _ in 0..ORDER.len() + 2 {
                match advance(&mut j, &path, &mut driver).await {
                    Ok(true) => break,
                    Ok(false) => (),
                    Err(_) => {
                        j = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap();
                    }
                }
            }
            assert!(j.complete(), "{failed:?}");
            assert_eq!(driver.effects.len(), ORDER.len());
            assert!(driver.effects.values().all(|n| *n == 1));
        }
    }
}