Skip to content
Python
3.6 KB
Raw Blame
//! Opt-in observer on its own bounded connection. Never gates agent work.
use openworker_store_postgres::{Config, PostgresStore};
use openworker_telemetry::{Gauge, Telemetry};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tokio_util::sync::CancellationToken;
#[derive(Clone, serde::Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct Options {
    pub sample_interval_seconds: u64,
    pub stalled_after_seconds: u64,
    pub recent_window_seconds: u64,
}
impl Default for Options {
    fn default() -> Self {
        Self {
            sample_interval_seconds: 10,
            stalled_after_seconds: 300,
            recent_window_seconds: 300,
        }
    }
}
impl Options {
    pub fn validate(&self) -> anyhow::Result<()> {
        anyhow::ensure!(
            (1..=60).contains(&self.sample_interval_seconds)
                && (1..=86400).contains(&self.stalled_after_seconds)
                && (1..=86400).contains(&self.recent_window_seconds),
            "Invalid operational observation bounds"
        );
        Ok(())
    }
}
pub async fn run(
    mut config: Config,
    options: Options,
    telemetry: Telemetry,
    stop: CancellationToken,
) {
    // Do not instrument our own sampling query as storage activity/errors.
    config.telemetry = Telemetry::default();
    let mut store = PostgresStore::open(config).ok();
    let mut tick = tokio::time::interval(Duration::from_secs(options.sample_interval_seconds));
    tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
    loop {
        tokio::select! {biased;_=stop.cancelled()=>break,_=tick.tick()=>{}}
        telemetry.observe(
            Gauge::SamplerHeartbeat,
            SystemTime::now()
                .duration_since(UNIX_EPOCH)
                .unwrap_or_default()
                .as_secs(),
        );
        let result = store.as_ref().and_then(|s| {
            s.operational_snapshot(options.stalled_after_seconds, options.recent_window_seconds)
                .ok()
        });
        if let Some(s) = result {
            telemetry.observe_many(&[
                (Gauge::SessionsActive, s.active),
                (Gauge::TurnsQueued, s.queued),
                (Gauge::ApprovalsPending, s.approvals),
                (Gauge::QuestionsPending, s.questions),
                (Gauge::OldestQueuedSeconds, s.oldest_queued_seconds),
                (Gauge::SessionsStalled, s.stalled),
                (Gauge::UnknownRecent, s.unknown_recent),
                (Gauge::StateObservedAt, s.observed_at),
                (Gauge::DatabaseHealthy, 1),
            ]);
        } else {
            telemetry.observe(Gauge::DatabaseHealthy, 0);
            // A poisoned connection is not reopened in a loop. Inspection stays
            // unhealthy/stale until operator restart, without blocking the API.
            store = None;
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    #[test]
    fn defaults_and_bounds_are_explicit() {
        let o: Options = serde_json::from_str("{}").unwrap();
        assert_eq!(o.sample_interval_seconds, 10);
        assert_eq!(o.stalled_after_seconds, 300);
        assert_eq!(o.recent_window_seconds, 300);
        o.validate().unwrap();
        assert!(Options {
            sample_interval_seconds: 0,
            ..o.clone()
        }
        .validate()
        .is_err());
        assert!(Options {
            stalled_after_seconds: 86401,
            ..o.clone()
        }
        .validate()
        .is_err());
        assert!(Options {
            recent_window_seconds: 0,
            ..o
        }
        .validate()
        .is_err());
        assert!(serde_json::from_str::<Options>(r#"{"prompt":"private"}"#).is_err());
    }
}