Skip to content
Python
11.6 KB
Raw Blame
use crate::{Conflict, Step, Store, Write};
use anyhow::{ensure, Result};
use openworker_telemetry::{Capture, Observation, Record, Sink};
use serde::Serialize;
use std::{
    sync::{
        atomic::{AtomicBool, AtomicU64, Ordering},
        mpsc, Arc, Mutex, OnceLock,
    },
    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
};

#[derive(Default)]
struct Counters {
    accepted: AtomicU64,
    persisted: AtomicU64,
    duplicates: AtomicU64,
    conflicts: AtomicU64,
    dropped: AtomicU64,
    failed_writes: AtomicU64,
    rejected: AtomicU64,
    abandoned: AtomicU64,
    pruned: AtomicU64,
    prune_failures: AtomicU64,
    last_success_ms: AtomicU64,
}
#[derive(Clone, Default)]
pub struct Statistics(Arc<Counters>);
#[derive(Clone, Debug, Default, Serialize)]
pub struct Snapshot {
    pub accepted: u64,
    pub persisted: u64,
    pub duplicates: u64,
    pub conflicts: u64,
    pub dropped: u64,
    pub failed_writes: u64,
    pub rejected: u64,
    pub abandoned: u64,
    pub pending: u64,
    pub pruned: u64,
    pub prune_failures: u64,
    pub last_success_ms: u64,
}
impl Statistics {
    pub fn snapshot(&self) -> Snapshot {
        let load = |v: &AtomicU64| v.load(Ordering::Relaxed);
        let mut s = Snapshot {
            accepted: load(&self.0.accepted),
            persisted: load(&self.0.persisted),
            duplicates: load(&self.0.duplicates),
            conflicts: load(&self.0.conflicts),
            dropped: load(&self.0.dropped),
            failed_writes: load(&self.0.failed_writes),
            rejected: load(&self.0.rejected),
            abandoned: load(&self.0.abandoned),
            pending: 0,
            pruned: load(&self.0.pruned),
            prune_failures: load(&self.0.prune_failures),
            last_success_ms: load(&self.0.last_success_ms),
        };
        s.pending = s.accepted.saturating_sub(
            s.persisted + s.duplicates + s.conflicts + s.failed_writes + s.abandoned,
        );
        s
    }
}
fn now_ms() -> u64 {
    SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .unwrap_or_default()
        .as_millis()
        .min(u64::MAX as u128) as u64
}
struct Shared {
    deployment: String,
    admission: Mutex<()>,
    stop: AtomicBool,
    deadline: OnceLock<Instant>,
    statistics: Statistics,
}
#[derive(Clone)]
pub struct Recorder {
    tx: mpsc::SyncSender<Step>,
    shared: Arc<Shared>,
}
pub struct Controller {
    shared: Arc<Shared>,
    done: mpsc::Receiver<()>,
}
impl Recorder {
    pub fn start(
        mut store: Box<dyn Store>,
        deployment: String,
        capacity: usize,
        retention_days: u16,
    ) -> Result<(Self, Controller)> {
        ensure!(
            crate::identifier(&deployment),
            "Invalid recording deployment"
        );
        ensure!(
            (1..=2048).contains(&capacity),
            "Recording capacity must be 1..=2048"
        );
        ensure!(
            (1..=365).contains(&retention_days),
            "Recording retention must be 1..=365 days"
        );
        let (tx, rx) = mpsc::sync_channel::<Step>(capacity);
        let (done, wait) = mpsc::sync_channel(1);
        let shared = Arc::new(Shared {
            deployment: deployment.clone(),
            admission: Mutex::new(()),
            stop: AtomicBool::new(false),
            deadline: OnceLock::new(),
            statistics: Statistics::default(),
        });
        let thread = shared.clone();
        std::thread::Builder::new()
            .name("ow-recording".into())
            .spawn(move || {
                let counts = &thread.statistics.0;
                let mut prune_at = Instant::now() + Duration::from_secs(60);
                loop {
                    if thread.deadline.get().is_some_and(|d| Instant::now() >= *d) {
                        while rx.try_recv().is_ok() {
                            counts.abandoned.fetch_add(1, Ordering::Relaxed);
                        }
                        break;
                    }
                    let next = if thread.stop.load(Ordering::Acquire) {
                        match rx.try_recv() {
                            Ok(s) => Some(s),
                            Err(_) => break,
                        }
                    } else {
                        match rx.recv_timeout(Duration::from_millis(200)) {
                            Ok(s) => Some(s),
                            Err(mpsc::RecvTimeoutError::Disconnected) => break,
                            Err(_) => None,
                        }
                    };
                    if let Some(step) = next {
                        // Stable insert identity makes explicit observation reconciliation
                        // safe. Never invoke/replay tools or automatically retry SQL here.
                        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
                            store.put(&step)
                        }));
                        match result {
                            Ok(Ok(Write::Inserted)) => {
                                counts.persisted.fetch_add(1, Ordering::Relaxed);
                                counts.last_success_ms.store(now_ms(), Ordering::Relaxed);
                            }
                            Ok(Ok(Write::Duplicate)) => {
                                counts.duplicates.fetch_add(1, Ordering::Relaxed);
                            }
                            Ok(Err(e)) if e.is::<Conflict>() => {
                                counts.conflicts.fetch_add(1, Ordering::Relaxed);
                            }
                            _ => {
                                counts.failed_writes.fetch_add(1, Ordering::Relaxed);
                            }
                        }
                    }
                    if !thread.stop.load(Ordering::Acquire) && Instant::now() >= prune_at {
                        let cutoff = now_ms().saturating_sub(u64::from(retention_days) * 86400000);
                        match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
                            store.prune(&deployment, cutoff, 1000)
                        })) {
                            Ok(Ok(n)) => {
                                counts.pruned.fetch_add(n, Ordering::Relaxed);
                            }
                            _ => {
                                counts.prune_failures.fetch_add(1, Ordering::Relaxed);
                            }
                        }
                        prune_at = Instant::now() + Duration::from_secs(60);
                    }
                }
                let _ = done.send(());
            })?;
        Ok((
            Self {
                tx,
                shared: shared.clone(),
            },
            Controller { shared, done: wait },
        ))
    }
    pub fn statistics(&self) -> Statistics {
        self.shared.statistics.clone()
    }
    pub fn record(&self, step: Step) -> bool {
        if step.deployment != self.shared.deployment || step.validate().is_err() {
            self.shared
                .statistics
                .0
                .rejected
                .fetch_add(1, Ordering::Relaxed);
            return false;
        }
        // Serialize only stop/enqueue, never validation or database work. Producer
        // contention drops visibly instead of blocking an agent thread.
        let Ok(_admission) = self.shared.admission.try_lock() else {
            self.shared
                .statistics
                .0
                .dropped
                .fetch_add(1, Ordering::Relaxed);
            return false;
        };
        if self.shared.stop.load(Ordering::Acquire) {
            self.shared
                .statistics
                .0
                .dropped
                .fetch_add(1, Ordering::Relaxed);
            return false;
        }
        match self.tx.try_send(step) {
            Ok(()) => {
                self.shared
                    .statistics
                    .0
                    .accepted
                    .fetch_add(1, Ordering::Relaxed);
                true
            }
            Err(_) => {
                self.shared
                    .statistics
                    .0
                    .dropped
                    .fetch_add(1, Ordering::Relaxed);
                false
            }
        }
    }
}
impl Controller {
    pub fn statistics(&self) -> Statistics {
        self.shared.statistics.clone()
    }
    pub fn shutdown(&self, budget: Duration) -> bool {
        let budget = budget.min(Duration::from_secs(5));
        {
            let _admission = self
                .shared
                .admission
                .lock()
                .unwrap_or_else(|e| e.into_inner());
            self.shared.deadline.get_or_init(|| Instant::now() + budget);
            self.shared.stop.store(true, Ordering::Release);
        }
        self.done.recv_timeout(budget).is_ok()
    }
}
impl Drop for Controller {
    fn drop(&mut self) {
        let _admission = self
            .shared
            .admission
            .lock()
            .unwrap_or_else(|e| e.into_inner());
        self.shared
            .deadline
            .get_or_init(|| Instant::now() + Duration::from_secs(5));
        self.shared.stop.store(true, Ordering::Release);
    }
}

/// Local recording is queued before vendor export and uses its own writer. Only
/// allowlisted metadata is persisted even when an exporter explicitly captures
/// full content. Raising local capture never changes the export privacy choice.
pub struct RecordingSink {
    recorder: Recorder,
    deployment: String,
    role: String,
    exporter: Option<Box<dyn Sink>>,
    export_capture: Capture,
}
impl RecordingSink {
    pub fn new(
        recorder: Recorder,
        deployment: String,
        role: String,
        exporter: Option<Box<dyn Sink>>,
        export_capture: Capture,
    ) -> Result<Self> {
        ensure!(
            crate::identifier(&deployment) && crate::identifier(&role),
            "Invalid recording scope"
        );
        Ok(Self {
            recorder,
            deployment,
            role,
            exporter,
            export_capture,
        })
    }
}
impl Sink for RecordingSink {
    fn export(&mut self, records: &[Record], stats: openworker_telemetry::Snapshot) -> bool {
        let mut ok = true;
        for r in records {
            match Step::from_record(&self.deployment, &self.role, r) {
                Ok(s) => ok = self.recorder.record(s) && ok,
                Err(_) => {
                    self.recorder
                        .shared
                        .statistics
                        .0
                        .rejected
                        .fetch_add(1, Ordering::Relaxed);
                    ok = false;
                }
            }
        }
        if let Some(exporter) = &mut self.exporter {
            let mut outgoing = records.to_vec();
            for record in &mut outgoing {
                match self.export_capture {
                    Capture::Off => record.attributes.clear(),
                    Capture::Metadata => record.attributes.retain(|(key, value)| {
                        openworker_telemetry::metadata_attribute(key, value)
                    }),
                    Capture::Full => record
                        .attributes
                        .retain(|(key, value)| openworker_telemetry::export_attribute(key, value)),
                }
            }
            ok = exporter.export(&outgoing, stats) && ok;
        }
        ok
    }
    fn observe(&mut self, samples: &[Observation], stats: openworker_telemetry::Snapshot) -> bool {
        self.exporter
            .as_mut()
            .is_none_or(|s| s.observe(samples, stats))
    }
}