Skip to content
Python
15.0 KB
Raw Blame
//! Metadata-only diagnostic recordings. Separate from execution state and audit.
//! Committed rows survive restart; the bounded input queue is not a durable outbox.
use anyhow::{ensure, Result};
use openworker_telemetry::Record;
use serde::{Deserialize, Serialize};
use std::time::UNIX_EPOCH;
#[cfg(feature = "evaluations")]
pub mod evaluations;
#[cfg(feature = "postgres")]
pub mod postgres;
pub mod query;
pub mod sqlite;
pub mod summary;
mod writer;
pub use writer::{Controller, Recorder, RecordingSink, Snapshot, Statistics};

pub const VERSION: u8 = 2;
pub const MAX_RECORD_BYTES: usize = 8192;

#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Usage {
    pub input: Option<u64>,
    pub output: Option<u64>,
    pub cache_read: Option<u64>,
    pub cache_write: Option<u64>,
    pub reasoning: Option<u64>,
}

/// Versioned recording with pseudonymous session/turn/action correlation.
/// A logical turn can contain multiple linked active segments across workers.
/// Arbitrary attributes/content deliberately have no field in this contract.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Step {
    pub version: u8,
    pub deployment: String,
    pub role: String,
    pub trace_id: String,
    pub span_id: String,
    pub parent_trace_id: Option<String>,
    pub parent_span_id: Option<String>,
    pub session_hash: Option<String>,
    pub action_hash: Option<String>,
    #[serde(default)]
    pub turn_hash: Option<String>,
    #[serde(default)]
    pub links: Vec<StepLink>,
    pub operation: String,
    pub outcome: String,
    pub started_ms: u64,
    pub ended_ms: u64,
    pub duration_us: Option<u64>,
    pub first_output_us: Option<u64>,
    pub attempt: Option<u32>,
    pub model: Option<String>,
    pub tool: Option<String>,
    pub purpose: Option<String>,
    pub usage: Option<Usage>,
    #[serde(default)]
    pub output_bytes: Option<u64>,
    #[serde(default)]
    pub sandbox: Option<String>,
    #[serde(default)]
    pub enforcement: Option<String>,
    #[serde(default)]
    pub policy_digest: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct StepLink {
    pub trace_id: String,
    pub span_id: String,
}
fn hex(bytes: &[u8]) -> String {
    bytes.iter().map(|b| format!("{b:02x}")).collect()
}
fn hex_id(value: &str, size: usize) -> bool {
    value.len() == size
        && value
            .bytes()
            .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase())
        && value.bytes().any(|b| b != b'0')
}
pub(crate) fn identifier(value: &str) -> bool {
    !value.is_empty()
        && value.len() <= 64
        && value
            .bytes()
            .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-'))
}
fn label(value: &str) -> bool {
    !value.is_empty()
        && value.len() <= 128
        && !value.starts_with('/')
        && !value.contains("://")
        && value
            .bytes()
            .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'.' | b'/' | b':'))
        && openworker_telemetry::redact(value) == value
}
impl Step {
    pub fn from_record(deployment: &str, role: &str, r: &Record) -> Result<Self> {
        let attribute = |name| {
            r.attributes
                .iter()
                .find(|(key, _)| *key == name)
                .map(|(_, value)| value)
                .filter(|v| label(v))
                .cloned()
        };
        let usage =
            (r.input_tokens.is_some() || r.output_tokens.is_some() || r.usage_details.present())
                .then_some(Usage {
                    input: r.input_tokens,
                    output: r.output_tokens,
                    cache_read: r.usage_details.cache_read,
                    cache_write: r.usage_details.cache_write,
                    reasoning: r.usage_details.reasoning,
                });
        let s = Self {
            version: VERSION,
            deployment: deployment.into(),
            role: role.into(),
            trace_id: hex(&r.context.trace),
            span_id: hex(&r.context.span),
            parent_trace_id: r.parent.map(|p| hex(&p.trace)),
            parent_span_id: r.parent.map(|p| hex(&p.span)),
            session_hash: r.identity.session.map(|s| hex(&s)),
            action_hash: r.identity.action.map(|s| hex(&s)),
            turn_hash: r.turn_hash.map(|s| hex(&s)),
            links: r
                .links
                .iter()
                .map(|c| StepLink {
                    trace_id: hex(&c.trace),
                    span_id: hex(&c.span),
                })
                .collect(),
            operation: r.operation.name().into(),
            outcome: r.outcome.name().into(),
            started_ms: u64::try_from(r.started.duration_since(UNIX_EPOCH)?.as_millis())?,
            ended_ms: u64::try_from(r.ended.duration_since(UNIX_EPOCH)?.as_millis())?,
            duration_us: r
                .timing_known
                .then(|| u64::try_from(r.elapsed.as_micros()))
                .transpose()?,
            first_output_us: r
                .first_output
                .map(|d| u64::try_from(d.as_micros()))
                .transpose()?,
            attempt: r.attempt,
            model: attribute("gen_ai.request.model"),
            tool: attribute("gen_ai.tool.name"),
            purpose: r.purpose.map(|p| p.name().to_owned()),
            usage,
            output_bytes: r.output_bytes,
            sandbox: attribute("openworker.sandbox.provider"),
            enforcement: attribute("openworker.sandbox.enforcement"),
            policy_digest: attribute("openworker.policy.digest").filter(|s| hex_id(s, 64)),
        };
        s.validate()?;
        Ok(s)
    }
    pub fn validate(&self) -> Result<()> {
        ensure!(
            (1..=VERSION).contains(&self.version),
            "Unsupported observation version"
        );
        ensure!(
            identifier(&self.deployment) && identifier(&self.role),
            "Invalid observation scope"
        );
        ensure!(
            hex_id(&self.trace_id, 32) && hex_id(&self.span_id, 16),
            "Invalid observation identity"
        );
        ensure!(
            self.parent_trace_id.is_some() == self.parent_span_id.is_some(),
            "Incomplete parent context"
        );
        ensure!(
            self.parent_trace_id.as_ref().is_none_or(|v| hex_id(v, 32))
                && self.parent_span_id.as_ref().is_none_or(|v| hex_id(v, 16)),
            "Invalid observation parent"
        );
        ensure!(
            self.parent_trace_id.as_ref() != Some(&self.trace_id)
                || self.parent_span_id.as_ref() != Some(&self.span_id),
            "Observation cannot parent itself"
        );
        ensure!(
            self.session_hash.as_ref().is_none_or(|v| hex_id(v, 64))
                && self.action_hash.as_ref().is_none_or(|v| hex_id(v, 64))
                && self.turn_hash.as_ref().is_none_or(|v| hex_id(v, 64)),
            "Invalid correlation hash"
        );
        ensure!(
            matches!(
                self.operation.as_str(),
                "api.request"
                    | "command.admission"
                    | "runtime.command"
                    | "agent.execute"
                    | "model.attempt"
                    | "model.requested"
                    | "approval.wait"
                    | "question.wait"
                    | "tool.execute"
                    | "policy.review"
                    | "safety.check"
                    | "environment.operation"
                    | "storage.call"
                    | "worker.activation"
                    | "worker.lease"
                    | "session.recovery"
                    | "session.recovery_scan"
                    | "approval.requested"
                    | "approval.resolved"
                    | "question.requested"
                    | "question.resolved"
                    | "turn.cancellation"
                    | "tool.unknown_outcome"
                    | "turn.budget_exceeded"
                    | "tool.validation_failed"
                    | "process.started"
                    | "process.stopped"
                    | "process.stale"
            ),
            "Invalid observation operation"
        );
        ensure!(
            matches!(
                self.outcome.as_str(),
                "ok" | "error" | "interrupted" | "denied" | "waiting" | "cancelled" | "unknown"
            ),
            "Invalid observation outcome"
        );
        ensure!(
            (self.duration_us.is_none() || self.ended_ms >= self.started_ms)
                && self.ended_ms <= i64::MAX as u64
                && self.started_ms <= i64::MAX as u64
                && self.duration_us.is_none_or(|n| n <= i64::MAX as u64),
            "Invalid observation timing"
        );
        ensure!(
            self.first_output_us
                .is_none_or(|v| self.duration_us.is_some_and(|d| v <= d)),
            "First output exceeds request duration"
        );
        ensure!(
            self.attempt.is_none_or(|v| (1..=10000).contains(&v)),
            "Invalid observation attempt"
        );
        ensure!(
            self.model.as_ref().is_none_or(|v| label(v))
                && self.tool.as_ref().is_none_or(|v| label(v)),
            "Invalid observation label"
        );
        ensure!(
            self.purpose.as_ref().is_none_or(|v| matches!(
                v.as_str(),
                "main"
                    | "auto_approver"
                    | "compaction"
                    | "helper"
                    | "title"
                    | "probe"
                    | "guardrail"
            )),
            "Invalid model purpose"
        );
        ensure!(
            self.operation == "model.attempt"
                || (self.usage.is_none()
                    && (self.operation == "model.requested" || self.purpose.is_none())),
            "Usage belongs to model requests"
        );
        ensure!(
            self.links.len() <= 4
                && self
                    .links
                    .iter()
                    .all(|l| hex_id(&l.trace_id, 32) && hex_id(&l.span_id, 16)),
            "Invalid observation links"
        );
        ensure!(
            self.output_bytes.is_none_or(|n| n <= i64::MAX as u64)
                && self.sandbox.as_ref().is_none_or(|s| label(s))
                && self.enforcement.as_ref().is_none_or(|s| label(s))
                && self.policy_digest.as_ref().is_none_or(|s| hex_id(s, 64)),
            "Invalid tool facts"
        );
        if let Some(u) = &self.usage {
            ensure!(
                [u.input, u.output, u.cache_read, u.cache_write, u.reasoning]
                    .into_iter()
                    .flatten()
                    .all(|v| v <= i64::MAX as u64),
                "Observation usage out of range"
            );
            ensure!(
                u.cache_read.zip(u.input).is_none_or(|(a, b)| a <= b)
                    && u.cache_write.zip(u.input).is_none_or(|(a, b)| a <= b)
                    && u.reasoning.zip(u.output).is_none_or(|(a, b)| a <= b),
                "Usage breakdown exceeds total"
            );
            if let (Some(a), Some(b), Some(total)) = (u.cache_read, u.cache_write, u.input) {
                ensure!(
                    a.checked_add(b).is_some_and(|v| v <= total),
                    "Cache breakdown exceeds input"
                );
            }
        }
        ensure!(
            serde_json::to_vec(self)?.len() <= MAX_RECORD_BYTES,
            "Observation too large"
        );
        Ok(())
    }
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Write {
    Inserted,
    Duplicate,
}
#[derive(Debug)]
pub struct Conflict;
impl std::fmt::Display for Conflict {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.write_str("Observation identity conflict")
    }
}
impl std::error::Error for Conflict {}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct StoredStep {
    pub cursor: u64,
    pub step: Step,
}

/// Trusted host interface only. A service must authorize the canonical session
/// before querying; the observation database is never an authorization authority.
pub trait Store: Send + 'static {
    #[cfg(feature = "evaluations")]
    fn register_evaluation_sink(
        &mut self,
        _deployment: &str,
        _sink: &str,
        _binding: &str,
    ) -> Result<()> {
        anyhow::bail!("Evaluation export unavailable")
    }

    #[cfg(feature = "evaluations")]
    fn prune_evaluations(
        &mut self,
        _deployment: &str,
        _before_ms: u64,
        _limit: u16,
    ) -> Result<u64> {
        anyhow::bail!("Evaluation pruning unsupported")
    }

    #[cfg(feature = "evaluations")]
    fn claim_evaluation_export(
        &mut self,
        _deployment: &str,
        _sink: &str,
        _binding: &str,
        _token: &str,
        _now: u64,
    ) -> Result<Option<evaluations::Delivery>> {
        anyhow::bail!("Evaluation export unavailable")
    }
    #[cfg(feature = "evaluations")]
    fn settle_evaluation_export(
        &mut self,
        _deployment: &str,
        _sink: &str,
        _token: &str,
        _now: u64,
        _success: bool,
    ) -> Result<()> {
        anyhow::bail!("Evaluation export unavailable")
    }

    #[cfg(feature = "evaluations")]
    fn evaluation(
        &mut self,
        _deployment: &str,
        _id: &str,
    ) -> Result<Option<evaluations::Evaluation>> {
        anyhow::bail!("Evaluation storage unavailable")
    }
    #[cfg(feature = "evaluations")]
    fn record_evaluation(&mut self, value: &evaluations::Evaluation) -> Result<Write> {
        self.record_evaluation_bounded(value, evaluations::DEFAULT_MAX_PENDING)
    }
    #[cfg(feature = "evaluations")]
    fn record_evaluation_bounded(
        &mut self,
        _value: &evaluations::Evaluation,
        _max_pending: u32,
    ) -> Result<Write> {
        anyhow::bail!("Evaluation storage unsupported")
    }
    #[cfg(feature = "evaluations")]
    fn evaluations(&mut self, _query: &evaluations::Query) -> Result<evaluations::Page> {
        anyhow::bail!("Evaluation storage unsupported")
    }

    /// Read-only bounded session/turn projection; no authorization authority.
    fn timeline(&mut self, _query: &query::Query) -> Result<query::Page> {
        anyhow::bail!("Recording queries unsupported by this adapter")
    }
    fn put(&mut self, step: &Step) -> Result<Write>;
    fn get(&mut self, deployment: &str, span_id: &str) -> Result<Option<Step>>;
    fn list(
        &mut self,
        deployment: &str,
        session_hash: Option<&str>,
        after: u64,
        limit: u16,
    ) -> Result<Vec<StoredStep>>;
    fn prune(&mut self, deployment: &str, ended_before_ms: u64, limit: u16) -> Result<u64>;
}
pub(crate) fn query(deployment: &str, session: Option<&str>, after: u64, limit: u16) -> Result<()> {
    ensure!(
        identifier(deployment)
            && session.is_none_or(|s| hex_id(s, 64))
            && after <= i64::MAX as u64
            && (1..=1000).contains(&limit),
        "Invalid observation query"
    );
    Ok(())
}