Skip to content
Python
7.4 KB
Raw Blame
//! Optional durable score delivery using the compliance checkpoint contract.
//! An export retry never submits a turn or executes a tool.
use crate::recording::{EvaluationSink, RecordingConfig};
use anyhow::{ensure, Context, Result};
use openworker_observability::evaluations::{Evaluation, Score, Status};
use serde_json::{json, Value};
use std::{
    io::Read,
    sync::{
        atomic::{AtomicBool, AtomicU64, Ordering},
        mpsc, Arc, Mutex,
    },
    time::Duration,
};
#[derive(Default)]
struct Counters {
    sent: AtomicU64,
    skipped: AtomicU64,
    failed: AtomicU64,
}
pub(crate) struct Controller {
    stop: Arc<AtomicBool>,
    wake: mpsc::SyncSender<()>,
    done: Mutex<mpsc::Receiver<()>>,
    counters: Arc<Counters>,
}
impl Controller {
    pub fn notify(&self) {
        let _ = self.wake.try_send(());
    }
    pub fn health(&self) -> Value {
        json!({"scope":"sdk_host","sent":self.counters.sent.load(Ordering::Relaxed),"skipped_not_exportable":self.counters.skipped.load(Ordering::Relaxed),"failed_or_uncertain":self.counters.failed.load(Ordering::Relaxed)})
    }
    pub fn shutdown(&self) {
        self.stop.store(true, Ordering::Release);
        self.notify();
        let _ = self
            .done
            .lock()
            .unwrap()
            .recv_timeout(Duration::from_secs(5));
    }
}
impl Drop for Controller {
    fn drop(&mut self) {
        self.stop.store(true, Ordering::Release);
        self.notify();
    }
}
fn now() -> u64 {
    openworker_runtime::store::now_ms()
}
pub(crate) fn start(options: RecordingConfig) -> Result<Option<Arc<Controller>>> {
    if options.evaluation_sinks.is_empty() {
        return Ok(None);
    }
    for sink in &options.evaluation_sinks {
        sink.validate()?;
        ensure!(
            std::env::var(&sink.authorization_env).is_ok(),
            "Evaluation export credential unavailable"
        );
    }
    let mut store = options.open()?;
    for sink in &options.evaluation_sinks {
        let binding = openworker_runtime::bundle::digest(serde_json::to_string(sink)?.as_bytes());
        store.register_evaluation_sink(&options.deployment, &sink.name, &binding)?;
    }
    drop(store);
    let (wake, rx) = mpsc::sync_channel(1);
    let (done, wait) = mpsc::sync_channel(1);
    let stop = Arc::new(AtomicBool::new(false));
    let stopped = stop.clone();
    let counters = Arc::new(Counters::default());
    let stats = counters.clone();
    std::thread::Builder::new()
        .name("ow-evaluation-export".into())
        .spawn(move || {
            let client = reqwest::blocking::Client::builder()
                .no_proxy()
                .redirect(reqwest::redirect::Policy::none())
                .timeout(Duration::from_secs(3))
                .build();
            if let Ok(client) = client {
                while !stopped.load(Ordering::Acquire) {
                    for sink in &options.evaluation_sinks {
                        if stopped.load(Ordering::Acquire) {
                            break;
                        }
                        if deliver_next(&options, sink, &client, &stats).is_err() {
                            stats.failed.fetch_add(1, Ordering::Relaxed);
                        }
                    }
                    let _ = rx.recv_timeout(Duration::from_secs(1));
                }
            } else {
                stats.failed.fetch_add(1, Ordering::Relaxed);
            }
            let _ = done.send(());
        })?;
    Ok(Some(Arc::new(Controller {
        stop,
        wake,
        done: Mutex::new(wait),
        counters,
    })))
}
fn deliver_next(
    options: &RecordingConfig,
    sink: &EvaluationSink,
    client: &reqwest::blocking::Client,
    stats: &Counters,
) -> Result<()> {
    let binding = openworker_runtime::bundle::digest(serde_json::to_string(sink)?.as_bytes());
    let token = openworker_runtime::id("eval");
    let mut store = options.open()?;
    let Some(delivery) =
        store.claim_evaluation_export(&options.deployment, &sink.name, &binding, &token, now())?
    else {
        return Ok(());
    };
    let value = &delivery.row.evaluation;
    // Non-quality outcomes stay available locally without fabricating a score.
    let result = if value.result.status != Status::Completed || value.target.trace_id.is_none() {
        stats.skipped.fetch_add(1, Ordering::Relaxed);
        Ok(())
    } else {
        send(client, sink, value).map(|_| {
            stats.sent.fetch_add(1, Ordering::Relaxed);
        })
    };
    // Stable score ID/name/timestamp survive lease expiry, restart and lost ACK.
    // This checkpoint advances only for an acknowledged score or explicit skip.
    store.settle_evaluation_export(
        &options.deployment,
        &sink.name,
        &token,
        now(),
        result.is_ok(),
    )?;
    result
}
pub(crate) fn payload(e: &Evaluation, explanation: bool) -> Result<Value> {
    e.validate()?;
    ensure!(e.result.status == Status::Completed, "Not a quality score");
    let trace = e.target.trace_id.as_ref().context("Score trace missing")?;
    let (kind, value) = match e.result.score.as_ref().context("Score missing")? {
        Score::Numeric(v) => ("NUMERIC", json!(v)),
        Score::Boolean(v) => ("BOOLEAN", json!(u8::from(*v))),
        Score::Categorical(v) => ("CATEGORICAL", json!(v)),
        Score::Text(v) => ("TEXT", json!(v)),
    };
    let id =
        openworker_runtime::bundle::digest(format!("{}\0{}", e.deployment, e.result.id).as_bytes());
    let timestamp = chrono::DateTime::from_timestamp_millis(e.result.created_ms as i64)
        .context("Invalid score time")?
        .to_rfc3339_opts(chrono::SecondsFormat::Millis, true);
    let mut body = json!({"id":id,"name":e.result.name,"traceId":trace,"dataType":kind,"value":value,"timestamp":timestamp,"metadata":{"openworker.evaluation.id":e.result.id,"evaluator":e.result.evaluator.name,"evaluatorVersion":e.result.evaluator.version,"evaluatorSource":e.result.evaluator.source,"evaluatorModel":e.result.evaluator.model,"evaluatorUsage":e.result.evaluator.usage,"supersedes":e.result.supersedes}});
    if let Some(span) = &e.target.span_id {
        body["observationId"] = json!(span);
    }
    if explanation {
        if let Some(comment) = &e.result.explanation {
            body["comment"] = json!(comment);
        }
    }
    Ok(body)
}
fn send(client: &reqwest::blocking::Client, sink: &EvaluationSink, e: &Evaluation) -> Result<()> {
    let body = payload(e, sink.include_explanation)?;
    let authorization = zeroize::Zeroizing::new(
        std::env::var(&sink.authorization_env).context("Evaluation credential unavailable")?,
    );
    let response = client
        .post(format!(
            "{}/api/public/scores",
            sink.url.trim_end_matches('/')
        ))
        .header("Authorization", authorization.as_str())
        .json(&body)
        .send()
        .map_err(|_| anyhow::anyhow!("Evaluation delivery outcome unknown"))?;
    ensure!(
        response.status().as_u16() == 200,
        "Evaluation delivery not acknowledged"
    );
    let mut bytes = Vec::new();
    response.take(65537).read_to_end(&mut bytes)?;
    ensure!(bytes.len() <= 65536, "Evaluation acknowledgement too large");
    let ack: Value = serde_json::from_slice(&bytes)
        .map_err(|_| anyhow::anyhow!("Invalid evaluation acknowledgement"))?;
    ensure!(
        ack["id"] == body["id"],
        "Evaluation acknowledgement identity mismatch"
    );
    Ok(())
}