Skip to content
Python
5.0 KB
Raw Blame
//! Acknowledged evaluation writes. No action or model call is replayed on failure.
use super::Host;
use anyhow::{ensure, Context, Result};
use openworker_observability::evaluations::{Evaluation, Input, Query, Target};
use serde::Deserialize;
use serde_json::{json, Value};
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Record {
    session_id: String,
    turn_id: String,
    result: Input,
    span_id: Option<String>,
}
impl Host {
    pub(super) async fn evaluation_dispatch(&self, method: &str, p: Value) -> Result<Value> {
        let recorder = self.recording.clone().context("Recording disabled")?;
        ensure!(recorder.options.evaluations, "Evaluation storage disabled");
        if method == "observability/evaluation-record" {
            let q: Record = serde_json::from_value(p)?;
            crate::TraceQuery::turn(&q.session_id, &q.turn_id).validate()?;
            ensure!(
                self.store
                    .lock()
                    .unwrap()
                    .has_turn(&q.session_id, &q.turn_id)?,
                "Evaluation target turn unavailable"
            );
            let session_hash = openworker_runtime::bundle::digest(q.session_id.as_bytes());
            let turn_hash = openworker_runtime::bundle::digest(q.turn_id.as_bytes());
            let permit = self
                .recording_readers
                .clone()
                .try_acquire_owned()
                .context("Recording readers busy")?;
            let value=tokio::task::spawn_blocking(move || -> Result<Value> {
                let _permit=permit;
                let mut store=recorder.options.open()?;
                // Trace retention is independent from result retention. A valid
                // duplicate must not need its old span to remain in raw storage.
                if let Some(old)=store.evaluation(&recorder.options.deployment,&q.result.id)? {
                    ensure!(old.target.session_hash==session_hash&&old.target.turn_hash==turn_hash&&old.target.span_id==q.span_id,"Evaluation identity conflict");
                    let candidate=Evaluation{result:q.result.clone(),..old.clone()}.redacted();candidate.validate()?;
                    ensure!(candidate==old,"Evaluation identity conflict");
                    return Ok(json!({"result":old,"duplicate":true}));
                }
                let trace_id=if let Some(span)=&q.span_id {
                    let step=store.get(&recorder.options.deployment,span)?.context("Evaluation span unavailable")?;
                    ensure!(step.session_hash.as_deref()==Some(&session_hash)&&step.turn_hash.as_deref()==Some(&turn_hash),"Evaluation span scope mismatch");Some(step.trace_id)
                } else {None};
                let e=Evaluation{version:1,deployment:recorder.options.deployment.clone(),target:Target{session_hash,turn_hash,trace_id,span_id:q.span_id},result:q.result}.redacted();
                e.validate()?;
                let cutoff=openworker_runtime::store::now_ms().saturating_sub(u64::from(recorder.options.evaluation_retention_days)*86_400_000);
                store.prune_evaluations(&recorder.options.deployment,cutoff,1000)?;
                let result=store.record_evaluation_bounded(&e,recorder.options.max_pending_evaluations)?;
                Ok(json!({"result":e,"duplicate":result==openworker_observability::Write::Duplicate}))
            }).await??;
            #[cfg(feature = "evaluation-export")]
            if let Some(exporter) = &self.evaluation_exporter {
                exporter.notify();
            }
            return Ok(value);
        }
        if method == "observability/evaluation-list" {
            let q: crate::TraceQuery = serde_json::from_value(p)?;
            q.validate()?;
            ensure!(q.limit <= 50, "Evaluation page limit is 50");
            ensure!(
                self.store
                    .lock()
                    .unwrap()
                    .has_turn(&q.session_id, &q.turn_id)?,
                "Evaluation target turn unavailable"
            );
            let query = Query {
                deployment: recorder.options.deployment.clone(),
                session_hash: openworker_runtime::bundle::digest(q.session_id.as_bytes()),
                turn_hash: openworker_runtime::bundle::digest(q.turn_id.as_bytes()),
                after: q.after,
                limit: q.limit,
            };
            let permit = self
                .recording_readers
                .clone()
                .try_acquire_owned()
                .context("Recording readers busy")?;
            let value = tokio::task::spawn_blocking(move || -> Result<Value> {
                let _permit = permit;
                Ok(json!(recorder.options.open()?.evaluations(&query)?))
            })
            .await??;
            ensure!(
                self.store
                    .lock()
                    .unwrap()
                    .has_turn(&q.session_id, &q.turn_id)?,
                "Evaluation target turn unavailable"
            );
            return Ok(value);
        }
        anyhow::bail!("Unknown evaluation operation")
    }
}