Skip to content
Python
8.6 KB
Raw Blame
use openworker_observability::{
    sqlite::Sqlite, Recorder, RecordingSink, Step, Store, StoredStep, Write,
};
use openworker_telemetry::{
    Capture, ContentPolicy, Operation, Outcome, Record, Sink, Snapshot, Telemetry,
};
use std::{
    sync::{mpsc, Arc, Mutex},
    time::{Duration, Instant},
};
struct Collect(Arc<Mutex<Vec<Record>>>);
impl Sink for Collect {
    fn export(&mut self, r: &[Record], _: Snapshot) -> bool {
        self.0.lock().unwrap().extend_from_slice(r);
        true
    }
}
fn record() -> Record {
    let out = Arc::new(Mutex::new(Vec::new()));
    let (t, c) = Telemetry::start_with_content(
        Collect(out.clone()),
        10,
        ContentPolicy {
            capture: Capture::Full,
            ..Default::default()
        },
    )
    .unwrap();
    let mut s = t.span(Operation::Model, Some("session-id"), Some("attempt-id"));
    s.metadata("gen_ai.request.model", "claude-sonnet");
    s.content("gen_ai.prompt", || {
        "private prompt sk-notarealsecret1234567890".into()
    });
    s.content("gen_ai.completion", || "private answer".into());
    s.usage(100, 20);
    s.first_output();
    s.finish(Outcome::Ok);
    assert!(c.shutdown(Duration::from_secs(1)));
    let r = out.lock().unwrap()[0].clone();
    r
}
#[test]
fn typed_purpose_and_reported_breakdowns_survive_conversion_without_inference() {
    let mut r = record();
    assert!(Step::from_record("test", "worker", &r)
        .unwrap()
        .purpose
        .is_none());
    r.purpose = Some(openworker_telemetry::ModelPurpose::Compaction);
    r.turn_hash = Some([7; 32]);
    r.links.push(openworker_telemetry::TraceContext {
        trace: [3; 16],
        span: [4; 8],
    });
    r.usage_details = openworker_telemetry::TokenUsage {
        input: Some(100),
        output: Some(20),
        cache_read: Some(40),
        cache_write: Some(10),
        reasoning: Some(5),
    };
    let step = Step::from_record("test", "worker", &r).unwrap();
    assert_eq!(step.version, 2);
    assert_eq!(step.purpose.as_deref(), Some("compaction"));
    assert_eq!(step.links.len(), 1);
    let usage = step.usage.as_ref().unwrap();
    assert_eq!(usage.cache_read, Some(40));
    assert_eq!(usage.cache_write, Some(10));
    assert_eq!(usage.reasoning, Some(5));
    let encoded = serde_json::to_string(&step).unwrap();
    assert!(!encoded.contains("private"));
    assert_eq!(serde_json::from_str::<Step>(&encoded).unwrap(), step);
}
#[test]
fn real_instrumentation_reopens_metadata_only_and_keeps_export_off() {
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("obs.db");
    let (store, done) = Recorder::start(
        Box::new(Sqlite::open(&path).unwrap()),
        "deployment".into(),
        16,
        30,
    )
    .unwrap();
    let stats = store.statistics();
    let exported = Arc::new(Mutex::new(Vec::new()));
    let mut sink = RecordingSink::new(
        store,
        "deployment".into(),
        "worker".into(),
        Some(Box::new(Collect(exported.clone()))),
        Capture::Off,
    )
    .unwrap();
    let r = record();
    assert!(sink.export(&[r.clone(), r.clone()], Snapshot::default()));
    let mut changed = r.clone();
    changed.input_tokens = Some(101);
    assert!(sink.export(&[changed], Snapshot::default()));
    assert!(done.shutdown(Duration::from_secs(2)));
    let s = stats.snapshot();
    assert_eq!(
        (s.persisted, s.duplicates, s.conflicts, s.pending),
        (1, 1, 1, 0)
    );
    assert!(exported
        .lock()
        .unwrap()
        .iter()
        .all(|r| r.attributes.is_empty()));
    let mut reopened = Sqlite::open(&path).unwrap();
    let rows = reopened.list("deployment", None, 0, 100).unwrap();
    assert_eq!(rows.len(), 1);
    assert_eq!(rows[0].step.model.as_deref(), Some("claude-sonnet"));
    let json = serde_json::to_string(&rows).unwrap();
    assert!(
        !json.contains("private prompt")
            && !json.contains("private answer")
            && !json.contains("sk-notareal")
            && !json.contains("session-id")
    );
    assert_eq!(rows[0].step.usage.as_ref().unwrap().input, Some(100));
    assert_eq!(rows[0].step.usage.as_ref().unwrap().cache_read, None);
}
struct Blocked {
    entered: mpsc::SyncSender<()>,
    release: mpsc::Receiver<()>,
}
impl Store for Blocked {
    fn put(&mut self, _: &Step) -> anyhow::Result<Write> {
        self.entered.send(()).unwrap();
        self.release.recv().unwrap();
        Ok(Write::Inserted)
    }
    fn get(&mut self, _: &str, _: &str) -> anyhow::Result<Option<Step>> {
        unreachable!()
    }
    fn list(
        &mut self,
        _: &str,
        _: Option<&str>,
        _: u64,
        _: u16,
    ) -> anyhow::Result<Vec<StoredStep>> {
        unreachable!()
    }
    fn prune(&mut self, _: &str, _: u64, _: u16) -> anyhow::Result<u64> {
        unreachable!()
    }
}
#[test]
fn blocked_writer_has_bounded_queue_and_shutdown_without_blocking_producer() {
    let (entered, rx) = mpsc::sync_channel(1);
    let (tx, release) = mpsc::sync_channel(1);
    let (recorder, c) = Recorder::start(
        Box::new(Blocked { entered, release }),
        "deployment".into(),
        1,
        30,
    )
    .unwrap();
    let step = Step::from_record("deployment", "test", &record()).unwrap();
    assert!(recorder.record(step.clone()));
    rx.recv_timeout(Duration::from_secs(1)).unwrap();
    assert!(recorder.record(step.clone()));
    let at = Instant::now();
    for _ in 0..1000 {
        assert!(!recorder.record(step.clone()));
    }
    assert!(at.elapsed() < Duration::from_secs(1));
    assert!(!c.shutdown(Duration::from_millis(10)));
    assert!(!recorder.record(step));
    tx.send(()).unwrap();
    let deadline = Instant::now() + Duration::from_secs(2);
    while recorder.statistics().snapshot().abandoned == 0 && Instant::now() < deadline {
        std::thread::sleep(Duration::from_millis(5));
    }
    let stats = recorder.statistics().snapshot();
    assert_eq!(stats.persisted, 1);
    assert_eq!(stats.abandoned, 1);
    assert_eq!(stats.dropped, 1001);
}
#[test]
fn invalid_scope_is_rejected_before_queue() {
    let d = tempfile::tempdir().unwrap();
    let (recorder, c) = Recorder::start(
        Box::new(Sqlite::open(&d.path().join("obs.db")).unwrap()),
        "deployment".into(),
        8,
        30,
    )
    .unwrap();
    assert!(!recorder.record(Step::from_record("other", "test", &record()).unwrap()));
    assert!(c.shutdown(Duration::from_secs(1)));
    assert_eq!(recorder.statistics().snapshot().rejected, 1);
}

struct StalledExporter {
    entered: mpsc::SyncSender<()>,
    release: mpsc::Receiver<()>,
}
impl Sink for StalledExporter {
    fn export(&mut self, _: &[Record], _: Snapshot) -> bool {
        self.entered.send(()).unwrap();
        self.release.recv().unwrap();
        false
    }
}
#[test]
fn local_commit_does_not_wait_for_a_stalled_vendor_exporter() {
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("obs.db");
    let (recorder, recording_done) = Recorder::start(
        Box::new(Sqlite::open(&path).unwrap()),
        "deployment".into(),
        16,
        30,
    )
    .unwrap();
    let stats = recorder.statistics();
    let (tx, entered) = mpsc::sync_channel(1);
    let (release, rx) = mpsc::sync_channel(1);
    let sink = RecordingSink::new(
        recorder,
        "deployment".into(),
        "test".into(),
        Some(Box::new(StalledExporter {
            entered: tx,
            release: rx,
        })),
        Capture::Off,
    )
    .unwrap();
    let (t, done) = Telemetry::start_with_content(
        sink,
        16,
        ContentPolicy {
            capture: Capture::Metadata,
            ..Default::default()
        },
    )
    .unwrap();
    let mut span = t.span(Operation::Tool, Some("session"), Some("call"));
    span.metadata("gen_ai.tool.name", "read_file");
    span.finish(Outcome::Ok);
    entered.recv_timeout(Duration::from_secs(1)).unwrap();
    let deadline = Instant::now() + Duration::from_secs(2);
    while stats.snapshot().persisted == 0 && Instant::now() < deadline {
        std::thread::sleep(Duration::from_millis(5));
    }
    let committed = stats.snapshot().persisted;
    release.send(()).unwrap();
    assert!(done.shutdown(Duration::from_secs(2)));
    assert!(recording_done.shutdown(Duration::from_secs(2)));
    assert_eq!(
        committed, 1,
        "Local commit must occur while exporter remains blocked"
    );
    assert_eq!(stats.snapshot().failed_writes, 0);
    assert_eq!(t.snapshot().export_failures, 1);
    assert_eq!(
        Sqlite::open(&path)
            .unwrap()
            .list("deployment", None, 0, 100)
            .unwrap()
            .len(),
        1
    );
}