Skip to content
Python
8.6 KB
Raw Blame
use openworker_telemetry::*;
use std::{
    sync::{mpsc, Arc, Mutex},
    time::{Duration, Instant},
};
struct Capture(Arc<Mutex<Vec<Record>>>);
impl Sink for Capture {
    fn export(&mut self, r: &[Record], _: Snapshot) -> bool {
        self.0.lock().unwrap().extend_from_slice(r);
        true
    }
}
#[tokio::test]
async fn hierarchy_is_task_local_and_only_hashes_cross_the_boundary() {
    let rows = Arc::new(Mutex::new(vec![]));
    let (t, c) = Telemetry::start(Capture(rows.clone()), 2048).unwrap();
    let root = t.span(
        Operation::Turn,
        Some("PRIVATE_SESSION"),
        Some("PRIVATE_TEXT"),
    );
    let context = root.context();
    scope(context, async {
        let mut model = t.span(Operation::Model, Some("PRIVATE_SESSION"), None);
        model.attempt(2);
        model.usage(40, 10);
        model.first_output();
        model.finish(Outcome::Ok);
        let other = t.clone();
        tokio::spawn(async move { other.event(Operation::Command, Outcome::Ok, None, None) })
            .await
            .unwrap();
    })
    .await;
    root.finish(Outcome::Waiting);
    assert!(c.shutdown(Duration::from_secs(2)));
    let records = rows.lock().unwrap();
    assert_eq!(records.len(), 3);
    let model = records
        .iter()
        .find(|r| r.operation == Operation::Model)
        .unwrap();
    assert_eq!(model.parent, Some(context));
    assert_eq!(model.context.trace, context.trace);
    let independent = records
        .iter()
        .find(|r| r.operation == Operation::Command)
        .unwrap();
    assert!(independent.parent.is_none());
    assert_ne!(independent.context.trace, context.trace);
    assert!(!format!("{records:?}").contains("PRIVATE"));
    assert_eq!(model.input_tokens, Some(40));
    assert!(model.first_output.is_some());
}
struct Block(mpsc::SyncSender<()>, mpsc::Receiver<()>);
#[tokio::test]
async fn durable_turn_identity_links_segments_without_counting_pause_as_execution() {
    let rows = Arc::new(Mutex::new(vec![]));
    let (t, c) = Telemetry::start(Capture(rows.clone()), 32).unwrap();
    let mut first = t.span(Operation::Turn, Some("session"), Some("turn"));
    first.turn("turn");
    let a = first.context();
    first.finish(Outcome::Waiting);
    let mut second = t.span(Operation::Turn, Some("session"), Some("turn"));
    second.turn("turn");
    second.link(a);
    let b = second.context();
    scope_turn("turn", b, async {
        let mut model = t.span(Operation::Model, Some("session"), None);
        model.purpose(ModelPurpose::Main);
        model.reported_usage(TokenUsage {
            input: Some(0),
            output: Some(0),
            ..Default::default()
        });
        model.finish(Outcome::Ok);
    })
    .await;
    second.finish(Outcome::Ok);
    t.interval(
        Operation::ApprovalWait,
        Outcome::Ok,
        "session",
        "turn",
        "approval",
        1000,
        6000,
    );
    t.interval(
        Operation::QuestionWait,
        Outcome::Ok,
        "session",
        "turn",
        "question",
        7000,
        6500,
    );
    assert!(c.shutdown(Duration::from_secs(2)));
    let r = rows.lock().unwrap();
    assert_eq!(a.trace, b.trace);
    assert_ne!(a.span, b.span);
    assert_eq!(r[0].turn_hash, r[1].turn_hash);
    let resumed = r.iter().find(|r| r.context == b).unwrap();
    assert_eq!(resumed.links, vec![a]);
    assert!(resumed.elapsed < Duration::from_secs(1));
    let m = r.iter().find(|r| r.operation == Operation::Model).unwrap();
    assert_eq!(m.parent, Some(b));
    assert_eq!(m.context.trace, a.trace);
    assert_eq!(m.input_tokens, Some(0));
    let wait = r
        .iter()
        .find(|r| r.operation == Operation::ApprovalWait)
        .unwrap();
    assert_eq!(wait.elapsed, Duration::from_secs(5));
    assert!(wait.timing_known);
    assert_eq!(wait.context.trace, a.trace);
    assert!(
        !r.iter()
            .find(|r| r.operation == Operation::QuestionWait)
            .unwrap()
            .timing_known
    );
}
impl Sink for Block {
    fn export(&mut self, _: &[Record], _: Snapshot) -> bool {
        let _ = self.0.try_send(());
        let _ = self.1.recv();
        false
    }
}
#[test]
fn observation_snapshot_is_enqueued_or_dropped_as_one_unit() {
    struct Samples(Arc<Mutex<Vec<Observation>>>);
    impl Sink for Samples {
        fn export(&mut self, _: &[Record], _: Snapshot) -> bool {
            panic!("Gauges must not create logs or spans")
        }
        fn observe(&mut self, s: &[Observation], _: Snapshot) -> bool {
            self.0.lock().unwrap().extend_from_slice(s);
            true
        }
    }
    let rows = Arc::new(Mutex::new(Vec::new()));
    let (t, c) = Telemetry::start(Samples(rows.clone()), 1).unwrap();
    t.observe_many(&[(Gauge::TurnsQueued, 5), (Gauge::StateObservedAt, 123)]);
    assert!(c.shutdown(Duration::from_secs(2)));
    assert_eq!(c.snapshot().accepted, 1);
    let rows = rows.lock().unwrap();
    assert_eq!(rows.len(), 2);
    assert_eq!((rows[0].value, rows[1].value), (5, 123));
}
#[test]
fn full_queue_and_blocked_exporter_never_block_producers_or_shutdown() {
    let (entered, wait) = mpsc::sync_channel(1);
    let (release, blocked) = mpsc::channel();
    let (t, c) = Telemetry::start(Block(entered, blocked), 2).unwrap();
    t.event(Operation::Tool, Outcome::Ok, None, None);
    wait.recv_timeout(Duration::from_secs(1)).unwrap();
    let now = Instant::now();
    for _ in 0..10000 {
        t.observe(Gauge::TurnsQueued, 1)
    }
    assert!(now.elapsed() < Duration::from_secs(2));
    assert_eq!(c.snapshot().accepted, 3);
    assert_eq!(c.snapshot().dropped, 9998);
    let now = Instant::now();
    assert!(!c.shutdown(Duration::from_millis(20)));
    assert!(now.elapsed() < Duration::from_millis(200));
    drop(release);
    assert!(c.shutdown(Duration::from_secs(2)));
    assert!(c.snapshot().export_failures >= 1);
}
#[test]
fn disabled_and_abandoned_spans_are_truthful() {
    assert!(
        std::mem::size_of::<Record>() <= 512,
        "Keep queue payloads bounded independently of strings/JSON"
    );
    let t = Telemetry::default();
    assert!(!t.enabled());
    t.event(Operation::Command, Outcome::Ok, None, None);
    assert_eq!(t.snapshot().accepted, 0);
    let rows = Arc::new(Mutex::new(vec![]));
    let (t, c) = Telemetry::start(Capture(rows.clone()), 2).unwrap();
    drop(t.span(Operation::Tool, None, None));
    drop(t.span(Operation::Model, None, None));
    assert!(c.shutdown(Duration::from_secs(2)));
    let rows = rows.lock().unwrap();
    assert_eq!(rows.len(), 2);
    assert_eq!(rows[0].outcome, Outcome::Unknown);
    assert_eq!(rows[1].outcome, Outcome::Interrupted);
    assert_eq!(rows[1].input_tokens, None);
}

#[test]
fn shared_registry_bounds_names_and_content_and_reserves_typed_fields() {
    let rows = Arc::new(Mutex::new(vec![]));
    let (t, c) = Telemetry::start_with_content(
        Capture(rows.clone()),
        16,
        ContentPolicy {
            capture: openworker_telemetry::Capture::Full,
            max_bytes: 64,
            redact_secrets: true,
        },
    )
    .unwrap();
    let mut model = t.span(Operation::Model, None, None);
    model.content("private.reasoning", || {
        panic!("Unknown/private content keys must stay lazy")
    });
    model.metadata("gen_ai.usage.input_tokens", "12345");
    model.metadata(
        "gen_ai.request.model",
        "https://user:[email protected]",
    );
    for _ in 0..100 {
        model.metadata("gen_ai.request.model", "safe-model");
    }
    model.content("gen_ai.prompt", || "a safe prompt".repeat(100));
    model.usage(0, 0);
    model.usage(u64::MAX, u64::MAX);
    model.finish(Outcome::Ok);
    assert!(c.shutdown(Duration::from_secs(2)));
    let rows = rows.lock().unwrap();
    let r = &rows[0];
    assert_eq!(r.attributes.len(), 2);
    assert_eq!(r.input_tokens, Some(0));
    assert_eq!(r.output_tokens, Some(0));
    assert!(r
        .attributes
        .iter()
        .all(|(k, v)| export_attribute(k, v) && v.len() <= 67));
}

#[test]
fn flush_is_a_bounded_barrier_without_stopping_instrumentation() {
    let rows = Arc::new(Mutex::new(vec![]));
    let (t, c) = Telemetry::start(Capture(rows.clone()), 32).unwrap();
    t.event(Operation::Command, Outcome::Ok, None, None);
    assert!(t.flush(Duration::from_secs(2)));
    assert_eq!(rows.lock().unwrap().len(), 1);
    t.event(Operation::Command, Outcome::Ok, None, None);
    assert!(t.flush(Duration::from_secs(2)));
    assert_eq!(rows.lock().unwrap().len(), 2);
    assert_eq!(t.snapshot().accepted, 2);
    assert!(c.shutdown(Duration::from_secs(2)));
    assert!(!t.flush(Duration::from_millis(10)));
    assert!(Telemetry::default().flush(Duration::ZERO));
}