Skip to content
Python
12.5 KB
Raw Blame
use openworker_observability::{
    query::{Page, Query, Scope},
    sqlite::Sqlite,
    summary::{self, Canonical},
    Step, Store, StoredStep,
};
use serde_json::json;
fn step(id: u64, op: &str, start: u64, duration: u64) -> Step {
    serde_json::from_value(json!({"version":2,"deployment":"test","role":"worker","trace_id":format!("{:032x}",1),"span_id":format!("{id:016x}"),"session_hash":format!("{:064x}",1),"turn_hash":format!("{:064x}",2),"operation":op,"outcome":"ok","started_ms":start,"ended_ms":start+duration/1000,"duration_us":duration})).unwrap()
}
fn query() -> Query {
    Query {
        deployment: "test".into(),
        scopes: vec![Scope {
            session_hash: format!("{:064x}", 1),
            turn_hash: format!("{:064x}", 2),
        }],
        after: 0,
        through: None,
        limit: 1,
    }
}
fn contract(store: &mut dyn Store) {
    let a = step(1, "agent.execute", 1000, 100000);
    let b = step(2, "tool.execute", 1010, 20000);
    store.put(&a).unwrap();
    store.put(&b).unwrap();
    let mut same_session = step(3, "tool.execute", 1010, 10000);
    same_session.turn_hash = Some(format!("{:064x}", 3));
    store.put(&same_session).unwrap();
    let mut foreign = step(4, "tool.execute", 1010, 10000);
    foreign.session_hash = Some(format!("{:064x}", 9));
    store.put(&foreign).unwrap();
    let mut other = step(5, "tool.execute", 1010, 10000);
    other.deployment = "other".into();
    store.put(&other).unwrap();
    let mut q = query();
    let first = store.timeline(&q).unwrap();
    assert_eq!(first.steps.len(), 1);
    assert_eq!(first.steps[0].step, a);
    assert!(first.next.is_some());
    store.put(&step(6, "tool.execute", 1030, 1000)).unwrap();
    q.after = first.next.unwrap();
    q.through = Some(first.through);
    let next = store.timeline(&q).unwrap();
    assert_eq!(next.steps.len(), 1);
    assert_eq!(next.steps[0].step, b);
    assert!(next.next.is_none());
    assert_eq!(next.through, first.through);
    q.after = 0;
    q.through = None;
    q.limit = 4096;
    q.scopes.push(Scope {
        session_hash: format!("{:064x}", 9),
        turn_hash: format!("{:064x}", 2),
    });
    let all = store.timeline(&q).unwrap();
    assert_eq!(all.steps.len(), 4);
    assert!(all
        .steps
        .iter()
        .all(|r| r.step.deployment == "test" && r.step.turn_hash == Some(format!("{:064x}", 2))));
    q.scopes.clear();
    assert!(store.timeline(&q).is_err());
    q = query();
    q.limit = 4097;
    assert!(store.timeline(&q).is_err());
    q = query();
    q.after = u64::MAX;
    assert!(store.timeline(&q).is_err());
    store.prune("test", 2000, 1000).unwrap();
    q = query();
    q.through = Some(all.through);
    assert!(store.timeline(&q).unwrap().steps.is_empty());
    assert_eq!(store.list("other", None, 0, 100).unwrap().len(), 1);
}
#[test]
fn sqlite_scoped_pages_watermark_retention_and_validation() {
    let root = tempfile::tempdir().unwrap();
    let mut store = Sqlite::open(&root.path().join("obs.db")).unwrap();
    contract(&mut store);
}
#[cfg(feature = "postgres")]
#[test]
#[ignore = "disposable OW_OBSERVABILITY_TEST_URL required"]
fn postgres_scoped_pages_watermark_retention_and_validation() {
    use openworker_observability::postgres::{Config, Postgres, Tls};
    let url = std::env::var("OW_OBSERVABILITY_TEST_URL").unwrap();
    let schema = format!(
        "ow_query_{}",
        std::time::SystemTime::now()
            .duration_since(std::time::UNIX_EPOCH)
            .unwrap()
            .as_nanos()
    );
    let mut c = Config::new(url.clone(), schema.clone());
    c.tls = Tls::LoopbackPlaintext;
    Postgres::migrate(&c).unwrap();
    let mut store = Postgres::open(&c).unwrap();
    contract(&mut store);
    drop(store);
    postgres::Client::connect(&url, postgres::NoTls)
        .unwrap()
        .batch_execute(&format!("DROP SCHEMA {schema} CASCADE"))
        .unwrap();
}
#[test]
fn corrupted_row_cannot_return_another_sessions_payload() {
    let root = tempfile::tempdir().unwrap();
    let path = root.path().join("obs.db");
    let mut store = Sqlite::open(&path).unwrap();
    store.put(&step(1, "tool.execute", 1000, 1000)).unwrap();
    let mut foreign = step(1, "tool.execute", 1000, 1000);
    foreign.session_hash = Some(format!("{:064x}", 9));
    let db = rusqlite::Connection::open(&path).unwrap();
    db.execute(
        "UPDATE ow_observation_steps SET data=?1",
        [serde_json::to_string(&foreign).unwrap()],
    )
    .unwrap();
    assert!(store.timeline(&query()).is_err());
}
fn page(steps: Vec<Step>) -> Page {
    Page {
        steps: steps
            .into_iter()
            .enumerate()
            .map(|(n, step)| StoredStep {
                cursor: n as u64 + 1,
                step,
            })
            .collect(),
        through: 100,
        next: None,
    }
}
fn model(id: u64, start: u64, duration: u64, usage: bool) -> Step {
    let mut s = step(id, "model.attempt", start, duration);
    s.model = Some("fixture".into());
    s.purpose = Some("main".into());
    s.first_output_us = Some(1000.min(duration));
    if usage {
        s.usage = Some(openworker_observability::Usage {
            input: Some(100),
            output: Some(20),
            cache_read: Some(40),
            cache_write: None,
            reasoning: Some(5),
        });
    }
    s
}
fn requested(id: u64, m: &Step) -> Step {
    let mut s = step(id, "model.requested", m.started_ms, 0);
    s.parent_trace_id = Some(m.trace_id.clone());
    s.parent_span_id = Some(m.span_id.clone());
    s
}
#[test]
fn summaries_reconcile_retries_usage_containers_parallel_time_and_waits() {
    let root = step(1, "agent.execute", 1000, 100000);
    let last = step(2, "agent.execute", 2000, 50000);
    let mut a = model(3, 1010, 40000, true);
    a.parent_trace_id = Some(a.trace_id.clone());
    a.parent_span_id = Some(format!("{:016x}", 9));
    let mut b = model(4, 2010, 10000, false);
    b.outcome = "error".into();
    b.purpose = Some("compaction".into());
    let mut x = step(5, "tool.execute", 1060, 20000);
    x.tool = Some("coding.read".into());
    x.output_bytes = Some(12);
    let mut y = step(6, "tool.execute", 2015, 20000);
    y.tool = x.tool.clone();
    let p = page(vec![
        root,
        last.clone(),
        a.clone(),
        b.clone(),
        x,
        y,
        requested(7, &a),
        requested(8, &b),
        step(9, "policy.review", 1005, 50000),
        step(10, "approval.wait", 1090, 910000),
    ]);
    for row in &p.steps {
        row.step.validate().unwrap();
    }
    let summary = summary::build(
        &p,
        Canonical {
            terminal: true,
            expected_segments: Some(2),
            first_started_ms: Some(1000),
            last_span_id: Some(last.span_id),
            pending_waits: Some(0),
        },
    );
    let golden: serde_json::Value =
        serde_json::from_str(include_str!("../../tests/contracts/observability.json")).unwrap();
    let expected = &golden["summary_overlap"];
    assert_eq!(
        serde_json::to_value(&summary.exclusive_observed_us).unwrap(),
        expected["exclusive_observed_us"]
    );
    assert_eq!(
        serde_json::to_value(summary.wall_elapsed_us).unwrap(),
        expected["wall_elapsed_us"]
    );
    assert_eq!(summary.model_accumulated_us.total, Some(50000));
    assert_eq!(summary.tool_accumulated_us.total, Some(40000));
    assert_eq!(summary.active_segments_accumulated_us.total, Some(150000));
    assert_eq!(summary.wall_elapsed_us, Some(1050000));
    assert_eq!(summary.observed_window_us, Some(1050000));
    assert_eq!(summary.exclusive_observed_us.values().sum::<u64>(), 1050000);
    assert_eq!(summary.exclusive_observed_us["model"], 45000);
    assert_eq!(summary.exclusive_observed_us["tool"], 35000);
    assert_eq!(summary.exclusive_observed_us["parallel"], 5000);
    assert_eq!(summary.exclusive_observed_us["policy"], 10000);
    assert_eq!(summary.exclusive_observed_us["approval_wait"], 910000);
    assert_eq!(summary.usage.input.total, Some(100));
    assert_eq!(summary.usage.input.known, 1);
    assert_eq!(summary.usage.input.unknown, 1);
    assert_eq!(summary.usage.cache_read.total, Some(40));
    assert_eq!(summary.usage.reasoning.total, Some(5));
    assert_eq!(summary.models.len(), 2);
    assert_eq!(summary.tools[0].calls, 2);
    assert_eq!(summary.tools[0].active_union_us, Some(40000));
    assert_eq!(summary.tools[0].returned_bytes.unknown, 1);
    assert_eq!(summary.coverage.requests_without_completion, 0);
    assert!(summary.coverage.all_visible_segments_closed);
    assert_eq!(summary.coverage.guarantee, "best_effort");
    let again = summary::build(&p, Canonical::default());
    assert_eq!(again.usage.input.total, summary.usage.input.total);
    assert!(again.wall_elapsed_us.is_none());
}
#[test]
fn union_is_not_accumulation_known_zero_is_not_missing_and_gaps_are_explicit() {
    let mut a = model(1, 1000, 100000, true);
    a.usage.as_mut().unwrap().input = Some(0);
    a.usage.as_mut().unwrap().cache_read = None;
    let b = model(2, 1050, 100000, false);
    let p = page(vec![a, b, step(3, "tool.execute", 1200, 10000)]);
    let s = summary::build(&p, Canonical::default());
    assert_eq!(s.models[0].accumulated_us.total, Some(200000));
    assert_eq!(s.models[0].active_union_us, Some(150000));
    assert_eq!(s.usage.input.total, Some(0));
    assert_eq!(s.usage.input.unknown, 1);
    assert_eq!(s.exclusive_observed_us["unobserved_gap"], 50000);
    assert_eq!(s.coverage.completions_without_start, 2);
}
#[test]
fn truncation_orphan_requests_clock_anomalies_and_overflow_never_fake_completeness() {
    let a = model(1, 1000, 1000, true);
    let mut skew = step(2, "approval.wait", 2000, 0);
    skew.duration_us = None;
    skew.ended_ms = 1999;
    skew.validate().unwrap();
    let mut p = page(vec![requested(3, &a), skew]);
    p.next = Some(2);
    let s = summary::build(
        &p,
        Canonical {
            terminal: true,
            expected_segments: Some(2),
            ..Default::default()
        },
    );
    assert!(s.wall_elapsed_us.is_none());
    assert!(s.coverage.scan_truncated);
    assert!(s.coverage.clock_or_range_anomaly);
    assert_eq!(s.coverage.requests_without_completion, 1);
    assert_eq!(s.coverage.unknown_timing_steps, 1);
    let p = page(
        (1..=3)
            .map(|n| {
                let mut s = model(n, 1000, 1000, true);
                s.usage.as_mut().unwrap().input = Some(i64::MAX as u64);
                s
            })
            .collect(),
    );
    let s = summary::build(&p, Canonical::default());
    assert!(s.usage.input.overflow);
    assert!(s.usage.input.total.is_none());
}
#[test]
fn groups_are_bounded_but_overall_usage_still_counts_the_whole_scanned_page() {
    let p = page(
        (1..=129)
            .map(|n| {
                let mut s = model(n, 1000, 1000, true);
                s.model = Some(format!("model-{n}"));
                s
            })
            .collect(),
    );
    let s = summary::build(&p, Canonical::default());
    assert_eq!(s.models.len(), 128);
    assert!(s.coverage.group_truncated);
    assert_eq!(s.usage.input.total, Some(12900));
}

#[test]
fn projection_enforces_its_own_row_budget_and_preserves_zero_duration() {
    let p = page((1..=4097).map(|n| model(n, 1000, 0, true)).collect());
    let s = summary::build(&p, Canonical::default());
    assert_eq!(s.coverage.scanned_steps, 4096);
    assert!(s.coverage.scan_truncated);
    assert_eq!(s.usage.input.total, Some(409600));
    assert_eq!(s.model_accumulated_us.total, Some(0));
    assert_eq!(s.observed_window_us, Some(0));
}

#[test]
fn unrelated_policy_work_is_parallel_and_missing_or_cyclic_parents_are_explicit() {
    let policy = step(1, "policy.review", 1000, 100000);
    let mut m = model(2, 1000, 100000, true);
    let independent = summary::build(&page(vec![policy.clone(), m.clone()]), Canonical::default());
    assert_eq!(independent.exclusive_observed_us["parallel"], 100000);
    m.parent_trace_id = Some(policy.trace_id.clone());
    m.parent_span_id = Some(policy.span_id.clone());
    let nested = summary::build(&page(vec![policy.clone(), m.clone()]), Canonical::default());
    assert_eq!(nested.exclusive_observed_us["model"], 100000);
    assert!(!nested.exclusive_observed_us.contains_key("parallel"));
    let missing = summary::build(&page(vec![m.clone()]), Canonical::default());
    assert!(missing.coverage.hierarchy_incomplete);
    let mut cycle = policy;
    cycle.parent_trace_id = Some(m.trace_id.clone());
    cycle.parent_span_id = Some(m.span_id.clone());
    let cycle = summary::build(&page(vec![cycle, m]), Canonical::default());
    assert!(cycle.coverage.hierarchy_incomplete);
}