Skip to content
Python
21.2 KB
Raw Blame
#![cfg(feature = "otlp")]
use axum::{
    body::Bytes,
    extract::{OriginalUri, State},
    routing::post,
    Router,
};
use openworker_telemetry::*;
use prost::Message;
use std::{
    sync::{Arc, Mutex},
    time::{Duration, Instant},
};
type Packets = Arc<Mutex<Vec<(String, Bytes)>>>;

#[tokio::test(flavor = "multi_thread")]
async fn slow_collector_cannot_extend_shutdown_budget() {
    async fn slow() -> &'static str {
        tokio::time::sleep(Duration::from_secs(15)).await;
        ""
    }
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
    let endpoint = format!("http://{}", listener.local_addr().unwrap());
    let server = tokio::spawn(async move {
        axum::serve(listener, Router::new().route("/*path", post(slow)))
            .await
            .unwrap()
    });
    let (t, c) = tokio::task::spawn_blocking(move || {
        otlp::Config {
            endpoint,
            deployment: "test".into(),
            ..Default::default()
        }
        .start("test")
        .unwrap()
    })
    .await
    .unwrap();
    t.event(Operation::Tool, Outcome::Ok, None, None);
    tokio::time::sleep(Duration::from_millis(100)).await;
    let now = Instant::now();
    let flushed = tokio::task::spawn_blocking(move || c.shutdown(Duration::from_millis(100)))
        .await
        .unwrap();
    assert!(!flushed);
    assert!(now.elapsed() < Duration::from_secs(1));
    let now = Instant::now();
    for _ in 0..1000 {
        t.event(Operation::Model, Outcome::Ok, None, None);
    }
    assert!(now.elapsed() < Duration::from_secs(1));
    assert!(t.snapshot().dropped >= 1000);
    server.abort();
}
async fn receive(
    State(p): State<Packets>,
    OriginalUri(uri): OriginalUri,
    bytes: Bytes,
) -> &'static str {
    p.lock().unwrap().push((uri.path().into(), bytes));
    ""
}
#[tokio::test(flavor = "multi_thread")]
async fn all_signals_have_valid_otlp_and_no_payload_or_metric_identifiers() {
    let packets = Packets::default();
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
    let endpoint = format!("http://{}", listener.local_addr().unwrap());
    let app = Router::new()
        .route("/v1/traces", post(receive))
        .route("/v1/metrics", post(receive))
        .route("/v1/logs", post(receive))
        .with_state(packets.clone());
    let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
    let (t, c) = tokio::task::spawn_blocking(move || {
        otlp::Config {
            endpoint,
            deployment: "test".into(),
            ..Default::default()
        }
        .start("test")
        .unwrap()
    })
    .await
    .unwrap();
    let mut root = t.span(
        Operation::Turn,
        Some("SECRET_SESSION"),
        Some("PROMPT_SECRET"),
    );
    root.turn("SECRET_TURN");
    let cx = root.context();
    scope_turn("SECRET_TURN", cx, async {
        let mut child = t.span(Operation::Model, Some("SECRET_SESSION"), None);
        child.reported_usage(TokenUsage {
            input: Some(42),
            output: Some(9),
            cache_read: Some(12),
            cache_write: Some(5),
            reasoning: Some(3),
        });
        child.purpose(ModelPurpose::Compaction);
        child.link(TraceContext {
            trace: [1; 16],
            span: [2; 8],
        });
        child.first_output();
        child.finish(Outcome::Error)
    })
    .await;
    root.finish(Outcome::Waiting);
    t.observe(Gauge::TurnsQueued, 7);
    t.observe(Gauge::StateObservedAt, 1234567890);
    assert!(
        tokio::task::spawn_blocking(move || c.shutdown(Duration::from_secs(5)))
            .await
            .unwrap()
    );
    let packets = packets.lock().unwrap();
    let mut trace_rows = vec![];
    let mut logs = 0;
    let mut metrics = vec![];
    for (path, data) in packets.iter() {
        assert!(!String::from_utf8_lossy(data).contains("SECRET"));
        match path.as_str() {
            "/v1/traces" => {
                let p=opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest::decode(data.clone()).unwrap();
                for r in p.resource_spans {
                    for s in r.scope_spans {
                        trace_rows.extend(s.spans)
                    }
                }
            }
            "/v1/logs" => {
                let p=opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest::decode(data.clone()).unwrap();
                for r in p.resource_logs {
                    for s in r.scope_logs {
                        logs += s.log_records.len()
                    }
                }
            }
            "/v1/metrics" => {
                let p=opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest::decode(data.clone()).unwrap();
                for r in p.resource_metrics {
                    assert!(format!("{:?}", r.resource).contains("openworker.deployment"));
                    for s in r.scope_metrics {
                        metrics.extend(s.metrics)
                    }
                }
            }
            _ => panic!(),
        }
    }
    assert_eq!(trace_rows.len(), 2);
    assert_eq!(logs, 2);
    let parent = trace_rows
        .iter()
        .find(|s| s.name == "agent.execute")
        .unwrap();
    let child = trace_rows
        .iter()
        .find(|s| s.name == "model.attempt")
        .unwrap();
    assert_eq!(child.parent_span_id, parent.span_id);
    assert_eq!(child.trace_id, parent.trace_id);
    assert_eq!(child.links.len(), 1);
    assert_eq!(child.links[0].span_id, vec![2; 8]);
    for (key, value) in [
        ("gen_ai.usage.input_tokens", 42),
        ("gen_ai.usage.cache_read.input_tokens", 12),
        ("gen_ai.usage.cache_creation.input_tokens", 5),
        ("gen_ai.usage.reasoning.output_tokens", 3),
    ] {
        assert!(child.attributes.iter().any(|a|a.key==key&&matches!(a.value.as_ref().and_then(|v|v.value.as_ref()),Some(opentelemetry_proto::tonic::common::v1::any_value::Value::IntValue(n)) if *n==value)));
    }
    assert!(child
        .attributes
        .iter()
        .any(|a| a.key == "openworker.turn.hash"));
    let m = format!("{metrics:?}");
    assert!(m.contains("openworker.model.tokens"));
    assert!(m.contains("openworker.operation.last_error_timestamp_seconds"));
    assert!(!m.contains("session.hash"));
    assert!(!m.contains("action.hash"));
    assert!(!m.contains("turn.hash"));
    let queued = metrics
        .iter()
        .find(|m| m.name == Gauge::TurnsQueued.name())
        .unwrap();
    let Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Gauge(g)) = &queued.data else {
        panic!("Expected gauge")
    };
    assert!(g.data_points.iter().all(|p| p.attributes.is_empty()));
    assert_eq!(
        g.data_points[0].value,
        Some(opentelemetry_proto::tonic::metrics::v1::number_data_point::Value::AsInt(7))
    );
    server.abort();
}
#[test]
fn deployment_configuration_is_backward_compatible_and_bounded() {
    let c: otlp::Config = serde_json::from_str(r#"{"endpoint":"http://127.0.0.1:4318"}"#).unwrap();
    assert_eq!(c.deployment, "default");
    for deployment in [
        "".into(),
        "x".repeat(65),
        "[email protected]".into(),
        "bad\nlabel".into(),
    ] {
        assert!(otlp::Config {
            endpoint: c.endpoint.clone(),
            deployment,
            ..Default::default()
        }
        .start("test")
        .is_err());
    }
}
#[tokio::test(flavor = "multi_thread")]
async fn collector_outage_and_bad_endpoints() {
    for endpoint in [
        "http://example.org",
        "https://user:[email protected]",
        "https://example.org?secret=x",
        "https://example.org/v1/traces",
    ] {
        let endpoint = endpoint.to_owned();
        assert!(tokio::task::spawn_blocking(move || otlp::Config {
            endpoint,
            deployment: "test".into(),
            ..Default::default()
        }
        .start("test"))
        .await
        .unwrap()
        .is_err());
    }
    let socket = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
    let endpoint = format!("http://{}", socket.local_addr().unwrap());
    drop(socket);
    let (t, c) = tokio::task::spawn_blocking(move || {
        otlp::Config {
            endpoint,
            deployment: "test".into(),
            ..Default::default()
        }
        .start("test")
        .unwrap()
    })
    .await
    .unwrap();
    let now = Instant::now();
    for _ in 0..10000 {
        t.event(Operation::Tool, Outcome::Ok, None, None)
    }
    assert!(now.elapsed() < Duration::from_secs(2));
    let until = Instant::now() + Duration::from_secs(5);
    while c.snapshot().export_failures == 0 && Instant::now() < until {
        tokio::time::sleep(Duration::from_millis(20)).await;
    }
    assert!(c.snapshot().export_failures > 0);
    let now = Instant::now();
    tokio::task::spawn_blocking(move || c.shutdown(Duration::from_secs(5)))
        .await
        .unwrap();
    assert!(now.elapsed() < Duration::from_secs(6));
}

type Seen = Arc<Mutex<Vec<(String, axum::http::HeaderMap, Bytes)>>>;
async fn receive_with_headers(
    State(p): State<Seen>,
    OriginalUri(uri): OriginalUri,
    headers: axum::http::HeaderMap,
    bytes: Bytes,
) -> &'static str {
    p.lock().unwrap().push((uri.path().into(), headers, bytes));
    ""
}

#[tokio::test(flavor = "multi_thread")]
async fn authenticated_base_path_traces_only_with_opt_in_content() {
    let seen = Seen::default();
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
    let endpoint = format!("http://{}/api/public/otel", listener.local_addr().unwrap());
    let app = Router::new()
        .route("/api/public/otel/v1/traces", post(receive_with_headers))
        .route("/api/public/otel/v1/logs", post(receive_with_headers))
        .route("/api/public/otel/v1/metrics", post(receive_with_headers))
        .with_state(seen.clone());
    let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
    std::env::set_var("OW_TEST_OTLP_AUTH", "Basic dGVzdDpzZWNyZXQ=");
    let config = otlp::Config {
        endpoint,
        deployment: "test".into(),
        headers: vec![
            otlp::Header::secret("Authorization", "OW_TEST_OTLP_AUTH"),
            otlp::Header::literal("x-langfuse-ingestion-version", "4"),
        ],
        signals: vec![otlp::Signal::Traces],
        acknowledgement: otlp::Acknowledgement::Otlp,
        content: ContentPolicy {
            capture: Capture::Full,
            max_bytes: 64,
            redact_secrets: true,
        },
    };
    assert!(
        !format!("{config:?}").contains("dGVzdDpzZWNyZXQ="),
        "config never holds the secret"
    );
    let (t, c) = tokio::task::spawn_blocking(move || config.start("test").unwrap())
        .await
        .unwrap();
    let mut span = t.span(Operation::Model, Some("session-1"), None);
    assert_eq!(span.capture(), Capture::Full);
    span.metadata("gen_ai.request.model", "gpt-test");
    span.content_tail("gen_ai.prompt", || {
        format!("{} use sk-abcdefghijklmnopqrstuvwxyz now", "x".repeat(200))
    });
    span.content("gen_ai.completion", || "done".into());
    span.finish(Outcome::Ok);
    assert!(
        tokio::task::spawn_blocking(move || c.shutdown(Duration::from_secs(5)))
            .await
            .unwrap()
    );
    server.abort();
    let seen = seen.lock().unwrap();
    assert!(!seen.is_empty());
    for (path, headers, body) in seen.iter() {
        assert_eq!(path, "/api/public/otel/v1/traces", "only selected signals");
        assert_eq!(headers["authorization"], "Basic dGVzdDpzZWNyZXQ=");
        assert_eq!(headers["accept"], "application/x-protobuf");
        assert_eq!(headers["x-langfuse-ingestion-version"], "4");
        let text = String::from_utf8_lossy(body);
        assert!(
            !text.contains("abcdefghijklmnopqrstuvwxyz"),
            "secret redacted"
        );
        assert!(text.contains("gen_ai.prompt") && text.contains("sk-[REDACTED] now"));
        assert!(text.contains("gpt-test") && text.contains("session.id"));
        assert!(!text.contains(&"x".repeat(100)), "kept only the tail");
    }
}

#[test]
fn content_off_by_default_and_header_rules_enforced() {
    let base = || otlp::Config {
        endpoint: "https://otlp.example.com".into(),
        ..Default::default()
    };
    assert_eq!(base().content.capture, Capture::Off);
    let cases = [
        vec![otlp::Header::secret(
            "Authorization",
            "OW_TEST_DEFINITELY_UNSET",
        )],
        vec![otlp::Header::literal("content-type", "text/plain")],
        vec![otlp::Header::literal("Authorization", "Basic fixture-only")],
        vec![otlp::Header::literal("proxy-authorization", "x")],
        vec![otlp::Header::secret("Authorization", "lower_case")],
        vec![otlp::Header {
            name: "x-both".into(),
            value: Some("a".into()),
            value_env: Some("B".into()),
        }],
        vec![otlp::Header::literal("bad header", "x")],
        vec![
            otlp::Header::literal("x-a", "1"),
            otlp::Header::literal("X-A", "2"),
        ],
    ];
    for headers in cases {
        let config = otlp::Config {
            headers: headers.clone(),
            ..base()
        };
        assert!(config.start("test").is_err(), "{headers:?}");
    }
    // Headers never travel over plaintext to a remote host.
    let plaintext = otlp::Config {
        endpoint: "http://otlp.example.com".into(),
        headers: vec![otlp::Header::literal("x-a", "1")],
        ..Default::default()
    };
    assert!(plaintext.start("test").is_err());
    let no_signals = otlp::Config {
        signals: vec![],
        ..base()
    };
    assert!(no_signals.start("test").is_err());
}

#[test]
fn redaction_masks_common_credentials() {
    let r = redact("key=sk-proj-abcdefghijklmnop1234 and ghp_ABCDEFGHIJKLMNOPQRSTUV ok"); // gitleaks:allow -- deliberately invalid synthetic redaction fixtures
    assert!(!r.contains("abcdefghijklmnop1234") && !r.contains("ABCDEFGHIJKLMNOPQRSTUV"));
    assert!(r.ends_with(" ok"));
    let r = redact("Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig done");
    assert_eq!(r, "Authorization: Bearer [REDACTED] done");
    let r = redact("a\n-----BEGIN PRIVATE KEY-----\nMIIE\n-----END PRIVATE KEY-----\nb");
    assert_eq!(r, "a\n[REDACTED PRIVATE KEY]\nb");
    assert_eq!(redact("ask-me about sk-short"), "ask-me about sk-short");
    assert_eq!(redact("naïve café"), "naïve café");
}

#[tokio::test(flavor = "multi_thread")]
async fn rejection_redirect_partial_malformed_and_large_acknowledgements_are_not_success_or_retried(
) {
    use axum::{
        body::Body,
        http::{Response, StatusCode},
    };
    use opentelemetry_proto::tonic::collector::trace::v1::{
        ExportTracePartialSuccess, ExportTraceServiceResponse,
    };
    use std::sync::atomic::{AtomicUsize, Ordering};
    for mode in [
        "unauthorized",
        "redirect",
        "partial",
        "warning",
        "malformed",
        "oversized",
        "json_ok",
        "json_partial",
        "json_malformed",
        "json_wrong_signal",
        "job_standard",
        "job_vendor",
        "job_invalid",
    ] {
        let calls = Arc::new(AtomicUsize::new(0));
        let seen = calls.clone();
        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
        let endpoint = format!("http://{}", listener.local_addr().unwrap());
        let app = Router::new().route(
            "/v1/traces",
            post(move || {
                let seen = seen.clone();
                async move {
                    seen.fetch_add(1, Ordering::SeqCst);
                    let (status, body) = match mode {
                        "unauthorized" => {
                            (StatusCode::UNAUTHORIZED, b"PRIVATE_COLLECTOR_ECHO".to_vec())
                        }
                        "redirect" => (StatusCode::TEMPORARY_REDIRECT, vec![]),
                        "partial" | "warning" => (
                            StatusCode::OK,
                            ExportTraceServiceResponse {
                                partial_success: Some(ExportTracePartialSuccess {
                                    rejected_spans: if mode == "partial" { 1 } else { 0 },
                                    error_message: "PRIVATE_COLLECTOR_ECHO".into(),
                                }),
                            }
                            .encode_to_vec(),
                        ),
                        "job_standard" | "job_vendor" | "job_invalid" => (StatusCode::OK, serde_json::to_vec(&serde_json::json!({"name":"otel-ingestion-job","id":"fixture","data":{"name":"otel-ingestion-job","payload":{"ingestionVersion":"4","authCheck":{"validKey":mode != "job_invalid"}}}})).unwrap()),
                        "json_ok" => (StatusCode::OK, b"{}".to_vec()),
                        "json_partial" => (StatusCode::OK, br#"{"partialSuccess":{"rejectedSpans":"1"}}"#.to_vec()),
                        "json_malformed" => (StatusCode::OK, b"not json".to_vec()),
                        "json_wrong_signal" => (StatusCode::OK, br#"{"partialSuccess":{"rejectedDataPoints":0}}"#.to_vec()),
                        "malformed" => (StatusCode::OK, vec![0xff]),
                        "oversized" => (StatusCode::OK, vec![0; 65537]),
                        _ => unreachable!(),
                    };
                    Response::builder()
                        .status(status)
                        .header("location", "http://127.0.0.1:1/not-followed")
                        .header("content-type", if mode.starts_with("json_") || mode.starts_with("job_") { "application/json; charset=utf-8" } else { "application/x-protobuf" })
                        .body(Body::from(body))
                        .unwrap()
                }
            }),
        );
        let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
        let (t, c) = tokio::task::spawn_blocking(move || {
            otlp::Config {
                endpoint,
                signals: vec![otlp::Signal::Traces],
                acknowledgement: if mode == "job_vendor" || mode == "job_invalid" {
                    otlp::Acknowledgement::LangfuseV4
                } else {
                    otlp::Acknowledgement::Otlp
                },
                ..Default::default()
            }
            .start("test")
            .unwrap()
        })
        .await
        .unwrap();
        t.event(
            Operation::Tool,
            Outcome::Unknown,
            Some("private-session"),
            None,
        );
        let c = tokio::task::spawn_blocking(move || {
            assert!(c.shutdown(Duration::from_secs(5)));
            c
        })
        .await
        .unwrap();
        assert_eq!(
            calls.load(Ordering::SeqCst),
            1,
            "{mode}: never replay a rejected/unknown export batch"
        );
        assert!(
            (c.snapshot().export_failures == 0) == (mode == "json_ok" || mode == "job_vendor"),
            "{mode}: report only complete acceptance: {:?}",
            c.snapshot()
        );
        server.abort();
    }
}
#[tokio::test(flavor = "multi_thread")]
async fn ambient_headers_resources_and_proxy_are_not_inherited() {
    struct Reset(Vec<(&'static str, Option<String>)>);
    impl Drop for Reset {
        fn drop(&mut self) {
            for (k, v) in &self.0 {
                if let Some(v) = v {
                    std::env::set_var(k, v)
                } else {
                    std::env::remove_var(k)
                }
            }
        }
    }
    let names = [
        (
            "OTEL_EXPORTER_OTLP_HEADERS",
            "x-ambient=PRIVATE_AMBIENT_SECRET",
        ),
        (
            "OTEL_EXPORTER_OTLP_TRACES_HEADERS",
            "x-signal=PRIVATE_SIGNAL_SECRET",
        ),
        ("OTEL_RESOURCE_ATTRIBUTES", "ambient.key=PRIVATE_RESOURCE"),
        ("HTTP_PROXY", "http://127.0.0.1:1"),
        ("HTTPS_PROXY", "http://127.0.0.1:1"),
    ];
    let _reset = Reset(
        names
            .iter()
            .map(|(k, v)| {
                let old = std::env::var(k).ok();
                std::env::set_var(k, v);
                (*k, old)
            })
            .collect(),
    );
    let seen = Seen::default();
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
    let endpoint = format!("http://{}", listener.local_addr().unwrap());
    let app = Router::new()
        .route("/v1/traces", post(receive_with_headers))
        .with_state(seen.clone());
    let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
    let (t, c) = tokio::task::spawn_blocking(move || {
        otlp::Config {
            endpoint,
            signals: vec![otlp::Signal::Traces],
            ..Default::default()
        }
        .start("test")
        .unwrap()
    })
    .await
    .unwrap();
    t.event(Operation::Model, Outcome::Ok, None, None);
    assert!(
        tokio::task::spawn_blocking(move || c.shutdown(Duration::from_secs(5)))
            .await
            .unwrap()
    );
    let rows = seen.lock().unwrap();
    assert_eq!(rows.len(), 1);
    for (_, headers, body) in rows.iter() {
        assert!(headers.get("x-ambient").is_none() && headers.get("x-signal").is_none());
        assert!(!String::from_utf8_lossy(body).contains("PRIVATE"));
    }
    server.abort();
}