Skip to content
Python
20.7 KB
Raw Blame
//! Opt-in OTLP/HTTP. No inherited proxy, environment headers, resource
//! detector or redirect. Authentication headers are explicit, and their secret
//! values are read from named environment variables at startup, never from the
//! configuration file, URL or process arguments.
use crate::{otel::OtelSink, Controller, Telemetry};
use anyhow::{ensure, Context, Result};
use opentelemetry_http::{HttpClient, HttpError};
use opentelemetry_otlp::{Protocol, RetryPolicy, WithExportConfig, WithHttpConfig};
use opentelemetry_sdk::{
    logs::{BatchLogProcessor, SdkLoggerProvider},
    metrics::{PeriodicReader, SdkMeterProvider},
    trace::{BatchConfigBuilder, BatchSpanProcessor, SpanProcessor},
    Resource,
};
use std::{sync::Arc, time::Duration};

pub use crate::config::{Acknowledgement, Config, Header, Signal};
impl Header {
    fn resolve(&self) -> Result<(http::HeaderName, http::HeaderValue)> {
        let name = http::HeaderName::from_bytes(self.name.trim().to_ascii_lowercase().as_bytes())
            .context("Invalid telemetry header name")?;
        ensure!(
            !matches!(
                name.as_str(),
                "host"
                    | "accept"
                    | "content-type"
                    | "content-length"
                    | "content-encoding"
                    | "transfer-encoding"
                    | "connection"
                    | "keep-alive"
                    | "te"
                    | "trailer"
                    | "upgrade"
                    | "expect"
            ) && !name.as_str().starts_with("proxy-"),
            "Telemetry header {} is managed by the exporter",
            name
        );
        let raw = match (&self.value, &self.value_env) {
            (Some(v), None) => {
                ensure!(
                    !matches!(name.as_str(), "authorization" | "api-key" | "x-api-key"),
                    "Telemetry authentication must use value_env"
                );
                v.clone()
            }
            (None, Some(env)) => {
                ensure!(
                    !env.is_empty()
                        && env.len() <= 100
                        && env
                            .bytes()
                            .all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_'),
                    "Invalid telemetry credential reference"
                );
                std::env::var(env)
                    .with_context(|| format!("Telemetry credential variable {env} is not set"))?
            }
            _ => anyhow::bail!("Telemetry header {name} needs exactly one of value/value_env"),
        };
        ensure!(raw.len() <= 4096, "Telemetry header {name} value too long");
        let mut value =
            http::HeaderValue::from_str(&raw).context("Invalid telemetry header value")?;
        value.set_sensitive(self.value_env.is_some());
        Ok((name, value))
    }
}
#[derive(Clone)]
struct Client {
    acknowledgement: Acknowledgement,
    client: reqwest::blocking::Client,
    headers: Arc<Vec<(http::HeaderName, http::HeaderValue)>>,
}
impl std::fmt::Debug for Client {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        // Never print header values; they may be credentials.
        f.debug_struct("Client")
            .field(
                "headers",
                &self.headers.iter().map(|(n, _)| n).collect::<Vec<_>>(),
            )
            .finish()
    }
}
#[async_trait::async_trait]
impl HttpClient for Client {
    async fn send_bytes(
        &self,
        request: http::Request<bytes::Bytes>,
    ) -> Result<http::Response<bytes::Bytes>, HttpError> {
        // The SDK also reads OTEL_*_HEADERS. Deliberately don't copy those headers
        // to the wire. The explicitly configured collector is the only authority.
        let mut post = self
            .client
            .post(request.uri().to_string())
            .header("accept", "application/x-protobuf")
            .header("content-type", "application/x-protobuf");
        let signal = request
            .uri()
            .path()
            .rsplit('/')
            .next()
            .unwrap_or_default()
            .to_owned();
        for (name, value) in self.headers.iter() {
            post = post.header(name.clone(), value.clone());
        }
        let response = post.body(request.into_body()).send()?;
        let status = response.status();
        let json_ack = response
            .headers()
            .get("content-type")
            .and_then(|v| v.to_str().ok())
            .is_some_and(|v| {
                v.split(';')
                    .next()
                    .unwrap_or_default()
                    .trim()
                    .eq_ignore_ascii_case("application/json")
            });
        // OTLP success responses are tiny. Bound an adversarial collector body.
        use std::io::Read;
        let mut body = Vec::new();
        response.take(65537).read_to_end(&mut body)?;
        if body.len() > 65536 {
            return Err("Collector response exceeds limit".into());
        }
        if status.is_success() {
            if json_ack {
                if self.acknowledgement == Acknowledgement::LangfuseV4 {
                    langfuse_acknowledgement(&signal, &body)?;
                } else {
                    json_acknowledgement(&signal, &body)?;
                }
                // The official exporter is configured for binary requests. A
                // validated JSON response (Langfuse) must not be decoded as proto.
                body.clear();
            } else {
                acknowledgement(&signal, &body)?;
            }
        } else {
            // The SDK may log rejection bodies. Do not let an untrusted collector
            // echo payloads/credentials into diagnostics. Status is sufficient.
            body.clear();
        }
        Ok(http::Response::builder()
            .status(status)
            .body(bytes::Bytes::from(body))?)
    }
}
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct JsonAck {
    #[serde(default, rename = "partialSuccess", alias = "partial_success")]
    partial: Option<JsonPartial>,
}
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct JsonPartial {
    #[serde(default, rename = "rejectedSpans", alias = "rejected_spans")]
    spans: Option<JsonCount>,
    #[serde(default, rename = "rejectedDataPoints", alias = "rejected_data_points")]
    metrics: Option<JsonCount>,
    #[serde(default, rename = "rejectedLogRecords", alias = "rejected_log_records")]
    logs: Option<JsonCount>,
    #[serde(default, rename = "errorMessage", alias = "error_message")]
    message: Option<String>,
}
#[derive(serde::Deserialize)]
#[serde(untagged)]
enum JsonCount {
    Int(i64),
    Text(String),
}
fn langfuse_acknowledgement(signal: &str, body: &[u8]) -> Result<(), HttpError> {
    if json_acknowledgement(signal, body).is_ok() {
        return Ok(());
    }
    // Never reinterpret a rejected OTLP partial-success response as a job receipt.
    let value: serde_json::Value = serde_json::from_slice(body)
        .map_err(|_| -> HttpError { "Invalid Langfuse acknowledgement".into() })?;
    if value.get("partialSuccess").is_some() || value.get("partial_success").is_some() {
        return Err("Invalid Langfuse acknowledgement".into());
    }
    // Validate only receipt markers. Do not retain/log server queue metadata,
    // storage references, authentication scope or receiver-supplied error text.
    #[derive(serde::Deserialize)]
    struct Receipt {
        name: String,
        id: String,
        data: Job,
    }
    #[derive(serde::Deserialize)]
    struct Job {
        name: String,
        payload: Payload,
    }
    #[derive(serde::Deserialize)]
    #[serde(rename_all = "camelCase")]
    struct Payload {
        ingestion_version: String,
        auth_check: Auth,
    }
    #[derive(serde::Deserialize)]
    #[serde(rename_all = "camelCase")]
    struct Auth {
        valid_key: bool,
    }
    let receipt: Receipt = serde_json::from_slice(body)
        .map_err(|_| -> HttpError { "Invalid Langfuse acknowledgement".into() })?;
    if signal != "traces"
        || receipt.name != "otel-ingestion-job"
        || receipt.data.name != "otel-ingestion-job"
        || receipt.id.is_empty()
        || receipt.id.len() > 256
        || receipt.data.payload.ingestion_version != "4"
        || !receipt.data.payload.auth_check.valid_key
    {
        return Err("Invalid Langfuse acknowledgement".into());
    }
    Ok(())
}
fn json_acknowledgement(signal: &str, body: &[u8]) -> Result<(), HttpError> {
    let invalid = || -> HttpError { "Invalid collector acknowledgement".into() };
    let ack: JsonAck = serde_json::from_slice(body).map_err(|_| invalid())?;
    if let Some(p) = ack.partial {
        let value = match signal {
            "traces" if p.metrics.is_none() && p.logs.is_none() => p.spans,
            "metrics" if p.spans.is_none() && p.logs.is_none() => p.metrics,
            "logs" if p.spans.is_none() && p.metrics.is_none() => p.logs,
            _ => return Err(invalid()),
        };
        let count = match value {
            None => 0,
            Some(JsonCount::Int(n)) => n,
            Some(JsonCount::Text(n)) => {
                if n.is_empty() || n.len() > 19 || !n.bytes().all(|b| b.is_ascii_digit()) {
                    return Err(invalid());
                }
                n.parse::<i64>().map_err(|_| invalid())?
            }
        };
        if count < 0 {
            return Err(invalid());
        }
        if count > 0 || p.message.is_some_and(|m| !m.is_empty()) {
            return Err("Collector reported incomplete acceptance".into());
        }
    }
    Ok(())
}
fn acknowledgement(signal: &str, body: &[u8]) -> Result<(), HttpError> {
    use opentelemetry_proto::tonic::collector::{
        logs::v1::ExportLogsServiceResponse, metrics::v1::ExportMetricsServiceResponse,
        trace::v1::ExportTraceServiceResponse,
    };
    use prost::Message;
    let rejected = match signal {
        "traces" => ExportTraceServiceResponse::decode(body).map(|r| {
            r.partial_success
                .is_some_and(|p| p.rejected_spans != 0 || !p.error_message.is_empty())
        }),
        "metrics" => ExportMetricsServiceResponse::decode(body).map(|r| {
            r.partial_success
                .is_some_and(|p| p.rejected_data_points != 0 || !p.error_message.is_empty())
        }),
        "logs" => ExportLogsServiceResponse::decode(body).map(|r| {
            r.partial_success
                .is_some_and(|p| p.rejected_log_records != 0 || !p.error_message.is_empty())
        }),
        _ => return Err("Invalid telemetry signal".into()),
    }
    .map_err(|_| -> HttpError { "Invalid collector acknowledgement".into() })?;
    if rejected {
        return Err("Collector reported incomplete acceptance".into());
    }
    Ok(())
}
impl Config {
    pub fn start(&self, role: &'static str) -> Result<(Telemetry, Controller)> {
        Telemetry::start_with_content(self.build_sink(role)?, 2048, self.content.clone())
    }
    /// Build instance-owned export components without installing a dispatcher.
    /// Trusted hosts can compose an independent local recorder before export.
    pub fn build_sink(&self, role: &'static str) -> Result<OtelSink> {
        ensure!(
            !self.deployment.is_empty()
                && self.deployment.len() <= 64
                && self
                    .deployment
                    .bytes()
                    .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-'),
            "Invalid telemetry deployment identifier"
        );
        let url = reqwest::Url::parse(&self.endpoint).context("Invalid telemetry endpoint")?;
        ensure!(
            url.username().is_empty()
                && url.password().is_none()
                && url.query().is_none()
                && url.fragment().is_none(),
            "Telemetry endpoint must not contain credentials/query/fragment"
        );
        let path = url.path().trim_end_matches('/');
        ensure!(
            !["/v1/traces", "/v1/metrics", "/v1/logs"]
                .iter()
                .any(|signal| path.ends_with(signal)),
            "Telemetry endpoint is a base URL; signal paths are appended"
        );
        ensure!(
            !self.signals.is_empty() && self.signals.len() <= 3,
            "Select at least one telemetry signal"
        );
        ensure!(self.headers.len() <= 8, "At most 8 telemetry headers");
        self.content.validate()?;
        let loopback = url.host_str().is_some_and(|h| {
            h == "localhost"
                || h.trim_matches(['[', ']'])
                    .parse::<std::net::IpAddr>()
                    .is_ok_and(|ip| ip.is_loopback())
        });
        ensure!(
            url.scheme() == "https" || (url.scheme() == "http" && loopback),
            "Telemetry requires HTTPS except explicit loopback HTTP"
        );
        let endpoint = url.as_str().trim_end_matches('/');
        let mut headers = Vec::with_capacity(self.headers.len());
        for h in &self.headers {
            let (name, value) = h.resolve()?;
            ensure!(
                headers.iter().all(|(n, _)| *n != name),
                "Duplicate telemetry header {name}"
            );
            headers.push((name, value));
        }
        ensure!(
            self.acknowledgement != Acknowledgement::LangfuseV4 || self.signals == [Signal::Traces],
            "Langfuse v4 acknowledgements require traces only"
        );
        let client = Client {
            acknowledgement: self.acknowledgement,
            client: reqwest::blocking::Client::builder()
                .no_proxy()
                .redirect(reqwest::redirect::Policy::none())
                .timeout(Duration::from_secs(3))
                .connect_timeout(Duration::from_secs(3))
                .build()?,
            headers: Arc::new(headers),
        };
        let resource = Resource::builder_empty()
            .with_attributes([
                opentelemetry::KeyValue::new(crate::attributes::SERVICE_NAME, "openworker"),
                opentelemetry::KeyValue::new(
                    crate::attributes::SERVICE_VERSION,
                    env!("CARGO_PKG_VERSION"),
                ),
                opentelemetry::KeyValue::new(
                    crate::attributes::SCHEMA_VERSION,
                    crate::attributes::VERSION,
                ),
                opentelemetry::KeyValue::new(
                    crate::attributes::SEMCONV_VERSION,
                    crate::attributes::SEMCONV,
                ),
                opentelemetry::KeyValue::new(
                    "service.instance.id",
                    uuid::Uuid::new_v4().to_string(),
                ),
                opentelemetry::KeyValue::new("openworker.role", role),
                opentelemetry::KeyValue::new("openworker.deployment", self.deployment.clone()),
            ])
            .build();
        let enabled = |signal| self.signals.contains(&signal);
        let traces = if enabled(Signal::Traces) {
            Some(self.traces(endpoint, &client, &resource)?)
        } else {
            None
        };
        let logs = if enabled(Signal::Logs) {
            self.logs(endpoint, &client, &resource)?
        } else {
            SdkLoggerProvider::builder().build()
        };
        let metrics = if enabled(Signal::Metrics) {
            self.metrics(endpoint, &client, &resource)?
        } else {
            SdkMeterProvider::builder().build()
        };
        Ok(OtelSink::with_parts(traces, logs, metrics))
    }
    fn traces(
        &self,
        endpoint: &str,
        client: &Client,
        resource: &Resource,
    ) -> Result<Arc<dyn SpanProcessor>> {
        let traces = opentelemetry_otlp::SpanExporter::builder()
            .with_http()
            .with_endpoint(format!("{endpoint}/v1/traces"))
            .with_protocol(Protocol::HttpBinary)
            .with_timeout(Duration::from_secs(3))
            .with_retry_policy(RetryPolicy::default().with_max_retries(0))
            .with_http_client(client.clone())
            .build()?;
        let mut traces = BatchSpanProcessor::builder(traces)
            .with_batch_config(
                BatchConfigBuilder::default()
                    .with_max_queue_size(2048)
                    .with_max_export_batch_size(128)
                    .with_scheduled_delay(Duration::from_secs(1))
                    .build(),
            )
            .build();
        traces.set_resource(resource);
        Ok(Arc::new(traces))
    }
    fn logs(
        &self,
        endpoint: &str,
        client: &Client,
        resource: &Resource,
    ) -> Result<SdkLoggerProvider> {
        let logs = opentelemetry_otlp::LogExporter::builder()
            .with_http()
            .with_endpoint(format!("{endpoint}/v1/logs"))
            .with_protocol(Protocol::HttpBinary)
            .with_timeout(Duration::from_secs(3))
            .with_retry_policy(RetryPolicy::default().with_max_retries(0))
            .with_http_client(client.clone())
            .build()?;
        Ok(SdkLoggerProvider::builder()
            .with_resource(resource.clone())
            .with_log_processor(
                BatchLogProcessor::builder(logs)
                    .with_batch_config(
                        opentelemetry_sdk::logs::BatchConfigBuilder::default()
                            .with_max_queue_size(2048)
                            .with_max_export_batch_size(128)
                            .with_scheduled_delay(Duration::from_secs(1))
                            .build(),
                    )
                    .build(),
            )
            .build())
    }
    fn metrics(
        &self,
        endpoint: &str,
        client: &Client,
        resource: &Resource,
    ) -> Result<SdkMeterProvider> {
        let metrics = opentelemetry_otlp::MetricExporter::builder()
            .with_http()
            .with_endpoint(format!("{endpoint}/v1/metrics"))
            .with_protocol(Protocol::HttpBinary)
            .with_timeout(Duration::from_secs(3))
            .with_retry_policy(RetryPolicy::default().with_max_retries(0))
            .with_http_client(client.clone())
            .build()?;
        Ok(SdkMeterProvider::builder()
            .with_resource(resource.clone())
            .with_reader(
                PeriodicReader::builder(metrics)
                    .with_interval(Duration::from_secs(10))
                    .build(),
            )
            .build())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use prost::Message;
    #[test]
    fn acknowledgements_are_typed_bounded_and_never_echo_receiver_text() {
        use opentelemetry_proto::tonic::collector::trace::v1::{
            ExportTracePartialSuccess, ExportTraceServiceResponse,
        };
        for signal in ["traces", "metrics", "logs"] {
            assert!(acknowledgement(signal, &[]).is_ok());
            assert!(acknowledgement(signal, &[0xff]).is_err());
        }
        let rejected = ExportTraceServiceResponse {
            partial_success: Some(ExportTracePartialSuccess {
                rejected_spans: 1,
                error_message: "PRIVATE_COLLECTOR_ECHO".into(),
            }),
        }
        .encode_to_vec();
        let error = acknowledgement("traces", &rejected)
            .unwrap_err()
            .to_string();
        assert_eq!(error, "Collector reported incomplete acceptance");
        assert!(!error.contains("PRIVATE"));
        assert!(json_acknowledgement("traces", br#"{}"#).is_ok());
        assert!(json_acknowledgement(
            "metrics",
            br#"{"partialSuccess":{"rejectedDataPoints":"0"}}"#
        )
        .is_ok());
        let receipt = br#"{"name":"otel-ingestion-job","id":"fixture","data":{"name":"otel-ingestion-job","payload":{"ingestionVersion":"4","authCheck":{"validKey":true}}}}"#;
        assert!(langfuse_acknowledgement("traces", receipt).is_ok());
        assert!(json_acknowledgement("traces", receipt).is_err());
        assert!(langfuse_acknowledgement("logs", receipt).is_err());
        let mut hybrid: serde_json::Value = serde_json::from_slice(receipt).unwrap();
        hybrid["partialSuccess"] = serde_json::json!({"rejectedSpans":1});
        assert!(langfuse_acknowledgement("traces", &serde_json::to_vec(&hybrid).unwrap()).is_err());
        assert!(Config {
            endpoint: "http://127.0.0.1:4318".into(),
            acknowledgement: Acknowledgement::LangfuseV4,
            ..Default::default()
        }
        .build_sink("test")
        .is_err());
        for body in [
            br#"{"success":true}"#.as_slice(),
            br#"{"partialSuccess":{"rejectedSpans":-1}}"#,
            br#"{"partialSuccess":{"rejectedDataPoints":0}}"#,
            br#"{"partialSuccess":{"rejectedSpans":"1","errorMessage":"PRIVATE_ECHO"}}"#,
        ] {
            assert!(json_acknowledgement("traces", body).is_err());
        }
    }
}