Python
//! Explicit OpenTelemetry component injection. Components remain app-owned;
//! this sink flushes but never shuts down an application's providers.
use crate::{attributes as a, Operation, Outcome, Record, Sink, Snapshot, TokenUsage};
use opentelemetry::{
logs::{LogRecord, Logger, LoggerProvider, Severity},
metrics::{Counter, Gauge, Histogram, MeterProvider},
trace::{SpanContext, SpanId, SpanKind, Status, TraceFlags, TraceId, TraceState},
InstrumentationScope, KeyValue,
};
use opentelemetry_sdk::{
logs::SdkLoggerProvider,
metrics::SdkMeterProvider,
trace::{SpanData, SpanProcessor},
};
use std::sync::Arc;
pub struct OtelSink {
traces: Option<Arc<dyn SpanProcessor>>,
logs: SdkLoggerProvider,
metrics: SdkMeterProvider,
operations: Counter<u64>,
duration: Histogram<f64>,
tokens: Counter<u64>,
retries: Counter<u64>,
first: Histogram<f64>,
drops: Gauge<u64>,
failures: Gauge<u64>,
observations: Vec<Gauge<u64>>,
error_at: Gauge<u64>,
latest_errors: [u64; 2],
}
pub(crate) fn hex(bytes: &[u8]) -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
impl OtelSink {
pub fn new(
traces: Arc<dyn SpanProcessor>,
logs: SdkLoggerProvider,
metrics: SdkMeterProvider,
) -> Self {
Self::with_parts(Some(traces), logs, metrics)
}
/// Same as [`Self::new`] with traces optional (e.g. a metrics-only backend).
pub fn with_parts(
traces: Option<Arc<dyn SpanProcessor>>,
logs: SdkLoggerProvider,
metrics: SdkMeterProvider,
) -> Self {
let meter = metrics.meter("openworker.harness");
Self {
traces,
logs,
metrics,
error_at: meter
.u64_gauge("openworker.operation.last_error_timestamp_seconds")
.build(),
latest_errors: [0; 2],
observations: crate::Gauge::ALL
.iter()
.map(|g| meter.u64_gauge(g.name()).build())
.collect(),
operations: meter.u64_counter("openworker.operation.count").build(),
duration: meter
.f64_histogram("openworker.operation.duration")
.with_unit("s")
.build(),
tokens: meter.u64_counter("openworker.model.tokens").build(),
retries: meter.u64_counter("openworker.model.retry.count").build(),
first: meter.f64_histogram(a::FIRST_OUTPUT).with_unit("s").build(),
drops: meter.u64_gauge("openworker.telemetry.dropped").build(),
failures: meter
.u64_gauge("openworker.telemetry.export_failures")
.build(),
}
}
}
impl Sink for OtelSink {
fn observe(&mut self, samples: &[crate::Observation], stats: Snapshot) -> bool {
for s in samples {
self.observations[s.gauge as usize].record(s.value, &[]);
}
self.drops.record(stats.dropped, &[]);
self.failures.record(stats.export_failures, &[]);
self.metrics.force_flush().is_ok()
}
fn export(&mut self, records: &[Record], stats: Snapshot) -> bool {
let logger = self.logs.logger("openworker.harness");
for r in records {
let labels = [
KeyValue::new(a::OPERATION, r.operation.name()),
KeyValue::new(a::OUTCOME, r.outcome.name()),
];
self.operations.add(1, &labels);
// A timestamp also catches the first error of a short-lived process,
// for which Prometheus may never have seen a zero counter baseline.
if r.outcome == Outcome::Error {
let index = match r.operation {
crate::Operation::Model => Some(0),
crate::Operation::Tool => Some(1),
_ => None,
};
if let Some(i) = index {
self.latest_errors[i] = self.latest_errors[i].max(
r.ended
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs(),
);
self.error_at
.record(self.latest_errors[i], &[labels[0].clone()]);
}
}
if !matches!(
r.operation,
crate::Operation::ApprovalRequested
| crate::Operation::ModelStarted
| crate::Operation::ApprovalResolved
| crate::Operation::Cancellation
| crate::Operation::UnknownOutcome
| crate::Operation::BudgetExceeded
| crate::Operation::ToolValidationFailed
| crate::Operation::ProcessStarted
| crate::Operation::ProcessStopped
| crate::Operation::ProcessStale
| crate::Operation::QuestionRequested
| crate::Operation::QuestionResolved
) && r.timing_known
{
self.duration.record(r.elapsed.as_secs_f64(), &labels);
}
if r.operation == crate::Operation::Model && r.attempt.is_some_and(|n| n > 1) {
self.retries.add(1, &[]);
}
// Only completed provider requests own usage. Parent/container spans
// and immutable open-request markers never duplicate those counters.
let usage = TokenUsage {
input: r.input_tokens,
output: r.output_tokens,
cache_read: r.usage_details.cache_read,
cache_write: r.usage_details.cache_write,
reasoning: r.usage_details.reasoning,
};
let model = r.operation == Operation::Model;
let valid_usage = model && usage.validate();
for (kind, value) in [("input", usage.input), ("output", usage.output)] {
if !valid_usage {
continue;
}
if let Some(n) = value {
let mut dimensions = vec![KeyValue::new(a::DIRECTION, kind)];
if let Some(p) = r.purpose {
dimensions.push(KeyValue::new(a::PURPOSE, p.name()));
}
self.tokens.add(n, &dimensions);
}
}
if let Some(d) = r
.first_output
.filter(|d| model && r.timing_known && *d <= r.elapsed)
{
self.first.record(d.as_secs_f64(), &[]);
}
let mut attrs = labels.to_vec();
if let Some(s) = r.identity.session {
attrs.push(KeyValue::new(a::SESSION_HASH, hex(&s)));
}
if let Some(s) = r.identity.action {
attrs.push(KeyValue::new(a::ACTION_HASH, hex(&s)));
}
if let Some(n) = r.attempt {
attrs.push(KeyValue::new(a::ATTEMPT, i64::from(n)));
}
if let Some(t) = r.turn_hash {
attrs.push(KeyValue::new(a::TURN_HASH, hex(&t)));
}
if let Some(p) = r.purpose {
attrs.push(KeyValue::new(a::PURPOSE, p.name()));
}
attrs.push(KeyValue::new(a::TIMING_KNOWN, r.timing_known));
let trace_id = TraceId::from_bytes(r.context.trace);
let span_id = SpanId::from_bytes(r.context.span);
// Opt-in metadata/content goes on spans only, never on metric
// labels or the lifecycle log stream.
let mut span_attrs = attrs.clone();
for (key, value) in [
(
a::INPUT_TOKENS,
valid_usage.then_some(usage.input).flatten(),
),
(
a::OUTPUT_TOKENS,
valid_usage.then_some(usage.output).flatten(),
),
(
a::CACHE_READ_TOKENS,
valid_usage.then_some(usage.cache_read).flatten(),
),
(
a::CACHE_WRITE_TOKENS,
valid_usage.then_some(usage.cache_write).flatten(),
),
(
a::REASONING_TOKENS,
valid_usage.then_some(usage.reasoning).flatten(),
),
(
a::RETURNED_BYTES,
(r.operation == Operation::Tool)
.then_some(r.output_bytes)
.flatten(),
),
] {
if let Some(n) = value.and_then(|n| i64::try_from(n).ok()) {
span_attrs.push(KeyValue::new(key, n));
}
}
// Classification is derived from the operation, not a model name:
// content-off generations still map correctly in Langfuse, while
// model.requested remains an event even if it carries a model name.
span_attrs.push(KeyValue::new(
a::OBSERVATION_TYPE,
r.operation.observation_type(),
));
if let Some(operation) = r.operation.gen_ai_name() {
span_attrs.push(KeyValue::new(a::GEN_AI_OPERATION, operation));
}
if let Some(session) = r.identity.session {
span_attrs.push(KeyValue::new(a::SESSION_ID, hex(&session)));
span_attrs.push(KeyValue::new(a::CONVERSATION_ID, hex(&session)));
}
if let Some(d) = r
.first_output
.filter(|d| model && r.timing_known && *d <= r.elapsed)
{
span_attrs.push(KeyValue::new(a::FIRST_OUTPUT, d.as_secs_f64()));
}
if r.outcome == Outcome::Error {
span_attrs.push(KeyValue::new(a::ERROR_TYPE, "operation_failed"));
}
let mut seen = std::collections::BTreeSet::new();
span_attrs.extend(
r.attributes
.iter()
.take(12)
.filter(|(key, value)| crate::export_attribute(key, value) && seen.insert(*key))
.map(|(key, value)| KeyValue::new(*key, value.clone())),
);
// Cross-worker wall clocks can disagree. Preserve the unknown wait in
// recording/logs; do not export an invalid negative-duration span.
if let Some(traces) = self.traces.as_ref().filter(|_| r.timing_known) {
traces.on_end(SpanData {
span_context: SpanContext::new(
trace_id,
span_id,
TraceFlags::SAMPLED,
false,
TraceState::default(),
),
parent_span_id: r
.parent
.filter(|p| p.trace == r.context.trace)
.map(|p| SpanId::from_bytes(p.span))
.unwrap_or(SpanId::INVALID),
parent_span_is_remote: false,
span_kind: if model {
SpanKind::Client
} else {
SpanKind::Internal
},
name: r.operation.name().into(),
start_time: r.started,
end_time: r.ended,
attributes: span_attrs,
dropped_attributes_count: 0,
events: Default::default(),
links: {
let mut links = opentelemetry_sdk::trace::SpanLinks::default();
for c in &r.links {
links.links.push(opentelemetry::trace::Link::new(
SpanContext::new(
TraceId::from_bytes(c.trace),
SpanId::from_bytes(c.span),
TraceFlags::SAMPLED,
false,
TraceState::default(),
),
vec![],
0,
));
}
links
},
status: if r.outcome == Outcome::Error {
Status::error("operation_failed")
} else {
Status::Unset
},
instrumentation_scope: InstrumentationScope::builder("openworker.harness")
.with_version("0.1.0")
.build(),
});
}
// Structured lifecycle diagnostics, never raw runtime logs or stderr.
let mut log = logger.create_log_record();
log.set_event_name(r.operation.name());
log.set_timestamp(r.ended);
log.set_severity_number(if r.outcome == Outcome::Error {
Severity::Error
} else {
Severity::Info
});
log.set_trace_context(trace_id, span_id, Some(TraceFlags::SAMPLED));
for a in attrs {
match a.value {
opentelemetry::Value::String(s) => log.add_attribute(a.key, s.to_string()),
opentelemetry::Value::I64(n) => log.add_attribute(a.key, n),
opentelemetry::Value::Bool(b) => log.add_attribute(a.key, b),
_ => {}
}
}
logger.emit(log);
}
self.drops.record(stats.dropped, &[]);
self.failures.record(stats.export_failures, &[]);
// Only this dedicated telemetry thread waits for exporters.
let a = self.traces.as_ref().is_none_or(|t| t.force_flush().is_ok());
let b = self.logs.force_flush().is_ok();
let c = self.metrics.force_flush().is_ok();
a && b && c
}
}