Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

6.8 KB
Raw Blame
use std::sync::Arc;

use opentelemetry::trace::TraceContextExt;
use tokio::{
    sync::{Semaphore, mpsc},
    task::JoinSet,
    time::Instant,
};
use tracing::Instrument;
use tracing_opentelemetry::OpenTelemetrySpanExt;

use super::{
    Grant, Stats,
    batch::{Batches, Flush, Pending},
    otlp::{ExportError, Payload, Uploader},
    retry::{RetryPolicy, random},
};

pub(super) enum Message {
    Record(Grant, Pending),
    Shutdown(Instant),
}

pub(super) async fn run_worker(
    mut queue: mpsc::Receiver<Message>,
    mut batches: Batches,
    mut uploads: Uploads,
) {
    let deadline = loop {
        let due = batches.next_due();
        tokio::select! {
            message = queue.recv() => match message {
                Some(Message::Record(grant, item)) => {
                    uploads.start_flushes(batches.push_record(grant, item, Instant::now()));
                }
                Some(Message::Shutdown(deadline)) => break deadline,
                None => break Instant::now(),
            },
            () = sleep_until_due(due) => uploads.start_flushes(batches.take_due(Instant::now())),
        }
    };
    queue.close();
    while let Some(message) = queue.recv().await {
        if let Message::Record(grant, item) = message {
            uploads.start_flushes(batches.push_record(grant, item, Instant::now()));
        }
    }
    uploads.start_flushes(batches.take_all());
    uploads.drain_until(deadline).await;
}

async fn sleep_until_due(due: Option<Instant>) {
    match due {
        Some(due) => tokio::time::sleep_until(due).await,
        None => std::future::pending().await,
    }
}

pub(super) struct Uploads {
    uploader: Arc<Uploader>,
    slots: Arc<Semaphore>,
    retry: RetryPolicy,
    tasks: JoinSet<()>,
    stats: Arc<Stats>,
}

impl Uploads {
    pub(super) fn new(
        uploader: Uploader,
        slots: usize,
        retry: RetryPolicy,
        stats: Arc<Stats>,
    ) -> Self {
        Self {
            uploader: Arc::new(uploader),
            slots: Arc::new(Semaphore::new(slots)),
            retry,
            tasks: JoinSet::new(),
            stats,
        }
    }

    fn start_flushes(&mut self, flushes: Vec<Flush>) {
        for flush in flushes {
            self.start_flush(flush);
        }
    }

    fn start_flush(&mut self, flush: Flush) {
        while self.tasks.try_join_next().is_some() {}
        let receipt = Receipt::new(flush.items.len(), self.stats.clone());
        let span = tracing::info_span!(
            parent: None,
            "telemetry.batch",
            otel.kind = "internal",
            gateway.telemetry.records = i64::try_from(flush.items.len()).unwrap_or(i64::MAX),
            gateway.telemetry.attempts = tracing::field::Empty,
            otel.status_code = tracing::field::Empty,
            error.type = tracing::field::Empty
        );
        for item in &flush.items {
            if item.link.is_valid() {
                span.add_link(item.link.clone());
            }
        }
        let uploader = self.uploader.clone();
        let slots = self.slots.clone();
        let retry = self.retry;
        self.tasks.spawn(
            async move {
                let mut flush = flush;
                let payload = match encode(&mut flush.items).await {
                    Ok(payload) => payload,
                    Err(error) => return receipt.settle_failed(error.reason()),
                };
                let mut attempt = 1;
                let outcome = loop {
                    let Ok(slot) = slots.acquire().await else {
                        return;
                    };
                    let result = uploader.export(&flush.grant, &payload).await;
                    drop(slot);
                    let Err(error) = result else { break Ok(()) };
                    let Some(delay) =
                        retry.backoff(attempt, &error, flush.grant.remaining(), random())
                    else {
                        break Err(error);
                    };
                    tracing::debug!(
                        attempt,
                        reason = error.reason(),
                        delay_ms = u64::try_from(delay.as_millis()).unwrap_or(u64::MAX),
                        "gateway telemetry upload retrying"
                    );
                    tokio::time::sleep(delay).await;
                    attempt += 1;
                };
                tracing::Span::current().record("gateway.telemetry.attempts", i64::from(attempt));
                match outcome {
                    Ok(()) => receipt.settle_accepted(),
                    Err(error) => receipt.settle_failed(error.reason()),
                }
            }
            .instrument(span),
        );
    }

    async fn drain_until(mut self, deadline: Instant) {
        loop {
            match tokio::time::timeout_at(deadline, self.tasks.join_next()).await {
                Ok(Some(_)) => {}
                Ok(None) => break,
                Err(_) => {
                    self.tasks.abort_all();
                    while self.tasks.join_next().await.is_some() {}
                    break;
                }
            }
        }
    }
}

async fn encode(items: &mut [Pending]) -> Result<Payload, ExportError> {
    let spans: Vec<_> = items.iter_mut().map(|item| item.span.take()).collect();
    tokio::task::spawn_blocking(move || Payload::encode(&spans))
        .await
        .unwrap_or(Err(ExportError::Payload))
}

struct Receipt {
    records: u64,
    stats: Arc<Stats>,
    settled: bool,
}

impl Receipt {
    fn new(records: usize, stats: Arc<Stats>) -> Self {
        Self {
            records: u64::try_from(records).unwrap_or(u64::MAX),
            stats,
            settled: false,
        }
    }

    fn settle_accepted(mut self) {
        self.settled = true;
        self.stats.record_accepted(self.records);
        tracing::debug!(records = self.records, "gateway telemetry accepted");
    }

    fn settle_failed(mut self, reason: &'static str) {
        self.settled = true;
        let current = tracing::Span::current();
        current.record("otel.status_code", "ERROR");
        current.record("error.type", reason);
        if self.stats.record_failed(self.records, reason) {
            let context = tracing::Span::current().context();
            let span = context.span();
            let context = span.span_context();
            tracing::warn!(
                trace_id = %context.trace_id(),
                span_id = %context.span_id(),
                reason,
                records = self.records,
                failed = self.stats.failed.load(std::sync::atomic::Ordering::Relaxed),
                "gateway telemetry upload failed"
            );
        }
    }
}

impl Drop for Receipt {
    fn drop(&mut self) {
        if !self.settled {
            self.stats.record_dropped(self.records, "shutdown");
        }
    }
}