Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

5.6 KB
Raw Blame
use crate::observation::ObservationSender;
use std::{
    future::Future,
    time::{SystemTime, UNIX_EPOCH},
};

use futures_util::TryStreamExt;

use crate::{call::CallOutput, interceptors::RawResponse};

/// Seconds since the Unix epoch, on one clock for every host.
pub fn epoch_seconds() -> f64 {
    SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .map(|duration| duration.as_secs_f64())
        .unwrap_or(0.0)
}

#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Timing {
    pub start_time: f64,
    pub end_time: f64,
}

/// Whether a failure surfaced inside the call, including a host op the call asked for,
/// or in a host step around it (preparing the arguments, finalizing the response).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum FailureOrigin {
    Call,
    Host,
}

#[derive(Clone, Debug, PartialEq)]
pub enum CallEvent<Response = (), Error = (), Raw = RawResponse> {
    Started {
        start_time: f64,
    },
    Execution(ExecutionEvent<Raw>),
    Succeeded {
        timing: Timing,
        response: Response,
    },
    Failed {
        timing: Timing,
        origin: FailureOrigin,
        error: Error,
    },
    Cancelled {
        timing: Timing,
    },
}

#[derive(Clone, Debug, PartialEq, Eq)]
pub enum ExecutionEvent<Raw = RawResponse> {
    ResultReady {
        facts: crate::interceptors::ExecutionFacts,
    },
    ProviderResponseReceived {
        raw: Raw,
    },
}

impl<Response, Error, Raw: std::borrow::Borrow<RawResponse>> CallEvent<Response, Error, Raw> {
    pub fn snapshot(&self) -> CallEvent {
        match self {
            Self::Started { start_time } => CallEvent::Started {
                start_time: *start_time,
            },
            Self::Execution(ExecutionEvent::ProviderResponseReceived { raw }) => {
                CallEvent::Execution(ExecutionEvent::ProviderResponseReceived {
                    raw: raw.borrow().clone(),
                })
            }
            Self::Execution(ExecutionEvent::ResultReady { facts }) => {
                CallEvent::Execution(ExecutionEvent::ResultReady {
                    facts: facts.clone(),
                })
            }
            Self::Succeeded { timing, .. } => CallEvent::Succeeded {
                timing: *timing,
                response: (),
            },
            Self::Failed { timing, origin, .. } => CallEvent::Failed {
                timing: *timing,
                origin: *origin,
                error: (),
            },
            Self::Cancelled { timing } => CallEvent::Cancelled { timing: *timing },
        }
    }
}

pub trait CallObserver: Send + Sync {
    fn observe(&self, event: CallEvent);
}

struct CallGuard {
    observers: Option<ObservationSender>,
    started_at: f64,
}

impl CallGuard {
    fn new(observers: ObservationSender) -> Self {
        let started_at = epoch_seconds();
        observers.emit(CallEvent::Started {
            start_time: started_at,
        });
        Self {
            observers: Some(observers),
            started_at,
        }
    }

    fn timing(&self) -> Timing {
        Timing {
            start_time: self.started_at,
            end_time: epoch_seconds(),
        }
    }

    fn finish(mut self, failed: bool) {
        if let Some(observers) = self.observers.take() {
            observers.emit(if failed {
                CallEvent::Failed {
                    timing: self.timing(),
                    origin: FailureOrigin::Call,
                    error: (),
                }
            } else {
                CallEvent::Succeeded {
                    timing: self.timing(),
                    response: (),
                }
            });
        }
    }
}

impl Drop for CallGuard {
    fn drop(&mut self) {
        if let Some(observers) = self.observers.take() {
            observers.emit(CallEvent::Cancelled {
                timing: self.timing(),
            });
        }
    }
}

pub async fn observe_call<R, H, C, E>(
    observers: Option<ObservationSender>,
    execute: impl Future<Output = Result<CallOutput<R, H, C, E>, E>>,
) -> Result<CallOutput<R, H, C, E>, E>
where
    C: Send + 'static,
    E: Send + 'static,
{
    let Some(observers) = observers else {
        return execute.await;
    };
    let guard = CallGuard::new(observers);
    match execute.await {
        Err(error) => {
            guard.finish(true);
            Err(error)
        }
        Ok(CallOutput::Complete(response)) => {
            guard.finish(false);
            Ok(CallOutput::Complete(response))
        }
        Ok(CallOutput::Stream { head, chunks }) => {
            let stream = futures_util::stream::try_unfold(
                (chunks, guard),
                |(mut chunks, guard)| async move {
                    match chunks.try_next().await {
                        Ok(Some(chunk)) => Ok(Some((chunk, (chunks, guard)))),
                        Ok(None) => {
                            guard.finish(false);
                            Ok(None)
                        }
                        Err(error) => {
                            guard.finish(true);
                            Err(error)
                        }
                    }
                },
            );
            Ok(CallOutput::Stream {
                head,
                chunks: Box::pin(stream),
            })
        }
    }
}

pub async fn observe_unary<R, E>(
    observers: Option<ObservationSender>,
    execute: impl Future<Output = Result<R, E>>,
) -> Result<R, E> {
    let Some(observers) = observers else {
        return execute.await;
    };
    let guard = CallGuard::new(observers);
    let result = execute.await;
    guard.finish(result.is_err());
    result
}