Skip to content

Use Claude Code with your ChatGPT, Kimi, Cursor or Grok subscription via a local Anthropic-compatible proxy

Rust

keep streaming requests active through response body

Raine Virtab6cfcf4c4d75
diff --git a/src/monitor.rs b/src/monitor.rs
index b1a1a2d..8d37c63 100644
--- a/src/monitor.rs
+++ b/src/monitor.rs
@@ -819,6 +819,20 @@ mod tests {
assert_eq!(state.active[0].model.as_deref(), Some("gpt-5.6-sol"));
}
+ #[test]
+ fn active_stream_progress_reports_token_rate() {
+ let monitor = MonitorHandle::new(10);
+ monitor.request_started("r1", None, None, EndpointKind::Messages);
+ monitor.stream_progress("r1", 50, 1, Some(1_225), Some(141));
+
+ let state = monitor.snapshot();
+ assert_eq!(state.active.len(), 1);
+ assert!(matches!(
+ state.active[0].rate(),
+ Throughput::TokensPerSecond(_)
+ ));
+ }
+
#[test]
fn late_stream_usage_updates_completed_request() {
let monitor = MonitorHandle::new(10);
diff --git a/src/server.rs b/src/server.rs
index 07aa7b2..dce16f7 100644
--- a/src/server.rs
+++ b/src/server.rs
@@ -15,7 +15,7 @@ use axum::{
response::Response,
routing::{get, post},
};
-use http_body_util::BodyExt;
+use http_body_util::{BodyExt, StreamBody};
use serde::de::DeserializeOwned;
use serde_json::{Map, Value, json};
use std::fs::{self, File};
@@ -152,7 +152,7 @@ async fn dispatch_request(
if let Some(monitor) = state.monitor.as_ref() {
monitor.request_started(&req_id, session_id.clone(), None, endpoint);
}
- let _request_guard = RequestMonitorGuard::new(state.monitor.clone(), req_id.clone());
+ let request_guard = RequestMonitorGuard::new(state.monitor.clone(), req_id.clone());
let now = current_millis();
let body_bytes = match axum::body::to_bytes(req.into_body(), usize::MAX).await {
Ok(bytes) => bytes,
@@ -432,10 +432,7 @@ async fn dispatch_request(
);
let status = response.status();
if status.is_success() {
- if let Some(monitor) = state.monitor.as_ref() {
- monitor.request_completed(&req_id, status.as_u16(), None, None);
- }
- return response;
+ return monitor_response_body(response, request_guard);
}
let (response, details) = record_failed_response(
@@ -468,6 +465,26 @@ async fn dispatch_request(
response
}
+fn monitor_response_body(response: Response, guard: RequestMonitorGuard) -> Response {
+ let status = response.status();
+ let (parts, body) = response.into_parts();
+ let stream =
+ futures_util::stream::unfold((body, guard), move |(mut body, mut guard)| async move {
+ match body.frame().await {
+ Some(Ok(frame)) => Some((Ok(frame), (body, guard))),
+ Some(Err(err)) => {
+ guard.failed(status, err.to_string());
+ Some((Err(err), (body, guard)))
+ }
+ None => {
+ guard.completed(status);
+ None
+ }
+ }
+ });
+ Response::from_parts(parts, Body::new(StreamBody::new(stream)))
+}
+
struct RequestLogContext<'a> {
req_id: &'a str,
provider: Option<&'a str>,
@@ -669,6 +686,18 @@ impl RequestMonitorGuard {
fn new(monitor: Option<MonitorHandle>, req_id: String) -> Self {
Self { monitor, req_id }
}
+
+ fn completed(&mut self, status: StatusCode) {
+ if let Some(monitor) = self.monitor.take() {
+ monitor.request_completed(&self.req_id, status.as_u16(), None, None);
+ }
+ }
+
+ fn failed(&mut self, status: StatusCode, error: String) {
+ if let Some(monitor) = self.monitor.take() {
+ monitor.request_failed(&self.req_id, Some(status.as_u16()), error);
+ }
+ }
}
impl Drop for RequestMonitorGuard {
diff --git a/tests/server.rs b/tests/server.rs
index 9f7ef4b..dac0597 100644
--- a/tests/server.rs
+++ b/tests/server.rs
@@ -237,6 +237,13 @@ async fn monitor_records_successful_request_events() {
assert_eq!(response.status(), StatusCode::OK);
let state = monitor.snapshot();
+ assert_eq!(state.active.len(), 1);
+ assert!(state.recent.is_empty());
+
+ let _body = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let state = monitor.snapshot();
assert!(state.active.is_empty());
assert_eq!(state.recent.len(), 1);
assert_eq!(state.recent[0].status, RequestStatus::Completed);