Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): add native Bedrock converse-stream route (#917)

Yasser Sheikhb08ec15b0d39
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a3e44c9d3..e67e3d517 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Bug Fixes
* **ccr:** make retrieval store TTL configurable with `HEADROOM_CCR_TTL_SECONDS`, expose the effective TTL in `/v1/retrieve/stats`, and distinguish expired retrievals from missing hashes.
+* **proxy:** add native Bedrock `/model/{id}/converse-stream` route and forward it through the existing streaming EventStream/SSE pipeline.
## [0.25.0](https://github.com/chopratejas/headroom/compare/v0.24.0...v0.25.0) (2026-06-12)
diff --git a/crates/headroom-proxy/src/bedrock/eventstream_to_sse.rs b/crates/headroom-proxy/src/bedrock/eventstream_to_sse.rs
index d4e5c8f17..f7e3107e6 100644
--- a/crates/headroom-proxy/src/bedrock/eventstream_to_sse.rs
+++ b/crates/headroom-proxy/src/bedrock/eventstream_to_sse.rs
@@ -154,12 +154,18 @@ pub fn translate_message(
event_type: event_type.to_string(),
})
}
- (OutputMode::Sse, "chunk") => {
+ (OutputMode::Sse, "chunk")
+ | (OutputMode::Sse, "messageStart")
+ | (OutputMode::Sse, "contentBlockStart")
+ | (OutputMode::Sse, "contentBlockDelta")
+ | (OutputMode::Sse, "contentBlockStop")
+ | (OutputMode::Sse, "messageStop")
+ | (OutputMode::Sse, "metadata") => {
tracing::info!(
event = "bedrock_eventstream_translated_to_sse",
event_type = event_type,
payload_bytes = message.payload.len(),
- "translated bedrock eventstream chunk to sse frame"
+ "translated bedrock eventstream message to sse frame"
);
Ok(TranslateOutcome::Emit(payload_to_sse_frame(
&message.payload,
@@ -380,6 +386,28 @@ mod tests {
}
}
+ #[test]
+ fn translate_converse_event_to_sse_frame() {
+ let bytes = crate::bedrock::eventstream::MessageBuilder::new()
+ .header_string(":event-type", "contentBlockDelta")
+ .header_string(":message-type", "event")
+ .payload(Bytes::from_static(
+ br#"{"contentBlockIndex":0,"delta":{"text":"hi"}}"#,
+ ))
+ .build();
+ let msg = parse(&bytes).unwrap();
+ let outcome = translate_message(&msg, OutputMode::Sse).unwrap();
+ match outcome {
+ TranslateOutcome::Emit(b) => {
+ let s = std::str::from_utf8(&b).unwrap();
+ assert!(s.starts_with("data: "));
+ assert!(s.ends_with("\n\n"));
+ assert!(s.contains("contentBlockIndex"));
+ }
+ other => panic!("expected Emit; got {other:?}"),
+ }
+ }
+
#[test]
fn missing_event_type_is_loud() {
// A message lacking :event-type must not silently translate.
diff --git a/crates/headroom-proxy/src/bedrock/invoke_streaming.rs b/crates/headroom-proxy/src/bedrock/invoke_streaming.rs
index ea52bee58..347195f6e 100644
--- a/crates/headroom-proxy/src/bedrock/invoke_streaming.rs
+++ b/crates/headroom-proxy/src/bedrock/invoke_streaming.rs
@@ -80,8 +80,9 @@ const ANTHROPIC_VENDOR_PREFIX: &str = "anthropic.";
/// AWS Bedrock Runtime DNS template.
const BEDROCK_RUNTIME_HOST_TEMPLATE: &str = "bedrock-runtime.{region}.amazonaws.com";
-/// Path action for the streaming route.
+/// Path action for the streaming routes.
const STREAMING_ACTION: &str = "invoke-with-response-stream";
+const CONVERSE_STREAM_ACTION: &str = "converse-stream";
/// RAII guard that observes the `bedrock_invoke_latency_seconds`
/// histogram on drop. Mirrors the [`crate::bedrock::invoke`] guard
@@ -166,8 +167,26 @@ pub async fn handle_invoke_streaming(
body.clone()
};
- // 2. Build upstream URL.
- let upstream_url = match build_bedrock_streaming_upstream(&state, &model_id, &uri) {
+ // 2. Resolve the Bedrock streaming action from the inbound path and
+ // build the upstream URL.
+ let action = match extract_streaming_action(uri.path()) {
+ Some(a) => a,
+ None => {
+ tracing::error!(
+ event = "bedrock_streaming_action_invalid",
+ request_id = %request_id,
+ path = %uri.path(),
+ "bedrock invoke-streaming: unrecognized streaming action path"
+ );
+ return error_response(
+ StatusCode::BAD_REQUEST,
+ "bedrock_streaming_action_invalid",
+ "Unsupported Bedrock streaming action path",
+ );
+ }
+ };
+
+ let upstream_url = match build_bedrock_streaming_upstream(&state, &model_id, &uri, action) {
Ok(u) => u,
Err(msg) => {
tracing::error!(
@@ -888,6 +907,7 @@ fn build_bedrock_streaming_upstream(
state: &AppState,
model_id: &str,
uri: &Uri,
+ action: &str,
) -> Result<Url, String> {
let base = match state.config.bedrock_endpoint.as_ref() {
Some(u) => u.clone(),
@@ -901,7 +921,7 @@ fn build_bedrock_streaming_upstream(
let path = format!(
"/model/{model_id}/{action}",
model_id = model_id,
- action = STREAMING_ACTION,
+ action = action,
);
let mut joined = base;
joined.set_path(&path);
@@ -911,6 +931,16 @@ fn build_bedrock_streaming_upstream(
Ok(joined)
}
+fn extract_streaming_action(path: &str) -> Option<&'static str> {
+ if path.ends_with(&format!("/{STREAMING_ACTION}")) {
+ Some(STREAMING_ACTION)
+ } else if path.ends_with(&format!("/{CONVERSE_STREAM_ACTION}")) {
+ Some(CONVERSE_STREAM_ACTION)
+ } else {
+ None
+ }
+}
+
fn collect_signed_headers(headers: &HeaderMap, upstream_url: &Url) -> Vec<(String, String)> {
let mut out: Vec<(String, String)> = Vec::with_capacity(headers.len() + 1);
for (name, value) in headers.iter() {
@@ -996,6 +1026,7 @@ mod tests {
&state,
"anthropic.claude-3-haiku-20240307-v1:0",
&uri,
+ STREAMING_ACTION,
)
.unwrap();
assert_eq!(
@@ -1012,4 +1043,54 @@ mod tests {
assert!(s.ends_with("\n\n"));
assert!(s.contains("bedrock_eventstream_crc_mismatch"));
}
+
+ #[test]
+ fn build_streaming_upstream_supports_converse_stream_action() {
+ use crate::config::Config;
+ let mut config = Config::for_test(Url::parse("http://up:8080").unwrap());
+ config.bedrock_region = "eu-west-1".to_string();
+ let state = AppState {
+ config: std::sync::Arc::new(config),
+ client: reqwest::Client::new(),
+ bedrock_credentials: None,
+ drift_state: crate::cache_stabilization::drift_detector::DriftState::new(8),
+ vertex_token_source: std::sync::Arc::new(crate::vertex::StaticTokenSource::new(
+ "test".to_string(),
+ )),
+ };
+ let uri: Uri = "/model/anthropic.claude-3-haiku-20240307-v1:0/converse-stream"
+ .parse()
+ .unwrap();
+ let url = build_bedrock_streaming_upstream(
+ &state,
+ "anthropic.claude-3-haiku-20240307-v1:0",
+ &uri,
+ CONVERSE_STREAM_ACTION,
+ )
+ .unwrap();
+ assert_eq!(
+ url.as_str(),
+ "https://bedrock-runtime.eu-west-1.amazonaws.com/model/anthropic.claude-3-haiku-20240307-v1:0/converse-stream"
+ );
+ }
+
+ #[test]
+ fn extract_streaming_action_supports_both_bedrock_paths() {
+ assert_eq!(
+ extract_streaming_action(
+ "/model/anthropic.claude-3-haiku-20240307-v1:0/invoke-with-response-stream"
+ ),
+ Some(STREAMING_ACTION)
+ );
+ assert_eq!(
+ extract_streaming_action(
+ "/model/anthropic.claude-3-haiku-20240307-v1:0/converse-stream"
+ ),
+ Some(CONVERSE_STREAM_ACTION)
+ );
+ assert_eq!(
+ extract_streaming_action("/model/anthropic.claude-3-haiku-20240307-v1:0/invoke"),
+ None
+ );
+ }
}
diff --git a/crates/headroom-proxy/src/proxy.rs b/crates/headroom-proxy/src/proxy.rs
index d3fe821cc..89820d6cc 100644
--- a/crates/headroom-proxy/src/proxy.rs
+++ b/crates/headroom-proxy/src/proxy.rs
@@ -209,15 +209,22 @@ pub fn build_app(state: AppState) -> Router {
"/model/:model_id/converse",
post(crate::bedrock::invoke::handle_invoke),
)
- // PR-D2: streaming counterpart. Bedrock's protocol is
+ // PR-D2/PR-D5: streaming counterparts. Bedrock's protocol is
// binary EventStream; the handler parses incrementally,
// optionally translates each chunk to an SSE frame, and
// tees translated frames into AnthropicStreamState for
- // telemetry. See `bedrock::invoke_streaming`.
+ // telemetry. `invoke-with-response-stream` and
+ // `converse-stream` share the same wire framing and
+ // processing pipeline, so both route to the same handler.
+ // See `bedrock::invoke_streaming`.
.route(
"/model/:model_id/invoke-with-response-stream",
post(crate::bedrock::invoke_streaming::handle_invoke_streaming),
)
+ .route(
+ "/model/:model_id/converse-stream",
+ post(crate::bedrock::invoke_streaming::handle_invoke_streaming),
+ )
.route_layer(axum::middleware::from_fn(
crate::bedrock::classify_and_attach_auth_mode,
))
diff --git a/crates/headroom-proxy/tests/integration_bedrock_streaming.rs b/crates/headroom-proxy/tests/integration_bedrock_streaming.rs
index 41ade8b40..5bb4ba099 100644
--- a/crates/headroom-proxy/tests/integration_bedrock_streaming.rs
+++ b/crates/headroom-proxy/tests/integration_bedrock_streaming.rs
@@ -218,6 +218,18 @@ async fn mount_eventstream_upstream(upstream: &MockServer, body: Bytes) {
.await;
}
+async fn mount_eventstream_upstream_for_action(upstream: &MockServer, body: Bytes, action: &str) {
+ Mock::given(method("POST"))
+ .and(path(format!("/model/{TEST_MODEL}/{action}")))
+ .respond_with(
+ ResponseTemplate::new(200)
+ .insert_header("content-type", "application/vnd.amazon.eventstream")
+ .set_body_bytes(body.to_vec()),
+ )
+ .mount(upstream)
+ .await;
+}
+
#[tokio::test]
async fn eventstream_translated_to_sse() {
let _ = tracing_subscriber::fmt()
@@ -465,6 +477,53 @@ async fn client_can_choose_eventstream_or_sse() {
proxy.shutdown().await;
}
+#[tokio::test]
+async fn converse_stream_route_translates_to_sse() {
+ let upstream = MockServer::start().await;
+ let bedrock_bytes = synthesize_bedrock_stream();
+ mount_eventstream_upstream_for_action(&upstream, bedrock_bytes, "converse-stream").await;
+ let proxy = bedrock_proxy(&upstream, |c| {
+ c.compression_mode = headroom_proxy::config::CompressionMode::Off;
+ })
+ .await;
+
+ let body = serde_json::to_vec(&json!({
+ "anthropic_version": "bedrock-2023-05-31",
+ "max_tokens": 16,
+ "messages": [{"role":"user","content":"hi"}]
+ }))
+ .unwrap();
+
+ let resp = reqwest::Client::new()
+ .post(format!(
+ "{}/model/{TEST_MODEL}/converse-stream",
+ proxy.url()
+ ))
+ .header("content-type", "application/json")
+ .header("accept", "text/event-stream")
+ .body(body)
+ .send()
+ .await
+ .unwrap();
+
+ assert_eq!(resp.status(), 200);
+ let ct = resp
+ .headers()
+ .get("content-type")
+ .and_then(|v| v.to_str().ok())
+ .unwrap_or("");
+ assert!(
+ ct.starts_with("text/event-stream"),
+ "converse-stream should emit SSE when client asks for SSE; got {ct}"
+ );
+
+ let text = resp.text().await.unwrap();
+ assert!(text.contains("event: content_block_delta"));
+ assert!(text.contains("\"text\":\"OK\""));
+
+ proxy.shutdown().await;
+}
+
// ─── Test 5: Property test — never panic on adversarial bytes ─────
proptest! {