Lightweight coding agent that runs in your terminal
Rust
Add TLS forwarding tests for MITM SigV4 requests (#56301)
viyatb-oaidc4844c907f8
diff --git a/codex-rs/network-proxy/src/certs.rs b/codex-rs/network-proxy/src/certs.rsindex fb3a7f56a7..1248a11052 100644--- a/codex-rs/network-proxy/src/certs.rs+++ b/codex-rs/network-proxy/src/certs.rs@@ -66,6 +66,11 @@ impl ManagedMitmCa {Ok(ca)}+ #[cfg(test)]+ pub(super) fn for_test(proxy_dir: &Path) -> Result<Arc<Self>> {+ Ok(Arc::new(Self::create(proxy_dir)?))+ }+fn create(proxy_dir: &Path) -> Result<Self> {fs::create_dir_all(proxy_dir).with_context(|| format!("failed to create {}", proxy_dir.display()))?;diff --git a/codex-rs/network-proxy/src/mitm.rs b/codex-rs/network-proxy/src/mitm.rsindex 7c21a85a56..a90045a875 100644--- a/codex-rs/network-proxy/src/mitm.rs+++ b/codex-rs/network-proxy/src/mitm.rs@@ -715,3 +715,7 @@ fn path_for_log(uri: &Uri) -> String {#[cfg(test)]#[path = "mitm_tests.rs"]mod tests;++#[cfg(test)]+#[path = "mitm_forwarding_tests.rs"]+mod forwarding_tests;diff --git a/codex-rs/network-proxy/src/mitm_forwarding_tests.rs b/codex-rs/network-proxy/src/mitm_forwarding_tests.rsnew file mode 100644index 0000000000..f52d792556--- /dev/null+++ b/codex-rs/network-proxy/src/mitm_forwarding_tests.rs@@ -0,0 +1,452 @@+//! Exercise authorized MITM forwarding over TLS, including transport-owned request bodies.++use super::*;+use crate::aws_sigv4::AwsSigV4Config;+use crate::aws_sigv4::ProxySecretSource;+use crate::aws_sigv4::tests::guest_authorization;+use crate::config::NetworkProxyConfig;+use crate::mitm_hook::MitmHookActionsConfig;+use crate::mitm_hook::MitmHookConfig;+use crate::mitm_hook::MitmHookMatchConfig;+use crate::runtime::network_proxy_state_for_policy;+use pretty_assertions::assert_eq;+use rama_tls_rustls::dep::pki_types::PrivateKeyDer;+use rama_tls_rustls::dep::pki_types::pem::PemObject;+use rama_tls_rustls::dep::rcgen::CertificateParams;+use rama_tls_rustls::dep::rcgen::KeyPair;+use rama_tls_rustls::dep::tokio_rustls::TlsAcceptor;+use sha2::Digest as _;+use std::collections::HashMap;+use std::path::Path;+use std::time::Duration;+use tempfile::tempdir;+use tokio::io::AsyncRead;+use tokio::io::AsyncReadExt;+use tokio::io::AsyncWriteExt;+use tokio::net::TcpListener;+use tokio::sync::mpsc;+use tokio::sync::watch;+use tokio::task::JoinSet;+use tokio::time::timeout;++const FORWARD_TIMEOUT: Duration = Duration::from_secs(15);+const RESPONSE: &[u8] = b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n";++async fn tls_origin(+ dir: &Path,+ service: &str,+ alpn: &[u8],+) -> (MitmRequestContext, TcpListener, TlsAcceptor) {+ ensure_rustls_crypto_provider();+ let key = KeyPair::generate().expect("generate origin key");+ let cert = CertificateParams::new(vec!["localhost".to_string()])+ .expect("origin certificate parameters")+ .self_signed(&key)+ .expect("origin certificate");+ let mut roots = rustls::RootCertStore::empty();+ roots+ .add(cert.der().clone())+ .expect("trust origin certificate");+ let key =+ PrivateKeyDer::from_pem_slice(key.serialize_pem().as_bytes()).expect("parse origin key");+ let mut server_config =+ rustls::ServerConfig::builder_with_protocol_versions(rustls::ALL_VERSIONS)+ .with_no_client_auth()+ .with_single_cert(vec![cert.der().clone()], key)+ .expect("origin TLS configuration");+ server_config.alpn_protocols = vec![alpn.to_vec()];+ let listener = TcpListener::bind("127.0.0.1:0")+ .await+ .expect("bind TLS origin");+ let port = listener.local_addr().expect("origin address").port();++ let access_key = dir.join("access-key");+ let secret_key = dir.join("secret-key");+ std::fs::write(&access_key, "AKIDEXAMPLE").expect("write access key");+ std::fs::write(&secret_key, "test-secret-key").expect("write secret key");+ let mut config = NetworkProxyConfig {+ mitm: true,+ mode: NetworkMode::Full,+ allow_local_binding: Some(true),+ mitm_hooks: vec![MitmHookConfig {+ host: "localhost".to_string(),+ matcher: MitmHookMatchConfig {+ schemes: vec!["https".to_string()],+ ports: vec![port],+ methods: vec!["GET".to_string(), "POST".to_string()],+ path_prefixes: vec!["/".to_string()],+ ..Default::default()+ },+ actions: MitmHookActionsConfig {+ strip_request_headers: vec!["x-amz-trailer".to_string()],+ aws_sigv4: Some(AwsSigV4Config {+ placeholder_access_key_id: "MASKEDACCESSKEY".to_string(),+ region: "us-east-1".to_string(),+ service: service.to_string(),+ access_key_id: ProxySecretSource::File(access_key.display().to_string()),+ secret_access_key: ProxySecretSource::File(secret_key.display().to_string()),+ session_token: None,+ port,+ max_body_bytes: 1024,+ }),+ ..Default::default()+ },+ }],+ ..Default::default()+ };+ config.set_allowed_domains(vec!["localhost".to_string()]);+ let app_state = Arc::new(network_proxy_state_for_policy(config));+ let roots = Arc::new(roots);+ let upstream =+ UpstreamClient::direct_with_tls_root_store(Arc::clone(&app_state), Arc::clone(&roots));+ let mitm = Arc::new(MitmState {+ ca: ManagedMitmCa::for_test(dir).expect("create test MITM CA"),+ allow_upstream_proxy: false,+ upstream_tls_root_store: roots,+ inspect: false,+ max_body_bytes: MITM_MAX_BODY_BYTES,+ });+ let context = MitmRequestContext {+ policy: MitmPolicyContext {+ target_host: "localhost".to_string(),+ target_port: port,+ scheme: Scheme::HTTPS,+ mode: NetworkMode::Full,+ app_state,+ },+ mitm,+ upstream,+ };+ (+ context,+ listener,+ TlsAcceptor::from(Arc::new(server_config)),+ )+}++struct ReceivedRequest {+ path: String,+ headers: HashMap<String, String>,+ body: Vec<u8>,+}++async fn read_request(stream: &mut (impl AsyncRead + Unpin)) -> ReceivedRequest {+ let mut head = Vec::new();+ while !head.ends_with(b"\r\n\r\n") {+ assert!(head.len() < 16 * 1024, "bounded origin request headers");+ head.push(stream.read_u8().await.expect("read request headers"));+ }+ let head = String::from_utf8(head).expect("ASCII request headers");+ let mut lines = head.split("\r\n");+ let path = lines+ .next()+ .expect("request line")+ .split_whitespace()+ .nth(1)+ .expect("request target")+ .to_string();+ let headers = lines+ .filter(|line| !line.is_empty())+ .map(|line| {+ let (name, value) = line.split_once(':').expect("header separator");+ (name.to_ascii_lowercase(), value.trim().to_string())+ })+ .collect::<HashMap<_, _>>();+ assert!(!headers.contains_key("transfer-encoding"));+ let length = headers+ .get("content-length")+ .map_or(0, |length| length.parse::<usize>().expect("content length"));+ assert!(length <= 1024, "bounded origin request body");+ let mut body = vec![0; length];+ stream+ .read_exact(&mut body)+ .await+ .expect("drain request body");+ ReceivedRequest {+ path,+ headers,+ body,+ }+}++fn request(+ context: &MitmRequestContext,+ service: &str,+ method: &str,+ path: &str,+ body: &[u8],+) -> Request {+ Request::builder()+ .method(method)+ .uri(path)+ .header(HOST, format!("localhost:{}", context.policy.target_port))+ .header(+ "authorization",+ guest_authorization(service, "host;x-amz-content-sha256;x-amz-date"),+ )+ .header("x-amz-date", "20150830T123600Z")+ .header("x-amz-content-sha256", "guest-hash")+ .body(Body::from(body.to_vec()))+ .expect("inner MITM request")+}++#[tokio::test]+async fn forwards_aoss_payload_hashes_and_signatures_over_tls() {+ let dir = tempdir().expect("TLS fixture directory");+ let (context, listener, acceptor) = tls_origin(dir.path(), "aoss", b"http/1.1").await;+ let cases: [(&str, &[u8]); 4] = [+ ("/unsigned", b""),+ ("/other-key", b""),+ ("/get", b""),+ ("/post", b"{\"message\":\"hello\"}"),+ ];+ let other_key =+ guest_authorization("aoss", "host;x-amz-date").replace("MASKEDACCESSKEY", "OTHERKEY");+ let expected_other = other_key.clone();+ let server = tokio::spawn(async move {+ for (path, body) in cases {+ let (stream, _) = listener.accept().await.expect("accept origin connection");+ let mut stream = acceptor.accept(stream).await.expect("origin TLS handshake");+ let received = read_request(&mut stream).await;+ assert_eq!(received.path, path);+ assert_eq!(received.body, body);+ match path {+ "/unsigned" => assert!(!received.headers.contains_key("authorization")),+ "/other-key" => assert_eq!(received.headers["authorization"], expected_other),+ _ => {+ assert_eq!(+ received.headers["x-amz-content-sha256"],+ format!("{:x}", sha2::Sha256::digest(body))+ );+ assert!(received.headers["authorization"].contains(+ "Credential=AKIDEXAMPLE/20150830/us-east-1/aoss/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date,"+ ));+ }+ }+ stream.write_all(RESPONSE).await.expect("origin response");+ }+ });++ for (path, body) in cases {+ let method = if body.is_empty() { "GET" } else { "POST" };+ let mut req = request(&context, "aoss", method, path, body);+ if path == "/unsigned" {+ req.headers_mut().remove("authorization");+ } else if path == "/other-key" {+ req.headers_mut()+ .insert("authorization", HeaderValue::from_str(&other_key).unwrap());+ }+ let response = timeout(FORWARD_TIMEOUT, forward_request(req, &context))+ .await+ .expect("MITM forwarding deadline")+ .expect("MITM forwarding");+ assert_eq!(response.status(), StatusCode::OK);+ }+ timeout(FORWARD_TIMEOUT, server)+ .await+ .expect("origin completion deadline")+ .expect("origin assertions");+}++#[tokio::test]+async fn signs_only_supported_http1_requests_for_an_http2_origin() {+ use rama_core::ServiceInput;+ use rama_http::body::util::BodyExt;++ let dir = tempdir().expect("TLS fixture directory");+ let (context, listener, acceptor) = tls_origin(dir.path(), "s3", b"h2").await;+ let context = Arc::new(context);+ let (observed, mut received) = mpsc::unbounded_channel();+ let origin = tokio::spawn(async move {+ loop {+ let (socket, _) = listener.accept().await.expect("accept origin");+ let stream = acceptor.accept(socket).await.expect("origin TLS");+ let observed = observed.clone();+ let service = service_fn(move |request: Request| {+ let observed = observed.clone();+ async move {+ let (parts, body) = request.into_parts();+ let body = body.collect().await.expect("origin body").to_bytes();+ observed.send((parts, body)).expect("report origin request");+ Ok::<_, std::convert::Infallible>(Response::new(Body::empty()))+ }+ });+ HttpServer::h2(Executor::default())+ .service(service)+ .serve(ServiceInput::new(stream))+ .await+ .expect("serve HTTP/2");+ }+ });++ let auth = guest_authorization("s3", "host;x-amz-date");+ for (extra_headers, status) in [+ ("Content-Length: 1\r\nTransfer-Encoding: chunked\r\n", 403),+ ("Transfer-Encoding: gzip, chunked\r\n", 403),+ (+ "Transfer-Encoding: chunked\r\nConnection: Authorization\r\n",+ 403,+ ),+ (+ "Transfer-Encoding: chunked\r\nX-Amz-Trailer: checksum\r\n",+ 403,+ ),+ (+ "Transfer-Encoding: chunked\r\nContent-Encoding: aws-chunked\r\nConnection: content-encoding\r\n",+ 403,+ ),+ // Rama discards Content-Length when it appears after Transfer-Encoding.+ // Verify that the retained chunk framing and the signed payload agree.+ (+ "Transfer-Encoding: chunked\r\nContent-Length: 1\r\nKeep-Alive: timeout=5\r\nSec-WebSocket-Key: test\r\n",+ 200,+ ),+ ] {+ let (mut guest, io) = tokio::io::duplex(4096);+ let handler = service_fn({+ let context = Arc::clone(&context);+ move |request| handle_mitm_request(request, Arc::clone(&context))+ });+ let guest_server = tokio::spawn(async move {+ HttpServer::http1()+ .service(handler)+ .serve(ServiceInput::new(io))+ .await+ .expect("serve guest");+ });+ let request = format!(+ "POST /key HTTP/1.1\r\nHost: localhost:{}\r\nAuthorization: {auth}\r\nX-Amz-Date: 20150830T123600Z\r\n{extra_headers}Connection: close\r\n\r\n3\r\nabc\r\n0\r\n\r\n",+ context.policy.target_port+ );+ guest+ .write_all(request.as_bytes())+ .await+ .expect("send guest request");+ let mut response = Vec::new();+ timeout(FORWARD_TIMEOUT, guest.read_to_end(&mut response))+ .await+ .expect("response deadline")+ .expect("read guest response");+ guest_server.await.expect("guest server");+ assert!(+ response.starts_with(format!("HTTP/1.1 {status} ").as_bytes()),+ "{extra_headers}: {}",+ String::from_utf8_lossy(&response)+ );+ }+ let (parts, body) = received.try_recv().expect("one allowed request");+ assert!(received.try_recv().is_err());+ assert_eq!(parts.version, rama_http::Version::HTTP_2);+ assert_eq!(body.as_ref(), b"abc");+ assert_eq!(+ parts.headers["x-amz-content-sha256"],+ format!("{:x}", sha2::Sha256::digest(&body))+ );+ let authorization = parts.headers["authorization"].to_str().expect("signature");+ let (fields, _) = authorization+ .split_once("SignedHeaders=")+ .expect("signed fields")+ .1+ .split_once(',')+ .expect("signature suffix");+ for name in fields.split(';') {+ // HTTP/2 transports Host in :authority.+ assert!(name == "host" || parts.headers.contains_key(name), "{name}");+ }+ origin.abort();+}++#[tokio::test]+async fn drained_signed_bodies_release_capacity_before_response_headers() {+ let dir = tempdir().expect("TLS fixture directory");+ let (context, listener, acceptor) = tls_origin(dir.path(), "sqs", b"http/1.1").await;+ let (arrived, mut arrivals) = mpsc::channel(9);+ let (release, released) = watch::channel(false);+ let server = tokio::spawn(async move {+ let mut requests = JoinSet::new();+ for _ in 0..9 {+ let (stream, _) = listener.accept().await.expect("accept origin connection");+ let acceptor = acceptor.clone();+ let arrived = arrived.clone();+ let mut released = released.clone();+ requests.spawn(async move {+ let mut stream = acceptor.accept(stream).await.expect("origin TLS handshake");+ let received = read_request(&mut stream).await;+ let wait = received.path.starts_with("/wait/");+ let expected_body = if wait {+ b"Action=ReceiveMessage".as_slice()+ } else {+ b"Action=SendMessage&MessageBody=hello".as_slice()+ };+ assert_eq!(received.body, expected_body);+ assert!(received.headers["authorization"].contains("/sqs/aws4_request"));+ arrived+ .send(received.path)+ .await+ .expect("report drained body");+ if wait {+ released+ .wait_for(|released| *released)+ .await+ .expect("release delayed headers");+ }+ stream.write_all(RESPONSE).await.expect("origin response");+ });+ }+ while let Some(result) = requests.join_next().await {+ result.expect("origin request assertions");+ }+ });++ let mut pending = JoinSet::new();+ for index in 0..8 {+ let context = context.clone();+ pending.spawn(async move {+ let req = request(+ &context,+ "sqs",+ "POST",+ &format!("/wait/{index}"),+ b"Action=ReceiveMessage",+ );+ forward_request(req, &context).await+ });+ }+ for _ in 0..8 {+ let path = timeout(FORWARD_TIMEOUT, arrivals.recv())+ .await+ .expect("eight request bodies must drain")+ .expect("origin remains connected");+ assert!(path.starts_with("/wait/"));+ }++ // All eight responses still lack headers; only their request bodies have completed.+ let ninth = timeout(+ FORWARD_TIMEOUT,+ forward_request(+ request(+ &context,+ "sqs",+ "POST",+ "/send",+ b"Action=SendMessage&MessageBody=hello",+ ),+ &context,+ ),+ )+ .await;+ release.send(true).expect("release long polls");+ while let Some(result) = timeout(FORWARD_TIMEOUT, pending.join_next())+ .await+ .expect("long poll completion deadline")+ {+ let response = result.expect("forward task").expect("long poll response");+ assert_eq!(response.status(), StatusCode::OK);+ }+ // Aborting the origin also aborts its request tasks if the ninth request never arrived.+ server.abort();+ let response = ninth+ .expect("ninth forwarding deadline")+ .expect("drained bodies must release buffer slots before response headers");+ assert_eq!(response.status(), StatusCode::OK);+}