Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

11.5 KB
Raw Blame
use std::{
    convert::Infallible,
    pin::Pin,
    sync::{
        Arc,
        atomic::{AtomicBool, AtomicUsize, Ordering},
    },
    task::{Context, Poll},
    time::Duration,
};

use bytes::Bytes;
use hyper::{
    Request, Response, StatusCode,
    body::{Frame, Incoming},
    service::service_fn,
};
use hyper_util::rt::{TokioExecutor, TokioIo};
use rustls::pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject};
use tokio::sync::watch;

use crate::{
    OpensearchClient,
    agent_session::{AgentSessionMessageAuthor, AgentSessionMessageDocument},
    date_format::EpochMillis,
    error::{OpensearchClientError, OpensearchOperation, TransportFailure},
    upsert::{agent_session::ReconcileAgentSessionArgs, email::UpsertEmailArgs},
};

const OK_BODY: &str = r#"{"errors":false,"items":[],"failures":[],"timed_out":false}"#;

/// When the server closes a connection with requests still waiting on it.
#[derive(Clone, Copy)]
enum Close {
    /// Close every connection opened so far once this many requests wait on
    /// them, then answer on new connections: a front end recycling the one
    /// HTTP/2 connection every request multiplexes over.
    OnceWhenWaiting(usize),
    /// Close each connection as soon as a request arrives on it.
    Always,
    /// Reset the first response's stream partway through its body, as a
    /// recycled connection does to a response still in flight.
    CutFirstBody,
    /// Answer every request.
    Never,
}

/// A response body, reset after its bytes when `cut` is set.
struct Reply {
    data: Option<Bytes>,
    cut: bool,
}

impl hyper::body::Body for Reply {
    type Data = Bytes;
    type Error = std::io::Error;

    fn poll_frame(
        mut self: Pin<&mut Self>,
        _: &mut Context<'_>,
    ) -> Poll<Option<Result<Frame<Bytes>, Self::Error>>> {
        if let Some(data) = self.data.take() {
            return Poll::Ready(Some(Ok(Frame::data(data))));
        }
        if std::mem::take(&mut self.cut) {
            return Poll::Ready(Some(Err(std::io::ErrorKind::ConnectionReset.into())));
        }
        Poll::Ready(None)
    }
}

struct Server {
    url: String,
    requests: Arc<AtomicUsize>,
    connections: Arc<AtomicUsize>,
}

/// HTTP/2 over TLS, which reqwest negotiates with the OpenSearch domain.
async fn server(close: Close, status: StatusCode) -> Server {
    let certs = include_bytes!("../../../../infra/local/certs/localhost.pem");
    let key = include_bytes!("../../../../infra/local/certs/localhost-key.pem");
    let mut config = rustls::ServerConfig::builder_with_provider(Arc::new(
        rustls::crypto::aws_lc_rs::default_provider(),
    ))
    .with_safe_default_protocol_versions()
    .unwrap()
    .with_no_client_auth()
    .with_single_cert(
        CertificateDer::pem_slice_iter(certs)
            .collect::<Result<_, _>>()
            .unwrap(),
        PrivateKeyDer::from_pem_slice(key).unwrap(),
    )
    .unwrap();
    config.alpn_protocols = vec![b"h2".to_vec()];
    let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(config));
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
    let url = format!(
        "https://localhost:{}",
        listener.local_addr().unwrap().port()
    );

    let requests = Arc::new(AtomicUsize::new(0));
    let connections = Arc::new(AtomicUsize::new(0));
    let waiting = Arc::new(AtomicUsize::new(0));
    let cut_pending = Arc::new(AtomicBool::new(true));
    let (closed_tx, closed_rx) = watch::channel(false);
    let closed_tx = Arc::new(closed_tx);
    let (server_requests, server_connections) = (requests.clone(), connections.clone());
    tokio::spawn(async move {
        loop {
            let (tcp, _) = listener.accept().await.unwrap();
            let tls = acceptor.accept(tcp).await.unwrap();
            server_connections.fetch_add(1, Ordering::SeqCst);
            let holds = match close {
                Close::OnceWhenWaiting(_) => !*closed_rx.borrow(),
                Close::Always => true,
                Close::CutFirstBody | Close::Never => false,
            };
            let (requests, waiting, closed_tx, cut_pending) = (
                server_requests.clone(),
                waiting.clone(),
                closed_tx.clone(),
                cut_pending.clone(),
            );
            let (arrived_tx, mut arrived_rx) = watch::channel(false);
            let mut closed_rx = closed_rx.clone();
            tokio::spawn(async move {
                let service = service_fn(move |_: Request<Incoming>| {
                    requests.fetch_add(1, Ordering::SeqCst);
                    let cut = matches!(close, Close::CutFirstBody)
                        && cut_pending.swap(false, Ordering::SeqCst);
                    let (waiting, closed_tx, arrived_tx) =
                        (waiting.clone(), closed_tx.clone(), arrived_tx.clone());
                    async move {
                        if holds {
                            arrived_tx.send_replace(true);
                            if let Close::OnceWhenWaiting(count) = close
                                && waiting.fetch_add(1, Ordering::SeqCst) + 1 == count
                            {
                                closed_tx.send_replace(true);
                            }
                            std::future::pending::<()>().await;
                        }
                        let body = if cut {
                            &OK_BODY[..OK_BODY.len() / 2]
                        } else {
                            OK_BODY
                        };
                        let mut response = Response::new(Reply {
                            data: Some(Bytes::from_static(body.as_bytes())),
                            cut,
                        });
                        *response.status_mut() = status;
                        Ok::<_, Infallible>(response)
                    }
                });
                let connection = hyper::server::conn::http2::Builder::new(TokioExecutor::new())
                    .serve_connection(TokioIo::new(tls), service);
                // Returning drops the connection, closing it under every waiting stream.
                let close_signal = async {
                    match close {
                        Close::OnceWhenWaiting(_) if holds => {
                            let _ = closed_rx.wait_for(|closed| *closed).await;
                        }
                        Close::Always => {
                            let _ = arrived_rx.wait_for(|arrived| *arrived).await;
                        }
                        _ => std::future::pending().await,
                    }
                };
                tokio::select! {
                    _ = connection => {}
                    () = close_signal => {}
                }
            });
        }
    });
    Server {
        url,
        requests,
        connections,
    }
}

fn client(server: &Server) -> OpensearchClient {
    OpensearchClient::new(server.url.clone(), "".into(), "".into()).unwrap()
}

fn session(id: &str) -> ReconcileAgentSessionArgs {
    ReconcileAgentSessionArgs {
        agent_session_id: id.to_owned(),
        name: "Investigate indexing".to_owned(),
        owner_id: "user-1".to_owned(),
        bot_id: "bot-1".to_owned(),
        thread_id: None,
        originating_message_id: None,
        created_at_millis: EpochMillis::new(10).unwrap(),
        updated_at_millis: EpochMillis::new(20).unwrap(),
        projection_generation: "generation-1".to_owned(),
        messages: vec![AgentSessionMessageDocument {
            turn: 1,
            author: AgentSessionMessageAuthor::Agent,
            author_user_id: None,
            content: "Folded output".to_owned(),
        }],
    }
}

fn email(message_id: &str) -> UpsertEmailArgs {
    UpsertEmailArgs {
        thread_id: "thread-1".to_owned(),
        message_id: message_id.to_owned(),
        sender: "[email protected]".to_owned(),
        sender_name: None,
        reply_to: None,
        recipients: vec![],
        recipient_names: vec![],
        cc: vec![],
        cc_names: vec![],
        bcc: vec![],
        bcc_names: vec![],
        labels: vec![],
        link_id: "link-1".to_owned(),
        user_id: "user-1".to_owned(),
        updated_at_millis: EpochMillis::new(20).unwrap(),
        subject: None,
        sent_at_millis: None,
        content: "Body".to_owned(),
        properties: vec![],
    }
}

/// The production burst: unrelated agent-session and email writes share one
/// connection, it closes under all of them at once, and each must still land.
#[tokio::test]
async fn writes_sharing_a_closed_connection_are_resent_on_a_new_one() {
    let server = server(Close::OnceWhenWaiting(4), StatusCode::OK).await;
    let client = client(&server);
    let (first_session, second_session) = (session("session-1"), session("session-2"));
    let (first_email, second_email) = ([email("message-1")], [email("message-2")]);

    let results = tokio::time::timeout(Duration::from_secs(20), async {
        tokio::join!(
            client.reconcile_agent_session(&first_session, None),
            client.reconcile_agent_session(&second_session, None),
            client.bulk_upsert_email_messages(&first_email, None),
            client.bulk_upsert_email_messages(&second_email, None),
        )
    })
    .await
    .unwrap();

    results.0.unwrap();
    results.1.unwrap();
    results.2.unwrap();
    results.3.unwrap();
    // Four first attempts lost, then each reconcile's parent, bulk, and prune
    // plus each email bulk on a fresh connection.
    assert_eq!(server.requests.load(Ordering::SeqCst), 4 + 3 * 2 + 2);
    assert!(server.connections.load(Ordering::SeqCst) >= 2);
}

/// A response cut off partway through its body is resent, not reported as
/// an undecodable body.
#[tokio::test]
async fn a_response_cut_off_mid_body_is_resent() {
    let server = server(Close::CutFirstBody, StatusCode::OK).await;
    let messages = [email("message-1")];

    let result = tokio::time::timeout(
        Duration::from_secs(20),
        client(&server).bulk_upsert_email_messages(&messages, None),
    )
    .await
    .unwrap();

    result.unwrap();
    assert_eq!(server.requests.load(Ordering::SeqCst), 2);
}

/// A domain that keeps dropping connections still surfaces as a failure.
#[tokio::test]
async fn a_connection_lost_on_every_attempt_fails_after_three_attempts() {
    let server = server(Close::Always, StatusCode::OK).await;
    let messages = [email("message-1")];

    let result = tokio::time::timeout(
        Duration::from_secs(20),
        client(&server).bulk_upsert_email_messages(&messages, None),
    )
    .await
    .unwrap();

    let Err(OpensearchClientError::RequestFailed {
        cause, operation, ..
    }) = result
    else {
        panic!("expected a lost connection, got {result:?}");
    };
    assert_eq!(cause, TransportFailure::ConnectionClosed);
    assert_eq!(operation, OpensearchOperation::BulkUpsertEmailMessages);
    assert_eq!(server.requests.load(Ordering::SeqCst), 3);
}

/// An answered request is never resent, so a server error stays visible.
#[tokio::test]
async fn an_error_response_is_not_resent() {
    let server = server(Close::Never, StatusCode::SERVICE_UNAVAILABLE).await;
    let messages = [email("message-1")];

    let result = client(&server)
        .bulk_upsert_email_messages(&messages, None)
        .await;

    assert!(
        matches!(
            result,
            Err(OpensearchClientError::Unknown {
                operation: OpensearchOperation::BulkUpsertEmailMessages,
                ..
            })
        ),
        "{result:?}"
    );
    assert_eq!(server.requests.load(Ordering::SeqCst), 1);
}