Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

18.7 KB
Raw Blame
//! Shared fixtures for remote-resolution integration tests.
//!
//! `tests/common/mod.rs` is the canonical place to put helpers that multiple
//! integration test binaries (`tests/remote_resolution.rs`,
//! `tests/remote_resolution_hardening.rs`,
//! `tests/remote_resolution_parity.rs`, …) reuse. Keep new fixtures here
//! rather than duplicating them per test file so failure-mode tests can be
//! extended without re-writing the in-process gRPC stub.

#![allow(dead_code)]

use std::net::SocketAddr;
use std::pin::Pin;
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};

use async_trait::async_trait;
use cymbal::error::{ResolveError, UnhandledError};
use cymbal::frames::{Frame, RawFrame};
use cymbal::langs::native::DebugImage;
use cymbal::stages::pipeline::ParsedPipelineItem;
use cymbal::stages::resolution::{
    event_release::ReleaseCache,
    remote::{
        config::RemoteResolutionConfig, pool::EndpointPool, resolver::RemoteResolutionContext,
    },
    ResolutionStage,
};
use cymbal::symbolication::symbol::SymbolResolver;
use cymbal::symbolication::symbol_store::chunk_id::OrChunkId;
use cymbal::symbolication::symbol_store::proguard::ProguardRef;
use cymbal::types::{
    batch::Batch,
    event::AnyEvent,
    exception_event::{ExceptionEvent, Parsed, Resolved},
    operator::TeamId,
    stage::Stage,
    Exception, Stacktrace,
};
use cymbal_proto::cymbal::resolution::v1::cymbal_resolution_server::{
    CymbalResolution, CymbalResolutionServer,
};
use cymbal_proto::cymbal::resolution::v1::{
    resolve_outcome, Accepted, Done, Error as ItemError, ErrorKind, LoadEvent, ResolveItem,
    ResolveOutcome, Retry, SubscribeRequest,
};
use futures::{Stream, StreamExt};
use tokio::sync::mpsc;
use tokio_stream::wrappers::ReceiverStream;
use tonic::{Request, Response, Status};
use uuid::Uuid;

pub type ResolveStream = Pin<Box<dyn Stream<Item = Result<ResolveOutcome, Status>> + Send>>;
pub type SubscribeStream = Pin<Box<dyn Stream<Item = Result<LoadEvent, Status>> + Send>>;

/// Fixture stub for the cymbal.resolution.v1 server. Streams are recorded
/// separately from items: the mux opens one long-lived bidi stream per
/// endpoint, while tests usually care about the individual ResolveItems that
/// actually crossed that stream.
#[derive(Clone)]
pub struct StubServer {
    pub behavior: ServerBehavior,
    pub streams: Arc<Mutex<Vec<SocketAddr>>>,
    pub items: Arc<Mutex<Vec<ResolveItem>>>,
    pub addr: SocketAddr,
    pub stream_counter: Arc<AtomicU32>,
}

/// Catalog of behaviors a test stub can advertise. Keep failure modes here so
/// the integration suite stays fixture-driven and the bidi protocol shape is
/// shared across remote-resolution tests.
#[derive(Clone)]
pub enum ServerBehavior {
    /// Echo each item back as Done with the input exception unchanged.
    Happy,
    /// Echo each item back as Done with the input exception unchanged but
    /// sleep `delay` before emitting each outcome.
    HappyDelayed { delay: Duration },
    /// Emit Accepted immediately, then emit Done after `delay` without blocking
    /// reads for subsequent items.
    AcceptedThenDoneDelayed { delay: Duration },
    /// Fail the Resolve stream during setup with Status::Unavailable.
    AlwaysUnavailable,
    /// Fail the Resolve stream during setup with Status::InvalidArgument.
    AlwaysInvalidArgument,
    /// Delay stream setup long enough for caller deadlines to fire.
    SlowerThanDeadline { sleep: Duration },
    /// Emit Retry outcomes for every item on this stream.
    Retry,
    /// Emit ErrorKind::Overloaded outcomes for every item on this stream.
    Overloaded,
    /// Emit one Done for the first item then break the stream with Internal.
    InterruptAfterFirst,
    /// Emit the first item as Done and every subsequent item as Error.
    ErrorAfterFirst { kind: ErrorKind },
}

impl StubServer {
    pub fn new(behavior: ServerBehavior, addr: SocketAddr) -> Self {
        Self {
            behavior,
            streams: Arc::new(Mutex::new(Vec::new())),
            items: Arc::new(Mutex::new(Vec::new())),
            addr,
            stream_counter: Arc::new(AtomicU32::new(0)),
        }
    }
}

#[tonic::async_trait]
impl CymbalResolution for StubServer {
    type ResolveStream = ResolveStream;
    type SubscribeStream = SubscribeStream;

    /// Default Subscribe behaviour: emit freshness/draining/load `LoadEvent`s
    /// on a fast tick so snapshot-required routing can warm up quickly.
    async fn subscribe(
        &self,
        _request: Request<SubscribeRequest>,
    ) -> Result<Response<Self::SubscribeStream>, Status> {
        let (tx, rx) = mpsc::channel(2);
        let instance = format!("stub-{}", self.addr);
        tokio::spawn(async move {
            let mut sequence = 0u64;
            let mut ticker = tokio::time::interval(Duration::from_millis(25));
            ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
            loop {
                ticker.tick().await;
                sequence += 1;
                let event = LoadEvent {
                    service_instance_id: instance.clone(),
                    draining: false,
                    sequence,
                    message: String::new(),
                    in_flight: 0,
                    max_in_flight: 64,
                };
                if tx.send(Ok(event)).await.is_err() {
                    return;
                }
            }
        });
        Ok(Response::new(Box::pin(ReceiverStream::new(rx))))
    }

    async fn resolve(
        &self,
        request: Request<tonic::Streaming<ResolveItem>>,
    ) -> Result<Response<Self::ResolveStream>, Status> {
        self.streams.lock().unwrap().push(self.addr);
        self.stream_counter.fetch_add(1, Ordering::SeqCst);

        match self.behavior.clone() {
            ServerBehavior::AlwaysUnavailable => Err(Status::unavailable("forced shed")),
            ServerBehavior::AlwaysInvalidArgument => {
                Err(Status::invalid_argument("forced bad request"))
            }
            ServerBehavior::SlowerThanDeadline { sleep } => {
                tokio::time::sleep(sleep).await;
                Err(Status::deadline_exceeded("server slow"))
            }
            behavior => {
                let mut inbound = request.into_inner();
                let (tx, rx) = mpsc::channel(16);
                let items = self.items.clone();
                tokio::spawn(async move {
                    let mut seen = 0usize;
                    while let Some(next) = inbound.next().await {
                        let item = match next {
                            Ok(item) => item,
                            Err(err) => {
                                // The client may close its request half as soon as every in-flight
                                // item has a terminal outcome. Tonic can surface that as an h2 body
                                // read error on the server task; don't turn client-side shutdown into
                                // an extra response-stream failure after the fixture has already sent
                                // the outcomes the test cares about.
                                drop(err);
                                return;
                            }
                        };
                        items.lock().unwrap().push(item.clone());
                        seen += 1;

                        match behavior {
                            ServerBehavior::Happy => {
                                send_outcome(&tx, accepted_outcome(&item)).await;
                                send_outcome(&tx, done_outcome(&item)).await;
                            }
                            ServerBehavior::HappyDelayed { delay } => {
                                send_outcome(&tx, accepted_outcome(&item)).await;
                                tokio::time::sleep(delay).await;
                                send_outcome(&tx, done_outcome(&item)).await;
                            }
                            ServerBehavior::AcceptedThenDoneDelayed { delay } => {
                                send_outcome(&tx, accepted_outcome(&item)).await;
                                let item_tx = tx.clone();
                                tokio::spawn(async move {
                                    tokio::time::sleep(delay).await;
                                    send_outcome(&item_tx, done_outcome(&item)).await;
                                });
                            }
                            ServerBehavior::Retry => {
                                send_outcome(&tx, retry_outcome(&item)).await;
                            }
                            ServerBehavior::Overloaded => {
                                send_outcome(&tx, error_outcome(&item, ErrorKind::Overloaded))
                                    .await;
                            }
                            ServerBehavior::InterruptAfterFirst => {
                                if seen == 1 {
                                    send_outcome(&tx, accepted_outcome(&item)).await;
                                    send_outcome(&tx, done_outcome(&item)).await;
                                    let _ignored = tx
                                        .send(Err(Status::internal("simulated interruption")))
                                        .await;
                                    return;
                                }
                                send_outcome(&tx, accepted_outcome(&item)).await;
                                send_outcome(&tx, done_outcome(&item)).await;
                            }
                            ServerBehavior::ErrorAfterFirst { kind } => {
                                if seen == 1 {
                                    send_outcome(&tx, accepted_outcome(&item)).await;
                                    send_outcome(&tx, done_outcome(&item)).await;
                                } else {
                                    send_outcome(&tx, accepted_outcome(&item)).await;
                                    send_outcome(&tx, error_outcome(&item, kind)).await;
                                }
                            }
                            ServerBehavior::AlwaysUnavailable
                            | ServerBehavior::AlwaysInvalidArgument
                            | ServerBehavior::SlowerThanDeadline { .. } => unreachable!(
                                "stream setup failures return before spawning the bidi task"
                            ),
                        }
                    }
                });
                Ok(Response::new(Box::pin(ReceiverStream::new(rx))))
            }
        }
    }
}

async fn send_outcome(tx: &mpsc::Sender<Result<ResolveOutcome, Status>>, outcome: ResolveOutcome) {
    let _ignored = tx.send(Ok(outcome)).await;
}

fn accepted_outcome(item: &ResolveItem) -> ResolveOutcome {
    ResolveOutcome {
        id: item.id,
        result: Some(resolve_outcome::Result::Accepted(Accepted {})),
    }
}

fn done_outcome(item: &ResolveItem) -> ResolveOutcome {
    ResolveOutcome {
        id: item.id,
        result: Some(resolve_outcome::Result::Done(Done {
            resolved_exception_json: item.exception_json.clone(),
            release_id: String::new(),
        })),
    }
}

fn retry_outcome(item: &ResolveItem) -> ResolveOutcome {
    ResolveOutcome {
        id: item.id,
        result: Some(resolve_outcome::Result::Retry(Retry {
            code: "retry".to_string(),
            message: "try later".to_string(),
            retry_after_ms: 0,
        })),
    }
}

fn error_outcome(item: &ResolveItem, kind: ErrorKind) -> ResolveOutcome {
    ResolveOutcome {
        id: item.id,
        result: Some(resolve_outcome::Result::Error(ItemError {
            kind: kind as i32,
            message: format!("forced {} from stub", kind.as_str_name()),
            details_json: Vec::new(),
        })),
    }
}

/// Bind to an OS-assigned local port, drop the listener, and spawn a tonic
/// server. Returns the address once a probe TCP connect succeeds, so callers
/// never dial before the bind has actually happened.
pub async fn spawn_stub_server(
    behavior: ServerBehavior,
) -> (SocketAddr, Arc<Mutex<Vec<ResolveItem>>>) {
    let (addr, _streams, items) = spawn_recording_stub_server(behavior).await;
    (addr, items)
}

pub async fn spawn_recording_stub_server(
    behavior: ServerBehavior,
) -> (
    SocketAddr,
    Arc<Mutex<Vec<SocketAddr>>>,
    Arc<Mutex<Vec<ResolveItem>>>,
) {
    let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
    let addr = listener.local_addr().unwrap();
    drop(listener);
    let stub = StubServer::new(behavior, addr);
    let streams = stub.streams.clone();
    let items = stub.items.clone();

    tokio::spawn(async move {
        let _outcome = tonic::transport::Server::builder()
            .add_service(CymbalResolutionServer::new(stub))
            .serve(addr)
            .await;
    });

    for _ in 0..40 {
        if std::net::TcpStream::connect_timeout(&addr, Duration::from_millis(50)).is_ok() {
            return (addr, streams, items);
        }
        tokio::time::sleep(Duration::from_millis(25)).await;
    }
    panic!("test stub server failed to come up at {addr}");
}

/// Reserve a local TCP port without keeping a listener bound to it. The
/// returned address is therefore guaranteed not to accept connections, which
/// is what `connection-refused` failure tests need.
pub fn unbound_addr() -> SocketAddr {
    let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
    let addr = listener.local_addr().unwrap();
    drop(listener);
    addr
}

pub fn make_config(max_retries: u32, deadline: Duration) -> RemoteResolutionConfig {
    RemoteResolutionConfig {
        host: "test-only".to_string(),
        port: 0,
        internal_api_secret: "test-secret".to_string(),
        dns_refresh: Duration::from_secs(60),
        request_deadline: deadline,
        connect_timeout: Duration::from_secs(1),
        max_retries,
        // Tests run with near-zero backoff so retries don't drag suite latency;
        // production defaults are tuned for thundering-herd mitigation.
        retry_backoff: Duration::from_millis(1),
        retry_max_backoff: Duration::from_millis(2),
        routing_jitter: 0.0,
        routing_acceptance_concurrency: 10,
        overload_ejection_initial: Duration::ZERO,
        overload_ejection_max: Duration::ZERO,
        overload_ejection_decay: Duration::from_secs(30),
        // Subscription cadence is irrelevant for tests that don't wire the
        // subscription client — kept short so tests that do consume it run
        // quickly when they opt in via a dedicated pool builder.
        subscribe_tick_hint: Duration::from_millis(50),
        subscribe_reconnect_backoff: Duration::from_millis(50),
    }
}

pub async fn make_ctx(
    addrs: &[SocketAddr],
    max_retries: u32,
    deadline: Duration,
) -> RemoteResolutionContext {
    let config = make_config(max_retries, deadline);
    let pool = EndpointPool::from_addrs(config.clone(), addrs).expect("build pool");
    if !addrs.is_empty() {
        wait_until_routable(&pool).await;
    }
    RemoteResolutionContext::new(pool, config)
}

/// Wait until the pool has at least one routable endpoint (a fresh
/// `LoadEvent` snapshot has arrived). Snapshot-required routing means an
/// empty pool persists until the per-endpoint Subscribe stream delivers its
/// first event; without this small warm-up tests race the first tick and
/// observe spurious `pool_empty` failures.
pub async fn wait_until_routable(pool: &Arc<cymbal::stages::resolution::remote::EndpointPool>) {
    let deadline = Instant::now() + Duration::from_secs(2);
    loop {
        if pool.select(&[]).await.is_ok() {
            return;
        }
        if Instant::now() >= deadline {
            panic!("pool never became routable within warm-up window");
        }
        tokio::time::sleep(Duration::from_millis(5)).await;
    }
}

#[derive(Default)]
pub struct NoopResolver;

#[async_trait]
impl SymbolResolver for NoopResolver {
    async fn resolve_raw_frame(
        &self,
        _team_id: TeamId,
        _frame: &RawFrame,
        _debug_images: &[DebugImage],
    ) -> Result<Vec<Frame>, UnhandledError> {
        Ok(Vec::new())
    }

    async fn resolve_java_class(
        &self,
        _team_id: TeamId,
        _symbolset_ref: OrChunkId<ProguardRef>,
        _class: String,
    ) -> Result<String, ResolveError> {
        unreachable!("integration fixtures do not exercise Java class resolution")
    }

    async fn resolve_dart_minified_name(
        &self,
        _team_id: TeamId,
        _symbolset_ref: String,
        _minified_name: &str,
    ) -> Result<String, ResolveError> {
        unreachable!("integration fixtures do not exercise Dart name resolution")
    }
}

pub fn remote_stage(ctx: RemoteResolutionContext) -> ResolutionStage {
    ResolutionStage {
        remote: ctx,
        // Never connected: fixture events carry no release identifiers, so the release
        // resolver never issues a query.
        posthog_pool: sqlx::postgres::PgPoolOptions::new()
            .connect_lazy("postgres://unused/unused")
            .expect("lazy pool construction does not connect"),
        release_cache: ReleaseCache::new(0, Duration::from_secs(0)),
        drop_code_variables_team_ids: Default::default(),
    }
}

pub async fn process_one(
    stage: ResolutionStage,
    evt: ExceptionEvent<Parsed>,
) -> Result<ExceptionEvent<Resolved>, UnhandledError> {
    let batch: Batch<ParsedPipelineItem> = Batch::from(vec![Ok(evt)]);
    let result = stage.process(batch).await?;
    let mut items: Vec<_> = result.into_iter().collect();
    assert_eq!(items.len(), 1, "single-event batch must produce one output");
    Ok(items.remove(0).expect("event must not be EventError"))
}

pub fn build_event(num_exceptions: usize) -> ExceptionEvent<Parsed> {
    build_event_with(num_exceptions, 7, Uuid::now_v7(), Vec::new())
}

pub fn build_event_with(
    num_exceptions: usize,
    team_id: i32,
    uuid: Uuid,
    debug_images: Vec<DebugImage>,
) -> ExceptionEvent<Parsed> {
    let exceptions: Vec<Exception> = (0..num_exceptions)
        .map(|i| Exception {
            exception_id: None,
            exception_type: format!("Boom{i}"),
            exception_message: format!("message {i}"),
            mechanism: None,
            module: None,
            thread_id: None,
            stack: Some(Stacktrace::Raw { frames: vec![] }),
        })
        .collect();
    AnyEvent {
        uuid,
        event: "$exception".to_string(),
        properties: serde_json::json!({
            "$exception_list": exceptions,
            "$debug_images": debug_images,
        }),
        timestamp: String::new(),
        team_id,
        others: Default::default(),
    }
    .try_into()
    .expect("test event is valid")
}