:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Resolution mode: serves `cymbal.resolution.v1` over gRPC, exposing cymbal's
//! symbol-resolution stack to the remote-resolution client
//! (`crate::stages::resolution::remote`). It starts only the symbol resolver
//! (Postgres + object storage) — no Kafka, Redis, issue cache, or signals.
//! See [`README.md`](./README.md) for the wire contract and operations guide.
use std::sync::{
atomic::{AtomicBool, Ordering},
Arc,
};
use std::time::Duration;
use axum::{http::StatusCode, routing::get, Router};
use common_metrics::Matcher;
use cymbal_proto::cymbal::resolution::v1::cymbal_resolution_server::CymbalResolutionServer;
use cymbal_proto::CYMBAL_RESOLUTION_MAX_MESSAGE_SIZE_BYTES;
use personhog_common::grpc::{tracked_tcp_incoming, GrpcLoadShedLayer, GrpcMetricsLayer};
use tokio::sync::watch;
use tokio::task::JoinHandle;
use tonic::service::interceptor::InterceptedService;
use tonic::transport::Server;
use tracing::{info, warn};
use crate::core::{config::ResolverConfig, shutdown::wait_for_shutdown};
use crate::metric_consts::{
BYTE_HISTOGRAM_BUCKETS, SYMBOL_SET_DECOMPRESSED_BYTES, SYMBOL_SET_PARSE_INPUT_BYTES,
};
pub mod app_context;
pub mod auth;
pub mod config;
pub mod load_monitor;
pub mod service;
pub use config::{Config, ResolutionConfig};
use app_context::ResolutionAppContext;
use auth::InternalApiSecretInterceptor;
use load_monitor::LoadMonitor;
use service::{CymbalResolutionService, ServiceConfig};
/// Boot the gRPC resolution service plus its metrics/health server and run
/// until shutdown. The `internal_api_secret` and symbol-resolution concurrency
/// come from the shared resolver config; everything else from the resolution
/// service config.
pub async fn serve(
resolver: &ResolverConfig,
service: &Config,
) -> Result<(), Box<dyn std::error::Error>> {
let res = service;
info!("Starting cymbal-resolution service");
info!("gRPC address: {}", res.grpc_address);
info!("Metrics port: {}", res.metrics_port);
info!("Max concurrent requests: {}", res.max_concurrent_requests);
let app_context = ResolutionAppContext::from_config(resolver, service).await?;
let (shutdown_tx, shutdown_rx) = watch::channel(false);
let draining = Arc::new(AtomicBool::new(false));
let drain_notice = Duration::from_millis(res.subscribe_tick_interval_ms).saturating_mul(2);
let _shutdown_handle = spawn_shutdown_listener(
shutdown_tx.clone(),
draining.clone(),
app_context.load_monitor.clone(),
drain_notice,
);
let metrics_handle =
spawn_metrics_server(res.metrics_port, shutdown_rx.clone(), draining.clone());
let service_config = ServiceConfig::from(res);
let service = CymbalResolutionService::new(
app_context.symbol_resolver.clone(),
app_context.symbol_resolution_limiter.clone(),
app_context.load_monitor.clone(),
app_context.service_instance_id.clone(),
service_config,
draining,
);
let internal_api_secret_fallbacks = resolver
.internal_api_secret_fallbacks
.split(',')
.map(str::to_string)
.collect::<Vec<_>>();
let auth_interceptor = InternalApiSecretInterceptor::new(
resolver.internal_api_secret.clone(),
internal_api_secret_fallbacks,
);
let listener = tokio::net::TcpListener::bind(res.grpc_address).await?;
let incoming = tracked_tcp_incoming(listener);
info!("gRPC server listening on {}", res.grpc_address);
let grpc_service = CymbalResolutionServer::new(service)
.max_decoding_message_size(CYMBAL_RESOLUTION_MAX_MESSAGE_SIZE_BYTES)
.max_encoding_message_size(CYMBAL_RESOLUTION_MAX_MESSAGE_SIZE_BYTES);
let grpc_service = InterceptedService::new(grpc_service, move |request| {
auth_interceptor.authenticate(request)
});
let server_result = Server::builder()
.http2_keepalive_interval(Some(Duration::from_secs(30)))
.http2_keepalive_timeout(Some(Duration::from_secs(20)))
.layer(GrpcMetricsLayer::default().with_processing_time_header())
.layer(GrpcLoadShedLayer::new(res.max_concurrent_requests))
.add_service(grpc_service)
.serve_with_incoming_shutdown(incoming, wait_for_shutdown(shutdown_rx))
.await;
let _ignored = shutdown_tx.send(true);
if let Err(err) = metrics_handle.await {
warn!(error = %err, "metrics server task failed during shutdown");
}
server_result?;
Ok(())
}
fn spawn_metrics_server(
port: u16,
shutdown_rx: watch::Receiver<bool>,
draining: Arc<AtomicBool>,
) -> JoinHandle<()> {
tokio::spawn(async move {
let readiness_draining = draining.clone();
let router = Router::new()
.route("/_liveness", get(|| async { "ok" }))
.route(
"/_readiness",
get(move || readiness(readiness_draining.clone())),
);
let router = common_metrics::setup_metrics_routes_for_product_with_overrides(
router,
"cymbal-resolution",
&[
(
Matcher::Full(SYMBOL_SET_DECOMPRESSED_BYTES.into()),
BYTE_HISTOGRAM_BUCKETS,
),
(
Matcher::Full(SYMBOL_SET_PARSE_INPUT_BYTES.into()),
BYTE_HISTOGRAM_BUCKETS,
),
],
);
let bind = format!("0.0.0.0:{port}");
info!("Metrics server listening on {}", bind);
let listener = match tokio::net::TcpListener::bind(&bind).await {
Ok(listener) => listener,
Err(e) => {
tracing::error!("Metrics server bind error: {e}");
return;
}
};
if let Err(e) = axum::serve(listener, router)
.with_graceful_shutdown(wait_for_shutdown(shutdown_rx))
.await
{
tracing::error!("Metrics server error: {e}");
}
})
}
fn spawn_shutdown_listener(
shutdown_tx: watch::Sender<bool>,
draining: Arc<AtomicBool>,
load_monitor: LoadMonitor,
drain_notice: Duration,
) -> JoinHandle<()> {
tokio::spawn(async move {
shutdown_signal().await;
info!(
drain_notice_ms = drain_notice.as_millis() as u64,
"shutdown signal received, marking cymbal-resolution as draining",
);
draining.store(true, Ordering::Relaxed);
load_monitor.set_draining(true);
tokio::time::sleep(drain_notice).await;
info!("drain notice elapsed, stopping cymbal-resolution");
let _ignored = shutdown_tx.send(true);
})
}
async fn readiness(draining: Arc<AtomicBool>) -> (StatusCode, &'static str) {
if draining.load(Ordering::Relaxed) {
return (StatusCode::SERVICE_UNAVAILABLE, "draining");
}
(StatusCode::OK, "ok")
}
#[cfg(unix)]
async fn shutdown_signal() {
use tokio::signal::unix::{signal, SignalKind};
let mut sigterm = signal(SignalKind::terminate()).expect("failed to listen for SIGTERM");
tokio::select! {
result = tokio::signal::ctrl_c() => {
if let Err(err) = result {
warn!(error = %err, "failed to listen for Ctrl+C");
}
}
_ = sigterm.recv() => {}
}
}
#[cfg(not(unix))]
async fn shutdown_signal() {
if let Err(err) = tokio::signal::ctrl_c().await {
warn!(error = %err, "failed to listen for Ctrl+C");
}
}