:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::sync::Arc;
use std::time::Duration;
use axum::{extract::DefaultBodyLimit, http::Method, routing::get, routing::post, Router};
use capture::metrics_middleware::track_metrics;
use capture_logs::authorizer::Authorizer;
use capture_logs::config::Config;
use capture_logs::endpoints::aws_firehose;
use capture_logs::endpoints::datadog;
use capture_logs::endpoints::prometheus;
use capture_logs::kafka::KafkaSink;
use capture_logs::middleware::translate_compression_query_param;
use capture_logs::service::Service;
use capture_logs::service::{
export_logs_http, export_metrics_http, export_traces_http, options_handler,
};
use common_metrics::setup_metrics_routes;
use std::future::ready;
use std::net::SocketAddr;
use health::HealthRegistry;
use tokio::signal;
use tower_http::cors::{AllowHeaders, AllowOrigin, CorsLayer};
use tower_http::decompression::RequestDecompressionLayer;
use tracing::level_filters::LevelFilter;
use tracing::{error, info};
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer};
use limiters::token_dropper::TokenDropper;
common_alloc::used!();
async fn shutdown() {
let mut term = signal::unix::signal(signal::unix::SignalKind::terminate())
.expect("failed to register SIGTERM handler");
let mut interrupt = signal::unix::signal(signal::unix::SignalKind::interrupt())
.expect("failed to register SIGINT handler");
tokio::select! {
_ = term.recv() => {},
_ = interrupt.recv() => {},
};
tracing::info!("Shutting down gracefully...");
}
fn setup_tracing() {
let log_layer = tracing_subscriber::fmt::layer()
.json()
.with_span_list(false)
.with_filter(
EnvFilter::builder()
.with_default_directive(LevelFilter::INFO.into())
.from_env_lossy()
.add_directive("pyroscope=warn".parse().unwrap()),
);
tracing_subscriber::registry().with(log_layer).init();
}
pub async fn index() -> &'static str {
"log hog hogs logs
.|||||||||.
||||||||||||| gimme ur logs 🔫
|||||||||||' .\\
`||||||||||_,__o
"
}
#[tokio::main]
async fn main() {
setup_tracing();
info!("Starting up...");
let config = Config::init_with_defaults().unwrap();
// Start continuous profiling if enabled (keep _agent alive for the duration of the program)
let _profiling_agent = match config.continuous_profiling.start_agent() {
Ok(agent) => agent,
Err(e) => {
tracing::warn!("Failed to start continuous profiling agent: {e}");
None
}
};
let health_registry = HealthRegistry::new("liveness");
let logs_sink_liveness = health_registry
.register("rdkafka".to_string(), Duration::from_secs(30))
.await;
let traces_sink_liveness = health_registry
.register("rdkafka_traces".to_string(), Duration::from_secs(30))
.await;
let metrics_sink_liveness = health_registry
.register("rdkafka_metrics".to_string(), Duration::from_secs(30))
.await;
let kafka_sink = KafkaSink::new(
config.kafka.clone(),
logs_sink_liveness,
traces_sink_liveness,
metrics_sink_liveness,
)
.await
.expect("failed to start Kafka sink");
let management_router = Router::new()
.route("/", get(index))
.route("/_readiness", get(index))
.route(
"/_liveness",
get(move || ready(health_registry.get_status())),
);
let management_router = setup_metrics_routes(management_router);
let management_bind = format!("{}:{}", config.management_host, config.management_port);
info!("Healthcheck and metrics listening on {}", management_bind);
let management_listener = tokio::net::TcpListener::bind(management_bind)
.await
.expect("could not bind management port");
let token_dropper = TokenDropper::new(&config.drop_events_by_token.unwrap_or_default());
let authorizer = Authorizer::new(Arc::new(token_dropper));
let logs_service = match Service::new(
kafka_sink,
authorizer,
config.max_request_body_size_bytes,
config.firehose_max_request_body_size_bytes,
config.max_backfill_days,
)
.await
{
Ok(service) => service,
Err(e) => {
error!("Failed to initialize log service: {}", e);
panic!("Could not start log capture service: {e}");
}
};
let http_bind = format!("{}:{}", config.host, config.port);
info!("Listening on {}", http_bind);
let http_listener = tokio::net::TcpListener::bind(http_bind)
.await
.expect("could not bind http port");
// Very permissive CORS policy
let cors = CorsLayer::new()
.allow_methods([Method::GET, Method::POST, Method::OPTIONS])
.allow_headers(AllowHeaders::mirror_request())
.allow_credentials(true)
.allow_origin(AllowOrigin::mirror_request());
let http_router = Router::new()
.route("/v1/logs", post(export_logs_http).options(options_handler))
.route(
"/i/v1/logs",
post(export_logs_http).options(options_handler),
)
.route(
"/v1/traces",
post(export_traces_http).options(options_handler),
)
.route(
"/i/v1/traces",
post(export_traces_http).options(options_handler),
)
.route(
"/v1/metrics",
post(export_metrics_http).options(options_handler),
)
.route(
"/i/v1/metrics",
post(export_metrics_http).options(options_handler),
)
.route(
"/i/v1/logs/datadog",
post(datadog::export_datadog_logs_http).options(options_handler),
)
.route(
"/i/v1/logs/datadog/api/v2/logs",
post(datadog::export_datadog_logs_http).options(options_handler),
)
// allow setting the token directly in the route path
// as the datadog agent allows custom paths but not query strings or headers
.route(
"/i/v1/logs/datadog/:token",
post(datadog::export_datadog_logs_http).options(options_handler),
)
.route(
"/i/v1/logs/datadog/:token/api/v2/logs",
post(datadog::export_datadog_logs_http).options(options_handler),
)
.with_state(logs_service.clone())
.layer(DefaultBodyLimit::max(config.max_request_body_size_bytes))
.layer(axum::middleware::from_fn(track_metrics))
.layer(RequestDecompressionLayer::new())
.layer(axum::middleware::from_fn(translate_compression_query_param));
// Prometheus remote-write sends `Content-Encoding: snappy` and vmagent
// sends `Content-Encoding: zstd`, which RequestDecompressionLayer rejects
// with 415 before the handler runs. This route is deliberately kept off
// that layer (and the gzip query-param shim); the handler decodes the body
// itself.
let prometheus_router = Router::new()
.route(
"/i/v1/prometheus/write",
post(prometheus::export_prometheus_remote_write_http).options(options_handler),
)
.route(
"/i/v1/prometheus/write/:token",
post(prometheus::export_prometheus_remote_write_http).options(options_handler),
)
.with_state(logs_service.clone())
.layer(DefaultBodyLimit::max(config.max_request_body_size_bytes))
.layer(axum::middleware::from_fn(track_metrics));
// Amazon Data Firehose HTTP endpoint destination. Its own router because Firehose buffers up to
// 64 MiB per delivery, so the body cap is separate, and because Firehose only understands a
// JSON error body: the outermost layer re-shapes rejections raised by the layers below it.
let firehose_router = Router::new()
.route(
"/i/v1/logs/aws/firehose",
post(aws_firehose::export_aws_firehose_logs_http).options(options_handler),
)
.with_state(logs_service)
.layer(DefaultBodyLimit::max(
config.firehose_max_request_body_size_bytes,
))
.layer(axum::middleware::from_fn(track_metrics))
.layer(RequestDecompressionLayer::new())
.layer(axum::middleware::from_fn(
aws_firehose::shape_layer_rejections,
));
let http_router = http_router
.merge(prometheus_router)
.merge(firehose_router)
.layer(cors);
let http_server = tokio::spawn(async move {
if let Err(e) = axum::serve(
http_listener,
http_router.into_make_service_with_connect_info::<SocketAddr>(),
)
.with_graceful_shutdown(shutdown())
.await
{
error!("HTTP server failed: {}", e);
}
});
let mgmt_server = tokio::spawn(async move {
if let Err(e) = axum::serve(
management_listener,
management_router.into_make_service_with_connect_info::<SocketAddr>(),
)
.await
{
error!("Management server failed: {}", e);
}
});
// Wait for any server to finish
tokio::select! {
_ = http_server => {
error!("HTTP server stopped");
}
_ = mgmt_server => {
error!("Management server stopped");
}
}
}