:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use crate::authorizer::{Authorizer, Signal};
use crate::log_record::{default_max_past, KafkaLogRow};
use crate::metric_record::{flatten_metric, KafkaMetricRow};
use crate::trace_record::KafkaTraceRow;
use axum::{
extract::Query,
extract::State,
http::{header::CONTENT_TYPE, HeaderMap, StatusCode},
response::{IntoResponse, Json, Response},
};
use bytes::Bytes;
use chrono::TimeDelta;
use common_compression::{decompress_gzip_capped, has_gzip_magic_header, CompressionError};
use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest;
use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest;
use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
use prost::Message;
use serde::Deserialize;
use serde_json::{json, Value};
use std::fs::File;
use std::io::Write;
use crate::kafka::KafkaSink;
use metrics::counter;
use tracing::{debug, error, instrument};
// `patch_otel_json` normalises an OTLP/JSON payload so it can be deserialized by
// upstream `opentelemetry-proto`'s generated types. This is a workaround layer
// for three known upstream gaps; each `FIXME` below is independently removable
// once its upstream issue lands. We string-match JSON keys to compensate for type
// system gaps in a third-party crate — it is not elegant. The function is bounded,
// idempotent, and side-effect-free, and the regression tests in tests/metrics_test.rs
// (the `#[ignore]`-d `edge_*_should_error` tests in particular) will surface upstream
// changes — un-ignore them when the silencing pattern is fixed.
//
// FIXME(upstream-1253): empty `value: {}` AnyValue objects fail to deserialize.
// https://github.com/open-telemetry/opentelemetry-rust/issues/1253
// Workaround: replace empty `value`/`body` objects with `null`.
//
// FIXME(upstream-3328): several fixed64/uint64/sfixed64 fields in the metrics
// protos lack the `deserialize_string_to_u64` annotation, so the OTLP/JSON
// spec-canonical string encoding fails. Combined with the silencing pattern
// below, affected metrics end up with `data: None` instead of erroring.
// https://github.com/open-telemetry/opentelemetry-rust/issues/3328
// https://github.com/open-telemetry/opentelemetry-rust/pull/3329 (stale)
// Workaround: coerce string-encoded integers to JSON numbers for the affected
// fields (`count`, `zeroCount`, `asInt`, `bucketCounts[*]`, and timestamp
// descendants under `exponentialHistogram` / `summary` / `exemplars`).
//
// FIXME(upstream-unreported): `ExponentialHistogram`, `ExponentialHistogramDataPoint`,
// `SummaryDataPoint`, `Buckets`, and `Exemplar` all lack `#[serde(default)]`
// upstream, so any missing non-Option proto field hard-errors and trips the
// silencing pattern. No upstream issue filed yet — open one alongside removing
// this workaround. Workaround: the `fill_*_defaults` functions inject defaults
// for every required-by-serde field below.
//
// Silencing pattern (the reason these gaps silently drop metrics rather than
// returning a 400): `Metric` declares `data` as `#[serde(flatten)] Option<Data>`
// on a `#[serde(default)]` struct, so any error deserializing the inner oneof
// variant is swallowed and `data` becomes `None`. `flatten_metric` then emits
// zero rows with no log line. Until upstream changes the silencing structure
// itself, the only defense is to never let the inner deserialization fail.
pub fn patch_otel_json(v: &mut Value) {
match v {
Value::Object(map) => {
// In OTel, AnyValue is usually a field named "value"
// If we find "value": {}, change it to "value": null
if let Some(inner) = map.get_mut("value") {
if inner.is_object() && inner.as_object().map(|obj| obj.is_empty()).unwrap_or(false)
{
*inner = Value::Null;
}
}
// Handle empty body objects - body should be an AnyValue or null
if let Some(inner) = map.get_mut("body") {
if inner.is_object() && inner.as_object().map(|obj| obj.is_empty()).unwrap_or(false)
{
*inner = Value::Null;
}
}
// Universal string→integer coercions for u64/i64 fields that have NO custom
// deserializer in upstream opentelemetry-proto. Safe in any context.
for key in ["count", "zeroCount", "asInt"] {
if let Some(inner) = map.get_mut(key) {
coerce_string_to_integer(inner);
}
}
if let Some(Value::Array(arr)) = map.get_mut("bucketCounts") {
for el in arr.iter_mut() {
coerce_string_to_integer(el);
}
}
// Context-aware fixes for the metric variants whose data point types lack both
// the string→u64 deserializer and #[serde(default)] upstream. We coerce
// timestamp strings to numbers AND inject defaults for every non-Option proto
// field, since any missing field hard-errors and trips the silencing pattern.
if let Some(inner) = map.get_mut("exponentialHistogram") {
coerce_unix_nano_descendants(inner);
fill_exponential_histogram_defaults(inner);
}
if let Some(inner) = map.get_mut("summary") {
coerce_unix_nano_descendants(inner);
fill_summary_defaults(inner);
}
// Exemplar also lacks #[serde(default)] upstream — missing fields silently drop
// the parent metric, not just the exemplar. Fill defaults wherever we see one.
if let Some(Value::Array(arr)) = map.get_mut("exemplars") {
for el in arr.iter_mut() {
fill_exemplar_defaults(el);
}
}
// Recurse through all other keys
for (_, val) in map.iter_mut() {
patch_otel_json(val);
}
}
Value::Array(arr) => {
for val in arr.iter_mut() {
patch_otel_json(val);
}
}
_ => {}
}
}
fn coerce_string_to_integer(v: &mut Value) {
let Value::String(s) = v else { return };
if let Ok(n) = s.parse::<i64>() {
*v = Value::Number(n.into());
} else if let Ok(n) = s.parse::<u64>() {
*v = Value::Number(n.into());
}
}
fn number_u64(n: u64) -> Value {
Value::Number(n.into())
}
fn number_zero_f64() -> Value {
// serde_json::Number::from_f64 only fails for NaN/Infinity — never for 0.0.
Value::Number(serde_json::Number::from_f64(0.0).expect("0.0 is finite"))
}
fn empty_string() -> Value {
Value::String(String::new())
}
fn empty_array() -> Value {
Value::Array(Vec::new())
}
fn fill_exponential_histogram_defaults(variant: &mut Value) {
let Value::Object(map) = variant else { return };
// ExponentialHistogram itself lacks #[serde(default)].
map.entry("aggregationTemporality".to_string())
.or_insert_with(|| number_u64(0));
map.entry("dataPoints".to_string())
.or_insert_with(empty_array);
if let Some(Value::Array(arr)) = map.get_mut("dataPoints") {
for dp in arr.iter_mut() {
fill_exponential_histogram_dp_defaults(dp);
}
}
}
fn fill_exponential_histogram_dp_defaults(dp: &mut Value) {
let Value::Object(obj) = dp else { return };
obj.entry("attributes".to_string())
.or_insert_with(empty_array);
obj.entry("startTimeUnixNano".to_string())
.or_insert_with(|| number_u64(0));
obj.entry("timeUnixNano".to_string())
.or_insert_with(|| number_u64(0));
obj.entry("count".to_string())
.or_insert_with(|| number_u64(0));
obj.entry("scale".to_string())
.or_insert_with(|| number_u64(0));
obj.entry("zeroCount".to_string())
.or_insert_with(|| number_u64(0));
obj.entry("flags".to_string())
.or_insert_with(|| number_u64(0));
obj.entry("exemplars".to_string())
.or_insert_with(empty_array);
obj.entry("zeroThreshold".to_string())
.or_insert_with(number_zero_f64);
if let Some(buckets) = obj.get_mut("positive") {
fill_buckets_defaults(buckets);
}
if let Some(buckets) = obj.get_mut("negative") {
fill_buckets_defaults(buckets);
}
}
fn fill_buckets_defaults(buckets: &mut Value) {
let Value::Object(obj) = buckets else { return };
obj.entry("offset".to_string())
.or_insert_with(|| number_u64(0));
obj.entry("bucketCounts".to_string())
.or_insert_with(empty_array);
}
fn fill_summary_defaults(variant: &mut Value) {
// Summary itself has #[serde(default)] upstream, so the variant-level fields
// (dataPoints, aggregationTemporality if present) are tolerated when missing.
// SummaryDataPoint is the level that needs help.
let Value::Object(map) = variant else { return };
if let Some(Value::Array(arr)) = map.get_mut("dataPoints") {
for dp in arr.iter_mut() {
fill_summary_dp_defaults(dp);
}
}
}
fn fill_summary_dp_defaults(dp: &mut Value) {
let Value::Object(obj) = dp else { return };
obj.entry("attributes".to_string())
.or_insert_with(empty_array);
obj.entry("startTimeUnixNano".to_string())
.or_insert_with(|| number_u64(0));
obj.entry("timeUnixNano".to_string())
.or_insert_with(|| number_u64(0));
obj.entry("count".to_string())
.or_insert_with(|| number_u64(0));
obj.entry("sum".to_string()).or_insert_with(number_zero_f64);
obj.entry("quantileValues".to_string())
.or_insert_with(empty_array);
obj.entry("flags".to_string())
.or_insert_with(|| number_u64(0));
}
fn fill_exemplar_defaults(exemplar: &mut Value) {
let Value::Object(obj) = exemplar else { return };
obj.entry("filteredAttributes".to_string())
.or_insert_with(empty_array);
obj.entry("timeUnixNano".to_string())
.or_insert_with(|| number_u64(0));
obj.entry("spanId".to_string()).or_insert_with(empty_string);
obj.entry("traceId".to_string())
.or_insert_with(empty_string);
if let Some(t) = obj.get_mut("timeUnixNano") {
coerce_string_to_integer(t);
}
}
fn coerce_unix_nano_descendants(v: &mut Value) {
match v {
Value::Object(map) => {
for key in ["timeUnixNano", "startTimeUnixNano"] {
if let Some(inner) = map.get_mut(key) {
coerce_string_to_integer(inner);
}
}
for (_, val) in map.iter_mut() {
coerce_unix_nano_descendants(val);
}
}
Value::Array(arr) => {
for val in arr.iter_mut() {
coerce_unix_nano_descendants(val);
}
}
_ => {}
}
}
/// Parse OpenTelemetry log message from JSON bytes.
///
/// Supports both single JSON objects and JSONL format (JSON Lines).
/// For JSONL, multiple ExportLogsServiceRequest objects are parsed and merged
/// into a single request by combining their resource_logs arrays.
pub fn parse_otel_message(json_bytes: &Bytes) -> Result<ExportLogsServiceRequest, anyhow::Error> {
// First, attempt to parse the entire payload as a single JSON object.
// If this succeeds, we treat it as a normal ExportLogsServiceRequest.
if let Ok(mut v) = serde_json::from_slice::<Value>(json_bytes) {
patch_otel_json(&mut v);
let result: ExportLogsServiceRequest = serde_json::from_value(v)?;
return Ok(result);
}
// If parsing as a single JSON object fails, fall back to JSONL (JSON Lines)
// where each non-empty line is expected to be a complete JSON object.
let json_str = std::str::from_utf8(json_bytes)?;
let lines: Vec<&str> = json_str
.lines()
.filter(|line| !line.trim().is_empty())
.collect();
// Handle JSONL format - parse each line and merge them
let mut merged_request = ExportLogsServiceRequest {
resource_logs: Vec::new(),
};
for line in lines {
let mut v: Value = serde_json::from_str(line)?;
patch_otel_json(&mut v);
let request: ExportLogsServiceRequest = serde_json::from_value(v)?;
merged_request.resource_logs.extend(request.resource_logs);
}
Ok(merged_request)
}
#[derive(Clone)]
pub struct Service {
pub(crate) sink: KafkaSink,
pub(crate) authorizer: Authorizer,
pub(crate) max_request_body_size_bytes: usize,
pub(crate) firehose_max_request_body_size_bytes: usize,
pub(crate) max_backfill_days: u32,
}
#[derive(Deserialize)]
pub struct QueryParams {
token: Option<String>,
}
/// Logs-only, so the traces and metrics handlers cannot accept a parameter they do not honor.
#[derive(Deserialize)]
pub struct BackfillParams {
backfill_days: Option<u32>,
}
impl Service {
pub async fn new(
kafka_sink: KafkaSink,
authorizer: Authorizer,
max_request_body_size_bytes: usize,
firehose_max_request_body_size_bytes: usize,
max_backfill_days: u32,
) -> Result<Self, anyhow::Error> {
Ok(Self {
sink: kafka_sink,
authorizer,
max_request_body_size_bytes,
firehose_max_request_body_size_bytes,
max_backfill_days,
})
}
}
/// Gunzip `bytes` when they start with the gzip magic header, capped at `limit` decoded bytes.
/// `Ok(None)` means the input was not gzip and can be used as is.
pub(crate) fn gunzip_if_magic(
bytes: &[u8],
limit: usize,
) -> Result<Option<Vec<u8>>, (StatusCode, String)> {
if !has_gzip_magic_header(bytes) {
return Ok(None);
}
match decompress_gzip_capped(bytes, limit) {
Ok(decompressed) => Ok(Some(decompressed)),
Err(CompressionError::OutputTooLarge {
decompressed,
limit,
}) => Err((
StatusCode::PAYLOAD_TOO_LARGE,
format!("Decompressed request body exceeds limit ({decompressed} > {limit} bytes)"),
)),
Err(e) => Err((
StatusCode::BAD_REQUEST,
format!("Failed to decompress gzip request body: {e}"),
)),
}
}
/// An over-wide request is rejected, not narrowed to the default. A quietly narrowed import
/// writes most of its records onto the ingest time and still returns 200, so nobody notices.
pub(crate) fn resolve_backfill_window(
requested_days: Option<u32>,
max_backfill_days: u32,
) -> Result<TimeDelta, String> {
let Some(days) = requested_days else {
return Ok(default_max_past());
};
if max_backfill_days == 0 {
return Err("This deployment does not accept backdated logs. Remove backfill_days, or contact PostHog support to turn on historical imports.".to_string());
}
if days == 0 || days > max_backfill_days {
return Err(format!(
"backfill_days {days} is outside the accepted range. Use a value between 1 and {max_backfill_days}."
));
}
Ok(TimeDelta::days(days as i64))
}
pub(crate) fn decode_body_if_gzip_magic(
body: Bytes,
max_request_body_size_bytes: usize,
) -> Result<Bytes, (StatusCode, Json<serde_json::Value>)> {
match gunzip_if_magic(&body, max_request_body_size_bytes) {
Ok(None) => Ok(body),
Ok(Some(decompressed)) => Ok(Bytes::from(decompressed)),
Err((status, message)) => Err((status, Json(json!({"error": message})))),
}
}
/// Protobuf clients can't decode a JSON body.
pub fn export_success_response(headers: &HeaderMap) -> Response {
let is_protobuf = headers
.get(CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.and_then(|v| v.split(';').next())
.is_some_and(|mime| mime.trim().eq_ignore_ascii_case("application/x-protobuf"));
if is_protobuf {
([(CONTENT_TYPE, "application/x-protobuf")], Vec::<u8>::new()).into_response()
} else {
Json(json!({})).into_response()
}
}
// Forwarded unchecked, because only the logs consumer knows the team the token resolves to.
const INTERNAL_LOGS_HTTP_HEADER: &str = "x-posthog-internal";
const MAX_INTERNAL_LOGS_SIGNATURE_LEN: usize = 256;
fn internal_logs_signature(headers: &HeaderMap) -> Option<&str> {
headers
.get(INTERNAL_LOGS_HTTP_HEADER)?
.to_str()
.ok()
.filter(|signature| signature.len() <= MAX_INTERNAL_LOGS_SIGNATURE_LEN)
}
#[instrument(skip_all, fields(
token = tracing::field::Empty,
content_type = %headers.get("content-type")
.and_then(|v| v.to_str().ok())
.unwrap_or(""),
user_agent = %headers.get("user-agent")
.and_then(|v| v.to_str().ok())
.unwrap_or(""),
content_length = %headers.get("content-length")
.and_then(|v| v.to_str().ok())
.unwrap_or(""),
content_encoding = %headers.get("content-encoding")
.and_then(|v| v.to_str().ok())
.unwrap_or("")))
]
pub async fn export_logs_http(
State(service): State<Service>,
Query(query_params): Query<QueryParams>,
Query(backfill_params): Query<BackfillParams>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, (StatusCode, Json<serde_json::Value>)> {
let token =
service
.authorizer
.authorize(&headers, query_params.token.as_deref(), Signal::Logs)?;
tracing::Span::current().record("token", token);
let max_past =
resolve_backfill_window(backfill_params.backfill_days, service.max_backfill_days)
.map_err(|message| (StatusCode::BAD_REQUEST, Json(json!({ "error": message }))))?;
let body = decode_body_if_gzip_magic(body, service.max_request_body_size_bytes)?;
// Try to decode as Protobuf, if this fails, try JSON.
// We do this over relying on Content-Type headers to be as permissive as possible in what we accept.
let export_request = match ExportLogsServiceRequest::decode(body.as_ref()) {
Ok(request) => request,
Err(proto_err) => match parse_otel_message(&body) {
Ok(request) => request,
Err(json_err) => {
// Write last failed event to a file
// To make this super simple, we literally write a single event to /tmp/last_failed_event.txt
//
if let Err(e) = File::create("/tmp/last_failed_event.txt").and_then(|mut file|
// write the raw message to a file prepended by the token for debugging
file.write_all(token.as_bytes()).and_then(|_| file.write_all(&body)))
{
error!("Failed to write last failed event to file: {}", e);
}
error!(
"Failed to decode JSON: {} or Protobuf: {}",
json_err, proto_err
);
return Err((
StatusCode::BAD_REQUEST,
Json(
json!({"error": format!("Failed to decode JSON: {} or Protobuf: {}", json_err, proto_err)}),
),
));
}
},
};
let mut rows: Vec<KafkaLogRow> = Vec::new();
let mut timestamps_overridden: u64 = 0;
for resource_logs in export_request.resource_logs {
for scope_logs in resource_logs.scope_logs {
for log_record in scope_logs.log_records {
let (row, was_overridden) = match KafkaLogRow::new(
log_record,
resource_logs.resource.clone(),
scope_logs.scope.clone(),
max_past,
) {
Ok(result) => result,
Err(e) => {
error!("Failed to create LogRow: {e}");
return Err((
StatusCode::BAD_REQUEST,
Json(json!({"error": format!("Bad input format provided")})),
));
}
};
if was_overridden {
timestamps_overridden += 1;
}
rows.push(row);
}
}
}
let row_count = rows.len();
if let Err(e) = service
.sink
.write(
token,
rows,
body.len() as u64,
timestamps_overridden,
backfill_params.backfill_days,
internal_logs_signature(&headers),
)
.await
{
error!("Failed to send logs to Kafka: {}", e);
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({"error": format!("Internal server error")})),
));
} else {
debug!("Successfully sent {} logs to Kafka", row_count);
}
Ok(export_success_response(&headers))
}
/// Handle CORS preflight requests (OPTIONS method) for all log endpoints.
///
/// This endpoint supports all preflight requests by returning an empty JSON response.
/// The actual CORS headers are handled by the CorsLayer middleware in main.rs,
/// which provides a very permissive policy allowing all origins, methods, and headers
/// to support various SDK versions and reverse proxy configurations.
pub async fn options_handler(
) -> Result<Json<serde_json::Value>, (StatusCode, Json<serde_json::Value>)> {
Ok(Json(json!({})))
}
/// Parse OpenTelemetry trace message from JSON bytes.
///
/// Supports both single JSON objects and JSONL format (JSON Lines).
/// For JSONL, multiple ExportTraceServiceRequest objects are parsed and merged
/// into a single request by combining their resource_spans arrays.
pub fn parse_otel_traces_message(
json_bytes: &Bytes,
) -> Result<ExportTraceServiceRequest, anyhow::Error> {
if let Ok(mut v) = serde_json::from_slice::<Value>(json_bytes) {
patch_otel_json(&mut v);
let result: ExportTraceServiceRequest = serde_json::from_value(v)?;
return Ok(result);
}
let json_str = std::str::from_utf8(json_bytes)?;
let lines: Vec<&str> = json_str
.lines()
.filter(|line| !line.trim().is_empty())
.collect();
let mut merged_request = ExportTraceServiceRequest {
resource_spans: Vec::new(),
};
for line in lines {
let mut v: Value = serde_json::from_str(line)?;
patch_otel_json(&mut v);
let request: ExportTraceServiceRequest = serde_json::from_value(v)?;
merged_request.resource_spans.extend(request.resource_spans);
}
Ok(merged_request)
}
#[instrument(skip_all, fields(
token = tracing::field::Empty,
content_type = %headers.get("content-type")
.and_then(|v| v.to_str().ok())
.unwrap_or(""),
user_agent = %headers.get("user-agent")
.and_then(|v| v.to_str().ok())
.unwrap_or(""),
content_length = %headers.get("content-length")
.and_then(|v| v.to_str().ok())
.unwrap_or(""),
content_encoding = %headers.get("content-encoding")
.and_then(|v| v.to_str().ok())
.unwrap_or("")))
]
pub async fn export_traces_http(
State(service): State<Service>,
Query(query_params): Query<QueryParams>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, (StatusCode, Json<serde_json::Value>)> {
let token =
service
.authorizer
.authorize(&headers, query_params.token.as_deref(), Signal::Traces)?;
tracing::Span::current().record("token", token);
let body = decode_body_if_gzip_magic(body, service.max_request_body_size_bytes)?;
let export_request = match ExportTraceServiceRequest::decode(body.as_ref()) {
Ok(request) => request,
Err(proto_err) => match parse_otel_traces_message(&body) {
Ok(request) => request,
Err(json_err) => {
if let Err(e) =
File::create("/tmp/last_failed_trace_event.txt").and_then(|mut file| {
file.write_all(token.as_bytes())
.and_then(|_| file.write_all(&body))
})
{
error!("Failed to write last failed trace event to file: {}", e);
}
error!(
"Failed to decode JSON: {} or Protobuf: {}",
json_err, proto_err
);
return Err((
StatusCode::BAD_REQUEST,
Json(
json!({"error": format!("Failed to decode JSON: {} or Protobuf: {}", json_err, proto_err)}),
),
));
}
},
};
let mut rows: Vec<KafkaTraceRow> = Vec::new();
let mut timestamps_overridden: u64 = 0;
for resource_spans in export_request.resource_spans {
for scope_spans in resource_spans.scope_spans {
for span in scope_spans.spans {
let (row, was_overridden) = match KafkaTraceRow::new(
span,
resource_spans.resource.clone(),
scope_spans.scope.clone(),
) {
Ok(result) => result,
Err(e) => {
error!("Failed to create TraceRow: {e}");
return Err((
StatusCode::BAD_REQUEST,
Json(json!({"error": "Bad input format provided"})),
));
}
};
if was_overridden {
timestamps_overridden += 1;
}
rows.push(row);
}
}
}
let row_count = rows.len();
if let Err(e) = service
.sink
.write_traces(token, rows, body.len() as u64, timestamps_overridden)
.await
{
error!("Failed to send traces to Kafka: {}", e);
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({"error": "Internal server error"})),
));
} else {
debug!("Successfully sent {} traces to Kafka", row_count);
}
Ok(export_success_response(&headers))
}
/// Parse OpenTelemetry metric message from JSON bytes.
///
/// Supports both single JSON objects and JSONL format (JSON Lines).
/// For JSONL, multiple ExportMetricsServiceRequest objects are parsed and merged
/// into a single request by combining their resource_metrics arrays.
pub fn parse_otel_metrics_message(
json_bytes: &Bytes,
) -> Result<ExportMetricsServiceRequest, anyhow::Error> {
if let Ok(mut v) = serde_json::from_slice::<Value>(json_bytes) {
patch_otel_json(&mut v);
let result: ExportMetricsServiceRequest = serde_json::from_value(v)?;
return Ok(result);
}
let json_str = std::str::from_utf8(json_bytes)?;
let lines: Vec<&str> = json_str
.lines()
.filter(|line| !line.trim().is_empty())
.collect();
let mut merged_request = ExportMetricsServiceRequest {
resource_metrics: Vec::new(),
};
for line in lines {
let mut v: Value = serde_json::from_str(line)?;
patch_otel_json(&mut v);
let request: ExportMetricsServiceRequest = serde_json::from_value(v)?;
merged_request
.resource_metrics
.extend(request.resource_metrics);
}
Ok(merged_request)
}
#[instrument(skip_all, fields(
token = tracing::field::Empty,
content_type = %headers.get("content-type")
.and_then(|v| v.to_str().ok())
.unwrap_or(""),
user_agent = %headers.get("user-agent")
.and_then(|v| v.to_str().ok())
.unwrap_or(""),
content_length = %headers.get("content-length")
.and_then(|v| v.to_str().ok())
.unwrap_or(""),
content_encoding = %headers.get("content-encoding")
.and_then(|v| v.to_str().ok())
.unwrap_or("")))
]
pub async fn export_metrics_http(
State(service): State<Service>,
Query(query_params): Query<QueryParams>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, (StatusCode, Json<serde_json::Value>)> {
let token =
service
.authorizer
.authorize(&headers, query_params.token.as_deref(), Signal::Metrics)?;
tracing::Span::current().record("token", token);
let body = decode_body_if_gzip_magic(body, service.max_request_body_size_bytes)?;
let export_request = match ExportMetricsServiceRequest::decode(body.as_ref()) {
Ok(request) => request,
Err(proto_err) => match parse_otel_metrics_message(&body) {
Ok(request) => request,
Err(json_err) => {
if let Err(e) =
File::create("/tmp/last_failed_metric_event.txt").and_then(|mut file| {
file.write_all(token.as_bytes())
.and_then(|_| file.write_all(&body))
})
{
error!("Failed to write last failed metric event to file: {}", e);
}
error!(
"Failed to decode JSON: {} or Protobuf: {}",
json_err, proto_err
);
return Err((
StatusCode::BAD_REQUEST,
Json(
json!({"error": format!("Failed to decode JSON: {} or Protobuf: {}", json_err, proto_err)}),
),
));
}
},
};
let mut rows: Vec<KafkaMetricRow> = Vec::new();
let mut timestamps_overridden: u64 = 0;
for resource_metrics in &export_request.resource_metrics {
for scope_metrics in &resource_metrics.scope_metrics {
for metric in &scope_metrics.metrics {
let (metric_rows, overridden) = match flatten_metric(
metric.clone(),
resource_metrics.resource.as_ref(),
scope_metrics.scope.as_ref(),
) {
Ok(result) => result,
Err(e) => {
error!("Failed to flatten metric: {e}");
return Err((
StatusCode::BAD_REQUEST,
Json(json!({"error": "Bad input format provided"})),
));
}
};
timestamps_overridden += overridden;
rows.extend(metric_rows);
}
}
}
let row_count = rows.len();
if let Err(e) = service
.sink
.write_metrics(token, rows, body.len() as u64, timestamps_overridden)
.await
{
error!("Failed to send metrics to Kafka: {}", e);
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({"error": "Internal server error"})),
));
} else {
counter!("capture_metrics_records_produced", "source" => "otlp")
.increment(row_count as u64);
debug!(
"Successfully sent {} metric data points to Kafka",
row_count
);
}
Ok(export_success_response(&headers))
}
#[cfg(test)]
mod tests {
use super::*;
use flate2::{write::GzEncoder, Compression};
fn gzip(data: &[u8]) -> Bytes {
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
encoder.write_all(data).unwrap();
Bytes::from(encoder.finish().unwrap())
}
#[tokio::test]
async fn export_success_response_matches_request_encoding() {
let cases = [
(
Some("application/x-protobuf"),
"application/x-protobuf",
&b""[..],
),
(
Some("Application/X-Protobuf; charset=binary"),
"application/x-protobuf",
&b""[..],
),
(Some("application/json"), "application/json", &b"{}"[..]),
(None, "application/json", &b"{}"[..]),
];
for (request_content_type, expected_content_type, expected_body) in cases {
let mut headers = HeaderMap::new();
if let Some(content_type) = request_content_type {
headers.insert(CONTENT_TYPE, content_type.parse().unwrap());
}
let response = export_success_response(&headers);
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.headers().get(CONTENT_TYPE).unwrap(),
expected_content_type,
"request content type {request_content_type:?}"
);
let body = axum::body::to_bytes(response.into_body(), usize::MAX)
.await
.unwrap();
assert_eq!(body.as_ref(), expected_body);
}
}
#[test]
fn decode_body_if_gzip_magic_decompresses_without_header() {
let body = gzip(br#"{"resourceLogs":[]}"#);
let decoded = decode_body_if_gzip_magic(body, 1024).unwrap();
assert_eq!(decoded, Bytes::from_static(br#"{"resourceLogs":[]}"#));
}
#[test]
fn resolve_backfill_window_defaults_when_no_window_is_requested() {
for (case, ceiling) in [("backfill off", 0), ("backfill on", 400)] {
let window = resolve_backfill_window(None, ceiling);
assert_eq!(window, Ok(default_max_past()), "{case}");
}
}
#[test]
fn resolve_backfill_window_accepts_a_window_within_the_ceiling() {
for (case, days, ceiling) in [
("at the ceiling", 400, 400),
("under the ceiling", 30, 400),
("the smallest window", 1, 400),
] {
let window = resolve_backfill_window(Some(days), ceiling);
assert_eq!(window, Ok(TimeDelta::days(days as i64)), "{case}");
}
}
#[test]
fn resolve_backfill_window_rejects_a_window_it_cannot_grant() {
for (case, days, ceiling) in [
("above the ceiling", 401, 400),
("zero is not a window", 0, 400),
("any window where backfill is off", 1, 0),
] {
let window = resolve_backfill_window(Some(days), ceiling);
assert!(window.is_err(), "{case}");
}
}
#[test]
fn internal_logs_signature_forwards_only_a_bounded_header() {
let at_cap = "a".repeat(MAX_INTERNAL_LOGS_SIGNATURE_LEN);
let over_cap = "a".repeat(MAX_INTERNAL_LOGS_SIGNATURE_LEN + 1);
for (case, value, expected) in [
(
"a signature",
Some("v1.hog_flow.abc"),
Some("v1.hog_flow.abc"),
),
(
"a signature at the cap",
Some(at_cap.as_str()),
Some(at_cap.as_str()),
),
("a signature over the cap", Some(over_cap.as_str()), None),
("no header", None, None),
] {
let mut headers = HeaderMap::new();
if let Some(value) = value {
headers.insert(INTERNAL_LOGS_HTTP_HEADER, value.parse().unwrap());
}
assert_eq!(internal_logs_signature(&headers), expected, "{case}");
}
}
#[test]
fn decode_body_if_gzip_magic_preserves_plain_body() {
let body = Bytes::from_static(br#"{"resourceLogs":[]}"#);
let decoded = decode_body_if_gzip_magic(body.clone(), 1024).unwrap();
assert_eq!(decoded, body);
}
#[test]
fn decode_body_if_gzip_magic_rejects_oversized_output() {
let body = gzip(&vec![b'a'; 2048]);
let err = decode_body_if_gzip_magic(body, 1024).unwrap_err();
assert_eq!(err.0, StatusCode::PAYLOAD_TOO_LARGE);
}
}