:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(logs): forward the internal logs signature in capture-logs (#123930)
Jon McCallum492a959ae355
diff --git a/rust/capture-logs/src/endpoints/aws_firehose.rs b/rust/capture-logs/src/endpoints/aws_firehose.rsindex 0ef169412629..686627e1d05b 100644--- a/rust/capture-logs/src/endpoints/aws_firehose.rs+++ b/rust/capture-logs/src/endpoints/aws_firehose.rs@@ -512,6 +512,7 @@ fn spawn_write(batch.bytes,batch.timestamps_overridden,None,+ None,).await});diff --git a/rust/capture-logs/src/endpoints/datadog.rs b/rust/capture-logs/src/endpoints/datadog.rsindex 94e29545906e..6555d07dbceb 100644--- a/rust/capture-logs/src/endpoints/datadog.rs+++ b/rust/capture-logs/src/endpoints/datadog.rs@@ -339,7 +339,14 @@ pub async fn export_datadog_logs_http(let rows: Vec<KafkaLogRow> = results.into_iter().map(|(row, _)| row).collect();if let Err(e) = service.sink- .write(&token, rows, body.len() as u64, timestamps_overridden, None)+ .write(+ &token,+ rows,+ body.len() as u64,+ timestamps_overridden,+ None,+ None,+ ).await{error!("Failed to send logs to Kafka: {}", e);diff --git a/rust/capture-logs/src/kafka.rs b/rust/capture-logs/src/kafka.rsindex 31ad1f3d1713..ebf0549d3d78 100644--- a/rust/capture-logs/src/kafka.rs+++ b/rust/capture-logs/src/kafka.rs@@ -22,6 +22,9 @@ use std::sync::Arc;use std::time::Duration;use tracing::log::{debug, info};+// The logs consumer verifies this signature against the team it resolves from the token.+pub const INTERNAL_LOGS_KAFKA_HEADER: &str = "posthog-internal";+struct KafkaContext {liveness: HealthHandle,}@@ -402,6 +405,7 @@ impl KafkaSink {timestamps_overridden: u64,min_timestamp_micros: Option<i64>,backfill_days: Option<u32>,+ internal_signature: Option<&str>,) -> Result<(), anyhow::Error> {let mut writer = Writer::with_codec(schema,@@ -454,6 +458,12 @@ impl KafkaSink {value: Some(&days.to_string()),});}+ if let Some(signature) = internal_signature {+ headers = headers.insert(Header {+ key: INTERNAL_LOGS_KAFKA_HEADER,+ value: Some(signature),+ });+ }headers.insert(Header {key: "bytes_compressed",@@ -494,6 +504,7 @@ impl KafkaSink {uncompressed_bytes: u64,timestamps_overridden: u64,backfill_days: Option<u32>,+ internal_signature: Option<&str>,) -> Result<(), anyhow::Error> {if rows.is_empty() {return Ok(());@@ -523,6 +534,7 @@ impl KafkaSink {timestamps_overridden,min_kafka_log_row_timestamp_micros(&rows),backfill_days,+ internal_signature,).await?;@@ -555,6 +567,7 @@ impl KafkaSink {timestamps_overridden,None,None,+ None,).await?;@@ -587,6 +600,7 @@ impl KafkaSink {timestamps_overridden,None,None,+ None,).await?;diff --git a/rust/capture-logs/src/service.rs b/rust/capture-logs/src/service.rsindex cbd3a0379bc0..602db2283c92 100644--- a/rust/capture-logs/src/service.rs+++ b/rust/capture-logs/src/service.rs@@ -413,6 +413,18 @@ pub fn export_success_response(headers: &HeaderMap) -> Response {}}+// Forwarded unchecked, because only the logs consumer knows the team the token resolves to.+const INTERNAL_LOGS_HTTP_HEADER: &str = "x-posthog-internal";+const MAX_INTERNAL_LOGS_SIGNATURE_LEN: usize = 256;++fn internal_logs_signature(headers: &HeaderMap) -> Option<&str> {+ headers+ .get(INTERNAL_LOGS_HTTP_HEADER)?+ .to_str()+ .ok()+ .filter(|signature| signature.len() <= MAX_INTERNAL_LOGS_SIGNATURE_LEN)+}+#[instrument(skip_all, fields(token = tracing::field::Empty,content_type = %headers.get("content-type")@@ -515,6 +527,7 @@ pub async fn export_logs_http(body.len() as u64,timestamps_overridden,backfill_params.backfill_days,+ internal_logs_signature(&headers),).await{@@ -912,6 +925,33 @@ mod tests {}}+ #[test]+ fn internal_logs_signature_forwards_only_a_bounded_header() {+ let at_cap = "a".repeat(MAX_INTERNAL_LOGS_SIGNATURE_LEN);+ let over_cap = "a".repeat(MAX_INTERNAL_LOGS_SIGNATURE_LEN + 1);+ for (case, value, expected) in [+ (+ "a signature",+ Some("v1.hog_flow.abc"),+ Some("v1.hog_flow.abc"),+ ),+ (+ "a signature at the cap",+ Some(at_cap.as_str()),+ Some(at_cap.as_str()),+ ),+ ("a signature over the cap", Some(over_cap.as_str()), None),+ ("no header", None, None),+ ] {+ let mut headers = HeaderMap::new();+ if let Some(value) = value {+ headers.insert(INTERNAL_LOGS_HTTP_HEADER, value.parse().unwrap());+ }++ assert_eq!(internal_logs_signature(&headers), expected, "{case}");+ }+ }+#[test]fn decode_body_if_gzip_magic_preserves_plain_body() {let body = Bytes::from_static(br#"{"resourceLogs":[]}"#);