Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(logs): forward the internal logs signature in capture-logs (#123930)

Jon McCallum492a959ae355
diff --git a/rust/capture-logs/src/endpoints/aws_firehose.rs b/rust/capture-logs/src/endpoints/aws_firehose.rs
index 0ef169412629..686627e1d05b 100644
--- a/rust/capture-logs/src/endpoints/aws_firehose.rs
+++ b/rust/capture-logs/src/endpoints/aws_firehose.rs
@@ -512,6 +512,7 @@ fn spawn_write(
batch.bytes,
batch.timestamps_overridden,
None,
+ None,
)
.await
});
diff --git a/rust/capture-logs/src/endpoints/datadog.rs b/rust/capture-logs/src/endpoints/datadog.rs
index 94e29545906e..6555d07dbceb 100644
--- a/rust/capture-logs/src/endpoints/datadog.rs
+++ b/rust/capture-logs/src/endpoints/datadog.rs
@@ -339,7 +339,14 @@ pub async fn export_datadog_logs_http(
let rows: Vec<KafkaLogRow> = results.into_iter().map(|(row, _)| row).collect();
if let Err(e) = service
.sink
- .write(&token, rows, body.len() as u64, timestamps_overridden, None)
+ .write(
+ &token,
+ rows,
+ body.len() as u64,
+ timestamps_overridden,
+ None,
+ None,
+ )
.await
{
error!("Failed to send logs to Kafka: {}", e);
diff --git a/rust/capture-logs/src/kafka.rs b/rust/capture-logs/src/kafka.rs
index 31ad1f3d1713..ebf0549d3d78 100644
--- a/rust/capture-logs/src/kafka.rs
+++ b/rust/capture-logs/src/kafka.rs
@@ -22,6 +22,9 @@ use std::sync::Arc;
use std::time::Duration;
use tracing::log::{debug, info};
+// The logs consumer verifies this signature against the team it resolves from the token.
+pub const INTERNAL_LOGS_KAFKA_HEADER: &str = "posthog-internal";
+
struct KafkaContext {
liveness: HealthHandle,
}
@@ -402,6 +405,7 @@ impl KafkaSink {
timestamps_overridden: u64,
min_timestamp_micros: Option<i64>,
backfill_days: Option<u32>,
+ internal_signature: Option<&str>,
) -> Result<(), anyhow::Error> {
let mut writer = Writer::with_codec(
schema,
@@ -454,6 +458,12 @@ impl KafkaSink {
value: Some(&days.to_string()),
});
}
+ if let Some(signature) = internal_signature {
+ headers = headers.insert(Header {
+ key: INTERNAL_LOGS_KAFKA_HEADER,
+ value: Some(signature),
+ });
+ }
headers
.insert(Header {
key: "bytes_compressed",
@@ -494,6 +504,7 @@ impl KafkaSink {
uncompressed_bytes: u64,
timestamps_overridden: u64,
backfill_days: Option<u32>,
+ internal_signature: Option<&str>,
) -> Result<(), anyhow::Error> {
if rows.is_empty() {
return Ok(());
@@ -523,6 +534,7 @@ impl KafkaSink {
timestamps_overridden,
min_kafka_log_row_timestamp_micros(&rows),
backfill_days,
+ internal_signature,
)
.await?;
@@ -555,6 +567,7 @@ impl KafkaSink {
timestamps_overridden,
None,
None,
+ None,
)
.await?;
@@ -587,6 +600,7 @@ impl KafkaSink {
timestamps_overridden,
None,
None,
+ None,
)
.await?;
diff --git a/rust/capture-logs/src/service.rs b/rust/capture-logs/src/service.rs
index cbd3a0379bc0..602db2283c92 100644
--- a/rust/capture-logs/src/service.rs
+++ b/rust/capture-logs/src/service.rs
@@ -413,6 +413,18 @@ pub fn export_success_response(headers: &HeaderMap) -> Response {
}
}
+// Forwarded unchecked, because only the logs consumer knows the team the token resolves to.
+const INTERNAL_LOGS_HTTP_HEADER: &str = "x-posthog-internal";
+const MAX_INTERNAL_LOGS_SIGNATURE_LEN: usize = 256;
+
+fn internal_logs_signature(headers: &HeaderMap) -> Option<&str> {
+ headers
+ .get(INTERNAL_LOGS_HTTP_HEADER)?
+ .to_str()
+ .ok()
+ .filter(|signature| signature.len() <= MAX_INTERNAL_LOGS_SIGNATURE_LEN)
+}
+
#[instrument(skip_all, fields(
token = tracing::field::Empty,
content_type = %headers.get("content-type")
@@ -515,6 +527,7 @@ pub async fn export_logs_http(
body.len() as u64,
timestamps_overridden,
backfill_params.backfill_days,
+ internal_logs_signature(&headers),
)
.await
{
@@ -912,6 +925,33 @@ mod tests {
}
}
+ #[test]
+ fn internal_logs_signature_forwards_only_a_bounded_header() {
+ let at_cap = "a".repeat(MAX_INTERNAL_LOGS_SIGNATURE_LEN);
+ let over_cap = "a".repeat(MAX_INTERNAL_LOGS_SIGNATURE_LEN + 1);
+ for (case, value, expected) in [
+ (
+ "a signature",
+ Some("v1.hog_flow.abc"),
+ Some("v1.hog_flow.abc"),
+ ),
+ (
+ "a signature at the cap",
+ Some(at_cap.as_str()),
+ Some(at_cap.as_str()),
+ ),
+ ("a signature over the cap", Some(over_cap.as_str()), None),
+ ("no header", None, None),
+ ] {
+ let mut headers = HeaderMap::new();
+ if let Some(value) = value {
+ headers.insert(INTERNAL_LOGS_HTTP_HEADER, value.parse().unwrap());
+ }
+
+ assert_eq!(internal_logs_signature(&headers), expected, "{case}");
+ }
+ }
+
#[test]
fn decode_body_if_gzip_magic_preserves_plain_body() {
let body = Bytes::from_static(br#"{"resourceLogs":[]}"#);