Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(capture): name the v1 field with the wrong JSON type in the 400 (#120820)

Eli Reismanbaf885696441
diff --git a/rust/capture/src/v1/analytics/handler.rs b/rust/capture/src/v1/analytics/handler.rs
index 194736b89327..fb64ffbd83bc 100644
--- a/rust/capture/src/v1/analytics/handler.rs
+++ b/rust/capture/src/v1/analytics/handler.rs
@@ -6,6 +6,7 @@ use axum_client_ip::InsecureClientIp;
use super::constants::{CAPTURE_V1_PATH, CAPTURE_V1_PATHS};
use super::context::Context;
+use super::lenient::parse_batch;
use super::types::Batch;
use tracing::Level;
@@ -86,8 +87,8 @@ pub async fn handle_request(
metrics::histogram!(CAPTURE_V1_PAYLOAD_SIZE, "stage" => "decompressed", "encoding" => v1::util::encoding_tag(context.content_encoding.as_deref()))
.record(payload.len() as f64);
- let batch: Batch = serde_json::from_slice(&payload).map_err(|e| {
- let err = v1::Error::RequestParsingError(e.to_string());
+ let batch: Batch = parse_batch(&payload).map_err(|failure| {
+ let err = v1::Error::RequestParsingError(failure);
log_stat_error!(err, &context);
err
})?;
@@ -191,6 +192,7 @@ mod tests {
use axum::body::Body;
use axum::http::{Request, StatusCode};
use axum::Router;
+ use rstest::rstest;
use tower::ServiceExt;
use uuid::Uuid;
@@ -328,18 +330,60 @@ mod tests {
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
}
+ #[rstest]
+ #[case::invalid_json(
+ "not json at all {{{",
+ "Failed to parse request body: invalid JSON at line 1, column 2."
+ )]
+ #[case::wrong_type_distinct_id(
+ r#"{"created_at":"2026-03-19T14:30:00Z","batch":[{"event":"e","uuid":"019a0000-0000-7000-8000-000000000001","distinct_id":12345678901234567890123,"timestamp":"2026-03-19T14:30:00Z"}]}"#,
+ "Failed to parse request body: batch[0].distinct_id must be a string, got number."
+ )]
+ #[case::missing_event_name_in_later_event(
+ r#"{"created_at":"2026-03-19T14:30:00Z","batch":[{"event":"e","uuid":"019a0000-0000-7000-8000-000000000001","distinct_id":"d","timestamp":"2026-03-19T14:30:00Z"},{"uuid":"019a0000-0000-7000-8000-000000000002","distinct_id":"[email protected]","timestamp":"2026-03-19T14:30:00Z"}]}"#,
+ "Failed to parse request body: batch[1].event is missing."
+ )]
+ #[case::wrong_type_created_at(
+ r#"{"created_at":{"secret":"[email protected]"},"batch":[]}"#,
+ "Failed to parse request body: created_at must be a string, got object."
+ )]
+ #[case::wrong_type_batch(
+ r#"{"created_at":"2026-03-19T14:30:00Z","batch":"[email protected]"}"#,
+ "Failed to parse request body: wrong JSON type or value at line 1, column 64."
+ )]
+ #[case::truncated(
+ r#"{"created_at":"2026-03-19T14:30:00Z","batch":[{"event":"e""#,
+ "Failed to parse request body: unexpected end of JSON at line 1, column 58."
+ )]
#[tokio::test]
- async fn malformed_json() {
+ async fn unparseable_body_describes_the_problem_without_content(
+ #[case] body: &str,
+ #[case] expected_description: &str,
+ ) {
let ts = TestStateBuilder::new().build();
let app = test_app(ts.state);
- let garbage = b"not json at all {{{".to_vec();
let resp = app
- .oneshot(valid_request().body(Body::from(garbage)).unwrap())
+ .oneshot(valid_request().body(Body::from(body.to_owned())).unwrap())
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
+ let json = response_json(resp).await;
+ let raw = json.to_string();
+ for value in [
+ "12345678901234567890123",
+ "1.2345678901234568e22",
+ "secret",
+ ] {
+ assert!(
+ !raw.contains(value),
+ "response echoes request content: {raw}"
+ );
+ }
+ assert_eq!(json["error"], "request_parsing_error");
+ assert_eq!(json["error_description"], expected_description);
}
#[tokio::test]
diff --git a/rust/capture/src/v1/analytics/lenient.rs b/rust/capture/src/v1/analytics/lenient.rs
new file mode 100644
index 000000000000..580dbff79642
--- /dev/null
+++ b/rust/capture/src/v1/analytics/lenient.rs
@@ -0,0 +1,530 @@
+use std::cell::Cell;
+use std::fmt;
+
+use serde::de::{self, Deserializer, IgnoredAny, MapAccess, SeqAccess, Visitor};
+use serde::Deserialize;
+use serde_json::value::RawValue;
+
+use super::types::{
+ deserialize_lenient_flag, deserialize_lenient_flag_or_false, empty_raw_object, trim_owned,
+ Batch, Event, RawOptions,
+};
+use crate::v1::error::ParseFailure;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum JsonKind {
+ Missing,
+ Null,
+ Number,
+ Boolean,
+ Array,
+ Object,
+}
+
+impl JsonKind {
+ pub fn as_str(self) -> &'static str {
+ match self {
+ Self::Missing => "missing",
+ Self::Null => "null",
+ Self::Number => "number",
+ Self::Boolean => "boolean",
+ Self::Array => "array",
+ Self::Object => "object",
+ }
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FieldTypeError {
+ pub index: Option<usize>,
+ pub field: &'static str,
+ pub kind: JsonKind,
+}
+
+impl fmt::Display for FieldTypeError {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self.index {
+ Some(i) => write!(f, "batch[{i}].{}", self.field)?,
+ None => f.write_str(self.field)?,
+ }
+ match self.kind {
+ JsonKind::Missing => f.write_str(" is missing"),
+ kind => write!(f, " must be a string, got {}", kind.as_str()),
+ }
+ }
+}
+
+/// Keeps only a non-string's JSON kind, so the 400 names the field without echoing the value.
+#[derive(Debug, Default)]
+enum JsonStr {
+ #[default]
+ Missing,
+ Null,
+ Str(String),
+ Wrong(JsonKind),
+}
+
+impl<'de> Deserialize<'de> for JsonStr {
+ fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+ deserializer.deserialize_any(JsonStrVisitor)
+ }
+}
+
+struct JsonStrVisitor;
+
+impl<'de> Visitor<'de> for JsonStrVisitor {
+ type Value = JsonStr;
+
+ fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.write_str("any JSON value")
+ }
+
+ fn visit_str<E: de::Error>(self, v: &str) -> Result<JsonStr, E> {
+ Ok(JsonStr::Str(v.to_owned()))
+ }
+
+ fn visit_string<E: de::Error>(self, v: String) -> Result<JsonStr, E> {
+ Ok(JsonStr::Str(v))
+ }
+
+ fn visit_unit<E: de::Error>(self) -> Result<JsonStr, E> {
+ Ok(JsonStr::Null)
+ }
+
+ fn visit_none<E: de::Error>(self) -> Result<JsonStr, E> {
+ Ok(JsonStr::Null)
+ }
+
+ fn visit_bool<E: de::Error>(self, _: bool) -> Result<JsonStr, E> {
+ Ok(JsonStr::Wrong(JsonKind::Boolean))
+ }
+
+ fn visit_i64<E: de::Error>(self, _: i64) -> Result<JsonStr, E> {
+ Ok(JsonStr::Wrong(JsonKind::Number))
+ }
+
+ fn visit_u64<E: de::Error>(self, _: u64) -> Result<JsonStr, E> {
+ Ok(JsonStr::Wrong(JsonKind::Number))
+ }
+
+ fn visit_f64<E: de::Error>(self, _: f64) -> Result<JsonStr, E> {
+ Ok(JsonStr::Wrong(JsonKind::Number))
+ }
+
+ fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<JsonStr, A::Error> {
+ while seq.next_element::<IgnoredAny>()?.is_some() {}
+ Ok(JsonStr::Wrong(JsonKind::Array))
+ }
+
+ fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<JsonStr, A::Error> {
+ while map.next_entry::<IgnoredAny, IgnoredAny>()?.is_some() {}
+ Ok(JsonStr::Wrong(JsonKind::Object))
+ }
+}
+
+/// Lenient mirror of [`Event`] so a wrong type gets a 400 naming the field; keep `Event`'s field order (serde also reads the array form).
+#[derive(Deserialize)]
+struct LenientEvent {
+ #[serde(default)]
+ event: JsonStr,
+ #[serde(default)]
+ uuid: JsonStr,
+ #[serde(default)]
+ distinct_id: JsonStr,
+ #[serde(default)]
+ timestamp: JsonStr,
+ // `Option` without a default, as in `Event`, so a short array form still fails in serde.
+ session_id: Option<JsonStr>,
+ window_id: Option<JsonStr>,
+ #[serde(default)]
+ options: RawOptions,
+ #[serde(default = "empty_raw_object")]
+ properties: Box<RawValue>,
+}
+
+/// Lenient mirror of [`Batch`] so a wrong-typed `created_at` gets a 400 naming the field.
+#[derive(Deserialize)]
+struct LenientBatch {
+ #[serde(default)]
+ created_at: JsonStr,
+ #[serde(default, deserialize_with = "deserialize_lenient_flag_or_false")]
+ historical_migration: bool,
+ #[serde(default, deserialize_with = "deserialize_lenient_flag")]
+ capture_internal: Option<bool>,
+ #[serde(deserialize_with = "deserialize_events")]
+ batch: Vec<Event>,
+}
+
+thread_local! {
+ // serde returns only its own error type, so the field error travels beside it.
+ static FIELD_ERROR: Cell<Option<FieldTypeError>> = const { Cell::new(None) };
+}
+
+/// Builds the [`Batch`] the strict types build, or names the first non-string field; event errors win over `created_at` whatever the key order.
+pub fn parse_batch(body: &[u8]) -> Result<Batch, ParseFailure> {
+ // Clear a value a panicked parse on this thread left behind.
+ FIELD_ERROR.set(None);
+ let lenient = serde_json::from_slice::<LenientBatch>(body).map_err(|err| {
+ FIELD_ERROR
+ .take()
+ .map_or_else(|| err.into(), ParseFailure::Field)
+ })?;
+ lenient.into_batch().map_err(ParseFailure::Field)
+}
+
+/// Converts each event as serde reads it and aborts at the first bad field, so a bad body holds no more than the strict parse would.
+fn deserialize_events<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Vec<Event>, D::Error> {
+ deserializer.deserialize_seq(EventsVisitor)
+}
+
+struct EventsVisitor;
+
+impl<'de> Visitor<'de> for EventsVisitor {
+ type Value = Vec<Event>;
+
+ fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.write_str("a sequence")
+ }
+
+ fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Vec<Event>, A::Error> {
+ let mut events = Vec::new();
+ while let Some(event) = seq.next_element::<LenientEvent>()? {
+ match event.into_event(events.len()) {
+ Ok(event) => events.push(event),
+ Err(field) => {
+ FIELD_ERROR.set(Some(field));
+ return Err(de::Error::custom("field type error"));
+ }
+ }
+ }
+ Ok(events)
+ }
+}
+
+fn required(
+ value: JsonStr,
+ index: Option<usize>,
+ field: &'static str,
+) -> Result<String, FieldTypeError> {
+ match value {
+ JsonStr::Str(s) => Ok(s),
+ JsonStr::Missing => Err(FieldTypeError {
+ index,
+ field,
+ kind: JsonKind::Missing,
+ }),
+ JsonStr::Null => Err(FieldTypeError {
+ index,
+ field,
+ kind: JsonKind::Null,
+ }),
+ JsonStr::Wrong(kind) => Err(FieldTypeError { index, field, kind }),
+ }
+}
+
+fn optional(
+ value: Option<JsonStr>,
+ index: usize,
+ field: &'static str,
+) -> Result<Option<String>, FieldTypeError> {
+ match value {
+ None | Some(JsonStr::Null) | Some(JsonStr::Missing) => Ok(None),
+ Some(JsonStr::Str(s)) => Ok(Some(s)),
+ Some(JsonStr::Wrong(kind)) => Err(FieldTypeError {
+ index: Some(index),
+ field,
+ kind,
+ }),
+ }
+}
+
+impl LenientEvent {
+ fn into_event(self, index: usize) -> Result<Event, FieldTypeError> {
+ let i = Some(index);
+ Ok(Event {
+ event: required(self.event, i, "event")?,
+ uuid: trim_owned(required(self.uuid, i, "uuid")?),
+ distinct_id: trim_owned(required(self.distinct_id, i, "distinct_id")?),
+ timestamp: required(self.timestamp, i, "timestamp")?,
+ session_id: optional(self.session_id, index, "session_id")?,
+ window_id: optional(self.window_id, index, "window_id")?,
+ options: self.options,
+ properties: self.properties,
+ })
+ }
+}
+
+impl LenientBatch {
+ fn into_batch(self) -> Result<Batch, FieldTypeError> {
+ Ok(Batch {
+ created_at: required(self.created_at, None, "created_at")?,
+ historical_migration: self.historical_migration,
+ capture_internal: self.capture_internal,
+ batch: self.batch,
+ })
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use rstest::rstest;
+ use serde_json::{json, Value};
+
+ const PII: &str = "[email protected]";
+ const BIG: u64 = 12345678901234567890;
+
+ fn field_error(body: &[u8]) -> FieldTypeError {
+ match parse_batch(body) {
+ Err(ParseFailure::Field(field)) => field,
+ other => panic!("expected a field error, got {:?}", other.map(|_| ())),
+ }
+ }
+
+ fn valid_event() -> Value {
+ json!({
+ "event": "$pageview",
+ "uuid": "019a0000-0000-7000-8000-000000000001",
+ "distinct_id": "user-1",
+ "timestamp": "2026-10-21T00:00:00.000Z",
+ })
+ }
+
+ fn body_with(
+ batch_fields: &[(&str, Option<Value>)],
+ event_fields: &[(&str, Option<Value>)],
+ ) -> Vec<u8> {
+ let mut event = valid_event();
+ for (key, value) in event_fields {
+ match value {
+ Some(v) => event[*key] = v.clone(),
+ None => {
+ event.as_object_mut().unwrap().remove(*key);
+ }
+ }
+ }
+ let mut batch = json!({ "created_at": "2026-10-21T00:00:01.000Z", "batch": [event] });
+ for (key, value) in batch_fields {
+ match value {
+ Some(v) => batch[*key] = v.clone(),
+ None => {
+ batch.as_object_mut().unwrap().remove(*key);
+ }
+ }
+ }
+ serde_json::to_vec(&batch).unwrap()
+ }
+
+ fn kind_value(kind: JsonKind) -> Option<Value> {
+ match kind {
+ JsonKind::Missing => None,
+ JsonKind::Null => Some(Value::Null),
+ JsonKind::Number => Some(json!(BIG)),
+ JsonKind::Boolean => Some(json!(true)),
+ JsonKind::Array => Some(json!([PII])),
+ JsonKind::Object => Some(json!({ "secret": PII })),
+ }
+ }
+
+ #[rstest]
+ fn required_field_errors_name_the_field_and_kind_only(
+ #[values("created_at", "event", "uuid", "distinct_id", "timestamp")] field: &'static str,
+ #[values(
+ JsonKind::Missing,
+ JsonKind::Null,
+ JsonKind::Number,
+ JsonKind::Boolean,
+ JsonKind::Array,
+ JsonKind::Object
+ )]
+ kind: JsonKind,
+ ) {
+ let (index, body) = if field == "created_at" {
+ (None, body_with(&[(field, kind_value(kind))], &[]))
+ } else {
+ (Some(0), body_with(&[], &[(field, kind_value(kind))]))
+ };
+
+ let err = field_error(&body);
+
+ let text = err.to_string();
+ assert!(
+ !text.contains(PII) && !text.contains(&BIG.to_string()),
+ "{text}"
+ );
+ assert_eq!(err, FieldTypeError { index, field, kind });
+ let path = index.map_or(field.to_owned(), |i| format!("batch[{i}].{field}"));
+ let expected = match kind {
+ JsonKind::Missing => format!("{path} is missing"),
+ _ => format!("{path} must be a string, got {}", kind.as_str()),
+ };
+ assert_eq!(text, expected);
+ }
+
+ #[rstest]
+ fn optional_field_wrong_types_name_the_field_and_kind_only(
+ #[values("session_id", "window_id")] field: &'static str,
+ #[values(JsonKind::Number, JsonKind::Boolean, JsonKind::Array, JsonKind::Object)]
+ kind: JsonKind,
+ ) {
+ let body = body_with(&[], &[(field, kind_value(kind))]);
+
+ let err = field_error(&body);
+
+ assert_eq!(
+ err,
+ FieldTypeError {
+ index: Some(0),
+ field,
+ kind
+ }
+ );
+ assert_eq!(
+ err.to_string(),
+ format!("batch[0].{field} must be a string, got {}", kind.as_str())
+ );
+ }
+
+ #[rstest]
+ #[case::created_at_key_first(
+ r#"{"created_at":1,"batch":[{"event":"e","uuid":"u","distinct_id":1,"timestamp":"t"}]}"#,
+ "batch[0].distinct_id"
+ )]
+ #[case::batch_key_first(
+ r#"{"batch":[{"event":"e","uuid":"u","distinct_id":1,"timestamp":"t"}],"created_at":1}"#,
+ "batch[0].distinct_id"
+ )]
+ #[case::field_order_within_an_event(
+ r#"{"created_at":"c","batch":[{"timestamp":1,"distinct_id":1,"event":"e","uuid":"u"}]}"#,
+ "batch[0].distinct_id"
+ )]
+ fn first_problem_wins(#[case] body: &str, #[case] path: &str) {
+ let err = field_error(body.as_bytes()).to_string();
+ assert!(err.starts_with(path), "{err}");
+ }
+
+ #[rstest]
+ #[case::empty_event("{}", "batch[0].event is missing")]
+ #[case::wrong_typed_array_event(
+ "[0,0,0,0,null,null]",
+ "batch[0].event must be a string, got number"
+ )]
+ #[case::bad_event_after_good_ones(
+ &format!(r#"{{{EVENT}}},{{{EVENT}}},{{"event":"e","uuid":"u","distinct_id":{{}},"timestamp":"t"}}"#),
+ "batch[2].distinct_id must be a string, got object"
+ )]
+ fn parse_stops_at_the_first_bad_event(#[case] events: &str, #[case] expected: &str) {
+ // Trailing garbage proves serde never reads past the bad event, so a huge bad body costs no more memory than the strict parse.
+ let body = format!(r#"{{"created_at":"c","batch":[{events},{{]]]"#);
+ assert_eq!(field_error(body.as_bytes()).to_string(), expected);
+ }
+
+ #[test]
+ fn stale_field_error_from_a_panicked_parse_is_ignored() {
+ FIELD_ERROR.set(Some(FieldTypeError {
+ index: Some(0),
+ field: "event",
+ kind: JsonKind::Missing,
+ }));
+ assert!(matches!(
+ parse_batch(br#"{"created_at":"c","batch":[{]}"#),
+ Err(ParseFailure::Json { .. })
+ ));
+ }
+
+ #[test]
+ fn first_bad_event_in_batch_order_wins() {
+ let mut bad = valid_event();
+ bad["distinct_id"] = json!(7);
+ let body = json!({
+ "created_at": "2026-10-21T00:00:01.000Z",
+ "batch": [valid_event(), bad.clone(), valid_event(), bad],
+ });
+ let err = field_error(&serde_json::to_vec(&body).unwrap());
+ assert_eq!(
+ err.to_string(),
+ "batch[1].distinct_id must be a string, got number"
+ );
+ }
+
+ fn assert_same_batch(strict: &Batch, lenient: &Batch, body: &str) {
+ assert_eq!(strict.created_at, lenient.created_at, "{body}");
+ assert_eq!(
+ strict.historical_migration, lenient.historical_migration,
+ "{body}"
+ );
+ assert_eq!(strict.capture_internal, lenient.capture_internal, "{body}");
+ assert_eq!(strict.batch.len(), lenient.batch.len(), "{body}");
+ for (s, l) in strict.batch.iter().zip(&lenient.batch) {
+ assert_eq!(s.event, l.event, "{body}");
+ assert_eq!(s.uuid, l.uuid, "{body}");
+ assert_eq!(s.distinct_id, l.distinct_id, "{body}");
+ assert_eq!(s.timestamp, l.timestamp, "{body}");
+ assert_eq!(s.session_id, l.session_id, "{body}");
+ assert_eq!(s.window_id, l.window_id, "{body}");
+ assert_eq!(s.options.0, l.options.0, "{body}");
+ assert_eq!(s.properties.get(), l.properties.get(), "{body}");
+ }
+ }
+
+ const EVENT: &str = r#""event":"e","uuid":"019a0000-0000-7000-8000-000000000001","distinct_id":"d","timestamp":"2026-10-21T00:00:00Z""#;
+
+ #[rstest]
+ #[case::minimal(true, format!(r#"{{"created_at":"c","batch":[{{{EVENT}}}]}}"#))]
+ #[case::padded_ids(true, r#"{"created_at":"c","batch":[{"event":"e","uuid":" 019a0000-0000-7000-8000-000000000001\t","distinct_id":"\n user 1 \r","timestamp":"t"}]}"#.to_owned())]
+ #[case::escapes_and_unicode(true, r#"{"created_at":"c\u00e9","batch":[{"event":"a \"q\" \\ \ud83d\ude00 é","uuid":"u","distinct_id":"\u0000x","timestamp":"t"}]}"#.to_owned())]
+ #[case::null_session_and_window(true, format!(r#"{{"created_at":"c","batch":[{{{EVENT},"session_id":null,"window_id":null}}]}}"#))]
+ #[case::string_session_and_window(true, format!(r#"{{"created_at":"c","batch":[{{{EVENT},"session_id":" s ","window_id":"w"}}]}}"#))]
+ #[case::unknown_fields(true, format!(r#"{{"created_at":"c","extra":{{"a":[1]}},"batch":[{{{EVENT},"other":7}}]}}"#))]
+ #[case::options_object(true, format!(r#"{{"created_at":"c","batch":[{{{EVENT},"options":{{"process_person_profile":"yes","x":[1]}}}}]}}"#))]
+ #[case::options_not_object(true, format!(r#"{{"created_at":"c","batch":[{{{EVENT},"options":"nope"}}]}}"#))]
+ #[case::options_null(true, format!(r#"{{"created_at":"c","batch":[{{{EVENT},"options":null}}]}}"#))]
+ #[case::properties_nested(true, format!(r#"{{"created_at":"c","batch":[{{{EVENT},"properties":{{"a":{{"b":[1,2.5e300,null]}},"n":123456789012345678901234567890}}}}]}}"#))]
+ #[case::properties_not_object(true, format!(r#"{{"created_at":"c","batch":[{{{EVENT},"properties":[1,2]}}]}}"#))]
+ #[case::flags_set(true, format!(r#"{{"created_at":"c","historical_migration":true,"capture_internal":true,"batch":[{{{EVENT}}}]}}"#))]
+ #[case::flags_unreadable(true, format!(r#"{{"created_at":"c","historical_migration":{{"x":1}},"capture_internal":"maybe","batch":[{{{EVENT}}}]}}"#))]
+ #[case::flags_lenient_strings(true, format!(r#"{{"created_at":"c","historical_migration":"yes","capture_internal":0,"batch":[{{{EVENT}}}]}}"#))]
+ #[case::empty_batch(true, r#"{"created_at":"c","batch":[]}"#.to_owned())]
+ #[case::empty_strings(true, r#"{"created_at":"","batch":[{"event":"","uuid":"","distinct_id":" ","timestamp":""}]}"#.to_owned())]
+ #[case::many_events(true, format!(r#"{{"created_at":"c","batch":[{{{EVENT}}},{{{EVENT},"session_id":"s"}},{{{EVENT}}}]}}"#))]
+ #[case::event_as_array(true, r#"{"created_at":"c","batch":[["e"," u ","d","t","s",null,{"cookieless_mode":true},{"p":1}]]}"#.to_owned())]
+ #[case::event_as_short_array(true, r#"{"created_at":"c","batch":[["e","u","d","t","s","w"]]}"#.to_owned())]
+ #[case::batch_as_array(true, format!(r#"["c",true,null,[{{{EVENT}}}]]"#))]
+ #[case::numeric_distinct_id(false, r#"{"created_at":"c","batch":[{"event":"e","uuid":"u","distinct_id":12345678901234567890,"timestamp":"t"}]}"#.to_owned())]
+ #[case::null_event(false, r#"{"created_at":"c","batch":[{"event":null,"uuid":"u","distinct_id":"d","timestamp":"t"}]}"#.to_owned())]
+ #[case::missing_uuid(false, r#"{"created_at":"c","batch":[{"event":"e","distinct_id":"d","timestamp":"t"}]}"#.to_owned())]
+ #[case::object_timestamp(false, r#"{"created_at":"c","batch":[{"event":"e","uuid":"u","distinct_id":"d","timestamp":{"a":"[email protected]"}}]}"#.to_owned())]
+ #[case::numeric_session_id(false, format!(r#"{{"created_at":"c","batch":[{{{EVENT},"session_id":5}}]}}"#))]
+ #[case::array_window_id(false, format!(r#"{{"created_at":"c","batch":[{{{EVENT},"window_id":["w"]}}]}}"#))]
+ #[case::missing_created_at(false, format!(r#"{{"batch":[{{{EVENT}}}]}}"#))]
+ #[case::numeric_created_at(false, format!(r#"{{"created_at":1,"batch":[{{{EVENT}}}]}}"#))]
+ #[case::missing_batch(false, r#"{"created_at":"c"}"#.to_owned())]
+ #[case::batch_not_array(false, r#"{"created_at":"c","batch":"[email protected]"}"#.to_owned())]
+ #[case::event_not_object(false, r#"{"created_at":"c","batch":["[email protected]"]}"#.to_owned())]
+ #[case::duplicate_event_key(false, r#"{"created_at":"c","batch":[{"event":"e","uuid":"u","distinct_id":"d","distinct_id":"d","timestamp":"t"}]}"#.to_owned())]
+ #[case::duplicate_batch_key(false, format!(r#"{{"created_at":"c","created_at":"c","batch":[{{{EVENT}}}]}}"#))]
+ #[case::event_as_too_short_array(false, r#"{"created_at":"c","batch":[["e","u","d","t"]]}"#.to_owned())]
+ #[case::syntax_error(false, r#"{"created_at":"c","batch":[{]}"#.to_owned())]
+ #[case::truncated(false, r#"{"created_at":"c","batch":[{"event":"e""#.to_owned())]
+ #[case::lone_surrogate(false, r#"{"created_at":"c","batch":[{"event":"e","uuid":"u","distinct_id":"\ud800","timestamp":"t"}]}"#.to_owned())]
+ #[case::number_out_of_range(false, r#"{"created_at":"c","batch":[{"event":"e","uuid":"u","distinct_id":1e400,"timestamp":"t"}]}"#.to_owned())]
+ #[case::trailing_garbage(false, format!(r#"{{"created_at":"c","batch":[{{{EVENT}}}]}} x"#))]
+ fn lenient_parse_accepts_and_builds_exactly_what_the_strict_types_do(
+ #[case] accepted: bool,
+ #[case] body: String,
+ ) {
+ let strict = serde_json::from_slice::<Batch>(body.as_bytes());
+ let lenient = parse_batch(body.as_bytes());
+ assert_eq!(strict.is_ok(), accepted, "strict parse of {body}");
+
+ match (strict, lenient) {
+ (Ok(strict), Ok(lenient)) => assert_same_batch(&strict, &lenient, &body),
+ (Err(_), Err(_)) => {}
+ (strict, lenient) => panic!(
+ "strict ok={} but lenient ok={} for {body}",
+ strict.is_ok(),
+ lenient.is_ok()
+ ),
+ }
+ }
+}
diff --git a/rust/capture/src/v1/analytics/mod.rs b/rust/capture/src/v1/analytics/mod.rs
index 65d620d0995a..c9c90db73432 100644
--- a/rust/capture/src/v1/analytics/mod.rs
+++ b/rust/capture/src/v1/analytics/mod.rs
@@ -1,6 +1,7 @@
pub mod constants;
pub mod context;
pub mod handler;
+pub mod lenient;
pub mod process;
pub mod query;
pub mod response;
diff --git a/rust/capture/src/v1/analytics/types.rs b/rust/capture/src/v1/analytics/types.rs
index 65d751bd1e91..f9b4472222db 100644
--- a/rust/capture/src/v1/analytics/types.rs
+++ b/rust/capture/src/v1/analytics/types.rs
@@ -28,7 +28,7 @@ use crate::ordering::{person_ordering, OrderingGuarantee};
use crate::v1::context::RequestContext;
use crate::v1::types::{Destination, Publishable};
-fn empty_raw_object() -> Box<RawValue> {
+pub(super) fn empty_raw_object() -> Box<RawValue> {
RawValue::from_string("{}".to_owned()).unwrap()
}
@@ -38,13 +38,17 @@ fn deserialize_trimmed_string<'de, D>(deserializer: D) -> Result<String, D::Erro
where
D: serde::Deserializer<'de>,
{
- let s = String::deserialize(deserializer)?;
+ Ok(trim_owned(String::deserialize(deserializer)?))
+}
+
+/// Shared by [`Event`] and the lenient parse so their trims can't diverge.
+pub(super) fn trim_owned(s: String) -> String {
let trimmed = s.trim();
- Ok(if trimmed.len() == s.len() {
+ if trimmed.len() == s.len() {
s
} else {
trimmed.to_owned()
- })
+ }
}
/// Per-event outcome in the batch response.
@@ -60,6 +64,7 @@ pub enum EventResult {
Retry,
}
+/// Keep fields in sync with `LenientBatch` in `super::lenient`.
#[derive(Debug, Deserialize)]
pub struct Batch {
pub created_at: String,
@@ -79,14 +84,14 @@ fn lenient_flag(value: &Value) -> Option<bool> {
}
}
-fn deserialize_lenient_flag<'de, D>(deserializer: D) -> Result<Option<bool>, D::Error>
+pub(super) fn deserialize_lenient_flag<'de, D>(deserializer: D) -> Result<Option<bool>, D::Error>
where
D: serde::Deserializer<'de>,
{
Ok(lenient_flag(&Value::deserialize(deserializer)?))
}
-fn deserialize_lenient_flag_or_false<'de, D>(deserializer: D) -> Result<bool, D::Error>
+pub(super) fn deserialize_lenient_flag_or_false<'de, D>(deserializer: D) -> Result<bool, D::Error>
where
D: serde::Deserializer<'de>,
{
@@ -300,6 +305,7 @@ fn coerce_product_tour_id(v: &Value) -> Parsed<String> {
}
}
+/// Keep fields in sync with `LenientEvent` in `super::lenient`.
#[derive(Debug, Deserialize)]
pub struct Event {
pub event: String,
diff --git a/rust/capture/src/v1/error.rs b/rust/capture/src/v1/error.rs
index 09cb6c639325..9509918e0755 100644
--- a/rust/capture/src/v1/error.rs
+++ b/rust/capture/src/v1/error.rs
@@ -3,10 +3,12 @@ use axum::response::{IntoResponse, Response};
use axum::Json;
use metrics::counter;
use serde::Serialize;
+use serde_json::error::Category;
use thiserror::Error;
use tracing::Level;
use crate::v1::analytics::constants::DEFAULT_RETRY_AFTER_SECS;
+use crate::v1::analytics::lenient::FieldTypeError;
use crate::v1::constants::{CAPTURE_V1_ERROR_METRIC, CAPTURE_V1_UNKNOWN_PATH};
use crate::v1::context::RequestContext;
@@ -17,6 +19,66 @@ pub struct ErrorResponse {
pub error_uri: String,
}
+#[derive(Debug)]
+pub enum ParseFailure {
+ Json {
+ /// Can quote request values: log it, never put it in a response.
+ message: String,
+ category: Category,
+ line: usize,
+ column: usize,
+ },
+ Field(FieldTypeError),
+}
+
+impl ParseFailure {
+ fn description(&self) -> String {
+ match self {
+ Self::Json {
+ category,
+ line,
+ column,
+ ..
+ } => {
+ let what = match category {
+ Category::Syntax => "invalid JSON",
+ Category::Data => "wrong JSON type or value",
+ Category::Eof => "unexpected end of JSON",
+ Category::Io => "unreadable body",
+ };
+ format!("Failed to parse request body: {what} at line {line}, column {column}.")
+ }
+ Self::Field(field) => format!("Failed to parse request body: {field}."),
+ }
+ }
+}
+
+impl std::fmt::Display for ParseFailure {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ Self::Json { message, .. } => f.write_str(message),
+ Self::Field(field) => field.fmt(f),
+ }
+ }
+}
+
+impl From<serde_json::Error> for ParseFailure {
+ fn from(e: serde_json::Error) -> Self {
+ Self::Json {
+ message: e.to_string(),
+ category: e.classify(),
+ line: e.line(),
+ column: e.column(),
+ }
+ }
+}
+
+impl From<FieldTypeError> for ParseFailure {
+ fn from(e: FieldTypeError) -> Self {
+ Self::Field(e)
+ }
+}
+
impl ErrorResponse {
fn new(error: &Error) -> Self {
Self {
@@ -41,7 +103,7 @@ pub enum Error {
#[error("failed to decode request: {0}")]
RequestDecodingError(String),
#[error("failed to parse request: {0}")]
- RequestParsingError(String),
+ RequestParsingError(ParseFailure),
#[error("request body is empty")]
EmptyBody,
#[error("request batch is empty")]
@@ -147,7 +209,7 @@ impl Error {
pub fn description(&self) -> String {
match self {
Self::RequestDecodingError(_) => "Failed to decode request body.".to_string(),
- Self::RequestParsingError(_) => "Failed to parse request body.".to_string(),
+ Self::RequestParsingError(failure) => failure.description(),
Self::InvalidApiToken(_) => "The provided API token is not valid.".to_string(),
Self::BillingLimitExceeded => "Billing quota exceeded. Events are being dropped. Upgrade your plan to resume ingestion.".to_string(),
Self::InternalError(_) | Self::ServiceUnavailable(_) | Self::GatewayTimeout => self
@@ -217,6 +279,20 @@ impl Error {
pub(crate) fn stat_error(&self, ctx: Option<&RequestContext>) {
let path = ctx.map(|c| c.path).unwrap_or(CAPTURE_V1_UNKNOWN_PATH);
let status = self.status_code().as_str().to_owned();
+ // Field errors add bounded `field` and `kind` labels; sums by the existing labels are unchanged.
+ if let Self::RequestParsingError(ParseFailure::Field(field)) = self {
+ counter!(
+ CAPTURE_V1_ERROR_METRIC,
+ "error" => self.tag(),
+ "level" => self.level_tag(),
+ "path" => path,
+ "status_code" => status,
+ "field" => field.field,
+ "kind" => field.kind.as_str(),
+ )
+ .increment(1);
+ return;
+ }
counter!(
CAPTURE_V1_ERROR_METRIC,
"error" => self.tag(),
@@ -357,7 +433,7 @@ macro_rules! log_stat_error {
impl From<serde_json::Error> for Error {
fn from(e: serde_json::Error) -> Self {
- Error::RequestParsingError(e.to_string())
+ Error::RequestParsingError(e.into())
}
}
@@ -373,6 +449,7 @@ impl IntoResponse for Error {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::v1::analytics::lenient::JsonKind;
use axum::body::to_bytes;
use axum::http::header::RETRY_AFTER;
use rstest::rstest;
@@ -411,6 +488,18 @@ mod tests {
assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR);
}
+ #[test]
+ fn parse_failure_logs_serde_message_but_describes_only_the_position() {
+ let serde_err = serde_json::from_str::<Vec<u32>>(r#"["[email protected]"]"#).unwrap_err();
+ let err = Error::from(serde_err);
+
+ assert!(err.to_string().contains("[email protected]"));
+ assert_eq!(
+ err.description(),
+ "Failed to parse request body: wrong JSON type or value at line 1, column 20."
+ );
+ }
+
#[tokio::test]
async fn billing_limit_custom_error_uri() {
let err = Error::BillingLimitExceeded;
@@ -496,4 +585,71 @@ mod tests {
assert_eq!(body["error"], expected_tag);
assert_eq!(body["error_description"], expected_description);
}
+
+ fn error_metric_labels(err: &Error) -> Vec<(String, String)> {
+ use metrics_util::debugging::{DebugValue, DebuggingRecorder};
+
+ let recorder = DebuggingRecorder::new();
+ let snapshotter = recorder.snapshotter();
+ let _guard = metrics::set_default_local_recorder(&recorder);
+ err.stat_error(None);
+
+ let mut series: Vec<_> = snapshotter
+ .snapshot()
+ .into_vec()
+ .into_iter()
+ .filter(|(key, ..)| key.key().name() == CAPTURE_V1_ERROR_METRIC)
+ .collect();
+ assert_eq!(series.len(), 1);
+ let (key, _, _, value) = series.remove(0);
+ assert_eq!(value, DebugValue::Counter(1));
+ let mut labels: Vec<_> = key
+ .key()
+ .labels()
+ .map(|l| (l.key().to_owned(), l.value().to_owned()))
+ .collect();
+ labels.sort();
+ labels
+ }
+
+ #[rstest]
+ #[case::field_type_error(
+ Error::RequestParsingError(ParseFailure::Field(FieldTypeError {
+ index: Some(3),
+ field: "distinct_id",
+ kind: JsonKind::Number,
+ })),
+ &[("field", "distinct_id"), ("kind", "number")]
+ )]
+ #[case::missing_field(
+ Error::RequestParsingError(ParseFailure::Field(FieldTypeError {
+ index: None,
+ field: "created_at",
+ kind: JsonKind::Missing,
+ })),
+ &[("field", "created_at"), ("kind", "missing")]
+ )]
+ #[case::corrupt_json(
+ Error::from(serde_json::from_str::<serde_json::Value>("{").unwrap_err()),
+ &[]
+ )]
+ fn parse_error_metric_adds_field_labels_only_for_field_errors(
+ #[case] err: Error,
+ #[case] extra: &[(&str, &str)],
+ ) {
+ let labels = error_metric_labels(&err);
+
+ let mut expected: Vec<(String, String)> = [
+ ("error", "request_parsing_error"),
+ ("level", err.level_tag()),
+ ("path", CAPTURE_V1_UNKNOWN_PATH),
+ ("status_code", "400"),
+ ]
+ .iter()
+ .chain(extra)
+ .map(|(k, v)| (k.to_string(), v.to_string()))
+ .collect();
+ expected.sort();
+ assert_eq!(labels, expected);
+ }
}