:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(capture): name the v1 field with the wrong JSON type in the 400 (#120820)
Eli Reismanbaf885696441
diff --git a/rust/capture/src/v1/analytics/handler.rs b/rust/capture/src/v1/analytics/handler.rsindex 194736b89327..fb64ffbd83bc 100644--- a/rust/capture/src/v1/analytics/handler.rs+++ b/rust/capture/src/v1/analytics/handler.rs@@ -6,6 +6,7 @@ use axum_client_ip::InsecureClientIp;use super::constants::{CAPTURE_V1_PATH, CAPTURE_V1_PATHS};use super::context::Context;+use super::lenient::parse_batch;use super::types::Batch;use tracing::Level;@@ -86,8 +87,8 @@ pub async fn handle_request(metrics::histogram!(CAPTURE_V1_PAYLOAD_SIZE, "stage" => "decompressed", "encoding" => v1::util::encoding_tag(context.content_encoding.as_deref())).record(payload.len() as f64);- let batch: Batch = serde_json::from_slice(&payload).map_err(|e| {- let err = v1::Error::RequestParsingError(e.to_string());+ let batch: Batch = parse_batch(&payload).map_err(|failure| {+ let err = v1::Error::RequestParsingError(failure);log_stat_error!(err, &context);err})?;@@ -191,6 +192,7 @@ mod tests {use axum::body::Body;use axum::http::{Request, StatusCode};use axum::Router;+ use rstest::rstest;use tower::ServiceExt;use uuid::Uuid;@@ -328,18 +330,60 @@ mod tests {assert_eq!(resp.status(), StatusCode::BAD_REQUEST);}+ #[rstest]+ #[case::invalid_json(+ "not json at all {{{",+ "Failed to parse request body: invalid JSON at line 1, column 2."+ )]+ #[case::wrong_type_distinct_id(+ r#"{"created_at":"2026-03-19T14:30:00Z","batch":[{"event":"e","uuid":"019a0000-0000-7000-8000-000000000001","distinct_id":12345678901234567890123,"timestamp":"2026-03-19T14:30:00Z"}]}"#,+ "Failed to parse request body: batch[0].distinct_id must be a string, got number."+ )]+ #[case::missing_event_name_in_later_event(+ r#"{"created_at":"2026-03-19T14:30:00Z","batch":[{"event":"e","uuid":"019a0000-0000-7000-8000-000000000001","distinct_id":"d","timestamp":"2026-03-19T14:30:00Z"},{"uuid":"019a0000-0000-7000-8000-000000000002","distinct_id":"[email protected]","timestamp":"2026-03-19T14:30:00Z"}]}"#,+ "Failed to parse request body: batch[1].event is missing."+ )]+ #[case::wrong_type_created_at(+ r#"{"created_at":{"secret":"[email protected]"},"batch":[]}"#,+ "Failed to parse request body: created_at must be a string, got object."+ )]+ #[case::wrong_type_batch(+ r#"{"created_at":"2026-03-19T14:30:00Z","batch":"[email protected]"}"#,+ "Failed to parse request body: wrong JSON type or value at line 1, column 64."+ )]+ #[case::truncated(+ r#"{"created_at":"2026-03-19T14:30:00Z","batch":[{"event":"e""#,+ "Failed to parse request body: unexpected end of JSON at line 1, column 58."+ )]#[tokio::test]- async fn malformed_json() {+ async fn unparseable_body_describes_the_problem_without_content(+ #[case] body: &str,+ #[case] expected_description: &str,+ ) {let ts = TestStateBuilder::new().build();let app = test_app(ts.state);- let garbage = b"not json at all {{{".to_vec();let resp = app- .oneshot(valid_request().body(Body::from(garbage)).unwrap())+ .oneshot(valid_request().body(Body::from(body.to_owned())).unwrap()).await.unwrap();assert_eq!(resp.status(), StatusCode::BAD_REQUEST);+ let json = response_json(resp).await;+ let raw = json.to_string();+ for value in [+ "[email protected]",+ "12345678901234567890123",+ "1.2345678901234568e22",+ "secret",+ ] {+ assert!(+ !raw.contains(value),+ "response echoes request content: {raw}"+ );+ }+ assert_eq!(json["error"], "request_parsing_error");+ assert_eq!(json["error_description"], expected_description);}#[tokio::test]diff --git a/rust/capture/src/v1/analytics/lenient.rs b/rust/capture/src/v1/analytics/lenient.rsnew file mode 100644index 000000000000..580dbff79642--- /dev/null+++ b/rust/capture/src/v1/analytics/lenient.rs@@ -0,0 +1,530 @@+use std::cell::Cell;+use std::fmt;++use serde::de::{self, Deserializer, IgnoredAny, MapAccess, SeqAccess, Visitor};+use serde::Deserialize;+use serde_json::value::RawValue;++use super::types::{+ deserialize_lenient_flag, deserialize_lenient_flag_or_false, empty_raw_object, trim_owned,+ Batch, Event, RawOptions,+};+use crate::v1::error::ParseFailure;++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum JsonKind {+ Missing,+ Null,+ Number,+ Boolean,+ Array,+ Object,+}++impl JsonKind {+ pub fn as_str(self) -> &'static str {+ match self {+ Self::Missing => "missing",+ Self::Null => "null",+ Self::Number => "number",+ Self::Boolean => "boolean",+ Self::Array => "array",+ Self::Object => "object",+ }+ }+}++#[derive(Debug, Clone, PartialEq, Eq)]+pub struct FieldTypeError {+ pub index: Option<usize>,+ pub field: &'static str,+ pub kind: JsonKind,+}++impl fmt::Display for FieldTypeError {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ match self.index {+ Some(i) => write!(f, "batch[{i}].{}", self.field)?,+ None => f.write_str(self.field)?,+ }+ match self.kind {+ JsonKind::Missing => f.write_str(" is missing"),+ kind => write!(f, " must be a string, got {}", kind.as_str()),+ }+ }+}++/// Keeps only a non-string's JSON kind, so the 400 names the field without echoing the value.+#[derive(Debug, Default)]+enum JsonStr {+ #[default]+ Missing,+ Null,+ Str(String),+ Wrong(JsonKind),+}++impl<'de> Deserialize<'de> for JsonStr {+ fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {+ deserializer.deserialize_any(JsonStrVisitor)+ }+}++struct JsonStrVisitor;++impl<'de> Visitor<'de> for JsonStrVisitor {+ type Value = JsonStr;++ fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.write_str("any JSON value")+ }++ fn visit_str<E: de::Error>(self, v: &str) -> Result<JsonStr, E> {+ Ok(JsonStr::Str(v.to_owned()))+ }++ fn visit_string<E: de::Error>(self, v: String) -> Result<JsonStr, E> {+ Ok(JsonStr::Str(v))+ }++ fn visit_unit<E: de::Error>(self) -> Result<JsonStr, E> {+ Ok(JsonStr::Null)+ }++ fn visit_none<E: de::Error>(self) -> Result<JsonStr, E> {+ Ok(JsonStr::Null)+ }++ fn visit_bool<E: de::Error>(self, _: bool) -> Result<JsonStr, E> {+ Ok(JsonStr::Wrong(JsonKind::Boolean))+ }++ fn visit_i64<E: de::Error>(self, _: i64) -> Result<JsonStr, E> {+ Ok(JsonStr::Wrong(JsonKind::Number))+ }++ fn visit_u64<E: de::Error>(self, _: u64) -> Result<JsonStr, E> {+ Ok(JsonStr::Wrong(JsonKind::Number))+ }++ fn visit_f64<E: de::Error>(self, _: f64) -> Result<JsonStr, E> {+ Ok(JsonStr::Wrong(JsonKind::Number))+ }++ fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<JsonStr, A::Error> {+ while seq.next_element::<IgnoredAny>()?.is_some() {}+ Ok(JsonStr::Wrong(JsonKind::Array))+ }++ fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<JsonStr, A::Error> {+ while map.next_entry::<IgnoredAny, IgnoredAny>()?.is_some() {}+ Ok(JsonStr::Wrong(JsonKind::Object))+ }+}++/// Lenient mirror of [`Event`] so a wrong type gets a 400 naming the field; keep `Event`'s field order (serde also reads the array form).+#[derive(Deserialize)]+struct LenientEvent {+ #[serde(default)]+ event: JsonStr,+ #[serde(default)]+ uuid: JsonStr,+ #[serde(default)]+ distinct_id: JsonStr,+ #[serde(default)]+ timestamp: JsonStr,+ // `Option` without a default, as in `Event`, so a short array form still fails in serde.+ session_id: Option<JsonStr>,+ window_id: Option<JsonStr>,+ #[serde(default)]+ options: RawOptions,+ #[serde(default = "empty_raw_object")]+ properties: Box<RawValue>,+}++/// Lenient mirror of [`Batch`] so a wrong-typed `created_at` gets a 400 naming the field.+#[derive(Deserialize)]+struct LenientBatch {+ #[serde(default)]+ created_at: JsonStr,+ #[serde(default, deserialize_with = "deserialize_lenient_flag_or_false")]+ historical_migration: bool,+ #[serde(default, deserialize_with = "deserialize_lenient_flag")]+ capture_internal: Option<bool>,+ #[serde(deserialize_with = "deserialize_events")]+ batch: Vec<Event>,+}++thread_local! {+ // serde returns only its own error type, so the field error travels beside it.+ static FIELD_ERROR: Cell<Option<FieldTypeError>> = const { Cell::new(None) };+}++/// Builds the [`Batch`] the strict types build, or names the first non-string field; event errors win over `created_at` whatever the key order.+pub fn parse_batch(body: &[u8]) -> Result<Batch, ParseFailure> {+ // Clear a value a panicked parse on this thread left behind.+ FIELD_ERROR.set(None);+ let lenient = serde_json::from_slice::<LenientBatch>(body).map_err(|err| {+ FIELD_ERROR+ .take()+ .map_or_else(|| err.into(), ParseFailure::Field)+ })?;+ lenient.into_batch().map_err(ParseFailure::Field)+}++/// Converts each event as serde reads it and aborts at the first bad field, so a bad body holds no more than the strict parse would.+fn deserialize_events<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Vec<Event>, D::Error> {+ deserializer.deserialize_seq(EventsVisitor)+}++struct EventsVisitor;++impl<'de> Visitor<'de> for EventsVisitor {+ type Value = Vec<Event>;++ fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.write_str("a sequence")+ }++ fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Vec<Event>, A::Error> {+ let mut events = Vec::new();+ while let Some(event) = seq.next_element::<LenientEvent>()? {+ match event.into_event(events.len()) {+ Ok(event) => events.push(event),+ Err(field) => {+ FIELD_ERROR.set(Some(field));+ return Err(de::Error::custom("field type error"));+ }+ }+ }+ Ok(events)+ }+}++fn required(+ value: JsonStr,+ index: Option<usize>,+ field: &'static str,+) -> Result<String, FieldTypeError> {+ match value {+ JsonStr::Str(s) => Ok(s),+ JsonStr::Missing => Err(FieldTypeError {+ index,+ field,+ kind: JsonKind::Missing,+ }),+ JsonStr::Null => Err(FieldTypeError {+ index,+ field,+ kind: JsonKind::Null,+ }),+ JsonStr::Wrong(kind) => Err(FieldTypeError { index, field, kind }),+ }+}++fn optional(+ value: Option<JsonStr>,+ index: usize,+ field: &'static str,+) -> Result<Option<String>, FieldTypeError> {+ match value {+ None | Some(JsonStr::Null) | Some(JsonStr::Missing) => Ok(None),+ Some(JsonStr::Str(s)) => Ok(Some(s)),+ Some(JsonStr::Wrong(kind)) => Err(FieldTypeError {+ index: Some(index),+ field,+ kind,+ }),+ }+}++impl LenientEvent {+ fn into_event(self, index: usize) -> Result<Event, FieldTypeError> {+ let i = Some(index);+ Ok(Event {+ event: required(self.event, i, "event")?,+ uuid: trim_owned(required(self.uuid, i, "uuid")?),+ distinct_id: trim_owned(required(self.distinct_id, i, "distinct_id")?),+ timestamp: required(self.timestamp, i, "timestamp")?,+ session_id: optional(self.session_id, index, "session_id")?,+ window_id: optional(self.window_id, index, "window_id")?,+ options: self.options,+ properties: self.properties,+ })+ }+}++impl LenientBatch {+ fn into_batch(self) -> Result<Batch, FieldTypeError> {+ Ok(Batch {+ created_at: required(self.created_at, None, "created_at")?,+ historical_migration: self.historical_migration,+ capture_internal: self.capture_internal,+ batch: self.batch,+ })+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use rstest::rstest;+ use serde_json::{json, Value};++ const PII: &str = "[email protected]";+ const BIG: u64 = 12345678901234567890;++ fn field_error(body: &[u8]) -> FieldTypeError {+ match parse_batch(body) {+ Err(ParseFailure::Field(field)) => field,+ other => panic!("expected a field error, got {:?}", other.map(|_| ())),+ }+ }++ fn valid_event() -> Value {+ json!({+ "event": "$pageview",+ "uuid": "019a0000-0000-7000-8000-000000000001",+ "distinct_id": "user-1",+ "timestamp": "2026-10-21T00:00:00.000Z",+ })+ }++ fn body_with(+ batch_fields: &[(&str, Option<Value>)],+ event_fields: &[(&str, Option<Value>)],+ ) -> Vec<u8> {+ let mut event = valid_event();+ for (key, value) in event_fields {+ match value {+ Some(v) => event[*key] = v.clone(),+ None => {+ event.as_object_mut().unwrap().remove(*key);+ }+ }+ }+ let mut batch = json!({ "created_at": "2026-10-21T00:00:01.000Z", "batch": [event] });+ for (key, value) in batch_fields {+ match value {+ Some(v) => batch[*key] = v.clone(),+ None => {+ batch.as_object_mut().unwrap().remove(*key);+ }+ }+ }+ serde_json::to_vec(&batch).unwrap()+ }++ fn kind_value(kind: JsonKind) -> Option<Value> {+ match kind {+ JsonKind::Missing => None,+ JsonKind::Null => Some(Value::Null),+ JsonKind::Number => Some(json!(BIG)),+ JsonKind::Boolean => Some(json!(true)),+ JsonKind::Array => Some(json!([PII])),+ JsonKind::Object => Some(json!({ "secret": PII })),+ }+ }++ #[rstest]+ fn required_field_errors_name_the_field_and_kind_only(+ #[values("created_at", "event", "uuid", "distinct_id", "timestamp")] field: &'static str,+ #[values(+ JsonKind::Missing,+ JsonKind::Null,+ JsonKind::Number,+ JsonKind::Boolean,+ JsonKind::Array,+ JsonKind::Object+ )]+ kind: JsonKind,+ ) {+ let (index, body) = if field == "created_at" {+ (None, body_with(&[(field, kind_value(kind))], &[]))+ } else {+ (Some(0), body_with(&[], &[(field, kind_value(kind))]))+ };++ let err = field_error(&body);++ let text = err.to_string();+ assert!(+ !text.contains(PII) && !text.contains(&BIG.to_string()),+ "{text}"+ );+ assert_eq!(err, FieldTypeError { index, field, kind });+ let path = index.map_or(field.to_owned(), |i| format!("batch[{i}].{field}"));+ let expected = match kind {+ JsonKind::Missing => format!("{path} is missing"),+ _ => format!("{path} must be a string, got {}", kind.as_str()),+ };+ assert_eq!(text, expected);+ }++ #[rstest]+ fn optional_field_wrong_types_name_the_field_and_kind_only(+ #[values("session_id", "window_id")] field: &'static str,+ #[values(JsonKind::Number, JsonKind::Boolean, JsonKind::Array, JsonKind::Object)]+ kind: JsonKind,+ ) {+ let body = body_with(&[], &[(field, kind_value(kind))]);++ let err = field_error(&body);++ assert_eq!(+ err,+ FieldTypeError {+ index: Some(0),+ field,+ kind+ }+ );+ assert_eq!(+ err.to_string(),+ format!("batch[0].{field} must be a string, got {}", kind.as_str())+ );+ }++ #[rstest]+ #[case::created_at_key_first(+ r#"{"created_at":1,"batch":[{"event":"e","uuid":"u","distinct_id":1,"timestamp":"t"}]}"#,+ "batch[0].distinct_id"+ )]+ #[case::batch_key_first(+ r#"{"batch":[{"event":"e","uuid":"u","distinct_id":1,"timestamp":"t"}],"created_at":1}"#,+ "batch[0].distinct_id"+ )]+ #[case::field_order_within_an_event(+ r#"{"created_at":"c","batch":[{"timestamp":1,"distinct_id":1,"event":"e","uuid":"u"}]}"#,+ "batch[0].distinct_id"+ )]+ fn first_problem_wins(#[case] body: &str, #[case] path: &str) {+ let err = field_error(body.as_bytes()).to_string();+ assert!(err.starts_with(path), "{err}");+ }++ #[rstest]+ #[case::empty_event("{}", "batch[0].event is missing")]+ #[case::wrong_typed_array_event(+ "[0,0,0,0,null,null]",+ "batch[0].event must be a string, got number"+ )]+ #[case::bad_event_after_good_ones(+ &format!(r#"{{{EVENT}}},{{{EVENT}}},{{"event":"e","uuid":"u","distinct_id":{{}},"timestamp":"t"}}"#),+ "batch[2].distinct_id must be a string, got object"+ )]+ fn parse_stops_at_the_first_bad_event(#[case] events: &str, #[case] expected: &str) {+ // Trailing garbage proves serde never reads past the bad event, so a huge bad body costs no more memory than the strict parse.+ let body = format!(r#"{{"created_at":"c","batch":[{events},{{]]]"#);+ assert_eq!(field_error(body.as_bytes()).to_string(), expected);+ }++ #[test]+ fn stale_field_error_from_a_panicked_parse_is_ignored() {+ FIELD_ERROR.set(Some(FieldTypeError {+ index: Some(0),+ field: "event",+ kind: JsonKind::Missing,+ }));+ assert!(matches!(+ parse_batch(br#"{"created_at":"c","batch":[{]}"#),+ Err(ParseFailure::Json { .. })+ ));+ }++ #[test]+ fn first_bad_event_in_batch_order_wins() {+ let mut bad = valid_event();+ bad["distinct_id"] = json!(7);+ let body = json!({+ "created_at": "2026-10-21T00:00:01.000Z",+ "batch": [valid_event(), bad.clone(), valid_event(), bad],+ });+ let err = field_error(&serde_json::to_vec(&body).unwrap());+ assert_eq!(+ err.to_string(),+ "batch[1].distinct_id must be a string, got number"+ );+ }++ fn assert_same_batch(strict: &Batch, lenient: &Batch, body: &str) {+ assert_eq!(strict.created_at, lenient.created_at, "{body}");+ assert_eq!(+ strict.historical_migration, lenient.historical_migration,+ "{body}"+ );+ assert_eq!(strict.capture_internal, lenient.capture_internal, "{body}");+ assert_eq!(strict.batch.len(), lenient.batch.len(), "{body}");+ for (s, l) in strict.batch.iter().zip(&lenient.batch) {+ assert_eq!(s.event, l.event, "{body}");+ assert_eq!(s.uuid, l.uuid, "{body}");+ assert_eq!(s.distinct_id, l.distinct_id, "{body}");+ assert_eq!(s.timestamp, l.timestamp, "{body}");+ assert_eq!(s.session_id, l.session_id, "{body}");+ assert_eq!(s.window_id, l.window_id, "{body}");+ assert_eq!(s.options.0, l.options.0, "{body}");+ assert_eq!(s.properties.get(), l.properties.get(), "{body}");+ }+ }++ const EVENT: &str = r#""event":"e","uuid":"019a0000-0000-7000-8000-000000000001","distinct_id":"d","timestamp":"2026-10-21T00:00:00Z""#;++ #[rstest]+ #[case::minimal(true, format!(r#"{{"created_at":"c","batch":[{{{EVENT}}}]}}"#))]+ #[case::padded_ids(true, r#"{"created_at":"c","batch":[{"event":"e","uuid":" 019a0000-0000-7000-8000-000000000001\t","distinct_id":"\n user 1 \r","timestamp":"t"}]}"#.to_owned())]+ #[case::escapes_and_unicode(true, r#"{"created_at":"c\u00e9","batch":[{"event":"a \"q\" \\ \ud83d\ude00 é","uuid":"u","distinct_id":"\u0000x","timestamp":"t"}]}"#.to_owned())]+ #[case::null_session_and_window(true, format!(r#"{{"created_at":"c","batch":[{{{EVENT},"session_id":null,"window_id":null}}]}}"#))]+ #[case::string_session_and_window(true, format!(r#"{{"created_at":"c","batch":[{{{EVENT},"session_id":" s ","window_id":"w"}}]}}"#))]+ #[case::unknown_fields(true, format!(r#"{{"created_at":"c","extra":{{"a":[1]}},"batch":[{{{EVENT},"other":7}}]}}"#))]+ #[case::options_object(true, format!(r#"{{"created_at":"c","batch":[{{{EVENT},"options":{{"process_person_profile":"yes","x":[1]}}}}]}}"#))]+ #[case::options_not_object(true, format!(r#"{{"created_at":"c","batch":[{{{EVENT},"options":"nope"}}]}}"#))]+ #[case::options_null(true, format!(r#"{{"created_at":"c","batch":[{{{EVENT},"options":null}}]}}"#))]+ #[case::properties_nested(true, format!(r#"{{"created_at":"c","batch":[{{{EVENT},"properties":{{"a":{{"b":[1,2.5e300,null]}},"n":123456789012345678901234567890}}}}]}}"#))]+ #[case::properties_not_object(true, format!(r#"{{"created_at":"c","batch":[{{{EVENT},"properties":[1,2]}}]}}"#))]+ #[case::flags_set(true, format!(r#"{{"created_at":"c","historical_migration":true,"capture_internal":true,"batch":[{{{EVENT}}}]}}"#))]+ #[case::flags_unreadable(true, format!(r#"{{"created_at":"c","historical_migration":{{"x":1}},"capture_internal":"maybe","batch":[{{{EVENT}}}]}}"#))]+ #[case::flags_lenient_strings(true, format!(r#"{{"created_at":"c","historical_migration":"yes","capture_internal":0,"batch":[{{{EVENT}}}]}}"#))]+ #[case::empty_batch(true, r#"{"created_at":"c","batch":[]}"#.to_owned())]+ #[case::empty_strings(true, r#"{"created_at":"","batch":[{"event":"","uuid":"","distinct_id":" ","timestamp":""}]}"#.to_owned())]+ #[case::many_events(true, format!(r#"{{"created_at":"c","batch":[{{{EVENT}}},{{{EVENT},"session_id":"s"}},{{{EVENT}}}]}}"#))]+ #[case::event_as_array(true, r#"{"created_at":"c","batch":[["e"," u ","d","t","s",null,{"cookieless_mode":true},{"p":1}]]}"#.to_owned())]+ #[case::event_as_short_array(true, r#"{"created_at":"c","batch":[["e","u","d","t","s","w"]]}"#.to_owned())]+ #[case::batch_as_array(true, format!(r#"["c",true,null,[{{{EVENT}}}]]"#))]+ #[case::numeric_distinct_id(false, r#"{"created_at":"c","batch":[{"event":"e","uuid":"u","distinct_id":12345678901234567890,"timestamp":"t"}]}"#.to_owned())]+ #[case::null_event(false, r#"{"created_at":"c","batch":[{"event":null,"uuid":"u","distinct_id":"d","timestamp":"t"}]}"#.to_owned())]+ #[case::missing_uuid(false, r#"{"created_at":"c","batch":[{"event":"e","distinct_id":"d","timestamp":"t"}]}"#.to_owned())]+ #[case::object_timestamp(false, r#"{"created_at":"c","batch":[{"event":"e","uuid":"u","distinct_id":"d","timestamp":{"a":"[email protected]"}}]}"#.to_owned())]+ #[case::numeric_session_id(false, format!(r#"{{"created_at":"c","batch":[{{{EVENT},"session_id":5}}]}}"#))]+ #[case::array_window_id(false, format!(r#"{{"created_at":"c","batch":[{{{EVENT},"window_id":["w"]}}]}}"#))]+ #[case::missing_created_at(false, format!(r#"{{"batch":[{{{EVENT}}}]}}"#))]+ #[case::numeric_created_at(false, format!(r#"{{"created_at":1,"batch":[{{{EVENT}}}]}}"#))]+ #[case::missing_batch(false, r#"{"created_at":"c"}"#.to_owned())]+ #[case::batch_not_array(false, r#"{"created_at":"c","batch":"[email protected]"}"#.to_owned())]+ #[case::event_not_object(false, r#"{"created_at":"c","batch":["[email protected]"]}"#.to_owned())]+ #[case::duplicate_event_key(false, r#"{"created_at":"c","batch":[{"event":"e","uuid":"u","distinct_id":"d","distinct_id":"d","timestamp":"t"}]}"#.to_owned())]+ #[case::duplicate_batch_key(false, format!(r#"{{"created_at":"c","created_at":"c","batch":[{{{EVENT}}}]}}"#))]+ #[case::event_as_too_short_array(false, r#"{"created_at":"c","batch":[["e","u","d","t"]]}"#.to_owned())]+ #[case::syntax_error(false, r#"{"created_at":"c","batch":[{]}"#.to_owned())]+ #[case::truncated(false, r#"{"created_at":"c","batch":[{"event":"e""#.to_owned())]+ #[case::lone_surrogate(false, r#"{"created_at":"c","batch":[{"event":"e","uuid":"u","distinct_id":"\ud800","timestamp":"t"}]}"#.to_owned())]+ #[case::number_out_of_range(false, r#"{"created_at":"c","batch":[{"event":"e","uuid":"u","distinct_id":1e400,"timestamp":"t"}]}"#.to_owned())]+ #[case::trailing_garbage(false, format!(r#"{{"created_at":"c","batch":[{{{EVENT}}}]}} x"#))]+ fn lenient_parse_accepts_and_builds_exactly_what_the_strict_types_do(+ #[case] accepted: bool,+ #[case] body: String,+ ) {+ let strict = serde_json::from_slice::<Batch>(body.as_bytes());+ let lenient = parse_batch(body.as_bytes());+ assert_eq!(strict.is_ok(), accepted, "strict parse of {body}");++ match (strict, lenient) {+ (Ok(strict), Ok(lenient)) => assert_same_batch(&strict, &lenient, &body),+ (Err(_), Err(_)) => {}+ (strict, lenient) => panic!(+ "strict ok={} but lenient ok={} for {body}",+ strict.is_ok(),+ lenient.is_ok()+ ),+ }+ }+}diff --git a/rust/capture/src/v1/analytics/mod.rs b/rust/capture/src/v1/analytics/mod.rsindex 65d620d0995a..c9c90db73432 100644--- a/rust/capture/src/v1/analytics/mod.rs+++ b/rust/capture/src/v1/analytics/mod.rs@@ -1,6 +1,7 @@pub mod constants;pub mod context;pub mod handler;+pub mod lenient;pub mod process;pub mod query;pub mod response;diff --git a/rust/capture/src/v1/analytics/types.rs b/rust/capture/src/v1/analytics/types.rsindex 65d751bd1e91..f9b4472222db 100644--- a/rust/capture/src/v1/analytics/types.rs+++ b/rust/capture/src/v1/analytics/types.rs@@ -28,7 +28,7 @@ use crate::ordering::{person_ordering, OrderingGuarantee};use crate::v1::context::RequestContext;use crate::v1::types::{Destination, Publishable};-fn empty_raw_object() -> Box<RawValue> {+pub(super) fn empty_raw_object() -> Box<RawValue> {RawValue::from_string("{}".to_owned()).unwrap()}@@ -38,13 +38,17 @@ fn deserialize_trimmed_string<'de, D>(deserializer: D) -> Result<String, D::ErrowhereD: serde::Deserializer<'de>,{- let s = String::deserialize(deserializer)?;+ Ok(trim_owned(String::deserialize(deserializer)?))+}++/// Shared by [`Event`] and the lenient parse so their trims can't diverge.+pub(super) fn trim_owned(s: String) -> String {let trimmed = s.trim();- Ok(if trimmed.len() == s.len() {+ if trimmed.len() == s.len() {s} else {trimmed.to_owned()- })+ }}/// Per-event outcome in the batch response.@@ -60,6 +64,7 @@ pub enum EventResult {Retry,}+/// Keep fields in sync with `LenientBatch` in `super::lenient`.#[derive(Debug, Deserialize)]pub struct Batch {pub created_at: String,@@ -79,14 +84,14 @@ fn lenient_flag(value: &Value) -> Option<bool> {}}-fn deserialize_lenient_flag<'de, D>(deserializer: D) -> Result<Option<bool>, D::Error>+pub(super) fn deserialize_lenient_flag<'de, D>(deserializer: D) -> Result<Option<bool>, D::Error>whereD: serde::Deserializer<'de>,{Ok(lenient_flag(&Value::deserialize(deserializer)?))}-fn deserialize_lenient_flag_or_false<'de, D>(deserializer: D) -> Result<bool, D::Error>+pub(super) fn deserialize_lenient_flag_or_false<'de, D>(deserializer: D) -> Result<bool, D::Error>whereD: serde::Deserializer<'de>,{@@ -300,6 +305,7 @@ fn coerce_product_tour_id(v: &Value) -> Parsed<String> {}}+/// Keep fields in sync with `LenientEvent` in `super::lenient`.#[derive(Debug, Deserialize)]pub struct Event {pub event: String,diff --git a/rust/capture/src/v1/error.rs b/rust/capture/src/v1/error.rsindex 09cb6c639325..9509918e0755 100644--- a/rust/capture/src/v1/error.rs+++ b/rust/capture/src/v1/error.rs@@ -3,10 +3,12 @@ use axum::response::{IntoResponse, Response};use axum::Json;use metrics::counter;use serde::Serialize;+use serde_json::error::Category;use thiserror::Error;use tracing::Level;use crate::v1::analytics::constants::DEFAULT_RETRY_AFTER_SECS;+use crate::v1::analytics::lenient::FieldTypeError;use crate::v1::constants::{CAPTURE_V1_ERROR_METRIC, CAPTURE_V1_UNKNOWN_PATH};use crate::v1::context::RequestContext;@@ -17,6 +19,66 @@ pub struct ErrorResponse {pub error_uri: String,}+#[derive(Debug)]+pub enum ParseFailure {+ Json {+ /// Can quote request values: log it, never put it in a response.+ message: String,+ category: Category,+ line: usize,+ column: usize,+ },+ Field(FieldTypeError),+}++impl ParseFailure {+ fn description(&self) -> String {+ match self {+ Self::Json {+ category,+ line,+ column,+ ..+ } => {+ let what = match category {+ Category::Syntax => "invalid JSON",+ Category::Data => "wrong JSON type or value",+ Category::Eof => "unexpected end of JSON",+ Category::Io => "unreadable body",+ };+ format!("Failed to parse request body: {what} at line {line}, column {column}.")+ }+ Self::Field(field) => format!("Failed to parse request body: {field}."),+ }+ }+}++impl std::fmt::Display for ParseFailure {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ Self::Json { message, .. } => f.write_str(message),+ Self::Field(field) => field.fmt(f),+ }+ }+}++impl From<serde_json::Error> for ParseFailure {+ fn from(e: serde_json::Error) -> Self {+ Self::Json {+ message: e.to_string(),+ category: e.classify(),+ line: e.line(),+ column: e.column(),+ }+ }+}++impl From<FieldTypeError> for ParseFailure {+ fn from(e: FieldTypeError) -> Self {+ Self::Field(e)+ }+}+impl ErrorResponse {fn new(error: &Error) -> Self {Self {@@ -41,7 +103,7 @@ pub enum Error {#[error("failed to decode request: {0}")]RequestDecodingError(String),#[error("failed to parse request: {0}")]- RequestParsingError(String),+ RequestParsingError(ParseFailure),#[error("request body is empty")]EmptyBody,#[error("request batch is empty")]@@ -147,7 +209,7 @@ impl Error {pub fn description(&self) -> String {match self {Self::RequestDecodingError(_) => "Failed to decode request body.".to_string(),- Self::RequestParsingError(_) => "Failed to parse request body.".to_string(),+ Self::RequestParsingError(failure) => failure.description(),Self::InvalidApiToken(_) => "The provided API token is not valid.".to_string(),Self::BillingLimitExceeded => "Billing quota exceeded. Events are being dropped. Upgrade your plan to resume ingestion.".to_string(),Self::InternalError(_) | Self::ServiceUnavailable(_) | Self::GatewayTimeout => self@@ -217,6 +279,20 @@ impl Error {pub(crate) fn stat_error(&self, ctx: Option<&RequestContext>) {let path = ctx.map(|c| c.path).unwrap_or(CAPTURE_V1_UNKNOWN_PATH);let status = self.status_code().as_str().to_owned();+ // Field errors add bounded `field` and `kind` labels; sums by the existing labels are unchanged.+ if let Self::RequestParsingError(ParseFailure::Field(field)) = self {+ counter!(+ CAPTURE_V1_ERROR_METRIC,+ "error" => self.tag(),+ "level" => self.level_tag(),+ "path" => path,+ "status_code" => status,+ "field" => field.field,+ "kind" => field.kind.as_str(),+ )+ .increment(1);+ return;+ }counter!(CAPTURE_V1_ERROR_METRIC,"error" => self.tag(),@@ -357,7 +433,7 @@ macro_rules! log_stat_error {impl From<serde_json::Error> for Error {fn from(e: serde_json::Error) -> Self {- Error::RequestParsingError(e.to_string())+ Error::RequestParsingError(e.into())}}@@ -373,6 +449,7 @@ impl IntoResponse for Error {#[cfg(test)]mod tests {use super::*;+ use crate::v1::analytics::lenient::JsonKind;use axum::body::to_bytes;use axum::http::header::RETRY_AFTER;use rstest::rstest;@@ -411,6 +488,18 @@ mod tests {assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR);}+ #[test]+ fn parse_failure_logs_serde_message_but_describes_only_the_position() {+ let serde_err = serde_json::from_str::<Vec<u32>>(r#"["[email protected]"]"#).unwrap_err();+ let err = Error::from(serde_err);++ assert!(err.to_string().contains("[email protected]"));+ assert_eq!(+ err.description(),+ "Failed to parse request body: wrong JSON type or value at line 1, column 20."+ );+ }+#[tokio::test]async fn billing_limit_custom_error_uri() {let err = Error::BillingLimitExceeded;@@ -496,4 +585,71 @@ mod tests {assert_eq!(body["error"], expected_tag);assert_eq!(body["error_description"], expected_description);}++ fn error_metric_labels(err: &Error) -> Vec<(String, String)> {+ use metrics_util::debugging::{DebugValue, DebuggingRecorder};++ let recorder = DebuggingRecorder::new();+ let snapshotter = recorder.snapshotter();+ let _guard = metrics::set_default_local_recorder(&recorder);+ err.stat_error(None);++ let mut series: Vec<_> = snapshotter+ .snapshot()+ .into_vec()+ .into_iter()+ .filter(|(key, ..)| key.key().name() == CAPTURE_V1_ERROR_METRIC)+ .collect();+ assert_eq!(series.len(), 1);+ let (key, _, _, value) = series.remove(0);+ assert_eq!(value, DebugValue::Counter(1));+ let mut labels: Vec<_> = key+ .key()+ .labels()+ .map(|l| (l.key().to_owned(), l.value().to_owned()))+ .collect();+ labels.sort();+ labels+ }++ #[rstest]+ #[case::field_type_error(+ Error::RequestParsingError(ParseFailure::Field(FieldTypeError {+ index: Some(3),+ field: "distinct_id",+ kind: JsonKind::Number,+ })),+ &[("field", "distinct_id"), ("kind", "number")]+ )]+ #[case::missing_field(+ Error::RequestParsingError(ParseFailure::Field(FieldTypeError {+ index: None,+ field: "created_at",+ kind: JsonKind::Missing,+ })),+ &[("field", "created_at"), ("kind", "missing")]+ )]+ #[case::corrupt_json(+ Error::from(serde_json::from_str::<serde_json::Value>("{").unwrap_err()),+ &[]+ )]+ fn parse_error_metric_adds_field_labels_only_for_field_errors(+ #[case] err: Error,+ #[case] extra: &[(&str, &str)],+ ) {+ let labels = error_metric_labels(&err);++ let mut expected: Vec<(String, String)> = [+ ("error", "request_parsing_error"),+ ("level", err.level_tag()),+ ("path", CAPTURE_V1_UNKNOWN_PATH),+ ("status_code", "400"),+ ]+ .iter()+ .chain(extra)+ .map(|(k, v)| (k.to_string(), v.to_string()))+ .collect();+ expected.sort();+ assert_eq!(labels, expected);+ }}