Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

22.7 KB
Raw Blame
use std::sync::Arc;

use anyhow::{Context, Error};
use chrono::Duration;
use common_types::{InternallyCapturedEvent, RawEvent};
use rayon::iter::IntoParallelIterator;
use rayon::prelude::*;
use serde::{de::DeserializeOwned, Deserialize, Serialize};

use crate::error::UserError;
use crate::{context::AppContext, job::model::JobModel};

use super::{
    content::{
        amplitude::AmplitudeEvent, captured::captured_parse_fn, mixpanel::MixpanelEvent,
        ContentType, TransformContext,
    },
    Parsed,
};

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum FormatConfig {
    JsonLines {
        skip_blanks: bool,
        content: ContentType,
    },
}

pub type ParserFnFor<T> = Box<dyn Fn(Vec<u8>) -> Result<Parsed<Vec<T>>, Error> + Send + Sync>;

pub type ParserFn = ParserFnFor<InternallyCapturedEvent>;

impl FormatConfig {
    pub async fn get_parser(
        &self,
        model: &JobModel,
        context: Arc<AppContext>,
    ) -> Result<ParserFn, Error> {
        // Only support json-lines for now
        let Self::JsonLines {
            skip_blanks,
            content,
        } = self;

        let transform_context = TransformContext {
            team_id: model.team_id,
            token: context.get_token_for_team_id(model.team_id).await?,
            job_id: model.id,
            identify_cache: context.identify_cache.clone(),
            group_cache: context.group_cache.clone(),
            import_events: model.import_config.import_events,
            generate_identify_events: model.import_config.generate_identify_events,
            generate_group_identify_events: model.import_config.generate_group_identify_events,
        };

        match content {
            ContentType::Mixpanel(config) => {
                let format_parse = json_nd(*skip_blanks);

                let event_transform = MixpanelEvent::parse_fn(
                    transform_context,
                    config.skip_no_distinct_id,
                    config
                        .timestamp_offset_seconds
                        .map(Duration::seconds)
                        .unwrap_or_default(),
                    skip_geoip(),
                );

                let parser = move |data| {
                    let parsed: Parsed<Vec<MixpanelEvent>> = format_parse(data)?;
                    let consumed = parsed.consumed;
                    let result: Result<_, Error> = parsed
                        .data
                        .into_par_iter()
                        .map(&event_transform)
                        .filter_map(|x| x.transpose())
                        .collect();

                    Ok(Parsed {
                        data: result?,
                        consumed,
                    })
                };

                Ok(Box::new(parser))
            }
            ContentType::Amplitude => {
                let format_parse = json_nd(*skip_blanks);
                let event_transform = AmplitudeEvent::parse_fn(transform_context, skip_geoip());
                let parser = move |data| {
                    let parsed: Parsed<Vec<AmplitudeEvent>> = format_parse(data)?;
                    let consumed = parsed.consumed;
                    // Propagate the first transform error rather than silently
                    // dropping events (matches the Mixpanel/Captured paths).
                    // Each Amplitude input event may produce multiple output
                    // events (the event itself plus optional identify and group
                    // identify events), so we collect into Vec<Vec<_>> and
                    // flatten after the error check.
                    let result: Result<Vec<Vec<_>>, Error> =
                        parsed.data.into_par_iter().map(&event_transform).collect();

                    Ok(Parsed {
                        data: result?.into_iter().flatten().collect(),
                        consumed,
                    })
                };

                Ok(Box::new(parser))
            }
            ContentType::Captured => {
                let format_parse = json_nd(*skip_blanks);
                let event_transform = captured_parse_fn(transform_context, skip_geoip());
                let parser = move |data| {
                    let parsed: Parsed<Vec<RawEvent>> = format_parse(data)?;
                    let consumed = parsed.consumed;
                    let result: Result<_, Error> = parsed
                        .data
                        .into_par_iter()
                        .map(&event_transform)
                        .filter_map(|x| x.transpose())
                        .collect();

                    Ok(Parsed {
                        data: result?,
                        consumed,
                    })
                };

                Ok(Box::new(parser))
            }
        }
    }
}

const NEWLINE_DELIM: u8 = b'\n';

pub fn newline_delim<T: Send>(
    skip_blank_lines: bool,
    inner: impl Fn(&str) -> Result<T, Error> + Sync,
) -> impl Fn(Vec<u8>) -> Result<Parsed<Vec<T>>, Error> {
    newline_delim_lines(skip_blank_lines, move |line, _is_complete| inner(line))
}

/// Like [`newline_delim`], but the inner closure also receives whether the line
/// was newline-terminated (`true`) or is the chunk's trailing remainder
/// (`false`). The remainder is usually a line split at the chunk boundary, so an
/// error-tolerant parser (the trial path) must still return `Err` for it — that
/// leaves it unconsumed for the next chunk to re-read — while recording parse
/// errors on complete lines inline instead of failing the whole chunk.
pub fn newline_delim_lines<T: Send>(
    skip_blank_lines: bool,
    inner: impl Fn(&str, bool) -> Result<T, Error> + Sync,
) -> impl Fn(Vec<u8>) -> Result<Parsed<Vec<T>>, Error> {
    move |data: Vec<u8>| {
        // A zero-length chunk (e.g. a read at or past the end of a part) has nothing
        // to consume. Reporting any consumed bytes for it would advance the caller's
        // offset past real data, so it could never converge on the part's end.
        if data.is_empty() {
            return Ok(Parsed {
                data: Vec::new(),
                consumed: 0,
            });
        }

        let mut cursor = 0;
        let mut last_consumed_byte = 0;

        let mut lines = Vec::new();

        // Note that we can't parse the entire buffer as a string and then iterate over the lines/characters here,
        // because it is possible for us to have split the file on a utf8 character boundary, meaning the last line in the
        // input will end in an invalid utf8 byte sequence. Instead, we iterate over the bytes directly, and only convert
        // them to utf8 when we've got a complete line.
        while cursor < data.len() {
            // The cursor != 0 bit here is because the "this might be the end of the file" handling below this can sometimes
            // cause the next chunk to start exactly on a newline. This does run the risk of accidentally skipping a blank line,
            // but we generally don't consider newlines important anyway (skip_blank_lines is generally only set to false to ensure
            // the presence of one in the input will cause the inner function to return an error, not because they're semantically
            // relevant)
            if data[cursor] == NEWLINE_DELIM && cursor != 0 {
                let line = std::str::from_utf8(&data[last_consumed_byte..cursor])
                    .context("Failed to parse line as utf8")?;
                if !skip_blank_lines || !line.trim().is_empty() {
                    lines.push((cursor, line.trim()));
                }
                last_consumed_byte = cursor;
            }

            cursor += 1;
        }

        // Sometimes we've split the file on a utf8 character boundary, and the last line is incomplete not just as
        // a line that can be fed to the inner parser, but as a utf8 string in general. If that's the case, just set
        // the remainder to be empty, and we can handle it in the next chunk.
        let remainder_str = std::str::from_utf8(&data[last_consumed_byte..]).ok();
        // A blank remainder (e.g. a chunk ending in "\n" or trailing blank lines at the
        // end of a part) has nothing to parse. When blank lines are skippable, count it
        // as consumed so the caller's offset can reach the end of the part - at the end
        // of a stream-decompressed part there is no next chunk to re-read it from.
        let remainder_is_blank = remainder_str.is_some_and(|r| r.trim().is_empty());
        let remainder = remainder_str.map(|r| inner(r, false));

        let mut output = Vec::with_capacity(lines.len());
        let intermediate: Vec<_> = lines
            .into_par_iter()
            .map(|(end_byte_idx, line)| (end_byte_idx, inner(line, true)))
            .collect();

        drop(data);

        let mut last_validly_consumed_byte = 0;
        for (byte_idx, res) in intermediate.into_iter() {
            match res {
                Ok(parsed) => {
                    output.push(parsed);
                    last_validly_consumed_byte = byte_idx;
                }
                Err(e) => {
                    return Err(e.context(format!(
                        "Starting at byte {last_validly_consumed_byte} of current chunk"
                    )));
                }
            }
        }

        // If we managed to parse the last line, add it too, but if we didn't, assume it's due to this chunk being partway through the file,
        // and carry on.
        if let Some(Ok(parsed)) = remainder {
            output.push(parsed);
            // -1 because at this point the cursor is pointing at the end of the data,
            // and we want to point at the last byte we actually consumed
            last_validly_consumed_byte = cursor - 1;
        } else if skip_blank_lines && remainder_is_blank {
            last_validly_consumed_byte = cursor - 1;
        }

        let parsed = Parsed {
            data: output,
            consumed: last_validly_consumed_byte + 1,
        };

        Ok(parsed)
    }
}

/// Trait for types that can provide user-facing JSON parse error messages.
/// Each event type (RawEvent, MixpanelEvent, AmplitudeEvent) implements this
/// to provide format-specific error messages that help users fix their data.
pub trait UserFacingParseError {
    /// Returns a user-facing error message for any JSON parse error.
    /// Dispatches to specific handlers based on error category.
    /// Override `user_facing_schema_error` to customize schema mismatch messages.
    fn user_facing_parse_error(err: &serde_json::Error) -> String {
        use serde_json::error::Category;

        let base_message = match err.classify() {
            Category::Eof => Self::user_facing_eof_error(err),
            Category::Syntax => Self::user_facing_syntax_error(err),
            Category::Data => Self::user_facing_schema_error(err),
            Category::Io => "Error reading the data. Please try again.".to_string(),
        };

        format!("{} (Error at column {})", base_message, err.column())
    }

    fn user_facing_eof_error(err: &serde_json::Error) -> String {
        let err_str = err.to_string();
        if err_str.contains("parsing a value") && err.column() == 0 {
            "The line appears to be empty. Each line should contain a valid JSON object."
                .to_string()
        } else {
            "The JSON appears to be truncated or incomplete. Check for missing closing braces or brackets.".to_string()
        }
    }

    fn user_facing_syntax_error(err: &serde_json::Error) -> String {
        let err_str = err.to_string();
        if err_str.contains("key must be a string") {
            "JSON keys must be quoted strings. Use double quotes (\") not single quotes (')."
                .to_string()
        } else if err_str.contains("trailing comma") {
            "Remove the trailing comma before the closing brace or bracket.".to_string()
        } else if err_str.contains("expected `,`") {
            "Missing comma between values or properties.".to_string()
        } else if err_str.contains("expected `:`") {
            "Missing colon after property name.".to_string()
        } else if err_str.contains("expected ident") {
            "Invalid JSON syntax. Make sure the line is valid JSON, not plain text.".to_string()
        } else {
            "Invalid JSON syntax. Please check for proper formatting.".to_string()
        }
    }

    fn user_facing_schema_error(err: &serde_json::Error) -> String {
        user_facing_schema_error_generic(err)
    }
}

fn user_facing_schema_error_generic(err: &serde_json::Error) -> String {
    let err_str = err.to_string();

    if err_str.contains("missing field") {
        if let Some(field_name) = extract_field_name(&err_str, "missing field `", "`") {
            return format!(
                "Missing required field '{field_name}'. Please check that your data includes this field."
            );
        }
    }

    if err_str.contains("invalid type:") {
        let got = extract_between(&err_str, "invalid type: ", ", expected");
        let expected = extract_between(&err_str, "expected ", " at line");

        if let (Some(got), Some(expected)) = (got, expected) {
            if expected.contains("map") {
                return format!(
                    "Expected an object/map but got {got}. This field must be a JSON object like {{\"key\": \"value\"}}."
                );
            }
            if expected == "a string" {
                return format!(
                    "Expected a string value but got {got}. String values must be quoted."
                );
            }
            if expected == "an integer" || expected == "a number" {
                return format!("Expected a number but got {got}.");
            }

            return format!(
                "Type mismatch: expected {expected} but got {got}. Please check your data format."
            );
        }
    }

    if err_str.contains("unknown field") {
        if let Some(field_name) = extract_field_name(&err_str, "unknown field `", "`") {
            return format!(
                "Unknown field '{field_name}'. This field is not recognized. Check for typos or remove this field."
            );
        }
    }

    // Fallback
    "The JSON structure doesn't match the expected format. Please check that your data matches the required schema.".to_string()
}

pub fn extract_field_name(s: &str, prefix: &str, suffix: &str) -> Option<String> {
    let start = s.find(prefix)? + prefix.len();
    let end = s[start..].find(suffix)? + start;
    Some(s[start..end].to_string())
}

pub fn extract_between(s: &str, start_marker: &str, end_marker: &str) -> Option<String> {
    let start = s.find(start_marker)? + start_marker.len();
    let end = s[start..].find(end_marker)? + start;
    Some(s[start..end].to_string())
}

pub fn json_nd<T>(skip_blank_lines: bool) -> impl Fn(Vec<u8>) -> Result<Parsed<Vec<T>>, Error>
where
    T: DeserializeOwned + Send + UserFacingParseError,
{
    newline_delim(skip_blank_lines, |line| {
        let parsed = serde_json::from_str(line)
            .map_err(|e| {
                let user_msg = T::user_facing_parse_error(&e);
                anyhow::Error::from(e).context(UserError::new(user_msg))
            })
            .context("Failed to json parse line")?;
        Ok(parsed)
    })
}

pub fn skip_geoip() -> impl Fn(RawEvent) -> Result<Option<RawEvent>, Error> {
    move |mut event| {
        event
            .properties
            .insert("$geoip_disable".to_string(), serde_json::Value::Bool(true));

        Ok(Some(event))
    }
}

#[cfg(test)]
mod tests {
    use crate::source::{folder::FolderSource, DataSource};

    use super::*;
    use serde::Deserialize;
    use std::fs;
    use tempfile::TempDir;

    #[derive(Deserialize, Debug, PartialEq)]
    struct TestData {
        id: i32,
        name: String,
    }

    impl UserFacingParseError for TestData {}

    async fn setup_test_files() -> (TempDir, FolderSource) {
        let temp_dir = TempDir::new().unwrap();
        fs::write(
            temp_dir.path().join("data.jsonl"),
            r#"{"id": 1, "name": "test1"}
{"id": 2, "name": "test2"}
{"id": 3, "name": "test3"}"#,
        )
        .unwrap();

        fs::write(
            temp_dir.path().join("blank_lines.jsonl"),
            r#"{"id": 1, "name": "test1"}

{"id": 2, "name": "test2"}
"#,
        )
        .unwrap();

        let source = FolderSource::new(temp_dir.path().to_str().unwrap().to_string())
            .await
            .unwrap();

        (temp_dir, source)
    }

    #[tokio::test]
    async fn test_json_nd_parsing() {
        let (_temp_dir, source) = setup_test_files().await;
        let chunk = source.get_chunk("data.jsonl", 0, 100).await.unwrap();
        let chunk_len = chunk.len();
        let parsed = json_nd::<TestData>(false)(chunk).unwrap();

        assert_eq!(parsed.data.len(), 3);
        assert_eq!(
            parsed.data[0],
            TestData {
                id: 1,
                name: "test1".to_string()
            }
        );
        assert_eq!(parsed.consumed, chunk_len);
    }

    #[tokio::test]
    async fn test_json_nd_with_blank_lines() {
        let (_temp_dir, source) = setup_test_files().await;
        let data = source.get_chunk("blank_lines.jsonl", 0, 100).await.unwrap();

        let parsed_with_blanks = json_nd::<TestData>(true)(data.clone()).unwrap();
        assert_eq!(parsed_with_blanks.data.len(), 2);

        // IF we're not skipping blank lines, an empty line will cause json parsing
        // to fail, and we should get an error
        let should_be_error = json_nd::<TestData>(false)(data);
        assert!(should_be_error.is_err());
    }

    #[tokio::test]
    async fn test_partial_line() {
        let (_temp_dir, source) = setup_test_files().await;
        let data = source.get_chunk("data.jsonl", 0, 30).await.unwrap();
        let parsed = json_nd::<TestData>(false)(data).unwrap();

        assert_eq!(parsed.data.len(), 1);
        // 26 "data" characters, plus the newline
        assert_eq!(parsed.consumed, 27);
    }

    #[test]
    fn test_newline_delim_empty_input_consumes_nothing() {
        // An empty chunk (e.g. a streaming source read past EOF) must report zero
        // bytes consumed. Reporting consumed > 0 for empty input makes the job's
        // part offset overshoot the part's total size and the job crawls forever.
        let parsed = json_nd::<TestData>(true)(Vec::new()).unwrap();
        assert!(parsed.data.is_empty());
        assert_eq!(
            parsed.consumed, 0,
            "empty input must consume 0 bytes, got {}",
            parsed.consumed
        );
    }

    #[test]
    fn test_newline_delim_blank_trailing_remainder() {
        // A part's final chunk can end in blank content ("\n", trailing blank
        // lines). With skip_blank_lines the blanks must count as consumed so the
        // offset can reach the end of the part - at the end of a stream-decompressed
        // part there is no next chunk to re-read them from.
        let line = br#"{"id": 1, "name": "test1"}"#;
        for tail in [b"\n\n".as_slice(), b"\n\n\n", b"\n \n"] {
            let mut data = line.to_vec();
            data.extend_from_slice(tail);
            let len = data.len();

            let parsed = json_nd::<TestData>(true)(data).unwrap();
            assert_eq!(parsed.data.len(), 1, "tail {tail:?}");
            assert_eq!(
                parsed.consumed, len,
                "blank tail {tail:?} must be fully consumed"
            );
        }

        // Without skip_blank_lines a blank line is still an error, preserving the
        // "blank lines in input should fail the inner parser" contract.
        let mut data = line.to_vec();
        data.extend_from_slice(b"\n\n");
        assert!(json_nd::<TestData>(false)(data).is_err());

        // An invalid-utf8 tail is not "blank" - it must be left unconsumed for the
        // next chunk, exactly as before.
        let mut data = line.to_vec();
        data.extend_from_slice(b"\n{\"id\": 2, \"name\": \"\xf0\x9f");
        let len = data.len();
        let parsed = json_nd::<TestData>(true)(data).unwrap();
        assert_eq!(parsed.data.len(), 1);
        assert_eq!(parsed.consumed, line.len() + 1);
        assert!(parsed.consumed < len);
    }

    #[test]
    fn test_json_parse_error_has_user_friendly_message() {
        use crate::error::get_user_message;

        let invalid_json = b"not valid json\n".to_vec();
        let result = json_nd::<TestData>(false)(invalid_json);

        assert!(result.is_err());
        let err = result.unwrap_err();

        let user_message = get_user_message(&err);
        assert!(
            user_message.contains("Invalid JSON syntax") || user_message.contains("plain text"),
            "Expected user message to give specific guidance, got: {user_message}"
        );
        assert!(
            user_message.contains("column"),
            "Expected user message to include column info, got: {user_message}"
        );
    }

    #[test]
    fn test_json_parse_error_preserves_underlying_error() {
        let invalid_json = b"not valid json\n".to_vec();
        let result = json_nd::<TestData>(false)(invalid_json);

        assert!(result.is_err());
        let err = result.unwrap_err();

        let full_error = format!("{err:#}");
        assert!(
            full_error.contains("expected value") || full_error.contains("expected ident"),
            "Expected full error to contain serde details, got: {full_error}"
        );
    }

    #[test]
    fn test_specific_error_messages_for_common_issues() {
        use crate::error::get_user_message;

        let truncated = b"{\"id\": 1, \"name\": \"test\n".to_vec();
        let err = json_nd::<TestData>(false)(truncated).unwrap_err();
        let msg = get_user_message(&err);
        assert!(
            msg.contains("truncated") || msg.contains("incomplete"),
            "Truncated JSON should mention truncation: {msg}"
        );

        let trailing_comma = b"{\"id\": 1,}\n".to_vec();
        let err = json_nd::<TestData>(false)(trailing_comma).unwrap_err();
        let msg = get_user_message(&err);
        assert!(
            msg.contains("trailing comma"),
            "Trailing comma should be mentioned: {msg}"
        );

        let single_quotes = b"{'id': 1}\n".to_vec();
        let err = json_nd::<TestData>(false)(single_quotes).unwrap_err();
        let msg = get_user_message(&err);
        assert!(
            msg.contains("double quotes") || msg.contains("quoted strings"),
            "Single quotes error should suggest double quotes: {msg}"
        );

        let missing_comma = b"{\"id\": 1 \"name\": \"test\"}\n".to_vec();
        let err = json_nd::<TestData>(false)(missing_comma).unwrap_err();
        let msg = get_user_message(&err);
        assert!(
            msg.contains("comma"),
            "Missing comma should be mentioned: {msg}"
        );
    }
}