Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

6.5 KB
Raw Blame
//! Replays the `pii_scrub`, `pattern` and `normalize` vectors that Node recorded in
//! `nodejs/src/logs/parity/fixtures/modules` (see `logs-module-parity.test.ts`).

mod common;

use std::sync::Arc;

use common::{array, cases, get, string, try_get};
use logs_ingestion::avro::{AvroMap, AvroValue};
use logs_ingestion::js::{JsObject, JsString, JsValue};
use logs_ingestion::normalize::{transform_decoded_log_records_in_place, LogsSettings};
use logs_ingestion::pattern::build_log_pattern;
use logs_ingestion::pii::scrub_plain_string_with_stats;
use logs_ingestion::record::LogRecord;

fn number(value: &JsValue) -> f64 {
    match value {
        JsValue::Number(n) => *n,
        other => panic!("not a number: {other:?}"),
    }
}

fn short(text: &str) -> String {
    text.chars().take(80).collect()
}

#[test]
fn pii_scrub_matches_node() {
    let cases = cases("pii_scrub");
    assert_eq!(cases.len(), 44);
    for (input, output) in cases {
        let input = string(&input);
        let (scrubbed, n) = scrub_plain_string_with_stats(&input);
        assert_eq!(
            scrubbed,
            string(get(&output, "output")),
            "{:?}",
            short(&input)
        );
        assert_eq!(
            n as f64,
            number(get(&output, "piiReplacements")),
            "{:?}",
            short(&input)
        );
    }
}

#[test]
fn pattern_matches_node() {
    let cases = cases("pattern");
    assert_eq!(cases.len(), 132);
    for (input, output) in cases {
        let body = string(get(&input, "body"));
        let keys: Vec<String> = array(get(&input, "keys")).iter().map(string).collect();
        let keys: Vec<&str> = keys.iter().map(String::as_str).collect();
        let result = build_log_pattern(Some(&body), &keys);
        let ctx = format!("{:?} keys {keys:?}", short(&body));

        assert_eq!(
            JsString::from(result.pattern.as_str()),
            *get(&output, "pattern").as_js_string().unwrap(),
            "pattern {ctx}"
        );
        assert_eq!(
            result.body_kind.as_str(),
            string(get(&output, "bodyKind")),
            "kind {ctx}"
        );
        assert_eq!(
            JsValue::Bool(result.input_capped),
            *get(&output, "inputCapped"),
            "capped {ctx}"
        );
        assert_eq!(
            result.masked_length as f64,
            number(get(&output, "maskedLength")),
            "len {ctx}"
        );
        assert_eq!(
            result.json_key_count.map(|n| n as f64),
            try_get(&output, "jsonKeyCount").map(number),
            "key count {ctx}"
        );
        let fires: Vec<f64> = array(get(&output, "ruleFires"))
            .iter()
            .map(number)
            .collect();
        let actual: Vec<f64> = result.rule_fires.iter().map(|&n| n as f64).collect();
        assert_eq!(actual, fires, "rule fires {ctx}");
    }
}

fn settings_from_js(value: &JsValue) -> LogsSettings {
    let bool_of = |key: &str| match try_get(value, key) {
        Some(JsValue::Bool(b)) => Some(*b),
        None => None,
        other => panic!("{key}: {other:?}"),
    };
    LogsSettings {
        json_parse_logs: bool_of("json_parse_logs"),
        pii_scrub_logs: bool_of("pii_scrub_logs"),
        json_parse_logs_attribute_key: try_get(value, "json_parse_logs_attribute_key").map(string),
        ..LogsSettings::default()
    }
}

fn hex_decode(text: &str) -> Vec<u8> {
    (0..text.len())
        .step_by(2)
        .map(|i| u8::from_str_radix(&text[i..i + 2], 16).unwrap())
        .collect()
}

fn avro_from_js(value: &JsValue) -> AvroValue {
    match value {
        JsValue::Null => AvroValue::Null,
        JsValue::Bool(b) => AvroValue::Boolean(*b),
        JsValue::Number(n) => AvroValue::Long(*n as i64),
        JsValue::String(s) => AvroValue::String(s.to_string()),
        JsValue::Object(o) => {
            if let Some(JsValue::String(hex)) = o.get("$bytes") {
                return AvroValue::Bytes(hex_decode(&hex.to_string()));
            }
            let mut map = AvroMap::new();
            for (k, v) in o.iter() {
                map.define(k.to_string(), avro_from_js(v));
            }
            AvroValue::Map(map)
        }
        JsValue::Array(_) => panic!("records hold no arrays"),
    }
}

/// The harness's `recordToJson`: bytes as `{ $bytes: hex }`, map key order kept.
fn avro_to_js(value: &AvroValue) -> JsValue {
    match value {
        AvroValue::Null => JsValue::Null,
        AvroValue::Boolean(b) => JsValue::Bool(*b),
        AvroValue::Int(n) | AvroValue::Long(n) => JsValue::Number(*n as f64),
        AvroValue::Float(f) | AvroValue::Double(f) => JsValue::Number(*f),
        AvroValue::String(s) | AvroValue::Enum(s) => JsValue::string(s),
        AvroValue::Bytes(b) | AvroValue::Fixed(b) => {
            let mut o = JsObject::new();
            let hex: String = b.iter().map(|x| format!("{x:02x}")).collect();
            o.insert_str("$bytes", JsValue::string(&hex));
            JsValue::Object(o)
        }
        AvroValue::Map(m) => {
            let mut o = JsObject::new();
            for (k, v) in m.iter() {
                o.insert_str(k, avro_to_js(v));
            }
            JsValue::Object(o)
        }
        other => panic!("unexpected record value {other:?}"),
    }
}

fn record_from_js(value: &JsValue) -> LogRecord {
    let JsValue::Object(o) = value else {
        panic!("record is not an object");
    };
    let fields = o
        .iter()
        .map(|(k, v)| (Arc::from(k.to_string().as_str()), avro_from_js(v)))
        .collect();
    LogRecord::from_value(AvroValue::Record(fields))
}

fn record_to_js(record: &LogRecord) -> JsValue {
    let mut o = JsObject::new();
    for (k, v) in record.fields.iter().chain(record.extra.iter()) {
        o.insert_str(k, avro_to_js(v));
    }
    JsValue::Object(o)
}

#[test]
fn normalize_matches_node() {
    let cases = cases("normalize");
    assert_eq!(cases.len(), 209);
    for (i, (input, output)) in cases.into_iter().enumerate() {
        let settings = settings_from_js(get(&input, "settings"));
        let mut records = vec![record_from_js(get(&input, "record"))];
        let pii = transform_decoded_log_records_in_place(&mut records, &settings);
        // Compared with key order: the attribute order is what flatten eviction depends on.
        assert_eq!(
            record_to_js(&records[0]),
            *get(&output, "record"),
            "case {i} settings {settings:?}"
        );
        assert_eq!(
            pii.pii_replacements as f64,
            number(get(&output, "pii_replacements")),
            "case {i}"
        );
    }
}