:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::collections::HashMap;
use anyhow::Error;
use celes::Country;
use chrono::{DateTime, Duration, Utc};
use common_types::{CapturedEvent, InternallyCapturedEvent, RawEvent};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use uuid::Uuid;
/// UUID namespace for generating deterministic UUIDs from Mixpanel $insert_id values.
/// This allows deduplication of events that may be imported multiple times.
/// Generated using `uuidgen` - this is a random UUID that serves as our namespace.
const MIXPANEL_INSERT_ID_NAMESPACE: Uuid = Uuid::from_bytes(*b"posthog_mixpanel");
use super::TransformContext;
use crate::parse::format::{extract_between, extract_field_name, UserFacingParseError};
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub struct MixpanelContentConfig {
#[serde(default)] // Defaults to false
pub skip_no_distinct_id: bool,
// We had a customer report that mixpanel used to have a timestamp offsets bug, and they wanted to
// update all event timestamps as they were being ingested.
pub timestamp_offset_seconds: Option<i64>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub struct MixpanelEvent {
event: String,
properties: MixpanelProperties,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub struct MixpanelProperties {
#[serde(rename = "time")]
timestamp: i64,
distinct_id: Option<String>,
#[serde(flatten)]
other: HashMap<String, Value>,
}
/// Implement schema-specific error messages for MixpanelEvent
/// That we can surface to the user to let them know what's wrong with the
/// data set they are trying to import
impl UserFacingParseError for MixpanelEvent {
fn user_facing_schema_error(err: &serde_json::Error) -> String {
let err_str = err.to_string();
if err_str.contains("missing field") {
if let Some(field_name) = extract_field_name(&err_str, "missing field `", "`") {
return match field_name.as_str() {
"event" => "Missing required field 'event'. Each Mixpanel event must have an 'event' field with the event name (e.g., \"event\": \"page_view\").".to_string(),
"properties" => "Missing required field 'properties'. Each Mixpanel event must have a 'properties' object containing at least the 'time' field.".to_string(),
"time" => "Missing required field 'time' in 'properties'. Each Mixpanel event must have a timestamp (e.g., \"properties\": {\"time\": 1697379000}).".to_string(),
_ => format!("Missing required field '{field_name}'. Please check that your Mixpanel export includes this field."),
};
}
}
if err_str.contains("invalid type:") {
let got = extract_between(&err_str, "invalid type: ", ", expected");
let expected = extract_between(&err_str, "expected ", " at line");
if let (Some(got), Some(expected)) = (got, expected) {
if err_str.contains("`event`") || (expected == "a string" && err.column() < 15) {
return format!(
"The 'event' field must be a string (e.g., \"event\": \"page_view\"), but got {got}."
);
}
if err_str.contains("`time`") || expected.contains("i64") {
return format!(
"The 'time' field must be a Unix timestamp (integer), but got {got}. Use seconds since epoch (e.g., 1697379000)."
);
}
if expected.contains("map") || expected.contains("struct") {
return format!(
"Expected an object/map but got {got}. The 'properties' field must be a JSON object like {{\"time\": 1697379000}}."
);
}
}
}
// Fallback to generic message
"The JSON structure doesn't match the expected Mixpanel event format. Required fields: 'event' (string), 'properties' (object with 'time' as integer timestamp).".to_string()
}
}
// Based off sample data provided by customer.
impl MixpanelEvent {
pub fn parse_fn(
context: TransformContext,
skip_no_distinct_id: bool,
timestamp_offset: Duration,
event_transform: impl Fn(RawEvent) -> Result<Option<RawEvent>, Error>,
) -> impl Fn(Self) -> Result<Option<InternallyCapturedEvent>, Error> {
move |mx| {
let token = context.token.clone();
let team_id = context.team_id;
let distinct_id = match (get_distinct_id(&mx.properties), skip_no_distinct_id) {
(Some(distinct_id), _) => distinct_id,
(None, true) => return Ok(None),
(None, false) => Uuid::now_v7().to_string(),
};
// Generate a deterministic UUID from $insert_id if present, otherwise use random UUIDv7.
// This allows deduplication of events that may be imported multiple times.
let event_uuid = mx
.properties
.other
.get("$insert_id")
.and_then(|v| v.as_str())
.map(|insert_id| Uuid::new_v5(&MIXPANEL_INSERT_ID_NAMESPACE, insert_id.as_bytes()))
.unwrap_or_else(Uuid::now_v7);
// Was seeing timestamp values come in that were in seconds, not milliseconds
// Do a quick heuristic check on the size of the timestamp to determine if it's in seconds or milliseconds
let timestamp_value = mx.properties.timestamp;
let timestamp_seconds = if timestamp_value > 10_000_000_000 {
timestamp_value / 1000
} else {
timestamp_value
};
// We don't support subsecond precision for historical imports
let timestamp = DateTime::<Utc>::from_timestamp(timestamp_seconds, 0)
.ok_or(Error::msg("Invalid timestamp"))?;
let timestamp = timestamp + timestamp_offset;
let properties = mx.properties.other;
let properties = map_geoip_props(properties);
let properties = remove_mp_props(properties);
let properties = add_source_data(properties, context.job_id);
let raw_event = RawEvent {
token: Some(token.clone()),
distinct_id: Some(Value::String(distinct_id.clone())),
uuid: Some(event_uuid),
event: map_event_names(mx.event),
properties,
// We send timestamps in iso 8601 format
timestamp: Some(timestamp.to_rfc3339()),
set: None,
set_once: None,
offset: None,
};
let Some(raw_event) = event_transform(raw_event)? else {
return Ok(None);
};
// Only return the event if import_events is enabled
if context.import_events {
let inner = CapturedEvent {
uuid: event_uuid,
distinct_id,
session_id: None,
ip: "127.0.0.1".to_string(),
data: serde_json::to_string(&raw_event)?,
now: Utc::now().to_rfc3339(),
sent_at: None,
token,
event: raw_event.event.clone(),
timestamp,
is_cookieless_mode: false,
historical_migration: true,
};
Ok(Some(InternallyCapturedEvent { team_id, inner }))
} else {
Ok(None)
}
}
}
}
// Maps mixpanel event names to posthog event names
pub fn map_event_names(event: String) -> String {
// TODO - add more as you find them
match event.as_str() {
"$mp_web_page_view" => "$pageview".to_string(),
_ => event,
}
}
fn get_distinct_id(props: &MixpanelProperties) -> Option<String> {
let distinct_id = props.distinct_id.clone();
let Some(before_identity) = props.other.get("$distinct_id_before_identity") else {
return distinct_id;
};
// If it's not a string, return the set distinct ID
let Some(before_identity) = before_identity.as_str() else {
return distinct_id;
};
// If we don't have a distinct ID, return the before identity
let Some(distinct_id) = distinct_id else {
return Some(before_identity.to_string());
};
// If the distinct_id starts with "$device:", it's an anonymous ID
if distinct_id.starts_with("$device:") {
return Some(before_identity.to_string());
}
// If the distinct_id contains only uppercase letters and dashes, it's an anonymous ID
if distinct_id
.chars()
.all(|c| c.is_ascii_uppercase() || c == '-' || c.is_ascii_digit())
{
return Some(before_identity.to_string());
}
// We default to using the distinct ID
Some(distinct_id)
}
const GEOIP_PROP_MAPPINGS: &[(&str, &str)] = &[
("$city", "$geoip_city_name"),
("$region", "$geoip_subdivision_1_name"),
("mp_country_code", "$geoip_country_code"),
];
fn map_geoip_props(mut props: HashMap<String, Value>) -> HashMap<String, Value> {
for (from, to) in GEOIP_PROP_MAPPINGS {
if let Some(value) = props.remove(*from) {
props.insert(to.to_string(), value);
}
}
if let Some(code) = props.get("$geoip_country_code").and_then(|c| c.as_str()) {
if let Some(country_name) = map_country_code(code) {
props.insert(
"$geoip_country_name".to_string(),
Value::String(country_name),
);
}
}
props
}
// We have to do some mapping because maxmind doesn't precisely follow ISO3166
// Names taken from: http://www.geonames.org/countries/
const LONG_NAME_MAP: &[(&str, &str)] = &[
("The United States of America", "United States"),
(
"The United Kingdom Of Great Britain And Northern Ireland",
"United Kingdom",
),
("The United Arab Emirates", "United Arab Emirates"),
];
fn map_country_code(code: &str) -> Option<String> {
let country = Country::from_alpha2(code).ok()?;
for (long_name, short_name) in LONG_NAME_MAP {
if country.long_name == *long_name {
return Some(short_name.to_string());
}
}
Some(country.long_name.to_string())
}
const MP_PROPS_TO_REMOVE: &[&str] = &[
"$mp_api_endpoint",
"mp_processing_time_ms",
"$insert_id",
"$geo_source",
"$mp_api_timestamp_ms",
];
fn remove_mp_props(mut props: HashMap<String, Value>) -> HashMap<String, Value> {
for prop in MP_PROPS_TO_REMOVE {
props.remove(*prop);
}
props
}
fn add_source_data(
mut props: HashMap<String, Value>,
job_id: uuid::Uuid,
) -> HashMap<String, Value> {
props.insert("historical_migration".to_string(), Value::Bool(true));
props.insert(
"analytics_source".to_string(),
Value::String("mixpanel".to_string()),
);
props.insert(
"$import_job_id".to_string(),
Value::String(job_id.to_string()),
);
props
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn identity_transform(event: RawEvent) -> Result<Option<RawEvent>, Error> {
Ok(Some(event))
}
#[test]
fn test_job_id_in_mixpanel_event() {
let test_job_id = Uuid::now_v7();
let mx_event = MixpanelEvent {
event: "test_event".to_string(),
properties: MixpanelProperties {
timestamp: 1697379000,
distinct_id: Some("user123".to_string()),
other: HashMap::new(),
},
};
let context = TransformContext {
team_id: 123,
token: "test_token".to_string(),
job_id: test_job_id,
identify_cache: std::sync::Arc::new(crate::cache::MockIdentifyCache::new()),
group_cache: std::sync::Arc::new(crate::cache::MockGroupCache::new()),
import_events: true,
generate_identify_events: false,
generate_group_identify_events: false,
};
let parser =
MixpanelEvent::parse_fn(context, false, Duration::seconds(0), identity_transform);
let result = parser(mx_event).unwrap().unwrap();
let data: RawEvent = serde_json::from_str(&result.inner.data).unwrap();
assert_eq!(
data.properties.get("$import_job_id"),
Some(&json!(test_job_id.to_string()))
);
assert_eq!(
data.properties.get("historical_migration"),
Some(&json!(true))
);
assert_eq!(
data.properties.get("analytics_source"),
Some(&json!("mixpanel"))
);
}
#[test]
fn test_mixpanel_event_has_historical_migration_and_now_fields() {
let test_job_id = Uuid::now_v7();
let before_test = Utc::now();
let mx_event = MixpanelEvent {
event: "test_event".to_string(),
properties: MixpanelProperties {
timestamp: 1697379000,
distinct_id: Some("user123".to_string()),
other: HashMap::new(),
},
};
let context = TransformContext {
team_id: 123,
token: "test_token".to_string(),
job_id: test_job_id,
identify_cache: std::sync::Arc::new(crate::cache::MockIdentifyCache::new()),
group_cache: std::sync::Arc::new(crate::cache::MockGroupCache::new()),
import_events: true,
generate_identify_events: false,
generate_group_identify_events: false,
};
let parser =
MixpanelEvent::parse_fn(context, false, Duration::seconds(0), identity_transform);
let result = parser(mx_event).unwrap().unwrap();
let after_test = Utc::now();
assert!(
result.inner.historical_migration,
"historical_migration field must be true for batch import events"
);
assert!(
!result.inner.now.is_empty(),
"now field must be set for events"
);
let now_timestamp = chrono::DateTime::parse_from_rfc3339(&result.inner.now)
.expect("now should be valid RFC3339 timestamp")
.with_timezone(&Utc);
assert!(
now_timestamp >= before_test && now_timestamp <= after_test,
"now timestamp should be current (between test start and end)"
);
let serialized = serde_json::to_value(&result.inner).unwrap();
assert_eq!(
serialized["historical_migration"],
json!(true),
"historical_migration must be in serialized output"
);
assert!(
serialized["now"].is_string(),
"now must be a string in serialized output"
);
}
#[test]
fn test_deterministic_uuid_from_insert_id() {
let test_job_id = Uuid::now_v7();
let context = TransformContext {
team_id: 123,
token: "test_token".to_string(),
job_id: test_job_id,
identify_cache: std::sync::Arc::new(crate::cache::MockIdentifyCache::new()),
group_cache: std::sync::Arc::new(crate::cache::MockGroupCache::new()),
import_events: true,
generate_identify_events: false,
generate_group_identify_events: false,
};
// Parse the same event twice with the same $insert_id
let parser1 = MixpanelEvent::parse_fn(
context.clone(),
false,
Duration::seconds(0),
identity_transform,
);
let parser2 =
MixpanelEvent::parse_fn(context, false, Duration::seconds(0), identity_transform);
let mut other1 = HashMap::new();
other1.insert("$insert_id".to_string(), json!("unique_insert_id_123"));
let mx_event1 = MixpanelEvent {
event: "test_event".to_string(),
properties: MixpanelProperties {
timestamp: 1697379000,
distinct_id: Some("user123".to_string()),
other: other1,
},
};
let mut other2 = HashMap::new();
other2.insert("$insert_id".to_string(), json!("unique_insert_id_123"));
let mx_event2 = MixpanelEvent {
event: "test_event".to_string(),
properties: MixpanelProperties {
timestamp: 1697379000,
distinct_id: Some("user123".to_string()),
other: other2,
},
};
let result1 = parser1(mx_event1).unwrap().unwrap();
let result2 = parser2(mx_event2).unwrap().unwrap();
// Both events should have the same UUID because they have the same $insert_id
assert_eq!(
result1.inner.uuid, result2.inner.uuid,
"Events with the same $insert_id should have the same deterministic UUID"
);
// The UUID should be deterministic (UUID v5), not random
let expected_uuid = Uuid::new_v5(&MIXPANEL_INSERT_ID_NAMESPACE, b"unique_insert_id_123");
assert_eq!(
result1.inner.uuid, expected_uuid,
"UUID should be generated using UUID v5 from $insert_id"
);
}
#[test]
fn test_random_uuid_without_insert_id() {
let test_job_id = Uuid::now_v7();
let mx_event1 = MixpanelEvent {
event: "test_event".to_string(),
properties: MixpanelProperties {
timestamp: 1697379000,
distinct_id: Some("user123".to_string()),
other: HashMap::new(), // No $insert_id
},
};
let mx_event2 = MixpanelEvent {
event: "test_event".to_string(),
properties: MixpanelProperties {
timestamp: 1697379000,
distinct_id: Some("user123".to_string()),
other: HashMap::new(), // No $insert_id
},
};
let context = TransformContext {
team_id: 123,
token: "test_token".to_string(),
job_id: test_job_id,
identify_cache: std::sync::Arc::new(crate::cache::MockIdentifyCache::new()),
group_cache: std::sync::Arc::new(crate::cache::MockGroupCache::new()),
import_events: true,
generate_identify_events: false,
generate_group_identify_events: false,
};
let parser =
MixpanelEvent::parse_fn(context, false, Duration::seconds(0), identity_transform);
let result1 = parser(mx_event1).unwrap().unwrap();
let result2 = parser(mx_event2).unwrap().unwrap();
// Without $insert_id, UUIDs should be random (different)
assert_ne!(
result1.inner.uuid, result2.inner.uuid,
"Events without $insert_id should have different random UUIDs"
);
}
}