:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use anyhow::{Context, Result};
use common_types::RawEvent;
use serde::{Deserialize, Serialize};
use crate::utils::timestamp::parse_timestamp;
const UNKNOWN_STR: &str = "unknown";
/// Timestamp-based deduplication key
/// Uses bincode serialization to handle arbitrary characters in fields
#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)]
pub struct TimestampKey {
pub timestamp: u64,
pub distinct_id: String,
pub token: String,
pub event_name: String,
}
impl TimestampKey {
pub fn new(timestamp: u64, distinct_id: String, token: String, event_name: String) -> Self {
Self {
timestamp,
distinct_id,
token,
event_name,
}
}
}
impl From<TimestampKey> for Vec<u8> {
fn from(key: TimestampKey) -> Vec<u8> {
// Custom serialization to ensure timestamp comes first for proper ordering
// Format: [8 bytes timestamp BE][rest as bincode]
let mut bytes = Vec::new();
bytes.extend_from_slice(&key.timestamp.to_be_bytes());
// Serialize the rest of the fields
let rest = (key.distinct_id, key.token, key.event_name);
let rest_bytes = bincode::serde::encode_to_vec(&rest, bincode::config::standard())
.expect("TimestampKey serialization should never fail");
bytes.extend_from_slice(&rest_bytes);
bytes
}
}
impl From<&TimestampKey> for Vec<u8> {
fn from(key: &TimestampKey) -> Vec<u8> {
// Custom serialization to ensure timestamp comes first for proper ordering
// Format: [8 bytes timestamp BE][rest as bincode]
let mut bytes = Vec::new();
bytes.extend_from_slice(&key.timestamp.to_be_bytes());
// Serialize the rest of the fields
let rest = (&key.distinct_id, &key.token, &key.event_name);
let rest_bytes = bincode::serde::encode_to_vec(rest, bincode::config::standard())
.expect("TimestampKey serialization should never fail");
bytes.extend_from_slice(&rest_bytes);
bytes
}
}
impl TryFrom<&[u8]> for TimestampKey {
type Error = anyhow::Error;
fn try_from(bytes: &[u8]) -> Result<Self> {
// Custom deserialization to match our serialization format
// Format: [8 bytes timestamp BE][rest as bincode]
if bytes.len() < 8 {
anyhow::bail!("TimestampKey bytes too short: {} bytes", bytes.len());
}
// Extract timestamp
let timestamp_bytes: [u8; 8] = bytes[..8].try_into()?;
let timestamp = u64::from_be_bytes(timestamp_bytes);
// Deserialize the rest
let (rest, _): ((String, String, String), usize) =
bincode::serde::decode_from_slice(&bytes[8..], bincode::config::standard())
.with_context(|| {
format!(
"Failed to deserialize TimestampKey fields from {} bytes",
bytes.len() - 8
)
})?;
Ok(TimestampKey {
timestamp,
distinct_id: rest.0,
token: rest.1,
event_name: rest.2,
})
}
}
impl From<&RawEvent> for TimestampKey {
fn from(raw_event: &RawEvent) -> Self {
let timestamp = raw_event
.timestamp
.as_ref()
.and_then(|t| parse_timestamp(t))
.unwrap_or_else(|| chrono::Utc::now().timestamp_millis() as u64);
let distinct_id = extract_distinct_id(raw_event);
let token = raw_event
.token
.clone()
.unwrap_or_else(|| UNKNOWN_STR.to_string());
Self::new(timestamp, distinct_id, token, raw_event.event.clone())
}
}
/// Helper function to extract distinct_id from RawEvent
fn extract_distinct_id(raw_event: &RawEvent) -> String {
raw_event
.distinct_id
.as_ref()
.and_then(|v| {
// Treat JSON null as None
if v.is_null() {
None
} else {
// Try to get as string first, if not possible stringify the JSON
Some(
v.as_str()
.map(|s| s.to_string())
.unwrap_or_else(|| v.to_string()),
)
}
})
.unwrap_or_else(|| UNKNOWN_STR.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
fn create_test_event() -> RawEvent {
RawEvent {
uuid: Some(uuid::Uuid::new_v4()),
event: "test_event".to_string(),
distinct_id: Some(serde_json::Value::String("user123".to_string())),
token: Some("token456".to_string()),
properties: std::collections::HashMap::new(),
timestamp: Some("2021-01-01T00:00:00Z".to_string()),
..Default::default()
}
}
#[test]
fn test_timestamp_key_serialization() {
let event = create_test_event();
let key = TimestampKey::from(&event);
// Test that we can serialize and deserialize
let serialized: Vec<u8> = (&key).into();
let deserialized = TimestampKey::try_from(serialized.as_slice()).unwrap();
assert_eq!(key, deserialized);
assert_eq!(deserialized.distinct_id, "user123");
assert_eq!(deserialized.token, "token456");
assert_eq!(deserialized.event_name, "test_event");
}
#[test]
fn test_timestamp_key_roundtrip() {
let event = create_test_event();
let key = TimestampKey::from(&event);
let key_bytes: Vec<u8> = (&key).into();
let parsed_key = TimestampKey::try_from(key_bytes.as_slice()).unwrap();
assert_eq!(key.timestamp, parsed_key.timestamp);
assert_eq!(key.distinct_id, parsed_key.distinct_id);
assert_eq!(key.token, parsed_key.token);
assert_eq!(key.event_name, parsed_key.event_name);
}
}