:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Domain layer: `ChunkAccumulator` — folds scanned events through the shared HogVM evaluator into
//! per-`(person, condition, day)` tiles. Depends on `plan`, `window`, `ids`, and `cohort-core`
//! (including its `seed::SeedTile` wire type).
use std::collections::hash_map::Entry;
use std::collections::HashMap;
use std::num::NonZeroU32;
use cohort_core::events::CohortStreamEvent;
use cohort_core::filters::{TeamFilters, TeamId};
use cohort_core::hogvm::{
build_behavioral_globals, classify_vm_error, CohortEvaluator, ConditionProgram, EvalOutcome,
GlobalsBuild, VmErrorClass,
};
use uuid::Uuid;
use super::ids::{ClaimEpoch, ConditionHash, RunId};
use super::plan::ActiveConditions;
use super::window::SeedDomain;
use cohort_core::seed::SeedTile;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RecordOutcome {
Evaluated(RecordStats),
SkippedGlobals,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct RecordStats {
pub matched: u32,
pub non_matched: u32,
pub unknown_functions: u32,
pub vm_failures: VmFailureCounts,
/// Conditions the person path decided from the run's cached vacuous verdict rather than by
/// running the VM. Always zero on the behavioral path.
pub shortcut_evaluations: u32,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct VmFailureCounts([u32; VmErrorClass::COUNT]);
impl Default for VmFailureCounts {
fn default() -> Self {
Self([0; VmErrorClass::COUNT])
}
}
impl VmFailureCounts {
pub fn get(&self, class: VmErrorClass) -> u32 {
self.0.get(class.index()).copied().unwrap_or(0)
}
pub fn iter(&self) -> impl Iterator<Item = (VmErrorClass, u32)> + '_ {
VmErrorClass::ALL
.into_iter()
.map(|class| (class, self.get(class)))
}
pub(super) fn increment(&mut self, class: VmErrorClass) -> Result<(), AggregateError> {
let count = self
.0
.get_mut(class.index())
.ok_or(AggregateError::VmErrorClassInvariant(class))?;
*count = count
.checked_add(1)
.ok_or(AggregateError::OutcomeCountOverflow(
OutcomeKind::VmFailures,
))?;
Ok(())
}
}
struct ActiveCandidate {
hash: ConditionHash,
program: ConditionProgram,
}
pub struct ChunkAccumulator {
team_id: TeamId,
active_by_event_name: HashMap<String, Vec<ActiveCandidate>>,
/// The team's whole behavioral plan, the same one the live path parses under — a seed that
/// parsed a payload live never reads would diverge from live on a malformed one.
globals_build: GlobalsBuild,
evaluator: CohortEvaluator,
counts: HashMap<(Uuid, ConditionHash), NonZeroU32>,
}
impl ChunkAccumulator {
pub fn new(
team_id: TeamId,
filters: &TeamFilters,
active: &ActiveConditions,
) -> Result<Self, AggregateError> {
let active_by_event_name = filters
.behavioral_by_event_name
.iter()
.map(|(event_name, bucket)| {
let candidates = bucket
.conditions
.iter()
.filter_map(|candidate| active.get(candidate).map(|hash| (candidate, hash)))
.map(|(candidate, hash)| {
let program = filters
.by_condition_to_program
.get(candidate)
.cloned()
.ok_or(AggregateError::MissingBytecode(hash))?;
Ok(ActiveCandidate { hash, program })
})
.collect::<Result<Vec<_>, AggregateError>>()?;
Ok((event_name.clone(), candidates))
})
.filter_map(|result| match result {
Ok((_, candidates)) if candidates.is_empty() => None,
other => Some(other),
})
.collect::<Result<HashMap<_, _>, AggregateError>>()?;
Ok(Self {
team_id,
active_by_event_name,
globals_build: GlobalsBuild::whole(filters.behavioral.plan),
evaluator: CohortEvaluator::new(),
counts: HashMap::new(),
})
}
pub fn record_event(
&mut self,
event: &CohortStreamEvent,
) -> Result<RecordOutcome, AggregateError> {
if event.team_id != self.team_id.0 {
return Err(AggregateError::TeamMismatch {
expected: self.team_id,
actual: event.team_id,
});
}
let person_id = Uuid::parse_str(&event.person_id).map_err(|source| {
AggregateError::InvalidPersonId {
value: event.person_id.clone(),
source,
}
})?;
if self.active_by_event_name.is_empty() {
return Ok(RecordOutcome::Evaluated(RecordStats::default()));
}
let Ok(globals) = build_behavioral_globals(event, self.globals_build) else {
return Ok(RecordOutcome::SkippedGlobals);
};
self.evaluator.set_globals(globals);
let candidates = self
.active_by_event_name
.get(&event.event)
.map_or(&[][..], Vec::as_slice);
let mut stats = RecordStats::default();
for candidate in candidates {
match self.evaluator.evaluate_detailed(&candidate.program) {
EvalOutcome::Matched(true) => {
increment_outcome(&mut stats.matched, OutcomeKind::Matched)?;
}
EvalOutcome::Matched(false) => {
increment_outcome(&mut stats.non_matched, OutcomeKind::NonMatched)?;
continue;
}
EvalOutcome::UnknownFunction(_) => {
increment_outcome(&mut stats.unknown_functions, OutcomeKind::UnknownFunctions)?;
continue;
}
EvalOutcome::VmError(error) => {
stats.vm_failures.increment(classify_vm_error(&error))?;
continue;
}
}
match self.counts.entry((person_id, candidate.hash)) {
Entry::Vacant(entry) => {
entry.insert(NonZeroU32::MIN);
}
Entry::Occupied(mut entry) => {
let next = entry
.get()
.checked_add(1)
.ok_or(AggregateError::CountOverflow {
person_id,
hash: candidate.hash,
})?;
entry.insert(next);
}
}
}
Ok(RecordOutcome::Evaluated(stats))
}
pub fn entry_count(&self) -> usize {
self.counts.len()
}
pub fn into_tiles(
self,
domain: &SeedDomain,
run_id: RunId,
claim_epoch: ClaimEpoch,
) -> Vec<SeedTile> {
let mut counts = self.counts.into_iter().collect::<Vec<_>>();
counts.sort_unstable_by_key(|((person_id, hash), _)| (*person_id, *hash));
counts
.into_iter()
.map(|((person_id, hash), count)| {
SeedTile::new(
self.team_id,
person_id,
hash,
count,
domain.day(),
domain.s_chunk(),
run_id,
claim_epoch,
)
})
.collect()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum OutcomeKind {
Matched,
NonMatched,
UnknownFunctions,
VmFailures,
}
impl OutcomeKind {
pub const fn as_str(self) -> &'static str {
match self {
Self::Matched => "matched",
Self::NonMatched => "non_matched",
Self::UnknownFunctions => "unknown_functions",
Self::VmFailures => "vm_failures",
}
}
}
fn increment_outcome(value: &mut u32, kind: OutcomeKind) -> Result<(), AggregateError> {
*value = value
.checked_add(1)
.ok_or(AggregateError::OutcomeCountOverflow(kind))?;
Ok(())
}
#[derive(Debug, thiserror::Error)]
pub enum AggregateError {
#[error("event team {actual} does not match accumulator team {}", .expected.0)]
TeamMismatch { expected: TeamId, actual: i32 },
#[error("invalid person UUID {value:?}: {source}")]
InvalidPersonId {
value: String,
#[source]
source: uuid::Error,
},
#[error("active condition {0} has no frozen bytecode")]
MissingBytecode(ConditionHash),
#[error("record outcome counter {} overflowed", .0.as_str())]
OutcomeCountOverflow(OutcomeKind),
#[error("VM error class {0:?} is outside the bounded counter set")]
VmErrorClassInvariant(VmErrorClass),
#[error("daily count overflowed for person {person_id}, condition {hash}")]
CountOverflow {
person_id: Uuid,
hash: ConditionHash,
},
}
#[cfg(test)]
mod tests {
use chrono_tz::UTC;
use cohort_core::filters::{CohortId, TeamFiltersBuilder};
use proptest::prelude::*;
use serde_json::json;
use super::*;
use crate::domain::SChunkMs;
const HASH_A: &str = "aaaaaaaaaaaaaaaa";
const HASH_B: &str = "bbbbbbbbbbbbbbbb";
const HASH_FALSE: &str = "ffffffffffffffff";
const HASH_UNKNOWN: &str = "uuuuuuuuuuuuuuuu";
const HASH_BROKEN: &str = "xxxxxxxxxxxxxxxx";
fn filters() -> TeamFilters {
let mut builder = TeamFiltersBuilder::default();
builder
.add_cohort(
CohortId(1),
TeamId(2),
&json!({
"properties": { "type": "AND", "values": [
{ "type": "behavioral", "value": "performed_event", "key": "a", "conditionHash": HASH_A, "time_value": 7, "time_interval": "day", "bytecode": ["_H", 1, 32, "a", 32, "event", 1, 1, 11] },
{ "type": "behavioral", "value": "performed_event", "key": "b", "conditionHash": HASH_B, "time_value": 7, "time_interval": "day", "bytecode": ["_H", 1, 32, "b", 32, "event", 1, 1, 11] },
]}
}),
)
.unwrap();
builder.freeze(UTC)
}
/// The same catalog, with condition `a` also reading `properties`. A team whose conditions name
/// no payload never parses one, so only this catalog can fail on a malformed `properties`.
fn filters_reading_properties() -> TeamFilters {
let mut builder = TeamFiltersBuilder::default();
builder
.add_cohort(
CohortId(1),
TeamId(2),
&json!({
"properties": { "type": "AND", "values": [
// event == "a" AND properties.x == "1"
{ "type": "behavioral", "value": "performed_event", "key": "a", "conditionHash": HASH_A, "time_value": 7, "time_interval": "day", "bytecode": ["_H", 1, 32, "a", 32, "event", 1, 1, 11, 32, "1", 32, "x", 32, "properties", 1, 2, 11, 3, 2] },
{ "type": "behavioral", "value": "performed_event", "key": "b", "conditionHash": HASH_B, "time_value": 7, "time_interval": "day", "bytecode": ["_H", 1, 32, "b", 32, "event", 1, 1, 11] },
]}
}),
)
.unwrap();
builder.freeze(UTC)
}
fn event(person: Uuid, event_name: &str) -> CohortStreamEvent {
CohortStreamEvent {
team_id: 2,
person_id: person.to_string(),
distinct_id: person.to_string(),
uuid: Uuid::now_v7().to_string(),
event: event_name.to_string(),
timestamp: "2026-03-08 12:00:00.000000".to_string(),
properties: Some("{}".to_string()),
person_properties: Some("{}".to_string()),
elements_chain: None,
source_offset: 0,
source_partition: -1,
redirected_from: None,
redirect_hops: 0,
}
}
fn outcome_filters() -> TeamFilters {
let mut builder = TeamFiltersBuilder::default();
builder
.add_cohort(
CohortId(2),
TeamId(2),
&json!({
"properties": { "type": "AND", "values": [
{ "type": "behavioral", "value": "performed_event", "key": "false", "conditionHash": HASH_FALSE, "time_value": 7, "time_interval": "day", "bytecode": ["_H", 1, 30] },
{ "type": "behavioral", "value": "performed_event", "key": "unknown", "conditionHash": HASH_UNKNOWN, "time_value": 7, "time_interval": "day", "bytecode": ["_H", 1, 2, "definitelyNotANative", 0] },
// Loads (the header is valid) but 9999 is no opcode, so it fails at run time.
{ "type": "behavioral", "value": "performed_event", "key": "broken", "conditionHash": HASH_BROKEN, "time_value": 7, "time_interval": "day", "bytecode": ["_H", 1, 9999] },
]}
}),
)
.unwrap();
builder.freeze(UTC)
}
fn evaluated(outcome: RecordOutcome) -> RecordStats {
match outcome {
RecordOutcome::Evaluated(stats) => stats,
RecordOutcome::SkippedGlobals => panic!("expected evaluated event"),
}
}
fn active() -> ActiveConditions {
ActiveConditions::new([
ConditionHash::parse(HASH_A).unwrap(),
ConditionHash::parse(HASH_B).unwrap(),
])
}
fn domain() -> SeedDomain {
SeedDomain::new(19, UTC, SChunkMs(20 * 86_400_000)).unwrap()
}
#[test]
fn fold_counts_each_matching_event_condition_pair_once() {
let filters = filters();
let active = active();
let mut accumulator = ChunkAccumulator::new(TeamId(2), &filters, &active).unwrap();
let person = Uuid::from_u128(1);
assert_eq!(
evaluated(accumulator.record_event(&event(person, "a")).unwrap()).matched,
1
);
assert_eq!(
evaluated(accumulator.record_event(&event(person, "a")).unwrap()).matched,
1
);
assert_eq!(
evaluated(accumulator.record_event(&event(person, "b")).unwrap()).matched,
1
);
assert_eq!(
evaluated(accumulator.record_event(&event(person, "other")).unwrap()),
RecordStats::default()
);
let tiles = accumulator.into_tiles(&domain(), RunId(Uuid::nil()), ClaimEpoch(1));
assert_eq!(tiles.iter().map(SeedTile::count).sum::<u32>(), 3);
assert_eq!(tiles.len(), 2);
assert_eq!(tiles[0].count(), 2);
assert_eq!(tiles[1].count(), 1);
}
/// The seeder's shadow compare merges two of these vectors on `(person, condition)`, so this
/// sort is a contract rather than an ordering convenience: reorder it and the merge pairs
/// unrelated tiles, reporting byte-identical scans as thousands of divergences.
#[test]
fn tiles_come_out_sorted_by_person_then_condition() {
let filters = filters();
let active = active();
let mut accumulator = ChunkAccumulator::new(TeamId(2), &filters, &active).unwrap();
for person in [2, 1] {
for event_name in ["b", "a"] {
accumulator
.record_event(&event(Uuid::from_u128(person), event_name))
.unwrap();
}
}
let tiles = accumulator.into_tiles(&domain(), RunId(Uuid::nil()), ClaimEpoch(1));
assert_eq!(tiles.len(), 4);
assert!(
tiles
.windows(2)
.all(|pair| (pair[0].person_id(), pair[0].condition_hash())
< (pair[1].person_id(), pair[1].condition_hash())),
"tiles are not in merge order: {tiles:?}"
);
}
#[test]
fn record_outcome_separates_false_unknown_function_and_vm_error() {
let filters = outcome_filters();
let active = ActiveConditions::new([
ConditionHash::parse(HASH_FALSE).unwrap(),
ConditionHash::parse(HASH_UNKNOWN).unwrap(),
ConditionHash::parse(HASH_BROKEN).unwrap(),
]);
let mut accumulator = ChunkAccumulator::new(TeamId(2), &filters, &active).unwrap();
let person = Uuid::from_u128(1);
let ordinary_false = evaluated(accumulator.record_event(&event(person, "false")).unwrap());
assert_eq!(ordinary_false.non_matched, 1);
assert_eq!(ordinary_false.unknown_functions, 0);
assert_eq!(ordinary_false.vm_failures, VmFailureCounts::default());
let unknown_function =
evaluated(accumulator.record_event(&event(person, "unknown")).unwrap());
assert_eq!(unknown_function.non_matched, 0);
assert_eq!(unknown_function.unknown_functions, 1);
assert_eq!(unknown_function.vm_failures, VmFailureCounts::default());
let vm_error = evaluated(accumulator.record_event(&event(person, "broken")).unwrap());
assert_eq!(vm_error.non_matched, 0);
assert_eq!(vm_error.unknown_functions, 0);
assert_eq!(vm_error.vm_failures.get(VmErrorClass::Program), 1);
assert_eq!(accumulator.entry_count(), 0);
}
#[test]
fn malformed_globals_skip_only_that_event_and_leave_counts_unchanged() {
let filters = filters_reading_properties();
let active = active();
let mut accumulator = ChunkAccumulator::new(TeamId(2), &filters, &active).unwrap();
let person = Uuid::from_u128(1);
let matching = || CohortStreamEvent {
properties: Some(r#"{"x":"1"}"#.to_string()),
..event(person, "a")
};
assert_eq!(
evaluated(accumulator.record_event(&matching()).unwrap()).matched,
1
);
let mut malformed = event(person, "a");
malformed.properties = Some("{".to_string());
assert_eq!(
accumulator.record_event(&malformed).unwrap(),
RecordOutcome::SkippedGlobals
);
assert_eq!(accumulator.entry_count(), 1);
assert_eq!(
evaluated(accumulator.record_event(&matching()).unwrap()).matched,
1
);
let tiles = accumulator.into_tiles(&domain(), RunId(Uuid::nil()), ClaimEpoch(1));
assert_eq!(tiles.iter().map(SeedTile::count).sum::<u32>(), 2);
}
proptest! {
#[test]
fn tile_sum_conserves_arbitrary_matched_pairs(
stream in prop::collection::vec((0u8..16, 0u8..4), 0..500),
) {
let filters = filters();
let active = active();
let mut accumulator = ChunkAccumulator::new(TeamId(2), &filters, &active).unwrap();
let expected = stream.iter().filter(|(_, event_kind)| *event_kind < 2).count() as u32;
for (person, event_kind) in stream {
let event_name = match event_kind {
0 => "a",
1 => "b",
_ => "other",
};
let outcome = accumulator
.record_event(&event(Uuid::from_u128(u128::from(person) + 1), event_name))
.unwrap();
prop_assert!(matches!(outcome, RecordOutcome::Evaluated(_)));
}
let tiles = accumulator.into_tiles(&domain(), RunId(Uuid::nil()), ClaimEpoch(1));
prop_assert_eq!(tiles.iter().map(SeedTile::count).sum::<u32>(), expected);
prop_assert!(tiles.iter().all(|tile| tile.count() > 0));
}
}
}