Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(cohorts): read "" and "null" person properties as null (#130476)

Matheus Batista148b91f58faa
diff --git a/docs/internal/realtime-cohorts/definitions-and-eligibility.md b/docs/internal/realtime-cohorts/definitions-and-eligibility.md
index 01f9f82cb280..24c3cfacecfb 100644
--- a/docs/internal/realtime-cohorts/definitions-and-eligibility.md
+++ b/docs/internal/realtime-cohorts/definitions-and-eligibility.md
@@ -318,6 +318,12 @@ Leaves are evaluated with the Rust HogVM, configured the same way in every proce
The batch calculation reads an untyped or String-typed property as its JSON text, so `$feature/<flag> = 'variant'` skips events where the flag was off on both paths.
It reads a Boolean-typed property through a case-sensitive `transform` of that text, so a stored string `"True"` does not equal the constant `true` on either path.
A bare read and an ordering of a boolean keep boolean semantics.
+- A top-level person property stored as `""` or `"null"` reads as null, as the batch calculation reads one through a non-nullable `pmat_` column or a Numeric, DateTime or Boolean property definition.
+ So `is_set` is false, `is_not_set` is true, and `exact ""` matches no person.
+ A text operator such as `icontains` or `regex` reads that null as the text `null`, as it reads a missing key.
+ On a null, the batch calculation matches no text operator and matches every negated one, so the two paths disagree on a pattern that matches `null`, such as `regex '.+'`.
+ For a person property with neither, which includes a String-typed one without a non-nullable `pmat_` column, the batch calculation reads the JSON blob or a nullable `pmat_` column, which keep both texts, and the two paths disagree.
+ Event properties keep both texts, because cloud reads an event property without a `mat_` column through its property group, which keeps them too.
- A program that fails at runtime, calls an unknown function, or returns a non-boolean counts as `false`.
Failures are counted by reason.
- An event with malformed `properties` JSON skips only the behavioral side.
diff --git a/rust/cohort-core/src/fingerprint.rs b/rust/cohort-core/src/fingerprint.rs
index 9aedc8e54f35..49a01d63f576 100644
--- a/rust/cohort-core/src/fingerprint.rs
+++ b/rust/cohort-core/src/fingerprint.rs
@@ -1,8 +1,9 @@
use sha2::{Digest, Sha256};
-/// Raise with any change to `cohort_execution_context` that can flip a verdict, so that every stored
-/// person record is evaluated again instead of keeping a verdict of the old rules.
-const EVALUATOR_VERSION: u8 = 1;
+/// Raise with any change to how the evaluator reads globals or compares values that can flip a
+/// verdict, so that every stored person record is evaluated again instead of keeping a verdict of
+/// the old rules.
+const EVALUATOR_VERSION: u8 = 2;
/// 128 bits of SHA-256 over [`EVALUATOR_VERSION`] and a team's sorted person condition hashes.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
diff --git a/rust/cohort-core/src/hogvm/globals.rs b/rust/cohort-core/src/hogvm/globals.rs
index 868a3980c2ae..9c5da996e25b 100644
--- a/rust/cohort-core/src/hogvm/globals.rs
+++ b/rust/cohort-core/src/hogvm/globals.rs
@@ -41,10 +41,7 @@ pub fn build_behavioral_globals(
} else {
Value::Object(Map::new())
};
- let person = object([
- ("id", text(&event.person_id)),
- ("properties", person_properties),
- ]);
+ let person = person(&event.person_id, person_properties);
let mut roots = Roots::new(build.materialize());
roots.set(GlobalRoot::Event, || text(&event.event));
@@ -127,6 +124,26 @@ fn empty_group() -> Value {
object([("properties", Value::Object(Map::new()))])
}
+/// `MAT_COL_NULL_SENTINELS` in `posthog/hogql/database/schema/util/null_sentinels.py`.
+const NULL_SENTINELS: [&str; 2] = ["", "null"];
+
+/// Legacy reads a top-level person property through its non-nullable `pmat_` column or its Numeric,
+/// DateTime or Boolean definition where one exists, and both read these texts as NULL. Raise
+/// `EVALUATOR_VERSION` in [`crate::fingerprint`] with any change here.
+fn person(id: &str, mut properties: Value) -> Value {
+ if let Some(map) = properties.as_object_mut() {
+ for value in map.values_mut() {
+ if value
+ .as_str()
+ .is_some_and(|text| NULL_SENTINELS.contains(&text))
+ {
+ *value = Value::Null;
+ }
+ }
+ }
+ object([("id", text(id)), ("properties", properties)])
+}
+
pub fn build_person_property_globals(event: &CohortStreamEvent) -> Result<Value, GlobalsError> {
let person_properties =
parse_optional_json(event.person_properties.as_deref(), "person_properties")?;
@@ -169,7 +186,7 @@ pub fn person_scan_globals(team_id: TeamId, person_id: Uuid, properties: Value)
/// The one constructor of the `{"person":{"id","properties"},"project":{"id"}}` shape.
fn person_scope_globals(team_id: i32, person_id: &str, person_properties: Value) -> Value {
json!({
- "person": { "id": person_id, "properties": person_properties },
+ "person": person(person_id, person_properties),
"project": { "id": team_id },
})
}
@@ -222,6 +239,7 @@ mod tests {
use hogvm::stl_map;
use crate::hogvm::analysis::{Projection, ReadPath};
+ use crate::hogvm::{CohortEvaluator, ConditionProgram, EvalOutcome};
use super::*;
@@ -469,12 +487,24 @@ mod tests {
assert_eq!(globals["person"]["properties"], json!({}));
}
+ const NULL_TEXT_PERSON_PROPERTIES: &str =
+ r#"{"a":"","b":"null","c":"NULL","d":" ","e":null,"f":{"g":""},"h":0}"#;
+
+ fn null_text_person_properties_read_as_null() -> Value {
+ json!({ "a": null, "b": null, "c": "NULL", "d": " ", "e": null, "f": { "g": "" }, "h": 0 })
+ }
+
/// Shape lock: the scan builder and the live event builder must produce byte-equal globals for
/// equal inputs — the correctness anchor that keeps seeder evals equivalent to live evals.
#[test]
fn scan_globals_are_byte_equal_to_event_globals_for_equal_inputs() {
let person_id = Uuid::parse_str("01928aaa-bbbb-cccc-dddd-eeeeeeeeeeee").unwrap();
- for properties in [r#"{"email":"[email protected]","plan":"paid"}"#, "", "{}"] {
+ for properties in [
+ r#"{"email":"[email protected]","plan":"paid"}"#,
+ "",
+ "{}",
+ NULL_TEXT_PERSON_PROPERTIES,
+ ] {
let mut e = event();
e.team_id = 42;
e.person_id = person_id.to_string();
@@ -496,9 +526,67 @@ mod tests {
build_person_scan_globals(TeamId(42), person_id, "").unwrap()["person"]["properties"],
json!({})
);
+ assert_eq!(
+ build_person_scan_globals(TeamId(42), person_id, NULL_TEXT_PERSON_PROPERTIES).unwrap()
+ ["person"]["properties"],
+ null_text_person_properties_read_as_null()
+ );
assert!(build_person_scan_globals(TeamId(42), person_id, "{not json").is_err());
}
+ #[test]
+ fn behavioral_globals_read_null_text_person_properties_as_null_and_keep_event_properties() {
+ let mut e = event();
+ e.properties = Some(r#"{"k":"","n":"null"}"#.to_string());
+ e.person_properties = Some(NULL_TEXT_PERSON_PROPERTIES.to_string());
+ let globals = build_behavioral_globals(&e, GlobalsBuild::whole(GlobalsPlan::FULL)).unwrap();
+ assert_eq!(
+ globals["person"]["properties"],
+ null_text_person_properties_read_as_null()
+ );
+ assert_eq!(
+ globals["pdi"]["person"]["properties"],
+ null_text_person_properties_read_as_null()
+ );
+ assert_eq!(globals["properties"], json!({ "k": "", "n": "null" }));
+ }
+
+ #[test]
+ fn a_null_text_person_property_is_not_set_and_equals_no_text() {
+ // As `generate_cohort_filter_bytecode` stores these person leaves on `email`.
+ let leaves = [
+ (
+ "is_set",
+ r#"["_H",1,31,32,"email",32,"properties",32,"person",1,3,12]"#,
+ false,
+ ),
+ (
+ "is_not_set",
+ r#"["_H",1,31,32,"email",32,"properties",32,"person",1,3,11]"#,
+ true,
+ ),
+ (
+ "exact ''",
+ r#"["_H",1,32,"",32,"email",32,"properties",32,"person",1,3,11]"#,
+ false,
+ ),
+ ];
+ let mut evaluator = CohortEvaluator::new();
+ for blob in [r#"{"email":""}"#, r#"{"email":"null"}"#] {
+ evaluator
+ .set_globals(build_person_scan_globals(TeamId(42), Uuid::nil(), blob).unwrap());
+ for (leaf, stored, expected) in leaves {
+ let stored: Vec<Value> = serde_json::from_str(stored).unwrap();
+ let program = ConditionProgram::from_stored(&stored).unwrap();
+ let outcome = evaluator.evaluate_detailed(&program);
+ assert!(
+ matches!(outcome, EvalOutcome::Matched(matched) if matched == expected),
+ "{leaf} on {blob}: {outcome:?}"
+ );
+ }
+ }
+ }
+
#[test]
fn malformed_properties_is_an_error() {
let mut e = event();