:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Coverage for the `toDateTime`/`toDate` natives and Hog-temporal comparison. This VM orders
//! temporals by `dt` seconds to match ClickHouse, matching the Python/TS reference VMs; naive
//! strings parse as UTC, not the process-local timezone.
use chrono::{NaiveDate, TimeZone, Utc};
use hogvm::{sync_execute, ExecutionContext, Program, VmError};
use serde_json::{json, Value};
// Opcode numeric values (mirror common/hogvm/python/operation.py).
const OP_CALL_GLOBAL: i64 = 2;
const OP_EQ: i64 = 11;
const OP_NOT_EQ: i64 = 12;
const OP_GT: i64 = 13;
const OP_LT: i64 = 15;
const OP_STRING: i64 = 32;
const OP_RETURN: i64 = 38;
fn run(bytecode: Vec<Value>) -> Value {
let program = Program::new(bytecode).expect("valid program");
let ctx = ExecutionContext::with_defaults(program)
.with_globals(json!({}))
.with_coercing_comparisons();
sync_execute(&ctx, false).expect("execution succeeds")
}
fn to_datetime(s: &str) -> Vec<Value> {
vec![
json!(OP_STRING),
json!(s),
json!(OP_CALL_GLOBAL),
json!("toDateTime"),
json!(1),
]
}
fn to_datetime_zoned(s: &str, zone: &str) -> Vec<Value> {
vec![
json!(OP_STRING),
json!(s),
json!(OP_STRING),
json!(zone),
json!(OP_CALL_GLOBAL),
json!("toDateTime"),
json!(2),
]
}
/// `left <op> right`. The compiler emits operands as `[right…, left…, op]`.
fn compare(left: &[Value], right: &[Value], op: i64) -> Vec<Value> {
let mut bc = vec![json!("_H"), json!(1)];
bc.extend_from_slice(right);
bc.extend_from_slice(left);
bc.push(json!(op));
bc.push(json!(OP_RETURN));
bc
}
fn returning(fragment: &[Value]) -> Vec<Value> {
let mut bc = vec![json!("_H"), json!(1)];
bc.extend_from_slice(fragment);
bc.push(json!(OP_RETURN));
bc
}
/// The shared date-like grammar. The canonical spec is above `parse_datetime_to_seconds` in
/// `src/stl.rs`; the same table is driven by `common/hogvm/python/test/test_date.py` and
/// `common/hogvm/typescript/src/__tests__/date.test.ts`. All three must agree — before this was
/// pinned, only 4 of the original 34 of these inputs produced the same answer in all three VMs.
const ACCEPTED: [(&str, f64); 16] = [
("2024-01-01", 1704067200.0),
("2024-01-01T00:00:00Z", 1704067200.0),
("2024-01-01t00:00:00z", 1704067200.0), // RFC3339 says the designators are case-insensitive
("2024-01-01T00:00:00.000Z", 1704067200.0),
("2024-01-01T00:00:00", 1704067200.0), // naive => UTC, never the host zone
("2024-01-01 00:00:00", 1704067200.0), // the ClickHouse form HogQL emits; luxon alone rejected it
("2024-01-01T00:00", 1704067200.0),
("2024-01-01T00:00:00+05:00", 1704049200.0),
("2024-01-01 00:00:00+05:00", 1704049200.0),
("2024-01-01T00:00:00-0500", 1704085200.0), // offset without the colon
("2024-01-01T00:00:00.123Z", 1704067200.123),
("2024-01-01T00:00:00.123456Z", 1704067200.123), // truncated to ms, not rounded
(" 2024-01-01 ", 1704067200.0),
("2024-01-01T00:00:00+05", 1704049200.0), // offset hours only
("2024-01-01T00:00:00,123Z", 1704067200.123), // comma fraction separator
("1960-01-01T00:00:00.123Z", -315619199.877), // pre-epoch: truncation must not flip sign
];
const REJECTED: [&str; 21] = [
"2024", // luxon accepted these five as instants; a string property could plausibly hold any
"2024-01",
"20240101", // Python's `fromisoformat` accepted this and `2024-W05`; the others never did
"2024-W05",
"2024-001",
"12:30", // luxon resolved this against *today's* date
"1700000000", // the explicit toDateTime native still reads this as unix seconds; the implicit
// comparison coercion must not, and neither reference VM accepts it anywhere
"not-a-date",
"",
"2024-13-01",
"2024-02-30",
"2024-01-01T24:00:00Z", // luxon normalized hour 24 to the next midnight; the others rejected
"2024-01-01T00:00.123Z", // a fraction requires seconds; luxon rejected what the others took
"2024-01-01T00:00:00+24:00", // made Python datetime.timezone RAISE out of the comparison path
"2024-01-01T00:00:00+99:99", // same
"0000-01-01", // valid to chrono and luxon, not to Python datetime
"2024-01-01T25:00:00Z", // out-of-range hour
"2024-01-01T00:60:00Z", // out-of-range minute
"٢٠٢٤-٠١-٠١", // Unicode digits. \d matches them in the Rust/Python regex engines (never in
// JS), so Python's int() read this one as a real date while the other two VMs rejected it...
"2024-01-01T00:00+0١", // ...and these two byte-sliced mid-codepoint in Rust — a panic, not an error
"2024-01-01T00:00:00.١٢Z",
];
/// Does the *implicit* string-vs-temporal coercion — the shared grammar — accept `input`?
///
/// `Lt(<bare string>, toDateTime("9999-12-31"))`: a date-like string coerces to `true`, while an
/// uncoerced String-vs-Object pair reaches the generic arms and is a `CannotCoerce` error. This is
/// the grammar the ACCEPTED/REJECTED tables define, and is deliberately *not* the explicit
/// `toDateTime` native, which accepts bare numeric strings on top of it (see
/// `numeric_strings_are_unix_seconds_to_the_native_but_never_to_the_comparison_coercion`).
fn grammar_accepts(input: &str) -> bool {
let program = Program::new(compare(
&[json!(OP_STRING), json!(input)],
&to_datetime("9999-12-31"),
OP_LT,
))
.expect("valid program");
let ctx = ExecutionContext::with_defaults(program)
.with_globals(json!({}))
.with_coercing_comparisons();
matches!(sync_execute(&ctx, false), Ok(Value::Bool(true)))
}
#[test]
fn date_like_accept_set_matches_the_shared_grammar() {
for (input, expected) in ACCEPTED {
assert!(grammar_accepts(input), "grammar should accept {input:?}");
// The native agrees with the grammar on every non-numeric string, so it can supply the value.
let dt = run(returning(&to_datetime(input)));
assert_eq!(
dt["dt"].as_f64().expect("parsed to a number"),
expected,
"should accept {input:?}"
);
}
for input in REJECTED {
assert!(!grammar_accepts(input), "grammar should reject {input:?}");
}
}
#[test]
fn to_datetime_parses_naive_string_as_utc_not_local() {
let result = run(returning(&to_datetime("2026-05-13 00:00:00")));
let expected = NaiveDate::from_ymd_opt(2026, 5, 13)
.unwrap()
.and_hms_opt(0, 0, 0)
.unwrap()
.and_utc()
.timestamp() as f64;
assert_eq!(result["__hogDateTime__"], json!(true));
assert_eq!(result["zone"], json!("UTC"));
assert_eq!(result["dt"].as_f64().unwrap(), expected);
}
#[test]
fn to_datetime_honors_an_explicit_iso_offset() {
let result = run(returning(&to_datetime("2026-03-19T14:00:00Z")));
let expected = Utc
.with_ymd_and_hms(2026, 3, 19, 14, 0, 0)
.unwrap()
.timestamp() as f64;
assert_eq!(result["dt"].as_f64().unwrap(), expected);
}
#[test]
fn to_datetime_arity2_interprets_naive_string_in_the_given_zone() {
let utc = run(returning(&to_datetime("2026-05-13 00:00:00")));
let ny = run(returning(&to_datetime_zoned(
"2026-05-13 00:00:00",
"America/New_York",
)));
assert_eq!(ny["zone"], json!("America/New_York"));
assert_eq!(
ny["dt"].as_f64().unwrap() - utc["dt"].as_f64().unwrap(),
4.0 * 3600.0,
);
}
#[test]
fn numeric_strings_are_unix_seconds_to_the_native_but_never_to_the_comparison_coercion() {
// ClickHouse — the oracle for the realtime cohort evaluator, the only production consumer of
// the coercing path — reads an all-digit string as a unix timestamp, and the compiled leaf
// shape `toDateTime(toString(person.properties.X))` depends on it for numeric date properties.
// So the explicit native keeps it.
assert_eq!(
run(returning(&to_datetime("1700000000")))["dt"]
.as_f64()
.unwrap(),
1_700_000_000.0
);
// But the *implicit* string-vs-temporal coercion must not: treating every numeric-looking
// property as an instant would silently change the meaning of comparisons nobody marked as date
// comparisons, and the Python/TS VMs reject it. `Lt` against a far-future date coerces to `true`
// for a date-like string; an uncoerced String-vs-Object pair is a `CannotCoerce` error instead.
let far_future = to_datetime("9999-12-31");
assert_eq!(
run(compare(
&[json!(OP_STRING), json!("2024-01-01")],
&far_future,
OP_LT
)),
Value::Bool(true),
"a date-like string coerces"
);
let program = Program::new(compare(
&[json!(OP_STRING), json!("1700000000")],
&far_future,
OP_LT,
))
.expect("valid program");
let ctx = ExecutionContext::with_defaults(program)
.with_globals(json!({}))
.with_coercing_comparisons();
assert!(
sync_execute(&ctx, false).is_err(),
"a bare numeric string must NOT be coerced to an instant by a comparison"
);
}
#[test]
fn to_datetime_passes_through_a_unix_number() {
let bc = returning(&[
json!(33), // OP_INTEGER
json!(1_700_000_000),
json!(OP_CALL_GLOBAL),
json!("toDateTime"),
json!(1),
]);
assert_eq!(run(bc)["dt"].as_f64().unwrap(), 1_700_000_000.0);
}
#[test]
fn unparseable_input_becomes_null_not_an_error() {
// `err_to_null` turns a parse failure into `Null` so the leaf's `if(isNull(...), false, …)` guard wins.
assert_eq!(
run(returning(&to_datetime("definitely not a date"))),
Value::Null
);
}
#[test]
fn datetime_less_than_orders_by_instant() {
let earlier = to_datetime("2020-01-01 00:00:00");
let later = to_datetime("2030-01-01 00:00:00");
assert_eq!(run(compare(&earlier, &later, OP_LT)), Value::Bool(true));
assert_eq!(run(compare(&later, &earlier, OP_LT)), Value::Bool(false));
}
#[test]
fn datetime_greater_than_orders_by_instant() {
let earlier = to_datetime("2020-06-15 12:00:00");
let later = to_datetime("2020-06-15 12:00:01");
assert_eq!(run(compare(&later, &earlier, OP_GT)), Value::Bool(true));
assert_eq!(run(compare(&earlier, &later, OP_GT)), Value::Bool(false));
}
#[test]
fn datetime_equality_is_by_instant() {
let naive = to_datetime("2026-03-19 14:00:00");
let iso_z = to_datetime("2026-03-19T14:00:00Z");
let other = to_datetime("2026-03-19 14:00:01");
assert_eq!(run(compare(&naive, &iso_z, OP_EQ)), Value::Bool(true));
assert_eq!(run(compare(&naive, &other, OP_EQ)), Value::Bool(false));
}
/// Run WITHOUT opting into coercing comparisons — the default every non-cohort consumer gets.
fn run_legacy(bytecode: Vec<Value>) -> Value {
let program = Program::new(bytecode).expect("valid program");
let ctx = ExecutionContext::with_defaults(program).with_globals(json!({}));
sync_execute(&ctx, false).expect("execution succeeds")
}
#[test]
fn legacy_default_compares_temporals_structurally_not_by_epoch() {
// Same instant (05:30 Kolkata == 00:00 UTC), different zone: epoch-equal but structurally
// distinct objects. The opt-in coercing path treats them as equal (ClickHouse `is_date_exact`);
// the default path every other consumer (e.g. cymbal) gets keeps the legacy structural
// comparison, so they are NOT equal — proving the temporal `Eq` change is gated.
let kolkata = to_datetime_zoned("2026-01-01 05:30:00", "Asia/Kolkata");
let utc = to_datetime("2026-01-01 00:00:00");
assert_eq!(
run(compare(&kolkata, &utc, OP_EQ)),
Value::Bool(true),
"coercing: epoch-equal"
);
assert_eq!(
run_legacy(compare(&kolkata, &utc, OP_EQ)),
Value::Bool(false),
"legacy: structurally distinct (zone differs)",
);
}
#[test]
fn bare_field_string_orders_against_a_datetime_by_parsing_it() {
// Regression: a hand-written SQL trigger filter like `timestamp > toDateTime('2026-01-01')`
// only wraps the RHS in toDateTime, so the VM compares a bare String literal (the filter globals'
// ISO `timestamp`) against a HogDateTime object. Before this fix, exactly one side being temporal
// fell straight to `CannotCoerce` — the trigger's filter would error, not silently pass, but it
// never matched by ordering either.
let bare_timestamp = vec![json!(OP_STRING), json!("2026-06-28T00:00:00.000Z")];
let threshold = to_datetime("2026-01-01 00:00:00");
assert_eq!(
run(compare(&bare_timestamp, &threshold, OP_GT)),
Value::Bool(true)
);
assert_eq!(
run(compare(&threshold, &bare_timestamp, OP_GT)),
Value::Bool(false)
);
}
#[test]
fn bare_field_string_equals_a_datetime_by_parsing_it() {
// Regression: ordering got the bare-field string treatment but equality did not, so
// `timestamp == toDateTime('…')` was silently false here while the Python/TS VMs said true —
// they route all six comparison opcodes through one coercion function. `Eq` and `Gt` must agree
// on which operands count as dates.
let bare = |s: &str| vec![json!(OP_STRING), json!(s)];
let threshold = to_datetime("2026-01-01 00:00:00");
for (label, string) in [
("naive", "2026-01-01 00:00:00"),
("iso Z", "2026-01-01T00:00:00.000Z"),
("date only", "2026-01-01"),
] {
assert_eq!(
run(compare(&bare(string), &threshold, OP_EQ)),
Value::Bool(true),
"{label}: string on the left"
);
assert_eq!(
run(compare(&threshold, &bare(string), OP_EQ)),
Value::Bool(true),
"{label}: string on the right"
);
assert_eq!(
run(compare(&bare(string), &threshold, OP_NOT_EQ)),
Value::Bool(false),
"{label}: NotEq is the negation"
);
}
// A different instant, and a string the grammar rejects, both stay unequal rather than erroring.
assert_eq!(
run(compare(&bare("2026-01-02"), &threshold, OP_EQ)),
Value::Bool(false)
);
assert_eq!(
run(compare(&bare("not-a-date"), &threshold, OP_EQ)),
Value::Bool(false)
);
}
#[test]
fn legacy_default_does_not_coerce_string_vs_temporal_eq() {
// The bare-field coercion is gated on `with_coercing_comparisons`, so cymbal's rules engine —
// which relies on the legacy structural path, and permanently disables a rule on any VmError —
// is unaffected by the `eq_op` change above.
let bare = vec![json!(OP_STRING), json!("2026-01-01 00:00:00")];
let threshold = to_datetime("2026-01-01 00:00:00");
assert_eq!(run(compare(&bare, &threshold, OP_EQ)), Value::Bool(true));
assert_eq!(
run_legacy(compare(&bare, &threshold, OP_EQ)),
Value::Bool(false),
"legacy: a String is never structurally equal to a HogDateTime object",
);
}
#[test]
fn in_does_not_coerce_strings_against_temporals() {
// Pinning a deliberate gap, not an endorsement: `in` skips the comparison coercion in all three
// VMs (TS `Array.includes`, Python `in`, Rust `contains` → structural `equals`), so this is
// consistent today. Rust's `contains` is also NOT gated on `coerce_comparisons` and is shared
// with `has()` and cymbal, so changing it is a bigger decision than it looks.
const OP_ARRAY: i64 = 43;
const OP_IN: i64 = 21;
let haystack = {
let mut bc = to_datetime("2026-01-01 00:00:00");
bc.extend_from_slice(&[json!(OP_ARRAY), json!(1)]);
bc
};
let needle = vec![json!(OP_STRING), json!("2026-01-01 00:00:00")];
assert_eq!(run(compare(&needle, &haystack, OP_IN)), Value::Bool(false));
}
#[test]
fn date_and_datetime_are_mutually_comparable() {
let date = vec![
json!(OP_STRING),
json!("2026-05-13"),
json!(OP_CALL_GLOBAL),
json!("toDate"),
json!(1),
];
let noon = to_datetime("2026-05-13 12:00:00");
assert_eq!(run(compare(&date, &noon, OP_LT)), Value::Bool(true));
}
#[test]
fn full_compiled_shape_to_string_to_datetime_lt() {
// Realistic leaf shape `Lt(toDateTime(toString(person.properties.signup_date)), toDateTime(const))`;
// the `toString → toDateTime` hop mirrors `_force_datetime`. `Program` is not `Clone`, so rebuild per iteration.
const OP_GET_GLOBAL: i64 = 1;
let bytecode = || {
let lhs = vec![
json!(OP_STRING),
json!("signup_date"),
json!(OP_STRING),
json!("properties"),
json!(OP_STRING),
json!("person"),
json!(OP_GET_GLOBAL),
json!(3),
json!(OP_CALL_GLOBAL),
json!("toString"),
json!(1),
json!(OP_CALL_GLOBAL),
json!("toDateTime"),
json!(1),
];
compare(&lhs, &to_datetime("2026-01-01 00:00:00"), OP_LT)
};
for (signup, expected) in [
("2024-09-09 08:30:00", true),
("2027-09-09 08:30:00", false),
] {
let program = Program::new(bytecode()).expect("valid program");
let ctx = ExecutionContext::with_defaults(program)
.with_globals(json!({ "person": { "properties": { "signup_date": signup } } }))
.with_coercing_comparisons();
assert_eq!(
sync_execute(&ctx, false).expect("execution succeeds"),
Value::Bool(expected),
"signup {signup}",
);
}
}
// Years 275760 and 262143 in luxon. The first is past chrono's calendar; the second is chrono's
// last UTC second and crosses the end once the +09:00 offset is added.
#[test]
fn extract_defers_instants_past_the_chrono_calendar_to_the_fallback() {
for (dt, zone) in [
(8_640_000_000_000_i64, "UTC"),
(8_210_266_876_799_i64, "Asia/Tokyo"),
] {
let bc = returning(&[
json!(OP_STRING),
json!("year"),
json!(33), // OP_INTEGER
json!(dt),
json!(OP_CALL_GLOBAL),
json!("fromUnixTimestamp"),
json!(1),
json!(OP_STRING),
json!(zone),
json!(OP_CALL_GLOBAL),
json!("toTimeZone"),
json!(2),
json!(OP_CALL_GLOBAL),
json!("extract"),
json!(2),
]);
let program = Program::new(bc).expect("valid program");
let ctx = ExecutionContext::with_defaults(program).with_globals(json!({}));
let failure = sync_execute(&ctx, false).expect_err("chrono cannot read this instant");
assert!(
matches!(failure.error, VmError::NotImplemented(_)),
"{dt} in {zone}: {:?}",
failure.error
);
}
}