:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Finds the materialized columns a behavioral scan's row filter and projection can read instead of
//! the `properties` blob.
//!
//! A column qualifies only when its stored default expression on the data table is exactly the
//! trim-quotes extraction the filter would otherwise evaluate. A typed `JSONExtract` column reads
//! different values for the same row, such as an empty value for a boolean that HogVM equality can
//! match. The column must also exist on the distributed table the scan reads.
use std::collections::{BTreeSet, HashMap};
use std::iter;
use clickhouse::Row;
use serde::Deserialize;
use super::client::ClickHouseClient;
use super::sql::{clickhouse_string_literal, fits_client_get};
use crate::domain::{MaterializedColumns, ScanTable};
impl MaterializedColumns {
pub async fn lookup(
client: &ClickHouseClient,
table: ScanTable,
keys: &BTreeSet<&str>,
) -> Result<Self, clickhouse::error::Error> {
let key_by_expression: HashMap<String, &str> = keys
.iter()
.map(|key| (trim_quotes_extraction(key), *key))
.collect();
let mut expressions = key_by_expression
.keys()
.map(String::as_str)
.collect::<Vec<_>>();
expressions.sort_unstable();
let mut rows = Vec::new();
for batch in lookup_batches(table, &expressions) {
rows.extend(
client
.query(&lookup_sql(table, &batch))
.fetch_all::<ColumnRow>()
.await?,
);
}
// Rows are sorted by name, so every chunk picks the same column for a key.
Ok(rows
.into_iter()
.filter_map(|row| Some((*key_by_expression.get(&row.expression)?, row.name)))
.collect())
}
}
#[derive(Debug, Row, Deserialize)]
struct ColumnRow {
name: String,
expression: String,
}
/// The materializer's default expression for a non-nullable property column, as ClickHouse stores
/// it. Only `'` and `\` are escaped, so a key holding another character ClickHouse escapes finds no
/// column and the filter reads the blob.
fn trim_quotes_extraction(key: &str) -> String {
let escaped = key.replace('\\', "\\\\").replace('\'', "\\'");
format!("replaceRegexpAll(JSONExtractRaw(properties, '{escaped}'), '^\"|\"$', '')")
}
/// Each batch fits a GET, because the `cohort_seeder` profile refuses the POST form.
fn lookup_batches<'a>(
table: ScanTable,
expressions: &'a [&'a str],
) -> impl Iterator<Item = Vec<&'a str>> + 'a {
let mut pending = expressions.iter().copied().peekable();
iter::from_fn(move || {
let mut batch = vec![pending.next()?];
while let Some(expression) = pending.next_if(|next| {
fits_client_get(&lookup_sql(table, &[batch.as_slice(), &[*next]].concat()))
}) {
batch.push(expression);
}
Some(batch)
})
}
fn lookup_sql(table: ScanTable, expressions: &[&str]) -> String {
let mut literals = expressions
.iter()
.map(|expression| clickhouse_string_literal(expression))
.collect::<Vec<_>>();
literals.sort();
format!(
"SELECT name, default_expression AS expression\nFROM system.columns\nWHERE database = currentDatabase() AND table = '{}'\n AND type = 'String' AND default_kind IN ('DEFAULT', 'MATERIALIZED')\n AND default_expression IN ({})\n AND name IN (SELECT name FROM system.columns WHERE database = currentDatabase() AND table = '{}' AND type = 'String')\nORDER BY name",
table.data_table(),
literals.join(", "),
table.as_str(),
)
}
#[cfg(test)]
mod tests {
use super::*;
/// Checks both tables because their names differ in length. A batch packed for one table can
/// overflow a GET for the other.
#[test]
fn a_long_lookup_is_split_into_queries_that_each_fit_a_get() {
let expressions = (0..300)
.map(|index| trim_quotes_extraction(&format!("$feature/a-long-flag-name-{index:03}")))
.collect::<Vec<_>>();
let mut sorted = expressions.iter().map(String::as_str).collect::<Vec<_>>();
sorted.sort_unstable();
for table in [ScanTable::Events, ScanTable::FlagEvaluations] {
let batches = lookup_batches(table, &sorted).collect::<Vec<_>>();
assert!(
batches.len() > 1,
"{table:?}: the keys fit one query, so nothing was split"
);
for batch in &batches {
assert!(fits_client_get(&lookup_sql(table, batch)), "{table:?}");
}
assert_eq!(batches.concat(), sorted, "{table:?}");
}
}
#[test]
fn the_expected_expression_is_the_stored_materializer_text() {
assert_eq!(
trim_quotes_extraction("$feature_flag"),
r#"replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag'), '^"|"$', '')"#
);
assert_eq!(
trim_quotes_extraction(r"it's\here"),
r#"replaceRegexpAll(JSONExtractRaw(properties, 'it\'s\\here'), '^"|"$', '')"#
);
}
}