:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! The queries, TTLs and fallbacks match their TypeScript counterparts.
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use async_trait::async_trait;
use futures::future::{BoxFuture, FutureExt, Shared};
use sqlx::{PgPool, Row};
use crate::js::{json_parse, JsObject, JsValue};
use crate::metric_rules::{
compile_metric_rules, CompiledMetricRule, MetricRuleRow, MetricRulesSource,
};
use crate::pattern::MESSAGE_KEYS;
use crate::pipeline::scenario::logs_settings;
use crate::pipeline::{
DependencyUnavailableError, PatternMessageKeysSource, RetentionRulesSource,
SamplingRulesSource, Team, TeamManager, TracingConfigSource,
};
use crate::retention::{
compile_retention_rule_set, CompiledRetentionRuleSet, RetentionRuleRow,
MAX_ENABLED_RETENTION_RULES,
};
use crate::sampling::{compile_rule_set, CompiledRuleSet, SamplingRuleRow};
use super::lazy_loader::{LazyLoader, LazyLoaderOptions};
const UNAVAILABLE_MESSAGES: &[&str] = &[
"connection to server at",
"could not translate host",
"server conn crashed",
"no more connections allowed",
"server closed the connection unexpectedly",
"getaddrinfo EAI_AGAIN",
"getaddrinfo ENOTFOUND",
"Connection terminated unexpectedly",
"ECONNREFUSED",
"ECONNRESET",
"EHOSTUNREACH",
"ETIMEDOUT",
"query_wait_timeout",
"server login has been failing",
"pooler is shutting down",
"Cannot use a pool after calling end on the pool",
"conflict with recovery",
];
/// sqlx reports a lost connection as an I/O or pool error instead.
pub fn classify_pg_error(error: sqlx::Error) -> anyhow::Error {
let transient = match &error {
sqlx::Error::Io(_)
| sqlx::Error::PoolTimedOut
| sqlx::Error::PoolClosed
| sqlx::Error::WorkerCrashed
| sqlx::Error::Tls(_) => true,
other => {
let text = other.to_string();
UNAVAILABLE_MESSAGES.iter().any(|m| text.contains(m))
}
};
if transient {
metrics::counter!("logs_ingestion_postgres_unavailable_total").increment(1);
anyhow::Error::new(DependencyUnavailableError {
message: error.to_string(),
dependency_name: "Postgres".to_string(),
})
} else {
anyhow::Error::new(error)
}
}
fn parse_json_column(text: Option<String>) -> JsValue {
text.and_then(|t| json_parse(&t).ok())
.unwrap_or(JsValue::Null)
}
/// Postgres team ids are int4 and start at 1 (`isValidTeamId`).
fn valid_team_id(raw: &str) -> Option<i32> {
raw.parse::<i32>().ok().filter(|n| *n >= 1)
}
/// A token lookup must never reach a team by id, so the two kinds of key never share a name.
fn id_key(team_id: impl std::fmt::Display) -> String {
format!("id:{team_id}")
}
fn token_key(token: &str) -> String {
format!("token:{token}")
}
#[derive(Debug, Clone)]
pub struct CachedTeam {
pub team: Team,
pub ingested_event: bool,
}
/// Without a `writer` it never writes `ingested_event`.
pub struct PgTeamManager {
loader: LazyLoader<CachedTeam>,
writer: Option<PgPool>,
}
impl PgTeamManager {
pub fn new(reader: PgPool, writer: Option<PgPool>) -> Self {
let mut options = LazyLoaderOptions::new("TeamManager", Duration::from_secs(120));
options.jitter = Some(Duration::from_secs(30));
let loader = LazyLoader::new(
options,
Arc::new(move |keys: Vec<String>| {
let pool = reader.clone();
async move { fetch_teams(&pool, keys).await }.boxed()
}),
);
Self { loader, writer }
}
async fn cached(&self, key: &str) -> anyhow::Result<Option<CachedTeam>> {
self.loader.get(key).await
}
async fn set_ingested_event(&self, writer: &PgPool, team: &CachedTeam) -> anyhow::Result<()> {
if team.ingested_event {
return Ok(());
}
sqlx::query(
"UPDATE posthog_team SET ingested_event = $1 WHERE id = $2 AND NOT ingested_event",
)
.bind(true)
.bind(team.team.id as i32)
.execute(writer)
.await
.map_err(classify_pg_error)?;
self.loader
.mark_for_refresh(&[id_key(team.team.id), token_key(&team.team.api_token)]);
Ok(())
}
}
async fn fetch_teams(
pool: &PgPool,
keys: Vec<String>,
) -> anyhow::Result<HashMap<String, Option<CachedTeam>>> {
let mut ids: Vec<i32> = Vec::new();
let mut tokens: Vec<String> = Vec::new();
for key in &keys {
if let Some(raw) = key.strip_prefix("id:") {
ids.extend(valid_team_id(raw));
} else if let Some(token) = key.strip_prefix("token:") {
tokens.push(token.to_string());
}
}
let rows = sqlx::query(
"SELECT t.id, t.api_token, t.logs_settings::text AS logs_settings, t.ingested_event \
FROM posthog_team t JOIN posthog_organization o ON o.id = t.organization_id \
WHERE t.id = ANY($1) OR t.api_token = ANY($2)",
)
.bind(&ids)
.bind(&tokens)
.fetch_all(pool)
.await
.map_err(classify_pg_error)?;
let mut out: HashMap<String, Option<CachedTeam>> =
keys.into_iter().map(|k| (k, None)).collect();
for row in rows {
let id: i32 = row.try_get("id")?;
let api_token: String = row.try_get("api_token")?;
let settings = parse_json_column(row.try_get("logs_settings")?);
let team = CachedTeam {
team: Team {
id: id as i64,
api_token: api_token.clone(),
logs_settings: logs_settings(Some(&settings)),
},
ingested_event: row.try_get("ingested_event")?,
};
out.insert(id_key(id), Some(team.clone()));
out.insert(token_key(&api_token), Some(team));
}
Ok(out)
}
#[async_trait]
impl TeamManager for PgTeamManager {
async fn get_team(&self, team_id: i64) -> anyhow::Result<Option<Team>> {
Ok(self.cached(&id_key(team_id)).await?.map(|t| t.team))
}
async fn get_team_by_token(&self, token: &str) -> anyhow::Result<Option<Team>> {
Ok(self.cached(&token_key(token)).await?.map(|t| t.team))
}
async fn set_teams_ingested_data(&self, team_ids: &[i64]) -> anyhow::Result<()> {
let Some(writer) = &self.writer else {
return Ok(());
};
let mut unique: Vec<i64> = team_ids.to_vec();
unique.sort_unstable();
unique.dedup();
futures::future::join_all(unique.into_iter().map(|team_id| async move {
let result = async {
if let Some(team) = self.cached(&id_key(team_id)).await? {
self.set_ingested_event(writer, &team).await?;
}
anyhow::Ok(())
}
.await;
if let Err(error) = result {
tracing::error!(team_id, error = %error, "Failed to set ingested_event");
}
}))
.await;
Ok(())
}
}
const RULES_REFRESH: Duration = Duration::from_secs(30);
struct Timed<V> {
value: Arc<V>,
fetched_at: Instant,
}
/// Concurrent misses each query, as in Node.
struct TtlCache<K, V>(Mutex<HashMap<K, Timed<V>>>);
impl<K: std::hash::Hash + Eq + Clone, V> TtlCache<K, V> {
fn new() -> Self {
Self(Mutex::new(HashMap::new()))
}
fn fresh(&self, key: &K) -> Option<Arc<V>> {
let cache = self.0.lock().unwrap();
cache
.get(key)
.filter(|e| e.fetched_at.elapsed() < RULES_REFRESH)
.map(|e| e.value.clone())
}
fn stale(&self, key: &K) -> Option<Arc<V>> {
self.0.lock().unwrap().get(key).map(|e| e.value.clone())
}
fn put(&self, key: K, value: Arc<V>, fetched_at: Instant) {
self.0
.lock()
.unwrap()
.insert(key, Timed { value, fetched_at });
}
}
/// A fetch or compile error propagates.
pub struct PgSamplingRules {
pool: PgPool,
cache: TtlCache<i64, CompiledRuleSet>,
}
impl PgSamplingRules {
pub fn new(pool: PgPool) -> Self {
Self {
pool,
cache: TtlCache::new(),
}
}
}
#[async_trait]
impl SamplingRulesSource for PgSamplingRules {
async fn get_compiled_rule_set(&self, team_id: i64) -> anyhow::Result<Arc<CompiledRuleSet>> {
if let Some(v) = self.cache.fresh(&team_id) {
return Ok(v);
}
let started = Instant::now();
let rows = sqlx::query(
"SELECT id::text AS id, rule_type, scope_service, scope_path_pattern, \
scope_attribute_filters::text AS scope_attribute_filters, config::text AS config, version \
FROM logs_logsexclusionrule \
WHERE team_id = $1 AND enabled = true \
ORDER BY priority ASC, created_at ASC",
)
.bind(team_id)
.fetch_all(&self.pool)
.await
.map_err(classify_pg_error)?;
let rows: Vec<SamplingRuleRow> = rows
.into_iter()
.map(|r| -> anyhow::Result<SamplingRuleRow> {
let config = match parse_json_column(r.try_get("config")?) {
JsValue::Null => JsValue::Object(JsObject::new()),
v => v,
};
Ok(SamplingRuleRow {
id: r.try_get("id")?,
rule_type: r.try_get("rule_type")?,
scope_service: r.try_get("scope_service")?,
scope_path_pattern: r.try_get("scope_path_pattern")?,
scope_attribute_filters: parse_json_column(
r.try_get("scope_attribute_filters")?,
),
config,
version: Some(r.try_get::<i32, _>("version")? as i64),
})
})
.collect::<anyhow::Result<_>>()?;
let compiled = Arc::new(compile_rule_set(&rows).map_err(anyhow::Error::new)?);
self.cache.put(team_id, compiled.clone(), started);
Ok(compiled)
}
}
/// A fetch error serves the last rules, or none.
pub struct PgRetentionRules {
pool: PgPool,
cache: TtlCache<(i64, String), CompiledRetentionRuleSet>,
}
impl PgRetentionRules {
pub fn new(pool: PgPool) -> Self {
Self {
pool,
cache: TtlCache::new(),
}
}
async fn fetch(&self, team_id: i64, source: &str) -> anyhow::Result<Vec<RetentionRuleRow>> {
let table = match source {
"spans" => "tracing_tracesretentionrule",
_ => "logs_logsretentionrule",
};
let sql = format!(
"SELECT id::text AS id, config::text AS config, version FROM {table} \
WHERE team_id = $1 AND enabled = true \
ORDER BY priority ASC, created_at ASC LIMIT {MAX_ENABLED_RETENTION_RULES}"
);
let rows = sqlx::query(&sql)
.bind(team_id)
.fetch_all(&self.pool)
.await
.map_err(classify_pg_error)?;
rows.into_iter()
.map(|r| {
let config = match parse_json_column(r.try_get("config")?) {
JsValue::Null => JsValue::Object(JsObject::new()),
v => v,
};
Ok(RetentionRuleRow {
id: r.try_get("id")?,
config,
version: Some(r.try_get::<i32, _>("version")? as i64),
})
})
.collect()
}
}
#[async_trait]
impl RetentionRulesSource for PgRetentionRules {
async fn get_compiled_rule_set(
&self,
team_id: i64,
source: &str,
) -> anyhow::Result<Arc<CompiledRetentionRuleSet>> {
let key = (team_id, source.to_string());
if let Some(v) = self.cache.fresh(&key) {
return Ok(v);
}
let started = Instant::now();
let rows = match self.fetch(team_id, source).await {
Ok(rows) => rows,
Err(error) => {
tracing::warn!(team_id, source, error = %error, "[logs-retention] rules fetch failed — falling back");
return match self.cache.stale(&key) {
Some(v) => Ok(v),
None => Ok(Arc::new(
compile_retention_rule_set(&[]).map_err(anyhow::Error::new)?,
)),
};
}
};
let compiled = compile_retention_rule_set(&rows).map_err(anyhow::Error::new)?;
let dropped = rows.len().saturating_sub(compiled.rules.len());
if dropped > 0 {
let name = if source == "spans" {
"traces_ingestion_retention_rules_dropped_total"
} else {
"logs_ingestion_retention_rules_dropped_total"
};
let team: Arc<str> = Arc::from(team_id.to_string());
metrics::counter!(name, "team_id" => team).increment(dropped as u64);
tracing::warn!(
team_id,
source,
fetched = rows.len(),
dropped,
"[logs-retention] enabled rules discarded at compile — check retention_days tier"
);
}
let compiled = Arc::new(compiled);
self.cache.put(key, compiled.clone(), started);
Ok(compiled)
}
}
/// A fetch or compile error propagates, and the consumer fails open on it.
pub struct PgMetricRules {
pool: PgPool,
cache: TtlCache<i64, Vec<CompiledMetricRule>>,
}
impl PgMetricRules {
pub fn new(pool: PgPool) -> Self {
Self {
pool,
cache: TtlCache::new(),
}
}
}
#[async_trait]
impl MetricRulesSource for PgMetricRules {
async fn get_compiled_rules(
&self,
team_id: i64,
) -> anyhow::Result<Arc<Vec<CompiledMetricRule>>> {
if let Some(v) = self.cache.fresh(&team_id) {
return Ok(v);
}
let started = Instant::now();
let rows = sqlx::query(
"SELECT id::text AS id, metric_name, filter_group::text AS filter_group, value_attribute, \
group_by::text[] AS group_by, source \
FROM logs_logsmetricrule \
WHERE team_id = $1 AND enabled = true \
ORDER BY created_at ASC",
)
.bind(team_id)
.fetch_all(&self.pool)
.await
.map_err(classify_pg_error)?;
let rows: Vec<MetricRuleRow> = rows
.into_iter()
.map(|r| -> anyhow::Result<MetricRuleRow> {
let opt_string =
|v: Option<String>| v.map(|s| JsValue::string(&s)).unwrap_or(JsValue::Null);
let mut row = JsObject::new();
row.insert_str("id", JsValue::string(&r.try_get::<String, _>("id")?));
row.insert_str("metric_name", opt_string(r.try_get("metric_name")?));
row.insert_str(
"filter_group",
parse_json_column(r.try_get("filter_group")?),
);
row.insert_str("value_attribute", opt_string(r.try_get("value_attribute")?));
let group_by: Vec<Option<String>> = r.try_get("group_by")?;
row.insert_str(
"group_by",
JsValue::Array(group_by.into_iter().map(opt_string).collect()),
);
row.insert_str("source", opt_string(r.try_get("source")?));
Ok(MetricRuleRow::from_js(&row))
})
.collect::<anyhow::Result<_>>()?;
let compiled = Arc::new(compile_metric_rules(&rows).map_err(anyhow::Error::new)?);
self.cache.put(team_id, compiled.clone(), started);
Ok(compiled)
}
}
type KeysLoad = Shared<BoxFuture<'static, Result<Arc<Vec<String>>, Arc<anyhow::Error>>>>;
struct KeysEntry {
keys: Arc<Vec<String>>,
expires_at: Instant,
}
/// Only an outage with nothing cached propagates.
pub struct PgPatternMessageKeys {
pool: PgPool,
cache: Arc<Mutex<HashMap<i64, KeysEntry>>>,
inflight: Arc<Mutex<HashMap<i64, KeysLoad>>>,
}
impl PgPatternMessageKeys {
pub fn new(pool: PgPool) -> Self {
Self {
pool,
cache: Default::default(),
inflight: Default::default(),
}
}
}
fn default_message_keys() -> Arc<Vec<String>> {
Arc::new(MESSAGE_KEYS.iter().map(|k| k.to_string()).collect())
}
async fn refresh_message_keys(
pool: PgPool,
cache: Arc<Mutex<HashMap<i64, KeysEntry>>>,
team_id: i64,
) -> anyhow::Result<Arc<Vec<String>>> {
let existing = cache.lock().unwrap().get(&team_id).map(|e| e.keys.clone());
let fetched = sqlx::query("SELECT logs_pattern_message_keys::text[] AS keys FROM logs_teamlogsconfig WHERE team_id = $1")
.bind(team_id)
.fetch_optional(&pool)
.await
.map_err(classify_pg_error)
.and_then(|row| match row {
None => Ok(default_message_keys()),
Some(row) => {
let keys: Option<Vec<Option<String>>> = row.try_get("keys")?;
Ok(match keys.and_then(|k| k.into_iter().collect::<Option<Vec<String>>>()) {
Some(keys) => Arc::new(keys),
None => default_message_keys(),
})
}
});
match fetched {
Ok(keys) => {
cache.lock().unwrap().insert(
team_id,
KeysEntry {
keys: keys.clone(),
expires_at: Instant::now() + RULES_REFRESH,
},
);
Ok(keys)
}
Err(error) => {
if existing.is_none() && crate::pipeline::errors::is_dependency_unavailable(&error) {
return Err(error);
}
tracing::warn!(team_id, error = %error, "[logs-patterns] message keys fetch failed — falling back");
let fallback = existing.unwrap_or_else(default_message_keys);
cache.lock().unwrap().insert(
team_id,
KeysEntry {
keys: fallback.clone(),
expires_at: Instant::now() + Duration::from_secs(5),
},
);
Ok(fallback)
}
}
}
#[async_trait]
impl PatternMessageKeysSource for PgPatternMessageKeys {
async fn get_message_keys(&self, team_id: i64) -> anyhow::Result<Vec<String>> {
if let Some(e) = self.cache.lock().unwrap().get(&team_id) {
if Instant::now() < e.expires_at {
return Ok(e.keys.to_vec());
}
}
let load = {
let mut inflight = self.inflight.lock().unwrap();
inflight
.entry(team_id)
.or_insert_with(|| {
let pool = self.pool.clone();
let cache = self.cache.clone();
let inflight = self.inflight.clone();
let task = tokio::spawn(async move {
let result = refresh_message_keys(pool, cache, team_id)
.await
.map_err(Arc::new);
inflight.lock().unwrap().remove(&team_id);
result
});
async move {
task.await.unwrap_or_else(|e| {
Err(Arc::new(anyhow::anyhow!("refresh task failed: {e}")))
})
}
.boxed()
.shared()
})
.clone()
};
match load.await {
Ok(keys) => Ok(keys.to_vec()),
Err(error) => Err(match error.downcast_ref::<DependencyUnavailableError>() {
Some(e) => anyhow::Error::new(e.clone()),
None => anyhow::anyhow!("{error:#}"),
}),
}
}
}
pub struct PgTracingConfig {
pool: PgPool,
cache: TtlCache<i64, f64>,
}
impl PgTracingConfig {
pub fn new(pool: PgPool) -> Self {
Self {
pool,
cache: TtlCache::new(),
}
}
}
#[async_trait]
impl TracingConfigSource for PgTracingConfig {
async fn get_retention_days(&self, team_id: i64) -> anyhow::Result<f64> {
if let Some(v) = self.cache.fresh(&team_id) {
return Ok(*v);
}
let started = Instant::now();
let fetched =
sqlx::query("SELECT retention_days FROM tracing_teamtracingconfig WHERE team_id = $1")
.bind(team_id)
.fetch_optional(&self.pool)
.await
.map_err(classify_pg_error)
.and_then(|row| {
let stored: Option<i32> = match row {
Some(r) => r.try_get("retention_days")?,
None => None,
};
Ok(stored
.filter(|d| *d > 0)
.map(f64::from)
.unwrap_or(crate::pipeline::types::DEFAULT_TRACES_RETENTION_DAYS))
});
match fetched {
Ok(days) => {
self.cache.put(team_id, Arc::new(days), started);
Ok(days)
}
Err(error) => {
let existing = self.cache.stale(&team_id);
if existing.is_none() && crate::pipeline::errors::is_dependency_unavailable(&error)
{
return Err(error);
}
tracing::warn!(team_id, error = %error, "[traces-retention] tracing config fetch failed — falling back");
Ok(existing
.map(|v| *v)
.unwrap_or(crate::pipeline::types::DEFAULT_TRACES_RETENTION_DAYS))
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn only_valid_team_ids_are_queried() {
assert_eq!(valid_team_id("42"), Some(42));
assert_eq!(valid_team_id("0"), None);
assert_eq!(valid_team_id("9999999999"), None);
assert_eq!(valid_team_id("phc_abc"), None);
}
#[test]
fn connection_failures_are_dependency_unavailable() {
let refused = classify_pg_error(sqlx::Error::Io(std::io::Error::new(
std::io::ErrorKind::ConnectionRefused,
"refused",
)));
assert!(crate::pipeline::errors::is_dependency_unavailable(&refused));
let bad = classify_pg_error(sqlx::Error::RowNotFound);
assert!(!crate::pipeline::errors::is_dependency_unavailable(&bad));
}
}