:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(cohorts): cap rocksdb info logs and gauge volume usage by file type (#114236)
posthog[bot]3de953df98e8
diff --git a/docs/internal/realtime-cohorts/state-store-and-durability.md b/docs/internal/realtime-cohorts/state-store-and-durability.mdindex 52ef9fa82f2a..17cd74490b5b 100644--- a/docs/internal/realtime-cohorts/state-store-and-durability.md+++ b/docs/internal/realtime-cohorts/state-store-and-durability.md@@ -327,6 +327,16 @@ Three metrics follow coverage:- **Merge markers and tombstones** are removed after their retention period.- **Orphaned behavioral rows**, left behind when a leaf's definition changes, a cohort is deleted, or a team leaves the catalog, are not collected.They are removed only by a merge drain or a partition wipe.+- **RocksDB's info logs** have a size cap.+ The info `LOG` rolls at `COHORT_MAX_LOG_FILE_SIZE_BYTES` (256 MiB), and RocksDB keeps `COHORT_KEEP_LOG_FILE_NUM` (10) files, the active one included, so the info logs use at most about 2.5 GiB.+ Each open and each roll delete the oldest rolled files beyond that count.+ The first boot under the cap keeps the newest 9 rolled files that an uncapped build left, and they can be much larger than 256 MiB.+ The next 9 rolls delete them.+ Both values must be greater than 0, or the processor does not start.+- **Usage by file type** of the store directory and the local checkpoints is the `store_file_bytes` gauge.+ Its `kind` label is `sst`, `wal`, `info_log`, `manifest`, `other` or `checkpoint`.+ The `checkpoint` value counts only local checkpoint files that the live store does not share, such as SSTs that compaction replaced.+ It is 0 when checkpoints are off.## Tuning that mattersdiff --git a/rust/cohort-stream-processor/src/config.rs b/rust/cohort-stream-processor/src/config.rsindex bed6e0db83a7..57e325039c00 100644--- a/rust/cohort-stream-processor/src/config.rs+++ b/rust/cohort-stream-processor/src/config.rs@@ -461,6 +461,14 @@ pub struct Config {#[envconfig(from = "COHORT_MAX_BACKGROUND_JOBS", default = "0")]pub cohort_max_background_jobs: i32,+ /// Size at which RocksDB rolls its info `LOG` file.+ #[envconfig(from = "COHORT_MAX_LOG_FILE_SIZE_BYTES", default = "268435456")]+ pub cohort_max_log_file_size_bytes: NonZeroUsize,++ /// Number of info `LOG` files RocksDB keeps.+ #[envconfig(from = "COHORT_KEEP_LOG_FILE_NUM", default = "10")]+ pub cohort_keep_log_file_num: NonZeroUsize,+/// TTL in days for `cf_person_records`: a compaction filter drops a person record whose/// `last_seen_ms` is older than this. `0` (the default) disables it — no filter is installed, so a/// persisted record never ages out. Attached to `cf_person_records` only; `cf_behavioral` eviction@@ -1130,6 +1138,8 @@ impl Config {compact_on_deletion: self.cohort_compact_on_deletion_enabled,periodic_compaction_seconds: self.cohort_periodic_compaction_seconds,max_background_jobs: self.cohort_max_background_jobs,+ max_log_file_size_bytes: self.cohort_max_log_file_size_bytes,+ keep_log_file_num: self.cohort_keep_log_file_num,person_record_ttl_days: self.cohort_person_record_ttl_days,ownership,..StoreConfig::default()@@ -1336,6 +1346,8 @@ mod tests {cohort_compact_on_deletion_enabled: true,cohort_periodic_compaction_seconds: 0,cohort_max_background_jobs: 0,+ cohort_max_log_file_size_bytes: NonZeroUsize::new(268_435_456).unwrap(),+ cohort_keep_log_file_num: NonZeroUsize::new(10).unwrap(),cohort_person_record_ttl_days: 0,cohort_store_offload_mode: OffloadMode::All,cohort_store_event_read_permits: 16,@@ -2611,6 +2623,8 @@ mod tests {assert!(defaults.cohort_compact_on_deletion_enabled);assert_eq!(defaults.cohort_periodic_compaction_seconds, 0);assert_eq!(defaults.cohort_max_background_jobs, 0);+ assert_eq!(defaults.cohort_max_log_file_size_bytes.get(), 268_435_456);+ assert_eq!(defaults.cohort_keep_log_file_num.get(), 10);assert_eq!(defaults.partition_channel_buffer, 128);let env: std::collections::HashMap<String, String> = [@@ -2619,6 +2633,8 @@ mod tests {("COHORT_COMPACT_ON_DELETION_ENABLED", "false"),("COHORT_PERIODIC_COMPACTION_SECONDS", "3600"),("COHORT_MAX_BACKGROUND_JOBS", "2"),+ ("COHORT_MAX_LOG_FILE_SIZE_BYTES", "1048576"),+ ("COHORT_KEEP_LOG_FILE_NUM", "3"),("PARTITION_CHANNEL_BUFFER", "256"),].into_iter()@@ -2630,6 +2646,8 @@ mod tests {assert!(!config.cohort_compact_on_deletion_enabled);assert_eq!(config.cohort_periodic_compaction_seconds, 3600);assert_eq!(config.cohort_max_background_jobs, 2);+ assert_eq!(config.cohort_max_log_file_size_bytes.get(), 1_048_576);+ assert_eq!(config.cohort_keep_log_file_num.get(), 3);assert_eq!(config.partition_channel_buffer, 256);}@@ -2641,6 +2659,8 @@ mod tests {config.cohort_compact_on_deletion_enabled = false;config.cohort_periodic_compaction_seconds = 3600;config.cohort_max_background_jobs = 2;+ config.cohort_max_log_file_size_bytes = NonZeroUsize::new(1_048_576).unwrap();+ config.cohort_keep_log_file_num = NonZeroUsize::new(3).unwrap();let store = config.store_config(PartitionOwnership::single(64));assert_eq!(store.block_cache_bytes, 3_221_225_472);@@ -2648,6 +2668,8 @@ mod tests {assert!(!store.compact_on_deletion);assert_eq!(store.periodic_compaction_seconds, 3600);assert_eq!(store.max_background_jobs, 2);+ assert_eq!(store.max_log_file_size_bytes.get(), 1_048_576);+ assert_eq!(store.keep_log_file_num.get(), 3);}#[test]diff --git a/rust/cohort-stream-processor/src/main.rs b/rust/cohort-stream-processor/src/main.rsindex 58a76969fc4c..03c2427f1b92 100644--- a/rust/cohort-stream-processor/src/main.rs+++ b/rust/cohort-stream-processor/src/main.rs@@ -23,6 +23,7 @@ use cohort_stream_processor::merge::gc::MergeGcSweeper;use cohort_stream_processor::merge::redrive::RedriveSweeper;use cohort_stream_processor::observability;use cohort_stream_processor::observability::disk::SharedDiskUtilization;+use cohort_stream_processor::observability::store_files::StoreFilesSweeper;use cohort_stream_processor::observability::store_stats::{DiskProbe, StoreStatsSweeper};use cohort_stream_processor::observability::tokio_monitor::TokioRuntimeMonitor;use cohort_stream_processor::partitions::{@@ -560,6 +561,17 @@ async fn async_main(config: Config) -> Result<()> {"store_stats",consumer_handle.shutdown_token(),));+ tokio::spawn(run_sweep_loop(+ StoreFilesSweeper::new(+ PathBuf::from(&config.store_path),+ config+ .checkpoint_enabled+ .then(|| PathBuf::from(&config.checkpoint_local_dir)),+ ),+ config.stats_publish_interval(),+ "store_files",+ consumer_handle.shutdown_token(),+ ));tokio::spawn(TokioRuntimeMonitor::new(&tokio::runtime::Handle::current(),diff --git a/rust/cohort-stream-processor/src/observability/metrics.rs b/rust/cohort-stream-processor/src/observability/metrics.rsindex 6da376469a19..8ec201630f3c 100644--- a/rust/cohort-stream-processor/src/observability/metrics.rs+++ b/rust/cohort-stream-processor/src/observability/metrics.rs@@ -222,6 +222,10 @@ pub const STORE_SST_BYTES: &str = "store_sst_bytes";pub const STORE_LIVE_DATA_BYTES: &str = "store_live_data_bytes";/// Estimated key count, labelled by `cf` (gauge).pub const STORE_ESTIMATE_NUM_KEYS: &str = "store_estimate_num_keys";+/// Bytes in the store directory and the local checkpoints, labelled by file `kind`+/// (`sst`|`wal`|`info_log`|`manifest`|`other`|`checkpoint`) (gauge). `checkpoint` counts only the+/// local checkpoint files that the live store does not share.+pub const STORE_FILE_BYTES: &str = "store_file_bytes";/// Size of the filesystem holding the store (gauge, bytes).pub const STORE_DISK_TOTAL_BYTES: &str = "store_disk_total_bytes";@@ -939,6 +943,7 @@ mod tests {assert_eq!(STORE_SST_BYTES, "store_sst_bytes");assert_eq!(STORE_LIVE_DATA_BYTES, "store_live_data_bytes");assert_eq!(STORE_ESTIMATE_NUM_KEYS, "store_estimate_num_keys");+ assert_eq!(STORE_FILE_BYTES, "store_file_bytes");assert_eq!(STORE_DISK_TOTAL_BYTES, "store_disk_total_bytes");assert_eq!(STORE_DISK_AVAILABLE_BYTES, "store_disk_available_bytes");assert_eq!(STORE_DISK_UTILIZATION_PCT, "store_disk_utilization_pct");diff --git a/rust/cohort-stream-processor/src/observability/mod.rs b/rust/cohort-stream-processor/src/observability/mod.rsindex 4007a0a9bdd8..aa526fc694cb 100644--- a/rust/cohort-stream-processor/src/observability/mod.rs+++ b/rust/cohort-stream-processor/src/observability/mod.rs@@ -4,5 +4,6 @@ pub mod canonical_log;pub mod disk;pub mod health;pub mod metrics;+pub mod store_files;pub mod store_stats;pub mod tokio_monitor;diff --git a/rust/cohort-stream-processor/src/observability/store_files.rs b/rust/cohort-stream-processor/src/observability/store_files.rsnew file mode 100644index 000000000000..3c0ca39ec378--- /dev/null+++ b/rust/cohort-stream-processor/src/observability/store_files.rs@@ -0,0 +1,215 @@+//! Per-file-type disk usage of the store directory and the local checkpoint directory. RocksDB+//! reports only SST sizes, so this walk is the only view of the WAL, info `LOG`, `MANIFEST`, and+//! checkpoint bytes on the store volume.++use std::collections::HashSet;+use std::fs::{self, DirEntry, Metadata};+use std::io;+use std::os::unix::fs::MetadataExt;+use std::path::{Path, PathBuf};++use async_trait::async_trait;+use metrics::gauge;+use tracing::{error, warn};++use crate::observability::metrics::STORE_FILE_BYTES;+use crate::sweep::Sweeper;++/// Publishes the `store_file_bytes` gauge each tick. It has its own sweep loop, apart from+/// [`DiskProbe`](crate::observability::store_stats::DiskProbe), so a slow walk cannot skip a+/// sample of the seed disk gate. The loop awaits each walk, so two walks never overlap.+pub struct StoreFilesSweeper {+ store_path: PathBuf,+ /// Set only when checkpoints are on, because only then does startup check that the directory+ /// is outside the store and on the store's filesystem.+ checkpoint_dir: Option<PathBuf>,+}++impl StoreFilesSweeper {+ pub fn new(store_path: PathBuf, checkpoint_dir: Option<PathBuf>) -> Self {+ Self {+ store_path,+ checkpoint_dir,+ }+ }+}++#[async_trait]+impl Sweeper for StoreFilesSweeper {+ async fn run_once(&self) {+ let store_path = self.store_path.clone();+ let checkpoint_dir = self.checkpoint_dir.clone();+ let sample = tokio::task::spawn_blocking(move || {+ sample_store_files(&store_path, checkpoint_dir.as_deref())+ })+ .await;+ match sample {+ Ok(Ok(usage)) => {+ for (kind, bytes) in usage.by_kind() {+ gauge!(STORE_FILE_BYTES, "kind" => kind).set(bytes as f64);+ }+ }+ Ok(Err(err)) => {+ warn!(error = %err, path = %self.store_path.display(), "store file usage sample failed");+ }+ Err(err) => error!(error = %err, "store file usage sample panicked"),+ }+ }+}++#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]+struct StoreFileUsage {+ sst_bytes: u64,+ wal_bytes: u64,+ info_log_bytes: u64,+ manifest_bytes: u64,+ other_bytes: u64,+ /// Checkpoint bytes that the store directory does not share: SSTs that the live store+ /// compacted away, and the files the checkpoint copies instead of hard-links. Each file counts+ /// once, however many checkpoint attempts hard-link it.+ checkpoint_bytes: u64,+}++impl StoreFileUsage {+ fn by_kind(&self) -> [(&'static str, u64); 6] {+ [+ ("sst", self.sst_bytes),+ ("wal", self.wal_bytes),+ ("info_log", self.info_log_bytes),+ ("manifest", self.manifest_bytes),+ ("other", self.other_bytes),+ ("checkpoint", self.checkpoint_bytes),+ ]+ }+}++fn sample_store_files(+ store_path: &Path,+ checkpoint_dir: Option<&Path>,+) -> io::Result<StoreFileUsage> {+ let mut usage = StoreFileUsage::default();+ // Inode numbers do not collide between the two trees, because startup checks that the+ // checkpoint directory is on the store's filesystem.+ let mut counted = HashSet::new();+ for entry in fs::read_dir(store_path)? {+ let entry = entry?;+ let Some(meta) = file_metadata(&entry)? else {+ continue;+ };+ let name = entry.file_name();+ let name = name.to_string_lossy();+ let bucket = if name.ends_with(".sst") {+ &mut usage.sst_bytes+ } else if name.ends_with(".log") {+ // RocksDB names WAL files `<number>.log`; the info log is `LOG`, with no extension.+ &mut usage.wal_bytes+ } else if name == "LOG" || name.starts_with("LOG.old") {+ &mut usage.info_log_bytes+ } else if name.starts_with("MANIFEST-") {+ &mut usage.manifest_bytes+ } else {+ &mut usage.other_bytes+ };+ *bucket += meta.len();+ counted.insert(meta.ino());+ }+ if let Some(dir) = checkpoint_dir {+ usage.checkpoint_bytes = unshared_bytes(dir, &mut counted)?;+ }+ Ok(usage)+}++fn unshared_bytes(root: &Path, counted: &mut HashSet<u64>) -> io::Result<u64> {+ let mut total = 0;+ let mut pending = vec![root.to_path_buf()];+ while let Some(dir) = pending.pop() {+ let entries = match fs::read_dir(&dir) {+ Ok(entries) => entries,+ // The checkpoint sweeper prunes old attempt directories while this walk runs.+ Err(err) if err.kind() == io::ErrorKind::NotFound => continue,+ Err(err) => return Err(err),+ };+ for entry in entries {+ let entry = entry?;+ if entry.file_type()?.is_dir() {+ pending.push(entry.path());+ continue;+ }+ if let Some(meta) = file_metadata(&entry)? {+ if counted.insert(meta.ino()) {+ total += meta.len();+ }+ }+ }+ }+ Ok(total)+}++/// `None` for a non-file entry or a file that compaction or pruning deleted after the listing.+fn file_metadata(entry: &DirEntry) -> io::Result<Option<Metadata>> {+ match entry.metadata() {+ Ok(meta) if meta.is_file() => Ok(Some(meta)),+ Ok(_) => Ok(None),+ Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None),+ Err(err) => Err(err),+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn classifies_store_files_by_rocksdb_name() {+ let dir = tempfile::tempdir().unwrap();+ for (name, len) in [+ ("000012.sst", 100),+ ("000013.sst", 50),+ ("000014.log", 7),+ ("LOG", 3),+ ("LOG.old.1700000000000000", 4),+ ("MANIFEST-000005", 9),+ ("OPTIONS-000007", 2),+ ("CURRENT", 1),+ ] {+ fs::write(dir.path().join(name), vec![0u8; len]).unwrap();+ }++ let usage = sample_store_files(dir.path(), None).unwrap();++ assert_eq!(+ usage,+ StoreFileUsage {+ sst_bytes: 150,+ wal_bytes: 7,+ info_log_bytes: 7,+ manifest_bytes: 9,+ other_bytes: 3,+ checkpoint_bytes: 0,+ },+ );+ }++ #[test]+ fn checkpoint_counts_only_files_the_live_store_no_longer_shares() {+ let root = tempfile::tempdir().unwrap();+ let store = root.path().join("store");+ let lineage = root.path().join("ckpt").join("topic").join("0");+ let (older, newer) = (lineage.join("a1"), lineage.join("a2"));+ fs::create_dir_all(&store).unwrap();+ fs::create_dir_all(&older).unwrap();+ fs::create_dir_all(&newer).unwrap();++ fs::write(store.join("000020.sst"), vec![0u8; 40]).unwrap();+ fs::hard_link(store.join("000020.sst"), newer.join("000020.sst")).unwrap();+ // The live store compacted this SST away, so only the two checkpoint attempts hold it.+ fs::write(older.join("000010.sst"), vec![0u8; 30]).unwrap();+ fs::hard_link(older.join("000010.sst"), newer.join("000010.sst")).unwrap();+ fs::write(newer.join("MANIFEST-000003"), vec![0u8; 5]).unwrap();++ let usage = sample_store_files(&store, Some(&root.path().join("ckpt"))).unwrap();++ assert_eq!(usage.sst_bytes, 40);+ assert_eq!(usage.checkpoint_bytes, 35);+ }+}diff --git a/rust/cohort-stream-processor/src/store/rocks.rs b/rust/cohort-stream-processor/src/store/rocks.rsindex cbbc00119371..50ffc595bf2f 100644--- a/rust/cohort-stream-processor/src/store/rocks.rs+++ b/rust/cohort-stream-processor/src/store/rocks.rs@@ -82,6 +82,11 @@ const DEFAULT_COMPACT_ON_DELETION_WINDOW: usize = 1000;const DEFAULT_COMPACT_ON_DELETION_NUM_DELS_TRIGGER: usize = 500;const DEFAULT_COMPACT_ON_DELETION_RATIO: f64 = 0.5;+/// RocksDB keeps the active info `LOG` and the newest rolled ones, `DEFAULT_KEEP_LOG_FILE_NUM` files+/// in total, so the info logs use at most about 10 × 256 MiB of the store volume.+const DEFAULT_MAX_LOG_FILE_SIZE_BYTES: NonZeroUsize = NonZeroUsize::new(256 * 1024 * 1024).unwrap();+const DEFAULT_KEEP_LOG_FILE_NUM: NonZeroUsize = NonZeroUsize::new(10).unwrap();+/// Resolved RocksDB settings.#[derive(Debug, Clone)]pub struct StoreConfig {@@ -125,6 +130,11 @@ pub struct StoreConfig {/// record does not cover. A store that predates coverage records adopts all of its partitions/// as complete.pub ownership: PartitionOwnership,+ /// Size at which RocksDB rolls the info `LOG` file. RocksDB reads `0` as "never roll", which+ /// lets one info log grow without bound.+ pub max_log_file_size_bytes: NonZeroUsize,+ /// Number of info `LOG` files RocksDB keeps. RocksDB refuses to open with `0`.+ pub keep_log_file_num: NonZeroUsize,}impl Default for StoreConfig {@@ -148,6 +158,8 @@ impl Default for StoreConfig {max_background_jobs: 0,person_record_ttl_days: 0,ownership: PartitionOwnership::single(COHORT_PARTITION_COUNT as u16),+ max_log_file_size_bytes: DEFAULT_MAX_LOG_FILE_SIZE_BYTES,+ keep_log_file_num: DEFAULT_KEEP_LOG_FILE_NUM,}}}@@ -1652,6 +1664,8 @@ fn db_options(config: &StoreConfig) -> Options {if config.max_background_jobs > 0 {opts.set_max_background_jobs(config.max_background_jobs);}+ opts.set_max_log_file_size(config.max_log_file_size_bytes.get());+ opts.set_keep_log_file_num(config.keep_log_file_num.get());opts}@@ -3303,6 +3317,37 @@ mod tests {);}+ #[test]+ fn info_log_caps_reach_the_persisted_rocksdb_options() {+ let dir = TempDir::new().unwrap();+ let path = dir.path().join("db");+ let _store = CohortStore::open(&StoreConfig {+ path: path.clone(),+ max_log_file_size_bytes: NonZeroUsize::new(1_048_576).unwrap(),+ keep_log_file_num: NonZeroUsize::new(3).unwrap(),+ ..StoreConfig::default()+ })+ .unwrap();++ let options_file = std::fs::read_dir(&path)+ .unwrap()+ .map(|entry| entry.unwrap().path())+ .find(|p| {+ p.file_name()+ .unwrap()+ .to_string_lossy()+ .starts_with("OPTIONS-")+ })+ .expect("RocksDB writes an OPTIONS file at open");+ let options = std::fs::read_to_string(options_file).unwrap();+ for line in ["max_log_file_size=1048576", "keep_log_file_num=3"] {+ assert!(+ options.lines().any(|l| l.trim() == line),+ "missing `{line}`"+ );+ }+ }+#[test]fn stats_snapshot_tickers_are_zero_when_statistics_disabled() {let dir = TempDir::new().unwrap();